# -*- coding: utf-8 -*-
import re
import sys

#reload(sys)
#sys.setdefaultencoding('utf-8')

class word_frequency:
    def __init__(self, dirname = ''):
        sentence_separator = ['.', '。', '?', '？', '!', '！', ';', '；']  
        sentence_separator = set([i.encode('utf-8').decode('utf-8') for i in sentence_separator])
        
        #将正则表达式编译成Pattern对象
        rx_latin      = re.compile("^[a-zA-Z0-9_]+$", re.UNICODE)
        rx_numeral    = re.compile(r"([-+]?[0-9]*\.?[0-9]+([eE][-+]?[0-9]+)?)", re.UNICODE)
        rx_url        = re.compile(r"((https?|ftp|file)://[-a-zA-Z0-9+&@#/%?=~_|!:,.;]*[-a-zA-Z0-9+&@#/%=~_|])", re.UNICODE)
        rx_emotion    = re.compile(r"(\[(.{1,16}?)\])", re.UNICODE)
        rx_at_mention = re.compile(u"@[\u4e00-\u9fa5a-zA-Z0-9_-]{4,30} ", re.UNICODE)
        rx_hashtag    = re.compile(r"#[^#]+#", re.UNICODE)
        rx_sentence   = re.compile(u"[.。?？!！;；]", re.UNICODE)
        
        self.is_sentence_separator = lambda term: term in sentence_separator
        self.is_latin              = lambda term: rx_latin.match(term) is not None
        
        #匹配
        self.find_numeral          = lambda string: [t[0] for t in rx_numeral.findall(string)]
        self.find_url              = lambda string: [t[0] for t in rx_url.findall(string)]
        self.find_emotions         = lambda string: [t[0] for t in rx_emotion.findall(string)]
        self.find_at_mention       = lambda string: [t[0] for t in rx_at_mention.findall(string.encode('utf-8').decode('utf-8'))]
        self.find_hashtag          = lambda string: [t[0] for t in rx_hashtag.findall(string)]
        self.find_sentence         = lambda string: [t[0] for t in rx_sentence.findall(string)]
        self.find_latin            = lambda string: [t[0] for t in rx_latin.findall(string)]

        self.IndexName = []
        self.Name = []
        self.NameIndex = {}

        self.termsMap = {}
        self.mapResult = {}

        self.n_sentence_total_number = 0
        self.n_term_total_number = 0
        self.n_term_numerals = 0
        self.n_term_in_dic = 0
        self.n_term_len_gte6 = 0
        self.n_term_len_gte4 = 0
        self.n_term_latin = 0
        self.n_term_at_mention = 0
        self.n_term_emotion = 0
        self.n_term_hashtag = 0
        self.n_term_url = 0
        self.word_max_len = 0
        self.dic_dir = dirname

        self.loadDic(dirname)

    def loadDic(self, dirname):
        #if dirname.endswith('\\') or dirname.endswith('/'):
        self.dic_dir = dirname

        self.IndexName = []
        self.Name = []
        self.NameIndex.clear()
        self.termsMap.clear()
        
        #载入map.xxxx.ALL文件
        self.load_map()
        
        #根据目录中的文件名前缀，判断是否为所需的文件；
        import os
        fileList = os.listdir(self.dic_dir)
        for fn in fileList:
            if fn.startswith('dic') or fn.startswith('sym'):
                fname = self.dic_dir+'/' +fn
                if os.path.isdir(fname):
                    continue
                self.load_dic_terms(fname)
            elif fn.startswith('lst'):
                fname = self.dic_dir+'/' +fn
                if os.path.isdir(fname):
                    continue
        


    #读取map.xxxx.ALL文件内容
    def load_map(self):
        self.IndexName = []
        self.Name = []
        
        import os
        fileLists = os.listdir(self.dic_dir)
        for fns in fileLists:
            if fns.startswith('map'):
                fnames = self.dic_dir+'/' +fns
                
                with open(fnames, 'r',encoding='utf-8') as fin:
                    #按行读取；
                    for line in fin:
                        line = line.strip().replace('\n', '').replace('\r', '')
                        #在map.xxxx.ALL中第一样有“ %【说明】内容” 
                        if line.__contains__('%'):
                            continue
                        
                        lines = re.split('\\s{1,}', line)
                        if len(lines) < 2:
                            print ('ERROR:load map length error')
                            continue
                        
                        self.IndexName.append(int(lines[0]))
                        self.Name.append(lines[1])
                        self.NameIndex[str(lines[1]).lower()] = int(lines[0])
                
                
    #将新载入的词典（新增的词汇和标注） 与map.xxxx.ALL文件中的类型名称进行对比，添加类型编号；
    def load_dic_terms(self,fname):
        with open(fname, 'r',encoding='utf-8') as fin:
            for line in fin:
                line = line.strip().replace('\n', '').replace('\r', '')
                if line.__contains__('%'):
                    continue
                
                lines = re.split('\\s{1,}', line)
                if len(lines) < 2:
                    print ('ERROR:load map length error')
                    continue

                #keyword为文字内容，而非词类；
                keyword = lines[0].strip().encode('utf-8').decode('utf-8')
                
                #word_max_len初始值为0，字符串长度
                if self.word_max_len < len(keyword.encode('utf-8').decode('utf-8')):
                    self.word_max_len = len(keyword.encode('utf-8').decode('utf-8'))
                
                #初始self.termsMap = {}；添加字典元素；{文字内容： }
                if not self.termsMap.__contains__(keyword):
                    self.termsMap[keyword] = []

                for i in range(1, len(lines)):
                    classname = lines[i].strip().lower()
                    #判断通过map.xxxx.ALL文件构造的NameIndex字典中，是否含有classname 键（类型）；
                    if self.NameIndex.__contains__(classname):
                        classId = self.NameIndex.get(classname)  #获取值（编号，1、2、...）
                    else:
                        print ("ERROR:can not find classname:", classname)
                        continue
                    
                    #补充termsMap字典中的值， {文字内容：map.xxxx.ALL文件中对应的类型编号，1、2、...}
                    if self.termsMap.get(keyword).__contains__(classId):
                        continue
                    else:
                        self.termsMap.get(keyword).append(classId)
        # print(self.NameIndex)
        # print(self.termsMap)
        # print(self.word_max_len)


    def getFeatureOriginal(self, original=[], ration=True):
        self.clear()

        for line in original:
            if len(line) < 3:
                continue
            
            #统计各项匹配的个数
            line = line.encode('utf-8').decode('utf-8')
            self.n_term_numerals += len(self.find_numeral(line))
            self.n_term_at_mention += len(self.find_at_mention(line))
            self.n_term_emotion += len(self.find_emotions(line))
            self.n_term_hashtag += len(self.find_hashtag(line))
            self.n_term_url += len(self.find_url(line))
            
            #以标点符号u"[.。?？!！;；]"匹配的结果，确定句子个数；
            if len(self.find_sentence(line)) == 0:
                self.n_sentence_total_number = 1
            else:
                self.n_sentence_total_number += len(self.find_sentence(line))
            #print (self.n_sentence_total_number)
        
            self.n_term_latin += len(self.find_latin(line))

            #字符总个数；
            self.n_term_total_number += len(line.encode('utf-8').decode('utf-8'))

            strbeg = 0
            strend = strbeg + self.word_max_len
            #line = line.decode('utf-8')
            while strend <= len(line):
                if strend > strbeg:
                    stmp = line[strbeg:strend]
                    #print(stmp)
                    #print(self.word_max_len)
                    if self.termsMap.__contains__(stmp):
                        self.n_term_in_dic += len(stmp)  
                        tags = self.termsMap.get(stmp)  #获取值（类型编号）
                        
                        #termsMap字典中的文字内容，可能对应多种类型，所以会有多个编号；
                        for index in tags:   
                            #初始化self.mapResult = {}，将对应的键（类型编号）增加相应的值（出现的次数）；
                            if self.mapResult.__contains__(index):
                                self.mapResult[index] = self.mapResult[index]+1
                            else:
                                self.mapResult[index] = 1
                        strbeg = strend
                        strend = strbeg + self.word_max_len
                    else:
                        strend -= 1
                else:
                    strbeg += 1
                    strend = strbeg + self.word_max_len
                    
            strend = len(line)
            while strbeg < len(line):
                if strend > strbeg:
                    stmp = line[strbeg:strend]
                    #print(stmp)
                    #print(self.word_max_len)
                    if self.termsMap.__contains__(stmp):
                        self.n_term_in_dic += len(stmp)  
                        tags = self.termsMap.get(stmp)  
    
                        for index in tags:   
                            if self.mapResult.__contains__(index):
                                self.mapResult[index] = self.mapResult[index]+1
                            else:
                                self.mapResult[index] = 1
                        strbeg = strend
                        strend = len(line)
                    else:
                        strend -= 1
                else:
                    strbeg += 1
                    strend = len(line)

        return self.getFrequencyResult(ration) + self.getStatisticResult()


    def getFeatureOriginalF(self, original=[], ration=True):
        self.clear()

        for line in original:
            if len(line) < 3:
                continue
            
            line = line.encode('utf-8').decode('utf-8')
            self.n_term_numerals += len(self.find_numeral(line))
            self.n_term_at_mention += len(self.find_at_mention(line))
            self.n_term_emotion += len(self.find_emotions(line))
            self.n_term_hashtag += len(self.find_hashtag(line))
            self.n_term_url += len(self.find_url(line))
            
            if len(self.find_sentence(line)) == 0:
                self.n_sentence_total_number = 1
            else:
                self.n_sentence_total_number += len(self.find_sentence(line))
            #print (self.n_sentence_total_number)
        
            self.n_term_latin += len(self.find_latin(line))
            self.n_term_total_number += len(line.encode('utf-8').decode('utf-8'))

            strbeg = 0
            strend = strbeg + self.word_max_len
            #line = line.decode('utf-8')
            while strend <= len(line):
                if strend > strbeg:
                    stmp = line[strbeg:strend]
                    #print(stmp)
                    #print(self.word_max_len)
                    if self.termsMap.__contains__(stmp):
                        self.n_term_in_dic += len(stmp)  
                        tags = self.termsMap.get(stmp)  
    
                        for index in tags:   
                            if self.mapResult.__contains__(index):
                                self.mapResult[index] = self.mapResult[index]+1
                            else:
                                self.mapResult[index] = 1
                        strbeg = strend
                        strend = strbeg + self.word_max_len
                    else:
                        strend -= 1
                else:
                    strbeg += 1
                    strend = strbeg + self.word_max_len
                    
            strend = len(line)
            while strbeg < len(line):
                if strend > strbeg:
                    stmp = line[strbeg:strend]
                    #print(stmp)
                    #print(self.word_max_len)
                    if self.termsMap.__contains__(stmp):
                        self.n_term_in_dic += len(stmp)  
                        tags = self.termsMap.get(stmp)  
    
                        for index in tags:   
                            if self.mapResult.__contains__(index):
                                self.mapResult[index] = self.mapResult[index]+1
                            else:
                                self.mapResult[index] = 1
                        strbeg = strend
                        strend = len(line)
                    else:
                        strend -= 1
                else:
                    strbeg += 1
                    strend = len(line)

        return self.getFrequencyResult(ration)
		
    def clear(self):
        self.n_sentence_total_number = 0
        self.n_term_total_number = 0
        self.n_term_numerals = 0
        self.n_term_in_dic = 0
        self.n_term_len_gte6 = 0
        self.n_term_len_gte4 = 0
        self.n_term_latin = 0
        self.n_term_at_mention = 0
        self.n_term_emotion = 0
        self.n_term_hashtag = 0
        self.n_term_url = 0

        self.mapResult.clear()

    def getFrequencyResult(self, ration=True):
        res = ''
        #总字符数
        if self.n_term_total_number == 0 :
            for i in range(0, len(self.IndexName)):
                res += '0,'
            return res

        for index in self.IndexName:
            if ration:
                if self.mapResult.__contains__(index):
                    #词频
                    res += str(float(self.mapResult.get(index))/float(self.n_term_total_number))+','
                else:
                    res += '0.0,'
            else:
                if self.mapResult.__contains__(index):
                    res += str(self.mapResult.get(index)) + ','
                else:
                    res += '0,'

        return res

    def getStatisticResult(self):
        if self.n_term_total_number == 0:
            self.n_term_total_number = sys.maxsize
        
        #额外的11个指标
        res = ''
        res += str(self.n_term_total_number) + ','
        if(self.n_sentence_total_number > 0):
            res += str(float(self.n_term_total_number)/float(self.n_sentence_total_number)) + ','
        else:
            res += '0.0,'
        res += str(float(self.n_term_in_dic)/float(self.n_term_total_number)) + ','
        res += str(float(self.n_term_numerals)/float(self.n_term_total_number)) + ','
        res += str(float(self.n_term_len_gte6)/float(self.n_term_total_number)) + ','  #未计算
        res += str(float(self.n_term_len_gte4)/float(self.n_term_total_number)) + ',' #未计算
        res += str(float(self.n_term_latin)/float(self.n_term_total_number)) + ','
        res += str(self.n_term_at_mention) + ','
        res += str(self.n_term_emotion) + ','
        res += str(self.n_term_hashtag) + ','
        res += str(self.n_term_url)

        return res

if __name__ == "__main__":
    # wf = word_frequency("dic_cn\\dic_test")
    # test = ["我和王小明小红在操场跑步。我们哈哈大笑，非常开心！一起打了套三十八式陈氏太极拳！"]
    # print (wf.getFeatureOriginal(test, False))

    #我 我们 不会重复计数；优先匹配更长的词汇；
    #分母为总字数n_term_total_number，包含所有字符（数字、字母、符号等）；
    #n_sentence_total_number句子个数；[.。?？!！;；]出现的频数为句子个数，上述符号在全文中一次也没出现时，算一句话；
    #n_term_in_dic表示字典中有多少个字匹配成功（是字数，而不是词数）；
    
    
    # wf = word_frequency("dic_cn\\dic_daodejichu")
    # test = ["守卫饕餮遭受危险一视同仁区别对待非正义一致反对危害国家尊重以下犯上目无法纪寺无节操有价值"]
    # print (wf.getFeatureOriginal(test, False))
    
    
    # wf = word_frequency("dic_cn\\dic_daodedongji")
    # test = ["定金小王，区分面值；诚挚且平易近人，非政府组织不偏不倚。"]
    # print (wf.getFeatureOriginal(test, False))

    
    # wf = word_frequency("dic_cn\\dic_getijitizhuyi")
    # test = ["张三坚持与众不同，一己之力；咱们一致同意；一个县城的人乐于助人；富有同情心！"]
    # print (wf.getFeatureOriginal(test, False))
    
    
    wf = word_frequency("dic_cn\\dic_weibojibenqingxu")
    test = ["high爱情.狗狗吵架诚惶诚恐还发慌。霸道狗out,耿耿于怀！"]
    print (wf.getFeatureOriginal(test, False))
    
    
    
    
    