
#利用结巴分词，统计自述中出现的词频，计算词频比率并导出
import re
import os
import jieba
import jieba.analyse
import pandas as pd
import joblib

import sys
sys.path.append("..")
from utils import ccpl_lib as ccpl

# ===================== 参数设置 =====================

#输入的数据文件所在目录
inpath = "../data/timeloc/"

#每个数据文件名在输出文件里的列名
filenameHeader = 'ID'


# 计算结果的输出路径
outpath = '../data/out/'

# ===================== 参数设置 =====================


#大连理工情感词典，21个情感分类
#快乐(PA)，安心(PE)，尊敬(PD)，赞扬(PH)，相信(PG)，喜爱(PB)，祝愿(PK)
#愤怒(NA)，悲伤(NB)，失望(NJ)，疚(NH)，思(PF)，慌(NI)，恐惧(NC)
#羞(NG)，烦闷(NE)，憎恶(ND)，贬责(NN)，妒忌(NK)，怀疑(NL)，惊奇(PC)
#微博客基本情绪词库，5个分类
#快乐(MH), 悲伤(MS), 愤怒(MA), 恐惧(MD), 厌恶(ME)
#复合情绪词类：3个词类
#P(Positive)、N(Negative)、Ne(Neutral)

affect_col_list = ['PA', 'PE','PD','PH','PG','PB','PK',
                   'NA','NB','NJ','NH','PF','NI','NC',
                   'NG','NE','ND','NN','NK','NL','PC',
                   'MH', 'MS', 'MA', 'MD', 'ME',
                   'P','N','Ne'
                   ]
# 载入情感词典
def load_affect_dict(filepath):
    m_affectdict = []
    for m_col in affect_col_list:
        m_col = []
        m_affectdict.append(m_col)
        
    for m_line in open(filepath, 'r', encoding='utf-8').readlines():
        m_line=m_line.strip()
        kwd = m_line.split('\t')[0].strip()
        col = m_line.split('\t')[1].strip()
        m_affectdict[affect_col_list.index(col)].append(kwd)
        
    return m_affectdict


# 创建停用词list
def load_stopwords(filepath):
    m_stopwords = [line.strip() for line in open(filepath, 'r', encoding='utf-8').readlines()]
    return m_stopwords
#清洗文本
def clean(text):
    text = re.sub(r"(回复)?(//)?\s*@\S*?\s*(:| |$)", " ", text)  # 去除正文中的@和回复/转发中的用户名
    text = re.sub(r"\[\S+\]", "", text)      # 去除表情符号
    text = re.sub(r"#\S+#", "", text)      # 保留话题内容
    URL_REGEX = re.compile(
        r'(?i)\b((?:https?://|www\d{0,3}[.]|[a-z0-9.\-]+[.][a-z]{2,4}/)(?:[^\s()<>]+|\(([^\s()<>]+|(\([^\s()<>]+\)))*\))+(?:\(([^\s()<>]+|(\([^\s()<>]+\)))*\)|[^\s`!()\[\]{};:\'".,<>?«»“”‘’]))',
        re.IGNORECASE)
    text = re.sub(URL_REGEX, "", text)       # 去除网址
    para= text.strip()
    return para


#结巴分词后，统计分词后的字词中，有多少出现在词典中
def cntkws_jieba_seg_wrds(c_desc,kws):
    #精确切分
    c_tags = jieba.cut(c_desc, cut_all=False)
    #切词后词的总数
    cnt_wrds = 0
    #切词后，其中包含情感词的总数
    affect_cnt = 0
    for s in c_tags:
        s = s.strip()

        # 删除停词
        if (s in stopwords) or (len(s) == 0):
            continue
        cnt_wrds += 1
        if(s in kws):
            affect_cnt += 1
            
    return affect_cnt/cnt_wrds

#读入一个数据文件，返回文本
def read_swls_file(fname):
    fr_swls = open(fname, 'r', encoding='UTF-8-sig')
    x_swls_strs = fr_swls.read()
    return x_swls_strs


if(not os.path.exists(inpath)):
    print("输入数据目录 ", inpath, " 不存在!")
    sys.exit(0)

if(not os.path.exists(outpath)):
    print("输出数据目录 ", outpath, " 不存在!")
    sys.exit(0)

#载入停词表
stop_word_file = '.\\res\\stop_words_cn.txt'
stopwords = load_stopwords(stop_word_file)

#载入情感词典
affect_dict_file = '.\\res\\dict-affect.txt'
affect_dict = load_affect_dict(affect_dict_file)

#载入情感词典中的词做为自定义词典
jieba.load_userdict('.\\res\\jiebaload_affect_dict.txt')

# 载入预测模型, 调用训练好的模型
myModel=".\\res\\swls.model"
clf_swls = joblib.load(myModel)

# 表头
headerstr = filenameHeader+','+','.join(affect_col_list)+'\n'

groupslist = ccpl.getSubFolderList(inpath)
for eachgroup in groupslist:
    inGroupDir = inpath + eachgroup
    # print(rL)
    
    #设置词频特征的输出文件，并打印表头
    ind_vars_file = outpath + eachgroup + '_WordFreqResults.csv'
    with open(ind_vars_file,'w',encoding='UTF-8-sig') as outf:
        outf.write(headerstr)
        outf.flush()
    
    # 读取目录下的每个数据文件
    for eachSample in os.listdir(inGroupDir):
        infile = inGroupDir + '/' + eachSample 
        # print(infile)
        
        # 获取数据文件的文件名
        sampleID = os.path.splitext(eachSample)[0]
        
        px_desc = read_swls_file(infile)
        
        str_export = sampleID

        idx = 0
        for g_col in affect_col_list:
            r_affect = cntkws_jieba_seg_wrds(px_desc, affect_dict[idx])
            str_export += ',' + str(r_affect)
            idx += 1

        with open(ind_vars_file,'a',encoding='UTF-8-sig') as outf:
            outf.write(str_export+'\n')
            outf.flush()


    datafile = ind_vars_file
    
    #输出文件
    swlsfile = outpath + eachgroup + '_result_SWLS.csv'

    labels = ['swls']
    feature_index=[1, 9, 16, 18, 22, 23, 28]
    data = pd.read_csv(datafile,header=None)
    df = pd.DataFrame()
    for label in labels:
        df['id'] = data.iloc[1:, 0]
        data_ready = data.loc[1:, feature_index]
        X = data_ready.values
        
        df['swls'] = clf_swls.predict(X)

    df.to_csv(swlsfile, encoding='UTF-8-sig', index=False)

print("DONE！")

        

