# 根据随机抽取出1/1000的微博数据作为样例，与原始数据格式一致

# 微博数据文件的格式：
#  用户ID 地址 微博ID unix时间戳 月数 周数 微博文本

import os 
import time 
import random

# ===================== 参数设置 =====================

# 原始数据文件的路径
inpath = "../data/weibo2209/"

# 抽取出来的数据文件保存的路径
outpath = "../data/weibo2209-samples/"

# 是否对UID和微博ID进行随机化处理
IS_RADOMIZE = True

# ===================== 参数设置 =====================

# 用户uid和微博wid的随机化生成，uid可能有重复，wid是单向增加
uid_array = []
uid_last = 2209
wid_last = 220901

def getRandomUID():
    global uid_last
    
    if(len(uid_array) == 0):
        idx = 0
        while(idx < 100):
            uid_last += random.randint(1, 9)
            uid_array.append(uid_last)
            idx += 1

    lrand = random.randint(0, int(len(uid_array)+10)-1)
    if(lrand < len(uid_array)):
        return uid_array[lrand]
    else:
        uid_last += random.randint(1, 9)
        uid_array.append(uid_last)
        return uid_last


def getRandomWID():
    global wid_last
    
    wid_last += random.randint(1, 9)
    return wid_last


# 将缓存中的微博数据wbuf写入文件fname中
def saveWeibo(fname, wbuf):
    if(len(wbuf) <= 0):
        return
    
    with open(fname, 'a', encoding='utf-8') as outf:
        outf.write(wbuf)
        outf.flush()

    return

    
# 筛选出符合地区要求的微博导出
def weiboExport(fname, fpath):
    buf_out = ''
    
    #从数据文件中读入，解析每行内容
    num_lines = 0
    
    outfile = outpath + fname
    
    with open(fpath, encoding='utf-8') as freader:
        for fline in freader:
            line_content = fline.rstrip('\r\n')
            seg = line_content.split('\t')
            col_num = len(seg)
            if(col_num != 7):
                continue
            
            lrand = random.randint(0, 100000)
            # 抽取出最多1/10000的微博数据作为样例数据
            if(lrand > 10):
                continue
            
            # 对用户uid和微博wid进行随机化处理
            if(IS_RADOMIZE):
                seg[0] = str(getRandomUID())
                seg[2] = str(getRandomWID())
                buf_out += '\t'.join(seg) + '\n'
            else:
                buf_out += line_content + '\n'
            
            num_lines += 1
            
            if(num_lines >= 10000):
                print('.', end='')
                saveWeibo(outfile, buf_out)
                buf_out = ''
                num_lines = 0
                
    print()
    
    saveWeibo(outfile, buf_out)
    buf_out = ''

    return

# 对inpath目录中的每个文件进行筛选，导出符合地区要求的数据
for f in os.listdir(inpath):
    st = time.time()
    infile = inpath + f 
    print(infile)
    weiboExport(f, infile)
           
    et = time.time()
    print(f,":",(et-st),"s")
    
    