预处理tencent词向量

本文主要是介绍预处理tencent词向量，希望对大家解决编程问题提供一定的参考价值，需要的开发者们随着小编来一起学习吧！

#数据预处理，去除数字，只保留中文
def load_tencent_word_embedding(input_file, output_file):n = 0with open('tencent.txt', 'a', encoding='utf-8', errors='ignore') as w_f:with open('Tencent_AILab_ChineseEmbedding.txt', 'r', encoding='utf-8', errors='ignore')as f:for i in tqdm(range(8824330)):# 原始data = f.readline()a = data.split()#listif len(a) == 201:if not a[0].isdigit():n = n + 1w_f.write(data)print('total lines:',n)  # 8748463#获取单个字词向量
def process_tencent_embedding_unigram(input_file, output_file):n = 0with open(output_file, 'a', encoding='utf-8') as g:with open(input_file, 'r', encoding='utf-8') as f:for i in tqdm(range(8748463)):data = f.readline()a = data.split()if i == 0:g.write(data)else:if len(a[0]) == 1:n += 1g.write(data)print('total unigram:', n)#获取一个词(两个字)的词向量
def process_tencent_embedding_bigrams(input_file, output_file):n = 0with open(output_file, 'a', encoding='utf-8') as g:with open(input_file, 'r', encoding='utf-8') as f:for i in tqdm(range(8748463)):data = f.readline()a = data.split()if i == 0:g.write(data)else:if len(a[0]) == 2:n += 1g.write(data)print('total bigrams:',n)if __name__ == '__main__':input_file = 'Tencent_AILab_ChineseEmbedding'output_file = 'tencent.txt'load_tencent_word_embedding(input_file, output_file)

这篇关于预处理tencent词向量的文章就介绍到这儿，希望我们推荐的文章对编程师们有所帮助！

---