NLP-情感分析

import os
from os.path import isfile, join
import re
import numpy as np
from random import randint
import tensorflow as tf
import config


batch_size = 24
lstm_units = 64
num_labels = 2
iterations = 100
lr = 0.001


def load_wordsList():
    """
    载入词典, 该词典包含 400000 个词汇
    """
    wordsList = np.load(os.path.join(config.root_dir, 'wordsList.npy'))
    print("-" * 20)
    print('载入word列表...')
    print("-" * 20)
    wordsList = wordsList.tolist()
    wordsList = [word.decode('UTF-8') for word in wordsList]

    return wordsList


def load_wordVectors():
    """
    载入已经训练好的词典向量模型, 该矩阵包含了的文本向量, 维度: (400000, 50)
    """
    wordVectors = np.load(os.path.join(config.root_dir, 'wordVectors.npy'))
    print("-" * 20)
    print('载入文本向量...')
    print("-" * 20)

    return wordVectors


def load_idsMatrix():
    """
    
    """
    ids = np.load(os.path.join(config.root_dir, 'idsMatrix.npy'))

    return ids


def postive_analysis():
    """
    载入正面数据集
    """
    pos_files = [config.pos_data_dir + f for f in os.listdir(config.pos_data_dir) if isfile(join(config.pos_data_dir, f))]
    num_words = []
    for pf in pos_files:
        with open(pf, "r", encoding='utf-8') as f:
            line = f.readline()
            counter = len(line.split())
            num_words.append(counter)
    print("-" * 20)
    print('正面评价数据加载完结...')
    print("-" * 20)
    num_files = len(num_words)
    print('正面评价数据文件总数', num_files)
    print('正面评价数据所有的词的数量', sum(num_words))
    print('正面评价数据平均文件词的长度', sum(num_words) / len(num_words))
    
    return pos_files


def negtive_analysis():
    """
    载入负面数据集
    """
    neg_files = [config.neg_data_dir + f for f in os.listdir(config.neg_data_dir) if isfile(join(config.neg_data_dir, f))]
    num_words = []
    for nf in neg_files:
        with open(nf, "r", encoding='utf-8') as f:
            line = f.readline()
            counter = len(line.split())
            num_words.append(counter)
    print("-" * 20)
    print('负面评价数据加载完结...')
    print("-" * 20)
    num_files = len(num_words)
    print('负面评价数据文件总数', num_files)
    print('负面评价数据所有的词的数量', sum(num_words))
    print('负面评价数据平均文件词的长度', sum(num_words) / len(num_words))

    return neg_files


if __name__ == "__main__":
    # 词典
    wordsList = load_wordsList()
    print("词典中词汇数量:", len(wordsList))
    home_index = wordsList.index("home")
    print("'home' 单词在词典中的索引:", home_index)
    
    # 词典向量模型矩阵
    wordVectors = load_wordVectors()
    print("词典向量模型矩阵:", wordVectors.shape)
    print("'home' 在词典向量模型矩阵中的向量表示:", wordVectors[home_index])

    # 正面、负面文本数据
    pos_files = postive_analysis()
    neg_files = negtive_analysis()

def data_visual(num_words):
    import matplotlib as mpl
    import matplotlib.pyplot as plt
    # mpl.use("qt4agg")
    # 指定默认字体
    mpl.rcParams["font.sans-serif"] = ["SimHei"]
    mpl.rcParams["font.family"] = "sans-serif"
    # %matplotlib inline
    plt.hist(num_words, 50, facecolor = "g")
    plt.xlabel("文本长度")
    plt.ylabel("频次")
    plt.axis([0, 1200, 0, 8000])
    plt.show()

if __name__ == "__main__":
    # 正面、负面文本数据
    pos_files, pos_num_words = postive_analysis()
    neg_files, neg_num_words = negtive_analysis()

    # 文本数据可视化
    num_total_words = pos_num_words + neg_num_words
    data_visual(num_total_words)

$ tensorflow --logdir=tensorboard