机器学习入门-贝叶斯中文新闻分类任务

import pandas as pd
import numpy as np
import jieba

# 1.导入数据语料的新闻数据
df_data = pd.read_table(‘data/val.txt‘, names=[‘category‘, ‘theme‘, ‘URL‘, ‘content‘], encoding=‘utf-8‘)

# 2.对语料库进行分词操作
df_contents = df_data.content.values.tolist()

# list of list 结构
Jie_content = []
for df_content in df_contents:
    split_content = jieba.lcut(df_content)
    if len(split_content) > 1 and split_content != ‘\t\n‘:
        Jie_content.append(split_content)

# 3. 导入停止词的语料库, sep=‘\t‘表示分隔符， quoting控制引号的常量， names=列名， index_col=False，不用第一列做为行的列名， encoding
stopwords = pd.read_csv(‘stopwords.txt‘, sep=‘\t‘, quoting=3, names=[‘stopwords‘], index_col=False, encoding=‘utf-8‘)
print(stopwords.head())

# 对文本进行停止词的去除
def drop_stops(Jie_content, stopwords):
    clean_content = []
    all_words = []
    for j_content in Jie_content:
        line_clean = []
        for line in j_content:
            if line in stopwords:
                continue
            line_clean.append(line)
            all_words.append(line)
        clean_content.append(line_clean)

    return clean_content, all_words
# 将DateFrame的stopwords数据转换为list形式
stopwords = stopwords.stopwords.values.tolist()
clean_content, all_words = drop_stops(Jie_content, stopwords)
print(clean_content[0])

# 4. 构造训练数据，变量是content，标签是category
df_content = pd.DataFrame({‘content‘:clean_content, ‘label‘:df_data[‘category‘]})
# 使用map将标签转换为数字形式
print(df_content.label.unique())
label_map = {‘汽车‘:1, ‘财经‘:2, ‘科技‘:3, ‘健康‘:4, ‘体育‘:5, ‘教育‘:6, ‘文化‘:7, ‘军事‘:8, ‘娱乐‘:9, ‘时尚‘:10}
df_content[‘label‘] = df_content[‘label‘].map(label_map)


# 5.使用train_test_split 分出训练集和测试集

from sklearn.cross_validation import train_test_split
train_x, test_x, train_y, test_y = train_test_split(df_content[‘content‘], df_content[‘label‘], random_state=1)
# 将样本特征转换为词频向量的形式

from sklearn.feature_extraction.text import CountVectorizer
# 6. 为了满足CountVectorizer的形式，我们需要对转换前的样本做转换

train_x_str = []
for line in train_x:
    str_line = ‘ ‘.join(line)
    train_x_str.append(str_line)
test_x_str = []
for line in test_x:
    str_line = ‘ ‘.join(line)
    test_x_str.append(str_line)


# 将文本数据根据词频转换为向量形式
vec = CountVectorizer(lowercase=False, max_features=4000)
vec.fit(train_x_str)

# 7步 使用贝叶斯模型进行训练和测试
from sklearn.naive_bayes import MultinomialNB

print(‘1‘, np.shape(vec.transform(train_x_str)))
classifier = MultinomialNB()
classifier.fit(vec.transform(train_x_str), train_y)
print(classifier.score(vec.transform(test_x_str), test_y))

# 8. 使用TF-IDF构建向量矩阵
from sklearn.feature_extraction.text import TfidfVectorizer

vec = TfidfVectorizer(lowercase=False, max_features=4000)
vec.fit(train_x_str)
classifier.fit(vec.transform(train_x_str), train_y)
print(classifier.score(vec.transform(test_x_str), test_y))


vec = TfidfVectorizer(lowercase=False, max_features=4000, ngram_range=(1, 3))
vec.fit(train_x_str)
classifier.fit(vec.transform(train_x_str), train_y)
print(classifier.score(vec.transform(test_x_str), test_y))