达观杯--特征工程5（特征组合）

前面已经用各种方法对数据集中‘Word’进行了处理，主要是向量化包括countVectorizer等，也有特征降维（特征选择）等工作。接下来将数据集中的article这一属性进行同样的处理之后，将其和Word处理之后的特征进行组合。

1 Word + article

使用tfidf，当然使用其他的向量化方法也是完全可以的。

import pickle 
import pandas as pd 
from sklearn.feature_extraction.text 
import TfidfVectorizer """===================================================================================================================== 1 数据预处理 """
 read_start_time = time.time() 
 df_train=pd.read_csv('train_set.csv') 
 df_test=pd.read_csv('test_set.csv') #df_train.drop(df_train.columns[0],axis=1,inplace=True) 
 df_train["word_article"] = df_train["article"].map(str) +''+df_train["word_seg"].map(str) df_test["word_article"] = df_test["article"].map(str) +' ' + df_test["word_seg"].map(str) y_train = (df_train['class'] - 1).values """===================================================================================================================== 2 特征工程 """ 
 vectorizer = TfidfVectorizer(ngram_range=(1, 2), min_df=3, max_df=0.9, sublinear_tf=True) 
vectorizer.fit(df_train['word_article']) 
x_train = vectorizer.transform(df_train['word_article']) 
x_test = vectorizer.transform(df_test['word_article'])
保存至本地
"""
data = (x_train, y_train, x_test)
with open('./tfidf(word+article).pkl', 'wb') as f：
	pickle.dump(data, f)

2 +lsa

""" 将tfidf(word+article)特征降维为lsa特征，并将结果保存至本地，并将结果保存到本地 """ 
from sklearn.decomposition import TruncatedSVD 
import pickle 
import time 
t_start = time.time() """===================================================================================================================== 0 读取tfidf(word+article)特征 """
 with open('tfidf(word+article).pkl.pkl', 'rb') as f: 
     x_train, y_train, x_test = pickle.load(f) """===================================================================================================================== 1 特征降维：
 lsa """ lsa = TruncatedSVD(n_components=200) 
 x_train = lsa.fit_transform(x_train) 
 x_test = lsa.transform(x_test) """===================================================================================================================== 2 将lsa特征保存至本地 """ 
 data = (x_train, y_train, x_test) 
 with open('tfidf(word+article)+lsa.pkl', 'wb') as f: 
     pickle.dump(data, f_data) 
 t_end = time.time() 
 print("共耗时：{}min".format((t_end-t_start)/60))