before svm blog

Author: WenDesi

.idea/workspace.xml

@@ -0,0 +1,151 @@
+Id,SepalLengthCm,SepalWidthCm,PetalLengthCm,PetalWidthCm,Species
+1,5.1,3.5,1.4,0.2,Iris-setosa
+2,4.9,3.0,1.4,0.2,Iris-setosa
+3,4.7,3.2,1.3,0.2,Iris-setosa
+4,4.6,3.1,1.5,0.2,Iris-setosa
+5,5.0,3.6,1.4,0.2,Iris-setosa
+6,5.4,3.9,1.7,0.4,Iris-setosa
+7,4.6,3.4,1.4,0.3,Iris-setosa
+8,5.0,3.4,1.5,0.2,Iris-setosa
+9,4.4,2.9,1.4,0.2,Iris-setosa
+10,4.9,3.1,1.5,0.1,Iris-setosa
+11,5.4,3.7,1.5,0.2,Iris-setosa
+12,4.8,3.4,1.6,0.2,Iris-setosa
+13,4.8,3.0,1.4,0.1,Iris-setosa
+14,4.3,3.0,1.1,0.1,Iris-setosa
+15,5.8,4.0,1.2,0.2,Iris-setosa
+16,5.7,4.4,1.5,0.4,Iris-setosa
+17,5.4,3.9,1.3,0.4,Iris-setosa
+18,5.1,3.5,1.4,0.3,Iris-setosa
+19,5.7,3.8,1.7,0.3,Iris-setosa
+20,5.1,3.8,1.5,0.3,Iris-setosa
+21,5.4,3.4,1.7,0.2,Iris-setosa
+22,5.1,3.7,1.5,0.4,Iris-setosa
+23,4.6,3.6,1.0,0.2,Iris-setosa
+24,5.1,3.3,1.7,0.5,Iris-setosa
+25,4.8,3.4,1.9,0.2,Iris-setosa
+26,5.0,3.0,1.6,0.2,Iris-setosa
+27,5.0,3.4,1.6,0.4,Iris-setosa
+28,5.2,3.5,1.5,0.2,Iris-setosa
+29,5.2,3.4,1.4,0.2,Iris-setosa
+30,4.7,3.2,1.6,0.2,Iris-setosa
+31,4.8,3.1,1.6,0.2,Iris-setosa
+32,5.4,3.4,1.5,0.4,Iris-setosa
+33,5.2,4.1,1.5,0.1,Iris-setosa
+34,5.5,4.2,1.4,0.2,Iris-setosa
+35,4.9,3.1,1.5,0.1,Iris-setosa
+36,5.0,3.2,1.2,0.2,Iris-setosa
+37,5.5,3.5,1.3,0.2,Iris-setosa
+38,4.9,3.1,1.5,0.1,Iris-setosa
+39,4.4,3.0,1.3,0.2,Iris-setosa
+40,5.1,3.4,1.5,0.2,Iris-setosa
+41,5.0,3.5,1.3,0.3,Iris-setosa
+42,4.5,2.3,1.3,0.3,Iris-setosa
+43,4.4,3.2,1.3,0.2,Iris-setosa
+44,5.0,3.5,1.6,0.6,Iris-setosa
+45,5.1,3.8,1.9,0.4,Iris-setosa
+46,4.8,3.0,1.4,0.3,Iris-setosa
+47,5.1,3.8,1.6,0.2,Iris-setosa
+48,4.6,3.2,1.4,0.2,Iris-setosa
+49,5.3,3.7,1.5,0.2,Iris-setosa
+50,5.0,3.3,1.4,0.2,Iris-setosa
+51,7.0,3.2,4.7,1.4,Iris-versicolor
+52,6.4,3.2,4.5,1.5,Iris-versicolor
+53,6.9,3.1,4.9,1.5,Iris-versicolor
+54,5.5,2.3,4.0,1.3,Iris-versicolor
+55,6.5,2.8,4.6,1.5,Iris-versicolor
+56,5.7,2.8,4.5,1.3,Iris-versicolor
+57,6.3,3.3,4.7,1.6,Iris-versicolor
+58,4.9,2.4,3.3,1.0,Iris-versicolor
+59,6.6,2.9,4.6,1.3,Iris-versicolor
+60,5.2,2.7,3.9,1.4,Iris-versicolor
+61,5.0,2.0,3.5,1.0,Iris-versicolor
+62,5.9,3.0,4.2,1.5,Iris-versicolor
+63,6.0,2.2,4.0,1.0,Iris-versicolor
+64,6.1,2.9,4.7,1.4,Iris-versicolor
+65,5.6,2.9,3.6,1.3,Iris-versicolor
+66,6.7,3.1,4.4,1.4,Iris-versicolor
+67,5.6,3.0,4.5,1.5,Iris-versicolor
+68,5.8,2.7,4.1,1.0,Iris-versicolor
+69,6.2,2.2,4.5,1.5,Iris-versicolor
+70,5.6,2.5,3.9,1.1,Iris-versicolor
+71,5.9,3.2,4.8,1.8,Iris-versicolor
+72,6.1,2.8,4.0,1.3,Iris-versicolor
+73,6.3,2.5,4.9,1.5,Iris-versicolor
+74,6.1,2.8,4.7,1.2,Iris-versicolor
+75,6.4,2.9,4.3,1.3,Iris-versicolor
+76,6.6,3.0,4.4,1.4,Iris-versicolor
+77,6.8,2.8,4.8,1.4,Iris-versicolor
+78,6.7,3.0,5.0,1.7,Iris-versicolor
+79,6.0,2.9,4.5,1.5,Iris-versicolor
+80,5.7,2.6,3.5,1.0,Iris-versicolor
+81,5.5,2.4,3.8,1.1,Iris-versicolor
+82,5.5,2.4,3.7,1.0,Iris-versicolor
+83,5.8,2.7,3.9,1.2,Iris-versicolor
+84,6.0,2.7,5.1,1.6,Iris-versicolor
+85,5.4,3.0,4.5,1.5,Iris-versicolor
+86,6.0,3.4,4.5,1.6,Iris-versicolor
+87,6.7,3.1,4.7,1.5,Iris-versicolor
+88,6.3,2.3,4.4,1.3,Iris-versicolor
+89,5.6,3.0,4.1,1.3,Iris-versicolor
+90,5.5,2.5,4.0,1.3,Iris-versicolor
+91,5.5,2.6,4.4,1.2,Iris-versicolor
+92,6.1,3.0,4.6,1.4,Iris-versicolor
+93,5.8,2.6,4.0,1.2,Iris-versicolor
+94,5.0,2.3,3.3,1.0,Iris-versicolor
+95,5.6,2.7,4.2,1.3,Iris-versicolor
+96,5.7,3.0,4.2,1.2,Iris-versicolor
+97,5.7,2.9,4.2,1.3,Iris-versicolor
+98,6.2,2.9,4.3,1.3,Iris-versicolor
+99,5.1,2.5,3.0,1.1,Iris-versicolor
+100,5.7,2.8,4.1,1.3,Iris-versicolor
+101,6.3,3.3,6.0,2.5,Iris-virginica
+102,5.8,2.7,5.1,1.9,Iris-virginica
+103,7.1,3.0,5.9,2.1,Iris-virginica
+104,6.3,2.9,5.6,1.8,Iris-virginica
+105,6.5,3.0,5.8,2.2,Iris-virginica
+106,7.6,3.0,6.6,2.1,Iris-virginica
+107,4.9,2.5,4.5,1.7,Iris-virginica
+108,7.3,2.9,6.3,1.8,Iris-virginica
+109,6.7,2.5,5.8,1.8,Iris-virginica
+110,7.2,3.6,6.1,2.5,Iris-virginica
+111,6.5,3.2,5.1,2.0,Iris-virginica
+112,6.4,2.7,5.3,1.9,Iris-virginica
+113,6.8,3.0,5.5,2.1,Iris-virginica
+114,5.7,2.5,5.0,2.0,Iris-virginica
+115,5.8,2.8,5.1,2.4,Iris-virginica
+116,6.4,3.2,5.3,2.3,Iris-virginica
+117,6.5,3.0,5.5,1.8,Iris-virginica
+118,7.7,3.8,6.7,2.2,Iris-virginica
+119,7.7,2.6,6.9,2.3,Iris-virginica
+120,6.0,2.2,5.0,1.5,Iris-virginica
+121,6.9,3.2,5.7,2.3,Iris-virginica
+122,5.6,2.8,4.9,2.0,Iris-virginica
+123,7.7,2.8,6.7,2.0,Iris-virginica
+124,6.3,2.7,4.9,1.8,Iris-virginica
+125,6.7,3.3,5.7,2.1,Iris-virginica
+126,7.2,3.2,6.0,1.8,Iris-virginica
+127,6.2,2.8,4.8,1.8,Iris-virginica
+128,6.1,3.0,4.9,1.8,Iris-virginica
+129,6.4,2.8,5.6,2.1,Iris-virginica
+130,7.2,3.0,5.8,1.6,Iris-virginica
+131,7.4,2.8,6.1,1.9,Iris-virginica
+132,7.9,3.8,6.4,2.0,Iris-virginica
+133,6.4,2.8,5.6,2.2,Iris-virginica
+134,6.3,2.8,5.1,1.5,Iris-virginica
+135,6.1,2.6,5.6,1.4,Iris-virginica
+136,7.7,3.0,6.1,2.3,Iris-virginica
+137,6.3,3.4,5.6,2.4,Iris-virginica
+138,6.4,3.1,5.5,1.8,Iris-virginica
+139,6.0,3.0,4.8,1.8,Iris-virginica
+140,6.9,3.1,5.4,2.1,Iris-virginica
+141,6.7,3.1,5.6,2.4,Iris-virginica
+142,6.9,3.1,5.1,2.3,Iris-virginica
+143,5.8,2.7,5.1,1.9,Iris-virginica
+144,6.8,3.2,5.9,2.3,Iris-virginica
+145,6.7,3.3,5.7,2.5,Iris-virginica
+146,6.7,3.0,5.2,2.3,Iris-virginica
+147,6.3,2.5,5.0,1.9,Iris-virginica
+148,6.5,3.0,5.2,2.0,Iris-virginica
+149,6.2,3.4,5.4,2.3,Iris-virginica
+150,5.9,3.0,5.1,1.8,Iris-virginica

data/adult-census-income.zip

@@ -0,0 +1,136 @@
+# encoding=utf8
+import numpy as np
+import random
+import matplotlib
+import matplotlib.pyplot as plt
+
+N = 10 #生成训练数据的个数
+
+# AX=0 相当于matlab中 null(a','r')
+def null(a, rtol=1e-5):
+    u, s, v = np.linalg.svd(a)
+    rank = (s > rtol*s[0]).sum()
+    return rank, v[rank:].T.copy()
+
+# 符号函数，之后要进行向量化
+def sign(x):
+    if x > 0:
+        return 1
+    elif x == 0:
+        return 0
+    elif x < 0:
+        return -1
+#noisy=False，那么就会生成N的dim维的线性可分数据X，标签为y
+#noisy=True, 那么生成的数据是线性不可分的,标签为y
+def mk_data(N, noisy=False):
+    rang = [-10,10]
+    dim = 2
+
+    X=np.random.rand(dim,N)*(rang[1]-rang[0])+rang[0]
+
+    while True:
+        Xsample = np.concatenate((np.ones((1,dim)), np.random.rand(dim,dim)*(rang[1]-rang[0])+rang[0]))
+        k,w=null(Xsample.T)
+        y = sign(np.dot(w.T,np.concatenate((np.ones((1,N)), X))))
+        if np.all(y):
+            break
+
+    if noisy == True:
+        idx = random.sample(range(1,N), N/10)
+
+        for id in idx:
+            y[0][id] = -y[0][id]
+
+    return (X,y,w)
+
+def data_visualization(X,y,title):
+    class_1 = [[],[]]
+    class_2 = [[],[]]
+
+    size = len(y)
+
+    for i in xrange(size):
+        X_1 = X[0][i]
+        X_2 = X[1][i]
+
+        if y[i] == 1:
+            class_1[0].append(X_1)
+            class_1[1].append(X_2)
+        else:
+            class_2[0].append(X_1)
+            class_2[1].append(X_2)
+
+
+    plt.figure(figsize=(8, 6), dpi=80)
+    plt.title(title)
+
+    axes = plt.subplot(111)
+
+    type1 = axes.scatter(class_1[0], class_1[1], s=20, c='red')
+    type2 = axes.scatter(class_2[0], class_2[1], s=20, c='green')
+
+
+    plt.show()
+
+def rebuild_features(features):
+    size = len(features[0])
+
+    new_features = []
+    for i in xrange(size):
+        new_features.append([features[0][i],features[1][i]])
+
+    return new_features
+
+def generate_dataset(size, noisy = False, visualization = True):
+    global sign
+    sign = np.vectorize(sign)
+    X,y,w = mk_data(size,False)
+    y = list(y[0])
+
+    if visualization:
+        data_visualization(X,y,'all data')         #数据可视化
+
+    testset_size = int(len(y)*0.333)
+
+    indexes = [i for i in xrange(len(y))]
+    test_indexes = random.sample(indexes,testset_size)
+    train_indexes = list(set(indexes)-set(test_indexes))
+
+    trainset_features = [[],[]]
+    trainset_labels = []
+
+    testset_features = [[],[]]
+    testset_labels = []
+
+    for i in test_indexes:
+        testset_features[0].append(X[0][i])
+        testset_features[1].append(X[1][i])
+        testset_labels.append(y[i])
+
+
+    if visualization:
+        data_visualization(testset_features,testset_labels,'test set')
+
+    for i in train_indexes:
+        trainset_features[0].append(X[0][i])
+        trainset_features[1].append(X[1][i])
+        trainset_labels.append(y[i])
+
+    if visualization:
+        data_visualization(trainset_features,trainset_labels,'train set')
+
+    return rebuild_features(trainset_features),trainset_labels,rebuild_features(testset_features),testset_labels
+
+
+
+if __name__ == '__main__':
+
+    size = 1000
+    generate_dataset(size)
+
+    # generate_dataset
+    # print sign
+    # sign = np.vectorize(sign)
+    # X,y,w = mk_data(size,False)
+    #
+    # data_visualization(X,y)

data/adult-census-income/adult.csv

@@ -0,0 +1,107 @@
+# encoding=utf-8
+# @Author: WenDesi
+# @Date:   08-11-16
+# @Email:  [email protected]
+# @Last modified by:   WenDesi
+# @Last modified time: 08-11-16
+
+import time
+import math
+import random
+
+import pandas as pd
+from sklearn.cross_validation import train_test_split
+from sklearn.metrics import accuracy_score
+
+
+class LogisticRegression(object):
+
+    def __init__(self):
+        self.learning_step = 0.00001
+        self.max_iteration = 5000
+
+    def predict_(self,x):
+        wx = sum([self.w[j] * x[j] for j in xrange(len(self.w))])
+        exp_wx = math.exp(wx)
+
+        predict1 = exp_wx / (1 + exp_wx)
+        predict0 = 1 / (1 + exp_wx)
+
+        if predict1 > predict0:
+            return 1
+        else:
+            return 0
+
+
+    def train(self,features, labels):
+        self.w = [0.0] * (len(features[0]) + 1)
+
+        correct_count = 0
+        time = 0
+
+        while time < self.max_iteration:
+            index = random.randint(0, len(labels) - 1)
+            x = list(features[index])
+            x.append(1.0)
+            y = labels[index]
+
+            if y == self.predict_(x):
+                correct_count += 1
+                if correct_count > self.max_iteration:
+                    break
+                continue
+
+            # print 'iterater times %d' % time
+            time += 1
+            correct_count = 0
+
+            wx = sum([self.w[i] * x[i] for i in xrange(len(self.w))])
+            exp_wx = math.exp(wx)
+
+            for i in xrange(len(self.w)):
+                self.w[i] -= self.learning_step * \
+                    (-y * x[i] + float(x[i] * exp_wx) / float(1 + exp_wx))
+
+
+    def predict(self,features):
+        labels = []
+
+        for feature in features:
+            x = list(feature)
+            x.append(1)
+            labels.append(self.predict_(x))
+
+        return labels
+
+if __name__ == "__main__":
+    print 'Start read data'
+
+    time_1 = time.time()
+
+    raw_data = pd.read_csv('../data/train_binary.csv',header=0)
+    data = raw_data.values
+
+    imgs = data[0::,1::]
+    labels = data[::,0]
+
+
+    # 选取 2/3 数据作为训练集， 1/3 数据作为测试集
+    train_features, test_features, train_labels, test_labels = train_test_split(imgs, labels, test_size=0.33, random_state=23323)
+
+    time_2 = time.time()
+    print 'read data cost ',time_2 - time_1,' second','\n'
+
+    print 'Start training'
+    lr = LogisticRegression()
+    lr.train(train_features, train_labels)
+
+    time_3 = time.time()
+    print 'training cost ',time_3 - time_2,' second','\n'
+
+    print 'Start predicting'
+    test_predict = lr.predict(test_features)
+    time_4 = time.time()
+    print 'predicting cost ',time_4 - time_3,' second','\n'
+
+    score = accuracy_score(test_labels,test_predict)
+    print "The accruacy socre is ", score