adaboost with threshold version complete

Author: WenDesi

.idea/workspace.xml

@@ -6,6 +6,7 @@
 # @Last modified time: 15-11-16
 
 import math
+import logging
 
 class Sign(object):
     def __init__(self,features,labels,w):
@@ -15,15 +16,19 @@ def __init__(self,features,labels,w):
 
         self.w = w
 
+        mmax = max(self.X)
+        self.indexes = self.X[:]
+        self.indexes.append(mmax+1)
+
     def _train_less_than_(self):
         index = -1
         error_score = 1000000
 
-        for i in xrange(self.N+1):
+        for i in self.indexes:
             score = 0
             for j in xrange(self.N):
                 val = -1
-                if j<i:
+                if self.X[j]<i:
                     val = 1
 
                 if val*self.Y[j]<0:
@@ -41,11 +46,11 @@ def _train_more_than_(self):
         index = -1
         error_score = 1000000
 
-        for i in xrange(self.N+1):
+        for i in self.indexes:
             score = 0
             for j in xrange(self.N):
                 val = 1
-                if j<i:
+                if self.X[j]<i:
                     val = -1
 
                 if val*self.Y[j]<0:
@@ -95,7 +100,7 @@ def _init_parameters_(self,features,labels):
 
         self.n = len(features[0])
         self.N = len(features)
-        self.M = 100                            # 分类器数目
+        self.M = 10000                            # 分类器数目
 
         self.w = [1.0/self.N]*self.N
         self.alpha = []
@@ -117,6 +122,7 @@ def train(self,features,labels):
         self._init_parameters_(features,labels)
 
         for times in xrange(self.M):
+            logging.debug('iterater %d' % times)
 
             best_classifier = (100000,None,None)        #(误差率,分类器,针对的特征)
             for i in xrange(self.n):
@@ -125,17 +131,44 @@ def train(self,features,labels):
                 error_score = classifier.train()
 
                 if error_score < best_classifier[0]:
-                    best_classifier = (error_score,classifier,i)
+                    best_classifier = (error_score,i,classifier)
 
             em = best_classifier[0]
-            self.alpha.append(1/2*math.log((1-em)/em))
+            if em==0:
+                self.alpha.append(100)
+            else:
+                self.alpha.append(0.5*math.log((1-em)/em))
+
             self.classifier.append(best_classifier[1:])
 
             Z = self._Z_(best_classifier[1],best_classifier[2])
 
             for i in xrange(self.N):
                 self.w[i] = self._w_(best_classifier[1],best_classifier[2],i)/Z
 
+    def _predict_(self,feature):
+
+        result = 0.0
+        for i in xrange(self.M):
+            index = self.classifier[i][0]
+            classifier = self.classifier[i][1]
+
+            result += self.alpha[i]*classifier.predict(feature[index])
+
+        if result>0:
+            return 1
+        return -1
+
+
+
+    def predict(self,features):
+        results = []
+
+        for feature in features:
+            results.append(self._predict_(feature))
+
+        return results
+
 if __name__ == '__main__':
     features = [[0],[1],[2],[3],[4],[5],[6],[7],[8],[9]]
     labels = [1,1,1,-1,-1,-1,1,1,1,-1]

AdaBoost/adaboost.py

@@ -4,3 +4,26 @@
 # @Email:  [email protected]
 # @Last modified by:   wendesi
 # @Last modified time: 15-11-16
+
+import logging
+
+from generate_dataset import *
+from adaboost import AdaBoost
+
+from sklearn.metrics import accuracy_score
+
+if __name__ == '__main__':
+    logger = logging.getLogger()
+    logger.setLevel(logging.DEBUG)
+
+    train_features, train_labels, test_features, test_labels = generate_dataset(200)
+
+    ada = AdaBoost()
+    ada.train(train_features,train_labels)
+
+    print 'end train'
+    test_predict = ada.predict(test_features)
+
+
+    score = accuracy_score(test_labels,test_predict)
+    print "ada boost the accruacy socre is ", score

AdaBoost/adaboost.pyc

@@ -134,3 +134,104 @@ def generate_dataset(size, noisy = False, visualization = True):
     # X,y,w = mk_data(size,False)
     #
     # data_visualization(X,y)
+# encoding=utf8
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+# #encoding=utf-8
+# # @Author: wendesi
+# # @Date:   15-11-16
+# # @Email:  [email protected]
+# # @Last modified by:   wendesi
+# # @Last modified time: 15-11-16
+#
+#
+#
+#
+# import random
+#
+# def generate(size,point,radius,label):
+#     results = []
+#
+#     for i in xrange(size):
+#         result = [label]
+#
+#         for j in xrange(len(point)):
+#             x = point[j]+random.randint(-radius,radius)
+#             result.append(x)
+#
+#         results.append(result)
+#
+#     return results
+#
+#
+# def generate_dataset(size):
+#     class1 = generate(size/2,(0,0),10,-1)
+#     class2 = generate(size-size/2,(21,21),10,1)
+#
+#     class_ = class1
+#     class_.extend(class2)
+#
+#     random.shuffle(class_)
+#
+#     split_point = int(float(size)*0.333)
+#     testset = class_[:split_point]
+#     trainset = class_[split_point:]
+#
+#     trainset_features = map(lambda x:x[1:],trainset)
+#     trainset_labels = map(lambda x:x[0],trainset)
+#
+#     testset_features = map(lambda x:x[1:],testset)
+#     testset_labels = map(lambda x:x[0],testset)
+#
+#     return trainset_features,trainset_labels,testset_features,testset_labels
+#
+#
+# if __name__ == '__main__':
+#
+#     size = 1000
+#     generate_dataset(size)
+#
+#     # generate_dataset
+#     # print sign
+#     # sign = np.vectorize(sign)
+#     # X,y,w = mk_data(size,False)
+#     #
+#     # data_visualization(X,y)

AdaBoost/adaboost_fakedata.py

@@ -134,3 +134,4 @@ def generate_dataset(size, noisy = False, visualization = True):
     # X,y,w = mk_data(size,False)
     #
     # data_visualization(X,y)
+# encoding=utf8