用代理池批量检查域名有效性完成。

Author: linjun

.gitignore

@@ -1 +1,2 @@
-.idea
+.idea
+*.pyc

domainChecker.py

@@ -4,20 +4,75 @@
 import sys
 import array
 
+import msvcrt
+
+from utils.proxyManager import ProxyManager
 from utils.utils import *
 from multiprocessing.dummy import Pool as ThreadPool
 
-DOMAIN_NAMES_FILE = "DomainNamesFile.txt"
-DomainCheckResult = "DomainCheckResult.txt"
-
+man=ProxyManager()
+class config:
+        MAX_THREAD_COUNT = 30
+        FailedRetryTimes = 20
+        DomainNamesList = "DomainNamesList.txt"
+        resultFilename = "DomainCheck_result.txt"
 
-def checkDomain(name):
+def checkWithBaiduWhois(name):
+    #http://whois.bj.baidubce.com/whois?format=javascript&domain=whatthesdfksdjf.com
+    domainName = name + ".com"
+    url = r"http://whois.bj.baidubce.com/whois?format=javascript&domain=" + domainName
+    # url=r"https://checkapi.aliyun.com/check/checkdomain?callback=result&domain=azzzz.shop&token=check-web-hichina-com%3A7s88yvbgwterj87mk5llo6k6owagkw4u"
+    return GetUrlContent2(url,proxies=man.popProxies())
+def checkWithBaiduApi(name):
+    # https://cloud.baidu.com/api/bcd/search/status
+    # {"domainNames":[{"label":"sz","tld":"com"}]}
+    domainName = name
+    url = r"https://cloud.baidu.com/api/bcd/search/status"
+
+    header = {
+        'Connection': 'keep-alive',
+        'Content-Length': '44',
+        'Pragma': 'no-cache',
+        'Cache-Control': 'no-cache',
+        'Accept': 'application/json, text/javascript, */*; q=0.01',
+        'Origin': 'https://cloud.baidu.com',
+        'X-Requested-With': 'XMLHttpRequest',
+        'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.94 Safari/537.36',
+        'Content-Type': 'application/json',
+        'Referer': 'https://cloud.baidu.com/product/bcd/search.html?keyword=sz',
+        'Accept-Encoding': 'gzip, deflate, br',
+        'Accept-Language': 'zh-CN,zh;q=0.9,en-US;q=0.8,en;q=0.7',
+        'Cookie': 'BAIDUID=ABDF12D91049CBD5AA0895DAFCA1144D:FG=1; PSTM=1491968808; BIDUPSID=D0C77823B4F11112E550D605C212D8D9; BDUSS=DVpUlk5eUZnWmdkM09WUHE3VmJQNVhCekY2bE9Wd0tRNmFBMnpCR3RKQ202d3RhTVFBQUFBJCQAAAAAAAAAAAEAAADG3h0rY29kZXJsaW4AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAKZe5FmmXuRZW; MCITY=-340%3A; BCLID=10955523308864486684; BDSFRCVID=5G_sJeC62Ra1LkRAhCCwhM3RN2K2PtoTH6aopK_Jf9Dcn3ODE-HlEG0PJU8g0KubVI2-ogKK3gOTH4nP; H_BDCLCKID_SF=tbAjVIPaf-3bfTrTKb55-P_3qxby26nfMmJeaJ5nJD_BSl6q5TJNqJ8Vjf7uJbkjWgcXKqTOQpP-HqTYLp3b2h-ghJoKah5eKjREKl0MLPbYbb0xyn_VKP_bDfnMBMPe52OnaIb8LIF-MK8xejK2en-W5gT0tC62aKDX3buQJlIMqpcNLTDK2Mty2R393CrWBmr32COytn5ZjnTIylO1j4_e3bjw54vmWmO0bRIEtfTbJh5jDh3Ub6ksD-Rte4on-6Ry0hvctb3cShPmhl00Djb-jN0qJ6FsKKJ03bk8KRREJt5kq4bohjn0QnneBtQmJJrN3Cj12MoNjhOJ5P7YDpDND44HQn3yQg-q3R7MWM7keCTe-PI5XU0kqt7Q0x-jLgQPVn0MW-5DSlI4qtnJyUnybPnnBT3XLnQ2QJ8BJDtKMCQP; PSINO=6; H_PS_PSSID=25245_1423_13289_21097_20697_25439_25178_20719; CAMPAIGN_TRACK=cp%3Aaladdin%7Ckw%3A139; CAMPAIGN_TRACK_TIME=2017-12-23+18%3A15%3A11; Hm_lvt_28a17f66627d87f1d046eae152a1c93d=1513070462,1514024132; Hm_lpvt_28a17f66627d87f1d046eae152a1c93d=1514024141',
+    }
+    data = "{\"domainNames\":[{\"label\":\"" + domainName + "\",\"tld\":\"com\"}]}"
+    return PostUrlWithDataAndHeader(url, data, header)
+
+
+def checkWithAliyunApi(name):
     domainName = name + ".com"
-    reprint("checking "+domainName+" ...")
     url = r"https://checkapi.aliyun.com/check/checkdomain?callback=result&domain=" + domainName + r"&token=check-web-hichina-com%3A7s88yvbgwterj87mk5llo6k6owagkw4u"
     # url=r"https://checkapi.aliyun.com/check/checkdomain?callback=result&domain=azzzz.shop&token=check-web-hichina-com%3A7s88yvbgwterj87mk5llo6k6owagkw4u"
-    response = GetUrlContent2(url)
-    appendStrToFile(response + "\n", DomainCheckResult)
+    return GetUrlContent2(url)
+
+
+def checkDomain(name):
+    reprint("checking " + name + " ...")
+    retryCount = 0
+    while True:
+        try:
+            # response= checkWithAliyunApi(name)
+            # response = checkWithBaiduApi(name)
+            response = checkWithBaiduWhois(name)
+            if "\"status\":0" in response:
+                break;
+        except BaseException as e:
+            pass
+        retryCount += 1
+        if retryCount > config.FailedRetryTimes:
+            response = "Failed:" + e.message
+            break
+        reprint("checking %s ... retry #%d"%(name,retryCount))
+    loglToFile("%s\t%d\t--->\t" % (name, retryCount) + response, config.resultFilename)
     # logl(response)
     return "ok"
 
@@ -38,8 +93,8 @@ def strPlusOne(t):
 
 
 def BuildDomainList(fileName):
-    startName = 'aaaaa'
-    endName = 'zzzzz'
+    startName = 'aa'
+    endName = 'zz'
     # nameList = []
     logl("Building name list:%s to %s..." % (startName, endName))
     while startName != endName:
@@ -61,17 +116,42 @@ def LoadDomainList(fileName):
 
 
 def CheckAllDomains(nameList):
-    pool = ThreadPool()
+    pool = ThreadPool(config.MAX_THREAD_COUNT)
+    logl(pool._processes)
     pool.map(checkDomain, nameList)
     pool.close()
     pool.join()
     logl('All done.')
 
 
+def usage():
+    logl("usage:%s DOMAIN_NAMES_FILE")
+
+
+def raise_test():
+    raise BaseException("test")
+
+
+def test():
+    try:
+        raise_test()
+    except BaseException as a:
+        logl("Exception:" + a.message)
+
+
 def main():
-    logl("Loading domain names from %s .." % DOMAIN_NAMES_FILE)
-    nameList = LoadDomainList(DOMAIN_NAMES_FILE)
-    logl("%d names loaded."%len(nameList))
+    # test()
+    # return
+    logl("test end.")
+    if len(sys.argv) == 2:
+        config.DomainNamesListFile = sys.argv[1]
+        config.resultFilename = "result_" + config.DomainNamesListFile
+    else:
+        usage()
+        return
+    logl("Loading domain names from %s .." % config.DomainNamesListFile)
+    nameList = LoadDomainList(config.DomainNamesListFile)
+    logl("%d names loaded." % len(nameList))
     logl("Checking...")
     CheckAllDomains(nameList)
     logl("All done.")
@@ -80,4 +160,6 @@ def main():
 if __name__ == '__main__':
     # BuildDomainList(DOMAIN_NAMES_FILE)
     main()
-    # checkDomain("aaaaa")
+    logl("<<")
+    msvcrt.getch()
+    # checkDomain("as")

utils/proxyManager.py

@@ -0,0 +1,91 @@
+# -*- coding: utf-8 -*-
+import json
+import threading
+from utils import *
+
+SingletonLock = threading.Lock()
+
+
+class Singleton(object):
+    # 定义静态变量实例
+    __instance = None
+
+    def __init__(self):
+        pass
+
+    def __new__(cls, *args, **kwargs):
+        if not cls.__instance:
+            try:
+                SingletonLock.acquire()
+                # double check
+                if not cls.__instance:
+                    cls.__instance = super(Singleton, cls).__new__(cls, *args, **kwargs)
+            finally:
+                SingletonLock.release()
+        return cls.__instance
+
+
+class ProxyManager(Singleton):
+    __first_init = True
+
+    def __init__(self):
+        if self.__first_init:
+            self.__first_init = False
+            self.proxyLock = threading.Lock()
+            self.proxies = []
+            self.proxy_pool_url = "http://127.0.0.1:8000"
+            self.refresh()
+            logl("init..")
+            self.count = 0
+
+    def refresh(self):
+        response = GetUrlContent2(self.proxy_pool_url)
+        self.proxies = json.loads(response)
+
+    def pop(self):
+        proxy = None
+        try:
+            self.proxyLock.acquire()
+            if len(self.proxies) < 1:
+                self.refresh()
+            proxy = self.proxies.pop(0)
+            self.proxies.append(proxy)
+            # self.count += 1
+            # logl(self.count)
+
+        finally:
+            self.proxyLock.release()
+        return proxy
+
+    def popProxies(self):
+        proxy = self.pop()
+        proxies = {"http": "http://%s:%s" % (proxy[0], proxy[1]), "https": "http://%s:%s" % (proxy[0], proxy[1]), }
+        return proxies
+
+
+man = ProxyManager()
+
+
+def test_singleton_in_thread():
+    # logl(id(man))
+    tp = man.popProxies()
+    logl(str(tp))
+    try:
+        response = GetUrlContent2("http://pv.sohu.com/cityjson", proxies=tp)
+    except BaseException as e:
+        logl(e.message)
+        response = str(e.message)
+    logl("proxies:%s result:%s" % (str(tp), response))
+
+
+if __name__ == "__main__":
+    man = ProxyManager()
+    threadList = []
+    for i in range(100):
+        t = threading.Thread(target=test_singleton_in_thread, args=[])
+        t.setDaemon(True)
+        t.start()
+        threadList.append(t)
+    for th in threadList:
+        th.join()
+    logl("all threads end.")

utils/utils.py

@@ -8,6 +8,8 @@
 
 import chardet
 import sys
+reload(sys)
+sys.setdefaultencoding('utf8')
 
 import requests
 from requests import ConnectionError
@@ -49,6 +51,7 @@
     "Mozilla/5.0 (X11; U; Linux x86_64; zh-CN; rv:1.9.2.10) Gecko/20100922 Ubuntu/10.10 (maverick) Firefox/3.6.10"
 ]
 
+
 def appendStrToFile(sstr, filepath):
     f = open(filepath, "a+")
     try:
@@ -62,7 +65,7 @@ def appendStrToFile(sstr, filepath):
 def logToFile(msg, filepath):
     prefix = "[" + time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) + "]"
     logtext = prefix + "\t" + msg
-    appendStrToFile(msg, filepath)
+    appendStrToFile(logtext, filepath)
 
 
 def loglToFile(msg, filepath):
@@ -97,13 +100,10 @@ def logldebug(msg):
 
 
 def logl(msg):
-    msg=str(msg)
+    msg = str(msg)
     log("[.] " + msg + "\n")
 
 
-
-
-
 def GetEncoding(data):
     chardit1 = chardet.detect(data)
     encoding = chardit1['encoding']
@@ -201,27 +201,31 @@ def writeFile(filename, data):
     f.write(data)
     f.close()
 
+
 def getFileMd5(filepath):
     return DoCmd("md5sum " + filepath + "|grep -oE '[a-zA-Z0-9]{32}'").strip()
     pass
 
+
 def reprint(msg):
-    sys.stdout.write("\r" + msg )
+    sys.stdout.write("\r" + msg)
 
 
-def urlencode( val):
+def urlencode(val):
     if isinstance(val, unicode):
         return urllib.quote(str(val), safe='/:?=')
     return urllib.quote(val, safe='/:?=')
 
-def GetUrlContent( url):
+
+def GetUrlContent(url):
     request = urllib2.Request(urlencode(url))
     response = urllib2.urlopen(request)
     page = response.read()
     logl(page)
     encoding = GetEncoding(page)
     return page.decode(encoding)
 
+
 def get_header():
     return {
         'User-Agent': random.choice(USER_AGENTS),
@@ -231,12 +235,37 @@ def get_header():
         'Accept-Encoding': 'gzip, deflate',
     }
 
+
 TIMEOUT = 5
-def GetUrlContent2(url):
-    r = requests.get(url=url, headers=get_header(), timeout=TIMEOUT)
-    r.encoding = chardet.detect(r.content)['encoding']
+
+
+def GetUrlContent2(url,proxies=None):
+    try:
+        if proxies is None:
+            r = requests.get(url=url, headers=get_header(), timeout=TIMEOUT)
+        else:
+            r = requests.get(url=url, headers=get_header(), timeout=TIMEOUT,proxies=proxies)
+        r.encoding = chardet.detect(r.content)['encoding']
+    except Exception as e:
+        raise BaseException("#ConnectionException:"+str(e.message))
+    if not r.ok:
+        # raise ConnectionError
+        raise BaseException("#ConnectionError:"+str(r.reason))
+    else:
+        return r.text
+
+
+def PostUrlWithDataAndHeader(url, data, header,proxies=None):
+    try:
+        if proxies is None:
+            r = requests.post(url=url, data=data, headers=header, timeout=TIMEOUT)
+        else:
+            r = requests.post(url=url, data=data, headers=header, timeout=TIMEOUT,proxies=proxies)
+        r.encoding = chardet.detect(r.content)['encoding']
+    except Exception as e:
+        raise BaseException("#ConnectionException:"+e.message)
     if not r.ok:
-        #raise ConnectionError
-        return "#ConnectionError"
+        # raise ConnectionError
+        raise BaseException("#ConnectionError"+str(r.reason))
     else:
-        return r.text
+        return r.text