Created Printer class for color printing

Author: WilHall

PyCrawler.py

@@ -1,7 +1,8 @@
 from query import CrawlerDb
 from content_processor import ContentProcessor
-from settings import VERBOSE
+from settings import VERBOSE, COLOR_ERROR, COLOR_SUCCESS
 import sys, urlparse, urllib2
+import cPrinter
 
 # ===== Init stuff =====
 
@@ -12,50 +13,44 @@
 # content processor init
 processor = ContentProcessor(None, None, None)
 
+# get cprinter
+printer = cPrinter.Printer(COLOR_SUCCESS, COLOR_ERROR)
+
 if len(sys.argv) < 2:
-	print "Error: No start url was passed"
+	printer.p("Error: No start url was passed", printer.error)
 	sys.exit()
 
 l = sys.argv[1:]
 
 cdb.enqueue(l)
 
 def crawl():
-	print "starting..."
+	printer.p("starting...", printer.success)
 	queue_empty = False
 	while True:
 		url = cdb.dequeue()
-		print url
 		if cdb.checkCrawled(url):
 			continue
 		if url is False:
 			queue_empty = True
-
-		# Get HTTPConnection
-		#connection = httplib.HTTPConnection(parsed_url.netloc)
-		# Make the request
-		#connection.request("GET", parsed_url.path)
-		# Get response
-		#response = connection.getresponse()
-		#data = response.read()
 		status = 0
 		request = None
 		try:
 			request = urllib2.urlopen(str(url))
 		except urllib2.URLError, e:
-			print e.reason
+			printer.p(e.reason, printer.error)
 		except urllib2.HTTPError, e:
 			status = e.code
 		if status == 0:
 			status = 200
 		data = request.read()
 
-		if VERBOSE:
-			print "Got %s status from %s" % (status, url)
 		processor.setInfo(str(url), status, data)
 		add_queue = processor.process()
 		l = len(add_queue)
-		print "Found %i links" % l
+		if VERBOSE:
+			printer.p("Got %s status from %s" % (status, url), printer.success)
+			printer.p("Found %i links" % l, printer.success)
 		if l > 0:
 			if queue_empty == True:
 				queue_empty = False
@@ -65,9 +60,9 @@ def crawl():
 		if queue_empty:
 			break
 
-	print "finishing..."
+	printer.p("finishing...", printer.success)
 	cdb.close()
-	print "done! goodbye!"
+	printer.p("done! goodbye!", printer.success)
 
 if __name__ == "__main__":
 	crawl()

cPrinter.py

@@ -0,0 +1,34 @@
+import curses
+
+class Printer():
+
+	def __init__(self, COLOR_SUCCESS, COLOR_ERROR):
+		# Define our types
+		self.success = 0;
+		self.error = 1;
+
+		# Initialize environment
+		curses.setupterm()
+
+		# Get the foreground color attribute for this environment
+		self.fcap = curses.tigetstr('setaf')
+
+		#Get the normal attribute
+		self.COLOR_NORMAL = curses.tigetstr('sgr0')
+
+		# Initialize custom colors to the first two slots
+		curses.initscr()
+		curses.start_color()
+		curses.init_color(0, COLOR_SUCCESS[0], COLOR_SUCCESS[1], COLOR_SUCCESS[2])
+		curses.init_color(1, COLOR_ERROR[0], COLOR_ERROR[1], COLOR_ERROR[2])
+		curses.endwin()
+
+		# Get + Save the color sequences
+		self.COLOR_SUCCESS = curses.tparm(self.fcap, 0)
+		self.COLOR_ERROR = curses.tparm(self.fcap, 1)
+
+	def p(self, text, type):
+		if type == self.success:
+			print "%s%s%s" % (self.COLOR_SUCCESS, text, self.COLOR_NORMAL)
+		elif type == self.error:
+			print "%s%s%s" % (self.COLOR_SUCCESS, text, self.COLOR_NORMAL)

content_processor.py

@@ -12,11 +12,9 @@ def rankKeywords(text):
 		if t in invalid_keywords:
 			continue
 		if not ranks.has_key(t):
-			print "adding %s" % t
 			ranks[t] = 1
 		else:
 			ranks[t] += 1
-			print "setting %s to %i" % (t, ranks[t])
 	return ranks
 
 def stripPunctuation(text):
@@ -67,38 +65,28 @@ def combineKeywordLists(self):
 			for k,v in l.items():
 				if self.keywords.has_key(k):
 					self.keywords[k] += v
-					print "setting %s to %i" %(k,self.keywords[k])
 				else:
 					self.keywords[k] = v
-					print "setting %s to %i" %(k,v)
 
 	# returns links to queue	
 	def processBody(self):
 		queue = ready_queue(self.url, self.body)
-		print "found %i links to queue" % len(queue)
 		self.text = stripPunctuation(self.remove_html_tags(self.body))
 		if len(self.text) > 5000:
 			offset = 0
 			i = 0
 			l = []
-			print "splitting text"
 			while True:
 				j = self.findnth(self.text[i:],' ',500)
 				offset += j
-				print "SPLIT: 500th space at %i" % j
 				if j == -1:
-					print "appending from %i on" % i
-					l.append(self.text[i:])
 					break
-				print "appending from %i to %i" % (i,j)
 				l.append(self.text[i:j])
 				i = offset + j+1
-			print "processing with %i threads" % len(l)
 			pool = Pool(processes=(len(l)))
 			self.keyword_dicts = pool.map(rankKeywords, l)
-			print "processed, returned %i dicts" % len(self.keyword_dicts)
 		else:
-			self.keyword_dicts.append(self.rankKeywords(self.text))
+			self.keyword_dicts.append(rankKeywords(self.text))
 		return queue
 
 	def processHead(self):
@@ -117,18 +105,11 @@ def findnth(self, haystack, needle, n):
 	# returns the queue from processBody
 	def process(self):
 		text_lower = self.text.lower()
-		print "Finding title"
 		self.title = self.text[text_lower.find('<title')+6:text_lower.find('</title>')]
-		print "Found title: %s" % self.title
-		print "Finding head"
 		self.head = self.text[text_lower.find('<head')+5:text_lower.find('</head>')]
-		print "Found head of length %i" % len(self.head)
 		self.processHead()
-		print "Finding body"
 		self.body = self.text[text_lower.find('<body'):text_lower.find('</body>')]
-		print "Found body of length %i" % len(self.body)
 		queue = self.processBody()
-		print "combining keyword lists"
 		self.combineKeywordLists()
 		return queue
 

query.py

@@ -77,7 +77,6 @@ def dequeue(self):
 			if not delres:
 				return False
 			# Return the row
-			print result
 			return result[0][1]
 		return False
 

settings.py

@@ -8,3 +8,9 @@
 DATABASE_PASS = ""				# Not used with sqlite
 
 VERBOSE = True
+
+# These values are for the text output colors.
+# List values are 0-255 RGB values, respectively.
+
+COLOR_SUCCESS = [0, 255, 0]		# Success Color (Green)
+COLOR_ERROR = [255, 0, 0]		# Error Color (Red)