Finished Week 4 (Parsing HTML with BeautifulSoup)

Author: jeremy-wl

Week 4 Programs that Surf the Web/Assignments/solution.py

@@ -0,0 +1,18 @@
+import urllib.request as ur
+from bs4 import *
+
+url = input('Enter the url to scrape - ')
+
+html = ur.urlopen(url).read()
+soup = BeautifulSoup(html, 'html.parser')
+
+count_of_spans = 0
+sum = 0
+
+spans = soup('span')
+for span in spans:
+    sum += int(span.contents[0])
+    count_of_spans += 1
+
+print('Count ', count_of_spans)
+print('Sum ', sum)

Week 4 Programs that Surf the Web/Assignments/solution2.py

@@ -0,0 +1,27 @@
+import urllib.request as ur
+from bs4 import *
+
+current_repeat_count = 0
+url = input('Enter URL: ')
+repeat_count = int(input('Enter count: '))
+position = int(input('Enter position: '))
+
+
+def parse_html(url):
+    html = ur.urlopen(url).read()
+    soup = BeautifulSoup(html, 'html.parser')
+    tags = soup('a')
+    return tags
+
+while current_repeat_count < repeat_count:
+    print('Retrieving: ', url)
+    tags = parse_html(url)
+    for index, item in enumerate(tags):
+        if index == position - 1:
+            url = item.get('href', None)
+            name = item.contents[0]
+            break
+        else:
+            continue
+    current_repeat_count += 1
+print('Last Url: ', url)

Week 4 Programs that Surf the Web/bs.py

@@ -0,0 +1,16 @@
+import urllib.request as ur
+from bs4 import *
+
+url = input('Enter the url to scrape - ')
+# http://www.dr-chuck.com
+
+html = ur.urlopen(url).read()
+soup = BeautifulSoup(html, 'html.parser')
+# wrapping the whole HTML into a single soup object
+
+tags = soup('a')
+# extracts all 'a' tag from the HTML object
+
+for tag in tags:
+    print(tag.get('href'), None)
+    # each tag is returned as a dictionary of its attributes