adopt

Author: James Lee

rdd/WordCount.py

@@ -0,0 +1,10 @@
+import sys
+from pyspark import SparkContext
+
+if __name__ == "__main__":
+	sc = SparkContext("local", "word count")
+  	lines = sc.textFile("in/word_count.text")
+  	words = lines.flatMap(lambda line: line.split(" "))
+  	wordCounts = words.countByValue()
+  	for word, count in wordCounts.items():
+  		print(word, count)

rdd/WordCount.scala

@@ -0,0 +1,11 @@
+
+import sys
+from pyspark import SparkContext
+
+if __name__ == "__main__":
+
+	'''
+    Create a Spark program to read the first 100 prime numbers from in/prime_nums.text,
+    print the sum of those numbers to console.
+    Each row of the input file contains 10 prime numbers separated by spaces.
+    '''

rdd/sumOfNumbers/SumOfNumbersProblem.py

@@ -0,0 +1,11 @@
+import sys
+from pyspark import SparkContext
+
+if __name__ == "__main__":
+    sc = SparkContext("local", "primeNumbers")
+    lines = sc.textFile("in/prime_nums.text")
+    numbers = lines.flatMap(lambda line: line.split("\t"))
+    validNumbers = numbers.filter(lambda number: number)
+    intNumbers = validNumbers.map(lambda number: int(number))
+    print("Sum is: ")
+    print(intNumbers.reduce(lambda x, y: x + y))

rdd/sumOfNumbers/SumOfNumbersProblem.scala

@@ -0,0 +1,10 @@
+import sys
+from pyspark import SparkContext
+
+if __name__ == "__main__":
+	sc = SparkContext("local", "take")
+	inputWords = ["spark", "hadoop", "spark", "hive", "pig", "cassandra", "hadoop"]
+	wordRdd = sc.parallelize(inputWords)
+	words = wordRdd.take(3)
+	for word in words: 
+		print(word)