Added sparkSql/join/UkMakerSpaces.py

Author: Pedro Bernardo

sparkSql/join/UkMakerSpaces.py

@@ -0,0 +1,27 @@
+from pyspark.sql import SparkSession, functions as fs
+
+if __name__ == "__main__":
+
+    session = SparkSession.builder.appName("UkMakerSpaces").master("local").getOrCreate()
+    sc = session.sparkContext
+    sc.setLogLevel("ERROR")
+
+    makerSpace = session.read \
+        .option("header", "true") \
+        .csv("in/uk-makerspaces-identifiable-data.csv")
+
+    postCode = session.read \
+        .option("header", "true") \
+        .csv("in/uk-postcode.csv") \
+        .withColumn("PostCode", fs.concat_ws("", fs.col("PostCode"), fs.lit(" ")))
+
+    print("=== Print 20 records of makerspace table ===")
+    makerSpace.select("Name of makerspace", "Postcode").show()
+
+    print("=== Print 20 records of postcode table ===")
+    postCode.show()
+
+    joined = makerSpace.join(postCode, makerSpace["Postcode"].startswith(postCode["Postcode"]), "left_outer")
+
+    print("=== Group by Region ===")
+    joined.groupBy("Region").count().show(200)