added pyspark UDF example

mahmoudparsian · Jan 1, 2022 · c04f3b5 · c04f3b5
1 parent 6a2d4ab
commit c04f3b5
Show file tree

Hide file tree

Showing 32 changed files with 57 additions and 0 deletions.
diff --git a/tutorial/.DS_Store b/tutorial/.DS_Store
diff --git a/tutorial/add-indices/add-indices.txt b/tutorial/add-indices/add-indices.txt
diff --git a/tutorial/basic-average/basic-average.txt b/tutorial/basic-average/basic-average.txt
diff --git a/tutorial/basic-filter/basic-filter.txt b/tutorial/basic-filter/basic-filter.txt
diff --git a/tutorial/basic-join/basicjoin.txt b/tutorial/basic-join/basicjoin.txt
diff --git a/tutorial/basic-map/basic-map.txt b/tutorial/basic-map/basic-map.txt
diff --git a/tutorial/basic-multiply/basic-multiply.txt b/tutorial/basic-multiply/basic-multiply.txt
diff --git a/tutorial/basic-sort/sort-by-key.txt b/tutorial/basic-sort/sort-by-key.txt
diff --git a/tutorial/basic-sum/basic-sum.txt b/tutorial/basic-sum/basic-sum.txt
diff --git a/tutorial/basic-union/basic-union.txt b/tutorial/basic-union/basic-union.txt
diff --git a/tutorial/bigrams/bigrams.txt b/tutorial/bigrams/bigrams.txt
diff --git a/tutorial/cartesian/cartesian.txt b/tutorial/cartesian/cartesian.txt
diff --git a/tutorial/combine-by-key/README.md b/tutorial/combine-by-key/README.md
diff --git a/tutorial/combine-by-key/combine-by-key.txt b/tutorial/combine-by-key/combine-by-key.txt
diff --git a/tutorial/combine-by-key/distributed_computing_with_spark_by_Javier_Santos_Paniego.pdf b/tutorial/combine-by-key/distributed_computing_with_spark_by_Javier_Santos_Paniego.pdf
diff --git a/tutorial/combine-by-key/spark-combineByKey.md b/tutorial/combine-by-key/spark-combineByKey.md
diff --git a/tutorial/combine-by-key/spark-combineByKey.txt b/tutorial/combine-by-key/spark-combineByKey.txt
diff --git a/tutorial/combine-by-key/standard_deviation_by_combineByKey.md b/tutorial/combine-by-key/standard_deviation_by_combineByKey.md
diff --git a/tutorial/dna-basecount/README.md b/tutorial/dna-basecount/README.md
diff --git a/tutorial/dna-basecount/dna-basecount.md b/tutorial/dna-basecount/dna-basecount.md
diff --git a/tutorial/dna-basecount/dna-basecount2.md b/tutorial/dna-basecount/dna-basecount2.md
diff --git a/tutorial/dna-basecount/dna-basecount3.md b/tutorial/dna-basecount/dna-basecount3.md
diff --git a/tutorial/dna-basecount/dna_seq.txt b/tutorial/dna-basecount/dna_seq.txt
diff --git a/tutorial/map-partitions/README.md b/tutorial/map-partitions/README.md
diff --git a/tutorial/pyspark-udf/pyspark_udf_maptype.txt b/tutorial/pyspark-udf/pyspark_udf_maptype.txt
@@ -0,0 +1,57 @@
+$SPARK_HOME/bin/pyspark
+Python 3.8.9 (default, Nov  9 2021, 04:26:29)
+Welcome to
+      ____              __
+     / __/__  ___ _____/ /__
+    _\ \/ _ \/ _ `/ __/  '_/
+   /__ / .__/\_,_/_/ /_/\_\   version 3.2.0
+      /_/
+
+Using Python version 3.8.9 (default, Nov  9 2021 04:26:29)
+Spark context Web UI available at http://10.0.0.232:4040
+Spark context available as 'sc' (master = local[*], app id = local-1641011178190).
+SparkSession available as 'spark'.
+
+>>> from pyspark.sql import Row
+
+>>> data = spark.createDataFrame(
+... [Row(zip_code='94087', city='Sunnyvale'),
+...  Row(zip_code='94088', city='Cupertino'),
+...  Row(zip_code='95055', city='Santa Clara'),
+...  Row(zip_code='95054', city='Palo Alto')])
+
+>>>
+>>> data.show()
++--------+-----------+
+|zip_code|       city|
++--------+-----------+
+|   94087|  Sunnyvale|
+|   94088|  Cupertino|
+|   95055|Santa Clara|
+|   95054|  Palo Alto|
++--------+-----------+
+
+>>> from pyspark.sql.functions import udf
+>>> from pyspark.sql import types as T
+>>>
+>>> @udf(T.MapType(T.StringType(), T.StringType()))
+... def create_structure(zip_code, city):
+...     return {zip_code: city}
+...
+>>> data.withColumn('structure', create_structure(data.zip_code, data.city)).toJSON().collect()
+[
+ '{"zip_code":"94087","city":"Sunnyvale","structure":{"94087":"Sunnyvale"}}', 
+ '{"zip_code":"94088","city":"Cupertino","structure":{"94088":"Cupertino"}}', 
+ '{"zip_code":"95055","city":"Santa Clara","structure":{"95055":"Santa Clara"}}', 
+ '{"zip_code":"95054","city":"Palo Alto","structure":{"95054":"Palo Alto"}}'
+]
+
+>>> data.withColumn('structure', create_structure(data.zip_code, data.city)).show(truncate=False)
++--------+-----------+----------------------+
+|zip_code|city       |structure             |
++--------+-----------+----------------------+
+|94087   |Sunnyvale  |{94087 -> Sunnyvale}  |
+|94088   |Cupertino  |{94088 -> Cupertino}  |
+|95055   |Santa Clara|{95055 -> Santa Clara}|
+|95054   |Palo Alto  |{95054 -> Palo Alto}  |
++--------+-----------+----------------------+
diff --git a/tutorial/split-function/README.md b/tutorial/split-function/README.md
diff --git a/tutorial/top-N/top-N.txt b/tutorial/top-N/top-N.txt
diff --git a/tutorial/wordcount/README.md b/tutorial/wordcount/README.md
diff --git a/tutorial/wordcount/word_count.py b/tutorial/wordcount/word_count.py
diff --git a/tutorial/wordcount/word_count_ver2.py b/tutorial/wordcount/word_count_ver2.py
diff --git a/tutorial/wordcount/wordcount-shorthand.txt b/tutorial/wordcount/wordcount-shorthand.txt
diff --git a/tutorial/wordcount/wordcount.txt b/tutorial/wordcount/wordcount.txt