using rdds to achieve computation

.gitignore

@@ -0,0 +1,3 @@
+# created by virtualenv automatically
+env
+output

main.py

@@ -1,7 +1,17 @@
 from pyspark.sql import SparkSession
+from pyspark.sql.functions import col, explode, from_json
+from pyspark.sql.types import ArrayType, StringType, StructField, StructType
 
 spark = SparkSession.builder.appName("FHIR conversoin").getOrCreate()
 
-df = spark.read.option("multiline", "true").json("./data")
+data = spark.read.option("multiline", "true").json("./data").rdd
 
-df.show(10)
+mappedLines = data.map(lambda x: x.asDict())
+
+entryLines = mappedLines.map(lambda x: x["entry"])
+
+flattendEntries = entryLines.flatMap(lambda x: x)
+
+requiredData = flattendEntries.map(lambda x: (x["fullUrl"], x["resource"]["id"]))
+
+requiredData.saveAsTextFile("./output")