+{"cells":[{"cell_type":"markdown","source":["# PySpark Code to Move Data from Bronze to Silver"],"metadata":{"nteract":{"transient":{"deleting":false}},"microsoft":{"language":"python","language_group":"synapse_pyspark"}},"id":"ab7f0b7c-b0cc-44ec-9948-61d68f4b0b13"},{"cell_type":"markdown","source":["This is an example on how to work with the medallion architecture. From Bronze to Silver"],"metadata":{"nteract":{"transient":{"deleting":false}},"microsoft":{"language":"python","language_group":"synapse_pyspark"}},"id":"44f47922-4e3b-45cc-81a6-c5de97634f73"},{"cell_type":"markdown","source":["## Working with 2020orders information"],"metadata":{"nteract":{"transient":{"deleting":false}},"microsoft":{"language":"python","language_group":"synapse_pyspark"}},"id":"4e1afae8-b2ef-4e4c-9ac9-485139f19e9c"},{"cell_type":"code","source":["from pyspark.sql.types import *\n","import pyspark.sql.functions \n","from pyspark.sql import *"],"outputs":[],"execution_count":null,"metadata":{"microsoft":{"language":"python","language_group":"synapse_pyspark"}},"id":"febb6c3e-6841-42c1-a633-0da056b7f69c"},{"cell_type":"code","source":["# Read the data from the bronze layer:\n","df_raw_2020orders = spark.read.format(\"delta\").load(\"abfss://fabric_medallion_arch_demo@onelake.dfs.fabric.microsoft.com/raw_Bronze.Lakehouse/Tables/2020orders\")\n","\n","df_raw_2020orders.head(2)"],"outputs":[{"output_type":"display_data","data":{"application/vnd.livy.statement-meta+json":{"spark_pool":null,"statement_id":4,"statement_ids":[4],"state":"finished","livy_statement_state":"available","session_id":"ecb846c9-e988-4906-95bc-af67b3aacd94","normalized_state":"finished","queued_time":"2024-10-25T19:11:30.8930212Z","session_start_time":null,"execution_start_time":"2024-10-25T19:14:18.2840528Z","execution_finish_time":"2024-10-25T19:14:24.7554027Z","parent_msg_id":"4d60fa03-18dd-4bbc-adab-cb12ff2ba6cf"},"text/plain":"StatementMeta(, ecb846c9-e988-4906-95bc-af67b3aacd94, 4, Finished, Available, Finished)"},"metadata":{}},{"output_type":"execute_result","execution_count":11,"data":{"text/plain":"[Row(ID='SO45376', Count=1, Date='1/5/2020', Name='Edgar Mehta', Style='Mountain-100 Silver, 38', price=3399.99, tax=271.9992),\n Row(ID='SO45381', Count=1, Date='1/6/2020', Name='Jordan Long', Style='Mountain-100 Silver, 38', price=3399.99, tax=271.9992)]"},"metadata":{}}],"execution_count":2,"metadata":{"jupyter":{"source_hidden":false,"outputs_hidden":false},"nteract":{"transient":{"deleting":false}},"microsoft":{"language":"python","language_group":"synapse_pyspark"}},"id":"6357dc47-1bb1-4391-8f38-b5d5a2abf5b2"},{"cell_type":"code","source":["# Clean the data (e.g., filter out rows with null values in the 'age' column):\n","df_cleaned = df_raw_2020orders.filter(df_raw_2020orders[\"Date\"].isNotNull())\n","print(df_cleaned)"],"outputs":[{"output_type":"display_data","data":{"application/vnd.livy.statement-meta+json":{"spark_pool":null,"statement_id":10,"statement_ids":[10],"state":"finished","livy_statement_state":"available","session_id":"ecb846c9-e988-4906-95bc-af67b3aacd94","normalized_state":"finished","queued_time":"2024-10-25T19:21:35.4162864Z","session_start_time":null,"execution_start_time":"2024-10-25T19:21:35.9099818Z","execution_finish_time":"2024-10-25T19:21:36.2079156Z","parent_msg_id":"d65f6fd9-d9ab-4498-ab5d-0710bab459be"},"text/plain":"StatementMeta(, ecb846c9-e988-4906-95bc-af67b3aacd94, 10, Finished, Available, Finished)"},"metadata":{}},{"output_type":"stream","name":"stdout","text":["DataFrame[ID: string, Count: int, Date: string, Name: string, Style: string, price: double, tax: double]\n"]}],"execution_count":8,"metadata":{"jupyter":{"source_hidden":false,"outputs_hidden":false},"nteract":{"transient":{"deleting":false}},"microsoft":{"language":"python","language_group":"synapse_pyspark"}},"id":"5f5c8125-cbf5-4e00-9d8f-0c437f25b37f"},{"cell_type":"code","source":["# Save the cleaned data to the \"cleansed_Silver\" table in the Silver lakehouse:\n","df_cleaned.write.format(\"delta\").mode(\"overwrite\").save(\"abfss://fabric_medallion_arch_demo@onelake.dfs.fabric.microsoft.com/cleansed_test_Silver.Lakehouse/Tables/2020orders_silver\")"],"outputs":[{"output_type":"display_data","data":{"application/vnd.livy.statement-meta+json":{"spark_pool":null,"statement_id":12,"statement_ids":[12],"state":"finished","livy_statement_state":"available","session_id":"ecb846c9-e988-4906-95bc-af67b3aacd94","normalized_state":"finished","queued_time":"2024-10-25T19:23:52.3238132Z","session_start_time":null,"execution_start_time":"2024-10-25T19:23:52.7414203Z","execution_finish_time":"2024-10-25T19:24:09.4412514Z","parent_msg_id":"8c92d669-7856-4961-a9d0-c38d54833ee4"},"text/plain":"StatementMeta(, ecb846c9-e988-4906-95bc-af67b3aacd94, 12, Finished, Available, Finished)"},"metadata":{}}],"execution_count":10,"metadata":{"jupyter":{"source_hidden":false,"outputs_hidden":false},"nteract":{"transient":{"deleting":false}},"microsoft":{"language":"python","language_group":"synapse_pyspark"}},"id":"c83d4e46-2b49-490f-aadb-87a350c85e89"},{"cell_type":"markdown","source":["## Working with products information"],"metadata":{"nteract":{"transient":{"deleting":false}},"microsoft":{"language":"python","language_group":"synapse_pyspark"}},"id":"f830afb3-2b02-4076-800a-85ca9fc33fea"},{"cell_type":"code","source":["# Read data from the Bronze layer\n","bronze_df = spark.read.format(\"delta\").load(\"abfss://fabric_medallion_arch_demo@onelake.dfs.fabric.microsoft.com/raw_Bronze.Lakehouse/Tables/products\")\n","# Perform transformations (if any)\n","silver_df = bronze_df # Assuming no transformations for simplicity\n","# Write data to the Silver layer\n","silver_df.write.mode(\"overwrite\").format(\"delta\").save(\"abfss://fabric_medallion_arch_demo@onelake.dfs.fabric.microsoft.com/cleansed_test_Silver.Lakehouse/Tables/products_silver\")"],"outputs":[{"output_type":"display_data","data":{"application/vnd.livy.statement-meta+json":{"spark_pool":null,"statement_id":15,"statement_ids":[15],"state":"finished","livy_statement_state":"available","session_id":"ecb846c9-e988-4906-95bc-af67b3aacd94","normalized_state":"finished","queued_time":"2024-10-25T19:27:20.1106109Z","session_start_time":null,"execution_start_time":"2024-10-25T19:27:20.5334249Z","execution_finish_time":"2024-10-25T19:27:25.4936309Z","parent_msg_id":"bf665ff4-43d5-4b02-90a6-6c28640576c3"},"text/plain":"StatementMeta(, ecb846c9-e988-4906-95bc-af67b3aacd94, 15, Finished, Available, Finished)"},"metadata":{}}],"execution_count":13,"metadata":{"jupyter":{"source_hidden":false,"outputs_hidden":false},"nteract":{"transient":{"deleting":false}},"microsoft":{"language":"python","language_group":"synapse_pyspark"}},"id":"7f72ac98-4ece-4a8a-a5c5-5e1fc7273382"}],"metadata":{"kernel_info":{"name":"synapse_pyspark"},"kernelspec":{"name":"synapse_pyspark","language":"Python","display_name":"Synapse PySpark"},"language_info":{"name":"python"},"microsoft":{"language":"python","language_group":"synapse_pyspark","ms_spell_check":{"ms_spell_check_language":"en"}},"widgets":{},"nteract":{"version":"nteract-front-end@1.0.0"},"spark_compute":{"compute_id":"/trident/default","session_options":{"conf":{"spark.synapse.nbs.session.timeout":"1200000"}}},"dependencies":{"lakehouse":{"default_lakehouse":"251cd515-16a3-4555-a3d2-dfd12adb2335","default_lakehouse_name":"raw_Bronze","default_lakehouse_workspace_id":"597e0afc-c8db-4f4d-8464-d13570f5b075"}}},"nbformat":4,"nbformat_minor":5}
0 commit comments