minor changes

Author: ritchie46

scripts/aws_glue.py

@@ -22,54 +22,55 @@
 sc = SparkContext()
 glueContext = GlueContext(sc)
 spark = glueContext.spark_session
+base = "s3://polars-pdsh/scale-factor-100.0/200/"
 
 # Read arbitrary dataset from S3
 lineitem = glueContext.create_dynamic_frame.from_options(
     connection_type="s3",
-    connection_options={"paths": ["s3://polars-pdsh/scale-factor-100.0/200/lineitem/"]},
-    format="parquet",  # Change to "json", "csv", etc.
+    connection_options={"paths": [base + "lineitem/"]},
+    format="parquet"  # Change to "json", "csv", etc.
 ).toDF()
 
 customer = glueContext.create_dynamic_frame.from_options(
     connection_type="s3",
-    connection_options={"paths": ["s3://polars-pdsh/scale-factor-100.0/200/customer/"]},
-    format="parquet",  # Change to "json", "csv", etc.
+    connection_options={"paths": [base + "customer/"]},
+    format="parquet"  # Change to "json", "csv", etc.
 ).toDF()
 
 nation = glueContext.create_dynamic_frame.from_options(
     connection_type="s3",
-    connection_options={"paths": ["s3://polars-pdsh/scale-factor-100.0/200/nation/"]},
-    format="parquet",  # Change to "json", "csv", etc.
+    connection_options={"paths": [base + "nation/"]},
+    format="parquet"  # Change to "json", "csv", etc.
 ).toDF()
 
 orders = glueContext.create_dynamic_frame.from_options(
     connection_type="s3",
-    connection_options={"paths": ["s3://polars-pdsh/scale-factor-100.0/200/orders/"]},
-    format="parquet",  # Change to "json", "csv", etc.
+    connection_options={"paths": [base + "orders/"]},
+    format="parquet"  # Change to "json", "csv", etc.
 ).toDF()
 
 part = glueContext.create_dynamic_frame.from_options(
     connection_type="s3",
-    connection_options={"paths": ["s3://polars-pdsh/scale-factor-100.0/200/part/"]},
-    format="parquet",  # Change to "json", "csv", etc.
+    connection_options={"paths": [base + "part/"]},
+    format="parquet"  # Change to "json", "csv", etc.
 ).toDF()
 
 partsupp = glueContext.create_dynamic_frame.from_options(
     connection_type="s3",
-    connection_options={"paths": ["s3://polars-pdsh/scale-factor-100.0/200/partsupp/"]},
-    format="parquet",  # Change to "json", "csv", etc.
+    connection_options={"paths": [base + "partsupp/"]},
+    format="parquet"  # Change to "json", "csv", etc.
 ).toDF()
 
 region = glueContext.create_dynamic_frame.from_options(
     connection_type="s3",
-    connection_options={"paths": ["s3://polars-pdsh/scale-factor-100.0/200/region/"]},
-    format="parquet",  # Change to "json", "csv", etc.
+    connection_options={"paths": [base + "region/"]},
+    format="parquet"  # Change to "json", "csv", etc.
 ).toDF()
 
 supplier = glueContext.create_dynamic_frame.from_options(
     connection_type="s3",
-    connection_options={"paths": ["s3://polars-pdsh/scale-factor-100.0/200/supplier/"]},
-    format="parquet",  # Change to "json", "csv", etc.
+    connection_options={"paths": [base + "supplier/"]},
+    format="parquet"  # Change to "json", "csv", etc.
 ).toDF()
 
 

scripts/pl_cloud.py

@@ -26,7 +26,7 @@
 from queries.polars.q21 import q as q21
 from queries.polars.q22 import q as q22
 
-pc.login()
+pc.authenticate()
 
 
 def _scan_ds(table_name: str) -> pl.LazyFrame:
@@ -81,8 +81,8 @@ def _scan_ds(table_name: str) -> pl.LazyFrame:
 
 
 ctx = pc.ComputeContext(
-    workspace="ritchie-workspace",
-    instance_type="t3.xlarge",
+    workspace="polars-ritchie-dev",
+    instance_type="m6.xlarge",
     cluster_size=10,
     interactive=True,
 )