diff --git a/src/upload_parquet.py b/src/upload_parquet.py index 8b49aef..b7aced3 100644 --- a/src/upload_parquet.py +++ b/src/upload_parquet.py @@ -1,4 +1,4 @@ # %% -df = spark.read.format("csv").load("s3a://data/", header=True) # noqa: F821 +df = spark.read.format("csv").load("s3a://data/raw", header=True) # noqa: F821 df.write.mode("overwrite").option("compression", "snappy").parquet("s3a://data/lake/spark_df.parquet")