From aa51d5b675b151cdcb1bff75813b5974e6bdd7cb Mon Sep 17 00:00:00 2001 From: Haeun-Oh Date: Sat, 7 Oct 2023 09:06:47 +0900 Subject: [PATCH] add: raw --- src/upload_parquet.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/upload_parquet.py b/src/upload_parquet.py index 8b49aef..b7aced3 100644 --- a/src/upload_parquet.py +++ b/src/upload_parquet.py @@ -1,4 +1,4 @@ # %% -df = spark.read.format("csv").load("s3a://data/", header=True) # noqa: F821 +df = spark.read.format("csv").load("s3a://data/raw", header=True) # noqa: F821 df.write.mode("overwrite").option("compression", "snappy").parquet("s3a://data/lake/spark_df.parquet")