Unique files in S3 including pid of target snowflake

dext · Aug 8, 2023 · dcdd330 · dcdd330
1 parent a67ccdc
commit dcdd330
Show file tree

Hide file tree

Showing 2 changed files with 2 additions and 1 deletion.
diff --git a/target_snowflake/__init__.py b/target_snowflake/__init__.py
@@ -527,6 +527,7 @@ def main():
     if args.config:
         with open(args.config, encoding="utf8") as config_input:
             config = json.load(config_input)
+            config["s3_key_prefix"] = f"{config['s3_key_prefix']}pid_{str(os.getpid())}/"
     else:
         config = {}
 

diff --git a/target_snowflake/upload_clients/s3_upload_client.py b/target_snowflake/upload_clients/s3_upload_client.py
@@ -50,7 +50,7 @@ def upload_file(self, file, stream, temp_dir=None):
         bucket = self.connection_config['s3_bucket']
         s3_acl = self.connection_config.get('s3_acl')
         s3_key_prefix = self.connection_config.get('s3_key_prefix', '')
-        timestamp = datetime.datetime.now().strftime("%Y%m%d-%H%M%S-%f")
+        timestamp = datetime.datetime.now().strftime("%Y%m%d-%H%M%S")
 
         s3_key = f"{s3_key_prefix}pipelinewise_{stream}_{timestamp}_{os.path.basename(file)}"
         self.logger.info('Target S3 bucket: %s, local file: %s, S3 key: %s', bucket, file, s3_key)