40 local push files from worker fails when cross filesystem (#41)

* Fix moving local files cross filesystem * Revert launch config change
adammcdonagh · Oct 6, 2023 · fc65afe · fc65afe
1 parent 29f2ab6
commit fc65afe
Show file tree

Hide file tree

Showing 3 changed files with 17 additions and 5 deletions.
diff --git a/.vscode/settings.json b/.vscode/settings.json
@@ -5,7 +5,7 @@
   "python.linting.enabled": true,
   "python.linting.flake8Enabled": false,
   "http.systemCertificates": true,
-  "python.formatting.provider": "black",
+  "python.formatting.provider": "none",
   "files.associations": {
     "*.json": "jsonc"
   },
@@ -27,5 +27,8 @@
   "python.linting.pylintEnabled": true,
   "editor.formatOnType": true,
   "python.testing.pytestArgs": [".", "--keepalive"],
-  "python.linting.pylintCategorySeverity.refactor": "Warning"
+  "python.linting.pylintCategorySeverity.refactor": "Warning",
+  "[python]": {
+    "editor.defaultFormatter": "ms-python.black-formatter"
+  }
 }
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,5 +1,9 @@
 # Changelog
 
+# v0.14.3
+
+- Fix issue with local file move when cross filesystem.
+
 # v0.14.2
 
 - Minor log verbosity update

diff --git a/src/opentaskpy/remotehandlers/local.py b/src/opentaskpy/remotehandlers/local.py
@@ -181,22 +181,27 @@ def push_files_from_worker(self, local_staging_directory: str) -> int:
         # Get list of files in local_staging_directory
         files = glob.glob(f"{local_staging_directory}/*")
         for file in files:
-            self.logger.info(f"[LOCALHOST] Moving file to new location: {file}")
             file_name = os.path.basename(file)
+            final_destination = f"{destination_directory}/{file_name}"
+            self.logger.info(
+                f"[LOCALHOST] Moving file to new location: {final_destination}"
+            )
 
             # Handle any rename that might be specified in the spec
             if "rename" in self.spec:
                 rename_regex = self.spec["rename"]["pattern"]
                 rename_sub = self.spec["rename"]["sub"]
 
                 file_name = re.sub(rename_regex, rename_sub, file_name)
+                final_destination = f"{destination_directory}/{file_name}"
 
             mode = self.spec["mode"] if "mode" in self.spec else None
 
             try:
-                os.rename(file, f"{destination_directory}/{file_name}")
+                shutil.copy(file, final_destination)
+                os.remove(file)
                 if mode:
-                    os.chmod(f"{destination_directory}/{file_name}", int(mode))
+                    os.chmod(final_destination, int(mode))
             except Exception as ex:  # pylint: disable=broad-exception-caught
                 self.logger.error(f"[LOCALHOST] Failed to move file: {ex}")
                 result = 1