From ce8e0060dbd4a044fd4991a01ccdf316ec614f0b Mon Sep 17 00:00:00 2001
From: Nicky Hochmuth <nicky.thecrag@gmail.com>
Date: Mon, 18 Nov 2024 14:28:38 +0100
Subject: [PATCH] Upgrade_watchdog (#401)

* bump all product versions to 2

* bump version

* fix version label

* pin common CONF version for end2end test

* ignore VERS_CFG in end2end testing

* add test for TM folder observing

* bump watchdog==6.0.0

* integrate review fixes
---
 setup.cfg                                    |  2 +-
 stixcore/processing/tests/test_pipline.py    | 62 ++++++++++++++++++++
 stixcore/processing/tests/test_processing.py |  3 +-
 stixcore/soop/tests/test_soop_manager.py     | 10 +---
 stixcore/version_conf.py                     |  2 +-
 5 files changed, 69 insertions(+), 10 deletions(-)

diff --git a/setup.cfg b/setup.cfg
index 5e5092c8..e454f0d2 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -28,7 +28,7 @@ install_requires =
     scp
     spiceypy
     sunpy>=4.0.1
-    watchdog
+    watchdog==6.0.0
     requests
 
 
diff --git a/stixcore/processing/tests/test_pipline.py b/stixcore/processing/tests/test_pipline.py
index e69de29b..7736a8bb 100644
--- a/stixcore/processing/tests/test_pipline.py
+++ b/stixcore/processing/tests/test_pipline.py
@@ -0,0 +1,62 @@
+import re
+import time
+import shutil
+
+import pytest
+from watchdog.observers import Observer
+
+from stixcore.processing.pipeline import GFTSFileHandler
+from stixcore.util.logging import get_logger
+
+logger = get_logger(__name__)
+
+
+@pytest.fixture
+def out_dir(tmp_path):
+    return tmp_path
+
+
+def processing_tm_file(path, **args):
+    logger.info(f"start processing tm file: {path}")
+    time.sleep(2)
+    logger.info(f"stop processing tm file: {path}")
+
+
+@pytest.fixture
+def gfts_manager():
+    return GFTSFileHandler(processing_tm_file, re.compile(r'.*test_[0-9]*.tm$'), name="w-dog-test")
+
+
+def test_gfts_manager_watchdog(out_dir, gfts_manager):
+    (out_dir / "test_1.tm").touch()
+    (out_dir / "test_2.tm").touch()
+    (out_dir / "test_3.tm").touch()
+    (out_dir / "test_4.tmp").touch()
+    (out_dir / "test_5.tmp").touch()
+    (out_dir / "test_6.tmp").touch()
+
+    observer = Observer()
+    observer.schedule(gfts_manager, out_dir,  recursive=False)
+    observer.start()
+
+    assert gfts_manager.queue.qsize() == 0
+
+    # 3 files are in the base dir
+    # simulate start with unprocessed
+    gfts_manager.add_to_queue(out_dir.glob("*.tm"))
+    assert gfts_manager.queue.qsize() == 3
+
+    # now some tm in coming in
+    time.sleep(1)
+    shutil.move((out_dir / "test_4.tmp"), (out_dir / "test_4.tm"))
+    time.sleep(1)
+    shutil.move((out_dir / "test_5.tmp"), (out_dir / "test_5.tm"))
+    time.sleep(1)
+    shutil.move((out_dir / "test_6.tmp"), (out_dir / "test_6.tm"))
+    time.sleep(1)
+
+    openfiles = gfts_manager.queue.qsize()
+    assert openfiles > 2
+    time.sleep(20)
+    assert gfts_manager.queue.qsize() < openfiles
+    observer.stop()
diff --git a/stixcore/processing/tests/test_processing.py b/stixcore/processing/tests/test_processing.py
index aabb1192..09ca9dd6 100644
--- a/stixcore/processing/tests/test_processing.py
+++ b/stixcore/processing/tests/test_processing.py
@@ -247,7 +247,8 @@ def test_single_vs_batch(out_dir):
 
         for i, f_b in enumerate(files_b):
             f_s = files_s[i]
-            diff = FITSDiff(f_b, f_s, ignore_keywords=['CHECKSUM', 'DATASUM', 'DATE', 'VERS_SW'])
+            diff = FITSDiff(f_b, f_s, ignore_keywords=['CHECKSUM', 'DATASUM', 'DATE',
+                                                       'VERS_SW', 'VERS_CFG'])
             assert diff.identical
     finally:
         CONFIG.set('Logging', 'stop_on_error', str(CONTINUE_ON_ERROR))
diff --git a/stixcore/soop/tests/test_soop_manager.py b/stixcore/soop/tests/test_soop_manager.py
index e3b43e35..fedad79d 100644
--- a/stixcore/soop/tests/test_soop_manager.py
+++ b/stixcore/soop/tests/test_soop_manager.py
@@ -2,7 +2,6 @@
 import sys
 import time
 import shutil
-import platform
 
 import dateutil.parser
 import pytest
@@ -61,12 +60,9 @@ def test_soop_manager_watchdog(soop_manager):
 
     time.sleep(3)
 
-    # currently not triggered on mac see: https://github.com/i4Ds/STIXCore/issues/149
-    if platform.system() != "Darwin":
-        # the new data should be integrated now
-        assert soop_manager.filecounter == 4
-        assert len(soop_manager.soops) == 129
-        assert len(soop_manager.observations) == 132
+    assert soop_manager.filecounter == 4
+    assert len(soop_manager.soops) == 129
+    assert len(soop_manager.observations) == 132
 
     observer.stop()
 
diff --git a/stixcore/version_conf.py b/stixcore/version_conf.py
index 1cf04707..01255d15 100644
--- a/stixcore/version_conf.py
+++ b/stixcore/version_conf.py
@@ -6,7 +6,7 @@ def get_conf_version():
         with open(Path(__file__).parent / "config" / "data" / "common" / "VERSION.TXT") as f:
             return f.readline().strip()
     except Exception:
-        return 'v0.1.3'
+        return 'v0.1.5code'
 
 
 __version_conf__ = get_conf_version()