Merge pull request #242 from discos/fix_summary

Fix summary

.github/workflows/ci_tests.yml

@@ -109,11 +109,17 @@ jobs:
             toxenv: py311-test-alldeps-cov
             experimental: false
 
-          - name: OS X - Python 3.8 with all optional dependencies at their 3.8-compatible versions
-            os: macos-latest
+          - name: Linux - Python 3.8 with all optional dependencies at their 3.8-compatible versions
+            os: ubuntu-latest
             python: '3.8'
             toxenv: py38-test-alldeps
-            experimental: false
+            experimental: true
+
+          - name: OS X - Python 3.10 with all optional dependencies
+            os: macos-latest
+            python: '3.10'
+            toxenv: py310-test-alldeps
+            experimental: true
 
           - name: Windows - Python 3.10 with all optional dependencies
             os: windows-latest

docs/changes/242.bugfix.rst

@@ -0,0 +1 @@
+Support new naming for summary file

srttools/convert.py

@@ -16,6 +16,7 @@
 from .converters.mbfits import MBFITS_creator
 from .converters.classfits import CLASSFITS_creator
 from .converters.sdfits import SDFITS_creator
+from .scan import _is_summary_file, find_summary_file_in_dir
 
 
 def convert_to_complete_fitszilla(fname, outname):
@@ -94,7 +95,7 @@ def launch_convert_coords(name, label, save_locally=False):
         allfiles += [name]
 
     for fname in allfiles:
-        if "summary.fits" in fname:
+        if _is_summary_file(fname):
             continue
         outroot = fname.replace(".fits", "_" + label)
         if save_locally:
@@ -115,12 +116,12 @@ def launch_mbfits_creator(name, label, test=False, wrap=False, detrend=False, sa
     name = name.rstrip("/")
     random_name = "tmp_" + str(np.random.random())
     mbfits = MBFITS_creator(random_name, test=test)
-    summary = os.path.join(name, "summary.fits")
+    summary = find_summary_file_in_dir(name)
     if os.path.exists(summary):
         mbfits.fill_in_summary(summary)
 
     for fname in sorted(glob.glob(os.path.join(name, "*.fits"))):
-        if "summary.fits" in fname:
+        if _is_summary_file(fname):
             continue
         mbfits.add_subscan(fname, detrend=detrend)
 

srttools/converters/classfits.py

@@ -11,7 +11,7 @@
 
 The conversion is performed as follows:
 
-1. Load all subscans from a given scan (=a subdirectory with a summary.fits file)
+1. Load all subscans from a given scan (=a subdirectory with a summary file)
     and classify each of them as ON, OFF, CALON, CALOFF.
 
 2. Try to infer patterns in the observation of the kind nON, mOFF, rCAL, and
@@ -75,6 +75,7 @@
 import glob
 from ..utils import get_mH2O
 from ..io import label_from_chan_name
+from ..scan import find_summary_file_in_dir
 from scipy.signal import medfilt
 import copy
 import warnings
@@ -478,7 +479,7 @@ def __init__(self, dirname, scandir=None, average=True, use_calon=False, test=Fa
         ----------------
         scandir : str
             Input data directory (to be clear, the directory containing a set
-            of subscans plus a summary.fits file)
+            of subscans plus a summary file)
         average : bool, default True
             Average all spectra of a given configuration?
         use_calon : bool, default False
@@ -500,7 +501,7 @@ def __init__(self, dirname, scandir=None, average=True, use_calon=False, test=Fa
             self.write_tables_to_disk()
 
     def fill_in_summary(self, summaryfile):
-        """Fill in the information contained in the summary.fits file."""
+        """Fill in the information contained in the summary file."""
         with fits.open(summaryfile) as hdul:
             self.summary.update(hdul[0].header)
 
@@ -515,7 +516,7 @@ def get_scan(self, scandir, average=False):
         ----------
         scandir : str
             Input data directory (to be clear, the directory containing a set
-            of subscans plus a summary.fits file)
+            of subscans plus a summary file)
 
         Other Parameters
         ----------------
@@ -527,7 +528,7 @@ def get_scan(self, scandir, average=False):
         tables
         """
         scandir = scandir.rstrip("/")
-        fname = os.path.join(scandir, "summary.fits")
+        fname = find_summary_file_in_dir(scandir)
         self.fill_in_summary(fname)
         for fname in sorted(glob.glob(os.path.join(scandir, "*.fits"))):
             if "summary" in fname:

srttools/converters/sdfits.py

@@ -12,6 +12,7 @@
     classify_chan_columns,
     interpret_chan_name,
 )
+from srttools.scan import find_summary_file_in_dir
 import glob
 
 
@@ -292,7 +293,7 @@ def __init__(self, dirname, scandir=None, average=True, use_calon=False, test=Fa
         ----------------
         scandir : str
             Input data directory (to be clear, the directory containing a set
-            of subscans plus a summary.fits file)
+            of subscans plus a summary file)
         average : bool, default True
             Average all spectra of a given configuration?
         use_calon : bool, default False
@@ -313,7 +314,7 @@ def __init__(self, dirname, scandir=None, average=True, use_calon=False, test=Fa
             self.write_tables_to_disk()
 
     def fill_in_summary(self, summaryfile):
-        """Fill in the information contained in the summary.fits file."""
+        """Fill in the information contained in the summary file."""
         with fits.open(summaryfile) as hdul:
             self.summary.update(hdul[0].header)
 
@@ -328,7 +329,7 @@ def get_scan(self, scandir, average=False):
         ----------
         scandir : str
             Input data directory (to be clear, the directory containing a set
-            of subscans plus a summary.fits file)
+            of subscans plus a summary file)
 
         Other Parameters
         ----------------
@@ -340,7 +341,7 @@ def get_scan(self, scandir, average=False):
         tables
         """
         scandir = scandir.rstrip("/")
-        fname = os.path.join(scandir, "summary.fits")
+        fname = find_summary_file_in_dir(scandir)
         self.fill_in_summary(fname)
         for fname in sorted(glob.glob(os.path.join(scandir, "*.fits"))):
             if "summary" in fname:

srttools/global_fit.py

@@ -1,6 +1,5 @@
 """Functions to clean images by fitting linear trends to the initial scans."""
 
-
 try:
     import matplotlib.pyplot as plt
     from matplotlib.gridspec import GridSpec

srttools/imager.py

@@ -26,7 +26,7 @@
 from astropy.table.np_utils import TableMergeError
 from astropy.time import Time
 
-from .scan import Scan, list_scans
+from .scan import Scan, list_scans, _is_summary_file
 from .read_config import read_config, sample_config_file
 from .utils import calculate_zernike_moments, calculate_beam_fom, HAS_MAHO
 from .utils import compare_anything, ds9_like_log_scale, njit
@@ -122,7 +122,7 @@ def merge_tables(tables):
         with warnings.catch_warnings():
             warnings.simplefilter("ignore", MergeConflictWarning)
             scan_table = vstack(tables)
-    except TableMergeError as e:
+    except TableMergeError:
         raise TableMergeError(
             "ERROR while merging tables. "
             "Tables:\n" + "\n".join([t.meta["filename"] for t in tables])
@@ -320,7 +320,7 @@ def __init__(
 
         super().__init__(data)
 
-        if data is not None and not "x" in self.colnames:
+        if data is not None and "x" not in self.colnames:
             self.convert_coordinates()
         self.current = None
         self._scan_list = None
@@ -441,7 +441,7 @@ def get_opacity(self, datadir=None, dirlist=None):
             return
 
         for s in scans:
-            if "summary.fits" in s:
+            if _is_summary_file(s):
                 continue
             try:
                 results = calculate_opacity(s, plot=False)
@@ -1499,7 +1499,7 @@ def save_ds9_images(
             is_stokes = ("Q" in ch) or ("U" in ch)
 
             do_moments = not (is_sdev or is_expo or is_stokes or is_outl)
-            do_moments = do_moments and frame and HAS_MAHO
+            do_moments = do_moments and frame == "altaz" and HAS_MAHO
 
             if is_sdev and not save_sdev:
                 continue

srttools/inspect_observations.py

@@ -7,7 +7,7 @@
 from astropy.time import Time
 import logging
 from .io import read_data, chan_re
-from .scan import list_scans
+from .scan import list_scans, _is_summary_file
 from .calibration import read_calibrator_config
 from .read_config import sample_config_file
 from .utils import remove_suffixes_and_prefixes
@@ -75,7 +75,7 @@ def inspect_directories(
     for d in directories:
         fits_files = list_scans(".", [d])
         for f in fits_files:
-            if "summary.fits" in f:
+            if _is_summary_file(f):
                 continue
             logging.info("Reading {}".format(f))
             try:

srttools/interactive_filter.py

@@ -1,6 +1,5 @@
 """Interactive operations."""
 
-
 import copy
 import warnings
 import logging

srttools/monitor/monitor.py

@@ -48,6 +48,7 @@ class MyEventHandler(PatternMatchingEventHandler):
         "*/tempfits/*",
         "*/*.fitstemp",
         "*/summary.fits",
+        "*/Sum_*.fits",
     ]
     patterns = ["*/*.fits"] + ["*/*.fits{}".format(x) for x in range(MAX_FEEDS)]
 

srttools/read_config.py

@@ -1,6 +1,5 @@
 """Read the configuration file."""
 
-
 import os
 import glob
 import warnings

srttools/scan.py

@@ -827,6 +827,41 @@ def frequency_filter(dynamical_spectrum, mask):
     return lc_corr
 
 
+def _is_summary_file(fname):
+    """Check if a file name pattern is compatible with a summary file.
+
+    Examples
+    --------
+    >>> _is_summary_file("summary.fits")
+    True
+    >>> _is_summary_file(os.path.join("bubub", "SuMmary.fits"))
+    True
+    >>> _is_summary_file("SuM_fae02ef0fajdasj.fits")
+    True
+    >>> _is_summary_file(os.path.join("bubub", "SuM_fae02ef0fajdasj.fits"))
+    True
+    >>> # Here "sum" is in the directory name
+    >>> _is_summary_file(os.path.join("Summ", "afaskdjgahga.fits"))
+    False
+    """
+    return os.path.basename(fname)[:4].lower() in ["summ", "sum_"]
+
+
+def find_summary_file_in_dir(directory, warn_if_absent=True):
+    """Find the summary file in an observation directory."""
+    summary_files = []
+    for pattern in ["summary.fits", "Sum_*.fits"]:
+        summary_files += glob.glob(os.path.join(directory, pattern))
+
+    if len(summary_files) > 1:
+        raise ValueError(f"Multiple summary files found: {summary_files}")
+    elif warn_if_absent and len(summary_files) == 0:
+        warnings.warn("No summary file in directory")
+    else:
+        summary_files = summary_files[0]
+    return summary_files
+
+
 def list_scans(datadir, dirlist):
     """List all scans contained in the directory listed in config."""
     scan_list = []
@@ -836,7 +871,7 @@ def list_scans(datadir, dirlist):
         list_of_files += glob.glob(os.path.join(datadir, d, "*.fits[0-9]"))
         list_of_files += glob.glob(os.path.join(datadir, d, "*.fits[0-9][0-9]"))
         for f in list_of_files:
-            if "summary.fits" in f:
+            if _is_summary_file(f):
                 continue
             scan_list.append(f)
     return scan_list

srttools/tests/test_imager.py

@@ -97,14 +97,14 @@ def setup_class(klass):
         )
 
     @pytest.mark.skipif("not HAS_SUNPY")
-    def test_sun_map(self):
+    def test_sun_map_sunpy(self):
         files = main_inspector([os.path.join(self.simdir, "*"), "-d"])
         main_imager(["-c", files[0], "--frame", "sun"])
         for f in files:
             os.unlink(f)
 
     @pytest.mark.skipif("HAS_SUNPY")
-    def test_sun_map(self):
+    def test_sun_map_no_sunpy(self):
         with pytest.raises(ValueError) as excinfo:
             _ = main_inspector([os.path.join(self.simdir, "*"), "-d"])
         assert "No valid observations found" in str(excinfo.value)

srttools/tests/test_io_and_scan.py

@@ -1,5 +1,6 @@
 # -*- coding: utf-8 -*-
 
+from pathlib import Path
 from srttools.read_config import read_config
 from astropy.time import Time
 from astropy.coordinates import SkyCoord
@@ -8,7 +9,12 @@
 import astropy.units as u
 import pytest
 
-from srttools.scan import Scan, HAS_MPL, clean_scan_using_variability
+from srttools.scan import (
+    Scan,
+    HAS_MPL,
+    clean_scan_using_variability,
+    find_summary_file_in_dir,
+)
 from srttools.io import print_obs_info_fitszilla, bulk_change, main_bulk_change
 from srttools.io import locations, read_data_fitszilla, mkdir_p
 from srttools.utils import compare_anything
@@ -438,3 +444,28 @@ def teardown_class(klass):
             for f in glob.glob(os.path.join(klass.datadir, "spectrum", "*.hdf5")):
                 os.unlink(f)
             shutil.rmtree(os.path.join(klass.datadir, "out_spectrum_test"))
+
+
+class TestSummary:
+    @classmethod
+    def setup_class(cls):
+        cls.testdir = "tmp-fake-dir"
+        os.makedirs(cls.testdir)
+
+    def test_summary_nofiles(self):
+        with pytest.warns(UserWarning, match="No summary file"):
+            find_summary_file_in_dir(self.testdir)
+
+    def test_summary_onefile(self):
+        fname = os.path.join(self.testdir, "summary.fits")
+        Path(fname).touch()
+        assert fname == find_summary_file_in_dir(self.testdir)
+
+    def test_summary_multiple(self):
+        Path(os.path.join(self.testdir, "summary.fits")).touch()
+        Path(os.path.join(self.testdir, "Sum_asdfhasldfjhal.fits")).touch()
+        with pytest.raises(ValueError, match="Multiple"):
+            find_summary_file_in_dir(self.testdir)
+
+    def teardown_class(cls):
+        shutil.rmtree(cls.testdir)

srttools/utils.py

@@ -448,8 +448,9 @@ def get_center_of_mass(im, radius=1, approx=None):
     npix = int(radius * min(im.shape))
     xmin, xmax = max(0, approx[0] - npix), min(approx[0] + npix, im.shape[0])
     ymin, ymax = max(0, approx[1] - npix), min(approx[1] + npix, im.shape[1])
-    good_x = slice(xmin, xmax)
-    good_y = slice(ymin, ymax)
+    good_x = slice(xmin, max(xmax, xmin + 1))
+    good_y = slice(ymin, max(ymax, ymin + 1))
+
     cm = np.asarray(scipy.ndimage.center_of_mass(im[good_x, good_y]))
     cm[0] += xmin
     cm[1] += ymin