Merge pull request #215 from askap-vast/v2.0.0-rc.4

v2.0.0-rc.4

README.md

@@ -25,6 +25,8 @@ pip install -r requirements.txt
 
 As of version v2.0 the module has been made 'notebook friendly' and can be used interactively. See the `notebook-examples` directory in this repository for examples on how to use the module in a notebook environment. Note that for large queries it is better to use the `find_sources.py` script as pre version v2.0.
 
+**Note**: Jupyter is not included in the requirements, hence please install the required packages to the environment if you wish to use a notebook interface.
+
 ## System Variables
 
 To save specifying the data directories in every call to VAST tools there are two system variables you can set that will be read by the module:

notebook-examples/planet-hunting-example.ipynb

@@ -426,7 +426,7 @@
     }
    ],
    "source": [
-    "jupiter.plot_all_cutouts(figsize=(20,8), zscale=True, contrast=0.1, force=True)"
+    "jupiter.show_all_png_cutouts(figsize=(20,8), zscale=True, contrast=0.1, force=True)"
    ]
   },
   {
@@ -455,7 +455,7 @@
     }
    ],
    "source": [
-    "venus.plot_all_cutouts(figsize=(20,8))"
+    "venus.show_all_png_cutouts(figsize=(20,8))"
    ]
   },
   {
@@ -484,7 +484,7 @@
     }
    ],
    "source": [
-    "moon.plot_all_cutouts(figsize=(20,8), force=True, size=Angle(35. * u.arcmin), zscale=True, contrast=0.4, no_selavy=True)"
+    "moon.show_all_png_cutouts(figsize=(20,8), force=True, size=Angle(35. * u.arcmin), zscale=True, contrast=0.4, no_selavy=True)"
    ]
   },
   {

notebook-examples/source-search-example.ipynb

@@ -1260,7 +1260,7 @@
     }
    ],
    "source": [
-    "all_cutouts = my_source.plot_all_cutouts(columns=5, figsize=(20, 8))\n",
+    "all_cutouts = my_source.show_all_png_cutouts(columns=5, figsize=(20, 8))\n",
     "all_cutouts"
    ]
   },
@@ -1297,7 +1297,7 @@
     }
    ],
    "source": [
-    "all_cutouts = my_source.plot_all_cutouts(columns=5, figsize=(20, 8), force=True, size=Angle(1*u.arcmin))\n",
+    "all_cutouts = my_source.show_all_png_cutouts(columns=5, figsize=(20, 8), force=True, size=Angle(1*u.arcmin))\n",
     "all_cutouts"
    ]
   },

notebook-examples/vast-pipeline-example.ipynb

@@ -1404,7 +1404,7 @@
     }
    ],
    "source": [
-    "all_cutouts = my_source.plot_all_cutouts(columns=6, figsize=(20,10))\n",
+    "all_cutouts = my_source.show_all_png_cutouts(columns=6, figsize=(20,10))\n",
     "all_cutouts"
    ]
   },
@@ -3155,9 +3155,9 @@
  ],
  "metadata": {
   "kernelspec": {
-   "display_name": "vast-tools",
+   "display_name": "Python 3",
    "language": "python",
-   "name": "vast-tools"
+   "name": "python3"
   },
   "language_info": {
    "codemirror_mode": {
@@ -3169,7 +3169,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.8.2"
+   "version": "3.8.3"
   }
  },
  "nbformat": 4,

requirements.txt

@@ -1,17 +1,17 @@
 astroML>=0.4.1,<0.5
-astropy<=4.2,!=4.0
-astroquery<=0.4.1
+astropy>=3.2,<4.2,!=4.0
+astroquery>=0.4.1,<0.5
 bokeh>=2.1.1,<2.2
-colorlog<=4.0.2
-dask[dataframe]<=2.20.0
+colorlog>=4.0.2,<5.0
+dask[dataframe]>=2.25.0,<2.30.0
 matplotlib>=3.1.2,<3.3.0
-mocpy>=0.8.4
-multiprocessing_logging<=0.3.1
+mocpy>=0.8.4,<0.9
+multiprocessing_logging>=0.3.1,<0.4
 numexpr>=2.7.1,<2.8
-numpy<=1.17.4
-pandas<1.1,>=1.0
-pyarrow<=0.17.1
+numpy>=1.17.4,<1.20.0
+pandas>=1.1,<1.5
+pyarrow>=0.17.1,<0.20.0
 radio_beam>=0.3.1,<0.4
-scipy<=1.4.0
-tables<=3.6.1
+scipy>=1.4.0,<1.5.0
+tables>=3.6.1,<3.7.0
 tabulate>=0.8.6,<0.9

setup.py

@@ -14,17 +14,17 @@
     install_requires=[
         "astroML>=0.4.1,<0.5",
         "astropy>=3.2,<4.2,!=4.0",
-        "astroquery<=0.4.1",
+        "astroquery>=0.4.1,<0.5",
         "bokeh>=2.1.1,<2.2",
         "colorlog>=4.0.2,<5.0",
-        "dask[dataframe]<=2.20.0",
+        "dask[dataframe]>=2.25.0,<2.30.0",
         "matplotlib>=3.1.2,<3.3.0",
-        "mocpy>=0.8.4",
-        "multiprocessing_logging<=0.3.1",
+        "mocpy>=0.8.4,<0.9",
+        "multiprocessing_logging>=0.3.1,<0.4",
         "numexpr>=2.7.1,<2.8",
-        "numpy>=1.17.4,<1.18",
-        "pandas<1.1,>=1.0",
-        "pyarrow<=0.17.1",
+        "numpy>=1.17.4,<1.20.0",
+        "pandas>=1.1,<1.5",
+        "pyarrow>=0.17.1,<0.20.0",
         "radio_beam>=0.3.1,<0.4",
         "scipy>=1.4.0,<1.5.0",
         "tables>=3.6.1,<3.7.0",

vasttools/__init__.py

@@ -9,4 +9,4 @@
 '''
 
 __author__ = 'Dougal Dobie, Adam Stewart'
-__version__ = "2.0.0-rc.3"
+__version__ = "2.0.0-rc.4"

vasttools/pipeline.py

@@ -275,7 +275,7 @@ def load_run(
                 'sources.parquet'
             ),
             engine='pyarrow'
-        )
+        ).set_index('id')
 
         m_files = images['measurements_path'].tolist()
         m_files += sorted(glob.glob(os.path.join(

vasttools/query.py

@@ -608,7 +608,11 @@ def _gen_all_source_products(
             sys.exit()
         finally:
             self.logger.debug("Closing workers.")
-            workers.close()
+            # Using terminate below as close was prone to hanging
+            # when join is called. I believe the hang comes from
+            # a child processes not getting returned properly
+            # because of the large number of file I/O.
+            workers.terminate()
             workers.join()
 
     def _produce_source_products(

vasttools/source.py

@@ -141,7 +141,7 @@ class Source:
     )
         Saves the FITS cutout of the epoch selected.
 
-    plot_all_cutouts(
+    show_all_png_cutouts(
         columns=4, percentile=99.9, zscale=False, contrast=0.1,
         outfile=None, save=False, size=None, figsize=(10, 5),
         force=False, no_selavy=False, disable_autoscaling=False
@@ -1272,7 +1272,7 @@ def save_all_png_cutouts(
             )
         )
 
-    def plot_all_cutouts(
+    def show_all_png_cutouts(
         self, columns=4, percentile=99.9, zscale=False,
         contrast=0.1, outfile=None, save=False, size=None, figsize=(10, 5),
         force=False, no_selavy=False, disable_autoscaling=False
@@ -1373,8 +1373,10 @@ def plot_all_cutouts(
                 cutout_row.data * 1.e3, norm=img_norms, cmap="gray_r"
             )
 
-            plots[i].set_title('Epoch {}'.format(
-                measurement_row.epoch
+            epoch_time = measurement_row.dateobs
+
+            plots[i].set_title('{}'.format(
+                epoch_time.strftime("%Y-%m-%d %H:%M:%S")
             ))
 
             cross_target_coords = cutout_row.wcs.wcs_world2pix(

vasttools/survey.py

@@ -39,7 +39,8 @@
     "8": "08",
     "9": "09",
     "10x": "10x",
-    "11x": "11x"
+    "11x": "11x",
+    "12": "12",
 }
 
 FIELD_FILES = {
@@ -66,7 +67,9 @@
     "10x": pkg_resources.resource_filename(
         __name__, "./data/csvs/vast_epoch10x_info.csv"),
     "11x": pkg_resources.resource_filename(
-        __name__, "./data/csvs/vast_epoch11x_info.csv")
+        __name__, "./data/csvs/vast_epoch11x_info.csv"),
+    "12": pkg_resources.resource_filename(
+        __name__, "./data/csvs/vast_epoch12_info.csv")
 }
 
 FIELD_CENTRES = pd.read_csv(