Merge pull request #52 from nsidc/simplify-cli-and-ops-instructions

Simplify cli and ops instructions

antarctica_today/__main__.py

@@ -1,4 +1,5 @@
 import click
+from loguru import logger
 
 from antarctica_today.generate_daily_melt_file import generate_new_daily_melt_files
 from antarctica_today.generate_gap_filled_melt_picklefile import (
@@ -10,6 +11,7 @@
 )
 from antarctica_today.melt_array_picklefile import save_model_array_picklefile
 from antarctica_today.nsidc_download_Tb_data import download_new_files
+from antarctica_today.update_data import update_everything_to_latest_date
 
 
 @click.group()
@@ -18,69 +20,79 @@ def cli():
     pass
 
 
-@click.option(
-    "--start-date",
-    default="2022-01-10",
-    help="TODO",
-)
-@cli.command()
-def download_tb(start_date: str):
-    """Download NSIDC-0080 brightness temperature granules.
-
-    The default start date is the day after the end of the .bin data available in
-    `/data/daily_melt_bin_files/` directory in this repo.
-    """
-    download_new_files(time_start=start_date)
+@cli.group()
+def init():
+    """Initialize the Antarctica Today database."""
+    pass
 
 
-@click.option(
-    "--start-date",
-    # TODO: Is this default right? The actual function we're calling has a default of
-    # 2021-10-01. Why?
-    default="2022-01-10",
-    help="TODO",
-)
-@cli.command()
-def generate_daily_melt(start_date: str):
-    """Generate daily melt file from brightness temperature granules."""
-    generate_new_daily_melt_files(
-        start_date=start_date,
-        overwrite=False,
-    )
+@init.command("01-download-tb")
+def download_tb():
+    """Download NSIDC-0080 brightness temperature granules.
 
+    By default, data is downloaded starting on 2022-01-10, as data before then is
+    committed to this repository. This pre-generated data was generated from NSIDC-0001
+    and NSIDC-0007 datasets.
+    """
+    download_new_files()
+    logger.success("Download of NSIDC-0080 granules complete.")
 
-@cli.command
-def melt_array_picklefile():
-    """Is this needed operationally?"""
-    save_model_array_picklefile()
 
+@init.command("02-generate-daily-melt")
+def generate_daily_melt():
+    """Generate daily melt files from brightness temperature granules.
 
-@cli.command
-def gap_filled_melt_picklefile():
-    """Is this needed operationally?"""
-    save_gap_filled_picklefile()
+    These files are written as binary files with ".bin" extension in the
+    `/data/daily_melt_bin_files/` directory in this repository.
+    """
+    generate_new_daily_melt_files(overwrite=False)
+    logger.success("Generation of daily melt files complete.")
 
 
-@cli.command()
+@init.command("03-preprocess")
 def preprocess():
     """Perform pre-processing steps for Antarctica Today data.
 
-    This includes:
-        - ...
-        - ...
+    This includes CSV, TIF, and pickled numpy arrays with ".pickle" extension.
     """
     preprocessing_main()
+    logger.success("Preprocess complete.")
+
+
+@init.command("all")
+def init_all():
+    """Initialize the Antarctica Today database."""
+    download_new_files()
+    generate_new_daily_melt_files(overwrite=False)
+    preprocessing_main()
+    logger.success("Antarctica Today database initialized.")
 
 
 @cli.command()
-def process():
-    """Perform processing steps for Antarctica Today data.
+def daily_update_and_plots():
+    """Perform a daily update of the Antarctica Today database and produce plots.
 
     This includes:
-        - ...
-        - ...
+
+    * A "daily melt" map of the most recent day's melt extent
+    * A "sum" map of the current season's total melt days
+    * An "anomaly" map of that season's total melt days in comparison to baseline
+      average values to-that-day-of-year
+    * A line plot of melt extent up do that date, compared to historical baseline
+      averages
+
+    It will copy these plots into a sub-directory `/plots/daily_plots_gathered/[date]/`
+    for easy collection.
     """
+    update_everything_to_latest_date(copy_to_gathered_dir=True)
+    logger.success("Database updated to the current date. New plots produced.")
+
+
+@cli.command()
+def all_plots():
+    """Generate all Antarctica Today plot images from the database."""
     generate_all_plots_and_maps_main()
+    logger.success("All plots generated.")
 
 
 if __name__ == "__main__":

antarctica_today/generate_daily_melt_file.py

@@ -31,7 +31,7 @@
 
 
 def generate_new_daily_melt_files(
-    start_date: str = "2021-10-01",
+    start_date: str = "2022-01-10",
     end_date: Optional[str] = None,
     overwrite: bool = True,
     warn_if_missing_files: bool = True,
@@ -40,6 +40,9 @@ def generate_new_daily_melt_files(
 
     This function assumes the necessary .bin Tb files from NSIDC are downloaded.
     If not, go to "nsidc_download_Tb_data.py" and update there first.
+
+    The default start date is the day after the end of the .bin data available in
+    `/data/daily_melt_bin_files/` directory in this repo.
     """
     start_dt = datetime.datetime(
         year=int(start_date[0:4]), month=int(start_date[5:7]), day=int(start_date[8:10])

antarctica_today/nsidc_download_Tb_data.py

@@ -264,13 +264,16 @@ def filter_data_only_in_melt_season(
 
 def download_new_files(
     *,
-    time_start="2021-02-17",
+    time_start="2022-01-10",
     time_end=datetime.datetime.now().strftime("%Y-%m-%d"),
     only_in_melt_season=True,
 ) -> List[str]:
     """Download new NSIDC-0080 files into the directory of your choice.
 
     Will download 25km resolution data files from the southern hemisphere.
+
+    The default start date is the day after the end of the .bin data available in
+    `/data/daily_melt_bin_files/` directory in this repo.
     """
     short_name = "NSIDC-0080"
     version = "2"

doc/operation.md

@@ -37,13 +37,42 @@ threshold binaries provided by Tom Mote. These are checked in to this repository
 > :bangbang: Steps must be performed in order :bangbang:
 
 
-## 1. Download NSIDC-0080
+## Quick start
+
+These steps will use Docker, as we expect that to be the main operation mode. See the
+"Detailed steps" section below for examples of running the code without Docker.
+
+
+### Initialize the database and generate all plots
+
+```bash
+docker compose run cli init all
+docker compose run cli plots
+```
+
+
+### Daily updates and plots
+
+```bash
+docker compose run cli daily-update-and-plots
+```
+
+Plots will be available in the repository subdirectory
+`/plots/daily_plots_gathered/[date]/`.
+
+
+## Detailed steps
+
+It's not necessary to run these steps separately, as they will be run by the "Quick
+start" commands above.
+
+### 1. Download NSIDC-0080
 
 Download NSIDC-0080 granules:
 
 ```bash
 PYTHONPATH=.
-python antarctica_today download-tb
+python antarctica_today init 01-download-tb
 ```
 
 > [!NOTE]
@@ -56,17 +85,17 @@ python antarctica_today download-tb
 > downloads that raw data.
 
 
-#### Creates data:
+##### Creates data:
 
 * `.nc` files in `Tb/` directory
 
 
-## 2. Generate all the daily melt binary files
+### 2. Generate all the daily melt binary files
 
 
 ```bash
 PYTHONPATH=.
-python antarctica_today generate-daily-melt
+python antarctica_today init 02-generate-daily-melt
 ```
 
 <details><summary>🛠️ _TODO_</summary>
@@ -81,7 +110,7 @@ Why?
 </details>
 
 
-#### Creates data:
+##### Creates data:
 
 * `.bin` files in `data/daily_melt_bin_files/` directory for dates on or after
   2022-01-10
@@ -92,14 +121,14 @@ Why?
 > note in the previous step: pre-generated data goes through to 2022-01-10.
 
 
-## 3. Generate the database
+### 3. Generate the database
 
 > [!NOTE]
 > This command may take up to tens of minutes.
 
 ```bash
 PYTHONPATH=.
-python antarctica_today preprocess
+python antarctica_today init 03-preprocess
 ```
 
 <details><summary>🛠️ _TODO_</summary>
@@ -120,7 +149,7 @@ python antarctica_today preprocess
 </details>
 
 
-#### Creates data:
+##### Creates data:
 
 * `database/v3_1979-present_gap_filled.pickle`
 * `database/v3_1979-present_raw.pickle`
@@ -131,33 +160,7 @@ python antarctica_today preprocess
 * `.tif` files in `data/annual_*_geotifs/` directories
 
 
-### Database initialization (?)
-
-<details><summary>🛠️ _TODO_</summary>
-Is this step necessary? New files aren't being created when this step is
-run.
-</details>
-
-Create the melt array picklefile, a file containing a 2d grid for each day:
-
-```bash
-PYTHONPATH=.
-python antarctica_today melt-array-picklefile
-```
-
-Create a gap-filled melt picklefile, This "fills the gaps" of missing data or missing
-days in the historical record with climatological averages. This is especially prevalent
-in the 1980s when composites are only tallied every other day. In the small 'pole hole'
-orbital gap, "no melt" (1) is filled. (NOTE: This assumption may need to be changed if
-melt ever reaches South Pole.)
-
-```bash
-PYTHONPATH=.
-python antarctica_today gap-filled-melt-picklefile
-```
-
-
-### Daily updates
+#### Daily updates
 
 > [!WARNING]
 > All initialization steps above must be completed first.
@@ -167,15 +170,16 @@ This step will download any new Tb data files from NSIDC since its last run, and
 2) A "sum" map of that season's total melt days
 3) An "anomaly" map of that season's total melt days in comparison to baseline average values to-that-day-of-year
 4) A line plot of melt extent up do that date, compared to historical baseline averages.
+
 It will copy these plots into a sub-directory /plots/daily_plots_gathered/[date]/ for easy collection.
 
 ```bash
 PYTHONPATH=.
-python antarctica_today/update_data.py
+python antarctica_today daily_update_and_plots
 ```
 
 
-## 4. Generate outputs (optional)
+### 4. Generate outputs (optional)
 
 > [!NOTE]
 > This command may take up to tens of minutes.
@@ -184,7 +188,7 @@ This will go through the entire database and produce summary maps and plots for
 
 ```bash
 PYTHONPATH=.
-python antarctica_today process
+python antarctica_today plots 
 ```
 
 <details><summary>🛠️ _TODO_</summary>
@@ -194,20 +198,6 @@ python antarctica_today process
 </details>
 
 
-#### Creates data:
+##### Creates data:
 
 * `.png` files in `plots/` subdirectories
-
-
-## Running in Docker
-
-This repository includes a `compose.yml` configuration which enables running this code
-with Docker. For example, the `download-tb` command can be run as follows:
-
-```bash
-docker compose run cli download-tb
-```
-
-> [!WARNING]
-> By default, outputs will be written as `root`! You can override the user (TODO: how?)
-> to match your desired production user.