diff --git a/CHANGELOG.md b/CHANGELOG.md index 5839dc0..20e4670 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## 1.0.1 - 2024-05-13 + +* Added new command line option `--download-dir` when using the `download` subcommand so you can specify the directory to download files. It defaults to `.`, if ommitted, which was the hardcoed default in previous releases of giant-squid. + ## 1.0.0 - 2024-05-13 * Increased MSRV to 1.70 diff --git a/Cargo.lock b/Cargo.lock index 512454d..8012a45 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -773,7 +773,7 @@ dependencies = [ [[package]] name = "mwa_giant_squid" -version = "1.0.0" +version = "1.0.1" dependencies = [ "anyhow", "backoff", diff --git a/Cargo.toml b/Cargo.toml index 0b13321..ffb87c0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "mwa_giant_squid" -version = "1.0.0" +version = "1.0.1" authors = [ "Christopher H. Jordan ", "Harrison Barlow ", diff --git a/README.md b/README.md index 365ae31..a789b01 100644 --- a/README.md +++ b/README.md @@ -7,15 +7,14 @@ ![Crates.io](https://img.shields.io/crates/d/mwa_giant_squid) ![Crates.io](https://img.shields.io/crates/l/mwa_giant_squid) [![docs](https://docs.rs/mwa_giant_squid/badge.svg)](https://docs.rs/crate/mwa_giant_squid/latest) -[![Rust Report Card](https://rust-reportcard.xuri.me/badge/github.com/MWATelescope/mwa_giant_squid)](https://rust-reportcard.xuri.me/report/github.com/MWATelescope/mwa_giant_squid) -An alternative [MWA ASVO](https://asvo.mwatelescope.org/) client. For general help on using +An alternative [MWA ASVO](https://asvo.mwatelescope.org/) client. For general help on using the MWA ASVO, please visit: [MWA ASVO wiki](https://mwatelescope.atlassian.net/wiki/spaces/MP/pages/24973129/Data+Access). --- NOTE FOR HPC USERS -Please read [this wiki article](https://mwatelescope.atlassian.net/wiki/spaces/MP/pages/65405030/MWA+ASVO+Use+with+HPC+Systems) +Please read [this wiki article](https://mwatelescope.atlassian.net/wiki/spaces/MP/pages/65405030/MWA+ASVO+Use+with+HPC+Systems) if you are running giant-squid on HPC systems. --- @@ -178,7 +177,7 @@ do < ready.tsv ### Download ASVO jobs -To download job ID 12345: +To download job ID 12345 to your current directory '.': ```bash giant-squid download 12345 @@ -186,7 +185,7 @@ giant-squid download 12345 giant-squid d 12345 ``` -To download obsid 1065880128: +To download obsid 1065880128 to your current directory '.': ```bash giant-squid download 1065880128 @@ -201,6 +200,17 @@ same applies if this code is still being used in the year 2296.) Text files containing job IDs or obsids may be used too. +You can specify the directory to download to by providing the `download_dir` parameter +to the `download` subcommand. Ommitting this will default to your current dir `.`. + +To download obsid 1065880128 to your `/tmp` directory: + +```bash +giant-squid download --download-dir /tmp 1065880128 +# or +giant-squid d -d /tmp 1065880128 +``` + By default, `giant-squid` will perform stream unzipping. Disable this with `-k` (or `--keep-zip`). diff --git a/src/asvo/mod.rs b/src/asvo/mod.rs index a224c8e..1b05288 100644 --- a/src/asvo/mod.rs +++ b/src/asvo/mod.rs @@ -112,6 +112,7 @@ impl AsvoClient { jobid: AsvoJobID, keep_tar: bool, hash: bool, + download_dir: &str ) -> Result<(), AsvoError> { let mut jobs = self.get_jobs()?; debug!("Attempting to download job {}", jobid); @@ -119,7 +120,7 @@ impl AsvoClient { jobs.0.retain(|j| j.jobid == jobid); match jobs.0.len() { 0 => Err(AsvoError::NoAsvoJob(jobid)), - 1 => self.download(&jobs.0[0], keep_tar, hash), + 1 => self.download(&jobs.0[0], keep_tar, hash, download_dir), // Hopefully there's never multiples of the same ASVO job ID in a // user's job listing... _ => unreachable!(), @@ -134,6 +135,7 @@ impl AsvoClient { obsid: Obsid, keep_tar: bool, hash: bool, + download_dir: &str ) -> Result<(), AsvoError> { let mut jobs = self.get_jobs()?; debug!("Attempting to download obsid {}", obsid); @@ -142,13 +144,13 @@ impl AsvoClient { jobs.0.retain(|j| j.obsid == obsid); match jobs.0.len() { 0 => Err(AsvoError::NoObsid(obsid)), - 1 => self.download(&jobs.0[0], keep_tar, hash), + 1 => self.download(&jobs.0[0], keep_tar, hash, download_dir), _ => Err(AsvoError::TooManyObsids(obsid)), } } /// Private function to actually do the work. - fn download(&self, job: &AsvoJob, keep_tar: bool, hash: bool) -> Result<(), AsvoError> { + fn download(&self, job: &AsvoJob, keep_tar: bool, hash: bool, download_dir: &str) -> Result<(), AsvoError> { // Is the job ready to download? if job.state != AsvoJobState::Ready { return Err(AsvoError::NotReady { @@ -186,7 +188,7 @@ impl AsvoClient { debug!("Downloading file {:?}", &url); let op = || { - self.try_download(url, keep_tar, hash, f, job) + self.try_download(url, keep_tar, hash, f, job, download_dir) .map_err(|e| match &e { &AsvoError::IO(_) => Error::permanent(e), _ => Error::transient(e), @@ -260,6 +262,7 @@ impl AsvoClient { hash: bool, f: &AsvoFilesArray, job: &AsvoJob, + download_dir: &str ) -> Result<(), AsvoError> { // How big should our in-memory download buffer be [MiB]? let buffer_size = match var("GIANT_SQUID_BUF_SIZE") { @@ -297,7 +300,7 @@ impl AsvoClient { } } else { // Stream-untar the response. - let unpack_path = Path::new("."); + let unpack_path = Path::new(download_dir); info!("Untarring to {:?}", unpack_path); let mut tar = Archive::new(&mut tee); diff --git a/src/bin/giant-squid.rs b/src/bin/giant-squid.rs index 1911f8f..b2b9857 100644 --- a/src/bin/giant-squid.rs +++ b/src/bin/giant-squid.rs @@ -68,6 +68,10 @@ enum Args { /// Download an ASVO job #[clap(alias = "d")] Download { + /// Which dir should downloads be written to. + #[clap(short, long, default_value=".")] + download_dir: String, + /// Don't unzip the contents from the ASVO. #[clap(short, long)] keep_zip: bool, @@ -383,6 +387,7 @@ fn main() -> Result<(), anyhow::Error> { dry_run, verbosity, jobids_or_obsids, + download_dir, .. } => { if jobids_or_obsids.is_empty() { @@ -409,10 +414,10 @@ fn main() -> Result<(), anyhow::Error> { } else { let client = AsvoClient::new()?; for j in jobids { - client.download_job(j, keep_zip, hash)?; + client.download_job(j, keep_zip, hash, &download_dir)?; } for o in obsids { - client.download_obsid(o, keep_zip, hash)?; + client.download_obsid(o, keep_zip, hash, &download_dir)?; } } }