From 943120e64a2c72538bf0fad6d4b081ea5bcf32c4 Mon Sep 17 00:00:00 2001 From: Jamie Kennea Date: Wed, 15 Nov 2023 12:25:34 -0500 Subject: [PATCH] Push to version 3.0.19 --- setup.cfg | 4 +- swifttools/swift_too/ChangeLog.md | 13 ++++- swifttools/swift_too/api_common.py | 11 +++-- swifttools/swift_too/swift_data.py | 47 +++++++++++++++---- swifttools/swift_too/version.py | 4 +- .../APIDocs/ukssdc/xrt_prods/ChangeLog.md | 7 ++- .../ukssdc/APIDocs/ukssdc/xrt_prods/README.md | 3 +- .../APIDocs/ukssdc/xrt_prods/RequestJob.md | 8 +++- swifttools/version.py | 2 +- 9 files changed, 75 insertions(+), 24 deletions(-) diff --git a/setup.cfg b/setup.cfg index 265f26d..1d8af9d 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = swifttools -version = 3.0.16 +version = 3.0.19 description = Tools for users of the Swift satellite long_description = file: README.md long_description_content_type = text/markdown @@ -32,7 +32,7 @@ install_requires = pandas tabulate numpy - + boto3 [bdist_wheel] universal = 0 python-tag = py36 diff --git a/swifttools/swift_too/ChangeLog.md b/swifttools/swift_too/ChangeLog.md index 87edcf0..388de23 100644 --- a/swifttools/swift_too/ChangeLog.md +++ b/swifttools/swift_too/ChangeLog.md @@ -6,13 +6,24 @@ #### Author: Jamie A. Kennea (Penn State) +## `swifttools` 3.0.18 / `swift_too` 1.2.30 + +**Oct 31, 2023**: Resolve issue with noisy warnings in `Data` even if `quiet=True` + +* When downloading files using `Data` module, if those files already exist on disk a `warning` is now issued, instead of a printed warning. If `quiet=True` no warnings are issued. + +## `swifttools` 3.0.17 / `swift_too` 1.2.29 + +**Oct 31, 2023**: Add AWS download support to `Data` class. + +* Added option to download data from AWS instead of HEASARC. Add argument `aws=True` to `Data` class arguments to default to AWS downlink. For details of AWS data hosting, see [here](https://heasarc.gsfc.nasa.gov/docs/archive/cloud.html). + ## `swifttools` 3.0.16 / `swift_too` 1.2.28 **March 22, 2023**: Warning about incorrectly formatted ISO8601 dates * ISO8601 formatted dates without a timezone was assuming localtime, causing confusion. ISO8601 formatted dates should include a timezone specifier, e.g. '2022-01-01T00:00:00Z'. If no timezone is given, the code now issues a warning about this issue. Please use the "Z" for UTC times, as required for for the ISO8601 specification. - ## `swifttools` 3.0.15 / `swift_too` 1.2.27 **March 17, 2023**: `TOORequests` fix diff --git a/swifttools/swift_too/api_common.py b/swifttools/swift_too/api_common.py index 39ac446..44f5842 100644 --- a/swifttools/swift_too/api_common.py +++ b/swifttools/swift_too/api_common.py @@ -12,8 +12,11 @@ # Make Warnings a little less weird formatwarning_orig = warnings.formatwarning -warnings.formatwarning = lambda message, category, filename, lineno, line=None: \ - formatwarning_orig(message, category, filename, lineno, line='') +warnings.formatwarning = ( + lambda message, category, filename, lineno, line=None: formatwarning_orig( + message, category, filename, lineno, line="" + ) +) # Configure for IPV4 only due to issue requests.packages.urllib3.util.connection.HAS_IPV6 = False @@ -96,7 +99,9 @@ def convert_to_dt(value, isutc=False, outfunc=datetime): elif re.match(_iso8601_regex, value): dtvalue = parser.parse(value) if dtvalue.tzinfo is None: - warnings.warn("ISO8601 formatted dates should be supplied with timezone. ISO8601 dates with no timezone will be assumed to be localtime and then converted to UTC.") + warnings.warn( + "ISO8601 formatted dates should be supplied with timezone. ISO8601 dates with no timezone will be assumed to be localtime and then converted to UTC." + ) dtvalue = dtvalue.astimezone(timezone.utc).replace(tzinfo=None) else: raise ValueError( diff --git a/swifttools/swift_too/swift_data.py b/swifttools/swift_too/swift_data.py index c5b56d5..dff094a 100644 --- a/swifttools/swift_too/swift_data.py +++ b/swifttools/swift_too/swift_data.py @@ -4,6 +4,10 @@ import requests import os from fnmatch import fnmatch +import warnings +import boto3 +from botocore import UNSIGNED +from botocore.client import Config try: from tqdm.auto import tqdm @@ -47,6 +51,7 @@ class Swift_DataFile(TOOAPI_Baseclass): quicklook = None type = None localpath = None + s3 = None outdir = "." # Core API definitions _parameters = ["filename", "path", "url", "quicklook", "type"] @@ -70,14 +75,20 @@ def download(self, outdir=None): # Download the data fullfilepath = os.path.join(self.outdir, self.path, self.filename) - r = requests.get(self.url, stream=True, allow_redirects=True) - if r.ok: - filedata = r.raw.read() - with open(fullfilepath, "wb") as outfile: - outfile.write(filedata) - self.localpath = fullfilepath + + if "heasarc" in self.url and self.quicklook is False and self.s3 is not None: + # Download HEASARC hosted data from AWS + key_name = self.url.replace("https://heasarc.gsfc.nasa.gov/FTP/", "") + self.s3.download_file("nasa-heasarc", key_name, fullfilepath) else: - return False + r = requests.get(self.url, stream=True, allow_redirects=True) + if r.ok: + filedata = r.raw.read() + with open(fullfilepath, "wb") as outfile: + outfile.write(filedata) + self.localpath = fullfilepath + else: + return False return True @@ -158,6 +169,7 @@ class Swift_Data(TOOAPI_Baseclass, TOOAPI_ObsID): "fetch", "match", "quiet", + "aws", ] _attributes = ["entries", "status"] @@ -211,12 +223,17 @@ def __init__(self, *args, **kwargs): username for TOO API (default 'anonymous') shared_secret : str shared secret for TOO API (default 'anonymous') + aws : boolean + Download data from AWS instead of HEASARC (note only if uksdc and itsdc + are False). (default 'False'). """ # Parameters self.username = "anonymous" self.obsid = None # Only look in quicklook self.quicklook = False + # Download data from AWS instead of HEASARC + self.aws = False # Download from UK SDC self.uksdc = None # Download from the Italian SDC @@ -294,6 +311,16 @@ def all(self, bool): def _post_process(self): """A place to do things to API results after they have been fetched.""" # Filter out files that don't match `match` expression + if not self.uksdc and not self.itsdc and self.aws is True: + # Set up S3 stuff + config = Config( + connect_timeout=5, + retries={"max_attempts": 0}, + signature_version=UNSIGNED, + ) + s3 = boto3.client("s3", config=config) + for file in self.entries: + file.s3 = s3 if self.match is not None: if type(self.match) is str: self.match = [self.match] @@ -372,9 +399,9 @@ def download(self, outdir=None): for dfile in dfiles: # Don't re-download a file unless clobber=True localfile = f"{self.outdir}/{dfile.path}/{dfile.filename}" - if not self.clobber and os.path.exists(localfile): - print( - f"WARNING: {dfile.filename} exists and not overwritten (set clobber=True to override this)." + if not self.clobber and os.path.exists(localfile) and not self.quiet: + warnings.warn( + f"{dfile.filename} exists and not overwritten (set clobber=True to override this)." ) elif not dfile.download(outdir=self.outdir): self.status.error(f"Error downloading {dfile.filename}") diff --git a/swifttools/swift_too/version.py b/swifttools/swift_too/version.py index 18763c4..68b77c4 100644 --- a/swifttools/swift_too/version.py +++ b/swifttools/swift_too/version.py @@ -1,2 +1,2 @@ -version = "1.2.28" -version_tuple = (1, 2, 28) +version = "1.2.30" +version_tuple = (1, 2, 30) diff --git a/swifttools/ukssdc/APIDocs/ukssdc/xrt_prods/ChangeLog.md b/swifttools/ukssdc/APIDocs/ukssdc/xrt_prods/ChangeLog.md index b49d005..49ad63c 100644 --- a/swifttools/ukssdc/APIDocs/ukssdc/xrt_prods/ChangeLog.md +++ b/swifttools/ukssdc/APIDocs/ukssdc/xrt_prods/ChangeLog.md @@ -1,8 +1,11 @@ -# Change history for the xrt_prods module +# Change history for the `swifttools.ukssdc.xrt_prods` module {#change-history-for-the-swifttoolsukssdcxrt_prods-module} Changes made to this module after its original release will be documented here. -* 2022 DATE TBC. xrt_prods v1.10 released, with swifttools 3.0. +* 2023 September 20. A change was made to the way the targetIDs are selected if you do not supply them but do set `getTargs=True`. This now +uses a cone search where possible and only falls back to getting the targets based on the observation name in the database, if you supplied no position +and the input name cannot be resolved. +* 2022 August 31. xrt_prods v1.10 released, with swifttools 3.0. * This release contains many changes **including deprecating some old behaviour**. Full details are given in the [release notes](ReleaseNotes_v110.md). * 2022 February 10, xrt_prods v 1.9 released, as part of swifttools 2.3.2 **users are encouraged to update ASAP** diff --git a/swifttools/ukssdc/APIDocs/ukssdc/xrt_prods/README.md b/swifttools/ukssdc/APIDocs/ukssdc/xrt_prods/README.md index 551a00d..a6990e5 100644 --- a/swifttools/ukssdc/APIDocs/ukssdc/xrt_prods/README.md +++ b/swifttools/ukssdc/APIDocs/ukssdc/xrt_prods/README.md @@ -1,6 +1,6 @@ # The `xrt_prods` Python module -**This documentation is for `xrt_prods` v1.10, in `swifttools` v3.0** ([Release notes](../ReleaseNotes_v110.md)) +**This documentation is for `xrt_prods` v1.10, in `swifttools` v3.0** ([Release notes](ReleaseNotes_v110.md)) The `swifttools.ukssdc/xrt_prods` Python module provides an interface to the [tools to build Swift-XRT data products for point sources](https://www.swift.ac.uk/user_objects). @@ -19,6 +19,7 @@ The documentation is organised as follows. * [How to retrieve the completed products](RetrieveProducts.md). * [A simple end-to-end tutorial](tutorial.md). * [Miscellaneous methods and advanced usage](advanced.md). + * [Change log](ChangeLog.md) ## Registration diff --git a/swifttools/ukssdc/APIDocs/ukssdc/xrt_prods/RequestJob.md b/swifttools/ukssdc/APIDocs/ukssdc/xrt_prods/RequestJob.md index 5161c10..fe4cdb9 100644 --- a/swifttools/ukssdc/APIDocs/ukssdc/xrt_prods/RequestJob.md +++ b/swifttools/ukssdc/APIDocs/ukssdc/xrt_prods/RequestJob.md @@ -332,8 +332,12 @@ Global parameters are set/retrieved with the `setGlobalPars()` and `getGlobalPar | getT0 | No** | bool | Whether to ask the server to complete the `T0` field automatically | False | | getCoords | No*** | bool | Whether to ask the server to complete the `RA` and `Dec` fields automatically | False | -*: The targetID(s) must be supplied; these tell the server which sets of data to include in your products, and it can be a comma-separated list if more than one targetID corresponds to your object. You can either supply the targetIDs in the `targ` field, or set `getTargs=True`. In the latter case, the server will select all targetIDs in the database where the object name matches that in the `name` field, **and** targets in the database where the XRT field of view will overlap the position in the (`RA`, `Dec`) fields. If `getCoords=True` then the targetID determination is carried out **after** the name has been resolved -into a position. +*: The targetID(s) must be supplied; these tell the server which sets of data to include in your products, and it can be +a comma-separated list if more than one targetID corresponds to your object. You can either supply the targetIDs in the +`targ` field, or set `getTargs=True`. In the latter case, the server will do a cone search around the position +you supplied in the `RA` and `Dec` fields and select all targets within 12′ of the position. If you did +not supply a position it will attempt to determine it by resolving the supplied name. If the name cannot be resolved, +than instead all targetIDs in the database where the object name matches that in the `name` field, will be selected. **: A start time is not mandatory, but is helpful, particularly to zero the time axis on light curve, but it can also be used as a reference point for all other input times. You can either supply it in the `T0` field, or set `getT0=True`. In the latter case the server will try to work it out, either as the trigger time (if the object is a GRB), or as the start time of the first observation of the object. In this case `T0` will be set in the [data returned by the server](ReturnData.md) diff --git a/swifttools/version.py b/swifttools/version.py index 8ee9dfb..9271447 100644 --- a/swifttools/version.py +++ b/swifttools/version.py @@ -1 +1 @@ -__version__ = '3.0.16' +__version__ = '3.0.19'