Merge pull request #1 from jak574/master

Push to version 3.0.19

setup.cfg

@@ -1,6 +1,6 @@
 [metadata]
 name = swifttools 
-version = 3.0.16
+version = 3.0.19
 description = Tools for users of the Swift satellite
 long_description = file: README.md
 long_description_content_type = text/markdown
@@ -32,7 +32,7 @@ install_requires =
     pandas 
     tabulate 
     numpy
-
+    boto3
 [bdist_wheel]
 universal = 0
 python-tag = py36

swifttools/swift_too/ChangeLog.md

@@ -6,13 +6,24 @@
 
 #### Author: Jamie A. Kennea (Penn State)
 
+## `swifttools` 3.0.18 / `swift_too` 1.2.30
+
+**Oct 31, 2023**: Resolve issue with noisy warnings in `Data` even if `quiet=True`
+
+* When downloading files using `Data` module, if those files already exist on disk a `warning` is now issued, instead of a printed warning. If `quiet=True` no warnings are issued.
+
+## `swifttools` 3.0.17 / `swift_too` 1.2.29
+
+**Oct 31, 2023**: Add AWS download support to `Data` class.
+
+* Added option to download data from AWS instead of HEASARC. Add argument `aws=True` to `Data` class arguments to default to AWS downlink. For details of AWS data hosting, see [here](https://heasarc.gsfc.nasa.gov/docs/archive/cloud.html).
+
 ## `swifttools` 3.0.16 / `swift_too` 1.2.28
 
 **March 22, 2023**: Warning about incorrectly formatted ISO8601 dates
 
 * ISO8601 formatted dates without a timezone was assuming localtime, causing confusion. ISO8601 formatted dates should include a timezone specifier, e.g. '2022-01-01T00:00:00Z'. If no timezone is given, the code now issues a warning about this issue. Please use the "Z" for UTC times, as required for for the ISO8601 specification.
 
-
 ## `swifttools` 3.0.15 / `swift_too` 1.2.27
 
 **March 17, 2023**: `TOORequests` fix

swifttools/swift_too/api_common.py

@@ -12,8 +12,11 @@
 
 # Make Warnings a little less weird
 formatwarning_orig = warnings.formatwarning
-warnings.formatwarning = lambda message, category, filename, lineno, line=None: \
-    formatwarning_orig(message, category, filename, lineno, line='')
+warnings.formatwarning = (
+    lambda message, category, filename, lineno, line=None: formatwarning_orig(
+        message, category, filename, lineno, line=""
+    )
+)
 
 # Configure for IPV4 only due to issue
 requests.packages.urllib3.util.connection.HAS_IPV6 = False
@@ -96,7 +99,9 @@ def convert_to_dt(value, isutc=False, outfunc=datetime):
         elif re.match(_iso8601_regex, value):
             dtvalue = parser.parse(value)
             if dtvalue.tzinfo is None:
-                warnings.warn("ISO8601 formatted dates should be supplied with timezone. ISO8601 dates with no timezone will be assumed to be localtime and then converted to UTC.")
+                warnings.warn(
+                    "ISO8601 formatted dates should be supplied with timezone. ISO8601 dates with no timezone will be assumed to be localtime and then converted to UTC."
+                )
             dtvalue = dtvalue.astimezone(timezone.utc).replace(tzinfo=None)
         else:
             raise ValueError(

swifttools/swift_too/swift_data.py

@@ -4,6 +4,10 @@
 import requests
 import os
 from fnmatch import fnmatch
+import warnings
+import boto3
+from botocore import UNSIGNED
+from botocore.client import Config
 
 try:
     from tqdm.auto import tqdm
@@ -47,6 +51,7 @@ class Swift_DataFile(TOOAPI_Baseclass):
     quicklook = None
     type = None
     localpath = None
+    s3 = None
     outdir = "."
     # Core API definitions
     _parameters = ["filename", "path", "url", "quicklook", "type"]
@@ -70,14 +75,20 @@ def download(self, outdir=None):
 
         # Download the data
         fullfilepath = os.path.join(self.outdir, self.path, self.filename)
-        r = requests.get(self.url, stream=True, allow_redirects=True)
-        if r.ok:
-            filedata = r.raw.read()
-            with open(fullfilepath, "wb") as outfile:
-                outfile.write(filedata)
-            self.localpath = fullfilepath
+
+        if "heasarc" in self.url and self.quicklook is False and self.s3 is not None:
+            # Download HEASARC hosted data from AWS
+            key_name = self.url.replace("https://heasarc.gsfc.nasa.gov/FTP/", "")
+            self.s3.download_file("nasa-heasarc", key_name, fullfilepath)
         else:
-            return False
+            r = requests.get(self.url, stream=True, allow_redirects=True)
+            if r.ok:
+                filedata = r.raw.read()
+                with open(fullfilepath, "wb") as outfile:
+                    outfile.write(filedata)
+                self.localpath = fullfilepath
+            else:
+                return False
 
         return True
 
@@ -158,6 +169,7 @@ class Swift_Data(TOOAPI_Baseclass, TOOAPI_ObsID):
         "fetch",
         "match",
         "quiet",
+        "aws",
     ]
     _attributes = ["entries", "status"]
 
@@ -211,12 +223,17 @@ def __init__(self, *args, **kwargs):
             username for TOO API (default 'anonymous')
         shared_secret : str
             shared secret for TOO API (default 'anonymous')
+        aws : boolean
+            Download data from AWS instead of HEASARC (note only if uksdc and itsdc
+            are False). (default 'False').
         """
         # Parameters
         self.username = "anonymous"
         self.obsid = None
         # Only look in quicklook
         self.quicklook = False
+        # Download data from AWS instead of HEASARC
+        self.aws = False
         # Download from UK SDC
         self.uksdc = None
         # Download from the Italian SDC
@@ -294,6 +311,16 @@ def all(self, bool):
     def _post_process(self):
         """A place to do things to API results after they have been fetched."""
         # Filter out files that don't match `match` expression
+        if not self.uksdc and not self.itsdc and self.aws is True:
+            # Set up S3 stuff
+            config = Config(
+                connect_timeout=5,
+                retries={"max_attempts": 0},
+                signature_version=UNSIGNED,
+            )
+            s3 = boto3.client("s3", config=config)
+            for file in self.entries:
+                file.s3 = s3
         if self.match is not None:
             if type(self.match) is str:
                 self.match = [self.match]
@@ -372,9 +399,9 @@ def download(self, outdir=None):
         for dfile in dfiles:
             # Don't re-download a file unless clobber=True
             localfile = f"{self.outdir}/{dfile.path}/{dfile.filename}"
-            if not self.clobber and os.path.exists(localfile):
-                print(
-                    f"WARNING: {dfile.filename} exists and not overwritten (set clobber=True to override this)."
+            if not self.clobber and os.path.exists(localfile) and not self.quiet:
+                warnings.warn(
+                    f"{dfile.filename} exists and not overwritten (set clobber=True to override this)."
                 )
             elif not dfile.download(outdir=self.outdir):
                 self.status.error(f"Error downloading {dfile.filename}")

swifttools/swift_too/version.py

@@ -1,2 +1,2 @@
-version = "1.2.28"
-version_tuple = (1, 2, 28)
+version = "1.2.30"
+version_tuple = (1, 2, 30)

swifttools/ukssdc/APIDocs/ukssdc/xrt_prods/ChangeLog.md

@@ -1,8 +1,11 @@
-# Change history for the xrt_prods module
+# Change history for the `swifttools.ukssdc.xrt_prods` module {#change-history-for-the-swifttoolsukssdcxrt_prods-module}
 
 Changes made to this module after its original release will be documented here.
 
-* 2022 DATE TBC. xrt_prods v1.10 released, with swifttools 3.0.
+* 2023 September 20. A change was made to the way the targetIDs are selected if you do not supply them but do set `getTargs=True`. This now
+uses a cone search where possible and only falls back to getting the targets based on the observation name in the database, if you supplied no position
+and the input name cannot be resolved.
+* 2022 August 31. xrt_prods v1.10 released, with swifttools 3.0.
   * This release contains many changes **including deprecating some old behaviour**. Full details
   are given in the [release notes](ReleaseNotes_v110.md).
 * 2022 February 10, xrt_prods v 1.9 released, as part of swifttools 2.3.2 **users are encouraged to update ASAP**

swifttools/ukssdc/APIDocs/ukssdc/xrt_prods/README.md

@@ -1,6 +1,6 @@
 # The `xrt_prods` Python module
 
-**This documentation is for `xrt_prods` v1.10, in `swifttools` v3.0** ([Release notes](../ReleaseNotes_v110.md))
+**This documentation is for `xrt_prods` v1.10, in `swifttools` v3.0** ([Release notes](ReleaseNotes_v110.md))
 
 The `swifttools.ukssdc/xrt_prods` Python module provides an interface to the [tools to build Swift-XRT data products for
 point sources](https://www.swift.ac.uk/user_objects).
@@ -19,6 +19,7 @@ The documentation is organised as follows.
  * [How to retrieve the completed products](RetrieveProducts.md).
  * [A simple end-to-end tutorial](tutorial.md).
  * [Miscellaneous methods and advanced usage](advanced.md).
+ * [Change log](ChangeLog.md)
 
 
 ## Registration

swifttools/ukssdc/APIDocs/ukssdc/xrt_prods/RequestJob.md

@@ -332,8 +332,12 @@ Global parameters are set/retrieved with the `setGlobalPars()` and `getGlobalPar
 | getT0  | No** | bool | Whether to ask the server to complete the `T0` field automatically | False |
 | getCoords  | No*** | bool | Whether to ask the server to complete the `RA` and `Dec` fields automatically | False |
 
-*: The targetID(s) must be supplied; these tell the server which sets of data to include in your products, and it can be a comma-separated list if more than one targetID corresponds to your object. You can either supply the targetIDs in the `targ` field, or set `getTargs=True`. In the latter case, the server will select all targetIDs in the database where the object name matches that in the `name` field, **and**  targets in the database where the XRT field of view will overlap the position in the (`RA`, `Dec`) fields. If `getCoords=True` then the targetID determination is carried out **after** the name has been resolved
-into a position.
+*: The targetID(s) must be supplied; these tell the server which sets of data to include in your products, and it can be
+a comma-separated list if more than one targetID corresponds to your object. You can either supply the targetIDs in the
+`targ` field, or set `getTargs=True`. In the latter case, the server will do a cone search around the position
+you supplied in the `RA` and `Dec` fields and select all targets within 12′ of the position. If you did
+not supply a position it will attempt to determine it by resolving the supplied name. If the name cannot be resolved,
+than instead all targetIDs in the database where the object name matches that in the `name` field, will be selected.
 
 **: A start time is not mandatory, but is helpful, particularly to zero the time axis on light curve, but it can also be used as a reference point for all other input times. You can either supply it in the `T0` field, or set `getT0=True`. In the latter case the server will try to work it out, either as the trigger time (if the object is a GRB), or as the start time of the first observation of the object. In this case `T0` will be set in the [data returned by the server](ReturnData.md)
 

swifttools/version.py

@@ -1 +1 @@
-__version__ = '3.0.16'
+__version__ = '3.0.19'