diff --git a/docs/howto.rst b/docs/howto.rst index a27db55ea..9bc17587f 100644 --- a/docs/howto.rst +++ b/docs/howto.rst @@ -18,3 +18,4 @@ Please feel free to change this by writing more; there are also some entries on editing-documentation working-with-notebooks controlling-logging + working-offline diff --git a/docs/working-offline.rst b/docs/working-offline.rst new file mode 100644 index 000000000..e98341715 --- /dev/null +++ b/docs/working-offline.rst @@ -0,0 +1,42 @@ +Working offline +=============== + +PINT takes advantage of the Internet to ensure that you are always working with +up-to-date clock correction and other files. Normally this is very convenient, +but sometimes one doesn't have access to the Internet. This could be because +you were on an airplane or a mountaintop or inside a secure facility and can't +wait to time your pulsar, or it could be because you are running PINT on the +nodes of a cluster that don't have connections to the general Internet. PINT +has some tools to support these situations. + +Here are some general guidelines: + +- You will need to ensure that you pre-download an appropriate collection of files before you disconnect from the Internet. +- You can find yourself using out-of-date clock files; PINT will by default emit warnings if this happens. +- You can find yourself using out-of-date IERS (Earth-rotation or leap-second) files. Newer versions of Astropy will warn you if this occurs, but older ones may not. + +The files that PINT and Astropy go looking for on the Internet get downloaded +and stored in the "Astropy cache". This is a location, usually in your home +directory, that depends on your operating system, version of Astropy, and +possibly environment variable settings. If you need to know where this is, you +can run ``...``. In general this directory will work and be accessible when you +need it to; the Astropy documentation has some more details on this. + +Pre-loading the Astropy cache should be easy: simply call +:func:`pint.utils.preload_cache` and it will download every file it thinks you +might need in the cache. You can often get away with less; if you have been +running your scripts fine, your cache probably already contains all the files +you need. + +When you want to ensure that PINT and Astropy do not reach out to the Internet, +you can call :func:`pint.utils.set_no_internet`; this will set several Astropy +config options for the duration of the current Python session, and together +they will ensure that your script does not reach out to the Internet and +out-of-date files result in warnings. + +If you want a longer-lasting setup to disable Internet access, you can set up +an Astropy config file. See ... for details on how to do this. Once the file +exists, you can set some options to achieve the same effect as +:func:`pint.utils.set_no_internet`. Unfortunately, which options are available +differs between Astropy 4.0, 4.3, 5.0, and 5.1; support for disconnected +operation has been gradually improving. diff --git a/src/pint/utils.py b/src/pint/utils.py index d4a56c13f..d96e44bd9 100644 --- a/src/pint/utils.py +++ b/src/pint/utils.py @@ -38,7 +38,7 @@ import sys import textwrap from collections import OrderedDict -from contextlib import contextmanager +from contextlib import contextmanager, ExitStack from copy import deepcopy from io import StringIO from pathlib import Path @@ -52,6 +52,7 @@ import scipy.optimize.zeros as zeros from loguru import logger as log from scipy.special import fdtrc +from astropy.utils.iers import IERS_Auto import pint import pint.pulsar_ecliptic @@ -1095,7 +1096,7 @@ def weighted_mean(arrin, weights_in, inputmean=None, calcerr=False, sdev=False): def ELL1_check( A1: u.cm, E: u.dimensionless_unscaled, TRES: u.us, NTOA: int, outstring=True ): - """Check for validity of assumptions in ELL1 binary model + r"""Check for validity of assumptions in ELL1 binary model Checks whether the assumptions that allow ELL1 to be safely used are satisfied. To work properly, we should have: @@ -1652,3 +1653,69 @@ def compute_hash(filename): while block := f.read(blocks * h.block_size): h.update(block) return h.digest() + + +def preload_cache(extra_ephemerides=None): + """Ensure that all files PINT needs are in the Astropy cache. + + This requests all clock corrections in the global repository, a list of + standard Solar System ephemerides, and up-to-date IERS tables. Once + complete, PINT should be able to function without an Internet connection. + + Note that you may need to set some Astropy configuration options to prevent + Astropy from requesting new IERS data after a month. + + For more information see http://docs.astropy.org/en/stable/utils/data.html#using-astropy-with-limited-or-no-internet-access + """ + from pint.observatory.global_clock_corrections import update_all + from pint.solar_system_ephemerides import load_kernel + + update_all() + IERS_Auto.open() + ephemerides = [ + "de200", + "de405", + "de421", + "de430", + "de430t", + "de432s", + "de434", + "de436", + "de436t", + "de440", + "de440s", + ] + if extra_ephemerides is not None: + ephemerides.extend(extra_ephemerides) + for e in ephemerides: + load_kernel(e) + + +def set_no_internet(mode="warn"): + """Set Astropy and PINT to run without Internet access. + + The sets up a number of Astropy configuration options. If you want to achieve + this effect without having to add this line to your scripts, you can create + an Astropy config file and edit it to contain these same options. See + https://docs.astropy.org/en/stable/config/index.html#astropy-config + for details of how to do this. + + Parameters + ---------- + mode : 'warn' or 'ignore' + What to do when files appear to be out of date + """ + import astropy.utils.data + import astropy.utils.iers + + if hasattr(astropy.utils.data.conf, "allow_internet"): + astropy.utils.data.conf.allow_internet = False + astropy.utils.iers.conf.remote_timeout = 0 + # if hasattr(astropy.utils.iers.conf, "auto_download"): + # astropy.utils.iers.conf.auto_download = False + # else: + # astropy.utils.iers.conf.remote_timeout = 0 + if hasattr(astropy.utils.iers.conf, "iers_degraded_accuracy"): + astropy.utils.iers.conf.iers_degraded_accuracy = "warn" + # else: + # astropy.utils.iers.conf.auto_max_age = None diff --git a/tests/test_utils.py b/tests/test_utils.py index 553b7d3d7..7c3214172 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -6,10 +6,12 @@ import astropy.constants as c import astropy.units as u +import astropy.utils.data import numpy as np import pytest import scipy.stats from astropy.time import Time +from astropy.config import set_temp_cache, set_temp_config from hypothesis import assume, example, given from hypothesis.extra.numpy import array_shapes, arrays, scalar_dtypes from hypothesis.strategies import ( @@ -51,6 +53,8 @@ open_or_use, taylor_horner, taylor_horner_deriv, + preload_cache, + set_no_internet, ) @@ -765,3 +769,100 @@ def test_compute_hash_accepts_no_change(a): h_b = compute_hash(g) assert h_a == h_b + + +def test_preload_cache_gives_all_expected_files(tmp_path): + with set_temp_cache(): + preload_cache() + expected = [ + # "ftp://ssd.jpl.nasa.gov/pub/eph/planets/bsp/de200.bsp", + "http://data.astropy.org/coordinates/sites.json", + "http://hpiers.obspm.fr/iers/eop/eopc04/eopc04_IAU2000.62-now", + # Ephemerides are sometimes pulled in via Astropy and sometimes via PINT + # Astropy + "https://naif.jpl.nasa.gov/pub/naif/generic_kernels/spk/planets/de430.bsp", + "https://naif.jpl.nasa.gov/pub/naif/generic_kernels/spk/planets/de432s.bsp", + "https://naif.jpl.nasa.gov/pub/naif/generic_kernels/spk/planets/de440.bsp", + "https://naif.jpl.nasa.gov/pub/naif/generic_kernels/spk/planets/de440s.bsp", + # PINT + "https://data.nanograv.org/static/data/ephem/de200.bsp", + "https://data.nanograv.org/static/data/ephem/de405.bsp", + "https://data.nanograv.org/static/data/ephem/de421.bsp", + "https://data.nanograv.org/static/data/ephem/de430t.bsp", + "https://data.nanograv.org/static/data/ephem/de434.bsp", + "https://data.nanograv.org/static/data/ephem/de436.bsp", + "https://data.nanograv.org/static/data/ephem/de436t.bsp", + # IERS A table + "https://datacenter.iers.org/data/9/finals2000A.all", + # "https://maia.usno.navy.mil/ser7/finals2000A.all", # This is the mirror address, should never be needed I think? + # "ftp://anonymous:mail%40astropy.org@gdc.cddis.eosdis.nasa.gov/pub/products/iers/finals2000A.all", # Another mirror that shouldn't be needed + "https://hpiers.obspm.fr/iers/bul/bulc/Leap_Second.dat", + # Clock corrections + "https://raw.githubusercontent.com/ipta/pulsar-clock-corrections/main/T2runtime/clock/ao2gps.clk", + "https://raw.githubusercontent.com/ipta/pulsar-clock-corrections/main/T2runtime/clock/ao2gps_tempo2.clk", + "https://raw.githubusercontent.com/ipta/pulsar-clock-corrections/main/T2runtime/clock/ao2nist.clk", + "https://raw.githubusercontent.com/ipta/pulsar-clock-corrections/main/T2runtime/clock/chime2gps.clk", + "https://raw.githubusercontent.com/ipta/pulsar-clock-corrections/main/T2runtime/clock/eff2gps.clk", + "https://raw.githubusercontent.com/ipta/pulsar-clock-corrections/main/T2runtime/clock/effix2gps.clk", + "https://raw.githubusercontent.com/ipta/pulsar-clock-corrections/main/T2runtime/clock/gbt2gps.clk", + "https://raw.githubusercontent.com/ipta/pulsar-clock-corrections/main/T2runtime/clock/gbt2gps_tempo2.clk", + "https://raw.githubusercontent.com/ipta/pulsar-clock-corrections/main/T2runtime/clock/gmrt2gps.clk", + "https://raw.githubusercontent.com/ipta/pulsar-clock-corrections/main/T2runtime/clock/gps2utc.clk", + "https://raw.githubusercontent.com/ipta/pulsar-clock-corrections/main/T2runtime/clock/gps2utc_c0p.clk", + "https://raw.githubusercontent.com/ipta/pulsar-clock-corrections/main/T2runtime/clock/gps2utc_cc.clk", + "https://raw.githubusercontent.com/ipta/pulsar-clock-corrections/main/T2runtime/clock/gps2utc_tempo2.clk", + "https://raw.githubusercontent.com/ipta/pulsar-clock-corrections/main/T2runtime/clock/jb2gps.clk", + "https://raw.githubusercontent.com/ipta/pulsar-clock-corrections/main/T2runtime/clock/jbdfb2jb.clk", + "https://raw.githubusercontent.com/ipta/pulsar-clock-corrections/main/T2runtime/clock/jbroach2jb.clk", + "https://raw.githubusercontent.com/ipta/pulsar-clock-corrections/main/T2runtime/clock/mk2utc.clk", + "https://raw.githubusercontent.com/ipta/pulsar-clock-corrections/main/T2runtime/clock/mk2utc_observatory.clk", + "https://raw.githubusercontent.com/ipta/pulsar-clock-corrections/main/T2runtime/clock/mo2gps.clk", + "https://raw.githubusercontent.com/ipta/pulsar-clock-corrections/main/T2runtime/clock/ncyobs2obspm.clk", + "https://raw.githubusercontent.com/ipta/pulsar-clock-corrections/main/T2runtime/clock/obspm2gps.clk", + "https://raw.githubusercontent.com/ipta/pulsar-clock-corrections/main/T2runtime/clock/pks2gps.clk", + "https://raw.githubusercontent.com/ipta/pulsar-clock-corrections/main/T2runtime/clock/srt2gps.clk", + "https://raw.githubusercontent.com/ipta/pulsar-clock-corrections/main/T2runtime/clock/tai2tt_bipm01.clk", + "https://raw.githubusercontent.com/ipta/pulsar-clock-corrections/main/T2runtime/clock/tai2tt_bipm05.clk", + "https://raw.githubusercontent.com/ipta/pulsar-clock-corrections/main/T2runtime/clock/tai2tt_bipm06.clk", + "https://raw.githubusercontent.com/ipta/pulsar-clock-corrections/main/T2runtime/clock/tai2tt_bipm2003.clk", + "https://raw.githubusercontent.com/ipta/pulsar-clock-corrections/main/T2runtime/clock/tai2tt_bipm2004.clk", + "https://raw.githubusercontent.com/ipta/pulsar-clock-corrections/main/T2runtime/clock/tai2tt_bipm2010.clk", + "https://raw.githubusercontent.com/ipta/pulsar-clock-corrections/main/T2runtime/clock/tai2tt_bipm2012.clk", + "https://raw.githubusercontent.com/ipta/pulsar-clock-corrections/main/T2runtime/clock/tai2tt_bipm2013.clk", + "https://raw.githubusercontent.com/ipta/pulsar-clock-corrections/main/T2runtime/clock/tai2tt_bipm2014.clk", + "https://raw.githubusercontent.com/ipta/pulsar-clock-corrections/main/T2runtime/clock/tai2tt_bipm2015.clk", + "https://raw.githubusercontent.com/ipta/pulsar-clock-corrections/main/T2runtime/clock/tai2tt_bipm2016.clk", + "https://raw.githubusercontent.com/ipta/pulsar-clock-corrections/main/T2runtime/clock/tai2tt_bipm2017.clk", + "https://raw.githubusercontent.com/ipta/pulsar-clock-corrections/main/T2runtime/clock/tai2tt_bipm2018.clk", + "https://raw.githubusercontent.com/ipta/pulsar-clock-corrections/main/T2runtime/clock/tai2tt_bipm2019.clk", + "https://raw.githubusercontent.com/ipta/pulsar-clock-corrections/main/T2runtime/clock/tai2tt_bipm2020.clk", + "https://raw.githubusercontent.com/ipta/pulsar-clock-corrections/main/T2runtime/clock/tai2tt_bipm2021.clk", + "https://raw.githubusercontent.com/ipta/pulsar-clock-corrections/main/T2runtime/clock/tai2tt_bipm92.clk", + "https://raw.githubusercontent.com/ipta/pulsar-clock-corrections/main/T2runtime/clock/vla2gps.clk", + "https://raw.githubusercontent.com/ipta/pulsar-clock-corrections/main/T2runtime/clock/vla2nist.clk", + "https://raw.githubusercontent.com/ipta/pulsar-clock-corrections/main/T2runtime/clock/wsrt2gps.clk", + "https://raw.githubusercontent.com/ipta/pulsar-clock-corrections/main/index.txt", + "https://raw.githubusercontent.com/ipta/pulsar-clock-corrections/main/tempo/clock/leap.sec", + "https://raw.githubusercontent.com/ipta/pulsar-clock-corrections/main/tempo/clock/time_ao.dat", + "https://raw.githubusercontent.com/ipta/pulsar-clock-corrections/main/tempo/clock/time_fast.dat", + "https://raw.githubusercontent.com/ipta/pulsar-clock-corrections/main/tempo/clock/time_gb140.dat", + "https://raw.githubusercontent.com/ipta/pulsar-clock-corrections/main/tempo/clock/time_gb853.dat", + "https://raw.githubusercontent.com/ipta/pulsar-clock-corrections/main/tempo/clock/time_gbt.dat", + "https://raw.githubusercontent.com/ipta/pulsar-clock-corrections/main/tempo/clock/time_jb.dat", + "https://raw.githubusercontent.com/ipta/pulsar-clock-corrections/main/tempo/clock/time_nuppi.dat", + "https://raw.githubusercontent.com/ipta/pulsar-clock-corrections/main/tempo/clock/time_pks.dat", + "https://raw.githubusercontent.com/ipta/pulsar-clock-corrections/main/tempo/clock/time_vla.dat", + "https://raw.githubusercontent.com/ipta/pulsar-clock-corrections/main/tempo/clock/time_wsrt.dat", + "https://raw.githubusercontent.com/ipta/pulsar-clock-corrections/main/tempo/clock/ut1.dat", + ] + for k in expected: + assert k in astropy.utils.data.cache_contents() + + +def test_no_internet_forbids_internet(tmp_path, monkeypatch): + monkeypatch.delattr("astropy.utils.data.download_file") + with set_temp_cache(tmp_path): + with set_temp_config(): + set_no_internet() + (Time.now() + 1 * u.year).utc.tai.ut1 + assert not astropy.utils.data.cache_contents() diff --git a/tox.ini b/tox.ini index d35d05e93..29c79b5d3 100644 --- a/tox.ini +++ b/tox.ini @@ -66,6 +66,58 @@ deps = numdifftools commands = {posargs:pytest} +[testenv:astropy40] +description = + Run known-tricky tests with Astropy 4.0 specifically +basepython = python3.8 +deps = + numpy==1.17.* + astropy==4.0 + pytest + coverage + hypothesis + numdifftools +commands = {posargs:pytest tests/test_utils.py} + +[testenv:astropy43] +description = + Run known-tricky tests with Astropy 4.3 specifically +basepython = python3.8 +deps = + numpy==1.17.* + astropy==4.3 + pytest + coverage + hypothesis + numdifftools +commands = {posargs:pytest tests/test_utils.py} + +[testenv:astropy50] +description = + Run known-tricky tests with Astropy 5.0 specifically +basepython = python3.8 +deps = + numpy==1.18.* + astropy==5.0 + pytest + coverage + hypothesis + numdifftools +commands = {posargs:pytest tests/test_utils.py} + +[testenv:astropy51] +description = + Run known-tricky tests with Astropy 5.1 specifically +basepython = python3.8 +deps = + numpy + astropy==5.1 + pytest + coverage + hypothesis + numdifftools +commands = {posargs:pytest tests/test_utils.py} + [testenv:report] skip_install = true deps = coverage