Merge pull request #41 from octoenergy/update

Update to tf 2.8 and python 3.9 and drop s3fs dependency

.circleci/config.yml

@@ -3,7 +3,7 @@ version: 2.1
 jobs:
   sync-lint:
     docker:
-      - image: circleci/python:3.6.9
+      - image: circleci/python:3.9.5
 
     working_directory: ~/timeserio
 
@@ -47,7 +47,7 @@ jobs:
 
   test:
     docker:
-      - image: circleci/python:3.6.9
+      - image: circleci/python:3.9.5
     parallelism: 4
 
     working_directory: ~/timeserio
@@ -74,7 +74,7 @@ jobs:
 
   doctest:
     docker:
-      - image: circleci/python:3.6.9
+      - image: circleci/python:3.9.5
 
     working_directory: ~/timeserio
 
@@ -95,7 +95,7 @@ jobs:
 
   build-release-test:
     docker:
-      - image: circleci/python:3.6.9
+      - image: circleci/python:3.9.5
 
     working_directory: ~/timeserio
 
@@ -128,7 +128,7 @@ jobs:
 
   release-prod:
     docker:
-      - image: circleci/python:3.6.9
+      - image: circleci/python:3.9.5
 
     working_directory: ~/timeserio
 
@@ -153,7 +153,7 @@ jobs:
 
   docs-build:
     docker:
-      - image: circleci/python:3.6.9
+      - image: circleci/python:3.9.5
 
     working_directory: ~/timeserio
 

Makefile

@@ -19,7 +19,7 @@ sync:
 	pipenv sync --dev
 
 lock:
-	pipenv lock --dev 
+	pipenv lock --dev
 
 test:
 	pipenv run pytest tests/

Pipfile

@@ -2,25 +2,25 @@
 name = "pypi"
 url = "https://pypi.org/simple"
 verify_ssl = true
-
 [dev-packages]
 mccabe           = "*"
 pep8-naming      = "*"
 flake8           = "*"
 pydocstyle       = "*"
 mypy             = "*"
 yapf             = "*"
-pytest           = ">=5.3, <6"
+pytest           = "*"
 pytest-cov       = "*"
 pytest-benchmark = "*"
-pytest-xdist     = "<2.0"
+pytest-xdist     = "*"
 pytest-mock      = "*"
+pytest-lazy-fixture = "*"
 moto             = "*"
 glob2            = "*"
 codecov          = "*"
 #
 dask             = {extras = ["dataframe", "distributed"], version = "*"}
-tensorflow       = "==1.15.2"
+tensorflow       = "==2.8.0"
 pyarrow          = ">=0.15.1"
 # sphinx docs
 sphinx-autodoc-typehints = ">=1.6.0"
@@ -32,10 +32,9 @@ nbsphinx-link            = ">=1.2.0"
 ipython                  = ">=7.5.0"
 # release tools
 twine = "*"
-tensorpandas = {git = "https://github.com/ig248/tensorpandas.git"}
 
 [packages]
 timeserio        = {editable = true,path = "."}
 
 [requires]
-python_version    = "3.6"
+python_version    = "3.9"

dockerfiles/Dockerfile.cpu

@@ -1,27 +1,28 @@
 # tensorflow no cpu and running python 3.6 
 # Loosely based in what tensorflow tools generate. 
 # More info
-ARG UBUNTU_VERSION=18.04
+ARG UBUNTU_VERSION=20.04
 
 FROM ubuntu:${UBUNTU_VERSION} as base
 
 # See http://bugs.python.org/issue19846
 ENV LANG C.UTF-8
+ENV DEBIAN_FRONTEND=noninteractive
 
 RUN apt-get update && apt-get install -y --no-install-recommends \
         build-essential \
         curl \
         software-properties-common \
-        python3.6 python3.6-dev python3-distutils\
+        python3.9 python3.9-dev python3-distutils\
         && \
     apt-get clean && \
     rm -rf /var/lib/apt/lists/*
 
-RUN curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py && python3.6 get-pip.py && rm get-pip.py
-RUN python3.6 -m pip install --no-cache-dir -U pip
-RUN python3.6 -m pip install --no-cache-dir -U setuptools
+RUN curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py && python3.9 get-pip.py && rm get-pip.py
+RUN python3.9 -m pip install --no-cache-dir -U pip
+RUN python3.9 -m pip install --no-cache-dir -U setuptools
 # Some TF tools expect a "python" binary
-RUN ln -s /usr/bin/python3.6 /usr/local/bin/python
+RUN ln -s /usr/bin/python3.9 /usr/local/bin/python
 
 ARG TF_PACKAGE=tensorflow
 RUN pip install ${TF_PACKAGE}

dockerfiles/Dockerfile.gpu

@@ -1,57 +1,73 @@
-# tensorflow with cuda 9 and running python 3.6 
-# Loosely based in what tensorflow tools generate. 
-# More info
-ARG UBUNTU_VERSION=16.04
+# tensorflow with cuda 11.2 and running python 3.9
+# Loosely based in what tensorflow tools generate.
+ARG UBUNTU_VERSION=20.04
+ARG CUDA=11.2
+FROM nvidia/cuda:${CUDA}.1-base-ubuntu${UBUNTU_VERSION} as base
 
-FROM nvidia/cuda:9.0-base-ubuntu${UBUNTU_VERSION} as base
+# CUDA is specified again because the FROM directive resets ARGs
+# (but the default value is retained if set previously)
+ARG CUDA
+ARG CUDNN=8.1.0.77-1
+ARG LIBNVINFER=7.2.2-1
+ARG LIBNVINFER_MAJOR_VERSION=7
 
+ENV DEBIAN_FRONTEND=noninteractive
+
+# Needed for string substitution
+SHELL ["/bin/bash", "-c"]
 RUN apt-get update && apt-get install -y --no-install-recommends \
         build-essential \
-        cuda-command-line-tools-9-0 \
-        cuda-cublas-9-0 \
-        cuda-cufft-9-0 \
-        cuda-curand-9-0 \
-        cuda-cusolver-9-0 \
-        cuda-cusparse-9-0 \
+        cuda-command-line-tools-${CUDA/./-} \
+        libcublas-${CUDA/./-} \
+        cuda-nvrtc-${CUDA/./-} \
+        libcufft-${CUDA/./-} \
+        libcurand-${CUDA/./-} \
+        libcusolver-${CUDA/./-} \
+        libcusparse-${CUDA/./-} \
         curl \
-        libcudnn7=7.2.1.38-1+cuda9.0 \
-        libnccl2=2.2.13-1+cuda9.0 \
+        libcudnn8=${CUDNN}+cuda${CUDA} \
         libfreetype6-dev \
         libhdf5-serial-dev \
-        libpng12-dev \
         libzmq3-dev \
         pkg-config \
-        rsync \
         software-properties-common \
-        unzip \
-        && \
-    apt-get clean && \
-    rm -rf /var/lib/apt/lists/*
+        unzip
 
 RUN apt-get update && \
-        apt-get install nvinfer-runtime-trt-repo-ubuntu1604-4.0.1-ga-cuda9.0 && \
+        apt-key adv --fetch-keys \
+        https://developer.download.nvidia.com/compute/machine-learning/repos/ubuntu1804/x86_64/7fa2af80.pub && \
+        echo "deb https://developer.download.nvidia.com/compute/machine-learning/repos/ubuntu1804/x86_64 /"  > /etc/apt/sources.list.d/tensorRT.list && \
         apt-get update && \
-        apt-get install libnvinfer4=4.1.2-1+cuda9.0
+        apt-get install -y --no-install-recommends \
+        libnvinfer${LIBNVINFER_MAJOR_VERSION}=${LIBNVINFER}+cuda11.0 \
+        libnvinfer-plugin${LIBNVINFER_MAJOR_VERSION}=${LIBNVINFER}+cuda11.0 \
+        && \
+        apt-get clean && \
+        rm -rf /var/lib/apt/lists/*
 
 # For CUDA profiling, TensorFlow requires CUPTI.
-ENV LD_LIBRARY_PATH /usr/local/cuda/extras/CUPTI/lib64:$LD_LIBRARY_PATH
+ENV LD_LIBRARY_PATH /usr/local/cuda-11.0/targets/x86_64-linux/lib:/usr/local/cuda/extras/CUPTI/lib64:/usr/local/cuda/lib64:$LD_LIBRARY_PATH
 
+# Link the libcuda stub to the location where tensorflow is searching for it and reconfigure
+# dynamic linker run-time bindings
+RUN ln -s /usr/local/cuda/lib64/stubs/libcuda.so /usr/local/cuda/lib64/stubs/libcuda.so.1 \
+    && echo "/usr/local/cuda/lib64/stubs" > /etc/ld.so.conf.d/z-cuda-stubs.conf \
+    && ldconfig
 
 # See http://bugs.python.org/issue19846
 ENV LANG C.UTF-8
 
 RUN apt-get update \
-       && add-apt-repository -y ppa:deadsnakes/ppa \
        && apt-get update \
-       && apt-get install -y python3.6 python3.6-dev python3.6-venv \
+       && apt-get install -y python3.9 python3.9-dev python3.9-venv \
        && apt-get clean \
        && rm -rf /var/lib/apt/lists/*
 
-RUN curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py && python3.6 get-pip.py && rm get-pip.py
-RUN python3.6 -m pip install --no-cache-dir -U pip
-RUN python3.6 -m pip install --no-cache-dir -U setuptools
+RUN curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py && python3.9 get-pip.py && rm get-pip.py
+RUN python3.9 -m pip install --no-cache-dir -U pip
+RUN python3.9 -m pip install --no-cache-dir -U setuptools
 # Some TF tools expect a "python" binary
-RUN ln -s /usr/bin/python3.6 /usr/local/bin/python
+RUN ln -s /usr/bin/python3.9 /usr/local/bin/python
 
 ARG TF_PACKAGE=tensorflow-gpu
 RUN pip install ${TF_PACKAGE}

dockerfiles/Makefile

@@ -1,5 +1,5 @@
-TF_VERSION=1.12.0
-PY_VERSION?=py36
+TF_VERSION=2.8.0
+PY_VERSION?=py39
 CPU_IMAGE?=tensorflow:${TF_VERSION}-${PY_VERSION}
 GPU_IMAGE?=tensorflow:${TF_VERSION}-gpu-${PY_VERSION}
 DOCKER_NAMESPACE?=krakentechnologies

dockerfiles/README.md

@@ -1,7 +1,12 @@
-# Tensorflow 1.12 + Python 3.6 images
+# Tensorflow 1.12 + Python 3.9 images
 
-Official tensorflow 1.12 docker images don't really fit our needs as they depend on python3.5 rather than pyton3.6. For this reason two docker files can be find in the `dockerfiles` folder, one for a cpu image and another for gpu image. 
+Official tensorflow 1.12 docker images don't really fit our needs as they depend on python3.8
+rather than python 3.9. For this reason two docker files can be find in the `dockerfiles` folder,
+one for a cpu image and another for gpu image.
 
-These images live publicly in our (docker hub)[https://hub.docker.com/r/krakentechnologies/tensorflow/].
+These images live publicly in our [docker hub][kraken_docker].
 
-More info about (docker and tensorflow)[https://github.com/tensorflow/tensorflow/tree/master/tensorflow/tools/dockerfiles]
+More info about [docker and tensorflow][tensorflow_dockerfiles]
+
+[kraken_docker]: https://hub.docker.com/r/krakentechnologies/tensorflow/.
+[tensorflow_dockerfiles]: https://github.com/tensorflow/tensorflow/tree/master/tensorflow/tools/dockerfiles

docs/source/conf.py

@@ -26,9 +26,9 @@
 # Read version
 exec(open('../../timeserio/version.py').read())
 # short X.Y version
-version = __version__
+version = __version__  # type: ignore
 # The full version, including alpha/beta/rc tags
-release = __version__
+release = __version__  # type: ignore
 
 # -- General configuration ---------------------------------------------------
 
@@ -162,7 +162,7 @@
 
 # -- Options for LaTeX output ------------------------------------------------
 
-latex_elements = {
+latex_elements = {  # type: ignore
     # The paper size ('letterpaper' or 'a4paper').
     #
     # 'papersize': 'letterpaper',

setup.py

@@ -38,18 +38,16 @@
         "numpy",
         "pandas",
         "scikit-learn>=0.23.1",
-        "s3fs<0.5",  # see [1] below
+        "tentaclio[s3]",
         "holidays",
     ],
-    # [1] this is where s3fs switches to async
-    #     https://github.com/aio-libs/aiobotocore/pull/766
     classifiers=[
         "Development Status :: 3 - Alpha",
         "Intended Audience :: Developers",
         "License :: OSI Approved :: MIT License",
         "Operating System :: OS Independent",
         "Programming Language :: Python",
         "Programming Language :: Python :: 3",
-        "Programming Language :: Python :: 3.6",
+        "Programming Language :: Python :: 3.9",
     ],
 )