Python 3.8 Support (#3295)

* In Python 3.8 docstrings are constants, not strings

* Update build requirements

* Updates sckit-learn API usage

* Update unit tests for new coremltools version

* Update dependencies in setup.py

* Add Python 3.8 to GitLab config

* Code review feedback

* Added instructions for pushing new docker images

* More requirements tweaking

* Be explicit about which Python version documentation build should use

* More requirements tweaking

* Only depend on beta version of coremltools when necessary

* Make unit tests agnostic of coremltool version; newer versions of coremltools add a key/value to the user metadata

* Sound Classifier fails on Linux with TensorFlow 2.3 and 2.2

* Try the first coremltools 4.0 beta

* And more dependency version tweaking

* More dependency expirementation

* Scenerio tests should not use minimal wheel. It doesn't have all required functionality

* Go back to using beta3 for coremltools; they have fixed some things. Also 'and' vs 'or' bug fix.

* Skip another test on Linux for Python 3.8

* Run 3.8 mac unit tests on 10.15; CoreMLTool beta is broken on 10.14

* Add Python 3.8 support to README

* Update unsupported_python/setup.py to include Python 3.8

* Add Python 3.8 classifier tags to setup.py files

.gitlab-ci.yml

@@ -108,6 +108,26 @@ build_wheel_linux_python37:
     paths:
       - target/
 
+build_wheel_linux_python38:
+  tags:
+    - docker
+  services:
+    - docker:19.03-dind
+  image: docker:19.03.8
+  stage: build
+  variables:
+    DOCKER_HOST: tcp://docker:2375
+    DOCKER_DRIVER: overlay2
+    DOCKER_TLS_CERTDIR: ""
+  script:
+    - apk add bash
+    - mkdir -p /etc/ssl/certs/
+    - bash -e scripts/make_wheel.sh --skip_test --skip_cpp_test --build_number=$CI_PIPELINE_ID --num_procs=4 --docker-python3.8
+  artifacts:
+    expire_in: 1 day
+    paths:
+      - target/
+
 build_wheel_mac_10_15_python27:
   tags:
     - macos10.15
@@ -172,6 +192,22 @@ build_wheel_mac_10_15_python37:
     paths:
       - target/
 
+build_wheel_mac_10_15_python38:
+  tags:
+    - macos10.15
+  only:
+    variables:
+      - $TC_HAS_MACOS_RUNNERS == "1"
+  stage: build
+  script:
+    - export VIRTUALENV="/Library/Frameworks/Python.framework/Versions/3.8/bin/virtualenv"
+    - source scripts/use_ccache.sh
+    - bash -e scripts/make_wheel.sh --skip_test --skip_cpp_test --build_number=$CI_PIPELINE_ID --num_procs=4
+  artifacts:
+    expire_in: 1 day
+    paths:
+      - target/
+
 build_dylib_macos_10_15:
   tags:
     - macos10.15
@@ -445,6 +481,28 @@ test_python_linux_python37:
     paths:
       - pytest.*
 
+test_python_linux_python38:
+  tags:
+    - docker
+  services:
+    - docker:19.03-dind
+  image: docker:19.03.8
+  stage: test
+  variables:
+    DOCKER_HOST: tcp://docker:2375
+    DOCKER_DRIVER: overlay2
+    DOCKER_TLS_CERTDIR: ""
+  dependencies:
+    - build_wheel_linux_python38
+  script:
+    - apk add bash
+    - bash -e scripts/test_wheel.sh --docker-python3.8
+  artifacts:
+    when: always
+    expire_in: 2 weeks
+    paths:
+      - pytest.*
+
 test_python_mac_python27_10_13:
   tags:
     - macos10.13
@@ -553,6 +611,42 @@ test_python_mac_python37_10_14_minimal:
     paths:
       - pytest.*
 
+test_python_mac_python38_10_15:
+  tags:
+    - macos10.15
+  only:
+    variables:
+      - $TC_HAS_MACOS_RUNNERS == "1"
+  stage: test
+  dependencies:
+    - build_wheel_mac_10_15_python38
+  script:
+    - export VIRTUALENV="/Library/Frameworks/Python.framework/Versions/3.8/bin/virtualenv"
+    - bash -e scripts/test_wheel.sh
+  artifacts:
+    when: always
+    expire_in: 2 weeks
+    paths:
+      - pytest.*
+
+test_python_mac_python38_10_14_minimal:
+  tags:
+    - macos10.14
+  only:
+    variables:
+      - $TC_HAS_MACOS_RUNNERS == "1"
+  stage: test
+  dependencies:
+    - build_wheel_mac_10_15_python38
+  script:
+    - export VIRTUALENV="/Library/Frameworks/Python.framework/Versions/3.8/bin/virtualenv"
+    - bash -e scripts/test_wheel.sh --minimal
+  artifacts:
+    when: always
+    expire_in: 2 weeks
+    paths:
+      - pytest.*
+
 scenario_tests_linux_python27:
   tags:
     - docker
@@ -570,7 +664,7 @@ scenario_tests_linux_python27:
     - apk add bash
     - cd scenario-tests
     - ./run_scenario_tests.sh --docker-python2.7 \
-      ../target/turicreate-*+minimal*.whl
+        "$(ls -1 ../target/turicreate-*.whl | grep -v minimal)"
 
 scenario_tests_linux_python35:
   tags:
@@ -589,7 +683,7 @@ scenario_tests_linux_python35:
     - apk add bash
     - cd scenario-tests
     - ./run_scenario_tests.sh --docker-python3.5 \
-      ../target/turicreate-*+minimal*.whl
+        "$(ls -1 ../target/turicreate-*.whl | grep -v minimal)"
 
 scenario_tests_linux_python36:
   tags:
@@ -608,7 +702,7 @@ scenario_tests_linux_python36:
     - apk add bash
     - cd scenario-tests
     - ./run_scenario_tests.sh --docker-python3.6 \
-      ../target/turicreate-*+minimal*.whl
+        "$(ls -1 ../target/turicreate-*.whl | grep -v minimal)"
 
 scenario_tests_mac_python27_10_13:
   tags:
@@ -650,10 +744,12 @@ collect_artifacts:
     - build_wheel_linux_python35
     - build_wheel_linux_python36
     - build_wheel_linux_python37
+    - build_wheel_linux_python38
     - build_wheel_mac_10_15_python27
     - build_wheel_mac_10_15_python35
     - build_wheel_mac_10_15_python36
     - build_wheel_mac_10_15_python37
+    - build_wheel_mac_10_15_python38
     - build_documentation
   script:
     - find release/src/deployment/ -name '*.dylib' -exec mv {} target/ \;

README.md

@@ -79,7 +79,7 @@ System Requirements
 
 Turi Create requires:
 
-* Python 2.7, 3.5, 3.6, 3.7
+* Python 2.7, 3.5, 3.6, 3.7, 3.8
 * x86\_64 architecture
 * At least 4 GB of RAM
 

scripts/Dockerfile-CentOS-6

@@ -7,7 +7,7 @@
 # Builds against libstdc++ from GCC 4.8 for compatibility with older libstdc++
 # runtime versions.
 
-FROM quay.io/pypa/manylinux2010_x86_64@sha256:191e8bc35089ce4ceda48224289b7475322ee6853faa4bf7d4b8e0d9c3b0358e
+FROM quay.io/pypa/manylinux2010_x86_64
 
 # Set env variables for tools to pick up
 ENV CC="gcc"
@@ -133,6 +133,17 @@ RUN ./configure --prefix=/usr/local --enable-unicode=ucs4 --enable-shared --enab
     ldconfig && \
     rm -rf /src/Python-3.7.3*
 
+# Install Python 3.8 from source
+WORKDIR /src
+RUN curl -O https://www.python.org/ftp/python/3.8.5/Python-3.8.5.tgz
+RUN tar xf Python-3.8.5.tgz
+WORKDIR /src/Python-3.8.5
+RUN ./configure --prefix=/usr/local --enable-unicode=ucs4 --enable-shared --enable-loadable-sqlite-extensions && \
+    make -j4 --quiet && \
+    make install && \
+    ldconfig && \
+    rm -rf /src/Python-3.8.5*
+
 ENV CC="/usr/local/bin/gcc"
 ENV CXX="/usr/local/bin/g++"
 
@@ -147,6 +158,8 @@ RUN python3.6 get-pip.py
 RUN pip3.6 install virtualenv
 RUN python3.7 get-pip.py
 RUN pip3.7 install virtualenv
+RUN python3.8 get-pip.py
+RUN pip3.8 install virtualenv
 RUN rm -rf /src/get-pip.py
 
 ENV LD="/usr/local/lib64"

scripts/Dockerfile-Ubuntu-18.04

@@ -17,6 +17,11 @@ RUN apt-get -y upgrade
 # turicreate and upstream dependencies
 RUN apt-get -y install python3.6 python3.6-distutils python3.7 python3.7-distutils libgomp1 lsb-release npm nodejs doxygen zip
 
+# Install Python 3.8
+RUN apt-get -y install software-properties-common
+RUN add-apt-repository -y ppa:deadsnakes/ppa
+RUN apt-get -y install python3.8
+
 # Install build-essential (needed by numpy, which tries to
 # build itself from source on 3.6 and above)
 RUN apt-get -y install build-essential libpython3.6-dev libpython3.7-dev
@@ -29,3 +34,5 @@ RUN python3.6 get-pip.py
 RUN pip3.6 install virtualenv
 RUN python3.7 get-pip.py
 RUN pip3.7 install virtualenv
+RUN python3.8 get-pip.py
+RUN pip3.8 install virtualenv

scripts/create_docker_images.sh

@@ -3,6 +3,10 @@
 set -e
 set -x
 
+# Once new docker images have been verified, you should manually push them to the GitLab registry.
+# To do this, first log: registry.gitlab.com`.
+# Then for each docker image: `docker push <image name>`.
+
 SCRIPT_DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
 WORKSPACE=${SCRIPT_DIR}/..
 cd ${WORKSPACE}

scripts/get_docker_image.sh

@@ -37,7 +37,7 @@ if [[ -z "${UBUNTU_VERSION}" && -z "${CENTOS_VERSION}" ]]; then
 fi
 
 # The build image version that will be used for building
-TC_BUILD_IMAGE_VERSION="1.0.6"
+TC_BUILD_IMAGE_VERSION="1.0.7"
 
 if [[ -z "${CENTOS_VERSION}" ]]; then
   # The base image name - using Gitlab CI registry

scripts/install_python_toolchain.sh

@@ -27,8 +27,9 @@ function linux_patch_sigfpe_handler {
   if [[ $OSTYPE == linux* ]]; then
     targfile=deps/local/include/pyfpe.h
     if [[ -f $targfile ]]; then
-      echo "#undef WANT_SIGFPE_HANDLER" | cat - $targfile > tmp
-      mv -f tmp $targfile
+      temp_file=/tmp/sigfpe_handler_temp.txt
+      echo "#undef WANT_SIGFPE_HANDLER" | cat - $targfile > $temp_file
+      mv -f $temp_file $targfile
     fi
   fi
 }

scripts/make_documentation.sh

@@ -56,7 +56,7 @@ fi
 
 TARGET_DIR=${WORKSPACE}/target
 
-virtualenv ./deps/env
+virtualenv --python=python3.7 ./deps/env
 source deps/env/bin/activate
 
 # ignore minimal build

scripts/make_wheel.sh

@@ -44,11 +44,13 @@ print_help() {
   echo
   echo "  --docker-python3.7       Use docker to build for Python 3.7 in CentOS 6 with Clang 8."
   echo
+  echo "  --docker-python3.8       Use docker to build for Python 3.8 in CentOS 6 with Clang 8."
+  echo
   echo "  --num_procs=n            Specify the number of proceses to run in parallel."
   echo
   echo "  --target-dir=[dir]       The directory where the wheel and associated files are put."
   echo
-  echo "Produce a local wheel and skip test and doc generation"
+  echo "Produce a local wheel and skip test"
   echo "Example: ./make_wheel.sh --skip_test"
   exit 1
 } # end of print help
@@ -70,6 +72,7 @@ while [ $# -gt 0 ]
     --docker-python3.5)     USE_DOCKER=1;DOCKER_PYTHON=3.5;;
     --docker-python3.6)     USE_DOCKER=1;DOCKER_PYTHON=3.6;;
     --docker-python3.7)     USE_DOCKER=1;DOCKER_PYTHON=3.7;;
+    --docker-python3.8)     USE_DOCKER=1;DOCKER_PYTHON=3.8;;
     --help)                 print_help ;;
     *) unknown_option $1 ;;
   esac

scripts/requirements-minimal.txt

@@ -1,7 +1,10 @@
-numpy==1.16.4
-cython==0.29.10
+numpy==1.16.4; python_version < '3.6'
+numpy==1.19.1; python_version >= '3.6'
+
+cython==0.29.21
 decorator==4.4.0
-pandas==0.24.2
+pandas==0.24.2; python_version <= '3.6'
+pandas==1.0.5; python_version > '3.6'
 pillow==6.2.2
 prettytable==0.7.2
 pytz==2019.1

scripts/requirements.txt

@@ -2,15 +2,22 @@
 
 ### extra for full pkg ###
 
-coremltools==3.3
-scipy==1.2.1
+coremltools==3.3; python_version < '3.8'
+coremltools==4.0b3; python_version == '3.8'
+
+scipy==1.2.1; python_version < '3.8'
+scipy==1.5.1; python_version == '3.8'
+
 resampy==0.2.1
-scikit-learn==0.20.3
-statsmodels==0.9.0
-tensorflow==2.0.1; python_version > '2.7'
-tensorflow==2.0.0; python_version < '3.5'   # Last version to support Python 2.7
+
+scikit-learn==0.20.3; python_version < '3.8'
+scikit-learn==0.23.1; python_version == '3.8'
+
+statsmodels==0.9.0; python_version < '3.8'
+statsmodels==0.11.1; python_version == '3.8'
+
 UISoup==2.5.7
-pyobjc==5.2; sys_platform == 'darwin'
+
 llvmlite==0.33.0; python_version >= '3.6'
 llvmlite==0.31.0; python_version < '3.6'
 

scripts/test_wheel.sh

@@ -30,6 +30,8 @@ print_help() {
   echo
   echo "  --docker-python3.7       Use docker to test on Python 3.7 in Ubuntu 18.04."
   echo
+  echo "  --docker-python3.8       Use docker to test on Python 3.8 in Ubuntu 18.04."
+  echo
   echo "  --minimal                test the turicreate+minimal package"
   echo
   exit 1
@@ -45,6 +47,7 @@ while [ $# -gt 0 ]
     --docker-python3.5)     USE_DOCKER=1;DOCKER_PYTHON=3.5;;
     --docker-python3.6)     USE_DOCKER=1;DOCKER_PYTHON=3.6;;
     --docker-python3.7)     USE_DOCKER=1;DOCKER_PYTHON=3.7;;
+    --docker-python3.8)     USE_DOCKER=1;DOCKER_PYTHON=3.8;;
     --minimal)              USE_MINIMAL=1;;
     --help)                 print_help ;;
     *) unknown_option "$1" ;;
@@ -81,7 +84,7 @@ if [[ -n "${USE_DOCKER}" ]]; then
       --env-file "${envlist}" \
       "${TC_BUILD_IMAGE_1404}" \
       /build/scripts/test_wheel.sh ${docker_extra_args}
-  elif [[ "${DOCKER_PYTHON}" == "3.6" ]] || [[ "${DOCKER_PYTHON}" == "3.7" ]]; then
+  elif [[ "${DOCKER_PYTHON}" == "3.6" ]] || [[ "${DOCKER_PYTHON}" == "3.7" ]] || [[ "${DOCKER_PYTHON}" == "3.8" ]]; then
     docker run --rm -m=8g \
       --mount type=bind,source="$WORKSPACE",target=/build,consistency=delegated \
       --env-file "${envlist}" \