From 31955c718f08197e6244c312eb9baa483df32c40 Mon Sep 17 00:00:00 2001 From: dumol Date: Thu, 20 May 2021 14:27:05 +0300 Subject: [PATCH] Containerized builds through GitHub Actions. (#21) Added containerized builds for: * Alpine 3.12 * CentOS 5.11 * CentOS 8.2 (which has OpenSSL 1.1.1c for backward compatibility with 8.0-8.2). **Drive-by changes**: * Forward ported the latest improvements from `python-package`. * Specific support for Amazon Linux 2 and RHEL 7 was removed. * Trimmed down Alpine deps, to have it work on minimal containers. * Updated embedded OpenSSL libs to 1.1.1k. * Updated `cryptography` to 3.4.7. * Updated `pip` to latest version. --- .github/workflows/{main.yaml => bare.yaml} | 76 +++++++----- .github/workflows/docker.yaml | 108 ++++++++++++++++++ .travis.yml | 40 ------- README.rst | 16 +-- brink.sh | 50 ++++---- os_quirks.sh | 10 +- pkg_checks.sh | 30 ++--- publish_dist.sh | 4 +- pythia | 9 +- pythia.conf | 8 +- src/Python/chevahbs | 5 + .../test_python_binary_dist.py | 13 ++- src/libffi/chevahbs | 17 ++- src/openssl/chevahbs | 12 +- src/openssl/sha512.sum | 2 +- 15 files changed, 264 insertions(+), 136 deletions(-) rename .github/workflows/{main.yaml => bare.yaml} (71%) create mode 100644 .github/workflows/docker.yaml delete mode 100644 .travis.yml mode change 100644 => 100755 publish_dist.sh diff --git a/.github/workflows/main.yaml b/.github/workflows/bare.yaml similarity index 71% rename from .github/workflows/main.yaml rename to .github/workflows/bare.yaml index d1b4223d..0335e53e 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/bare.yaml @@ -4,7 +4,7 @@ # Don't use `-latest` for runners, pin specific OS versions instead. More at # https://help.github.com/en/actions/reference/virtual-environments-for-github-hosted-runners. -name: GitHub-CI +name: Bare on: push: @@ -12,10 +12,16 @@ on: pull_request: branches: [ main ] +concurrency: + group: bare-${{ github.ref }} + cancel-in-progress: true + # Set to 'yes' to open a tunnel to GitHub's VMs through ngrok/tmate on failures. # Also increase timeout-minutes for the relevant OS when debugging remotely. +# Version 3 of mxschmitt/action-tmate should also work on Windows. env: - CHEVAH_DEBUG: 'no' + TMATE_DEBUG: 'no' + NGROK_DEBUG: 'no' jobs: @@ -27,22 +33,20 @@ jobs: fail-fast: false matrix: runs-on: [ windows-2019, windows-2016 ] - timeout-minutes: 20 + # Steps represent a sequence of tasks that will be executed as part of the job steps: - # Make sure there are no concurrent jobs - - uses: chevah/auto-cancel-redundant-job@v1 # Checks-out the repository under $GITHUB_WORKSPACE, so the job can access it - uses: actions/checkout@v2 with: fetch-depth: 2 # Explicitly run our scripts with Bash, not PowerShell (GitHub's default). - - name: Build Python + - name: Build Pythia run: bash ./pythia build - - name: Test Python + - name: Test Pythia run: bash ./pythia test # To use an RSA key with SFTPPlus, install upstream OpenSSH package, @@ -58,14 +62,24 @@ jobs: echo "${{ secrets.SFTPPLUS_BIN_HOST_KEY }}" > ~/.ssh/known_hosts choco install --yes --no-progress openssh - # Uploads using a (per-OS selected) sftp command, then shows final links. + # Upload using a (per-OS selected) sftp command, then show final links. + # Remove key in same step to avoid leaving it on disk if publishing fails. - name: Upload testing package - run: bash ./publish_dist.sh + run: bash -c './publish_dist.sh ; rm priv_key' - # Per https://github.com/nelsonjchen/reverse-rdp-windows-github-actions, - # but using the Ngrok token as password for the runnneradmin user. + # Command line debugging through Tmate. v3 works on Windows too. + - name: Tmate debug on failure + if: failure() && env.TMATE_DEBUG == 'yes' + uses: mxschmitt/action-tmate@v3 + with: + limit-access-to-actor: true + + # To access the Windows desktop for advanced debugging, as per + # https://github.com/nelsonjchen/reverse-rdp-windows-github-actions, + # but using the ngrok token as password for the runnneradmin user. + # Basically use the ngrok token and the ngrok URL (from ngrok's dashboard). - name: Ngrok debugging on failure - if: failure() && env.CHEVAH_DEBUG == 'yes' + if: failure() && env.NGROK_DEBUG == 'yes' env: NGROK_TOKEN: ${{ secrets.NGROK_AUTH_TOKEN }} run: | @@ -86,7 +100,6 @@ jobs: runs-on: [ ubuntu-20.04, ubuntu-18.04 ] timeout-minutes: 40 steps: - - uses: chevah/auto-cancel-redundant-job@v1 - uses: actions/checkout@v2 with: fetch-depth: 2 @@ -94,11 +107,11 @@ jobs: - name: Install required packages run: sudo apt-get install libncurses5-dev - - name: Build Python - run: bash ./pythia build + - name: Build Pythia + run: ./pythia build - - name: Test Python - run: bash ./pythia test + - name: Test Pythia + run: ./pythia test - name: Upload testing package run: | @@ -107,19 +120,20 @@ jobs: chmod 600 priv_key echo "${{ secrets.SFTPPLUS_BIN_PRIV_KEY }}" > priv_key echo "${{ secrets.SFTPPLUS_BIN_HOST_KEY }}" > ~/.ssh/known_hosts - bash ./publish_dist.sh + ./publish_dist.sh ; rm priv_key # If one of the above steps fails, fire up tmate for remote debugging. - name: Tmate debug on failure - if: failure() && env.CHEVAH_DEBUG == 'yes' - uses: mxschmitt/action-tmate@v2 - + if: failure() && env.TMATE_DEBUG == 'yes' + uses: mxschmitt/action-tmate@v3 + with: + limit-access-to-actor: true macos: runs-on: macos-10.15 + needs: [ windows, linux ] timeout-minutes: 60 steps: - - uses: chevah/auto-cancel-redundant-job@v1 - uses: actions/checkout@v2 with: fetch-depth: 2 @@ -132,11 +146,11 @@ jobs: sudo chmod -v a-r /usr/local/include/libintl.h sudo chmod -v a-r /usr/local/opt/gettext/lib/libintl.* - - name: Build Python - run: bash ./pythia build + - name: Build Pythia + run: ./pythia build - - name: Test Python - run: bash ./pythia test + - name: Test Pythia + run: ./pythia test - name: Upload testing package run: | @@ -145,16 +159,18 @@ jobs: chmod 600 priv_key echo "${{ secrets.SFTPPLUS_BIN_PRIV_KEY }}" > priv_key echo "${{ secrets.SFTPPLUS_BIN_HOST_KEY }}" > ~/.ssh/known_hosts - bash ./publish_dist.sh + ./publish_dist.sh ; rm priv_key # Fix back Homebrew, to make everything functional for tmate debugging. - name: Unhack Homebrew - if: failure() && env.CHEVAH_DEBUG == 'yes' + if: failure() && env.TMATE_DEBUG == 'yes' run: | sudo chmod -v a+r /usr/local/opt/gettext/lib/libintl* sudo chmod -v a+r /usr/local/include/libintl.h sudo mv -v /usr/local/bin/git{.saved,} - name: Tmate debug on failure - if: failure() && env.CHEVAH_DEBUG == 'yes' - uses: mxschmitt/action-tmate@v2 + if: failure() && env.TMATE_DEBUG == 'yes' + uses: mxschmitt/action-tmate@v3 + with: + limit-access-to-actor: true diff --git a/.github/workflows/docker.yaml b/.github/workflows/docker.yaml new file mode 100644 index 00000000..928be65b --- /dev/null +++ b/.github/workflows/docker.yaml @@ -0,0 +1,108 @@ +# +# GitHub actions for building and testing. +# +# For best support, use `-latest` for runners spinning up containers. More at +# https://help.github.com/en/actions/reference/virtual-environments-for-github-hosted-runners. + +name: Docker + +on: + push: + branches: [ main ] + pull_request: + branches: [ main ] + +concurrency: + group: docker-${{ github.ref }} + cancel-in-progress: true + +# Set to 'yes' to open a tunnel to GitHub's VMs through tmate on failures. +# Also increase timeout-minutes for the relevant OS when debugging remotely. +env: + TMATE_DEBUG: 'no' + +# Using a job name that doesn't contain the OS name, to minimize the risk of +# confusion with the OS names of the containers, which are the relevant ones. +jobs: + latest: + runs-on: ubuntu-latest + container: ${{ matrix.container }} + strategy: + fail-fast: false + matrix: + container: [ 'alpine:3.12', 'centos:8.2.2004', 'centos:5.11' ] + timeout-minutes: 30 + steps: + + # OpenSSL gets updated by apk, but that is the Alpine way, so it's fine. + - name: Alpine 3.12 setup + if: matrix.container == 'alpine:3.12' + run: | + apk update + apk upgrade + apk add git curl bash openssh-client + curl -o /usr/local/bin/paxctl https://binary.chevah.com/third-party-stuff/alpine/paxctl-3.12 + chmod +x /usr/local/bin/paxctl + + # Stick to CentOS 8.2 as OpenSSL got updated in 8.3 from 1.1.1c to 1.1.1g. + - name: CentOS 8.2 setup + if: matrix.container == 'centos:8.2.2004' + run: | + sed -i s/^mirrorlist=/#mirrorlist=/ /etc/yum.repos.d/*.repo + sed -i s@^#baseurl=http://mirror.centos.org/\$contentdir/\$releasever/@baseurl=https://vault.centos.org/8.2.2004/@ /etc/yum.repos.d/*.repo + yum -y upgrade + yum -y install git curl openssh-clients + + # Final CentOS 5 version is used to build the generic Linux package. + - name: CentOS 5.11 setup + if: matrix.container == 'centos:5.11' + run: | + sed -i s/^mirrorlist=/#mirrorlist=/ /etc/yum.repos.d/*.repo + sed -i s@^#baseurl=http://mirror.centos.org/centos/\$releasever/@baseurl=http://vault.centos.org/5.11/@ /etc/yum.repos.d/*.repo + yum -y upgrade + # Use http://binary.chevah.com/third-party-stuff/centos5/tuxad/ + # when tuxad.de dissapears, it has the minimum required stuff. + rpm -i http://www.tuxad.de/rpms/tuxad-release-5-1.noarch.rpm + yum -y install curl openssh-clients gcc44 make m4 patch unzip wget + ln -s /usr/bin/gcc44 /usr/local/bin/gcc + wget --mirror --no-parent https://binary.chevah.com/third-party-stuff/centos5/endpoint/ + cd binary.chevah.com/third-party-stuff/centos5/endpoint/ + rpm -i local-perl-*.rpm + rpm -i --nodeps git{-core,}-2.5.0-1.ep.x86_64.rpm + + - name: Clone repo independently + run: | + git clone https://github.com/chevah/pythia.git + cd pythia + git checkout ${GITHUB_HEAD_REF} + + - name: Build Pythia + run: | + cd pythia + ./pythia build + + - name: Test Pythia + run: | + cd pythia + ./pythia test + + # Using `~/` is problematic under Docker, use `/root/`. + # Remove key in same step to avoid leaving it on disk if publishing fails. + - name: Upload testing package + run: | + mkdir -pv /root/.ssh/ + cd pythia + touch priv_key + chmod 600 priv_key + echo "${{ secrets.SFTPPLUS_BIN_PRIV_KEY }}" > priv_key + echo "${{ secrets.SFTPPLUS_BIN_HOST_KEY }}" > /root/.ssh/known_hosts + ./publish_dist.sh ; rm priv_key + + # If one of the above steps fails, fire up tmate for remote debugging. + # Not working on Alpine (not supported) and CentOS 5 (glibc too old). + - name: Tmate debug on failure + if: failure() && env.TMATE_DEBUG == 'yes' + uses: mxschmitt/action-tmate@v3 + with: + sudo: false + limit-access-to-actor: true diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index c0af6792..00000000 --- a/.travis.yml +++ /dev/null @@ -1,40 +0,0 @@ -# -# Experimental Travis CI. -# -# Linux ARM64 target is pinned on the oldest distribution available on Travis, -# thus matching a generic ARM64 Linux build. - -language: shell - -git: - depth: false - -jobs: - include: - # Ubuntu Server 16.04 on ARM64 - - os: linux - dist: xenial - arch: arm64 - -install: - - | - case $TRAVIS_OS_NAME in - linux) - # Only generic 'lnx' is targeted, thus no automatic package management. - sudo apt install -y unzip - ;; - osx) - # Some Homebrew libs pollute the build. - sudo chmod a-r /usr/local/opt/libffi/lib/libffi.6.dylib - ;; - esac - -before_script: - - ./brink.sh detect_os - -script: - - ./pythia build && ./pythia test - -notifications: - email: - - dumol@gmx.co.uk diff --git a/README.rst b/README.rst index 82e8afe8..c54dbeab 100644 --- a/README.rst +++ b/README.rst @@ -1,7 +1,7 @@ -Pythia - a Python Portable Package +Pythia - a portable Python package ================================== -Build system for a portable Python distribution. +Build system for a portable Python package. A derivative of https://github.com/chevah/python-package/. Building: @@ -22,12 +22,15 @@ Supported platforms ------------------- * Windows Server 2012 R2 and newer (x86 and x64) -* Red Hat Linux Enterprise 7 and 8 (including derivatives such as CentOS) +* macOS 10.13 and newer. +* all glibc-based Linux distributions (glibc 2.5+ for x64, 2.23+ for arm64) + +Platforms on which the system OpenSSL is used: + +* Red Hat Linux Enterprise 8 and newer (including derivatives such as CentOS) * Amazon Linux 2 * Ubuntu Server 18.04 and 20.04 -* all glibc-based Linux distributions (glibc 2.5+ for x64, 2.23+ for arm64) * Alpine Linux 3.12 -* macOS 10.13 and newer. Platforms that should work, but are not regularly tested: @@ -70,8 +73,5 @@ right, then save the diff to the sources patched with the preceding patch. .. image:: https://github.com/chevah/pythia/workflows/GitHub-CI/badge.svg :target: https://github.com/chevah/pythia/actions -.. image:: https://travis-ci.com/chevah/pythia.svg?branch=main - :target: https://travis-ci.com/github/chevah/pythia - .. image:: https://img.shields.io/github/issues/chevah/pythia.svg :target: https://github.com/chevah/pythia/issues diff --git a/brink.sh b/brink.sh index 84c635bf..b6dbd29e 100755 --- a/brink.sh +++ b/brink.sh @@ -373,6 +373,7 @@ test_version_exists() { local remote_base_url=$1 local target_file=python-${PYTHON_VERSION}-${OS}-${ARCH}.tar.gz + echo "Checking $remote_base_url/${OS}/${ARCH}/$target_file" $ONLINETEST_CMD $remote_base_url/${PYTHON_VERSION}/$target_file return $? } @@ -395,7 +396,7 @@ get_python_dist() { # We have the requested python version. get_binary_dist $python_distributable $remote_base_url/${PYTHON_VERSION} else - (>&2 echo "Couldn't find package on remote server. Full link below...") + (>&2 echo "Couldn't find package on remote server. Full link:") echo "$remote_base_url/$PYTHON_VERSION/$python_distributable.tar.gz" exit 4 fi @@ -514,8 +515,8 @@ install_dependencies(){ # Check version of current OS to see if it is supported. # If it's too old, exit with a nice informative message. # If it's supported, return through eval the version numbers to be used for -# naming the package, for example: '7' for RHEL 7.7, '2' for Amazon 2, -# '2004' for Ubuntu 20.04', '312' for Alpine Linux 3.12, '11' for Solaris 11. +# naming the package, for example: '8' for RHEL 8.2, '2004' for Ubuntu 20.04, +# '312' for Alpine Linux 3.12, '114' for Solaris 11.4. # check_os_version() { # First parameter should be the human-readable name for the current OS. @@ -582,6 +583,8 @@ check_linux_glibc() { local glibc_version local glibc_version_array local supported_glibc2_version + # Output to a file to avoid "write error: Broken pipe" with grep/head. + local ldd_output_file="/tmp/.chevah_glibc_version" # Supported minimum minor glibc 2.X versions for various arches. # For x64, we build on CentOS 5.11 (Final) with glibc 2.5. @@ -607,14 +610,15 @@ check_linux_glibc() { exit 18 fi - ldd --version | egrep "GNU\ libc|GLIBC" > /dev/null + ldd --version > $ldd_output_file + egrep "GNU\ libc|GLIBC" $ldd_output_file > /dev/null if [ $? -ne 0 ]; then (>&2 echo "No glibc reported by ldd... Unsupported Linux libc?") exit 19 fi # Tested with glibc 2.5/2.11.3/2.12/2.23/2.28-31 and eglibc 2.13/2.19. - glibc_version=$(ldd --version | head -n 1 | rev | cut -d\ -f1 | rev) + glibc_version=$(head -n 1 $ldd_output_file | rev | cut -d\ -f1 | rev) if [[ $glibc_version =~ [^[:digit:]\.] ]]; then (>&2 echo "Glibc version should only have numbers and periods, but:") @@ -683,23 +687,11 @@ detect_os() { # Some rolling-release distros (eg. Arch Linux) have # no VERSION_ID here, so don't count on it unconditionally. case "$linux_distro" in - rhel|centos) + rhel|centos|ol) os_version_raw="$VERSION_ID" - check_os_version "Red Hat Enterprise Linux" 7 \ + check_os_version "Red Hat Enterprise Linux" 8 \ "$os_version_raw" os_version_chevah set_os_if_not_generic "rhel" $os_version_chevah - if [ "$os_version_chevah" -eq 7 ]; then - if openssl version | grep -F -q "1.0.1"; then - # 7.0-7.3 has OpenSSL 1.0.1, use generic build. - check_linux_glibc - fi - fi - ;; - amzn) - os_version_raw="$VERSION_ID" - check_os_version "$distro_fancy_name" 2 \ - "$os_version_raw" os_version_chevah - set_os_if_not_generic "amzn" $os_version_chevah ;; ubuntu|ubuntu-core) os_version_raw="$VERSION_ID" @@ -721,7 +713,8 @@ detect_os() { set_os_if_not_generic "alpine" $os_version_chevah ;; *) - # Unsupported modern distros such as SLES, Debian, etc. + # Supported distros with unsupported OpenSSL versions or + # distros not specifically supported: SLES, Debian, etc. check_linux_glibc ;; esac @@ -748,7 +741,22 @@ detect_os() { ;; SunOS) ARCH=$(isainfo -n) - os_version_raw=$(uname -v) + ver_major=$(uname -r | cut -d'.' -f2) + case $ver_major in + 10) + ver_minor=$(\ + head -1 /etc/release | cut -d_ -f2 | sed s/[^0-9]*//g) + ;; + 11) + ver_minor=$(uname -v | cut -d'.' -f2) + ;; + *) + # Not sure if $ver_minor detection works on other versions. + (>&2 echo "Unsupported Solaris version: ${ver_major}.") + exit 15 + ;; + esac + os_version_raw="${ver_major}.${ver_minor}" check_os_version "Solaris" 11.4 "$os_version_raw" os_version_chevah OS="sol${os_version_chevah}" ;; diff --git a/os_quirks.sh b/os_quirks.sh index 6e0f2f11..7a023b30 100644 --- a/os_quirks.sh +++ b/os_quirks.sh @@ -2,6 +2,9 @@ # # OS quirks for the Pythia build system. +# Avoid Rust for now, it brings more trouble. +export CRYPTOGRAPHY_DONT_BUILD_RUST=1 + case $OS in win) # On Windows, python executable is installed at a different path. @@ -16,9 +19,12 @@ case $OS in export SHA_CMD="$SHA_CMD --ignore-missing" ;; alpine*) - # By default, the busybox ersatz binaries are used. - export GET_CMD="wget -q -O" + # The busybox ersatz binary is different. export SHA_CMD="sha512sum -csw" + # Do not depend on libffi and ncurses-libs Alpine packages. + # It's better to run on minimal Alpine containers. + export BUILD_LIBFFI="yes" + export BUILD_LIBEDIT="no" ;; lnx) # Build as portable as possible, only glibc 2.x should be needed. diff --git a/pkg_checks.sh b/pkg_checks.sh index 3000d354..51dea800 100644 --- a/pkg_checks.sh +++ b/pkg_checks.sh @@ -10,17 +10,19 @@ # * git (for patching Python's version, if actually building it) # * automake, libtool, headers of a curses library (if building libedit) # * perl 5.10.0 or newer, Test::More 0.96 or newer (if building OpenSSL) -# * curl/wget, sha512sum, tar, unzip (for downloading and unpacking) +# * curl, sha512sum, tar, unzip (for downloading and unpacking) # # On platforms with multiple C compilers, choose by setting CC in os_quirks.sh. # List of OS packages required for building Python/pyOpenSSL/cryptography etc. -BASE_PKGS="gcc make m4 automake libtool texinfo patch curl tar coreutils unzip" -DPKG_PKGS="$BASE_PKGS git libssl-dev zlib1g-dev libffi-dev libncurses5-dev" -RPM_PKGS="$BASE_PKGS git openssl-devel zlib-devel libffi-devel ncurses-devel" -# Alpine's ersatz wget/tar/sha51sum binaries from Busybox are good enough. -APK_PKGS="gcc make m4 automake libtool texinfo patch unzip file musl-dev \ - git openssl-dev zlib-dev libffi-dev ncurses-dev" +BASE_PKGS="gcc make m4 automake libtool patch unzip" +DPKG_PKGS="$BASE_PKGS tar diffutils \ + git libssl-dev zlib1g-dev libffi-dev libncurses5-dev" +RPM_PKGS="$BASE_PKGS tar diffutils \ + git-core openssl-devel zlib-devel libffi-devel ncurses-devel" +# Alpine's ersatz tar/sha51sum binaries from Busybox are good enough. +APK_PKGS="$BASE_PKGS file lddtree \ + git openssl-dev zlib-dev libffi-dev musl-dev" # Windows is special, but package management is possible through Chocolatey. # Curl, sha512sum, and unzip are bundled with MINGW. CHOCO_PKGS="" @@ -101,18 +103,18 @@ if [ -n "$MISSING_PACKAGES" ]; then case "$OS" in ubuntu*) echo "## Installing missing dpkg packages... ##" - execute sudo apt install -y $MISSING_PACKAGES + execute $SUDO_CMD apt install -y $MISSING_PACKAGES ;; rhel*|amzn*) echo "## Installing missing rpm packages... ##" - execute sudo yum install -y $MISSING_PACKAGES + execute $SUDO_CMD yum install -y $MISSING_PACKAGES ;; alpine*) echo "## Installing missing apk packages... ##" - execute sudo apk add $MISSING_PACKAGES + execute $SUDO_CMD apk add $MISSING_PACKAGES ;; *) - (>&2 echo "Don't know how to install missing dependencies.") + (>&2 echo "Don't know how to install those on the current OS.") exit 149 ;; esac @@ -141,13 +143,13 @@ fi echo "# Checking if it's possible to avoid linking to system uuid libs... #" case "$OS" in ubuntu*) - execute sudo apt remove -y uuid-dev + execute $SUDO_CMD apt remove -y uuid-dev ;; rhel*|amzn*) - execute sudo yum remove -y e2fsprogs-devel libuuid-devel + execute $SUDO_CMD yum remove -y e2fsprogs-devel libuuid-devel ;; alpine*) - execute sudo apk del util-linux-dev + execute $SUDO_CMD apk del util-linux-dev ;; *) (>&2 echo "Not guarding against linking to uuid libs on this system!") diff --git a/publish_dist.sh b/publish_dist.sh old mode 100644 new mode 100755 index e6453acc..2a40a685 --- a/publish_dist.sh +++ b/publish_dist.sh @@ -13,8 +13,8 @@ root_link="https://${dest_server}:20443/testing" # The build/publish_dist_sftp_batch file is generated by the build process. # The private key comes from GitHub Secrets through the configured workflow. -sftp_opts="\ - -b build/publish_dist_sftp_batch -i priv_key -o StrictHostKeyChecking=yes" +sftp_opts="-b build/publish_dist_sftp_batch -o IdentityFile=priv_key \ + -o StrictHostKeyChecking=yes" # Get $OS var and set sftp command accordingly. source BUILD_ENV_VARS diff --git a/pythia b/pythia index 203f7a88..cc90f807 100755 --- a/pythia +++ b/pythia @@ -49,6 +49,9 @@ export GET_CMD="curl --silent --output" export SHA_CMD="sha512sum --check --status --warn" export TAR_CMD="tar xfz" export ZIP_CMD="unzip -q" +if [ x$(id -u) != "x0" ]; then + export SUDO_CMD="sudo" +fi # OS quirks. source os_quirks.sh @@ -122,11 +125,9 @@ build_dep() { if [ $dep_name = "openssl" ]; then if [ "$OS" = "lnx" ]; then # On RHEL5/SLES11 x64, OpenSSL instals only to lib64/ sub-dir. + # More so, under Docker installing fails, so it's done manually. # '-Wl,-rpath' voodoo is needed to build cryptography w/ pip. - export LDFLAGS="-L${INSTALL_DIR}/lib64/ \ - -Wl,-rpath,${INSTALL_DIR}/lib64/ ${LDFLAGS}" - export PKG_CONFIG_PATH="\ - ${INSTALL_DIR}/lib64/pkgconfig/:${PKG_CONFIG_PATH}" + export LDFLAGS="-Wl,-rpath,${INSTALL_DIR}/lib/ ${LDFLAGS}" fi # Still needed for building cryptography. export CPPFLAGS="$CPPFLAGS -I${INSTALL_DIR}/include" diff --git a/pythia.conf b/pythia.conf index 33c46563..ae2d5db5 100644 --- a/pythia.conf +++ b/pythia.conf @@ -12,18 +12,18 @@ BZIP2_VERSION="1.0.8" # Statically build the BSD libedit on selected platforms to get the # readline module available without linking to the GPL-only readline libs. LIBEDIT_VERSION="20170329-3.1" -OPENSSL_VERSION="1.1.1h" +OPENSSL_VERSION="1.1.1k" # To match the unusual naming scheme for SQLite downloads, eliminate dots from # the regular SQLite version, then add 3 zeros. E.g. "3.33.0" -> "3330000". # When updating this, also update the year in src/sqlite/chevahbs, if needed. SQLITE_VERSION="3330000" # These are used by get-pip.py. -PIP_VERSION="20.2.4" -SETUPTOOLS_VERSION="50.3.2" +PIP_VERSION="21.1.1" +SETUPTOOLS_VERSION="53.0.0" # Python modules versions to be used everywhere possible. CFFI_VERSION="1.14.3" -CRYPTOGRAPHY_VERSION="3.2.1" +CRYPTOGRAPHY_VERSION="3.4.7" SUBPROCESS32_VERSION="3.5.4" BCRYPT_VERSION="3.1.7" PSUTIL_VERSION="5.7.3" diff --git a/src/Python/chevahbs b/src/Python/chevahbs index 807d356c..f4c9e8f0 100755 --- a/src/Python/chevahbs +++ b/src/Python/chevahbs @@ -50,6 +50,11 @@ chevahbs_configure() { fi case $OS in + alpine312) + # Ugly hack for Python 3's configure issues. + execute $SUDO_CMD rm -f /include + execute $SUDO_CMD ln -s "${INSTALL_DIR}/include" /include + ;; fbsd*|sol11*) LDFLAGS="$LDFLAGS -lncurses" ;; diff --git a/src/chevah-python-test/test_python_binary_dist.py b/src/chevah-python-test/test_python_binary_dist.py index 92ed2d53..974f0bc3 100644 --- a/src/chevah-python-test/test_python_binary_dist.py +++ b/src/chevah-python-test/test_python_binary_dist.py @@ -145,8 +145,6 @@ def get_allowed_deps(): '/lib/libcrypto.so.1.1', '/lib/libssl.so.1.1', '/lib/libz.so.1', - '/usr/lib/libffi.so.7', - '/usr/lib/libncursesw.so.6', ] elif platform_system == 'sunos': # This is the list of deps for Solaris 11 64bit builds. @@ -387,9 +385,14 @@ def main(): from cryptography.hazmat.backends.openssl.backend import backend import cryptography openssl_version = backend.openssl_version_text() - if CHEVAH_OS in [ "win", "lnx", "macos" ]: - # Check OpenSSL version on OS'es with static OpenSSL libs. - expecting = u'OpenSSL 1.1.1h 22 Sep 2020' + if CHEVAH_OS in [ "win", "macos", "lnx", "rhel-8" ]: + if CHEVAH_OS == "rhel-8": + # On RHEL 8.3, OpenSSL got updated to 1.1.1g. To keep backward + # compatibility, link to version 1.1.1c from CentOS 8.2.2004. + expecting = u'OpenSSL 1.1.1c FIPS 28 May 2019' + else: + # Use latest OpenSSL version when building it from source. + expecting = u'OpenSSL 1.1.1k 25 Mar 2021' if openssl_version != expecting: sys.stderr.write('Expecting %s, got %s.\n' % ( expecting, openssl_version)) diff --git a/src/libffi/chevahbs b/src/libffi/chevahbs index 7a07c185..00b8c3ba 100755 --- a/src/libffi/chevahbs +++ b/src/libffi/chevahbs @@ -29,10 +29,19 @@ chevahbs_compile() { chevahbs_install() { - execute $MAKE install DESTDIR=$INSTALL_DIR - # Ugly hack for Python 3's configure issues. - execute sudo rm -f /include - execute sudo ln -s "${INSTALL_DIR}/include" /include + case $OS in + lnx) + # Installing manually to avoid messing with a lib64/ sub-dir. + execute cp -v *-linux-gnu/.libs/libffi.a "$INSTALL_DIR"/lib/ + execute cp -v *-linux-gnu/libffi.la "$INSTALL_DIR"/lib/ + execute cp -v *-linux-gnu/include/*.h "$INSTALL_DIR"/include/ + execute mkdir "$INSTALL_DIR"/pkgconfig/ + execute cp -v *-linux-gnu/libffi.pc "$INSTALL_DIR"/pkgconfig/ + ;; + *) + execute $MAKE install DESTDIR=$INSTALL_DIR + ;; + esac } select_chevahbs_command $@ diff --git a/src/openssl/chevahbs b/src/openssl/chevahbs index c978b2fb..3bfab7ca 100755 --- a/src/openssl/chevahbs +++ b/src/openssl/chevahbs @@ -33,7 +33,17 @@ chevahbs_compile() { chevahbs_install() { - execute $MAKE install + case $OS in + lnx) + echo "Installing manually to avoid messing with a lib64/ sub-dir:" + execute cp -v libcrypto.a libssl.a "$INSTALL_DIR"/lib/ + execute cp -rv include/openssl/ "$INSTALL_DIR"/include/ + execute cp -v *.pc "$INSTALL_DIR"/pkgconfig/ + ;; + *) + execute $MAKE install + ;; + esac } diff --git a/src/openssl/sha512.sum b/src/openssl/sha512.sum index 4491194b..5aba0b90 100644 --- a/src/openssl/sha512.sum +++ b/src/openssl/sha512.sum @@ -1 +1 @@ -da50fd99325841ed7a4367d9251c771ce505a443a73b327d8a46b2c6a7d2ea99e43551a164efc86f8743b22c2bdb0020bf24a9cbd445e9d68868b2dc1d34033a openssl-1.1.1h.tar.gz +73cd042d4056585e5a9dd7ab68e7c7310a3a4c783eafa07ab0b560e7462b924e4376436a6d38a155c687f6942a881cfc0c1b9394afcde1d8c46bf396e7d51121 openssl-1.1.1k.tar.gz