Detect OS automatically. (#20)

Run detect_os from pythia if BUILD_ENV_VARS file is missing. Drive-by changes : - Group the uploading logic in a separate script. - Show links for testing packages after uploading. - Simplify local `dist/` hierarchy to match uploads. - Minor cleanups.
chevah · Dec 21, 2020 · 2480fe3 · 2480fe3
1 parent e939358
commit 2480fe3
Show file tree

Hide file tree

Showing 9 changed files with 108 additions and 90 deletions.
diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml
@@ -22,7 +22,6 @@ jobs:
   windows:
     # The type of runner that the job will run on
     runs-on: ${{ matrix.runs-on }}
-    if: github.event_name == 'push' && contains(toJson(github.event.commits), '[skip ci]') == false
     strategy:
       # Workflow won't be cancelled at the first failed job.
       fail-fast: false
@@ -40,9 +39,6 @@ jobs:
         fetch-depth: 2
 
     # Explicitly run our scripts with Bash, not PowerShell (GitHub's default).
-    - name: Detect current OS
-      run: bash ./brink.sh detect_os
-
     - name: Build Python
       run: bash ./pythia build
 
@@ -62,11 +58,9 @@ jobs:
         echo "${{ secrets.SFTPPLUS_BIN_HOST_KEY }}" > ~/.ssh/known_hosts
         choco install --yes --no-progress openssh
 
-    # The publish_dist_sftp_batch file is generated by the build process.
+    # Uploads using a (per-OS selected) sftp command, then shows final links.
     - name: Upload testing package
-      run: |
-        C:\Progra~1\OpenSSH-Win64\sftp.exe -b publish_dist_sftp_batch -i priv_key -o StrictHostKeyChecking=yes [email protected]
-        echo "Package uploaded to https://bin.chevah.com:20443/testing/"
+      run: bash ./publish_dist.sh
 
     # Per https://github.com/nelsonjchen/reverse-rdp-windows-github-actions,
     # but using the Ngrok token as password for the runnneradmin user.
@@ -86,7 +80,6 @@ jobs:
 
   linux:
     runs-on: ${{ matrix.runs-on }}
-    if: github.event_name == 'push' && contains(toJson(github.event.commits), '[skip ci]') == false
     strategy:
       fail-fast: false
       matrix:
@@ -101,9 +94,6 @@ jobs:
     - name: Install required packages
       run: sudo apt-get install libncurses5-dev
 
-    - name: Detect current OS
-      run: bash ./brink.sh detect_os
-
     - name: Build Python
       run: bash ./pythia build
 
@@ -117,8 +107,7 @@ jobs:
         chmod 600 priv_key
         echo "${{ secrets.SFTPPLUS_BIN_PRIV_KEY }}" > priv_key
         echo "${{ secrets.SFTPPLUS_BIN_HOST_KEY }}" > ~/.ssh/known_hosts
-        sftp -b publish_dist_sftp_batch -i priv_key -o StrictHostKeyChecking=yes [email protected]
-        echo "Package uploaded to https://bin.chevah.com:20443/testing/"
+        bash ./publish_dist.sh
 
     # If one of the above steps fails, fire up tmate for remote debugging.
     - name: Tmate debug on failure
@@ -128,17 +117,13 @@ jobs:
 
   macos:
     runs-on: macos-10.15
-    if: github.event_name == 'push' && contains(toJson(github.event.commits), '[skip ci]') == false
     timeout-minutes: 60
     steps:
     - uses: chevah/auto-cancel-redundant-job@v1
     - uses: actions/checkout@v2
       with:
         fetch-depth: 2
 
-    - name: Detect current OS
-      run: ./brink.sh detect_os
-
     # Some Homebrew libs pollute the build.
     # Also make sure binaries depending on these libs are out of the way.
     - name: Hack Homebrew
@@ -160,8 +145,7 @@ jobs:
         chmod 600 priv_key
         echo "${{ secrets.SFTPPLUS_BIN_PRIV_KEY }}" > priv_key
         echo "${{ secrets.SFTPPLUS_BIN_HOST_KEY }}" > ~/.ssh/known_hosts
-        sftp -b publish_dist_sftp_batch -i priv_key -o StrictHostKeyChecking=yes [email protected]
-        echo "Package uploaded to https://bin.chevah.com:20443/testing/"
+        bash ./publish_dist.sh
 
     # Fix back Homebrew, to make everything functional for tmate debugging.
     - name: Unhack Homebrew

diff --git a/README.rst b/README.rst
@@ -4,12 +4,11 @@ Pythia - a Python Portable Package
 Build system for a portable Python distribution. 
 A derivative of https://github.com/chevah/python-package/.
 
-Building steps:
+Building:
 
-* ``./brink.sh detect_os``
 * ``./pythia build``
 
-Testing steps:
+Testing:
 
 * ``./pythia test``
 * ``./pythia compat``

diff --git a/brink.conf b/brink.conf
@@ -1,13 +1,13 @@
 BASE_REQUIREMENTS='pip==20.2.4 chevah-brink==0.78.1 paver==1.2.4'
-PYTHON_CONFIGURATION='[email protected].b1fa3c6'
+PYTHON_CONFIGURATION='[email protected].e939358'
 # For production packages there are 2 options:
 BINARY_DIST_URI='https://github.com/chevah/pythia/releases/download'
-BINARY_DIST_URI='https://bin.chevah.com:20443/production'
+#BINARY_DIST_URI='https://bin.chevah.com:20443/production'
 # For testing packages, make sure this one is the last uncommented instance:
 #BINARY_DIST_URI='https://bin.chevah.com:20443/testing'
 PIP_INDEX='https://pypi.chevah.com/simple'
 
 # There are 2 build directories used in this repo:
-#   * 'build' is a sub-dir used for building libffi, OpenSSL, Python, etc.
-#   * $CHEVAH_BUILD_DIR is used by the Python that builds the above.
+#   * $BUILD_DIR (set as 'build') is used for building this Python package.
+#   * $CHEVAH_BUILD_DIR is used for Buildbot's Python when building through it.
 CHEVAH_BUILD_DIR='build-pythia'
diff --git a/brink.sh b/brink.sh
@@ -432,7 +432,7 @@ copy_python() {
             # We have a cached distributable.
             # Check if is at the right version.
             local cache_ver_file
-            cache_ver_file=${python_distributable}/lib/PYTHON_PACKAGE_VERSION
+            cache_ver_file=${python_distributable}/lib/PYTHIA_VERSION
             cache_version='UNVERSIONED'
             if [ -f $cache_ver_file ]; then
                 cache_version=`cat $cache_ver_file`
@@ -461,7 +461,7 @@ copy_python() {
         WAS_PYTHON_JUST_INSTALLED=1
     else
         # We have a Python, but we are not sure if is the right version.
-        local version_file=${BUILD_FOLDER}/lib/PYTHON_PACKAGE_VERSION
+        local version_file=${BUILD_FOLDER}/lib/PYTHIA_VERSION
 
         python_installed_version=`cat $version_file`
         if [ "$PYTHON_VERSION" != "$python_installed_version" ]; then

diff --git a/functions_pythia.sh b/functions_pythia.sh
@@ -158,7 +158,7 @@ cleanup_install_dir() {
 
     echo "::group::Clean up Python install dir"
     execute pushd ${BUILD_DIR}/${PYTHON_BUILD_DIR}
-        echo "    Cleaning up Python's caches and compiled files..."
+        echo "Cleaning up Python's caches and compiled files..."
         find lib/ | grep -E "(__pycache__|\.pyc|\.pyo$)" | xargs rm -rf
     execute popd
     case $OS in
@@ -218,8 +218,8 @@ cleanup_install_dir() {
     esac
 
     # Output Pythia's own version to a dedicated file in the archive.
-    echo "${PYTHON_BUILD_VERSION}.${PYTHON_PACKAGE_VERSION}" \
-        > ${BUILD_DIR}/${PYTHON_BUILD_DIR}/lib/PYTHON_PACKAGE_VERSION
+    echo "${PYTHON_BUILD_VERSION}.${PYTHIA_VERSION}" \
+        > "${BUILD_DIR}/${PYTHON_BUILD_DIR}/lib/PYTHIA_VERSION"
 
     echo "::endgroup::"
 }
@@ -228,47 +228,41 @@ cleanup_install_dir() {
 #
 # Create the distributable archive.
 #
-# It also generates the symlink to latest build.
-#
 # Args:
-#  * kind = python3.8 (for example)
 #  * target_dir = name of the dir to be archived.
 #
 make_dist(){
-    kind=$1
-    target_dir=$2
-
-    target_path=../dist/${kind}/${OS}/${ARCH}
-    target_common=python-${PYTHON_BUILD_VERSION}.${PYTHON_PACKAGE_VERSION}-${OS}-${ARCH}
-    target_tar=${target_path}/${target_common}.tar
-    target_tar_gz=${target_tar}.gz
-
-    tar_gz_file=${target_dir}.tar.gz
-    tar_gz_source_file=${target_common}.tar.gz
+    local target_dir=$1
+    local full_ver="${PYTHON_BUILD_VERSION}.${PYTHIA_VERSION}"
+    local target_path="../${DIST_DIR}/${full_ver}"
+    local target_tar="${target_path}/python-${full_ver}-${OS}-${ARCH}.tar"
 
-    # Create a clean dist dir.
-    execute rm -rf ${DIST_DIR}
-    execute mkdir -p ${DIST_DIR}/${kind}/${OS}/${ARCH}
+    # Clean dist dir and only create a sub-dir for current version.
+    execute rm -rf "${DIST_DIR}"
+    execute mkdir -p "${DIST_DIR}/${full_ver}"
 
-    # Create tar inside dist dir.
-    execute pushd ${BUILD_DIR}
-        echo "#### Creating $target_tar_gz from $target_dir. ####"
-        execute tar -cf $target_tar $target_dir
-        execute gzip $target_tar
+    execute pushd "${BUILD_DIR}"
+        echo "#### Creating ${target_tar}.gz from $target_dir. ####"
+        execute tar -cf "$target_tar" "$target_dir"
+        execute gzip "$target_tar"
     execute popd
 }
 
 #
 # Construct a SFTP batch file for uploading testing packages.
-# Commands prefixed with a '-' are allowed to fail.
+# Files are uploaded with a temporary name and then renamed to final name.
 #
 build_publish_dist_sftp_batch() {
-    # This matches the GitHub's hierarchy for releases of production packages.
-    local upload_version_dir="$PYTHON_BUILD_VERSION.$PYTHON_PACKAGE_VERSION"
+    local full_ver="${PYTHON_BUILD_VERSION}.${PYTHIA_VERSION}"
+    local local_dir="${DIST_DIR}/${full_ver}"
+    local upload_dir="testing/${full_ver}"
+    local pkg_file="python-${full_ver}-${OS}-${ARCH}.tar.gz"
+    local local_file="${local_dir}/${pkg_file}"
+    local dest_file="${upload_dir}/${pkg_file}"
 
-    # Files are uploaded with a temp name and then renamed to final name.
-    echo "lcd dist/python/$OS/$ARCH/" > publish_dist_sftp_batch
-    echo "-mkdir testing/$upload_version_dir" >> publish_dist_sftp_batch
-    echo "put python-$PYTHON_BUILD_VERSION.$PYTHON_PACKAGE_VERSION-$OS-$ARCH.tar.gz testing/$upload_version_dir/python-$PYTHON_BUILD_VERSION.$PYTHON_PACKAGE_VERSION-$OS-$ARCH.tar.gz.part" >> publish_dist_sftp_batch
-    echo "rename testing/$upload_version_dir/python-$PYTHON_BUILD_VERSION.$PYTHON_PACKAGE_VERSION-$OS-$ARCH.tar.gz.part testing/$upload_version_dir/python-$PYTHON_BUILD_VERSION.$PYTHON_PACKAGE_VERSION-$OS-$ARCH.tar.gz" >> publish_dist_sftp_batch
+    # The mkdir command is prefixed with '-' to allow it to fail because
+    # $upload_dir exists if this is not the first upload for this version.
+    echo "-mkdir $upload_dir"                    > build/publish_dist_sftp_batch
+    echo "put $local_file ${dest_file}.part"    >> build/publish_dist_sftp_batch
+    echo "rename ${dest_file}.part $dest_file"  >> build/publish_dist_sftp_batch
 }
diff --git a/publish_dist.sh b/publish_dist.sh
@@ -0,0 +1,42 @@
+#!/usr/bin/env bash
+#
+# Uploads dist packages to testing/ on Chevah's server, then shows final links.
+# To be used through GitHub actions.
+
+set -o nounset
+set -o errexit
+set -o pipefail
+
+dest_server="bin.chevah.com"
+dest_user="github-upload"
+root_link="https://${dest_server}:20443/testing"
+
+# The build/publish_dist_sftp_batch file is generated by the build process.
+# The private key comes from GitHub Secrets through the configured workflow.
+sftp_opts="\
+    -b build/publish_dist_sftp_batch -i priv_key -o StrictHostKeyChecking=yes"
+
+# Get $OS var and set sftp command accordingly.
+source BUILD_ENV_VARS
+case $OS in
+    win)
+        # To use an RSA key, upstream SFTP is installed through GitHub actions.
+        sftp_cmd="/c/Progra~1/OpenSSH-Win64/sftp.exe"
+        ;;
+    *)
+        sftp_cmd="sftp"
+        ;;
+esac
+
+$sftp_cmd $sftp_opts ${dest_user}@${dest_server}
+
+# Get $DIST_DIR.
+source pythia.conf
+
+# As dist/ is rebuilt on every build, it should only have 1 sub-dir with 1 pkg.
+upload_dir=$(cd $DIST_DIR && ls -1)
+pkg_name=$(cd $DIST_DIR/$upload_dir && ls -1)
+
+# Local hierarchy matches the remote one.
+echo "Package $pkg_name uploaded to: ${root_link}/${upload_dir}/"
+echo "Direct link: ${root_link}/${upload_dir}/${pkg_name}"
diff --git a/pythia b/pythia
@@ -2,7 +2,7 @@
 #
 # Pythia's script for building Python.
 
-# Set versions for the software to be built.
+# Set versions for the software to be built and other defaults.
 source pythia.conf
 
 # Import shared and specific code.
@@ -11,37 +11,32 @@ source ./functions_pythia.sh
 
 # Git revision to inject into Python's sys.version string through chevahbs
 # on non-Windows platforms. Also used for compat tests and archived in the dist.
-PYTHON_PACKAGE_VERSION="$(git log -n 1 --no-merges --pretty=format:%h)"
+PYTHIA_VERSION="$(git log -n 1 --no-merges --pretty=format:%h)"
 exit_on_error $? 250
 
 # Export the variables needed by the chevahbs scripts and the test phase.
-export PYTHON_BUILD_VERSION PYTHON_PACKAGE_VERSION
+export PYTHON_BUILD_VERSION PYTHIA_VERSION
 export BUILD_ZLIB BUILD_BZIP2 BUILD_LIBEDIT BUILD_LIBFFI BUILD_OPENSSL
 
-# Import build env vars set by "./brink.sh detect_os" and export needed ones.
-if [ -f ./BUILD_ENV_VARS ]; then
-    source ./BUILD_ENV_VARS
-else
-    (>&2 echo "Missing BUILD_ENV_VARS file!")
-    echo "Run './brink.sh detect_os' first, as per the README file..."
-    exit 249
+
+# OS detection is slow on Windows, only execute it when the file is missing.
+if [ ! -r ./BUILD_ENV_VARS ]; then
+    execute ./brink.sh detect_os
 fi
+# Import build env vars as set by brink.sh.
+source ./BUILD_ENV_VARS
+
 # On Unix, use $ARCH to choose between 32bit or 64bit packages. It's possible
 # to force a 32bit build on a 64bit machine, e.g. by setting ARCH in brink.sh
 # as "x86" instead of "x64" for a certain platform.
 # $ARCH is also used when "building" Python on Windows and for testing.
 # $OS is used when patching/configuring/building/testing.
 export ARCH OS
 
-# Sub-directories for building from sources and the final distributable.
-BUILD_DIR="build"
-DIST_DIR="dist"
-
 # Local variables for the build process.
-LOCAL_PYTHON_BINARY_DIST="$PYTHON_VERSION-$OS-$ARCH"
-INSTALL_DIR="$PWD/$BUILD_DIR/$LOCAL_PYTHON_BINARY_DIST"
-PYTHON_BIN="$INSTALL_DIR/bin/$PYTHON_VERSION"
 PYTHON_BUILD_DIR="$PYTHON_VERSION-$OS-$ARCH"
+INSTALL_DIR="$PWD/$BUILD_DIR/$PYTHON_BUILD_DIR"
+PYTHON_BIN="$INSTALL_DIR/bin/$PYTHON_VERSION"
 
 # Explicitly choose the C compiler in order to make it possible to switch
 # between native compilers and GCC on platforms such as the BSDs and Solaris.
@@ -107,7 +102,7 @@ command_build() {
     cleanup_install_dir
 
     # Build the new package.
-    make_dist 'python' ${PYTHON_BUILD_DIR}
+    make_dist ${PYTHON_BUILD_DIR}
 
     # Generate a SFTP batch for uploading the package.
     build_publish_dist_sftp_batch
@@ -150,7 +145,7 @@ build_python() {
     if [ $OS = "win" ]; then
         # Python "build" is a very special case under Windows.
         execute pushd src/Python-Windows
-        ./chevahbs Python $PYTHON_BUILD_VERSION $INSTALL_DIR
+        execute ./chevahbs Python $PYTHON_BUILD_VERSION $INSTALL_DIR
         execute popd
     else
         build Python $PYTHON_BUILD_VERSION
@@ -237,6 +232,7 @@ command_test() {
 help_text_compat=\
 "Run the test suite from chevah/compat master."
 command_compat() {
+    local new_python_conf="${PYTHON_BUILD_VERSION}.${PYTHIA_VERSION}"
     execute pushd "$BUILD_DIR"
 
     # This is quite hackish, as compat is arm-twisted to use the local version.
@@ -249,14 +245,12 @@ command_compat() {
     # Copy over current brink stuff, as some changes might require it.
     execute cp ../../brink.{conf,sh} ./
     # Patch compat to use the newly-built Python, then copy it to cache/.
-    execute echo -e \
-        "\nPYTHON_CONFIGURATION=default@${PYTHON_BUILD_VERSION}.${PYTHON_PACKAGE_VERSION}" \
-        >> brink.conf
+    echo -e "\nPYTHON_CONFIGURATION=default@${new_python_conf}" >> brink.conf
     execute mkdir cache
-    execute cp -r ../$LOCAL_PYTHON_BINARY_DIST cache/
+    execute cp -r ../"$PYTHON_BUILD_DIR" cache/
     # Make sure everything is done from scratch in the current dir.
     unset CHEVAH_CACHE CHEVAH_BUILD
-    # Some tests might fail due to causes which are not related to python.
+    # Some tests might still fail due to causes not related to the new Python.
     execute ./brink.sh deps
     execute ./brink.sh test_ci