deploy: make builds repeatable and reproducible

All package build must "lock" and get the same commit in order to make the process of creating a package repeatable. This will force us to change the build commit hash every time a new build occurs. For dev packages, a temporary change will occur in the 'release' branch whenever a new package is released. In the end, for "real releases", this will mean committing the final commit hash to 'main'. Signed-off-by: Claudio André <[email protected]>
openwall · May 14, 2024 · 6b790e7 · 6b790e7
1 parent e0a639d
commit 6b790e7
Show file tree

Hide file tree

Showing 12 changed files with 34 additions and 38 deletions.
diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml
@@ -147,7 +147,8 @@ jobs:
             (github.event.inputs.push || false)
             }}
           build-args: |
-            TYPE="ALL"
+            TYPE=ALL
+            RELEASE_COMMIT=f9fedd238b0b1d69181c1fef033b85c787e96e57
           tags: ${{ steps.data.outputs.image_tags }}
           labels: |
             ${{ steps.meta.outputs.labels }}

diff --git a/deploy/Mac_ARM-Delivery.yml b/deploy/Mac_ARM-Delivery.yml
@@ -37,6 +37,7 @@ macOS_task:
     TEST: "macOS;"
     BASE: "Apple macOS M2"
     DEPLOY_PAK: "Yes"
+    RELEASE_COMMIT: "f9fedd238b0b1d69181c1fef033b85c787e96e57"
 
   info_script: |
     git clone --depth 10 https://github.com/openwall/john.git ~/tmp

diff --git a/deploy/Solaris-Delivery.yml b/deploy/Solaris-Delivery.yml
@@ -40,15 +40,15 @@ jobs:
       EXTRA: "full;extra"
       BUILD_OPTS: ""
       BASE: "Oracle Solaris"
-      _JUMBO_RELEASE: ${{ github.event.inputs.release }}
+      RELEASE_COMMIT: ${{ github.event.inputs.hash }}
 
     steps:
       - uses: actions/checkout@v4
       - name: Build on Solaris
         id: test
         uses: vmactions/solaris-vm@v1
         with:
-          envs: "TARGET_ARCH EXTRA BUILD_OPTS TEST BASE"
+          envs: "TARGET_ARCH EXTRA BUILD_OPTS TEST BASE RELEASE_COMMIT"
           usesh: true
           copyback: false
           prepare: |

diff --git a/deploy/Windows-Delivery.yml b/deploy/Windows-Delivery.yml
@@ -25,26 +25,19 @@ steps:
       # Setup image environment ##############################################
       $cygwinFolder = "C:\$env:cygwinPath"
 
-      if ($env:Test -eq "package") {
-          # Use upstream source code
-          cd $env:PROJ_FOLDER
-          Remove-Item ./JtR -Force -Recurse -ErrorAction SilentlyContinue
-          git clone -q --depth 1 https://github.com/openwall/john.git JtR
-      }
+      # Get upstream JtR source code
+      cd $env:PROJ_FOLDER
+      Remove-Item ./JtR -Force -Recurse -ErrorAction SilentlyContinue
+      git clone https://github.com/openwall/john.git JtR
+
       cd $env:BUILD_FOLDER
-      git describe --dirty=+ --always
-      echo '--------------------------------'
 
-      # $JUMBO_RELEASE = "f9fedd238b0b1d69181c1fef033b85c787e96e57"
+      # Force the build to be repeatable
+      git checkout "f9fedd238b0b1d69181c1fef033b85c787e96e57"
 
-      # Make it a reproducible build
-      if ($JUMBO_RELEASE) {
-          Write-Host "--------------------------------------------------------"
-          Write-Host "   =====  Deploying the release $JUMBO_RELEASE  =====   " -ForegroundColor White
-          Write-Host "--------------------------------------------------------"
-          git pull --unshallow
-          git checkout $JUMBO_RELEASE
-      }
+      Write-Host "--------------------------------------------------------"
+      Write-Host "   =====  Deploying the release $RELEASE_COMMIT  =====  " -ForegroundColor White
+      Write-Host "--------------------------------------------------------"
 
       # If needed (Cygwin is not installed) or as a workaround (Cygwin update is required)
       if ($env:Cygwin -eq "install") {

diff --git a/deploy/docker/Dockerfile b/deploy/docker/Dockerfile
@@ -29,8 +29,9 @@ COPY build.sh .
 
 # To build a release run a (docker build [...] --build-arg TYPE=ALL)
 ARG TYPE=ALL
+ARG RELEASE_COMMIT
 
-RUN ./build.sh "${TYPE}"
+RUN ./build.sh "$TYPE" "$RELEASE_COMMIT"
 
 # ==================================================================
 # Build the final lean Docker image

diff --git a/deploy/docker/build.sh b/deploy/docker/build.sh
@@ -27,13 +27,14 @@ function install_nvidia_opencl() {
 		nvidia-opencl-dev=*
 }
 type="$1"
+RELEASE_COMMIT="$2"
 
 # Required defines
 arch=$(uname -m)
 DEPLOY_PAK="Yes"
 BASE="CUDA on Ubuntu"
 TASK_RUNNING="Docker build"
-export -p DEPLOY_PAK BASE TASK_RUNNING
+export -p DEPLOY_PAK BASE TASK_RUNNING RELEASE_COMMIT
 
 # Build options (system wide, disable checks, etc.)
 SYSTEM_WIDE='--with-systemwide'
@@ -64,18 +65,17 @@ source ../helper.sh
 
 if true; then
 	# Get upstream JtR source code and the version string
-	#RELEASE="f9fedd238b0b1d69181c1fef033b85c787e96e57" # Remove line comment for a release
 	(
 		cd .. || exit 1
 		rm -rf tmp
 		git clone --depth 10 https://github.com/openwall/john.git tmp
 		cp -r tmp/. . && rm -rf tmp/
 
 		# Make it a reproducible build
-		if [[ -n "$RELEASE" ]]; then
-			echo "Deploying the release $RELEASE"
+		if [[ -n "$RELEASE_COMMIT" ]]; then
+			echo "Deploying the release $RELEASE_COMMIT"
 			git pull --unshallow
-			git checkout "$RELEASE"
+			git checkout "$RELEASE_COMMIT"
 		fi
 	)
 	do_get_version

diff --git a/deploy/flatpak/com.openwall.John.json b/deploy/flatpak/com.openwall.John.json
@@ -44,7 +44,7 @@
                 {
                     "type": "git",
                     "url": "https://github.com/openwall/john.git",
-                    "branch": "bleeding-jumbo"
+                    "commit": "f9fedd238b0b1d69181c1fef033b85c787e96e57"
                 },
                 {
                     "type": "file",

diff --git a/deploy/snap/build.sh b/deploy/snap/build.sh
@@ -54,18 +54,17 @@ source ../helper.sh
 
 if [[ "$1" == "PULL" ]]; then
 	# The pull phase will get upstream JtR source code and the version string
-	#RELEASE="f9fedd238b0b1d69181c1fef033b85c787e96e57" # Remove line comment for a release
 	(
 		cd .. || exit 1
 		rm -rf tmp
 		git clone --depth 10 https://github.com/openwall/john.git tmp
 		cp -r tmp/. .
 
 		# Make it a reproducible build
-		if [[ -n "$RELEASE" ]]; then
-			echo "Deploying the release $RELEASE"
+		if [[ -n "$RELEASE_COMMIT" ]]; then
+			echo "Deploying the release $RELEASE_COMMIT"
 			git pull --unshallow
-			git checkout "$RELEASE"
+			git checkout "$RELEASE_COMMIT"
 		fi
 	)
 	do_get_version

diff --git a/deploy/snap/snapcraft.yaml b/deploy/snap/snapcraft.yaml
@@ -94,6 +94,9 @@ parts:
     override-pull: |
       craftctl default
 
+      # Force the build to be repeatable
+      export RELEASE_COMMIT="f9fedd238b0b1d69181c1fef033b85c787e96e57"
+
       # Get latest files from upstream repository
       wget https://raw.githubusercontent.com/openwall/john-packages/release/deploy/snap/build.sh -O build.sh
       chmod +x build.sh

diff --git a/requirements.hash b/requirements.hash
@@ -1,8 +1,8 @@
-0d253abba47e4fcf07c59f2b01b00fb2b73f84ca1eaaca19930a33f9ef9223f2  ./ci_controller.sh
+bc092ea2321580e3199003febacba83d8204a8590db5b24788632958ad3440bc  ./ci_controller.sh
 539ff1db5a1cee05533a1f431560be008239e1facf930fb4242bc21eeaf0920b  ./clean_package.sh
 0fb3d5f7c4271a102b1f14c859e127ed1db013f011cb9b48a46e357614831c93  ./helper.sh
 e1a7e9691bfaba3398eb28ac724a79df5e76f66d243c97f142b2aa415b9bc27f  ./package_version.sh
 6877e23f9225f4d80cbc98de68e37784817e0a9f96b0ca2831f62533bb15f80e  ./run_tests.sh
-de6aab236ca5dd5e3f1b647b540d65a5740953e8d7c206755848fbfb65634cdb  ./show_info.sh
+5e4629cce7b4552876dc308a659e878b1b196df60327df50f462ec899d6d28ed  ./show_info.sh
 aab7868a06d5a06745a234907f4e26cbe794610fe14198674d595a638529e7bd  ./Handle-self-confined-system-wide-build.patch
 1f93d80dd97a5450c6e36759148d2767e29bbf987c078d5e5089b702e77aece5  ./Remove-peflags-from-the-default-target.patch
diff --git a/scripts/ci_controller.sh b/scripts/ci_controller.sh
@@ -61,10 +61,10 @@ if [[ "$2" == "BUILD" ]]; then
 	do_get_version
 
 	# Make it a reproducible build
-	if [[ -n "$_JUMBO_RELEASE" ]]; then
-		echo "Deploying the release $_JUMBO_RELEASE"
+	if [[ -n "$RELEASE_COMMIT" ]]; then
+		echo "Deploying the release $RELEASE_COMMIT"
 		git pull --unshallow
-		git checkout "$_JUMBO_RELEASE"
+		git checkout "$RELEASE_COMMIT"
 	fi
 	echo ""
 	echo "---------------------------- BUILDING -----------------------------"

diff --git a/scripts/show_info.sh b/scripts/show_info.sh
@@ -100,5 +100,3 @@ echo '------ Task ------'
 echo "Running on: ${BASE-unknown}"
 echo "Doing: ${TASK_RUNNING-unknown}"
 echo '--------------------------------'
-echo "Release: ${RELEASE-none}"
-echo '--------------------------------'