Jenkins WIP

globus · Apr 26, 2024 · 036d40e · 036d40e
1 parent e5e2212
commit 036d40e
Show file tree

Hide file tree

Showing 4 changed files with 114 additions and 19 deletions.
diff --git a/compute_endpoint/packaging/JenkinsFile b/compute_endpoint/packaging/JenkinsFile
@@ -0,0 +1,89 @@
+#! groovy
+
+@Library('gcs-build-scripts') _
+
+pipeline {
+    agent none
+    options {
+        buildDiscarder(
+            logRotator(
+                numToKeepStr: '5',
+                artifactNumToKeepStr: '1',
+                artifactDaysToKeepStr: '7'
+            )
+        )
+        timeout(time: 3, unit: 'HOURS')
+        disableConcurrentBuilds()
+    }
+    stages {
+        stage ("Prep source") {
+            agent {label "package_creator"}
+            steps {
+                checkout scm
+                script {
+                    def py_full_version = sh(script: "/opt/globus-python/bin/python3 -c 'import sys; print(\"{}.{}.{}\".format(*sys.version_info))'", returnStdout: true).trim()
+                    def (py_epoch_version, py_major_version, py_minor_version) = py_full_version.tokenize(".")
+                    def pkg_version = "not yet set"
+
+                    def venv_py = "${env.WORKSPACE}/compute_endpoint/packaging/venv-${py_epoch_version}${py_major_version}/bin/python"
+                    sh "git clean -fdx"
+                    sh "git checkout kevin_package_test"
+//                     dir("compute_endpoint/") {
+//                         env.PKG_TARBALL = "globus_compute_endpoint-${env.PKG_VERSION.replace("~", "")}.tar.gz"
+//                         env.PREREQS_TARBALL = "globus_compute_endpoint-prereqs-py${env.PY_MAJOR_VERSION}-${env.PKG_VERSION}.tar.xz"
+//                     }
+
+                    dir("compute_endpoint/packaging/") {
+                        env.DEB_SOURCE_STASH_NAME = "${UUID.randomUUID()}";
+//                         env.RPM_SOURCE_STASH_NAME = "${UUID.randomUUID()}";
+                        sh(script: "make show_vars")
+                        sh(script: "make setup_dist_for_deb setup_dist_for_rpm")
+                        sh(script: "make show_vars")
+                        echo "PY_MAJOR_VERSION = ${env.PY_MAJOR_VERSION}"
+                    }
+                    dir("compute_endpoint/") {
+                        pkg_version = sh(script: "${venv_py} setup.py --version", returnStdout: true).trim().replace("-", "~")
+                        echo "pkg_version = ${pkg_version}"
+                    }
+                    env.PKG_TARBALL = "globus_compute_endpoint-${pkg_version.replace("~", "")}.tar.gz"
+                    env.PREREQS_TARBALL = "globus_compute_endpoint-prereqs-py${py_epoch_version}${py_major_version}-${pkg_version}.tar.xz"
+                    dir("compute_endpoint/packaging/dist/") {
+                        sh "ls -l ./"
+                        sh "cp ${env.PKG_TARBALL} ${env.PREREQS_TARBALL} debbuild/globus-compute-agent/"
+                        sh "ls -lR debbuild/globus-compute-agent/"
+                        sh "mv debbuild/globus-compute-agent/ ${env.DEB_SOURCE_STASH_NAME}"
+//                         sh "mv dist/rpmbuild/globus-compute-agent/ ${env.DEB_SOURCE_STASH_NAME}"
+                        stash(name: env.DEB_SOURCE_STASH_NAME, includes: "${env.DEB_SOURCE_STASH_NAME}/**/*")
+//                         stash(name: env.RPM_SOURCE_STASH_NAME, includes: "${env.RPM_SOURCE_STASH_NAME}/**/*")
+                    }
+                }
+            }
+        }
+        stage ("Build packages") {
+            steps {
+                script {
+                    lock(resource: 'globus-compute-agent-build') {
+                        parallel "debian": {
+                            def extra_tarball_map = [
+                                prereqs: env.PREREQS_TARBALL
+                            ]
+                            echo "extra_tarball_map = ${extra_tarball_map}"
+                            env.DEB_ARTIFACTS_STASH = buildDebian(
+                                    env.DEB_SOURCE_STASH_NAME,
+                                    env.PKG_TARBALL,
+                                    require_gcs5_repo: true,
+                                    epic: null,
+                                    exclude: null,
+                                    extra_tarball_map: extra_tarball_map)
+//                         }, "rpm": {
+//                             env.RPM_ARTIFACTS_STASH = buildMock(
+//                                 env.RPM_SOURCE_STASH_NAME,
+//                                 env.PKG_TARBALL,
+//                                 true,
+//                                 null)
+                        }, "failFast": true
+                    }
+                }
+            }
+        }    }
+}
diff --git a/compute_endpoint/packaging/Makefile b/compute_endpoint/packaging/Makefile
@@ -27,6 +27,7 @@ PIP_NAME_U := $(shell echo $(PIP_NAME_D) | tr '-' '_')
 
 PKG_VERSION := $(shell cd ../; "$(VENV_PY)" setup.py --version | tr '-' '~')
 PKG_WHEEL = $(PIP_NAME_U)-$(PKG_VERSION)-py$(PY_MAJOR_VERSION)-none-any.whl
+PKG_TARBALL = $(PIP_NAME_U)-$(PKG_VERSION).tar.gz
 PREREQS_TARBALL_NAME = $(PIP_NAME_U)-prereqs-py$(PY_VERSION)-$(PKG_VERSION).tar.xz
 
 OS_CODENAME := $(shell test -f /etc/os-release && . /etc/os-release; echo $${VERSION_CODENAME:-focal})
@@ -107,7 +108,7 @@ $(PKG_WHEEL): $(VENV_PY)
 	   { /bin/echo -e "\nBUILD COMPUTE FROM A RELEASE TAG (current branch: $$(git branch --show-current))"; exit 1; } \
 	    fi \
 	 && rm -rf tests/ \
-	 && "$(VENV_PY)" -m build --wheel -o ../../ \
+	 && "$(VENV_PY)" -m build -o ../../ \
 	)
 
 wheel: $(PKG_WHEEL)  ##-Make the wheel (note that this does *not* include dependencies)
@@ -120,17 +121,13 @@ prereq_tarball: $(PREREQS_TARBALL_NAME)  ##-Make a tarball of wheel dependencies
 dist: $(PREREQS_TARBALL_NAME)  ##-Make the dist/ directory with prereqs and wheel ready for packaging step
 	rm -rf dist/ \
 	  && mkdir dist/ \
-	  && cp $(PREREQS_TARBALL_NAME) $(PKG_WHEEL) dist/
+	  && mv $(PREREQS_TARBALL_NAME) $(PKG_WHEEL) $(PKG_TARBALL) dist/
 
-deb_build_needs:  ##-Check that necessary executables are available before starting the DEB build.
-	@[ -x "$$(command -v dpkg-checkbuilddeps)" ] || { echo "'dpkg-checkbuilddeps' not found; missing 'dpkg-dev' package?"; exit 1; }
-	@dpkg-checkbuilddeps
-
-deb: deb_build_needs dist  ##-Build a Debian package of the Globus Compute Endpoint (.deb)
+setup_dist_for_deb: dist  ##-Place Debian-build required files in dist/
 	(   cd dist/ \
 	 && rm -rf debbuild/ \
 	 && mkdir -p debbuild/$(PKG_NAME)/wheels/ \
-	 && tar -C debbuild/$(PKG_NAME)/wheels/ -xf "$(PREREQS_TARBALL_NAME)" \
+	 && tar -C debbuild/$(PKG_NAME)/wheels/ --strip 1 -xf "$(PREREQS_TARBALL_NAME)" \
 	 && cp $(PKG_WHEEL) debbuild/$(PKG_NAME)/wheels/ \
 	 && cp -R ../debian debbuild/$(PKG_NAME)/ \
 	 && cp ../package_shim.sh debbuild/$(PKG_NAME)/ \
@@ -148,20 +145,22 @@ deb: deb_build_needs dist  ##-Build a Debian package of the Globus Compute Endpo
 	    -e "s/@PACKAGE_NAME@/$(PKG_NAME)/g" \
 	    -e "s/@PACKAGE_VERSION@/$(PKG_VERSION)/g" \
 	    -e "s/@PIP_NAME@/$(PIP_NAME_D)/g" \
-	 && dpkg-buildpackage -uc -us \
 	)
+
+deb_build_needs:  ##-Check that necessary executables are available before starting the DEB build.
+	@[ -x "$$(command -v dpkg-checkbuilddeps)" ] || { echo "'dpkg-checkbuilddeps' not found; missing 'dpkg-dev' package?"; exit 1; }
+	@dpkg-checkbuilddeps
+
+deb: deb_build_needs dist setup_dist_for_deb  ##-Build a Debian package of the Globus Compute Endpoint (.deb)
+	(cd dist/debbuild/$(PKG_NAME)/; dpkg-buildpackage -uc -us)
 	@echo "\nDEB package successfully built:"
 	@ls -lh dist/debbuild/*deb
 
-rpm_build_needs:  ##-Check that necessary executables are available before starting the RPM build.
-	@[ -x "$$(command -v rpmbuild)" ] || { echo "'rpmbuild' not found; missing 'rpmdevtools' or 'rpm-build' package(s)?"; exit 1; }
-
-rpm: rpm_build_needs dist  ##-Build an RPM package of the Globus Compute Endpoint (.rpm)
+setup_dist_for_rpm: dist  ##-Place RPM-build required files in dist/ (separated for CI building purposes)
 	(   cd dist/ \
-	 && pwd && ls \
 	 && rm -rf rpmbuild/ \
 	 && mkdir -p rpmbuild/{BUILD,BUILDROOT,RPMS,SPECS,SRPMS} rpmbuild/SOURCES/$(PKG_NAME)/wheels \
-	 && tar -C rpmbuild/SOURCES/$(PKG_NAME)/wheels -xf "$(PREREQS_TARBALL_NAME)" \
+	 && tar -C rpmbuild/SOURCES/$(PKG_NAME)/wheels --strip 1 -xf "$(PREREQS_TARBALL_NAME)" \
 	 && cp $(PKG_WHEEL) rpmbuild/SOURCES/$(PKG_NAME)/wheels/ \
 	 && cp ../package_shim.sh rpmbuild/SOURCES/$(PKG_NAME)/ \
 	 && sed \
@@ -170,7 +169,12 @@ rpm: rpm_build_needs dist  ##-Build an RPM package of the Globus Compute Endpoin
 	    -e "s/@PACKAGE_WHEEL@/$(PKG_WHEEL)/g" \
 	    -e "s/@PIP_NAME@/$(PIP_NAME_D)/g" \
 	    < ../fedora/$(PKG_NAME).spec.in > ./$(PKG_NAME).spec \
-	 && HOME="$$(pwd)" rpmbuild --define "_topdir $$(pwd)/rpmbuild" -ba ./$(PKG_NAME).spec \
 	)
+
+rpm_build_needs:  ##-Check that necessary executables are available before starting the RPM build.
+	@[ -x "$$(command -v rpmbuild)" ] || { echo "'rpmbuild' not found; missing 'rpmdevtools' or 'rpm-build' package(s)?"; exit 1; }
+
+rpm: rpm_build_needs dist setup_dist_for_rpm  ##-Build an RPM package of the Globus Compute Endpoint (.rpm)
+	(cd dist/; HOME="$$(pwd)" rpmbuild --define "_topdir $$(pwd)/rpmbuild" -ba ./$(PKG_NAME).spec)
 	@echo -e "\nRPM package successfully built:"
 	@ls -lh dist/rpmbuild/RPMS/**/*rpm
diff --git a/compute_endpoint/packaging/create-prereqs-tarball.sh b/compute_endpoint/packaging/create-prereqs-tarball.sh
@@ -90,12 +90,12 @@ done > dependent-prereqs.txt
 
 modified_time="$(TZ=UTC0 date --rfc-3339=seconds)"
 
-TZ=UTC0 tar -C "$download_dir/" --format=posix \
+TZ=UTC0 tar --format=posix \
   --pax-option=exthdr.name=%d/PaxHeaders/%f,delete=atime,delete=ctime,delete=mtime \
   --mtime="$modified_time" \
   --numeric-owner \
   --owner=0 \
   --group=0 \
   --mode="go-rwx,u+rw" \
-  -cf - . \
-  | xz -9
+  -cf - "$download_dir/" \
+  | gzip -9
diff --git a/compute_endpoint/packaging/debian/rules b/compute_endpoint/packaging/debian/rules
@@ -32,6 +32,8 @@ override_dh_auto_configure:
 	$(PYTHON3) -mvenv $${PWD}/$(TMP_VIRTUAL_ENV)
 	. "$(TMP_VIRTUAL_ENV)/bin/activate"; \
 	set -x; \
+	pwd
+	ls -lR
 	python3 -mpip install --no-index --no-cache-dir -I --compile -U wheels/pip-*.whl; \
 	python3 -mpip install --no-index --no-cache-dir -I --pre --compile --find-links=file://$${PWD}/wheels/ "@PIP_NAME@"