gh: bump deps and use `faasmctl` (#19) #146

Workflow file for this run

	name: Test example applications

	on:
	push:
	branches: [main]
	pull_request:
	branches: [main]
	types: [opened, synchronize, reopened, ready_for_review]

	jobs:
	# Cancel previous running actions for the same PR
	cancel_previous:
	if: github.event.pull_request.draft == false
	runs-on: ubuntu-latest
	steps:
	- name: Cancel Workflow Action
	uses: styfle/[email protected]

	checks:
	if: github.event.pull_request.draft == false
	runs-on: ubuntu-latest
	steps:
	# --- Update code ---
	- name: "Checkout code"
	uses: actions/checkout@v4
	with:
	submodules: true
	# --- Formatting ---
	- name: "Format code"
	run: ./bin/inv_wrapper.sh format-code --check

	# Work-out whether we need to re-build the examples. We need to re-build the
	# examples if either _any_ fo the examples has changed, or the WASM cache
	# has expired
	needs-build:
	if: github.event.pull_request.draft == false
	runs-on: ubuntu-latest
	outputs:
	needs-wasm: ${{ (steps.filter.outputs.examples-changed == 'true') \|\| (steps.wasm-cache.outputs.cache-hit != 'true') }}
	steps:
	- name: "Checkout code"
	uses: actions/checkout@v4
	# Check if any of the submodules have been modified
	- uses: dorny/paths-filter@v2
	id: filter
	with:
	filters: \|
	examples-changed:
	- './examples/**'
	# Check if we have a WASM cache hit
	- name: "Get CPP/Python commits"
	id: submodule-commit
	run: \|
	git submodule update --init ./cpp
	git submodule update --init ./python
	sudo apt install -y zstd
	git config --global --add safe.directory "$GITHUB_WORKSPACE"
	echo "cpp-commit=$(git submodule status ./cpp \| cut '-d ' -f 2)" >> $GITHUB_OUTPUT
	echo "py-commit=$(git submodule status ./python \| cut '-d ' -f 2)" >> $GITHUB_OUTPUT
	- uses: actions/cache/[email protected]
	id: wasm-cache
	with:
	path: ./wasm
	key: wasm-${{ steps.submodule-commit.outputs.cpp-commit }}-${{ steps.submodule-commit.outputs.py-commit }}
	lookup-only: true

	build-examples:
	needs: needs-build
	if: ${{ needs.needs-build.outputs.needs-wasm == 'true' }}
	runs-on: ubuntu-latest
	container:
	image: faasm.azurecr.io/examples-build:0.3.1_0.3.1
	credentials:
	username: ${{ secrets.ACR_SERVICE_PRINCIPAL_ID }}
	password: ${{ secrets.ACR_SERVICE_PRINCIPAL_PASSWORD }}
	steps:
	- name: "Checkout code"
	uses: actions/checkout@v4
	with:
	submodules: true
	- name: "Build FFmpeg"
	run: ./bin/inv_wrapper.sh ffmpeg
	- name: "Build ImageMagick"
	run: ./bin/inv_wrapper.sh libpng imagemagick
	- name: "Build Kernels"
	run: ./bin/inv_wrapper.sh kernels kernels --native
	- name: "Build LAMMPS"
	run: ./bin/inv_wrapper.sh lammps lammps --native
	- name: "Build Lulesh"
	run: ./bin/inv_wrapper.sh lulesh lulesh --native
	- name: "Build TensorFlow"
	run: ./bin/inv_wrapper.sh tensorflow
	- name: "Build PolyBench/C"
	run: ./bin/inv_wrapper.sh polybench polybench --native
	- name: "Build functions used in the tests"
	run: ./bin/inv_wrapper.sh func.tests
	- name: "Get CPP/Python commits"
	id: submodule-commit
	run: \|
	apt install -y zstd
	git config --global --add safe.directory "$GITHUB_WORKSPACE"
	echo "cpp-commit=$(git submodule status ./cpp \| cut '-d ' -f 2)" >> $GITHUB_OUTPUT
	echo "py-commit=$(git submodule status ./python \| cut '-d ' -f 2)" >> $GITHUB_OUTPUT
	# Also move to a different path to restore from
	mv /usr/local/faasm/wasm ./wasm
	# If we are here we _always_ want to overwrite the cache
	- name: "Prepare WASM cache"
	uses: actions/cache/[email protected]
	if: always()
	with:
	path: ./wasm
	key: wasm-${{ steps.submodule-commit.outputs.cpp-commit }}-${{ steps.submodule-commit.outputs.py-commit }}

	run-examples-faasmctl:
	needs: [build-examples]
	if:
	always() &&
	!cancelled() &&
	github.event.pull_request.draft == false &&
	(needs.build-examples.result == 'success' \|\| needs.build-examples.result == 'skipped')
	runs-on: ubuntu-latest
	strategy:
	fail-fast: false
	matrix:
	wasm_vm: [wavm, wamr]
	defaults:
	run:
	shell: bash
	env:
	CONAN_CACHE_MOUNT_SOURCE: ~/.conan
	FAASM_INI_FILE: ./faasm.ini
	FAASM_VERSION: 0.15.0
	WASM_VM: ${{ matrix.wasm_vm }}
	steps:
	- uses: csegarragonz/set-compose-version-action@main
	with:
	compose-version: "2.22.0"
	# A regular `faasmctl` deployment pulls a lot of docker images, so we
	# may run out of disk space
	- name: "Maximize build space"
	uses: easimon/maximize-build-space@master
	with:
	# Leave 25 GB for the / partition for docker images (stored under
	# /var/lib/docker)
	root-reserve-mb: 25600
	remove-android: 'true'
	remove-codeql: 'true'
	remove-docker-images: 'true'
	remove-dotnet: 'true'
	remove-haskell: 'true'
	- name: "Checkout code"
	uses: actions/checkout@v4
	- name: "Install faasmctl"
	run: pip3 install $(cat ./requirements.txt \| grep faasmctl \| tr -d '\n')
	- name: "Start a Faasm cluster to run the examples"
	run: faasmctl deploy.compose --workers=2
	# This can fail when the container isn't ready, so we want to retry
	- name: "Wait for upload server to be available"
	run: \|
	(echo "Attempt 1" && faasmctl cli.faasm --cmd "./deploy/local/wait_for_upload.sh upload 8002") \|\| \
	(echo "Attempt 2" && faasmctl cli.faasm --cmd "./deploy/local/wait_for_upload.sh upload 8002") \|\| \
	(echo "Attempt 3" && faasmctl cli.faasm --cmd "./deploy/local/wait_for_upload.sh upload 8002") \|\| \
	(echo "Wait for upload failed after retries" && faasmctl logs -s upload && exit 1)
	- name: "Get CPP/Python commits"
	id: submodule-commit
	run: \|
	sudo apt install -y zstd
	git config --global --add safe.directory "$GITHUB_WORKSPACE"
	git submodule update --init ./cpp
	git submodule update --init ./python
	echo "cpp-commit=$(git submodule status ./cpp \| cut '-d ' -f 2)" >> $GITHUB_OUTPUT
	echo "py-commit=$(git submodule status ./python \| cut '-d ' -f 2)" >> $GITHUB_OUTPUT
	- name: "Get WASM cache"
	uses: actions/cache/[email protected]
	id: cpp-wasm-cache
	with:
	path: ./wasm
	key: wasm-${{ steps.submodule-commit.outputs.cpp-commit }}-${{ steps.submodule-commit.outputs.py-commit }}
	fail-on-cache-miss: true
	- name: "Upload all WASM files to the cluster"
	run: ./bin/inv_wrapper.sh wasm.upload
	- name: "Run MPI kernels"
	run: \|
	# The global MPI kernel stopped working when enabling SIMD
	# faasmctl invoke kernels-mpi global --cmdline '10 1024'
	faasmctl invoke kernels-mpi p2p --cmdline '10 1024 1024' --mpi-world-size 4
	faasmctl invoke kernels-mpi sparse --cmdline '10 10 5' --mpi-world-size 4
	faasmctl invoke kernels-mpi transpose --cmdline '10 1024 32' --mpi-world-size 4
	faasmctl invoke kernels-mpi stencil --cmdline '10 1000' --mpi-world-size 4
	# The dgemm MPI kernel does not work because we are missing
	# implementations for MPI_Comm_group, MPI_Group_incl, and MPI_Comm_create
	# faasmctl invoke kernels-mpi dgemm --cmdline '10 1024 32 1' --mpi-world-size 4
	faasmctl invoke kernels-mpi nstream --cmdline '10 1024 32' --mpi-world-size 4
	faasmctl invoke kernels-mpi reduce --cmdline '10 1024' --mpi-world-size 4
	# The random MPI kernel does not work because we are missing
	# implementations for MPI_Alltoallv
	# faasmctl invoke kernels-mpi random --cmdline '32 20' --mpi-world-size 4
	- name: "Run OpenMP kernels"
	if: "contains(env.WASM_VM, 'wavm')"
	run: \|
	faasmctl invoke kernels-omp global --cmdline '2 10 1024'
	faasmctl invoke kernels-omp p2p --cmdline '2 10 1024 1024'
	# The sparse OpenMP kernel fails during the result integrity checks
	# faasmctl invoke kernels-omp sparse --cmdline '2 10 10 5'
	# The transpose OpenMP kernel can not be cross-compiled due to problems
	# with the static_for signature and symbol definition
	# faasmctl invoke kernels-omp transpose --cmdline '10 1024 32'
	# The stencil OpenMP kernel can not be cross-compiled due to a problem
	# with the faasm_sm_reduce signature
	# faasmctl invoke kernels-omp stencil --cmdline '10 1000'
	faasmctl invoke kernels-omp dgemm --cmdline '2 2 1024 32'
	faasmctl invoke kernels-omp nstream --cmdline '2 10 1024 32'
	faasmctl invoke kernels-omp reduce --cmdline '2 10 1024'
	# The random OpenMP kernel can not be cross-compiled due to a problem
	# with the faasm_sm_reduce signature
	# faasmctl invoke kernels-omp random --cmdline '32 20'
	- name: "Run PolyBench/C"
	# We deliberately enumerate all the supported functions here and
	# execute them individually, rather than using the microbench runner.
	# This is so that we can more easily track what functions are supported
	# and what functions are not
	run: \|
	faasmctl invoke polybench poly_covariance
	faasmctl invoke polybench poly_correlation
	faasmctl invoke polybench poly_2mm
	faasmctl invoke polybench poly_3mm
	faasmctl invoke polybench poly_atax
	faasmctl invoke polybench poly_bicg
	faasmctl invoke polybench poly_doitgen
	faasmctl invoke polybench poly_mvt
	faasmctl invoke polybench poly_cholesky
	faasmctl invoke polybench poly_durbin
	faasmctl invoke polybench poly_gramschmidt
	faasmctl invoke polybench poly_lu
	faasmctl invoke polybench poly_ludcmp
	faasmctl invoke polybench poly_trisolv
	faasmctl invoke polybench poly_deriche
	faasmctl invoke polybench poly_floyd-warshall
	faasmctl invoke polybench poly_nussinov
	faasmctl invoke polybench poly_adi
	faasmctl invoke polybench poly_fdtd-2d
	faasmctl invoke polybench poly_heat-3d
	faasmctl invoke polybench poly_jacobi-1d
	faasmctl invoke polybench poly_jacobi-2d
	faasmctl invoke polybench poly_seidel-2d
	- name: "Run ImageMagick"
	if: "contains(env.WASM_VM, 'wavm')"
	run: faasmctl invoke imagemagick main --cmdline 'faasm://im/sample_image.png -flip faasm://im/image_out.png'
	- name: "Run LAMMPS"
	run: faasmctl invoke lammps main --cmdline '-in faasm://lammps-data/in.controller.wall'
	- name: "Run LULESH"
	if: "contains(env.WASM_VM, 'wavm')"
	run: faasmctl invoke lulesh main --cmdline '-i 5 -s 20 -r 11 -c 1 -b 1' --data '4'
	- name: "Run TFlite check"
	if: "contains(env.WASM_VM, 'wavm')"
	run: faasmctl invoke tf check
	- name: "Run FFmpeg check"
	run: faasmctl invoke ffmpeg check

	# run-examples:
	# needs: [build-examples]
	# if:
	# always() &&
	# !cancelled() &&
	# github.event.pull_request.draft == false &&
	# (needs.build-examples.result == 'success' \|\| needs.build-examples.result == 'skipped')
	# runs-on: ubuntu-latest
	# strategy:
	# fail-fast: false
	# matrix:
	# wasm_vm: [wavm, wamr]
	# defaults:
	# run:
	# working-directory: /usr/local/code/faasm
	# env:
	# CGROUP_MODE: off
	# DEFAULT_MPI_WORLD_SIZE: 4
	# HOST_TYPE: ci
	# LOG_LEVEL: info
	# NETNS_MODE: off
	# OVERRIDE_CPU_COUNT: 30
	# PLANNER_HOST: planner
	# PLANNER_PORT: 8080
	# REDIS_QUEUE_HOST: redis
	# REDIS_STATE_HOST: redis
	# WASM_VM: ${{ matrix.wasm_vm }}
	# container:
	# image: faasm.azurecr.io/examples-run:0.15.0
	# credentials:
	# username: ${{ secrets.ACR_SERVICE_PRINCIPAL_ID }}
	# password: ${{ secrets.ACR_SERVICE_PRINCIPAL_PASSWORD }}
	# services:
	# redis:
	# image: faasm.azurecr.io/redis:0.15.0
	# credentials:
	# username: ${{ secrets.ACR_SERVICE_PRINCIPAL_ID }}
	# password: ${{ secrets.ACR_SERVICE_PRINCIPAL_PASSWORD }}
	# minio:
	# image: faasm.azurecr.io/minio:0.15.0
	# credentials:
	# username: ${{ secrets.ACR_SERVICE_PRINCIPAL_ID }}
	# password: ${{ secrets.ACR_SERVICE_PRINCIPAL_PASSWORD }}
	# env:
	# MINIO_ROOT_USER: minio
	# MINIO_ROOT_PASSWORD: minio123
	# planner:
	# image: faasm.azurecr.io/planner:0.10.0
	# steps:
	# - name: "Download examples wasm"
	# uses: actions/[email protected]
	# with:
	# name: examples-wasm
	# path: /usr/local/faasm/wasm
	# - name: "Run codegen for all the examples"
	# run: \|
	# ./bin/inv_wrapper.sh codegen.user kernels-mpi
	# ./bin/inv_wrapper.sh codegen.user kernels-omp
	# ./bin/inv_wrapper.sh codegen.user polybench
	# ./bin/inv_wrapper.sh codegen imagemagick main
	# ./bin/inv_wrapper.sh codegen lammps main
	# ./bin/inv_wrapper.sh codegen lulesh main
	# ./bin/inv_wrapper.sh codegen ffmpeg check
	# ./bin/inv_wrapper.sh codegen tf check
	# - name: "Put all data files in place"
	# run: ./bin/inv_wrapper.sh data
	# working-directory: /code/examples
	# - name: "Run MPI kernels"
	# run: \|
	# # The global MPI kernel stopped working when enabling SIMD
	# # ./bin/inv_wrapper.sh run.pool kernels-mpi global --cmdline '10 1024'
	# ./bin/inv_wrapper.sh run.pool kernels-mpi p2p --cmdline '10 1024 1024'
	# ./bin/inv_wrapper.sh run.pool kernels-mpi sparse --cmdline '10 10 5'
	# ./bin/inv_wrapper.sh run.pool kernels-mpi transpose --cmdline '10 1024 32'
	# ./bin/inv_wrapper.sh run.pool kernels-mpi stencil --cmdline '10 1000'
	# # The dgemm MPI kernel does not work because we are missing
	# # implementations for MPI_Comm_group, MPI_Group_incl, and MPI_Comm_create
	# # ./bin/inv_wrapper.sh run.pool kernels-mpi dgemm --cmdline '10 1024 32 1'
	# ./bin/inv_wrapper.sh run.pool kernels-mpi nstream --cmdline '10 1024 32'
	# ./bin/inv_wrapper.sh run.pool kernels-mpi reduce --cmdline '10 1024'
	# # The random MPI kernel does not work because we are missing
	# # implementations for MPI_Alltoallv
	# # ./bin/inv_wrapper.sh run.pool kernels-mpi random --cmdline '32 20'
	# - name: "Run OpenMP kernels"
	# if: "contains(env.WASM_VM, 'wavm')"
	# run: \|
	# ./bin/inv_wrapper.sh run.pool kernels-omp global --cmdline '2 10 1024'
	# ./bin/inv_wrapper.sh run.pool kernels-omp p2p --cmdline '2 10 1024 1024'
	# # The sparse OpenMP kernel fails during the result integrity checks
	# # ./bin/inv_wrapper.sh run.pool kernels-omp sparse --cmdline '2 10 10 5'
	# # The transpose OpenMP kernel can not be cross-compiled due to problems
	# # with the static_for signature and symbol definition
	# # ./bin/inv_wrapper.sh run.pool kernels-omp transpose --cmdline '10 1024 32'
	# # The stencil OpenMP kernel can not be cross-compiled due to a problem
	# # with the faasm_sm_reduce signature
	# # ./bin/inv_wrapper.sh run.pool kernels-omp stencil --cmdline '10 1000'
	# ./bin/inv_wrapper.sh run.pool kernels-omp dgemm --cmdline '2 2 1024 32'
	# ./bin/inv_wrapper.sh run.pool kernels-omp nstream --cmdline '2 10 1024 32'
	# ./bin/inv_wrapper.sh run.pool kernels-omp reduce --cmdline '2 10 1024'
	# # The random OpenMP kernel can not be cross-compiled due to a problem
	# # with the faasm_sm_reduce signature
	# # ./bin/inv_wrapper.sh run.pool kernels-omp random --cmdline '32 20'
	# - name: "Run PolyBench/C"
	# # We deliberately enumerate all the supported functions here and
	# # execute them individually, rather than using the microbench runner.
	# # This is so that we can more easily track what functions are supported
	# # and what functions are not
	# run: \|
	# ./bin/inv_wrapper.sh run polybench poly_covariance
	# ./bin/inv_wrapper.sh run polybench poly_correlation
	# ./bin/inv_wrapper.sh run polybench poly_2mm
	# ./bin/inv_wrapper.sh run polybench poly_3mm
	# ./bin/inv_wrapper.sh run polybench poly_atax
	# ./bin/inv_wrapper.sh run polybench poly_bicg
	# ./bin/inv_wrapper.sh run polybench poly_doitgen
	# ./bin/inv_wrapper.sh run polybench poly_mvt
	# ./bin/inv_wrapper.sh run polybench poly_cholesky
	# ./bin/inv_wrapper.sh run polybench poly_durbin
	# ./bin/inv_wrapper.sh run polybench poly_gramschmidt
	# ./bin/inv_wrapper.sh run polybench poly_lu
	# ./bin/inv_wrapper.sh run polybench poly_ludcmp
	# ./bin/inv_wrapper.sh run polybench poly_trisolv
	# ./bin/inv_wrapper.sh run polybench poly_deriche
	# ./bin/inv_wrapper.sh run polybench poly_floyd-warshall
	# ./bin/inv_wrapper.sh run polybench poly_nussinov
	# ./bin/inv_wrapper.sh run polybench poly_adi
	# ./bin/inv_wrapper.sh run polybench poly_fdtd-2d
	# ./bin/inv_wrapper.sh run polybench poly_heat-3d
	# ./bin/inv_wrapper.sh run polybench poly_jacobi-1d
	# ./bin/inv_wrapper.sh run polybench poly_jacobi-2d
	# ./bin/inv_wrapper.sh run polybench poly_seidel-2d
	# - name: "Run ImageMagick"
	# if: "contains(env.WASM_VM, 'wavm')"
	# run: ./bin/inv_wrapper.sh run imagemagick main --cmdline 'faasm://im/sample_image.png -flip faasm://im/image_out.png'
	# - name: "Run LAMMPS"
	# run: ./bin/inv_wrapper.sh run.pool lammps main --cmdline '-in faasm://lammps-data/in.controller.wall'
	# - name: "Run LULESH"
	# if: "contains(env.WASM_VM, 'wavm')"
	# run: ./bin/inv_wrapper.sh run.pool lulesh main --cmdline '-i 5 -s 20 -r 11 -c 1 -b 1' --data '4'
	# - name: "Run TFlite check"
	# if: "contains(env.WASM_VM, 'wavm')"
	# run: ./bin/inv_wrapper.sh run tf check
	# - name: "Run FFmpeg check"
	# run: ./bin/inv_wrapper.sh run ffmpeg check

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

gh: bump deps and use `faasmctl` (#19) #146

Workflow file

gh: bump deps and use `faasmctl` (#19) #146

Jobs

Run details

Workflow file for this run

gh: bump deps and use faasmctl (#19) #146

Workflow file

Workflow file for this run

gh: bump deps and use `faasmctl` (#19) #146