From dbadaf57ea52a3fd1b0530b9d5ae6054e55facea Mon Sep 17 00:00:00 2001 From: Jean Privat Date: Thu, 27 Jun 2024 10:10:55 -0400 Subject: [PATCH 1/2] update misc/jenkins/readme Signed-off-by: Jean Privat --- misc/jenkins/README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/misc/jenkins/README.md b/misc/jenkins/README.md index 7fbe77c882..42c74296ca 100644 --- a/misc/jenkins/README.md +++ b/misc/jenkins/README.md @@ -1 +1,2 @@ -Some scripts used by the Nit Jenkins for continuous integration, but you can use them also. +Some scripts used by the continuous integration, but you can use them also. +While Jenkins is not used anymore, we keep it as a legacy name and memento. From 9c251d64e86439a32baa6b65af259f92ccdd365e Mon Sep 17 00:00:00 2001 From: Jean Privat Date: Wed, 26 Jun 2024 09:37:18 -0400 Subject: [PATCH 2/2] github action: initial makefile CI Signed-off-by: Jean Privat --- .github/workflows/ci-pr-report.yml | 27 +++ .github/workflows/ci-pr.yml | 258 +++++++++++++++++++++++++++++ Makefile | 86 +++++++++- misc/jenkins/ensuregit.sh | 35 ++++ misc/jenkins/nitunit-some.sh | 25 +++ 5 files changed, 429 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/ci-pr-report.yml create mode 100644 .github/workflows/ci-pr.yml create mode 100755 misc/jenkins/ensuregit.sh create mode 100755 misc/jenkins/nitunit-some.sh diff --git a/.github/workflows/ci-pr-report.yml b/.github/workflows/ci-pr-report.yml new file mode 100644 index 0000000000..49f3d10798 --- /dev/null +++ b/.github/workflows/ci-pr-report.yml @@ -0,0 +1,27 @@ +name: CI Report +on: + workflow_run: + workflows: [ CI ] + types: [ completed ] + +permissions: + checks: write + pull-requests: write + actions: read + +jobs: + checks: + runs-on: ubuntu-latest + steps: + - name: Download Test Results + uses: dawidd6/action-download-artifact@v2 + with: + workflow: ${{ github.event.workflow.id }} + run_id: ${{ github.event.workflow_run.id }} + - name: Publish Test Results + uses: EnricoMi/publish-unit-test-result-action@v2 + with: + commit: ${{ github.event.workflow_run.head_sha }} + event_file: Event File/event.json + event_name: ${{ github.event.workflow_run.event }} + files: "**/*.xml" diff --git a/.github/workflows/ci-pr.yml b/.github/workflows/ci-pr.yml new file mode 100644 index 0000000000..ca23efdb1f --- /dev/null +++ b/.github/workflows/ci-pr.yml @@ -0,0 +1,258 @@ +name: CI + +on: [ pull_request ] +env: + # BASIC_PACKAGES is used fo the basic tools (make) + BASIC_PACKAGES: git build-essential ccache libgc-dev graphviz libunwind-dev libreadline-dev pkg-config + # MORE_PACKAGES is used for more tools and libs (make more) + MORE_PACKAGES: libgmp-dev libcurl4-openssl-dev libevent-dev openjdk-17-jdk-headless libgles-dev libegl-dev libsdl1.2-dev libsdl2-dev libsdl2-mixer-dev libsdl2-gfx-dev libsdl2-image-dev libsdl2-net-dev libsdl2-ttf-dev libgtk-3-dev inkscape postgresql libxdg-basedir-dev gettext libsqlite3-dev libpq-dev libmongoc-dev openmpi-bin libopenmpi-dev + JAVA_HOME: /usr/lib/jvm/java-17-openjdk-amd64/ + JNI_LIB_PATH: /usr/lib/jvm/java-17-openjdk-amd64/lib/server/ + LD_LIBRARY_PATH: /usr/lib/jvm/java-17-openjdk-amd64/lib/server/ + +jobs: + build: + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v4 + - name: Install dependencies + run: sudo apt-get update && sudo apt-get install -y $BASIC_PACKAGES + - run: make + - uses: actions/upload-artifact@v4 + with: + path: bin + name: bin + + sanity-check: + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v4 + - run: misc/jenkins/ensuregit.sh + - run: misc/jenkins/checkwhitespaces.sh + if: ${{ !cancelled() }} + - run: misc/jenkins/checklicense.sh + if: ${{ !cancelled() }} + - run: misc/jenkins/checksignedoffby.sh + if: ${{ !cancelled() }} + + tests-some: + runs-on: ubuntu-24.04 + needs: [build] + container: "ubuntu:24.04" + services: + postgres: + image: postgres + env: + POSTGRES_HOST_AUTH_METHOD: trust + mongo: + image: mongo + neo4j: + image: 'neo4j:2.3' + env: + NEO4J_AUTH: none + strategy: + fail-fast: false + matrix: + engine: [nitcg, nitcs, nitcsg, nitce, niti] + steps: + - name: Install dependencies + run: apt-get update && apt-get install -y $BASIC_PACKAGES $MORE_PACKAGES + - uses: actions/checkout@v4 + - run: misc/jenkins/ensuregit.sh + - uses: actions/download-artifact@v4 + - run: chmod +x bin/* + - run: make src/version.nit + - run: cd tests && ./search_tests_git.sh origin/master HEAD > list + - run: cd tests && ./tests.sh --engine ${{ matrix.engine }} `cat list` + - uses: actions/upload-artifact@v4 + if: ${{ !cancelled() }} + with: + path: 'tests/*.xml' + name: tests-some-${{ matrix.engine }} + - uses: actions/upload-artifact@v4 + if: ${{ !cancelled() }} + with: + path: 'tests/out/*.res' + name: tests-some-${{ matrix.engine }}-res + + tests-full: + runs-on: ubuntu-24.04 + needs: [tests-some] + container: "ubuntu:24.04" + services: + postgres: + image: postgres + env: + POSTGRES_HOST_AUTH_METHOD: trust + mongo: + image: mongo + neo4j: + image: 'neo4j:2.3' + env: + NEO4J_AUTH: none + strategy: + fail-fast: false + matrix: + engine: [nitcg, nitcs, nitcsg, nitce, niti] + steps: + - name: Install dependencies + run: apt-get update && apt-get install -y $BASIC_PACKAGES $MORE_PACKAGES + - uses: actions/checkout@v4 + - run: misc/jenkins/ensuregit.sh + - uses: actions/download-artifact@v4 + - run: chmod +x bin/* + - run: make src/version.nit + - name: Run tests + run: cd tests && ./testfull.sh --engine ${{ matrix.engine }} + - uses: actions/upload-artifact@v4 + if: ${{ !cancelled() }} + with: + path: 'tests/*.xml' + name: tests-full-${{ matrix.engine }} + - uses: actions/upload-artifact@v4 + if: ${{ !cancelled() }} + with: + path: 'tests/out/*.res`' + name: tests-full-${{ matrix.engine }}-res + + nitunit-some: + runs-on: ubuntu-24.04 + needs: [build] + container: "ubuntu:24.04" + services: + postgres: + image: postgres + env: + POSTGRES_HOST_AUTH_METHOD: trust + mongo: + image: mongo + neo4j: + image: 'neo4j:2.3' + env: + NEO4J_AUTH: none + steps: + - name: Install dependencies + run: apt-get update && apt-get install -y $BASIC_PACKAGES $MORE_PACKAGES + - uses: actions/checkout@v4 + - run: misc/jenkins/ensuregit.sh + - uses: actions/download-artifact@v4 + - run: chmod +x bin/* + - run: make nitunit-some + - uses: actions/upload-artifact@v4 + if: ${{ !cancelled() }} + with: + path: '*.xml' + name: nitunit-some + + nitunit-full: + runs-on: ubuntu-24.04 + needs: [nitunit-some] + container: "ubuntu:24.04" + services: + postgres: + image: postgres + env: + POSTGRES_HOST_AUTH_METHOD: trust + mongo: + image: mongo + neo4j: + image: 'neo4j:2.3' + env: + NEO4J_AUTH: none + steps: + - name: Install dependencies + run: apt-get update && apt-get install -y $BASIC_PACKAGES $MORE_PACKAGES + - uses: actions/checkout@v4 + - run: misc/jenkins/ensuregit.sh + - uses: actions/download-artifact@v4 + - run: chmod +x bin/* + - run: make nitunit-full + - uses: actions/upload-artifact@v4 + if: ${{ !cancelled() }} + with: + path: '*.xml' + name: nitunit-full + + test-contrib: + runs-on: ubuntu-24.04 + needs: [build] + steps: + - uses: actions/checkout@v4 + - name: Install dependencies + run: sudo apt-get update && sudo apt-get install -y $BASIC_PACKAGES $MORE_PACKAGES + - uses: actions/download-artifact@v4 + - run: chmod +x bin/* + - run: 'PATH=$PWD/bin:$PATH make test-contrib' + - uses: actions/upload-artifact@v4 + if: ${{ !cancelled() }} + with: + path: '*.xml' + name: test-contrib + + bootstrap-full: + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v4 + - name: Install dependencies + run: sudo apt-get update && sudo apt-get install -y $BASIC_PACKAGES + - run: make bootstrap-full + + bench-fast: + runs-on: ubuntu-24.04 + needs: [build] + steps: + - uses: actions/checkout@v4 + - name: Install dependencies + run: sudo apt-get update && sudo apt-get install -y $BASIC_PACKAGES gnuplot + - uses: actions/download-artifact@v4 + - run: chmod +x bin/* + - run: 'PATH=$PWD/bin:$PATH make bench-fast' + - uses: actions/upload-artifact@v4 + if: ${{ !cancelled() }} + with: + path: |- + benchmarks/*.dat + benchmarks/*.gnu + benchmarks/*.png + benchmarks/*.html + name: bench-fast + + valgrind: + runs-on: ubuntu-24.04 + needs: [build] + steps: + - uses: actions/checkout@v4 + - name: Install dependencies + run: sudo apt-get update && sudo apt-get install -y $BASIC_PACKAGES valgrind + - uses: actions/download-artifact@v4 + - run: chmod +x bin/* + - run: make valgrind + - uses: actions/upload-artifact@v4 + if: ${{ !cancelled() }} + with: + path: 'valgrind.out' + name: valgrind + + info: + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v4 + - run: pwd + - run: id + - run: ls -al + - run: git branch -av || true + - run: lscpu || true + - run: git log --oneline --graph --decorate | head -n 50 + - run: cat /etc/issue + - run: cat /etc/os-release + - run: env + + event_file: + name: "Event File" + runs-on: ubuntu-latest + steps: + - name: Upload + uses: actions/upload-artifact@v4 + with: + name: Event File + path: ${{ github.event_path }} diff --git a/Makefile b/Makefile index 4f4f71b96d..48c3fe7cab 100644 --- a/Makefile +++ b/Makefile @@ -49,7 +49,16 @@ bin/nitdoc: cd src; make ../bin/nitdoc bin/nitls: - cd src; make ../bin/nitls + cd src; make + +bin/nitunit: + cd src; make + +bin/nitpick: + cd src; make + +src/version.nit: + cd src && ./git-gen-version.sh doc/stdlib/index.html: bin/nitdoc bin/nitls @echo '***************************************************************' @@ -87,4 +96,77 @@ clean: (echo "CLEANING $$directory" && cd "$$directory" && $(MAKE) clean); \ done @echo "" - @echo "Nit was succesfully cleaned." \ No newline at end of file + @echo "Nit was succesfully cleaned." + +# BASIC CI-CD JOBS ######################################################## + +sanity-check: + misc/jenkins/checkwhitespaces.sh + misc/jenkins/checksignedoffby.sh + misc/jenkins/checkbinaryfiles.sh + misc/jenkins/checklicense.sh + +nitunit-some: bin/nitls bin/nitunit src/version.nit + misc/jenkins/nitunit-some.sh + +nitpick-full: bin/nitls bin/nitpick src/version.nit + bin/nitls lib src examples contrib + bin/nitls -Pp lib src examples | grep -v -f tests/gitlab_ci.skip > list.txt || true # filter what is skipped by tests.sh + xargs bin/nitpick < list.txt + + +# LIB, CONTRIB AND OOT ############################################## + +nitunit-full: bin/nitls bin/nitunit src/version.nit + bin/nitls -Pp lib src examples | grep -v -f tests/gitlab_ci.skip > list.txt || true # filter what is skipped by tests.sh + ls -1 doc/manual/*.md >> list.txt + xargs bin/nitunit -v < list.txt + +nitunit-src: bin/nitls bin/nitunit src/version.nit + bin/nitls -Pp src examples | grep -v -f tests/gitlab_ci.skip > list.txt || true # filter what is skipped by tests.sh + xargs bin/nitunit -v < list.txt + +test-contrib: src/version.nit + misc/jenkins/check_contrib.sh all check + grep 'error message' *.xml > status.txt || true + test ! -s status.txt # no lines, no errors + +# MISC ############################################################## + +bootstrap-full: + cd src && ./full_bootstrap && ./ncall.sh + +bench-fast: + cd benchmarks && ./bench_engines.sh --fast --fast --html options + +nitunit-manual: bin/nitunit + bin/nitunit doc/manual/*.md + +# MORE TOOLS ######################################################## + +build-more-tools: more + $(MAKE) -C contrib/nitcc + +valgrind: src/version.nit + mkdir -p valgrind.out + bin/nitc src/nitc.nit # To warm-up the cache + src/valgrind.sh --callgrind-out-file=valgrind.out/nitc.nitc.out bin/nitc src/nitc.nit -vv + callgrind_annotate valgrind.out/nitc.nitc.out > valgrind.out/nitc.nitc.txt + src/valgrind.sh --callgrind-out-file=valgrind.out/niti.niti.out bin/nit -- src/nit.nit tests/base_simple3.nit -vv + callgrind_annotate valgrind.out/niti.niti.out > valgrind.out/niti.niti.txt + +build-doc: + nitdoc -d nitdoc.out --keep-going lib src + +build-manual: + # apt-get update && apt-get install --yes --no-install-recommends pandoc texlive texlive-latex-extra lmodern + $(make) -C doc/manual + +nitmetrics: + mkdir -p nitmetrics.out + nitmetrics --all --log --log-dir nitmetrics.out --dir nitmetrics.out --keep-going lib src | tee nitmetrics.out/metrics.txt + +build-catalog: + misc/jenkins/check_contrib.sh pre-build + cd contrib && ./oot.sh update && ./oot.sh pre-build + nitcatalog -d catalog.out lib/ examples/ contrib/ contrib/oot/ diff --git a/misc/jenkins/ensuregit.sh b/misc/jenkins/ensuregit.sh new file mode 100755 index 0000000000..9fa7fb6d1f --- /dev/null +++ b/misc/jenkins/ensuregit.sh @@ -0,0 +1,35 @@ +#!/bin/sh +# This file is part of NIT ( http://www.nitlanguage.org ). +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This script ensure that the root direcroty of nit is a git repository with a fully fetched origin. +# It helps CI/CD in environments (like github action) that trim and hack the initial fetch. + +if [ ! -f src/nitc.nit ]; then + echo "Error: run the script in the root directory of nit" + exit 1 +fi +if [ ! -d .git ]; then + echo "No git directory" + exit 1 +fi + +set -x +git config --global --add safe.directory "$PWD" # Because github messes with the homedir +git fetch --unshallow # Because github does --depth=1 +git fetch origin # Ensure origin/master is here +# Outputs some info... +git remote -v +git branch -av +git log --oneline --graph --decorate origin/master..HEAD diff --git a/misc/jenkins/nitunit-some.sh b/misc/jenkins/nitunit-some.sh new file mode 100755 index 0000000000..54560a0850 --- /dev/null +++ b/misc/jenkins/nitunit-some.sh @@ -0,0 +1,25 @@ +#!/bin/sh +# This file is part of NIT ( http://www.nitlanguage.org ). +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This script run nitunit of nit packages modified between revisions + +from=${1:-origin/master} +to=${2:-HEAD} + +git diff --name-only "$from..$to" -- "*.nit" "*.res" "README.*" | grep -v "^tests/\|contrib/" > list0.txt || true +test -s list0.txt || exit 0 +xargs bin/nitls -pP < list0.txt > list.txt +test -s list.txt || exit 0 +xargs bin/nitunit -v < list.txt