.gitlab-ci.yml

#### Stages for this pipeline
stages:
  - image      # build updated image for runner
  - lint       # do basic lint'ing to nudge towards common formatting
  - build      # build libraries, frontends, etc...
  - test       # test PEPC for correctness and performance
    #   - html      # run doxygen (prefer FORD)
  - deploy     # create HTML from results and push to pages

variables:
  FORCE_DOCKER_IMAGE:
    value: "false"
    options:
      - "false"
      - "true"
    description: "Force the creation of a new Docker image for this branch."
  SBATCH_ACCOUNT:
    # use this variable for SBATCH/JUBE to define accounting (or hard code within JUBE file)
    value: cstma
    description: "Account name of compute project to use for JUBE runs"
  SCHEDULER_PARAMETERS: -A$SBATCH_ACCOUNT
    # this needs to include the accounting name if JACAMAR is meant to run on compute nodes, i.e. -A<account>

#### Build PEPC and it's libraries
# Template for building
.build_template: &build_template
  tags:
    - public-docker
  image: "$CI_REGISTRY_IMAGE/$CI_COMMIT_REF_SLUG-build-image:latest"

# Build libsl and libOPA ahead of frontends
libraries:
  stage: build
  <<: *build_template
  script:
    # Build for GCC
    - ln -s makefiles/makefile.defs.GCC makefile.defs
    - make libsl libopa
  artifacts:
    when: on_success
    expire_in: 1h
    paths:
      - lib

# Build a number of frontends
frontends:
  stage: build
  parallel:
    matrix:
      - TARGET: "pepc-mini"
      - TARGET: "pepc-benchmark"
      - TARGET: "pepc-breakup"
      - TARGET: "pepc-v"
      # Build pepc-essential with all available walks to see if any of those fail
      - TARGET: pepc-essential
        WALK: [pthreads, pthreads_clustered, openmp, simple]
      # Build all available frontends to see a list of working ones
      # (ditch others in favour of this step?)
      - TARGET: "all"
  <<: *build_template
  needs: ["libraries"]
  dependencies:
    - libraries
  script:
    - echo "Building PEPC with frontend $TARGET and walk $WALK"
    # Build for GCC
    - ln -s makefiles/makefile.defs.GCC makefile.defs
    # Avoid building libs again
    - touch lib/*
    # Possibly force the walk we want to build for
    - |
      if [ -z $WALK ]
      then
         echo "using walk from sources"
      else
         echo "WALK = $WALK" >> src/frontends/$TARGET/makefile.frontend
      fi
    - make $TARGET

# If the 'public-docker' builds work, test toolchains on a cluster
toolchains:
  stage: build
  id_tokens:
    CI_JOB_JWT:
      aud: https://gitlab.jsc.fz-juelich.de
  parallel:
    matrix:
      - SYSTEM: [jusuf]
        COMPILER: GCC
        MPI: [OpenMPI, ParaStationMPI]
      - SYSTEM: [juwels]
        COMPILER: Intel
        MPI: [ParaStationMPI, IntelMPI]
  tags:
    - ${SYSTEM}
    - shell
  needs: ["frontends: [pepc-essential, pthreads]"]
  script:
    - echo "Building PEPC (all frontends) on $SYSTEMNAME using $HOSTNAME"
    - echo "Loading modules..."
    - ml ${COMPILER}
    - ml ${MPI}
    - ml Autotools
    - ln -s makefiles/makefile.defs.${COMPILER} makefile.defs
    - make all
  artifacts:
    when: on_failure
    paths:
      - bin/*
  rules:
    # Only run when on repository in JSC
    - if: $CI_PROJECT_URL =~ /^https:\/\/gitlab.jsc.fz-juelich.de/

# If the 'toolchain' builds work, test PEPC correctness on a cluster
correctness:
  stage: test
  id_tokens:
    CI_JOB_JWT:
      aud: https://gitlab.jsc.fz-juelich.de
  tags:
    - jusuf
    - shell
  needs: ["toolchains: [jusuf, GCC, OpenMPI]"]
  script:
    - echo "Running pepc-benchmark on $SYSTEMNAME using $HOSTNAME"
    - echo "Loading modules..."
    # load JUBE, the rest will be done from there
    - ml JUBE
    # start JUBE
    - echo "Running JUBE... This will take some time..."
    - jube-autorun -o -r "-t check CI" benchmark/benchmark.xml
    - echo "Checking for success..."
    - jube result run/benchmark/ | grep -q failed && exit 128
    - echo "Check seems to have passed..."
  artifacts:
    when: on_failure
    paths:
      - run
  rules:
    # Only run when on repository in JSC
    - if: $CI_PROJECT_URL =~ /^https:\/\/gitlab.jsc.fz-juelich.de/

# If correctness is 'established', test performance
performance:
  stage: test
  id_tokens:
    CI_JOB_JWT:
      aud: https://gitlab.jsc.fz-juelich.de
  tags:
    - jusuf
    - shell
  needs: ["correctness"]
  script:
    - echo "Running pepc-benchmark on $SYSTEMNAME using $HOSTNAME"
    - echo "Loading modules..."
    # load JUBE, the rest will be done from there
    - ml JUBE git
    # get previous results
    - git describe --long --all
    - git fetch
    - git describe --long --all
    - git checkout --force origin/continuous_benchmarking --
    - git describe --long --all
    #- git pull --ff-only # in case there have been overlapping changes
    - test -d ${CI_COMMIT_BRANCH} || mkdir ${CI_COMMIT_BRANCH}
    - test -f ${CI_COMMIT_BRANCH}/result_database.sqlite || touch ${CI_COMMIT_BRANCH}/result_database.sqlite
    - cp ${CI_COMMIT_BRANCH}/result_database.sqlite database.sqlite
    - git checkout $CI_COMMIT_BRANCH --
    - git describe --long --all
    # create empty run directory for JUBE with last number for benchmarks - can be RACY but may preserve some history
    - RID=`(sqlite3 database.sqlite 'select jube_benchmark_padid from full_results order by jube_benchmark_padid;' || echo '000000' ) | tail -n 1`
    - test -d "run/benchmark/${RID}" || mkdir -p "run/benchmark/${RID}"
    # start JUBE
    - echo "Running JUBE... This will take some time..."
    - jube-autorun -o -r "-t CI" benchmark/benchmark.xml
    # store results again
    - git checkout --force origin/continuous_benchmarking --
    - git describe --long --all
    - cp database.sqlite ${CI_COMMIT_BRANCH}/result_database.sqlite
    - git remote set-url --push origin "https://$CI_COMMITTER_USER_AND_TOKEN@$CI_SERVER_HOST/$CI_PROJECT_PATH.git"
    - git add ${CI_COMMIT_BRANCH}/result_database.sqlite
    - git commit -m "Committing results from CI to sqlite database [${CI_COMMIT_BRANCH}]" --author='CI Bot <anon@nowhere.net>'
    - git push origin HEAD:continuous_benchmarking
  artifacts:
    when: on_failure
    paths:
      - run
  rules:
    # Only run when on repository in JSC
    - if: $CI_PROJECT_URL =~ /^https:\/\/gitlab.jsc.fz-juelich.de/

pages:
  stage: deploy
  tags:
    - public-docker
  image: "$CI_REGISTRY_IMAGE/continuous_benchmarking-pages-image:latest"
  needs: ["performance"]
  script:
    # get results branch
    - cp DESCRIPTION.md /tmp/.
    - git fetch
    - git checkout --force origin/continuous_benchmarking --
    - echo "Converting jupyter notebook..."
    - |
      for BRANCHES in `ls -l | grep ^d | grep -vE 'public|mkdocs' | awk '{print $9}'`
      do
         export BRN=${BRANCHES}
         sed -e "s/branch=\\\\\"master/branch=\\\\\"${BRN}/" -e "s/px_renderer = \\\\\"jupyterlab/px_renderer = \\\\\"plotly_mimetype+notebook_connected/" plot_results.ipynb > "mkdocs/${BRN}.ipynb"
         echo "  - 'Branch: ${BRN}': ${BRN}.ipynb" >> nav.yml
      done
    - |
      grep -v ipynb mkdocs.yml | sed '/nav:/r nav.yml' > mkdocs.yml_
      mv mkdocs.yml_ mkdocs.yml
      cat /tmp/DESCRIPTION.md >> mkdocs/index.md
    - echo "Building HTML pages w/ results."
    - mkdir public
    - mkdocs build
    - |
      for BRANCHES in `ls -l | grep ^d | grep -vE 'public|mkdocs' | awk '{print $9}'`
      do
         cp mkdocs/${BRANCHES}-cb-badge.svg public/${BRANCHES}/cb-badge.svg
      done
  artifacts:
    paths:
      - public

#### Lint source Fortran source files
fprettify:
  stage: lint
  <<: *build_template
  script:
    - echo "Running fprettify for all Fortran source files (all frontends)"
    - cd src
    - fprettify -r -c .fprettify.rc *
    - git diff --stat=132 --color
    - COUNT=`git diff --shortstat | awk '{print $1}'`
    # generate exit code
    - exit $COUNT
  allow_failure: true

#### Create new image from a changed Dockerfile
docker_image:
  stage: image
  tags:
    - public-docker
  image:
    name: gcr.io/kaniko-project/executor:debug
    entrypoint: [""]
  script:
    - mkdir -p /kaniko/.docker
    - |
      echo "{\"auths\":{\"$CI_REGISTRY\":{\"username\":\"$CI_REGISTRY_USER\",\"password\":\"$CI_REGISTRY_PASSWORD\"}}}" \
      > /kaniko/.docker/config.json
    - |
      /kaniko/executor --context $CI_PROJECT_DIR \
      --dockerfile $CI_PROJECT_DIR/Dockerfile \
      --destination $CI_REGISTRY_IMAGE/$CI_COMMIT_REF_SLUG-build-image:$CI_COMMIT_SHORT_SHA \
      --destination $CI_REGISTRY_IMAGE/$CI_COMMIT_REF_SLUG-build-image:latest
  rules:
    # Run when triggered and a varible is set
    - if: $FORCE_DOCKER_IMAGE == "true"
    # Run when committing to a branch and changing Dockerfile
    - if: $CI_COMMIT_BRANCH
      changes:
        - Dockerfile
    # Run when merging and changing Dockerfile
    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
      changes:
        - Dockerfile