.gitlab-ci.yml

stages:
  - lint
  - unittests
  - mirror
  - release
  - deploy

include:
  # Make sure the CHANGELOG is always updated
  - project: 'rml/util/ci-templates'
    ref: main
    file: 'CHANGELOG.gitlab-ci.yml'
  # Make a Github Release on new tags
  - project: 'rml/util/ci-templates'
    ref: main
    file: 'Github-Release.gitlab-ci.yml'
  # Push a Docker Image to Docker Hub on new tags
  - project: 'rml/util/ci-templates'
    ref: main
    file: 'Docker-Hub.gitlab-ci.yml'
  # Push a build to Maven Central on new tags
  - project: 'rml/util/ci-templates'
    ref: main
    file: 'Maven-Central.gitlab-ci.yml'

variables:
  # This will suppress any download for dependencies and plugins or upload messages which would clutter the console log.
  # `showDateTime` will show the passed time in milliseconds.
  MAVEN_OPTS: "-Dhttps.protocols=TLSv1.2 -Dmaven.repo.local=$CI_PROJECT_DIR/.m2/repository -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=WARN -Dorg.slf4j.simpleLogger.showDateTime=true -Djava.awt.headless=true"
  MAVEN_CLI_OPTS: "-s $CI_PROJECT_DIR/.m2/settings.xml --batch-mode --errors --fail-at-end --show-version -DinstallAtEnd=true -DdeployAtEnd=true"
  # Postgres
  POSTGRES_DB: postgres
  POSTGRES_USER: postgres
  POSTGRES_PASSWORD: "YourSTRONG!Passw0rd"
  # SQLServer
  ACCEPT_EULA: "Y"
  SA_PASSWORD: "YourSTRONG!Passw0rd"
  # https://hub.docker.com/r/alpine/git/tags
  GIT_VERSION: v2.32.0

cache:
  key: ${CI_JOB_NAME}
  paths:
    - .m2/repository

# Cancel pipeline if a newer pipeline is running
default:
    interruptible: true

###############################
#                             #
#     Unittests               #
#                             #
###############################

General:
  stage: unittests
  image: maven:3.8.6-openjdk-11
  services:
    - name: docker:19.03.12-dind
      # explicitly disable tls to avoid docker startup interruption
      command: ["--tls=false"]
  variables:
    # Instruct Testcontainers to use the daemon of DinD.
    DOCKER_HOST: "tcp://docker:2375"
    # Instruct Docker not to start over TLS.
    DOCKER_TLS_CERTDIR: ""
    # Improve performance with overlayfs.
    DOCKER_DRIVER: overlay2
  script:
    - 'mvn $MAVEN_CLI_OPTS -Dtest=$TEST test'
  parallel:
    matrix:
      - TEST: [ArgumentsTest, MapperCSVTest, MapperJSONTest, MapperMySQLTest, MapperPostgresR2RMLTest, MapperWoTTest, ArgumentsTestMySQLTest, MapperCSVWTest, MapperLDESTest, MapperODSTest, MapperOracleDBTest, MapperPostgresXMLTest, MapperXMLTest, CustomRMLFnOMapperCSVTest, MapperEXCELTest, MapperMappingFileURLTest, MapperSPARQLTest, MetadataTest, CustomRMLFnOMapperJSONTest, MapperHTMLTest, MapperMySQLR2RMLTest, MapperPostgresCSVTest, MapperSQLServerTest, OptimizationsTest, R2RMLConverterTest, QuadTest, ReadmeTest, ReadmeFunctionTest]
  except:
    - master
    - development

Docker Build:
  stage: unittests
  image: docker:latest
  services:
    - docker:19.03.12-dind
  before_script:
    - docker info
  script:
    - docker build -t rmlmapper .
  except:
    - master
    - development

###############################
#                             #
#     Automated Releases      #
#                             #
###############################

# Bump version, create changelog, commit changes to master
# Only happens when manually clicked in the CI pipeline
# Thanks to: https://www.benjaminrancourt.ca/how-to-push-to-a-git-repository-from-a-gitlab-ci-pipeline
# Description
#   This script allows to store the artefacts of a step into the current
#   repository, to improve the efficiency of the next build process.

# Set up this script
#   1. Create a new personal access token (https://gitlab.com/-/profile/personal_access_tokens)
#      with the following scopes:
#        - read_repository
#        - write_repository
#   2. Inside Settings -> CI / CD -> Variables, create the following variables:
#
#   GITLAB_TOKEN          Personal access token previously created.             XGE2-k445hd5fbs94v9d
#     (masked)
#   GITLAB_USERNAME       Username associated with the personal access token.   ranb2002
#   COMMIT_MESSAGE        Commit message                                        Automatic update from the weekly schedule

# Other variables used by this script
#   The following variables are defined automatically by GitLab CI. Thus, you
#   don't need to override them.
#
#   CI_COMMIT_SHA         Commit SHA, to use a unique directory name.           e46f153dd47ce5f3ca8c56be3fb5d55039853655
#   CI_DEFAULT_BRANCH     Default branch.                                       main
#   CI_PROJECT_PATH       Current project path.                                 ranb2002/benjaminrancourt.ca
#   CI_SERVER_HOST        Hostname of the current GitLab instance.              gitlab.com
#   GITLAB_USER_EMAIL     Email of the user used to commit the changes to the   ranb2002@gitlab.com
#                         secondary repository.
#   GITLAB_USER_NAME      User name of the user used to commit the changes to   Benjamin Rancourt
#                         the secondary repository.
#
Create Release:
  image:
    entrypoint: ['']
    name: alpine/git:${GIT_VERSION}
  stage: release
  before_script:
    # Dependencies
    - apk add maven java-jdk
    # Clone the repository via HTTPS inside a new directory
    - git clone "https://${GITLAB_USERNAME}:${GITLAB_TOKEN_RMLMAPPER}@${CI_SERVER_HOST}/${CI_PROJECT_PATH}.git" "${CI_COMMIT_SHA}"

    # Set the displayed user with the commits that are about to be made
    - git config --global user.email "${GIT_USER_EMAIL:-$GITLAB_USER_EMAIL}"
    - git config --global user.name "${GIT_USER_NAME:-$GITLAB_USER_NAME}"
  script:
    # Using before_script would override .git:push template
    # User has to set the RELEASE_TAG_NAME variable
    - if [ -z "$RELEASE_TAG_NAME" ]; then exit 1; else echo "Creating release v$RELEASE_TAG_NAME"; fi
    # Install dependencies
    - apk add nodejs npm
    - npm install -g changefrog

    # Update pom.xml
    - head -n6 pom.xml | sed "/<version>/s/>.*</>$RELEASE_TAG_NAME</" > pom_updated.xml
    - tail -n +7 pom.xml >> pom_updated.xml 
    - mv pom_updated.xml pom.xml  
    # Update changelog. Changefrog does not like vX.X.X so drop 'v'
    - changefrog -n "$RELEASE_TAG_NAME"
    # Build a Jar
    - mvn install -DskipTests=true
    # Stage changes for commit to master
    - cp -u pom.xml "${CI_COMMIT_SHA}/pom.xml"
    - cp -u CHANGELOG.md "${CI_COMMIT_SHA}/CHANGELOG.md"
    - cp -u buildNumber.properties "${CI_COMMIT_SHA}/buildNumber.properties"
  after_script:
    # Go to the new directory
    - cd "${CI_COMMIT_SHA}"

    # Add all generated files to Git
    - git add .
    - |-
      # Check if we have modifications to commit
      CHANGES=$(git status --porcelain | wc -l)

      if [ "$CHANGES" -gt "0" ]; then
        # Show the status of files that are about to be created, updated or deleted
        git diff --cached --shortstat

        # Commit all changes
        COMMIT_MESSAGE="CHANGELOG: release v$RELEASE_TAG_NAME"
        echo "Commit message: ${COMMIT_MESSAGE}"
        git commit -m "${COMMIT_MESSAGE}"

        # Create git tag
        git tag "v$RELEASE_TAG_NAME"

        # Update the repository and make sure to skip the pipeline create for this commit
        git push origin "${CI_DEFAULT_BRANCH}"
        # Push new tags and trigger the pipeline since we're deploying
        git push --tags origin "${CI_DEFAULT_BRANCH}"

        # Also release on the master branch
        git checkout master
        git rebase "${CI_DEFAULT_BRANCH}"
        git push origin master
      fi
  when: manual

# Generate R2RML test report and send MR to rml.io website
# Push options: https://docs.gitlab.com/ee/user/project/push_options.html
R2RML Test Report:
  stage: deploy
  image: docker:latest
  services:
    - docker:19.03.12-dind
  before_script:
    # Dependencies
    - apk add git python3 python3-dev py3-pip postgresql-dev postgresql libpq gcc musl-dev docker docker-compose java-jre-headless maven java-jdk

    # Clone the repository via HTTPS inside a new directory
    - git clone "https://${GITLAB_USERNAME}:${GITLAB_TOKEN_WEBSITE}@${CI_SERVER_HOST}/rml/doc/rmlio-website.git"
    - git clone "https://github.com/kg-construct/r2rml-test-cases-support"

    # Set the displayed user with the commits that are about to be made
    - git config --global user.email "${GIT_USER_EMAIL:-$GITLAB_USER_EMAIL}"
    - git config --global user.name "${GIT_USER_NAME:-$GITLAB_USER_NAME}"

    # Show docker
    - docker info
  script:
    # Variables
    - RELEASE_TAG_NAME=$(git tag -l "v*" --sort=-creatordate | head -n1)
    - RELEASE_TAG_NAME=$(echo "$RELEASE_TAG_NAME" | cut -c2-)
    - TODAY=$(date +"%Y-%m-%d")

    # Build RMLMapper jar
    - mvn install -DskipTests=true

    # Setup R2RML test cases configs
    - cp data/r2rml-test-cases-config-postgresql.ini r2rml-test-cases-support/config-postgresql.ini
    - cp data/r2rml-test-cases-config-mysql.ini r2rml-test-cases-support/config-mysql.ini
    - cd r2rml-test-cases-support
    - echo "Generating test report for release v${RELEASE_TAG_NAME} on ${TODAY}"
    - sed -i "s/VERSION/${RELEASE_TAG_NAME}/" config-postgresql.ini
    - sed -i "s/VERSION/${RELEASE_TAG_NAME}/" config-mysql.ini
    - sed -i "s/DATE/${TODAY}/" config-postgresql.ini
    - sed -i "s/DATE/${TODAY}/" config-mysql.ini

    # Install R2RML test cases dependencies
    - python3 -m pip install -r requirements.txt

    # Execute R2RML test cases
    - HOST=docker python3 test.py config-postgresql.ini
    - HOST=$(getent hosts docker | cut -f1 -d " ") python3 test.py config-mysql.ini # MySQL wants an IP address

    # Prepare MR for rml.io website
    - cp results.ttl ../rmlio-website/tools/rmlmapper/r2rml-test-cases-results.ttl
    - cd ..
  after_script:
    - RELEASE_TAG_NAME=$(git tag -l "v*" --sort=-creatordate | head -n1)
    - RELEASE_TAG_NAME=$(echo "$RELEASE_TAG_NAME" | cut -c2-)

    # Go to the new directory
    - cd rmlio-website

    # Add all generated files to Git
    - git add .
    - |-
      # Check if we have modifications to commit
      CHANGES=$(git status --porcelain | wc -l)
      BRANCH="rmlmapper/${RELEASE_TAG_NAME}"

      if [ "$CHANGES" -gt "0" ]; then
        # Show the status of files that are about to be created, updated or deleted
        git diff --cached --shortstat

        # Create new branch
        git checkout -b "${BRANCH}"

        # Commit all changes
        COMMIT_MESSAGE="tools/rmlmapper: update R2RML test report for RMLMapper ${RELEASE_TAG_NAME}"
        echo "Commit message: ${COMMIT_MESSAGE}"
        git commit -m "${COMMIT_MESSAGE}"

        # Update the repository and make sure to skip the pipeline create for this commit
        echo "Creating Merge Request of branch ${BRANCH}"
        git push origin "${BRANCH}" -o merge_request.create -o merge_request.remove_source_branch -o merge_request.title="${COMMIT_MESSAGE}"
      fi
  only:
    - tags