From 78027c57bd0d0691c47e005d72fabc071867fe4d Mon Sep 17 00:00:00 2001 From: Jeong Han Lee Date: Thu, 16 Jul 2020 12:11:22 -0700 Subject: [PATCH 01/89] add dockerignore --- .dockerignore | 7 +++++++ configure/RELEASE | 8 ++++---- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/.dockerignore b/.dockerignore index 4cf6474..9fc4204 100644 --- a/.dockerignore +++ b/.dockerignore @@ -5,3 +5,10 @@ temp? *.md !README.md **/*.class +examples +elastic-stack +*-src +*.md +LICENSE +docs +*Dockerfile* diff --git a/configure/RELEASE b/configure/RELEASE index 4fb7bea..129757f 100644 --- a/configure/RELEASE +++ b/configure/RELEASE @@ -1,14 +1,14 @@ -## Where the Source repository SRC_URL/SRC_NAME -## +##- Where the Source repository SRC_URL/SRC_NAME +##- CF_SRC_URL:=https://github.com/ChannelFinder CF_SRC_NAME:=ChannelFinder-SpringBoot -## Which the source tag / branch / hash id would like to use +##- Which the source tag / branch / hash id would like to use CF_SRC_TAG:=537bf2c -## Placeholder for the site-specific version control +##- Placeholder for the site-specific version control CF_SRC_VERSION:=4.0.0 From 16cf63402168bd9ba88dba5dc2e22d157d0d9a45 Mon Sep 17 00:00:00 2001 From: Jeong Han Lee Date: Thu, 16 Jul 2020 17:35:25 -0700 Subject: [PATCH 02/89] update bash to follow shellcheck --- .github/workflows/linter.yml | 1 - examples/configuration.bash | 15 ++++++++----- examples/delete_channels.bash | 25 ++++++++++++--------- examples/delete_properties.bash | 26 +++++++++++++--------- examples/delete_tags.bash | 25 ++++++++++++--------- examples/put_channels.bash | 27 +++++++++++++---------- examples/put_properties.bash | 35 +++++++++++++++-------------- examples/put_tags.bash | 39 +++++++++++++++++++-------------- examples/update_channels.bash | 39 ++++++++++++++++----------------- 9 files changed, 131 insertions(+), 101 deletions(-) diff --git a/.github/workflows/linter.yml b/.github/workflows/linter.yml index 8047879..69c72a1 100644 --- a/.github/workflows/linter.yml +++ b/.github/workflows/linter.yml @@ -17,6 +17,5 @@ jobs: - name: Lint Code Base uses: docker://github/super-linter:latest env: -# VALIDATE_ALL_CODEBASE: true VALIDATE_MD: true VALIDATE_BASH: true diff --git a/examples/configuration.bash b/examples/configuration.bash index 3f9e650..7318ef5 100644 --- a/examples/configuration.bash +++ b/examples/configuration.bash @@ -1,9 +1,15 @@ + cf_host=localhost cf_port=8080 -cf_url=http://${cf_host}:${cf_port}/ChannelFinder/resources +# shellcheck disable=SC2034 +cf_url=http://"${cf_host}:${cf_port}/ChannelFinder/resources" + +# admin, cfuser, channel +# username, and its password should be matched with the running ldif file. +# function print_help { @@ -12,10 +18,8 @@ function print_help printf "\n>>> %s : %s ....\n" "$a" "$b"; } - function get_list_from_a_file { - local empty_string=""; declare -a entry=(); while IFS= read -r line_data; do @@ -25,8 +29,7 @@ function get_list_from_a_file entry[i]="${line_data}" ((++i)) fi - done < $1 + done < "$1" - echo ${entry[@]} + echo "${entry[@]}" } - diff --git a/examples/delete_channels.bash b/examples/delete_channels.bash index 8fcded6..ff5815c 100644 --- a/examples/delete_channels.bash +++ b/examples/delete_channels.bash @@ -1,30 +1,35 @@ #!/usr/bin/env bash -declare -gr SC_SCRIPT="$(realpath "$0")" -declare -gr SC_SCRIPTNAME=${0##*/} -declare -gr SC_TOP="${SC_SCRIPT%/*}" +declare -g SC_SCRIPT; +declare -g SC_TOP; -. ${SC_TOP}/configuration.bash +SC_SCRIPT="$(realpath "$0")"; +SC_TOP="${SC_SCRIPT%/*}" -list="$(get_list_from_a_file ${SC_TOP}/CHANNELS)" +# shellcheck disable=SC1090 +. "${SC_TOP}/configuration.bash" -URL=${cf_url}/channels + +list=$(get_list_from_a_file "${SC_TOP}"/CHANNELS) + +# shellcheck disable=SC2154 +URL="${cf_url}"/channels # admin, cfuser, channel # username, and its password should be matched with the running ldif file. # cf_userid=channel cf_passwd=1234 -cf_user=${cf_userid}:${cf_passwd} +cf_user="${cf_userid}:${cf_passwd}" -for a_chan in ${list[@]}; do +for a_chan in "${list[@]}"; do print_help "DELETE" "$a_chan" - curl -u $cf_user -X DELETE ${URL}/${a_chan} + curl -u $cf_user -X DELETE "${URL}/${a_chan}" printf "\n" done print_help "GET" "CHANNELS" -curl -X GET ${URL} +curl -X GET "${URL}" printf "\n"; diff --git a/examples/delete_properties.bash b/examples/delete_properties.bash index a5e7e40..832fd7f 100644 --- a/examples/delete_properties.bash +++ b/examples/delete_properties.bash @@ -1,28 +1,34 @@ #!/usr/bin/env bash -declare -gr SC_SCRIPT="$(realpath "$0")" -declare -gr SC_SCRIPTNAME=${0##*/} -declare -gr SC_TOP="${SC_SCRIPT%/*}" -. ${SC_TOP}/configuration.bash +declare -g SC_SCRIPT; +declare -g SC_TOP; -list="$(get_list_from_a_file ${SC_TOP}/PROPERTIES)" +SC_SCRIPT="$(realpath "$0")"; +SC_TOP="${SC_SCRIPT%/*}" -URL=${cf_url}/properties + +# shellcheck disable=SC1090 +. "${SC_TOP}"/configuration.bash + +list=$(get_list_from_a_file "${SC_TOP}"/PROPERTIES) + +# shellcheck disable=SC2154 +URL="${cf_url}"/properties # admin, cfuser, property # username, and its password should be matched with the running ldif file. # cf_userid=property cf_passwd=1234 -cf_user=${cf_userid}:${cf_passwd} +cf_user="${cf_userid}:${cf_passwd}" -for a_property in ${list[@]}; do +for a_property in "${list[@]}"; do print_help "DELETE" "$a_property" - curl -u $cf_user -X DELETE ${URL}/${a_property} + curl -u $cf_user -X DELETE "${URL}/${a_property}" printf "\n" done print_help "GET" "Properties" -curl -X GET ${URL} +curl -X GET "${URL}" printf "\n"; diff --git a/examples/delete_tags.bash b/examples/delete_tags.bash index 4afa0a9..bd30d4c 100644 --- a/examples/delete_tags.bash +++ b/examples/delete_tags.bash @@ -1,13 +1,18 @@ #!/usr/bin/env bash -declare -gr SC_SCRIPT="$(realpath "$0")" -declare -gr SC_SCRIPTNAME=${0##*/} -declare -gr SC_TOP="${SC_SCRIPT%/*}" -. ${SC_TOP}/configuration.bash +declare -g SC_SCRIPT; +declare -g SC_TOP; -list="$(get_list_from_a_file ${SC_TOP}/TAGS)" +SC_SCRIPT="$(realpath "$0")"; +SC_TOP="${SC_SCRIPT%/*}" -URL=${cf_url}/tags +# shellcheck disable=SC1090 +. "${SC_TOP}"/configuration.bash + +list=$(get_list_from_a_file "${SC_TOP}"/TAGS) + +# shellcheck disable=SC2154 +URL="${cf_url}"/tags # admin, cfuser, property # username, and its password should be matched with the running ldif file. @@ -16,15 +21,15 @@ URL=${cf_url}/tags cf_userid=tag cf_passwd=1234 -cf_user=${cf_userid}:${cf_passwd} +cf_user="${cf_userid}:${cf_passwd}" -for a_tag in ${list[@]}; do +for a_tag in "${list[@]}"; do print_help "DELETE" "$a_tag" - curl -u $cf_user -X DELETE ${URL}/${a_tag} + curl -u $cf_user -X DELETE "${URL}/${a_tag}" printf "\n" done print_help "GET" "tags" -curl -X GET ${URL} +curl -X GET "${URL}" printf "\n"; diff --git a/examples/put_channels.bash b/examples/put_channels.bash index 6f8c7df..199b6a4 100644 --- a/examples/put_channels.bash +++ b/examples/put_channels.bash @@ -1,14 +1,19 @@ #!/usr/bin/env bash -declare -gr SC_SCRIPT="$(realpath "$0")" -declare -gr SC_SCRIPTNAME=${0##*/} -declare -gr SC_TOP="${SC_SCRIPT%/*}" +declare -g SC_SCRIPT; +declare -g SC_TOP; -. ${SC_TOP}/configuration.bash +SC_SCRIPT="$(realpath "$0")"; +SC_TOP="${SC_SCRIPT%/*}" -list="$(get_list_from_a_file ${SC_TOP}/CHANNELS)" -URL=${cf_url}/channels +# shellcheck disable=SC1090 +. "${SC_TOP}"/configuration.bash + +list=$(get_list_from_a_file "${SC_TOP}"/CHANNELS) + +# shellcheck disable=SC2154 +URL="${cf_url}"/channels # admin, cfuser, channel @@ -18,7 +23,7 @@ cf_userid=channel cf_passwd=1234 cf_user=${cf_userid}:${cf_passwd} -for a_chan in ${list[@]}; do +for a_chan in "${list[@]}"; do print_help "PUT" "$a_chan" temp_json=$(mktemp) echo " @@ -28,7 +33,7 @@ for a_chan in ${list[@]}; do \"owner\": \"cf-channels\" } ] - " > ${temp_json} + " > "${temp_json}" #echo " # [ # { @@ -54,12 +59,12 @@ for a_chan in ${list[@]}; do # " > ${temp_json} - curl -u $cf_user -H 'Content-Type: application/json' -X PUT ${URL} -d @$temp_json - rm -f $temp_json + curl -u $cf_user -H 'Content-Type: application/json' -X PUT "${URL}" -d @"$temp_json" + rm -f "$temp_json" printf "\n" done print_help "GET" "CHANNELS" -curl -X GET ${URL} +curl -X GET "${URL}" printf "\n"; diff --git a/examples/put_properties.bash b/examples/put_properties.bash index 4be9bed..a588c15 100644 --- a/examples/put_properties.bash +++ b/examples/put_properties.bash @@ -1,26 +1,32 @@ #!/usr/bin/env bash -declare -gr SC_SCRIPT="$(realpath "$0")" -declare -gr SC_SCRIPTNAME=${0##*/} -declare -gr SC_TOP="${SC_SCRIPT%/*}" -. ${SC_TOP}/configuration.bash +declare -g SC_SCRIPT; +declare -g SC_TOP; -list="$(get_list_from_a_file ${SC_TOP}/PROPERTIES)" +SC_SCRIPT="$(realpath "$0")"; +SC_TOP="${SC_SCRIPT%/*}" + + +# shellcheck disable=SC1090 +. "${SC_TOP}"/configuration.bash + +list=$(get_list_from_a_file "${SC_TOP}"/PROPERTIES) #cf_url=https://${cf_host}:8443/ChannelFinder/resources -URL=${cf_url}/properties +# shellcheck disable=SC2154 +URL="${cf_url}"/properties # admin, cfuser, property # username, and its password should be matched with the running ldif file. # -cf_userid=admin +cf_userid="admin" cf_passwd=1234 -cf_user=${cf_userid}:${cf_passwd} +cf_user="${cf_userid}:${cf_passwd}" temp_json=$(mktemp) -for a_property in ${list[@]}; do +for a_property in "${list[@]}"; do print_help "PUT" "$a_property" temp_json=$(mktemp) echo " @@ -30,16 +36,13 @@ for a_property in ${list[@]}; do \"owner\": \"cf-properties\" } ] - " > ${temp_json} + " > "${temp_json}" - curl -u $cf_user -H 'Content-Type: application/json' -X PUT ${URL} -d @$temp_json - rm -f $temp_json + curl -u $cf_user -H 'Content-Type: application/json' -X PUT "${URL}" -d @"$temp_json" + rm -f "$temp_json" printf "\n" done print_help "GET" "Properties" -curl -X GET ${URL} +curl -X GET "${URL}" printf "\n"; - - - diff --git a/examples/put_tags.bash b/examples/put_tags.bash index e469d9c..b30ea9d 100644 --- a/examples/put_tags.bash +++ b/examples/put_tags.bash @@ -1,28 +1,36 @@ #!/usr/bin/env bash -declare -gr SC_SCRIPT="$(realpath "$0")" -declare -gr SC_SCRIPTNAME=${0##*/} -declare -gr SC_TOP="${SC_SCRIPT%/*}" -. ${SC_TOP}/configuration.bash -list="$(get_list_from_a_file ${SC_TOP}/TAGS)" +declare -g SC_SCRIPT; +declare -g SC_TOP; + +SC_SCRIPT="$(realpath "$0")"; +SC_TOP="${SC_SCRIPT%/*}" + + +# shellcheck disable=SC1090 +. "${SC_TOP}"/configuration.bash + + +list=$(get_list_from_a_file"${SC_TOP}"/TAGS) #cf_url=https://${cf_host}:8443/ChannelFinder/resources -URL=${cf_url}/tags +# shellcheck disable=SC2154 +URL="${cf_url}"/tags # admin, cfuser, tag, operator # username, and its password should be matched with the running ldif file. # -cf_userid=admin +cf_userid="admin" cf_passwd=1234 -cf_user=${cf_userid}:${cf_passwd} +cf_user="${cf_userid}:${cf_passwd}" temp_json=$(mktemp) -a_chan="SR04C:BPM1:SA:X1" +#a_chan="SR04C:BPM1:SA:X1" -for a_tag in ${list[@]}; do +for a_tag in "${list[@]}"; do print_help "PUT" "$a_tag" temp_json=$(mktemp) echo " @@ -32,16 +40,13 @@ for a_tag in ${list[@]}; do \"owner\": \"cf-tags\" } ] - " > ${temp_json} + " > "${temp_json}" - curl -u $cf_user -H 'Content-Type: application/json' -X PUT ${URL} -d @$temp_json - rm -f $temp_json + curl -u "$cf_user" -H 'Content-Type: application/json' -X PUT "${URL}" -d @"$temp_json" + rm -f "$temp_json" printf "\n" done print_help "GET" "tags" -curl -X GET ${URL} +curl -X GET "${URL}" printf "\n"; - - - diff --git a/examples/update_channels.bash b/examples/update_channels.bash index f02870e..4c0bef2 100644 --- a/examples/update_channels.bash +++ b/examples/update_channels.bash @@ -1,28 +1,29 @@ #!/usr/bin/env bash -declare -gr SC_SCRIPT="$(realpath "$0")" -declare -gr SC_SCRIPTNAME=${0##*/} -declare -gr SC_TOP="${SC_SCRIPT%/*}" +declare -g SC_SCRIPT; +declare -g SC_TOP; -. ${SC_TOP}/configuration.bash +SC_SCRIPT="$(realpath "$0")"; +SC_TOP="${SC_SCRIPT%/*}" -list="$(get_list_from_a_file ${SC_TOP}/CHANNELS)" -URL=${cf_url}/channels +# shellcheck disable=SC1090 +. "${SC_TOP}"/configuration.bash -# admin, cfuser, channel -# username, and its password should be matched with the running ldif file. -# -cf_userid=admin +list=$(get_list_from_a_file "${SC_TOP}"/CHANNELS) +# shellcheck disable=SC2154 +URL="${cf_url}"/channels + +cf_userid="admin" cf_passwd=1234 +cf_user="${cf_userid}:${cf_passwd}" -cf_user=${cf_userid}:${cf_passwd} filterA="AR" filterB="SA:X" -for a_chan in ${list[@]}; do +for a_chan in "${list[@]}"; do print_help "PUT" "$a_chan" temp_json=$(mktemp) if [[ $a_chan =~ $filterA ]]; then @@ -61,26 +62,24 @@ for a_chan in ${list[@]}; do ] } ] - " > ${temp_json} + " > "${temp_json}" - curl -u $cf_user -H 'Content-Type: application/json' -X PUT ${URL} -d @$temp_json - rm -f $temp_json + curl -u $cf_user -H 'Content-Type: application/json' -X PUT "${URL}" -d @"$temp_json" + rm -f "$temp_json" printf "\n" done print_help "GET" "SR domain CHANNELS" -curl -X GET ${URL}?domain=SR +curl -X GET "${URL}"?domain=SR printf "\n"; print_help "GET" "AR domain CHANNELS" -curl -X GET ${URL}?domain=AR +curl -X GET "${URL}"?domain=AR printf "\n"; - print_help "GET" "ALL CHANNELS" -curl -X GET ${URL} +curl -X GET "${URL}" printf "\n"; - From 40be7a4533c81b61e0f6bc91be3084f178c3a9ca Mon Sep 17 00:00:00 2001 From: Jeong Han Lee Date: Thu, 24 Jun 2021 19:18:32 -0700 Subject: [PATCH 03/89] replace docker --- .github/workflows/docker-image.yml | 19 --------- .github/workflows/docker.yml | 67 ++++++++++++++++++++++++++++++ 2 files changed, 67 insertions(+), 19 deletions(-) delete mode 100644 .github/workflows/docker-image.yml create mode 100644 .github/workflows/docker.yml diff --git a/.github/workflows/docker-image.yml b/.github/workflows/docker-image.yml deleted file mode 100644 index 8b1c4a9..0000000 --- a/.github/workflows/docker-image.yml +++ /dev/null @@ -1,19 +0,0 @@ -name: Docker Image CI - -on: - push: - branches: [ master ] - pull_request: - branches: [ master ] - -jobs: - - build: - - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v2 - - name: Build the Docker image - run: make build.docker - diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml new file mode 100644 index 0000000..85fe453 --- /dev/null +++ b/.github/workflows/docker.yml @@ -0,0 +1,67 @@ +# 0. change name +name: ChannelFinder-Springboot +# change path name +on: + push: + branches: [ main ] + paths: + - 'docker/**' + pull_request: + branches: [ main ] +# +jobs: +# change job name + debian10: + runs-on: ubuntu-latest + env: + # change the following 4 variables + DOCKER_FILE: docker/Dockerfile + DOCKER_ACCOUNT: alscontrols + DOCKER_REPO: channelfinder + DOCKER_TAG: latest + + steps: + - + name: checkout + uses: actions/checkout@v2 + - + name: Docker meta + id: meta + uses: docker/metadata-action@v3 + with: + # list of Docker images to use as base name for tags + images: | + ${{ env.DOCKER_ACCOUNT }}/${{ env.DOCKER_REPO }} + # generate Docker tags based on the following events/attributes + tags: | + type=schedule + type=ref,event=branch + type=ref,event=pr + type=semver,pattern={{version}} + type=semver,pattern={{major}}.{{minor}} + type=semver,pattern={{major}} + type=sha + - + name: Set up QEMU + uses: docker/setup-qemu-action@v1 + - + name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + - + name: Login to DockerHub + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + - + name: Build and push + id: docker_build + uses: docker/build-push-action@v2 + with: + context: . + file: ${{ github.workspace }}/${{ env.DOCKER_FILE }} + push: ${{ github.event_name != 'pull_request' }} + #tags: ${{ steps.meta.outputs.tags }} + tags: ${{ env.DOCKER_ACCOUNT }}/${{ env.DOCKER_REPO }}:${{ env.DOCKER_TAG }} + labels: ${{ steps.meta.outputs.labels }} + From 5f5f121e342700d0a57664d31b318cf40acd858b Mon Sep 17 00:00:00 2001 From: Jeong Han Lee Date: Thu, 24 Jun 2021 19:20:56 -0700 Subject: [PATCH 04/89] set alpine as default --- docker/Dockerfile | 98 +++++++++++++++++++++++++++++- docker/Dockerfile.alpine | 97 ----------------------------- docker/{ => old}/Dockerfile.buster | 0 3 files changed, 97 insertions(+), 98 deletions(-) mode change 120000 => 100644 docker/Dockerfile delete mode 100644 docker/Dockerfile.alpine rename docker/{ => old}/Dockerfile.buster (100%) diff --git a/docker/Dockerfile b/docker/Dockerfile deleted file mode 120000 index 45cc04d..0000000 --- a/docker/Dockerfile +++ /dev/null @@ -1 +0,0 @@ -Dockerfile.alpine \ No newline at end of file diff --git a/docker/Dockerfile b/docker/Dockerfile new file mode 100644 index 0000000..52460da --- /dev/null +++ b/docker/Dockerfile @@ -0,0 +1,97 @@ +# +# author : Jeong Han Lee +# email : jeonghan.lee@gmail.com +# date : Wednesday, July 1 14:32:24 PDT 2020 +# version : 0.0.2 +# +# + +## BuildEnv docker image size : 672MB +FROM alpine:3.12 AS builder +ENV WorkPath /home/ChannelFinder-env +ARG RS_SERVER_PORT=5075 +ARG RS_PORT_BROADCAST=5076 +ARG CF_PORT_HTTP=8080 +ARG CF_PORT_HTTPS=8443 +ARG CF_INSTALL_LOCATION=/opt/channelfinder + +ENV JAVA_HOME="/usr/lib/jvm/default-jvm/" +RUN apk update && apk add openjdk11 maven git make + +WORKDIR $WorkPath +COPY . . +RUN echo "CF_PORT:=${CF_PORT_HTTP}" > configure/CONFIG_COMMON.local +RUN echo "CF_INSTALL_LOCATION:=${CF_INSTALL_LOCATION}" > configure/CONFIG_SITE.local +RUN make distclean && \ + make init && \ + make conf && \ + make build && \ + make install.docker + # make build MVN_OPTS=dependency:go-offline + + +## Multi-Stages build +## Running docker image size : 240MB +FROM alpine:3.12 +LABEL maintainer="Jeong Han Lee " + +ARG BUILD_DATE +ARG BUILD_VERSION +ARG RS_SERVER_PORT=5075 +ARG RS_PORT_BROADCAST=5076 +ARG CF_PORT_HTTP=8080 +ARG CF_PORT_HTTPS=8443 +ARG CF_INSTALL_LOCATION=/opt/channelfinder +ARG TZ=America/Los_Angeles + +# Labels. +LABEL org.label-schema.schema-version="1.0" +LABEL org.label-schema.build-date=$BUILD_DATE +LABEL org.label-schema.name="jeonghanlee/channelfinder" +LABEL org.label-schema.description="EPICS ChannelFinder Docker Image" +LABEL org.label-schema.url="https://github.com/ChannelFinder/ChannelFinder-env/" +LABEL org.label-schema.version=$BUILD_VERSION +LABEL org.label-schema.docker.cmd="docker run --network=host --detach --rm --name=channelfinder jeonghanlee/channelfinder:latest" + + +### RecCeiver PORT configuration. +ENV RS_SERVER_PORT ${RS_SERVER_PORT} +ENV RS_PORT_BROADCAST ${RS_PORT_BROADCAST} +# + +# +### DO NOT CHANGE IT +ENV LDAP_SERVER_PORT 8389 +# +### ChannelFinder HTTP PORT through Tomcat +ENV CF_PORT_HTTP ${CF_PORT_HTTP} +### ChannelFinder HTTPS PORT through Tomcat +ENV CF_PORT_HTTPS ${CF_PORT_HTTPS} +### ChannelFinder Installation Loation +ENV CF_INSTALL_LOCATION ${CF_INSTALL_LOCATION} + +### YOUR TIME ZONE +ENV TZ ${TZ} +RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone +### Install minimal packages for ChannelFinder +RUN apk add --no-cache bash openjdk11-jre tomcat-native tzdata + +WORKDIR ${CF_INSTALL_LOCATION} +COPY --from=builder ${CF_INSTALL_LOCATION} . +## +# CF Port : http +EXPOSE $CF_PORT_HTTP/tcp +EXPOSE $CF_PORT_HTTP/udp +# CF PORT : https +EXPOSE $CF_PORT_HTTPS/tcp +EXPOSE $CF_PORT_HTTPS/udp +# RecCeiver Broadcast Port +EXPOSE $RS_PORT_BROADCAST/udp +# RecCeiver Server Port +EXPOSE $RS_SERVER_PORT/udp +# Embedded LDAP Port +EXPOSE $LDAP_SERVER_PORT/tcp +EXPOSE $LDAP_SERVER_PORT/udp +# +COPY ./docker/scripts/entrypoint.sh /entrypoint.sh +ENTRYPOINT ["/entrypoint.sh"] diff --git a/docker/Dockerfile.alpine b/docker/Dockerfile.alpine deleted file mode 100644 index 52460da..0000000 --- a/docker/Dockerfile.alpine +++ /dev/null @@ -1,97 +0,0 @@ -# -# author : Jeong Han Lee -# email : jeonghan.lee@gmail.com -# date : Wednesday, July 1 14:32:24 PDT 2020 -# version : 0.0.2 -# -# - -## BuildEnv docker image size : 672MB -FROM alpine:3.12 AS builder -ENV WorkPath /home/ChannelFinder-env -ARG RS_SERVER_PORT=5075 -ARG RS_PORT_BROADCAST=5076 -ARG CF_PORT_HTTP=8080 -ARG CF_PORT_HTTPS=8443 -ARG CF_INSTALL_LOCATION=/opt/channelfinder - -ENV JAVA_HOME="/usr/lib/jvm/default-jvm/" -RUN apk update && apk add openjdk11 maven git make - -WORKDIR $WorkPath -COPY . . -RUN echo "CF_PORT:=${CF_PORT_HTTP}" > configure/CONFIG_COMMON.local -RUN echo "CF_INSTALL_LOCATION:=${CF_INSTALL_LOCATION}" > configure/CONFIG_SITE.local -RUN make distclean && \ - make init && \ - make conf && \ - make build && \ - make install.docker - # make build MVN_OPTS=dependency:go-offline - - -## Multi-Stages build -## Running docker image size : 240MB -FROM alpine:3.12 -LABEL maintainer="Jeong Han Lee " - -ARG BUILD_DATE -ARG BUILD_VERSION -ARG RS_SERVER_PORT=5075 -ARG RS_PORT_BROADCAST=5076 -ARG CF_PORT_HTTP=8080 -ARG CF_PORT_HTTPS=8443 -ARG CF_INSTALL_LOCATION=/opt/channelfinder -ARG TZ=America/Los_Angeles - -# Labels. -LABEL org.label-schema.schema-version="1.0" -LABEL org.label-schema.build-date=$BUILD_DATE -LABEL org.label-schema.name="jeonghanlee/channelfinder" -LABEL org.label-schema.description="EPICS ChannelFinder Docker Image" -LABEL org.label-schema.url="https://github.com/ChannelFinder/ChannelFinder-env/" -LABEL org.label-schema.version=$BUILD_VERSION -LABEL org.label-schema.docker.cmd="docker run --network=host --detach --rm --name=channelfinder jeonghanlee/channelfinder:latest" - - -### RecCeiver PORT configuration. -ENV RS_SERVER_PORT ${RS_SERVER_PORT} -ENV RS_PORT_BROADCAST ${RS_PORT_BROADCAST} -# - -# -### DO NOT CHANGE IT -ENV LDAP_SERVER_PORT 8389 -# -### ChannelFinder HTTP PORT through Tomcat -ENV CF_PORT_HTTP ${CF_PORT_HTTP} -### ChannelFinder HTTPS PORT through Tomcat -ENV CF_PORT_HTTPS ${CF_PORT_HTTPS} -### ChannelFinder Installation Loation -ENV CF_INSTALL_LOCATION ${CF_INSTALL_LOCATION} - -### YOUR TIME ZONE -ENV TZ ${TZ} -RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone -### Install minimal packages for ChannelFinder -RUN apk add --no-cache bash openjdk11-jre tomcat-native tzdata - -WORKDIR ${CF_INSTALL_LOCATION} -COPY --from=builder ${CF_INSTALL_LOCATION} . -## -# CF Port : http -EXPOSE $CF_PORT_HTTP/tcp -EXPOSE $CF_PORT_HTTP/udp -# CF PORT : https -EXPOSE $CF_PORT_HTTPS/tcp -EXPOSE $CF_PORT_HTTPS/udp -# RecCeiver Broadcast Port -EXPOSE $RS_PORT_BROADCAST/udp -# RecCeiver Server Port -EXPOSE $RS_SERVER_PORT/udp -# Embedded LDAP Port -EXPOSE $LDAP_SERVER_PORT/tcp -EXPOSE $LDAP_SERVER_PORT/udp -# -COPY ./docker/scripts/entrypoint.sh /entrypoint.sh -ENTRYPOINT ["/entrypoint.sh"] diff --git a/docker/Dockerfile.buster b/docker/old/Dockerfile.buster similarity index 100% rename from docker/Dockerfile.buster rename to docker/old/Dockerfile.buster From 79424bb8548f3b41a01e355aaf11d67ce737fe1d Mon Sep 17 00:00:00 2001 From: Jeong Han Lee Date: Thu, 24 Jun 2021 19:24:58 -0700 Subject: [PATCH 05/89] rename it to Service --- configure/RELEASE | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/configure/RELEASE b/configure/RELEASE index 129757f..1249499 100644 --- a/configure/RELEASE +++ b/configure/RELEASE @@ -2,14 +2,14 @@ ##- Where the Source repository SRC_URL/SRC_NAME ##- CF_SRC_URL:=https://github.com/ChannelFinder -CF_SRC_NAME:=ChannelFinder-SpringBoot +CF_SRC_NAME:=ChannelFinderService ##- Which the source tag / branch / hash id would like to use -CF_SRC_TAG:=537bf2c +CF_SRC_TAG:=ef86257 ##- Placeholder for the site-specific version control -CF_SRC_VERSION:=4.0.0 +CF_SRC_VERSION:=4.0.1 -include $(TOP)/../RELEASE.local From 77fe326e7fde977ff51f2fd2989fb46245eb1ed0 Mon Sep 17 00:00:00 2001 From: Jeong Han Lee Date: Thu, 24 Jun 2021 19:26:35 -0700 Subject: [PATCH 06/89] swich to master --- .github/workflows/docker.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 85fe453..02bc494 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -1,13 +1,13 @@ # 0. change name -name: ChannelFinder-Springboot +name: ChannelFinderService # change path name on: push: - branches: [ main ] + branches: [ master ] paths: - 'docker/**' pull_request: - branches: [ main ] + branches: [ master ] # jobs: # change job name From c1101ccf63837bb13ec49c60016562f70f571f60 Mon Sep 17 00:00:00 2001 From: Jeong Han Lee Date: Thu, 24 Jun 2021 19:30:04 -0700 Subject: [PATCH 07/89] udate docker and linter ci --- .github/workflows/docker.yml | 1 + .github/workflows/linter.yml | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 02bc494..9072228 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -6,6 +6,7 @@ on: branches: [ master ] paths: - 'docker/**' + - '.github/workflows/docker.yml' pull_request: branches: [ master ] # diff --git a/.github/workflows/linter.yml b/.github/workflows/linter.yml index 69c72a1..5493769 100644 --- a/.github/workflows/linter.yml +++ b/.github/workflows/linter.yml @@ -3,6 +3,10 @@ name: Linter Run on: push: branches: [master] + paths: + - '*.md' + - 'docker/scripts/**' + - '.github/workflows/linter.yml' pull_request: branches: [master] From be0d35ea2528777c13a85add2d796a4f6f016ae6 Mon Sep 17 00:00:00 2001 From: Jeong Han Lee Date: Thu, 24 Jun 2021 19:48:11 -0700 Subject: [PATCH 08/89] remove lint --- .github/workflows/linter.yml | 25 ------------------------- 1 file changed, 25 deletions(-) delete mode 100644 .github/workflows/linter.yml diff --git a/.github/workflows/linter.yml b/.github/workflows/linter.yml deleted file mode 100644 index 5493769..0000000 --- a/.github/workflows/linter.yml +++ /dev/null @@ -1,25 +0,0 @@ -name: Linter Run - -on: - push: - branches: [master] - paths: - - '*.md' - - 'docker/scripts/**' - - '.github/workflows/linter.yml' - pull_request: - branches: [master] - -jobs: - build: - name: Lint Code Base - runs-on: ubuntu-latest - - steps: - - name: Checkout Code - uses: actions/checkout@v2 - - name: Lint Code Base - uses: docker://github/super-linter:latest - env: - VALIDATE_MD: true - VALIDATE_BASH: true From 53ea4cdf38df2f15b798f0b96ab96321ec2dc05e Mon Sep 17 00:00:00 2001 From: Jeong Han Lee Date: Thu, 24 Jun 2021 19:59:06 -0700 Subject: [PATCH 09/89] use alscontrols --- docker/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 52460da..2d2586f 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -47,11 +47,11 @@ ARG TZ=America/Los_Angeles # Labels. LABEL org.label-schema.schema-version="1.0" LABEL org.label-schema.build-date=$BUILD_DATE -LABEL org.label-schema.name="jeonghanlee/channelfinder" +LABEL org.label-schema.name="alscontrols/channelfinder" LABEL org.label-schema.description="EPICS ChannelFinder Docker Image" LABEL org.label-schema.url="https://github.com/ChannelFinder/ChannelFinder-env/" LABEL org.label-schema.version=$BUILD_VERSION -LABEL org.label-schema.docker.cmd="docker run --network=host --detach --rm --name=channelfinder jeonghanlee/channelfinder:latest" +LABEL org.label-schema.docker.cmd="docker run --network=host --detach --rm --name=channelfinder alscontrols/channelfinder:latest" ### RecCeiver PORT configuration. From ef307695c83ed85acd1a10aac9e5e1546b7c8131 Mon Sep 17 00:00:00 2001 From: Jeong Han Lee Date: Sat, 17 Sep 2022 16:02:46 -0700 Subject: [PATCH 10/89] preparation for the next release, gitlab merging --- .gitignore | 3 + README.md | 166 +++----------------------------- configure/CONFIG_COMMON | 7 +- configure/CONFIG_SITE | 21 +++- configure/CONFIG_SRC | 8 +- configure/CONFIG_VARS | 7 ++ configure/RELEASE | 4 +- configure/RULES | 4 + configure/RULES_PROPERTIES | 32 ++++++ configure/RULES_SRC | 14 +-- configure/RULES_VARS | 44 ++++++--- examples/delete_channels.bash | 2 +- examples/delete_properties.bash | 2 +- examples/delete_tags.bash | 2 +- examples/put_channels.bash | 2 +- examples/put_properties.bash | 2 +- examples/put_tags.bash | 2 +- examples/update_channels.bash | 2 +- 18 files changed, 130 insertions(+), 194 deletions(-) create mode 100644 configure/RULES_PROPERTIES diff --git a/.gitignore b/.gitignore index c59be69..d78d4ae 100644 --- a/.gitignore +++ b/.gitignore @@ -39,3 +39,6 @@ site-template/application.properties +.*.swp +*.swp +*.swo diff --git a/README.md b/README.md index 14384ce..d0d1308 100644 --- a/README.md +++ b/README.md @@ -4,175 +4,35 @@ ![Docker Image CI](https://github.com/ChannelFinder/ChannelFinder-env/workflows/Docker%20Image%20CI/badge.svg) ![Linter Run](https://github.com/ChannelFinder/ChannelFinder-env/workflows/Linter%20Run/badge.svg) -Configuration Environment for ChannelFinder-SpringBoot at +Configuration Environment for ChannelFinderService at -## Role +## Pre-requirement packages -In order to download, install, setup all relevant components, one should do many steps manually. This repository was designed for the easy-to-reproducible environment for ChannelFinder-SprintBoot. - -## Requirements - -**Note that** this implementation is valid only for `ChannelFinder-SpringBoot` - -### Apache Tomcat Native Library and Maven - -* Debian 10 - -```bash -apt install maven libtcnative-1 -``` - -* CentOS 7 - -```bash -yum install maven tomcat-native -``` - -* CentOS 8 \& Fedora 31 - -One can use `yum` instead of `dnf` for CentOS 8. However, `epel-release` is needed. - -```bash -dnf install maven tomcat-native -``` - -### JDK 8 or newer - -* Debian 10 - -```bash -openjdk version "11.0.6" 2020-01-14 -OpenJDK Runtime Environment (build 11.0.6+10-post-Debian-1deb10u1) -OpenJDK 64-Bit Server VM (build 11.0.6+10-post-Debian-1deb10u1, mixed mode, sharing) ``` - -* Fedora 32 - -```bash -$ update-alternatives --config java -There is 1 program that provides 'java'. - - Selection Command ------------------------------------------------ -*+ 1 java-1.8.0-openjdk.x86_64 (/usr/lib/jvm/java-1.8.0-openjdk-1.8.0.242.b08-1.fc32.x86_64/jre/bin/java) - -$ java -version - -openjdk version "1.8.0_242" -OpenJDK Runtime Environment (build 1.8.0_242-b08) -OpenJDK 64-Bit Server VM (build 25.242-b08, mixed mode) +git make sudo which ``` -### ElasticSearch 6.3.1 +## Debian 10/11 (EOL: 2024-06-01/2026-08-15) -Please use the exact version of Elasticsearch **6.3.1**. -* Debian 10 +### Build, install, and Service -```bash -wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-6.3.1.deb -sudo apt install ./elasticsearch-6.3.1.deb -``` - -* CentOS 7 +## CentOS7 (EOL: 2024-06-30) -```bash -wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-6.3.1.rpm -sudo yum install ./elasticsearch-6.3.1.rpm -``` +See [docs/README.centos7.md](docs/README.centos7.md). -* CentOS 8 & Fedora 32 +## ~~CentOS8 (EOL: 2021-12-31)~~ -```bash -wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-6.3.1.rpm -sudo dnf install ./elasticsearch-6.3.1.rpm -``` - -* ES Systemd service - -```bash -sudo systemctl daemon-reload -sudo systemctl enable elasticsearch -sudo systemctl start elasticsearch -``` +~~See [docs/README.centos8.md](docs/README.centos8.md).~~ -Please make sure the service is running via `systemctl status elasticsearch`. One can do the same things with embedded make rules such as +## Rocky8 (EOL: 2029-05-31) -### LDAP +See [docs/README.rocky8.md](docs/README.rocky8.md). -* Configuration is needed if an external one. -* Embedded LDAP configuration [Local LDIF configuration](site-template/LDIF_CONFIG.md) - -## Few Makefile Rules - -### `make init` - -* Download the ChannelFinder-SpringBoot -* Switch to a specific version defined in `$(SRC_TAG)` in `configure/RELEASE` - -### `make es_mapping` - -* Do maping CF index into ElasticSearch -* For CentOS7, one should disable SELINUX before doing this. Please check `/etc/selinux/config`. - -### `make es_mapping_clean` - -* Remove all existent CF related mapping from ElasticSearch - -### `make conf` - -* Apply several site-specific files into the downloaded sources. Please see `site-template` - -### `make build` - -* Build `ChannelFinder-*.jar` as a Spring Boot jar file, located in `target` path. - -### `make install` - -* `sudo` permission is required. -* Install `ChannelFinder-*.jar` in `target` path into `INSTALL_LOCATION` defined in `configure/CONFIG_SITE` -* Install `cf.conf` into `INSTALL_LOCATION`. This file contains java options, defined in `site-template/cf.conf` -* Install `channelfinder.service` into a default systemd path `/etc/systemd/system`. Note that `channelfinder.service` is generated from `site-template/cf.service.in` file with `INSTAL_LOCATION`, `JAVA_PATH`, and `CF_JAR_NAME` which are defined in `configure/CONFIG_SITE`. -* Run `systemctl daemon-reload` for the updated `channelfinder.serive` systemd file. -* Enable the channelfinder systemd service. - -### `make distclean` - -* Remove the downloaded ChannelFinder-SprintBoot source file - -### `make vars` - -* Print out interesting variables -* One can use `make PRINT.VARIABLE_NAME` to print out them. For example, `make PRINT.INSTALL_LOCATION`. - -## A typical example to configure the ChannelFinder service - -Note that this example has the assumption which ES service is running. - -```bash -make init -make es_mapping -make conf -make build -make install -sudo systemctl start channelfinder -``` - -Note that we create the alias name of the `channelfinder.service` as cf.servie. So one can start it `sudo systemctl start cf` also. Sometimes with CentOS8, the service doesn't start properly. In this case, please try with `SELINUX=disabled` in `/etc/selinux/config`. - -For Fedora 32, one should disable `SELINUX` before `make es_mapping`. - -## While evaluating its configuration - -Modify `application.properties` or `ldif` file, and then run the following command: - -```bash -make restart -``` +## macOS (BigSur 11.6.1, 20G224) -## Customize site-specific configuration +See [docs/README.macos.md](docs/README.macos.md). -Please consult two files in `configure` path, such as `RELEASE` and `CONFIG_SITE`. There are few comments on there. If you are familiar with the standard EPICS building system [1], it should be easy to understand them, because we mimic that concept into this repository. ## Docker Image diff --git a/configure/CONFIG_COMMON b/configure/CONFIG_COMMON index da704b5..f665398 100644 --- a/configure/CONFIG_COMMON +++ b/configure/CONFIG_COMMON @@ -1,6 +1,11 @@ +# Debian +JAVA_HOME:=/usr/lib/jvm/default-java +MAVEN_HOME:=/usr/share/maven +TOMCAT_HOME:=/usr/share/tomcat9 -JAVA_PATH=/usr/bin +JAVA_PATH:=$(JAVA_HOME)/bin +MAVEN_PATH:=$(MAVEN_HOME)/bin ## Elasticsearch Server ES_HOST:=localhost diff --git a/configure/CONFIG_SITE b/configure/CONFIG_SITE index 9dd450d..c6edf62 100644 --- a/configure/CONFIG_SITE +++ b/configure/CONFIG_SITE @@ -1,15 +1,19 @@ ### channelfinder installation location -CF_INSTALL_LOCATION=/opt/channelfinder +CF_INSTALL_PATH=/opt +CF_INSTALL_LOCATION=${CF_INSTALL_PATH}/channelfinder + ### Placeholder ### This variable will be replaced by reading a pom.xml file or others ### It may be linked with SRC_VERSION in RELEASE file ### -CF_JAR_FILENAME:=ChannelFinder-4.0.0.jar +CF_JAR_FILENAME:=ChannelFinder-$(CF_SRC_VERSION).jar +CF_PROPERTIES:=application.properties +SYSTEMD_PATH:=/etc/systemd/system ### ### These two parameters are used to generate application.properties from application.properties.in ### @@ -25,7 +29,18 @@ CF_JAVA_OPTS:=-Xms512m -Xmx512m CF_MVN_OPTS:= -SYSTEMD_PATH:=/etc/systemd/system + +CF_SITE_TEMPLATE_PATH=$(TOP)/site-template + +# SYSTEDM Required and After Service List +# This is After and Required services for the systemd archiver appliance systemd service. +# mariadb.service was defined in there, but sometimes, we need to check our directories where +# our data will be saved. +# one can check the mounting service name through `systemctl list-units --type=mount` +# Here is the example for the ALS environment +# CONFIG_SITE.local can be used to hold this information locally +# Each service need one space between them, and backslash must be double backslash +CF_SYSTEMD_SERVICES:= # These allow developers to override the CONFIG_SITE variable # settings without having to modify the configure/CONFIG_SITE diff --git a/configure/CONFIG_SRC b/configure/CONFIG_SRC index 0b96c85..aa51d75 100644 --- a/configure/CONFIG_SRC +++ b/configure/CONFIG_SRC @@ -12,8 +12,14 @@ ifeq "$(SRC_PATH_INFO)" "1" INIT_SRC = 1 endif + +PATH:=$(MAVEN_PATH):$(JAVA_PATH):${PATH} + +MAVEN_CMD:=$(MAVE_PATH)/mvn +JAVA_CMD:=$(JAVA_PATH)/java + + CF_SITE_SPECIFIC_FILES_PATH:=$(CF_SRC_PATH)/src/main/resources -CF_SITE_TEMPLATE_PATH:=$(TOP)/site-template VARS_EXCLUDES+=SRC_PATH_INFO VARS_EXCLUDES+=INSTALL_DATA diff --git a/configure/CONFIG_VARS b/configure/CONFIG_VARS index e1398dc..db7acac 100644 --- a/configure/CONFIG_VARS +++ b/configure/CONFIG_VARS @@ -10,3 +10,10 @@ CF_SRC_GITURL:=$(CF_SRC_URL)/$(CF_SRC_NAME) ### CF_SRC_PATH used for a local directory which is a clone git repository CF_SRC_PATH:=$(CF_SRC_NAME)-src + +#DOCURL:=https://docs.payara.fish/community/docs/$(SRC_VERSION)/README.html +DOCURL:=$(CF_SRC_GITURL) + +# These allow developers to override the variable +-include $(TOP)/../CONFIG_VARS.local +-include $(TOP)/configure/CONFIG_VARS.local diff --git a/configure/RELEASE b/configure/RELEASE index 1249499..5cc57a6 100644 --- a/configure/RELEASE +++ b/configure/RELEASE @@ -6,10 +6,10 @@ CF_SRC_NAME:=ChannelFinderService ##- Which the source tag / branch / hash id would like to use -CF_SRC_TAG:=ef86257 +CF_SRC_TAG:=ChannelFinder-4.7.0 ##- Placeholder for the site-specific version control -CF_SRC_VERSION:=4.0.1 +CF_SRC_VERSION:=4.7.0 -include $(TOP)/../RELEASE.local diff --git a/configure/RULES b/configure/RULES index 12f7b69..737f3b9 100644 --- a/configure/RULES +++ b/configure/RULES @@ -1,9 +1,13 @@ +vpath %.in $(CF_SITE_TEMPLATE_PATH) +vpath %.properties $(CF_SITE_TEMPLATE_PATH) +# include $(TOP)/configure/RULES_FUNC include $(TOP)/configure/RULES_REQ include $(TOP)/configure/RULES_SRC include $(TOP)/configure/RULES_INSTALL +include $(TOP)/configure/RULES_PROPERTIES include $(TOP)/configure/RULES_DOCKER include $(TOP)/configure/RULES_VARS diff --git a/configure/RULES_PROPERTIES b/configure/RULES_PROPERTIES new file mode 100644 index 0000000..82a8283 --- /dev/null +++ b/configure/RULES_PROPERTIES @@ -0,0 +1,32 @@ + + +properties_RULES_NAMES:=properties +conf_properties_RULES:=$(addprefix conf., $(properties_RULES_NAMES)) +show_properties_RULES:=$(addsuffix .show, $(conf_properties_RULES)) + +RULES_VARS+=conf.cfproperties conf.cfproperties.show + +conf.cfproperties: $(conf_properties_RULES) + +conf.cfproperties.show: $(show_properties_RULES) + +conf.properties: $(CF_PROPERTIES).in + $(QUIET)echo ">>> Generate $(basename $<) from $<" + $(QUIET)sed -e "s|@SSHKEY@|$(CF_SSHKEY)|g" \ + -e "s|@SSHKEYALIAS@|$(CF_SSHKEYALIAS)|g" \ + -e "s|@CFLDIF@|$(CF_LDIF)|g" \ + -e "s|@CF_PORT@|$(CF_PORT)|g" \ + -e "s|@ES_HOST@|$(ES_HOST)|g" \ + -e "s|@ES_PORT@|$(ES_PORT)|g" \ + -e "s|@CF_QUERY_SIZE@|$(CF_QUERY_SIZE)|g" \ + < $< > $(basename $<) + rm -f $(CF_SITE_SPECIFIC_FILES_PATH)/$(notdir $(basename $<)) + $(QUIET)echo ">>> Copy $(basename $<) to $(CF_SITE_SPECIFIC_FILES_PATH)" + $(QUIET)$(INSTALL_DATA) $(basename $<) $(CF_SITE_SPECIFIC_FILES_PATH)/ +# $(QUIET)cp $(CF_SITE_TEMPLATE_PATH)/$(CF_LDIF) $(CF_SITE_SPECIFIC_FILES_PATH)/ +# $(QUIET)cp $(CF_SITE_TEMPLATE_PATH)/$(CF_SSHKEY) $(CF_SITE_SPECIFIC_FILES_PATH)/keystore/ +# $(QUIET)cp $(CF_SITE_TEMPLATE_PATH)/application.properties $(CF_SITE_SPECIFIC_FILES_PATH)/ + +conf.properties.show: $(CF_PROPERTIES).in + $(QUIET)cat -b $(basename $<) + diff --git a/configure/RULES_SRC b/configure/RULES_SRC index f661373..dcb6b3f 100644 --- a/configure/RULES_SRC +++ b/configure/RULES_SRC @@ -53,15 +53,5 @@ build: $(QUIET) mvn $(CF_MVN_OPTS) -f $(CF_SRC_PATH)/pom.xml clean install -Dmaven.test.skip=true ## Copy the site specific files into sources -conf: - $(QUIET)sed -e "s:@SSHKEY@:$(CF_SSHKEY):g" \ - -e "s:@SSHKEYALIAS@:$(CF_SSHKEYALIAS):g" \ - -e "s:@CFLDIF@:$(CF_LDIF):g" \ - -e "s:@CF_PORT@:$(CF_PORT):g" \ - -e "s:@ES_HOST@:$(ES_HOST):g" \ - -e "s:@ES_PORT@:$(ES_PORT):g" \ - -e "s:@CF_QUERY_SIZE@:$(CF_QUERY_SIZE):g" \ - < $(CF_SITE_TEMPLATE_PATH)/application.properties.in > $(CF_SITE_TEMPLATE_PATH)/application.properties - $(QUIET)cp $(CF_SITE_TEMPLATE_PATH)/$(CF_LDIF) $(CF_SITE_SPECIFIC_FILES_PATH)/ - $(QUIET)cp $(CF_SITE_TEMPLATE_PATH)/$(CF_SSHKEY) $(CF_SITE_SPECIFIC_FILES_PATH)/keystore/ - $(QUIET)cp $(CF_SITE_TEMPLATE_PATH)/application.properties $(CF_SITE_SPECIFIC_FILES_PATH)/ +conf: conf.cfproperties + diff --git a/configure/RULES_VARS b/configure/RULES_VARS index 015647d..436d19e 100644 --- a/configure/RULES_VARS +++ b/configure/RULES_VARS @@ -1,18 +1,21 @@ +RULES_VARS+=$(foreach v, $(filter %_RULES,$(.VARIABLES)), $(v)) +RULES_VARS+=$(foreach v, $(filter %_RULES_NAMES,$(.VARIABLES)), $(v)) + VARS_EXCLUDES+=.SHELLSTATUS VARS_EXCLUDES+=INIT_SRC -VARS_EXCLUDES+=SITE_SPECIFIC_FILES_PATH -VARS_EXCLUDES+=RS_% -VARS_EXCLUDES+=AS_% -VARS_EXCLUDES+=OG_% -VARS_EXCLUDES+=PS_% -VARS_EXCLUDES+=KAFKA_% -VARS_EXCLUDES+=ALARM_% -VARS_EXCLUDES+=AL_% -VARS_EXCLUDES+=AA_% +VARS_EXCLUDES+=FILTER +VARS_EXCLUDES+=LEVEL +VARS_EXCLUDES+=$(filter JAR_%, $(.VARIABLES)) +VARS_EXCLUDES+=$(RULES_VARS) +VARS_EXCLUDES+=%_RULES_VARS +VARS_EXCLUDES+=a_service_BUIDER buildrules_BUILDER -LEVEL?=1 +.PHONY: $(RULES_VARS) +LEVEL?=1 FILTER?=1 +LSOPTS?="-lta" + ifeq "$(FILTER)" "1" SRC_VARIABLES:=$(sort $(filter-out $(VARS_EXCLUDES) VARS_EXCLUDES,$(.VARIABLES))) @@ -20,8 +23,6 @@ else SRC_VARIABLES:=$(filter $(FILTER)%, $(sort $(filter-out $(VARS_EXCLUDES) VARS_EXCLUDES,$(.VARIABLES)))) endif - - .PHONY : env vars header env: vars @@ -45,15 +46,28 @@ print-%: $(QUIET)echo $* = $($*) $(QUIET)echo $*\'s origin is $(origin $*) +ls.%: + $(QUIET) ls $(LSOPTS) $($*) + +tree.%: + $(QUIET) tree -aL $(LEVEL) $($*) + +cat.%: + $(QUIET) cat -b $($*) + +FORCE: + .PHONY : exist exist: - $(if $(wildcard $(CF_INSTALL_LOCATION)), tree -aL $(LEVEL) $(CF_INSTALL_LOCATION), $(QUIET)echo "No $(CF_INSTALL_LOCATION)") +ifeq (, $(shell which tree)) + $(if $(wildcard $(CF_INSTALL_LOCATION)), @ ls -d $(CF_INSTALL_LOCATION) && find $(CF_INSTALL_LOCATION)/ -maxdepth $(LEVEL) | sed -e "s/[^-][^\/]*\// |/g" -e "s/|\([^ ]\)/:---\1/") +else + $(if $(wildcard $(CF_INSTALL_LOCATION)), tree -pugaL $(LEVEL) $(CF_INSTALL_LOCATION), $(QUIET)echo "No $(CF_INSTALL_LOCATION)") +endif .DELETE_ON_ERROR: .NOTPARALLEL: - -VARS_EXCLUDES+=FILTER \ No newline at end of file diff --git a/examples/delete_channels.bash b/examples/delete_channels.bash index ff5815c..55675c6 100644 --- a/examples/delete_channels.bash +++ b/examples/delete_channels.bash @@ -6,7 +6,7 @@ declare -g SC_TOP; SC_SCRIPT="$(realpath "$0")"; SC_TOP="${SC_SCRIPT%/*}" -# shellcheck disable=SC1090 +# shellcheck disable=SC1090,SC1091 . "${SC_TOP}/configuration.bash" diff --git a/examples/delete_properties.bash b/examples/delete_properties.bash index 832fd7f..ef22f73 100644 --- a/examples/delete_properties.bash +++ b/examples/delete_properties.bash @@ -7,7 +7,7 @@ SC_SCRIPT="$(realpath "$0")"; SC_TOP="${SC_SCRIPT%/*}" -# shellcheck disable=SC1090 +# shellcheck disable=SC1090,SC1091 . "${SC_TOP}"/configuration.bash list=$(get_list_from_a_file "${SC_TOP}"/PROPERTIES) diff --git a/examples/delete_tags.bash b/examples/delete_tags.bash index bd30d4c..d448995 100644 --- a/examples/delete_tags.bash +++ b/examples/delete_tags.bash @@ -6,7 +6,7 @@ declare -g SC_TOP; SC_SCRIPT="$(realpath "$0")"; SC_TOP="${SC_SCRIPT%/*}" -# shellcheck disable=SC1090 +# shellcheck disable=SC1090,SC1091 . "${SC_TOP}"/configuration.bash list=$(get_list_from_a_file "${SC_TOP}"/TAGS) diff --git a/examples/put_channels.bash b/examples/put_channels.bash index 199b6a4..7a6d220 100644 --- a/examples/put_channels.bash +++ b/examples/put_channels.bash @@ -7,7 +7,7 @@ SC_SCRIPT="$(realpath "$0")"; SC_TOP="${SC_SCRIPT%/*}" -# shellcheck disable=SC1090 +# shellcheck disable=SC1090,SC1091 . "${SC_TOP}"/configuration.bash list=$(get_list_from_a_file "${SC_TOP}"/CHANNELS) diff --git a/examples/put_properties.bash b/examples/put_properties.bash index a588c15..185a7b6 100644 --- a/examples/put_properties.bash +++ b/examples/put_properties.bash @@ -7,7 +7,7 @@ SC_SCRIPT="$(realpath "$0")"; SC_TOP="${SC_SCRIPT%/*}" -# shellcheck disable=SC1090 +# shellcheck disable=SC1090,SC1091 . "${SC_TOP}"/configuration.bash list=$(get_list_from_a_file "${SC_TOP}"/PROPERTIES) diff --git a/examples/put_tags.bash b/examples/put_tags.bash index b30ea9d..e1a628e 100644 --- a/examples/put_tags.bash +++ b/examples/put_tags.bash @@ -8,7 +8,7 @@ SC_SCRIPT="$(realpath "$0")"; SC_TOP="${SC_SCRIPT%/*}" -# shellcheck disable=SC1090 +# shellcheck disable=SC1090,SC1091 . "${SC_TOP}"/configuration.bash diff --git a/examples/update_channels.bash b/examples/update_channels.bash index 4c0bef2..5b7a2b5 100644 --- a/examples/update_channels.bash +++ b/examples/update_channels.bash @@ -7,7 +7,7 @@ SC_SCRIPT="$(realpath "$0")"; SC_TOP="${SC_SCRIPT%/*}" -# shellcheck disable=SC1090 +# shellcheck disable=SC1090,SC1091 . "${SC_TOP}"/configuration.bash From 63985b2a0a1f8b28e5e4ad1fcd3e088ce7838f85 Mon Sep 17 00:00:00 2001 From: Jeong Han Lee Date: Sat, 17 Sep 2022 16:13:26 -0700 Subject: [PATCH 11/89] update github action --- .github/workflows/maven.yml | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml index a2880e7..2f2d92e 100644 --- a/.github/workflows/maven.yml +++ b/.github/workflows/maven.yml @@ -1,30 +1,30 @@ -# This workflow will build a Java project with Maven -# For more information see: https://help.github.com/actions/language-and-framework-guides/building-and-testing-java-with-maven - -name: ChannelFinder-env Build Test +--- +name: Ubuntu Latest on: push: branches: [ master ] + paths-ignores: + - '*.md' + - 'examples/*.bash' + pull_request: branches: [ master ] jobs: - build: + Ubuntu: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 - - uses: actions/setup-java@v1 + - uses: actions/checkout@v3 + - uses: actions/setup-java@v3 with: - java-version: '11.0.x' - java-package: jdk - architecture: x64 + distribution: 'zulu' + java-version: '18' + cache: 'maven' - run: | java -version - echo $JAVA_HOME - echo $PATH mvn --version - run: sudo apt-get install make tree git - run: | @@ -36,4 +36,3 @@ jobs: make install - run: make exist - From 8abd793533c4076c0c92208e36aec39eb47d2cf0 Mon Sep 17 00:00:00 2001 From: Jeong Han Lee Date: Sat, 17 Sep 2022 16:16:31 -0700 Subject: [PATCH 12/89] test java actions --- .github/workflows/{maven.yml => ubuntu.yml} | 1 - 1 file changed, 1 deletion(-) rename .github/workflows/{maven.yml => ubuntu.yml} (96%) diff --git a/.github/workflows/maven.yml b/.github/workflows/ubuntu.yml similarity index 96% rename from .github/workflows/maven.yml rename to .github/workflows/ubuntu.yml index 2f2d92e..e02a013 100644 --- a/.github/workflows/maven.yml +++ b/.github/workflows/ubuntu.yml @@ -22,7 +22,6 @@ jobs: with: distribution: 'zulu' java-version: '18' - cache: 'maven' - run: | java -version mvn --version From 08812b8066b2fb8b55e3741e7618d9419ab97ecf Mon Sep 17 00:00:00 2001 From: Jeong Han Lee Date: Sat, 17 Sep 2022 16:20:23 -0700 Subject: [PATCH 13/89] to get JAVA_HOME,MVN_HOME --- .github/workflows/ubuntu.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/ubuntu.yml b/.github/workflows/ubuntu.yml index e02a013..7250833 100644 --- a/.github/workflows/ubuntu.yml +++ b/.github/workflows/ubuntu.yml @@ -25,6 +25,8 @@ jobs: - run: | java -version mvn --version + echo $JAVA_HOME + echo $MVN_HOME - run: sudo apt-get install make tree git - run: | make init From b8cad42913e3ad0420b108587e402eb0d4991ab8 Mon Sep 17 00:00:00 2001 From: Jeong Han Lee Date: Sat, 17 Sep 2022 18:39:54 -0700 Subject: [PATCH 14/89] github java test --- .github/workflows/ubuntu.yml | 11 +++--- configure/RULES | 1 + configure/RULES_CI | 68 ++++++++++++++++++++++++++++++++++++ configure/RULES_SRC | 2 +- 4 files changed, 75 insertions(+), 7 deletions(-) create mode 100644 configure/RULES_CI diff --git a/.github/workflows/ubuntu.yml b/.github/workflows/ubuntu.yml index 7250833..9314bdc 100644 --- a/.github/workflows/ubuntu.yml +++ b/.github/workflows/ubuntu.yml @@ -21,16 +21,15 @@ jobs: - uses: actions/setup-java@v3 with: distribution: 'zulu' - java-version: '18' - - run: | - java -version - mvn --version - echo $JAVA_HOME - echo $MVN_HOME + java-version: '18.0.2' - run: sudo apt-get install make tree git + - run: | + which java + which mvn - run: | make init make vars + make github.conf make conf - run: | make build diff --git a/configure/RULES b/configure/RULES index 737f3b9..69b43f0 100644 --- a/configure/RULES +++ b/configure/RULES @@ -8,6 +8,7 @@ include $(TOP)/configure/RULES_REQ include $(TOP)/configure/RULES_SRC include $(TOP)/configure/RULES_INSTALL include $(TOP)/configure/RULES_PROPERTIES +include $(TOP)/configure/RULES_CI include $(TOP)/configure/RULES_DOCKER include $(TOP)/configure/RULES_VARS diff --git a/configure/RULES_CI b/configure/RULES_CI new file mode 100644 index 0000000..f2da7ac --- /dev/null +++ b/configure/RULES_CI @@ -0,0 +1,68 @@ +.PHONY: centos8.conf rocky8.conf centos7.conf + +centos7.conf: rocky8.conf + +github.conf: + $(QUIET)echo ">>> Generating $(TOP)/configure/CONFIG_COMMON.local" + $(QUIET)-rm -f $(TOP)/configure/CONFIG_COMMON.local + $(QUIET)echo "JAVA_HOME:=${JAVA_HOME}" > $(TOP)/configure/CONFIG_COMMON.local + $(QUIET)echo "JAVA_PATH:=${JAVA_HOME}/bin" >> $(TOP)/configure/CONFIG_COMMON.local + $(QUIET)echo "MAVEN_HOME:=/usr/share/apache-maven-3.8.6" >> $(TOP)/configure/CONFIG_COMMON.local + $(QUIET)echo "MAVEN_PATH:=/usr/share/apache-maven-3.8.6/bin" >> $(TOP)/configure/CONFIG_COMMON.local + $(QUIET)cat -b $(TOP)/configure/CONFIG_COMMON.local + +rocky8.conf: + $(QUIET)echo ">>> Generating $(TOP)/configure/CONFIG_COMMON.local" + $(QUIET)-rm -f $(TOP)/configure/CONFIG_COMMON.local + $(QUIET)echo "JAVA_HOME:=/usr/lib/jvm/jre-11-openjdk" > $(TOP)/configure/CONFIG_COMMON.local + $(QUIET)echo "JAVA_PATH:=/usr/lib/jvm/jre-11-openjdk/bin" >> $(TOP)/configure/CONFIG_COMMON.local + $(QUIET)echo "TOMCAT_HOME:=/opt/tomcat9" >> $(TOP)/configure/CONFIG_COMMON.local + $(QUIET)cat -b $(TOP)/configure/CONFIG_COMMON.local + +macos.conf: + $(QUIET)echo ">>> Generating $(TOP)/configure/CONFIG_COMMON.local" + $(QUIET)-rm -f $(TOP)/configure/CONFIG_COMMON.local + $(QUIET)echo "JAVA_HOME:=/Library/Java/JavaVirtualMachines/openjdk17-zulu/Contents/Home" > $(TOP)/configure/CONFIG_COMMON.local + $(QUIET)echo "JAVA_PATH:=/Library/Java/JavaVirtualMachines/openjdk17-zulu/Contents/Home/bin" >> $(TOP)/configure/CONFIG_COMMON.local + $(QUIET)echo "ANT_HOME:=/opt/local/share/java/apache-ant" >> $(TOP)/configure/CONFIG_COMMON.local + $(QUIET)echo "ANT_PATH:=/opt/local/share/java/apache-ant/bin" >> $(TOP)/configure/CONFIG_COMMON.local + $(QUIET)echo "TOMCAT_HOME:=/opt/tomcat9" >> $(TOP)/configure/CONFIG_COMMON.local + $(QUIET)cat -b $(TOP)/configure/CONFIG_COMMON.local + $(QUIET)echo ">>> Generating $(TOP)/configure/CONFIG_SITE.local" + $(QUIET)-rm -f $(TOP)/configure/CONFIG_SITE.local + $(QUIET)echo "USERID:=${USER}" > $(TOP)/configure/CONFIG_SITE.local + $(QUIET)echo "GROUPID:=20" >> $(TOP)/configure/CONFIG_SITE.local + $(QUIET)echo "ARCHAPPL_LONG_TERM_FOLDER=$(ARCHAPPL_STORAGE_TOP)/lts/ArchiverStore" >> $(TOP)/configure/CONFIG_SITE.local + $(QUIET)cat -b $(TOP)/configure/CONFIG_SITE.local + + +macos.conf: macbrew.conf + +macbrew.conf: + $(QUIET)echo ">>> Generating $(TOP)/configure/CONFIG_COMMON.local" + $(QUIET)-rm -f $(TOP)/configure/CONFIG_COMMON.local + $(QUIET)echo "JAVA_HOME:=/opt/homebrew/opt/openjdk" > $(TOP)/configure/CONFIG_COMMON.local + $(QUIET)echo "JAVA_PATH:=/opt/homebrew/opt/openjdk/bin" >> $(TOP)/configure/CONFIG_COMMON.local + $(QUIET)echo "ANT_HOME:=/opt/homebrew/opt/ant" >> $(TOP)/configure/CONFIG_COMMON.local + $(QUIET)echo "ANT_PATH:=/opt/homebrew/opt/ant/bin" >> $(TOP)/configure/CONFIG_COMMON.local + $(QUIET)cat -b $(TOP)/configure/CONFIG_COMMON.local + $(QUIET)echo ">>> Generating $(TOP)/configure/CONFIG_SITE.local" + $(QUIET)-rm -f $(TOP)/configure/CONFIG_SITE.local + $(QUIET)echo "USERID:=${USER}" > $(TOP)/configure/CONFIG_SITE.local + $(QUIET)echo "GROUPID:=admin" >> $(TOP)/configure/CONFIG_SITE.local + $(QUIET)cat -b $(TOP)/configure/CONFIG_SITE.local + +githubmac.conf: + $(QUIET)echo ">>> Generating $(TOP)/configure/CONFIG_COMMON.local" + $(QUIET)-rm -f $(TOP)/configure/CONFIG_COMMON.local + $(QUIET)echo "JAVA_HOME:=/usr/local/opt/openjdk@17" > $(TOP)/configure/CONFIG_COMMON.local + $(QUIET)echo "JAVA_PATH:=/usr/local/opt/openjdk@17/bin" >> $(TOP)/configure/CONFIG_COMMON.local + $(QUIET)echo "ANT_HOME:=/usr/local/Cellar/ant/1.10.12" >> $(TOP)/configure/CONFIG_COMMON.local + $(QUIET)echo "ANT_PATH:=/usr/local/Cellar/ant/1.10.12/bin" >> $(TOP)/configure/CONFIG_COMMON.local + $(QUIET)cat -b $(TOP)/configure/CONFIG_COMMON.local + $(QUIET)echo ">>> Generating $(TOP)/configure/CONFIG_SITE.local" + $(QUIET)-rm -f $(TOP)/configure/CONFIG_SITE.local + $(QUIET)echo "USERID:=${USER}" > $(TOP)/configure/CONFIG_SITE.local + $(QUIET)echo "GROUPID:=admin" >> $(TOP)/configure/CONFIG_SITE.local + $(QUIET)cat -b $(TOP)/configure/CONFIG_SITE.local + diff --git a/configure/RULES_SRC b/configure/RULES_SRC index dcb6b3f..8c7d290 100644 --- a/configure/RULES_SRC +++ b/configure/RULES_SRC @@ -50,7 +50,7 @@ endif ## Build Source build: - $(QUIET) mvn $(CF_MVN_OPTS) -f $(CF_SRC_PATH)/pom.xml clean install -Dmaven.test.skip=true + $(QUIET) JAVA_HOME=$(JAVA_HOME) $(MAVEN_CMD) $(CF_MVN_OPTS) -f $(CF_SRC_PATH)/pom.xml clean install -Dmaven.test.skip=true ## Copy the site specific files into sources conf: conf.cfproperties From 20dd3c7fa5e41bafc6efefc25013ba9e98049f82 Mon Sep 17 00:00:00 2001 From: Jeong Han Lee Date: Sat, 17 Sep 2022 18:41:47 -0700 Subject: [PATCH 15/89] update mvn path for github --- configure/RULES_CI | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/configure/RULES_CI b/configure/RULES_CI index f2da7ac..6f326d8 100644 --- a/configure/RULES_CI +++ b/configure/RULES_CI @@ -7,8 +7,8 @@ github.conf: $(QUIET)-rm -f $(TOP)/configure/CONFIG_COMMON.local $(QUIET)echo "JAVA_HOME:=${JAVA_HOME}" > $(TOP)/configure/CONFIG_COMMON.local $(QUIET)echo "JAVA_PATH:=${JAVA_HOME}/bin" >> $(TOP)/configure/CONFIG_COMMON.local - $(QUIET)echo "MAVEN_HOME:=/usr/share/apache-maven-3.8.6" >> $(TOP)/configure/CONFIG_COMMON.local - $(QUIET)echo "MAVEN_PATH:=/usr/share/apache-maven-3.8.6/bin" >> $(TOP)/configure/CONFIG_COMMON.local + $(QUIET)echo "MAVEN_HOME:=/usr/" >> $(TOP)/configure/CONFIG_COMMON.local + $(QUIET)echo "MAVEN_PATH:=/usr/bin" >> $(TOP)/configure/CONFIG_COMMON.local $(QUIET)cat -b $(TOP)/configure/CONFIG_COMMON.local rocky8.conf: From 12920992898c8ed97d922596cdd51d28fe30f267 Mon Sep 17 00:00:00 2001 From: Jeong Han Lee Date: Sat, 17 Sep 2022 18:51:01 -0700 Subject: [PATCH 16/89] fixed typo in maven --- configure/CONFIG_SRC | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/configure/CONFIG_SRC b/configure/CONFIG_SRC index aa51d75..0c421fa 100644 --- a/configure/CONFIG_SRC +++ b/configure/CONFIG_SRC @@ -15,8 +15,8 @@ endif PATH:=$(MAVEN_PATH):$(JAVA_PATH):${PATH} -MAVEN_CMD:=$(MAVE_PATH)/mvn -JAVA_CMD:=$(JAVA_PATH)/java +MAVEN_CMD=$(MAVEN_PATH)/mvn +JAVA_CMD=$(JAVA_PATH)/java CF_SITE_SPECIFIC_FILES_PATH:=$(CF_SRC_PATH)/src/main/resources From 775627bfbb86f3406ad2ab75ab80f6bdaadd04bd Mon Sep 17 00:00:00 2001 From: Jeong Han Lee Date: Sat, 17 Sep 2022 18:55:28 -0700 Subject: [PATCH 17/89] github action with java --- .github/workflows/ubuntu.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ubuntu.yml b/.github/workflows/ubuntu.yml index 9314bdc..10cff19 100644 --- a/.github/workflows/ubuntu.yml +++ b/.github/workflows/ubuntu.yml @@ -28,8 +28,11 @@ jobs: which mvn - run: | make init + echo "JAVA_HOME:=${JAVA_HOME}" > ./configure/CONFIG_COMMON.local + echo "JAVA_PATH:=${JAVA_HOME}/bin" >> ./configure/CONFIG_COMMON.local + echo "MAVEN_HOME:=/usr/" >> ./configure/CONFIG_COMMON.local + echo "MAVEN_PATH:=/usr/bin" >> ./configure/CONFIG_COMMON.local make vars - make github.conf make conf - run: | make build From dad6aad3c45d3b720e4093997ed6efc1a8507464 Mon Sep 17 00:00:00 2001 From: Jeong Han Lee Date: Sat, 17 Sep 2022 19:09:28 -0700 Subject: [PATCH 18/89] add rocky8 and debian11 --- .github/workflows/debian11.yml | 51 ++++++++++++++++++++++++++++++++++ .github/workflows/rocky8.yml | 50 +++++++++++++++++++++++++++++++++ .github/workflows/ubuntu.yml | 18 ++++++++---- configure/RULES_CI | 9 ------ 4 files changed, 114 insertions(+), 14 deletions(-) create mode 100644 .github/workflows/debian11.yml create mode 100644 .github/workflows/rocky8.yml diff --git a/.github/workflows/debian11.yml b/.github/workflows/debian11.yml new file mode 100644 index 0000000..a420f6a --- /dev/null +++ b/.github/workflows/debian11.yml @@ -0,0 +1,51 @@ +--- +name: Debian 11 + +on: + push: + branches: [ master ] + paths-ignores: + - '*.md' + - 'examples/*.bash' + - 'docker/**' + - '.github/workflows/docker.yml' + - '.github/workflows/ubuntu.yml' + - '.github/workflows/rocky8.yml' + + pull_request: + branches: [ master ] + +jobs: + Debian11: + + runs-on: ubuntu-latest + container: debian:bullseye + + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-java@v3 + with: + distribution: 'zulu' + java-version: '18.0.2' + - name: Install required packages + run: apt install -y git make sudo tree + - name: Check JAVA configuration + run: | + which java + which mvn + - name: Configuration + run: | + make init + echo "JAVA_HOME:=${JAVA_HOME}" > ./configure/CONFIG_COMMON.local + echo "JAVA_PATH:=${JAVA_HOME}/bin" >> ./configure/CONFIG_COMMON.local + echo "MAVEN_HOME:=/usr/" >> ./configure/CONFIG_COMMON.local + echo "MAVEN_PATH:=/usr/bin" >> ./configure/CONFIG_COMMON.local + make vars + make conf + - name: Build + run: | + make build + make install + - name: Environment Check + run: make exist + diff --git a/.github/workflows/rocky8.yml b/.github/workflows/rocky8.yml new file mode 100644 index 0000000..db29a44 --- /dev/null +++ b/.github/workflows/rocky8.yml @@ -0,0 +1,50 @@ +--- +name: Rocky8 + +on: + push: + branches: [ master ] + paths-ignores: + - '*.md' + - 'examples/*.bash' + - 'docker/**' + - '.github/workflows/ubuntu.yml' + - '.github/workflows/dockyer.yml' + + pull_request: + branches: [ master ] + +jobs: + Rocky8: + + runs-on: ubuntu-latest + container: rockylinux/rockylinux:8 + + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-java@v3 + with: + distribution: 'zulu' + java-version: '18.0.2' + - name: Install required packages + run: dnf install -y git make sudo which + - name: Check JAVA configuration + run: | + which java + which mvn + - name: Configuration + run: | + make init + echo "JAVA_HOME:=${JAVA_HOME}" > ./configure/CONFIG_COMMON.local + echo "JAVA_PATH:=${JAVA_HOME}/bin" >> ./configure/CONFIG_COMMON.local + echo "MAVEN_HOME:=/usr/" >> ./configure/CONFIG_COMMON.local + echo "MAVEN_PATH:=/usr/bin" >> ./configure/CONFIG_COMMON.local + make vars + make conf + - name: Build + run: | + make build + make install + - name: Environment Check + run: make exist + diff --git a/.github/workflows/ubuntu.yml b/.github/workflows/ubuntu.yml index 10cff19..0983f67 100644 --- a/.github/workflows/ubuntu.yml +++ b/.github/workflows/ubuntu.yml @@ -7,6 +7,9 @@ on: paths-ignores: - '*.md' - 'examples/*.bash' + - 'docker/**' + - '.github/workflows/rocky8.yml' + - '.github/workflows/docker.yml' pull_request: branches: [ master ] @@ -22,11 +25,14 @@ jobs: with: distribution: 'zulu' java-version: '18.0.2' - - run: sudo apt-get install make tree git - - run: | + - name: Install required packages + run: sudo apt install -y make tree git + - name: Check JAVA configuration + run: | which java which mvn - - run: | + - name: Configuration + run: | make init echo "JAVA_HOME:=${JAVA_HOME}" > ./configure/CONFIG_COMMON.local echo "JAVA_PATH:=${JAVA_HOME}/bin" >> ./configure/CONFIG_COMMON.local @@ -34,8 +40,10 @@ jobs: echo "MAVEN_PATH:=/usr/bin" >> ./configure/CONFIG_COMMON.local make vars make conf - - run: | + - name: Build + run: | make build make install - - run: make exist + - name: Environment Check + run: make exist diff --git a/configure/RULES_CI b/configure/RULES_CI index 6f326d8..ab0108e 100644 --- a/configure/RULES_CI +++ b/configure/RULES_CI @@ -2,15 +2,6 @@ centos7.conf: rocky8.conf -github.conf: - $(QUIET)echo ">>> Generating $(TOP)/configure/CONFIG_COMMON.local" - $(QUIET)-rm -f $(TOP)/configure/CONFIG_COMMON.local - $(QUIET)echo "JAVA_HOME:=${JAVA_HOME}" > $(TOP)/configure/CONFIG_COMMON.local - $(QUIET)echo "JAVA_PATH:=${JAVA_HOME}/bin" >> $(TOP)/configure/CONFIG_COMMON.local - $(QUIET)echo "MAVEN_HOME:=/usr/" >> $(TOP)/configure/CONFIG_COMMON.local - $(QUIET)echo "MAVEN_PATH:=/usr/bin" >> $(TOP)/configure/CONFIG_COMMON.local - $(QUIET)cat -b $(TOP)/configure/CONFIG_COMMON.local - rocky8.conf: $(QUIET)echo ">>> Generating $(TOP)/configure/CONFIG_COMMON.local" $(QUIET)-rm -f $(TOP)/configure/CONFIG_COMMON.local From 54637cd531249ddbd682e5b5b8ed5f71b19a73d2 Mon Sep 17 00:00:00 2001 From: Jeong Han Lee Date: Sat, 17 Sep 2022 19:47:01 -0700 Subject: [PATCH 19/89] add maven --- .github/workflows/debian11.yml | 4 +++- .github/workflows/rocky8.yml | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/debian11.yml b/.github/workflows/debian11.yml index a420f6a..eb2f0df 100644 --- a/.github/workflows/debian11.yml +++ b/.github/workflows/debian11.yml @@ -28,7 +28,9 @@ jobs: distribution: 'zulu' java-version: '18.0.2' - name: Install required packages - run: apt install -y git make sudo tree + run: | + apt update -y + apt install -y git make sudo tree maven - name: Check JAVA configuration run: | which java diff --git a/.github/workflows/rocky8.yml b/.github/workflows/rocky8.yml index db29a44..a19620a 100644 --- a/.github/workflows/rocky8.yml +++ b/.github/workflows/rocky8.yml @@ -27,7 +27,9 @@ jobs: distribution: 'zulu' java-version: '18.0.2' - name: Install required packages - run: dnf install -y git make sudo which + run: | + dnf update -y + dnf install -y git make sudo which maven - name: Check JAVA configuration run: | which java From 591a9644706907f43e2ca33b38aee1e05de5c763 Mon Sep 17 00:00:00 2001 From: Jeong Han Lee Date: Sat, 17 Sep 2022 20:23:15 -0700 Subject: [PATCH 20/89] remove systemd rules from container --- .github/workflows/debian11.yml | 2 +- .github/workflows/rocky8.yml | 2 +- README.md | 7 ++++--- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/.github/workflows/debian11.yml b/.github/workflows/debian11.yml index eb2f0df..2c31088 100644 --- a/.github/workflows/debian11.yml +++ b/.github/workflows/debian11.yml @@ -47,7 +47,7 @@ jobs: - name: Build run: | make build - make install + make cf_install - name: Environment Check run: make exist diff --git a/.github/workflows/rocky8.yml b/.github/workflows/rocky8.yml index a19620a..4cd6280 100644 --- a/.github/workflows/rocky8.yml +++ b/.github/workflows/rocky8.yml @@ -46,7 +46,7 @@ jobs: - name: Build run: | make build - make install + make cf_install - name: Environment Check run: make exist diff --git a/README.md b/README.md index d0d1308..9b88472 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,8 @@ # ChannelFinder-env - -![ChannelFinder-env Build Test](https://github.com/ChannelFinder/ChannelFinder-env/workflows/ChannelFinder-env%20Build%20Test/badge.svg) -![Docker Image CI](https://github.com/ChannelFinder/ChannelFinder-env/workflows/Docker%20Image%20CI/badge.svg) +[![Debian 11](https://github.com/jeonghanlee/ChannelFinder-env/actions/workflows/debian11.yml/badge.svg)](https://github.com/jeonghanlee/ChannelFinder-env/actions/workflows/debian11.yml) +[![Rocky8](https://github.com/jeonghanlee/ChannelFinder-env/actions/workflows/rocky8.yml/badge.svg)](https://github.com/jeonghanlee/ChannelFinder-env/actions/workflows/rocky8.yml) +[![Ubuntu Latest](https://github.com/jeonghanlee/ChannelFinder-env/actions/workflows/ubuntu.yml/badge.svg)](https://github.com/jeonghanlee/ChannelFinder-env/actions/workflows/ubuntu.yml) +[![ChannelFinderService](https://github.com/jeonghanlee/ChannelFinder-env/actions/workflows/docker.yml/badge.svg)](https://github.com/jeonghanlee/ChannelFinder-env/actions/workflows/docker.yml) ![Linter Run](https://github.com/ChannelFinder/ChannelFinder-env/workflows/Linter%20Run/badge.svg) Configuration Environment for ChannelFinderService at From 001c91fccc40ab3b59f3d0877222040ae61cdc9d Mon Sep 17 00:00:00 2001 From: Jeong Han Lee Date: Sat, 17 Sep 2022 20:27:27 -0700 Subject: [PATCH 21/89] add linter --- .github/workflows/linter.yml | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 .github/workflows/linter.yml diff --git a/.github/workflows/linter.yml b/.github/workflows/linter.yml new file mode 100644 index 0000000..2349b01 --- /dev/null +++ b/.github/workflows/linter.yml @@ -0,0 +1,22 @@ +name: Linter Run + +on: + push: + branches: [master] + pull_request: + branches: [master] + +jobs: + build: + name: Lint Code Base + runs-on: ubuntu-latest + + steps: + - name: Checkout Code + uses: actions/checkout@v3 + - name: Lint Code Base + uses: docker://github/super-linter:v4.8.1 + env: + VALIDATE_MD: true + VALIDATE_BASH: true + From fe6a6c4a7c215936cbfa56051884ef82d5aede2a Mon Sep 17 00:00:00 2001 From: Jeong Han Lee Date: Sat, 17 Sep 2022 20:42:00 -0700 Subject: [PATCH 22/89] indepdent github actions, and add shellcheck options --- .github/workflows/debian11.yml | 1 + .github/workflows/rocky8.yml | 4 +++- .github/workflows/ubuntu.yml | 2 ++ site-template/mapping_definitions.bash | 20 +++++++++------- site-template/mapping_definitions_es7x.bash | 26 +++++++++++++-------- 5 files changed, 34 insertions(+), 19 deletions(-) diff --git a/.github/workflows/debian11.yml b/.github/workflows/debian11.yml index 2c31088..d46bc14 100644 --- a/.github/workflows/debian11.yml +++ b/.github/workflows/debian11.yml @@ -11,6 +11,7 @@ on: - '.github/workflows/docker.yml' - '.github/workflows/ubuntu.yml' - '.github/workflows/rocky8.yml' + - '.github/workflows/linter.yml' pull_request: branches: [ master ] diff --git a/.github/workflows/rocky8.yml b/.github/workflows/rocky8.yml index 4cd6280..5fc9795 100644 --- a/.github/workflows/rocky8.yml +++ b/.github/workflows/rocky8.yml @@ -9,7 +9,9 @@ on: - 'examples/*.bash' - 'docker/**' - '.github/workflows/ubuntu.yml' - - '.github/workflows/dockyer.yml' + - '.github/workflows/docker.yml' + - '.github/workflows/ubuntu.yml' + - '.github/workflows/linter.yml' pull_request: branches: [ master ] diff --git a/.github/workflows/ubuntu.yml b/.github/workflows/ubuntu.yml index 0983f67..bf3b477 100644 --- a/.github/workflows/ubuntu.yml +++ b/.github/workflows/ubuntu.yml @@ -8,7 +8,9 @@ on: - '*.md' - 'examples/*.bash' - 'docker/**' + - '.github/workflows/docker.yml' - '.github/workflows/rocky8.yml' + - '.github/workflows/linter.yml' - '.github/workflows/docker.yml' pull_request: diff --git a/site-template/mapping_definitions.bash b/site-template/mapping_definitions.bash index 6d19f47..06ee6e0 100755 --- a/site-template/mapping_definitions.bash +++ b/site-template/mapping_definitions.bash @@ -19,18 +19,20 @@ # date : Thursday, February 20 14:39:35 PST 2020 # version : 0.0.1 -declare -gr SC_SCRIPT="$(realpath "$0")" -declare -gr SC_SCRIPTNAME=${0##*/} -declare -gr SC_TOP="${SC_SCRIPT%/*}" +declare -g SC_SCRIPT; +declare -g SC_TOP; +SC_SCRIPT="$(realpath "$0")"; +SC_TOP="${SC_SCRIPT%/*}" -. ${SC_TOP}/es_host.cfg +# shellcheck disable=SC1090,SC1091 +. "${SC_TOP}"/es_host.cfg #Create the Index print_help "index/mapping" "cf_tags" - -curl -H 'Content-Type: application/json' -XPUT http://${es_host}:${es_port}/cf_tags -d' +# shellcheck disable=SC2153,SC2154 +curl -H 'Content-Type: application/json' -XPUT http://"${es_host}":"${es_port}"/cf_tags -d' { "mappings":{ "cf_tag" : { @@ -47,7 +49,8 @@ curl -H 'Content-Type: application/json' -XPUT http://${es_host}:${es_port}/cf_t }' print_help "index/mapping" "cf_properties" -curl -H 'Content-Type: application/json' -XPUT http://${es_host}:${es_port}/cf_properties -d' +# shellcheck disable=SC2153,SC2154 +curl -H 'Content-Type: application/json' -XPUT http://"${es_host}":"${es_port}"/cf_properties -d' { "mappings":{ "cf_property" : { @@ -64,7 +67,8 @@ curl -H 'Content-Type: application/json' -XPUT http://${es_host}:${es_port}/cf_p }' print_help "index/mapping" "channelfinder" -curl -H 'Content-Type: application/json' -XPUT http://${es_host}:${es_port}/channelfinder -d' +# shellcheck disable=SC2153,SC2154 +curl -H 'Content-Type: application/json' -XPUT http://"${es_host}":"${es_port}"/channelfinder -d' { "mappings":{ "cf_channel" : { diff --git a/site-template/mapping_definitions_es7x.bash b/site-template/mapping_definitions_es7x.bash index 8ff51a6..bc37295 100644 --- a/site-template/mapping_definitions_es7x.bash +++ b/site-template/mapping_definitions_es7x.bash @@ -19,24 +19,29 @@ # date : Wednesday, April 22 23:00:18 PDT 2020 # version : 0.0.2 -declare -gr SC_SCRIPT="$(realpath "$0")" -declare -gr SC_SCRIPTNAME=${0##*/} -declare -gr SC_TOP="${SC_SCRIPT%/*}" +declare -g SC_SCRIPT; +declare -g SC_TOP; +SC_SCRIPT="$(realpath "$0")"; +SC_TOP="${SC_SCRIPT%/*}" -. ${SC_TOP}/es_host.cfg +# shellcheck disable=SC1090,SC1091 +. "${SC_TOP}/es_host.cfg" +# shellcheck disable=SC2153,SC2154 +ES_URL="http://${es_host}:${es_port}" + #Create the Index : three print_help "index" "cf_tags" -curl -XPUT http://${es_host}:${es_port}/cf_tags +curl -XPUT "$ES_URL"/cf_tags print_help "index" "cf_properties" -curl -XPUT http://${es_host}:${es_port}/cf_properties +curl -XPUT "$ES_URL"/cf_properties print_help "index" "channelfinder" -curl -XPUT http://${es_host}:${es_port}/channelfinder +curl -XPUT "$ES_URL"/channelfinder print_help "mapping" "cf_tags" -curl -H 'Content-Type: application/json' -XPUT http://${es_host}:${es_port}/cf_tags/_mapping/cf_tag?include_type_name=true -d' +curl -H 'Content-Type: application/json' -XPUT "${ES_URL}"/cf_tags/_mapping/cf_tag?include_type_name=true -d' { "cf_tag" : { "properties" : { @@ -47,7 +52,8 @@ curl -H 'Content-Type: application/json' -XPUT http://${es_host}:${es_port}/cf_t }' print_help "mapping" "cf_properties" -curl -H 'Content-Type: application/json' -XPUT http://${es_host}:${es_port}/cf_properties/_mapping/cf_property?include_type_name=true -d' + +curl -H 'Content-Type: application/json' -XPUT "${ES_URL}"/cf_properties/_mapping/cf_property?include_type_name=true -d' { "cf_property" : { "properties" : { @@ -58,7 +64,7 @@ curl -H 'Content-Type: application/json' -XPUT http://${es_host}:${es_port}/cf_p }' print_help "mapping" "channelfinder" -curl -H 'Content-Type: application/json' -XPUT http://${es_host}:${es_port}/channelfinder/_mapping/cf_channel?include_type_name=true -d' +curl -H 'Content-Type: application/json' -XPUT "${ES_URL}"/channelfinder/_mapping/cf_channel?include_type_name=true -d' { "cf_channel" : { "properties" : { From 29f40d0770f99a436bcdbb09e51458910ec7c00f Mon Sep 17 00:00:00 2001 From: Jeong Han Lee Date: Sat, 17 Sep 2022 20:50:46 -0700 Subject: [PATCH 23/89] update linter for clean_mapping_definitions.bash, and readme --- README.md | 19 ++---------------- site-template/clean_mapping_definitions.bash | 21 +++++++++++++------- 2 files changed, 16 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index 9b88472..64de221 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ [![Rocky8](https://github.com/jeonghanlee/ChannelFinder-env/actions/workflows/rocky8.yml/badge.svg)](https://github.com/jeonghanlee/ChannelFinder-env/actions/workflows/rocky8.yml) [![Ubuntu Latest](https://github.com/jeonghanlee/ChannelFinder-env/actions/workflows/ubuntu.yml/badge.svg)](https://github.com/jeonghanlee/ChannelFinder-env/actions/workflows/ubuntu.yml) [![ChannelFinderService](https://github.com/jeonghanlee/ChannelFinder-env/actions/workflows/docker.yml/badge.svg)](https://github.com/jeonghanlee/ChannelFinder-env/actions/workflows/docker.yml) -![Linter Run](https://github.com/ChannelFinder/ChannelFinder-env/workflows/Linter%20Run/badge.svg) +[![Linter Run](https://github.com/jeonghanlee/ChannelFinder-env/actions/workflows/linter.yml/badge.svg)](https://github.com/jeonghanlee/ChannelFinder-env/actions/workflows/linter.yml) Configuration Environment for ChannelFinderService at @@ -13,28 +13,13 @@ Configuration Environment for ChannelFinderService at Date: Sat, 17 Sep 2022 21:04:52 -0700 Subject: [PATCH 24/89] add docker v2.0.0 --- .github/workflows/docker.yml | 14 +++++++------- docker/Dockerfile | 13 +++++++++---- 2 files changed, 16 insertions(+), 11 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 9072228..60f0317 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -19,16 +19,16 @@ jobs: DOCKER_FILE: docker/Dockerfile DOCKER_ACCOUNT: alscontrols DOCKER_REPO: channelfinder - DOCKER_TAG: latest + DOCKER_TAG: v2.0.0 steps: - name: checkout - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Docker meta id: meta - uses: docker/metadata-action@v3 + uses: docker/metadata-action@v4 with: # list of Docker images to use as base name for tags images: | @@ -44,20 +44,20 @@ jobs: type=sha - name: Set up QEMU - uses: docker/setup-qemu-action@v1 + uses: docker/setup-qemu-action@v2 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 + uses: docker/setup-buildx-action@v2 - name: Login to DockerHub - uses: docker/login-action@v1 + uses: docker/login-action@v2 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Build and push id: docker_build - uses: docker/build-push-action@v2 + uses: docker/build-push-action@v3 with: context: . file: ${{ github.workspace }}/${{ env.DOCKER_FILE }} diff --git a/docker/Dockerfile b/docker/Dockerfile index 2d2586f..d07d8db 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -7,7 +7,7 @@ # ## BuildEnv docker image size : 672MB -FROM alpine:3.12 AS builder +FROM alpine:3.16 AS builder ENV WorkPath /home/ChannelFinder-env ARG RS_SERVER_PORT=5075 ARG RS_PORT_BROADCAST=5076 @@ -16,12 +16,17 @@ ARG CF_PORT_HTTPS=8443 ARG CF_INSTALL_LOCATION=/opt/channelfinder ENV JAVA_HOME="/usr/lib/jvm/default-jvm/" -RUN apk update && apk add openjdk11 maven git make +RUN apk update && apk add openjdk17 maven git make WORKDIR $WorkPath COPY . . -RUN echo "CF_PORT:=${CF_PORT_HTTP}" > configure/CONFIG_COMMON.local +RUN echo "CF_PORT:=${CF_PORT_HTTP}" > configure/CONFIG_COMMON.local RUN echo "CF_INSTALL_LOCATION:=${CF_INSTALL_LOCATION}" > configure/CONFIG_SITE.local +RUN echo "JAVA_HOME:=/usr/lib/jvm/default-jvm/" >> configure/CONFIG_COMMON.local +RUN echo "JAVA_PATH:=/usr/lib/jvm/default-jvm/bin" >> configure/CONFIG_COMMON.local +RUN echo "MAVEN_HOME:=/usr/" >> configure/CONFIG_COMMON.local +RUN echo "MAVEN_PATH:=/usr//bin" >> configure/CONFIG_COMMON.local + RUN make distclean && \ make init && \ make conf && \ @@ -74,7 +79,7 @@ ENV CF_INSTALL_LOCATION ${CF_INSTALL_LOCATION} ENV TZ ${TZ} RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone ### Install minimal packages for ChannelFinder -RUN apk add --no-cache bash openjdk11-jre tomcat-native tzdata +RUN apk add --no-cache bash openjdk17-jre tomcat-native tzdata chrony WORKDIR ${CF_INSTALL_LOCATION} COPY --from=builder ${CF_INSTALL_LOCATION} . From fcd659ab431b95430aaa3ce6cf02dfb07cc121ad Mon Sep 17 00:00:00 2001 From: Jeong Han Lee Date: Sat, 17 Sep 2022 21:11:56 -0700 Subject: [PATCH 25/89] update docker --- .github/workflows/docker.yml | 2 +- docker/Dockerfile | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 60f0317..2cd52e3 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -1,5 +1,5 @@ # 0. change name -name: ChannelFinderService +name: Docker # change path name on: push: diff --git a/docker/Dockerfile b/docker/Dockerfile index d07d8db..c3c8045 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -37,7 +37,7 @@ RUN make distclean && \ ## Multi-Stages build ## Running docker image size : 240MB -FROM alpine:3.12 +FROM alpine:3.16 LABEL maintainer="Jeong Han Lee " ARG BUILD_DATE @@ -79,7 +79,7 @@ ENV CF_INSTALL_LOCATION ${CF_INSTALL_LOCATION} ENV TZ ${TZ} RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone ### Install minimal packages for ChannelFinder -RUN apk add --no-cache bash openjdk17-jre tomcat-native tzdata chrony +RUN apk add --no-cache bash openjdk17-jre tomcat-native tzdata WORKDIR ${CF_INSTALL_LOCATION} COPY --from=builder ${CF_INSTALL_LOCATION} . From c0f90626b63a228328585c93cdb6f0a8fb3d6cd6 Mon Sep 17 00:00:00 2001 From: Jeong Han Lee Date: Sat, 17 Sep 2022 21:14:53 -0700 Subject: [PATCH 26/89] update README and docker job name --- .github/workflows/docker.yml | 2 +- README.md | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 2cd52e3..42fe2b4 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -12,7 +12,7 @@ on: # jobs: # change job name - debian10: + Docker: runs-on: ubuntu-latest env: # change the following 4 variables diff --git a/README.md b/README.md index 64de221..1773911 100644 --- a/README.md +++ b/README.md @@ -4,6 +4,7 @@ [![Ubuntu Latest](https://github.com/jeonghanlee/ChannelFinder-env/actions/workflows/ubuntu.yml/badge.svg)](https://github.com/jeonghanlee/ChannelFinder-env/actions/workflows/ubuntu.yml) [![ChannelFinderService](https://github.com/jeonghanlee/ChannelFinder-env/actions/workflows/docker.yml/badge.svg)](https://github.com/jeonghanlee/ChannelFinder-env/actions/workflows/docker.yml) [![Linter Run](https://github.com/jeonghanlee/ChannelFinder-env/actions/workflows/linter.yml/badge.svg)](https://github.com/jeonghanlee/ChannelFinder-env/actions/workflows/linter.yml) +[![Docker](https://github.com/jeonghanlee/ChannelFinder-env/actions/workflows/docker.yml/badge.svg)](https://github.com/jeonghanlee/ChannelFinder-env/actions/workflows/docker.yml) Configuration Environment for ChannelFinderService at From cd0c761b4e49c33139c1296319470301bf1efa67 Mon Sep 17 00:00:00 2001 From: Jeong Han Lee Date: Sat, 17 Sep 2022 21:19:39 -0700 Subject: [PATCH 27/89] Update README.md --- README.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/README.md b/README.md index 1773911..9a71498 100644 --- a/README.md +++ b/README.md @@ -4,14 +4,13 @@ [![Ubuntu Latest](https://github.com/jeonghanlee/ChannelFinder-env/actions/workflows/ubuntu.yml/badge.svg)](https://github.com/jeonghanlee/ChannelFinder-env/actions/workflows/ubuntu.yml) [![ChannelFinderService](https://github.com/jeonghanlee/ChannelFinder-env/actions/workflows/docker.yml/badge.svg)](https://github.com/jeonghanlee/ChannelFinder-env/actions/workflows/docker.yml) [![Linter Run](https://github.com/jeonghanlee/ChannelFinder-env/actions/workflows/linter.yml/badge.svg)](https://github.com/jeonghanlee/ChannelFinder-env/actions/workflows/linter.yml) -[![Docker](https://github.com/jeonghanlee/ChannelFinder-env/actions/workflows/docker.yml/badge.svg)](https://github.com/jeonghanlee/ChannelFinder-env/actions/workflows/docker.yml) Configuration Environment for ChannelFinderService at ## Pre-requirement packages ``` -git make sudo which +git make sudo tree maven ``` ## Debian 11 (EOL: 2024-06-01/2026-08-15) From 0483f7f54eeb9e346b54eedbdd573a179357b13e Mon Sep 17 00:00:00 2001 From: Jeong Han Lee Date: Sat, 17 Sep 2022 21:30:00 -0700 Subject: [PATCH 28/89] update Docker.md --- docker/Dockerfile | 5 +++-- docs/Docker.md | 10 +++++----- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index c3c8045..cc2ba9b 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,8 +1,8 @@ # # author : Jeong Han Lee # email : jeonghan.lee@gmail.com -# date : Wednesday, July 1 14:32:24 PDT 2020 -# version : 0.0.2 +# date : Sat Sep 17 21:29:34 PDT 2022 +# version : 0.0.3 # # @@ -16,6 +16,7 @@ ARG CF_PORT_HTTPS=8443 ARG CF_INSTALL_LOCATION=/opt/channelfinder ENV JAVA_HOME="/usr/lib/jvm/default-jvm/" +# We can check the package availbilities through docker run -t -i --entrypoint /bin/sh alpine:3.16 RUN apk update && apk add openjdk17 maven git make WORKDIR $WorkPath diff --git a/docs/Docker.md b/docs/Docker.md index d58b7d1..ab0b996 100644 --- a/docs/Docker.md +++ b/docs/Docker.md @@ -12,7 +12,7 @@ Note that one will have to log out and to log in to take effect! One should look ## Pull the release image from a registry (hub.docker.com) ```bash -docker pull jeonghanlee/channelfinder:4-v0.1.0 +docker pull alscontrols/channelfinder:v2.0.0 ``` ## Run @@ -20,25 +20,25 @@ docker pull jeonghanlee/channelfinder:4-v0.1.0 ### Run with console outputs ```bash -docker run --network=host --name=channelfinder jeonghanlee/channelfinder:4-v0.1.0 +docker run --network=host --name=channelfinder alscontrols/channelfinder:v2.0.0 ``` ### Run in the detach mode ```bash -docker run --network=host --detach --rm --name=channelfinder jeonghanlee/channelfinder:4-v0.1.0 +docker run --network=host --detach --rm --name=channelfinder alscontrols/channelfinder:v2.0.0 ``` ### Run in order to access the container without the channelfinder service ```bash -docker run -i -t --entrypoint /bin/bash jeonghanlee/channelfinder:4-v0.1.0 +docker run -i -t --entrypoint /bin/bash alscontrols/channelfinder:v2.0.0 ``` ### Run with the local disk mount ```bash -docker run -i -t -v ${HOME}/docker_data:/data --entrypoint /bin/bash jeonghanlee/channelfinder:4-v0.1.0 +docker run -i -t -v ${HOME}/docker_data:/data --entrypoint /bin/bash alscontrols/channelfinder:v2.0.0 ``` , where `${HOME}/docker_data` is the host path, which will be created if it doesn't exist, and `/data` is a volume in the docker container. From 3d0053a6cde2acd29e46cfa4e12af3a96809a6aa Mon Sep 17 00:00:00 2001 From: Jeong Han Lee Date: Sat, 17 Sep 2022 21:35:52 -0700 Subject: [PATCH 29/89] update README --- README.md | 29 ++++++++++++++++++++++------- 1 file changed, 22 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 9a71498..ce8802d 100644 --- a/README.md +++ b/README.md @@ -13,17 +13,32 @@ Configuration Environment for ChannelFinderService at ./configure/CONFIG_COMMON.local + echo "JAVA_PATH:=/usr/bin" >> ./configure/CONFIG_COMMON.local + echo "MAVEN_HOME:=/usr/" >> ./configure/CONFIG_COMMON.local + echo "MAVEN_PATH:=/usr/bin" >> ./configure/CONFIG_COMMON.local +``` -See [docs/README.rocky8.md](docs/README.rocky8.md). +## Support OS [Debian 11 (EOL: 2024-06-01/2026-08-15), Rocky8 (EOL: 2029-05-31)] -## Docker Image +It will works with other systems. Please check github action workflows. -See [docs/Docker.md](docs/Docker.md) :whale: +```bash +make init +make conf +make build +make install +make sd_start +make sd_status +``` + +## Docker Image -## Reference +The Docker image is hosted at https://hub.docker.com/orgs/alscontrols +And for further information, please see [docs/Docker.md](docs/Docker.md) :whale: -[1] From cb3be5359d283aaaea1f83de0263398ed973b7b6 Mon Sep 17 00:00:00 2001 From: Jeong Han Lee Date: Sun, 18 Sep 2022 00:36:34 -0700 Subject: [PATCH 30/89] add SSH KEY and LDIF to the jar --- configure/CONFIG_COMMON | 1 - configure/RULES_CI | 3 --- configure/RULES_PROPERTIES | 9 +++++---- 3 files changed, 5 insertions(+), 8 deletions(-) diff --git a/configure/CONFIG_COMMON b/configure/CONFIG_COMMON index f665398..276d3a5 100644 --- a/configure/CONFIG_COMMON +++ b/configure/CONFIG_COMMON @@ -2,7 +2,6 @@ # Debian JAVA_HOME:=/usr/lib/jvm/default-java MAVEN_HOME:=/usr/share/maven -TOMCAT_HOME:=/usr/share/tomcat9 JAVA_PATH:=$(JAVA_HOME)/bin MAVEN_PATH:=$(MAVEN_HOME)/bin diff --git a/configure/RULES_CI b/configure/RULES_CI index ab0108e..b336a5d 100644 --- a/configure/RULES_CI +++ b/configure/RULES_CI @@ -7,7 +7,6 @@ rocky8.conf: $(QUIET)-rm -f $(TOP)/configure/CONFIG_COMMON.local $(QUIET)echo "JAVA_HOME:=/usr/lib/jvm/jre-11-openjdk" > $(TOP)/configure/CONFIG_COMMON.local $(QUIET)echo "JAVA_PATH:=/usr/lib/jvm/jre-11-openjdk/bin" >> $(TOP)/configure/CONFIG_COMMON.local - $(QUIET)echo "TOMCAT_HOME:=/opt/tomcat9" >> $(TOP)/configure/CONFIG_COMMON.local $(QUIET)cat -b $(TOP)/configure/CONFIG_COMMON.local macos.conf: @@ -17,13 +16,11 @@ macos.conf: $(QUIET)echo "JAVA_PATH:=/Library/Java/JavaVirtualMachines/openjdk17-zulu/Contents/Home/bin" >> $(TOP)/configure/CONFIG_COMMON.local $(QUIET)echo "ANT_HOME:=/opt/local/share/java/apache-ant" >> $(TOP)/configure/CONFIG_COMMON.local $(QUIET)echo "ANT_PATH:=/opt/local/share/java/apache-ant/bin" >> $(TOP)/configure/CONFIG_COMMON.local - $(QUIET)echo "TOMCAT_HOME:=/opt/tomcat9" >> $(TOP)/configure/CONFIG_COMMON.local $(QUIET)cat -b $(TOP)/configure/CONFIG_COMMON.local $(QUIET)echo ">>> Generating $(TOP)/configure/CONFIG_SITE.local" $(QUIET)-rm -f $(TOP)/configure/CONFIG_SITE.local $(QUIET)echo "USERID:=${USER}" > $(TOP)/configure/CONFIG_SITE.local $(QUIET)echo "GROUPID:=20" >> $(TOP)/configure/CONFIG_SITE.local - $(QUIET)echo "ARCHAPPL_LONG_TERM_FOLDER=$(ARCHAPPL_STORAGE_TOP)/lts/ArchiverStore" >> $(TOP)/configure/CONFIG_SITE.local $(QUIET)cat -b $(TOP)/configure/CONFIG_SITE.local diff --git a/configure/RULES_PROPERTIES b/configure/RULES_PROPERTIES index 82a8283..14a39ff 100644 --- a/configure/RULES_PROPERTIES +++ b/configure/RULES_PROPERTIES @@ -22,10 +22,11 @@ conf.properties: $(CF_PROPERTIES).in < $< > $(basename $<) rm -f $(CF_SITE_SPECIFIC_FILES_PATH)/$(notdir $(basename $<)) $(QUIET)echo ">>> Copy $(basename $<) to $(CF_SITE_SPECIFIC_FILES_PATH)" - $(QUIET)$(INSTALL_DATA) $(basename $<) $(CF_SITE_SPECIFIC_FILES_PATH)/ -# $(QUIET)cp $(CF_SITE_TEMPLATE_PATH)/$(CF_LDIF) $(CF_SITE_SPECIFIC_FILES_PATH)/ -# $(QUIET)cp $(CF_SITE_TEMPLATE_PATH)/$(CF_SSHKEY) $(CF_SITE_SPECIFIC_FILES_PATH)/keystore/ -# $(QUIET)cp $(CF_SITE_TEMPLATE_PATH)/application.properties $(CF_SITE_SPECIFIC_FILES_PATH)/ + $(INSTALL_DATA) $(basename $<) $(CF_SITE_SPECIFIC_FILES_PATH)/ + rm -rf $(CF_SITE_SPECIFIC_FILES_PATH)/$(CF_LDIF) + $(INSTALL_DATA) $(CF_SITE_TEMPLATE_PATH)/$(CF_LDIF) $(CF_SITE_SPECIFIC_FILES_PATH)/ + rm -rf $(CF_SITE_SPECIFIC_FILES_PATH)/keystore/$(CF_SSHKEY) + $(INSTALL_DATA) $(CF_SITE_TEMPLATE_PATH)/$(CF_SSHKEY) $(CF_SITE_SPECIFIC_FILES_PATH)/keystore/ conf.properties.show: $(CF_PROPERTIES).in $(QUIET)cat -b $(basename $<) From 49a35dbc5c1aa3df253af9042bbf19740b9b41e8 Mon Sep 17 00:00:00 2001 From: Jeong Han Lee Date: Sun, 18 Sep 2022 00:46:47 -0700 Subject: [PATCH 31/89] update docker.yml --- .github/workflows/docker.yml | 44 ++++++++++++++---------------------- 1 file changed, 17 insertions(+), 27 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 42fe2b4..76fb3a2 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -1,32 +1,27 @@ +--- # 0. change name name: Docker # change path name on: push: - branches: [ master ] + branches: [master] paths: - - 'docker/**' - - '.github/workflows/docker.yml' + - 'docker/**' + - '.github/workflows/docker.yml' pull_request: - branches: [ master ] -# + branches: [master] jobs: -# change job name Docker: runs-on: ubuntu-latest env: - # change the following 4 variables - DOCKER_FILE: docker/Dockerfile - DOCKER_ACCOUNT: alscontrols - DOCKER_REPO: channelfinder - DOCKER_TAG: v2.0.0 - + DOCKER_FILE: docker/Dockerfile + DOCKER_ACCOUNT: alscontrols + DOCKER_REPO: channelfinder + DOCKER_TAG: v2.0.0 steps: - - - name: checkout + - name: checkout uses: actions/checkout@v3 - - - name: Docker meta + - name: Docker meta id: meta uses: docker/metadata-action@v4 with: @@ -42,27 +37,22 @@ jobs: type=semver,pattern={{major}}.{{minor}} type=semver,pattern={{major}} type=sha - - - name: Set up QEMU + - name: Set up QEMU uses: docker/setup-qemu-action@v2 - - - name: Set up Docker Buildx + - name: Set up Docker Buildx uses: docker/setup-buildx-action@v2 - - - name: Login to DockerHub + - name: Login to DockerHub uses: docker/login-action@v2 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - - - name: Build and push + - name: Build and push id: docker_build uses: docker/build-push-action@v3 with: context: . file: ${{ github.workspace }}/${{ env.DOCKER_FILE }} push: ${{ github.event_name != 'pull_request' }} - #tags: ${{ steps.meta.outputs.tags }} - tags: ${{ env.DOCKER_ACCOUNT }}/${{ env.DOCKER_REPO }}:${{ env.DOCKER_TAG }} + tags: + ${{env.DOCKER_ACCOUNT}}/${{env.DOCKER_REPO}}:${{env.DOCKER_TAG} labels: ${{ steps.meta.outputs.labels }} - From 3f16370e809659bed19712eac50c598af714cac3 Mon Sep 17 00:00:00 2001 From: Jeong Han Lee Date: Sun, 18 Sep 2022 10:15:50 -0700 Subject: [PATCH 32/89] Update docker.yml --- .github/workflows/docker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 76fb3a2..a3dba12 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -54,5 +54,5 @@ jobs: file: ${{ github.workspace }}/${{ env.DOCKER_FILE }} push: ${{ github.event_name != 'pull_request' }} tags: - ${{env.DOCKER_ACCOUNT}}/${{env.DOCKER_REPO}}:${{env.DOCKER_TAG} + ${{env.DOCKER_ACCOUNT}}/${{env.DOCKER_REPO}}:${{env.DOCKER_TAG}} labels: ${{ steps.meta.outputs.labels }} From 936d94a8187382f419648b3806f11b210fd6ef11 Mon Sep 17 00:00:00 2001 From: Jeong Han Lee Date: Mon, 19 Sep 2022 13:00:54 -0700 Subject: [PATCH 33/89] refactoring systemd --- .gitignore | 1 + configure/CONFIG | 1 + configure/CONFIG_SITE | 5 +- configure/CONFIG_SYSTEMD | 3 + configure/RULES | 7 +- configure/RULES_INSTALL | 33 -------- configure/RULES_SYSTEMD | 75 +++++++++++++++++++ configure/RULES_VARS | 2 + site-template/cf.service.in | 22 ------ .../systemd/channelfinder.service.in | 22 ++++++ 10 files changed, 112 insertions(+), 59 deletions(-) create mode 100644 configure/CONFIG_SYSTEMD create mode 100644 configure/RULES_SYSTEMD delete mode 100644 site-template/cf.service.in create mode 100644 site-template/systemd/channelfinder.service.in diff --git a/.gitignore b/.gitignore index d78d4ae..6c99897 100644 --- a/.gitignore +++ b/.gitignore @@ -42,3 +42,4 @@ site-template/application.properties .*.swp *.swp *.swo +channelfinder.service diff --git a/configure/CONFIG b/configure/CONFIG index 4e029a5..48caf95 100644 --- a/configure/CONFIG +++ b/configure/CONFIG @@ -3,5 +3,6 @@ VARS_EXCLUDES := $(.VARIABLES) include $(TOP)/configure/CONFIG_COMMON include $(TOP)/configure/RELEASE include $(TOP)/configure/CONFIG_SITE +include $(TOP)/configure/CONFIG_SYSTEMD include $(TOP)/configure/CONFIG_VARS include $(TOP)/configure/CONFIG_SRC diff --git a/configure/CONFIG_SITE b/configure/CONFIG_SITE index c6edf62..727eb5d 100644 --- a/configure/CONFIG_SITE +++ b/configure/CONFIG_SITE @@ -13,7 +13,6 @@ CF_JAR_FILENAME:=ChannelFinder-$(CF_SRC_VERSION).jar CF_PROPERTIES:=application.properties -SYSTEMD_PATH:=/etc/systemd/system ### ### These two parameters are used to generate application.properties from application.properties.in ### @@ -40,8 +39,8 @@ CF_SITE_TEMPLATE_PATH=$(TOP)/site-template # Here is the example for the ALS environment # CONFIG_SITE.local can be used to hold this information locally # Each service need one space between them, and backslash must be double backslash -CF_SYSTEMD_SERVICES:= - +CF_SYSTEMD_SERVICES:=elasticsearch.service +CF_KILL_PATH:=/usr/bin # These allow developers to override the CONFIG_SITE variable # settings without having to modify the configure/CONFIG_SITE # file itself. diff --git a/configure/CONFIG_SYSTEMD b/configure/CONFIG_SYSTEMD new file mode 100644 index 0000000..4e12c46 --- /dev/null +++ b/configure/CONFIG_SYSTEMD @@ -0,0 +1,3 @@ +SYSTEMD_PATH:=/etc/systemd/system +CF_SYSTEMD_FILENAME:=channelfinder.service + diff --git a/configure/RULES b/configure/RULES index 69b43f0..629c201 100644 --- a/configure/RULES +++ b/configure/RULES @@ -1,14 +1,19 @@ +vpath %.local $(TOP)/configure vpath %.in $(CF_SITE_TEMPLATE_PATH) vpath %.properties $(CF_SITE_TEMPLATE_PATH) +vpath %.in $(CF_SITE_TEMPLATE_PATH)/systemd +vpath %.services $(CF_SITE_TEMPLATE_PATH)/systemd + # include $(TOP)/configure/RULES_FUNC include $(TOP)/configure/RULES_REQ include $(TOP)/configure/RULES_SRC include $(TOP)/configure/RULES_INSTALL +include $(TOP)/configure/RULES_SYSTEMD +include $(TOP)/configure/RULES_DOCKER include $(TOP)/configure/RULES_PROPERTIES include $(TOP)/configure/RULES_CI -include $(TOP)/configure/RULES_DOCKER include $(TOP)/configure/RULES_VARS diff --git a/configure/RULES_INSTALL b/configure/RULES_INSTALL index 71a0184..220e82f 100644 --- a/configure/RULES_INSTALL +++ b/configure/RULES_INSTALL @@ -1,37 +1,4 @@ -.PHONY: sd_config sd_install sd_status sd_stop sd_clean sd_enable sd_disable - -sd_config: - $(QUIET)sed -e "s:_JAVAPATH_:$(CF_JAVA_PATH):g" -e "s:_JAVAOPTS_:$(CF_JAVA_OPTS):g" -e "s:_CFPATH_:$(CF_INSTALL_LOCATION):g" -e "s:_CHANNELFINDER_JAR_NAME_:$(CF_JAR_FILENAME):g" < $(CF_SITE_TEMPLATE_PATH)/cf.service.in > $(CF_SITE_TEMPLATE_PATH)/channelfinder.service - - -sd_install: sd_config - $(QUIET)$(SUDO) install -b -m 644 $(CF_SITE_TEMPLATE_PATH)/channelfinder.service $(SYSTEMD_PATH)/ - $(QUIET)$(SUDO) systemctl daemon-reload - -sd_status: - $(QUIET) systemctl status -l channelfinder.service | cat -b - -sd_start: - $(QUIET)$(SUDO) systemctl start channelfinder.service - -sd_stop: - $(QUIET)$(SUDO) systemctl stop channelfinder.service - -sd_restart: - $(QUIET)$(SUDO) systemctl restart channelfinder.service - - -sd_clean: - $(QUIET)$(SUDO) systemctl disable channelfinder.service - $(QUIET)$(SUDO) rm -f $(SYSTEMD_PATH)/channelfinder.service - -sd_enable: - $(if $(wildcard $(SYSTEMD_PATH)/channelfinder.service), $(SUDO) systemctl enable channelfinder.service ) - -sd_disable: - $(if $(wildcard $(SYSTEMD_PATH)/channelfinder.service), $(SUDO) systemctl disable channelfinder.service ) - .PHONY: cf_status cf_start cf_stop cf_restart cf_install install uninstall cf_status: sd_status diff --git a/configure/RULES_SYSTEMD b/configure/RULES_SYSTEMD new file mode 100644 index 0000000..4d75e7b --- /dev/null +++ b/configure/RULES_SYSTEMD @@ -0,0 +1,75 @@ +# +# +.PHONY: sd_install sd_status sd_start sd_stop sd_restart sd_clean sd_enable sd_disable +# +# + +systemd_RULES_NAMES:=systemd0 +conf_systemd_RULES:=$(addprefix conf., $(systemd_RULES_NAMES)) +show_systemd_RULES:=$(addsuffix .show, $(conf_systemd_RULES)) +install_systemd_RULES:=$(addprefix install., $(systemd_RULES_NAMES)) +show_install_systemd_RULES:=$(addsuffix .show, $(install_systemd_RULES)) + +sd_install: conf.systemd install.systemd + $(QUIET)$(SUDO) systemctl daemon-reload + +.PHONY: conf.systemd conf.systemd.show + +conf.systemd: $(conf_systemd_RULES) + +conf.systemd.show: $(show_systemd_RULES) + +install.systemd: $(install_systemd_RULES) + +install.systemd.show: $(show_install_systemd_RULES) + +conf.systemd0: $(CF_SYSTEMD_FILENAME).in + $(QUIET)echo ">>> Generate $(basename $<) from $<" + $(QUIET)echo ">>> Required Services : $(CF_SYSTEMD_SERVICES)" + $(QUIET)sed -e "s|@DOCURL@|$(DOCURL)|g" \ + -e 's|@SYSTEMD_SERVICES@|$(CF_SYSTEMD_SERVICES)|g' \ + -e "s|@CF_JAVA_PATH@|$(CF_JAVA_PATH)|g" \ + -e "s|@CF_JAVA_OPTS@|$(CF_JAVA_OPTS)|g" \ + -e "s|@CF_INSTALL_LOCATION@|$(CF_INSTALL_LOCATION)|g" \ + -e "s|@CF_KILL_PATH@|$(CF_KILL_PATH)|g" \ + -e "s|@CF_JAR_NAME@|$(CF_JAR_FILENAME)|g" \ + < $< > $(basename $<) + +conf.systemd0.show: $(CF_SYSTEMD_FILENAME).in + @cat -b $(basename $<) + +install.systemd0: $(CF_SYSTEMD_FILENAME).in + $(QUIET)$(SUDO) $(INSTALL_DATA) -b $(basename $<) $(SYSTEMD_PATH)/ + +install.systemd0.show: + $(QUIET)cat -b $(SYSTEMD_PATH)/$(CF_SYSTEMD_FILENAME) +# +# +sd_status: + $(QUIET) systemctl status -l $(CF_SYSTEMD_FILENAME) | cat -b + +# +sd_start: + $(QUIET)$(SUDO) systemctl start $(CF_SYSTEMD_FILENAME) +# +# We ignore its error +sd_stop: + -$(QUIET)$(SUDO) systemctl stop $(CF_SYSTEMD_FILENAME) +# +# +sd_restart: + $(QUIET)$(SUDO) systemctl restart $(CF_SYSTEMD_FILENAME) +# +# We ignore its error +sd_clean: + $(if $(wildcard $(SYSTEMD_PATH)/$(CF_SYSTEMD_FILENAME)), $(QUIET)$(SUDO) rm -f $(SYSTEMD_PATH)/$(CF_SYSTEMD_FILENAME)) +# +# +sd_enable: + $(if $(wildcard $(SYSTEMD_PATH)/$(CF_SYSTEMD_FILENAME)), $(SUDO) systemctl enable $(CF_SYSTEMD_FILENAME)) +# +# +sd_disable: + $(if $(wildcard $(SYSTEMD_PATH)/$(CF_SYSTEMD_FILENAME)), $(SUDO) systemctl disable $(CF_SYSTEMD_FILENAME)) +# +# diff --git a/configure/RULES_VARS b/configure/RULES_VARS index 436d19e..397a6b1 100644 --- a/configure/RULES_VARS +++ b/configure/RULES_VARS @@ -9,6 +9,8 @@ VARS_EXCLUDES+=$(filter JAR_%, $(.VARIABLES)) VARS_EXCLUDES+=$(RULES_VARS) VARS_EXCLUDES+=%_RULES_VARS VARS_EXCLUDES+=a_service_BUIDER buildrules_BUILDER +VARS_EXCLUDES+=RULES_VARS +VARS_EXCLUDES+=LSOPTS .PHONY: $(RULES_VARS) diff --git a/site-template/cf.service.in b/site-template/cf.service.in deleted file mode 100644 index 976c928..0000000 --- a/site-template/cf.service.in +++ /dev/null @@ -1,22 +0,0 @@ -[Unit] -Description=ChannelFinder Service -Documentation=https://github.com/ChannelFinder/ChannelFinder-SpringBoot -After=network.target elasticsearch.service -Requires=elasticsearch.service - -[Service] -# User=elasticsearch -# Group=elasticsearch - -ExecStart=_JAVAPATH_/java _JAVAOPTS_ -jar _CFPATH_/_CHANNELFINDER_JAR_NAME_ -SuccessExitStatus=143 - -ExecReload=/usr/bin/kill -SIGINT $MAINPID -KillMode=process - -Restart=on-failure -RestartSec=10s - -[Install] -WantedBy=multi-user.target -Alias=cf.service diff --git a/site-template/systemd/channelfinder.service.in b/site-template/systemd/channelfinder.service.in new file mode 100644 index 0000000..32a6e21 --- /dev/null +++ b/site-template/systemd/channelfinder.service.in @@ -0,0 +1,22 @@ +[Unit] +Description=ChannelFinder Service +Documentation=@DOCURL@ +After=syslog.target network.target @SYSTEMD_SERVICES@ +Requires=@SYSTEMD_SERVICES@ + +[Service] +# User=elasticsearch +# Group=elasticsearch + +ExecStart=@CF_JAVA_PATH@/java @CF_JAVA_OPTS@ -jar @CF_INSTALL_LOCATION@/@CF_JAR_NAME@ +SuccessExitStatus=143 + +ExecReload=@CF_KILL_PATH@/kill -SIGINT $MAINPID +KillMode=process + +Restart=on-failure +RestartSec=10s + +[Install] +WantedBy=multi-user.target +Alias=cf.service From 00cefcd21044ebe7914b663e62ce87cbedccef15 Mon Sep 17 00:00:00 2001 From: Jeong Han Lee Date: Mon, 19 Sep 2022 17:37:33 -0700 Subject: [PATCH 34/89] system is integrated, but elasticsearch and cf do not run properly --- README.md | 8 +++++++ configure/RELEASE | 2 +- configure/RULES_REQ | 54 +++++++++++++++++++++++++++++++++++---------- 3 files changed, 51 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index ce8802d..6f85d8c 100644 --- a/README.md +++ b/README.md @@ -13,6 +13,14 @@ Configuration Environment for ChannelFinderService at Date: Tue, 20 Sep 2022 22:48:32 -0700 Subject: [PATCH 35/89] JDK 17 + ES 8.2.0, Indexing, and mapping, running on macOS --- .gitignore | 1 + README.md | 2 + configure/CONFIG_SITE | 5 +- configure/RULES_CI | 48 ++---------------- configure/RULES_INSTALL | 10 ++-- configure/RULES_PROPERTIES | 5 ++ configure/RULES_REQ | 40 +++++++++------ configure/RULES_VARS | 3 +- site-template/ChannelFinderConf.md | 12 +++++ site-template/application.properties.in | 30 ++++++----- site-template/clean_mapping_definitions.bash | 28 ----------- site-template/create_indexes.bash | 52 ++++++++++++++++++++ site-template/delete_indexes.bash | 35 +++++++++++++ 13 files changed, 169 insertions(+), 102 deletions(-) create mode 100644 site-template/ChannelFinderConf.md delete mode 100644 site-template/clean_mapping_definitions.bash create mode 100644 site-template/create_indexes.bash create mode 100644 site-template/delete_indexes.bash diff --git a/.gitignore b/.gitignore index 6c99897..f6201e6 100644 --- a/.gitignore +++ b/.gitignore @@ -43,3 +43,4 @@ site-template/application.properties *.swp *.swo channelfinder.service +elasticsearch-8.2.0 diff --git a/README.md b/README.md index 6f85d8c..d6c314d 100644 --- a/README.md +++ b/README.md @@ -43,6 +43,8 @@ make build make install make sd_start make sd_status +make mapping +make mapping.clean ``` ## Docker Image diff --git a/configure/CONFIG_SITE b/configure/CONFIG_SITE index 727eb5d..751dac8 100644 --- a/configure/CONFIG_SITE +++ b/configure/CONFIG_SITE @@ -28,8 +28,11 @@ CF_JAVA_OPTS:=-Xms512m -Xmx512m CF_MVN_OPTS:= - +CF_LOG_PATH:=$(CF_INSTALL_LOCATION) +CF_LOG_FILE:=channelfinder.log +CF_LOG_LEVEL:=WARN CF_SITE_TEMPLATE_PATH=$(TOP)/site-template +CF_CREATE_INDEX:=false # SYSTEDM Required and After Service List # This is After and Required services for the systemd archiver appliance systemd service. diff --git a/configure/RULES_CI b/configure/RULES_CI index b336a5d..14d7328 100644 --- a/configure/RULES_CI +++ b/configure/RULES_CI @@ -1,56 +1,18 @@ -.PHONY: centos8.conf rocky8.conf centos7.conf +.PHONY: conf.macos conf.macbrew -centos7.conf: rocky8.conf - -rocky8.conf: - $(QUIET)echo ">>> Generating $(TOP)/configure/CONFIG_COMMON.local" - $(QUIET)-rm -f $(TOP)/configure/CONFIG_COMMON.local - $(QUIET)echo "JAVA_HOME:=/usr/lib/jvm/jre-11-openjdk" > $(TOP)/configure/CONFIG_COMMON.local - $(QUIET)echo "JAVA_PATH:=/usr/lib/jvm/jre-11-openjdk/bin" >> $(TOP)/configure/CONFIG_COMMON.local - $(QUIET)cat -b $(TOP)/configure/CONFIG_COMMON.local - -macos.conf: - $(QUIET)echo ">>> Generating $(TOP)/configure/CONFIG_COMMON.local" - $(QUIET)-rm -f $(TOP)/configure/CONFIG_COMMON.local - $(QUIET)echo "JAVA_HOME:=/Library/Java/JavaVirtualMachines/openjdk17-zulu/Contents/Home" > $(TOP)/configure/CONFIG_COMMON.local - $(QUIET)echo "JAVA_PATH:=/Library/Java/JavaVirtualMachines/openjdk17-zulu/Contents/Home/bin" >> $(TOP)/configure/CONFIG_COMMON.local - $(QUIET)echo "ANT_HOME:=/opt/local/share/java/apache-ant" >> $(TOP)/configure/CONFIG_COMMON.local - $(QUIET)echo "ANT_PATH:=/opt/local/share/java/apache-ant/bin" >> $(TOP)/configure/CONFIG_COMMON.local - $(QUIET)cat -b $(TOP)/configure/CONFIG_COMMON.local - $(QUIET)echo ">>> Generating $(TOP)/configure/CONFIG_SITE.local" - $(QUIET)-rm -f $(TOP)/configure/CONFIG_SITE.local - $(QUIET)echo "USERID:=${USER}" > $(TOP)/configure/CONFIG_SITE.local - $(QUIET)echo "GROUPID:=20" >> $(TOP)/configure/CONFIG_SITE.local - $(QUIET)cat -b $(TOP)/configure/CONFIG_SITE.local +conf.macos: conf.macbrew -macos.conf: macbrew.conf - -macbrew.conf: +conf.macbrew: $(QUIET)echo ">>> Generating $(TOP)/configure/CONFIG_COMMON.local" $(QUIET)-rm -f $(TOP)/configure/CONFIG_COMMON.local $(QUIET)echo "JAVA_HOME:=/opt/homebrew/opt/openjdk" > $(TOP)/configure/CONFIG_COMMON.local $(QUIET)echo "JAVA_PATH:=/opt/homebrew/opt/openjdk/bin" >> $(TOP)/configure/CONFIG_COMMON.local - $(QUIET)echo "ANT_HOME:=/opt/homebrew/opt/ant" >> $(TOP)/configure/CONFIG_COMMON.local - $(QUIET)echo "ANT_PATH:=/opt/homebrew/opt/ant/bin" >> $(TOP)/configure/CONFIG_COMMON.local + $(QUIET)echo "MAVEN_HOME:=/opt/homebrew/opt/maven" >> $(TOP)/configure/CONFIG_COMMON.local + $(QUIET)echo "MAVEN_PATH:=/opt/homebrew/opt/maven/bin" >> $(TOP)/configure/CONFIG_COMMON.local $(QUIET)cat -b $(TOP)/configure/CONFIG_COMMON.local $(QUIET)echo ">>> Generating $(TOP)/configure/CONFIG_SITE.local" $(QUIET)-rm -f $(TOP)/configure/CONFIG_SITE.local $(QUIET)echo "USERID:=${USER}" > $(TOP)/configure/CONFIG_SITE.local $(QUIET)echo "GROUPID:=admin" >> $(TOP)/configure/CONFIG_SITE.local $(QUIET)cat -b $(TOP)/configure/CONFIG_SITE.local - -githubmac.conf: - $(QUIET)echo ">>> Generating $(TOP)/configure/CONFIG_COMMON.local" - $(QUIET)-rm -f $(TOP)/configure/CONFIG_COMMON.local - $(QUIET)echo "JAVA_HOME:=/usr/local/opt/openjdk@17" > $(TOP)/configure/CONFIG_COMMON.local - $(QUIET)echo "JAVA_PATH:=/usr/local/opt/openjdk@17/bin" >> $(TOP)/configure/CONFIG_COMMON.local - $(QUIET)echo "ANT_HOME:=/usr/local/Cellar/ant/1.10.12" >> $(TOP)/configure/CONFIG_COMMON.local - $(QUIET)echo "ANT_PATH:=/usr/local/Cellar/ant/1.10.12/bin" >> $(TOP)/configure/CONFIG_COMMON.local - $(QUIET)cat -b $(TOP)/configure/CONFIG_COMMON.local - $(QUIET)echo ">>> Generating $(TOP)/configure/CONFIG_SITE.local" - $(QUIET)-rm -f $(TOP)/configure/CONFIG_SITE.local - $(QUIET)echo "USERID:=${USER}" > $(TOP)/configure/CONFIG_SITE.local - $(QUIET)echo "GROUPID:=admin" >> $(TOP)/configure/CONFIG_SITE.local - $(QUIET)cat -b $(TOP)/configure/CONFIG_SITE.local - diff --git a/configure/RULES_INSTALL b/configure/RULES_INSTALL index 220e82f..bdf0be1 100644 --- a/configure/RULES_INSTALL +++ b/configure/RULES_INSTALL @@ -1,5 +1,5 @@ -.PHONY: cf_status cf_start cf_stop cf_restart cf_install install uninstall +.PHONY: cf_status cf_start cf_stop cf_restart cf_install src_install install uninstall cf_status: sd_status @@ -9,14 +9,15 @@ cf_stop: sd_stop cf_restart: sd_restart - cf_install: $(QUIET)$(SUDO) install -d $(CF_INSTALL_LOCATION) $(QUIET)$(SUDO) install -m 744 $(CF_SRC_PATH)/target/$(CF_JAR_FILENAME) $(CF_INSTALL_LOCATION)/ +src_install: cf_install + ## Install Service related files -install: cf_install sd_install sd_enable +install: src_install sd_install sd_enable $(QUIET)echo "----- Note that one should start it and check its status via " $(QUIET)echo "----- systemctl start channelfinder" $(QUIET)echo "----- systemctl status channelfinder" @@ -31,3 +32,6 @@ uninstall: sd_stop sd_disable sd_clean reinstall: conf build install restart: uninstall reinstall cf_start cf_status + +start.local: + $(QUIET)mvn -f $(CF_SRC_PATH) spring-boot:run diff --git a/configure/RULES_PROPERTIES b/configure/RULES_PROPERTIES index 14a39ff..cdb3b7d 100644 --- a/configure/RULES_PROPERTIES +++ b/configure/RULES_PROPERTIES @@ -19,6 +19,11 @@ conf.properties: $(CF_PROPERTIES).in -e "s|@ES_HOST@|$(ES_HOST)|g" \ -e "s|@ES_PORT@|$(ES_PORT)|g" \ -e "s|@CF_QUERY_SIZE@|$(CF_QUERY_SIZE)|g" \ + -e "s|@CF_LOG_PATH@|$(CF_LOG_PATH)|g" \ + -e "s|@CF_LOG_FILE@|$(CF_LOG_FILE)|g" \ + -e "s|@CF_LOG_LEVEL@|$(CF_LOG_LEVEL)|g" \ + -e "s|@CF_CREATE_INDEX@|$(CF_CREATE_INDEX)|g" \ + -e "s|@CF_VERSION@|$(CF_SRC_VERSION)|g" \ < $< > $(basename $<) rm -f $(CF_SITE_SPECIFIC_FILES_PATH)/$(notdir $(basename $<)) $(QUIET)echo ">>> Copy $(basename $<) to $(CF_SITE_SPECIFIC_FILES_PATH)" diff --git a/configure/RULES_REQ b/configure/RULES_REQ index 57bcfea..b371bd0 100644 --- a/configure/RULES_REQ +++ b/configure/RULES_REQ @@ -1,23 +1,27 @@ -es_RULES_NAMES:=esdeb esrpm +es_RULES_NAMES:=esdeb esrpm esmac install_es_RULES:=$(addprefix install., $(es_RULES_NAMES)) setup_es_RULES:=$(addprefix setup., $(es_RULES_NAMES)) -.PHONY: start.es status.es mapping.es clean.mapping.es all.es stop.es disable.es +.PHONY: start.es status.es all.es stop.es disable.es mapping mapping.clean mapping.settings -#https://www.elastic.co/guide/en/elasticsearch/reference/current/release-notes-8.2.3.html +#https://www.elastic.co/guide/en/elasticsearch/reference/current/release-notes-8.2.0.html ES_NAME:=elasticsearch ES_VER:=8.2.0 -ES_DEB:=$(ES_NAME)-$(ES_VER)-amd64.deb -ES_RPM:=$(ES_NAME)-$(ES_VER)-x86_64.rpm +ES_FILE:=$(ES_NAME)-$(ES_VER) +ES_DEB:=$(ES_FILE)-amd64.deb +ES_RPM:=$(ES_FILE)-x86_64.rpm +ES_MAC:=$(ES_FILE)-darwin-aarch64.tar.gz -# Debian 11 +# Debian 11 setup.esdeb: install.esdeb enable.es setup.esrpm: install.esrpm enable.es +setup.esmac: install.esmac + install.esdeb: $(QUIET) wget -c https://artifacts.elastic.co/downloads/elasticsearch/$(ES_DEB) $(QUIET)$(SUDO) dpkg -i ./$(ES_DEB) @@ -27,6 +31,13 @@ install.esrpm: $(QUIET) wget -c https://artifacts.elastic.co/downloads/elasticsearch/$(ES_RPM) $(QUIET)$(SUDO) dnf localinstall $(ES_DEB) + +# macOS aarch64 +install.esmac: + $(QUIET) curl -O https://artifacts.elastic.co/downloads/elasticsearch/$(ES_MAC) + $(QUIET) curl https://artifacts.elastic.co/downloads/elasticsearch/$(ES_MAC).sha512 | shasum -a 512 -c - + $(QUIET) tar -xzf $(ES_MAC) + enable.es: $(QUIET)$(SUDO) systemctl daemon-reload $(QUIET)$(SUDO) systemctl enable elasticsearch.service @@ -44,16 +55,17 @@ disable.es: $(QUIET)systemctl disable elasticsearch ## Create the ES indexes and set up their mapping -mapping.es: - $(QUIET)bash $(CF_SITE_TEMPLATE_PATH)/mapping_definitions.bash - $(QUIET)curl -XPUT "http://$(ES_HOST):$(ES_PORT)/channelfinder/_settings" -d '{ "index" : { "max_result_window" : $(CF_QUERY_SIZE) } }' -H "Content-Type: application/json" - $(QUIET)curl -XPUT "http://$(ES_HOST):$(ES_PORT)/cf_properties/_settings" -d '{ "index" : { "max_result_window" : $(CF_QUERY_SIZE) } }' -H "Content-Type: application/json" - $(QUIET)curl -XPUT "http://$(ES_HOST):$(ES_PORT)/cf_tags/_settings" -d '{ "index" : { "max_result_window" : $(CF_QUERY_SIZE) } }' -H "Content-Type: application/json" +mapping: + $(QUIET)bash $(CF_SITE_TEMPLATE_PATH)/create_indexes.bash ## Clean all ES indexes and their mapping -clean.mapping.es: - $(QUIET)bash $(CF_SITE_TEMPLATE_PATH)/clean_mapping_definitions.bash +mapping.clean: + $(QUIET)bash $(CF_SITE_TEMPLATE_PATH)/delete_indexes.bash +mapping.settings: + $(QUIET)curl -XPUT "http://$(ES_HOST):$(ES_PORT)/channelfinder/_settings" -d '{ "index" : { "max_result_window" : $(CF_QUERY_SIZE) } }' -H "Content-Type: application/jsoin" + $(QUIET)curl -XPUT "http://$(ES_HOST):$(ES_PORT)/cf_properties/_settings" -d '{ "index" : { "max_result_window" : $(CF_QUERY_SIZE) } }' -H "Content-Type: application/json" + $(QUIET)curl -XPUT "http://$(ES_HOST):$(ES_PORT)/cf_tags/_settings" -d '{ "index" : { "max_result_window" : $(CF_QUERY_SIZE) } }' -H "Content-Type: application/json" -all.es: install.esdeb start.es status.es mapping.es +all.es: install.esdeb start.es status.es mapping diff --git a/configure/RULES_VARS b/configure/RULES_VARS index 397a6b1..9ee2641 100644 --- a/configure/RULES_VARS +++ b/configure/RULES_VARS @@ -45,8 +45,7 @@ PRINT.%: $(QUIET)echo $*\'s origin is $(origin $*) print-%: - $(QUIET)echo $* = $($*) - $(QUIET)echo $*\'s origin is $(origin $*) + $(QUIET)echo $($*) ls.%: $(QUIET) ls $(LSOPTS) $($*) diff --git a/site-template/ChannelFinderConf.md b/site-template/ChannelFinderConf.md new file mode 100644 index 0000000..779797f --- /dev/null +++ b/site-template/ChannelFinderConf.md @@ -0,0 +1,12 @@ +# ChannelFinder Configuratino + +## Indexes + +```bash +channelfinder +cf_tags +cf_properties +``` + + + diff --git a/site-template/application.properties.in b/site-template/application.properties.in index 01d3dc6..1031a07 100644 --- a/site-template/application.properties.in +++ b/site-template/application.properties.in @@ -13,12 +13,13 @@ server.ssl.key-alias=@SSHKEYALIAS@ security.require-ssl=true -logging.level.org.springframework.web=DEBUG +logging.level.org.springframework.web=@CF_LOG_LEVEL@ -logging.file=/var/log/ChannelFinder.log +spring.http.log-request-details=true + +logging.file=@CF_LOG_PATH@/@CF_LOG_FILE@ # Default # logging.file.max-size=10MB -spring.http.log-request-details=true ############## LDAP - External ############## ldap.enabled = false @@ -44,6 +45,11 @@ spring.ldap.embedded.validation.enabled=false ############## Demo Auth ############## demo_auth.enabled = false +demo_auth.delimiter.roles = : +demo_auth.users = admin,user +demo_auth.pwds = adminPass,userPass +demo_auth.roles = ADMIN,USER + ############## Role --> group Mapping ############## # Customize group names here @@ -55,21 +61,23 @@ tag-groups=cf-tags,USER ############################## Elastic Network And HTTP ############################### # Elasticsearch host -elasticsearch.network.host: @ES_HOST@ +#elasticsearch.network.host: @ES_HOST@ # Set a custom port for the node to node communication (9300 by default): -elasticsearch.transport.tcp.port: 9300 +# elasticsearch.transport.tcp.port: 9300 # Set a custom port to listen for HTTP traffic: elasticsearch.http.port: @ES_PORT@ # Elasticsearch index names and types used by channelfinder, ensure that any changes here should be replicated in the mapping_definitions.sh elasticsearch.tag.index = cf_tags -elasticsearch.tag.type = cf_tag - elasticsearch.property.index = cf_properties -elasticsearch.property.type = cf_property - elasticsearch.channel.index = channelfinder -elasticsearch.channel.type = cf_channel # maximum query result size -elasticsearch.query.size = @CF_QUERY_SIZE@ +elasticsearch.query.size = @CF_QUERY_SIZE@ + +# Create the Channel Finder indices if they do not exist +elasticsearch.create.indices: @CF_CREATE_INDEX@ + +############################## Service Info ############################### +channelfinder.version = @CF_VERSION@ + diff --git a/site-template/clean_mapping_definitions.bash b/site-template/clean_mapping_definitions.bash deleted file mode 100644 index 01e3736..0000000 --- a/site-template/clean_mapping_definitions.bash +++ /dev/null @@ -1,28 +0,0 @@ -#!/usr/bin/env bash -# -# author : Jeong Han Lee -# email : jeonghan.lee@gmail.com -# date : Thursday, February 20 14:38:29 PST 2020 -# version : 0.0.1 - - -declare -g SC_SCRIPT; -declare -g SC_TOP; - -SC_SCRIPT="$(realpath "$0")"; -SC_TOP="${SC_SCRIPT%/*}" - -# shellcheck disable=SC1090,SC1091 -. "${SC_TOP}"/es_host.cfg - - -# shellcheck disable=SC2153,SC2154 -ES_URL="http://${es_host}:${es_port}" - - -curl -XDELETE "${ES_URL}"/cf_tags -echo "" -curl -XDELETE "${ES_URL}"/channelfinder -echo "" -curl -XDELETE "${ES_URL}"/cf_properties -echo "" diff --git a/site-template/create_indexes.bash b/site-template/create_indexes.bash new file mode 100644 index 0000000..c969a00 --- /dev/null +++ b/site-template/create_indexes.bash @@ -0,0 +1,52 @@ +#!/usr/bin/env bash +# +# author : Jeong Han Lee +# email : jeonghan.lee@gmail.com +# date : Tue Sep 20 22:24:45 PDT 2022 +# version : 0.0.1 + + +declare -g SC_SCRIPT; +declare -g SC_TOP; +declare -g ENV_TOP; + +SC_SCRIPT="$(realpath "$0")"; +SC_TOP="${SC_SCRIPT%/*}" +ENV_TOP="${SC_TOP}/.." + +SITE_TEMPLATE_PATH=$(make -C "${ENV_TOP}" -s print-CF_SITE_TEMPLATE_PATH) +CF_JSON_PATH="${ENV_TOP}/$(make -C "${ENV_TOP}" -s print-CF_SRC_PATH)" +CF_JSON_PATH+="/src/main/resources" +CF_QUERY_SIZE=$(make -C "${ENV_TOP}" -s print-CF_QUERY_SIZE) + +# shellcheck disable=SC1090,SC1091 +. "${SC_TOP}"/es_host.cfg + +function curl_put +{ + local index="$1"; shift; + local json="$1"; shift; + + local json_file="@${CF_JSON_PATH}/${json}" + + printf "Creating %s from %s\n" "$index" "$json"; + if [ -x "`which jq`" ]; then + curl -s -H 'Content-Type: application/json' -XPUT http://${es_host}:${es_port}/${index} -d ${json_file} | jq + else + curl -s -H 'Content-Type: application/json' -XPUT http://${es_host}:${es_port}/${index} -d ${json_file} + fi; + + printf ">>> Settings max_result_window of index %s : %s\n" "$index" "$CF_QUERY_SIZE"; + if [ -x "`which jq`" ]; then + curl -s -H 'Content-Type: application/json' -XPUT http://${es_host}:${es_port}/${index}/_settings -d '{ "index" : { "max_result_window" : "'"${CF_QUERY_SIZE}"'" }}' | jq + else + curl -s -H 'Content-Type: application/json' -XPUT http://${es_host}:${es_port}/${index}/_settings -d '{ "index" : { "max_result_window" : "'"${CF_QUERY_SIZE}"'" }}' + fi; +} + +#Create the Index : cf_tags, cf_properties, and channelfinder + +curl_put "cf_tags" "tag_mapping.json" +curl_put "cf_properties" "properties_mapping.json" +curl_put "channelfinder" "channelfinder_mapping.json" + diff --git a/site-template/delete_indexes.bash b/site-template/delete_indexes.bash new file mode 100644 index 0000000..30497e3 --- /dev/null +++ b/site-template/delete_indexes.bash @@ -0,0 +1,35 @@ +#!/usr/bin/env bash +# +# author : Jeong Han Lee +# email : jeonghan.lee@gmail.com +# date : Tue Sep 20 22:24:45 PDT 2022 +# version : 0.0.1 + +declare -g SC_SCRIPT; +declare -g SC_TOP; + +SC_SCRIPT="$(realpath "$0")"; +SC_TOP="${SC_SCRIPT%/*}" + +# shellcheck disable=SC1090,SC1091 +. "${SC_TOP}"/es_host.cfg + + +function curl_delete +{ + local index="$1"; shift; + + printf "Deleting %s \n" "$index"; + if [ -x "`which jq`" ]; then + curl -s -XDELETE http://${es_host}:${es_port}/${index} | jq + else + curl -s -XDELETE http://${es_host}:${es_port}/${index} + fi; +} + +#Create the Index : cf_tags, cf_properties, and channelfinder + +curl_delete "cf_tags" +curl_delete "cf_properties" +curl_delete "channelfinder" + From d727dd2e7c476894a574dfc7126c731728e2c28a Mon Sep 17 00:00:00 2001 From: Jeong Han Lee Date: Tue, 20 Sep 2022 22:50:20 -0700 Subject: [PATCH 36/89] update ChannelFinderConf.md --- site-template/ChannelFinderConf.md | 39 ++++++++++++++++++++++++++++-- 1 file changed, 37 insertions(+), 2 deletions(-) diff --git a/site-template/ChannelFinderConf.md b/site-template/ChannelFinderConf.md index 779797f..99cbdb8 100644 --- a/site-template/ChannelFinderConf.md +++ b/site-template/ChannelFinderConf.md @@ -1,6 +1,8 @@ -# ChannelFinder Configuratino +# ChannelFinder Configuration -## Indexes +## Create Indexes and add mapping information + +We have three indexes such as ```bash channelfinder @@ -8,5 +10,38 @@ cf_tags cf_properties ``` +This can be created via `make mapping` or set `elasticsearch.create.indices: true` in `applicaton.properties` + +* Before `make build` + +``` +echo "CF_CREATE_INDEX:=true" >> configure/CONFIG_SITE.local +``` + +After the channelfinder service starts, one can see the following log + +```bash +2022-09-20 22:40:21.084 INFO 78852 --- [ main] org.phoebus.channelfinder.ElasticConfig : Initializing a new Transport clients. +2022-09-20 22:40:22.079 INFO 78852 --- [ main] org.phoebus.channelfinder.ElasticConfig : Created index: channelfinder : acknowledged true +2022-09-20 22:40:22.666 INFO 78852 --- [ main] org.phoebus.channelfinder.ElasticConfig : Created index: cf_tags : acknowledged true +2022-09-20 22:40:23.305 INFO 78852 --- [ main] org.phoebus.channelfinder.ElasticConfig : Created index: cf_properties : acknowledged true +``` + +And please execute the following rule also + +```bash +make mapping.settings +``` + + +* After `make build` [Default] + +```bash +make mapping +``` + +Please check `create_indexes.bash` file in `site-template` for further information. + +* Index will be deleted via `make mapping.clean` From 1ce38a8af8f5d0ee6d9af00adf158e76e7c9f423 Mon Sep 17 00:00:00 2001 From: Jeong Han Lee Date: Tue, 20 Sep 2022 23:01:25 -0700 Subject: [PATCH 37/89] shellceck ... --- site-template/create_indexes.bash | 17 ++++++++++------- site-template/delete_indexes.bash | 8 +++++--- 2 files changed, 15 insertions(+), 10 deletions(-) diff --git a/site-template/create_indexes.bash b/site-template/create_indexes.bash index c969a00..efb488b 100644 --- a/site-template/create_indexes.bash +++ b/site-template/create_indexes.bash @@ -14,7 +14,6 @@ SC_SCRIPT="$(realpath "$0")"; SC_TOP="${SC_SCRIPT%/*}" ENV_TOP="${SC_TOP}/.." -SITE_TEMPLATE_PATH=$(make -C "${ENV_TOP}" -s print-CF_SITE_TEMPLATE_PATH) CF_JSON_PATH="${ENV_TOP}/$(make -C "${ENV_TOP}" -s print-CF_SRC_PATH)" CF_JSON_PATH+="/src/main/resources" CF_QUERY_SIZE=$(make -C "${ENV_TOP}" -s print-CF_QUERY_SIZE) @@ -30,17 +29,21 @@ function curl_put local json_file="@${CF_JSON_PATH}/${json}" printf "Creating %s from %s\n" "$index" "$json"; - if [ -x "`which jq`" ]; then - curl -s -H 'Content-Type: application/json' -XPUT http://${es_host}:${es_port}/${index} -d ${json_file} | jq + if [ -x "$(which jq)" ]; then + # shellcheck disable=SC2154 + curl -s -H 'Content-Type: application/json' -XPUT http://"${es_host}:${es_port}/${index}" -d "${json_file}" | jq else - curl -s -H 'Content-Type: application/json' -XPUT http://${es_host}:${es_port}/${index} -d ${json_file} + # shellcheck disable=SC2154 + curl -s -H 'Content-Type: application/json' -XPUT http://"${es_host}:${es_port}/${index}" -d "${json_file}" fi; printf ">>> Settings max_result_window of index %s : %s\n" "$index" "$CF_QUERY_SIZE"; - if [ -x "`which jq`" ]; then - curl -s -H 'Content-Type: application/json' -XPUT http://${es_host}:${es_port}/${index}/_settings -d '{ "index" : { "max_result_window" : "'"${CF_QUERY_SIZE}"'" }}' | jq + if [ -x "$(which jq)" ]; then + # shellcheck disable=SC2154 + curl -s -H 'Content-Type: application/json' -XPUT http://"${es_host}:${es_port}/${index}"/_settings -d '{ "index" : { "max_result_window" : "'"${CF_QUERY_SIZE}"'" }}' | jq else - curl -s -H 'Content-Type: application/json' -XPUT http://${es_host}:${es_port}/${index}/_settings -d '{ "index" : { "max_result_window" : "'"${CF_QUERY_SIZE}"'" }}' + # shellcheck disable=SC2154 + curl -s -H 'Content-Type: application/json' -XPUT http://"${es_host}:${es_port}/${index}"/_settings -d '{ "index" : { "max_result_window" : "'"${CF_QUERY_SIZE}"'" }}' fi; } diff --git a/site-template/delete_indexes.bash b/site-template/delete_indexes.bash index 30497e3..743606e 100644 --- a/site-template/delete_indexes.bash +++ b/site-template/delete_indexes.bash @@ -20,10 +20,12 @@ function curl_delete local index="$1"; shift; printf "Deleting %s \n" "$index"; - if [ -x "`which jq`" ]; then - curl -s -XDELETE http://${es_host}:${es_port}/${index} | jq + if [ -x "$(which jq)" ]; then + # shellcheck disable=SC2154 + curl -s -XDELETE http://"${es_host}:${es_port}/${index}" | jq else - curl -s -XDELETE http://${es_host}:${es_port}/${index} + # shellcheck disable=SC2154 + curl -s -XDELETE http://"${es_host}:${es_port}/${index}" fi; } From ec8c268a06d7c5660d011b58250ec3aa04322479 Mon Sep 17 00:00:00 2001 From: Jeong Han Lee Date: Tue, 20 Sep 2022 23:03:22 -0700 Subject: [PATCH 38/89] update superlinter to 4.9.5 --- .github/workflows/linter.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/linter.yml b/.github/workflows/linter.yml index 2349b01..fbd5f9e 100644 --- a/.github/workflows/linter.yml +++ b/.github/workflows/linter.yml @@ -15,7 +15,7 @@ jobs: - name: Checkout Code uses: actions/checkout@v3 - name: Lint Code Base - uses: docker://github/super-linter:v4.8.1 + uses: docker://github/super-linter:v4.9.5 env: VALIDATE_MD: true VALIDATE_BASH: true From f2358aae998787171bba30bcb93bb25f765306d6 Mon Sep 17 00:00:00 2001 From: Jeong Han Lee Date: Wed, 21 Sep 2022 11:25:02 -0700 Subject: [PATCH 39/89] unified shell to interact with CF elasticsearch --- configure/RULES_REQ | 7 +- site-template/cf_es_configuration.bash | 156 +++++++++++++++++++++++++ site-template/create_indexes.bash | 55 --------- site-template/delete_indexes.bash | 37 ------ 4 files changed, 161 insertions(+), 94 deletions(-) create mode 100755 site-template/cf_es_configuration.bash delete mode 100644 site-template/create_indexes.bash delete mode 100644 site-template/delete_indexes.bash diff --git a/configure/RULES_REQ b/configure/RULES_REQ index b371bd0..34b93a6 100644 --- a/configure/RULES_REQ +++ b/configure/RULES_REQ @@ -56,11 +56,14 @@ disable.es: ## Create the ES indexes and set up their mapping mapping: - $(QUIET)bash $(CF_SITE_TEMPLATE_PATH)/create_indexes.bash + $(QUIET)bash $(CF_SITE_TEMPLATE_PATH)/cf_es_configuration.bash mapping ## Clean all ES indexes and their mapping mapping.clean: - $(QUIET)bash $(CF_SITE_TEMPLATE_PATH)/delete_indexes.bash + $(QUIET)bash $(CF_SITE_TEMPLATE_PATH)/cf_es_configuration.bash mappingClean + +mapping.verify: + $(QUIET)bash $(CF_SITE_TEMPLATE_PATH)/cf_es_configuration.bash mappingVerify mapping.settings: $(QUIET)curl -XPUT "http://$(ES_HOST):$(ES_PORT)/channelfinder/_settings" -d '{ "index" : { "max_result_window" : $(CF_QUERY_SIZE) } }' -H "Content-Type: application/jsoin" diff --git a/site-template/cf_es_configuration.bash b/site-template/cf_es_configuration.bash new file mode 100755 index 0000000..6dd76a6 --- /dev/null +++ b/site-template/cf_es_configuration.bash @@ -0,0 +1,156 @@ +#!/usr/bin/env bash +# +# author : Jeong Han Lee +# email : jeonghan.lee@gmail.com +# date : Wed Sep 21 11:12:34 PDT 2022 +# version : 0.0.1 + + +declare -g SC_SCRIPT; +declare -g SC_TOP; +declare -g ENV_TOP; + +SC_SCRIPT="$(realpath "$0")"; +SC_TOP="${SC_SCRIPT%/*}" +ENV_TOP="${SC_TOP}/.." + +CF_JSON_PATH="${ENV_TOP}/$(make -C "${ENV_TOP}" -s print-CF_SRC_PATH)" +CF_JSON_PATH+="/src/main/resources" +CF_QUERY_SIZE=$(make -C "${ENV_TOP}" -s print-CF_QUERY_SIZE) + +# shellcheck disable=SC1090,SC1091 +. "${SC_TOP}"/es_host.cfg + +function curl_put +{ + local index="$1"; shift; + local json="$1"; shift; + + local json_file="${CF_JSON_PATH}/${json}" + + if [ ! -f "$json_file" ]; then + printf "Warning::File not found %s\n" "$json_file"; + printf "Please check %s\n" "$json_file"; + exit; + fi + + printf ">>> Creating ...%s... from ...%s...\n" "$index" "$json"; + if [ -x "$(which jq)" ]; then + # shellcheck disable=SC2154 + curl -s -H 'Content-Type: application/json' -XPUT http://"${es_host}:${es_port}/${index}" -d "@${json_file}" | jq + else + # shellcheck disable=SC2154 + curl -s -H 'Content-Type: application/json' -XPUT http://"${es_host}:${es_port}/${index}" -d "@${json_file}" + fi; + printf ">>>\n" + + printf ">>> Settings max_result_window of index ...%s... : ...%s...\n" "$index" "$CF_QUERY_SIZE"; + if [ -x "$(which jq)" ]; then + # shellcheck disable=SC2154 + curl -s -H 'Content-Type: application/json' -XPUT http://"${es_host}:${es_port}/${index}"/_settings -d '{ "index" : { "max_result_window" : "'"${CF_QUERY_SIZE}"'" }}' | jq + else + # shellcheck disable=SC2154 + curl -s -H 'Content-Type: application/json' -XPUT http://"${es_host}:${es_port}/${index}"/_settings -d '{ "index" : { "max_result_window" : "'"${CF_QUERY_SIZE}"'" }}' + fi; + printf ">------------\n"; + +} + +function curl_get +{ + local index="$1"; shift; + local meta="$1"; shift; + + printf ">>> Checking ...%s... with ...%s...\n" "$index" "$meta"; + if [ -x "$(which jq)" ]; then + # shellcheck disable=SC2154 + curl -s -XGET http://"${es_host}:${es_port}/${index}/_${meta}" | jq + else + # shellcheck disable=SC2154 + curl -s -XGET http://"${es_host}:${es_port}/${index}/_${meta}" + fi; + printf ">----------\n"; +} + +function curl_delete +{ + local index="$1"; shift; + + printf "Deleting %s \n" "$index"; + if [ -x "$(which jq)" ]; then + # shellcheck disable=SC2154 + curl -s -XDELETE http://"${es_host}:${es_port}/${index}" | jq + else + # shellcheck disable=SC2154 + curl -s -XDELETE http://"${es_host}:${es_port}/${index}" + fi; +} + + +function IndexMapping +{ + local index="$1"; shift; + local json="$1"; shift; + curl_put "$index" "$json" + curl_get "$index" "mapping" +} + +function MetaSearching +{ + local meta="$1"; shift; + curl_get "cf_tags" "$meta" + curl_get "cf_properties" "$meta" + curl_get "channelfinder" "$meta" +} + + +function usage +{ + { + echo ""; + echo "Usage : $0 args" + echo ""; + echo " possbile args"; + echo ""; + echo " mapping : CF index mapping"; + echo " mappingClean : Delete all CF index"; + echo " mappingVerify : Verify the existing mappings"; + echo " metaField 'search' : mapping, search, .... "; + echo " h : this screen"; + echo ""; + echo " bash $0 mapping " + echo "" + } 1>&2; + exit 1; +} + + + +case "$1" in + mapping) + IndexMapping "channelfinder" "channel_mapping.json" + IndexMapping "cf_tags" "tag_mapping.json" + IndexMapping "cf_properties" "properties_mapping.json" + ;; + mappingClean) + curl_delete "cf_tags" + curl_delete "cf_properties" + curl_delete "channelfinder" + ;; + mappingVerify) + curl_get "cf_tags" "mapping" + curl_get "cf_properties" "mapping" + curl_get "channelfinder" "mapping" + ;; + metaField) + MetaSearching "$2" + ;; + h);; + help) + usage; + ;; + *) + usage; + ;; +esac + diff --git a/site-template/create_indexes.bash b/site-template/create_indexes.bash deleted file mode 100644 index efb488b..0000000 --- a/site-template/create_indexes.bash +++ /dev/null @@ -1,55 +0,0 @@ -#!/usr/bin/env bash -# -# author : Jeong Han Lee -# email : jeonghan.lee@gmail.com -# date : Tue Sep 20 22:24:45 PDT 2022 -# version : 0.0.1 - - -declare -g SC_SCRIPT; -declare -g SC_TOP; -declare -g ENV_TOP; - -SC_SCRIPT="$(realpath "$0")"; -SC_TOP="${SC_SCRIPT%/*}" -ENV_TOP="${SC_TOP}/.." - -CF_JSON_PATH="${ENV_TOP}/$(make -C "${ENV_TOP}" -s print-CF_SRC_PATH)" -CF_JSON_PATH+="/src/main/resources" -CF_QUERY_SIZE=$(make -C "${ENV_TOP}" -s print-CF_QUERY_SIZE) - -# shellcheck disable=SC1090,SC1091 -. "${SC_TOP}"/es_host.cfg - -function curl_put -{ - local index="$1"; shift; - local json="$1"; shift; - - local json_file="@${CF_JSON_PATH}/${json}" - - printf "Creating %s from %s\n" "$index" "$json"; - if [ -x "$(which jq)" ]; then - # shellcheck disable=SC2154 - curl -s -H 'Content-Type: application/json' -XPUT http://"${es_host}:${es_port}/${index}" -d "${json_file}" | jq - else - # shellcheck disable=SC2154 - curl -s -H 'Content-Type: application/json' -XPUT http://"${es_host}:${es_port}/${index}" -d "${json_file}" - fi; - - printf ">>> Settings max_result_window of index %s : %s\n" "$index" "$CF_QUERY_SIZE"; - if [ -x "$(which jq)" ]; then - # shellcheck disable=SC2154 - curl -s -H 'Content-Type: application/json' -XPUT http://"${es_host}:${es_port}/${index}"/_settings -d '{ "index" : { "max_result_window" : "'"${CF_QUERY_SIZE}"'" }}' | jq - else - # shellcheck disable=SC2154 - curl -s -H 'Content-Type: application/json' -XPUT http://"${es_host}:${es_port}/${index}"/_settings -d '{ "index" : { "max_result_window" : "'"${CF_QUERY_SIZE}"'" }}' - fi; -} - -#Create the Index : cf_tags, cf_properties, and channelfinder - -curl_put "cf_tags" "tag_mapping.json" -curl_put "cf_properties" "properties_mapping.json" -curl_put "channelfinder" "channelfinder_mapping.json" - diff --git a/site-template/delete_indexes.bash b/site-template/delete_indexes.bash deleted file mode 100644 index 743606e..0000000 --- a/site-template/delete_indexes.bash +++ /dev/null @@ -1,37 +0,0 @@ -#!/usr/bin/env bash -# -# author : Jeong Han Lee -# email : jeonghan.lee@gmail.com -# date : Tue Sep 20 22:24:45 PDT 2022 -# version : 0.0.1 - -declare -g SC_SCRIPT; -declare -g SC_TOP; - -SC_SCRIPT="$(realpath "$0")"; -SC_TOP="${SC_SCRIPT%/*}" - -# shellcheck disable=SC1090,SC1091 -. "${SC_TOP}"/es_host.cfg - - -function curl_delete -{ - local index="$1"; shift; - - printf "Deleting %s \n" "$index"; - if [ -x "$(which jq)" ]; then - # shellcheck disable=SC2154 - curl -s -XDELETE http://"${es_host}:${es_port}/${index}" | jq - else - # shellcheck disable=SC2154 - curl -s -XDELETE http://"${es_host}:${es_port}/${index}" - fi; -} - -#Create the Index : cf_tags, cf_properties, and channelfinder - -curl_delete "cf_tags" -curl_delete "cf_properties" -curl_delete "channelfinder" - From 2c25b92a2dab4484834e752aabe09fd57cb39230 Mon Sep 17 00:00:00 2001 From: Jeong Han Lee Date: Wed, 21 Sep 2022 11:43:11 -0700 Subject: [PATCH 40/89] update ChannelFinderConf.md --- site-template/ChannelFinderConf.md | 45 +++++++++++++++++++----------- 1 file changed, 29 insertions(+), 16 deletions(-) diff --git a/site-template/ChannelFinderConf.md b/site-template/ChannelFinderConf.md index 99cbdb8..12f3dfa 100644 --- a/site-template/ChannelFinderConf.md +++ b/site-template/ChannelFinderConf.md @@ -2,7 +2,7 @@ ## Create Indexes and add mapping information -We have three indexes such as +We have three index(s), which I don't use `indices`, because it is a slightly different definition in ES, such as ```bash channelfinder @@ -10,7 +10,34 @@ cf_tags cf_properties ``` -This can be created via `make mapping` or set `elasticsearch.create.indices: true` in `applicaton.properties` +This can be created via `make mapping`, which is our preference, or set `elasticsearch.create.indices: true` in `applicaton.properties` + + +### `make mapping` + +This is the Makefile rule, to call `cf_es_configuration.bash` in `site-template` path. It uses the same json files to create three index(s) with `CF_QUERY_SIZE` configuration. We have the following Makefile rules: + +``` +make mapping +make mapping.clean +make mapping.verify +``` + +And the script will allow us to explore them in detail. + +``` +Usage : site-template/cf_es_configuration.bash args + + possbile args + + mapping : CF index mapping + mappingClean : Delete all CF index + mappingVerify : Verify the existing mappings + metaField 'search' : mapping, search, .... + h : this screen +``` + +### `application.properties` * Before `make build` @@ -31,17 +58,3 @@ And please execute the following rule also ```bash make mapping.settings -``` - - -* After `make build` [Default] - -```bash -make mapping -``` - -Please check `create_indexes.bash` file in `site-template` for further information. - -* Index will be deleted via `make mapping.clean` - - From 9f3b799b0109f0b467dc75801a3a1643b2885e5c Mon Sep 17 00:00:00 2001 From: Jeong Han Lee Date: Wed, 21 Sep 2022 11:44:38 -0700 Subject: [PATCH 41/89] remove uncessary and obsolete mapping shell scripts --- site-template/mapping_definitions.bash | 115 -------------------- site-template/mapping_definitions_es7x.bash | 95 ---------------- 2 files changed, 210 deletions(-) delete mode 100755 site-template/mapping_definitions.bash delete mode 100644 site-template/mapping_definitions_es7x.bash diff --git a/site-template/mapping_definitions.bash b/site-template/mapping_definitions.bash deleted file mode 100755 index 06ee6e0..0000000 --- a/site-template/mapping_definitions.bash +++ /dev/null @@ -1,115 +0,0 @@ -#!/usr/bin/env bash - -### -# #%L -# ChannelFinder Directory Service -# %% -# Copyright (C) 2010 - 2016 Helmholtz-Zentrum Berlin für Materialien und Energie GmbH -# %% -# Copyright (C) 2010 - 2012 Brookhaven National Laboratory -# All rights reserved. Use is subject to license terms. -# %% -# Copyright (c) 2020 Jeong Han Lee -# #L% -### -# The mapping definition for the Indexes associated with the channelfinder v4 - -# author : Jeong Han Lee -# email : jeonghan.lee@gmail.com -# date : Thursday, February 20 14:39:35 PST 2020 -# version : 0.0.1 - -declare -g SC_SCRIPT; -declare -g SC_TOP; - -SC_SCRIPT="$(realpath "$0")"; -SC_TOP="${SC_SCRIPT%/*}" - -# shellcheck disable=SC1090,SC1091 -. "${SC_TOP}"/es_host.cfg - - -#Create the Index -print_help "index/mapping" "cf_tags" -# shellcheck disable=SC2153,SC2154 -curl -H 'Content-Type: application/json' -XPUT http://"${es_host}":"${es_port}"/cf_tags -d' -{ -"mappings":{ - "cf_tag" : { - "properties" : { - "name" : { - "type" : "keyword" - }, - "owner" : { - "type" : "keyword" - } - } - } - } -}' - -print_help "index/mapping" "cf_properties" -# shellcheck disable=SC2153,SC2154 -curl -H 'Content-Type: application/json' -XPUT http://"${es_host}":"${es_port}"/cf_properties -d' -{ -"mappings":{ - "cf_property" : { - "properties" : { - "name" : { - "type" : "keyword" - }, - "owner" : { - "type" : "keyword" - } - } - } - } -}' - -print_help "index/mapping" "channelfinder" -# shellcheck disable=SC2153,SC2154 -curl -H 'Content-Type: application/json' -XPUT http://"${es_host}":"${es_port}"/channelfinder -d' -{ -"mappings":{ - "cf_channel" : { - "properties" : { - "name" : { - "type" : "keyword" - }, - "owner" : { - "type" : "keyword" - }, - "script" : { - "type" : "keyword" - }, - "properties" : { - "type" : "nested", - "properties" : { - "name" : { - "type" : "keyword" - }, - "owner" : { - "type" : "keyword" - }, - "value" : { - "type" : "keyword" - } - } - }, - "tags" : { - "type" : "nested", - "properties" : { - "name" : { - "type" : "keyword" - }, - "owner" : { - "type" : "keyword" - } - } - } - } - } - } -}' - -printf "\n"; diff --git a/site-template/mapping_definitions_es7x.bash b/site-template/mapping_definitions_es7x.bash deleted file mode 100644 index bc37295..0000000 --- a/site-template/mapping_definitions_es7x.bash +++ /dev/null @@ -1,95 +0,0 @@ -#!/usr/bin/env bash - -### -# #%L -# ChannelFinder Directory Service -# %% -# Copyright (C) 2010 - 2016 Helmholtz-Zentrum Berlin für Materialien und Energie GmbH -# %% -# Copyright (C) 2010 - 2012 Brookhaven National Laboratory -# All rights reserved. Use is subject to license terms. -# %% -# Copyright (c) 2020 Jeong Han Lee -# #L% -### -# The mapping definition for the Indexes associated with the channelfinder v4 - -# author : Jeong Han Lee -# email : jeonghan.lee@gmail.com -# date : Wednesday, April 22 23:00:18 PDT 2020 -# version : 0.0.2 - -declare -g SC_SCRIPT; -declare -g SC_TOP; - -SC_SCRIPT="$(realpath "$0")"; -SC_TOP="${SC_SCRIPT%/*}" - -# shellcheck disable=SC1090,SC1091 -. "${SC_TOP}/es_host.cfg" - - -# shellcheck disable=SC2153,SC2154 -ES_URL="http://${es_host}:${es_port}" - -#Create the Index : three -print_help "index" "cf_tags" -curl -XPUT "$ES_URL"/cf_tags -print_help "index" "cf_properties" -curl -XPUT "$ES_URL"/cf_properties -print_help "index" "channelfinder" -curl -XPUT "$ES_URL"/channelfinder - -print_help "mapping" "cf_tags" -curl -H 'Content-Type: application/json' -XPUT "${ES_URL}"/cf_tags/_mapping/cf_tag?include_type_name=true -d' -{ - "cf_tag" : { - "properties" : { - "name" : { "type" : "keyword" }, - "owner" : { "type" : "keyword" } - } - } -}' - -print_help "mapping" "cf_properties" - -curl -H 'Content-Type: application/json' -XPUT "${ES_URL}"/cf_properties/_mapping/cf_property?include_type_name=true -d' -{ - "cf_property" : { - "properties" : { - "name" : { "type" : "keyword" }, - "owner" : { "type" : "keyword" } - } - } -}' - -print_help "mapping" "channelfinder" -curl -H 'Content-Type: application/json' -XPUT "${ES_URL}"/channelfinder/_mapping/cf_channel?include_type_name=true -d' -{ - "cf_channel" : { - "properties" : { - "name" : { "type" : "keyword" }, - "owner" : { "type" : "keyword" }, - "script" : { "type" : "keyword" }, - "cf_properties" : { - "type" : "nested", - "include_in_parent" : true, - "properties" : { - "name" : { "type" : "keyword" }, - "owner" : { "type" : "keyword" }, - "value" : { "type" : "keyword" } - } - }, - "cf_tags" : { - "type" : "nested", - "include_in_parent" : true, - "properties" : { - "name" : { "type" : "keyword" }, - "owner" : { "type" : "keyword" } - } - } - } - } -}' - -printf "\n"; From 8c9ac2afd40cd4b472209a8c1a603393692aca6b Mon Sep 17 00:00:00 2001 From: Jeong Han Lee Date: Wed, 21 Sep 2022 15:32:08 -0700 Subject: [PATCH 42/89] tested with Demo --- README.md | 29 ++++- configure/RELEASE | 3 +- configure/RULES | 1 + configure/RULES_INSTALL | 2 - configure/RULES_REQ | 6 +- configure/RULES_RUN | 9 ++ configure/RULES_SRC | 2 +- {site-template => docs}/ChannelFinderConf.md | 0 docs/ChannelFinderDemo.md | 117 ++++++++++++++++++ .../cf_es_configuration.bash | 20 ++- scripts/cf_queries.bash | 112 +++++++++++++++++ scripts/tag_foo.json | 1 + 12 files changed, 290 insertions(+), 12 deletions(-) create mode 100644 configure/RULES_RUN rename {site-template => docs}/ChannelFinderConf.md (100%) create mode 100644 docs/ChannelFinderDemo.md rename {site-template => scripts}/cf_es_configuration.bash (92%) create mode 100755 scripts/cf_queries.bash create mode 100644 scripts/tag_foo.json diff --git a/README.md b/README.md index d6c314d..3ecfeed 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ Configuration Environment for ChannelFinderService at > ./configure/CONFIG_COMMON.local ``` -## Support OS [Debian 11 (EOL: 2024-06-01/2026-08-15), Rocky8 (EOL: 2029-05-31)] +## Support OS + +### Debian 11 (EOL: 2024-06-01/2026-08-15), Rocky8 (EOL: 2029-05-31) It will works with other systems. Please check github action workflows. @@ -47,6 +49,29 @@ make mapping make mapping.clean ``` +## macOS (tested with aarch64 with brew) + +```bash +make init +make conf.macos +make conf +make build +make run +make mapping +make mapping.clean +``` + +## Run Demo + +Please see [docs/ChannelFinderDemo.md](docs/ChannelFinderDemo.md). + + +``` +make run +make demo +make demo.clean +``` + ## Docker Image The Docker image is hosted at https://hub.docker.com/orgs/alscontrols diff --git a/configure/RELEASE b/configure/RELEASE index a6ad8f9..b3af4ea 100644 --- a/configure/RELEASE +++ b/configure/RELEASE @@ -6,7 +6,8 @@ CF_SRC_NAME:=ChannelFinderService ##- Which the source tag / branch / hash id would like to use -CF_SRC_TAG:=tags/ChannelFinder-4.7.0 +# 2022-09-21 +CF_SRC_TAG:=fc9d536 ##- Placeholder for the site-specific version control CF_SRC_VERSION:=4.7.0 diff --git a/configure/RULES b/configure/RULES index 629c201..852eb04 100644 --- a/configure/RULES +++ b/configure/RULES @@ -11,6 +11,7 @@ include $(TOP)/configure/RULES_FUNC include $(TOP)/configure/RULES_REQ include $(TOP)/configure/RULES_SRC include $(TOP)/configure/RULES_INSTALL +include $(TOP)/configure/RULES_RUN include $(TOP)/configure/RULES_SYSTEMD include $(TOP)/configure/RULES_DOCKER include $(TOP)/configure/RULES_PROPERTIES diff --git a/configure/RULES_INSTALL b/configure/RULES_INSTALL index bdf0be1..d355aa4 100644 --- a/configure/RULES_INSTALL +++ b/configure/RULES_INSTALL @@ -33,5 +33,3 @@ reinstall: conf build install restart: uninstall reinstall cf_start cf_status -start.local: - $(QUIET)mvn -f $(CF_SRC_PATH) spring-boot:run diff --git a/configure/RULES_REQ b/configure/RULES_REQ index 34b93a6..617be9e 100644 --- a/configure/RULES_REQ +++ b/configure/RULES_REQ @@ -56,14 +56,14 @@ disable.es: ## Create the ES indexes and set up their mapping mapping: - $(QUIET)bash $(CF_SITE_TEMPLATE_PATH)/cf_es_configuration.bash mapping + $(QUIET)bash $(TOP)/scripts/cf_es_configuration.bash mapping ## Clean all ES indexes and their mapping mapping.clean: - $(QUIET)bash $(CF_SITE_TEMPLATE_PATH)/cf_es_configuration.bash mappingClean + $(QUIET)bash $(TOP)/scripts/cf_es_configuration.bash mappingClean mapping.verify: - $(QUIET)bash $(CF_SITE_TEMPLATE_PATH)/cf_es_configuration.bash mappingVerify + $(QUIET)bash $(TOP)/scripts/cf_es_configuration.bash mappingVerify mapping.settings: $(QUIET)curl -XPUT "http://$(ES_HOST):$(ES_PORT)/channelfinder/_settings" -d '{ "index" : { "max_result_window" : $(CF_QUERY_SIZE) } }' -H "Content-Type: application/jsoin" diff --git a/configure/RULES_RUN b/configure/RULES_RUN new file mode 100644 index 0000000..0c8d5af --- /dev/null +++ b/configure/RULES_RUN @@ -0,0 +1,9 @@ + +run: + $(QUIET)mvn -f $(CF_SRC_PATH) spring-boot:run + +demo: + $(QUIET)mvn -f $(CF_SRC_PATH) spring-boot:run -Dspring-boot.run.arguments="--demo-data=1" + +demo.clean: + $(QUIET)mvn -f $(CF_SRC_PATH) spring-boot:run -Dspring-boot.run.arguments="--cleanup=1" diff --git a/configure/RULES_SRC b/configure/RULES_SRC index 8c7d290..f532ccc 100644 --- a/configure/RULES_SRC +++ b/configure/RULES_SRC @@ -49,7 +49,7 @@ endif .PHONY: build conf ## Build Source -build: +build: conf $(QUIET) JAVA_HOME=$(JAVA_HOME) $(MAVEN_CMD) $(CF_MVN_OPTS) -f $(CF_SRC_PATH)/pom.xml clean install -Dmaven.test.skip=true ## Copy the site specific files into sources diff --git a/site-template/ChannelFinderConf.md b/docs/ChannelFinderConf.md similarity index 100% rename from site-template/ChannelFinderConf.md rename to docs/ChannelFinderConf.md diff --git a/docs/ChannelFinderDemo.md b/docs/ChannelFinderDemo.md new file mode 100644 index 0000000..043801f --- /dev/null +++ b/docs/ChannelFinderDemo.md @@ -0,0 +1,117 @@ +# Built-in Demo Testing + +## Run Demo + +This demo is not available within the systemd service. Thus, please use it without systemd services. + +One should follow the strict order to run a channelfinder built-in demo service if the channelfinder runs locally, i.e., no systemd service. + +* Check the system is not running now. + +```bash +systemctl status channelfinder.service +systemctl stop channelfinder.service +systemctl disable channelfinder.service +``` + +* One should be in the `ChannelFinder-env` + +``` +make run +make mapping +make demo +``` + +## Check the Demo service + +If one doesn't have `jq`, it is highly recommended to install it. For example, `apt install jq` in Debian system. + +``` +curl -s http://localhost:8080/ChannelFinder/resources/tags | jq +curl -s http://localhost:8080/ChannelFinder/resources/properties | jq +curl -s http://localhost:8080/ChannelFinder/resources/channels | jq +``` + +## Check, Add, and Remove a tag + + +```bash +$ curl -s --request GET http://localhost:8080/ChannelFinder/resources/tags/foo +{"name":"foo","owner":"admin","channels":[]} + +$ curl -s -u admin:1234 -H 'Content-Type: application/json' --request PUT http://localhost:8080/ChannelFinder/resources/tags/foo -d '{"name":"foo", "owner":"admin"}' +{"name":"foo","owner":"admin","channels":[]} + +$ curl -s --request GET http://localhost:8080/ChannelFinder/resources/tags/foo +{"name":"foo","owner":"admin","channels":[]} + +$curl --basic -u admin:1234 -X DELETE http://localhost:8080/ChannelFinder/resources/tags/foo +$ curl -s --request GET http://localhost:8080/ChannelFinder/resources/tags/foo +{"timestamp":"2022-09-21T21:35:46.677+00:00","status":404,"error":"Not Found","path":"/ChannelFinder/resources/tags/foo"} +``` + +One can see not very kind log messages, which cannot tell exactly what kind of performance they did, in ChannelFinder log as follows: + +```bash +2022-09-21 14:24:05.289 INFO 96837 --- [nio-8080-exec-1] o.p.channelfinder.TagManager.audit : getting tag: foo +2022-09-21 14:24:05.325 INFO 96837 --- [nio-8080-exec-1] org.phoebus.channelfinder.TagRepository : Tag name foo +2022-09-21 14:34:45.959 INFO 96837 --- [nio-8080-exec-9] o.p.channelfinder.TagManager.audit : getting tag: foo +2022-09-21 14:34:45.980 INFO 96837 --- [nio-8080-exec-9] org.phoebus.channelfinder.TagRepository : Tag name foo +2022-09-21 14:35:10.958 INFO 96837 --- [nio-8080-exec-1] o.p.channelfinder.TagManager.audit : client initialization: 0 +2022-09-21 14:35:10.979 INFO 96837 --- [nio-8080-exec-1] org.phoebus.channelfinder.TagRepository : Tag name foo +2022-09-21 14:35:11.182 INFO 96837 --- [nio-8080-exec-1] org.phoebus.channelfinder.TagRepository : Tag name foo +2022-09-21 14:35:23.583 INFO 96837 --- [nio-8080-exec-2] o.p.channelfinder.TagManager.audit : getting tag: foo +2022-09-21 14:35:23.594 INFO 96837 --- [nio-8080-exec-2] org.phoebus.channelfinder.TagRepository : Tag name foo +2022-09-21 14:35:37.897 INFO 96837 --- [nio-8080-exec-4] org.phoebus.channelfinder.TagRepository : Tag name foo +2022-09-21 14:35:46.649 INFO 96837 --- [nio-8080-exec-6] o.p.channelfinder.TagManager.audit : getting tag: foo +2022-09-21 14:35:46.664 INFO 96837 --- [nio-8080-exec-6] org.phoebus.channelfinder.TagRepository : Tag not found +2022-09-21 14:35:46.668 ERROR 96837 --- [nio-8080-exec-6] org.phoebus.channelfinder.TagManager : The tag with the name foo does not exist +``` + +## `cf_query.bash` + +The script makes our life a bit easy to verify the CF server is running correctly. + +```bash +$ bash scripts/cf_queries.bash get tags foo +>>> Getting ...tags... with ...foo... +{ + "name": "foo", + "owner": "admin", + "channels": [] +} +>---------- + +$ bash scripts/cf_queries.bash delete tags foo +>>> Deleting...tags... with ...foo... +>---------- + +$ bash scripts/cf_queries.bash get tags foo +>>> Getting ...tags... with ...foo... +{ + "timestamp": "2022-09-21T22:28:08.966+00:00", + "status": 404, + "error": "Not Found", + "path": "/ChannelFinder/resources/tags/foo" +} +>---------- + +$ bash scripts/cf_queries.bash put tags foo scripts/tag_foo.json +>>> Putting ...... ...... +{ + "name": "foo", + "owner": "admin", + "channels": [] +} +>------------ + +$ bash scripts/cf_queries.bash get tags foo +>>> Getting ...tags... with ...foo... +{ + "name": "foo", + "owner": "admin", + "channels": [] +} +>---------- +``` + diff --git a/site-template/cf_es_configuration.bash b/scripts/cf_es_configuration.bash similarity index 92% rename from site-template/cf_es_configuration.bash rename to scripts/cf_es_configuration.bash index 6dd76a6..d21a729 100755 --- a/site-template/cf_es_configuration.bash +++ b/scripts/cf_es_configuration.bash @@ -10,6 +10,9 @@ declare -g SC_SCRIPT; declare -g SC_TOP; declare -g ENV_TOP; +declare -g es_host; +declare -g es_port; + SC_SCRIPT="$(realpath "$0")"; SC_TOP="${SC_SCRIPT%/*}" ENV_TOP="${SC_TOP}/.." @@ -18,8 +21,9 @@ CF_JSON_PATH="${ENV_TOP}/$(make -C "${ENV_TOP}" -s print-CF_SRC_PATH)" CF_JSON_PATH+="/src/main/resources" CF_QUERY_SIZE=$(make -C "${ENV_TOP}" -s print-CF_QUERY_SIZE) -# shellcheck disable=SC1090,SC1091 -. "${SC_TOP}"/es_host.cfg + +es_host=$(make -C "${ENV_TOP}" -s print-ES_HOST) +es_port=$(make -C "${ENV_TOP}" -s print-ES_PORT) function curl_put { @@ -96,6 +100,13 @@ function IndexMapping } function MetaSearching +{ + local index="$1"; shift; + local meta="$1"; shift; + curl_get "$index" "$meta"; +} + +function MetaAllSearching { local meta="$1"; shift; curl_get "cf_tags" "$meta" @@ -143,7 +154,10 @@ case "$1" in curl_get "channelfinder" "mapping" ;; metaField) - MetaSearching "$2" + MetaAllSearching "$2" + ;; + search) + MetaSearching "$2" "$3" ;; h);; help) diff --git a/scripts/cf_queries.bash b/scripts/cf_queries.bash new file mode 100755 index 0000000..37cdc2b --- /dev/null +++ b/scripts/cf_queries.bash @@ -0,0 +1,112 @@ +#!/usr/bin/env bash +# +# author : Jeong Han Lee +# email : jeonghan.lee@gmail.com +# date : Wed Sep 21 11:12:34 PDT 2022 +# version : 0.0.1 + + +declare -g SC_SCRIPT; +declare -g SC_TOP; +declare -g ENV_TOP; +declare -g CF_URL; +declare -g ADMIN; + +SC_SCRIPT="$(realpath "$0")"; +SC_TOP="${SC_SCRIPT%/*}" +ENV_TOP="${SC_TOP}/.." + +# shellcheck disable=SC2154 +CF_URL="$(make -s -C "${ENV_TOP}" print-CF_FULL_URL)/resources"; + +ADMIN="admin:1234" + +function cf_put +{ + local arg1="$1"; shift; + local arg2="$1"; shift; + local arg3="$1"; shift; + + printf ">>> Putting ...%s... ...%s...\n" "$index" "$json"; + if [ -x "$(which jq)" ]; then + curl -s -H 'Content-Type: application/json' -u "${ADMIN}" --request PUT "${CF_URL}/${arg1}/${arg2}" -d "@${arg3}" | jq + else + curl -s -H 'Content-Type: application/json' -u "${ADMIN}" --request PUT "${CF_URL}/${arg1}/${arg2}" -d "@${arg3}" + fi; + printf ">------------\n"; + +} + +function cf_get +{ + local arg1="$1"; shift; + local arg2="$1"; shift; + + printf ">>> Getting ...%s... with ...%s...\n" "$arg1" "$arg2"; + if [ -x "$(which jq)" ]; then +# echo "curl -s --request GET "${CF_URL}/${arg1}/${arg2}" | jq" + curl -s --request GET "${CF_URL}/${arg1}/${arg2}" | jq + else + curl -s -XGET "${CF_URL}/${arg1}/${arg2}" + fi; + printf ">----------\n"; +} + +function cf_delete +{ + local arg1="$1"; shift; + local arg2="$1"; shift; + + printf ">>> Deleting...%s... with ...%s...\n" "$arg1" "$arg2"; + if [ -x "$(which jq)" ]; then + curl --basic -u "${ADMIN}" -s --request DELETE "${CF_URL}/${arg1}/${arg2}" | jq + else + curl --basic -u "${ADMIN}" -s --request DELETE "${CF_URL}/${arg1}/${arg2}" + fi; + printf ">----------\n" +} + + +function usage +{ + { + echo ""; + echo "Usage : $0 args" + echo ""; + echo " possbile args"; + echo ""; + echo " get arg1 arg2 : Get "; + echo " delete arg1 arg2 : Delete "; + echo " put arg1 arg2 arg3 : Put, arg3 is a json file"; + echo " h : this screen"; + echo ""; + echo " bash $0 mapping " + echo "" + } 1>&2; + exit 1; +} + +arg0="$1"; shift; +arg1="$1"; shift; +arg2="$1"; shift; +arg3="$1"; shift; + +case "$arg0" in + get) + cf_get "$arg1" "$arg2" + ;; + delete) + cf_delete "$arg1" "$arg2" + ;; + put) + cf_put "$arg1" "$arg2" "$arg3" + ;; + h);; + help) + usage; + ;; + *) + usage; + ;; +esac + diff --git a/scripts/tag_foo.json b/scripts/tag_foo.json new file mode 100644 index 0000000..fd2b249 --- /dev/null +++ b/scripts/tag_foo.json @@ -0,0 +1 @@ +{"name":"foo", "owner":"admin"} From fbcb7ca691dfc81efc251703ca0d65f5475d1ac5 Mon Sep 17 00:00:00 2001 From: Jeong Han Lee Date: Wed, 21 Sep 2022 15:49:03 -0700 Subject: [PATCH 43/89] fixed to work around a specific jar file --- configure/CONFIG_SITE | 6 ++++-- configure/RELEASE | 6 ++++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/configure/CONFIG_SITE b/configure/CONFIG_SITE index 751dac8..82858ee 100644 --- a/configure/CONFIG_SITE +++ b/configure/CONFIG_SITE @@ -6,10 +6,12 @@ CF_INSTALL_LOCATION=${CF_INSTALL_PATH}/channelfinder ### Placeholder -### This variable will be replaced by reading a pom.xml file or others +### This variable will be replaced by reading a pom.xml file. The version in pom.xml will define +### the exact version of the jar file. It is difficult to track that version to implement into here. ### It may be linked with SRC_VERSION in RELEASE file ### -CF_JAR_FILENAME:=ChannelFinder-$(CF_SRC_VERSION).jar +### So, we use * to cover all version information with the jar file +CF_JAR_FILENAME:=ChannelFinder-*.jar CF_PROPERTIES:=application.properties diff --git a/configure/RELEASE b/configure/RELEASE index b3af4ea..7b7a55d 100644 --- a/configure/RELEASE +++ b/configure/RELEASE @@ -1,4 +1,4 @@ - +: ##- Where the Source repository SRC_URL/SRC_NAME ##- CF_SRC_URL:=https://github.com/ChannelFinder @@ -10,7 +10,9 @@ CF_SRC_NAME:=ChannelFinderService CF_SRC_TAG:=fc9d536 ##- Placeholder for the site-specific version control -CF_SRC_VERSION:=4.7.0 +##- This information is used for application.properties +##- +CF_SRC_VERSION:=4.7.0-$(CF_SRC_TAG) -include $(TOP)/../RELEASE.local From 6eb928a86f6b00e2a679aeaf40c35ebf294dbe40 Mon Sep 17 00:00:00 2001 From: Jeong Han Lee Date: Wed, 21 Sep 2022 21:40:34 -0700 Subject: [PATCH 44/89] works with the Debian 11 systemd --- .gitignore | 1 + README.md | 27 ++++++++++++- configure/CONFIG_COMMON | 2 + configure/CONFIG_SITE | 15 ++++++-- configure/RULES_INSTALL | 1 + configure/RULES_PROPERTIES | 11 +++--- configure/RULES_REQ | 38 ++++++++++++++++--- configure/RULES_SYSTEMD | 2 + site-template/application.properties.in | 16 +++++--- .../systemd/channelfinder.service.in | 4 +- 10 files changed, 94 insertions(+), 23 deletions(-) diff --git a/.gitignore b/.gitignore index f6201e6..028d1fb 100644 --- a/.gitignore +++ b/.gitignore @@ -44,3 +44,4 @@ site-template/application.properties *.swo channelfinder.service elasticsearch-8.2.0 +*.options diff --git a/README.md b/README.md index 3ecfeed..cbac741 100644 --- a/README.md +++ b/README.md @@ -15,12 +15,27 @@ git make sudo tree maven jq ## Elasticsearch +Three configurations are changed for the elasticsearch. + +* Elasticsearch JVM options : [default] `-Xms1g -Xmx1g` +* Elasticsearch `pack.security.enabled` option : [default] false +* Elasticsearch configuration location : [default] `/etc/elasticsearch` + +These options can be changed through the variables `CF_ES_JAVA_OPTS`, `CF_ES_CONF_PATH`, and `CF_ES_XPACK_SECURITY` in `configure/CONFIG_SITE`. + ``` make install.esdeb or install.esrpm +make conf.es +make conf.es.show make start.es make status.es ``` +Note that `conf.es` will use the `pack.security.enabled: false` in `elasticsearch.yml`, where is in `/etc/elasticsearch`. +The log file `elasticsearch.log` is located in `/etc/elasticsearch`. + + + ## JAVA The following four variables must be defined. Please setup them according to one's systems configuration. @@ -40,15 +55,19 @@ It will works with other systems. Please check github action workflows. ```bash make init -make conf make build make install make sd_start make sd_status make mapping -make mapping.clean ``` +ChannelFinder log is shown in `/var/log/syslog`. + +```bash +tail -f /var/log/syslog +``` + ## macOS (tested with aarch64 with brew) ```bash @@ -61,6 +80,10 @@ make mapping make mapping.clean ``` +## ChannelFinder Configuration + +Please see [docs/ChannelFinderConf.md](docs/ChannelFinderConf.md). + ## Run Demo Please see [docs/ChannelFinderDemo.md](docs/ChannelFinderDemo.md). diff --git a/configure/CONFIG_COMMON b/configure/CONFIG_COMMON index 276d3a5..338d0e5 100644 --- a/configure/CONFIG_COMMON +++ b/configure/CONFIG_COMMON @@ -10,6 +10,8 @@ MAVEN_PATH:=$(MAVEN_HOME)/bin ES_HOST:=localhost ES_PORT:=9200 +ES_USERID:=elasticsearch +ES_GROUPID:=elasticsearch CF_PRTO:=http CF_HOST:=localhost diff --git a/configure/CONFIG_SITE b/configure/CONFIG_SITE index 82858ee..5b9383f 100644 --- a/configure/CONFIG_SITE +++ b/configure/CONFIG_SITE @@ -4,14 +4,17 @@ CF_INSTALL_PATH=/opt CF_INSTALL_LOCATION=${CF_INSTALL_PATH}/channelfinder +### Elasticsearch specific configuration +CF_ES_JAVA_OPTS:=-Xms1g -Xmx1g +CF_ES_CONF_PATH:=/etc/elasticsearch +CF_ES_XPACK_SECURITY:=false ### Placeholder ### This variable will be replaced by reading a pom.xml file. The version in pom.xml will define ### the exact version of the jar file. It is difficult to track that version to implement into here. ### It may be linked with SRC_VERSION in RELEASE file ### -### So, we use * to cover all version information with the jar file -CF_JAR_FILENAME:=ChannelFinder-*.jar +CF_JAR_FILENAME:=ChannelFinder-4.7.1-SNAPSHOT.jar CF_PROPERTIES:=application.properties @@ -20,7 +23,8 @@ CF_PROPERTIES:=application.properties ### CF_LDIF:=cf4als.ldif CF_SSHKEYALIAS:=cf -CF_SSHKEY:=$(CF_SSHKEYALIAS).p12 +CF_SSHKEY:=new$(CF_SSHKEYALIAS).p12 +CF_SSL_REQUIRED:=true CF_JAVA_PATH:=$(JAVA_PATH) CF_JAVA_OPTS:=-Xms512m -Xmx512m @@ -30,8 +34,13 @@ CF_JAVA_OPTS:=-Xms512m -Xmx512m CF_MVN_OPTS:= +# it will be good to have its own, reserve them for the future expansion +# currnetly, channelfinder log is in the /var/log/syslog +# CF_LOG_PATH:=$(CF_INSTALL_LOCATION) CF_LOG_FILE:=channelfinder.log + +# CF_LOG_LEVEL:=WARN CF_SITE_TEMPLATE_PATH=$(TOP)/site-template CF_CREATE_INDEX:=false diff --git a/configure/RULES_INSTALL b/configure/RULES_INSTALL index d355aa4..f5c28cb 100644 --- a/configure/RULES_INSTALL +++ b/configure/RULES_INSTALL @@ -12,6 +12,7 @@ cf_restart: sd_restart cf_install: $(QUIET)$(SUDO) install -d $(CF_INSTALL_LOCATION) $(QUIET)$(SUDO) install -m 744 $(CF_SRC_PATH)/target/$(CF_JAR_FILENAME) $(CF_INSTALL_LOCATION)/ + $(QUIET)$(SUDO) chown -R $(ES_USERID):$(ES_GROUPID) $(CF_INSTALL_LOCATION)/ src_install: cf_install diff --git a/configure/RULES_PROPERTIES b/configure/RULES_PROPERTIES index cdb3b7d..c8c871e 100644 --- a/configure/RULES_PROPERTIES +++ b/configure/RULES_PROPERTIES @@ -19,19 +19,20 @@ conf.properties: $(CF_PROPERTIES).in -e "s|@ES_HOST@|$(ES_HOST)|g" \ -e "s|@ES_PORT@|$(ES_PORT)|g" \ -e "s|@CF_QUERY_SIZE@|$(CF_QUERY_SIZE)|g" \ + -e "s|@CF_SSL_REQUIRED@|$(CF_SSL_REQUIRED)|g" \ -e "s|@CF_LOG_PATH@|$(CF_LOG_PATH)|g" \ -e "s|@CF_LOG_FILE@|$(CF_LOG_FILE)|g" \ -e "s|@CF_LOG_LEVEL@|$(CF_LOG_LEVEL)|g" \ -e "s|@CF_CREATE_INDEX@|$(CF_CREATE_INDEX)|g" \ -e "s|@CF_VERSION@|$(CF_SRC_VERSION)|g" \ < $< > $(basename $<) - rm -f $(CF_SITE_SPECIFIC_FILES_PATH)/$(notdir $(basename $<)) +# rm -f $(CF_SITE_SPECIFIC_FILES_PATH)/$(notdir $(basename $<)) $(QUIET)echo ">>> Copy $(basename $<) to $(CF_SITE_SPECIFIC_FILES_PATH)" $(INSTALL_DATA) $(basename $<) $(CF_SITE_SPECIFIC_FILES_PATH)/ - rm -rf $(CF_SITE_SPECIFIC_FILES_PATH)/$(CF_LDIF) - $(INSTALL_DATA) $(CF_SITE_TEMPLATE_PATH)/$(CF_LDIF) $(CF_SITE_SPECIFIC_FILES_PATH)/ - rm -rf $(CF_SITE_SPECIFIC_FILES_PATH)/keystore/$(CF_SSHKEY) - $(INSTALL_DATA) $(CF_SITE_TEMPLATE_PATH)/$(CF_SSHKEY) $(CF_SITE_SPECIFIC_FILES_PATH)/keystore/ +# rm -rf $(CF_SITE_SPECIFIC_FILES_PATH)/$(CF_LDIF) +# $(INSTALL_DATA) $(CF_SITE_TEMPLATE_PATH)/$(CF_LDIF) $(CF_SITE_SPECIFIC_FILES_PATH)/ +# rm -rf $(CF_SITE_SPECIFIC_FILES_PATH)/keystore/$(CF_SSHKEY) +# $(INSTALL_DATA) $(CF_SITE_TEMPLATE_PATH)/$(CF_SSHKEY) $(CF_SITE_SPECIFIC_FILES_PATH)/keystore/ conf.properties.show: $(CF_PROPERTIES).in $(QUIET)cat -b $(basename $<) diff --git a/configure/RULES_REQ b/configure/RULES_REQ index 617be9e..0e64306 100644 --- a/configure/RULES_REQ +++ b/configure/RULES_REQ @@ -3,8 +3,9 @@ es_RULES_NAMES:=esdeb esrpm esmac install_es_RULES:=$(addprefix install., $(es_RULES_NAMES)) setup_es_RULES:=$(addprefix setup., $(es_RULES_NAMES)) +remove_es_RULES:=$(addprefix remove., $(es_RULES_NAMES)) -.PHONY: start.es status.es all.es stop.es disable.es mapping mapping.clean mapping.settings +.PHONY: clean.conf.es conf.es start.es status.es restart.es stop.es disable.es mapping mapping.clean mapping.settings conf.es.show #https://www.elastic.co/guide/en/elasticsearch/reference/current/release-notes-8.2.0.html ES_NAME:=elasticsearch @@ -22,15 +23,36 @@ setup.esrpm: install.esrpm enable.es setup.esmac: install.esmac + +clean.conf.es: + $(QUIET) rm -f $(CF_SITE_TEMPLATE_PATH)/01.CF_ES_JVMS.options + +$(CF_ES_JAVA_OPTS): clean.conf.es + $(QUIET)echo "$@" >> $(CF_SITE_TEMPLATE_PATH)/01.CF_ES_JVMS.options + +## Is this only way? +conf.es: $(CF_ES_JAVA_OPTS) + $(QUIET)$(SUDO) $(INSTALL) -p -m 660 $(CF_SITE_TEMPLATE_PATH)/01.CF_ES_JVMS.options $(CF_ES_CONF_PATH)/jvm.options.d/ + $(QUIET)$(SUDO) sed -i~ 's/^xpack.security.enabled:.*/xpack.security.enabled: $(CF_ES_XPACK_SECURITY)/g' $(CF_ES_CONF_PATH)/elasticsearch.yml + +conf.es.show: + $(SUDO) cat -b $(CF_ES_CONF_PATH)/jvm.options.d/01.CF_ES_JVMS.options + $(SUDO) cat -b $(CF_ES_CONF_PATH)/elasticsearch.yml + install.esdeb: $(QUIET) wget -c https://artifacts.elastic.co/downloads/elasticsearch/$(ES_DEB) $(QUIET)$(SUDO) dpkg -i ./$(ES_DEB) +remove.esdeb: + $(QUIET)$(SUDO) apt purge elasticsearch + # Rocky 8 install.esrpm: $(QUIET) wget -c https://artifacts.elastic.co/downloads/elasticsearch/$(ES_RPM) $(QUIET)$(SUDO) dnf localinstall $(ES_DEB) +remove.esrpm: + $(QUIET)$(SUDO) dnf remove elasticsearch # macOS aarch64 install.esmac: @@ -38,6 +60,9 @@ install.esmac: $(QUIET) curl https://artifacts.elastic.co/downloads/elasticsearch/$(ES_MAC).sha512 | shasum -a 512 -c - $(QUIET) tar -xzf $(ES_MAC) +remove.esmac: + $(QUIET) rm -rf $(ES_FILE)* + enable.es: $(QUIET)$(SUDO) systemctl daemon-reload $(QUIET)$(SUDO) systemctl enable elasticsearch.service @@ -46,13 +71,16 @@ start.es: $(QUIET)$(SUDO) systemctl start elasticsearch.service status.es: - $(QUIET)systemctl status -l elasticsearch | cat + $(QUIET)systemctl status -l elasticsearch.service | cat -b stop.es: - $(QUIET)systemctl stop elasticsearch + $(QUIET)systemctl stop elasticsearch.service disable.es: - $(QUIET)systemctl disable elasticsearch + $(QUIET)systemctl disable elasticsearch.service + +restart.es: + $(QUIET)$(SUDO) systemctl restart elasticsearch.service ## Create the ES indexes and set up their mapping mapping: @@ -70,5 +98,3 @@ mapping.settings: $(QUIET)curl -XPUT "http://$(ES_HOST):$(ES_PORT)/cf_properties/_settings" -d '{ "index" : { "max_result_window" : $(CF_QUERY_SIZE) } }' -H "Content-Type: application/json" $(QUIET)curl -XPUT "http://$(ES_HOST):$(ES_PORT)/cf_tags/_settings" -d '{ "index" : { "max_result_window" : $(CF_QUERY_SIZE) } }' -H "Content-Type: application/json" -all.es: install.esdeb start.es status.es mapping - diff --git a/configure/RULES_SYSTEMD b/configure/RULES_SYSTEMD index 4d75e7b..8f27e7f 100644 --- a/configure/RULES_SYSTEMD +++ b/configure/RULES_SYSTEMD @@ -28,6 +28,8 @@ conf.systemd0: $(CF_SYSTEMD_FILENAME).in $(QUIET)echo ">>> Required Services : $(CF_SYSTEMD_SERVICES)" $(QUIET)sed -e "s|@DOCURL@|$(DOCURL)|g" \ -e 's|@SYSTEMD_SERVICES@|$(CF_SYSTEMD_SERVICES)|g' \ + -e "s|@ES_USERID@|$(ES_USERID)|g" \ + -e "s|@ES_GROUPID@|$(ES_GROUPID)|g" \ -e "s|@CF_JAVA_PATH@|$(CF_JAVA_PATH)|g" \ -e "s|@CF_JAVA_OPTS@|$(CF_JAVA_OPTS)|g" \ -e "s|@CF_INSTALL_LOCATION@|$(CF_INSTALL_LOCATION)|g" \ diff --git a/site-template/application.properties.in b/site-template/application.properties.in index 1031a07..196e9da 100644 --- a/site-template/application.properties.in +++ b/site-template/application.properties.in @@ -11,13 +11,12 @@ server.ssl.key-store=classpath:keystore/@SSHKEY@ server.ssl.key-store-password=password server.ssl.key-alias=@SSHKEYALIAS@ -security.require-ssl=true +security.require-ssl=@CF_SSL_REQUIRED@ logging.level.org.springframework.web=@CF_LOG_LEVEL@ - spring.http.log-request-details=true -logging.file=@CF_LOG_PATH@/@CF_LOG_FILE@ +#logging.file=@CF_LOG_PATH@/@CF_LOG_FILE@ # Default # logging.file.max-size=10MB @@ -31,7 +30,7 @@ ldap.groups.search.base = ou=Group ldap.groups.search.pattern = (memberUid= {1}) ############## LDAP - Embedded ############## -embedded_ldap.enabled = true +embedded_ldap.enabled = false embedded_ldap.urls = ldap://localhost:8389/dc=cf,dc=local embedded_ldap.base.dn = dc=cf,dc=local embedded_ldap.user.dn.pattern = uid={0},ou=People @@ -44,7 +43,14 @@ spring.ldap.embedded.validation.enabled=false ############## Demo Auth ############## -demo_auth.enabled = false +# users, pwds, roles - lists of comma-separated values (same length) +# roles may contain multiple roles for user separated by delimiter +# e.g. +# demo_auth.users = user1,user2 +# demo_auth.pwds = pwd1,pwd2 +# demo_auth.roles = role1,role2 +# demo_auth.roles = role1,role21:role22 +demo_auth.enabled = true demo_auth.delimiter.roles = : demo_auth.users = admin,user demo_auth.pwds = adminPass,userPass diff --git a/site-template/systemd/channelfinder.service.in b/site-template/systemd/channelfinder.service.in index 32a6e21..4edb9f5 100644 --- a/site-template/systemd/channelfinder.service.in +++ b/site-template/systemd/channelfinder.service.in @@ -5,8 +5,8 @@ After=syslog.target network.target @SYSTEMD_SERVICES@ Requires=@SYSTEMD_SERVICES@ [Service] -# User=elasticsearch -# Group=elasticsearch +User=@ES_USERID@ +Group=@ES_GROUPID@ ExecStart=@CF_JAVA_PATH@/java @CF_JAVA_OPTS@ -jar @CF_INSTALL_LOCATION@/@CF_JAR_NAME@ SuccessExitStatus=143 From dd8a52be0be676f235f7d9194bda019e48ff5c9a Mon Sep 17 00:00:00 2001 From: Jeong Han Lee Date: Wed, 21 Sep 2022 21:47:41 -0700 Subject: [PATCH 45/89] add elasticsearch installation, and linter for shell script --- .github/workflows/debian11.yml | 5 +++++ .github/workflows/rocky8.yml | 5 +++++ .github/workflows/ubuntu.yml | 7 ++++++- scripts/cf_queries.bash | 2 +- 4 files changed, 17 insertions(+), 2 deletions(-) diff --git a/.github/workflows/debian11.yml b/.github/workflows/debian11.yml index d46bc14..a443ecc 100644 --- a/.github/workflows/debian11.yml +++ b/.github/workflows/debian11.yml @@ -36,6 +36,11 @@ jobs: run: | which java which mvn + - name: Elasticsearch + run: | + make install.esdeb + make conf.es + make conf.es.show - name: Configuration run: | make init diff --git a/.github/workflows/rocky8.yml b/.github/workflows/rocky8.yml index 5fc9795..c0da264 100644 --- a/.github/workflows/rocky8.yml +++ b/.github/workflows/rocky8.yml @@ -36,6 +36,11 @@ jobs: run: | which java which mvn + - name: Elasticsearch + run: | + make install.esrpm + make conf.es + make conf.es.show - name: Configuration run: | make init diff --git a/.github/workflows/ubuntu.yml b/.github/workflows/ubuntu.yml index bf3b477..a78b4b7 100644 --- a/.github/workflows/ubuntu.yml +++ b/.github/workflows/ubuntu.yml @@ -28,11 +28,16 @@ jobs: distribution: 'zulu' java-version: '18.0.2' - name: Install required packages - run: sudo apt install -y make tree git + run: sudo apt install -y make tree git elasticsearch - name: Check JAVA configuration run: | which java which mvn + - name: Elasticsearch + run: | + make install.esdeb + make conf.es + make conf.es.show - name: Configuration run: | make init diff --git a/scripts/cf_queries.bash b/scripts/cf_queries.bash index 37cdc2b..cd68026 100755 --- a/scripts/cf_queries.bash +++ b/scripts/cf_queries.bash @@ -27,7 +27,7 @@ function cf_put local arg2="$1"; shift; local arg3="$1"; shift; - printf ">>> Putting ...%s... ...%s...\n" "$index" "$json"; + printf ">>> Putting ...%s... ...%s...\n" "$arg1" "$arg2"; if [ -x "$(which jq)" ]; then curl -s -H 'Content-Type: application/json' -u "${ADMIN}" --request PUT "${CF_URL}/${arg1}/${arg2}" -d "@${arg3}" | jq else From 17c68fef2141752731c8e74ec3024c5c0c62c309 Mon Sep 17 00:00:00 2001 From: Jeong Han Lee Date: Wed, 21 Sep 2022 21:48:41 -0700 Subject: [PATCH 46/89] ubuntu action --- .github/workflows/ubuntu.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ubuntu.yml b/.github/workflows/ubuntu.yml index a78b4b7..80a40d3 100644 --- a/.github/workflows/ubuntu.yml +++ b/.github/workflows/ubuntu.yml @@ -28,7 +28,7 @@ jobs: distribution: 'zulu' java-version: '18.0.2' - name: Install required packages - run: sudo apt install -y make tree git elasticsearch + run: sudo apt install -y make tree git - name: Check JAVA configuration run: | which java From a5b33cbfd35d7e0432127c6ee71cbf8e20b67860 Mon Sep 17 00:00:00 2001 From: Jeong Han Lee Date: Wed, 21 Sep 2022 21:51:06 -0700 Subject: [PATCH 47/89] rocky8 --- .github/workflows/rocky8.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/rocky8.yml b/.github/workflows/rocky8.yml index c0da264..0742303 100644 --- a/.github/workflows/rocky8.yml +++ b/.github/workflows/rocky8.yml @@ -31,7 +31,7 @@ jobs: - name: Install required packages run: | dnf update -y - dnf install -y git make sudo which maven + dnf install -y git make sudo which maven wget tree - name: Check JAVA configuration run: | which java From dddb2db530f89f7e800506cf90b2ccdc2dfb3193 Mon Sep 17 00:00:00 2001 From: Jeong Han Lee Date: Wed, 21 Sep 2022 21:51:27 -0700 Subject: [PATCH 48/89] add wget --- .github/workflows/debian11.yml | 2 +- .github/workflows/ubuntu.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/debian11.yml b/.github/workflows/debian11.yml index a443ecc..36bd422 100644 --- a/.github/workflows/debian11.yml +++ b/.github/workflows/debian11.yml @@ -31,7 +31,7 @@ jobs: - name: Install required packages run: | apt update -y - apt install -y git make sudo tree maven + apt install -y git make sudo tree maven wget - name: Check JAVA configuration run: | which java diff --git a/.github/workflows/ubuntu.yml b/.github/workflows/ubuntu.yml index 80a40d3..ad77292 100644 --- a/.github/workflows/ubuntu.yml +++ b/.github/workflows/ubuntu.yml @@ -28,7 +28,7 @@ jobs: distribution: 'zulu' java-version: '18.0.2' - name: Install required packages - run: sudo apt install -y make tree git + run: sudo apt install -y make tree git wget - name: Check JAVA configuration run: | which java From 5c71586be5bc5b48600d346343d51ee4374af91a Mon Sep 17 00:00:00 2001 From: Jeong Han Lee Date: Wed, 21 Sep 2022 21:54:53 -0700 Subject: [PATCH 49/89] fixed RPM name --- configure/RULES_REQ | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configure/RULES_REQ b/configure/RULES_REQ index 0e64306..929c843 100644 --- a/configure/RULES_REQ +++ b/configure/RULES_REQ @@ -49,7 +49,7 @@ remove.esdeb: # Rocky 8 install.esrpm: $(QUIET) wget -c https://artifacts.elastic.co/downloads/elasticsearch/$(ES_RPM) - $(QUIET)$(SUDO) dnf localinstall $(ES_DEB) + $(QUIET)$(SUDO) dnf localinstall $(ES_RPM) remove.esrpm: $(QUIET)$(SUDO) dnf remove elasticsearch From 8bc590f16a59dbc0cfc3382b8fbe19e5f7aa6c59 Mon Sep 17 00:00:00 2001 From: Jeong Han Lee Date: Wed, 21 Sep 2022 21:58:18 -0700 Subject: [PATCH 50/89] fixed dnf installation option --- configure/RULES_REQ | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/configure/RULES_REQ b/configure/RULES_REQ index 929c843..c79d96c 100644 --- a/configure/RULES_REQ +++ b/configure/RULES_REQ @@ -44,12 +44,12 @@ install.esdeb: $(QUIET)$(SUDO) dpkg -i ./$(ES_DEB) remove.esdeb: - $(QUIET)$(SUDO) apt purge elasticsearch + $(QUIET)$(SUDO) apt purge -y elasticsearch # Rocky 8 install.esrpm: $(QUIET) wget -c https://artifacts.elastic.co/downloads/elasticsearch/$(ES_RPM) - $(QUIET)$(SUDO) dnf localinstall $(ES_RPM) + $(QUIET)$(SUDO) dnf localinstall -y $(ES_RPM) remove.esrpm: $(QUIET)$(SUDO) dnf remove elasticsearch From bead8ce0349267055a0fbecac01d5d479509800c Mon Sep 17 00:00:00 2001 From: Jeong Han Lee Date: Wed, 21 Sep 2022 22:01:22 -0700 Subject: [PATCH 51/89] add tomcat warning to ChannelFinderConf.md --- docs/ChannelFinderConf.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/docs/ChannelFinderConf.md b/docs/ChannelFinderConf.md index 12f3dfa..7415e37 100644 --- a/docs/ChannelFinderConf.md +++ b/docs/ChannelFinderConf.md @@ -1,5 +1,15 @@ # ChannelFinder Configuration +## Tomcat generic service + +It is always better to check the Tomcat service first, because the current configuration uses `8080` as the channelfinder port. + +```bash +systemctl status tomcat{9} +systemctl stop tomcat{9} +systemctl disable tomcat{9} +``` + ## Create Indexes and add mapping information We have three index(s), which I don't use `indices`, because it is a slightly different definition in ES, such as From 35ac1d2a9ab1071e41c0c01afba37a048a19cf35 Mon Sep 17 00:00:00 2001 From: Jeong Han Lee Date: Thu, 22 Sep 2022 13:32:07 -0700 Subject: [PATCH 52/89] work with macOS now --- README.md | 15 +++++++++++++++ configure/RULES_CI | 12 ++++++------ configure/RULES_REQ | 15 ++++++++++++++- 3 files changed, 35 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index cbac741..bab9c29 100644 --- a/README.md +++ b/README.md @@ -23,6 +23,8 @@ Three configurations are changed for the elasticsearch. These options can be changed through the variables `CF_ES_JAVA_OPTS`, `CF_ES_CONF_PATH`, and `CF_ES_XPACK_SECURITY` in `configure/CONFIG_SITE`. +### Debian 11 / Rocky 8 + ``` make install.esdeb or install.esrpm make conf.es @@ -34,6 +36,19 @@ make status.es Note that `conf.es` will use the `pack.security.enabled: false` in `elasticsearch.yml`, where is in `/etc/elasticsearch`. The log file `elasticsearch.log` is located in `/etc/elasticsearch`. +### macOS (M1) + + +```bash +make install.esmac +make conf.macos +make conf.esmac +make conf.es.show +make start.esmac +make stop.esmac +``` + +The log file `elasticsearch.log` is located in the installation location log folder of the elastisserch. ## JAVA diff --git a/configure/RULES_CI b/configure/RULES_CI index 14d7328..5125996 100644 --- a/configure/RULES_CI +++ b/configure/RULES_CI @@ -5,14 +5,14 @@ conf.macos: conf.macbrew conf.macbrew: $(QUIET)echo ">>> Generating $(TOP)/configure/CONFIG_COMMON.local" - $(QUIET)-rm -f $(TOP)/configure/CONFIG_COMMON.local $(QUIET)echo "JAVA_HOME:=/opt/homebrew/opt/openjdk" > $(TOP)/configure/CONFIG_COMMON.local $(QUIET)echo "JAVA_PATH:=/opt/homebrew/opt/openjdk/bin" >> $(TOP)/configure/CONFIG_COMMON.local - $(QUIET)echo "MAVEN_HOME:=/opt/homebrew/opt/maven" >> $(TOP)/configure/CONFIG_COMMON.local - $(QUIET)echo "MAVEN_PATH:=/opt/homebrew/opt/maven/bin" >> $(TOP)/configure/CONFIG_COMMON.local + $(QUIET)echo "MAVEN_HOME:=/opt/homebrew/opt/maven" >> $(TOP)/configure/CONFIG_COMMON.local + $(QUIET)echo "MAVEN_PATH:=/opt/homebrew/opt/maven/bin" >> $(TOP)/configure/CONFIG_COMMON.local $(QUIET)cat -b $(TOP)/configure/CONFIG_COMMON.local $(QUIET)echo ">>> Generating $(TOP)/configure/CONFIG_SITE.local" - $(QUIET)-rm -f $(TOP)/configure/CONFIG_SITE.local - $(QUIET)echo "USERID:=${USER}" > $(TOP)/configure/CONFIG_SITE.local - $(QUIET)echo "GROUPID:=admin" >> $(TOP)/configure/CONFIG_SITE.local + $(QUIET)echo "USERID:=${USER}" > $(TOP)/configure/CONFIG_SITE.local + $(QUIET)echo "GROUPID:=admin" >> $(TOP)/configure/CONFIG_SITE.local + $(QUIET)echo "CF_ES_CONF_PATH:=$(TOP)/$(ES_HOME_MAC)/config" >> $(TOP)/configure/CONFIG_SITE.local $(QUIET)cat -b $(TOP)/configure/CONFIG_SITE.local + diff --git a/configure/RULES_REQ b/configure/RULES_REQ index c79d96c..191a4d4 100644 --- a/configure/RULES_REQ +++ b/configure/RULES_REQ @@ -5,7 +5,8 @@ install_es_RULES:=$(addprefix install., $(es_RULES_NAMES)) setup_es_RULES:=$(addprefix setup., $(es_RULES_NAMES)) remove_es_RULES:=$(addprefix remove., $(es_RULES_NAMES)) -.PHONY: clean.conf.es conf.es start.es status.es restart.es stop.es disable.es mapping mapping.clean mapping.settings conf.es.show +.PHONY: clean.conf.es conf.es start.es status.es restart.es stop.es disable.es mapping mapping.clean mapping.settings conf.es.show +.PHONE: start.esmac stop.esmac #https://www.elastic.co/guide/en/elasticsearch/reference/current/release-notes-8.2.0.html ES_NAME:=elasticsearch @@ -14,6 +15,8 @@ ES_FILE:=$(ES_NAME)-$(ES_VER) ES_DEB:=$(ES_FILE)-amd64.deb ES_RPM:=$(ES_FILE)-x86_64.rpm ES_MAC:=$(ES_FILE)-darwin-aarch64.tar.gz +ES_HOME_MAC:=$(ES_FILE) +ES_PID_MAC:=$(TOP)/$(ES_HOME_MAC)/es_mac_pid # Debian 11 @@ -35,6 +38,10 @@ conf.es: $(CF_ES_JAVA_OPTS) $(QUIET)$(SUDO) $(INSTALL) -p -m 660 $(CF_SITE_TEMPLATE_PATH)/01.CF_ES_JVMS.options $(CF_ES_CONF_PATH)/jvm.options.d/ $(QUIET)$(SUDO) sed -i~ 's/^xpack.security.enabled:.*/xpack.security.enabled: $(CF_ES_XPACK_SECURITY)/g' $(CF_ES_CONF_PATH)/elasticsearch.yml +conf.esmac: $(CF_ES_JAVA_OPTS) + $(QUIET) $(INSTALL) -p -m 660 $(CF_SITE_TEMPLATE_PATH)/01.CF_ES_JVMS.options $(CF_ES_CONF_PATH)/jvm.options.d/ + $(QUIET) echo "xpack.security.enabled: $(CF_ES_XPACK_SECURITY)" >> $(CF_ES_CONF_PATH)/elasticsearch.yml + conf.es.show: $(SUDO) cat -b $(CF_ES_CONF_PATH)/jvm.options.d/01.CF_ES_JVMS.options $(SUDO) cat -b $(CF_ES_CONF_PATH)/elasticsearch.yml @@ -70,12 +77,18 @@ enable.es: start.es: $(QUIET)$(SUDO) systemctl start elasticsearch.service +start.esmac: + $(QUIET) $(TOP)/$(ES_HOME_MAC)/bin/elasticsearch -d -p $(ES_PID_MAC) + status.es: $(QUIET)systemctl status -l elasticsearch.service | cat -b stop.es: $(QUIET)systemctl stop elasticsearch.service +stop.esmac: + $(QUIET)pkill -F $(ES_PID_MAC) + disable.es: $(QUIET)systemctl disable elasticsearch.service From 9dbeeb3e6134264b69fade2ce27c28056c856c32 Mon Sep 17 00:00:00 2001 From: Jeong Han Lee Date: Thu, 22 Sep 2022 14:11:34 -0700 Subject: [PATCH 53/89] release docker v2.0.0: --- docker/Dockerfile | 1 - 1 file changed, 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index cc2ba9b..3c2a6a5 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -35,7 +35,6 @@ RUN make distclean && \ make install.docker # make build MVN_OPTS=dependency:go-offline - ## Multi-Stages build ## Running docker image size : 240MB FROM alpine:3.16 From 756e1e7e1f9a409c153c07e29c0f354fb738bb28 Mon Sep 17 00:00:00 2001 From: Jeong Han Lee Date: Thu, 22 Sep 2022 15:03:39 -0700 Subject: [PATCH 54/89] release latet docker image --- .github/workflows/docker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index a3dba12..df5b3ec 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -17,7 +17,7 @@ jobs: DOCKER_FILE: docker/Dockerfile DOCKER_ACCOUNT: alscontrols DOCKER_REPO: channelfinder - DOCKER_TAG: v2.0.0 + DOCKER_TAG: latest steps: - name: checkout uses: actions/checkout@v3 From c9dd6f2b5c141ba4aea02b171a3de9d99f90e8f6 Mon Sep 17 00:00:00 2001 From: Jeong Han Lee Date: Thu, 22 Sep 2022 20:15:47 -0700 Subject: [PATCH 55/89] enable local ldap --- configure/RULES_PROPERTIES | 10 ++++++++-- configure/RULES_SRC | 2 -- site-template/application.properties.in | 2 +- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/configure/RULES_PROPERTIES b/configure/RULES_PROPERTIES index c8c871e..7e969d2 100644 --- a/configure/RULES_PROPERTIES +++ b/configure/RULES_PROPERTIES @@ -6,6 +6,12 @@ show_properties_RULES:=$(addsuffix .show, $(conf_properties_RULES)) RULES_VARS+=conf.cfproperties conf.cfproperties.show +.PHONY: conf conf.show + +conf: conf.cfproperties + +conf.show: conf.cfproperties.show + conf.cfproperties: $(conf_properties_RULES) conf.cfproperties.show: $(show_properties_RULES) @@ -30,9 +36,9 @@ conf.properties: $(CF_PROPERTIES).in $(QUIET)echo ">>> Copy $(basename $<) to $(CF_SITE_SPECIFIC_FILES_PATH)" $(INSTALL_DATA) $(basename $<) $(CF_SITE_SPECIFIC_FILES_PATH)/ # rm -rf $(CF_SITE_SPECIFIC_FILES_PATH)/$(CF_LDIF) -# $(INSTALL_DATA) $(CF_SITE_TEMPLATE_PATH)/$(CF_LDIF) $(CF_SITE_SPECIFIC_FILES_PATH)/ + $(INSTALL_DATA) $(CF_SITE_TEMPLATE_PATH)/$(CF_LDIF) $(CF_SITE_SPECIFIC_FILES_PATH)/ # rm -rf $(CF_SITE_SPECIFIC_FILES_PATH)/keystore/$(CF_SSHKEY) -# $(INSTALL_DATA) $(CF_SITE_TEMPLATE_PATH)/$(CF_SSHKEY) $(CF_SITE_SPECIFIC_FILES_PATH)/keystore/ + $(INSTALL_DATA) $(CF_SITE_TEMPLATE_PATH)/$(CF_SSHKEY) $(CF_SITE_SPECIFIC_FILES_PATH)/keystore/ conf.properties.show: $(CF_PROPERTIES).in $(QUIET)cat -b $(basename $<) diff --git a/configure/RULES_SRC b/configure/RULES_SRC index f532ccc..a8af6fe 100644 --- a/configure/RULES_SRC +++ b/configure/RULES_SRC @@ -52,6 +52,4 @@ endif build: conf $(QUIET) JAVA_HOME=$(JAVA_HOME) $(MAVEN_CMD) $(CF_MVN_OPTS) -f $(CF_SRC_PATH)/pom.xml clean install -Dmaven.test.skip=true -## Copy the site specific files into sources -conf: conf.cfproperties diff --git a/site-template/application.properties.in b/site-template/application.properties.in index 196e9da..eb42b6b 100644 --- a/site-template/application.properties.in +++ b/site-template/application.properties.in @@ -30,7 +30,7 @@ ldap.groups.search.base = ou=Group ldap.groups.search.pattern = (memberUid= {1}) ############## LDAP - Embedded ############## -embedded_ldap.enabled = false +embedded_ldap.enabled = true embedded_ldap.urls = ldap://localhost:8389/dc=cf,dc=local embedded_ldap.base.dn = dc=cf,dc=local embedded_ldap.user.dn.pattern = uid={0},ou=People From 20994422dc3b2a695853b86f3b240534253a7cfe Mon Sep 17 00:00:00 2001 From: Jeong Han Lee Date: Thu, 22 Sep 2022 20:16:29 -0700 Subject: [PATCH 56/89] fixed the local ldap configuration --- .github/workflows/docker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index df5b3ec..a3dba12 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -17,7 +17,7 @@ jobs: DOCKER_FILE: docker/Dockerfile DOCKER_ACCOUNT: alscontrols DOCKER_REPO: channelfinder - DOCKER_TAG: latest + DOCKER_TAG: v2.0.0 steps: - name: checkout uses: actions/checkout@v3 From 6b7171735c254091d0fb2a79acbc212e4a2d5cc9 Mon Sep 17 00:00:00 2001 From: Jeong Han Lee Date: Thu, 22 Sep 2022 20:17:07 -0700 Subject: [PATCH 57/89] release the latest version of docker --- .github/workflows/docker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index a3dba12..df5b3ec 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -17,7 +17,7 @@ jobs: DOCKER_FILE: docker/Dockerfile DOCKER_ACCOUNT: alscontrols DOCKER_REPO: channelfinder - DOCKER_TAG: v2.0.0 + DOCKER_TAG: latest steps: - name: checkout uses: actions/checkout@v3 From 60dedaedea21fd529fd4a85791108a32291f77df Mon Sep 17 00:00:00 2001 From: Jeong Han Lee Date: Sat, 24 Sep 2022 15:05:37 -0700 Subject: [PATCH 58/89] test with macOS --- .github/workflows/macOS.yml | 40 +++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 .github/workflows/macOS.yml diff --git a/.github/workflows/macOS.yml b/.github/workflows/macOS.yml new file mode 100644 index 0000000..6dc945c --- /dev/null +++ b/.github/workflows/macOS.yml @@ -0,0 +1,40 @@ +--- +name: macOS build + +on: + push: + branches: [master] + paths-ignores: + - '**.md' + - '.github/workflows/build.yml' + + pull_request: + branches: [master] + +jobs: + macOS11: + runs-on: macos-12 + steps: + - uses: actions/checkout@v3 + - name: Install required packages + run: | + brew install tree wget curl bashi jq + - name: ES Configuration + run: | + make install.esmac + make conf.macos + make conf.esmac + make start.esmac + - name: CF Configuration and build + run: | + make confi + make mapping + make mapping.verify + make build + - name: Environment Check + run: | + make demo + bash scripts/cf_queries.bash get tags foo + bash scripts/cf_queries.bash put tags foo scripts/tag_foo.json + bash scripts/cf_queries.bash get tags foo + From f440639cc74518277365c0bc70112ea232980e5d Mon Sep 17 00:00:00 2001 From: Jeong Han Lee Date: Sat, 24 Sep 2022 15:07:56 -0700 Subject: [PATCH 59/89] fixed typo macOS.yml --- .github/workflows/macOS.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/macOS.yml b/.github/workflows/macOS.yml index 6dc945c..896c115 100644 --- a/.github/workflows/macOS.yml +++ b/.github/workflows/macOS.yml @@ -18,7 +18,7 @@ jobs: - uses: actions/checkout@v3 - name: Install required packages run: | - brew install tree wget curl bashi jq + brew install tree wget curl bash jq - name: ES Configuration run: | make install.esmac From 3b57894ee469761ba722f5c999fd81a6de05b31b Mon Sep 17 00:00:00 2001 From: Jeong Han Lee Date: Sat, 24 Sep 2022 15:08:48 -0700 Subject: [PATCH 60/89] fixed typo again macOS --- .github/workflows/macOS.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/macOS.yml b/.github/workflows/macOS.yml index 896c115..bb7a3db 100644 --- a/.github/workflows/macOS.yml +++ b/.github/workflows/macOS.yml @@ -27,7 +27,7 @@ jobs: make start.esmac - name: CF Configuration and build run: | - make confi + make conf make mapping make mapping.verify make build From 5426ac9649e8a8af93fd46f2320b5885ff5db7f1 Mon Sep 17 00:00:00 2001 From: Jeong Han Lee Date: Sat, 24 Sep 2022 15:16:51 -0700 Subject: [PATCH 61/89] add the minimal intel mac support for github --- .github/workflows/macOS.yml | 2 +- configure/CONFIG | 3 +++ configure/RULES_REQ | 4 ++++ 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/.github/workflows/macOS.yml b/.github/workflows/macOS.yml index bb7a3db..b20273d 100644 --- a/.github/workflows/macOS.yml +++ b/.github/workflows/macOS.yml @@ -12,7 +12,7 @@ on: branches: [master] jobs: - macOS11: + macOS12: runs-on: macos-12 steps: - uses: actions/checkout@v3 diff --git a/configure/CONFIG b/configure/CONFIG index 48caf95..d0ef341 100644 --- a/configure/CONFIG +++ b/configure/CONFIG @@ -1,5 +1,8 @@ VARS_EXCLUDES := $(.VARIABLES) +UNAME_S:=$(shell uname -s) +UNAME_M:=$(shell uname -m) + include $(TOP)/configure/CONFIG_COMMON include $(TOP)/configure/RELEASE include $(TOP)/configure/CONFIG_SITE diff --git a/configure/RULES_REQ b/configure/RULES_REQ index 191a4d4..acf5419 100644 --- a/configure/RULES_REQ +++ b/configure/RULES_REQ @@ -14,7 +14,11 @@ ES_VER:=8.2.0 ES_FILE:=$(ES_NAME)-$(ES_VER) ES_DEB:=$(ES_FILE)-amd64.deb ES_RPM:=$(ES_FILE)-x86_64.rpm +ifeq ($(UNAME_M), arm64) ES_MAC:=$(ES_FILE)-darwin-aarch64.tar.gz +else +ES_MAC:=$(ES_FILE)-darwin-x86_64.tar.gz +endif ES_HOME_MAC:=$(ES_FILE) ES_PID_MAC:=$(TOP)/$(ES_HOME_MAC)/es_mac_pid From b9bf71244afa02da6adc6c589b945255b717a685 Mon Sep 17 00:00:00 2001 From: Jeong Han Lee Date: Sat, 24 Sep 2022 15:18:56 -0700 Subject: [PATCH 62/89] add make init for github macOS --- .github/workflows/macOS.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/macOS.yml b/.github/workflows/macOS.yml index b20273d..3375a78 100644 --- a/.github/workflows/macOS.yml +++ b/.github/workflows/macOS.yml @@ -21,6 +21,7 @@ jobs: brew install tree wget curl bash jq - name: ES Configuration run: | + make init make install.esmac make conf.macos make conf.esmac From c64ed6520c2b8853c35556ff7c40d80c5d4b637e Mon Sep 17 00:00:00 2001 From: Jeong Han Lee Date: Sat, 24 Sep 2022 15:24:46 -0700 Subject: [PATCH 63/89] add coreutils for macOS --- .github/workflows/macOS.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/macOS.yml b/.github/workflows/macOS.yml index 3375a78..e320656 100644 --- a/.github/workflows/macOS.yml +++ b/.github/workflows/macOS.yml @@ -18,7 +18,7 @@ jobs: - uses: actions/checkout@v3 - name: Install required packages run: | - brew install tree wget curl bash jq + brew install tree wget curl bash jq coreutils - name: ES Configuration run: | make init From d4b401e41453d29ba312db0e2f394a6c6a589c50 Mon Sep 17 00:00:00 2001 From: Jeong Han Lee Date: Sat, 24 Sep 2022 15:31:15 -0700 Subject: [PATCH 64/89] macOS github, brew java and maven --- .github/workflows/macOS.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/macOS.yml b/.github/workflows/macOS.yml index e320656..70d2ebd 100644 --- a/.github/workflows/macOS.yml +++ b/.github/workflows/macOS.yml @@ -18,7 +18,7 @@ jobs: - uses: actions/checkout@v3 - name: Install required packages run: | - brew install tree wget curl bash jq coreutils + brew install tree wget curl bash jq coreutils openjdk@18 maven - name: ES Configuration run: | make init From 24ab7c13e426c8ca0df9ded525c9a939674229b6 Mon Sep 17 00:00:00 2001 From: Jeong Han Lee Date: Sat, 24 Sep 2022 15:32:20 -0700 Subject: [PATCH 65/89] update README for macOS --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index bab9c29..0d47a88 100644 --- a/README.md +++ b/README.md @@ -39,6 +39,10 @@ The log file `elasticsearch.log` is located in `/etc/elasticsearch`. ### macOS (M1) +``` +brew install openjdk@18 +``` + ```bash make install.esmac make conf.macos From 82271bd02a8525b49f3dac2e63781dc4efdc51b1 Mon Sep 17 00:00:00 2001 From: Jeong Han Lee Date: Sat, 24 Sep 2022 15:40:35 -0700 Subject: [PATCH 66/89] macOS WIP --- .github/workflows/macOS.yml | 12 ++++++++++-- configure/RULES_REQ | 3 +++ 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/.github/workflows/macOS.yml b/.github/workflows/macOS.yml index 70d2ebd..f157a6e 100644 --- a/.github/workflows/macOS.yml +++ b/.github/workflows/macOS.yml @@ -18,17 +18,25 @@ jobs: - uses: actions/checkout@v3 - name: Install required packages run: | - brew install tree wget curl bash jq coreutils openjdk@18 maven - - name: ES Configuration + brew install tree wget curl bash jq coreutils + - name : Check JAVA Environment run: | + which java + which mvn + tree -L 2 /usr/local/opt/ + tree -L 2 /usr/local/Cellar/ + - name: ES Configuration + run: |i make init make install.esmac make conf.macos make conf.esmac make start.esmac + make status.esmac - name: CF Configuration and build run: | make conf + make status.esmac make mapping make mapping.verify make build diff --git a/configure/RULES_REQ b/configure/RULES_REQ index acf5419..6c339f8 100644 --- a/configure/RULES_REQ +++ b/configure/RULES_REQ @@ -87,6 +87,9 @@ start.esmac: status.es: $(QUIET)systemctl status -l elasticsearch.service | cat -b +status.esmac: + $(QUIET) cat -b $(TOP)/$(ES_HOME_MAC)/elasticsearch.log + stop.es: $(QUIET)systemctl stop elasticsearch.service From e2deac6ceed6ff740be3c02a7fc1cd279b588461 Mon Sep 17 00:00:00 2001 From: Jeong Han Lee Date: Sat, 24 Sep 2022 15:41:10 -0700 Subject: [PATCH 67/89] macOS WIP --- .github/workflows/macOS.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/macOS.yml b/.github/workflows/macOS.yml index f157a6e..02b7a52 100644 --- a/.github/workflows/macOS.yml +++ b/.github/workflows/macOS.yml @@ -26,7 +26,7 @@ jobs: tree -L 2 /usr/local/opt/ tree -L 2 /usr/local/Cellar/ - name: ES Configuration - run: |i + run: | make init make install.esmac make conf.macos From c2e98488fd3dd735c33a29121143d50476778f7d Mon Sep 17 00:00:00 2001 From: Jeong Han Lee Date: Sat, 24 Sep 2022 15:46:01 -0700 Subject: [PATCH 68/89] macos github action wip --- configure/RULES_CI | 8 ++++---- configure/RULES_REQ | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/configure/RULES_CI b/configure/RULES_CI index 5125996..ce69457 100644 --- a/configure/RULES_CI +++ b/configure/RULES_CI @@ -5,10 +5,10 @@ conf.macos: conf.macbrew conf.macbrew: $(QUIET)echo ">>> Generating $(TOP)/configure/CONFIG_COMMON.local" - $(QUIET)echo "JAVA_HOME:=/opt/homebrew/opt/openjdk" > $(TOP)/configure/CONFIG_COMMON.local - $(QUIET)echo "JAVA_PATH:=/opt/homebrew/opt/openjdk/bin" >> $(TOP)/configure/CONFIG_COMMON.local - $(QUIET)echo "MAVEN_HOME:=/opt/homebrew/opt/maven" >> $(TOP)/configure/CONFIG_COMMON.local - $(QUIET)echo "MAVEN_PATH:=/opt/homebrew/opt/maven/bin" >> $(TOP)/configure/CONFIG_COMMON.local + $(QUIET)echo "JAVA_HOME:=/usr/" > $(TOP)/configure/CONFIG_COMMON.local + $(QUIET)echo "JAVA_PATH:=/usr/bin" >> $(TOP)/configure/CONFIG_COMMON.local + $(QUIET)echo "MAVEN_HOME:=/usr/local" >> $(TOP)/configure/CONFIG_COMMON.local + $(QUIET)echo "MAVEN_PATH:=/user/local/bin" >> $(TOP)/configure/CONFIG_COMMON.local $(QUIET)cat -b $(TOP)/configure/CONFIG_COMMON.local $(QUIET)echo ">>> Generating $(TOP)/configure/CONFIG_SITE.local" $(QUIET)echo "USERID:=${USER}" > $(TOP)/configure/CONFIG_SITE.local diff --git a/configure/RULES_REQ b/configure/RULES_REQ index 6c339f8..124cc48 100644 --- a/configure/RULES_REQ +++ b/configure/RULES_REQ @@ -88,7 +88,7 @@ status.es: $(QUIET)systemctl status -l elasticsearch.service | cat -b status.esmac: - $(QUIET) cat -b $(TOP)/$(ES_HOME_MAC)/elasticsearch.log + $(QUIET) cat -b $(TOP)/$(ES_HOME_MAC)/logs/elasticsearch.log stop.es: $(QUIET)systemctl stop elasticsearch.service From 300e67a4024523ac3e50d72f6d433221cfc728ed Mon Sep 17 00:00:00 2001 From: Jeong Han Lee Date: Sat, 24 Sep 2022 15:52:43 -0700 Subject: [PATCH 69/89] macos github action wip --- .github/workflows/macOS.yml | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/.github/workflows/macOS.yml b/.github/workflows/macOS.yml index 02b7a52..63125fe 100644 --- a/.github/workflows/macOS.yml +++ b/.github/workflows/macOS.yml @@ -31,18 +31,19 @@ jobs: make install.esmac make conf.macos make conf.esmac - make start.esmac - make status.esmac - name: CF Configuration and build run: | make conf - make status.esmac - make mapping - make mapping.verify make build - name: Environment Check run: | + make start.esmac + sleep 10 + make status.esmac + make mapping + make mapping.verify make demo + sleep 10 bash scripts/cf_queries.bash get tags foo bash scripts/cf_queries.bash put tags foo scripts/tag_foo.json bash scripts/cf_queries.bash get tags foo From eca17cc6e1207858004347400f804dd02ea4282b Mon Sep 17 00:00:00 2001 From: Jeong Han Lee Date: Sat, 24 Sep 2022 16:04:12 -0700 Subject: [PATCH 70/89] macos action wip --- configure/RULES_CI | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configure/RULES_CI b/configure/RULES_CI index ce69457..6e1ef8f 100644 --- a/configure/RULES_CI +++ b/configure/RULES_CI @@ -8,7 +8,7 @@ conf.macbrew: $(QUIET)echo "JAVA_HOME:=/usr/" > $(TOP)/configure/CONFIG_COMMON.local $(QUIET)echo "JAVA_PATH:=/usr/bin" >> $(TOP)/configure/CONFIG_COMMON.local $(QUIET)echo "MAVEN_HOME:=/usr/local" >> $(TOP)/configure/CONFIG_COMMON.local - $(QUIET)echo "MAVEN_PATH:=/user/local/bin" >> $(TOP)/configure/CONFIG_COMMON.local + $(QUIET)echo "MAVEN_PATH:=/usr/local/bin" >> $(TOP)/configure/CONFIG_COMMON.local $(QUIET)cat -b $(TOP)/configure/CONFIG_COMMON.local $(QUIET)echo ">>> Generating $(TOP)/configure/CONFIG_SITE.local" $(QUIET)echo "USERID:=${USER}" > $(TOP)/configure/CONFIG_SITE.local From 8ffe8b31c0d4fd65288f962384cfe064c5a9e804 Mon Sep 17 00:00:00 2001 From: Jeong Han Lee Date: Sat, 24 Sep 2022 18:54:47 -0700 Subject: [PATCH 71/89] just compile, and install on macOS --- .github/workflows/macOS.yml | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/.github/workflows/macOS.yml b/.github/workflows/macOS.yml index 63125fe..06a3619 100644 --- a/.github/workflows/macOS.yml +++ b/.github/workflows/macOS.yml @@ -34,17 +34,9 @@ jobs: - name: CF Configuration and build run: | make conf + make vars make build + make cf_install - name: Environment Check - run: | - make start.esmac - sleep 10 - make status.esmac - make mapping - make mapping.verify - make demo - sleep 10 - bash scripts/cf_queries.bash get tags foo - bash scripts/cf_queries.bash put tags foo scripts/tag_foo.json - bash scripts/cf_queries.bash get tags foo - + run: make exist + From 1c103452e147edb79172ba1df677449baafdec41 Mon Sep 17 00:00:00 2001 From: Jeong Han Lee Date: Sat, 24 Sep 2022 21:20:09 -0700 Subject: [PATCH 72/89] add customized elasticsearch configuration --- .gitignore | 1 + README.md | 2 +- configure/CONFIG_SITE | 17 +++- configure/RULES_CI | 2 +- configure/RULES_REQ | 36 +++++++-- site-template/elasticsearch.yml.in | 121 +++++++++++++++++++++++++++++ 6 files changed, 169 insertions(+), 10 deletions(-) create mode 100644 site-template/elasticsearch.yml.in diff --git a/.gitignore b/.gitignore index 028d1fb..d8d449e 100644 --- a/.gitignore +++ b/.gitignore @@ -45,3 +45,4 @@ site-template/application.properties channelfinder.service elasticsearch-8.2.0 *.options +*.yml diff --git a/README.md b/README.md index 0d47a88..c9d7341 100644 --- a/README.md +++ b/README.md @@ -47,7 +47,7 @@ brew install openjdk@18 make install.esmac make conf.macos make conf.esmac -make conf.es.show +make conf.esmac.show make start.esmac make stop.esmac ``` diff --git a/configure/CONFIG_SITE b/configure/CONFIG_SITE index 5b9383f..37c65c4 100644 --- a/configure/CONFIG_SITE +++ b/configure/CONFIG_SITE @@ -6,8 +6,21 @@ CF_INSTALL_LOCATION=${CF_INSTALL_PATH}/channelfinder ### Elasticsearch specific configuration CF_ES_JAVA_OPTS:=-Xms1g -Xmx1g -CF_ES_CONF_PATH:=/etc/elasticsearch -CF_ES_XPACK_SECURITY:=false +CF_ES_PATH:=/etc/elasticsearch +CF_ES_CONF_NAME:=elasticsearch.yml + +CF_ES_CLUSTER_NAME:=ALSCluster1 +CF_ES_NODE_NAME:=ALS ES Node +CF_ES_CONF_PATH=$(CF_ES_PATH)/config +CF_ES_PATH_DATA=$(CF_ES_PATH)/data +CF_ES_PATH_LOGS=$(CF_ES_PATH)/logs +CF_ES_DISCOVERY_TYPE:="single-node" +CF_ES_XPACK_SECURITY_ENABLED:=false +CF_ES_XPACK_SECURITY_ENROLL_ENABLED:=true +CF_ES_XPACK_SECURITY_HTTP_SSL:=false +CF_ES_XPACK_SECURITY_TRANSPORT_SSL:=false +CF_ES_XPACK_SECURITY_ENABLED:=false +CF_ES_HTTP_HOST:=0.0.0.0 ### Placeholder ### This variable will be replaced by reading a pom.xml file. The version in pom.xml will define diff --git a/configure/RULES_CI b/configure/RULES_CI index 6e1ef8f..44feb5b 100644 --- a/configure/RULES_CI +++ b/configure/RULES_CI @@ -13,6 +13,6 @@ conf.macbrew: $(QUIET)echo ">>> Generating $(TOP)/configure/CONFIG_SITE.local" $(QUIET)echo "USERID:=${USER}" > $(TOP)/configure/CONFIG_SITE.local $(QUIET)echo "GROUPID:=admin" >> $(TOP)/configure/CONFIG_SITE.local - $(QUIET)echo "CF_ES_CONF_PATH:=$(TOP)/$(ES_HOME_MAC)/config" >> $(TOP)/configure/CONFIG_SITE.local + $(QUIET)echo "CF_ES_PATH:=$(TOP)/$(ES_HOME_MAC)" >> $(TOP)/configure/CONFIG_SITE.local $(QUIET)cat -b $(TOP)/configure/CONFIG_SITE.local diff --git a/configure/RULES_REQ b/configure/RULES_REQ index 124cc48..1c5f474 100644 --- a/configure/RULES_REQ +++ b/configure/RULES_REQ @@ -1,11 +1,11 @@ -es_RULES_NAMES:=esdeb esrpm esmac +es_RULES_NAMES:=esdeb esrpm esmac esyml install_es_RULES:=$(addprefix install., $(es_RULES_NAMES)) setup_es_RULES:=$(addprefix setup., $(es_RULES_NAMES)) remove_es_RULES:=$(addprefix remove., $(es_RULES_NAMES)) -.PHONY: clean.conf.es conf.es start.es status.es restart.es stop.es disable.es mapping mapping.clean mapping.settings conf.es.show +.PHONY: clean.conf.es conf.es start.es status.es restart.es stop.es disable.es mapping mapping.clean mapping.settings conf.es.show conf.esmac.show .PHONE: start.esmac stop.esmac #https://www.elastic.co/guide/en/elasticsearch/reference/current/release-notes-8.2.0.html @@ -37,14 +37,35 @@ clean.conf.es: $(CF_ES_JAVA_OPTS): clean.conf.es $(QUIET)echo "$@" >> $(CF_SITE_TEMPLATE_PATH)/01.CF_ES_JVMS.options -## Is this only way? conf.es: $(CF_ES_JAVA_OPTS) $(QUIET)$(SUDO) $(INSTALL) -p -m 660 $(CF_SITE_TEMPLATE_PATH)/01.CF_ES_JVMS.options $(CF_ES_CONF_PATH)/jvm.options.d/ - $(QUIET)$(SUDO) sed -i~ 's/^xpack.security.enabled:.*/xpack.security.enabled: $(CF_ES_XPACK_SECURITY)/g' $(CF_ES_CONF_PATH)/elasticsearch.yml + $(QUIET)$(SUDO) $(INSTALL) -p -m 640 $(CF_SITE_TEMPLATE_PATH)/$(CF_ES_CONF_NAME) $(CF_ES_CONF_PATH)/ -conf.esmac: $(CF_ES_JAVA_OPTS) +conf.esmac: $(CF_ES_JAVA_OPTS) conf.esyml + $(QUIET) echo ">>> Installing 01.CF_ES_JVM.options and $(CF_ES_CONF_NAME) to $(CF_ES_CONF_PATH)" $(QUIET) $(INSTALL) -p -m 660 $(CF_SITE_TEMPLATE_PATH)/01.CF_ES_JVMS.options $(CF_ES_CONF_PATH)/jvm.options.d/ - $(QUIET) echo "xpack.security.enabled: $(CF_ES_XPACK_SECURITY)" >> $(CF_ES_CONF_PATH)/elasticsearch.yml + $(QUIET) $(INSTALL) -p -m 640 $(CF_SITE_TEMPLATE_PATH)/$(CF_ES_CONF_NAME) $(CF_ES_CONF_PATH)/ + +conf.esyml: $(CF_ES_CONF_NAME).in + $(QUIET)echo ">>> Generate $(basename $<) from $<" + $(QUIET)sed -e "s|@CF_ES_CLUSTER_NAME@|$(CF_ES_CLUSTER_NAME)|g" \ + -e "s|@CF_ES_NODE_NAME@|$(CF_ES_NODE_NAME)|g" \ + -e "s|@CF_ES_PATH_DATA@|$(CF_ES_PATH_DATA)|g" \ + -e "s|@CF_ES_PATH_LOGS@|$(CF_ES_PATH_LOGS)|g" \ + -e "s|@CF_ES_DISCOVERY_TYPE@|$(CF_ES_DISCOVERY_TYPE)|g" \ + -e "s|@CF_ES_XPACK_SECURITY_ENABLED@|$(CF_ES_XPACK_SECURITY_ENABLED)|g" \ + -e "s|@CF_ES_XPACK_SECURITY_ENROLL_ENABLED@|$(CF_ES_XPACK_SECURITY_ENROLL_ENABLED)|g" \ + -e "s|@CF_ES_XPACK_SECURITY_HTTP_SSL@|$(CF_ES_XPACK_SECURITY_HTTP_SSL)|g" \ + -e "s|@CF_ES_XPACK_SECURITY_TRANSPORT_SSL@|$(CF_ES_XPACK_SECURITY_TRANSPORT_SSL)|g" \ + -e "s|@CF_ES_HTTP_HOST@|$(CF_ES_HTTP_HOST)|g" \ + < $< > $(basename $<) + +conf.esyml.show: $(CF_ES_CONF_NAME).in + $(QUIET)cat -b $(basename $<) + +conf.esmac.show: + cat -b $(CF_ES_CONF_PATH)/jvm.options.d/01.CF_ES_JVMS.options + cat -b $(CF_ES_CONF_PATH)/elasticsearch.yml conf.es.show: $(SUDO) cat -b $(CF_ES_CONF_PATH)/jvm.options.d/01.CF_ES_JVMS.options @@ -84,6 +105,9 @@ start.es: start.esmac: $(QUIET) $(TOP)/$(ES_HOME_MAC)/bin/elasticsearch -d -p $(ES_PID_MAC) +start.esmac.console: + $(QUIET) $(TOP)/$(ES_HOME_MAC)/bin/elasticsearch -p $(ES_PID_MAC) + status.es: $(QUIET)systemctl status -l elasticsearch.service | cat -b diff --git a/site-template/elasticsearch.yml.in b/site-template/elasticsearch.yml.in new file mode 100644 index 0000000..30fb9d1 --- /dev/null +++ b/site-template/elasticsearch.yml.in @@ -0,0 +1,121 @@ +# ======================== Elasticsearch Configuration ========================= +# +# NOTE: Elasticsearch comes with reasonable defaults for most settings. +# Before you set out to tweak and tune the configuration, make sure you +# understand what are you trying to accomplish and the consequences. +# +# The primary way of configuring a node is via this file. This template lists +# the most important settings you may want to configure for a production cluster. +# +# Please consult the documentation for further information on configuration options: +# https://www.elastic.co/guide/en/elasticsearch/reference/index.html +# +# ---------------------------------- Cluster ----------------------------------- +# +# Use a descriptive name for your cluster: +# +cluster.name: @CF_ES_CLUSTER_NAME@ +# +# ------------------------------------ Node ------------------------------------ +# +# Use a descriptive name for the node: +# +node.name: "@CF_ES_NODE_NAME@" +#node.roles: [ master ] +# +# ----------------------------------- Paths ------------------------------------ +# +# Path to directory where to store the data (separate multiple locations by comma): +# +path.data: @CF_ES_PATH_DATA@ +# +# Path to log files: +# +path.logs: @CF_ES_PATH_LOGS@ +# +# ----------------------------------- Memory ----------------------------------- +# +# Lock the memory on startup: +# +#bootstrap.memory_lock: true +# +# Make sure that the heap size is set to about half the memory available +# on the system and that the owner of the process is allowed to use this +# limit. +# +# Elasticsearch performs poorly when the system is swapping the memory. +# +# ---------------------------------- Network ----------------------------------- +# +# By default Elasticsearch is only accessible on localhost. Set a different +# address here to expose this node on the network: +# +#network.host: 192.168.0.1 +# +# By default Elasticsearch listens for HTTP traffic on the first free port it +# finds starting at 9200. Set a specific HTTP port here: +# +#http.port: 9200 +# +# For more information, consult the network module documentation. +# +# --------------------------------- Discovery ---------------------------------- +# +# Pass an initial list of hosts to perform discovery when this node is started: +# The default list of hosts is ["127.0.0.1", "[::1]"] +# +#discovery.seed_hosts: ["host1", "host2"] +discovery.type: "@CF_ES_DISCOVERY_TYPE@" +# +# Bootstrap the cluster using an initial set of master-eligible nodes: +# +#cluster.initial_master_nodes: ["node-1", "node-2"] +# +# For more information, consult the discovery and cluster formation module documentation. +# +# --------------------------------- Readiness ---------------------------------- +# +# Enable an unauthenticated TCP readiness endpoint on localhost +# +#readiness.port: 9399 +# +# ---------------------------------- Various ----------------------------------- +# +# Allow wildcard deletion of indices: +# +#action.destructive_requires_name: false + +#----------------------- BEGIN SECURITY AUTO CONFIGURATION ----------------------- +# +# The following settings, TLS certificates, and keys have been automatically +# generated to configure Elasticsearch security features on 02-08-2022 16:30:54 +# +# -------------------------------------------------------------------------------- + +# Enable security features +xpack.security.enabled: @CF_ES_XPACK_SECURITY_ENABLED@ +xpack.security.enrollment.enabled: @CF_ES_XPACK_SECURITY_ENROLL_ENABLED@ + +# Enable encryption for HTTP API client connections, such as Kibana, Logstash, and Agents +xpack.security.http.ssl.enabled: @CF_ES_XPACK_SECURITY_HTTP_SSL@ +# config/certs +#xpack.security.http.ssl.keystore.path: certs/http.p12 + +# Enable encryption and mutual authentication between cluster nodes +xpack.security.transport.ssl.enabled: @CF_ES_XPACK_SECURITY_TRANSPORT_SSL@ +#xpack.security.transport.ssl.verification_mode: certificate +#xpack.security.transport.ssl.keystore.path: certs/transport.p12 +#xpack.security.transport.ssl.truststore.path: certs/transport.p12 +# Create a new cluster with the current node only +# Additional nodes can still join the cluster later +#cluster.initial_master_nodes: ["fedora"] + +# Allow HTTP API connections from anywhere +# Connections are encrypted and require user authentication +http.host: @CF_ES_HTTP_HOST@ + +# Allow other nodes to join the cluster from anywhere +# Connections are encrypted and mutually authenticated +#transport.host: 0.0.0.0 + +#----------------------- END SECURITY AUTO CONFIGURATION ------------------------- From 40b2a3acca6e3bf3fa3bdb73a2c5361fa8f7f0f2 Mon Sep 17 00:00:00 2001 From: Jeong Han Lee Date: Sat, 24 Sep 2022 22:34:08 -0700 Subject: [PATCH 73/89] change default elasticsearch path --- configure/CONFIG_SITE | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configure/CONFIG_SITE b/configure/CONFIG_SITE index 37c65c4..e91b817 100644 --- a/configure/CONFIG_SITE +++ b/configure/CONFIG_SITE @@ -6,7 +6,7 @@ CF_INSTALL_LOCATION=${CF_INSTALL_PATH}/channelfinder ### Elasticsearch specific configuration CF_ES_JAVA_OPTS:=-Xms1g -Xmx1g -CF_ES_PATH:=/etc/elasticsearch +CF_ES_PATH:=/usr/share/elasticsearch CF_ES_CONF_NAME:=elasticsearch.yml CF_ES_CLUSTER_NAME:=ALSCluster1 From 2263a48b9295254bb63dd1ec131fbad472385790 Mon Sep 17 00:00:00 2001 From: Jeong Han Lee Date: Sat, 24 Sep 2022 22:47:01 -0700 Subject: [PATCH 74/89] fixed Linux elasticsearch.yml configuration --- configure/CONFIG_SITE | 24 ++++++++++++------------ configure/RULES_REQ | 2 +- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/configure/CONFIG_SITE b/configure/CONFIG_SITE index e91b817..8d67dba 100644 --- a/configure/CONFIG_SITE +++ b/configure/CONFIG_SITE @@ -6,21 +6,21 @@ CF_INSTALL_LOCATION=${CF_INSTALL_PATH}/channelfinder ### Elasticsearch specific configuration CF_ES_JAVA_OPTS:=-Xms1g -Xmx1g -CF_ES_PATH:=/usr/share/elasticsearch +CF_ES_PATH:=/etc/elasticsearch CF_ES_CONF_NAME:=elasticsearch.yml -CF_ES_CLUSTER_NAME:=ALSCluster1 -CF_ES_NODE_NAME:=ALS ES Node -CF_ES_CONF_PATH=$(CF_ES_PATH)/config -CF_ES_PATH_DATA=$(CF_ES_PATH)/data -CF_ES_PATH_LOGS=$(CF_ES_PATH)/logs -CF_ES_DISCOVERY_TYPE:="single-node" +#CF_ES_CLUSTER_NAME:=ALSCluster1 +#CF_ES_NODE_NAME:=ALS ES Node +CF_ES_CONF_PATH=$(CF_ES_PATH) +#CF_ES_PATH_DATA=$(CF_ES_PATH)/data +#CF_ES_PATH_LOGS=$(CF_ES_PATH)/logs +#CF_ES_DISCOVERY_TYPE:="single-node" CF_ES_XPACK_SECURITY_ENABLED:=false -CF_ES_XPACK_SECURITY_ENROLL_ENABLED:=true -CF_ES_XPACK_SECURITY_HTTP_SSL:=false -CF_ES_XPACK_SECURITY_TRANSPORT_SSL:=false -CF_ES_XPACK_SECURITY_ENABLED:=false -CF_ES_HTTP_HOST:=0.0.0.0 +#CF_ES_XPACK_SECURITY_ENROLL_ENABLED:=true +#CF_ES_XPACK_SECURITY_HTTP_SSL:=false +#CF_ES_XPACK_SECURITY_TRANSPORT_SSL:=false +#CF_ES_XPACK_SECURITY_ENABLED:=false +#CF_ES_HTTP_HOST:=0.0.0.0 ### Placeholder ### This variable will be replaced by reading a pom.xml file. The version in pom.xml will define diff --git a/configure/RULES_REQ b/configure/RULES_REQ index 1c5f474..d87f6da 100644 --- a/configure/RULES_REQ +++ b/configure/RULES_REQ @@ -39,7 +39,7 @@ $(CF_ES_JAVA_OPTS): clean.conf.es conf.es: $(CF_ES_JAVA_OPTS) $(QUIET)$(SUDO) $(INSTALL) -p -m 660 $(CF_SITE_TEMPLATE_PATH)/01.CF_ES_JVMS.options $(CF_ES_CONF_PATH)/jvm.options.d/ - $(QUIET)$(SUDO) $(INSTALL) -p -m 640 $(CF_SITE_TEMPLATE_PATH)/$(CF_ES_CONF_NAME) $(CF_ES_CONF_PATH)/ + $(QUIET)$(SUDO) sed -i~ 's/^xpack.security.enabled:.*/xpack.security.enabled: $(CF_ES_XPACK_SECURITY_ENABLED)/g' $(CF_ES_CONF_PATH)/$(CF_ES_CONF_NAME) conf.esmac: $(CF_ES_JAVA_OPTS) conf.esyml $(QUIET) echo ">>> Installing 01.CF_ES_JVM.options and $(CF_ES_CONF_NAME) to $(CF_ES_CONF_PATH)" From c77d676e6da148c617bdb39442405ce996907da4 Mon Sep 17 00:00:00 2001 From: Jeong Han Lee Date: Tue, 27 Sep 2022 14:47:55 -0700 Subject: [PATCH 75/89] update fully elasticsearch.yml configuration, and readme in several area --- README.md | 2 ++ configure/CONFIG_COMMON | 7 ++++- configure/CONFIG_SITE | 39 +++++++++++++++++-------- configure/RULES_REQ | 46 ++++++++++++++++++------------ configure/RULES_RUN | 8 +++++- docs/ChannelFinderConf.md | 18 ++++++++++++ docs/ESUpgrade.md | 21 ++++++++++++++ site-template/elasticsearch.yml.in | 20 ++++++------- 8 files changed, 117 insertions(+), 44 deletions(-) create mode 100644 docs/ESUpgrade.md diff --git a/README.md b/README.md index c9d7341..99c8769 100644 --- a/README.md +++ b/README.md @@ -110,6 +110,8 @@ Please see [docs/ChannelFinderDemo.md](docs/ChannelFinderDemo.md). ``` make run +make run.status +make run.kill make demo make demo.clean ``` diff --git a/configure/CONFIG_COMMON b/configure/CONFIG_COMMON index 338d0e5..6a03a8b 100644 --- a/configure/CONFIG_COMMON +++ b/configure/CONFIG_COMMON @@ -1,6 +1,11 @@ # Debian -JAVA_HOME:=/usr/lib/jvm/default-java + +# /usr/lib/jvm/java-17-openjdk-amd64/ +# Sometime, debian default-java symlink doesn't update even if +# update-alternative --config java or update-java-alternative +# one should change it manually. +JAVA_HOME:=/usr MAVEN_HOME:=/usr/share/maven JAVA_PATH:=$(JAVA_HOME)/bin diff --git a/configure/CONFIG_SITE b/configure/CONFIG_SITE index 8d67dba..a77e605 100644 --- a/configure/CONFIG_SITE +++ b/configure/CONFIG_SITE @@ -5,23 +5,38 @@ CF_INSTALL_LOCATION=${CF_INSTALL_PATH}/channelfinder ### Elasticsearch specific configuration -CF_ES_JAVA_OPTS:=-Xms1g -Xmx1g CF_ES_PATH:=/etc/elasticsearch CF_ES_CONF_NAME:=elasticsearch.yml -#CF_ES_CLUSTER_NAME:=ALSCluster1 -#CF_ES_NODE_NAME:=ALS ES Node -CF_ES_CONF_PATH=$(CF_ES_PATH) -#CF_ES_PATH_DATA=$(CF_ES_PATH)/data -#CF_ES_PATH_LOGS=$(CF_ES_PATH)/logs -#CF_ES_DISCOVERY_TYPE:="single-node" +# Debian +# https://www.elastic.co/guide/en/elasticsearch/reference/8.2/important-settings.html#path-settings +CF_ES_CONF_PATH=/etc/elasticsearch + + +# +# elasticsearch.yml.in +CF_ES_NETWORK_HOST=127.0.0.1 +CF_ES_HTTP_HOST:=0.0.0.0 +CF_ES_CLUSTER_NAME:="als-es-prod" +CF_ES_NODE_NAME:="als-es-node" +CF_ES_DATA_PATH=/var/lib/elasticsearch +CF_ES_LOGS_PATH=/var/log/elasticsearch +CF_ES_DISCOVERY_TYPE:="single-node" + CF_ES_XPACK_SECURITY_ENABLED:=false -#CF_ES_XPACK_SECURITY_ENROLL_ENABLED:=true -#CF_ES_XPACK_SECURITY_HTTP_SSL:=false -#CF_ES_XPACK_SECURITY_TRANSPORT_SSL:=false -#CF_ES_XPACK_SECURITY_ENABLED:=false -#CF_ES_HTTP_HOST:=0.0.0.0 +CF_ES_XPACK_SECURITY_ENROLL_ENABLED:=true +CF_ES_XPACK_SECURITY_HTTP_SSL:=false +CF_ES_XPACK_SECURITY_HTTP_SSL_KEYSTORE_PATH:=certs/http.p12 +CF_ES_XPACK_SECURITY_TRANSPORT_SSL:=false +CF_ES_XPACK_SECURITY_TRANSPORT_SSL_VERIFICATION:=certificate +CF_ES_XPACK_SECURITY_TRANSPORT_SSL_KEYSTORE_PATH:=certs/transport.p12 +CF_ES_XPACK_SECURITY_TRANSPORT_SSL_TRUSTSTORE_PATH=certs/transport.p12 +# JVM heap dump path : +# CF logging disable : +# +CF_ES_JAVA_OPTS:=-Xms1g -Xmx1g +#-XX\:HeapDumpPath=$(CF_ES_DATA_PATH) -Xlog\:disable ### Placeholder ### This variable will be replaced by reading a pom.xml file. The version in pom.xml will define ### the exact version of the jar file. It is difficult to track that version to implement into here. diff --git a/configure/RULES_REQ b/configure/RULES_REQ index d87f6da..333be4b 100644 --- a/configure/RULES_REQ +++ b/configure/RULES_REQ @@ -21,7 +21,7 @@ ES_MAC:=$(ES_FILE)-darwin-x86_64.tar.gz endif ES_HOME_MAC:=$(ES_FILE) ES_PID_MAC:=$(TOP)/$(ES_HOME_MAC)/es_mac_pid - +ES_JAVA_OPTS_FILE:=cf_es_jvm.options # Debian 11 setup.esdeb: install.esdeb enable.es @@ -32,31 +32,41 @@ setup.esmac: install.esmac clean.conf.es: - $(QUIET) rm -f $(CF_SITE_TEMPLATE_PATH)/01.CF_ES_JVMS.options + $(QUIET) rm -f $(CF_SITE_TEMPLATE_PATH)/$(ES_JAVA_OPTS_FILE) $(CF_ES_JAVA_OPTS): clean.conf.es - $(QUIET)echo "$@" >> $(CF_SITE_TEMPLATE_PATH)/01.CF_ES_JVMS.options + $(QUIET)echo "$@" >> $(CF_SITE_TEMPLATE_PATH)/$(ES_JAVA_OPTS_FILE) + +#conf.es: $(CF_ES_JAVA_OPTS) +# $(QUIET)$(SUDO) $(INSTALL) -p -m 660 $(CF_SITE_TEMPLATE_PATH)/$(ES_JAVA_OPTS_FILE) $(CF_ES_CONF_PATH)/jvm.options.d/ +# $(QUIET)$(SUDO) sed -i~ 's/^xpack.security.enabled:.*/xpack.security.enabled: $(CF_ES_XPACK_SECURITY_ENABLED)/g' $(CF_ES_CONF_PATH)/$(CF_ES_CONF_NAME) -conf.es: $(CF_ES_JAVA_OPTS) - $(QUIET)$(SUDO) $(INSTALL) -p -m 660 $(CF_SITE_TEMPLATE_PATH)/01.CF_ES_JVMS.options $(CF_ES_CONF_PATH)/jvm.options.d/ - $(QUIET)$(SUDO) sed -i~ 's/^xpack.security.enabled:.*/xpack.security.enabled: $(CF_ES_XPACK_SECURITY_ENABLED)/g' $(CF_ES_CONF_PATH)/$(CF_ES_CONF_NAME) +conf.es: $(CF_ES_JAVA_OPTS) conf.esyml + $(QUIET)$(SUDO) $(INSTALL) -p -m 660 $(CF_SITE_TEMPLATE_PATH)/$(ES_JAVA_OPTS_FILE) $(CF_ES_CONF_PATH)/jvm.options.d/ + $(QUIET)$(SUDO) $(INSTALL) -p -m 660 $(CF_SITE_TEMPLATE_PATH)/$(CF_ES_CONF_NAME) $(CF_ES_CONF_PATH)/ conf.esmac: $(CF_ES_JAVA_OPTS) conf.esyml $(QUIET) echo ">>> Installing 01.CF_ES_JVM.options and $(CF_ES_CONF_NAME) to $(CF_ES_CONF_PATH)" - $(QUIET) $(INSTALL) -p -m 660 $(CF_SITE_TEMPLATE_PATH)/01.CF_ES_JVMS.options $(CF_ES_CONF_PATH)/jvm.options.d/ - $(QUIET) $(INSTALL) -p -m 640 $(CF_SITE_TEMPLATE_PATH)/$(CF_ES_CONF_NAME) $(CF_ES_CONF_PATH)/ + $(QUIET) $(INSTALL) -p -m 660 $(CF_SITE_TEMPLATE_PATH)/$(ES_JAVA_OPTS_FILE) $(CF_ES_CONF_PATH)/jvm.options.d/ + $(QUIET) $(INSTALL) -p -m 640 $(CF_SITE_TEMPLATE_PATH)/$(CF_ES_CONF_NAME) $(CF_ES_CONF_PATH)/ conf.esyml: $(CF_ES_CONF_NAME).in $(QUIET)echo ">>> Generate $(basename $<) from $<" - $(QUIET)sed -e "s|@CF_ES_CLUSTER_NAME@|$(CF_ES_CLUSTER_NAME)|g" \ + $(QUIET)sed -e "s|@CF_ES_NETWORK_HOST@|$(CF_ES_NETWORK_HOST)|g" \ + -e "s|@CF_ES_HTTP_HOST@|$(CF_ES_HTTP_HOST)|g" \ + -e "s|@CF_ES_CLUSTER_NAME@|$(CF_ES_CLUSTER_NAME)|g" \ -e "s|@CF_ES_NODE_NAME@|$(CF_ES_NODE_NAME)|g" \ - -e "s|@CF_ES_PATH_DATA@|$(CF_ES_PATH_DATA)|g" \ - -e "s|@CF_ES_PATH_LOGS@|$(CF_ES_PATH_LOGS)|g" \ + -e "s|@CF_ES_DATA_PATH@|$(CF_ES_DATA_PATH)|g" \ + -e "s|@CF_ES_LOGS_PATH@|$(CF_ES_LOGS_PATH)|g" \ -e "s|@CF_ES_DISCOVERY_TYPE@|$(CF_ES_DISCOVERY_TYPE)|g" \ -e "s|@CF_ES_XPACK_SECURITY_ENABLED@|$(CF_ES_XPACK_SECURITY_ENABLED)|g" \ -e "s|@CF_ES_XPACK_SECURITY_ENROLL_ENABLED@|$(CF_ES_XPACK_SECURITY_ENROLL_ENABLED)|g" \ -e "s|@CF_ES_XPACK_SECURITY_HTTP_SSL@|$(CF_ES_XPACK_SECURITY_HTTP_SSL)|g" \ + -e "s|@CF_ES_XPACK_SECURITY_HTTP_SSL_KEYSTORE_PATH@|$(CF_ES_XPACK_SECURITY_HTTP_SSL_KEYSTORE_PATH)|g" \ -e "s|@CF_ES_XPACK_SECURITY_TRANSPORT_SSL@|$(CF_ES_XPACK_SECURITY_TRANSPORT_SSL)|g" \ + -e "s|@CF_ES_XPACK_SECURITY_TRANSPORT_SSL_VERIFICATION@|$(CF_ES_XPACK_SECURITY_TRANSPORT_SSL_VERIFICATION)|g" \ + -e "s|@CF_ES_XPACK_SECURITY_TRANSPORT_SSL_KEYSTORE_PATH@|$(CF_ES_XPACK_SECURITY_TRANSPORT_SSL_KEYSTORE_PATH)|g" \ + -e "s|@CF_ES_XPACK_SECURITY_TRANSPORT_SSL_TRUSTSTORE_PATH@|$(CF_ES_XPACK_SECURITY_TRANSPORT_SSL_TRUSTSTORE_PATH)|g" \ -e "s|@CF_ES_HTTP_HOST@|$(CF_ES_HTTP_HOST)|g" \ < $< > $(basename $<) @@ -64,16 +74,16 @@ conf.esyml.show: $(CF_ES_CONF_NAME).in $(QUIET)cat -b $(basename $<) conf.esmac.show: - cat -b $(CF_ES_CONF_PATH)/jvm.options.d/01.CF_ES_JVMS.options - cat -b $(CF_ES_CONF_PATH)/elasticsearch.yml + cat -b $(CF_ES_CONF_PATH)/jvm.options.d/$(ES_JAVA_OPTS_FILE) + cat -b $(CF_ES_CONF_PATH)/$(CF_ES_CONF_NAME) conf.es.show: - $(SUDO) cat -b $(CF_ES_CONF_PATH)/jvm.options.d/01.CF_ES_JVMS.options - $(SUDO) cat -b $(CF_ES_CONF_PATH)/elasticsearch.yml + $(SUDO) cat -b $(CF_ES_CONF_PATH)/jvm.options.d/$(ES_JAVA_OPTS_FILE) + $(SUDO) cat -b $(CF_ES_CONF_PATH)/$(CF_ES_CONF_NAME) install.esdeb: $(QUIET) wget -c https://artifacts.elastic.co/downloads/elasticsearch/$(ES_DEB) - $(QUIET)$(SUDO) dpkg -i ./$(ES_DEB) + $(SUDO) dpkg -i ./$(ES_DEB) remove.esdeb: $(QUIET)$(SUDO) apt purge -y elasticsearch @@ -81,7 +91,7 @@ remove.esdeb: # Rocky 8 install.esrpm: $(QUIET) wget -c https://artifacts.elastic.co/downloads/elasticsearch/$(ES_RPM) - $(QUIET)$(SUDO) dnf localinstall -y $(ES_RPM) + $(SUDO) dnf localinstall -y $(ES_RPM) remove.esrpm: $(QUIET)$(SUDO) dnf remove elasticsearch @@ -90,7 +100,7 @@ remove.esrpm: install.esmac: $(QUIET) curl -O https://artifacts.elastic.co/downloads/elasticsearch/$(ES_MAC) $(QUIET) curl https://artifacts.elastic.co/downloads/elasticsearch/$(ES_MAC).sha512 | shasum -a 512 -c - - $(QUIET) tar -xzf $(ES_MAC) + tar -xzf $(ES_MAC) remove.esmac: $(QUIET) rm -rf $(ES_FILE)* diff --git a/configure/RULES_RUN b/configure/RULES_RUN index 0c8d5af..6b904db 100644 --- a/configure/RULES_RUN +++ b/configure/RULES_RUN @@ -1,6 +1,12 @@ run: - $(QUIET)mvn -f $(CF_SRC_PATH) spring-boot:run + $(QUIET)mvn -f $(CF_SRC_PATH) spring-boot:run -Dspring.pid.file=$(TOP)/channelfinder.pid + +run.check: + $(QUIET)lsof -i:8080 | xargs -I {} echo {} + +run.kill: + $(QUIET)lsof -i:8080 -t | xargs -I {} kill -9 {} demo: $(QUIET)mvn -f $(CF_SRC_PATH) spring-boot:run -Dspring-boot.run.arguments="--demo-data=1" diff --git a/docs/ChannelFinderConf.md b/docs/ChannelFinderConf.md index 7415e37..ac659aa 100644 --- a/docs/ChannelFinderConf.md +++ b/docs/ChannelFinderConf.md @@ -10,6 +10,24 @@ systemctl stop tomcat{9} systemctl disable tomcat{9} ``` +## Check other services hold 8080 + +```bash +lsof -i:8080 +``` + +``` +kill -i pid_number +``` +### a few rules + +The local run rule has the more options to check the process status, and kill the process which holds the port 8080. + +``` +make run.check +make run.kill +``` + ## Create Indexes and add mapping information We have three index(s), which I don't use `indices`, because it is a slightly different definition in ES, such as diff --git a/docs/ESUpgrade.md b/docs/ESUpgrade.md new file mode 100644 index 0000000..e9a383c --- /dev/null +++ b/docs/ESUpgrade.md @@ -0,0 +1,21 @@ +# New Elasticsearch configuration + +The following error tells us we have old elasticsearch configuration exists. + +```bash +[2022-09-27T11:22:59,729][ERROR][o.e.b.Bootstrap ] [als-es-node] Exception +java.lang.IllegalArgumentException: Could not load codec 'Lucene92'. Did you forget to add lucene-backward-codecs.jar? +``` + +* Solution + +We have to remove all existing file in `CF_ES_DATA_PATH`. For Debian, it is `/var/lib/elasticsearc` + +```bash +rm -rf /var/lib/elasticsearch +mkdir -p /var/lib/elasticsearch +chown elasticsearch:elasticsearch /var/lib/elasticsearch +systemctl restart elasticsearch +``` + + diff --git a/site-template/elasticsearch.yml.in b/site-template/elasticsearch.yml.in index 30fb9d1..8432760 100644 --- a/site-template/elasticsearch.yml.in +++ b/site-template/elasticsearch.yml.in @@ -21,17 +21,16 @@ cluster.name: @CF_ES_CLUSTER_NAME@ # Use a descriptive name for the node: # node.name: "@CF_ES_NODE_NAME@" -#node.roles: [ master ] # # ----------------------------------- Paths ------------------------------------ # # Path to directory where to store the data (separate multiple locations by comma): # -path.data: @CF_ES_PATH_DATA@ +path.data: @CF_ES_DATA_PATH@ # # Path to log files: # -path.logs: @CF_ES_PATH_LOGS@ +path.logs: @CF_ES_LOGS_PATH@ # # ----------------------------------- Memory ----------------------------------- # @@ -50,7 +49,7 @@ path.logs: @CF_ES_PATH_LOGS@ # By default Elasticsearch is only accessible on localhost. Set a different # address here to expose this node on the network: # -#network.host: 192.168.0.1 +network.host: @CF_ES_NETWORK_HOST@ # # By default Elasticsearch listens for HTTP traffic on the first free port it # finds starting at 9200. Set a specific HTTP port here: @@ -64,7 +63,6 @@ path.logs: @CF_ES_PATH_LOGS@ # Pass an initial list of hosts to perform discovery when this node is started: # The default list of hosts is ["127.0.0.1", "[::1]"] # -#discovery.seed_hosts: ["host1", "host2"] discovery.type: "@CF_ES_DISCOVERY_TYPE@" # # Bootstrap the cluster using an initial set of master-eligible nodes: @@ -95,17 +93,15 @@ discovery.type: "@CF_ES_DISCOVERY_TYPE@" # Enable security features xpack.security.enabled: @CF_ES_XPACK_SECURITY_ENABLED@ xpack.security.enrollment.enabled: @CF_ES_XPACK_SECURITY_ENROLL_ENABLED@ - -# Enable encryption for HTTP API client connections, such as Kibana, Logstash, and Agents xpack.security.http.ssl.enabled: @CF_ES_XPACK_SECURITY_HTTP_SSL@ -# config/certs -#xpack.security.http.ssl.keystore.path: certs/http.p12 +xpack.security.http.ssl.keystore.path: @CF_ES_XPACK_SECURITY_HTTP_SSL_KEYSTORE_PATH@ # Enable encryption and mutual authentication between cluster nodes xpack.security.transport.ssl.enabled: @CF_ES_XPACK_SECURITY_TRANSPORT_SSL@ -#xpack.security.transport.ssl.verification_mode: certificate -#xpack.security.transport.ssl.keystore.path: certs/transport.p12 -#xpack.security.transport.ssl.truststore.path: certs/transport.p12 +xpack.security.transport.ssl.verification_mode: @CF_ES_XPACK_SECURITY_TRANSPORT_SSL_VERIFICATION@ +xpack.security.transport.ssl.keystore.path: @CF_ES_XPACK_SECURITY_TRANSPORT_SSL_KEYSTORE_PATH@ +xpack.security.transport.ssl.truststore.path: @CF_ES_XPACK_SECURITY_TRANSPORT_SSL_TRUSTSTORE_PATH@ + # Create a new cluster with the current node only # Additional nodes can still join the cluster later #cluster.initial_master_nodes: ["fedora"] From c375dd9185a3f149af38c99fc734843613bc75ec Mon Sep 17 00:00:00 2001 From: Jeong Han Lee Date: Tue, 27 Sep 2022 15:55:24 -0700 Subject: [PATCH 76/89] add the different ES configuration path for macOS --- configure/RULES_CI | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/configure/RULES_CI b/configure/RULES_CI index 44feb5b..9560cb5 100644 --- a/configure/RULES_CI +++ b/configure/RULES_CI @@ -11,8 +11,9 @@ conf.macbrew: $(QUIET)echo "MAVEN_PATH:=/usr/local/bin" >> $(TOP)/configure/CONFIG_COMMON.local $(QUIET)cat -b $(TOP)/configure/CONFIG_COMMON.local $(QUIET)echo ">>> Generating $(TOP)/configure/CONFIG_SITE.local" - $(QUIET)echo "USERID:=${USER}" > $(TOP)/configure/CONFIG_SITE.local - $(QUIET)echo "GROUPID:=admin" >> $(TOP)/configure/CONFIG_SITE.local - $(QUIET)echo "CF_ES_PATH:=$(TOP)/$(ES_HOME_MAC)" >> $(TOP)/configure/CONFIG_SITE.local - $(QUIET)cat -b $(TOP)/configure/CONFIG_SITE.local + $(QUIET)echo "USERID:=${USER}" > $(TOP)/configure/CONFIG_SITE.local + $(QUIET)echo "GROUPID:=admin" >> $(TOP)/configure/CONFIG_SITE.local + $(QUIET)echo "CF_ES_PATH:=$(TOP)/$(ES_HOME_MAC)" >> $(TOP)/configure/CONFIG_SITE.local + $(QUIET)echo "CF_ES_CONFIG_PATH:=$(TOP)/$(ES_HOME_MAC)/config" >> $(TOP)/configure/CONFIG_SITE.local +$(QUIET)cat -b $(TOP)/configure/CONFIG_SITE.local From 60f921561f476cd7dbf4702a19657ae4a8afe31d Mon Sep 17 00:00:00 2001 From: Jeong Han Lee Date: Tue, 27 Sep 2022 16:00:17 -0700 Subject: [PATCH 77/89] fixed RULES_CI --- configure/RULES_CI | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configure/RULES_CI b/configure/RULES_CI index 9560cb5..c0b1216 100644 --- a/configure/RULES_CI +++ b/configure/RULES_CI @@ -15,5 +15,5 @@ conf.macbrew: $(QUIET)echo "GROUPID:=admin" >> $(TOP)/configure/CONFIG_SITE.local $(QUIET)echo "CF_ES_PATH:=$(TOP)/$(ES_HOME_MAC)" >> $(TOP)/configure/CONFIG_SITE.local $(QUIET)echo "CF_ES_CONFIG_PATH:=$(TOP)/$(ES_HOME_MAC)/config" >> $(TOP)/configure/CONFIG_SITE.local -$(QUIET)cat -b $(TOP)/configure/CONFIG_SITE.local + $(QUIET)cat -b $(TOP)/configure/CONFIG_SITE.local From 4b7d4b82af75bdd1d8332ea61020a57c5cc91782 Mon Sep 17 00:00:00 2001 From: Jeong Han Lee Date: Tue, 27 Sep 2022 16:40:01 -0700 Subject: [PATCH 78/89] add ALS changes into, and define macos CONFIG_SITE env --- configure/CONFIG_COMMON | 3 ++- configure/CONFIG_SITE | 3 ++- configure/RULES_CI | 5 +++-- configure/RULES_PROPERTIES | 4 +++- configure/RULES_REQ | 2 +- site-template/application.properties.in | 14 +++++++------- 6 files changed, 18 insertions(+), 13 deletions(-) diff --git a/configure/CONFIG_COMMON b/configure/CONFIG_COMMON index 6a03a8b..c299a26 100644 --- a/configure/CONFIG_COMMON +++ b/configure/CONFIG_COMMON @@ -20,7 +20,8 @@ ES_GROUPID:=elasticsearch CF_PRTO:=http CF_HOST:=localhost -CF_PORT:=8080 +CF_HTTP_PORT:=8080 +CF_HTTPS_PORT:=8443 CF_QUERY_SIZE:=10000 diff --git a/configure/CONFIG_SITE b/configure/CONFIG_SITE index a77e605..321f5a6 100644 --- a/configure/CONFIG_SITE +++ b/configure/CONFIG_SITE @@ -5,7 +5,7 @@ CF_INSTALL_LOCATION=${CF_INSTALL_PATH}/channelfinder ### Elasticsearch specific configuration -CF_ES_PATH:=/etc/elasticsearch +CF_ES_PATH=/etc/elasticsearch CF_ES_CONF_NAME:=elasticsearch.yml # Debian @@ -49,6 +49,7 @@ CF_PROPERTIES:=application.properties ### ### These two parameters are used to generate application.properties from application.properties.in ### +CF_HTTP_ENABLE:=true CF_LDIF:=cf4als.ldif CF_SSHKEYALIAS:=cf CF_SSHKEY:=new$(CF_SSHKEYALIAS).p12 diff --git a/configure/RULES_CI b/configure/RULES_CI index c0b1216..855fc13 100644 --- a/configure/RULES_CI +++ b/configure/RULES_CI @@ -14,6 +14,7 @@ conf.macbrew: $(QUIET)echo "USERID:=${USER}" > $(TOP)/configure/CONFIG_SITE.local $(QUIET)echo "GROUPID:=admin" >> $(TOP)/configure/CONFIG_SITE.local $(QUIET)echo "CF_ES_PATH:=$(TOP)/$(ES_HOME_MAC)" >> $(TOP)/configure/CONFIG_SITE.local - $(QUIET)echo "CF_ES_CONFIG_PATH:=$(TOP)/$(ES_HOME_MAC)/config" >> $(TOP)/configure/CONFIG_SITE.local + $(QUIET)echo "CF_ES_CONF_PATH:=$(TOP)/$(ES_HOME_MAC)/config" >> $(TOP)/configure/CONFIG_SITE.local + $(QUIET)echo "CF_ES_DATA_PATH:=$(TOP)/$(ES_HOME_MAC)" >> $(TOP)/configure/CONFIG_SITE.local + $(QUIET)echo "CF_ES_LOGS_PATH:=$(TOP)/$(ES_HOME_MAC)/logs" >> $(TOP)/configure/CONFIG_SITE.local $(QUIET)cat -b $(TOP)/configure/CONFIG_SITE.local - diff --git a/configure/RULES_PROPERTIES b/configure/RULES_PROPERTIES index 7e969d2..4a51ddb 100644 --- a/configure/RULES_PROPERTIES +++ b/configure/RULES_PROPERTIES @@ -21,7 +21,9 @@ conf.properties: $(CF_PROPERTIES).in $(QUIET)sed -e "s|@SSHKEY@|$(CF_SSHKEY)|g" \ -e "s|@SSHKEYALIAS@|$(CF_SSHKEYALIAS)|g" \ -e "s|@CFLDIF@|$(CF_LDIF)|g" \ - -e "s|@CF_PORT@|$(CF_PORT)|g" \ + -e "s|@CF_HTTP_PORT@|$(CF_HTTP_PORT)|g" \ + -e "s|@CF_HTTPS_PORT@|$(CF_HTTPS_PORT)|g" \ + -e "s|@CF_HTTP_ENABLE@|$(CF_HTTP_ENABLE)|g" \ -e "s|@ES_HOST@|$(ES_HOST)|g" \ -e "s|@ES_PORT@|$(ES_PORT)|g" \ -e "s|@CF_QUERY_SIZE@|$(CF_QUERY_SIZE)|g" \ diff --git a/configure/RULES_REQ b/configure/RULES_REQ index 333be4b..a23d7c9 100644 --- a/configure/RULES_REQ +++ b/configure/RULES_REQ @@ -46,7 +46,7 @@ conf.es: $(CF_ES_JAVA_OPTS) conf.esyml $(QUIET)$(SUDO) $(INSTALL) -p -m 660 $(CF_SITE_TEMPLATE_PATH)/$(CF_ES_CONF_NAME) $(CF_ES_CONF_PATH)/ conf.esmac: $(CF_ES_JAVA_OPTS) conf.esyml - $(QUIET) echo ">>> Installing 01.CF_ES_JVM.options and $(CF_ES_CONF_NAME) to $(CF_ES_CONF_PATH)" + $(QUIET) echo ">>> Installing $(ES_JAVA_OPTS_FILE) and $(CF_ES_CONF_NAME) to $(CF_ES_CONF_PATH)" $(QUIET) $(INSTALL) -p -m 660 $(CF_SITE_TEMPLATE_PATH)/$(ES_JAVA_OPTS_FILE) $(CF_ES_CONF_PATH)/jvm.options.d/ $(QUIET) $(INSTALL) -p -m 640 $(CF_SITE_TEMPLATE_PATH)/$(CF_ES_CONF_NAME) $(CF_ES_CONF_PATH)/ diff --git a/site-template/application.properties.in b/site-template/application.properties.in index eb42b6b..8355aa0 100644 --- a/site-template/application.properties.in +++ b/site-template/application.properties.in @@ -1,10 +1,10 @@ ################## ChannelFinder Server #################### # ChannelFinder https port -server.port=8443 +server.port=@CF_HTTPS_PORT@ # Options support for unsecure http -server.http.enable=true -server.http.port=@CF_PORT@ +server.http.enable=@CF_HTTP_ENABLE@ +server.http.port=@CF_HTTP_PORT@ server.ssl.key-store-type=PKCS12 server.ssl.key-store=classpath:keystore/@SSHKEY@ @@ -59,10 +59,10 @@ demo_auth.roles = ADMIN,USER ############## Role --> group Mapping ############## # Customize group names here -admin-groups=cf-admins,sys-admins,ADMIN -channel-groups=cf-channels,USER -property-groups=cf-properties,USER -tag-groups=cf-tags,USER +admin-groups=cf-admins +channel-groups=cf-channels +property-groups=cf-properties +tag-groups=cf-tags ############################## Elastic Network And HTTP ############################### From 07e603818c83465004cb17ba46013be9f875572a Mon Sep 17 00:00:00 2001 From: Jeong Han Lee Date: Tue, 27 Sep 2022 16:43:31 -0700 Subject: [PATCH 79/89] update v2.0.0 with the als merge --- .github/workflows/docker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index df5b3ec..a3dba12 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -17,7 +17,7 @@ jobs: DOCKER_FILE: docker/Dockerfile DOCKER_ACCOUNT: alscontrols DOCKER_REPO: channelfinder - DOCKER_TAG: latest + DOCKER_TAG: v2.0.0 steps: - name: checkout uses: actions/checkout@v3 From ea6937804346c1b848179063141b3b2a2eab7bd1 Mon Sep 17 00:00:00 2001 From: Jeong Han Lee Date: Tue, 27 Sep 2022 16:44:12 -0700 Subject: [PATCH 80/89] update the latest one with the als merge --- .github/workflows/docker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index a3dba12..df5b3ec 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -17,7 +17,7 @@ jobs: DOCKER_FILE: docker/Dockerfile DOCKER_ACCOUNT: alscontrols DOCKER_REPO: channelfinder - DOCKER_TAG: v2.0.0 + DOCKER_TAG: latest steps: - name: checkout uses: actions/checkout@v3 From 491cffec236a7878203ffe59825d97077b603034 Mon Sep 17 00:00:00 2001 From: Jeong Han Lee Date: Tue, 27 Sep 2022 18:53:16 -0700 Subject: [PATCH 81/89] update mac --- README.md | 2 +- configure/RULES_CI | 17 ++++++++++++++--- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index c9d7341..bb6c1c0 100644 --- a/README.md +++ b/README.md @@ -91,7 +91,7 @@ tail -f /var/log/syslog ```bash make init -make conf.macos +make conf.macbrew make conf make build make run diff --git a/configure/RULES_CI b/configure/RULES_CI index 44feb5b..f092071 100644 --- a/configure/RULES_CI +++ b/configure/RULES_CI @@ -1,9 +1,7 @@ .PHONY: conf.macos conf.macbrew -conf.macos: conf.macbrew - -conf.macbrew: +conf.macos: $(QUIET)echo ">>> Generating $(TOP)/configure/CONFIG_COMMON.local" $(QUIET)echo "JAVA_HOME:=/usr/" > $(TOP)/configure/CONFIG_COMMON.local $(QUIET)echo "JAVA_PATH:=/usr/bin" >> $(TOP)/configure/CONFIG_COMMON.local @@ -16,3 +14,16 @@ conf.macbrew: $(QUIET)echo "CF_ES_PATH:=$(TOP)/$(ES_HOME_MAC)" >> $(TOP)/configure/CONFIG_SITE.local $(QUIET)cat -b $(TOP)/configure/CONFIG_SITE.local +conf.macbrew: + $(QUIET)echo ">>> Generating $(TOP)/configure/CONFIG_COMMON.local" + $(QUIET)echo "JAVA_HOME:=/opt/homebrew/opt/openjdk@17/libexec/openjdk.jdk/Contents/Home" > $(TOP)/configure/CONFIG_COMMON.local + $(QUIET)echo "JAVA_PATH:=/opt/homebrew/opt/openjdk@17/libexec/openjdk.jdk/Contents/Home/bin" >> $(TOP)/configure/CONFIG_COMMON.local + $(QUIET)echo "MAVEN_HOME:=/opt/homebrew" >> $(TOP)/configure/CONFIG_COMMON.local + $(QUIET)echo "MAVEN_PATH:=/opt/homebrew/bin" >> $(TOP)/configure/CONFIG_COMMON.local + $(QUIET)cat -b $(TOP)/configure/CONFIG_COMMON.local + $(QUIET)echo ">>> Generating $(TOP)/configure/CONFIG_SITE.local" + $(QUIET)echo "USERID:=${USER}" > $(TOP)/configure/CONFIG_SITE.local + $(QUIET)echo "GROUPID:=admin" >> $(TOP)/configure/CONFIG_SITE.local + $(QUIET)echo "CF_ES_PATH:=$(TOP)/$(ES_HOME_MAC)" >> $(TOP)/configure/CONFIG_SITE.local + $(QUIET)cat -b $(TOP)/configure/CONFIG_SITE.local + From d679655e1b007a25dba4f31c07f96ab946cafc59 Mon Sep 17 00:00:00 2001 From: Tynan Ford Date: Mon, 3 Oct 2022 16:40:07 -0700 Subject: [PATCH 82/89] add es install rule for centos 7 --- configure/RULES_REQ | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/configure/RULES_REQ b/configure/RULES_REQ index a23d7c9..71f5423 100644 --- a/configure/RULES_REQ +++ b/configure/RULES_REQ @@ -96,6 +96,14 @@ install.esrpm: remove.esrpm: $(QUIET)$(SUDO) dnf remove elasticsearch +# Centos 7 +install.esrpm-yum: + $(QUIET) wget -c https://artifacts.elastic.co/downloads/elasticsearch/$(ES_RPM) + $(SUDO) yum localinstall -y $(ES_RPM) + +remove.esrpm-yum: + $(QUIET)$(SUDO) yum remove elasticsearch + # macOS aarch64 install.esmac: $(QUIET) curl -O https://artifacts.elastic.co/downloads/elasticsearch/$(ES_MAC) From 636e74f9ecb11ea2a6c7e5268da1ab6940e89753 Mon Sep 17 00:00:00 2001 From: Tynan Ford Date: Tue, 4 Oct 2022 09:19:32 -0700 Subject: [PATCH 83/89] Make CF_JAVA_HOME var for building CF --- configure/CONFIG_SITE | 1 + configure/RULES_SRC | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/configure/CONFIG_SITE b/configure/CONFIG_SITE index 321f5a6..4f76eaf 100644 --- a/configure/CONFIG_SITE +++ b/configure/CONFIG_SITE @@ -56,6 +56,7 @@ CF_SSHKEY:=new$(CF_SSHKEYALIAS).p12 CF_SSL_REQUIRED:=true CF_JAVA_PATH:=$(JAVA_PATH) +CF_JAVA_HOME:=$(JAVA_HOME) CF_JAVA_OPTS:=-Xms512m -Xmx512m # Two more potential interesting options are diff --git a/configure/RULES_SRC b/configure/RULES_SRC index a8af6fe..e51b5aa 100644 --- a/configure/RULES_SRC +++ b/configure/RULES_SRC @@ -50,6 +50,6 @@ endif ## Build Source build: conf - $(QUIET) JAVA_HOME=$(JAVA_HOME) $(MAVEN_CMD) $(CF_MVN_OPTS) -f $(CF_SRC_PATH)/pom.xml clean install -Dmaven.test.skip=true + $(QUIET) JAVA_HOME=$(CF_JAVA_HOME) $(MAVEN_CMD) $(CF_MVN_OPTS) -f $(CF_SRC_PATH)/pom.xml clean install -Dmaven.test.skip=true From 35f563e306da7e3ac0f0850e9ab667396537ea4b Mon Sep 17 00:00:00 2001 From: Tynan Ford Date: Tue, 4 Oct 2022 11:09:34 -0700 Subject: [PATCH 84/89] update USERID variable prefix to CF --- configure/CONFIG_COMMON | 4 ++-- configure/RULES_INSTALL | 2 +- configure/RULES_SYSTEMD | 2 +- site-template/systemd/channelfinder.service.in | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/configure/CONFIG_COMMON b/configure/CONFIG_COMMON index c299a26..32c5842 100644 --- a/configure/CONFIG_COMMON +++ b/configure/CONFIG_COMMON @@ -15,8 +15,8 @@ MAVEN_PATH:=$(MAVEN_HOME)/bin ES_HOST:=localhost ES_PORT:=9200 -ES_USERID:=elasticsearch -ES_GROUPID:=elasticsearch +CF_USERID:=elasticsearch +CF_GROUPID:=elasticsearch CF_PRTO:=http CF_HOST:=localhost diff --git a/configure/RULES_INSTALL b/configure/RULES_INSTALL index f5c28cb..d6ce955 100644 --- a/configure/RULES_INSTALL +++ b/configure/RULES_INSTALL @@ -12,7 +12,7 @@ cf_restart: sd_restart cf_install: $(QUIET)$(SUDO) install -d $(CF_INSTALL_LOCATION) $(QUIET)$(SUDO) install -m 744 $(CF_SRC_PATH)/target/$(CF_JAR_FILENAME) $(CF_INSTALL_LOCATION)/ - $(QUIET)$(SUDO) chown -R $(ES_USERID):$(ES_GROUPID) $(CF_INSTALL_LOCATION)/ + $(QUIET)$(SUDO) chown -R $(CF_USERID):$(CF_GROUPID) $(CF_INSTALL_LOCATION)/ src_install: cf_install diff --git a/configure/RULES_SYSTEMD b/configure/RULES_SYSTEMD index 8f27e7f..7d460e9 100644 --- a/configure/RULES_SYSTEMD +++ b/configure/RULES_SYSTEMD @@ -28,7 +28,7 @@ conf.systemd0: $(CF_SYSTEMD_FILENAME).in $(QUIET)echo ">>> Required Services : $(CF_SYSTEMD_SERVICES)" $(QUIET)sed -e "s|@DOCURL@|$(DOCURL)|g" \ -e 's|@SYSTEMD_SERVICES@|$(CF_SYSTEMD_SERVICES)|g' \ - -e "s|@ES_USERID@|$(ES_USERID)|g" \ + -e "s|@CF_USERID@|$(CF_USERID)|g" \ -e "s|@ES_GROUPID@|$(ES_GROUPID)|g" \ -e "s|@CF_JAVA_PATH@|$(CF_JAVA_PATH)|g" \ -e "s|@CF_JAVA_OPTS@|$(CF_JAVA_OPTS)|g" \ diff --git a/site-template/systemd/channelfinder.service.in b/site-template/systemd/channelfinder.service.in index 4edb9f5..c98317d 100644 --- a/site-template/systemd/channelfinder.service.in +++ b/site-template/systemd/channelfinder.service.in @@ -5,8 +5,8 @@ After=syslog.target network.target @SYSTEMD_SERVICES@ Requires=@SYSTEMD_SERVICES@ [Service] -User=@ES_USERID@ -Group=@ES_GROUPID@ +User=@CF_USERID@ +Group=@CF_GROUPID@ ExecStart=@CF_JAVA_PATH@/java @CF_JAVA_OPTS@ -jar @CF_INSTALL_LOCATION@/@CF_JAR_NAME@ SuccessExitStatus=143 From b74a592b26b9e45c6c79592234fa1127de13dbfd Mon Sep 17 00:00:00 2001 From: Tynan Ford Date: Tue, 4 Oct 2022 11:24:21 -0700 Subject: [PATCH 85/89] Fix missing ES_GROUPID --- configure/RULES_SYSTEMD | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configure/RULES_SYSTEMD b/configure/RULES_SYSTEMD index 7d460e9..994df25 100644 --- a/configure/RULES_SYSTEMD +++ b/configure/RULES_SYSTEMD @@ -29,7 +29,7 @@ conf.systemd0: $(CF_SYSTEMD_FILENAME).in $(QUIET)sed -e "s|@DOCURL@|$(DOCURL)|g" \ -e 's|@SYSTEMD_SERVICES@|$(CF_SYSTEMD_SERVICES)|g' \ -e "s|@CF_USERID@|$(CF_USERID)|g" \ - -e "s|@ES_GROUPID@|$(ES_GROUPID)|g" \ + -e "s|@CF_GROUPID@|$(CF_GROUPID)|g" \ -e "s|@CF_JAVA_PATH@|$(CF_JAVA_PATH)|g" \ -e "s|@CF_JAVA_OPTS@|$(CF_JAVA_OPTS)|g" \ -e "s|@CF_INSTALL_LOCATION@|$(CF_INSTALL_LOCATION)|g" \ From ad0d1a523e6733d70eca160049aea48bbd3b7dd6 Mon Sep 17 00:00:00 2001 From: Jeong Han Lee Date: Tue, 4 Oct 2022 12:30:11 -0700 Subject: [PATCH 86/89] update README, add cflog.show, and use the latest source 22-10-03 --- README.md | 6 ++++-- configure/RELEASE | 4 ++-- configure/RULES_RUN | 3 +++ 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index f02f1f9..a43595d 100644 --- a/README.md +++ b/README.md @@ -76,15 +76,17 @@ It will works with other systems. Please check github action workflows. make init make build make install +make mapping make sd_start make sd_status -make mapping ``` ChannelFinder log is shown in `/var/log/syslog`. ```bash -tail -f /var/log/syslog +tail -f /var/log/syslog |grep java + +make cflog.show ``` ## macOS (tested with aarch64 with brew) diff --git a/configure/RELEASE b/configure/RELEASE index 7b7a55d..70c06c1 100644 --- a/configure/RELEASE +++ b/configure/RELEASE @@ -6,8 +6,8 @@ CF_SRC_NAME:=ChannelFinderService ##- Which the source tag / branch / hash id would like to use -# 2022-09-21 -CF_SRC_TAG:=fc9d536 +# 2022-10-03 +CF_SRC_TAG:=4a17eea ##- Placeholder for the site-specific version control ##- This information is used for application.properties diff --git a/configure/RULES_RUN b/configure/RULES_RUN index 6b904db..9fc138e 100644 --- a/configure/RULES_RUN +++ b/configure/RULES_RUN @@ -13,3 +13,6 @@ demo: demo.clean: $(QUIET)mvn -f $(CF_SRC_PATH) spring-boot:run -Dspring-boot.run.arguments="--cleanup=1" + +cflog.show: + $(SUDO) tail -f /var/log/syslog |grep java From 7b9fd164e011eae897dd317f09105223f2577cdf Mon Sep 17 00:00:00 2001 From: Jeong Han Lee Date: Sun, 16 Oct 2022 00:38:00 -0700 Subject: [PATCH 87/89] add tomcat service comments --- README.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/README.md b/README.md index a43595d..7b76dee 100644 --- a/README.md +++ b/README.md @@ -89,6 +89,17 @@ tail -f /var/log/syslog |grep java make cflog.show ``` +Please check Tomcat generic service, if CF doesn't start. + +``` +lsof -i:8080 +systemctl status tomcat{9} +systemctl stop tomcat{9} +systemctl disable tomcat{9} +``` + + + ## macOS (tested with aarch64 with brew) ```bash From c7377e511ff86aa0f5ae6b6ad7898812163a17c6 Mon Sep 17 00:00:00 2001 From: Jeong Han Lee Date: Fri, 4 Nov 2022 16:04:48 -0700 Subject: [PATCH 88/89] update http.host to use localhost due to the network security issue --- configure/CONFIG_SITE | 46 +++++++++++++++++++++---------------------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/configure/CONFIG_SITE b/configure/CONFIG_SITE index 4f76eaf..b5fade4 100644 --- a/configure/CONFIG_SITE +++ b/configure/CONFIG_SITE @@ -6,7 +6,7 @@ CF_INSTALL_LOCATION=${CF_INSTALL_PATH}/channelfinder ### Elasticsearch specific configuration CF_ES_PATH=/etc/elasticsearch -CF_ES_CONF_NAME:=elasticsearch.yml +CF_ES_CONF_NAME=elasticsearch.yml # Debian # https://www.elastic.co/guide/en/elasticsearch/reference/8.2/important-settings.html#path-settings @@ -16,53 +16,53 @@ CF_ES_CONF_PATH=/etc/elasticsearch # # elasticsearch.yml.in CF_ES_NETWORK_HOST=127.0.0.1 -CF_ES_HTTP_HOST:=0.0.0.0 +CF_ES_HTTP_HOST=127.0.0.1 CF_ES_CLUSTER_NAME:="als-es-prod" CF_ES_NODE_NAME:="als-es-node" CF_ES_DATA_PATH=/var/lib/elasticsearch CF_ES_LOGS_PATH=/var/log/elasticsearch -CF_ES_DISCOVERY_TYPE:="single-node" - -CF_ES_XPACK_SECURITY_ENABLED:=false -CF_ES_XPACK_SECURITY_ENROLL_ENABLED:=true -CF_ES_XPACK_SECURITY_HTTP_SSL:=false -CF_ES_XPACK_SECURITY_HTTP_SSL_KEYSTORE_PATH:=certs/http.p12 -CF_ES_XPACK_SECURITY_TRANSPORT_SSL:=false -CF_ES_XPACK_SECURITY_TRANSPORT_SSL_VERIFICATION:=certificate -CF_ES_XPACK_SECURITY_TRANSPORT_SSL_KEYSTORE_PATH:=certs/transport.p12 +CF_ES_DISCOVERY_TYPE="single-node" + +CF_ES_XPACK_SECURITY_ENABLED=false +CF_ES_XPACK_SECURITY_ENROLL_ENABLED=true +CF_ES_XPACK_SECURITY_HTTP_SSL=false +CF_ES_XPACK_SECURITY_HTTP_SSL_KEYSTORE_PATH=certs/http.p12 +CF_ES_XPACK_SECURITY_TRANSPORT_SSL=false +CF_ES_XPACK_SECURITY_TRANSPORT_SSL_VERIFICATION=certificate +CF_ES_XPACK_SECURITY_TRANSPORT_SSL_KEYSTORE_PATH=certs/transport.p12 CF_ES_XPACK_SECURITY_TRANSPORT_SSL_TRUSTSTORE_PATH=certs/transport.p12 # JVM heap dump path : # CF logging disable : # -CF_ES_JAVA_OPTS:=-Xms1g -Xmx1g +CF_ES_JAVA_OPTS=-Xms1g -Xmx1g #-XX\:HeapDumpPath=$(CF_ES_DATA_PATH) -Xlog\:disable ### Placeholder ### This variable will be replaced by reading a pom.xml file. The version in pom.xml will define ### the exact version of the jar file. It is difficult to track that version to implement into here. ### It may be linked with SRC_VERSION in RELEASE file ### -CF_JAR_FILENAME:=ChannelFinder-4.7.1-SNAPSHOT.jar +CF_JAR_FILENAME=ChannelFinder-4.7.1-SNAPSHOT.jar -CF_PROPERTIES:=application.properties +CF_PROPERTIES=application.properties ### ### These two parameters are used to generate application.properties from application.properties.in ### -CF_HTTP_ENABLE:=true -CF_LDIF:=cf4als.ldif -CF_SSHKEYALIAS:=cf -CF_SSHKEY:=new$(CF_SSHKEYALIAS).p12 -CF_SSL_REQUIRED:=true +CF_HTTP_ENABLE=true +CF_LDIF=cf4als.ldif +CF_SSHKEYALIAS=cf +CF_SSHKEY=new$(CF_SSHKEYALIAS).p12 +CF_SSL_REQUIRED=true -CF_JAVA_PATH:=$(JAVA_PATH) -CF_JAVA_HOME:=$(JAVA_HOME) -CF_JAVA_OPTS:=-Xms512m -Xmx512m +CF_JAVA_PATH=$(JAVA_PATH) +CF_JAVA_HOME=$(JAVA_HOME) +CF_JAVA_OPTS=-Xms512m -Xmx512m # Two more potential interesting options are # CF_JAVA_OPTS=-XX:MaxMetaspaceSize=256M -XX:+UseG1GC -Xms2G -Xmx2G -CF_MVN_OPTS:= +CF_MVN_OPTS= # it will be good to have its own, reserve them for the future expansion # currnetly, channelfinder log is in the /var/log/syslog From 7cf8cb248d23068b9c54c7aac65d973c1946fe15 Mon Sep 17 00:00:00 2001 From: Jeong Han Lee Date: Sun, 13 Nov 2022 19:26:00 -0800 Subject: [PATCH 89/89] Delete macOS.yml --- .github/workflows/macOS.yml | 42 ------------------------------------- 1 file changed, 42 deletions(-) delete mode 100644 .github/workflows/macOS.yml diff --git a/.github/workflows/macOS.yml b/.github/workflows/macOS.yml deleted file mode 100644 index 06a3619..0000000 --- a/.github/workflows/macOS.yml +++ /dev/null @@ -1,42 +0,0 @@ ---- -name: macOS build - -on: - push: - branches: [master] - paths-ignores: - - '**.md' - - '.github/workflows/build.yml' - - pull_request: - branches: [master] - -jobs: - macOS12: - runs-on: macos-12 - steps: - - uses: actions/checkout@v3 - - name: Install required packages - run: | - brew install tree wget curl bash jq coreutils - - name : Check JAVA Environment - run: | - which java - which mvn - tree -L 2 /usr/local/opt/ - tree -L 2 /usr/local/Cellar/ - - name: ES Configuration - run: | - make init - make install.esmac - make conf.macos - make conf.esmac - - name: CF Configuration and build - run: | - make conf - make vars - make build - make cf_install - - name: Environment Check - run: make exist -