diff --git a/.dockerignore b/.dockerignore index 4cf6474..9fc4204 100644 --- a/.dockerignore +++ b/.dockerignore @@ -5,3 +5,10 @@ temp? *.md !README.md **/*.class +examples +elastic-stack +*-src +*.md +LICENSE +docs +*Dockerfile* diff --git a/.github/workflows/debian11.yml b/.github/workflows/debian11.yml new file mode 100644 index 0000000..36bd422 --- /dev/null +++ b/.github/workflows/debian11.yml @@ -0,0 +1,59 @@ +--- +name: Debian 11 + +on: + push: + branches: [ master ] + paths-ignores: + - '*.md' + - 'examples/*.bash' + - 'docker/**' + - '.github/workflows/docker.yml' + - '.github/workflows/ubuntu.yml' + - '.github/workflows/rocky8.yml' + - '.github/workflows/linter.yml' + + pull_request: + branches: [ master ] + +jobs: + Debian11: + + runs-on: ubuntu-latest + container: debian:bullseye + + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-java@v3 + with: + distribution: 'zulu' + java-version: '18.0.2' + - name: Install required packages + run: | + apt update -y + apt install -y git make sudo tree maven wget + - name: Check JAVA configuration + run: | + which java + which mvn + - name: Elasticsearch + run: | + make install.esdeb + make conf.es + make conf.es.show + - name: Configuration + run: | + make init + echo "JAVA_HOME:=${JAVA_HOME}" > ./configure/CONFIG_COMMON.local + echo "JAVA_PATH:=${JAVA_HOME}/bin" >> ./configure/CONFIG_COMMON.local + echo "MAVEN_HOME:=/usr/" >> ./configure/CONFIG_COMMON.local + echo "MAVEN_PATH:=/usr/bin" >> ./configure/CONFIG_COMMON.local + make vars + make conf + - name: Build + run: | + make build + make cf_install + - name: Environment Check + run: make exist + diff --git a/.github/workflows/docker-image.yml b/.github/workflows/docker-image.yml deleted file mode 100644 index 8b1c4a9..0000000 --- a/.github/workflows/docker-image.yml +++ /dev/null @@ -1,19 +0,0 @@ -name: Docker Image CI - -on: - push: - branches: [ master ] - pull_request: - branches: [ master ] - -jobs: - - build: - - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v2 - - name: Build the Docker image - run: make build.docker - diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml new file mode 100644 index 0000000..df5b3ec --- /dev/null +++ b/.github/workflows/docker.yml @@ -0,0 +1,58 @@ +--- +# 0. change name +name: Docker +# change path name +on: + push: + branches: [master] + paths: + - 'docker/**' + - '.github/workflows/docker.yml' + pull_request: + branches: [master] +jobs: + Docker: + runs-on: ubuntu-latest + env: + DOCKER_FILE: docker/Dockerfile + DOCKER_ACCOUNT: alscontrols + DOCKER_REPO: channelfinder + DOCKER_TAG: latest + steps: + - name: checkout + uses: actions/checkout@v3 + - name: Docker meta + id: meta + uses: docker/metadata-action@v4 + with: + # list of Docker images to use as base name for tags + images: | + ${{ env.DOCKER_ACCOUNT }}/${{ env.DOCKER_REPO }} + # generate Docker tags based on the following events/attributes + tags: | + type=schedule + type=ref,event=branch + type=ref,event=pr + type=semver,pattern={{version}} + type=semver,pattern={{major}}.{{minor}} + type=semver,pattern={{major}} + type=sha + - name: Set up QEMU + uses: docker/setup-qemu-action@v2 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + - name: Login to DockerHub + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + - name: Build and push + id: docker_build + uses: docker/build-push-action@v3 + with: + context: . + file: ${{ github.workspace }}/${{ env.DOCKER_FILE }} + push: ${{ github.event_name != 'pull_request' }} + tags: + ${{env.DOCKER_ACCOUNT}}/${{env.DOCKER_REPO}}:${{env.DOCKER_TAG}} + labels: ${{ steps.meta.outputs.labels }} diff --git a/.github/workflows/linter.yml b/.github/workflows/linter.yml index 8047879..fbd5f9e 100644 --- a/.github/workflows/linter.yml +++ b/.github/workflows/linter.yml @@ -13,10 +13,10 @@ jobs: steps: - name: Checkout Code - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Lint Code Base - uses: docker://github/super-linter:latest + uses: docker://github/super-linter:v4.9.5 env: -# VALIDATE_ALL_CODEBASE: true - VALIDATE_MD: true - VALIDATE_BASH: true + VALIDATE_MD: true + VALIDATE_BASH: true + diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml deleted file mode 100644 index a2880e7..0000000 --- a/.github/workflows/maven.yml +++ /dev/null @@ -1,39 +0,0 @@ -# This workflow will build a Java project with Maven -# For more information see: https://help.github.com/actions/language-and-framework-guides/building-and-testing-java-with-maven - -name: ChannelFinder-env Build Test - -on: - push: - branches: [ master ] - pull_request: - branches: [ master ] - -jobs: - build: - - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v2 - - uses: actions/setup-java@v1 - with: - java-version: '11.0.x' - java-package: jdk - architecture: x64 - - run: | - java -version - echo $JAVA_HOME - echo $PATH - mvn --version - - run: sudo apt-get install make tree git - - run: | - make init - make vars - make conf - - run: | - make build - make install - - run: make exist - - diff --git a/.github/workflows/rocky8.yml b/.github/workflows/rocky8.yml new file mode 100644 index 0000000..0742303 --- /dev/null +++ b/.github/workflows/rocky8.yml @@ -0,0 +1,59 @@ +--- +name: Rocky8 + +on: + push: + branches: [ master ] + paths-ignores: + - '*.md' + - 'examples/*.bash' + - 'docker/**' + - '.github/workflows/ubuntu.yml' + - '.github/workflows/docker.yml' + - '.github/workflows/ubuntu.yml' + - '.github/workflows/linter.yml' + + pull_request: + branches: [ master ] + +jobs: + Rocky8: + + runs-on: ubuntu-latest + container: rockylinux/rockylinux:8 + + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-java@v3 + with: + distribution: 'zulu' + java-version: '18.0.2' + - name: Install required packages + run: | + dnf update -y + dnf install -y git make sudo which maven wget tree + - name: Check JAVA configuration + run: | + which java + which mvn + - name: Elasticsearch + run: | + make install.esrpm + make conf.es + make conf.es.show + - name: Configuration + run: | + make init + echo "JAVA_HOME:=${JAVA_HOME}" > ./configure/CONFIG_COMMON.local + echo "JAVA_PATH:=${JAVA_HOME}/bin" >> ./configure/CONFIG_COMMON.local + echo "MAVEN_HOME:=/usr/" >> ./configure/CONFIG_COMMON.local + echo "MAVEN_PATH:=/usr/bin" >> ./configure/CONFIG_COMMON.local + make vars + make conf + - name: Build + run: | + make build + make cf_install + - name: Environment Check + run: make exist + diff --git a/.github/workflows/ubuntu.yml b/.github/workflows/ubuntu.yml new file mode 100644 index 0000000..ad77292 --- /dev/null +++ b/.github/workflows/ubuntu.yml @@ -0,0 +1,56 @@ +--- +name: Ubuntu Latest + +on: + push: + branches: [ master ] + paths-ignores: + - '*.md' + - 'examples/*.bash' + - 'docker/**' + - '.github/workflows/docker.yml' + - '.github/workflows/rocky8.yml' + - '.github/workflows/linter.yml' + - '.github/workflows/docker.yml' + + pull_request: + branches: [ master ] + +jobs: + Ubuntu: + + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-java@v3 + with: + distribution: 'zulu' + java-version: '18.0.2' + - name: Install required packages + run: sudo apt install -y make tree git wget + - name: Check JAVA configuration + run: | + which java + which mvn + - name: Elasticsearch + run: | + make install.esdeb + make conf.es + make conf.es.show + - name: Configuration + run: | + make init + echo "JAVA_HOME:=${JAVA_HOME}" > ./configure/CONFIG_COMMON.local + echo "JAVA_PATH:=${JAVA_HOME}/bin" >> ./configure/CONFIG_COMMON.local + echo "MAVEN_HOME:=/usr/" >> ./configure/CONFIG_COMMON.local + echo "MAVEN_PATH:=/usr/bin" >> ./configure/CONFIG_COMMON.local + make vars + make conf + - name: Build + run: | + make build + make install + - name: Environment Check + run: make exist + diff --git a/.gitignore b/.gitignore index c59be69..d8d449e 100644 --- a/.gitignore +++ b/.gitignore @@ -39,3 +39,10 @@ site-template/application.properties +.*.swp +*.swp +*.swo +channelfinder.service +elasticsearch-8.2.0 +*.options +*.yml diff --git a/README.md b/README.md index 14384ce..7b76dee 100644 --- a/README.md +++ b/README.md @@ -1,183 +1,136 @@ # ChannelFinder-env +[![Debian 11](https://github.com/jeonghanlee/ChannelFinder-env/actions/workflows/debian11.yml/badge.svg)](https://github.com/jeonghanlee/ChannelFinder-env/actions/workflows/debian11.yml) +[![Rocky8](https://github.com/jeonghanlee/ChannelFinder-env/actions/workflows/rocky8.yml/badge.svg)](https://github.com/jeonghanlee/ChannelFinder-env/actions/workflows/rocky8.yml) +[![Ubuntu Latest](https://github.com/jeonghanlee/ChannelFinder-env/actions/workflows/ubuntu.yml/badge.svg)](https://github.com/jeonghanlee/ChannelFinder-env/actions/workflows/ubuntu.yml) +[![ChannelFinderService](https://github.com/jeonghanlee/ChannelFinder-env/actions/workflows/docker.yml/badge.svg)](https://github.com/jeonghanlee/ChannelFinder-env/actions/workflows/docker.yml) +[![Linter Run](https://github.com/jeonghanlee/ChannelFinder-env/actions/workflows/linter.yml/badge.svg)](https://github.com/jeonghanlee/ChannelFinder-env/actions/workflows/linter.yml) -![ChannelFinder-env Build Test](https://github.com/ChannelFinder/ChannelFinder-env/workflows/ChannelFinder-env%20Build%20Test/badge.svg) -![Docker Image CI](https://github.com/ChannelFinder/ChannelFinder-env/workflows/Docker%20Image%20CI/badge.svg) -![Linter Run](https://github.com/ChannelFinder/ChannelFinder-env/workflows/Linter%20Run/badge.svg) +Configuration Environment for ChannelFinderService at -Configuration Environment for ChannelFinder-SpringBoot at +## Pre-requirement packages -## Role +``` +git make sudo tree maven jq +``` -In order to download, install, setup all relevant components, one should do many steps manually. This repository was designed for the easy-to-reproducible environment for ChannelFinder-SprintBoot. +## Elasticsearch -## Requirements +Three configurations are changed for the elasticsearch. -**Note that** this implementation is valid only for `ChannelFinder-SpringBoot` +* Elasticsearch JVM options : [default] `-Xms1g -Xmx1g` +* Elasticsearch `pack.security.enabled` option : [default] false +* Elasticsearch configuration location : [default] `/etc/elasticsearch` -### Apache Tomcat Native Library and Maven +These options can be changed through the variables `CF_ES_JAVA_OPTS`, `CF_ES_CONF_PATH`, and `CF_ES_XPACK_SECURITY` in `configure/CONFIG_SITE`. -* Debian 10 +### Debian 11 / Rocky 8 -```bash -apt install maven libtcnative-1 ``` - -* CentOS 7 - -```bash -yum install maven tomcat-native +make install.esdeb or install.esrpm +make conf.es +make conf.es.show +make start.es +make status.es ``` -* CentOS 8 \& Fedora 31 +Note that `conf.es` will use the `pack.security.enabled: false` in `elasticsearch.yml`, where is in `/etc/elasticsearch`. +The log file `elasticsearch.log` is located in `/etc/elasticsearch`. -One can use `yum` instead of `dnf` for CentOS 8. However, `epel-release` is needed. +### macOS (M1) -```bash -dnf install maven tomcat-native -``` - -### JDK 8 or newer -* Debian 10 - -```bash -openjdk version "11.0.6" 2020-01-14 -OpenJDK Runtime Environment (build 11.0.6+10-post-Debian-1deb10u1) -OpenJDK 64-Bit Server VM (build 11.0.6+10-post-Debian-1deb10u1, mixed mode, sharing) ``` - -* Fedora 32 +brew install openjdk@18 +``` ```bash -$ update-alternatives --config java -There is 1 program that provides 'java'. - - Selection Command ------------------------------------------------ -*+ 1 java-1.8.0-openjdk.x86_64 (/usr/lib/jvm/java-1.8.0-openjdk-1.8.0.242.b08-1.fc32.x86_64/jre/bin/java) - -$ java -version - -openjdk version "1.8.0_242" -OpenJDK Runtime Environment (build 1.8.0_242-b08) -OpenJDK 64-Bit Server VM (build 25.242-b08, mixed mode) +make install.esmac +make conf.macos +make conf.esmac +make conf.esmac.show +make start.esmac +make stop.esmac ``` -### ElasticSearch 6.3.1 - -Please use the exact version of Elasticsearch **6.3.1**. +The log file `elasticsearch.log` is located in the installation location log folder of the elastisserch. -* Debian 10 -```bash -wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-6.3.1.deb -sudo apt install ./elasticsearch-6.3.1.deb -``` +## JAVA -* CentOS 7 +The following four variables must be defined. Please setup them according to one's systems configuration. ```bash -wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-6.3.1.rpm -sudo yum install ./elasticsearch-6.3.1.rpm + echo "JAVA_HOME:=/usr/" > ./configure/CONFIG_COMMON.local + echo "JAVA_PATH:=/usr/bin" >> ./configure/CONFIG_COMMON.local + echo "MAVEN_HOME:=/usr/" >> ./configure/CONFIG_COMMON.local + echo "MAVEN_PATH:=/usr/bin" >> ./configure/CONFIG_COMMON.local ``` -* CentOS 8 & Fedora 32 +## Support OS -```bash -wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-6.3.1.rpm -sudo dnf install ./elasticsearch-6.3.1.rpm -``` +### Debian 11 (EOL: 2024-06-01/2026-08-15), Rocky8 (EOL: 2029-05-31) -* ES Systemd service +It will works with other systems. Please check github action workflows. ```bash -sudo systemctl daemon-reload -sudo systemctl enable elasticsearch -sudo systemctl start elasticsearch +make init +make build +make install +make mapping +make sd_start +make sd_status ``` -Please make sure the service is running via `systemctl status elasticsearch`. One can do the same things with embedded make rules such as +ChannelFinder log is shown in `/var/log/syslog`. -### LDAP - -* Configuration is needed if an external one. -* Embedded LDAP configuration [Local LDIF configuration](site-template/LDIF_CONFIG.md) - -## Few Makefile Rules - -### `make init` - -* Download the ChannelFinder-SpringBoot -* Switch to a specific version defined in `$(SRC_TAG)` in `configure/RELEASE` - -### `make es_mapping` - -* Do maping CF index into ElasticSearch -* For CentOS7, one should disable SELINUX before doing this. Please check `/etc/selinux/config`. - -### `make es_mapping_clean` - -* Remove all existent CF related mapping from ElasticSearch - -### `make conf` - -* Apply several site-specific files into the downloaded sources. Please see `site-template` - -### `make build` - -* Build `ChannelFinder-*.jar` as a Spring Boot jar file, located in `target` path. - -### `make install` - -* `sudo` permission is required. -* Install `ChannelFinder-*.jar` in `target` path into `INSTALL_LOCATION` defined in `configure/CONFIG_SITE` -* Install `cf.conf` into `INSTALL_LOCATION`. This file contains java options, defined in `site-template/cf.conf` -* Install `channelfinder.service` into a default systemd path `/etc/systemd/system`. Note that `channelfinder.service` is generated from `site-template/cf.service.in` file with `INSTAL_LOCATION`, `JAVA_PATH`, and `CF_JAR_NAME` which are defined in `configure/CONFIG_SITE`. -* Run `systemctl daemon-reload` for the updated `channelfinder.serive` systemd file. -* Enable the channelfinder systemd service. +```bash +tail -f /var/log/syslog |grep java -### `make distclean` +make cflog.show +``` -* Remove the downloaded ChannelFinder-SprintBoot source file +Please check Tomcat generic service, if CF doesn't start. -### `make vars` +``` +lsof -i:8080 +systemctl status tomcat{9} +systemctl stop tomcat{9} +systemctl disable tomcat{9} +``` -* Print out interesting variables -* One can use `make PRINT.VARIABLE_NAME` to print out them. For example, `make PRINT.INSTALL_LOCATION`. -## A typical example to configure the ChannelFinder service -Note that this example has the assumption which ES service is running. +## macOS (tested with aarch64 with brew) ```bash make init -make es_mapping +make conf.macbrew make conf make build -make install -sudo systemctl start channelfinder +make run +make mapping +make mapping.clean ``` -Note that we create the alias name of the `channelfinder.service` as cf.servie. So one can start it `sudo systemctl start cf` also. Sometimes with CentOS8, the service doesn't start properly. In this case, please try with `SELINUX=disabled` in `/etc/selinux/config`. +## ChannelFinder Configuration -For Fedora 32, one should disable `SELINUX` before `make es_mapping`. +Please see [docs/ChannelFinderConf.md](docs/ChannelFinderConf.md). -## While evaluating its configuration +## Run Demo -Modify `application.properties` or `ldif` file, and then run the following command: +Please see [docs/ChannelFinderDemo.md](docs/ChannelFinderDemo.md). -```bash -make restart -``` - -## Customize site-specific configuration -Please consult two files in `configure` path, such as `RELEASE` and `CONFIG_SITE`. There are few comments on there. If you are familiar with the standard EPICS building system [1], it should be easy to understand them, because we mimic that concept into this repository. +``` +make run +make run.status +make run.kill +make demo +make demo.clean +``` ## Docker Image -See [docs/Docker.md](docs/Docker.md) :whale: - -## Reference +The Docker image is hosted at https://hub.docker.com/orgs/alscontrols +And for further information, please see [docs/Docker.md](docs/Docker.md) :whale: -[1] diff --git a/configure/CONFIG b/configure/CONFIG index 4e029a5..d0ef341 100644 --- a/configure/CONFIG +++ b/configure/CONFIG @@ -1,7 +1,11 @@ VARS_EXCLUDES := $(.VARIABLES) +UNAME_S:=$(shell uname -s) +UNAME_M:=$(shell uname -m) + include $(TOP)/configure/CONFIG_COMMON include $(TOP)/configure/RELEASE include $(TOP)/configure/CONFIG_SITE +include $(TOP)/configure/CONFIG_SYSTEMD include $(TOP)/configure/CONFIG_VARS include $(TOP)/configure/CONFIG_SRC diff --git a/configure/CONFIG_COMMON b/configure/CONFIG_COMMON index da704b5..32c5842 100644 --- a/configure/CONFIG_COMMON +++ b/configure/CONFIG_COMMON @@ -1,15 +1,27 @@ +# Debian -JAVA_PATH=/usr/bin +# /usr/lib/jvm/java-17-openjdk-amd64/ +# Sometime, debian default-java symlink doesn't update even if +# update-alternative --config java or update-java-alternative +# one should change it manually. +JAVA_HOME:=/usr +MAVEN_HOME:=/usr/share/maven + +JAVA_PATH:=$(JAVA_HOME)/bin +MAVEN_PATH:=$(MAVEN_HOME)/bin ## Elasticsearch Server ES_HOST:=localhost ES_PORT:=9200 +CF_USERID:=elasticsearch +CF_GROUPID:=elasticsearch CF_PRTO:=http CF_HOST:=localhost -CF_PORT:=8080 +CF_HTTP_PORT:=8080 +CF_HTTPS_PORT:=8443 CF_QUERY_SIZE:=10000 diff --git a/configure/CONFIG_SITE b/configure/CONFIG_SITE index 9dd450d..b5fade4 100644 --- a/configure/CONFIG_SITE +++ b/configure/CONFIG_SITE @@ -1,32 +1,90 @@ ### channelfinder installation location -CF_INSTALL_LOCATION=/opt/channelfinder +CF_INSTALL_PATH=/opt +CF_INSTALL_LOCATION=${CF_INSTALL_PATH}/channelfinder +### Elasticsearch specific configuration +CF_ES_PATH=/etc/elasticsearch +CF_ES_CONF_NAME=elasticsearch.yml + +# Debian +# https://www.elastic.co/guide/en/elasticsearch/reference/8.2/important-settings.html#path-settings +CF_ES_CONF_PATH=/etc/elasticsearch + + +# +# elasticsearch.yml.in +CF_ES_NETWORK_HOST=127.0.0.1 +CF_ES_HTTP_HOST=127.0.0.1 +CF_ES_CLUSTER_NAME:="als-es-prod" +CF_ES_NODE_NAME:="als-es-node" +CF_ES_DATA_PATH=/var/lib/elasticsearch +CF_ES_LOGS_PATH=/var/log/elasticsearch +CF_ES_DISCOVERY_TYPE="single-node" + +CF_ES_XPACK_SECURITY_ENABLED=false +CF_ES_XPACK_SECURITY_ENROLL_ENABLED=true +CF_ES_XPACK_SECURITY_HTTP_SSL=false +CF_ES_XPACK_SECURITY_HTTP_SSL_KEYSTORE_PATH=certs/http.p12 +CF_ES_XPACK_SECURITY_TRANSPORT_SSL=false +CF_ES_XPACK_SECURITY_TRANSPORT_SSL_VERIFICATION=certificate +CF_ES_XPACK_SECURITY_TRANSPORT_SSL_KEYSTORE_PATH=certs/transport.p12 +CF_ES_XPACK_SECURITY_TRANSPORT_SSL_TRUSTSTORE_PATH=certs/transport.p12 + +# JVM heap dump path : +# CF logging disable : +# +CF_ES_JAVA_OPTS=-Xms1g -Xmx1g +#-XX\:HeapDumpPath=$(CF_ES_DATA_PATH) -Xlog\:disable ### Placeholder -### This variable will be replaced by reading a pom.xml file or others +### This variable will be replaced by reading a pom.xml file. The version in pom.xml will define +### the exact version of the jar file. It is difficult to track that version to implement into here. ### It may be linked with SRC_VERSION in RELEASE file ### -CF_JAR_FILENAME:=ChannelFinder-4.0.0.jar +CF_JAR_FILENAME=ChannelFinder-4.7.1-SNAPSHOT.jar +CF_PROPERTIES=application.properties ### ### These two parameters are used to generate application.properties from application.properties.in ### -CF_LDIF:=cf4als.ldif -CF_SSHKEYALIAS:=cf -CF_SSHKEY:=$(CF_SSHKEYALIAS).p12 +CF_HTTP_ENABLE=true +CF_LDIF=cf4als.ldif +CF_SSHKEYALIAS=cf +CF_SSHKEY=new$(CF_SSHKEYALIAS).p12 +CF_SSL_REQUIRED=true -CF_JAVA_PATH:=$(JAVA_PATH) -CF_JAVA_OPTS:=-Xms512m -Xmx512m +CF_JAVA_PATH=$(JAVA_PATH) +CF_JAVA_HOME=$(JAVA_HOME) +CF_JAVA_OPTS=-Xms512m -Xmx512m # Two more potential interesting options are # CF_JAVA_OPTS=-XX:MaxMetaspaceSize=256M -XX:+UseG1GC -Xms2G -Xmx2G -CF_MVN_OPTS:= +CF_MVN_OPTS= + +# it will be good to have its own, reserve them for the future expansion +# currnetly, channelfinder log is in the /var/log/syslog +# +CF_LOG_PATH:=$(CF_INSTALL_LOCATION) +CF_LOG_FILE:=channelfinder.log -SYSTEMD_PATH:=/etc/systemd/system +# +CF_LOG_LEVEL:=WARN +CF_SITE_TEMPLATE_PATH=$(TOP)/site-template +CF_CREATE_INDEX:=false +# SYSTEDM Required and After Service List +# This is After and Required services for the systemd archiver appliance systemd service. +# mariadb.service was defined in there, but sometimes, we need to check our directories where +# our data will be saved. +# one can check the mounting service name through `systemctl list-units --type=mount` +# Here is the example for the ALS environment +# CONFIG_SITE.local can be used to hold this information locally +# Each service need one space between them, and backslash must be double backslash +CF_SYSTEMD_SERVICES:=elasticsearch.service +CF_KILL_PATH:=/usr/bin # These allow developers to override the CONFIG_SITE variable # settings without having to modify the configure/CONFIG_SITE # file itself. diff --git a/configure/CONFIG_SRC b/configure/CONFIG_SRC index 0b96c85..0c421fa 100644 --- a/configure/CONFIG_SRC +++ b/configure/CONFIG_SRC @@ -12,8 +12,14 @@ ifeq "$(SRC_PATH_INFO)" "1" INIT_SRC = 1 endif + +PATH:=$(MAVEN_PATH):$(JAVA_PATH):${PATH} + +MAVEN_CMD=$(MAVEN_PATH)/mvn +JAVA_CMD=$(JAVA_PATH)/java + + CF_SITE_SPECIFIC_FILES_PATH:=$(CF_SRC_PATH)/src/main/resources -CF_SITE_TEMPLATE_PATH:=$(TOP)/site-template VARS_EXCLUDES+=SRC_PATH_INFO VARS_EXCLUDES+=INSTALL_DATA diff --git a/configure/CONFIG_SYSTEMD b/configure/CONFIG_SYSTEMD new file mode 100644 index 0000000..4e12c46 --- /dev/null +++ b/configure/CONFIG_SYSTEMD @@ -0,0 +1,3 @@ +SYSTEMD_PATH:=/etc/systemd/system +CF_SYSTEMD_FILENAME:=channelfinder.service + diff --git a/configure/CONFIG_VARS b/configure/CONFIG_VARS index e1398dc..db7acac 100644 --- a/configure/CONFIG_VARS +++ b/configure/CONFIG_VARS @@ -10,3 +10,10 @@ CF_SRC_GITURL:=$(CF_SRC_URL)/$(CF_SRC_NAME) ### CF_SRC_PATH used for a local directory which is a clone git repository CF_SRC_PATH:=$(CF_SRC_NAME)-src + +#DOCURL:=https://docs.payara.fish/community/docs/$(SRC_VERSION)/README.html +DOCURL:=$(CF_SRC_GITURL) + +# These allow developers to override the variable +-include $(TOP)/../CONFIG_VARS.local +-include $(TOP)/configure/CONFIG_VARS.local diff --git a/configure/RELEASE b/configure/RELEASE index 4fb7bea..70c06c1 100644 --- a/configure/RELEASE +++ b/configure/RELEASE @@ -1,15 +1,18 @@ - -## Where the Source repository SRC_URL/SRC_NAME -## +: +##- Where the Source repository SRC_URL/SRC_NAME +##- CF_SRC_URL:=https://github.com/ChannelFinder -CF_SRC_NAME:=ChannelFinder-SpringBoot +CF_SRC_NAME:=ChannelFinderService -## Which the source tag / branch / hash id would like to use -CF_SRC_TAG:=537bf2c +##- Which the source tag / branch / hash id would like to use +# 2022-10-03 +CF_SRC_TAG:=4a17eea -## Placeholder for the site-specific version control -CF_SRC_VERSION:=4.0.0 +##- Placeholder for the site-specific version control +##- This information is used for application.properties +##- +CF_SRC_VERSION:=4.7.0-$(CF_SRC_TAG) -include $(TOP)/../RELEASE.local diff --git a/configure/RULES b/configure/RULES index 12f7b69..852eb04 100644 --- a/configure/RULES +++ b/configure/RULES @@ -1,9 +1,20 @@ +vpath %.local $(TOP)/configure +vpath %.in $(CF_SITE_TEMPLATE_PATH) +vpath %.properties $(CF_SITE_TEMPLATE_PATH) +vpath %.in $(CF_SITE_TEMPLATE_PATH)/systemd +vpath %.services $(CF_SITE_TEMPLATE_PATH)/systemd + +# include $(TOP)/configure/RULES_FUNC include $(TOP)/configure/RULES_REQ include $(TOP)/configure/RULES_SRC include $(TOP)/configure/RULES_INSTALL +include $(TOP)/configure/RULES_RUN +include $(TOP)/configure/RULES_SYSTEMD include $(TOP)/configure/RULES_DOCKER +include $(TOP)/configure/RULES_PROPERTIES +include $(TOP)/configure/RULES_CI include $(TOP)/configure/RULES_VARS diff --git a/configure/RULES_CI b/configure/RULES_CI new file mode 100644 index 0000000..8cb1387 --- /dev/null +++ b/configure/RULES_CI @@ -0,0 +1,36 @@ +.PHONY: conf.macos conf.macbrew + + +conf.macos: + $(QUIET)echo ">>> Generating $(TOP)/configure/CONFIG_COMMON.local" + $(QUIET)echo "JAVA_HOME:=/usr/" > $(TOP)/configure/CONFIG_COMMON.local + $(QUIET)echo "JAVA_PATH:=/usr/bin" >> $(TOP)/configure/CONFIG_COMMON.local + $(QUIET)echo "MAVEN_HOME:=/usr/local" >> $(TOP)/configure/CONFIG_COMMON.local + $(QUIET)echo "MAVEN_PATH:=/usr/local/bin" >> $(TOP)/configure/CONFIG_COMMON.local + $(QUIET)cat -b $(TOP)/configure/CONFIG_COMMON.local + $(QUIET)echo ">>> Generating $(TOP)/configure/CONFIG_SITE.local" + $(QUIET)echo "USERID:=${USER}" > $(TOP)/configure/CONFIG_SITE.local + $(QUIET)echo "GROUPID:=admin" >> $(TOP)/configure/CONFIG_SITE.local + $(QUIET)echo "CF_ES_PATH:=$(TOP)/$(ES_HOME_MAC)" >> $(TOP)/configure/CONFIG_SITE.local + $(QUIET)echo "CF_ES_CONF_PATH:=$(TOP)/$(ES_HOME_MAC)/config" >> $(TOP)/configure/CONFIG_SITE.local + $(QUIET)echo "CF_ES_DATA_PATH:=$(TOP)/$(ES_HOME_MAC)" >> $(TOP)/configure/CONFIG_SITE.local + $(QUIET)echo "CF_ES_LOGS_PATH:=$(TOP)/$(ES_HOME_MAC)/logs" >> $(TOP)/configure/CONFIG_SITE.local + $(QUIET)cat -b $(TOP)/configure/CONFIG_SITE.local + + +conf.macbrew: + $(QUIET)echo ">>> Generating $(TOP)/configure/CONFIG_COMMON.local" + $(QUIET)echo "JAVA_HOME:=/opt/homebrew/opt/openjdk@17/libexec/openjdk.jdk/Contents/Home" > $(TOP)/configure/CONFIG_COMMON.local + $(QUIET)echo "JAVA_PATH:=/opt/homebrew/opt/openjdk@17/libexec/openjdk.jdk/Contents/Home/bin" >> $(TOP)/configure/CONFIG_COMMON.local + $(QUIET)echo "MAVEN_HOME:=/opt/homebrew" >> $(TOP)/configure/CONFIG_COMMON.local + $(QUIET)echo "MAVEN_PATH:=/opt/homebrew/bin" >> $(TOP)/configure/CONFIG_COMMON.local + $(QUIET)cat -b $(TOP)/configure/CONFIG_COMMON.local + $(QUIET)echo ">>> Generating $(TOP)/configure/CONFIG_SITE.local" + $(QUIET)echo "USERID:=${USER}" > $(TOP)/configure/CONFIG_SITE.local + $(QUIET)echo "GROUPID:=admin" >> $(TOP)/configure/CONFIG_SITE.local + $(QUIET)echo "CF_ES_PATH:=$(TOP)/$(ES_HOME_MAC)" >> $(TOP)/configure/CONFIG_SITE.local + $(QUIET)echo "CF_ES_CONF_PATH:=$(TOP)/$(ES_HOME_MAC)/config" >> $(TOP)/configure/CONFIG_SITE.local + $(QUIET)echo "CF_ES_DATA_PATH:=$(TOP)/$(ES_HOME_MAC)" >> $(TOP)/configure/CONFIG_SITE.local + $(QUIET)echo "CF_ES_LOGS_PATH:=$(TOP)/$(ES_HOME_MAC)/logs" >> $(TOP)/configure/CONFIG_SITE.local + $(QUIET)cat -b $(TOP)/configure/CONFIG_SITE.local + diff --git a/configure/RULES_INSTALL b/configure/RULES_INSTALL index 71a0184..d6ce955 100644 --- a/configure/RULES_INSTALL +++ b/configure/RULES_INSTALL @@ -1,38 +1,5 @@ -.PHONY: sd_config sd_install sd_status sd_stop sd_clean sd_enable sd_disable - -sd_config: - $(QUIET)sed -e "s:_JAVAPATH_:$(CF_JAVA_PATH):g" -e "s:_JAVAOPTS_:$(CF_JAVA_OPTS):g" -e "s:_CFPATH_:$(CF_INSTALL_LOCATION):g" -e "s:_CHANNELFINDER_JAR_NAME_:$(CF_JAR_FILENAME):g" < $(CF_SITE_TEMPLATE_PATH)/cf.service.in > $(CF_SITE_TEMPLATE_PATH)/channelfinder.service - - -sd_install: sd_config - $(QUIET)$(SUDO) install -b -m 644 $(CF_SITE_TEMPLATE_PATH)/channelfinder.service $(SYSTEMD_PATH)/ - $(QUIET)$(SUDO) systemctl daemon-reload - -sd_status: - $(QUIET) systemctl status -l channelfinder.service | cat -b - -sd_start: - $(QUIET)$(SUDO) systemctl start channelfinder.service - -sd_stop: - $(QUIET)$(SUDO) systemctl stop channelfinder.service - -sd_restart: - $(QUIET)$(SUDO) systemctl restart channelfinder.service - - -sd_clean: - $(QUIET)$(SUDO) systemctl disable channelfinder.service - $(QUIET)$(SUDO) rm -f $(SYSTEMD_PATH)/channelfinder.service - -sd_enable: - $(if $(wildcard $(SYSTEMD_PATH)/channelfinder.service), $(SUDO) systemctl enable channelfinder.service ) - -sd_disable: - $(if $(wildcard $(SYSTEMD_PATH)/channelfinder.service), $(SUDO) systemctl disable channelfinder.service ) - -.PHONY: cf_status cf_start cf_stop cf_restart cf_install install uninstall +.PHONY: cf_status cf_start cf_stop cf_restart cf_install src_install install uninstall cf_status: sd_status @@ -42,14 +9,16 @@ cf_stop: sd_stop cf_restart: sd_restart - cf_install: $(QUIET)$(SUDO) install -d $(CF_INSTALL_LOCATION) $(QUIET)$(SUDO) install -m 744 $(CF_SRC_PATH)/target/$(CF_JAR_FILENAME) $(CF_INSTALL_LOCATION)/ + $(QUIET)$(SUDO) chown -R $(CF_USERID):$(CF_GROUPID) $(CF_INSTALL_LOCATION)/ + +src_install: cf_install ## Install Service related files -install: cf_install sd_install sd_enable +install: src_install sd_install sd_enable $(QUIET)echo "----- Note that one should start it and check its status via " $(QUIET)echo "----- systemctl start channelfinder" $(QUIET)echo "----- systemctl status channelfinder" @@ -64,3 +33,4 @@ uninstall: sd_stop sd_disable sd_clean reinstall: conf build install restart: uninstall reinstall cf_start cf_status + diff --git a/configure/RULES_PROPERTIES b/configure/RULES_PROPERTIES new file mode 100644 index 0000000..4a51ddb --- /dev/null +++ b/configure/RULES_PROPERTIES @@ -0,0 +1,47 @@ + + +properties_RULES_NAMES:=properties +conf_properties_RULES:=$(addprefix conf., $(properties_RULES_NAMES)) +show_properties_RULES:=$(addsuffix .show, $(conf_properties_RULES)) + +RULES_VARS+=conf.cfproperties conf.cfproperties.show + +.PHONY: conf conf.show + +conf: conf.cfproperties + +conf.show: conf.cfproperties.show + +conf.cfproperties: $(conf_properties_RULES) + +conf.cfproperties.show: $(show_properties_RULES) + +conf.properties: $(CF_PROPERTIES).in + $(QUIET)echo ">>> Generate $(basename $<) from $<" + $(QUIET)sed -e "s|@SSHKEY@|$(CF_SSHKEY)|g" \ + -e "s|@SSHKEYALIAS@|$(CF_SSHKEYALIAS)|g" \ + -e "s|@CFLDIF@|$(CF_LDIF)|g" \ + -e "s|@CF_HTTP_PORT@|$(CF_HTTP_PORT)|g" \ + -e "s|@CF_HTTPS_PORT@|$(CF_HTTPS_PORT)|g" \ + -e "s|@CF_HTTP_ENABLE@|$(CF_HTTP_ENABLE)|g" \ + -e "s|@ES_HOST@|$(ES_HOST)|g" \ + -e "s|@ES_PORT@|$(ES_PORT)|g" \ + -e "s|@CF_QUERY_SIZE@|$(CF_QUERY_SIZE)|g" \ + -e "s|@CF_SSL_REQUIRED@|$(CF_SSL_REQUIRED)|g" \ + -e "s|@CF_LOG_PATH@|$(CF_LOG_PATH)|g" \ + -e "s|@CF_LOG_FILE@|$(CF_LOG_FILE)|g" \ + -e "s|@CF_LOG_LEVEL@|$(CF_LOG_LEVEL)|g" \ + -e "s|@CF_CREATE_INDEX@|$(CF_CREATE_INDEX)|g" \ + -e "s|@CF_VERSION@|$(CF_SRC_VERSION)|g" \ + < $< > $(basename $<) +# rm -f $(CF_SITE_SPECIFIC_FILES_PATH)/$(notdir $(basename $<)) + $(QUIET)echo ">>> Copy $(basename $<) to $(CF_SITE_SPECIFIC_FILES_PATH)" + $(INSTALL_DATA) $(basename $<) $(CF_SITE_SPECIFIC_FILES_PATH)/ +# rm -rf $(CF_SITE_SPECIFIC_FILES_PATH)/$(CF_LDIF) + $(INSTALL_DATA) $(CF_SITE_TEMPLATE_PATH)/$(CF_LDIF) $(CF_SITE_SPECIFIC_FILES_PATH)/ +# rm -rf $(CF_SITE_SPECIFIC_FILES_PATH)/keystore/$(CF_SSHKEY) + $(INSTALL_DATA) $(CF_SITE_TEMPLATE_PATH)/$(CF_SSHKEY) $(CF_SITE_SPECIFIC_FILES_PATH)/keystore/ + +conf.properties.show: $(CF_PROPERTIES).in + $(QUIET)cat -b $(basename $<) + diff --git a/configure/RULES_REQ b/configure/RULES_REQ index 47a3d4f..71f5423 100644 --- a/configure/RULES_REQ +++ b/configure/RULES_REQ @@ -1,29 +1,162 @@ -.PHONY: es_install es_start es_status es_mapping es_mapping_clean es_all -## Build Source for Debian 10 -es_install: - $(QUIET) wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-6.3.1.deb - $(QUIET)$(SUDO) dpkg -i elasticsearch-6.3.1.deb - $(QUIET)$(SUDO) systemctl enable elasticsearch +es_RULES_NAMES:=esdeb esrpm esmac esyml +install_es_RULES:=$(addprefix install., $(es_RULES_NAMES)) +setup_es_RULES:=$(addprefix setup., $(es_RULES_NAMES)) +remove_es_RULES:=$(addprefix remove., $(es_RULES_NAMES)) -es_start: - $(QUIET)$(SUDO) systemctl start elasticsearch +.PHONY: clean.conf.es conf.es start.es status.es restart.es stop.es disable.es mapping mapping.clean mapping.settings conf.es.show conf.esmac.show +.PHONE: start.esmac stop.esmac -es_status: - $(QUIET)systemctl status -l elasticsearch | cat +#https://www.elastic.co/guide/en/elasticsearch/reference/current/release-notes-8.2.0.html +ES_NAME:=elasticsearch +ES_VER:=8.2.0 +ES_FILE:=$(ES_NAME)-$(ES_VER) +ES_DEB:=$(ES_FILE)-amd64.deb +ES_RPM:=$(ES_FILE)-x86_64.rpm +ifeq ($(UNAME_M), arm64) +ES_MAC:=$(ES_FILE)-darwin-aarch64.tar.gz +else +ES_MAC:=$(ES_FILE)-darwin-x86_64.tar.gz +endif +ES_HOME_MAC:=$(ES_FILE) +ES_PID_MAC:=$(TOP)/$(ES_HOME_MAC)/es_mac_pid +ES_JAVA_OPTS_FILE:=cf_es_jvm.options + +# Debian 11 +setup.esdeb: install.esdeb enable.es + +setup.esrpm: install.esrpm enable.es + +setup.esmac: install.esmac + + +clean.conf.es: + $(QUIET) rm -f $(CF_SITE_TEMPLATE_PATH)/$(ES_JAVA_OPTS_FILE) + +$(CF_ES_JAVA_OPTS): clean.conf.es + $(QUIET)echo "$@" >> $(CF_SITE_TEMPLATE_PATH)/$(ES_JAVA_OPTS_FILE) + +#conf.es: $(CF_ES_JAVA_OPTS) +# $(QUIET)$(SUDO) $(INSTALL) -p -m 660 $(CF_SITE_TEMPLATE_PATH)/$(ES_JAVA_OPTS_FILE) $(CF_ES_CONF_PATH)/jvm.options.d/ +# $(QUIET)$(SUDO) sed -i~ 's/^xpack.security.enabled:.*/xpack.security.enabled: $(CF_ES_XPACK_SECURITY_ENABLED)/g' $(CF_ES_CONF_PATH)/$(CF_ES_CONF_NAME) + +conf.es: $(CF_ES_JAVA_OPTS) conf.esyml + $(QUIET)$(SUDO) $(INSTALL) -p -m 660 $(CF_SITE_TEMPLATE_PATH)/$(ES_JAVA_OPTS_FILE) $(CF_ES_CONF_PATH)/jvm.options.d/ + $(QUIET)$(SUDO) $(INSTALL) -p -m 660 $(CF_SITE_TEMPLATE_PATH)/$(CF_ES_CONF_NAME) $(CF_ES_CONF_PATH)/ + +conf.esmac: $(CF_ES_JAVA_OPTS) conf.esyml + $(QUIET) echo ">>> Installing $(ES_JAVA_OPTS_FILE) and $(CF_ES_CONF_NAME) to $(CF_ES_CONF_PATH)" + $(QUIET) $(INSTALL) -p -m 660 $(CF_SITE_TEMPLATE_PATH)/$(ES_JAVA_OPTS_FILE) $(CF_ES_CONF_PATH)/jvm.options.d/ + $(QUIET) $(INSTALL) -p -m 640 $(CF_SITE_TEMPLATE_PATH)/$(CF_ES_CONF_NAME) $(CF_ES_CONF_PATH)/ + +conf.esyml: $(CF_ES_CONF_NAME).in + $(QUIET)echo ">>> Generate $(basename $<) from $<" + $(QUIET)sed -e "s|@CF_ES_NETWORK_HOST@|$(CF_ES_NETWORK_HOST)|g" \ + -e "s|@CF_ES_HTTP_HOST@|$(CF_ES_HTTP_HOST)|g" \ + -e "s|@CF_ES_CLUSTER_NAME@|$(CF_ES_CLUSTER_NAME)|g" \ + -e "s|@CF_ES_NODE_NAME@|$(CF_ES_NODE_NAME)|g" \ + -e "s|@CF_ES_DATA_PATH@|$(CF_ES_DATA_PATH)|g" \ + -e "s|@CF_ES_LOGS_PATH@|$(CF_ES_LOGS_PATH)|g" \ + -e "s|@CF_ES_DISCOVERY_TYPE@|$(CF_ES_DISCOVERY_TYPE)|g" \ + -e "s|@CF_ES_XPACK_SECURITY_ENABLED@|$(CF_ES_XPACK_SECURITY_ENABLED)|g" \ + -e "s|@CF_ES_XPACK_SECURITY_ENROLL_ENABLED@|$(CF_ES_XPACK_SECURITY_ENROLL_ENABLED)|g" \ + -e "s|@CF_ES_XPACK_SECURITY_HTTP_SSL@|$(CF_ES_XPACK_SECURITY_HTTP_SSL)|g" \ + -e "s|@CF_ES_XPACK_SECURITY_HTTP_SSL_KEYSTORE_PATH@|$(CF_ES_XPACK_SECURITY_HTTP_SSL_KEYSTORE_PATH)|g" \ + -e "s|@CF_ES_XPACK_SECURITY_TRANSPORT_SSL@|$(CF_ES_XPACK_SECURITY_TRANSPORT_SSL)|g" \ + -e "s|@CF_ES_XPACK_SECURITY_TRANSPORT_SSL_VERIFICATION@|$(CF_ES_XPACK_SECURITY_TRANSPORT_SSL_VERIFICATION)|g" \ + -e "s|@CF_ES_XPACK_SECURITY_TRANSPORT_SSL_KEYSTORE_PATH@|$(CF_ES_XPACK_SECURITY_TRANSPORT_SSL_KEYSTORE_PATH)|g" \ + -e "s|@CF_ES_XPACK_SECURITY_TRANSPORT_SSL_TRUSTSTORE_PATH@|$(CF_ES_XPACK_SECURITY_TRANSPORT_SSL_TRUSTSTORE_PATH)|g" \ + -e "s|@CF_ES_HTTP_HOST@|$(CF_ES_HTTP_HOST)|g" \ + < $< > $(basename $<) + +conf.esyml.show: $(CF_ES_CONF_NAME).in + $(QUIET)cat -b $(basename $<) + +conf.esmac.show: + cat -b $(CF_ES_CONF_PATH)/jvm.options.d/$(ES_JAVA_OPTS_FILE) + cat -b $(CF_ES_CONF_PATH)/$(CF_ES_CONF_NAME) + +conf.es.show: + $(SUDO) cat -b $(CF_ES_CONF_PATH)/jvm.options.d/$(ES_JAVA_OPTS_FILE) + $(SUDO) cat -b $(CF_ES_CONF_PATH)/$(CF_ES_CONF_NAME) + +install.esdeb: + $(QUIET) wget -c https://artifacts.elastic.co/downloads/elasticsearch/$(ES_DEB) + $(SUDO) dpkg -i ./$(ES_DEB) + +remove.esdeb: + $(QUIET)$(SUDO) apt purge -y elasticsearch + +# Rocky 8 +install.esrpm: + $(QUIET) wget -c https://artifacts.elastic.co/downloads/elasticsearch/$(ES_RPM) + $(SUDO) dnf localinstall -y $(ES_RPM) + +remove.esrpm: + $(QUIET)$(SUDO) dnf remove elasticsearch + +# Centos 7 +install.esrpm-yum: + $(QUIET) wget -c https://artifacts.elastic.co/downloads/elasticsearch/$(ES_RPM) + $(SUDO) yum localinstall -y $(ES_RPM) + +remove.esrpm-yum: + $(QUIET)$(SUDO) yum remove elasticsearch + +# macOS aarch64 +install.esmac: + $(QUIET) curl -O https://artifacts.elastic.co/downloads/elasticsearch/$(ES_MAC) + $(QUIET) curl https://artifacts.elastic.co/downloads/elasticsearch/$(ES_MAC).sha512 | shasum -a 512 -c - + tar -xzf $(ES_MAC) + +remove.esmac: + $(QUIET) rm -rf $(ES_FILE)* + +enable.es: + $(QUIET)$(SUDO) systemctl daemon-reload + $(QUIET)$(SUDO) systemctl enable elasticsearch.service + +start.es: + $(QUIET)$(SUDO) systemctl start elasticsearch.service + +start.esmac: + $(QUIET) $(TOP)/$(ES_HOME_MAC)/bin/elasticsearch -d -p $(ES_PID_MAC) + +start.esmac.console: + $(QUIET) $(TOP)/$(ES_HOME_MAC)/bin/elasticsearch -p $(ES_PID_MAC) + +status.es: + $(QUIET)systemctl status -l elasticsearch.service | cat -b + +status.esmac: + $(QUIET) cat -b $(TOP)/$(ES_HOME_MAC)/logs/elasticsearch.log + +stop.es: + $(QUIET)systemctl stop elasticsearch.service + +stop.esmac: + $(QUIET)pkill -F $(ES_PID_MAC) + +disable.es: + $(QUIET)systemctl disable elasticsearch.service + +restart.es: + $(QUIET)$(SUDO) systemctl restart elasticsearch.service ## Create the ES indexes and set up their mapping -es_mapping: - $(QUIET)bash $(CF_SITE_TEMPLATE_PATH)/mapping_definitions.bash - $(QUIET)curl -XPUT "http://$(ES_HOST):$(ES_PORT)/channelfinder/_settings" -d '{ "index" : { "max_result_window" : $(CF_QUERY_SIZE) } }' -H "Content-Type: application/json" - $(QUIET)curl -XPUT "http://$(ES_HOST):$(ES_PORT)/cf_properties/_settings" -d '{ "index" : { "max_result_window" : $(CF_QUERY_SIZE) } }' -H "Content-Type: application/json" - $(QUIET)curl -XPUT "http://$(ES_HOST):$(ES_PORT)/cf_tags/_settings" -d '{ "index" : { "max_result_window" : $(CF_QUERY_SIZE) } }' -H "Content-Type: application/json" +mapping: + $(QUIET)bash $(TOP)/scripts/cf_es_configuration.bash mapping ## Clean all ES indexes and their mapping -es_mapping_clean: - $(QUIET)bash $(CF_SITE_TEMPLATE_PATH)/clean_mapping_definitions.bash +mapping.clean: + $(QUIET)bash $(TOP)/scripts/cf_es_configuration.bash mappingClean + +mapping.verify: + $(QUIET)bash $(TOP)/scripts/cf_es_configuration.bash mappingVerify +mapping.settings: + $(QUIET)curl -XPUT "http://$(ES_HOST):$(ES_PORT)/channelfinder/_settings" -d '{ "index" : { "max_result_window" : $(CF_QUERY_SIZE) } }' -H "Content-Type: application/jsoin" + $(QUIET)curl -XPUT "http://$(ES_HOST):$(ES_PORT)/cf_properties/_settings" -d '{ "index" : { "max_result_window" : $(CF_QUERY_SIZE) } }' -H "Content-Type: application/json" + $(QUIET)curl -XPUT "http://$(ES_HOST):$(ES_PORT)/cf_tags/_settings" -d '{ "index" : { "max_result_window" : $(CF_QUERY_SIZE) } }' -H "Content-Type: application/json" -es_all: es_install es_start es_start es_mapping - diff --git a/configure/RULES_RUN b/configure/RULES_RUN new file mode 100644 index 0000000..9fc138e --- /dev/null +++ b/configure/RULES_RUN @@ -0,0 +1,18 @@ + +run: + $(QUIET)mvn -f $(CF_SRC_PATH) spring-boot:run -Dspring.pid.file=$(TOP)/channelfinder.pid + +run.check: + $(QUIET)lsof -i:8080 | xargs -I {} echo {} + +run.kill: + $(QUIET)lsof -i:8080 -t | xargs -I {} kill -9 {} + +demo: + $(QUIET)mvn -f $(CF_SRC_PATH) spring-boot:run -Dspring-boot.run.arguments="--demo-data=1" + +demo.clean: + $(QUIET)mvn -f $(CF_SRC_PATH) spring-boot:run -Dspring-boot.run.arguments="--cleanup=1" + +cflog.show: + $(SUDO) tail -f /var/log/syslog |grep java diff --git a/configure/RULES_SRC b/configure/RULES_SRC index f661373..e51b5aa 100644 --- a/configure/RULES_SRC +++ b/configure/RULES_SRC @@ -49,19 +49,7 @@ endif .PHONY: build conf ## Build Source -build: - $(QUIET) mvn $(CF_MVN_OPTS) -f $(CF_SRC_PATH)/pom.xml clean install -Dmaven.test.skip=true +build: conf + $(QUIET) JAVA_HOME=$(CF_JAVA_HOME) $(MAVEN_CMD) $(CF_MVN_OPTS) -f $(CF_SRC_PATH)/pom.xml clean install -Dmaven.test.skip=true + -## Copy the site specific files into sources -conf: - $(QUIET)sed -e "s:@SSHKEY@:$(CF_SSHKEY):g" \ - -e "s:@SSHKEYALIAS@:$(CF_SSHKEYALIAS):g" \ - -e "s:@CFLDIF@:$(CF_LDIF):g" \ - -e "s:@CF_PORT@:$(CF_PORT):g" \ - -e "s:@ES_HOST@:$(ES_HOST):g" \ - -e "s:@ES_PORT@:$(ES_PORT):g" \ - -e "s:@CF_QUERY_SIZE@:$(CF_QUERY_SIZE):g" \ - < $(CF_SITE_TEMPLATE_PATH)/application.properties.in > $(CF_SITE_TEMPLATE_PATH)/application.properties - $(QUIET)cp $(CF_SITE_TEMPLATE_PATH)/$(CF_LDIF) $(CF_SITE_SPECIFIC_FILES_PATH)/ - $(QUIET)cp $(CF_SITE_TEMPLATE_PATH)/$(CF_SSHKEY) $(CF_SITE_SPECIFIC_FILES_PATH)/keystore/ - $(QUIET)cp $(CF_SITE_TEMPLATE_PATH)/application.properties $(CF_SITE_SPECIFIC_FILES_PATH)/ diff --git a/configure/RULES_SYSTEMD b/configure/RULES_SYSTEMD new file mode 100644 index 0000000..994df25 --- /dev/null +++ b/configure/RULES_SYSTEMD @@ -0,0 +1,77 @@ +# +# +.PHONY: sd_install sd_status sd_start sd_stop sd_restart sd_clean sd_enable sd_disable +# +# + +systemd_RULES_NAMES:=systemd0 +conf_systemd_RULES:=$(addprefix conf., $(systemd_RULES_NAMES)) +show_systemd_RULES:=$(addsuffix .show, $(conf_systemd_RULES)) +install_systemd_RULES:=$(addprefix install., $(systemd_RULES_NAMES)) +show_install_systemd_RULES:=$(addsuffix .show, $(install_systemd_RULES)) + +sd_install: conf.systemd install.systemd + $(QUIET)$(SUDO) systemctl daemon-reload + +.PHONY: conf.systemd conf.systemd.show + +conf.systemd: $(conf_systemd_RULES) + +conf.systemd.show: $(show_systemd_RULES) + +install.systemd: $(install_systemd_RULES) + +install.systemd.show: $(show_install_systemd_RULES) + +conf.systemd0: $(CF_SYSTEMD_FILENAME).in + $(QUIET)echo ">>> Generate $(basename $<) from $<" + $(QUIET)echo ">>> Required Services : $(CF_SYSTEMD_SERVICES)" + $(QUIET)sed -e "s|@DOCURL@|$(DOCURL)|g" \ + -e 's|@SYSTEMD_SERVICES@|$(CF_SYSTEMD_SERVICES)|g' \ + -e "s|@CF_USERID@|$(CF_USERID)|g" \ + -e "s|@CF_GROUPID@|$(CF_GROUPID)|g" \ + -e "s|@CF_JAVA_PATH@|$(CF_JAVA_PATH)|g" \ + -e "s|@CF_JAVA_OPTS@|$(CF_JAVA_OPTS)|g" \ + -e "s|@CF_INSTALL_LOCATION@|$(CF_INSTALL_LOCATION)|g" \ + -e "s|@CF_KILL_PATH@|$(CF_KILL_PATH)|g" \ + -e "s|@CF_JAR_NAME@|$(CF_JAR_FILENAME)|g" \ + < $< > $(basename $<) + +conf.systemd0.show: $(CF_SYSTEMD_FILENAME).in + @cat -b $(basename $<) + +install.systemd0: $(CF_SYSTEMD_FILENAME).in + $(QUIET)$(SUDO) $(INSTALL_DATA) -b $(basename $<) $(SYSTEMD_PATH)/ + +install.systemd0.show: + $(QUIET)cat -b $(SYSTEMD_PATH)/$(CF_SYSTEMD_FILENAME) +# +# +sd_status: + $(QUIET) systemctl status -l $(CF_SYSTEMD_FILENAME) | cat -b + +# +sd_start: + $(QUIET)$(SUDO) systemctl start $(CF_SYSTEMD_FILENAME) +# +# We ignore its error +sd_stop: + -$(QUIET)$(SUDO) systemctl stop $(CF_SYSTEMD_FILENAME) +# +# +sd_restart: + $(QUIET)$(SUDO) systemctl restart $(CF_SYSTEMD_FILENAME) +# +# We ignore its error +sd_clean: + $(if $(wildcard $(SYSTEMD_PATH)/$(CF_SYSTEMD_FILENAME)), $(QUIET)$(SUDO) rm -f $(SYSTEMD_PATH)/$(CF_SYSTEMD_FILENAME)) +# +# +sd_enable: + $(if $(wildcard $(SYSTEMD_PATH)/$(CF_SYSTEMD_FILENAME)), $(SUDO) systemctl enable $(CF_SYSTEMD_FILENAME)) +# +# +sd_disable: + $(if $(wildcard $(SYSTEMD_PATH)/$(CF_SYSTEMD_FILENAME)), $(SUDO) systemctl disable $(CF_SYSTEMD_FILENAME)) +# +# diff --git a/configure/RULES_VARS b/configure/RULES_VARS index 015647d..9ee2641 100644 --- a/configure/RULES_VARS +++ b/configure/RULES_VARS @@ -1,18 +1,23 @@ +RULES_VARS+=$(foreach v, $(filter %_RULES,$(.VARIABLES)), $(v)) +RULES_VARS+=$(foreach v, $(filter %_RULES_NAMES,$(.VARIABLES)), $(v)) + VARS_EXCLUDES+=.SHELLSTATUS VARS_EXCLUDES+=INIT_SRC -VARS_EXCLUDES+=SITE_SPECIFIC_FILES_PATH -VARS_EXCLUDES+=RS_% -VARS_EXCLUDES+=AS_% -VARS_EXCLUDES+=OG_% -VARS_EXCLUDES+=PS_% -VARS_EXCLUDES+=KAFKA_% -VARS_EXCLUDES+=ALARM_% -VARS_EXCLUDES+=AL_% -VARS_EXCLUDES+=AA_% +VARS_EXCLUDES+=FILTER +VARS_EXCLUDES+=LEVEL +VARS_EXCLUDES+=$(filter JAR_%, $(.VARIABLES)) +VARS_EXCLUDES+=$(RULES_VARS) +VARS_EXCLUDES+=%_RULES_VARS +VARS_EXCLUDES+=a_service_BUIDER buildrules_BUILDER +VARS_EXCLUDES+=RULES_VARS +VARS_EXCLUDES+=LSOPTS -LEVEL?=1 +.PHONY: $(RULES_VARS) +LEVEL?=1 FILTER?=1 +LSOPTS?="-lta" + ifeq "$(FILTER)" "1" SRC_VARIABLES:=$(sort $(filter-out $(VARS_EXCLUDES) VARS_EXCLUDES,$(.VARIABLES))) @@ -20,8 +25,6 @@ else SRC_VARIABLES:=$(filter $(FILTER)%, $(sort $(filter-out $(VARS_EXCLUDES) VARS_EXCLUDES,$(.VARIABLES)))) endif - - .PHONY : env vars header env: vars @@ -42,18 +45,30 @@ PRINT.%: $(QUIET)echo $*\'s origin is $(origin $*) print-%: - $(QUIET)echo $* = $($*) - $(QUIET)echo $*\'s origin is $(origin $*) + $(QUIET)echo $($*) + +ls.%: + $(QUIET) ls $(LSOPTS) $($*) + +tree.%: + $(QUIET) tree -aL $(LEVEL) $($*) + +cat.%: + $(QUIET) cat -b $($*) + +FORCE: .PHONY : exist exist: - $(if $(wildcard $(CF_INSTALL_LOCATION)), tree -aL $(LEVEL) $(CF_INSTALL_LOCATION), $(QUIET)echo "No $(CF_INSTALL_LOCATION)") +ifeq (, $(shell which tree)) + $(if $(wildcard $(CF_INSTALL_LOCATION)), @ ls -d $(CF_INSTALL_LOCATION) && find $(CF_INSTALL_LOCATION)/ -maxdepth $(LEVEL) | sed -e "s/[^-][^\/]*\// |/g" -e "s/|\([^ ]\)/:---\1/") +else + $(if $(wildcard $(CF_INSTALL_LOCATION)), tree -pugaL $(LEVEL) $(CF_INSTALL_LOCATION), $(QUIET)echo "No $(CF_INSTALL_LOCATION)") +endif .DELETE_ON_ERROR: .NOTPARALLEL: - -VARS_EXCLUDES+=FILTER \ No newline at end of file diff --git a/docker/Dockerfile b/docker/Dockerfile deleted file mode 120000 index 45cc04d..0000000 --- a/docker/Dockerfile +++ /dev/null @@ -1 +0,0 @@ -Dockerfile.alpine \ No newline at end of file diff --git a/docker/Dockerfile b/docker/Dockerfile new file mode 100644 index 0000000..3c2a6a5 --- /dev/null +++ b/docker/Dockerfile @@ -0,0 +1,102 @@ +# +# author : Jeong Han Lee +# email : jeonghan.lee@gmail.com +# date : Sat Sep 17 21:29:34 PDT 2022 +# version : 0.0.3 +# +# + +## BuildEnv docker image size : 672MB +FROM alpine:3.16 AS builder +ENV WorkPath /home/ChannelFinder-env +ARG RS_SERVER_PORT=5075 +ARG RS_PORT_BROADCAST=5076 +ARG CF_PORT_HTTP=8080 +ARG CF_PORT_HTTPS=8443 +ARG CF_INSTALL_LOCATION=/opt/channelfinder + +ENV JAVA_HOME="/usr/lib/jvm/default-jvm/" +# We can check the package availbilities through docker run -t -i --entrypoint /bin/sh alpine:3.16 +RUN apk update && apk add openjdk17 maven git make + +WORKDIR $WorkPath +COPY . . +RUN echo "CF_PORT:=${CF_PORT_HTTP}" > configure/CONFIG_COMMON.local +RUN echo "CF_INSTALL_LOCATION:=${CF_INSTALL_LOCATION}" > configure/CONFIG_SITE.local +RUN echo "JAVA_HOME:=/usr/lib/jvm/default-jvm/" >> configure/CONFIG_COMMON.local +RUN echo "JAVA_PATH:=/usr/lib/jvm/default-jvm/bin" >> configure/CONFIG_COMMON.local +RUN echo "MAVEN_HOME:=/usr/" >> configure/CONFIG_COMMON.local +RUN echo "MAVEN_PATH:=/usr//bin" >> configure/CONFIG_COMMON.local + +RUN make distclean && \ + make init && \ + make conf && \ + make build && \ + make install.docker + # make build MVN_OPTS=dependency:go-offline + +## Multi-Stages build +## Running docker image size : 240MB +FROM alpine:3.16 +LABEL maintainer="Jeong Han Lee " + +ARG BUILD_DATE +ARG BUILD_VERSION +ARG RS_SERVER_PORT=5075 +ARG RS_PORT_BROADCAST=5076 +ARG CF_PORT_HTTP=8080 +ARG CF_PORT_HTTPS=8443 +ARG CF_INSTALL_LOCATION=/opt/channelfinder +ARG TZ=America/Los_Angeles + +# Labels. +LABEL org.label-schema.schema-version="1.0" +LABEL org.label-schema.build-date=$BUILD_DATE +LABEL org.label-schema.name="alscontrols/channelfinder" +LABEL org.label-schema.description="EPICS ChannelFinder Docker Image" +LABEL org.label-schema.url="https://github.com/ChannelFinder/ChannelFinder-env/" +LABEL org.label-schema.version=$BUILD_VERSION +LABEL org.label-schema.docker.cmd="docker run --network=host --detach --rm --name=channelfinder alscontrols/channelfinder:latest" + + +### RecCeiver PORT configuration. +ENV RS_SERVER_PORT ${RS_SERVER_PORT} +ENV RS_PORT_BROADCAST ${RS_PORT_BROADCAST} +# + +# +### DO NOT CHANGE IT +ENV LDAP_SERVER_PORT 8389 +# +### ChannelFinder HTTP PORT through Tomcat +ENV CF_PORT_HTTP ${CF_PORT_HTTP} +### ChannelFinder HTTPS PORT through Tomcat +ENV CF_PORT_HTTPS ${CF_PORT_HTTPS} +### ChannelFinder Installation Loation +ENV CF_INSTALL_LOCATION ${CF_INSTALL_LOCATION} + +### YOUR TIME ZONE +ENV TZ ${TZ} +RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone +### Install minimal packages for ChannelFinder +RUN apk add --no-cache bash openjdk17-jre tomcat-native tzdata + +WORKDIR ${CF_INSTALL_LOCATION} +COPY --from=builder ${CF_INSTALL_LOCATION} . +## +# CF Port : http +EXPOSE $CF_PORT_HTTP/tcp +EXPOSE $CF_PORT_HTTP/udp +# CF PORT : https +EXPOSE $CF_PORT_HTTPS/tcp +EXPOSE $CF_PORT_HTTPS/udp +# RecCeiver Broadcast Port +EXPOSE $RS_PORT_BROADCAST/udp +# RecCeiver Server Port +EXPOSE $RS_SERVER_PORT/udp +# Embedded LDAP Port +EXPOSE $LDAP_SERVER_PORT/tcp +EXPOSE $LDAP_SERVER_PORT/udp +# +COPY ./docker/scripts/entrypoint.sh /entrypoint.sh +ENTRYPOINT ["/entrypoint.sh"] diff --git a/docker/Dockerfile.alpine b/docker/Dockerfile.alpine deleted file mode 100644 index 52460da..0000000 --- a/docker/Dockerfile.alpine +++ /dev/null @@ -1,97 +0,0 @@ -# -# author : Jeong Han Lee -# email : jeonghan.lee@gmail.com -# date : Wednesday, July 1 14:32:24 PDT 2020 -# version : 0.0.2 -# -# - -## BuildEnv docker image size : 672MB -FROM alpine:3.12 AS builder -ENV WorkPath /home/ChannelFinder-env -ARG RS_SERVER_PORT=5075 -ARG RS_PORT_BROADCAST=5076 -ARG CF_PORT_HTTP=8080 -ARG CF_PORT_HTTPS=8443 -ARG CF_INSTALL_LOCATION=/opt/channelfinder - -ENV JAVA_HOME="/usr/lib/jvm/default-jvm/" -RUN apk update && apk add openjdk11 maven git make - -WORKDIR $WorkPath -COPY . . -RUN echo "CF_PORT:=${CF_PORT_HTTP}" > configure/CONFIG_COMMON.local -RUN echo "CF_INSTALL_LOCATION:=${CF_INSTALL_LOCATION}" > configure/CONFIG_SITE.local -RUN make distclean && \ - make init && \ - make conf && \ - make build && \ - make install.docker - # make build MVN_OPTS=dependency:go-offline - - -## Multi-Stages build -## Running docker image size : 240MB -FROM alpine:3.12 -LABEL maintainer="Jeong Han Lee " - -ARG BUILD_DATE -ARG BUILD_VERSION -ARG RS_SERVER_PORT=5075 -ARG RS_PORT_BROADCAST=5076 -ARG CF_PORT_HTTP=8080 -ARG CF_PORT_HTTPS=8443 -ARG CF_INSTALL_LOCATION=/opt/channelfinder -ARG TZ=America/Los_Angeles - -# Labels. -LABEL org.label-schema.schema-version="1.0" -LABEL org.label-schema.build-date=$BUILD_DATE -LABEL org.label-schema.name="jeonghanlee/channelfinder" -LABEL org.label-schema.description="EPICS ChannelFinder Docker Image" -LABEL org.label-schema.url="https://github.com/ChannelFinder/ChannelFinder-env/" -LABEL org.label-schema.version=$BUILD_VERSION -LABEL org.label-schema.docker.cmd="docker run --network=host --detach --rm --name=channelfinder jeonghanlee/channelfinder:latest" - - -### RecCeiver PORT configuration. -ENV RS_SERVER_PORT ${RS_SERVER_PORT} -ENV RS_PORT_BROADCAST ${RS_PORT_BROADCAST} -# - -# -### DO NOT CHANGE IT -ENV LDAP_SERVER_PORT 8389 -# -### ChannelFinder HTTP PORT through Tomcat -ENV CF_PORT_HTTP ${CF_PORT_HTTP} -### ChannelFinder HTTPS PORT through Tomcat -ENV CF_PORT_HTTPS ${CF_PORT_HTTPS} -### ChannelFinder Installation Loation -ENV CF_INSTALL_LOCATION ${CF_INSTALL_LOCATION} - -### YOUR TIME ZONE -ENV TZ ${TZ} -RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone -### Install minimal packages for ChannelFinder -RUN apk add --no-cache bash openjdk11-jre tomcat-native tzdata - -WORKDIR ${CF_INSTALL_LOCATION} -COPY --from=builder ${CF_INSTALL_LOCATION} . -## -# CF Port : http -EXPOSE $CF_PORT_HTTP/tcp -EXPOSE $CF_PORT_HTTP/udp -# CF PORT : https -EXPOSE $CF_PORT_HTTPS/tcp -EXPOSE $CF_PORT_HTTPS/udp -# RecCeiver Broadcast Port -EXPOSE $RS_PORT_BROADCAST/udp -# RecCeiver Server Port -EXPOSE $RS_SERVER_PORT/udp -# Embedded LDAP Port -EXPOSE $LDAP_SERVER_PORT/tcp -EXPOSE $LDAP_SERVER_PORT/udp -# -COPY ./docker/scripts/entrypoint.sh /entrypoint.sh -ENTRYPOINT ["/entrypoint.sh"] diff --git a/docker/Dockerfile.buster b/docker/old/Dockerfile.buster similarity index 100% rename from docker/Dockerfile.buster rename to docker/old/Dockerfile.buster diff --git a/docs/ChannelFinderConf.md b/docs/ChannelFinderConf.md new file mode 100644 index 0000000..ac659aa --- /dev/null +++ b/docs/ChannelFinderConf.md @@ -0,0 +1,88 @@ +# ChannelFinder Configuration + +## Tomcat generic service + +It is always better to check the Tomcat service first, because the current configuration uses `8080` as the channelfinder port. + +```bash +systemctl status tomcat{9} +systemctl stop tomcat{9} +systemctl disable tomcat{9} +``` + +## Check other services hold 8080 + +```bash +lsof -i:8080 +``` + +``` +kill -i pid_number +``` +### a few rules + +The local run rule has the more options to check the process status, and kill the process which holds the port 8080. + +``` +make run.check +make run.kill +``` + +## Create Indexes and add mapping information + +We have three index(s), which I don't use `indices`, because it is a slightly different definition in ES, such as + +```bash +channelfinder +cf_tags +cf_properties +``` + +This can be created via `make mapping`, which is our preference, or set `elasticsearch.create.indices: true` in `applicaton.properties` + + +### `make mapping` + +This is the Makefile rule, to call `cf_es_configuration.bash` in `site-template` path. It uses the same json files to create three index(s) with `CF_QUERY_SIZE` configuration. We have the following Makefile rules: + +``` +make mapping +make mapping.clean +make mapping.verify +``` + +And the script will allow us to explore them in detail. + +``` +Usage : site-template/cf_es_configuration.bash args + + possbile args + + mapping : CF index mapping + mappingClean : Delete all CF index + mappingVerify : Verify the existing mappings + metaField 'search' : mapping, search, .... + h : this screen +``` + +### `application.properties` + +* Before `make build` + +``` +echo "CF_CREATE_INDEX:=true" >> configure/CONFIG_SITE.local +``` + +After the channelfinder service starts, one can see the following log + +```bash +2022-09-20 22:40:21.084 INFO 78852 --- [ main] org.phoebus.channelfinder.ElasticConfig : Initializing a new Transport clients. +2022-09-20 22:40:22.079 INFO 78852 --- [ main] org.phoebus.channelfinder.ElasticConfig : Created index: channelfinder : acknowledged true +2022-09-20 22:40:22.666 INFO 78852 --- [ main] org.phoebus.channelfinder.ElasticConfig : Created index: cf_tags : acknowledged true +2022-09-20 22:40:23.305 INFO 78852 --- [ main] org.phoebus.channelfinder.ElasticConfig : Created index: cf_properties : acknowledged true +``` + +And please execute the following rule also + +```bash +make mapping.settings diff --git a/docs/ChannelFinderDemo.md b/docs/ChannelFinderDemo.md new file mode 100644 index 0000000..043801f --- /dev/null +++ b/docs/ChannelFinderDemo.md @@ -0,0 +1,117 @@ +# Built-in Demo Testing + +## Run Demo + +This demo is not available within the systemd service. Thus, please use it without systemd services. + +One should follow the strict order to run a channelfinder built-in demo service if the channelfinder runs locally, i.e., no systemd service. + +* Check the system is not running now. + +```bash +systemctl status channelfinder.service +systemctl stop channelfinder.service +systemctl disable channelfinder.service +``` + +* One should be in the `ChannelFinder-env` + +``` +make run +make mapping +make demo +``` + +## Check the Demo service + +If one doesn't have `jq`, it is highly recommended to install it. For example, `apt install jq` in Debian system. + +``` +curl -s http://localhost:8080/ChannelFinder/resources/tags | jq +curl -s http://localhost:8080/ChannelFinder/resources/properties | jq +curl -s http://localhost:8080/ChannelFinder/resources/channels | jq +``` + +## Check, Add, and Remove a tag + + +```bash +$ curl -s --request GET http://localhost:8080/ChannelFinder/resources/tags/foo +{"name":"foo","owner":"admin","channels":[]} + +$ curl -s -u admin:1234 -H 'Content-Type: application/json' --request PUT http://localhost:8080/ChannelFinder/resources/tags/foo -d '{"name":"foo", "owner":"admin"}' +{"name":"foo","owner":"admin","channels":[]} + +$ curl -s --request GET http://localhost:8080/ChannelFinder/resources/tags/foo +{"name":"foo","owner":"admin","channels":[]} + +$curl --basic -u admin:1234 -X DELETE http://localhost:8080/ChannelFinder/resources/tags/foo +$ curl -s --request GET http://localhost:8080/ChannelFinder/resources/tags/foo +{"timestamp":"2022-09-21T21:35:46.677+00:00","status":404,"error":"Not Found","path":"/ChannelFinder/resources/tags/foo"} +``` + +One can see not very kind log messages, which cannot tell exactly what kind of performance they did, in ChannelFinder log as follows: + +```bash +2022-09-21 14:24:05.289 INFO 96837 --- [nio-8080-exec-1] o.p.channelfinder.TagManager.audit : getting tag: foo +2022-09-21 14:24:05.325 INFO 96837 --- [nio-8080-exec-1] org.phoebus.channelfinder.TagRepository : Tag name foo +2022-09-21 14:34:45.959 INFO 96837 --- [nio-8080-exec-9] o.p.channelfinder.TagManager.audit : getting tag: foo +2022-09-21 14:34:45.980 INFO 96837 --- [nio-8080-exec-9] org.phoebus.channelfinder.TagRepository : Tag name foo +2022-09-21 14:35:10.958 INFO 96837 --- [nio-8080-exec-1] o.p.channelfinder.TagManager.audit : client initialization: 0 +2022-09-21 14:35:10.979 INFO 96837 --- [nio-8080-exec-1] org.phoebus.channelfinder.TagRepository : Tag name foo +2022-09-21 14:35:11.182 INFO 96837 --- [nio-8080-exec-1] org.phoebus.channelfinder.TagRepository : Tag name foo +2022-09-21 14:35:23.583 INFO 96837 --- [nio-8080-exec-2] o.p.channelfinder.TagManager.audit : getting tag: foo +2022-09-21 14:35:23.594 INFO 96837 --- [nio-8080-exec-2] org.phoebus.channelfinder.TagRepository : Tag name foo +2022-09-21 14:35:37.897 INFO 96837 --- [nio-8080-exec-4] org.phoebus.channelfinder.TagRepository : Tag name foo +2022-09-21 14:35:46.649 INFO 96837 --- [nio-8080-exec-6] o.p.channelfinder.TagManager.audit : getting tag: foo +2022-09-21 14:35:46.664 INFO 96837 --- [nio-8080-exec-6] org.phoebus.channelfinder.TagRepository : Tag not found +2022-09-21 14:35:46.668 ERROR 96837 --- [nio-8080-exec-6] org.phoebus.channelfinder.TagManager : The tag with the name foo does not exist +``` + +## `cf_query.bash` + +The script makes our life a bit easy to verify the CF server is running correctly. + +```bash +$ bash scripts/cf_queries.bash get tags foo +>>> Getting ...tags... with ...foo... +{ + "name": "foo", + "owner": "admin", + "channels": [] +} +>---------- + +$ bash scripts/cf_queries.bash delete tags foo +>>> Deleting...tags... with ...foo... +>---------- + +$ bash scripts/cf_queries.bash get tags foo +>>> Getting ...tags... with ...foo... +{ + "timestamp": "2022-09-21T22:28:08.966+00:00", + "status": 404, + "error": "Not Found", + "path": "/ChannelFinder/resources/tags/foo" +} +>---------- + +$ bash scripts/cf_queries.bash put tags foo scripts/tag_foo.json +>>> Putting ...... ...... +{ + "name": "foo", + "owner": "admin", + "channels": [] +} +>------------ + +$ bash scripts/cf_queries.bash get tags foo +>>> Getting ...tags... with ...foo... +{ + "name": "foo", + "owner": "admin", + "channels": [] +} +>---------- +``` + diff --git a/docs/Docker.md b/docs/Docker.md index d58b7d1..ab0b996 100644 --- a/docs/Docker.md +++ b/docs/Docker.md @@ -12,7 +12,7 @@ Note that one will have to log out and to log in to take effect! One should look ## Pull the release image from a registry (hub.docker.com) ```bash -docker pull jeonghanlee/channelfinder:4-v0.1.0 +docker pull alscontrols/channelfinder:v2.0.0 ``` ## Run @@ -20,25 +20,25 @@ docker pull jeonghanlee/channelfinder:4-v0.1.0 ### Run with console outputs ```bash -docker run --network=host --name=channelfinder jeonghanlee/channelfinder:4-v0.1.0 +docker run --network=host --name=channelfinder alscontrols/channelfinder:v2.0.0 ``` ### Run in the detach mode ```bash -docker run --network=host --detach --rm --name=channelfinder jeonghanlee/channelfinder:4-v0.1.0 +docker run --network=host --detach --rm --name=channelfinder alscontrols/channelfinder:v2.0.0 ``` ### Run in order to access the container without the channelfinder service ```bash -docker run -i -t --entrypoint /bin/bash jeonghanlee/channelfinder:4-v0.1.0 +docker run -i -t --entrypoint /bin/bash alscontrols/channelfinder:v2.0.0 ``` ### Run with the local disk mount ```bash -docker run -i -t -v ${HOME}/docker_data:/data --entrypoint /bin/bash jeonghanlee/channelfinder:4-v0.1.0 +docker run -i -t -v ${HOME}/docker_data:/data --entrypoint /bin/bash alscontrols/channelfinder:v2.0.0 ``` , where `${HOME}/docker_data` is the host path, which will be created if it doesn't exist, and `/data` is a volume in the docker container. diff --git a/docs/ESUpgrade.md b/docs/ESUpgrade.md new file mode 100644 index 0000000..e9a383c --- /dev/null +++ b/docs/ESUpgrade.md @@ -0,0 +1,21 @@ +# New Elasticsearch configuration + +The following error tells us we have old elasticsearch configuration exists. + +```bash +[2022-09-27T11:22:59,729][ERROR][o.e.b.Bootstrap ] [als-es-node] Exception +java.lang.IllegalArgumentException: Could not load codec 'Lucene92'. Did you forget to add lucene-backward-codecs.jar? +``` + +* Solution + +We have to remove all existing file in `CF_ES_DATA_PATH`. For Debian, it is `/var/lib/elasticsearc` + +```bash +rm -rf /var/lib/elasticsearch +mkdir -p /var/lib/elasticsearch +chown elasticsearch:elasticsearch /var/lib/elasticsearch +systemctl restart elasticsearch +``` + + diff --git a/examples/configuration.bash b/examples/configuration.bash index 3f9e650..7318ef5 100644 --- a/examples/configuration.bash +++ b/examples/configuration.bash @@ -1,9 +1,15 @@ + cf_host=localhost cf_port=8080 -cf_url=http://${cf_host}:${cf_port}/ChannelFinder/resources +# shellcheck disable=SC2034 +cf_url=http://"${cf_host}:${cf_port}/ChannelFinder/resources" + +# admin, cfuser, channel +# username, and its password should be matched with the running ldif file. +# function print_help { @@ -12,10 +18,8 @@ function print_help printf "\n>>> %s : %s ....\n" "$a" "$b"; } - function get_list_from_a_file { - local empty_string=""; declare -a entry=(); while IFS= read -r line_data; do @@ -25,8 +29,7 @@ function get_list_from_a_file entry[i]="${line_data}" ((++i)) fi - done < $1 + done < "$1" - echo ${entry[@]} + echo "${entry[@]}" } - diff --git a/examples/delete_channels.bash b/examples/delete_channels.bash index 8fcded6..55675c6 100644 --- a/examples/delete_channels.bash +++ b/examples/delete_channels.bash @@ -1,30 +1,35 @@ #!/usr/bin/env bash -declare -gr SC_SCRIPT="$(realpath "$0")" -declare -gr SC_SCRIPTNAME=${0##*/} -declare -gr SC_TOP="${SC_SCRIPT%/*}" +declare -g SC_SCRIPT; +declare -g SC_TOP; -. ${SC_TOP}/configuration.bash +SC_SCRIPT="$(realpath "$0")"; +SC_TOP="${SC_SCRIPT%/*}" -list="$(get_list_from_a_file ${SC_TOP}/CHANNELS)" +# shellcheck disable=SC1090,SC1091 +. "${SC_TOP}/configuration.bash" -URL=${cf_url}/channels + +list=$(get_list_from_a_file "${SC_TOP}"/CHANNELS) + +# shellcheck disable=SC2154 +URL="${cf_url}"/channels # admin, cfuser, channel # username, and its password should be matched with the running ldif file. # cf_userid=channel cf_passwd=1234 -cf_user=${cf_userid}:${cf_passwd} +cf_user="${cf_userid}:${cf_passwd}" -for a_chan in ${list[@]}; do +for a_chan in "${list[@]}"; do print_help "DELETE" "$a_chan" - curl -u $cf_user -X DELETE ${URL}/${a_chan} + curl -u $cf_user -X DELETE "${URL}/${a_chan}" printf "\n" done print_help "GET" "CHANNELS" -curl -X GET ${URL} +curl -X GET "${URL}" printf "\n"; diff --git a/examples/delete_properties.bash b/examples/delete_properties.bash index a5e7e40..ef22f73 100644 --- a/examples/delete_properties.bash +++ b/examples/delete_properties.bash @@ -1,28 +1,34 @@ #!/usr/bin/env bash -declare -gr SC_SCRIPT="$(realpath "$0")" -declare -gr SC_SCRIPTNAME=${0##*/} -declare -gr SC_TOP="${SC_SCRIPT%/*}" -. ${SC_TOP}/configuration.bash +declare -g SC_SCRIPT; +declare -g SC_TOP; -list="$(get_list_from_a_file ${SC_TOP}/PROPERTIES)" +SC_SCRIPT="$(realpath "$0")"; +SC_TOP="${SC_SCRIPT%/*}" -URL=${cf_url}/properties + +# shellcheck disable=SC1090,SC1091 +. "${SC_TOP}"/configuration.bash + +list=$(get_list_from_a_file "${SC_TOP}"/PROPERTIES) + +# shellcheck disable=SC2154 +URL="${cf_url}"/properties # admin, cfuser, property # username, and its password should be matched with the running ldif file. # cf_userid=property cf_passwd=1234 -cf_user=${cf_userid}:${cf_passwd} +cf_user="${cf_userid}:${cf_passwd}" -for a_property in ${list[@]}; do +for a_property in "${list[@]}"; do print_help "DELETE" "$a_property" - curl -u $cf_user -X DELETE ${URL}/${a_property} + curl -u $cf_user -X DELETE "${URL}/${a_property}" printf "\n" done print_help "GET" "Properties" -curl -X GET ${URL} +curl -X GET "${URL}" printf "\n"; diff --git a/examples/delete_tags.bash b/examples/delete_tags.bash index 4afa0a9..d448995 100644 --- a/examples/delete_tags.bash +++ b/examples/delete_tags.bash @@ -1,13 +1,18 @@ #!/usr/bin/env bash -declare -gr SC_SCRIPT="$(realpath "$0")" -declare -gr SC_SCRIPTNAME=${0##*/} -declare -gr SC_TOP="${SC_SCRIPT%/*}" -. ${SC_TOP}/configuration.bash +declare -g SC_SCRIPT; +declare -g SC_TOP; -list="$(get_list_from_a_file ${SC_TOP}/TAGS)" +SC_SCRIPT="$(realpath "$0")"; +SC_TOP="${SC_SCRIPT%/*}" -URL=${cf_url}/tags +# shellcheck disable=SC1090,SC1091 +. "${SC_TOP}"/configuration.bash + +list=$(get_list_from_a_file "${SC_TOP}"/TAGS) + +# shellcheck disable=SC2154 +URL="${cf_url}"/tags # admin, cfuser, property # username, and its password should be matched with the running ldif file. @@ -16,15 +21,15 @@ URL=${cf_url}/tags cf_userid=tag cf_passwd=1234 -cf_user=${cf_userid}:${cf_passwd} +cf_user="${cf_userid}:${cf_passwd}" -for a_tag in ${list[@]}; do +for a_tag in "${list[@]}"; do print_help "DELETE" "$a_tag" - curl -u $cf_user -X DELETE ${URL}/${a_tag} + curl -u $cf_user -X DELETE "${URL}/${a_tag}" printf "\n" done print_help "GET" "tags" -curl -X GET ${URL} +curl -X GET "${URL}" printf "\n"; diff --git a/examples/put_channels.bash b/examples/put_channels.bash index 6f8c7df..7a6d220 100644 --- a/examples/put_channels.bash +++ b/examples/put_channels.bash @@ -1,14 +1,19 @@ #!/usr/bin/env bash -declare -gr SC_SCRIPT="$(realpath "$0")" -declare -gr SC_SCRIPTNAME=${0##*/} -declare -gr SC_TOP="${SC_SCRIPT%/*}" +declare -g SC_SCRIPT; +declare -g SC_TOP; -. ${SC_TOP}/configuration.bash +SC_SCRIPT="$(realpath "$0")"; +SC_TOP="${SC_SCRIPT%/*}" -list="$(get_list_from_a_file ${SC_TOP}/CHANNELS)" -URL=${cf_url}/channels +# shellcheck disable=SC1090,SC1091 +. "${SC_TOP}"/configuration.bash + +list=$(get_list_from_a_file "${SC_TOP}"/CHANNELS) + +# shellcheck disable=SC2154 +URL="${cf_url}"/channels # admin, cfuser, channel @@ -18,7 +23,7 @@ cf_userid=channel cf_passwd=1234 cf_user=${cf_userid}:${cf_passwd} -for a_chan in ${list[@]}; do +for a_chan in "${list[@]}"; do print_help "PUT" "$a_chan" temp_json=$(mktemp) echo " @@ -28,7 +33,7 @@ for a_chan in ${list[@]}; do \"owner\": \"cf-channels\" } ] - " > ${temp_json} + " > "${temp_json}" #echo " # [ # { @@ -54,12 +59,12 @@ for a_chan in ${list[@]}; do # " > ${temp_json} - curl -u $cf_user -H 'Content-Type: application/json' -X PUT ${URL} -d @$temp_json - rm -f $temp_json + curl -u $cf_user -H 'Content-Type: application/json' -X PUT "${URL}" -d @"$temp_json" + rm -f "$temp_json" printf "\n" done print_help "GET" "CHANNELS" -curl -X GET ${URL} +curl -X GET "${URL}" printf "\n"; diff --git a/examples/put_properties.bash b/examples/put_properties.bash index 4be9bed..185a7b6 100644 --- a/examples/put_properties.bash +++ b/examples/put_properties.bash @@ -1,26 +1,32 @@ #!/usr/bin/env bash -declare -gr SC_SCRIPT="$(realpath "$0")" -declare -gr SC_SCRIPTNAME=${0##*/} -declare -gr SC_TOP="${SC_SCRIPT%/*}" -. ${SC_TOP}/configuration.bash +declare -g SC_SCRIPT; +declare -g SC_TOP; -list="$(get_list_from_a_file ${SC_TOP}/PROPERTIES)" +SC_SCRIPT="$(realpath "$0")"; +SC_TOP="${SC_SCRIPT%/*}" + + +# shellcheck disable=SC1090,SC1091 +. "${SC_TOP}"/configuration.bash + +list=$(get_list_from_a_file "${SC_TOP}"/PROPERTIES) #cf_url=https://${cf_host}:8443/ChannelFinder/resources -URL=${cf_url}/properties +# shellcheck disable=SC2154 +URL="${cf_url}"/properties # admin, cfuser, property # username, and its password should be matched with the running ldif file. # -cf_userid=admin +cf_userid="admin" cf_passwd=1234 -cf_user=${cf_userid}:${cf_passwd} +cf_user="${cf_userid}:${cf_passwd}" temp_json=$(mktemp) -for a_property in ${list[@]}; do +for a_property in "${list[@]}"; do print_help "PUT" "$a_property" temp_json=$(mktemp) echo " @@ -30,16 +36,13 @@ for a_property in ${list[@]}; do \"owner\": \"cf-properties\" } ] - " > ${temp_json} + " > "${temp_json}" - curl -u $cf_user -H 'Content-Type: application/json' -X PUT ${URL} -d @$temp_json - rm -f $temp_json + curl -u $cf_user -H 'Content-Type: application/json' -X PUT "${URL}" -d @"$temp_json" + rm -f "$temp_json" printf "\n" done print_help "GET" "Properties" -curl -X GET ${URL} +curl -X GET "${URL}" printf "\n"; - - - diff --git a/examples/put_tags.bash b/examples/put_tags.bash index e469d9c..e1a628e 100644 --- a/examples/put_tags.bash +++ b/examples/put_tags.bash @@ -1,28 +1,36 @@ #!/usr/bin/env bash -declare -gr SC_SCRIPT="$(realpath "$0")" -declare -gr SC_SCRIPTNAME=${0##*/} -declare -gr SC_TOP="${SC_SCRIPT%/*}" -. ${SC_TOP}/configuration.bash -list="$(get_list_from_a_file ${SC_TOP}/TAGS)" +declare -g SC_SCRIPT; +declare -g SC_TOP; + +SC_SCRIPT="$(realpath "$0")"; +SC_TOP="${SC_SCRIPT%/*}" + + +# shellcheck disable=SC1090,SC1091 +. "${SC_TOP}"/configuration.bash + + +list=$(get_list_from_a_file"${SC_TOP}"/TAGS) #cf_url=https://${cf_host}:8443/ChannelFinder/resources -URL=${cf_url}/tags +# shellcheck disable=SC2154 +URL="${cf_url}"/tags # admin, cfuser, tag, operator # username, and its password should be matched with the running ldif file. # -cf_userid=admin +cf_userid="admin" cf_passwd=1234 -cf_user=${cf_userid}:${cf_passwd} +cf_user="${cf_userid}:${cf_passwd}" temp_json=$(mktemp) -a_chan="SR04C:BPM1:SA:X1" +#a_chan="SR04C:BPM1:SA:X1" -for a_tag in ${list[@]}; do +for a_tag in "${list[@]}"; do print_help "PUT" "$a_tag" temp_json=$(mktemp) echo " @@ -32,16 +40,13 @@ for a_tag in ${list[@]}; do \"owner\": \"cf-tags\" } ] - " > ${temp_json} + " > "${temp_json}" - curl -u $cf_user -H 'Content-Type: application/json' -X PUT ${URL} -d @$temp_json - rm -f $temp_json + curl -u "$cf_user" -H 'Content-Type: application/json' -X PUT "${URL}" -d @"$temp_json" + rm -f "$temp_json" printf "\n" done print_help "GET" "tags" -curl -X GET ${URL} +curl -X GET "${URL}" printf "\n"; - - - diff --git a/examples/update_channels.bash b/examples/update_channels.bash index f02870e..5b7a2b5 100644 --- a/examples/update_channels.bash +++ b/examples/update_channels.bash @@ -1,28 +1,29 @@ #!/usr/bin/env bash -declare -gr SC_SCRIPT="$(realpath "$0")" -declare -gr SC_SCRIPTNAME=${0##*/} -declare -gr SC_TOP="${SC_SCRIPT%/*}" +declare -g SC_SCRIPT; +declare -g SC_TOP; -. ${SC_TOP}/configuration.bash +SC_SCRIPT="$(realpath "$0")"; +SC_TOP="${SC_SCRIPT%/*}" -list="$(get_list_from_a_file ${SC_TOP}/CHANNELS)" -URL=${cf_url}/channels +# shellcheck disable=SC1090,SC1091 +. "${SC_TOP}"/configuration.bash -# admin, cfuser, channel -# username, and its password should be matched with the running ldif file. -# -cf_userid=admin +list=$(get_list_from_a_file "${SC_TOP}"/CHANNELS) +# shellcheck disable=SC2154 +URL="${cf_url}"/channels + +cf_userid="admin" cf_passwd=1234 +cf_user="${cf_userid}:${cf_passwd}" -cf_user=${cf_userid}:${cf_passwd} filterA="AR" filterB="SA:X" -for a_chan in ${list[@]}; do +for a_chan in "${list[@]}"; do print_help "PUT" "$a_chan" temp_json=$(mktemp) if [[ $a_chan =~ $filterA ]]; then @@ -61,26 +62,24 @@ for a_chan in ${list[@]}; do ] } ] - " > ${temp_json} + " > "${temp_json}" - curl -u $cf_user -H 'Content-Type: application/json' -X PUT ${URL} -d @$temp_json - rm -f $temp_json + curl -u $cf_user -H 'Content-Type: application/json' -X PUT "${URL}" -d @"$temp_json" + rm -f "$temp_json" printf "\n" done print_help "GET" "SR domain CHANNELS" -curl -X GET ${URL}?domain=SR +curl -X GET "${URL}"?domain=SR printf "\n"; print_help "GET" "AR domain CHANNELS" -curl -X GET ${URL}?domain=AR +curl -X GET "${URL}"?domain=AR printf "\n"; - print_help "GET" "ALL CHANNELS" -curl -X GET ${URL} +curl -X GET "${URL}" printf "\n"; - diff --git a/scripts/cf_es_configuration.bash b/scripts/cf_es_configuration.bash new file mode 100755 index 0000000..d21a729 --- /dev/null +++ b/scripts/cf_es_configuration.bash @@ -0,0 +1,170 @@ +#!/usr/bin/env bash +# +# author : Jeong Han Lee +# email : jeonghan.lee@gmail.com +# date : Wed Sep 21 11:12:34 PDT 2022 +# version : 0.0.1 + + +declare -g SC_SCRIPT; +declare -g SC_TOP; +declare -g ENV_TOP; + +declare -g es_host; +declare -g es_port; + +SC_SCRIPT="$(realpath "$0")"; +SC_TOP="${SC_SCRIPT%/*}" +ENV_TOP="${SC_TOP}/.." + +CF_JSON_PATH="${ENV_TOP}/$(make -C "${ENV_TOP}" -s print-CF_SRC_PATH)" +CF_JSON_PATH+="/src/main/resources" +CF_QUERY_SIZE=$(make -C "${ENV_TOP}" -s print-CF_QUERY_SIZE) + + +es_host=$(make -C "${ENV_TOP}" -s print-ES_HOST) +es_port=$(make -C "${ENV_TOP}" -s print-ES_PORT) + +function curl_put +{ + local index="$1"; shift; + local json="$1"; shift; + + local json_file="${CF_JSON_PATH}/${json}" + + if [ ! -f "$json_file" ]; then + printf "Warning::File not found %s\n" "$json_file"; + printf "Please check %s\n" "$json_file"; + exit; + fi + + printf ">>> Creating ...%s... from ...%s...\n" "$index" "$json"; + if [ -x "$(which jq)" ]; then + # shellcheck disable=SC2154 + curl -s -H 'Content-Type: application/json' -XPUT http://"${es_host}:${es_port}/${index}" -d "@${json_file}" | jq + else + # shellcheck disable=SC2154 + curl -s -H 'Content-Type: application/json' -XPUT http://"${es_host}:${es_port}/${index}" -d "@${json_file}" + fi; + printf ">>>\n" + + printf ">>> Settings max_result_window of index ...%s... : ...%s...\n" "$index" "$CF_QUERY_SIZE"; + if [ -x "$(which jq)" ]; then + # shellcheck disable=SC2154 + curl -s -H 'Content-Type: application/json' -XPUT http://"${es_host}:${es_port}/${index}"/_settings -d '{ "index" : { "max_result_window" : "'"${CF_QUERY_SIZE}"'" }}' | jq + else + # shellcheck disable=SC2154 + curl -s -H 'Content-Type: application/json' -XPUT http://"${es_host}:${es_port}/${index}"/_settings -d '{ "index" : { "max_result_window" : "'"${CF_QUERY_SIZE}"'" }}' + fi; + printf ">------------\n"; + +} + +function curl_get +{ + local index="$1"; shift; + local meta="$1"; shift; + + printf ">>> Checking ...%s... with ...%s...\n" "$index" "$meta"; + if [ -x "$(which jq)" ]; then + # shellcheck disable=SC2154 + curl -s -XGET http://"${es_host}:${es_port}/${index}/_${meta}" | jq + else + # shellcheck disable=SC2154 + curl -s -XGET http://"${es_host}:${es_port}/${index}/_${meta}" + fi; + printf ">----------\n"; +} + +function curl_delete +{ + local index="$1"; shift; + + printf "Deleting %s \n" "$index"; + if [ -x "$(which jq)" ]; then + # shellcheck disable=SC2154 + curl -s -XDELETE http://"${es_host}:${es_port}/${index}" | jq + else + # shellcheck disable=SC2154 + curl -s -XDELETE http://"${es_host}:${es_port}/${index}" + fi; +} + + +function IndexMapping +{ + local index="$1"; shift; + local json="$1"; shift; + curl_put "$index" "$json" + curl_get "$index" "mapping" +} + +function MetaSearching +{ + local index="$1"; shift; + local meta="$1"; shift; + curl_get "$index" "$meta"; +} + +function MetaAllSearching +{ + local meta="$1"; shift; + curl_get "cf_tags" "$meta" + curl_get "cf_properties" "$meta" + curl_get "channelfinder" "$meta" +} + + +function usage +{ + { + echo ""; + echo "Usage : $0 args" + echo ""; + echo " possbile args"; + echo ""; + echo " mapping : CF index mapping"; + echo " mappingClean : Delete all CF index"; + echo " mappingVerify : Verify the existing mappings"; + echo " metaField 'search' : mapping, search, .... "; + echo " h : this screen"; + echo ""; + echo " bash $0 mapping " + echo "" + } 1>&2; + exit 1; +} + + + +case "$1" in + mapping) + IndexMapping "channelfinder" "channel_mapping.json" + IndexMapping "cf_tags" "tag_mapping.json" + IndexMapping "cf_properties" "properties_mapping.json" + ;; + mappingClean) + curl_delete "cf_tags" + curl_delete "cf_properties" + curl_delete "channelfinder" + ;; + mappingVerify) + curl_get "cf_tags" "mapping" + curl_get "cf_properties" "mapping" + curl_get "channelfinder" "mapping" + ;; + metaField) + MetaAllSearching "$2" + ;; + search) + MetaSearching "$2" "$3" + ;; + h);; + help) + usage; + ;; + *) + usage; + ;; +esac + diff --git a/scripts/cf_queries.bash b/scripts/cf_queries.bash new file mode 100755 index 0000000..cd68026 --- /dev/null +++ b/scripts/cf_queries.bash @@ -0,0 +1,112 @@ +#!/usr/bin/env bash +# +# author : Jeong Han Lee +# email : jeonghan.lee@gmail.com +# date : Wed Sep 21 11:12:34 PDT 2022 +# version : 0.0.1 + + +declare -g SC_SCRIPT; +declare -g SC_TOP; +declare -g ENV_TOP; +declare -g CF_URL; +declare -g ADMIN; + +SC_SCRIPT="$(realpath "$0")"; +SC_TOP="${SC_SCRIPT%/*}" +ENV_TOP="${SC_TOP}/.." + +# shellcheck disable=SC2154 +CF_URL="$(make -s -C "${ENV_TOP}" print-CF_FULL_URL)/resources"; + +ADMIN="admin:1234" + +function cf_put +{ + local arg1="$1"; shift; + local arg2="$1"; shift; + local arg3="$1"; shift; + + printf ">>> Putting ...%s... ...%s...\n" "$arg1" "$arg2"; + if [ -x "$(which jq)" ]; then + curl -s -H 'Content-Type: application/json' -u "${ADMIN}" --request PUT "${CF_URL}/${arg1}/${arg2}" -d "@${arg3}" | jq + else + curl -s -H 'Content-Type: application/json' -u "${ADMIN}" --request PUT "${CF_URL}/${arg1}/${arg2}" -d "@${arg3}" + fi; + printf ">------------\n"; + +} + +function cf_get +{ + local arg1="$1"; shift; + local arg2="$1"; shift; + + printf ">>> Getting ...%s... with ...%s...\n" "$arg1" "$arg2"; + if [ -x "$(which jq)" ]; then +# echo "curl -s --request GET "${CF_URL}/${arg1}/${arg2}" | jq" + curl -s --request GET "${CF_URL}/${arg1}/${arg2}" | jq + else + curl -s -XGET "${CF_URL}/${arg1}/${arg2}" + fi; + printf ">----------\n"; +} + +function cf_delete +{ + local arg1="$1"; shift; + local arg2="$1"; shift; + + printf ">>> Deleting...%s... with ...%s...\n" "$arg1" "$arg2"; + if [ -x "$(which jq)" ]; then + curl --basic -u "${ADMIN}" -s --request DELETE "${CF_URL}/${arg1}/${arg2}" | jq + else + curl --basic -u "${ADMIN}" -s --request DELETE "${CF_URL}/${arg1}/${arg2}" + fi; + printf ">----------\n" +} + + +function usage +{ + { + echo ""; + echo "Usage : $0 args" + echo ""; + echo " possbile args"; + echo ""; + echo " get arg1 arg2 : Get "; + echo " delete arg1 arg2 : Delete "; + echo " put arg1 arg2 arg3 : Put, arg3 is a json file"; + echo " h : this screen"; + echo ""; + echo " bash $0 mapping " + echo "" + } 1>&2; + exit 1; +} + +arg0="$1"; shift; +arg1="$1"; shift; +arg2="$1"; shift; +arg3="$1"; shift; + +case "$arg0" in + get) + cf_get "$arg1" "$arg2" + ;; + delete) + cf_delete "$arg1" "$arg2" + ;; + put) + cf_put "$arg1" "$arg2" "$arg3" + ;; + h);; + help) + usage; + ;; + *) + usage; + ;; +esac + diff --git a/scripts/tag_foo.json b/scripts/tag_foo.json new file mode 100644 index 0000000..fd2b249 --- /dev/null +++ b/scripts/tag_foo.json @@ -0,0 +1 @@ +{"name":"foo", "owner":"admin"} diff --git a/site-template/application.properties.in b/site-template/application.properties.in index 01d3dc6..8355aa0 100644 --- a/site-template/application.properties.in +++ b/site-template/application.properties.in @@ -1,24 +1,24 @@ ################## ChannelFinder Server #################### # ChannelFinder https port -server.port=8443 +server.port=@CF_HTTPS_PORT@ # Options support for unsecure http -server.http.enable=true -server.http.port=@CF_PORT@ +server.http.enable=@CF_HTTP_ENABLE@ +server.http.port=@CF_HTTP_PORT@ server.ssl.key-store-type=PKCS12 server.ssl.key-store=classpath:keystore/@SSHKEY@ server.ssl.key-store-password=password server.ssl.key-alias=@SSHKEYALIAS@ -security.require-ssl=true +security.require-ssl=@CF_SSL_REQUIRED@ -logging.level.org.springframework.web=DEBUG +logging.level.org.springframework.web=@CF_LOG_LEVEL@ +spring.http.log-request-details=true -logging.file=/var/log/ChannelFinder.log +#logging.file=@CF_LOG_PATH@/@CF_LOG_FILE@ # Default # logging.file.max-size=10MB -spring.http.log-request-details=true ############## LDAP - External ############## ldap.enabled = false @@ -43,33 +43,47 @@ spring.ldap.embedded.validation.enabled=false ############## Demo Auth ############## -demo_auth.enabled = false +# users, pwds, roles - lists of comma-separated values (same length) +# roles may contain multiple roles for user separated by delimiter +# e.g. +# demo_auth.users = user1,user2 +# demo_auth.pwds = pwd1,pwd2 +# demo_auth.roles = role1,role2 +# demo_auth.roles = role1,role21:role22 +demo_auth.enabled = true +demo_auth.delimiter.roles = : +demo_auth.users = admin,user +demo_auth.pwds = adminPass,userPass +demo_auth.roles = ADMIN,USER + ############## Role --> group Mapping ############## # Customize group names here -admin-groups=cf-admins,sys-admins,ADMIN -channel-groups=cf-channels,USER -property-groups=cf-properties,USER -tag-groups=cf-tags,USER +admin-groups=cf-admins +channel-groups=cf-channels +property-groups=cf-properties +tag-groups=cf-tags ############################## Elastic Network And HTTP ############################### # Elasticsearch host -elasticsearch.network.host: @ES_HOST@ +#elasticsearch.network.host: @ES_HOST@ # Set a custom port for the node to node communication (9300 by default): -elasticsearch.transport.tcp.port: 9300 +# elasticsearch.transport.tcp.port: 9300 # Set a custom port to listen for HTTP traffic: elasticsearch.http.port: @ES_PORT@ # Elasticsearch index names and types used by channelfinder, ensure that any changes here should be replicated in the mapping_definitions.sh elasticsearch.tag.index = cf_tags -elasticsearch.tag.type = cf_tag - elasticsearch.property.index = cf_properties -elasticsearch.property.type = cf_property - elasticsearch.channel.index = channelfinder -elasticsearch.channel.type = cf_channel # maximum query result size -elasticsearch.query.size = @CF_QUERY_SIZE@ +elasticsearch.query.size = @CF_QUERY_SIZE@ + +# Create the Channel Finder indices if they do not exist +elasticsearch.create.indices: @CF_CREATE_INDEX@ + +############################## Service Info ############################### +channelfinder.version = @CF_VERSION@ + diff --git a/site-template/cf.service.in b/site-template/cf.service.in deleted file mode 100644 index 976c928..0000000 --- a/site-template/cf.service.in +++ /dev/null @@ -1,22 +0,0 @@ -[Unit] -Description=ChannelFinder Service -Documentation=https://github.com/ChannelFinder/ChannelFinder-SpringBoot -After=network.target elasticsearch.service -Requires=elasticsearch.service - -[Service] -# User=elasticsearch -# Group=elasticsearch - -ExecStart=_JAVAPATH_/java _JAVAOPTS_ -jar _CFPATH_/_CHANNELFINDER_JAR_NAME_ -SuccessExitStatus=143 - -ExecReload=/usr/bin/kill -SIGINT $MAINPID -KillMode=process - -Restart=on-failure -RestartSec=10s - -[Install] -WantedBy=multi-user.target -Alias=cf.service diff --git a/site-template/clean_mapping_definitions.bash b/site-template/clean_mapping_definitions.bash deleted file mode 100644 index 7c432d0..0000000 --- a/site-template/clean_mapping_definitions.bash +++ /dev/null @@ -1,21 +0,0 @@ -#!/usr/bin/env bash -# -# author : Jeong Han Lee -# email : jeonghan.lee@gmail.com -# date : Thursday, February 20 14:38:29 PST 2020 -# version : 0.0.1 - - -declare -gr SC_SCRIPT="$(realpath "$0")" -declare -gr SC_SCRIPTNAME=${0##*/} -declare -gr SC_TOP="${SC_SCRIPT%/*}" - -. ${SC_TOP}/es_host.cfg - - -curl -XDELETE http://${es_host}:${es_port}/cf_tags -echo "" -curl -XDELETE http://${es_host}:${es_port}/channelfinder -echo "" -curl -XDELETE http://${es_host}:${es_port}/cf_properties -echo "" diff --git a/site-template/elasticsearch.yml.in b/site-template/elasticsearch.yml.in new file mode 100644 index 0000000..8432760 --- /dev/null +++ b/site-template/elasticsearch.yml.in @@ -0,0 +1,117 @@ +# ======================== Elasticsearch Configuration ========================= +# +# NOTE: Elasticsearch comes with reasonable defaults for most settings. +# Before you set out to tweak and tune the configuration, make sure you +# understand what are you trying to accomplish and the consequences. +# +# The primary way of configuring a node is via this file. This template lists +# the most important settings you may want to configure for a production cluster. +# +# Please consult the documentation for further information on configuration options: +# https://www.elastic.co/guide/en/elasticsearch/reference/index.html +# +# ---------------------------------- Cluster ----------------------------------- +# +# Use a descriptive name for your cluster: +# +cluster.name: @CF_ES_CLUSTER_NAME@ +# +# ------------------------------------ Node ------------------------------------ +# +# Use a descriptive name for the node: +# +node.name: "@CF_ES_NODE_NAME@" +# +# ----------------------------------- Paths ------------------------------------ +# +# Path to directory where to store the data (separate multiple locations by comma): +# +path.data: @CF_ES_DATA_PATH@ +# +# Path to log files: +# +path.logs: @CF_ES_LOGS_PATH@ +# +# ----------------------------------- Memory ----------------------------------- +# +# Lock the memory on startup: +# +#bootstrap.memory_lock: true +# +# Make sure that the heap size is set to about half the memory available +# on the system and that the owner of the process is allowed to use this +# limit. +# +# Elasticsearch performs poorly when the system is swapping the memory. +# +# ---------------------------------- Network ----------------------------------- +# +# By default Elasticsearch is only accessible on localhost. Set a different +# address here to expose this node on the network: +# +network.host: @CF_ES_NETWORK_HOST@ +# +# By default Elasticsearch listens for HTTP traffic on the first free port it +# finds starting at 9200. Set a specific HTTP port here: +# +#http.port: 9200 +# +# For more information, consult the network module documentation. +# +# --------------------------------- Discovery ---------------------------------- +# +# Pass an initial list of hosts to perform discovery when this node is started: +# The default list of hosts is ["127.0.0.1", "[::1]"] +# +discovery.type: "@CF_ES_DISCOVERY_TYPE@" +# +# Bootstrap the cluster using an initial set of master-eligible nodes: +# +#cluster.initial_master_nodes: ["node-1", "node-2"] +# +# For more information, consult the discovery and cluster formation module documentation. +# +# --------------------------------- Readiness ---------------------------------- +# +# Enable an unauthenticated TCP readiness endpoint on localhost +# +#readiness.port: 9399 +# +# ---------------------------------- Various ----------------------------------- +# +# Allow wildcard deletion of indices: +# +#action.destructive_requires_name: false + +#----------------------- BEGIN SECURITY AUTO CONFIGURATION ----------------------- +# +# The following settings, TLS certificates, and keys have been automatically +# generated to configure Elasticsearch security features on 02-08-2022 16:30:54 +# +# -------------------------------------------------------------------------------- + +# Enable security features +xpack.security.enabled: @CF_ES_XPACK_SECURITY_ENABLED@ +xpack.security.enrollment.enabled: @CF_ES_XPACK_SECURITY_ENROLL_ENABLED@ +xpack.security.http.ssl.enabled: @CF_ES_XPACK_SECURITY_HTTP_SSL@ +xpack.security.http.ssl.keystore.path: @CF_ES_XPACK_SECURITY_HTTP_SSL_KEYSTORE_PATH@ + +# Enable encryption and mutual authentication between cluster nodes +xpack.security.transport.ssl.enabled: @CF_ES_XPACK_SECURITY_TRANSPORT_SSL@ +xpack.security.transport.ssl.verification_mode: @CF_ES_XPACK_SECURITY_TRANSPORT_SSL_VERIFICATION@ +xpack.security.transport.ssl.keystore.path: @CF_ES_XPACK_SECURITY_TRANSPORT_SSL_KEYSTORE_PATH@ +xpack.security.transport.ssl.truststore.path: @CF_ES_XPACK_SECURITY_TRANSPORT_SSL_TRUSTSTORE_PATH@ + +# Create a new cluster with the current node only +# Additional nodes can still join the cluster later +#cluster.initial_master_nodes: ["fedora"] + +# Allow HTTP API connections from anywhere +# Connections are encrypted and require user authentication +http.host: @CF_ES_HTTP_HOST@ + +# Allow other nodes to join the cluster from anywhere +# Connections are encrypted and mutually authenticated +#transport.host: 0.0.0.0 + +#----------------------- END SECURITY AUTO CONFIGURATION ------------------------- diff --git a/site-template/mapping_definitions.bash b/site-template/mapping_definitions.bash deleted file mode 100755 index 6d19f47..0000000 --- a/site-template/mapping_definitions.bash +++ /dev/null @@ -1,111 +0,0 @@ -#!/usr/bin/env bash - -### -# #%L -# ChannelFinder Directory Service -# %% -# Copyright (C) 2010 - 2016 Helmholtz-Zentrum Berlin für Materialien und Energie GmbH -# %% -# Copyright (C) 2010 - 2012 Brookhaven National Laboratory -# All rights reserved. Use is subject to license terms. -# %% -# Copyright (c) 2020 Jeong Han Lee -# #L% -### -# The mapping definition for the Indexes associated with the channelfinder v4 - -# author : Jeong Han Lee -# email : jeonghan.lee@gmail.com -# date : Thursday, February 20 14:39:35 PST 2020 -# version : 0.0.1 - -declare -gr SC_SCRIPT="$(realpath "$0")" -declare -gr SC_SCRIPTNAME=${0##*/} -declare -gr SC_TOP="${SC_SCRIPT%/*}" - - -. ${SC_TOP}/es_host.cfg - - -#Create the Index -print_help "index/mapping" "cf_tags" - -curl -H 'Content-Type: application/json' -XPUT http://${es_host}:${es_port}/cf_tags -d' -{ -"mappings":{ - "cf_tag" : { - "properties" : { - "name" : { - "type" : "keyword" - }, - "owner" : { - "type" : "keyword" - } - } - } - } -}' - -print_help "index/mapping" "cf_properties" -curl -H 'Content-Type: application/json' -XPUT http://${es_host}:${es_port}/cf_properties -d' -{ -"mappings":{ - "cf_property" : { - "properties" : { - "name" : { - "type" : "keyword" - }, - "owner" : { - "type" : "keyword" - } - } - } - } -}' - -print_help "index/mapping" "channelfinder" -curl -H 'Content-Type: application/json' -XPUT http://${es_host}:${es_port}/channelfinder -d' -{ -"mappings":{ - "cf_channel" : { - "properties" : { - "name" : { - "type" : "keyword" - }, - "owner" : { - "type" : "keyword" - }, - "script" : { - "type" : "keyword" - }, - "properties" : { - "type" : "nested", - "properties" : { - "name" : { - "type" : "keyword" - }, - "owner" : { - "type" : "keyword" - }, - "value" : { - "type" : "keyword" - } - } - }, - "tags" : { - "type" : "nested", - "properties" : { - "name" : { - "type" : "keyword" - }, - "owner" : { - "type" : "keyword" - } - } - } - } - } - } -}' - -printf "\n"; diff --git a/site-template/mapping_definitions_es7x.bash b/site-template/mapping_definitions_es7x.bash deleted file mode 100644 index 8ff51a6..0000000 --- a/site-template/mapping_definitions_es7x.bash +++ /dev/null @@ -1,89 +0,0 @@ -#!/usr/bin/env bash - -### -# #%L -# ChannelFinder Directory Service -# %% -# Copyright (C) 2010 - 2016 Helmholtz-Zentrum Berlin für Materialien und Energie GmbH -# %% -# Copyright (C) 2010 - 2012 Brookhaven National Laboratory -# All rights reserved. Use is subject to license terms. -# %% -# Copyright (c) 2020 Jeong Han Lee -# #L% -### -# The mapping definition for the Indexes associated with the channelfinder v4 - -# author : Jeong Han Lee -# email : jeonghan.lee@gmail.com -# date : Wednesday, April 22 23:00:18 PDT 2020 -# version : 0.0.2 - -declare -gr SC_SCRIPT="$(realpath "$0")" -declare -gr SC_SCRIPTNAME=${0##*/} -declare -gr SC_TOP="${SC_SCRIPT%/*}" - - -. ${SC_TOP}/es_host.cfg - - -#Create the Index : three -print_help "index" "cf_tags" -curl -XPUT http://${es_host}:${es_port}/cf_tags -print_help "index" "cf_properties" -curl -XPUT http://${es_host}:${es_port}/cf_properties -print_help "index" "channelfinder" -curl -XPUT http://${es_host}:${es_port}/channelfinder - -print_help "mapping" "cf_tags" -curl -H 'Content-Type: application/json' -XPUT http://${es_host}:${es_port}/cf_tags/_mapping/cf_tag?include_type_name=true -d' -{ - "cf_tag" : { - "properties" : { - "name" : { "type" : "keyword" }, - "owner" : { "type" : "keyword" } - } - } -}' - -print_help "mapping" "cf_properties" -curl -H 'Content-Type: application/json' -XPUT http://${es_host}:${es_port}/cf_properties/_mapping/cf_property?include_type_name=true -d' -{ - "cf_property" : { - "properties" : { - "name" : { "type" : "keyword" }, - "owner" : { "type" : "keyword" } - } - } -}' - -print_help "mapping" "channelfinder" -curl -H 'Content-Type: application/json' -XPUT http://${es_host}:${es_port}/channelfinder/_mapping/cf_channel?include_type_name=true -d' -{ - "cf_channel" : { - "properties" : { - "name" : { "type" : "keyword" }, - "owner" : { "type" : "keyword" }, - "script" : { "type" : "keyword" }, - "cf_properties" : { - "type" : "nested", - "include_in_parent" : true, - "properties" : { - "name" : { "type" : "keyword" }, - "owner" : { "type" : "keyword" }, - "value" : { "type" : "keyword" } - } - }, - "cf_tags" : { - "type" : "nested", - "include_in_parent" : true, - "properties" : { - "name" : { "type" : "keyword" }, - "owner" : { "type" : "keyword" } - } - } - } - } -}' - -printf "\n"; diff --git a/site-template/systemd/channelfinder.service.in b/site-template/systemd/channelfinder.service.in new file mode 100644 index 0000000..c98317d --- /dev/null +++ b/site-template/systemd/channelfinder.service.in @@ -0,0 +1,22 @@ +[Unit] +Description=ChannelFinder Service +Documentation=@DOCURL@ +After=syslog.target network.target @SYSTEMD_SERVICES@ +Requires=@SYSTEMD_SERVICES@ + +[Service] +User=@CF_USERID@ +Group=@CF_GROUPID@ + +ExecStart=@CF_JAVA_PATH@/java @CF_JAVA_OPTS@ -jar @CF_INSTALL_LOCATION@/@CF_JAR_NAME@ +SuccessExitStatus=143 + +ExecReload=@CF_KILL_PATH@/kill -SIGINT $MAINPID +KillMode=process + +Restart=on-failure +RestartSec=10s + +[Install] +WantedBy=multi-user.target +Alias=cf.service