diff --git a/CHANGELOG-1.0.md b/CHANGELOG-1.0.md index 0c6a72071e..c1222e6144 100644 --- a/CHANGELOG-1.0.md +++ b/CHANGELOG-1.0.md @@ -5,6 +5,7 @@ ### Fixed - [#3189](https://github.com/epiphany-platform/epiphany/issues/3189) - Fix configuration/feature-mapping enabling +- [#3243](https://github.com/epiphany-platform/epiphany/issues/3243) - Repositories for OpenDistro are shutdown ## [1.0.5] 2022-06-24 diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/centos-7/download-requirements.sh b/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/centos-7/download-requirements.sh index bedfec63ac..0202bcc662 100644 --- a/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/centos-7/download-requirements.sh +++ b/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/centos-7/download-requirements.sh @@ -544,10 +544,11 @@ echo $$ > $PID_FILE_PATH || exit_with_error "Command failed: echo $$ > $PID_FILE # --- Parse requirements file --- -# Requirements are grouped using sections: [packages-repo-prereqs], [packages], [files], [images] +# Requirements are grouped using sections: [packages-repo-prereqs], [packages], [packagesfromurl], [files], [images] get_requirements_from_group 'REPO_PREREQ_PACKAGES' 'packages-repo-prereqs' "$REQUIREMENTS_FILE_PATH" get_requirements_from_group 'CRANE' 'crane' "$REQUIREMENTS_FILE_PATH" get_requirements_from_group 'PACKAGES' 'packages' "$REQUIREMENTS_FILE_PATH" +get_requirements_from_group 'PACKAGESFROMURL' 'packagesfromurl' "$REQUIREMENTS_FILE_PATH" get_requirements_from_group 'FILES' 'files' "$REQUIREMENTS_FILE_PATH" get_requirements_from_group 'IMAGES' 'images' "$REQUIREMENTS_FILE_PATH" @@ -681,19 +682,6 @@ gpgkey=https://packages.cloud.google.com/yum/doc/yum-key.gpg https://packages.cl EOF ) -OPENDISTRO_REPO_CONF=$(cat <<'EOF' -[opendistroforelasticsearch-artifacts-repo] -name=Release RPM artifacts of OpenDistroForElasticsearch -baseurl=https://d3g5vo6xdbdb9a.cloudfront.net/yum/noarch/ -enabled=1 -gpgkey=https://d3g5vo6xdbdb9a.cloudfront.net/GPG-KEY-opendistroforelasticsearch -gpgcheck=1 -repo_gpgcheck=1 -autorefresh=1 -type=rpm-md -EOF -) - POSTGRESQL_REPO_CONF=$(cat <<'EOF' [pgdg10] name=PostgreSQL 10 for RHEL/CentOS $releasever - $basearch @@ -729,7 +717,6 @@ add_repo_as_file 'elasticsearch-7' "$ELASTICSEARCH_7_REPO_CONF" add_repo_as_file 'elasticsearch-curator-5' "$ELASTICSEARCH_CURATOR_REPO_CONF" add_repo_as_file 'grafana' "$GRAFANA_REPO_CONF" add_repo_as_file 'kubernetes' "$KUBERNETES_REPO_CONF" -add_repo_as_file 'opendistroforelasticsearch' "$OPENDISTRO_REPO_CONF" add_repo_as_file 'postgresql-10' "$POSTGRESQL_REPO_CONF" add_repo_as_file 'bintray-rabbitmq-rpm' "$RABBITMQ_SERVER_REPO_CONF" add_repo_from_script 'https://dl.2ndquadrant.com/default/release/get/10/rpm' @@ -817,6 +804,16 @@ else exit_with_error "Extracting tar failed: $YUM_CONFIG_BACKUP_FILE_PATH" fi +# === Packages from URL === + +check_connection wget $PACKAGESFROMURL + +create_directory "$PACKAGES_DIR" + +for file in $PACKAGESFROMURL; do + download_file "$file" "$PACKAGES_DIR" +done + # === Files === check_connection wget $FILES diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/centos-7/requirements.txt b/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/centos-7/requirements.txt index 585f86b74f..45316aaba9 100644 --- a/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/centos-7/requirements.txt +++ b/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/centos-7/requirements.txt @@ -63,14 +63,6 @@ logstash-oss-7.8.1 net-tools nfs-utils nmap-ncat -# Open Distro for Elasticsearch plugins are installed individually to not download them twice in different versions (as dependencies of opendistroforelasticsearch package) -opendistro-alerting-1.13.1.* -opendistro-index-management-1.13.1.* -opendistro-job-scheduler-1.13.0.* -opendistro-performance-analyzer-1.13.0.* -opendistro-security-1.13.1.* -opendistro-sql-1.13.0.* -opendistroforelasticsearch-kibana-1.13.1 # kibana has shorter version openssl perl # for vim perl-Getopt-Long # for vim @@ -151,6 +143,15 @@ kubernetes-cni-0.7.5-0 # kubernetes-cni-0.8.6-0 since K8s v1.18.6 kubernetes-cni-0.8.6-0 +[packagesfromurl] +https://github.com/epiphany-platform/alternative-repo/raw/main/1.0.x/redhat/packages/opendistro-alerting-1.13.1.0.rpm +https://github.com/epiphany-platform/alternative-repo/raw/main/1.0.x/redhat/packages/opendistro-index-management-1.13.1.0.rpm +https://github.com/epiphany-platform/alternative-repo/raw/main/1.0.x/redhat/packages/opendistro-job-scheduler-1.13.0.0.rpm +https://github.com/epiphany-platform/alternative-repo/raw/main/1.0.x/redhat/packages/opendistro-performance-analyzer-1.13.0.0.rpm +https://github.com/epiphany-platform/alternative-repo/raw/main/1.0.x/redhat/packages/opendistro-security-1.13.1.0.rpm +https://github.com/epiphany-platform/alternative-repo/raw/main/1.0.x/redhat/packages/opendistro-sql-1.13.0.0.rpm +https://github.com/epiphany-platform/alternative-repo/raw/main/1.0.x/redhat/packages/opendistroforelasticsearch-kibana-1.13.1-linux-x64.rpm + [files] # Github repository for erlang rpm is used since packagecloud repository is limited to a certain number of versions and erlang package from erlang-solutions repository is much more complex and bigger https://github.com/rabbitmq/erlang-rpm/releases/download/v23.1.5/erlang-23.1.5-1.el7.x86_64.rpm diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/redhat-7/download-requirements.sh b/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/redhat-7/download-requirements.sh index bd4a693a2d..1b33e4103b 100644 --- a/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/redhat-7/download-requirements.sh +++ b/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/redhat-7/download-requirements.sh @@ -561,10 +561,11 @@ echo $$ > $PID_FILE_PATH || exit_with_error "Command failed: echo $$ > $PID_FILE # --- Parse requirements file --- -# Requirements are grouped using sections: [packages-repo-prereqs], [packages], [files], [images] +# Requirements are grouped using sections: [packages-repo-prereqs], [packages], [packagesfromurl], [files], [images] get_requirements_from_group 'REPO_PREREQ_PACKAGES' 'packages-repo-prereqs' "$REQUIREMENTS_FILE_PATH" get_requirements_from_group 'CRANE' 'crane' "$REQUIREMENTS_FILE_PATH" get_requirements_from_group 'PACKAGES' 'packages' "$REQUIREMENTS_FILE_PATH" +get_requirements_from_group 'PACKAGESFROMURL' 'packagesfromurl' "$REQUIREMENTS_FILE_PATH" get_requirements_from_group 'FILES' 'files' "$REQUIREMENTS_FILE_PATH" get_requirements_from_group 'IMAGES' 'images' "$REQUIREMENTS_FILE_PATH" @@ -716,19 +717,6 @@ gpgkey=https://packages.cloud.google.com/yum/doc/yum-key.gpg https://packages.cl EOF ) -OPENDISTRO_REPO_CONF=$(cat <<'EOF' -[opendistroforelasticsearch-artifacts-repo] -name=Release RPM artifacts of OpenDistroForElasticsearch -baseurl=https://d3g5vo6xdbdb9a.cloudfront.net/yum/noarch/ -enabled=1 -gpgkey=https://d3g5vo6xdbdb9a.cloudfront.net/GPG-KEY-opendistroforelasticsearch -gpgcheck=1 -repo_gpgcheck=1 -autorefresh=1 -type=rpm-md -EOF -) - POSTGRESQL_REPO_CONF=$(cat <<'EOF' [pgdg10] name=PostgreSQL 10 for RHEL/CentOS $releasever - $basearch @@ -764,7 +752,6 @@ add_repo_as_file 'elasticsearch-7' "$ELASTICSEARCH_7_REPO_CONF" add_repo_as_file 'elasticsearch-curator-5' "$ELASTICSEARCH_CURATOR_REPO_CONF" add_repo_as_file 'grafana' "$GRAFANA_REPO_CONF" add_repo_as_file 'kubernetes' "$KUBERNETES_REPO_CONF" -add_repo_as_file 'opendistroforelasticsearch' "$OPENDISTRO_REPO_CONF" add_repo_as_file 'postgresql-10' "$POSTGRESQL_REPO_CONF" add_repo_as_file 'bintray-rabbitmq-rpm' "$RABBITMQ_SERVER_REPO_CONF" add_repo_from_script 'https://dl.2ndquadrant.com/default/release/get/10/rpm' @@ -845,6 +832,34 @@ else exit_with_error "Extracting tar failed: $YUM_CONFIG_BACKUP_FILE_PATH" fi +# === Packages from URL === + +create_directory "$PACKAGES_DIR" + +check_connection wget $PACKAGESFROMURL + +if [[ -z "$PACKAGESFROMURL" ]]; then + echol "No packages from URL to download" +else + + # list of all files that will be downloaded + echol "Package from URL to be downloaded:" + cat -n <<< "${PACKAGESFROMURL}" + + printf "\n" + + while IFS=' ' read -r url new_filename; do + # download files, check if new filename is provided + if [[ -z $new_filename ]]; then + download_file "$url" "$PACKAGES_DIR" + elif [[ $new_filename = *" "* ]]; then + exit_with_error "wrong new filename for file: $url" + else + download_file "$url" "$PACKAGES_DIR" "$new_filename" + fi + done <<< "$PACKAGESFROMURL" +fi + # === Files === create_directory "$FILES_DIR" diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/redhat-7/requirements.txt b/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/redhat-7/requirements.txt index 33dbf844d4..78eaf55bfe 100644 --- a/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/redhat-7/requirements.txt +++ b/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/redhat-7/requirements.txt @@ -61,14 +61,6 @@ logstash-oss-7.8.1 net-tools nfs-utils nmap-ncat -# Open Distro for Elasticsearch plugins are installed individually to not download them twice in different versions (as dependencies of opendistroforelasticsearch package) -opendistro-alerting-1.13.1.* -opendistro-index-management-1.13.1.* -opendistro-job-scheduler-1.13.0.* -opendistro-performance-analyzer-1.13.0.* -opendistro-security-1.13.1.* -opendistro-sql-1.13.0.* -opendistroforelasticsearch-kibana-1.13.1 # kibana has shorter version openssl perl # for vim perl-Getopt-Long # for vim @@ -147,6 +139,15 @@ kubernetes-cni-0.7.5-0 # kubernetes-cni-0.8.6-0 since K8s v1.18.6 kubernetes-cni-0.8.6-0 +[packagesfromurl] +https://github.com/epiphany-platform/alternative-repo/raw/main/1.0.x/redhat/packages/opendistro-alerting-1.13.1.0.rpm +https://github.com/epiphany-platform/alternative-repo/raw/main/1.0.x/redhat/packages/opendistro-index-management-1.13.1.0.rpm +https://github.com/epiphany-platform/alternative-repo/raw/main/1.0.x/redhat/packages/opendistro-job-scheduler-1.13.0.0.rpm +https://github.com/epiphany-platform/alternative-repo/raw/main/1.0.x/redhat/packages/opendistro-performance-analyzer-1.13.0.0.rpm +https://github.com/epiphany-platform/alternative-repo/raw/main/1.0.x/redhat/packages/opendistro-security-1.13.1.0.rpm +https://github.com/epiphany-platform/alternative-repo/raw/main/1.0.x/redhat/packages/opendistro-sql-1.13.0.0.rpm +https://github.com/epiphany-platform/alternative-repo/raw/main/1.0.x/redhat/packages/opendistroforelasticsearch-kibana-1.13.1-linux-x64.rpm + [files] # Github repository for erlang rpm is used since packagecloud repository is limited to a certain number of versions and erlang package from erlang-solutions repository is much more complex and bigger https://github.com/rabbitmq/erlang-rpm/releases/download/v23.1.5/erlang-23.1.5-1.el7.x86_64.rpm diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/ubuntu-18.04/add-repositories.sh b/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/ubuntu-18.04/add-repositories.sh index 300732be1a..3dcfbee401 100644 --- a/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/ubuntu-18.04/add-repositories.sh +++ b/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/ubuntu-18.04/add-repositories.sh @@ -21,8 +21,5 @@ echo "deb [arch=amd64] https://download.docker.com/linux/ubuntu bionic stable" | wget -qO - https://artifacts.elastic.co/GPG-KEY-elasticsearch | apt-key add - echo "deb https://artifacts.elastic.co/packages/oss-7.x/apt stable main" | tee /etc/apt/sources.list.d/elastic-7.x.list -wget -qO - https://d3g5vo6xdbdb9a.cloudfront.net/GPG-KEY-opendistroforelasticsearch | sudo apt-key add - -echo "deb https://d3g5vo6xdbdb9a.cloudfront.net/apt stable main" | tee -a /etc/apt/sources.list.d/opendistroforelasticsearch.list - wget -qO - https://dl.2ndquadrant.com/gpg-key.asc | apt-key add - echo "deb https://dl.2ndquadrant.com/default/release/apt bionic-2ndquadrant main" | tee -a /etc/apt/sources.list.d/2ndquadrant-dl-default-release.list diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/ubuntu-18.04/download-requirements.sh b/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/ubuntu-18.04/download-requirements.sh index 01d26073a6..7004c2460a 100644 --- a/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/ubuntu-18.04/download-requirements.sh +++ b/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/ubuntu-18.04/download-requirements.sh @@ -84,9 +84,10 @@ shopt -u nullglob check_connection apt $(ls /etc/apt/sources.list.d) apt update -# parse the input file, separete by tags: [crane], [packages], [files], [images] +# parse the input file, separete by tags: [crane], [packages], [packagesfromurl], [files], [images] crane=$(awk '/^$/ || /^#/ {next}; /\[crane\]/ {f=1; next}; /^\[/ {f=0}; f {print $0}' "${input_file}") packages=$(awk '/^$/ || /^#/ {next}; /\[packages\]/ {f=1; next}; /^\[/ {f=0}; f {print $0}' "${input_file}") +packagesfromurl=$(awk '/^$/ || /^#/ {next}; /\[packagesfromurl\]/ {f=1; next}; /^\[/ {f=0}; f {print $0}' "${input_file}") files=$(awk '/^$/ || /^#/ {next}; /\[files\]/ {f=1; next}; /^\[/ {f=0}; f {print $0}' "${input_file}") images=$(awk '/^$/ || /^#/ {next}; /\[images\]/ {f=1; next}; /^\[/ {f=0}; f {print $0}' "${input_file}") @@ -159,10 +160,36 @@ fi printf "\n" -check_connection wget $(for file in $files; do echo "$file"; done) +# PACKAGES AS URL +# process files + +check_connection wget $(for file in $packagesfromurl; do echo "$file"; done) + + +if [[ -z "${packagesfromurl}" ]]; then + echol "No packages from URL to download" +else + # be verbose, show what will be downloaded + # TODO: this is the list of all files shows on every run, not only the files that will be downloaded this run + echol "Packages from URL to be downloaded:" + cat -n <<< "${packagesfromurl}" + + printf "\n" + # download files using wget + while IFS= read -r file; do + # download files, skip if exists + #wget --no-verbose --continue --directory-prefix="${dst_dir_files}" "${file}" + #wget --continue --show-progress --directory-prefix="${dst_dir_files}" "${file}" + download_file "${file}" "${dst_dir_packages}" + done <<< "${packagesfromurl}" +fi # FILES # process files + +check_connection wget $(for file in $files; do echo "$file"; done) + + if [[ -z "${files}" ]]; then echol "No files to download" else diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/ubuntu-18.04/requirements.txt b/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/ubuntu-18.04/requirements.txt index 1e41ea4cf6..85ee922de2 100644 --- a/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/ubuntu-18.04/requirements.txt +++ b/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/ubuntu-18.04/requirements.txt @@ -55,13 +55,6 @@ logstash-oss 1:7.8.1 netcat net-tools nfs-common -opendistro-alerting 1.13.1 -opendistro-index-management 1.13.1 -opendistro-job-scheduler 1.13.0 -opendistro-performance-analyzer 1.13.0 -opendistro-security 1.13.1 -opendistro-sql 1.13.0 -opendistroforelasticsearch-kibana 1.13.1 openjdk-8-jre-headless openssl postgresql-10 @@ -203,6 +196,15 @@ kubernetes-cni 0.7.5-00 # kubernetes-cni-0.8.6-0 since K8s v1.18.6 kubernetes-cni 0.8.6-00 +[packagesfromurl] +https://github.com/epiphany-platform/alternative-repo/raw/main/1.0.x/ubuntu/packages/opendistro-alerting_1.13.1.0-1_all.deb +https://github.com/epiphany-platform/alternative-repo/raw/main/1.0.x/ubuntu/packages/opendistro-index-management_1.13.1.0-1_all.deb +https://github.com/epiphany-platform/alternative-repo/raw/main/1.0.x/ubuntu/packages/opendistro-job-scheduler_1.13.0.0-1_all.deb +https://github.com/epiphany-platform/alternative-repo/raw/main/1.0.x/ubuntu/packages/opendistro-performance-analyzer_1.13.0.0-1_all.deb +https://github.com/epiphany-platform/alternative-repo/raw/main/1.0.x/ubuntu/packages/opendistro-security_1.13.1.0-1_all.deb +https://github.com/epiphany-platform/alternative-repo/raw/main/1.0.x/ubuntu/packages/opendistro-sql_1.13.0.0-1_all.deb +https://github.com/epiphany-platform/alternative-repo/raw/main/1.0.x/ubuntu/packages/opendistroforelasticsearch-kibana_1.13.1_amd64.deb + [files] https://repo1.maven.org/maven2/io/prometheus/jmx/jmx_prometheus_javaagent/0.14.0/jmx_prometheus_javaagent-0.14.0.jar https://archive.apache.org/dist/kafka/2.6.0/kafka_2.12-2.6.0.tgz