Fix for #3243 (#3244)

* Added filesaspackages concept * Rename filesaspackages to packagesfromurl * Added support for Ubuntu packages. * Added changelog * Update core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/ubuntu-18.04/download-requirements.sh Co-authored-by: Tomasz Baran <[email protected]> * Update core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/ubuntu-18.04/download-requirements.sh Co-authored-by: Tomasz Baran <[email protected]> * Update core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/ubuntu-18.04/download-requirements.sh Co-authored-by: Tomasz Baran <[email protected]> Co-authored-by: Tomasz Baran <[email protected]>
hitachienergy · Aug 4, 2022 · 3765fa6 · 3765fa6
1 parent 84f4fc6
commit 3765fa6
Show file tree

Hide file tree

Showing 8 changed files with 99 additions and 58 deletions.
diff --git a/CHANGELOG-1.0.md b/CHANGELOG-1.0.md
@@ -5,6 +5,7 @@
 ### Fixed
 
 - [#3189](https://github.com/epiphany-platform/epiphany/issues/3189) - Fix configuration/feature-mapping enabling
+- [#3243](https://github.com/epiphany-platform/epiphany/issues/3243) - Repositories for OpenDistro are shutdown
 
 ## [1.0.5] 2022-06-24
 

diff --git a/.../playbooks/roles/repository/files/download-requirements/centos-7/download-requirements.sh b/.../playbooks/roles/repository/files/download-requirements/centos-7/download-requirements.sh
@@ -544,10 +544,11 @@ echo $$ > $PID_FILE_PATH || exit_with_error "Command failed: echo $$ > $PID_FILE
 
 # --- Parse requirements file ---
 
-# Requirements are grouped using sections: [packages-repo-prereqs], [packages], [files], [images]
+# Requirements are grouped using sections: [packages-repo-prereqs], [packages], [packagesfromurl], [files], [images]
 get_requirements_from_group 'REPO_PREREQ_PACKAGES' 'packages-repo-prereqs' "$REQUIREMENTS_FILE_PATH"
 get_requirements_from_group 'CRANE'                'crane'                 "$REQUIREMENTS_FILE_PATH"
 get_requirements_from_group 'PACKAGES'             'packages'              "$REQUIREMENTS_FILE_PATH"
+get_requirements_from_group 'PACKAGESFROMURL'      'packagesfromurl'       "$REQUIREMENTS_FILE_PATH"
 get_requirements_from_group 'FILES'                'files'                 "$REQUIREMENTS_FILE_PATH"
 get_requirements_from_group 'IMAGES'               'images'                "$REQUIREMENTS_FILE_PATH"
 
@@ -681,19 +682,6 @@ gpgkey=https://packages.cloud.google.com/yum/doc/yum-key.gpg https://packages.cl
 EOF
 )
 
-OPENDISTRO_REPO_CONF=$(cat <<'EOF'
-[opendistroforelasticsearch-artifacts-repo]
-name=Release RPM artifacts of OpenDistroForElasticsearch
-baseurl=https://d3g5vo6xdbdb9a.cloudfront.net/yum/noarch/
-enabled=1
-gpgkey=https://d3g5vo6xdbdb9a.cloudfront.net/GPG-KEY-opendistroforelasticsearch
-gpgcheck=1
-repo_gpgcheck=1
-autorefresh=1
-type=rpm-md
-EOF
-)
-
 POSTGRESQL_REPO_CONF=$(cat <<'EOF'
 [pgdg10]
 name=PostgreSQL 10 for RHEL/CentOS $releasever - $basearch
@@ -729,7 +717,6 @@ add_repo_as_file 'elasticsearch-7' "$ELASTICSEARCH_7_REPO_CONF"
 add_repo_as_file 'elasticsearch-curator-5' "$ELASTICSEARCH_CURATOR_REPO_CONF"
 add_repo_as_file 'grafana' "$GRAFANA_REPO_CONF"
 add_repo_as_file 'kubernetes' "$KUBERNETES_REPO_CONF"
-add_repo_as_file 'opendistroforelasticsearch' "$OPENDISTRO_REPO_CONF"
 add_repo_as_file 'postgresql-10' "$POSTGRESQL_REPO_CONF"
 add_repo_as_file 'bintray-rabbitmq-rpm' "$RABBITMQ_SERVER_REPO_CONF"
 add_repo_from_script 'https://dl.2ndquadrant.com/default/release/get/10/rpm'
@@ -817,6 +804,16 @@ else
 	exit_with_error "Extracting tar failed: $YUM_CONFIG_BACKUP_FILE_PATH"
 fi
 
+# === Packages from URL ===
+
+check_connection wget $PACKAGESFROMURL
+
+create_directory "$PACKAGES_DIR"
+
+for file in $PACKAGESFROMURL; do
+	download_file "$file" "$PACKAGES_DIR"
+done
+
 # === Files ===
 
 check_connection wget $FILES

diff --git a/.../ansible/playbooks/roles/repository/files/download-requirements/centos-7/requirements.txt b/.../ansible/playbooks/roles/repository/files/download-requirements/centos-7/requirements.txt
@@ -63,14 +63,6 @@ logstash-oss-7.8.1
 net-tools
 nfs-utils
 nmap-ncat
-# Open Distro for Elasticsearch plugins are installed individually to not download them twice in different versions (as dependencies of opendistroforelasticsearch package)
-opendistro-alerting-1.13.1.*
-opendistro-index-management-1.13.1.*
-opendistro-job-scheduler-1.13.0.*
-opendistro-performance-analyzer-1.13.0.*
-opendistro-security-1.13.1.*
-opendistro-sql-1.13.0.*
-opendistroforelasticsearch-kibana-1.13.1 # kibana has shorter version
 openssl
 perl # for vim
 perl-Getopt-Long # for vim
@@ -151,6 +143,15 @@ kubernetes-cni-0.7.5-0
 # kubernetes-cni-0.8.6-0 since K8s v1.18.6
 kubernetes-cni-0.8.6-0
 
+[packagesfromurl]
+https://github.com/epiphany-platform/alternative-repo/raw/main/1.0.x/redhat/packages/opendistro-alerting-1.13.1.0.rpm
+https://github.com/epiphany-platform/alternative-repo/raw/main/1.0.x/redhat/packages/opendistro-index-management-1.13.1.0.rpm
+https://github.com/epiphany-platform/alternative-repo/raw/main/1.0.x/redhat/packages/opendistro-job-scheduler-1.13.0.0.rpm
+https://github.com/epiphany-platform/alternative-repo/raw/main/1.0.x/redhat/packages/opendistro-performance-analyzer-1.13.0.0.rpm
+https://github.com/epiphany-platform/alternative-repo/raw/main/1.0.x/redhat/packages/opendistro-security-1.13.1.0.rpm
+https://github.com/epiphany-platform/alternative-repo/raw/main/1.0.x/redhat/packages/opendistro-sql-1.13.0.0.rpm
+https://github.com/epiphany-platform/alternative-repo/raw/main/1.0.x/redhat/packages/opendistroforelasticsearch-kibana-1.13.1-linux-x64.rpm
+
 [files]
 # Github repository for erlang rpm is used since packagecloud repository is limited to a certain number of versions and erlang package from erlang-solutions repository is much more complex and bigger
 https://github.com/rabbitmq/erlang-rpm/releases/download/v23.1.5/erlang-23.1.5-1.el7.x86_64.rpm

diff --git a/.../playbooks/roles/repository/files/download-requirements/redhat-7/download-requirements.sh b/.../playbooks/roles/repository/files/download-requirements/redhat-7/download-requirements.sh
@@ -561,10 +561,11 @@ echo $$ > $PID_FILE_PATH || exit_with_error "Command failed: echo $$ > $PID_FILE
 
 # --- Parse requirements file ---
 
-# Requirements are grouped using sections: [packages-repo-prereqs], [packages], [files], [images]
+# Requirements are grouped using sections: [packages-repo-prereqs], [packages], [packagesfromurl], [files], [images]
 get_requirements_from_group 'REPO_PREREQ_PACKAGES' 'packages-repo-prereqs' "$REQUIREMENTS_FILE_PATH"
 get_requirements_from_group 'CRANE'                'crane'                 "$REQUIREMENTS_FILE_PATH"
 get_requirements_from_group 'PACKAGES'             'packages'              "$REQUIREMENTS_FILE_PATH"
+get_requirements_from_group 'PACKAGESFROMURL'      'packagesfromurl'       "$REQUIREMENTS_FILE_PATH"
 get_requirements_from_group 'FILES'                'files'                 "$REQUIREMENTS_FILE_PATH"
 get_requirements_from_group 'IMAGES'               'images'                "$REQUIREMENTS_FILE_PATH"
 
@@ -716,19 +717,6 @@ gpgkey=https://packages.cloud.google.com/yum/doc/yum-key.gpg https://packages.cl
 EOF
 )
 
-OPENDISTRO_REPO_CONF=$(cat <<'EOF'
-[opendistroforelasticsearch-artifacts-repo]
-name=Release RPM artifacts of OpenDistroForElasticsearch
-baseurl=https://d3g5vo6xdbdb9a.cloudfront.net/yum/noarch/
-enabled=1
-gpgkey=https://d3g5vo6xdbdb9a.cloudfront.net/GPG-KEY-opendistroforelasticsearch
-gpgcheck=1
-repo_gpgcheck=1
-autorefresh=1
-type=rpm-md
-EOF
-)
-
 POSTGRESQL_REPO_CONF=$(cat <<'EOF'
 [pgdg10]
 name=PostgreSQL 10 for RHEL/CentOS $releasever - $basearch
@@ -764,7 +752,6 @@ add_repo_as_file 'elasticsearch-7' "$ELASTICSEARCH_7_REPO_CONF"
 add_repo_as_file 'elasticsearch-curator-5' "$ELASTICSEARCH_CURATOR_REPO_CONF"
 add_repo_as_file 'grafana' "$GRAFANA_REPO_CONF"
 add_repo_as_file 'kubernetes' "$KUBERNETES_REPO_CONF"
-add_repo_as_file 'opendistroforelasticsearch' "$OPENDISTRO_REPO_CONF"
 add_repo_as_file 'postgresql-10' "$POSTGRESQL_REPO_CONF"
 add_repo_as_file 'bintray-rabbitmq-rpm' "$RABBITMQ_SERVER_REPO_CONF"
 add_repo_from_script 'https://dl.2ndquadrant.com/default/release/get/10/rpm'
@@ -845,6 +832,34 @@ else
 	exit_with_error "Extracting tar failed: $YUM_CONFIG_BACKUP_FILE_PATH"
 fi
 
+# === Packages from URL  ===
+
+create_directory "$PACKAGES_DIR"
+
+check_connection wget $PACKAGESFROMURL
+
+if [[ -z "$PACKAGESFROMURL" ]]; then
+    echol "No packages from URL to download"
+else
+
+    # list of all files that will be downloaded
+    echol "Package from URL to be downloaded:"
+    cat -n <<< "${PACKAGESFROMURL}"
+
+    printf "\n"
+
+    while IFS=' ' read -r url new_filename; do
+        # download files, check if new filename is provided
+        if [[ -z $new_filename ]]; then
+            download_file "$url" "$PACKAGES_DIR"
+        elif [[ $new_filename = *" "* ]]; then
+            exit_with_error "wrong new filename for file: $url"
+        else
+            download_file "$url" "$PACKAGES_DIR" "$new_filename"
+        fi
+    done <<< "$PACKAGESFROMURL"
+fi
+
 # === Files ===
 
 create_directory "$FILES_DIR"

diff --git a/.../ansible/playbooks/roles/repository/files/download-requirements/redhat-7/requirements.txt b/.../ansible/playbooks/roles/repository/files/download-requirements/redhat-7/requirements.txt
@@ -61,14 +61,6 @@ logstash-oss-7.8.1
 net-tools
 nfs-utils
 nmap-ncat
-# Open Distro for Elasticsearch plugins are installed individually to not download them twice in different versions (as dependencies of opendistroforelasticsearch package)
-opendistro-alerting-1.13.1.*
-opendistro-index-management-1.13.1.*
-opendistro-job-scheduler-1.13.0.*
-opendistro-performance-analyzer-1.13.0.*
-opendistro-security-1.13.1.*
-opendistro-sql-1.13.0.*
-opendistroforelasticsearch-kibana-1.13.1 # kibana has shorter version
 openssl
 perl # for vim
 perl-Getopt-Long # for vim
@@ -147,6 +139,15 @@ kubernetes-cni-0.7.5-0
 # kubernetes-cni-0.8.6-0 since K8s v1.18.6
 kubernetes-cni-0.8.6-0
 
+[packagesfromurl]
+https://github.com/epiphany-platform/alternative-repo/raw/main/1.0.x/redhat/packages/opendistro-alerting-1.13.1.0.rpm
+https://github.com/epiphany-platform/alternative-repo/raw/main/1.0.x/redhat/packages/opendistro-index-management-1.13.1.0.rpm
+https://github.com/epiphany-platform/alternative-repo/raw/main/1.0.x/redhat/packages/opendistro-job-scheduler-1.13.0.0.rpm
+https://github.com/epiphany-platform/alternative-repo/raw/main/1.0.x/redhat/packages/opendistro-performance-analyzer-1.13.0.0.rpm
+https://github.com/epiphany-platform/alternative-repo/raw/main/1.0.x/redhat/packages/opendistro-security-1.13.1.0.rpm
+https://github.com/epiphany-platform/alternative-repo/raw/main/1.0.x/redhat/packages/opendistro-sql-1.13.0.0.rpm
+https://github.com/epiphany-platform/alternative-repo/raw/main/1.0.x/redhat/packages/opendistroforelasticsearch-kibana-1.13.1-linux-x64.rpm
+
 [files]
 # Github repository for erlang rpm is used since packagecloud repository is limited to a certain number of versions and erlang package from erlang-solutions repository is much more complex and bigger
 https://github.com/rabbitmq/erlang-rpm/releases/download/v23.1.5/erlang-23.1.5-1.el7.x86_64.rpm

diff --git a/...e/playbooks/roles/repository/files/download-requirements/ubuntu-18.04/add-repositories.sh b/...e/playbooks/roles/repository/files/download-requirements/ubuntu-18.04/add-repositories.sh
@@ -21,8 +21,5 @@ echo "deb [arch=amd64] https://download.docker.com/linux/ubuntu bionic stable" |
 wget -qO - https://artifacts.elastic.co/GPG-KEY-elasticsearch | apt-key add -
 echo "deb https://artifacts.elastic.co/packages/oss-7.x/apt stable main" | tee /etc/apt/sources.list.d/elastic-7.x.list
 
-wget -qO - https://d3g5vo6xdbdb9a.cloudfront.net/GPG-KEY-opendistroforelasticsearch | sudo apt-key add -
-echo "deb https://d3g5vo6xdbdb9a.cloudfront.net/apt stable main" | tee -a /etc/apt/sources.list.d/opendistroforelasticsearch.list
-
 wget -qO - https://dl.2ndquadrant.com/gpg-key.asc | apt-key add -
 echo "deb https://dl.2ndquadrant.com/default/release/apt bionic-2ndquadrant main" | tee -a /etc/apt/sources.list.d/2ndquadrant-dl-default-release.list
diff --git a/...ybooks/roles/repository/files/download-requirements/ubuntu-18.04/download-requirements.sh b/...ybooks/roles/repository/files/download-requirements/ubuntu-18.04/download-requirements.sh
@@ -84,9 +84,10 @@ shopt -u nullglob
 check_connection apt $(ls /etc/apt/sources.list.d)
 apt update
 
-# parse the input file, separete by tags: [crane], [packages], [files], [images]
+# parse the input file, separete by tags: [crane], [packages], [packagesfromurl], [files], [images]
 crane=$(awk '/^$/ || /^#/ {next}; /\[crane\]/ {f=1; next}; /^\[/ {f=0}; f {print $0}' "${input_file}")
 packages=$(awk '/^$/ || /^#/ {next}; /\[packages\]/ {f=1; next}; /^\[/ {f=0}; f {print $0}' "${input_file}")
+packagesfromurl=$(awk '/^$/ || /^#/ {next}; /\[packagesfromurl\]/ {f=1; next}; /^\[/ {f=0}; f {print $0}' "${input_file}")
 files=$(awk '/^$/ || /^#/ {next}; /\[files\]/ {f=1; next}; /^\[/ {f=0}; f {print $0}' "${input_file}")
 images=$(awk '/^$/ || /^#/ {next}; /\[images\]/ {f=1; next}; /^\[/ {f=0}; f {print $0}' "${input_file}")
 
@@ -159,10 +160,36 @@ fi
 
 printf "\n"
 
-check_connection wget $(for file in $files; do echo "$file"; done)
+# PACKAGES AS URL
+# process files
+
+check_connection wget $(for file in $packagesfromurl; do echo "$file"; done)
+
+
+if [[ -z "${packagesfromurl}" ]]; then
+    echol "No packages from URL to download"
+else
+    # be verbose, show what will be downloaded
+    # TODO: this is the list of all files shows on every run, not only the files that will be downloaded this run
+    echol "Packages from URL to be downloaded:"
+    cat -n <<< "${packagesfromurl}"
+
+    printf "\n"
+    # download files using wget
+    while IFS= read -r file; do
+        # download files, skip if exists
+        #wget --no-verbose --continue --directory-prefix="${dst_dir_files}" "${file}"
+        #wget --continue --show-progress --directory-prefix="${dst_dir_files}" "${file}"
+        download_file "${file}" "${dst_dir_packages}"
+    done <<< "${packagesfromurl}"
+fi
 
 # FILES
 # process files
+
+check_connection wget $(for file in $files; do echo "$file"; done)
+
+
 if [[ -z "${files}" ]]; then
     echol "No files to download"
 else

diff --git a/...ible/playbooks/roles/repository/files/download-requirements/ubuntu-18.04/requirements.txt b/...ible/playbooks/roles/repository/files/download-requirements/ubuntu-18.04/requirements.txt
@@ -55,13 +55,6 @@ logstash-oss 1:7.8.1
 netcat
 net-tools
 nfs-common
-opendistro-alerting 1.13.1
-opendistro-index-management 1.13.1
-opendistro-job-scheduler 1.13.0
-opendistro-performance-analyzer 1.13.0
-opendistro-security 1.13.1
-opendistro-sql 1.13.0
-opendistroforelasticsearch-kibana 1.13.1
 openjdk-8-jre-headless
 openssl
 postgresql-10
@@ -203,6 +196,15 @@ kubernetes-cni 0.7.5-00
 # kubernetes-cni-0.8.6-0 since K8s v1.18.6
 kubernetes-cni 0.8.6-00
 
+[packagesfromurl]
+https://github.com/epiphany-platform/alternative-repo/raw/main/1.0.x/ubuntu/packages/opendistro-alerting_1.13.1.0-1_all.deb
+https://github.com/epiphany-platform/alternative-repo/raw/main/1.0.x/ubuntu/packages/opendistro-index-management_1.13.1.0-1_all.deb
+https://github.com/epiphany-platform/alternative-repo/raw/main/1.0.x/ubuntu/packages/opendistro-job-scheduler_1.13.0.0-1_all.deb
+https://github.com/epiphany-platform/alternative-repo/raw/main/1.0.x/ubuntu/packages/opendistro-performance-analyzer_1.13.0.0-1_all.deb
+https://github.com/epiphany-platform/alternative-repo/raw/main/1.0.x/ubuntu/packages/opendistro-security_1.13.1.0-1_all.deb
+https://github.com/epiphany-platform/alternative-repo/raw/main/1.0.x/ubuntu/packages/opendistro-sql_1.13.0.0-1_all.deb
+https://github.com/epiphany-platform/alternative-repo/raw/main/1.0.x/ubuntu/packages/opendistroforelasticsearch-kibana_1.13.1_amd64.deb
+
 [files]
 https://repo1.maven.org/maven2/io/prometheus/jmx/jmx_prometheus_javaagent/0.14.0/jmx_prometheus_javaagent-0.14.0.jar
 https://archive.apache.org/dist/kafka/2.6.0/kafka_2.12-2.6.0.tgz