Skip to content

Commit

Permalink
Merge branch 'master' of https://github.com/apache/beam into tox_2_again
Browse files Browse the repository at this point in the history
  • Loading branch information
caneff committed Nov 14, 2023
2 parents 3a6cb26 + 38db0e5 commit 095f60e
Show file tree
Hide file tree
Showing 1,422 changed files with 62,366 additions and 34,694 deletions.
1 change: 1 addition & 0 deletions .asf.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,7 @@ github:

protected_branches:
master: {}
release-2.52.0: {}
release-2.51.0: {}
release-2.50.0: {}
release-2.49.0: {}
Expand Down
5 changes: 5 additions & 0 deletions .github/REVIEWERS.yml
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@ labels:
- kennknowles
- robertwb
- bvolpato
- m-trieu
exclusionList: []
- name: IO
reviewers:
Expand All @@ -52,6 +53,10 @@ labels:
- name: spanner
reviewers:
- nielm
- name: bigtable
reviewers:
- igorbernstein2
- mutianf
exclusionList: []
- name: Build
reviewers:
Expand Down
10 changes: 5 additions & 5 deletions .github/actions/common-rc-validation/action.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -36,19 +36,19 @@ runs:
shell: bash
run: |
echo "---------------------Downloading Python Staging RC----------------------------"
wget ${PYTHON_RC_DOWNLOAD_URL}/${RELEASE_VER}/python/apache-beam-${RELEASE_VER}.zip
wget ${PYTHON_RC_DOWNLOAD_URL}/${RELEASE_VER}/python/apache-beam-${RELEASE_VER}.zip.sha512
if [[ ! -f apache-beam-$RELEASE_VER.zip ]]; then
wget ${PYTHON_RC_DOWNLOAD_URL}/${RELEASE_VER}/python/apache-beam-${RELEASE_VER}.tar.gz
wget ${PYTHON_RC_DOWNLOAD_URL}/${RELEASE_VER}/python/apache-beam-${RELEASE_VER}.tar.gz.sha512
if [[ ! -f apache-beam-$RELEASE_VER.tar.gz ]]; then
{ echo "Fail to download Python Staging RC files." ;exit 1; }
fi
echo "--------------------------Verifying Hashes------------------------------------"
sha512sum -c apache-beam-${RELEASE_VER}.zip.sha512
sha512sum -c apache-beam-${RELEASE_VER}.tar.gz.sha512
`which pip` install --upgrade pip
`which pip` install --upgrade setuptools
- name: Installing python SDK
shell: bash
run: pip install apache-beam-${RELEASE_VER}.zip[gcp]
run: pip install apache-beam-${RELEASE_VER}.tar.gz[gcp]

11 changes: 3 additions & 8 deletions .github/actions/gradle-command-self-hosted-action/action.yml
Original file line number Diff line number Diff line change
Expand Up @@ -24,12 +24,6 @@ inputs:
required: false
description: 'Gradle options'
default: ''
default-arguments:
required: false
description: 'Default gradle switches' # Copied from CommonJobProperties.groovy'
default: |
--continue -Dorg.gradle.jvmargs=-Xms2g -Dorg.gradle.jvmargs=-Xmx6g \
-Dorg.gradle.vfs.watch=false -Pdocker-pull-licenses
max-workers:
required: false
description: 'Max number of workers'
Expand All @@ -47,5 +41,6 @@ runs:
if [ -f ~/.m2/settings.xml ]; then
rm ~/.m2/settings.xml
fi
./gradlew ${{ inputs.gradle-command }} --max-workers=${{ inputs.max-workers }} ${{ inputs.arguments }} \
${{ inputs.default-arguments }}
./gradlew ${{ inputs.gradle-command }} --max-workers=${{ inputs.max-workers }} --continue \
-Dorg.gradle.jvmargs=-Xms2g -Dorg.gradle.jvmargs=-Xmx6g -Dorg.gradle.vfs.watch=false -Pdocker-pull-licenses \
${{ inputs.arguments }}
4 changes: 1 addition & 3 deletions .github/actions/setup-action/action.yml
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,4 @@ runs:
- name: expose gcloud path
shell: bash
run: |
echo KUBELET_GCLOUD_CONFIG_PATH=/var/lib/kubelet/pods/$POD_UID/volumes/kubernetes.io~empty-dir/gcloud >> $GITHUB_ENV
- name: Setup environment
uses: ./.github/actions/setup-environment-action
echo KUBELET_GCLOUD_CONFIG_PATH=/var/lib/kubelet/pods/$POD_UID/volumes/kubernetes.io~empty-dir/gcloud >> $GITHUB_ENV
6 changes: 5 additions & 1 deletion .github/actions/setup-environment-action/action.yml
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,10 @@ inputs:
required: false
description: 'Install Go version'
default: ''
disable-cache:
required: false
description: 'Whether to disable the gradle cache'
default: false

runs:
using: "composite"
Expand All @@ -48,7 +52,7 @@ runs:
- name: Setup Gradle
uses: gradle/gradle-build-action@v2
with:
cache-read-only: false
cache-read-only: ${{ inputs.disable-cache }}
- name: Install Go
if: ${{ inputs.go-version != '' }}
uses: actions/setup-go@v3
Expand Down
2 changes: 2 additions & 0 deletions .github/autolabeler.yml
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@ python: ["sdks/python/**/*", "learning/katas/python/**/*"]
typescript: ["sdks/typescript/**/*"]
vendor: ["vendor/**/*"]
website: ["website/**/*"]
yaml: ["sdks/python/apache_beam/yaml/**"]

# Extensions
extensions: ["sdks/java/extensions/**/*", "runners/extensions-java/**/*"]
Expand Down Expand Up @@ -68,6 +69,7 @@ io: ["sdks/go/pkg/beam/io/**/*", "sdks/java/io/**/*", "sdks/python/apache_beam/
"redis": ["sdks/java/io/redis/**/*"]
"solr": ["sdks/java/io/solr/**/*"]
"spanner": ["sdks/go/pkg/beam/io/spannerio/**/*", "sdks/python/apache_beam/io/gcp/spanner.py", "sdks/python/apache_beam/io/gcp/experimental/spannerio.py", "sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/**/*"]
"bigtable": ["sdks/go/pkg/beam/io/bigtableio/**/*", "sdks/go/pkg/beam/io/xlang/bigtableio/**/*", "sdks/python/apache_beam/io/gcp/bigtableio.py", "sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/**/*"]
"synthetic": ["sdks/java/io/synthetic/**/*"]
"tests": ["sdks/java/io/file-based-io-tests/**/*"]
"thrift": ["sdks/java/io/thrift/**/*"]
Expand Down
61 changes: 61 additions & 0 deletions .github/build.gradle
Original file line number Diff line number Diff line change
@@ -0,0 +1,61 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* License); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an AS IS BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

buildscript {
repositories {
mavenCentral()
}
dependencies {
classpath group: 'org.yaml', name: 'snakeyaml', version: '2.2'
}
}

/** check that yml are valid */
task check {
doLast {
fileTree("${project.projectDir}/workflows").matching {
include "*.yml"
include "*.yaml"
}.each {
def fname = it.getName()
// attempt load yml to make sure its valid
def workflow = new org.yaml.snakeyaml.Yaml().load(it.newInputStream())

// additional guards for running all tests functionality
// TODO(yathu) expand this also to post commits prior teardown Jenkins postcommits
if ( fname.startsWith("beam_PreCommit")) {
List paths
try {
paths = workflow.getAt(true).pull_request_target.paths as List
} catch (Exception e) {
throw new GradleException("Fail to get the trigger path for ${fname}. " +
"Make sure precommit has a pull_request_target trigger.", e)
}
// precommit should triggered by this specific file
if (paths != null && !paths.contains('release/trigger_all_tests.json')) {
throw new GradleException("Error validating ${fname}: " +
"Please add 'release/trigger_all_tests.json' to the trigger path for release verification run properly")
}
}
}
}
}

task preCommit {
dependsOn check
}
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ spec:
name: ${name}
minReplicas: ${min_runners}
maxReplicas: ${max_runners}
%{~ if webhook_scaling == "true" ~}
%{~ if webhook_scaling ~}
scaleUpTriggers:
- githubEvent:
workflowJob: {}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,11 @@ metadata:
name: ${name}
spec:
template:
metadata:
annotations:
cluster-autoscaler.kubernetes.io/safe-to-evict: "false"
spec:
dockerMTU: 1460
%{~ if selector == true ~}
nodeSelector:
runner-pool: ${name}
Expand Down
24 changes: 14 additions & 10 deletions .github/gh-actions-self-hosted-runners/arc/environments/beam.env
Original file line number Diff line number Diff line change
Expand Up @@ -20,24 +20,28 @@
project_id = "apache-beam-testing"
region = "us-central1"
zone = "us-central1-b"
environment = "beam"
environment = "beam-prod"
ingress_domain = "action.beam.apache.org"
organization = "apache"
repository = "beam"
github_app_id_secret_name = "gh-app_id"
github_app_install_id_secret_name = "gh-app_installation_id"
github_private_key_secret_name = "gh-pem_key"
deploy_webhook = "true"
existing_vpc_name = "default"
existing_ip_name = "beam-arc-webhook-ip"
subnetwork_cidr_range = "10.119.0.0/20"
service_account_id = "[email protected]"
runner_group = "beam"
main_runner = {
name = "main-runner"
runner_image = "us-central1-docker.pkg.dev/apache-beam-testing/beam-github-actions/beam-arc-runner:2b20e26bb3b99d8e4f41a3d1d9d2e7080043de5c"
machine_type = "e2-standard-16"
min_node_count = "1"
max_node_count = "24"
max_node_count = "30"
min_replicas = "1"
max_replicas = "200"
webhook_scaling = true
max_replicas = "240"
webhook_scaling = false
disk_size_gb = 200
requests = {
cpu = "2"
Expand All @@ -49,10 +53,10 @@ additional_runner_pools = [{
machine_type = "e2-standard-2"
runner_image = "us-central1-docker.pkg.dev/apache-beam-testing/beam-github-actions/beam-arc-runner:2b20e26bb3b99d8e4f41a3d1d9d2e7080043de5c"
min_node_count = "1"
max_node_count = "10"
max_node_count = "15"
min_replicas = "1"
max_replicas = "10"
webhook_scaling = "true"
max_replicas = "15"
webhook_scaling = false
requests = {
cpu = "1500m"
memory = "5Gi"
Expand All @@ -66,10 +70,10 @@ additional_runner_pools = [{
machine_type = "c3-highmem-8"
runner_image = "us-central1-docker.pkg.dev/apache-beam-testing/beam-github-actions/beam-arc-runner:2b20e26bb3b99d8e4f41a3d1d9d2e7080043de5c"
min_node_count = "1"
max_node_count = "10"
max_node_count = "15"
min_replicas = "1"
max_replicas = "10"
webhook_scaling = "true"
max_replicas = "15"
webhook_scaling = false
requests = {
cpu = "7.5"
memory = "5Gi"
Expand Down
16 changes: 14 additions & 2 deletions .github/gh-actions-self-hosted-runners/arc/gke.tf
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ resource "google_container_cluster" "actions-runner-gke" {
project = var.project_id
location = var.zone
initial_node_count = 1
network = google_compute_network.actions-runner-network.id
network = data.google_compute_network.actions-runner-network.id
subnetwork = google_compute_subnetwork.actions-runner-subnetwork.id
remove_default_node_pool = true

Expand All @@ -45,6 +45,7 @@ resource "google_container_node_pool" "main-actions-runner-pool" {
oauth_scopes = [
"https://www.googleapis.com/auth/cloud-platform"
]
service_account = data.google_service_account.service_account.email
tags = ["actions-runner-pool"]
}
}
Expand Down Expand Up @@ -72,6 +73,7 @@ resource "google_container_node_pool" "additional_runner_pools" {
oauth_scopes = [
"https://www.googleapis.com/auth/cloud-platform"
]
service_account = data.google_service_account.service_account.email
tags = ["actions-runner-pool"]
labels = {
"runner-pool" = each.value.name
Expand All @@ -90,5 +92,15 @@ resource "google_container_node_pool" "additional_runner_pools" {


resource "google_compute_global_address" "actions-runner-ip" {
name = "${var.environment}-actions-runner-ip"
count = var.deploy_webhook == "true" && var.existing_ip_name == "" ? 1 : 0
name = "${var.environment}-actions-runner-ip"
}

data "google_compute_global_address" "actions-runner-ip" {
count = var.deploy_webhook == "true" ? 1 : 0
name = var.existing_ip_name == "" ? google_compute_global_address.actions-runner-ip[0].name : var.existing_ip_name
}

data google_service_account "service_account" {
account_id = var.service_account_id
}
1 change: 1 addition & 0 deletions .github/gh-actions-self-hosted-runners/arc/kubernetes.tf
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ resource "kubectl_manifest" "arc_autoscaler" {
depends_on = [helm_release.arc]
}
resource "kubectl_manifest" "arc_webhook_certificate" {
count = var.deploy_webhook != "false" ? 1 : 0
yaml_body = templatefile("config/arc_certificate.tpl", { ingress_domain = var.ingress_domain })
override_namespace = "arc"
depends_on = [helm_release.arc]
Expand Down
8 changes: 3 additions & 5 deletions .github/gh-actions-self-hosted-runners/arc/locals.tf
Original file line number Diff line number Diff line change
Expand Up @@ -19,20 +19,18 @@


locals {
subnetwork_cidr_range = "10.128.0.0/20"
arc_values = {
"githubWebhookServer.enabled" = "true"
"githubWebhookServer.enabled" = "${var.deploy_webhook}"
"authSecret.create" = "true"
"authSecret.github_app_id" = data.google_secret_manager_secret_version.github_app_id.secret_data
"authSecret.github_app_installation_id" = data.google_secret_manager_secret_version.github_app_install_id.secret_data
"authSecret.github_app_private_key" = data.google_secret_manager_secret_version.github_private_key.secret_data
"githubWebhookServer.ingress.enabled" = "true"
"githubWebhookServer.ingress.enabled" = "${var.deploy_webhook}"
"githubWebhookServer.ingress.hosts[0].host" = var.ingress_domain
"githubWebhookServer.ingress.hosts[0].paths[0].path" = "/"
"githubWebhookServer.ingress.hosts[0].paths[0].pathType" = "ImplementationSpecific"
"githubWebhookServer.service.type" = "NodePort"
#"githubWebhookServer.ingress.tls[0].hosts[0]" = var.ingress_domain
"githubWebhookServer.ingress.annotations.kubernetes\\.io/ingress\\.global-static-ip-name" = google_compute_global_address.actions-runner-ip.name
"githubWebhookServer.ingress.annotations.kubernetes\\.io/ingress\\.global-static-ip-name" = var.deploy_webhook != "false" ? data.google_compute_global_address.actions-runner-ip[0].name : "not-configured"
"githubWebhookServer.ingress.annotations.networking\\.gke\\.io/managed-certificates" = "managed-cert"
"githubWebhookServer.ingress.annotations.kubernetes\\.io/ingress\\.class" = "gce"
}
Expand Down
10 changes: 8 additions & 2 deletions .github/gh-actions-self-hosted-runners/arc/network.tf
Original file line number Diff line number Diff line change
Expand Up @@ -18,15 +18,21 @@
#

resource "google_compute_network" "actions-runner-network" {
count = var.existing_vpc_name == "" ? 1 : 0
project = var.project_id
name = "${var.environment}-actions-runner-network"
auto_create_subnetworks = false
}
data "google_compute_network" "actions-runner-network" {
name = var.existing_vpc_name == "" ? google_compute_network.actions-runner-network[0].name : var.existing_vpc_name
project = var.project_id
}


resource "google_compute_subnetwork" "actions-runner-subnetwork" {
ip_cidr_range = local.subnetwork_cidr_range
ip_cidr_range = var.subnetwork_cidr_range
name = "${var.environment}-actions-runner-subnetwork"
network = google_compute_network.actions-runner-network.id
network = data.google_compute_network.actions-runner-network.id
region = var.region
project = var.project_id
}
3 changes: 1 addition & 2 deletions .github/gh-actions-self-hosted-runners/arc/outputs.tf
Original file line number Diff line number Diff line change
Expand Up @@ -24,9 +24,8 @@ output "cluster_endpoint" {
value = google_container_cluster.actions-runner-gke.endpoint
}
output "ingress_ip" {
value = google_compute_global_address.actions-runner-ip.address
value = var.deploy_webhook != "false" ? data.google_compute_global_address.actions-runner-ip[0].address : "Not Configured"
}

output "get_kubeconfig_command" {
value = "gcloud container clusters get-credentials ${google_container_cluster.actions-runner-gke.name} --region ${var.zone} --project ${var.project_id}"
}
Expand Down
6 changes: 3 additions & 3 deletions .github/gh-actions-self-hosted-runners/arc/provider.tf
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@

terraform {
backend "gcs" {
prefix = "test-state"
prefix = "prod"
}

required_providers {
Expand All @@ -28,8 +28,8 @@ terraform {
version = "~> 4.62.0"
}
kubectl = {
source = "gavinbunney/kubectl"
version = ">= 1.7.0"
source = "alekc/kubectl"
version = ">= 2.0.2"
}
}
}
Expand Down
Loading

0 comments on commit 095f60e

Please sign in to comment.