Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Using single temp maven local repository for release validation tasks #32841

Merged
merged 1 commit into from
Oct 18, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions .github/workflows/beam_PostRelease_NightlySnapshot.yml
Original file line number Diff line number Diff line change
Expand Up @@ -59,10 +59,17 @@ jobs:
uses: ./.github/actions/setup-environment-action
with:
java-version: default
- name: Setup temp local maven
id: setup_local_maven
run: echo "NEW_TEMP_DIR=$(mktemp -d)" >> $GITHUB_OUTPUT
- name: run PostRelease validation script
uses: ./.github/actions/gradle-command-self-hosted-action
with:
gradle-command: :release:runJavaExamplesValidationTask
arguments: |
-Pver='${{ github.event.inputs.RELEASE }}' \
-Prepourl='${{ github.event.inputs.SNAPSHOT_URL }}' \
-PmavenLocalPath='${{ steps.setup_local_maven.outputs.NEW_TEMP_DIR }}'
- name: Clean up local maven
if: steps.setup_local_maven.outcome == 'success'
run: rm -rf '${{ steps.setup_local_maven.outputs.NEW_TEMP_DIR }}'
Original file line number Diff line number Diff line change
Expand Up @@ -2513,6 +2513,8 @@ class BeamModulePlugin implements Plugin<Project> {
def taskName = "run${config.type}Java${config.runner}"
def releaseVersion = project.findProperty('ver') ?: project.version
def releaseRepo = project.findProperty('repourl') ?: 'https://repository.apache.org/content/repositories/snapshots'
// shared maven local path for maven archetype projects
def sharedMavenLocal = project.findProperty('mavenLocalPath') ?: ''
def argsNeeded = [
"--ver=${releaseVersion}",
"--repourl=${releaseRepo}"
Expand All @@ -2532,6 +2534,9 @@ class BeamModulePlugin implements Plugin<Project> {
if (config.pubsubTopic) {
argsNeeded.add("--pubsubTopic=${config.pubsubTopic}")
}
if (sharedMavenLocal) {
argsNeeded.add("--mavenLocalPath=${sharedMavenLocal}")
}
project.evaluationDependsOn(':release')
project.task(taskName, dependsOn: ':release:classes', type: JavaExec) {
group = "Verification"
Expand Down
38 changes: 25 additions & 13 deletions release/src/main/groovy/TestScripts.groovy
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@ class TestScripts {
static String gcsBucket
static String bqDataset
static String pubsubTopic
static String mavenLocalPath
}

def TestScripts(String[] args) {
Expand All @@ -47,6 +48,7 @@ class TestScripts {
cli.gcsBucket(args:1, 'Google Cloud Storage Bucket')
cli.bqDataset(args:1, "BigQuery Dataset")
cli.pubsubTopic(args:1, "PubSub Topic")
cli.mavenLocalPath(args:1, "Maven local path")

def options = cli.parse(args)
var.repoUrl = options.repourl
Expand All @@ -73,6 +75,10 @@ class TestScripts {
var.pubsubTopic = options.pubsubTopic
println "PubSub Topic: ${var.pubsubTopic}"
}
if (options.mavenLocalPath) {
var.mavenLocalPath = options.mavenLocalPath
println "Maven local path: ${var.mavenLocalPath}"
}
}

def ver() {
Expand Down Expand Up @@ -189,11 +195,16 @@ class TestScripts {
}
}

// Run a maven command, setting up a new local repository and a settings.xml with a custom repository
// Run a maven command, setting up a new local repository and a settings.xml with a custom repository if needed
private String _mvn(String args) {
def m2 = new File(var.startDir, ".m2/repository")
String mvnlocalPath = var.mavenLocalPath
if (!(var.mavenLocalPath)) {
mvnlocalPath = var.startDir
}
def m2 = new File(mvnlocalPath, ".m2/repository")
m2.mkdirs()
def settings = new File(var.startDir, "settings.xml")
def settings = new File(mvnlocalPath, "settings.xml")
if(!settings.exists()) {
settings.write """
<settings>
<localRepository>${m2.absolutePath}</localRepository>
Expand All @@ -209,16 +220,17 @@ class TestScripts {
</profile>
</profiles>
</settings>
"""
def cmd = "mvn ${args} -s ${settings.absolutePath} -Ptestrel -B"
String path = System.getenv("PATH");
// Set the path on jenkins executors to use a recent maven
// MAVEN_HOME is not set on some executors, so default to 3.5.2
String maven_home = System.getenv("MAVEN_HOME") ?: '/home/jenkins/tools/maven/apache-maven-3.5.4'
println "Using maven ${maven_home}"
def mvnPath = "${maven_home}/bin"
def setPath = "export PATH=\"${mvnPath}:${path}\" && "
return _execute(setPath + cmd)
"""
}
def cmd = "mvn ${args} -s ${settings.absolutePath} -Ptestrel -B"
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Shall we also add "-U" in addition to this change ?

Copy link
Contributor Author

@Abacn Abacn Oct 18, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Sorry later on I found my initial investigation in #32824 (comment) was wrong. In particular, the test workflow runs 8 tasks in parallel, each spin up an individual maven local repository, and delete after finish. So there isn't exist scenario that bad local repository cached can interfere the next run.

Using -U tag will cause the workflow making more requests to maven snapshot repository, which is known to be less stable in terms of availability, and I am afraid it would worsen rate limiting

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ack. Thanks.

String path = System.getenv("PATH");
// Set the path on jenkins executors to use a recent maven
// MAVEN_HOME is not set on some executors, so default to 3.5.2
String maven_home = System.getenv("MAVEN_HOME") ?: '/usr/local/maven'
println "Using maven ${maven_home}"
def mvnPath = "${maven_home}/bin"
def setPath = "export PATH=\"${mvnPath}:${path}\" && "
return _execute(setPath + cmd)
}

// Clean up and report error
Expand Down
Loading