Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix V3 scans for client-side & generate separate VSCode notices bundle #149

Merged
Merged
Show file tree
Hide file tree
Changes from 8 commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .dockerfiles/ort.Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@ RUN rustup install stable && rustup default stable
RUN cargo install cargo-license
RUN cargo install get-license-helper

ARG ORT_VERSION=15.1.0
ARG ORT_VERSION=33.1.0

RUN git clone https://github.com/oss-review-toolkit/ort
WORKDIR /home/build/ort
Expand Down
76 changes: 55 additions & 21 deletions .github/workflows/license-generation.yml
Original file line number Diff line number Diff line change
Expand Up @@ -28,10 +28,10 @@ on:
type: string
required: false
default: ''
zowe_sources_branch:
zowe_sources_branch:
description: The branch of zowe-install-packaging used to determine sources included in the scan
required: true
default: 'v2.x/rc'
default: 'v2.x/rc'
dummy_build:
description: Creates empty zip files, bypassing license scans. For test purposes only.
required: false
Expand All @@ -52,26 +52,30 @@ on:
- 'debug'

env:
PUBLISH_RELEASE: ${{ github.event.inputs.publish_release }}
PUBLISH_RELEASE: ${{ github.event.inputs.publish_release }}
RELEASE_SUFFIX: ${{ github.event.inputs.release_suffix }}
REPLACE_EXISTING_RELEASE: ${{ github.event.inputs.replace_release }}
REPLACE_EXISTING_RELEASE: ${{ github.event.inputs.replace_release }}
ZOWE_RELEASE_BRANCH: ${{ github.event.inputs.zowe_sources_branch }}
PENDING_APPROVAL_REPORT_NAME: dependency_approval_action_aggregates.json
DEPENDENCY_SCAN_HOME: licenses/dependency-scan
MARKDOWN_REPORT_NAME: markdown_dependency_report.md
MARKDOWN_CLI_REPORT: cli_dependency_report.md
MARKDOWN_VSCODE_REPORT: vscode_dependency_report.md
MARKDOWN_ZOS_REPORT: zos_dependency_report.md
NOTICES_AGGREGATE_FILE: notices_aggregate.txt
NOTICES_CLI_FILE: notices_cli.txt
NOTICES_VSCODE_FILE: notices_vscode.txt
NOTICES_ZOS_FILE: notices_zos.txt
ARTIFACT_PATH: org/zowe/licenses
ARTIFACT_PATH_SBOM: init_in_step_one
VERSION: ${{ github.event.inputs.zowe_version }}
AGG_ARTIFACT_NAME: zowe_licenses_full.zip
CLI_ARTIFACT_NAME: zowe_licenses_cli.zip
VSCODE_ARTIFACT_NAME: zowe_licenses_vscode.zip
ZOS_ARTIFACT_NAME: zowe_licenses_zos.zip
AGG_SBOM_ARTIFACT_NAME: sbom_aggregate.spdx.yml
CLI_SBOM_ARTIFACT_NAME: sbom_cli.spdx.yml
VSCODE_SBOM_ARTIFACT_NAME: sbom_vscode.spdx.yml
ZOS_SBOM_ARTIFACT_NAME: sbom_zos.spdx.yml
FILENAME_PATTERN: init_in_step_one
ARTIFACT_REPO: init_in_step_one
Expand All @@ -84,13 +88,13 @@ jobs:
create-licenses:

runs-on: ubuntu-latest

container:
image: zowe-docker-release.jfrog.io/ompzowe/zowecicd-license-base:latest
image: zowe-docker-snapshot.jfrog.io/ompzowe/zowecicd-license-base:test-timothy
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@MarkAckert I'm happy to update the zowe-docker-release latest tag to include the updated ORT and pnpm install step if you'd like 😋


steps:
- name: Update variables if releasing
run: |
run: |
if [ "$PUBLISH_RELEASE" = true ]; then
echo "ARTIFACT_REPO=libs-release-local" >> $GITHUB_ENV
echo "ARTIFACT_VERSION=$VERSION" >> $GITHUB_ENV
Expand All @@ -102,68 +106,75 @@ jobs:
echo "ARTIFACT_PATH_SBOM=org/zowe/${{ env.VERSION }}-SNAPSHOT/sbom" >> $GITHUB_ENV
echo "FILENAME_PATTERN={filename}-${{ env.VERSION }}-SNAPSHOT{timestamp}{fileext}" >> $GITHUB_ENV
fi

- name: Checkout current repo
uses: actions/checkout@v4

- uses: actions/setup-node@v4
with:
node-version: '20'

- name: '[Zowe Actions] Prepare workflow'
uses: zowe-actions/shared-actions/prepare-workflow@main

- name: 'Setup jFrog CLI'
uses: jfrog/setup-jfrog-cli@v4
env:
JF_ENV_1: ${{ secrets.JF_ARTIFACTORY_TOKEN }}

- name: '[TEST-ONLY] Dummy scan step'
if: ${{ github.event.inputs.dummy_build == 'true' }}
if: ${{ github.event.inputs.dummy_build == 'true' }}
working-directory: ${{ env.DEPENDENCY_SCAN_HOME }}
run: |
run: |
mkdir -p zowe_licenses
mkdir -p zowe_cli_licenses
mkdir -p zowe_vscode_licenses
mkdir -p zowe_zos_licenses
echo "HI" >> dummy.txt
cp dummy.txt zowe_licenses
cp dummy.txt zowe_cli_licenses
cp dummy.txt zowe_vscode_licenses
cp dummy.txt zowe_zos_licenses

zip -j ${{ env.AGG_ARTIFACT_NAME }} zowe_licenses/*
zip -j ${{ env.CLI_ARTIFACT_NAME }} zowe_cli_licenses/*
zip -j ${{ env.VSCODE_ARTIFACT_NAME }} zowe_vscode_licenses/*
zip -j ${{ env.ZOS_ARTIFACT_NAME }} zowe_zos_licenses/*
echo "" > ${{ env.AGG_SBOM_ARTIFACT_NAME }}
echo "" > ${{ env.ZOS_SBOM_ARTIFACT_NAME }}
echo "" > ${{ env.CLI_SBOM_ARTIFACT_NAME }}
echo "" > ${{ env.VSCODE_SBOM_ARTIFACT_NAME }}
echo "" > ${{ env.ZOS_SBOM_ARTIFACT_NAME }}


- name: Scan Licenses on Branch ${{ env.ZOWE_RELEASE_BRANCH }}
if: ${{ github.event.inputs.dummy_build == 'false' }}
if: ${{ github.event.inputs.dummy_build == 'false' }}
env:
APP_NOTICES_SCAN: true
APP_LICENSE_SCAN: true
ZOWE_MANIFEST_BRANCH: ${{ env.ZOWE_RELEASE_BRANCH }}
working-directory: ${{ env.DEPENDENCY_SCAN_HOME }}
run: |
# Rustup is set to default in the container, but it's not picked up in this run block
rustup default stable
rustup default stable
npm install -g pnpm@8
yarn install && yarn build
node lib/index.js
cd build
zip -r logs.zip logs/
zip -r logs.zip logs/
zip -r license_reports.zip license_reports/
zip -r notice_reports.zip notice_reports/
cd ..
mkdir -p zowe_licenses
mkdir -p zowe_cli_licenses
mkdir -p zowe_vscode_licenses
mkdir -p zowe_zos_licenses
cp ../resources/* zowe_licenses/
cp ../resources/* zowe_cli_licenses/
cp ../resources/* zowe_vscode_licenses/
cp ../resources/* zowe_zos_licenses/

zip -r logs.zip build/logs/*

# Aggregate
cp build/notice_reports/${{ env.NOTICES_AGGREGATE_FILE }} zowe_licenses/zowe_full_notices.txt
cp build/license_reports/${{ env.MARKDOWN_REPORT_NAME }} zowe_licenses/zowe_full_dependency_list.md
Expand All @@ -173,6 +184,12 @@ jobs:
cp build/notice_reports/${{ env.NOTICES_CLI_FILE }} zowe_cli_licenses/zowe_cli_notices.txt
cp build/license_reports/${{ env.MARKDOWN_CLI_REPORT }} zowe_cli_licenses/zowe_cli_dependency_list.md
zip -j ${{ env.CLI_ARTIFACT_NAME }} zowe_cli_licenses/*

# VSCode
cp build/notice_reports/${{ env.NOTICES_VSCODE_FILE }} zowe_vscode_licenses/zowe_vscode_notices.txt
cp build/license_reports/${{ env.MARKDOWN_VSCODE_REPORT }} zowe_vscode_licenses/zowe_vscode_dependency_list.md
zip -j ${{ env.VSCODE_ARTIFACT_NAME }} zowe_vscode_licenses/*

# z/OS
cp build/notice_reports/${{ env.NOTICES_ZOS_FILE }} zowe_zos_licenses/zowe_zos_notices.txt
cp build/license_reports/${{ env.MARKDOWN_ZOS_REPORT }} zowe_zos_licenses/zowe_zos_dependency_list.md
Expand All @@ -181,6 +198,7 @@ jobs:
# SBOMs
cp build/sbom_reports/${{ env.AGG_SBOM_ARTIFACT_NAME }} ${{ env.AGG_SBOM_ARTIFACT_NAME }}
cp build/sbom_reports/${{ env.CLI_SBOM_ARTIFACT_NAME }} ${{ env.CLI_SBOM_ARTIFACT_NAME }}
cp build/sbom_reports/${{ env.VSCODE_SBOM_ARTIFACT_NAME }} ${{ env.VSCODE_SBOM_ARTIFACT_NAME }}
cp build/sbom_reports/${{ env.ZOS_SBOM_ARTIFACT_NAME }} ${{ env.ZOS_SBOM_ARTIFACT_NAME }}


Expand All @@ -199,6 +217,11 @@ jobs:
--password ${{secrets.ZOWE_JFROG_ELEVATED_KEY }} \
--url https://zowe.jfrog.io/artifactory \
${{ env.ARTIFACT_REPO }}/${{ env.ARTIFACT_PATH }}/${{ env.ARTIFACT_VERSION }}/${{ env.CLI_ARTIFACT_NAME }}
jfrog rt del\
--user ${{ secrets.ZOWE_JFROG_ELEVATED_USER }} \
--password ${{secrets.ZOWE_JFROG_ELEVATED_KEY }} \
--url https://zowe.jfrog.io/artifactory \
${{ env.ARTIFACT_REPO }}/${{ env.ARTIFACT_PATH }}/${{ env.ARTIFACT_VERSION }}/${{ env.VSCODE_ARTIFACT_NAME }}
jfrog rt del\
--user ${{ secrets.ZOWE_JFROG_ELEVATED_USER }} \
--password ${{secrets.ZOWE_JFROG_ELEVATED_KEY }} \
Expand All @@ -214,6 +237,11 @@ jobs:
--password ${{secrets.ZOWE_JFROG_ELEVATED_KEY }} \
--url https://zowe.jfrog.io/artifactory \
${{ env.ARTIFACT_REPO }}/${{ env.ARTIFACT_PATH }}/${{ env.ARTIFACT_VERSION }}/${{ env.CLI_SBOM_ARTIFACT_NAME }}
jfrog rt del\
--user ${{ secrets.ZOWE_JFROG_ELEVATED_USER }} \
--password ${{secrets.ZOWE_JFROG_ELEVATED_KEY }} \
--url https://zowe.jfrog.io/artifactory \
${{ env.ARTIFACT_REPO }}/${{ env.ARTIFACT_PATH }}/${{ env.ARTIFACT_VERSION }}/${{ env.VSCODE_SBOM_ARTIFACT_NAME }}
jfrog rt del\
--user ${{ secrets.ZOWE_JFROG_ELEVATED_USER }} \
--password ${{secrets.ZOWE_JFROG_ELEVATED_KEY }} \
Expand All @@ -237,7 +265,8 @@ jobs:
sigstore-sign-artifacts: true
artifacts: |
${{ env.DEPENDENCY_SCAN_HOME }}/${{ env.AGG_ARTIFACT_NAME }}
${{ env.DEPENDENCY_SCAN_HOME }}/${{ env.CLI_ARTIFACT_NAME }}
${{ env.DEPENDENCY_SCAN_HOME }}/${{ env.CLI_ARTIFACT_NAME }}
${{ env.DEPENDENCY_SCAN_HOME }}/${{ env.VSCODE_ARTIFACT_NAME }}
${{ env.DEPENDENCY_SCAN_HOME }}/${{ env.ZOS_ARTIFACT_NAME }}

- name: Publish to Artifactory
Expand All @@ -252,23 +281,28 @@ jobs:
artifacts: |
${{ env.DEPENDENCY_SCAN_HOME }}/${{ env.AGG_SBOM_ARTIFACT_NAME }}
${{ env.DEPENDENCY_SCAN_HOME }}/${{ env.CLI_SBOM_ARTIFACT_NAME }}
${{ env.DEPENDENCY_SCAN_HOME }}/${{ env.VSCODE_SBOM_ARTIFACT_NAME }}
${{ env.DEPENDENCY_SCAN_HOME }}/${{ env.ZOS_SBOM_ARTIFACT_NAME }}

- name: Archive Aggregates
- name: Archive Aggregates
uses: actions/upload-artifact@v4
if: ${{ always() }}
with:
path: |
${{ env.DEPENDENCY_SCAN_HOME }}/logs.zip
${{ env.DEPENDENCY_SCAN_HOME }}/${{ env.AGG_ARTIFACT_NAME }}
${{ env.DEPENDENCY_SCAN_HOME }}/${{ env.AGG_ARTIFACT_NAME }}.bundle
${{ env.DEPENDENCY_SCAN_HOME }}/${{ env.CLI_ARTIFACT_NAME }}
${{ env.DEPENDENCY_SCAN_HOME }}/${{ env.CLI_ARTIFACT_NAME }}.bundle
${{ env.DEPENDENCY_SCAN_HOME }}/${{ env.CLI_ARTIFACT_NAME }}
${{ env.DEPENDENCY_SCAN_HOME }}/${{ env.CLI_ARTIFACT_NAME }}.bundle
${{ env.DEPENDENCY_SCAN_HOME }}/${{ env.VSCODE_ARTIFACT_NAME }}
${{ env.DEPENDENCY_SCAN_HOME }}/${{ env.VSCODE_ARTIFACT_NAME }}.bundle
${{ env.DEPENDENCY_SCAN_HOME }}/${{ env.ZOS_ARTIFACT_NAME }}
${{ env.DEPENDENCY_SCAN_HOME }}/${{ env.ZOS_ARTIFACT_NAME }}.bundle
${{ env.DEPENDENCY_SCAN_HOME }}/${{ env.AGG_SBOM_ARTIFACT_NAME }}
${{ env.DEPENDENCY_SCAN_HOME }}/${{ env.AGG_SBOM_ARTIFACT_NAME }}.bundle
${{ env.DEPENDENCY_SCAN_HOME }}/${{ env.CLI_SBOM_ARTIFACT_NAME }}
${{ env.DEPENDENCY_SCAN_HOME }}/${{ env.CLI_SBOM_ARTIFACT_NAME }}.bundle
${{ env.DEPENDENCY_SCAN_HOME }}/${{ env.VSCODE_SBOM_ARTIFACT_NAME }}
${{ env.DEPENDENCY_SCAN_HOME }}/${{ env.VSCODE_SBOM_ARTIFACT_NAME }}.bundle
${{ env.DEPENDENCY_SCAN_HOME }}/${{ env.ZOS_SBOM_ARTIFACT_NAME }}
${{ env.DEPENDENCY_SCAN_HOME }}/${{ env.ZOS_SBOM_ARTIFACT_NAME }}.bundle
7 changes: 6 additions & 1 deletion licenses/dependency-scan/resources/repoRules.json
Original file line number Diff line number Diff line change
Expand Up @@ -57,8 +57,13 @@
"pattern": "**/samples/**",
"reason": "EXAMPLE_OF",
"comment": "Sample APIs"
},
{
"pattern": "**/zedc/**",
"reason": "TEST_TOOL_OF",
"comment": "Development CLI"
}
]
}
}
}
}
8 changes: 4 additions & 4 deletions licenses/dependency-scan/src/actions/base/InstallAction.ts
Original file line number Diff line number Diff line change
Expand Up @@ -88,15 +88,15 @@ export class InstallAction implements IAction {
//default npm install prod
let installCmd = "npm";
let installArgs = ["install", "--omit=dev", ...registry];

if (Utilities.hasPnpmLockFile(`${absDir}`)) {
installCmd = "pnpm";
installArgs = ["install", "--frozen-lockfile", "--prod", ...registry]
installArgs = ["install", "--frozen-lockfile", "--prod", "--ignore-scripts", ...registry]
} else if (Utilities.hasNpmLockfile(`${absDir}`)) {
installArgs = ["ci", "--omit=dev", ...registry]
installArgs = ["ci", "--omit=dev", "--ignore-scripts", ...registry]
} else if (Utilities.hasYarnLockfile(`${absDir}`)) {
installCmd = "yarn";
installArgs = ["install", "--production", "--frozen-lockfile", "--ignore-engines", ...registry]
installArgs = ["install", "--production", "--frozen-lockfile", "--ignore-engines", "--ignore-scripts", ...registry]
}

if (fs.existsSync(path.join(absDir, "node_modules"))) {
Expand Down
22 changes: 21 additions & 1 deletion licenses/dependency-scan/src/actions/ort/OrtReportAction.ts
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@ export class OrtReportAction implements IAction {

private readonly AGG_REPORT_MARKDOWN_FILE = path.resolve(Constants.LICENSE_REPORTS_DIR, "markdown_dependency_report.md");
private readonly CLI_REPORT_MARKDOWN_FILE = path.resolve(Constants.LICENSE_REPORTS_DIR, "cli_dependency_report.md")
private readonly VSCODE_REPORT_MARKDOWN_FILE = path.resolve(Constants.LICENSE_REPORTS_DIR, "vscode_dependency_report.md")
private readonly ZOS_REPORT_MARKDOWN_FILE = path.resolve(Constants.LICENSE_REPORTS_DIR, "zos_dependency_report.md")

private reportQueue: async.AsyncQueue<any> = async.queue(this.reportProject.bind(this), Constants.PARALLEL_REPORT_COUNT);
Expand Down Expand Up @@ -91,6 +92,7 @@ export class OrtReportAction implements IAction {
const sourceDependencies: ZoweManifestSourceDependency[] = this.zoweManifest.sourceDependencies;
const aggregateNoticesFile = path.join(Constants.NOTICE_REPORTS_DIR, "notices_aggregate.txt");
const cliNoticesFile = path.join(Constants.NOTICE_REPORTS_DIR, "notices_cli.txt");
const vscodeNoticesFile = path.join(Constants.NOTICE_REPORTS_DIR, "notices_vscode.txt");
const zosNoticesFile = path.join(Constants.NOTICE_REPORTS_DIR, "notices_zos.txt");

(sourceDependencies).forEach((dependency: ZoweManifestSourceDependency) => {
Expand All @@ -106,6 +108,8 @@ export class OrtReportAction implements IAction {
fs.appendFileSync(aggregateNoticesFile, fs.readFileSync(noticesTxtFile).toString() + "\n");
if (noticeInstance.destinations.join(",").includes("CLI")) {
fs.appendFileSync(cliNoticesFile, fs.readFileSync(noticesTxtFile).toString() + "\n");
} else if (noticeInstance.destinations.join(",").includes("Visual Studio Code")) {
fs.appendFileSync(vscodeNoticesFile, fs.readFileSync(noticesTxtFile).toString() + "\n");
} else {
fs.appendFileSync(zosNoticesFile, fs.readFileSync(noticesTxtFile).toString() + "\n");
}
Expand All @@ -125,9 +129,11 @@ export class OrtReportAction implements IAction {

const aggregateReportFile = fs.createWriteStream(this.AGG_REPORT_MARKDOWN_FILE, { flags: "a" });
const cliReportFile = fs.createWriteStream(this.CLI_REPORT_MARKDOWN_FILE)
const vscodeReportFile = fs.createWriteStream(this.VSCODE_REPORT_MARKDOWN_FILE)
const zosReportFile = fs.createWriteStream(this.ZOS_REPORT_MARKDOWN_FILE)
aggregateReportFile.write("# Zowe Third Party Library Usage\n\n");
cliReportFile.write("# Zowe CLI Third Party Library Usage\n\n");
vscodeReportFile.write("# Zowe Explorer for VS Code Third Party Library Usage\n\n");
zosReportFile.write("# Zowe z/OS Third Party Library Usage\n\n");
(sourceDependencies).forEach((dependency) => {
aggregateReportFile.write("* [" + dependency.componentGroup + "](#" + dependency.componentGroup.replace(/\s/g, "-").toLowerCase()
Expand All @@ -136,15 +142,19 @@ export class OrtReportAction implements IAction {
if (dependency.entries[0].destinations.join(",").includes("CLI")) {
cliReportFile.write("* [" + dependency.componentGroup + "](#" + dependency.componentGroup.replace(/\s/g, "-").toLowerCase()
+ "-dependency-attributions)" + "\n");
} else if (dependency.entries[0].destinations.join(",").includes("Visual Studio Code")) {
vscodeReportFile.write("* [" + dependency.componentGroup + "](#" + dependency.componentGroup.replace(/\s/g, "-").toLowerCase()
+ "-dependency-attributions)" + "\n");
} else {
zosReportFile.write("* [" + dependency.componentGroup + "](#" + dependency.componentGroup.replace(/\s/g, "-").toLowerCase()
+ "-dependency-attributions)" + "\n");
}
}
});
aggregateReportFile.write("\n");
zosReportFile.write("\n");
cliReportFile.write("\n");
vscodeReportFile.write("\n");
zosReportFile.write("\n");

(sourceDependencies).forEach((dependency: ZoweManifestSourceDependency) => {
const reports: ReportInfo[] = (dependency.entries.map((depEntry): ReportInfo => {
Expand All @@ -157,10 +167,12 @@ export class OrtReportAction implements IAction {

let totalDepCt = 0;
let cliDepCt = 0;
let vscodeDepCt = 0;
let zosDepCt = 0;
let missingReport: boolean = false;
let fullReportString = "";
let cliReportString = fullReportString
let vscodeReportString = fullReportString
let zosReportString = fullReportString
reports.forEach((reportInstance: ReportInfo) => {
try {
Expand All @@ -176,6 +188,10 @@ export class OrtReportAction implements IAction {
cliDepCt += reportDepCt
cliReportString += `### ${dependency.componentGroup} Dependency Attributions\n`
cliReportString += lines.join("\n");
} else if (reportInstance.destinations.join(",").includes("Visual Studio Code")) {
vscodeDepCt += reportDepCt
vscodeReportString += `### ${dependency.componentGroup} Dependency Attributions\n`
vscodeReportString += lines.join("\n");
} else {
zosDepCt += reportDepCt
zosReportString += `### ${dependency.componentGroup} Dependency Attributions\n`
Expand All @@ -196,6 +212,10 @@ export class OrtReportAction implements IAction {
cliReportFile.write(cliReportString);
cliReportFile.write("\n\n")
}
if (vscodeDepCt > 0) {
vscodeReportFile.write(vscodeReportString);
vscodeReportFile.write("\n\n")
}
if (zosDepCt > 0) {
zosReportFile.write(zosReportString);
zosReportFile.write("\n\n")
Expand Down
Loading