From a023e0be4d9bde8924ab03aecc1c64b75ad439cb Mon Sep 17 00:00:00 2001 From: Frank Viernau Date: Mon, 25 Sep 2023 14:38:57 +0200 Subject: [PATCH 01/47] chore(SpdxDocumentModelMapper): Remove some magic values Use named arguments to make it more obvious was the `1` is about. Also, use named arguments for `id` for consistency. Signed-off-by: Frank Viernau --- .../spdx/src/main/kotlin/SpdxDocumentModelMapper.kt | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/plugins/reporters/spdx/src/main/kotlin/SpdxDocumentModelMapper.kt b/plugins/reporters/spdx/src/main/kotlin/SpdxDocumentModelMapper.kt index b01d0de2a9e86..7213ecfad326a 100644 --- a/plugins/reporters/spdx/src/main/kotlin/SpdxDocumentModelMapper.kt +++ b/plugins/reporters/spdx/src/main/kotlin/SpdxDocumentModelMapper.kt @@ -70,7 +70,10 @@ internal object SpdxDocumentModelMapper : Logging { ortResult ) - ortResult.getDependencies(project.id, 1).mapTo(relationships) { dependency -> + ortResult.getDependencies( + id = project.id, + maxLevel = 1 + ).mapTo(relationships) { dependency -> SpdxRelationship( spdxElementId = spdxProjectPackage.spdxId, relationshipType = SpdxRelationship.Type.DEPENDS_ON, @@ -91,7 +94,10 @@ internal object SpdxDocumentModelMapper : Logging { ortResult ) - ortResult.getDependencies(pkg.id, 1).mapTo(relationships) { dependency -> + ortResult.getDependencies( + id = pkg.id, + maxLevel = 1 + ).mapTo(relationships) { dependency -> SpdxRelationship( spdxElementId = binaryPackage.spdxId, relationshipType = SpdxRelationship.Type.DEPENDS_ON, From 37a0894488ea5ba9fb7884e1b44b9464b79b2e06 Mon Sep 17 00:00:00 2001 From: Frank Viernau Date: Mon, 25 Sep 2023 14:34:31 +0200 Subject: [PATCH 02/47] feat(OrtResult): Allow `getDependencies()` to omit excluded IDs This is needed in a following change. Signed-off-by: Frank Viernau --- model/src/main/kotlin/OrtResult.kt | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/model/src/main/kotlin/OrtResult.kt b/model/src/main/kotlin/OrtResult.kt index bdb482e5778f7..0e20010746b8d 100644 --- a/model/src/main/kotlin/OrtResult.kt +++ b/model/src/main/kotlin/OrtResult.kt @@ -188,17 +188,19 @@ data class OrtResult( /** * Return the dependencies of the given [id] (which can refer to a [Project] or a [Package]), up to and including a * depth of [maxLevel] where counting starts at 0 (for the [Project] or [Package] itself) and 1 are direct - * dependencies etc. A value below 0 means to not limit the depth. + * dependencies etc. A value below 0 means to not limit the depth. If [omitExcluded] is set to true, identifiers of + * excluded projects / packages are omitted from the result. */ - fun getDependencies(id: Identifier, maxLevel: Int = -1): Set { + fun getDependencies(id: Identifier, maxLevel: Int = -1, omitExcluded: Boolean = false): Set { val dependencies = mutableSetOf() + val matcher = DependencyNavigator.MATCH_ALL.takeUnless { omitExcluded } ?: { !isExcluded(it.id) } getProjects().forEach { project -> if (project.id == id) { - dependencies += dependencyNavigator.projectDependencies(project, maxLevel) + dependencies += dependencyNavigator.projectDependencies(project, maxLevel, matcher) } - dependencies += dependencyNavigator.packageDependencies(project, id, maxLevel) + dependencies += dependencyNavigator.packageDependencies(project, id, maxLevel, matcher) } return dependencies From 74ba431e301be2b42d6500584c139e388f984017 Mon Sep 17 00:00:00 2001 From: Frank Viernau Date: Tue, 12 Sep 2023 11:33:11 +0200 Subject: [PATCH 03/47] fix(reporter): Fix the creation of first level dependency relationships As of [1] the SPDX document was changed to have separate entries for all projects and sub-projects instead of a single artificial root project containing all dependencies. While excluded packages are not included in the package, the implementation [1] accidentally creates (dangling) relationships to such excluded packages, see [2]. Fix the issue visible in [2] by the code change further up and an analog issue not visible in the expected result diff with the code change some lines further down. Fixes #7487. [1] https://github.com/oss-review-toolkit/ort/commit/b47154482299b9b49c0e0c692e3f2dacdd53c395 [2] https://github.com/oss-review-toolkit/ort/commit/b47154482299b9b49c0e0c692e3f2dacdd53c395#diff-6de35dd2aff1f92b7f5ea558d3f77e02d0d596dd4ce2a8199056cfb31b47fcabR181-R184 Signed-off-by: Frank Viernau --- .../assets/spdx-document-reporter-expected-output.spdx.json | 4 ---- .../assets/spdx-document-reporter-expected-output.spdx.yml | 3 --- .../spdx/src/main/kotlin/SpdxDocumentModelMapper.kt | 6 ++++-- 3 files changed, 4 insertions(+), 9 deletions(-) diff --git a/plugins/reporters/spdx/src/funTest/assets/spdx-document-reporter-expected-output.spdx.json b/plugins/reporters/spdx/src/funTest/assets/spdx-document-reporter-expected-output.spdx.json index ba9f4876ed8ec..9ddc9622e09ef 100644 --- a/plugins/reporters/spdx/src/funTest/assets/spdx-document-reporter-expected-output.spdx.json +++ b/plugins/reporters/spdx/src/funTest/assets/spdx-document-reporter-expected-output.spdx.json @@ -240,10 +240,6 @@ "spdxElementId" : "SPDXRef-Package-Maven-seventh-package-group-seventh-package-0.0.1", "relationshipType" : "GENERATED_FROM", "relatedSpdxElement" : "SPDXRef-Package-Maven-seventh-package-group-seventh-package-0.0.1-source-artifact" - }, { - "spdxElementId" : "SPDXRef-Project-Maven-first-project-group-first-project-name-0.0.1", - "relationshipType" : "DEPENDS_ON", - "relatedSpdxElement" : "SPDXRef-Package-Maven-fifth-package-group-fifth-package-0.0.1" }, { "spdxElementId" : "SPDXRef-Project-Maven-first-project-group-first-project-name-0.0.1", "relationshipType" : "DEPENDS_ON", diff --git a/plugins/reporters/spdx/src/funTest/assets/spdx-document-reporter-expected-output.spdx.yml b/plugins/reporters/spdx/src/funTest/assets/spdx-document-reporter-expected-output.spdx.yml index e2ad5bdfedfc6..9eefa75bb61a2 100644 --- a/plugins/reporters/spdx/src/funTest/assets/spdx-document-reporter-expected-output.spdx.yml +++ b/plugins/reporters/spdx/src/funTest/assets/spdx-document-reporter-expected-output.spdx.yml @@ -241,9 +241,6 @@ relationships: - spdxElementId: "SPDXRef-Package-Maven-seventh-package-group-seventh-package-0.0.1" relationshipType: "GENERATED_FROM" relatedSpdxElement: "SPDXRef-Package-Maven-seventh-package-group-seventh-package-0.0.1-source-artifact" -- spdxElementId: "SPDXRef-Project-Maven-first-project-group-first-project-name-0.0.1" - relationshipType: "DEPENDS_ON" - relatedSpdxElement: "SPDXRef-Package-Maven-fifth-package-group-fifth-package-0.0.1" - spdxElementId: "SPDXRef-Project-Maven-first-project-group-first-project-name-0.0.1" relationshipType: "DEPENDS_ON" relatedSpdxElement: "SPDXRef-Package-Maven-first-package-group-first-package-0.0.1" diff --git a/plugins/reporters/spdx/src/main/kotlin/SpdxDocumentModelMapper.kt b/plugins/reporters/spdx/src/main/kotlin/SpdxDocumentModelMapper.kt index 7213ecfad326a..219389650419f 100644 --- a/plugins/reporters/spdx/src/main/kotlin/SpdxDocumentModelMapper.kt +++ b/plugins/reporters/spdx/src/main/kotlin/SpdxDocumentModelMapper.kt @@ -72,7 +72,8 @@ internal object SpdxDocumentModelMapper : Logging { ortResult.getDependencies( id = project.id, - maxLevel = 1 + maxLevel = 1, + omitExcluded = true ).mapTo(relationships) { dependency -> SpdxRelationship( spdxElementId = spdxProjectPackage.spdxId, @@ -96,7 +97,8 @@ internal object SpdxDocumentModelMapper : Logging { ortResult.getDependencies( id = pkg.id, - maxLevel = 1 + maxLevel = 1, + omitExcluded = true ).mapTo(relationships) { dependency -> SpdxRelationship( spdxElementId = binaryPackage.spdxId, From 416c421f1b2bd1df383586b48bcb8753a62fb24f Mon Sep 17 00:00:00 2001 From: Sebastian Schuberth Date: Tue, 26 Sep 2023 11:34:39 +0200 Subject: [PATCH 04/47] refactor(CocoaPods): Slightly generalize parsing of dependencies Make use of `parseNameAndVersion()` to align with similar code. Also add some more comments while at it. Signed-off-by: Sebastian Schuberth --- .../package-managers/cocoapods/src/main/kotlin/CocoaPods.kt | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/plugins/package-managers/cocoapods/src/main/kotlin/CocoaPods.kt b/plugins/package-managers/cocoapods/src/main/kotlin/CocoaPods.kt index 49df4c82423e8..f7ffb8ad7a92d 100644 --- a/plugins/package-managers/cocoapods/src/main/kotlin/CocoaPods.kt +++ b/plugins/package-managers/cocoapods/src/main/kotlin/CocoaPods.kt @@ -304,8 +304,11 @@ private fun parseLockfile(podfileLock: File): LockfileData { dependencies = dependenciesForName[name].orEmpty().mapTo(mutableSetOf()) { createPackageReference(it) } ) + // The "DEPENDENCIES" section lists direct dependencies, but only along with version constraints, not with their + // resolved versions, and eventually additional information about the source. val dependencies = root.get("DEPENDENCIES").mapTo(mutableSetOf()) { node -> - val name = node.textValue().substringBefore(" ") + // Discard the version (which is only a constraint in this case) and just take the name. + val (name, _) = parseNameAndVersion(node.textValue()) createPackageReference(name) } From 933c3fccd235c2ec98a5250b99eae36b45be9a0f Mon Sep 17 00:00:00 2001 From: Sebastian Schuberth Date: Tue, 26 Sep 2023 11:36:26 +0200 Subject: [PATCH 05/47] refactor(CocoaPods): Rename two variables for clarity The new names better reflect their meaning which will become more apparent in another upcoming refactoring that actually changes behavior. Signed-off-by: Sebastian Schuberth --- .../cocoapods/src/main/kotlin/CocoaPods.kt | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/plugins/package-managers/cocoapods/src/main/kotlin/CocoaPods.kt b/plugins/package-managers/cocoapods/src/main/kotlin/CocoaPods.kt index f7ffb8ad7a92d..3445ed0f8b8ff 100644 --- a/plugins/package-managers/cocoapods/src/main/kotlin/CocoaPods.kt +++ b/plugins/package-managers/cocoapods/src/main/kotlin/CocoaPods.kt @@ -248,8 +248,8 @@ private data class LockfileData( ) private fun parseLockfile(podfileLock: File): LockfileData { - val versionForName = mutableMapOf() - val dependenciesForName = mutableMapOf>() + val resolvedVersions = mutableMapOf() + val dependencyConstraints = mutableMapOf>() val root = yamlMapper.readTree(podfileLock) // The "PODS" section lists the direct dependencies and, if applicable, their direct dependencies each. That is, the @@ -262,14 +262,14 @@ private fun parseLockfile(podfileLock: File): LockfileData { } val (name, version) = parseNameAndVersion(entry) - versionForName[name] = checkNotNull(version) + resolvedVersions[name] = checkNotNull(version) val dependencies = node[entry]?.map { depNode -> val (depName, depVersion) = parseNameAndVersion(depNode.textValue()) - depName.also { if (depVersion != null) versionForName[it] = depVersion } + depName.also { if (depVersion != null) resolvedVersions[it] = depVersion } }.orEmpty() - dependenciesForName.getOrPut(name) { mutableSetOf() } += dependencies + dependencyConstraints.getOrPut(name) { mutableSetOf() } += dependencies } val externalSources = root.get("CHECKOUT OPTIONS")?.fields()?.asSequence()?.mapNotNull { @@ -280,12 +280,12 @@ private fun parseLockfile(podfileLock: File): LockfileData { // The version written to the lockfile matches the version specified in the project's ".podspec" file at the // given revision, so the same version might be used in different revisions. To still get a unique identifier, // append the revision to the version. - val versionFromPodspec = checkNotNull(versionForName[it.key]) + val versionFromPodspec = checkNotNull(resolvedVersions[it.key]) val uniqueVersion = "$versionFromPodspec-$revision" val id = Identifier("Pod", "", it.key, uniqueVersion) // Write the unique version back for correctly associating dependencies below. - versionForName[it.key] = uniqueVersion + resolvedVersions[it.key] = uniqueVersion id to Package( id = id, @@ -300,8 +300,8 @@ private fun parseLockfile(podfileLock: File): LockfileData { fun createPackageReference(name: String): PackageReference = PackageReference( - id = Identifier("Pod", "", name, versionForName.getValue(name)), - dependencies = dependenciesForName[name].orEmpty().mapTo(mutableSetOf()) { createPackageReference(it) } + id = Identifier("Pod", "", name, resolvedVersions.getValue(name)), + dependencies = dependencyConstraints[name].orEmpty().mapTo(mutableSetOf()) { createPackageReference(it) } ) // The "DEPENDENCIES" section lists direct dependencies, but only along with version constraints, not with their From 69db3b319080a13e8cef5b7c9bef9ebaa4bc66d8 Mon Sep 17 00:00:00 2001 From: Sebastian Schuberth Date: Tue, 26 Sep 2023 11:24:41 +0200 Subject: [PATCH 06/47] fix(CocoaPods): Stop taking pure version constraints as dependencies Second-level entries in the "PODS" section are just "pure" constraints which should not be added as dependencies at all unless their names also appear with resolved versions as first-level entries. This partly reverts 18f9be2 which introduced this over-reporting by unconditionally adding constraints as dependencies. However, that did not surface until the test added in 4bbd26a. Signed-off-by: Sebastian Schuberth --- .../external-sources-expected-output.yml | 29 - .../version-resolution-expected-output.yml | 2011 +++++++++++++++++ .../synthetic/version-resolution/Podfile | 158 ++ .../synthetic/version-resolution/Podfile.lock | 246 ++ .../src/funTest/kotlin/CocoaPodsFunTest.kt | 9 + .../cocoapods/src/main/kotlin/CocoaPods.kt | 43 +- 6 files changed, 2448 insertions(+), 48 deletions(-) create mode 100644 plugins/package-managers/cocoapods/src/funTest/assets/projects/synthetic/version-resolution-expected-output.yml create mode 100644 plugins/package-managers/cocoapods/src/funTest/assets/projects/synthetic/version-resolution/Podfile create mode 100644 plugins/package-managers/cocoapods/src/funTest/assets/projects/synthetic/version-resolution/Podfile.lock diff --git a/plugins/package-managers/cocoapods/src/funTest/assets/projects/synthetic/external-sources-expected-output.yml b/plugins/package-managers/cocoapods/src/funTest/assets/projects/synthetic/external-sources-expected-output.yml index 6bfaf20bc6571..1720c1b2d2547 100644 --- a/plugins/package-managers/cocoapods/src/funTest/assets/projects/synthetic/external-sources-expected-output.yml +++ b/plugins/package-managers/cocoapods/src/funTest/assets/projects/synthetic/external-sources-expected-output.yml @@ -28,7 +28,6 @@ project: - id: "Pod::AFNetworking/Reachability:3.2.1" - id: "Pod::AFNetworking/Security:3.2.1" - id: "Pod::AFNetworking/Serialization:3.2.1" - - id: "Pod::AFNetworking/UIKit:3.2.1" - id: "Pod::CocoaSecurity:1.2.4" - id: "Pod::MLHudAlert:0.0.4-3f23fa41632f8935fed028b04335576633ec53a2" - id: "Pod::Ono:2.1.1" @@ -173,34 +172,6 @@ packages: url: "https://github.com/AFNetworking/AFNetworking.git" revision: "3.2.1" path: "" -- id: "Pod::AFNetworking/UIKit:3.2.1" - purl: "pkg:cocoapods/AFNetworking%2FUIKit@3.2.1" - declared_licenses: - - "MIT" - declared_licenses_processed: - spdx_expression: "MIT" - description: "A delightful iOS and OS X networking framework." - homepage_url: "https://github.com/AFNetworking/AFNetworking" - binary_artifact: - url: "" - hash: - value: "" - algorithm: "" - source_artifact: - url: "" - hash: - value: "" - algorithm: "" - vcs: - type: "Git" - url: "https://github.com/AFNetworking/AFNetworking.git" - revision: "3.2.1" - path: "" - vcs_processed: - type: "Git" - url: "https://github.com/AFNetworking/AFNetworking.git" - revision: "3.2.1" - path: "" - id: "Pod::CocoaSecurity:1.2.4" purl: "pkg:cocoapods/CocoaSecurity@1.2.4" declared_licenses: diff --git a/plugins/package-managers/cocoapods/src/funTest/assets/projects/synthetic/version-resolution-expected-output.yml b/plugins/package-managers/cocoapods/src/funTest/assets/projects/synthetic/version-resolution-expected-output.yml new file mode 100644 index 0000000000000..e0bd17ec787f2 --- /dev/null +++ b/plugins/package-managers/cocoapods/src/funTest/assets/projects/synthetic/version-resolution-expected-output.yml @@ -0,0 +1,2011 @@ +--- +project: + id: "CocoaPods::src/funTest/assets/projects/synthetic/version-resolution/Podfile:" + definition_file_path: "plugins/package-managers/cocoapods/src/funTest/assets/projects/synthetic/version-resolution/Podfile" + declared_licenses: [] + declared_licenses_processed: {} + vcs: + type: "" + url: "" + revision: "" + path: "" + vcs_processed: + type: "Git" + url: "" + revision: "" + path: "" + homepage_url: "" + scopes: + - name: "dependencies" + dependencies: + - id: "Pod::AnalyticsEvents:0.1.0-53ad46ba1ea1ee8f21139dda3c351890846a202f" + - id: "Pod::DSBottomSheet:0.3.0" + - id: "Pod::DSWaveformImage:6.1.1" + - id: "Pod::DTCoreText:1.6.26" + dependencies: + - id: "Pod::DTCoreText/Core:1.6.26" + dependencies: + - id: "Pod::DTFoundation/Core:1.7.18" + - id: "Pod::DTFoundation/DTAnimatedGIF:1.7.18" + - id: "Pod::DTFoundation/DTHTMLParser:1.7.18" + dependencies: + - id: "Pod::DTFoundation/Core:1.7.18" + - id: "Pod::DTFoundation/UIKit:1.7.18" + dependencies: + - id: "Pod::DTFoundation/Core:1.7.18" + - id: "Pod::DTFoundation/Core:1.7.18" + - id: "Pod::DTFoundation/DTAnimatedGIF:1.7.18" + - id: "Pod::DTFoundation/DTHTMLParser:1.7.18" + dependencies: + - id: "Pod::DTFoundation/Core:1.7.18" + - id: "Pod::DTFoundation/UIKit:1.7.18" + dependencies: + - id: "Pod::DTFoundation/Core:1.7.18" + - id: "Pod::Down:0.11.0" + - id: "Pod::FLEX:4.5.0" + - id: "Pod::FlowCommoniOS:1.12.2" + - id: "Pod::GBDeviceInfo:6.6.0" + dependencies: + - id: "Pod::GBDeviceInfo/Core:6.6.0" + - id: "Pod::Introspect:0.1.4" + - id: "Pod::KTCenterFlowLayout:1.3.1" + - id: "Pod::KeychainAccess:4.2.2" + - id: "Pod::MatrixSDK:0.23.18" + dependencies: + - id: "Pod::MatrixSDK/Core:0.23.18" + dependencies: + - id: "Pod::AFNetworking:4.0.1" + dependencies: + - id: "Pod::AFNetworking/NSURLSession:4.0.1" + dependencies: + - id: "Pod::AFNetworking/Reachability:4.0.1" + - id: "Pod::AFNetworking/Security:4.0.1" + - id: "Pod::AFNetworking/Serialization:4.0.1" + - id: "Pod::AFNetworking/Reachability:4.0.1" + - id: "Pod::AFNetworking/Security:4.0.1" + - id: "Pod::AFNetworking/Serialization:4.0.1" + - id: "Pod::AFNetworking/UIKit:4.0.1" + dependencies: + - id: "Pod::AFNetworking/NSURLSession:4.0.1" + dependencies: + - id: "Pod::AFNetworking/Reachability:4.0.1" + - id: "Pod::AFNetworking/Security:4.0.1" + - id: "Pod::AFNetworking/Serialization:4.0.1" + - id: "Pod::GZIP:1.3.0" + - id: "Pod::MatrixSDK/CryptoSDK:0.23.18" + dependencies: + - id: "Pod::MatrixSDKCrypto:0.1.0" + - id: "Pod::OLMKit:3.2.12" + dependencies: + - id: "Pod::OLMKit/olmc:3.2.12" + - id: "Pod::OLMKit/olmcpp:3.2.12" + - id: "Pod::Realm:10.27.0" + dependencies: + - id: "Pod::Realm/Headers:10.27.0" + - id: "Pod::SwiftyBeaver:1.9.5" + - id: "Pod::libbase58:0.1.4" + - id: "Pod::MatrixSDK/JingleCallStack:0.23.18" + dependencies: + - id: "Pod::JitsiMeetSDK:5.0.2" + - id: "Pod::MatrixSDK/Core:0.23.18" + dependencies: + - id: "Pod::AFNetworking:4.0.1" + dependencies: + - id: "Pod::AFNetworking/NSURLSession:4.0.1" + dependencies: + - id: "Pod::AFNetworking/Reachability:4.0.1" + - id: "Pod::AFNetworking/Security:4.0.1" + - id: "Pod::AFNetworking/Serialization:4.0.1" + - id: "Pod::AFNetworking/Reachability:4.0.1" + - id: "Pod::AFNetworking/Security:4.0.1" + - id: "Pod::AFNetworking/Serialization:4.0.1" + - id: "Pod::AFNetworking/UIKit:4.0.1" + dependencies: + - id: "Pod::AFNetworking/NSURLSession:4.0.1" + dependencies: + - id: "Pod::AFNetworking/Reachability:4.0.1" + - id: "Pod::AFNetworking/Security:4.0.1" + - id: "Pod::AFNetworking/Serialization:4.0.1" + - id: "Pod::GZIP:1.3.0" + - id: "Pod::MatrixSDK/CryptoSDK:0.23.18" + dependencies: + - id: "Pod::MatrixSDKCrypto:0.1.0" + - id: "Pod::OLMKit:3.2.12" + dependencies: + - id: "Pod::OLMKit/olmc:3.2.12" + - id: "Pod::OLMKit/olmcpp:3.2.12" + - id: "Pod::Realm:10.27.0" + dependencies: + - id: "Pod::Realm/Headers:10.27.0" + - id: "Pod::SwiftyBeaver:1.9.5" + - id: "Pod::libbase58:0.1.4" + - id: "Pod::OLMKit:3.2.12" + dependencies: + - id: "Pod::OLMKit/olmc:3.2.12" + - id: "Pod::OLMKit/olmcpp:3.2.12" + - id: "Pod::PostHog:1.4.4" + - id: "Pod::ReadMoreTextView:3.0.1" + - id: "Pod::Reusable:4.1.2" + dependencies: + - id: "Pod::Reusable/Storyboard:4.1.2" + - id: "Pod::Reusable/View:4.1.2" + - id: "Pod::Sentry:7.15.0" + dependencies: + - id: "Pod::Sentry/Core:7.15.0" + - id: "Pod::SideMenu:6.5.0" + - id: "Pod::SwiftBase32:0.9.0" + - id: "Pod::SwiftGen:6.6.2" + - id: "Pod::SwiftJWT:3.6.200" + dependencies: + - id: "Pod::BlueCryptor:1.0.32" + - id: "Pod::BlueECC:1.2.5" + - id: "Pod::BlueRSA:1.0.200" + - id: "Pod::KituraContracts:1.2.1" + dependencies: + - id: "Pod::LoggerAPI:1.9.200" + dependencies: + - id: "Pod::Logging:1.4.0" + - id: "Pod::LoggerAPI:1.9.200" + dependencies: + - id: "Pod::Logging:1.4.0" + - id: "Pod::SwiftLint:0.44.0" + - id: "Pod::UICollectionViewLeftAlignedLayout:1.0.2" + - id: "Pod::UICollectionViewRightAlignedLayout:0.0.3" + - id: "Pod::WeakDictionary:2.0.2" + - id: "Pod::ZXingObjC:3.6.5" + dependencies: + - id: "Pod::ZXingObjC/All:3.6.5" + - id: "Pod::ffmpeg-kit-ios-audio:4.5.1" + - id: "Pod::libPhoneNumber-iOS:0.9.15" + - id: "Pod::zxcvbn-ios:1.0.4" +packages: +- id: "Pod::AFNetworking:4.0.1" + purl: "pkg:cocoapods/AFNetworking@4.0.1" + declared_licenses: + - "MIT" + declared_licenses_processed: + spdx_expression: "MIT" + description: "A delightful networking framework for Apple platforms." + homepage_url: "https://github.com/AFNetworking/AFNetworking" + binary_artifact: + url: "" + hash: + value: "" + algorithm: "" + source_artifact: + url: "" + hash: + value: "" + algorithm: "" + vcs: + type: "Git" + url: "https://github.com/AFNetworking/AFNetworking.git" + revision: "4.0.1" + path: "" + vcs_processed: + type: "Git" + url: "https://github.com/AFNetworking/AFNetworking.git" + revision: "4.0.1" + path: "" +- id: "Pod::AFNetworking/NSURLSession:4.0.1" + purl: "pkg:cocoapods/AFNetworking%2FNSURLSession@4.0.1" + declared_licenses: + - "MIT" + declared_licenses_processed: + spdx_expression: "MIT" + description: "A delightful networking framework for Apple platforms." + homepage_url: "https://github.com/AFNetworking/AFNetworking" + binary_artifact: + url: "" + hash: + value: "" + algorithm: "" + source_artifact: + url: "" + hash: + value: "" + algorithm: "" + vcs: + type: "Git" + url: "https://github.com/AFNetworking/AFNetworking.git" + revision: "4.0.1" + path: "" + vcs_processed: + type: "Git" + url: "https://github.com/AFNetworking/AFNetworking.git" + revision: "4.0.1" + path: "" +- id: "Pod::AFNetworking/Reachability:4.0.1" + purl: "pkg:cocoapods/AFNetworking%2FReachability@4.0.1" + declared_licenses: + - "MIT" + declared_licenses_processed: + spdx_expression: "MIT" + description: "A delightful networking framework for Apple platforms." + homepage_url: "https://github.com/AFNetworking/AFNetworking" + binary_artifact: + url: "" + hash: + value: "" + algorithm: "" + source_artifact: + url: "" + hash: + value: "" + algorithm: "" + vcs: + type: "Git" + url: "https://github.com/AFNetworking/AFNetworking.git" + revision: "4.0.1" + path: "" + vcs_processed: + type: "Git" + url: "https://github.com/AFNetworking/AFNetworking.git" + revision: "4.0.1" + path: "" +- id: "Pod::AFNetworking/Security:4.0.1" + purl: "pkg:cocoapods/AFNetworking%2FSecurity@4.0.1" + declared_licenses: + - "MIT" + declared_licenses_processed: + spdx_expression: "MIT" + description: "A delightful networking framework for Apple platforms." + homepage_url: "https://github.com/AFNetworking/AFNetworking" + binary_artifact: + url: "" + hash: + value: "" + algorithm: "" + source_artifact: + url: "" + hash: + value: "" + algorithm: "" + vcs: + type: "Git" + url: "https://github.com/AFNetworking/AFNetworking.git" + revision: "4.0.1" + path: "" + vcs_processed: + type: "Git" + url: "https://github.com/AFNetworking/AFNetworking.git" + revision: "4.0.1" + path: "" +- id: "Pod::AFNetworking/Serialization:4.0.1" + purl: "pkg:cocoapods/AFNetworking%2FSerialization@4.0.1" + declared_licenses: + - "MIT" + declared_licenses_processed: + spdx_expression: "MIT" + description: "A delightful networking framework for Apple platforms." + homepage_url: "https://github.com/AFNetworking/AFNetworking" + binary_artifact: + url: "" + hash: + value: "" + algorithm: "" + source_artifact: + url: "" + hash: + value: "" + algorithm: "" + vcs: + type: "Git" + url: "https://github.com/AFNetworking/AFNetworking.git" + revision: "4.0.1" + path: "" + vcs_processed: + type: "Git" + url: "https://github.com/AFNetworking/AFNetworking.git" + revision: "4.0.1" + path: "" +- id: "Pod::AFNetworking/UIKit:4.0.1" + purl: "pkg:cocoapods/AFNetworking%2FUIKit@4.0.1" + declared_licenses: + - "MIT" + declared_licenses_processed: + spdx_expression: "MIT" + description: "A delightful networking framework for Apple platforms." + homepage_url: "https://github.com/AFNetworking/AFNetworking" + binary_artifact: + url: "" + hash: + value: "" + algorithm: "" + source_artifact: + url: "" + hash: + value: "" + algorithm: "" + vcs: + type: "Git" + url: "https://github.com/AFNetworking/AFNetworking.git" + revision: "4.0.1" + path: "" + vcs_processed: + type: "Git" + url: "https://github.com/AFNetworking/AFNetworking.git" + revision: "4.0.1" + path: "" +- id: "Pod::AnalyticsEvents:0.1.0-53ad46ba1ea1ee8f21139dda3c351890846a202f" + purl: "pkg:cocoapods/AnalyticsEvents@0.1.0-53ad46ba1ea1ee8f21139dda3c351890846a202f" + declared_licenses: [] + declared_licenses_processed: {} + description: "" + homepage_url: "https://github.com/matrix-org/matrix-analytics-events.git" + binary_artifact: + url: "" + hash: + value: "" + algorithm: "" + source_artifact: + url: "" + hash: + value: "" + algorithm: "" + vcs: + type: "Git" + url: "https://github.com/matrix-org/matrix-analytics-events.git" + revision: "53ad46ba1ea1ee8f21139dda3c351890846a202f" + path: "" + vcs_processed: + type: "Git" + url: "https://github.com/matrix-org/matrix-analytics-events.git" + revision: "53ad46ba1ea1ee8f21139dda3c351890846a202f" + path: "" +- id: "Pod::BlueCryptor:1.0.32" + purl: "pkg:cocoapods/BlueCryptor@1.0.32" + declared_licenses: + - "Apache License, Version 2.0" + declared_licenses_processed: + spdx_expression: "Apache-2.0" + mapped: + Apache License, Version 2.0: "Apache-2.0" + description: "Swift cross-platform crypto library using CommonCrypto/libcrypto via\ + \ Package Manager." + homepage_url: "https://github.com/IBM-Swift/BlueCryptor" + binary_artifact: + url: "" + hash: + value: "" + algorithm: "" + source_artifact: + url: "" + hash: + value: "" + algorithm: "" + vcs: + type: "Git" + url: "https://github.com/IBM-Swift/BlueCryptor.git" + revision: "1.0.32" + path: "" + vcs_processed: + type: "Git" + url: "https://github.com/IBM-Swift/BlueCryptor.git" + revision: "1.0.32" + path: "" +- id: "Pod::BlueECC:1.2.5" + purl: "pkg:cocoapods/BlueECC@1.2.5" + declared_licenses: + - "Apache License, Version 2.0" + declared_licenses_processed: + spdx_expression: "Apache-2.0" + mapped: + Apache License, Version 2.0: "Apache-2.0" + description: "Swift cross-platform ECC crypto library using CommonCrypto/libcrypto\ + \ via Package Manager." + homepage_url: "https://github.com/IBM-Swift/BlueECC" + binary_artifact: + url: "" + hash: + value: "" + algorithm: "" + source_artifact: + url: "" + hash: + value: "" + algorithm: "" + vcs: + type: "Git" + url: "https://github.com/IBM-Swift/BlueECC.git" + revision: "1.2.5" + path: "" + vcs_processed: + type: "Git" + url: "https://github.com/IBM-Swift/BlueECC.git" + revision: "1.2.5" + path: "" +- id: "Pod::BlueRSA:1.0.200" + purl: "pkg:cocoapods/BlueRSA@1.0.200" + declared_licenses: + - "Apache License, Version 2.0" + declared_licenses_processed: + spdx_expression: "Apache-2.0" + mapped: + Apache License, Version 2.0: "Apache-2.0" + description: "Swift cross-platform RSA crypto library using CommonCrypto/libcrypto\ + \ via Package Manager." + homepage_url: "https://github.com/Kitura/BlueRSA" + binary_artifact: + url: "" + hash: + value: "" + algorithm: "" + source_artifact: + url: "" + hash: + value: "" + algorithm: "" + vcs: + type: "Git" + url: "https://github.com/Kitura/BlueRSA.git" + revision: "1.0.200" + path: "" + vcs_processed: + type: "Git" + url: "https://github.com/Kitura/BlueRSA.git" + revision: "1.0.200" + path: "" +- id: "Pod::DSBottomSheet:0.3.0" + purl: "pkg:cocoapods/DSBottomSheet@0.3.0" + declared_licenses: + - "MIT" + declared_licenses_processed: + spdx_expression: "MIT" + description: "DSBottomSheet makes it easy to add custom bottom sheets to your SwiftUI\ + \ apps." + homepage_url: "https://github.com/danielsaidi" + binary_artifact: + url: "" + hash: + value: "" + algorithm: "" + source_artifact: + url: "" + hash: + value: "" + algorithm: "" + vcs: + type: "Git" + url: "https://github.com/danielsaidi/BottomSheet.git" + revision: "0.3.0" + path: "" + vcs_processed: + type: "Git" + url: "https://github.com/danielsaidi/BottomSheet.git" + revision: "0.3.0" + path: "" +- id: "Pod::DSWaveformImage:6.1.1" + purl: "pkg:cocoapods/DSWaveformImage@6.1.1" + declared_licenses: + - "MIT" + declared_licenses_processed: + spdx_expression: "MIT" + description: "generate waveform images from audio files in iOS" + homepage_url: "https://github.com/dmrschmidt/DSWaveformImage" + binary_artifact: + url: "" + hash: + value: "" + algorithm: "" + source_artifact: + url: "" + hash: + value: "" + algorithm: "" + vcs: + type: "Git" + url: "https://github.com/dmrschmidt/DSWaveformImage.git" + revision: "6.1.1" + path: "" + vcs_processed: + type: "Git" + url: "https://github.com/dmrschmidt/DSWaveformImage.git" + revision: "6.1.1" + path: "" +- id: "Pod::DTCoreText:1.6.26" + purl: "pkg:cocoapods/DTCoreText@1.6.26" + declared_licenses: + - "BSD" + declared_licenses_processed: + spdx_expression: "BSD-3-Clause" + mapped: + BSD: "BSD-3-Clause" + description: "Methods to allow using HTML code with CoreText." + homepage_url: "https://github.com/Cocoanetics/DTCoreText" + binary_artifact: + url: "" + hash: + value: "" + algorithm: "" + source_artifact: + url: "" + hash: + value: "" + algorithm: "" + vcs: + type: "Git" + url: "https://github.com/Cocoanetics/DTCoreText.git" + revision: "1.6.26" + path: "" + vcs_processed: + type: "Git" + url: "https://github.com/Cocoanetics/DTCoreText.git" + revision: "1.6.26" + path: "" +- id: "Pod::DTCoreText/Core:1.6.26" + purl: "pkg:cocoapods/DTCoreText%2FCore@1.6.26" + declared_licenses: + - "BSD" + declared_licenses_processed: + spdx_expression: "BSD-3-Clause" + mapped: + BSD: "BSD-3-Clause" + description: "Methods to allow using HTML code with CoreText." + homepage_url: "https://github.com/Cocoanetics/DTCoreText" + binary_artifact: + url: "" + hash: + value: "" + algorithm: "" + source_artifact: + url: "" + hash: + value: "" + algorithm: "" + vcs: + type: "Git" + url: "https://github.com/Cocoanetics/DTCoreText.git" + revision: "1.6.26" + path: "" + vcs_processed: + type: "Git" + url: "https://github.com/Cocoanetics/DTCoreText.git" + revision: "1.6.26" + path: "" +- id: "Pod::DTFoundation/Core:1.7.18" + purl: "pkg:cocoapods/DTFoundation%2FCore@1.7.18" + declared_licenses: + - "BSD" + declared_licenses_processed: + spdx_expression: "BSD-3-Clause" + mapped: + BSD: "BSD-3-Clause" + description: "Standard toolset classes and categories." + homepage_url: "https://github.com/Cocoanetics/DTFoundation" + binary_artifact: + url: "" + hash: + value: "" + algorithm: "" + source_artifact: + url: "" + hash: + value: "" + algorithm: "" + vcs: + type: "Git" + url: "https://github.com/Cocoanetics/DTFoundation.git" + revision: "1.7.18" + path: "" + vcs_processed: + type: "Git" + url: "https://github.com/Cocoanetics/DTFoundation.git" + revision: "1.7.18" + path: "" +- id: "Pod::DTFoundation/DTAnimatedGIF:1.7.18" + purl: "pkg:cocoapods/DTFoundation%2FDTAnimatedGIF@1.7.18" + declared_licenses: + - "BSD" + declared_licenses_processed: + spdx_expression: "BSD-3-Clause" + mapped: + BSD: "BSD-3-Clause" + description: "Standard toolset classes and categories." + homepage_url: "https://github.com/Cocoanetics/DTFoundation" + binary_artifact: + url: "" + hash: + value: "" + algorithm: "" + source_artifact: + url: "" + hash: + value: "" + algorithm: "" + vcs: + type: "Git" + url: "https://github.com/Cocoanetics/DTFoundation.git" + revision: "1.7.18" + path: "" + vcs_processed: + type: "Git" + url: "https://github.com/Cocoanetics/DTFoundation.git" + revision: "1.7.18" + path: "" +- id: "Pod::DTFoundation/DTHTMLParser:1.7.18" + purl: "pkg:cocoapods/DTFoundation%2FDTHTMLParser@1.7.18" + declared_licenses: + - "BSD" + declared_licenses_processed: + spdx_expression: "BSD-3-Clause" + mapped: + BSD: "BSD-3-Clause" + description: "Standard toolset classes and categories." + homepage_url: "https://github.com/Cocoanetics/DTFoundation" + binary_artifact: + url: "" + hash: + value: "" + algorithm: "" + source_artifact: + url: "" + hash: + value: "" + algorithm: "" + vcs: + type: "Git" + url: "https://github.com/Cocoanetics/DTFoundation.git" + revision: "1.7.18" + path: "" + vcs_processed: + type: "Git" + url: "https://github.com/Cocoanetics/DTFoundation.git" + revision: "1.7.18" + path: "" +- id: "Pod::DTFoundation/UIKit:1.7.18" + purl: "pkg:cocoapods/DTFoundation%2FUIKit@1.7.18" + declared_licenses: + - "BSD" + declared_licenses_processed: + spdx_expression: "BSD-3-Clause" + mapped: + BSD: "BSD-3-Clause" + description: "Standard toolset classes and categories." + homepage_url: "https://github.com/Cocoanetics/DTFoundation" + binary_artifact: + url: "" + hash: + value: "" + algorithm: "" + source_artifact: + url: "" + hash: + value: "" + algorithm: "" + vcs: + type: "Git" + url: "https://github.com/Cocoanetics/DTFoundation.git" + revision: "1.7.18" + path: "" + vcs_processed: + type: "Git" + url: "https://github.com/Cocoanetics/DTFoundation.git" + revision: "1.7.18" + path: "" +- id: "Pod::Down:0.11.0" + purl: "pkg:cocoapods/Down@0.11.0" + declared_licenses: + - "MIT" + declared_licenses_processed: + spdx_expression: "MIT" + description: "Blazing fast Markdown rendering in Swift, built upon cmark." + homepage_url: "https://github.com/johnxnguyen/Down" + binary_artifact: + url: "" + hash: + value: "" + algorithm: "" + source_artifact: + url: "" + hash: + value: "" + algorithm: "" + vcs: + type: "Git" + url: "https://github.com/johnxnguyen/Down.git" + revision: "v0.11.0" + path: "" + vcs_processed: + type: "Git" + url: "https://github.com/johnxnguyen/Down.git" + revision: "v0.11.0" + path: "" +- id: "Pod::FLEX:4.5.0" + purl: "pkg:cocoapods/FLEX@4.5.0" + declared_licenses: + - "BSD" + declared_licenses_processed: + spdx_expression: "BSD-3-Clause" + mapped: + BSD: "BSD-3-Clause" + description: "A set of in-app debugging and exploration tools for iOS" + homepage_url: "https://github.com/Flipboard/FLEX" + binary_artifact: + url: "" + hash: + value: "" + algorithm: "" + source_artifact: + url: "" + hash: + value: "" + algorithm: "" + vcs: + type: "Git" + url: "https://github.com/Flipboard/FLEX.git" + revision: "4.5.0" + path: "" + vcs_processed: + type: "Git" + url: "https://github.com/Flipboard/FLEX.git" + revision: "4.5.0" + path: "" +- id: "Pod::FlowCommoniOS:1.12.2" + purl: "pkg:cocoapods/FlowCommoniOS@1.12.2" + declared_licenses: + - "MIT" + declared_licenses_processed: + spdx_expression: "MIT" + description: "Common files required for running any iOS project that uses Flow timelines." + homepage_url: "https://github.com/createwithflow/FlowCommoniOS.git" + binary_artifact: + url: "" + hash: + value: "" + algorithm: "" + source_artifact: + url: "" + hash: + value: "" + algorithm: "" + vcs: + type: "Git" + url: "https://github.com/createwithflow/FlowCommoniOS.git" + revision: "1.12.2" + path: "" + vcs_processed: + type: "Git" + url: "https://github.com/createwithflow/FlowCommoniOS.git" + revision: "1.12.2" + path: "" +- id: "Pod::GBDeviceInfo:6.6.0" + purl: "pkg:cocoapods/GBDeviceInfo@6.6.0" + declared_licenses: + - "Apache License, Version 2.0" + declared_licenses_processed: + spdx_expression: "Apache-2.0" + mapped: + Apache License, Version 2.0: "Apache-2.0" + description: "Detects the hardware, software and display of the current iOS or Mac\ + \ OS X device at runtime." + homepage_url: "https://github.com/lmirosevic/GBDeviceInfo" + binary_artifact: + url: "" + hash: + value: "" + algorithm: "" + source_artifact: + url: "" + hash: + value: "" + algorithm: "" + vcs: + type: "Git" + url: "https://github.com/lmirosevic/GBDeviceInfo.git" + revision: "6.6.0" + path: "" + vcs_processed: + type: "Git" + url: "https://github.com/lmirosevic/GBDeviceInfo.git" + revision: "6.6.0" + path: "" +- id: "Pod::GBDeviceInfo/Core:6.6.0" + purl: "pkg:cocoapods/GBDeviceInfo%2FCore@6.6.0" + declared_licenses: + - "Apache License, Version 2.0" + declared_licenses_processed: + spdx_expression: "Apache-2.0" + mapped: + Apache License, Version 2.0: "Apache-2.0" + description: "Detects the hardware, software and display of the current iOS or Mac\ + \ OS X device at runtime." + homepage_url: "https://github.com/lmirosevic/GBDeviceInfo" + binary_artifact: + url: "" + hash: + value: "" + algorithm: "" + source_artifact: + url: "" + hash: + value: "" + algorithm: "" + vcs: + type: "Git" + url: "https://github.com/lmirosevic/GBDeviceInfo.git" + revision: "6.6.0" + path: "" + vcs_processed: + type: "Git" + url: "https://github.com/lmirosevic/GBDeviceInfo.git" + revision: "6.6.0" + path: "" +- id: "Pod::GZIP:1.3.0" + purl: "pkg:cocoapods/GZIP@1.3.0" + declared_licenses: + - "zlib" + declared_licenses_processed: + spdx_expression: "Zlib" + mapped: + zlib: "Zlib" + description: "GZIP is category on NSData that provides simple gzip compression and\ + \ decompression functionality." + homepage_url: "https://github.com/nicklockwood/GZIP" + binary_artifact: + url: "" + hash: + value: "" + algorithm: "" + source_artifact: + url: "" + hash: + value: "" + algorithm: "" + vcs: + type: "Git" + url: "https://github.com/nicklockwood/GZIP.git" + revision: "1.3.0" + path: "" + vcs_processed: + type: "Git" + url: "https://github.com/nicklockwood/GZIP.git" + revision: "1.3.0" + path: "" +- id: "Pod::Introspect:0.1.4" + purl: "pkg:cocoapods/Introspect@0.1.4" + declared_licenses: + - "MIT" + declared_licenses_processed: + spdx_expression: "MIT" + description: "Introspect the underlying UIKit element of a SwiftUI view." + homepage_url: "https://github.com/siteline/SwiftUI-Introspect.git" + binary_artifact: + url: "" + hash: + value: "" + algorithm: "" + source_artifact: + url: "" + hash: + value: "" + algorithm: "" + vcs: + type: "Git" + url: "https://github.com/siteline/SwiftUI-Introspect.git" + revision: "0.1.4" + path: "" + vcs_processed: + type: "Git" + url: "https://github.com/siteline/SwiftUI-Introspect.git" + revision: "0.1.4" + path: "" +- id: "Pod::JitsiMeetSDK:5.0.2" + purl: "pkg:cocoapods/JitsiMeetSDK@5.0.2" + declared_licenses: + - "Apache 2" + declared_licenses_processed: + spdx_expression: "Apache-2.0" + mapped: + Apache 2: "Apache-2.0" + description: "Jitsi Meet iOS SDK" + homepage_url: "https://github.com/jitsi/jitsi-meet-ios-sdk-releases" + binary_artifact: + url: "" + hash: + value: "" + algorithm: "" + source_artifact: + url: "" + hash: + value: "" + algorithm: "" + vcs: + type: "Git" + url: "https://github.com/jitsi/jitsi-meet-ios-sdk-releases.git" + revision: "5.0.2" + path: "" + vcs_processed: + type: "Git" + url: "https://github.com/jitsi/jitsi-meet-ios-sdk-releases.git" + revision: "5.0.2" + path: "" +- id: "Pod::KTCenterFlowLayout:1.3.1" + purl: "pkg:cocoapods/KTCenterFlowLayout@1.3.1" + declared_licenses: + - "MIT" + declared_licenses_processed: + spdx_expression: "MIT" + description: "Aligns cells to the center of a collection view." + homepage_url: "https://github.com/keighl/KTCenterFlowLayout" + binary_artifact: + url: "" + hash: + value: "" + algorithm: "" + source_artifact: + url: "" + hash: + value: "" + algorithm: "" + vcs: + type: "Git" + url: "https://github.com/keighl/KTCenterFlowLayout.git" + revision: "1.3.1" + path: "" + vcs_processed: + type: "Git" + url: "https://github.com/keighl/KTCenterFlowLayout.git" + revision: "1.3.1" + path: "" +- id: "Pod::KeychainAccess:4.2.2" + purl: "pkg:cocoapods/KeychainAccess@4.2.2" + declared_licenses: + - "MIT" + declared_licenses_processed: + spdx_expression: "MIT" + description: "KeychainAccess is a simple Swift wrapper for Keychain that works on\ + \ iOS and OS X." + homepage_url: "https://github.com/kishikawakatsumi/KeychainAccess" + binary_artifact: + url: "" + hash: + value: "" + algorithm: "" + source_artifact: + url: "" + hash: + value: "" + algorithm: "" + vcs: + type: "Git" + url: "https://github.com/kishikawakatsumi/KeychainAccess.git" + revision: "v4.2.2" + path: "" + vcs_processed: + type: "Git" + url: "https://github.com/kishikawakatsumi/KeychainAccess.git" + revision: "v4.2.2" + path: "" +- id: "Pod::KituraContracts:1.2.1" + purl: "pkg:cocoapods/KituraContracts@1.2.1" + declared_licenses: + - "Apache License, Version 2.0" + declared_licenses_processed: + spdx_expression: "Apache-2.0" + mapped: + Apache License, Version 2.0: "Apache-2.0" + description: "KituraContracts is a library containing type definitions shared by\ + \ client and server code." + homepage_url: "https://github.com/IBM-Swift/KituraContracts" + binary_artifact: + url: "" + hash: + value: "" + algorithm: "" + source_artifact: + url: "" + hash: + value: "" + algorithm: "" + vcs: + type: "Git" + url: "https://github.com/IBM-Swift/KituraContracts.git" + revision: "1.2.1" + path: "" + vcs_processed: + type: "Git" + url: "https://github.com/IBM-Swift/KituraContracts.git" + revision: "1.2.1" + path: "" +- id: "Pod::LoggerAPI:1.9.200" + purl: "pkg:cocoapods/LoggerAPI@1.9.200" + declared_licenses: + - "Apache License, Version 2.0" + declared_licenses_processed: + spdx_expression: "Apache-2.0" + mapped: + Apache License, Version 2.0: "Apache-2.0" + description: "A logger protocol that provides a common logging interface for different\ + \ kinds of loggers." + homepage_url: "https://github.com/Kitura/LoggerAPI" + binary_artifact: + url: "" + hash: + value: "" + algorithm: "" + source_artifact: + url: "" + hash: + value: "" + algorithm: "" + vcs: + type: "Git" + url: "https://github.com/Kitura/LoggerAPI.git" + revision: "1.9.200" + path: "" + vcs_processed: + type: "Git" + url: "https://github.com/Kitura/LoggerAPI.git" + revision: "1.9.200" + path: "" +- id: "Pod::Logging:1.4.0" + purl: "pkg:cocoapods/Logging@1.4.0" + declared_licenses: + - "Apache 2.0" + declared_licenses_processed: + spdx_expression: "Apache-2.0" + mapped: + Apache 2.0: "Apache-2.0" + description: "A Logging API for Swift." + homepage_url: "https://github.com/apple/swift-log" + binary_artifact: + url: "" + hash: + value: "" + algorithm: "" + source_artifact: + url: "" + hash: + value: "" + algorithm: "" + vcs: + type: "Git" + url: "https://github.com/apple/swift-log.git" + revision: "1.4.0" + path: "" + vcs_processed: + type: "Git" + url: "https://github.com/apple/swift-log.git" + revision: "1.4.0" + path: "" +- id: "Pod::MatrixSDK:0.23.18" + purl: "pkg:cocoapods/MatrixSDK@0.23.18" + declared_licenses: + - "Apache License, Version 2.0" + declared_licenses_processed: + spdx_expression: "Apache-2.0" + mapped: + Apache License, Version 2.0: "Apache-2.0" + description: "The iOS SDK to build apps compatible with Matrix (https://www.matrix.org)" + homepage_url: "https://www.matrix.org" + binary_artifact: + url: "" + hash: + value: "" + algorithm: "" + source_artifact: + url: "" + hash: + value: "" + algorithm: "" + vcs: + type: "Git" + url: "https://github.com/matrix-org/matrix-ios-sdk.git" + revision: "v0.23.18" + path: "" + vcs_processed: + type: "Git" + url: "https://github.com/matrix-org/matrix-ios-sdk.git" + revision: "v0.23.18" + path: "" +- id: "Pod::MatrixSDK/Core:0.23.18" + purl: "pkg:cocoapods/MatrixSDK%2FCore@0.23.18" + declared_licenses: + - "Apache License, Version 2.0" + declared_licenses_processed: + spdx_expression: "Apache-2.0" + mapped: + Apache License, Version 2.0: "Apache-2.0" + description: "The iOS SDK to build apps compatible with Matrix (https://www.matrix.org)" + homepage_url: "https://www.matrix.org" + binary_artifact: + url: "" + hash: + value: "" + algorithm: "" + source_artifact: + url: "" + hash: + value: "" + algorithm: "" + vcs: + type: "Git" + url: "https://github.com/matrix-org/matrix-ios-sdk.git" + revision: "v0.23.18" + path: "" + vcs_processed: + type: "Git" + url: "https://github.com/matrix-org/matrix-ios-sdk.git" + revision: "v0.23.18" + path: "" +- id: "Pod::MatrixSDK/CryptoSDK:0.23.18" + purl: "pkg:cocoapods/MatrixSDK%2FCryptoSDK@0.23.18" + declared_licenses: + - "Apache License, Version 2.0" + declared_licenses_processed: + spdx_expression: "Apache-2.0" + mapped: + Apache License, Version 2.0: "Apache-2.0" + description: "The iOS SDK to build apps compatible with Matrix (https://www.matrix.org)" + homepage_url: "https://www.matrix.org" + binary_artifact: + url: "" + hash: + value: "" + algorithm: "" + source_artifact: + url: "" + hash: + value: "" + algorithm: "" + vcs: + type: "Git" + url: "https://github.com/matrix-org/matrix-ios-sdk.git" + revision: "v0.23.18" + path: "" + vcs_processed: + type: "Git" + url: "https://github.com/matrix-org/matrix-ios-sdk.git" + revision: "v0.23.18" + path: "" +- id: "Pod::MatrixSDK/JingleCallStack:0.23.18" + purl: "pkg:cocoapods/MatrixSDK%2FJingleCallStack@0.23.18" + declared_licenses: + - "Apache License, Version 2.0" + declared_licenses_processed: + spdx_expression: "Apache-2.0" + mapped: + Apache License, Version 2.0: "Apache-2.0" + description: "The iOS SDK to build apps compatible with Matrix (https://www.matrix.org)" + homepage_url: "https://www.matrix.org" + binary_artifact: + url: "" + hash: + value: "" + algorithm: "" + source_artifact: + url: "" + hash: + value: "" + algorithm: "" + vcs: + type: "Git" + url: "https://github.com/matrix-org/matrix-ios-sdk.git" + revision: "v0.23.18" + path: "" + vcs_processed: + type: "Git" + url: "https://github.com/matrix-org/matrix-ios-sdk.git" + revision: "v0.23.18" + path: "" +- id: "Pod::MatrixSDKCrypto:0.1.0" + purl: "pkg:cocoapods/MatrixSDKCrypto@0.1.0" + declared_licenses: + - "Apache License, Version 2.0" + declared_licenses_processed: + spdx_expression: "Apache-2.0" + mapped: + Apache License, Version 2.0: "Apache-2.0" + description: "Uniffi based bindings for the Rust SDK crypto crate." + homepage_url: "https://github.com/matrix-org/matrix-rust-sdk" + binary_artifact: + url: "" + hash: + value: "" + algorithm: "" + source_artifact: + url: "https://github.com/matrix-org/matrix-rust-sdk/releases/download/matrix-sdk-crypto-ffi-0.1.0/MatrixSDKCryptoFFI.zip" + hash: + value: "" + algorithm: "" + vcs: + type: "" + url: "" + revision: "" + path: "" + vcs_processed: + type: "Git" + url: "https://github.com/matrix-org/matrix-rust-sdk.git" + revision: "" + path: "" +- id: "Pod::OLMKit:3.2.12" + purl: "pkg:cocoapods/OLMKit@3.2.12" + declared_licenses: + - "Apache License, Version 2.0" + declared_licenses_processed: + spdx_expression: "Apache-2.0" + mapped: + Apache License, Version 2.0: "Apache-2.0" + description: "An Objective-C wrapper of olm (http://matrix.org/git/olm)" + homepage_url: "https://gitlab.matrix.org/matrix-org/olm" + binary_artifact: + url: "" + hash: + value: "" + algorithm: "" + source_artifact: + url: "" + hash: + value: "" + algorithm: "" + vcs: + type: "Git" + url: "https://gitlab.matrix.org/matrix-org/olm.git" + revision: "3.2.12" + path: "" + vcs_processed: + type: "Git" + url: "https://gitlab.matrix.org/matrix-org/olm.git" + revision: "3.2.12" + path: "" +- id: "Pod::OLMKit/olmc:3.2.12" + purl: "pkg:cocoapods/OLMKit%2Folmc@3.2.12" + declared_licenses: + - "Apache License, Version 2.0" + declared_licenses_processed: + spdx_expression: "Apache-2.0" + mapped: + Apache License, Version 2.0: "Apache-2.0" + description: "An Objective-C wrapper of olm (http://matrix.org/git/olm)" + homepage_url: "https://gitlab.matrix.org/matrix-org/olm" + binary_artifact: + url: "" + hash: + value: "" + algorithm: "" + source_artifact: + url: "" + hash: + value: "" + algorithm: "" + vcs: + type: "Git" + url: "https://gitlab.matrix.org/matrix-org/olm.git" + revision: "3.2.12" + path: "" + vcs_processed: + type: "Git" + url: "https://gitlab.matrix.org/matrix-org/olm.git" + revision: "3.2.12" + path: "" +- id: "Pod::OLMKit/olmcpp:3.2.12" + purl: "pkg:cocoapods/OLMKit%2Folmcpp@3.2.12" + declared_licenses: + - "Apache License, Version 2.0" + declared_licenses_processed: + spdx_expression: "Apache-2.0" + mapped: + Apache License, Version 2.0: "Apache-2.0" + description: "An Objective-C wrapper of olm (http://matrix.org/git/olm)" + homepage_url: "https://gitlab.matrix.org/matrix-org/olm" + binary_artifact: + url: "" + hash: + value: "" + algorithm: "" + source_artifact: + url: "" + hash: + value: "" + algorithm: "" + vcs: + type: "Git" + url: "https://gitlab.matrix.org/matrix-org/olm.git" + revision: "3.2.12" + path: "" + vcs_processed: + type: "Git" + url: "https://gitlab.matrix.org/matrix-org/olm.git" + revision: "3.2.12" + path: "" +- id: "Pod::PostHog:1.4.4" + purl: "pkg:cocoapods/PostHog@1.4.4" + declared_licenses: + - "MIT" + declared_licenses_processed: + spdx_expression: "MIT" + description: "The hassle-free way to add posthog to your iOS app." + homepage_url: "http://posthog.com/" + binary_artifact: + url: "" + hash: + value: "" + algorithm: "" + source_artifact: + url: "" + hash: + value: "" + algorithm: "" + vcs: + type: "Git" + url: "https://github.com/PostHog/posthog-ios.git" + revision: "1.4.4" + path: "" + vcs_processed: + type: "Git" + url: "https://github.com/PostHog/posthog-ios.git" + revision: "1.4.4" + path: "" +- id: "Pod::ReadMoreTextView:3.0.1" + purl: "pkg:cocoapods/ReadMoreTextView@3.0.1" + declared_licenses: + - "MIT" + declared_licenses_processed: + spdx_expression: "MIT" + description: "UITextView subclass with \"read more\"/\"read less\" capabilities\ + \ and UITextView extensions to handle touches in characters range." + homepage_url: "http://ilya.puchka.me/custom-uitextview-in-swift/" + binary_artifact: + url: "" + hash: + value: "" + algorithm: "" + source_artifact: + url: "" + hash: + value: "" + algorithm: "" + vcs: + type: "Git" + url: "https://github.com/ilyapuchka/ReadMoreTextView.git" + revision: "3.0.1" + path: "" + vcs_processed: + type: "Git" + url: "https://github.com/ilyapuchka/ReadMoreTextView.git" + revision: "3.0.1" + path: "" +- id: "Pod::Realm:10.27.0" + purl: "pkg:cocoapods/Realm@10.27.0" + declared_licenses: + - "Apache 2.0" + declared_licenses_processed: + spdx_expression: "Apache-2.0" + mapped: + Apache 2.0: "Apache-2.0" + description: "Realm is a modern data framework & database for iOS, macOS, tvOS &\ + \ watchOS." + homepage_url: "https://realm.io" + binary_artifact: + url: "" + hash: + value: "" + algorithm: "" + source_artifact: + url: "" + hash: + value: "" + algorithm: "" + vcs: + type: "Git" + url: "https://github.com/realm/realm-swift.git" + revision: "v10.27.0" + path: "" + vcs_processed: + type: "Git" + url: "https://github.com/realm/realm-swift.git" + revision: "v10.27.0" + path: "" +- id: "Pod::Realm/Headers:10.27.0" + purl: "pkg:cocoapods/Realm%2FHeaders@10.27.0" + declared_licenses: + - "Apache 2.0" + declared_licenses_processed: + spdx_expression: "Apache-2.0" + mapped: + Apache 2.0: "Apache-2.0" + description: "Realm is a modern data framework & database for iOS, macOS, tvOS &\ + \ watchOS." + homepage_url: "https://realm.io" + binary_artifact: + url: "" + hash: + value: "" + algorithm: "" + source_artifact: + url: "" + hash: + value: "" + algorithm: "" + vcs: + type: "Git" + url: "https://github.com/realm/realm-swift.git" + revision: "v10.27.0" + path: "" + vcs_processed: + type: "Git" + url: "https://github.com/realm/realm-swift.git" + revision: "v10.27.0" + path: "" +- id: "Pod::Reusable:4.1.2" + purl: "pkg:cocoapods/Reusable@4.1.2" + declared_licenses: + - "MIT" + declared_licenses_processed: + spdx_expression: "MIT" + description: "A Swift Mixin to deal with reusable UITableView & UICollectionView\ + \ cells and XIB-based views" + homepage_url: "https://github.com/AliSoftware/Reusable" + binary_artifact: + url: "" + hash: + value: "" + algorithm: "" + source_artifact: + url: "" + hash: + value: "" + algorithm: "" + vcs: + type: "Git" + url: "https://github.com/AliSoftware/Reusable.git" + revision: "4.1.2" + path: "" + vcs_processed: + type: "Git" + url: "https://github.com/AliSoftware/Reusable.git" + revision: "4.1.2" + path: "" +- id: "Pod::Reusable/Storyboard:4.1.2" + purl: "pkg:cocoapods/Reusable%2FStoryboard@4.1.2" + declared_licenses: + - "MIT" + declared_licenses_processed: + spdx_expression: "MIT" + description: "A Swift Mixin to deal with reusable UITableView & UICollectionView\ + \ cells and XIB-based views" + homepage_url: "https://github.com/AliSoftware/Reusable" + binary_artifact: + url: "" + hash: + value: "" + algorithm: "" + source_artifact: + url: "" + hash: + value: "" + algorithm: "" + vcs: + type: "Git" + url: "https://github.com/AliSoftware/Reusable.git" + revision: "4.1.2" + path: "" + vcs_processed: + type: "Git" + url: "https://github.com/AliSoftware/Reusable.git" + revision: "4.1.2" + path: "" +- id: "Pod::Reusable/View:4.1.2" + purl: "pkg:cocoapods/Reusable%2FView@4.1.2" + declared_licenses: + - "MIT" + declared_licenses_processed: + spdx_expression: "MIT" + description: "A Swift Mixin to deal with reusable UITableView & UICollectionView\ + \ cells and XIB-based views" + homepage_url: "https://github.com/AliSoftware/Reusable" + binary_artifact: + url: "" + hash: + value: "" + algorithm: "" + source_artifact: + url: "" + hash: + value: "" + algorithm: "" + vcs: + type: "Git" + url: "https://github.com/AliSoftware/Reusable.git" + revision: "4.1.2" + path: "" + vcs_processed: + type: "Git" + url: "https://github.com/AliSoftware/Reusable.git" + revision: "4.1.2" + path: "" +- id: "Pod::Sentry:7.15.0" + purl: "pkg:cocoapods/Sentry@7.15.0" + declared_licenses: + - "mit" + declared_licenses_processed: + spdx_expression: "MIT" + mapped: + mit: "MIT" + description: "Sentry client for cocoa" + homepage_url: "https://github.com/getsentry/sentry-cocoa" + binary_artifact: + url: "" + hash: + value: "" + algorithm: "" + source_artifact: + url: "" + hash: + value: "" + algorithm: "" + vcs: + type: "Git" + url: "https://github.com/getsentry/sentry-cocoa.git" + revision: "7.15.0" + path: "" + vcs_processed: + type: "Git" + url: "https://github.com/getsentry/sentry-cocoa.git" + revision: "7.15.0" + path: "" +- id: "Pod::Sentry/Core:7.15.0" + purl: "pkg:cocoapods/Sentry%2FCore@7.15.0" + declared_licenses: + - "mit" + declared_licenses_processed: + spdx_expression: "MIT" + mapped: + mit: "MIT" + description: "Sentry client for cocoa" + homepage_url: "https://github.com/getsentry/sentry-cocoa" + binary_artifact: + url: "" + hash: + value: "" + algorithm: "" + source_artifact: + url: "" + hash: + value: "" + algorithm: "" + vcs: + type: "Git" + url: "https://github.com/getsentry/sentry-cocoa.git" + revision: "7.15.0" + path: "" + vcs_processed: + type: "Git" + url: "https://github.com/getsentry/sentry-cocoa.git" + revision: "7.15.0" + path: "" +- id: "Pod::SideMenu:6.5.0" + purl: "pkg:cocoapods/SideMenu@6.5.0" + declared_licenses: + - "MIT" + declared_licenses_processed: + spdx_expression: "MIT" + description: "Simple side menu control for iOS in Swift inspired by Facebook. Right\ + \ and Left sides. No coding required." + homepage_url: "https://github.com/jonkykong/SideMenu" + binary_artifact: + url: "" + hash: + value: "" + algorithm: "" + source_artifact: + url: "" + hash: + value: "" + algorithm: "" + vcs: + type: "Git" + url: "https://github.com/jonkykong/SideMenu.git" + revision: "6.5.0" + path: "" + vcs_processed: + type: "Git" + url: "https://github.com/jonkykong/SideMenu.git" + revision: "6.5.0" + path: "" +- id: "Pod::SwiftBase32:0.9.0" + purl: "pkg:cocoapods/SwiftBase32@0.9.0" + declared_licenses: + - "MIT" + declared_licenses_processed: + spdx_expression: "MIT" + description: "Base32 implementation for Swift." + homepage_url: "https://github.com/norio-nomura/Base32" + binary_artifact: + url: "" + hash: + value: "" + algorithm: "" + source_artifact: + url: "" + hash: + value: "" + algorithm: "" + vcs: + type: "Git" + url: "https://github.com/norio-nomura/Base32.git" + revision: "0.9.0" + path: "" + vcs_processed: + type: "Git" + url: "https://github.com/norio-nomura/Base32.git" + revision: "0.9.0" + path: "" +- id: "Pod::SwiftGen:6.6.2" + purl: "pkg:cocoapods/SwiftGen@6.6.2" + declared_licenses: + - "MIT" + declared_licenses_processed: + spdx_expression: "MIT" + description: "A collection of Swift tools to generate Swift code for your assets,\ + \ storyboards, strings, …" + homepage_url: "https://github.com/SwiftGen/SwiftGen" + binary_artifact: + url: "" + hash: + value: "" + algorithm: "" + source_artifact: + url: "https://github.com/SwiftGen/SwiftGen/releases/download/6.6.2/swiftgen-6.6.2.zip" + hash: + value: "" + algorithm: "" + vcs: + type: "" + url: "" + revision: "" + path: "" + vcs_processed: + type: "Git" + url: "https://github.com/SwiftGen/SwiftGen.git" + revision: "" + path: "" +- id: "Pod::SwiftJWT:3.6.200" + purl: "pkg:cocoapods/SwiftJWT@3.6.200" + declared_licenses: + - "Apache License, Version 2.0" + declared_licenses_processed: + spdx_expression: "Apache-2.0" + mapped: + Apache License, Version 2.0: "Apache-2.0" + description: "An implementation of JSON Web Token using Swift." + homepage_url: "https://github.com/Kitura/Swift-JWT" + binary_artifact: + url: "" + hash: + value: "" + algorithm: "" + source_artifact: + url: "" + hash: + value: "" + algorithm: "" + vcs: + type: "Git" + url: "https://github.com/Kitura/Swift-JWT.git" + revision: "3.6.200" + path: "" + vcs_processed: + type: "Git" + url: "https://github.com/Kitura/Swift-JWT.git" + revision: "3.6.200" + path: "" +- id: "Pod::SwiftLint:0.44.0" + purl: "pkg:cocoapods/SwiftLint@0.44.0" + declared_licenses: + - "MIT" + declared_licenses_processed: + spdx_expression: "MIT" + description: "A tool to enforce Swift style and conventions." + homepage_url: "https://github.com/realm/SwiftLint" + binary_artifact: + url: "" + hash: + value: "" + algorithm: "" + source_artifact: + url: "https://github.com/realm/SwiftLint/releases/download/0.44.0/portable_swiftlint.zip" + hash: + value: "" + algorithm: "" + vcs: + type: "" + url: "" + revision: "" + path: "" + vcs_processed: + type: "Git" + url: "https://github.com/realm/SwiftLint.git" + revision: "" + path: "" +- id: "Pod::SwiftyBeaver:1.9.5" + purl: "pkg:cocoapods/SwiftyBeaver@1.9.5" + declared_licenses: + - "MIT" + declared_licenses_processed: + spdx_expression: "MIT" + description: "Convenient logging during development & release in Swift 4 & 5." + homepage_url: "https://github.com/SwiftyBeaver/SwiftyBeaver" + binary_artifact: + url: "" + hash: + value: "" + algorithm: "" + source_artifact: + url: "" + hash: + value: "" + algorithm: "" + vcs: + type: "Git" + url: "https://github.com/SwiftyBeaver/SwiftyBeaver.git" + revision: "1.9.5" + path: "" + vcs_processed: + type: "Git" + url: "https://github.com/SwiftyBeaver/SwiftyBeaver.git" + revision: "1.9.5" + path: "" +- id: "Pod::UICollectionViewLeftAlignedLayout:1.0.2" + purl: "pkg:cocoapods/UICollectionViewLeftAlignedLayout@1.0.2" + declared_licenses: + - "MIT" + declared_licenses_processed: + spdx_expression: "MIT" + description: "A layout for UICollectionView that aligns the cells to the left" + homepage_url: "https://github.com/mokagio/UICollectionViewLeftAlignedLayout" + binary_artifact: + url: "" + hash: + value: "" + algorithm: "" + source_artifact: + url: "" + hash: + value: "" + algorithm: "" + vcs: + type: "Git" + url: "https://github.com/mokagio/UICollectionViewLeftAlignedLayout.git" + revision: "1.0.2" + path: "" + vcs_processed: + type: "Git" + url: "https://github.com/mokagio/UICollectionViewLeftAlignedLayout.git" + revision: "1.0.2" + path: "" +- id: "Pod::UICollectionViewRightAlignedLayout:0.0.3" + purl: "pkg:cocoapods/UICollectionViewRightAlignedLayout@0.0.3" + declared_licenses: + - "MIT" + declared_licenses_processed: + spdx_expression: "MIT" + description: "A layout for UICollectionView that aligns the cells to the right." + homepage_url: "https://github.com/mokagio/UICollectionViewRightAlignedLayout" + binary_artifact: + url: "" + hash: + value: "" + algorithm: "" + source_artifact: + url: "" + hash: + value: "" + algorithm: "" + vcs: + type: "Git" + url: "https://github.com/mokagio/UICollectionViewRightAlignedLayout.git" + revision: "0.0.3" + path: "" + vcs_processed: + type: "Git" + url: "https://github.com/mokagio/UICollectionViewRightAlignedLayout.git" + revision: "0.0.3" + path: "" +- id: "Pod::WeakDictionary:2.0.2" + purl: "pkg:cocoapods/WeakDictionary@2.0.2" + declared_licenses: + - "MIT" + declared_licenses_processed: + spdx_expression: "MIT" + description: "A naive (strong key/weak value) dictionary & (weak key/weak value)\ + \ dictionary implementations in swift" + homepage_url: "https://github.com/nicholascross/WeakDictionary" + binary_artifact: + url: "" + hash: + value: "" + algorithm: "" + source_artifact: + url: "" + hash: + value: "" + algorithm: "" + vcs: + type: "Git" + url: "https://github.com/nicholascross/WeakDictionary.git" + revision: "2.0.2" + path: "" + vcs_processed: + type: "Git" + url: "https://github.com/nicholascross/WeakDictionary.git" + revision: "2.0.2" + path: "" +- id: "Pod::ZXingObjC:3.6.5" + purl: "pkg:cocoapods/ZXingObjC@3.6.5" + declared_licenses: + - "Apache License 2.0" + declared_licenses_processed: + spdx_expression: "Apache-2.0" + mapped: + Apache License 2.0: "Apache-2.0" + description: "An Objective-C Port of the ZXing barcode framework." + homepage_url: "https://github.com/zxingify/zxingify-objc" + binary_artifact: + url: "" + hash: + value: "" + algorithm: "" + source_artifact: + url: "" + hash: + value: "" + algorithm: "" + vcs: + type: "Git" + url: "https://github.com/zxingify/zxingify-objc.git" + revision: "3.6.5" + path: "" + vcs_processed: + type: "Git" + url: "https://github.com/zxingify/zxingify-objc.git" + revision: "3.6.5" + path: "" +- id: "Pod::ZXingObjC/All:3.6.5" + purl: "pkg:cocoapods/ZXingObjC%2FAll@3.6.5" + declared_licenses: + - "Apache License 2.0" + declared_licenses_processed: + spdx_expression: "Apache-2.0" + mapped: + Apache License 2.0: "Apache-2.0" + description: "An Objective-C Port of the ZXing barcode framework." + homepage_url: "https://github.com/zxingify/zxingify-objc" + binary_artifact: + url: "" + hash: + value: "" + algorithm: "" + source_artifact: + url: "" + hash: + value: "" + algorithm: "" + vcs: + type: "Git" + url: "https://github.com/zxingify/zxingify-objc.git" + revision: "3.6.5" + path: "" + vcs_processed: + type: "Git" + url: "https://github.com/zxingify/zxingify-objc.git" + revision: "3.6.5" + path: "" +- id: "Pod::ffmpeg-kit-ios-audio:4.5.1" + purl: "pkg:cocoapods/ffmpeg-kit-ios-audio@4.5.1" + declared_licenses: + - "LGPL-3.0" + declared_licenses_processed: + spdx_expression: "LGPL-3.0-only" + mapped: + LGPL-3.0: "LGPL-3.0-only" + description: "FFmpeg Kit iOS Audio Shared Framework" + homepage_url: "https://github.com/tanersener/ffmpeg-kit" + binary_artifact: + url: "" + hash: + value: "" + algorithm: "" + source_artifact: + url: "https://github.com/tanersener/ffmpeg-kit/releases/download/v4.5.1/ffmpeg-kit-audio-4.5.1-ios-xcframework.zip" + hash: + value: "" + algorithm: "" + vcs: + type: "" + url: "" + revision: "" + path: "" + vcs_processed: + type: "Git" + url: "https://github.com/tanersener/ffmpeg-kit.git" + revision: "" + path: "" +- id: "Pod::libPhoneNumber-iOS:0.9.15" + purl: "pkg:cocoapods/libPhoneNumber-iOS@0.9.15" + declared_licenses: + - "Apache License, Version 2.0 (http://www.apache.org/licenses/LICENSE-2.0)" + declared_licenses_processed: + spdx_expression: "Apache-2.0" + mapped: + Apache License, Version 2.0 (http://www.apache.org/licenses/LICENSE-2.0): "Apache-2.0" + description: "iOS library for parsing, formatting, storing and validating international\ + \ phone numbers from libphonenumber library." + homepage_url: "https://github.com/iziz/libPhoneNumber-iOS.git" + binary_artifact: + url: "" + hash: + value: "" + algorithm: "" + source_artifact: + url: "" + hash: + value: "" + algorithm: "" + vcs: + type: "Git" + url: "https://github.com/iziz/libPhoneNumber-iOS.git" + revision: "0.9.15" + path: "" + vcs_processed: + type: "Git" + url: "https://github.com/iziz/libPhoneNumber-iOS.git" + revision: "0.9.15" + path: "" +- id: "Pod::libbase58:0.1.4" + purl: "pkg:cocoapods/libbase58@0.1.4" + declared_licenses: + - "MIT" + declared_licenses_processed: + spdx_expression: "MIT" + description: "C library for Bitcoin's base58 encoding" + homepage_url: "https://github.com/luke-jr/libbase58" + binary_artifact: + url: "" + hash: + value: "" + algorithm: "" + source_artifact: + url: "" + hash: + value: "" + algorithm: "" + vcs: + type: "Git" + url: "https://github.com/luke-jr/libbase58.git" + revision: "v0.1.4" + path: "" + vcs_processed: + type: "Git" + url: "https://github.com/luke-jr/libbase58.git" + revision: "v0.1.4" + path: "" +- id: "Pod::zxcvbn-ios:1.0.4" + purl: "pkg:cocoapods/zxcvbn-ios@1.0.4" + declared_licenses: + - "MIT" + declared_licenses_processed: + spdx_expression: "MIT" + description: "A realistic password strength estimator." + homepage_url: "https://github.com/dropbox/zxcvbn-ios" + binary_artifact: + url: "" + hash: + value: "" + algorithm: "" + source_artifact: + url: "" + hash: + value: "" + algorithm: "" + vcs: + type: "Git" + url: "https://github.com/dropbox/zxcvbn-ios.git" + revision: "v1.0.4" + path: "" + vcs_processed: + type: "Git" + url: "https://github.com/dropbox/zxcvbn-ios.git" + revision: "v1.0.4" + path: "" diff --git a/plugins/package-managers/cocoapods/src/funTest/assets/projects/synthetic/version-resolution/Podfile b/plugins/package-managers/cocoapods/src/funTest/assets/projects/synthetic/version-resolution/Podfile new file mode 100644 index 0000000000000..78aaa6b7e7fc0 --- /dev/null +++ b/plugins/package-managers/cocoapods/src/funTest/assets/projects/synthetic/version-resolution/Podfile @@ -0,0 +1,158 @@ +source 'https://cdn.cocoapods.org/' + +# Uncomment this line to define a global platform for your project +platform :ios, '14.0' + +# By default, ignore all warnings from any pod +inhibit_all_warnings! + +# Use frameworks to allow usage of pods written in Swift +use_frameworks! + +# Different flavours of pods to MatrixSDK. Can be one of: +# - a String indicating an official MatrixSDK released version number +# - `:local` (to use Development Pods) +# - `{ :branch => 'sdk branch name'}` to depend on specific branch of MatrixSDK repo +# - `{ :specHash => {sdk spec hash}` to depend on specific pod options (:git => …, :podspec => …) for MatrixSDK repo. Used by Fastfile during CI +# +# Warning: our internal tooling depends on the name of this variable name, so be sure not to change it +$matrixSDKVersion = '= 0.23.18' +# $matrixSDKVersion = :local +# $matrixSDKVersion = { :branch => 'develop'} +# $matrixSDKVersion = { :specHash => { git: 'https://git.io/fork123', branch: 'fix' } } + +######################################## + +case $matrixSDKVersion +when :local +$matrixSDKVersionSpec = { :path => '../matrix-ios-sdk/MatrixSDK.podspec' } +when Hash +spec_mode, sdk_spec = $matrixSDKVersion.first # extract first and only key/value pair; key is spec_mode, value is sdk_spec + + case spec_mode + when :branch + # :branch => sdk branch name + sdk_spec = { :git => 'https://github.com/matrix-org/matrix-ios-sdk.git', :branch => sdk_spec.to_s } unless sdk_spec.is_a?(Hash) + when :specHash + # :specHash => {sdk spec Hash} + sdk_spec = sdk_spec + end + +$matrixSDKVersionSpec = sdk_spec +when String # specific MatrixSDK released version +$matrixSDKVersionSpec = $matrixSDKVersion +end + +# Method to import the MatrixSDK +def import_MatrixSDK + pod 'MatrixSDK', $matrixSDKVersionSpec, :inhibit_warnings => false + pod 'MatrixSDK/JingleCallStack', $matrixSDKVersionSpec, :inhibit_warnings => false +end + +######################################## + +def import_MatrixKit_pods + pod 'libPhoneNumber-iOS', '~> 0.9.13' + pod 'DTCoreText', '~> 1.6.25' + #pod 'DTCoreText/Extension', '~> 1.6.25' + pod 'Down', '~> 0.11.0' +end + +def import_SwiftUI_pods + pod 'Introspect', '~> 0.1' + pod 'DSBottomSheet', '~> 0.3' +end + +abstract_target 'RiotPods' do + + pod 'GBDeviceInfo', '~> 6.6.0' + pod 'Reusable', '~> 4.1' + pod 'KeychainAccess', '~> 4.2.2' + pod 'WeakDictionary', '~> 2.0' + + # PostHog for analytics + pod 'PostHog', '~> 1.4.4' + pod 'Sentry', '~> 7.15.0' + pod 'AnalyticsEvents', :git => 'https://github.com/matrix-org/matrix-analytics-events.git', :branch => 'release/swift', :inhibit_warnings => false + # pod 'AnalyticsEvents', :path => '../matrix-analytics-events/AnalyticsEvents.podspec' + + pod 'OLMKit' + pod 'zxcvbn-ios' + + # Tools + pod 'SwiftGen', '~> 6.3' + pod 'SwiftLint', '~> 0.44.0' + + target "Riot" do + import_MatrixSDK + import_MatrixKit_pods + + import_SwiftUI_pods + + pod 'UICollectionViewRightAlignedLayout', '~> 0.0.3' + pod 'UICollectionViewLeftAlignedLayout', '~> 1.0.2' + pod 'KTCenterFlowLayout', '~> 1.3.1' + pod 'ZXingObjC', '~> 3.6.5' + pod 'FlowCommoniOS', '~> 1.12.0' + pod 'ReadMoreTextView', '~> 3.0.1' + pod 'SwiftBase32', '~> 0.9.0' + pod 'SwiftJWT', '~> 3.6.200' + pod 'SideMenu', '~> 6.5' + pod 'DSWaveformImage', '~> 6.1.1' + pod 'ffmpeg-kit-ios-audio', '4.5.1' + + pod 'FLEX', '~> 4.5.0', :configurations => ['Debug'], :inhibit_warnings => true + + target 'RiotTests' do + inherit! :search_paths + end + end + + target "RiotShareExtension" do + import_MatrixSDK + import_MatrixKit_pods + end + + target "RiotSwiftUI" do + import_SwiftUI_pods + end + + target "RiotSwiftUITests" do + import_SwiftUI_pods + end + + target "SiriIntents" do + import_MatrixSDK + import_MatrixKit_pods + end + + target "RiotNSE" do + import_MatrixSDK + import_MatrixKit_pods + end + +end + +post_install do |installer| + installer.pods_project.targets.each do |target| + + target.build_configurations.each do |config| + # Disable bitcode for each pod framework + # Because the WebRTC pod (included by the JingleCallStack pod) does not support it. + # Plus the app does not enable it + config.build_settings['ENABLE_BITCODE'] = 'NO' + + # Force ReadMoreTextView to use Swift 5.2 version (as there is no code changes to perform) + if target.name.include? 'ReadMoreTextView' + config.build_settings['SWIFT_VERSION'] = '5.2' + end + + # Stop Xcode 12 complaining about old IPHONEOS_DEPLOYMENT_TARGET from pods + config.build_settings.delete 'IPHONEOS_DEPLOYMENT_TARGET' + + # Disable nullability checks + config.build_settings['WARNING_CFLAGS'] ||= ['$(inherited)','-Wno-nullability-completeness'] + config.build_settings['OTHER_SWIFT_FLAGS'] ||= ['$(inherited)', '-Xcc', '-Wno-nullability-completeness'] + end + end +end diff --git a/plugins/package-managers/cocoapods/src/funTest/assets/projects/synthetic/version-resolution/Podfile.lock b/plugins/package-managers/cocoapods/src/funTest/assets/projects/synthetic/version-resolution/Podfile.lock new file mode 100644 index 0000000000000..7f62c51f7002b --- /dev/null +++ b/plugins/package-managers/cocoapods/src/funTest/assets/projects/synthetic/version-resolution/Podfile.lock @@ -0,0 +1,246 @@ +PODS: + - AFNetworking (4.0.1): + - AFNetworking/NSURLSession (= 4.0.1) + - AFNetworking/Reachability (= 4.0.1) + - AFNetworking/Security (= 4.0.1) + - AFNetworking/Serialization (= 4.0.1) + - AFNetworking/UIKit (= 4.0.1) + - AFNetworking/NSURLSession (4.0.1): + - AFNetworking/Reachability + - AFNetworking/Security + - AFNetworking/Serialization + - AFNetworking/Reachability (4.0.1) + - AFNetworking/Security (4.0.1) + - AFNetworking/Serialization (4.0.1) + - AFNetworking/UIKit (4.0.1): + - AFNetworking/NSURLSession + - AnalyticsEvents (0.1.0) + - BlueCryptor (1.0.32) + - BlueECC (1.2.5) + - BlueRSA (1.0.200) + - Down (0.11.0) + - DSBottomSheet (0.3.0) + - DSWaveformImage (6.1.1) + - DTCoreText (1.6.26): + - DTCoreText/Core (= 1.6.26) + - DTFoundation/Core (~> 1.7.5) + - DTFoundation/DTAnimatedGIF (~> 1.7.5) + - DTFoundation/DTHTMLParser (~> 1.7.5) + - DTFoundation/UIKit (~> 1.7.5) + - DTCoreText/Core (1.6.26): + - DTFoundation/Core (~> 1.7.5) + - DTFoundation/DTAnimatedGIF (~> 1.7.5) + - DTFoundation/DTHTMLParser (~> 1.7.5) + - DTFoundation/UIKit (~> 1.7.5) + - DTFoundation/Core (1.7.18) + - DTFoundation/DTAnimatedGIF (1.7.18) + - DTFoundation/DTHTMLParser (1.7.18): + - DTFoundation/Core + - DTFoundation/UIKit (1.7.18): + - DTFoundation/Core + - ffmpeg-kit-ios-audio (4.5.1) + - FLEX (4.5.0) + - FlowCommoniOS (1.12.2) + - GBDeviceInfo (6.6.0): + - GBDeviceInfo/Core (= 6.6.0) + - GBDeviceInfo/Core (6.6.0) + - GZIP (1.3.0) + - Introspect (0.1.4) + - JitsiMeetSDK (5.0.2) + - KeychainAccess (4.2.2) + - KituraContracts (1.2.1): + - LoggerAPI (~> 1.7) + - KTCenterFlowLayout (1.3.1) + - libbase58 (0.1.4) + - libPhoneNumber-iOS (0.9.15) + - LoggerAPI (1.9.200): + - Logging (~> 1.1) + - Logging (1.4.0) + - MatrixSDK (0.23.18): + - MatrixSDK/Core (= 0.23.18) + - MatrixSDK/Core (0.23.18): + - AFNetworking (~> 4.0.0) + - GZIP (~> 1.3.0) + - libbase58 (~> 0.1.4) + - MatrixSDK/CryptoSDK + - OLMKit (~> 3.2.5) + - Realm (= 10.27.0) + - SwiftyBeaver (= 1.9.5) + - MatrixSDK/CryptoSDK (0.23.18): + - MatrixSDKCrypto (= 0.1.0) + - MatrixSDK/JingleCallStack (0.23.18): + - JitsiMeetSDK (= 5.0.2) + - MatrixSDK/Core + - MatrixSDKCrypto (0.1.0) + - OLMKit (3.2.12): + - OLMKit/olmc (= 3.2.12) + - OLMKit/olmcpp (= 3.2.12) + - OLMKit/olmc (3.2.12) + - OLMKit/olmcpp (3.2.12) + - PostHog (1.4.4) + - ReadMoreTextView (3.0.1) + - Realm (10.27.0): + - Realm/Headers (= 10.27.0) + - Realm/Headers (10.27.0) + - Reusable (4.1.2): + - Reusable/Storyboard (= 4.1.2) + - Reusable/View (= 4.1.2) + - Reusable/Storyboard (4.1.2) + - Reusable/View (4.1.2) + - Sentry (7.15.0): + - Sentry/Core (= 7.15.0) + - Sentry/Core (7.15.0) + - SideMenu (6.5.0) + - SwiftBase32 (0.9.0) + - SwiftGen (6.6.2) + - SwiftJWT (3.6.200): + - BlueCryptor (~> 1.0) + - BlueECC (~> 1.1) + - BlueRSA (~> 1.0) + - KituraContracts (~> 1.2) + - LoggerAPI (~> 1.7) + - SwiftLint (0.44.0) + - SwiftyBeaver (1.9.5) + - UICollectionViewLeftAlignedLayout (1.0.2) + - UICollectionViewRightAlignedLayout (0.0.3) + - WeakDictionary (2.0.2) + - zxcvbn-ios (1.0.4) + - ZXingObjC (3.6.5): + - ZXingObjC/All (= 3.6.5) + - ZXingObjC/All (3.6.5) + +DEPENDENCIES: + - AnalyticsEvents (from `https://github.com/matrix-org/matrix-analytics-events.git`, branch `release/swift`) + - Down (~> 0.11.0) + - DSBottomSheet (~> 0.3) + - DSWaveformImage (~> 6.1.1) + - DTCoreText (~> 1.6.25) + - ffmpeg-kit-ios-audio (= 4.5.1) + - FLEX (~> 4.5.0) + - FlowCommoniOS (~> 1.12.0) + - GBDeviceInfo (~> 6.6.0) + - Introspect (~> 0.1) + - KeychainAccess (~> 4.2.2) + - KTCenterFlowLayout (~> 1.3.1) + - libPhoneNumber-iOS (~> 0.9.13) + - MatrixSDK (= 0.23.18) + - MatrixSDK/JingleCallStack (= 0.23.18) + - OLMKit + - PostHog (~> 1.4.4) + - ReadMoreTextView (~> 3.0.1) + - Reusable (~> 4.1) + - Sentry (~> 7.15.0) + - SideMenu (~> 6.5) + - SwiftBase32 (~> 0.9.0) + - SwiftGen (~> 6.3) + - SwiftJWT (~> 3.6.200) + - SwiftLint (~> 0.44.0) + - UICollectionViewLeftAlignedLayout (~> 1.0.2) + - UICollectionViewRightAlignedLayout (~> 0.0.3) + - WeakDictionary (~> 2.0) + - zxcvbn-ios + - ZXingObjC (~> 3.6.5) + +SPEC REPOS: + trunk: + - AFNetworking + - BlueCryptor + - BlueECC + - BlueRSA + - Down + - DSBottomSheet + - DSWaveformImage + - DTCoreText + - DTFoundation + - ffmpeg-kit-ios-audio + - FLEX + - FlowCommoniOS + - GBDeviceInfo + - GZIP + - Introspect + - JitsiMeetSDK + - KeychainAccess + - KituraContracts + - KTCenterFlowLayout + - libbase58 + - libPhoneNumber-iOS + - LoggerAPI + - Logging + - MatrixSDK + - MatrixSDKCrypto + - OLMKit + - PostHog + - ReadMoreTextView + - Realm + - Reusable + - Sentry + - SideMenu + - SwiftBase32 + - SwiftGen + - SwiftJWT + - SwiftLint + - SwiftyBeaver + - UICollectionViewLeftAlignedLayout + - UICollectionViewRightAlignedLayout + - WeakDictionary + - zxcvbn-ios + - ZXingObjC + +EXTERNAL SOURCES: + AnalyticsEvents: + :branch: release/swift + :git: https://github.com/matrix-org/matrix-analytics-events.git + +CHECKOUT OPTIONS: + AnalyticsEvents: + :commit: 53ad46ba1ea1ee8f21139dda3c351890846a202f + :git: https://github.com/matrix-org/matrix-analytics-events.git + +SPEC CHECKSUMS: + AFNetworking: 7864c38297c79aaca1500c33288e429c3451fdce + AnalyticsEvents: 0cc8cf52da2fd464a2f39b788a295988151116ce + BlueCryptor: b0aee3d9b8f367b49b30de11cda90e1735571c24 + BlueECC: 0d18e93347d3ec6d41416de21c1ffa4d4cd3c2cc + BlueRSA: dfeef51db96bcc4edec654956c1581adbda4e6a3 + Down: b6ba1bc985c9d2f4e15e3b293d2207766fa12612 + DSBottomSheet: ca0ac37eb5af2dd54663f86b84382ed90a59be2a + DSWaveformImage: 3c718a0cf99291887ee70d1d0c18d80101d3d9ce + DTCoreText: ec749e013f2e1f76de5e7c7634642e600a7467ce + DTFoundation: a53f8cda2489208cbc71c648be177f902ee17536 + ffmpeg-kit-ios-audio: 662ce2064e56733ca7d8216705efbc38d9e1c3fe + FLEX: e51461dd6f0bfb00643c262acdfea5d5d12c596b + FlowCommoniOS: ca92071ab526dc89905495a37844fd7e78d1a7f2 + GBDeviceInfo: ed0db16230d2fa280e1cbb39a5a7f60f6946aaec + GZIP: 416858efbe66b41b206895ac6dfd5493200d95b3 + Introspect: b62c4dd2063072327c21d618ef2bedc3c87bc366 + JitsiMeetSDK: edcac8e2b92ee0c7f3e75bd0aefefbe9faccfc93 + KeychainAccess: c0c4f7f38f6fc7bbe58f5702e25f7bd2f65abf51 + KituraContracts: e845e60dc8627ad0a76fa55ef20a45451d8f830b + KTCenterFlowLayout: 6e02b50ab2bd865025ae82fe266ed13b6d9eaf97 + libbase58: 7c040313537b8c44b6e2d15586af8e21f7354efd + libPhoneNumber-iOS: 0a32a9525cf8744fe02c5206eb30d571e38f7d75 + LoggerAPI: ad9c4a6f1e32f518fdb43a1347ac14d765ab5e3d + Logging: beeb016c9c80cf77042d62e83495816847ef108b + MatrixSDK: 26da2e3a9f3b02fc6ea67f5bc311d30f06f9ffba + MatrixSDKCrypto: 4b9146d5ef484550341be056a164c6930038028e + OLMKit: da115f16582e47626616874e20f7bb92222c7a51 + PostHog: 4b6321b521569092d4ef3a02238d9435dbaeb99f + ReadMoreTextView: 19147adf93abce6d7271e14031a00303fe28720d + Realm: 9ca328bd7e700cc19703799785e37f77d1a130f2 + Reusable: 6bae6a5e8aa793c9c441db0213c863a64bce9136 + Sentry: 63ca44f5e0c8cea0ee5a07686b02e56104f41ef7 + SideMenu: f583187d21c5b1dd04c72002be544b555a2627a2 + SwiftBase32: 9399c25a80666dc66b51e10076bf591e3bbb8f17 + SwiftGen: 1366a7f71aeef49954ca5a63ba4bef6b0f24138c + SwiftJWT: 88c412708f58c169d431d344c87bc79a87c830ae + SwiftLint: e96c0a8c770c7ebbc4d36c55baf9096bb65c4584 + SwiftyBeaver: 84069991dd5dca07d7069100985badaca7f0ce82 + UICollectionViewLeftAlignedLayout: 830bf6fa5bab9f9b464f62e3384f9d2e00b3c0f6 + UICollectionViewRightAlignedLayout: 823eef8c567eba4a44c21bc2ffcb0d0d5f361e2d + WeakDictionary: 8cd038acd77e5d54ca4ebaec3d20853d732b45e0 + zxcvbn-ios: fef98b7c80f1512ff0eec47ac1fa399fc00f7e3c + ZXingObjC: fdbb269f25dd2032da343e06f10224d62f537bdb + +PODFILE CHECKSUM: 45176df406c18b0c23321a308f58535fbe425a93 + +COCOAPODS: 1.11.2 diff --git a/plugins/package-managers/cocoapods/src/funTest/kotlin/CocoaPodsFunTest.kt b/plugins/package-managers/cocoapods/src/funTest/kotlin/CocoaPodsFunTest.kt index ef67e9666b0df..89bc3ca5d28e1 100644 --- a/plugins/package-managers/cocoapods/src/funTest/kotlin/CocoaPodsFunTest.kt +++ b/plugins/package-managers/cocoapods/src/funTest/kotlin/CocoaPodsFunTest.kt @@ -58,6 +58,15 @@ class CocoaPodsFunTest : WordSpec({ result.toYaml() should matchExpectedResult(expectedResultFile, definitionFile) } + "determine dependencies from a Podfile that upgrades a version during resolution" { + val definitionFile = getAssetFile("projects/synthetic/version-resolution/Podfile") + val expectedResultFile = getAssetFile("projects/synthetic/version-resolution-expected-output.yml") + + val result = create("CocoaPods").resolveSingleProject(definitionFile) + + result.toYaml() should matchExpectedResult(expectedResultFile, definitionFile) + } + "return no dependencies along with an issue if the lockfile is absent" { val definitionFile = getAssetFile("projects/synthetic/no-lockfile/Podfile") val expectedResultFile = getAssetFile("projects/synthetic/no-lockfile-expected-output.yml") diff --git a/plugins/package-managers/cocoapods/src/main/kotlin/CocoaPods.kt b/plugins/package-managers/cocoapods/src/main/kotlin/CocoaPods.kt index 3445ed0f8b8ff..cc6ef0a644126 100644 --- a/plugins/package-managers/cocoapods/src/main/kotlin/CocoaPods.kt +++ b/plugins/package-managers/cocoapods/src/main/kotlin/CocoaPods.kt @@ -237,7 +237,7 @@ private fun parseNameAndVersion(entry: String): Pair { // A version entry could look something like "(6.3.0)", "(= 2021.06.28.00-v2)", "(~> 8.15.0)", etc. Also see // https://guides.cocoapods.org/syntax/podfile.html#pod. - val version = info.getOrNull(1)?.removeSurrounding("(", ")")?.substringAfterLast(' ') + val version = info.getOrNull(1)?.removeSurrounding("(", ")") return name to version } @@ -252,24 +252,24 @@ private fun parseLockfile(podfileLock: File): LockfileData { val dependencyConstraints = mutableMapOf>() val root = yamlMapper.readTree(podfileLock) - // The "PODS" section lists the direct dependencies and, if applicable, their direct dependencies each. That is, the - // nesting never goes deeper than two levels, and in order to build up a full dependency tree, one needs to - // recursively step through all direct dependencies. - root.get("PODS").asIterable().forEach { node -> - val entry = when (node) { - is ObjectNode -> node.fieldNames().asSequence().first() - else -> node.textValue() - } - - val (name, version) = parseNameAndVersion(entry) - resolvedVersions[name] = checkNotNull(version) - - val dependencies = node[entry]?.map { depNode -> - val (depName, depVersion) = parseNameAndVersion(depNode.textValue()) - depName.also { if (depVersion != null) resolvedVersions[it] = depVersion } - }.orEmpty() + // The "PODS" section lists the resolved dependencies and, nested by one level, any version constraints of their + // direct dependencies. That is, the nesting never goes deeper than two levels. + root.get("PODS").forEach { node -> + when (node) { + is ObjectNode -> { + val (name, version) = parseNameAndVersion(node.fieldNames().asSequence().single()) + resolvedVersions[name] = checkNotNull(version) + dependencyConstraints[name] = node.single().mapTo(mutableSetOf()) { + // Discard the version (which is only a constraint in this case) and just take the name. + parseNameAndVersion(it.textValue()).first + } + } - dependencyConstraints.getOrPut(name) { mutableSetOf() } += dependencies + else -> { + val (name, version) = parseNameAndVersion(node.textValue()) + resolvedVersions[name] = checkNotNull(version) + } + } } val externalSources = root.get("CHECKOUT OPTIONS")?.fields()?.asSequence()?.mapNotNull { @@ -301,7 +301,12 @@ private fun parseLockfile(podfileLock: File): LockfileData { fun createPackageReference(name: String): PackageReference = PackageReference( id = Identifier("Pod", "", name, resolvedVersions.getValue(name)), - dependencies = dependencyConstraints[name].orEmpty().mapTo(mutableSetOf()) { createPackageReference(it) } + dependencies = dependencyConstraints[name].orEmpty().filter { + // Only use a constraint as a dependency if it has a resolved version. + it in resolvedVersions + }.mapTo(mutableSetOf()) { + createPackageReference(it) + } ) // The "DEPENDENCIES" section lists direct dependencies, but only along with version constraints, not with their From 4df30940dfd1c7000d21d4c65fb6d73f40594c13 Mon Sep 17 00:00:00 2001 From: Sebastian Schuberth Date: Tue, 26 Sep 2023 17:42:51 +0200 Subject: [PATCH 07/47] test(SpmFunTest): Update expected results Signed-off-by: Sebastian Schuberth --- .../projects/synthetic/spm-expected-output-lib.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/plugins/package-managers/spm/src/funTest/assets/projects/synthetic/spm-expected-output-lib.yml b/plugins/package-managers/spm/src/funTest/assets/projects/synthetic/spm-expected-output-lib.yml index 5fe13559a259a..a47366f019313 100644 --- a/plugins/package-managers/spm/src/funTest/assets/projects/synthetic/spm-expected-output-lib.yml +++ b/plugins/package-managers/spm/src/funTest/assets/projects/synthetic/spm-expected-output-lib.yml @@ -90,7 +90,7 @@ project: dependencies: - id: "SPM:apple:swift-atomics:1.1.0" - id: "SPM:apple:swift-collections:1.0.4" - - id: "SPM:vapor:console-kit:4.8.0" + - id: "SPM:vapor:console-kit:4.8.1" dependencies: - id: "SPM:apple:swift-log:1.5.3" - id: "SPM:apple:swift-nio:2.59.0" @@ -554,8 +554,8 @@ packages: url: "https://github.com/vapor/async-kit.git" revision: "1.18.0" path: "" -- id: "SPM:vapor:console-kit:4.8.0" - purl: "pkg:swift/vapor/console-kit@4.8.0" +- id: "SPM:vapor:console-kit:4.8.1" + purl: "pkg:swift/vapor/console-kit@4.8.1" authors: - "vapor" declared_licenses: [] @@ -575,12 +575,12 @@ packages: vcs: type: "Git" url: "https://github.com/vapor/console-kit.git" - revision: "4.8.0" + revision: "4.8.1" path: "" vcs_processed: type: "Git" url: "https://github.com/vapor/console-kit.git" - revision: "4.8.0" + revision: "4.8.1" path: "" - id: "SPM:vapor:multipart-kit:4.5.4" purl: "pkg:swift/vapor/multipart-kit@4.5.4" From f75d2003e8e6065a7791d1d8134bf346faa7ad0b Mon Sep 17 00:00:00 2001 From: Helio Chissini de Castro Date: Mon, 18 Sep 2023 11:01:32 +0200 Subject: [PATCH 08/47] fix(docs): Mention Apache 2.0 license Signed-off-by: Helio Chissini de Castro --- website/docs/license.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/website/docs/license.md b/website/docs/license.md index 96c9fe2e4b238..55d5906b2cdcb 100644 --- a/website/docs/license.md +++ b/website/docs/license.md @@ -6,8 +6,7 @@ sidebar_position: 8 Copyright (C) 2017-2023 [The ORT Project Authors](https://github.com/oss-review-toolkit/ort/blob/main/NOTICE). -See the [LICENSE](https://github.com/oss-review-toolkit/ort/blob/main/LICENSE) file in the root of this project for -license details. +Licensed under the [Apache License, Version 2.0](https://github.com/oss-review-toolkit/ort/blob/main/LICENSE). OSS Review Toolkit (ORT) is a [Linux Foundation project](https://www.linuxfoundation.org) and part of [ACT](https://automatecompliance.org/). From 2b813d0c1e42432c05f03858906b4165f7ddd803 Mon Sep 17 00:00:00 2001 From: Helio Chissini de Castro Date: Sun, 3 Sep 2023 16:34:56 +0200 Subject: [PATCH 09/47] feat(docker): Use per language container strategy ORT language components are built now in independent images. This enables the possibility to create custom tailored containers with only the required language components, reducing final size and improving build process. Main runtime contains Python, NodeJS, Rust, Ruby and Golang components. Other language components have their own images built, and can be included on custom containers. Signed-off-by: Helio Chissini de Castro --- .github/workflows/docker-ort-android.yml | 91 ++++++++ .github/workflows/docker-ort-base.yml | 83 +++++++ .github/workflows/docker-ort-binaries.yml | 87 ++++++++ .github/workflows/docker-ort-dart.yml | 91 ++++++++ .github/workflows/docker-ort-dotnet.yml | 89 ++++++++ .github/workflows/docker-ort-golang.yml | 92 ++++++++ .github/workflows/docker-ort-haskell.yml | 91 ++++++++ .github/workflows/docker-ort-nodejs.yml | 93 ++++++++ .github/workflows/docker-ort-python.yml | 96 +++++++++ .github/workflows/docker-ort-ruby.yml | 91 ++++++++ .github/workflows/docker-ort-runtime.yml | 98 +++++++++ .github/workflows/docker-ort-rust.yml | 89 ++++++++ .github/workflows/docker-ort-sbt.yml | 91 ++++++++ .github/workflows/docker-ort-swift.yml | 91 ++++++++ .ortversions/android.versions | 1 + .ortversions/base.versions | 2 + .ortversions/dart.versions | 1 + .ortversions/dotnet.versions | 2 + .ortversions/golang.versions | 2 + .ortversions/haskell.versions | 1 + .ortversions/nodejs.versions | 5 + .ortversions/php.versions | 2 + .ortversions/python.versions | 8 + .ortversions/ruby.versions | 2 + .ortversions/rust.versions | 1 + .ortversions/sbt.versions | 1 + .ortversions/swift.versions | 1 + Dockerfile | 251 ++++++++++------------ scripts/custom_docker.sh | 100 +++++++++ scripts/docker_build.sh | 148 ++++++++++++- scripts/docker_build_all.sh | 144 +++++++++++++ scripts/docker_snippets/android.snippet | 26 +++ scripts/docker_snippets/dart.snippet | 22 ++ scripts/docker_snippets/dotnet.snippet | 24 +++ scripts/docker_snippets/haskell.snippet | 32 +++ scripts/docker_snippets/php.snippet | 34 +++ scripts/docker_snippets/sbt.snippet | 22 ++ scripts/docker_snippets/swift.snippet | 22 ++ 38 files changed, 1986 insertions(+), 141 deletions(-) create mode 100644 .github/workflows/docker-ort-android.yml create mode 100644 .github/workflows/docker-ort-base.yml create mode 100644 .github/workflows/docker-ort-binaries.yml create mode 100644 .github/workflows/docker-ort-dart.yml create mode 100644 .github/workflows/docker-ort-dotnet.yml create mode 100644 .github/workflows/docker-ort-golang.yml create mode 100644 .github/workflows/docker-ort-haskell.yml create mode 100644 .github/workflows/docker-ort-nodejs.yml create mode 100644 .github/workflows/docker-ort-python.yml create mode 100644 .github/workflows/docker-ort-ruby.yml create mode 100644 .github/workflows/docker-ort-runtime.yml create mode 100644 .github/workflows/docker-ort-rust.yml create mode 100644 .github/workflows/docker-ort-sbt.yml create mode 100644 .github/workflows/docker-ort-swift.yml create mode 100644 .ortversions/android.versions create mode 100644 .ortversions/base.versions create mode 100644 .ortversions/dart.versions create mode 100644 .ortversions/dotnet.versions create mode 100644 .ortversions/golang.versions create mode 100644 .ortversions/haskell.versions create mode 100644 .ortversions/nodejs.versions create mode 100644 .ortversions/php.versions create mode 100644 .ortversions/python.versions create mode 100644 .ortversions/ruby.versions create mode 100644 .ortversions/rust.versions create mode 100644 .ortversions/sbt.versions create mode 100644 .ortversions/swift.versions create mode 100755 scripts/custom_docker.sh create mode 100755 scripts/docker_build_all.sh create mode 100644 scripts/docker_snippets/android.snippet create mode 100644 scripts/docker_snippets/dart.snippet create mode 100644 scripts/docker_snippets/dotnet.snippet create mode 100644 scripts/docker_snippets/haskell.snippet create mode 100644 scripts/docker_snippets/php.snippet create mode 100644 scripts/docker_snippets/sbt.snippet create mode 100644 scripts/docker_snippets/swift.snippet diff --git a/.github/workflows/docker-ort-android.yml b/.github/workflows/docker-ort-android.yml new file mode 100644 index 0000000000000..e16b7d79b45a4 --- /dev/null +++ b/.github/workflows/docker-ort-android.yml @@ -0,0 +1,91 @@ +# Copyright (C) 2023 The ORT Project Authors (see ) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 +# License-Filename: LICENSE + +name: Ort Docker Android Container + +on: + workflow_dispatch: + pull_request: + branches: + - main + push: + paths: + - '.ortversions/android.versions' + - '.github/workflows/docker-ort-base.yml' + workflow_run: + workflows: ["Ort Docker Base Image"] + types: + - completed + +env: + REGISTRY: ghcr.io + +permissions: write-all + +jobs: + build: + name: Build ORT Android Image + runs-on: ubuntu-22.04 + permissions: + contents: read + packages: write + + steps: + - name: Checkout main repository + uses: actions/checkout@v3 + + - name: Set environment variables + run: | + cat .ortversions/android.versions >> $GITHUB_ENV + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + + - name: Login to GitHub Container Registry + uses: docker/login-action@v2 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract components metadata (tags, labels) for base image + id: meta + uses: docker/metadata-action@v4 + with: + images: | + ${{ env.REGISTRY }}/${{ github.repository }}/android + tags: | + type=semver,pattern={{version}},value=${{ env.ANDROID_CMD_VERSION }} + + + - name: Build ORT Android container + uses: docker/build-push-action@v4 + with: + context: . + target: android + push: true + load: false + build-args: | + ANDROID_CMD_VERSION=${{ env.ANDROID_CMD_VERSION }} + tags: | + ${{ steps.meta.outputs.tags }} + ${{ env.REGISTRY }}/${{ github.repository }}/android:latest + labels: ${{ steps.meta.outputs.labels }} + build-contexts: | + ort-base-image=docker-image://${{ env.REGISTRY }}/${{ github.repository }}/base:latest + cache-from: type=gha,scope=ortandroid + cache-to: type=gha,scope=ortandroid,mode=max diff --git a/.github/workflows/docker-ort-base.yml b/.github/workflows/docker-ort-base.yml new file mode 100644 index 0000000000000..dbd88acf322f6 --- /dev/null +++ b/.github/workflows/docker-ort-base.yml @@ -0,0 +1,83 @@ +# Copyright (C) 2023 The ORT Project Authors (see ) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 +# License-Filename: LICENSE + +name: Ort Docker Base Container + +on: + workflow_dispatch: + pull_request: + branches: + - main + push: + paths: + - 'Dockerfile' + - '.ortversions/base.versions' + +env: + REGISTRY: ghcr.io + +jobs: + build: + name: Build ORT Base image + runs-on: ubuntu-22.04 + permissions: + contents: read + packages: write + + steps: + - name: Checkout main repository + uses: actions/checkout@v3 + + - name: Set environment variables + run: | + cat .ortversions/base.versions >> $GITHUB_ENV + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + + - name: Login to GitHub Container Registry + uses: docker/login-action@v2 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract components metadata (tags, labels) for base image + id: meta + uses: docker/metadata-action@v4 + with: + images: | + ${{ env.REGISTRY }}/${{ github.repository }}/base + tags: + type=raw,value=${{ env.JAVA_VERSION }}-jdk-${{ env.UBUNTU_VERSION }} + + - name: Build ORT Base container + uses: docker/build-push-action@v4 + with: + context: . + target: ort-base-image + push: true + load: false + build-args: | + JAVA_VERSION=${{ env.JAVA_VERSION }} + UBUNTU_VERSION=${{ env.UBUNTU_VERSION }} + tags: | + ${{ steps.meta.outputs.tags }} + ${{ env.REGISTRY }}/${{ github.repository }}/base:latest + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=gha,scope=ortbase + cache-to: type=gha,scope=ortbase,mode=max diff --git a/.github/workflows/docker-ort-binaries.yml b/.github/workflows/docker-ort-binaries.yml new file mode 100644 index 0000000000000..2d7494a8f9cf7 --- /dev/null +++ b/.github/workflows/docker-ort-binaries.yml @@ -0,0 +1,87 @@ +# Copyright (C) 2023 The ORT Project Authors (see ) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 +# License-Filename: LICENSE + +name: Ort Binaries Container + +on: + push: + branches: + - main + pull_request: + branches: + - main + workflow_dispatch: + workflow_run: + workflows: ["Ort Docker Base Image"] + types: + - completed + +env: + REGISTRY: ghcr.io + +permissions: write-all + +jobs: + build: + name: Build ORT Binary Image + runs-on: ubuntu-22.04 + permissions: + contents: read + packages: write + + steps: + - name: Checkout main repository + uses: actions/checkout@v3 + with: + ref: main + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + + - name: Login to GitHub Container Registry + uses: docker/login-action@v2 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract components metadata (tags, labels) for base image + id: meta + uses: docker/metadata-action@v4 + with: + images: | + ${{ env.REGISTRY }}/${{ github.repository }}/ortbin + tags: | + type=raw,sha,enable=true,format=short + + - name: Build ORT ort container + uses: docker/build-push-action@v4 + with: + context: . + target: ort + push: true + load: false + build-args: | + ORT_VERSION=${{ env.ORT_VERSION }} + tags: | + ${{ steps.meta.outputs.tags }} + ${{ env.REGISTRY }}/${{ github.repository }}/ortbin:latest + labels: ${{ steps.meta.outputs.labels }} + build-contexts: | + ort-base-image=docker-image://${{ env.REGISTRY }}/${{ github.repository }}/base:latest + cache-from: type=gha,scope=ortort + cache-to: type=gha,scope=ortort,mode=max diff --git a/.github/workflows/docker-ort-dart.yml b/.github/workflows/docker-ort-dart.yml new file mode 100644 index 0000000000000..d4e3967828e99 --- /dev/null +++ b/.github/workflows/docker-ort-dart.yml @@ -0,0 +1,91 @@ +# Copyright (C) 2023 The ORT Project Authors (see ) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 +# License-Filename: LICENSE + +name: Ort Docker Dart Container + +on: + workflow_dispatch: + pull_request: + branches: + - main + push: + paths: + - '.ortversions/dart.versions' + - '.github/workflows/docker-ort-base.yml' + workflow_run: + workflows: ["Ort Docker Base Image"] + types: + - completed + +env: + REGISTRY: ghcr.io + +permissions: write-all + +jobs: + build: + name: Build ORT Dart Image + runs-on: ubuntu-22.04 + permissions: + contents: read + packages: write + + steps: + - name: Checkout main repository + uses: actions/checkout@v3 + + - name: Set environment variables + run: | + cat .ortversions/dart.versions >> $GITHUB_ENV + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + + - name: Login to GitHub Container Registry + uses: docker/login-action@v2 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract components metadata (tags, labels) for base image + id: meta + uses: docker/metadata-action@v4 + with: + images: | + ${{ env.REGISTRY }}/${{ github.repository }}/dart + tags: | + type=semver,pattern={{version}},value=${{ env.DART_VERSION }} + + + - name: Build ORT Dart container + uses: docker/build-push-action@v4 + with: + context: . + target: dart + push: ${{ github.event_name == 'push' }} + load: false + build-args: | + DART_VERSION=${{ env.DART_VERSION }} + tags: | + ${{ steps.meta.outputs.tags }} + ${{ env.REGISTRY }}/${{ github.repository }}/dart:latest + labels: ${{ steps.meta.outputs.labels }} + build-contexts: | + ort-base-image=docker-image://${{ env.REGISTRY }}/${{ github.repository }}/base:latest + cache-from: type=gha,scope=ortdart + cache-to: type=gha,scope=ortdart,mode=max diff --git a/.github/workflows/docker-ort-dotnet.yml b/.github/workflows/docker-ort-dotnet.yml new file mode 100644 index 0000000000000..98e81c3a814b1 --- /dev/null +++ b/.github/workflows/docker-ort-dotnet.yml @@ -0,0 +1,89 @@ +# Copyright (C) 2023 The ORT Project Authors (see ) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 +# License-Filename: LICENSE + +name: Ort Docker Dotnet Container + +on: + workflow_dispatch: + pull_request: + branches: + - main + push: + paths: + - '.ortversions/dotnet.versions' + - '.github/workflows/docker-ort-base.yml' + workflow_run: + workflows: ["Ort Docker Base Image"] + +env: + REGISTRY: ghcr.io + +permissions: write-all + +jobs: + build: + name: Build ORT Dotnet Image + runs-on: ubuntu-22.04 + permissions: + contents: read + packages: write + + steps: + - name: Checkout main repository + uses: actions/checkout@v3 + + - name: Set environment variables + run: | + cat .ortversions/dotnet.versions >> $GITHUB_ENV + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + + - name: Login to GitHub Container Registry + uses: docker/login-action@v2 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract components metadata (tags, labels) for base image + id: meta + uses: docker/metadata-action@v4 + with: + images: | + ${{ env.REGISTRY }}/${{ github.repository }}/dotnet + tags: | + type=raw,value=${{ env.DOTNET_VERSION }} + + - name: Build ORT Dotnet container + uses: docker/build-push-action@v4 + with: + context: . + target: dotnet + push: ${{ github.event_name == 'push' }} + load: false + build-args: | + DOTNET_VERSION=${{ env.DOTNET_VERSION }} + NUGET_INSPECTOR_VERSION=${{ env.NUGET_INSPECTOR_VERSION }} + tags: | + ${{ steps.meta.outputs.tags }} + ${{ env.REGISTRY }}/${{ github.repository }}/dotnet:latest + labels: ${{ steps.meta.outputs.labels }} + build-contexts: | + ort-base-image=docker-image://${{ env.REGISTRY }}/${{ github.repository }}/base:latest + cache-from: type=gha,scope=ortdotnet + cache-to: type=gha,scope=ortdotnet,mode=max diff --git a/.github/workflows/docker-ort-golang.yml b/.github/workflows/docker-ort-golang.yml new file mode 100644 index 0000000000000..a2220af15ec0e --- /dev/null +++ b/.github/workflows/docker-ort-golang.yml @@ -0,0 +1,92 @@ +# Copyright (C) 2023 The ORT Project Authors (see ) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 +# License-Filename: LICENSE + +name: Ort Golang Container + +on: + workflow_dispatch: + pull_request: + branches: + - main + push: + paths: + - '.ortversions/golang.versions' + - '.github/workflows/docker-ort-base.yml' + workflow_run: + workflows: ["Ort Docker Base Image"] + types: + - completed + +env: + REGISTRY: ghcr.io + +permissions: write-all + +jobs: + build: + name: Build ORT Golang Image + runs-on: ubuntu-22.04 + permissions: + contents: read + packages: write + + steps: + - name: Checkout main repository + uses: actions/checkout@v3 + + - name: Set environment variables + run: | + cat .ortversions/golang.versions >> $GITHUB_ENV + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + + - name: Login to GitHub Container Registry + uses: docker/login-action@v2 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract components metadata (tags, labels) for base image + id: meta + uses: docker/metadata-action@v4 + with: + images: | + ${{ env.REGISTRY }}/${{ github.repository }}/golang + tags: | + type=semver,pattern={{version}},value=${{ env.GO_VERSION }} + + + - name: Build ORT Golang container + uses: docker/build-push-action@v4 + with: + context: . + target: golang + push: ${{ github.event_name == 'push' }} + load: false + build-args: | + GO_DEP_VERSION=${{ env.GO_DEP_VERSION }} + GO_VERSION=${{ env.GO_VERSION }} + tags: | + ${{ steps.meta.outputs.tags }} + ${{ env.REGISTRY }}/${{ github.repository }}/golang:latest + labels: ${{ steps.meta.outputs.labels }} + build-contexts: | + ort-base-image=docker-image://${{ env.REGISTRY }}/${{ github.repository }}/base:latest + cache-from: type=gha,scope=ortgolang + cache-to: type=gha,scope=ortgolang,mode=max diff --git a/.github/workflows/docker-ort-haskell.yml b/.github/workflows/docker-ort-haskell.yml new file mode 100644 index 0000000000000..db9077e99d93b --- /dev/null +++ b/.github/workflows/docker-ort-haskell.yml @@ -0,0 +1,91 @@ +# Copyright (C) 2023 The ORT Project Authors (see ) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 +# License-Filename: LICENSE + +name: Ort Docker Haskell Container + +on: + workflow_dispatch: + pull_request: + branches: + - main + push: + paths: + - '.ortversions/haskell.versions' + - '.github/workflows/docker-ort-base.yml' + workflow_run: + workflows: ["Ort Docker Base Image"] + types: + - completed + +env: + REGISTRY: ghcr.io + +permissions: write-all + +jobs: + build: + name: Build ORT Haskell Image + runs-on: ubuntu-22.04 + permissions: + contents: read + packages: write + + steps: + - name: Checkout main repository + uses: actions/checkout@v3 + + - name: Set environment variables + run: | + cat .ortversions/haskell.versions >> $GITHUB_ENV + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + + - name: Login to GitHub Container Registry + uses: docker/login-action@v2 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract components metadata (tags, labels) for base image + id: meta + uses: docker/metadata-action@v4 + with: + images: | + ${{ env.REGISTRY }}/${{ github.repository }}/haskell + tags: | + type=semver,pattern={{version}},value=${{ env.HASKELL_STACK_VERSION }} + + + - name: Build ORT Haskell container + uses: docker/build-push-action@v4 + with: + context: . + target: haskell + push: true + load: false + build-args: | + HASKELL_STACK_VERSION=${{ env.HASKELL_STACK_VERSION }} + tags: | + ${{ steps.meta.outputs.tags }} + ${{ env.REGISTRY }}/${{ github.repository }}/haskell:latest + labels: ${{ steps.meta.outputs.labels }} + build-contexts: | + ort-base-image=docker-image://${{ env.REGISTRY }}/${{ github.repository }}/base:latest + cache-from: type=gha,scope=orthaskell + cache-to: type=gha,scope=orthaskell,mode=max diff --git a/.github/workflows/docker-ort-nodejs.yml b/.github/workflows/docker-ort-nodejs.yml new file mode 100644 index 0000000000000..bfe500ba0234c --- /dev/null +++ b/.github/workflows/docker-ort-nodejs.yml @@ -0,0 +1,93 @@ +# Copyright (C) 2023 The ORT Project Authors (see ) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 +# License-Filename: LICENSE + +name: Ort NodeJS Container + +on: + workflow_dispatch: + pull_request: + branches: + - main + push: + paths: + - '.ortversions/nodejs.versions' + - '.github/workflows/docker-ort-base.yml' + workflow_run: + workflows: ["Ort Docker Base Image"] + types: + - completed + +env: + REGISTRY: ghcr.io + +permissions: write-all + +jobs: + build: + name: Build ORT NodeJS Image + runs-on: ubuntu-22.04 + permissions: + contents: read + packages: write + + steps: + - name: Checkout main repository + uses: actions/checkout@v3 + + - name: Set environment variables + run: | + cat .ortversions/nodejs.versions >> $GITHUB_ENV + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + + - name: Login to GitHub Container Registry + uses: docker/login-action@v2 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract components metadata (tags, labels) for base image + id: meta + uses: docker/metadata-action@v4 + with: + images: | + ${{ env.REGISTRY }}/${{ github.repository }}/nodejs + tags: | + type=raw,value=${{ env.NODEJS_VERSION }} + + - name: Build ORT NodeJS container + uses: docker/build-push-action@v4 + with: + context: . + target: nodejs + push: ${{ github.event_name == 'push' }} + load: false + build-args: | + NODEJS_VERSION=${{ env.NODEJS_VERSION }} + NPM_VERSION=${{ env.NPM_VERSION }} + PNPM_VERSION=${{ env.PNPM_VERSION }} + YARN_VERSION=${{ env.YARN_VERSION }} + tags: | + ${{ steps.meta.outputs.tags }} + ${{ env.REGISTRY }}/${{ github.repository }}/nodejs:latest + labels: ${{ steps.meta.outputs.labels }} + build-contexts: | + ort-base-image=docker-image://${{ env.REGISTRY }}/${{ github.repository }}/base:latest + cache-from: type=gha,scope=ortnodejs + cache-to: type=gha,scope=ortnodejs,mode=max diff --git a/.github/workflows/docker-ort-python.yml b/.github/workflows/docker-ort-python.yml new file mode 100644 index 0000000000000..f82be976e3837 --- /dev/null +++ b/.github/workflows/docker-ort-python.yml @@ -0,0 +1,96 @@ +# Copyright (C) 2023 The ORT Project Authors (see ) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 +# License-Filename: LICENSE + +name: Ort Python Container + +on: + workflow_dispatch: + pull_request: + branches: + - main + push: + paths: + - '.ortversions/python.versions' + - '.github/workflows/docker-ort-base.yml' + workflow_run: + workflows: ["Ort Docker Base Image"] + types: + - completed + +env: + REGISTRY: ghcr.io + +permissions: write-all + +jobs: + build: + name: Build ORT Python Image + runs-on: ubuntu-22.04 + permissions: + contents: read + packages: write + + steps: + - name: Checkout main repository + uses: actions/checkout@v3 + + - name: Set environment variables + run: | + cat .ortversions/python.versions >> $GITHUB_ENV + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + + - name: Login to GitHub Container Registry + uses: docker/login-action@v2 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract components metadata (tags, labels) for base image + id: meta + uses: docker/metadata-action@v4 + with: + images: | + ${{ env.REGISTRY }}/${{ github.repository }}/python + tags: | + type=raw,value=${{ env.PYTHON_VERSION }} + + - name: Build ORT Python container + uses: docker/build-push-action@v4 + with: + context: . + target: python + push: ${{ github.event_name == 'push' }} + load: false + build-args: | + CONAN_VERSION=${{ env.CONAN_VERSION }} + PIPTOOL_VERSION=${{ env.PIPTOOL_VERSION }} + PYENV_GIT_TAG=${{ env.PYENV_GIT_TAG }} + PYTHON_INSPECTOR_VERSION=${{ env.PYTHON_INSPECTOR_VERSION }} + PYTHON_PIPENV_VERSION=${{ env.PYTHON_PIPENV_VERSION }} + PYTHON_POETRY_VERSION=${{ env.PYTHON_POETRY_VERSION }} + PYTHON_VERSION=${{ env.PYTHON_VERSION }} + SCANCODE_VERSION=${{ env.SCANCODE_VERSION }} + tags: | + ${{ steps.meta.outputs.tags }} + ${{ env.REGISTRY }}/${{ github.repository }}/python:${{ env.PYTHON_VERSION }} + ${{ env.REGISTRY }}/${{ github.repository }}/python:latest + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=gha,scope=ortpython + cache-to: type=gha,scope=ortpython,mode=max diff --git a/.github/workflows/docker-ort-ruby.yml b/.github/workflows/docker-ort-ruby.yml new file mode 100644 index 0000000000000..5848bf1875e98 --- /dev/null +++ b/.github/workflows/docker-ort-ruby.yml @@ -0,0 +1,91 @@ +# Copyright (C) 2023 The ORT Project Authors (see ) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 +# License-Filename: LICENSE + +name: Ort Ruby Container + +on: + workflow_run: + workflows: ["Ort Docker Base Image"] + workflow_dispatch: + pull_request: + branches: + - main + push: + paths: + - '.ortversions/ruby.versions' + - '.github/workflows/docker-ort-base.yml' + +env: + REGISTRY: ghcr.io + +permissions: write-all + +jobs: + build: + name: Build ORT Ruby Image + runs-on: ubuntu-22.04 + permissions: + contents: read + packages: write + + steps: + - name: Checkout main repository + uses: actions/checkout@v3 + + - name: Set environment variables + run: | + cat .ortversions/ruby.versions >> $GITHUB_ENV + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + + - name: Login to GitHub Container Registry + uses: docker/login-action@v2 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract components metadata (tags, labels) for base image + id: meta + uses: docker/metadata-action@v4 + with: + images: | + ${{ env.REGISTRY }}/${{ github.repository }}/ruby + tags: | + type=semver,pattern={{version}},value=${{ env.RUBY_VERSION }} + + + - name: Build ORT Ruby container + if: github.event_name == 'push' + uses: docker/build-push-action@v4 + with: + context: . + target: ruby + push: ${{ github.event_name == 'push' }} + load: false + build-args: | + RUBY_VERSION=${{ env.RUBY_VERSION }} + COCOAPODS_VERSION=${{ env.COCOAPODS_VERSION }} + tags: | + ${{ steps.meta.outputs.tags }} + ${{ env.REGISTRY }}/${{ github.repository }}/ruby:latest + labels: ${{ steps.meta.outputs.labels }} + build-contexts: | + ort-base-image=docker-image://${{ env.REGISTRY }}/${{ github.repository }}/base:latest + cache-from: type=gha,scope=ortruby + cache-to: type=gha,scope=ortruby,mode=max diff --git a/.github/workflows/docker-ort-runtime.yml b/.github/workflows/docker-ort-runtime.yml new file mode 100644 index 0000000000000..38dbbedd8b593 --- /dev/null +++ b/.github/workflows/docker-ort-runtime.yml @@ -0,0 +1,98 @@ + # Copyright (C) 2023 The ORT Project Authors (see ) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 +# License-Filename: LICENSE + +name: Ort Runtime Container + +on: + workflow_dispatch: + pull_request: + branches: + - main + push: + branches: + - main + workflow_run: + workflows: + - 'Ort Binaries Container' + - 'Ort Python Container' + - 'Ort NodeJS Container' + - 'Ort Ruby Container' + - 'Ort Rust Container' + - 'Ort Golang Container' + types: + - completed + +env: + REGISTRY: ghcr.io + +permissions: write-all + +jobs: + build: + name: Build ORT runtime Image + runs-on: ubuntu-22.04 + permissions: + contents: read + packages: write + + steps: + - name: Checkout default branch + uses: actions/checkout@v3 + + - name: Set environment variables + run: | + cat .ortversions/nodejs.versions >> $GITHUB_ENV + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + + - name: Login to GitHub Container Registry + uses: docker/login-action@v2 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract components metadata (tags, labels) for Ort runtime image + id: meta-ort + uses: docker/metadata-action@v4 + with: + images: | + ${{ env.REGISTRY }}/${{ github.repository_owner }}/ort + tags: | + type=raw,sha,enable=true,format=short + + - name: Build ORT runtime container + uses: docker/build-push-action@v4 + with: + context: . + target: run + push: ${{ github.event_name == 'push' }} + load: false + build-args: | + NODEJS_VERSION=${{ env.NODEJS_VERSION }} + tags: | + ${{ steps.meta-ort.outputs.tags }} + ${{ env.REGISTRY }}/${{ github.repository_owner }}/ort:latest + labels: ${{ steps.meta.outputs.labels }} + build-contexts: | + ort=docker-image://${{ env.REGISTRY }}/${{ github.repository }}/ortbin:latest + nodejs=docker-image://${{ env.REGISTRY }}/${{ github.repository }}/nodejs:latest + python=docker-image://${{ env.REGISTRY }}/${{ github.repository }}/python:latest + rust=docker-image://${{ env.REGISTRY }}/${{ github.repository }}/rust:latest + ruby=docker-image://${{ env.REGISTRY }}/${{ github.repository }}/ruby:latest + golang=docker-image://${{ env.REGISTRY }}/${{ github.repository }}/golang:latest diff --git a/.github/workflows/docker-ort-rust.yml b/.github/workflows/docker-ort-rust.yml new file mode 100644 index 0000000000000..8c94225689f7b --- /dev/null +++ b/.github/workflows/docker-ort-rust.yml @@ -0,0 +1,89 @@ +# Copyright (C) 2023 The ORT Project Authors (see ) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 +# License-Filename: LICENSE + +name: Ort Rust Container + +on: + workflow_dispatch: + pull_request: + branches: + - main + push: + paths: + - '.ortversions/rust.versions' + - '.github/workflows/docker-ort-base.yml' + workflow_run: + workflows: ["Ort Docker Base Image"] + +env: + REGISTRY: ghcr.io + +permissions: write-all + +jobs: + build: + name: Build ORT Rust Image + runs-on: ubuntu-22.04 + permissions: + contents: read + packages: write + + steps: + - name: Checkout main repository + uses: actions/checkout@v3 + + - name: Set environment variables + run: | + cat .ortversions/rust.versions >> $GITHUB_ENV + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + + - name: Login to GitHub Container Registry + uses: docker/login-action@v2 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract components metadata (tags, labels) for base image + id: meta + uses: docker/metadata-action@v4 + with: + images: | + ${{ env.REGISTRY }}/${{ github.repository }}/rust + tags: | + type=raw,value=${{ env.RUST_VERSION }} + + + - name: Build ORT Rust container + uses: docker/build-push-action@v4 + with: + context: . + target: rust + push: ${{ github.event_name == 'push' }} + load: false + build-args: | + RUST_VERSION=${{ env.RUST_VERSION }} + tags: | + ${{ steps.meta.outputs.tags }} + ${{ env.REGISTRY }}/${{ github.repository }}/rust:latest + labels: ${{ steps.meta.outputs.labels }} + build-contexts: | + ort-base-image=docker-image://${{ env.REGISTRY }}/${{ github.repository }}/base:latest + cache-from: type=gha,scope=ortrust + cache-to: type=gha,scope=ortrust,mode=max diff --git a/.github/workflows/docker-ort-sbt.yml b/.github/workflows/docker-ort-sbt.yml new file mode 100644 index 0000000000000..d8f32ce425b49 --- /dev/null +++ b/.github/workflows/docker-ort-sbt.yml @@ -0,0 +1,91 @@ +# Copyright (C) 2023 The ORT Project Authors (see ) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 +# License-Filename: LICENSE + +name: Ort Docker Sbt Container + +on: + workflow_dispatch: + pull_request: + branches: + - main + push: + paths: + - '.ortversions/sbt.versions' + - '.github/workflows/docker-ort-base.yml' + workflow_run: + workflows: ["Ort Docker Base Image"] + types: + - completed + +env: + REGISTRY: ghcr.io + +permissions: write-all + +jobs: + build: + name: Build ORT sbt Image + runs-on: ubuntu-22.04 + permissions: + contents: read + packages: write + + steps: + - name: Checkout main repository + uses: actions/checkout@v3 + + - name: Set environment variables + run: | + cat .ortversions/sbt.versions >> $GITHUB_ENV + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + + - name: Login to GitHub Container Registry + uses: docker/login-action@v2 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract components metadata (tags, labels) for base image + id: meta + uses: docker/metadata-action@v4 + with: + images: | + ${{ env.REGISTRY }}/${{ github.repository }}/sbt + tags: | + type=semver,pattern={{version}},value=${{ env.SBT_VERSION }} + + + - name: Build ORT sbt container + uses: docker/build-push-action@v4 + with: + context: . + target: sbt + push: ${{ github.event_name == 'push' }} + load: false + build-args: | + SBT_VERSION=${{ env.SBT_VERSION }} + tags: | + ${{ steps.meta.outputs.tags }} + ${{ env.REGISTRY }}/${{ github.repository }}/sbt:latest + labels: ${{ steps.meta.outputs.labels }} + build-contexts: | + ort-base-image=docker-image://${{ env.REGISTRY }}/${{ github.repository }}/base:latest + cache-from: type=gha,scope=ortsbt + cache-to: type=gha,scope=ortsbt,mode=max diff --git a/.github/workflows/docker-ort-swift.yml b/.github/workflows/docker-ort-swift.yml new file mode 100644 index 0000000000000..90d4a9fd26255 --- /dev/null +++ b/.github/workflows/docker-ort-swift.yml @@ -0,0 +1,91 @@ +# Copyright (C) 2023 The ORT Project Authors (see ) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 +# License-Filename: LICENSE + +name: Ort Docker Swift Container + +on: + workflow_dispatch: + pull_request: + branches: + - main + push: + paths: + - '.ortversions/swift.versions' + - '.github/workflows/docker-ort-base.yml' + workflow_run: + workflows: ["Ort Docker Base Image"] + types: + - completed + +env: + REGISTRY: ghcr.io + +permissions: write-all + +jobs: + build: + name: Build ORT Swift Image + runs-on: ubuntu-22.04 + permissions: + contents: read + packages: write + + steps: + - name: Checkout main repository + uses: actions/checkout@v3 + + - name: Set environment variables + run: | + cat .ortversions/swift.versions >> $GITHUB_ENV + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + + - name: Login to GitHub Container Registry + uses: docker/login-action@v2 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract components metadata (tags, labels) for base image + id: meta + uses: docker/metadata-action@v4 + with: + images: | + ${{ env.REGISTRY }}/${{ github.repository }}/swift + tags: | + type=raw,value=${{ env.SWIFT_VERSION }} + + + - name: Build ORT Swift container + uses: docker/build-push-action@v4 + with: + context: . + target: swift + push: ${{ github.event_name == 'push' }} + load: false + build-args: | + SWIFT_VERSION=${{ env.SWIFT_VERSION }} + tags: | + ${{ steps.meta.outputs.tags }} + ${{ env.REGISTRY }}/${{ github.repository }}/swift:latest + labels: ${{ steps.meta.outputs.labels }} + build-contexts: | + ort-base-image=docker-image://${{ env.REGISTRY }}/${{ github.repository }}/base:latest + cache-from: type=gha,scope=ortswift + cache-to: type=gha,scope=ortswift,mode=max diff --git a/.ortversions/android.versions b/.ortversions/android.versions new file mode 100644 index 0000000000000..6a895889da72a --- /dev/null +++ b/.ortversions/android.versions @@ -0,0 +1 @@ +ANDROID_CMD_VERSION=9477386 \ No newline at end of file diff --git a/.ortversions/base.versions b/.ortversions/base.versions new file mode 100644 index 0000000000000..abc7cc3fac010 --- /dev/null +++ b/.ortversions/base.versions @@ -0,0 +1,2 @@ +JAVA_VERSION=17 +UBUNTU_VERSION=jammy diff --git a/.ortversions/dart.versions b/.ortversions/dart.versions new file mode 100644 index 0000000000000..31191f8ff697a --- /dev/null +++ b/.ortversions/dart.versions @@ -0,0 +1 @@ +DART_VERSION=2.18.4 \ No newline at end of file diff --git a/.ortversions/dotnet.versions b/.ortversions/dotnet.versions new file mode 100644 index 0000000000000..29159cfc2b228 --- /dev/null +++ b/.ortversions/dotnet.versions @@ -0,0 +1,2 @@ +DOTNET_VERSION=6.0 +NUGET_INSPECTOR_VERSION=0.9.12 diff --git a/.ortversions/golang.versions b/.ortversions/golang.versions new file mode 100644 index 0000000000000..8625a6c84e258 --- /dev/null +++ b/.ortversions/golang.versions @@ -0,0 +1,2 @@ +GO_DEP_VERSION=0.5.4 +GO_VERSION=1.20.5 diff --git a/.ortversions/haskell.versions b/.ortversions/haskell.versions new file mode 100644 index 0000000000000..9e794390d449f --- /dev/null +++ b/.ortversions/haskell.versions @@ -0,0 +1 @@ +HASKELL_STACK_VERSION=2.7.5 \ No newline at end of file diff --git a/.ortversions/nodejs.versions b/.ortversions/nodejs.versions new file mode 100644 index 0000000000000..a278ab79d394e --- /dev/null +++ b/.ortversions/nodejs.versions @@ -0,0 +1,5 @@ +BOWER_VERSION=1.8.12 +NODEJS_VERSION=18.17.1 +NPM_VERSION=8.15.1 +PNPM_VERSION=7.8.0 +YARN_VERSION=1.22.17 diff --git a/.ortversions/php.versions b/.ortversions/php.versions new file mode 100644 index 0000000000000..8114d4c2b7b06 --- /dev/null +++ b/.ortversions/php.versions @@ -0,0 +1,2 @@ +PHP_VERSION=8.1 +COMPOSER_VERSION=2.2 \ No newline at end of file diff --git a/.ortversions/python.versions b/.ortversions/python.versions new file mode 100644 index 0000000000000..8225ff2022fa1 --- /dev/null +++ b/.ortversions/python.versions @@ -0,0 +1,8 @@ +CONAN_VERSION=1.57.0 +PIPTOOL_VERSION=22.2.2 +PYENV_GIT_TAG=v2.3.25 +PYTHON_INSPECTOR_VERSION=0.9.8 +PYTHON_PIPENV_VERSION=2022.9.24 +PYTHON_POETRY_VERSION=1.6.1 +PYTHON_VERSION=3.10.13 +SCANCODE_VERSION=31.2.4 diff --git a/.ortversions/ruby.versions b/.ortversions/ruby.versions new file mode 100644 index 0000000000000..71b8026d9091e --- /dev/null +++ b/.ortversions/ruby.versions @@ -0,0 +1,2 @@ +COCOAPODS_VERSION=1.11.2 +RUBY_VERSION=3.1.2 diff --git a/.ortversions/rust.versions b/.ortversions/rust.versions new file mode 100644 index 0000000000000..2e59451ab4622 --- /dev/null +++ b/.ortversions/rust.versions @@ -0,0 +1 @@ +RUST_VERSION=1.72.0 diff --git a/.ortversions/sbt.versions b/.ortversions/sbt.versions new file mode 100644 index 0000000000000..9e0c411cab528 --- /dev/null +++ b/.ortversions/sbt.versions @@ -0,0 +1 @@ +SBT_VERSION=1.6.1 diff --git a/.ortversions/swift.versions b/.ortversions/swift.versions new file mode 100644 index 0000000000000..46c2142b6fdb0 --- /dev/null +++ b/.ortversions/swift.versions @@ -0,0 +1 @@ +SWIFT_VERSION=5.8.1 diff --git a/Dockerfile b/Dockerfile index b021c4a055a00..3793af8203a25 100644 --- a/Dockerfile +++ b/Dockerfile @@ -19,9 +19,10 @@ # Set this to the Java version to use in the base image (and to build and run ORT with). ARG JAVA_VERSION=17 +ARG UBUNTU_VERSION=jammy # Use OpenJDK Eclipe Temurin Ubuntu LTS -FROM eclipse-temurin:$JAVA_VERSION-jdk-jammy as ort-base-image +FROM eclipse-temurin:$JAVA_VERSION-jdk-$UBUNTU_VERSION as ort-base-image ENV LANG=en_US.UTF-8 ENV LANGUAGE=en_US:en @@ -102,6 +103,12 @@ COPY "$CRT_FILES" /tmp/certificates/ RUN /etc/scripts/export_proxy_certificates.sh /tmp/certificates/ \ && /etc/scripts/import_certificates.sh /tmp/certificates/ +# Add syft to use as primary spdx docker scanner +# Create docs dir to store future spdxs +RUN curl -sSfL https://raw.githubusercontent.com/anchore/syft/main/install.sh | sudo sh -s -- -b /usr/local/bin \ + && mkdir -p /usr/share/doc/ort \ + && chown $USER:$USER /usr/share/doc/ort + USER $USER WORKDIR $HOME @@ -109,7 +116,7 @@ ENTRYPOINT [ "/bin/bash" ] #------------------------------------------------------------------------ # PYTHON - Build Python as a separate component with pyenv -FROM ort-base-image as pythonbuild +FROM ort-base-image AS pythonbuild SHELL ["/bin/bash", "-o", "pipefail", "-c"] @@ -126,8 +133,8 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ tk-dev \ && sudo rm -rf /var/lib/apt/lists/* -ARG PYTHON_VERSION=3.10.8 -ARG PYENV_GIT_TAG=v2.3.7 +ARG PYTHON_VERSION=3.10.13 +ARG PYENV_GIT_TAG=v2.3.25 ENV PYENV_ROOT=/opt/python ENV PATH=$PATH:$PYENV_ROOT/shims:$PYENV_ROOT/bin @@ -138,7 +145,7 @@ RUN curl -kSs https://pyenv.run | bash \ ARG CONAN_VERSION=1.57.0 ARG PYTHON_INSPECTOR_VERSION=0.9.8 ARG PYTHON_PIPENV_VERSION=2022.9.24 -ARG PYTHON_POETRY_VERSION=1.2.2 +ARG PYTHON_POETRY_VERSION=1.6.1 ARG PIPTOOL_VERSION=22.2.2 ARG SCANCODE_VERSION=32.0.6 @@ -148,22 +155,46 @@ RUN pip install --no-cache-dir -U \ && pip install --no-cache-dir -U \ Mercurial \ conan=="$CONAN_VERSION" \ + pip \ pipenv=="$PYTHON_PIPENV_VERSION" \ poetry=="$PYTHON_POETRY_VERSION" \ python-inspector=="$PYTHON_INSPECTOR_VERSION" RUN ARCH=$(arch | sed s/aarch64/arm64/) \ && if [ "$ARCH" == "arm64" ]; then \ - pip install -U scancode-toolkit-mini==$SCANCODE_VERSION; \ - else \ - curl -Os https://raw.githubusercontent.com/nexB/scancode-toolkit/v$SCANCODE_VERSION/requirements.txt; \ - pip install -U --constraint requirements.txt scancode-toolkit==$SCANCODE_VERSION; \ - rm requirements.txt; \ - fi + pip install -U scancode-toolkit-mini==$SCANCODE_VERSION; \ + else \ + curl -Os https://raw.githubusercontent.com/nexB/scancode-toolkit/v$SCANCODE_VERSION/requirements.txt; \ + pip install -U --constraint requirements.txt scancode-toolkit==$SCANCODE_VERSION; \ + rm requirements.txt; \ + fi FROM scratch AS python COPY --from=pythonbuild /opt/python /opt/python +#------------------------------------------------------------------------ +# NODEJS - Build NodeJS as a separate component with nvm +FROM ort-base-image AS nodejsbuild + +ARG BOWER_VERSION=1.8.12 +ARG NODEJS_VERSION=18.14.2 +ARG NPM_VERSION=8.15.1 +ARG PNPM_VERSION=7.8.0 +ARG YARN_VERSION=1.22.17 + +ENV NVM_DIR=/opt/nvm +ENV PATH=$PATH:$NVM_DIR/versions/node/v$NODEJS_VERSION/bin + +RUN git clone --depth 1 https://github.com/nvm-sh/nvm.git $NVM_DIR +RUN . $NVM_DIR/nvm.sh \ + && nvm install "$NODEJS_VERSION" \ + && nvm alias default "$NODEJS_VERSION" \ + && nvm use default \ + && npm install --global npm@$NPM_VERSION bower@$BOWER_VERSION pnpm@$PNPM_VERSION yarn@$YARN_VERSION + +FROM scratch AS nodejs +COPY --from=nodejsbuild /opt/nvm /opt/nvm + #------------------------------------------------------------------------ # RUBY - Build Ruby as a separate component with rbenv FROM ort-base-image AS rubybuild @@ -183,6 +214,7 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ ARG COCOAPODS_VERSION=1.11.2 ARG RUBY_VERSION=3.1.2 + ENV RBENV_ROOT=/opt/rbenv ENV PATH=$RBENV_ROOT/bin:$RBENV_ROOT/shims/:$RBENV_ROOT/plugins/ruby-build/bin:$PATH @@ -198,37 +230,15 @@ RUN rbenv install $RUBY_VERSION -v \ FROM scratch AS ruby COPY --from=rubybuild /opt/rbenv /opt/rbenv -#------------------------------------------------------------------------ -# NODEJS - Build NodeJS as a separate component with nvm -FROM ort-base-image AS nodebuild - -ARG BOWER_VERSION=1.8.12 -ARG NODEJS_VERSION=18.14.2 -ARG NPM_VERSION=8.15.1 -ARG PNPM_VERSION=7.8.0 -ARG YARN_VERSION=1.22.17 - -ENV NVM_DIR=/opt/nvm -ENV PATH=$PATH:$NVM_DIR/versions/node/v$NODEJS_VERSION/bin - -RUN git clone --depth 1 https://github.com/nvm-sh/nvm.git $NVM_DIR -RUN . $NVM_DIR/nvm.sh \ - && nvm install "$NODEJS_VERSION" \ - && nvm alias default "$NODEJS_VERSION" \ - && nvm use default \ - && npm install --global npm@$NPM_VERSION bower@$BOWER_VERSION pnpm@$PNPM_VERSION yarn@$YARN_VERSION - -FROM scratch AS node -COPY --from=nodebuild /opt/nvm /opt/nvm - #------------------------------------------------------------------------ # RUST - Build as a separate component FROM ort-base-image AS rustbuild -ARG RUST_HOME=/opt/rust -ARG CARGO_HOME=$RUST_HOME/cargo -ARG RUSTUP_HOME=$RUST_HOME/rustup -ARG RUST_VERSION=1.64.0 +ARG RUST_VERSION=1.72.0 + +ENV RUST_HOME=/opt/rust +ENV CARGO_HOME=$RUST_HOME/cargo +ENV RUSTUP_HOME=$RUST_HOME/rustup RUN curl -ksSf https://sh.rustup.rs | sh -s -- -y --profile minimal --default-toolchain $RUST_VERSION FROM scratch AS rust @@ -293,8 +303,8 @@ RUN --mount=type=tmpfs,target=/android \ && PROXY_HOST_AND_PORT=${https_proxy#*://} \ && PROXY_HOST_AND_PORT=${PROXY_HOST_AND_PORT%/} \ && if [ -n "$PROXY_HOST_AND_PORT" ]; then \ - # While sdkmanager uses HTTPS by default, the proxy type is still called "http". - SDK_MANAGER_PROXY_OPTIONS="--proxy=http --proxy_host=${PROXY_HOST_AND_PORT%:*} --proxy_port=${PROXY_HOST_AND_PORT##*:}"; \ + # While sdkmanager uses HTTPS by default, the proxy type is still called "http". + SDK_MANAGER_PROXY_OPTIONS="--proxy=http --proxy_host=${PROXY_HOST_AND_PORT%:*} --proxy_port=${PROXY_HOST_AND_PORT##*:}"; \ fi \ && yes | $ANDROID_HOME/cmdline-tools/bin/sdkmanager $SDK_MANAGER_PROXY_OPTIONS --sdk_root=$ANDROID_HOME "platform-tools" "cmdline-tools;latest" @@ -339,20 +349,55 @@ FROM scratch AS sbt COPY --from=sbtbuild /opt/sbt /opt/sbt #------------------------------------------------------------------------ -# SPM -FROM ort-base-image AS spmbuild +# SWIFT +FROM ort-base-image AS swiftbuild ARG SWIFT_VERSION=5.8.1 ENV SWIFT_HOME=/opt/swift ENV PATH=$PATH:$SWIFT_HOME/bin -RUN mkdir $SWIFT_HOME \ - && curl -L https://download.swift.org/swift-$SWIFT_VERSION-release/ubuntu2204/swift-$SWIFT_VERSION-RELEASE/swift-$SWIFT_VERSION-RELEASE-ubuntu22.04.tar.gz \ +RUN mkdir -p $SWIFT_HOME \ + && echo $SWIFT_VERSION \ + && if [ "$(arch)" = "aarch64" ]; then \ + SWIFT_PACKAGE="ubuntu2204-aarch64/swift-$SWIFT_VERSION-RELEASE/swift-$SWIFT_VERSION-RELEASE-ubuntu22.04-aarch64.tar.gz"; \ + else \ + SWIFT_PACKAGE="ubuntu2204/swift-$SWIFT_VERSION-RELEASE/swift-$SWIFT_VERSION-RELEASE-ubuntu22.04.tar.gz"; \ + fi \ + && curl -L https://download.swift.org/swift-$SWIFT_VERSION-release/$SWIFT_PACKAGE \ | tar -xz -C $SWIFT_HOME --strip-components=2 -FROM scratch AS spm -COPY --from=spmbuild /opt/swift /opt/swift +FROM scratch AS swift +COPY --from=swiftbuild /opt/swift /opt/swift + +#------------------------------------------------------------------------ +# DOTNET +FROM ort-base-image AS dotnetbuild + +ARG DOTNET_VERSION=6.0 +ARG NUGET_INSPECTOR_VERSION=0.9.12 + +ENV DOTNET_HOME=/opt/dotnet +ENV NUGET_INSPECTOR_HOME=$DOTNET_HOME +ENV PATH=$PATH:$DOTNET_HOME:$DOTNET_HOME/tools:$DOTNET_HOME/bin + +# Note: We are not installing a dotnet package directly because +# debian packages from Ubuntu and Microsoft are incomplete + +RUN mkdir -p $DOTNET_HOME \ + && echo $SWIFT_VERSION \ + && if [ "$(arch)" = "aarch64" ]; then \ + curl -L https://aka.ms/dotnet/$DOTNET_VERSION/dotnet-sdk-linux-arm64.tar.gz | tar -C $DOTNET_HOME -xz; \ + else \ + curl -L https://aka.ms/dotnet/$DOTNET_VERSION/dotnet-sdk-linux-x64.tar.gz | tar -C $DOTNET_HOME -xz; \ + fi + +RUN mkdir -p $DOTNET_HOME/bin \ + && curl -L https://github.com/nexB/nuget-inspector/releases/download/v$NUGET_INSPECTOR_VERSION/nuget-inspector-v$NUGET_INSPECTOR_VERSION-linux-x64.tar.gz \ + | tar --strip-components=1 -C $DOTNET_HOME/bin -xz + +FROM scratch AS dotnet +COPY --from=dotnetbuild /opt/dotnet /opt/dotnet #------------------------------------------------------------------------ # ORT @@ -370,135 +415,71 @@ RUN --mount=type=cache,target=/var/tmp/gradle \ && sudo chown -R "$USER". $HOME/src/ort /var/tmp/gradle \ && scripts/set_gradle_proxy.sh \ && ./gradlew --no-daemon --stacktrace \ - -Pversion=$ORT_VERSION \ - :cli:installDist \ - :helper-cli:startScripts \ + -Pversion=$ORT_VERSION \ + :cli:installDist \ + :helper-cli:startScripts \ && mkdir /opt/ort \ && cp -a $HOME/src/ort/cli/build/install/ort /opt/ \ && cp -a $HOME/src/ort/scripts/*.sh /opt/ort/bin/ \ && cp -a $HOME/src/ort/helper-cli/build/scripts/orth /opt/ort/bin/ \ && cp -a $HOME/src/ort/helper-cli/build/libs/helper-cli-*.jar /opt/ort/lib/ -FROM scratch AS ort -COPY --from=ortbuild /opt/ort /opt/ort +FROM scratch AS ortbin +COPY --from=ortbuild /opt/ort /opt/ort #------------------------------------------------------------------------ -# Components container -FROM ort-base-image as components +# Main Minimal Runtime container +FROM ort-base-image as run # Remove ort build scripts RUN [ -d /etc/scripts ] && sudo rm -rf /etc/scripts -# Apt install commands. +# Minor requirements RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ --mount=type=cache,target=/var/lib/apt,sharing=locked \ sudo apt-get update && \ DEBIAN_FRONTEND=noninteractive sudo apt-get install -y --no-install-recommends \ - php \ - subversion \ - # dotnet requirements - libc6 \ - libgcc1 \ - libgcc-s1 \ - libgssapi-krb5-2 \ - libicu70 \ - liblttng-ust1 \ - libssl3 \ - libstdc++6 \ - libunwind8 \ - zlib1g \ + subversion \ && sudo rm -rf /var/lib/apt/lists/* +RUN syft / --exclude '*/usr/share/doc' --exclude '*/etc' -o spdx-json --file /usr/share/doc/ort/ort-base.spdx.json + # Python ENV PYENV_ROOT=/opt/python ENV PATH=$PATH:$PYENV_ROOT/shims:$PYENV_ROOT/bin COPY --from=python --chown=$USER:$USER $PYENV_ROOT $PYENV_ROOT - -# Ruby -ENV RBENV_ROOT=/opt/rbenv/ -ENV GEM_HOME=/var/tmp/gem -ENV PATH=$PATH:$RBENV_ROOT/bin:$RBENV_ROOT/shims:$RBENV_ROOT/plugins/ruby-install/bin -COPY --from=ruby --chown=$USER:$USER $RBENV_ROOT $RBENV_ROOT +RUN syft $PYENV_ROOT -o spdx-json --file /usr/share/doc/ort/ort-python.spdx.json # NodeJS ARG NODEJS_VERSION=18.14.2 ENV NVM_DIR=/opt/nvm ENV PATH=$PATH:$NVM_DIR/versions/node/v$NODEJS_VERSION/bin -COPY --from=node --chown=$USER:$USER $NVM_DIR $NVM_DIR +COPY --from=nodejs --chown=$USER:$USER $NVM_DIR $NVM_DIR +RUN syft $NVM_DIR -o spdx-json --file /usr/share/doc/ort/ort-nodejs.spdx.json # Rust ENV RUST_HOME=/opt/rust ENV CARGO_HOME=$RUST_HOME/cargo ENV RUSTUP_HOME=$RUST_HOME/rustup ENV PATH=$PATH:$CARGO_HOME/bin:$RUSTUP_HOME/bin -COPY --from=rust --chown=$USER:$USER $RUST_HOME $RUST_HOME +COPY --from=ghcr.io/oss-review-toolkit/rust --chown=$USER:$USER $RUST_HOME $RUST_HOME RUN chmod o+rwx $CARGO_HOME +RUN syft $RUST_HOME -o spdx-json --file /usr/share/doc/ort/ort-rust.spdx.json # Golang ENV PATH=$PATH:/opt/go/bin -COPY --from=golang --chown=$USER:$USER /opt/go /opt/go - -# Haskell -ENV HASKELL_HOME=/opt/haskell -ENV PATH=$PATH:$HASKELL_HOME/bin -COPY --from=haskell --chown=$USER:$USER $HASKELL_HOME $HASKELL_HOME - -# Repo and Android -ENV ANDROID_HOME=/opt/android-sdk -ENV ANDROID_USER_HOME=$HOME/.android -ENV PATH=$PATH:$ANDROID_HOME/cmdline-tools/latest/bin:$ANDROID_HOME/cmdline-tools/bin -ENV PATH=$PATH:$ANDROID_HOME/platform-tools -COPY --from=android --chown=$USER:$USER $ANDROID_HOME $ANDROID_HOME -RUN chmod -R o+rw $ANDROID_HOME - -# Dart -ENV DART_SDK=/opt/dart-sdk -ENV PATH=$PATH:$DART_SDK/bin -COPY --from=dart --chown=$USER:$USER $DART_SDK $DART_SDK +COPY --from=ghcr.io/oss-review-toolkit/golang --chown=$USER:$USER /opt/go /opt/go +RUN syft /opt/go -o spdx-json --file /usr/share/doc/ort/ort-golang.spdx.json -# SBT -ENV SBT_HOME=/opt/sbt -ENV PATH=$PATH:$SBT_HOME/bin -COPY --from=sbt --chown=$USER:$USER $SBT_HOME $SBT_HOME - -# SPM -ENV SWIFT_HOME=/opt/swift -ENV PATH=$PATH:$SWIFT_HOME/bin -COPY --from=spm --chown=$USER:$USER $SWIFT_HOME $SWIFT_HOME - -# PHP composer -ARG COMPOSER_VERSION=2.2 - -ENV PATH=$PATH:/opt/php/bin -RUN mkdir -p /opt/php/bin \ - && curl -ksS https://getcomposer.org/installer | php -- --install-dir=/opt/php/bin --filename=composer --$COMPOSER_VERSION - -# nuget-inspector -ENV NUGET_INSPECTOR_HOME=/opt/nuget-inspector -ENV NUGET_INSPECTOR_BIN=$NUGET_INSPECTOR_HOME/bin -ENV DOTNET_HOME=$NUGET_INSPECTOR_HOME/dotnet - -ENV PATH=$PATH:$DOTNET_HOME:$DOTNET_HOME/tools:$NUGET_INSPECTOR_BIN - -# Note: We are not installing a dotnet package directly because -# debian packages from Ubuntu and Microsoft are incomplete -RUN mkdir -p $DOTNET_HOME \ - && curl -L https://aka.ms/dotnet/6.0/dotnet-sdk-linux-x64.tar.gz \ - | tar -C $DOTNET_HOME -xz - -ARG NUGET_INSPECTOR_VERSION=0.9.12 -RUN mkdir -p $NUGET_INSPECTOR_BIN \ - && curl -L https://github.com/nexB/nuget-inspector/releases/download/v$NUGET_INSPECTOR_VERSION/nuget-inspector-v$NUGET_INSPECTOR_VERSION-linux-x64.tar.gz \ - | tar --strip-components=1 -C $NUGET_INSPECTOR_BIN -xz - -ENTRYPOINT ["/bin/bash"] - -#------------------------------------------------------------------------ -# Main Runtime container -FROM components AS run +# Ruby +ENV RBENV_ROOT=/opt/rbenv/ +ENV GEM_HOME=/var/tmp/gem +ENV PATH=$PATH:$RBENV_ROOT/bin:$RBENV_ROOT/shims:$RBENV_ROOT/plugins/ruby-install/bin +COPY --from=ghcr.io/oss-review-toolkit/ruby --chown=$USER:$USER $RBENV_ROOT $RBENV_ROOT +RUN syft $RBENV_ROOT -o spdx-json --file /usr/share/doc/ort/ort-ruby.spdx.json # ORT -COPY --from=ort --chown=$USER:$USER /opt/ort /opt/ort +COPY --from=ortbin --chown=$USER:$USER /opt/ort /opt/ort ENV PATH=$PATH:/opt/ort/bin USER $USER diff --git a/scripts/custom_docker.sh b/scripts/custom_docker.sh new file mode 100755 index 0000000000000..f0d1ca575218f --- /dev/null +++ b/scripts/custom_docker.sh @@ -0,0 +1,100 @@ +#!/usr/bin/env bash + +# Copyright (C) 2023 The ORT Project Authors (see ) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 +# License-Filename: LICENSE + +# Current script dir +SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd) + +# Docker root +DOCKER_IMAGE_ROOT="${DOCKER_IMAGE_ROOT:-ghcr.io/oss-review-toolkit}" + +# Define the list of valid components +valid_components=("android" "swift" "sbt" "dart" "dotnet" "php" "haskell") + +# Define the Dockerfile template +dockerfile_template="FROM ghcr.io/oss-review-toolkit/ort\n" + +# Default output file +output_file="Dockerfile.custom" + +function usage() { + echo "Usage: $0 -c [ ...] -output " + echo "Options:" + echo " -c [ ...]: List of language components to include in the Dockerfile: ${valid_components[*]}" + echo " -output : Output file for the generated Dockerfile, Defaults to Dockerfile.custom." + echo " -h: Display this help message" +} + +# Parse the command-line options +while [[ $# -gt 0 ]]; do + case "$1" in + -c) + shift + components=("$@") + break + ;; + -output) + shift + output_file=$1 + ;; + -h) + usage + exit 0 + ;; + *) + echo "Invalid option: $1" + usage + exit 1 + ;; + esac + shift +done + +# Check if the required options are present +if [[ -z "${components[*]}" ]]; then + echo "Missing required options" + usage + exit 1 +fi + +# Check if the components are valid +for component in "${components[@]}"; do + valid=false + for valid_component in "${valid_components[@]}"; do + if [[ "${valid_component}" == "${component}" ]]; then + valid=true + break + fi + done + + if [[ "${valid}" == false ]]; then + echo "Invalid component: ${component}" + usage + exit 1 + fi +done + +# Write the Dockerfile to the output file +echo -e "${dockerfile_template}" >"${output_file}" + +# Add the components to the Dockerfile template +for component in "${components[@]}"; do + # Add the component to the custom Dockerfile + cat "${SCRIPT_DIR}/docker_snippets/${component}.snippet" >>"${output_file}" + echo -e "\n" >>"${output_file}" +done diff --git a/scripts/docker_build.sh b/scripts/docker_build.sh index b4ed2b0894c81..cae24125d0dc3 100755 --- a/scripts/docker_build.sh +++ b/scripts/docker_build.sh @@ -17,15 +17,151 @@ # SPDX-License-Identifier: Apache-2.0 # License-Filename: LICENSE -DOCKER_ARGS=$@ +set -e -o pipefail GIT_ROOT=$(git rev-parse --show-toplevel) GIT_REVISION=$($GIT_ROOT/gradlew -q properties --property version | sed -nr "s/version: (.+)/\1/p") echo "Setting ORT_VERSION to $GIT_REVISION." -docker buildx build \ - -f "$GIT_ROOT/Dockerfile" \ - -t "${ORT_DOCKER_TAG:-ort}" \ + +# --------------------------- +# image_build function +# Usage ( position paramenters): +# image_build + +image_build() { + local target + local name + local version + target="$1" + shift + name="$1" + shift + version="$1" + shift + + docker buildx build \ + -f "$GIT_ROOT/Dockerfile" \ + --target "$target" \ + --tag "${DOCKER_IMAGE_ROOT}/$name:$version" \ + --tag "${DOCKER_IMAGE_ROOT}/$name:latest" \ + --build-context "base=docker-image://${DOCKER_IMAGE_ROOT}/base:latest" \ + "$@" . +} + +# Base +# shellcheck disable=SC1091 +. .ortversions/base.versions +image_build ort-base-image base "${JAVA_VERSION}-jdk-${UBUNTU_VERSION}" \ + --build-arg UBUNTU_VERSION="$UBUNTU_VERSION" \ + --build-arg JAVA_VERSION="$JAVA_VERSION" \ + "$@" + +# Python +# shellcheck disable=SC1091 +. .ortversions/python.versions +image_build python python "$PYTHON_VERSION" \ + --build-arg PYTHON_VERSION="$PYTHON_VERSION" \ + --build-arg CONAN_VERSION="$CONAN_VERSION" \ + --build-arg PYTHON_INSPECTOR_VERSION="$PYTHON_INSPECTOR_VERSION" \ + --build-arg PYTHON_PIPENV_VERSION="$PYTHON_PIPENV_VERSION" \ + --build-arg PYTHON_POETRY_VERSION="$PYTHON_POETRY_VERSION" \ + --build-arg PIPTOOL_VERSION="$PIPTOOL_VERSION" \ + --build-arg SCANCODE_VERSION="$SCANCODE_VERSION" \ + "$@" + +# Nodejs +# shellcheck disable=SC1091 +. .ortversions/nodejs.versions +image_build nodejs nodejs "$NODEJS_VERSION" \ + --build-arg NODEJS_VERSION="$NODEJS_VERSION" \ + --build-arg BOWER_VERSION="$BOWER_VERSION" \ + --build-arg NPM_VERSION="$NPM_VERSION" \ + --build-arg PNPM_VERSION="$PNPM_VERSION" \ + --build-arg YARN_VERSION="$YARN_VERSION" \ + "$@" + +# Rust +# shellcheck disable=SC1091 +. .ortversions/rust.versions +image_build rust rust "$RUST_VERSION" \ + --build-arg RUST_VERSION="$RUST_VERSION" \ + "$@" + +# Ruby +# shellcheck disable=SC1091 +. .ortversions/ruby.versions +image_build ruby ruby "$RUBY_VERSION" \ + --build-arg RUBY_VERSION="$RUBY_VERSION" \ + --build-arg COCOAPODS_VERSION="$COCOAPODS_VERSION" \ + "$@" + +# Golang +# shellcheck disable=SC1091 +. .ortversions/golang.versions +image_build golang golang "$GO_VERSION" \ + --build-arg GO_VERSION="$GO_VERSION" \ + --build-arg GO_DEP_VERSION="$GO_DEP_VERSION" \ + "$@" + +# Ort +image_build ortbin ortbin "$GIT_REVISION" \ --build-arg ORT_VERSION="$GIT_REVISION" \ - $DOCKER_ARGS \ - "$GIT_ROOT" + "$@" + +# Runtime ORT image +image_build run ort "$GIT_REVISION" \ + --build-context "python=docker-image://${DOCKER_IMAGE_ROOT}/python:latest" \ + --build-arg NODEJS_VERSION="$NODEJS_VERSION" \ + --build-context "nodejs=docker-image://${DOCKER_IMAGE_ROOT}/nodejs:latest" \ + --build-context "rust=docker-image://${DOCKER_IMAGE_ROOT}/rust:latest" \ + --build-context "golang=docker-image://${DOCKER_IMAGE_ROOT}/golang:latest" \ + --build-context "ruby=docker-image://${DOCKER_IMAGE_ROOT}/ruby:latest" \ + --build-context "ortbin=docker-image://${DOCKER_IMAGE_ROOT}/ortbin:latest" \ + "$@" + +# Build adjacent language containers if ALL_LANGUAGES is set. +[ -z "$ALL_LANGUAGES" ] && exit 0 + +# Android +# shellcheck disable=SC1091 +. .ortversions/android.versions +image_build android android "$ANDROID_CMD_VERSION" \ + --build-arg ANDROID_CMD_VERSION="$ANDROID_CMD_VERSION" \ + "$@" + +# Swift +# shellcheck disable=SC1091 +. .ortversions/swift.versions +image_build swift swift "$SWIFT_VERSION" \ + --build-arg SWIFT_VERSION="$SWIFT_VERSION" \ + "$@" + +# SBT +# shellcheck disable=SC1091 +. .ortversions/sbt.versions +image_build sbt sbt "$SBT_VERSION" \ + --build-arg SBT_VERSION="$SBT_VERSION" \ + "$@" + +# Dart +# shellcheck disable=SC1091 +. .ortversions/dart.versions +image_build dart dart "$DART_VERSION" \ + --build-arg DART_VERSION="$DART_VERSION" \ + "$@" + +# Dotnet +# shellcheck disable=SC1091 +. .ortversions/dotnet.versions +image_build dotnet dotnet "$DOTNET_VERSION" \ + --build-arg DOTNET_VERSION="$DOTNET_VERSION" \ + --build-arg NUGET_INSPECTOR_VERSION="$NUGET_INSPECTOR_VERSION" \ + "$@" + +# Dotnet +# shellcheck disable=SC1091 +. .ortversions/haskell.versions +image_build dotnet dotnet "$HASKELL_STACK_VERSION" \ + --build-arg HASKELL_STACK_VERSION="$HASKELL_STACK_VERSION" \ + "$@" diff --git a/scripts/docker_build_all.sh b/scripts/docker_build_all.sh new file mode 100755 index 0000000000000..e99e86eee37f3 --- /dev/null +++ b/scripts/docker_build_all.sh @@ -0,0 +1,144 @@ +#!/bin/bash +# +# Copyright (C) 2020 The ORT Project Authors (see ) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 +# License-Filename: LICENSE + +set -e -o pipefail + +GIT_ROOT=$(git rev-parse --show-toplevel) +GIT_REVISION=$(git describe --abbrev=10 --always --tags --dirty --match=[0-9]*) +DOCKER_IMAGE_ROOT="${DOCKER_IMAGE_ROOT:-ort}" + +echo "Setting ORT_VERSION to $GIT_REVISION." + +# --------------------------- +# image_build function +# Usage ( position paramenters): +# image_build + +image_build() { + local target + local name + local version + target="$1" + shift + name="$1" + shift + version="$1" + shift + + docker buildx build \ + -f "$GIT_ROOT/Dockerfile" \ + --target "$target" \ + --tag "${DOCKER_IMAGE_ROOT}/$name:$version" \ + --tag "${DOCKER_IMAGE_ROOT}/$name:latest" \ + --tag "ghcr.io/${DOCKER_IMAGE_ROOT}/$name:$version" \ + --tag "ghcr.io/${DOCKER_IMAGE_ROOT}/$name:latest" \ + --build-context "base=docker-image://${DOCKER_IMAGE_ROOT}/base:latest" \ + "$@" . +} + +#for containers in base android dart golang haskell nodejs python ruby rust sbt spm binaries runtime; do +# Base +image_build ort-base-image base "$UBUNTU_VERSION" --build-arg JAVA_VERSION="$JAVA_VERSION" "$@" + +# Python +image_build python python "$PYTHON_VERSION" \ + --build-arg PYTHON_VERSION="$PYTHON_VERSION" \ + --build-arg CONAN_VERSION="$CONAN_VERSION" \ + --build-arg PYTHON_INSPECTOR_VERSION="$PYTHON_INSPECTOR_VERSION" \ + --build-arg PYTHON_PIPENV_VERSION="$PYTHON_PIPENV_VERSION" \ + --build-arg PYTHON_POETRY_VERSION="$PYTHON_POETRY_VERSION" \ + --build-arg PIPTOOL_VERSION="$PIPTOOL_VERSION" \ + --build-arg SCANCODE_VERSION="$SCANCODE_VERSION" \ + "$@" + +# Ruby +image_build ruby ruby "$RUBY_VERSION" \ + --build-arg RUBY_VERSION="$RUBY_VERSION" \ + --build-arg COCOAPODS_VERSION="$COCOAPODS_VERSION" \ + "$@" + +# Nodejs +image_build nodejs nodejs "$NODEJS_VERSION" \ + --build-arg NODEJS_VERSION="$NODEJS_VERSION" \ + --build-arg BOWER_VERSION="$BOWER_VERSION" \ + --build-arg NPM_VERSION="$NPM_VERSION" \ + --build-arg PNPM_VERSION="$PNPM_VERSION" \ + --build-arg YARN_VERSION="$YARN_VERSION" \ + "$@" + +# Rust +image_build rust rust "$RUST_VERSION" \ + --build-arg RUST_VERSION="$RUST_VERSION" \ + "$@" + +# Golang +image_build golang golang "$GO_VERSION" \ + --build-arg GO_VERSION="$GO_VERSION" \ + --build-arg GO_DEP_VERSION="$GO_DEP_VERSION" \ + "$@" + +# Haskell +image_build haskell haskell "$HASKELL_STACK_VERSION" \ + --build-arg HASKELL_STACK_VERSION="$HASKELL_STACK_VERSION" \ + "$@" + +# Android +image_build android android "$ANDROID_CMD_VERSION" \ + --build-arg ANDROID_CMD_VERSION="$ANDROID_CMD_VERSION" \ + "$@" + +# Dart +image_build dart dart "$DART_VERSION" \ + --build-arg DART_VERSION="$DART_VERSION" \ + "$@" + +# SBT +image_build sbt sbt "$SBT_VERSION" \ + --build-arg SBT_VERSION="$SBT_VERSION" \ + "$@" + +# Spm +image_build spm spm "$SWIFT_VERSION" \ + --build-arg SWIFT_VERSION="$SWIFT_VERSION" \ + "$@" + +# Ort +image_build ort binaries "$GIT_REVISION" \ + --build-arg ORT_VERSION="$GIT_REVISION" \ + "$@" + +# Runtime ORT image +image_build run ort "$GIT_REVISION" \ + --build-context "python=docker-image://${DOCKER_IMAGE_ROOT}/python:latest" \ + --build-context "ruby=docker-image://${DOCKER_IMAGE_ROOT}/ruby:latest" \ + --build-context "python=docker-image://${DOCKER_IMAGE_ROOT}/python:latest" \ + --build-arg NODEJS_VERSION="$NODEJS_VERSION" \ + --build-context "nodejs=docker-image://${DOCKER_IMAGE_ROOT}/nodejs:latest" \ + --build-context "rust=docker-image://${DOCKER_IMAGE_ROOT}/rust:latest" \ + --build-context "golang=docker-image://${DOCKER_IMAGE_ROOT}/golang:latest" \ + --build-context "haskell=docker-image://${DOCKER_IMAGE_ROOT}/haskell:latest" \ + --build-context "android=docker-image://${DOCKER_IMAGE_ROOT}/android:latest" \ + --build-context "dart=docker-image://${DOCKER_IMAGE_ROOT}/dart:latest" \ + --build-context "sbt=docker-image://${DOCKER_IMAGE_ROOT}/sbt:latest" \ + --build-context "spm=docker-image://${DOCKER_IMAGE_ROOT}/spm:latest" \ + --build-context "ort=docker-image://${DOCKER_IMAGE_ROOT}/binaries:latest" \ + --build-arg COMPOSER_VERSION="$COMPOSER_VERSION" \ + --build-arg NUGET_INSPECTOR_VERSION="$NUGET_INSPECTOR_VERSION" \ + --tag "ghcr.io/vwdfive/ort:latest" +"$@" diff --git a/scripts/docker_snippets/android.snippet b/scripts/docker_snippets/android.snippet new file mode 100644 index 0000000000000..59b33f357e088 --- /dev/null +++ b/scripts/docker_snippets/android.snippet @@ -0,0 +1,26 @@ +# Copyright (C) 2023 The ORT Project Authors (see ) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 +# License-Filename: LICENSE + +# Repo and Android +ENV ANDROID_HOME=/opt/android-sdk +ENV ANDROID_USER_HOME=$HOME/.android +ENV PATH=$PATH:$ANDROID_HOME/cmdline-tools/latest/bin:$ANDROID_HOME/cmdline-tools/bin +ENV PATH=$PATH:$ANDROID_HOME/platform-tools +COPY --from=ghcr.io/oss-review-toolkit/android --chown=$USER:$USER $ANDROID_HOME $ANDROID_HOME +RUN chmod -R o+rw $ANDROID_HOME + +RUN syft $ANDROID_HOME -o spdx-json --file /usr/share/doc/ort/ort-android.spdx.json diff --git a/scripts/docker_snippets/dart.snippet b/scripts/docker_snippets/dart.snippet new file mode 100644 index 0000000000000..cb6869c6cef38 --- /dev/null +++ b/scripts/docker_snippets/dart.snippet @@ -0,0 +1,22 @@ +# Copyright (C) 2023 The ORT Project Authors (see ) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 +# License-Filename: LICENSE + +ENV DART_SDK=/opt/dart-sdk +ENV PATH=$PATH:$DART_SDK/bin +COPY --from=ghcr.io/oss-review-toolkit/dart --chown=$USER:$USER $DART_SDK $DART_SDK + +RUN syft $DART_SDK -o spdx-json --file /usr/share/doc/ort/ort-golang.dart.json diff --git a/scripts/docker_snippets/dotnet.snippet b/scripts/docker_snippets/dotnet.snippet new file mode 100644 index 0000000000000..a5285e9fa6234 --- /dev/null +++ b/scripts/docker_snippets/dotnet.snippet @@ -0,0 +1,24 @@ +# Copyright (C) 2023 The ORT Project Authors (see ) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 +# License-Filename: LICENSE + +ENV DOTNET_HOME=/opt/dotnet +ENV NUGET_INSPECTOR_HOME=$DOTNET_HOME +ENV PATH=$PATH:$DOTNET_HOME:$DOTNET_HOME/tools:$DOTNET_HOME/bin + +COPY --from=ghcr.io/oss-review-toolkit/dotnet --chown=$USER:$USER $DOTNET_HOME $DOTNET_HOME + +RUN syft $DOTNET_HOME -o spdx-json --file /usr/share/doc/ort/ort-dotnet.spdx.json diff --git a/scripts/docker_snippets/haskell.snippet b/scripts/docker_snippets/haskell.snippet new file mode 100644 index 0000000000000..b974d2937c661 --- /dev/null +++ b/scripts/docker_snippets/haskell.snippet @@ -0,0 +1,32 @@ +# Copyright (C) 2023 The ORT Project Authors (see ) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 +# License-Filename: LICENSE + +ARG HASKELL_STACK_VERSION=2.7.5 +ENV HASKELL_HOME=/opt/haskell +ENV PATH=$PATH:$HASKELL_HOME/bin + +RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ + --mount=type=cache,target=/var/lib/apt,sharing=locked \ + sudo apt-get update -qq \ + && DEBIAN_FRONTEND=noninteractive sudo apt-get install -y --no-install-recommends \ + zlib1g-dev \ + && sudo rm -rf /var/lib/apt/lists/* + +RUN curl -sSL https://get.haskellstack.org/ | bash -s -- -d $HASKELL_HOME/bin + +FROM scratch AS haskell +COPY --from=haskellbuild /opt/haskell /opt/haskell \ No newline at end of file diff --git a/scripts/docker_snippets/php.snippet b/scripts/docker_snippets/php.snippet new file mode 100644 index 0000000000000..cba7338b7ea9a --- /dev/null +++ b/scripts/docker_snippets/php.snippet @@ -0,0 +1,34 @@ +# Copyright (C) 2023 The ORT Project Authors (see ) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 +# License-Filename: LICENSE + +ARG PHP_VERSION=8.1 +ARG COMPOSER_VERSION=2.2 + +# Apt install commands. +RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ + --mount=type=cache,target=/var/lib/apt,sharing=locked \ + sudo apt-get update && \ + DEBIAN_FRONTEND=noninteractive sudo apt-get install -y --no-install-recommends \ + php${PHP_VERSION} \ + && sudo rm -rf /var/lib/apt/lists/* + +RUN mkdir -p /opt/php/bin \ + && curl -ksS https://getcomposer.org/installer | php -- --install-dir=/opt/php/bin --filename=composer --$COMPOSER_VERSION + +ENV PATH=$PATH:/opt/php/bin + +RUN syft /opt/php -o spdx-json --file /usr/share/doc/ort/ort-php.spdx.json \ No newline at end of file diff --git a/scripts/docker_snippets/sbt.snippet b/scripts/docker_snippets/sbt.snippet new file mode 100644 index 0000000000000..d73e03296cdde --- /dev/null +++ b/scripts/docker_snippets/sbt.snippet @@ -0,0 +1,22 @@ +# Copyright (C) 2023 The ORT Project Authors (see ) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 +# License-Filename: LICENSE + +ENV SBT_HOME=/opt/sbt +ENV PATH=$PATH:$SBT_HOME/bin +COPY --from=ghcr.io/oss-review-toolkit/sbt --chown=$USER:$USER $SBT_HOME $SBT_HOME + +RUN syft $SBT_HOME -o spdx-json --file /usr/share/doc/ort/ort-sbt.spdx.json diff --git a/scripts/docker_snippets/swift.snippet b/scripts/docker_snippets/swift.snippet new file mode 100644 index 0000000000000..e4956513c53d2 --- /dev/null +++ b/scripts/docker_snippets/swift.snippet @@ -0,0 +1,22 @@ +# Copyright (C) 2023 The ORT Project Authors (see ) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 +# License-Filename: LICENSE + +ENV SWIFT_HOME=/opt/swift +ENV PATH=$PATH:$SWIFT_HOME/bin +COPY --from=ghcr.io/oss-review-toolkit/swift --chown=$USER:$USER $SWIFT_HOME $SWIFT_HOME + +RUN syft $SWIFT_HOME -o spdx-json --file /usr/share/doc/ort/ort-swift.spdx.json From 19a5ee27310d42b64d452686e898e1f93e8cbdd2 Mon Sep 17 00:00:00 2001 From: Helio Chissini de Castro Date: Thu, 7 Sep 2023 12:23:53 +0200 Subject: [PATCH 10/47] feat(docker): Use runtime to do all the work for binaries Signed-off-by: Helio Chissini de Castro --- .github/workflows/docker-ort-android.yml | 12 +- .github/workflows/docker-ort-base.yml | 12 +- .github/workflows/docker-ort-binaries.yml | 87 ------------- .github/workflows/docker-ort-dart.yml | 10 +- .github/workflows/docker-ort-dotnet.yml | 10 +- .github/workflows/docker-ort-golang.yml | 10 +- .github/workflows/docker-ort-haskell.yml | 10 +- .github/workflows/docker-ort-nodejs.yml | 10 +- .github/workflows/docker-ort-python.yml | 10 +- .github/workflows/docker-ort-ruby.yml | 13 +- .github/workflows/docker-ort-runtime.yml | 16 ++- .github/workflows/docker-ort-rust.yml | 10 +- .github/workflows/docker-ort-sbt.yml | 10 +- .github/workflows/docker-ort-swift.yml | 10 +- .ortversions/python.versions | 2 +- Dockerfile | 6 +- Dockerfile-extended | 86 +++++++++++++ scripts/custom_docker.sh | 4 +- scripts/docker_build.sh | 5 +- scripts/docker_build_all.sh | 144 ---------------------- scripts/docker_snippets/android.snippet | 2 +- scripts/docker_snippets/haskell.snippet | 14 +-- 22 files changed, 170 insertions(+), 323 deletions(-) delete mode 100644 .github/workflows/docker-ort-binaries.yml create mode 100644 Dockerfile-extended delete mode 100755 scripts/docker_build_all.sh diff --git a/.github/workflows/docker-ort-android.yml b/.github/workflows/docker-ort-android.yml index e16b7d79b45a4..02d6914d75fa6 100644 --- a/.github/workflows/docker-ort-android.yml +++ b/.github/workflows/docker-ort-android.yml @@ -15,19 +15,19 @@ # SPDX-License-Identifier: Apache-2.0 # License-Filename: LICENSE -name: Ort Docker Android Container +name: Android Image on: workflow_dispatch: pull_request: - branches: - - main - push: paths: - '.ortversions/android.versions' - '.github/workflows/docker-ort-base.yml' + push: + branches: + - main workflow_run: - workflows: ["Ort Docker Base Image"] + workflows: ["Base Image"] types: - completed @@ -77,7 +77,7 @@ jobs: with: context: . target: android - push: true + push: ${{ github.event_name == 'push' || github.event_name == 'workflow_dispatch' }} load: false build-args: | ANDROID_CMD_VERSION=${{ env.ANDROID_CMD_VERSION }} diff --git a/.github/workflows/docker-ort-base.yml b/.github/workflows/docker-ort-base.yml index dbd88acf322f6..5f0589ce94d18 100644 --- a/.github/workflows/docker-ort-base.yml +++ b/.github/workflows/docker-ort-base.yml @@ -15,17 +15,17 @@ # SPDX-License-Identifier: Apache-2.0 # License-Filename: LICENSE -name: Ort Docker Base Container +name: Base Image on: workflow_dispatch: pull_request: - branches: - - main - push: paths: - - 'Dockerfile' - '.ortversions/base.versions' + - 'Dockerfile' + push: + branches: + - main env: REGISTRY: ghcr.io @@ -70,7 +70,7 @@ jobs: with: context: . target: ort-base-image - push: true + push: ${{ github.event_name == 'push' || github.event_name == 'workflow_dispatch' }} load: false build-args: | JAVA_VERSION=${{ env.JAVA_VERSION }} diff --git a/.github/workflows/docker-ort-binaries.yml b/.github/workflows/docker-ort-binaries.yml deleted file mode 100644 index 2d7494a8f9cf7..0000000000000 --- a/.github/workflows/docker-ort-binaries.yml +++ /dev/null @@ -1,87 +0,0 @@ -# Copyright (C) 2023 The ORT Project Authors (see ) -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# SPDX-License-Identifier: Apache-2.0 -# License-Filename: LICENSE - -name: Ort Binaries Container - -on: - push: - branches: - - main - pull_request: - branches: - - main - workflow_dispatch: - workflow_run: - workflows: ["Ort Docker Base Image"] - types: - - completed - -env: - REGISTRY: ghcr.io - -permissions: write-all - -jobs: - build: - name: Build ORT Binary Image - runs-on: ubuntu-22.04 - permissions: - contents: read - packages: write - - steps: - - name: Checkout main repository - uses: actions/checkout@v3 - with: - ref: main - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 - - - name: Login to GitHub Container Registry - uses: docker/login-action@v2 - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Extract components metadata (tags, labels) for base image - id: meta - uses: docker/metadata-action@v4 - with: - images: | - ${{ env.REGISTRY }}/${{ github.repository }}/ortbin - tags: | - type=raw,sha,enable=true,format=short - - - name: Build ORT ort container - uses: docker/build-push-action@v4 - with: - context: . - target: ort - push: true - load: false - build-args: | - ORT_VERSION=${{ env.ORT_VERSION }} - tags: | - ${{ steps.meta.outputs.tags }} - ${{ env.REGISTRY }}/${{ github.repository }}/ortbin:latest - labels: ${{ steps.meta.outputs.labels }} - build-contexts: | - ort-base-image=docker-image://${{ env.REGISTRY }}/${{ github.repository }}/base:latest - cache-from: type=gha,scope=ortort - cache-to: type=gha,scope=ortort,mode=max diff --git a/.github/workflows/docker-ort-dart.yml b/.github/workflows/docker-ort-dart.yml index d4e3967828e99..b84ee98981924 100644 --- a/.github/workflows/docker-ort-dart.yml +++ b/.github/workflows/docker-ort-dart.yml @@ -15,19 +15,19 @@ # SPDX-License-Identifier: Apache-2.0 # License-Filename: LICENSE -name: Ort Docker Dart Container +name: Dart Image on: workflow_dispatch: - pull_request: + push: branches: - main - push: + pull_request: paths: - '.ortversions/dart.versions' - '.github/workflows/docker-ort-base.yml' workflow_run: - workflows: ["Ort Docker Base Image"] + workflows: ["Base Image"] types: - completed @@ -77,7 +77,7 @@ jobs: with: context: . target: dart - push: ${{ github.event_name == 'push' }} + push: ${{ github.event_name == 'push' || github.event_name == 'workflow_dispatch' }} load: false build-args: | DART_VERSION=${{ env.DART_VERSION }} diff --git a/.github/workflows/docker-ort-dotnet.yml b/.github/workflows/docker-ort-dotnet.yml index 98e81c3a814b1..38a574922574a 100644 --- a/.github/workflows/docker-ort-dotnet.yml +++ b/.github/workflows/docker-ort-dotnet.yml @@ -15,19 +15,19 @@ # SPDX-License-Identifier: Apache-2.0 # License-Filename: LICENSE -name: Ort Docker Dotnet Container +name: Dotnet Image on: workflow_dispatch: - pull_request: + push: branches: - main - push: + pull_request: paths: - '.ortversions/dotnet.versions' - '.github/workflows/docker-ort-base.yml' workflow_run: - workflows: ["Ort Docker Base Image"] + workflows: ["Base Image"] env: REGISTRY: ghcr.io @@ -74,7 +74,7 @@ jobs: with: context: . target: dotnet - push: ${{ github.event_name == 'push' }} + push: ${{ github.event_name == 'push' || github.event_name == 'workflow_dispatch' }} load: false build-args: | DOTNET_VERSION=${{ env.DOTNET_VERSION }} diff --git a/.github/workflows/docker-ort-golang.yml b/.github/workflows/docker-ort-golang.yml index a2220af15ec0e..bedfd252ae83c 100644 --- a/.github/workflows/docker-ort-golang.yml +++ b/.github/workflows/docker-ort-golang.yml @@ -15,19 +15,19 @@ # SPDX-License-Identifier: Apache-2.0 # License-Filename: LICENSE -name: Ort Golang Container +name: Golang Image on: workflow_dispatch: - pull_request: + push: branches: - main - push: + pull_request: paths: - '.ortversions/golang.versions' - '.github/workflows/docker-ort-base.yml' workflow_run: - workflows: ["Ort Docker Base Image"] + workflows: ["Base Image"] types: - completed @@ -77,7 +77,7 @@ jobs: with: context: . target: golang - push: ${{ github.event_name == 'push' }} + push: ${{ github.event_name == 'push' || github.event_name == 'workflow_dispatch' }} load: false build-args: | GO_DEP_VERSION=${{ env.GO_DEP_VERSION }} diff --git a/.github/workflows/docker-ort-haskell.yml b/.github/workflows/docker-ort-haskell.yml index db9077e99d93b..01a7c9c3365e6 100644 --- a/.github/workflows/docker-ort-haskell.yml +++ b/.github/workflows/docker-ort-haskell.yml @@ -15,19 +15,19 @@ # SPDX-License-Identifier: Apache-2.0 # License-Filename: LICENSE -name: Ort Docker Haskell Container +name: Haskell Image on: workflow_dispatch: - pull_request: + push: branches: - main - push: + pull_request: paths: - '.ortversions/haskell.versions' - '.github/workflows/docker-ort-base.yml' workflow_run: - workflows: ["Ort Docker Base Image"] + workflows: ["Base Image"] types: - completed @@ -77,7 +77,7 @@ jobs: with: context: . target: haskell - push: true + push: ${{ github.event_name == 'push' || github.event_name == 'workflow_dispatch' }} load: false build-args: | HASKELL_STACK_VERSION=${{ env.HASKELL_STACK_VERSION }} diff --git a/.github/workflows/docker-ort-nodejs.yml b/.github/workflows/docker-ort-nodejs.yml index bfe500ba0234c..437da65c1e6b9 100644 --- a/.github/workflows/docker-ort-nodejs.yml +++ b/.github/workflows/docker-ort-nodejs.yml @@ -15,19 +15,19 @@ # SPDX-License-Identifier: Apache-2.0 # License-Filename: LICENSE -name: Ort NodeJS Container +name: NodeJS Image on: workflow_dispatch: - pull_request: + push: branches: - main - push: + pull_request: paths: - '.ortversions/nodejs.versions' - '.github/workflows/docker-ort-base.yml' workflow_run: - workflows: ["Ort Docker Base Image"] + workflows: ["Base Image"] types: - completed @@ -76,7 +76,7 @@ jobs: with: context: . target: nodejs - push: ${{ github.event_name == 'push' }} + push: ${{ github.event_name == 'push' || github.event_name == 'workflow_dispatch' }} load: false build-args: | NODEJS_VERSION=${{ env.NODEJS_VERSION }} diff --git a/.github/workflows/docker-ort-python.yml b/.github/workflows/docker-ort-python.yml index f82be976e3837..783e3fc4d3bec 100644 --- a/.github/workflows/docker-ort-python.yml +++ b/.github/workflows/docker-ort-python.yml @@ -15,19 +15,19 @@ # SPDX-License-Identifier: Apache-2.0 # License-Filename: LICENSE -name: Ort Python Container +name: Python Image on: workflow_dispatch: - pull_request: + push: branches: - main - push: + pull_request: paths: - '.ortversions/python.versions' - '.github/workflows/docker-ort-base.yml' workflow_run: - workflows: ["Ort Docker Base Image"] + workflows: ["Base Image"] types: - completed @@ -76,7 +76,7 @@ jobs: with: context: . target: python - push: ${{ github.event_name == 'push' }} + push: ${{ github.event_name == 'push' || github.event_name == 'workflow_dispatch' }} load: false build-args: | CONAN_VERSION=${{ env.CONAN_VERSION }} diff --git a/.github/workflows/docker-ort-ruby.yml b/.github/workflows/docker-ort-ruby.yml index 5848bf1875e98..1bb9afc8d87be 100644 --- a/.github/workflows/docker-ort-ruby.yml +++ b/.github/workflows/docker-ort-ruby.yml @@ -15,19 +15,20 @@ # SPDX-License-Identifier: Apache-2.0 # License-Filename: LICENSE -name: Ort Ruby Container +name: Ruby Image on: - workflow_run: - workflows: ["Ort Docker Base Image"] workflow_dispatch: - pull_request: + push: branches: - main - push: + pull_request: paths: - '.ortversions/ruby.versions' - '.github/workflows/docker-ort-base.yml' + workflow_run: + workflows: ["Base Image"] + env: REGISTRY: ghcr.io @@ -76,7 +77,7 @@ jobs: with: context: . target: ruby - push: ${{ github.event_name == 'push' }} + push: ${{ github.event_name == 'push' || github.event_name == 'workflow_dispatch' }} load: false build-args: | RUBY_VERSION=${{ env.RUBY_VERSION }} diff --git a/.github/workflows/docker-ort-runtime.yml b/.github/workflows/docker-ort-runtime.yml index 38dbbedd8b593..1eb2eb050067c 100644 --- a/.github/workflows/docker-ort-runtime.yml +++ b/.github/workflows/docker-ort-runtime.yml @@ -15,7 +15,7 @@ # SPDX-License-Identifier: Apache-2.0 # License-Filename: LICENSE -name: Ort Runtime Container +name: Runtime Image on: workflow_dispatch: @@ -27,12 +27,11 @@ on: - main workflow_run: workflows: - - 'Ort Binaries Container' - - 'Ort Python Container' - - 'Ort NodeJS Container' - - 'Ort Ruby Container' - - 'Ort Rust Container' - - 'Ort Golang Container' + - 'Python Image' + - 'NodeJS Image' + - 'Ruby Image' + - 'Rust Image' + - 'Golang Image' types: - completed @@ -81,7 +80,7 @@ jobs: with: context: . target: run - push: ${{ github.event_name == 'push' }} + push: ${{ github.event_name == 'push' || github.event_name == 'workflow_dispatch' }} load: false build-args: | NODEJS_VERSION=${{ env.NODEJS_VERSION }} @@ -90,7 +89,6 @@ jobs: ${{ env.REGISTRY }}/${{ github.repository_owner }}/ort:latest labels: ${{ steps.meta.outputs.labels }} build-contexts: | - ort=docker-image://${{ env.REGISTRY }}/${{ github.repository }}/ortbin:latest nodejs=docker-image://${{ env.REGISTRY }}/${{ github.repository }}/nodejs:latest python=docker-image://${{ env.REGISTRY }}/${{ github.repository }}/python:latest rust=docker-image://${{ env.REGISTRY }}/${{ github.repository }}/rust:latest diff --git a/.github/workflows/docker-ort-rust.yml b/.github/workflows/docker-ort-rust.yml index 8c94225689f7b..295a6351c196e 100644 --- a/.github/workflows/docker-ort-rust.yml +++ b/.github/workflows/docker-ort-rust.yml @@ -15,19 +15,19 @@ # SPDX-License-Identifier: Apache-2.0 # License-Filename: LICENSE -name: Ort Rust Container +name: Rust Image on: workflow_dispatch: - pull_request: + push: branches: - main - push: + pull_request: paths: - '.ortversions/rust.versions' - '.github/workflows/docker-ort-base.yml' workflow_run: - workflows: ["Ort Docker Base Image"] + workflows: ["Base Image"] env: REGISTRY: ghcr.io @@ -75,7 +75,7 @@ jobs: with: context: . target: rust - push: ${{ github.event_name == 'push' }} + push: ${{ github.event_name == 'push' || github.event_name == 'workflow_dispatch' }} load: false build-args: | RUST_VERSION=${{ env.RUST_VERSION }} diff --git a/.github/workflows/docker-ort-sbt.yml b/.github/workflows/docker-ort-sbt.yml index d8f32ce425b49..b4395ce1ee800 100644 --- a/.github/workflows/docker-ort-sbt.yml +++ b/.github/workflows/docker-ort-sbt.yml @@ -15,19 +15,19 @@ # SPDX-License-Identifier: Apache-2.0 # License-Filename: LICENSE -name: Ort Docker Sbt Container +name: Scala Image on: workflow_dispatch: - pull_request: + push: branches: - main - push: + pull_request: paths: - '.ortversions/sbt.versions' - '.github/workflows/docker-ort-base.yml' workflow_run: - workflows: ["Ort Docker Base Image"] + workflows: ["Base Image"] types: - completed @@ -77,7 +77,7 @@ jobs: with: context: . target: sbt - push: ${{ github.event_name == 'push' }} + push: ${{ github.event_name == 'push' || github.event_name == 'workflow_dispatch' }} load: false build-args: | SBT_VERSION=${{ env.SBT_VERSION }} diff --git a/.github/workflows/docker-ort-swift.yml b/.github/workflows/docker-ort-swift.yml index 90d4a9fd26255..24e4ecec9347d 100644 --- a/.github/workflows/docker-ort-swift.yml +++ b/.github/workflows/docker-ort-swift.yml @@ -15,19 +15,19 @@ # SPDX-License-Identifier: Apache-2.0 # License-Filename: LICENSE -name: Ort Docker Swift Container +name: Swift Image on: workflow_dispatch: - pull_request: + push: branches: - main - push: + pull_request: paths: - '.ortversions/swift.versions' - '.github/workflows/docker-ort-base.yml' workflow_run: - workflows: ["Ort Docker Base Image"] + workflows: ["Base Image"] types: - completed @@ -77,7 +77,7 @@ jobs: with: context: . target: swift - push: ${{ github.event_name == 'push' }} + push: ${{ github.event_name == 'push' || github.event_name == 'workflow_dispatch' }} load: false build-args: | SWIFT_VERSION=${{ env.SWIFT_VERSION }} diff --git a/.ortversions/python.versions b/.ortversions/python.versions index 8225ff2022fa1..35b83ed1ce6b0 100644 --- a/.ortversions/python.versions +++ b/.ortversions/python.versions @@ -5,4 +5,4 @@ PYTHON_INSPECTOR_VERSION=0.9.8 PYTHON_PIPENV_VERSION=2022.9.24 PYTHON_POETRY_VERSION=1.6.1 PYTHON_VERSION=3.10.13 -SCANCODE_VERSION=31.2.4 +SCANCODE_VERSION=32.0.6 diff --git a/Dockerfile b/Dockerfile index 3793af8203a25..9234dc5b3a4b7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -462,20 +462,20 @@ ENV RUST_HOME=/opt/rust ENV CARGO_HOME=$RUST_HOME/cargo ENV RUSTUP_HOME=$RUST_HOME/rustup ENV PATH=$PATH:$CARGO_HOME/bin:$RUSTUP_HOME/bin -COPY --from=ghcr.io/oss-review-toolkit/rust --chown=$USER:$USER $RUST_HOME $RUST_HOME +COPY --from=rust --chown=$USER:$USER $RUST_HOME $RUST_HOME RUN chmod o+rwx $CARGO_HOME RUN syft $RUST_HOME -o spdx-json --file /usr/share/doc/ort/ort-rust.spdx.json # Golang ENV PATH=$PATH:/opt/go/bin -COPY --from=ghcr.io/oss-review-toolkit/golang --chown=$USER:$USER /opt/go /opt/go +COPY --from=golang --chown=$USER:$USER /opt/go /opt/go RUN syft /opt/go -o spdx-json --file /usr/share/doc/ort/ort-golang.spdx.json # Ruby ENV RBENV_ROOT=/opt/rbenv/ ENV GEM_HOME=/var/tmp/gem ENV PATH=$PATH:$RBENV_ROOT/bin:$RBENV_ROOT/shims:$RBENV_ROOT/plugins/ruby-install/bin -COPY --from=ghcr.io/oss-review-toolkit/ruby --chown=$USER:$USER $RBENV_ROOT $RBENV_ROOT +COPY --from=ruby --chown=$USER:$USER $RBENV_ROOT $RBENV_ROOT RUN syft $RBENV_ROOT -o spdx-json --file /usr/share/doc/ort/ort-ruby.spdx.json # ORT diff --git a/Dockerfile-extended b/Dockerfile-extended new file mode 100644 index 0000000000000..81a007d6cd639 --- /dev/null +++ b/Dockerfile-extended @@ -0,0 +1,86 @@ +FROM ghcr.io/oss-review-toolkit/ort + +# Copyright (C) 2023 The ORT Project Authors (see ) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 +# License-Filename: LICENSE + +# Repo and Android +ENV ANDROID_HOME=/opt/android-sdk +ENV ANDROID_USER_HOME=$HOME/.android +ENV PATH=$PATH:$ANDROID_HOME/cmdline-tools/latest/bin:$ANDROID_HOME/cmdline-tools/bin +ENV PATH=$PATH:$ANDROID_HOME/platform-tools +COPY --from=ghcr.io/oss-review-toolkit/android --chown=$USER:$USER $ANDROID_HOME $ANDROID_HOME +RUN chmod -R o+rw $ANDROID_HOME + +RUN syft $ANDROID_HOME -o spdx-json --file /usr/share/doc/ort/ort-android.spdx.json + +# Swift +ENV SWIFT_HOME=/opt/swift +ENV PATH=$PATH:$SWIFT_HOME/bin +COPY --from=ghcr.io/oss-review-toolkit/swift --chown=$USER:$USER $SWIFT_HOME $SWIFT_HOME + +RUN syft $SWIFT_HOME -o spdx-json --file /usr/share/doc/ort/ort-swift.spdx.json + + +# Scala +ENV SBT_HOME=/opt/sbt +ENV PATH=$PATH:$SBT_HOME/bin +COPY --from=ghcr.io/oss-review-toolkit/sbt --chown=$USER:$USER $SBT_HOME $SBT_HOME + +RUN syft $SBT_HOME -o spdx-json --file /usr/share/doc/ort/ort-sbt.spdx.json + +# Dart +ENV DART_SDK=/opt/dart-sdk +ENV PATH=$PATH:$DART_SDK/bin +COPY --from=ghcr.io/oss-review-toolkit/dart --chown=$USER:$USER $DART_SDK $DART_SDK + +RUN syft $DART_SDK -o spdx-json --file /usr/share/doc/ort/ort-golang.dart.json + +# Dotnet +ENV DOTNET_HOME=/opt/dotnet +ENV NUGET_INSPECTOR_HOME=$DOTNET_HOME +ENV PATH=$PATH:$DOTNET_HOME:$DOTNET_HOME/tools:$DOTNET_HOME/bin + +COPY --from=ghcr.io/oss-review-toolkit/dotnet --chown=$USER:$USER $DOTNET_HOME $DOTNET_HOME + +RUN syft $DOTNET_HOME -o spdx-json --file /usr/share/doc/ort/ort-dotnet.spdx.json + +# PHP +ARG PHP_VERSION=8.1 +ARG COMPOSER_VERSION=2.2 + +RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ + --mount=type=cache,target=/var/lib/apt,sharing=locked \ + sudo apt-get update && \ + DEBIAN_FRONTEND=noninteractive sudo apt-get install -y --no-install-recommends \ + php${PHP_VERSION} \ + && sudo rm -rf /var/lib/apt/lists/* + +RUN mkdir -p /opt/php/bin \ + && curl -ksS https://getcomposer.org/installer | php -- --install-dir=/opt/php/bin --filename=composer --$COMPOSER_VERSION + +ENV PATH=$PATH:/opt/php/bin + +RUN syft /opt/php -o spdx-json --file /usr/share/doc/ort/ort-php.spdx.json + +# Haskell +ARG HASKELL_STACK_VERSION=2.7.5 +ENV HASKELL_HOME=/opt/haskell +ENV PATH=$PATH:$HASKELL_HOME/bin + +COPY --from=ghcr.io/oss-review-toolkit/haskell /opt/haskell /opt/haskell + +RUN syft /opt/haskell -o spdx-json --file /usr/share/doc/ort/ort-haskell.spdx.json diff --git a/scripts/custom_docker.sh b/scripts/custom_docker.sh index f0d1ca575218f..0e08cefe4d50a 100755 --- a/scripts/custom_docker.sh +++ b/scripts/custom_docker.sh @@ -33,7 +33,7 @@ dockerfile_template="FROM ghcr.io/oss-review-toolkit/ort\n" output_file="Dockerfile.custom" function usage() { - echo "Usage: $0 -c [ ...] -output " + echo "Usage: $0 -c [ ...] -o " echo "Options:" echo " -c [ ...]: List of language components to include in the Dockerfile: ${valid_components[*]}" echo " -output : Output file for the generated Dockerfile, Defaults to Dockerfile.custom." @@ -48,7 +48,7 @@ while [[ $# -gt 0 ]]; do components=("$@") break ;; - -output) + -o) shift output_file=$1 ;; diff --git a/scripts/docker_build.sh b/scripts/docker_build.sh index cae24125d0dc3..ee58e3392a234 100755 --- a/scripts/docker_build.sh +++ b/scripts/docker_build.sh @@ -21,6 +21,7 @@ set -e -o pipefail GIT_ROOT=$(git rev-parse --show-toplevel) GIT_REVISION=$($GIT_ROOT/gradlew -q properties --property version | sed -nr "s/version: (.+)/\1/p") +DOCKER_IMAGE_ROOT="${DOCKER_IMAGE_ROOT:-ghcr.io/oss-review-toolkit}" echo "Setting ORT_VERSION to $GIT_REVISION." @@ -159,9 +160,9 @@ image_build dotnet dotnet "$DOTNET_VERSION" \ --build-arg NUGET_INSPECTOR_VERSION="$NUGET_INSPECTOR_VERSION" \ "$@" -# Dotnet +# Haskell # shellcheck disable=SC1091 . .ortversions/haskell.versions -image_build dotnet dotnet "$HASKELL_STACK_VERSION" \ +image_build haskell haskell "$HASKELL_STACK_VERSION" \ --build-arg HASKELL_STACK_VERSION="$HASKELL_STACK_VERSION" \ "$@" diff --git a/scripts/docker_build_all.sh b/scripts/docker_build_all.sh deleted file mode 100755 index e99e86eee37f3..0000000000000 --- a/scripts/docker_build_all.sh +++ /dev/null @@ -1,144 +0,0 @@ -#!/bin/bash -# -# Copyright (C) 2020 The ORT Project Authors (see ) -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# SPDX-License-Identifier: Apache-2.0 -# License-Filename: LICENSE - -set -e -o pipefail - -GIT_ROOT=$(git rev-parse --show-toplevel) -GIT_REVISION=$(git describe --abbrev=10 --always --tags --dirty --match=[0-9]*) -DOCKER_IMAGE_ROOT="${DOCKER_IMAGE_ROOT:-ort}" - -echo "Setting ORT_VERSION to $GIT_REVISION." - -# --------------------------- -# image_build function -# Usage ( position paramenters): -# image_build - -image_build() { - local target - local name - local version - target="$1" - shift - name="$1" - shift - version="$1" - shift - - docker buildx build \ - -f "$GIT_ROOT/Dockerfile" \ - --target "$target" \ - --tag "${DOCKER_IMAGE_ROOT}/$name:$version" \ - --tag "${DOCKER_IMAGE_ROOT}/$name:latest" \ - --tag "ghcr.io/${DOCKER_IMAGE_ROOT}/$name:$version" \ - --tag "ghcr.io/${DOCKER_IMAGE_ROOT}/$name:latest" \ - --build-context "base=docker-image://${DOCKER_IMAGE_ROOT}/base:latest" \ - "$@" . -} - -#for containers in base android dart golang haskell nodejs python ruby rust sbt spm binaries runtime; do -# Base -image_build ort-base-image base "$UBUNTU_VERSION" --build-arg JAVA_VERSION="$JAVA_VERSION" "$@" - -# Python -image_build python python "$PYTHON_VERSION" \ - --build-arg PYTHON_VERSION="$PYTHON_VERSION" \ - --build-arg CONAN_VERSION="$CONAN_VERSION" \ - --build-arg PYTHON_INSPECTOR_VERSION="$PYTHON_INSPECTOR_VERSION" \ - --build-arg PYTHON_PIPENV_VERSION="$PYTHON_PIPENV_VERSION" \ - --build-arg PYTHON_POETRY_VERSION="$PYTHON_POETRY_VERSION" \ - --build-arg PIPTOOL_VERSION="$PIPTOOL_VERSION" \ - --build-arg SCANCODE_VERSION="$SCANCODE_VERSION" \ - "$@" - -# Ruby -image_build ruby ruby "$RUBY_VERSION" \ - --build-arg RUBY_VERSION="$RUBY_VERSION" \ - --build-arg COCOAPODS_VERSION="$COCOAPODS_VERSION" \ - "$@" - -# Nodejs -image_build nodejs nodejs "$NODEJS_VERSION" \ - --build-arg NODEJS_VERSION="$NODEJS_VERSION" \ - --build-arg BOWER_VERSION="$BOWER_VERSION" \ - --build-arg NPM_VERSION="$NPM_VERSION" \ - --build-arg PNPM_VERSION="$PNPM_VERSION" \ - --build-arg YARN_VERSION="$YARN_VERSION" \ - "$@" - -# Rust -image_build rust rust "$RUST_VERSION" \ - --build-arg RUST_VERSION="$RUST_VERSION" \ - "$@" - -# Golang -image_build golang golang "$GO_VERSION" \ - --build-arg GO_VERSION="$GO_VERSION" \ - --build-arg GO_DEP_VERSION="$GO_DEP_VERSION" \ - "$@" - -# Haskell -image_build haskell haskell "$HASKELL_STACK_VERSION" \ - --build-arg HASKELL_STACK_VERSION="$HASKELL_STACK_VERSION" \ - "$@" - -# Android -image_build android android "$ANDROID_CMD_VERSION" \ - --build-arg ANDROID_CMD_VERSION="$ANDROID_CMD_VERSION" \ - "$@" - -# Dart -image_build dart dart "$DART_VERSION" \ - --build-arg DART_VERSION="$DART_VERSION" \ - "$@" - -# SBT -image_build sbt sbt "$SBT_VERSION" \ - --build-arg SBT_VERSION="$SBT_VERSION" \ - "$@" - -# Spm -image_build spm spm "$SWIFT_VERSION" \ - --build-arg SWIFT_VERSION="$SWIFT_VERSION" \ - "$@" - -# Ort -image_build ort binaries "$GIT_REVISION" \ - --build-arg ORT_VERSION="$GIT_REVISION" \ - "$@" - -# Runtime ORT image -image_build run ort "$GIT_REVISION" \ - --build-context "python=docker-image://${DOCKER_IMAGE_ROOT}/python:latest" \ - --build-context "ruby=docker-image://${DOCKER_IMAGE_ROOT}/ruby:latest" \ - --build-context "python=docker-image://${DOCKER_IMAGE_ROOT}/python:latest" \ - --build-arg NODEJS_VERSION="$NODEJS_VERSION" \ - --build-context "nodejs=docker-image://${DOCKER_IMAGE_ROOT}/nodejs:latest" \ - --build-context "rust=docker-image://${DOCKER_IMAGE_ROOT}/rust:latest" \ - --build-context "golang=docker-image://${DOCKER_IMAGE_ROOT}/golang:latest" \ - --build-context "haskell=docker-image://${DOCKER_IMAGE_ROOT}/haskell:latest" \ - --build-context "android=docker-image://${DOCKER_IMAGE_ROOT}/android:latest" \ - --build-context "dart=docker-image://${DOCKER_IMAGE_ROOT}/dart:latest" \ - --build-context "sbt=docker-image://${DOCKER_IMAGE_ROOT}/sbt:latest" \ - --build-context "spm=docker-image://${DOCKER_IMAGE_ROOT}/spm:latest" \ - --build-context "ort=docker-image://${DOCKER_IMAGE_ROOT}/binaries:latest" \ - --build-arg COMPOSER_VERSION="$COMPOSER_VERSION" \ - --build-arg NUGET_INSPECTOR_VERSION="$NUGET_INSPECTOR_VERSION" \ - --tag "ghcr.io/vwdfive/ort:latest" -"$@" diff --git a/scripts/docker_snippets/android.snippet b/scripts/docker_snippets/android.snippet index 59b33f357e088..bbf9908b0fe8f 100644 --- a/scripts/docker_snippets/android.snippet +++ b/scripts/docker_snippets/android.snippet @@ -21,6 +21,6 @@ ENV ANDROID_USER_HOME=$HOME/.android ENV PATH=$PATH:$ANDROID_HOME/cmdline-tools/latest/bin:$ANDROID_HOME/cmdline-tools/bin ENV PATH=$PATH:$ANDROID_HOME/platform-tools COPY --from=ghcr.io/oss-review-toolkit/android --chown=$USER:$USER $ANDROID_HOME $ANDROID_HOME -RUN chmod -R o+rw $ANDROID_HOME +RUN sudo chmod -R o+rw $ANDROID_HOME RUN syft $ANDROID_HOME -o spdx-json --file /usr/share/doc/ort/ort-android.spdx.json diff --git a/scripts/docker_snippets/haskell.snippet b/scripts/docker_snippets/haskell.snippet index b974d2937c661..4594de4e4e1de 100644 --- a/scripts/docker_snippets/haskell.snippet +++ b/scripts/docker_snippets/haskell.snippet @@ -15,18 +15,10 @@ # SPDX-License-Identifier: Apache-2.0 # License-Filename: LICENSE -ARG HASKELL_STACK_VERSION=2.7.5 +# Haskell ENV HASKELL_HOME=/opt/haskell ENV PATH=$PATH:$HASKELL_HOME/bin -RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ - --mount=type=cache,target=/var/lib/apt,sharing=locked \ - sudo apt-get update -qq \ - && DEBIAN_FRONTEND=noninteractive sudo apt-get install -y --no-install-recommends \ - zlib1g-dev \ - && sudo rm -rf /var/lib/apt/lists/* +COPY --from=ghcr.io/oss-review-toolkit/haskell /opt/haskell /opt/haskell -RUN curl -sSL https://get.haskellstack.org/ | bash -s -- -d $HASKELL_HOME/bin - -FROM scratch AS haskell -COPY --from=haskellbuild /opt/haskell /opt/haskell \ No newline at end of file +RUN syft /opt/haskell -o spdx-json --file /usr/share/doc/ort/ort-haskell.spdx.json \ No newline at end of file From f55ca2eab9df13a86563b9db43dddda8400a842d Mon Sep 17 00:00:00 2001 From: Helio Chissini de Castro Date: Fri, 8 Sep 2023 09:53:14 +0200 Subject: [PATCH 11/47] feat(docker): Provide extended image with all components Signed-off-by: Helio Chissini de Castro --- .github/workflows/docker-ort-android.yml | 2 +- .github/workflows/docker-ort-dart.yml | 2 +- .github/workflows/docker-ort-dotnet.yml | 2 +- .github/workflows/docker-ort-golang.yml | 2 +- .github/workflows/docker-ort-haskell.yml | 2 +- .github/workflows/docker-ort-nodejs.yml | 2 +- .github/workflows/docker-ort-ruby.yml | 5 +- .github/workflows/docker-ort-runtime-ext.yml | 88 +++++++++++++++++++ .github/workflows/docker-ort-runtime.yml | 4 - .github/workflows/docker-ort-rust.yml | 4 +- .github/workflows/docker-ort-scala.yml | 91 ++++++++++++++++++++ .github/workflows/docker-ort-swift.yml | 2 +- .gitignore | 1 + Dockerfile-extended | 15 ++-- scripts/docker_build.sh | 16 +++- 15 files changed, 215 insertions(+), 23 deletions(-) create mode 100644 .github/workflows/docker-ort-runtime-ext.yml create mode 100644 .github/workflows/docker-ort-scala.yml diff --git a/.github/workflows/docker-ort-android.yml b/.github/workflows/docker-ort-android.yml index 02d6914d75fa6..d46c897ddb520 100644 --- a/.github/workflows/docker-ort-android.yml +++ b/.github/workflows/docker-ort-android.yml @@ -27,7 +27,7 @@ on: branches: - main workflow_run: - workflows: ["Base Image"] + workflows: ["Runtime Image"] types: - completed diff --git a/.github/workflows/docker-ort-dart.yml b/.github/workflows/docker-ort-dart.yml index b84ee98981924..ac84045f67032 100644 --- a/.github/workflows/docker-ort-dart.yml +++ b/.github/workflows/docker-ort-dart.yml @@ -27,7 +27,7 @@ on: - '.ortversions/dart.versions' - '.github/workflows/docker-ort-base.yml' workflow_run: - workflows: ["Base Image"] + workflows: ["Scale Image"] types: - completed diff --git a/.github/workflows/docker-ort-dotnet.yml b/.github/workflows/docker-ort-dotnet.yml index 38a574922574a..158eb7ff5bd64 100644 --- a/.github/workflows/docker-ort-dotnet.yml +++ b/.github/workflows/docker-ort-dotnet.yml @@ -27,7 +27,7 @@ on: - '.ortversions/dotnet.versions' - '.github/workflows/docker-ort-base.yml' workflow_run: - workflows: ["Base Image"] + workflows: ["Dart Image"] env: REGISTRY: ghcr.io diff --git a/.github/workflows/docker-ort-golang.yml b/.github/workflows/docker-ort-golang.yml index bedfd252ae83c..2b79707ff5a64 100644 --- a/.github/workflows/docker-ort-golang.yml +++ b/.github/workflows/docker-ort-golang.yml @@ -27,7 +27,7 @@ on: - '.ortversions/golang.versions' - '.github/workflows/docker-ort-base.yml' workflow_run: - workflows: ["Base Image"] + workflows: ["Ruby Image"] types: - completed diff --git a/.github/workflows/docker-ort-haskell.yml b/.github/workflows/docker-ort-haskell.yml index 01a7c9c3365e6..0fdf52cb681a3 100644 --- a/.github/workflows/docker-ort-haskell.yml +++ b/.github/workflows/docker-ort-haskell.yml @@ -27,7 +27,7 @@ on: - '.ortversions/haskell.versions' - '.github/workflows/docker-ort-base.yml' workflow_run: - workflows: ["Base Image"] + workflows: ["Dotnet Image"] types: - completed diff --git a/.github/workflows/docker-ort-nodejs.yml b/.github/workflows/docker-ort-nodejs.yml index 437da65c1e6b9..7429a049d6324 100644 --- a/.github/workflows/docker-ort-nodejs.yml +++ b/.github/workflows/docker-ort-nodejs.yml @@ -27,7 +27,7 @@ on: - '.ortversions/nodejs.versions' - '.github/workflows/docker-ort-base.yml' workflow_run: - workflows: ["Base Image"] + workflows: ["Python Image"] types: - completed diff --git a/.github/workflows/docker-ort-ruby.yml b/.github/workflows/docker-ort-ruby.yml index 1bb9afc8d87be..b8de1d088dd6e 100644 --- a/.github/workflows/docker-ort-ruby.yml +++ b/.github/workflows/docker-ort-ruby.yml @@ -27,8 +27,9 @@ on: - '.ortversions/ruby.versions' - '.github/workflows/docker-ort-base.yml' workflow_run: - workflows: ["Base Image"] - + workflows: ["Rust Image"] + types: + - completed env: REGISTRY: ghcr.io diff --git a/.github/workflows/docker-ort-runtime-ext.yml b/.github/workflows/docker-ort-runtime-ext.yml new file mode 100644 index 0000000000000..5e067ad073cbe --- /dev/null +++ b/.github/workflows/docker-ort-runtime-ext.yml @@ -0,0 +1,88 @@ + # Copyright (C) 2023 The ORT Project Authors (see ) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 +# License-Filename: LICENSE + +name: Extended Image + +on: + workflow_dispatch: + pull_request: + branches: + - main + push: + branches: + - main + workflow_run: + workflows: + - 'Haskell Image' + types: + - completed + +env: + REGISTRY: ghcr.io + +permissions: write-all + +jobs: + build: + name: Build ORT Extended Image + runs-on: ubuntu-22.04 + permissions: + contents: read + packages: write + + steps: + - name: Checkout default branch + uses: actions/checkout@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + + - name: Login to GitHub Container Registry + uses: docker/login-action@v2 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract components metadata (tags, labels) for Ort runtime image + id: meta-ort + uses: docker/metadata-action@v4 + with: + images: | + ${{ env.REGISTRY }}/${{ github.repository_owner }}/ort-extended + tags: | + type=raw,sha,enable=true,format=short + + - name: Build ORT runtime container + uses: docker/build-push-action@v4 + with: + context: . + file: Dockerfile-extended + push: ${{ github.event_name == 'push' || github.event_name == 'workflow_dispatch' }} + load: false + tags: | + ${{ steps.meta-ort.outputs.tags }} + ${{ env.REGISTRY }}/${{ github.repository_owner }}/ort-extended:latest + labels: ${{ steps.meta.outputs.labels }} + build-contexts: | + android=docker-image://${{ env.REGISTRY }}/${{ github.repository }}/android:latest + swift=docker-image://${{ env.REGISTRY }}/${{ github.repository }}/swift:latest + sbt=docker-image://${{ env.REGISTRY }}/${{ github.repository }}/sbt:latest + dart=docker-image://${{ env.REGISTRY }}/${{ github.repository }}/dart:latest + dotnet=docker-image://${{ env.REGISTRY }}/${{ github.repository }}/dotnet:latest + haskell=docker-image://${{ env.REGISTRY }}/${{ github.repository }}/haskell:latest + dotnet=docker-image://${{ env.REGISTRY }}/${{ github.repository }}/dotnet:latest diff --git a/.github/workflows/docker-ort-runtime.yml b/.github/workflows/docker-ort-runtime.yml index 1eb2eb050067c..76e34226d2680 100644 --- a/.github/workflows/docker-ort-runtime.yml +++ b/.github/workflows/docker-ort-runtime.yml @@ -27,10 +27,6 @@ on: - main workflow_run: workflows: - - 'Python Image' - - 'NodeJS Image' - - 'Ruby Image' - - 'Rust Image' - 'Golang Image' types: - completed diff --git a/.github/workflows/docker-ort-rust.yml b/.github/workflows/docker-ort-rust.yml index 295a6351c196e..654c122e36594 100644 --- a/.github/workflows/docker-ort-rust.yml +++ b/.github/workflows/docker-ort-rust.yml @@ -27,7 +27,9 @@ on: - '.ortversions/rust.versions' - '.github/workflows/docker-ort-base.yml' workflow_run: - workflows: ["Base Image"] + workflows: ["NodeJS Image"] + types: + - completed env: REGISTRY: ghcr.io diff --git a/.github/workflows/docker-ort-scala.yml b/.github/workflows/docker-ort-scala.yml new file mode 100644 index 0000000000000..f091a60eb3c34 --- /dev/null +++ b/.github/workflows/docker-ort-scala.yml @@ -0,0 +1,91 @@ +# Copyright (C) 2023 The ORT Project Authors (see ) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 +# License-Filename: LICENSE + +name: Scala Image + +on: + workflow_dispatch: + push: + branches: + - main + pull_request: + paths: + - '.ortversions/sbt.versions' + - '.github/workflows/docker-ort-base.yml' + workflow_run: + workflows: ["Swift Image"] + types: + - completed + +env: + REGISTRY: ghcr.io + +permissions: write-all + +jobs: + build: + name: Build ORT sbt Image + runs-on: ubuntu-22.04 + permissions: + contents: read + packages: write + + steps: + - name: Checkout main repository + uses: actions/checkout@v3 + + - name: Set environment variables + run: | + cat .ortversions/sbt.versions >> $GITHUB_ENV + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + + - name: Login to GitHub Container Registry + uses: docker/login-action@v2 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract components metadata (tags, labels) for base image + id: meta + uses: docker/metadata-action@v4 + with: + images: | + ${{ env.REGISTRY }}/${{ github.repository }}/sbt + tags: | + type=semver,pattern={{version}},value=${{ env.SBT_VERSION }} + + + - name: Build ORT sbt container + uses: docker/build-push-action@v4 + with: + context: . + target: sbt + push: ${{ github.event_name == 'push' || github.event_name == 'workflow_dispatch' }} + load: false + build-args: | + SBT_VERSION=${{ env.SBT_VERSION }} + tags: | + ${{ steps.meta.outputs.tags }} + ${{ env.REGISTRY }}/${{ github.repository }}/sbt:latest + labels: ${{ steps.meta.outputs.labels }} + build-contexts: | + ort-base-image=docker-image://${{ env.REGISTRY }}/${{ github.repository }}/base:latest + cache-from: type=gha,scope=ortsbt + cache-to: type=gha,scope=ortsbt,mode=max diff --git a/.github/workflows/docker-ort-swift.yml b/.github/workflows/docker-ort-swift.yml index 24e4ecec9347d..eeeac6d0e7c35 100644 --- a/.github/workflows/docker-ort-swift.yml +++ b/.github/workflows/docker-ort-swift.yml @@ -27,7 +27,7 @@ on: - '.ortversions/swift.versions' - '.github/workflows/docker-ort-base.yml' workflow_run: - workflows: ["Base Image"] + workflows: ["Android Image"] types: - completed diff --git a/.gitignore b/.gitignore index 02693a0aeb0d1..a7071979d01d1 100644 --- a/.gitignore +++ b/.gitignore @@ -6,6 +6,7 @@ build/ out/ /local.properties /*.iml +Dockerfile.custom # Kotest temporary directory .kotest/ diff --git a/Dockerfile-extended b/Dockerfile-extended index 81a007d6cd639..3dce2517060ce 100644 --- a/Dockerfile-extended +++ b/Dockerfile-extended @@ -22,15 +22,15 @@ ENV ANDROID_HOME=/opt/android-sdk ENV ANDROID_USER_HOME=$HOME/.android ENV PATH=$PATH:$ANDROID_HOME/cmdline-tools/latest/bin:$ANDROID_HOME/cmdline-tools/bin ENV PATH=$PATH:$ANDROID_HOME/platform-tools -COPY --from=ghcr.io/oss-review-toolkit/android --chown=$USER:$USER $ANDROID_HOME $ANDROID_HOME -RUN chmod -R o+rw $ANDROID_HOME +COPY --from=android --chown=$USER:$USER $ANDROID_HOME $ANDROID_HOME +RUN sudo chmod -R o+rw $ANDROID_HOME RUN syft $ANDROID_HOME -o spdx-json --file /usr/share/doc/ort/ort-android.spdx.json # Swift ENV SWIFT_HOME=/opt/swift ENV PATH=$PATH:$SWIFT_HOME/bin -COPY --from=ghcr.io/oss-review-toolkit/swift --chown=$USER:$USER $SWIFT_HOME $SWIFT_HOME +COPY --from=swift --chown=$USER:$USER $SWIFT_HOME $SWIFT_HOME RUN syft $SWIFT_HOME -o spdx-json --file /usr/share/doc/ort/ort-swift.spdx.json @@ -38,14 +38,14 @@ RUN syft $SWIFT_HOME -o spdx-json --file /usr/share/doc/ort/ort-swift.spdx.json # Scala ENV SBT_HOME=/opt/sbt ENV PATH=$PATH:$SBT_HOME/bin -COPY --from=ghcr.io/oss-review-toolkit/sbt --chown=$USER:$USER $SBT_HOME $SBT_HOME +COPY --from=sbt --chown=$USER:$USER $SBT_HOME $SBT_HOME RUN syft $SBT_HOME -o spdx-json --file /usr/share/doc/ort/ort-sbt.spdx.json # Dart ENV DART_SDK=/opt/dart-sdk ENV PATH=$PATH:$DART_SDK/bin -COPY --from=ghcr.io/oss-review-toolkit/dart --chown=$USER:$USER $DART_SDK $DART_SDK +COPY --from=dart --chown=$USER:$USER $DART_SDK $DART_SDK RUN syft $DART_SDK -o spdx-json --file /usr/share/doc/ort/ort-golang.dart.json @@ -54,7 +54,7 @@ ENV DOTNET_HOME=/opt/dotnet ENV NUGET_INSPECTOR_HOME=$DOTNET_HOME ENV PATH=$PATH:$DOTNET_HOME:$DOTNET_HOME/tools:$DOTNET_HOME/bin -COPY --from=ghcr.io/oss-review-toolkit/dotnet --chown=$USER:$USER $DOTNET_HOME $DOTNET_HOME +COPY --from=dotnet --chown=$USER:$USER $DOTNET_HOME $DOTNET_HOME RUN syft $DOTNET_HOME -o spdx-json --file /usr/share/doc/ort/ort-dotnet.spdx.json @@ -77,10 +77,9 @@ ENV PATH=$PATH:/opt/php/bin RUN syft /opt/php -o spdx-json --file /usr/share/doc/ort/ort-php.spdx.json # Haskell -ARG HASKELL_STACK_VERSION=2.7.5 ENV HASKELL_HOME=/opt/haskell ENV PATH=$PATH:$HASKELL_HOME/bin -COPY --from=ghcr.io/oss-review-toolkit/haskell /opt/haskell /opt/haskell +COPY --from=haskell /opt/haskell /opt/haskell RUN syft /opt/haskell -o spdx-json --file /usr/share/doc/ort/ort-haskell.spdx.json diff --git a/scripts/docker_build.sh b/scripts/docker_build.sh index ee58e3392a234..3bf5622a4b4a1 100755 --- a/scripts/docker_build.sh +++ b/scripts/docker_build.sh @@ -20,7 +20,7 @@ set -e -o pipefail GIT_ROOT=$(git rev-parse --show-toplevel) -GIT_REVISION=$($GIT_ROOT/gradlew -q properties --property version | sed -nr "s/version: (.+)/\1/p") +GIT_REVISION=$("$GIT_ROOT/gradlew" -q properties --property version | sed -nr "s/version: (.+)/\1/p") DOCKER_IMAGE_ROOT="${DOCKER_IMAGE_ROOT:-ghcr.io/oss-review-toolkit}" echo "Setting ORT_VERSION to $GIT_REVISION." @@ -166,3 +166,17 @@ image_build dotnet dotnet "$DOTNET_VERSION" \ image_build haskell haskell "$HASKELL_STACK_VERSION" \ --build-arg HASKELL_STACK_VERSION="$HASKELL_STACK_VERSION" \ "$@" + +# Runtime Extended ORT image +docker buildx build \ + --file Dockerfile-extended \ + --tag "${DOCKER_IMAGE_ROOT}/ort-extended:$GIT_REVISION" \ + --tag "${DOCKER_IMAGE_ROOT}/ort-extended:latest" \ + --build-context "sbt=docker-image://${DOCKER_IMAGE_ROOT}/sbt:latest" \ + --build-context "dotnet=docker-image://${DOCKER_IMAGE_ROOT}/dotnet:latest" \ + --build-context "swift=docker-image://${DOCKER_IMAGE_ROOT}/swift:latest" \ + --build-context "android=docker-image://${DOCKER_IMAGE_ROOT}/android:latest" \ + --build-context "dart=docker-image://${DOCKER_IMAGE_ROOT}/dart:latest" \ + --build-context "haskell=docker-image://${DOCKER_IMAGE_ROOT}/haskell:latest" \ + "$@" . + From 46061d053abd7693983a6d03a6eb88bd27375e4d Mon Sep 17 00:00:00 2001 From: Helio Chissini de Castro Date: Sat, 16 Sep 2023 17:36:20 +0200 Subject: [PATCH 12/47] feat(docker): Use jobs over workflow dependency Workflow dependencies relies on a single workflow, which make build pipelines for multiple docker containers complex and most of the time erratic. Move to single jobs dependency based with a internal action to check and build images if needed improve the logic of the process and allow to visually see the running pipeline. The pipeline will run on following cases: - On pull_request if .versions file is changed, but not publishing images on the registry. - On push or workflow_dispatch, publishing to defined registry. Signed-off-by: Helio Chissini de Castro --- .github/actions/ortdocker/action.yml | 102 +++++++++++++ .github/actions/ortdocker/check_image.py | 46 ++++++ .github/workflows/docker-ort-android.yml | 91 ------------ .github/workflows/docker-ort-base.yml | 83 ----------- .github/workflows/docker-ort-dart.yml | 91 ------------ .github/workflows/docker-ort-dotnet.yml | 89 ----------- .github/workflows/docker-ort-golang.yml | 92 ------------ .github/workflows/docker-ort-haskell.yml | 91 ------------ .github/workflows/docker-ort-nodejs.yml | 93 ------------ .github/workflows/docker-ort-python.yml | 96 ------------ .github/workflows/docker-ort-ruby.yml | 93 ------------ .github/workflows/docker-ort-runtime-ext.yml | 129 +++++++++++++++- .github/workflows/docker-ort-runtime.yml | 148 +++++++++++++++++-- .github/workflows/docker-ort-rust.yml | 91 ------------ .github/workflows/docker-ort-sbt.yml | 91 ------------ .github/workflows/docker-ort-scala.yml | 91 ------------ .github/workflows/docker-ort-swift.yml | 91 ------------ .ortversions/android.versions | 1 - .ortversions/base.versions | 2 - .ortversions/dart.versions | 1 - .ortversions/dotnet.versions | 2 - .ortversions/golang.versions | 2 - .ortversions/haskell.versions | 1 - .ortversions/nodejs.versions | 5 - .ortversions/php.versions | 2 - .ortversions/python.versions | 8 - .ortversions/ruby.versions | 2 - .ortversions/rust.versions | 1 - .ortversions/sbt.versions | 1 - .ortversions/swift.versions | 1 - .versions | 29 ++++ Dockerfile | 34 ++--- Dockerfile-extended | 4 +- scripts/docker_build.sh | 47 ++---- 34 files changed, 471 insertions(+), 1280 deletions(-) create mode 100644 .github/actions/ortdocker/action.yml create mode 100644 .github/actions/ortdocker/check_image.py delete mode 100644 .github/workflows/docker-ort-android.yml delete mode 100644 .github/workflows/docker-ort-base.yml delete mode 100644 .github/workflows/docker-ort-dart.yml delete mode 100644 .github/workflows/docker-ort-dotnet.yml delete mode 100644 .github/workflows/docker-ort-golang.yml delete mode 100644 .github/workflows/docker-ort-haskell.yml delete mode 100644 .github/workflows/docker-ort-nodejs.yml delete mode 100644 .github/workflows/docker-ort-python.yml delete mode 100644 .github/workflows/docker-ort-ruby.yml delete mode 100644 .github/workflows/docker-ort-rust.yml delete mode 100644 .github/workflows/docker-ort-sbt.yml delete mode 100644 .github/workflows/docker-ort-scala.yml delete mode 100644 .github/workflows/docker-ort-swift.yml delete mode 100644 .ortversions/android.versions delete mode 100644 .ortversions/base.versions delete mode 100644 .ortversions/dart.versions delete mode 100644 .ortversions/dotnet.versions delete mode 100644 .ortversions/golang.versions delete mode 100644 .ortversions/haskell.versions delete mode 100644 .ortversions/nodejs.versions delete mode 100644 .ortversions/php.versions delete mode 100644 .ortversions/python.versions delete mode 100644 .ortversions/ruby.versions delete mode 100644 .ortversions/rust.versions delete mode 100644 .ortversions/sbt.versions delete mode 100644 .ortversions/swift.versions create mode 100644 .versions diff --git a/.github/actions/ortdocker/action.yml b/.github/actions/ortdocker/action.yml new file mode 100644 index 0000000000000..fe9380c71f029 --- /dev/null +++ b/.github/actions/ortdocker/action.yml @@ -0,0 +1,102 @@ +# Copyright (C) 2023 The ORT Project Authors (see ) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 +# License-Filename: LICENSE + +name: "ORT Docker Image" +description: "Check and create docker image for Ort components" +author: "Helio Chissini de Castro " + +inputs: + registry: + description: "GitHub Container Registry" + default: "ghcr.io" + token: + description: "GitHub token" + required: true + name: + description: "Image name" + required: true + version: + description: "Image version" + required: true + build-args: + description: "List of build-time variables" + required: false + +runs: + using: "composite" + + steps: + - name: Install Python + uses: actions/setup-python@v4 + with: + python-version: '3.10' + cache: 'pip' + + - name: Check if Docker image tag exists + id: check_image + shell: bash + env: + INPUT_REGISTRY: ${{ inputs.registry }} + INPUT_TOKEN: ${{ inputs.token }} + INPUT_NAME: ${{ inputs.name }} + INPUT_VERSION: ${{ inputs.version }} + run: | + pip install -q -U pip requests + + result=$(python ./.github/actions/ortdocker/check_image.py) + echo $result + echo "result=$result" >> $GITHUB_OUTPUT + + - name: Set up Docker Buildx + if: steps.check_image.outputs.result != 'found' + uses: docker/setup-buildx-action@v2 + + - name: Login to GitHub Container Registry + if: steps.check_image.outputs.result != 'found' + uses: docker/login-action@v2 + with: + registry: ${{ inputs.registry }} + username: ${{ github.actor }} + password: ${{ inputs.token }} + + - name: Extract components metadata (tags, labels) for base image + if: steps.check_image.outputs.result != 'found' + id: meta + uses: docker/metadata-action@v4 + with: + images: | + ${{ env.REGISTRY }}/${{ github.repository }}/${{ inputs.name }} + tags: + type=raw,value=${{ inputs.version }} + + - name: Build ORT Base container + if: steps.check_image.outputs.result != 'found' + uses: docker/build-push-action@v4 + with: + context: . + target: ${{ inputs.name }} + push: ${{ github.event_name == 'push' || github.event_name == 'workflow_dispatch' }} + load: false + build-args: ${{ inputs.build-args }} + tags: | + ${{ steps.meta.outputs.tags }} + ${{ env.REGISTRY }}/${{ github.repository }}/${{ inputs.name }}:latest + labels: ${{ steps.meta.outputs.labels }} + build-contexts: | + base=docker-image://${{ inputs.registry }}/${{ github.repository }}/base:latest + + diff --git a/.github/actions/ortdocker/check_image.py b/.github/actions/ortdocker/check_image.py new file mode 100644 index 0000000000000..98846b0671d93 --- /dev/null +++ b/.github/actions/ortdocker/check_image.py @@ -0,0 +1,46 @@ +# Copyright (C) 2023 The ORT Project Authors (see ) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 +# License-Filename: LICENSE + +import os + +import requests + +token = os.getenv("INPUT_TOKEN") +org = os.getenv("GITHUB_REPOSITORY_OWNER") +name = os.getenv("INPUT_NAME") +version = os.getenv("INPUT_VERSION") + +url = f"https://api.github.com/orgs/{org}/packages/container/ort%2F{name}/versions" + +headers = { + "Accept": "application/vnd.github+json", + "Authorization": f"Bearer {token}", +} +response = requests.get(url, headers=headers) +if response.status_code == 404: + print("none") +else: + versions = [ + item + for sublist in [v["metadata"]["container"]["tags"] for v in response.json()] + if sublist + for item in sublist + ] + if version in versions: + print("found") + else: + print("none") diff --git a/.github/workflows/docker-ort-android.yml b/.github/workflows/docker-ort-android.yml deleted file mode 100644 index d46c897ddb520..0000000000000 --- a/.github/workflows/docker-ort-android.yml +++ /dev/null @@ -1,91 +0,0 @@ -# Copyright (C) 2023 The ORT Project Authors (see ) -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# SPDX-License-Identifier: Apache-2.0 -# License-Filename: LICENSE - -name: Android Image - -on: - workflow_dispatch: - pull_request: - paths: - - '.ortversions/android.versions' - - '.github/workflows/docker-ort-base.yml' - push: - branches: - - main - workflow_run: - workflows: ["Runtime Image"] - types: - - completed - -env: - REGISTRY: ghcr.io - -permissions: write-all - -jobs: - build: - name: Build ORT Android Image - runs-on: ubuntu-22.04 - permissions: - contents: read - packages: write - - steps: - - name: Checkout main repository - uses: actions/checkout@v3 - - - name: Set environment variables - run: | - cat .ortversions/android.versions >> $GITHUB_ENV - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 - - - name: Login to GitHub Container Registry - uses: docker/login-action@v2 - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Extract components metadata (tags, labels) for base image - id: meta - uses: docker/metadata-action@v4 - with: - images: | - ${{ env.REGISTRY }}/${{ github.repository }}/android - tags: | - type=semver,pattern={{version}},value=${{ env.ANDROID_CMD_VERSION }} - - - - name: Build ORT Android container - uses: docker/build-push-action@v4 - with: - context: . - target: android - push: ${{ github.event_name == 'push' || github.event_name == 'workflow_dispatch' }} - load: false - build-args: | - ANDROID_CMD_VERSION=${{ env.ANDROID_CMD_VERSION }} - tags: | - ${{ steps.meta.outputs.tags }} - ${{ env.REGISTRY }}/${{ github.repository }}/android:latest - labels: ${{ steps.meta.outputs.labels }} - build-contexts: | - ort-base-image=docker-image://${{ env.REGISTRY }}/${{ github.repository }}/base:latest - cache-from: type=gha,scope=ortandroid - cache-to: type=gha,scope=ortandroid,mode=max diff --git a/.github/workflows/docker-ort-base.yml b/.github/workflows/docker-ort-base.yml deleted file mode 100644 index 5f0589ce94d18..0000000000000 --- a/.github/workflows/docker-ort-base.yml +++ /dev/null @@ -1,83 +0,0 @@ -# Copyright (C) 2023 The ORT Project Authors (see ) -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# SPDX-License-Identifier: Apache-2.0 -# License-Filename: LICENSE - -name: Base Image - -on: - workflow_dispatch: - pull_request: - paths: - - '.ortversions/base.versions' - - 'Dockerfile' - push: - branches: - - main - -env: - REGISTRY: ghcr.io - -jobs: - build: - name: Build ORT Base image - runs-on: ubuntu-22.04 - permissions: - contents: read - packages: write - - steps: - - name: Checkout main repository - uses: actions/checkout@v3 - - - name: Set environment variables - run: | - cat .ortversions/base.versions >> $GITHUB_ENV - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 - - - name: Login to GitHub Container Registry - uses: docker/login-action@v2 - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Extract components metadata (tags, labels) for base image - id: meta - uses: docker/metadata-action@v4 - with: - images: | - ${{ env.REGISTRY }}/${{ github.repository }}/base - tags: - type=raw,value=${{ env.JAVA_VERSION }}-jdk-${{ env.UBUNTU_VERSION }} - - - name: Build ORT Base container - uses: docker/build-push-action@v4 - with: - context: . - target: ort-base-image - push: ${{ github.event_name == 'push' || github.event_name == 'workflow_dispatch' }} - load: false - build-args: | - JAVA_VERSION=${{ env.JAVA_VERSION }} - UBUNTU_VERSION=${{ env.UBUNTU_VERSION }} - tags: | - ${{ steps.meta.outputs.tags }} - ${{ env.REGISTRY }}/${{ github.repository }}/base:latest - labels: ${{ steps.meta.outputs.labels }} - cache-from: type=gha,scope=ortbase - cache-to: type=gha,scope=ortbase,mode=max diff --git a/.github/workflows/docker-ort-dart.yml b/.github/workflows/docker-ort-dart.yml deleted file mode 100644 index ac84045f67032..0000000000000 --- a/.github/workflows/docker-ort-dart.yml +++ /dev/null @@ -1,91 +0,0 @@ -# Copyright (C) 2023 The ORT Project Authors (see ) -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# SPDX-License-Identifier: Apache-2.0 -# License-Filename: LICENSE - -name: Dart Image - -on: - workflow_dispatch: - push: - branches: - - main - pull_request: - paths: - - '.ortversions/dart.versions' - - '.github/workflows/docker-ort-base.yml' - workflow_run: - workflows: ["Scale Image"] - types: - - completed - -env: - REGISTRY: ghcr.io - -permissions: write-all - -jobs: - build: - name: Build ORT Dart Image - runs-on: ubuntu-22.04 - permissions: - contents: read - packages: write - - steps: - - name: Checkout main repository - uses: actions/checkout@v3 - - - name: Set environment variables - run: | - cat .ortversions/dart.versions >> $GITHUB_ENV - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 - - - name: Login to GitHub Container Registry - uses: docker/login-action@v2 - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Extract components metadata (tags, labels) for base image - id: meta - uses: docker/metadata-action@v4 - with: - images: | - ${{ env.REGISTRY }}/${{ github.repository }}/dart - tags: | - type=semver,pattern={{version}},value=${{ env.DART_VERSION }} - - - - name: Build ORT Dart container - uses: docker/build-push-action@v4 - with: - context: . - target: dart - push: ${{ github.event_name == 'push' || github.event_name == 'workflow_dispatch' }} - load: false - build-args: | - DART_VERSION=${{ env.DART_VERSION }} - tags: | - ${{ steps.meta.outputs.tags }} - ${{ env.REGISTRY }}/${{ github.repository }}/dart:latest - labels: ${{ steps.meta.outputs.labels }} - build-contexts: | - ort-base-image=docker-image://${{ env.REGISTRY }}/${{ github.repository }}/base:latest - cache-from: type=gha,scope=ortdart - cache-to: type=gha,scope=ortdart,mode=max diff --git a/.github/workflows/docker-ort-dotnet.yml b/.github/workflows/docker-ort-dotnet.yml deleted file mode 100644 index 158eb7ff5bd64..0000000000000 --- a/.github/workflows/docker-ort-dotnet.yml +++ /dev/null @@ -1,89 +0,0 @@ -# Copyright (C) 2023 The ORT Project Authors (see ) -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# SPDX-License-Identifier: Apache-2.0 -# License-Filename: LICENSE - -name: Dotnet Image - -on: - workflow_dispatch: - push: - branches: - - main - pull_request: - paths: - - '.ortversions/dotnet.versions' - - '.github/workflows/docker-ort-base.yml' - workflow_run: - workflows: ["Dart Image"] - -env: - REGISTRY: ghcr.io - -permissions: write-all - -jobs: - build: - name: Build ORT Dotnet Image - runs-on: ubuntu-22.04 - permissions: - contents: read - packages: write - - steps: - - name: Checkout main repository - uses: actions/checkout@v3 - - - name: Set environment variables - run: | - cat .ortversions/dotnet.versions >> $GITHUB_ENV - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 - - - name: Login to GitHub Container Registry - uses: docker/login-action@v2 - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Extract components metadata (tags, labels) for base image - id: meta - uses: docker/metadata-action@v4 - with: - images: | - ${{ env.REGISTRY }}/${{ github.repository }}/dotnet - tags: | - type=raw,value=${{ env.DOTNET_VERSION }} - - - name: Build ORT Dotnet container - uses: docker/build-push-action@v4 - with: - context: . - target: dotnet - push: ${{ github.event_name == 'push' || github.event_name == 'workflow_dispatch' }} - load: false - build-args: | - DOTNET_VERSION=${{ env.DOTNET_VERSION }} - NUGET_INSPECTOR_VERSION=${{ env.NUGET_INSPECTOR_VERSION }} - tags: | - ${{ steps.meta.outputs.tags }} - ${{ env.REGISTRY }}/${{ github.repository }}/dotnet:latest - labels: ${{ steps.meta.outputs.labels }} - build-contexts: | - ort-base-image=docker-image://${{ env.REGISTRY }}/${{ github.repository }}/base:latest - cache-from: type=gha,scope=ortdotnet - cache-to: type=gha,scope=ortdotnet,mode=max diff --git a/.github/workflows/docker-ort-golang.yml b/.github/workflows/docker-ort-golang.yml deleted file mode 100644 index 2b79707ff5a64..0000000000000 --- a/.github/workflows/docker-ort-golang.yml +++ /dev/null @@ -1,92 +0,0 @@ -# Copyright (C) 2023 The ORT Project Authors (see ) -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# SPDX-License-Identifier: Apache-2.0 -# License-Filename: LICENSE - -name: Golang Image - -on: - workflow_dispatch: - push: - branches: - - main - pull_request: - paths: - - '.ortversions/golang.versions' - - '.github/workflows/docker-ort-base.yml' - workflow_run: - workflows: ["Ruby Image"] - types: - - completed - -env: - REGISTRY: ghcr.io - -permissions: write-all - -jobs: - build: - name: Build ORT Golang Image - runs-on: ubuntu-22.04 - permissions: - contents: read - packages: write - - steps: - - name: Checkout main repository - uses: actions/checkout@v3 - - - name: Set environment variables - run: | - cat .ortversions/golang.versions >> $GITHUB_ENV - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 - - - name: Login to GitHub Container Registry - uses: docker/login-action@v2 - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Extract components metadata (tags, labels) for base image - id: meta - uses: docker/metadata-action@v4 - with: - images: | - ${{ env.REGISTRY }}/${{ github.repository }}/golang - tags: | - type=semver,pattern={{version}},value=${{ env.GO_VERSION }} - - - - name: Build ORT Golang container - uses: docker/build-push-action@v4 - with: - context: . - target: golang - push: ${{ github.event_name == 'push' || github.event_name == 'workflow_dispatch' }} - load: false - build-args: | - GO_DEP_VERSION=${{ env.GO_DEP_VERSION }} - GO_VERSION=${{ env.GO_VERSION }} - tags: | - ${{ steps.meta.outputs.tags }} - ${{ env.REGISTRY }}/${{ github.repository }}/golang:latest - labels: ${{ steps.meta.outputs.labels }} - build-contexts: | - ort-base-image=docker-image://${{ env.REGISTRY }}/${{ github.repository }}/base:latest - cache-from: type=gha,scope=ortgolang - cache-to: type=gha,scope=ortgolang,mode=max diff --git a/.github/workflows/docker-ort-haskell.yml b/.github/workflows/docker-ort-haskell.yml deleted file mode 100644 index 0fdf52cb681a3..0000000000000 --- a/.github/workflows/docker-ort-haskell.yml +++ /dev/null @@ -1,91 +0,0 @@ -# Copyright (C) 2023 The ORT Project Authors (see ) -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# SPDX-License-Identifier: Apache-2.0 -# License-Filename: LICENSE - -name: Haskell Image - -on: - workflow_dispatch: - push: - branches: - - main - pull_request: - paths: - - '.ortversions/haskell.versions' - - '.github/workflows/docker-ort-base.yml' - workflow_run: - workflows: ["Dotnet Image"] - types: - - completed - -env: - REGISTRY: ghcr.io - -permissions: write-all - -jobs: - build: - name: Build ORT Haskell Image - runs-on: ubuntu-22.04 - permissions: - contents: read - packages: write - - steps: - - name: Checkout main repository - uses: actions/checkout@v3 - - - name: Set environment variables - run: | - cat .ortversions/haskell.versions >> $GITHUB_ENV - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 - - - name: Login to GitHub Container Registry - uses: docker/login-action@v2 - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Extract components metadata (tags, labels) for base image - id: meta - uses: docker/metadata-action@v4 - with: - images: | - ${{ env.REGISTRY }}/${{ github.repository }}/haskell - tags: | - type=semver,pattern={{version}},value=${{ env.HASKELL_STACK_VERSION }} - - - - name: Build ORT Haskell container - uses: docker/build-push-action@v4 - with: - context: . - target: haskell - push: ${{ github.event_name == 'push' || github.event_name == 'workflow_dispatch' }} - load: false - build-args: | - HASKELL_STACK_VERSION=${{ env.HASKELL_STACK_VERSION }} - tags: | - ${{ steps.meta.outputs.tags }} - ${{ env.REGISTRY }}/${{ github.repository }}/haskell:latest - labels: ${{ steps.meta.outputs.labels }} - build-contexts: | - ort-base-image=docker-image://${{ env.REGISTRY }}/${{ github.repository }}/base:latest - cache-from: type=gha,scope=orthaskell - cache-to: type=gha,scope=orthaskell,mode=max diff --git a/.github/workflows/docker-ort-nodejs.yml b/.github/workflows/docker-ort-nodejs.yml deleted file mode 100644 index 7429a049d6324..0000000000000 --- a/.github/workflows/docker-ort-nodejs.yml +++ /dev/null @@ -1,93 +0,0 @@ -# Copyright (C) 2023 The ORT Project Authors (see ) -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# SPDX-License-Identifier: Apache-2.0 -# License-Filename: LICENSE - -name: NodeJS Image - -on: - workflow_dispatch: - push: - branches: - - main - pull_request: - paths: - - '.ortversions/nodejs.versions' - - '.github/workflows/docker-ort-base.yml' - workflow_run: - workflows: ["Python Image"] - types: - - completed - -env: - REGISTRY: ghcr.io - -permissions: write-all - -jobs: - build: - name: Build ORT NodeJS Image - runs-on: ubuntu-22.04 - permissions: - contents: read - packages: write - - steps: - - name: Checkout main repository - uses: actions/checkout@v3 - - - name: Set environment variables - run: | - cat .ortversions/nodejs.versions >> $GITHUB_ENV - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 - - - name: Login to GitHub Container Registry - uses: docker/login-action@v2 - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Extract components metadata (tags, labels) for base image - id: meta - uses: docker/metadata-action@v4 - with: - images: | - ${{ env.REGISTRY }}/${{ github.repository }}/nodejs - tags: | - type=raw,value=${{ env.NODEJS_VERSION }} - - - name: Build ORT NodeJS container - uses: docker/build-push-action@v4 - with: - context: . - target: nodejs - push: ${{ github.event_name == 'push' || github.event_name == 'workflow_dispatch' }} - load: false - build-args: | - NODEJS_VERSION=${{ env.NODEJS_VERSION }} - NPM_VERSION=${{ env.NPM_VERSION }} - PNPM_VERSION=${{ env.PNPM_VERSION }} - YARN_VERSION=${{ env.YARN_VERSION }} - tags: | - ${{ steps.meta.outputs.tags }} - ${{ env.REGISTRY }}/${{ github.repository }}/nodejs:latest - labels: ${{ steps.meta.outputs.labels }} - build-contexts: | - ort-base-image=docker-image://${{ env.REGISTRY }}/${{ github.repository }}/base:latest - cache-from: type=gha,scope=ortnodejs - cache-to: type=gha,scope=ortnodejs,mode=max diff --git a/.github/workflows/docker-ort-python.yml b/.github/workflows/docker-ort-python.yml deleted file mode 100644 index 783e3fc4d3bec..0000000000000 --- a/.github/workflows/docker-ort-python.yml +++ /dev/null @@ -1,96 +0,0 @@ -# Copyright (C) 2023 The ORT Project Authors (see ) -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# SPDX-License-Identifier: Apache-2.0 -# License-Filename: LICENSE - -name: Python Image - -on: - workflow_dispatch: - push: - branches: - - main - pull_request: - paths: - - '.ortversions/python.versions' - - '.github/workflows/docker-ort-base.yml' - workflow_run: - workflows: ["Base Image"] - types: - - completed - -env: - REGISTRY: ghcr.io - -permissions: write-all - -jobs: - build: - name: Build ORT Python Image - runs-on: ubuntu-22.04 - permissions: - contents: read - packages: write - - steps: - - name: Checkout main repository - uses: actions/checkout@v3 - - - name: Set environment variables - run: | - cat .ortversions/python.versions >> $GITHUB_ENV - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 - - - name: Login to GitHub Container Registry - uses: docker/login-action@v2 - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Extract components metadata (tags, labels) for base image - id: meta - uses: docker/metadata-action@v4 - with: - images: | - ${{ env.REGISTRY }}/${{ github.repository }}/python - tags: | - type=raw,value=${{ env.PYTHON_VERSION }} - - - name: Build ORT Python container - uses: docker/build-push-action@v4 - with: - context: . - target: python - push: ${{ github.event_name == 'push' || github.event_name == 'workflow_dispatch' }} - load: false - build-args: | - CONAN_VERSION=${{ env.CONAN_VERSION }} - PIPTOOL_VERSION=${{ env.PIPTOOL_VERSION }} - PYENV_GIT_TAG=${{ env.PYENV_GIT_TAG }} - PYTHON_INSPECTOR_VERSION=${{ env.PYTHON_INSPECTOR_VERSION }} - PYTHON_PIPENV_VERSION=${{ env.PYTHON_PIPENV_VERSION }} - PYTHON_POETRY_VERSION=${{ env.PYTHON_POETRY_VERSION }} - PYTHON_VERSION=${{ env.PYTHON_VERSION }} - SCANCODE_VERSION=${{ env.SCANCODE_VERSION }} - tags: | - ${{ steps.meta.outputs.tags }} - ${{ env.REGISTRY }}/${{ github.repository }}/python:${{ env.PYTHON_VERSION }} - ${{ env.REGISTRY }}/${{ github.repository }}/python:latest - labels: ${{ steps.meta.outputs.labels }} - cache-from: type=gha,scope=ortpython - cache-to: type=gha,scope=ortpython,mode=max diff --git a/.github/workflows/docker-ort-ruby.yml b/.github/workflows/docker-ort-ruby.yml deleted file mode 100644 index b8de1d088dd6e..0000000000000 --- a/.github/workflows/docker-ort-ruby.yml +++ /dev/null @@ -1,93 +0,0 @@ -# Copyright (C) 2023 The ORT Project Authors (see ) -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# SPDX-License-Identifier: Apache-2.0 -# License-Filename: LICENSE - -name: Ruby Image - -on: - workflow_dispatch: - push: - branches: - - main - pull_request: - paths: - - '.ortversions/ruby.versions' - - '.github/workflows/docker-ort-base.yml' - workflow_run: - workflows: ["Rust Image"] - types: - - completed - -env: - REGISTRY: ghcr.io - -permissions: write-all - -jobs: - build: - name: Build ORT Ruby Image - runs-on: ubuntu-22.04 - permissions: - contents: read - packages: write - - steps: - - name: Checkout main repository - uses: actions/checkout@v3 - - - name: Set environment variables - run: | - cat .ortversions/ruby.versions >> $GITHUB_ENV - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 - - - name: Login to GitHub Container Registry - uses: docker/login-action@v2 - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Extract components metadata (tags, labels) for base image - id: meta - uses: docker/metadata-action@v4 - with: - images: | - ${{ env.REGISTRY }}/${{ github.repository }}/ruby - tags: | - type=semver,pattern={{version}},value=${{ env.RUBY_VERSION }} - - - - name: Build ORT Ruby container - if: github.event_name == 'push' - uses: docker/build-push-action@v4 - with: - context: . - target: ruby - push: ${{ github.event_name == 'push' || github.event_name == 'workflow_dispatch' }} - load: false - build-args: | - RUBY_VERSION=${{ env.RUBY_VERSION }} - COCOAPODS_VERSION=${{ env.COCOAPODS_VERSION }} - tags: | - ${{ steps.meta.outputs.tags }} - ${{ env.REGISTRY }}/${{ github.repository }}/ruby:latest - labels: ${{ steps.meta.outputs.labels }} - build-contexts: | - ort-base-image=docker-image://${{ env.REGISTRY }}/${{ github.repository }}/base:latest - cache-from: type=gha,scope=ortruby - cache-to: type=gha,scope=ortruby,mode=max diff --git a/.github/workflows/docker-ort-runtime-ext.yml b/.github/workflows/docker-ort-runtime-ext.yml index 5e067ad073cbe..1b8e68567af52 100644 --- a/.github/workflows/docker-ort-runtime-ext.yml +++ b/.github/workflows/docker-ort-runtime-ext.yml @@ -15,19 +15,23 @@ # SPDX-License-Identifier: Apache-2.0 # License-Filename: LICENSE -name: Extended Image +name: MegaDocker Extended Image on: workflow_dispatch: pull_request: - branches: - - main + paths: + - '.versions' + - 'Dockerfile' + - 'Dockerfile-extended' push: branches: - main + tags: + - '*' workflow_run: workflows: - - 'Haskell Image' + - 'MegaDocker Runtime Image' types: - completed @@ -37,8 +41,119 @@ env: permissions: write-all jobs: - build: + android_image: + name: Android Image + runs-on: ubuntu-22.04 + steps: + - name: Checkout default branch + uses: actions/checkout@v3 + - name: Set environment variables + run: | + cat .versions >> $GITHUB_ENV + - name: Build Android Image + uses: ./.github/actions/ortdocker + with: + name: android + token: ${{ secrets.GITHUB_TOKEN }} + version: "${{ env.ANDROID_CMD_VERSION }}" + build-args: | + ANDROID_CMD_VERSION=${{ env.ANDROID_CMD_VERSION }} + + dart_image: + name: Dart Image + runs-on: ubuntu-22.04 + steps: + - name: Checkout default branch + uses: actions/checkout@v3 + - name: Set environment variables + run: | + cat .versions >> $GITHUB_ENV + - name: Build Dart Image + uses: ./.github/actions/ortdocker + with: + name: dart + token: ${{ secrets.GITHUB_TOKEN }} + version: "${{ env.DART_VERSION }}" + build-args: | + DART_VERSION=${{ env.DART_VERSION }} + + dotnet_image: + name: Dotnet Image + runs-on: ubuntu-22.04 + steps: + - name: Checkout default branch + uses: actions/checkout@v3 + - name: Set environment variables + run: | + cat .versions >> $GITHUB_ENV + - name: Build Dotnet Image + uses: ./.github/actions/ortdocker + with: + name: dotnet + token: ${{ secrets.GITHUB_TOKEN }} + version: "${{ env.DOTNET_VERSION }}" + build-args: | + DOTNET_VERSION=${{ env.DOTNET_VERSION }} + NUGET_INSPECTOR_VERSION=${{ env.NUGET_INSPECTOR_VERSION }} + + haskell_image: + name: Haskell Image + runs-on: ubuntu-22.04 + steps: + - name: Checkout default branch + uses: actions/checkout@v3 + - name: Set environment variables + run: | + cat .versions >> $GITHUB_ENV + - name: Build Haskell Image + uses: ./.github/actions/ortdocker + with: + name: haskell + token: ${{ secrets.GITHUB_TOKEN }} + version: "${{ env.HASKELL_STACK_VERSION }}" + build-args: | + HASKELL_STACK_VERSION=${{ env.HASKELL_STACK_VERSION }} + + scala_image: + name: Scala Image + runs-on: ubuntu-22.04 + steps: + - name: Checkout default branch + uses: actions/checkout@v3 + - name: Set environment variables + run: | + cat .versions >> $GITHUB_ENV + - name: Build Scala Image + uses: ./.github/actions/ortdocker + with: + name: scala + token: ${{ secrets.GITHUB_TOKEN }} + version: "${{ env.SBT_VERSION }}" + build-args: | + SBT_VERSION=${{ env.SBT_VERSION }} + + swift_image: + name: Swift Image + runs-on: ubuntu-22.04 + steps: + - name: Checkout default branch + uses: actions/checkout@v3 + - name: Set environment variables + run: | + cat .versions >> $GITHUB_ENV + - name: Build Swift Image + uses: ./.github/actions/ortdocker + with: + name: swift + token: ${{ secrets.GITHUB_TOKEN }} + version: "${{ env.SWIFT_VERSION }}" + build-args: | + SWIFT_VERSION=${{ env.SWIFT_VERSION }} + + runtime_extended_image: + if: ${{ github.event_name == 'push' || github.event_name == 'workflow_dispatch' }} name: Build ORT Extended Image + needs: [ android_image, dart_image, dotnet_image, haskell_image, scala_image, swift_image ] runs-on: ubuntu-22.04 permissions: contents: read @@ -79,10 +194,10 @@ jobs: ${{ env.REGISTRY }}/${{ github.repository_owner }}/ort-extended:latest labels: ${{ steps.meta.outputs.labels }} build-contexts: | + ort=docker-image://${{ env.REGISTRY }}/${{ github.repository_owner }}/ort:latest android=docker-image://${{ env.REGISTRY }}/${{ github.repository }}/android:latest swift=docker-image://${{ env.REGISTRY }}/${{ github.repository }}/swift:latest - sbt=docker-image://${{ env.REGISTRY }}/${{ github.repository }}/sbt:latest + scala=docker-image://${{ env.REGISTRY }}/${{ github.repository }}/scala:latest dart=docker-image://${{ env.REGISTRY }}/${{ github.repository }}/dart:latest dotnet=docker-image://${{ env.REGISTRY }}/${{ github.repository }}/dotnet:latest haskell=docker-image://${{ env.REGISTRY }}/${{ github.repository }}/haskell:latest - dotnet=docker-image://${{ env.REGISTRY }}/${{ github.repository }}/dotnet:latest diff --git a/.github/workflows/docker-ort-runtime.yml b/.github/workflows/docker-ort-runtime.yml index 76e34226d2680..f467e7d46dd01 100644 --- a/.github/workflows/docker-ort-runtime.yml +++ b/.github/workflows/docker-ort-runtime.yml @@ -15,21 +15,19 @@ # SPDX-License-Identifier: Apache-2.0 # License-Filename: LICENSE -name: Runtime Image +name: MegaDocker Runtime Image on: workflow_dispatch: pull_request: - branches: - - main + paths: + - '.versions' + - 'Dockerfile' push: branches: - main - workflow_run: - workflows: - - 'Golang Image' - types: - - completed + tags: + - '*' env: REGISTRY: ghcr.io @@ -37,7 +35,134 @@ env: permissions: write-all jobs: - build: + base_image: + name: Base Image + runs-on: ubuntu-22.04 + steps: + - name: Checkout default branch + uses: actions/checkout@v3 + - name: Set environment variables + run: | + cat .versions >> $GITHUB_ENV + - name: Build Base Image + uses: ./.github/actions/ortdocker + with: + name: base + token: ${{ secrets.GITHUB_TOKEN }} + version: "${{ env.JAVA_VERSION }}-jdk-${{ env.UBUNTU_VERSION }}" + build-args: | + JAVA_VERSION=${{ env.JAVA_VERSION }} + UBUNTU_VERSION=${{ env.UBUNTU_VERSION }} + + nodejs_image: + name: NodeJS Image + needs: [base_image] + runs-on: ubuntu-22.04 + steps: + - name: Checkout default branch + uses: actions/checkout@v3 + - name: Set environment variables + run: | + cat .versions >> $GITHUB_ENV + - name: Build NodeJS Image + uses: ./.github/actions/ortdocker + with: + name: nodejs + token: ${{ secrets.GITHUB_TOKEN }} + version: "${{ env.NODEJS_VERSION }}" + build-args: | + NODEJS_VERSION=${{ env.NODEJS_VERSION }} + NPM_VERSION=${{ env.NPM_VERSION }} + PNPM_VERSION=${{ env.PNPM_VERSION }} + YARN_VERSION=${{ env.YARN_VERSION }} + + python_image: + name: Python Image + needs: [base_image] + runs-on: ubuntu-22.04 + steps: + - name: Checkout default branch + uses: actions/checkout@v3 + - name: Set environment variables + run: | + cat .versions >> $GITHUB_ENV + - name: Build Python Image + uses: ./.github/actions/ortdocker + with: + name: python + token: ${{ secrets.GITHUB_TOKEN }} + version: "${{ env.PYTHON_VERSION }}" + build-args: | + CONAN_VERSION=${{ env.CONAN_VERSION }} + PIPTOOL_VERSION=${{ env.PIPTOOL_VERSION }} + PYENV_GIT_TAG=${{ env.PYENV_GIT_TAG }} + PYTHON_INSPECTOR_VERSION=${{ env.PYTHON_INSPECTOR_VERSION }} + PYTHON_PIPENV_VERSION=${{ env.PYTHON_PIPENV_VERSION }} + PYTHON_POETRY_VERSION=${{ env.PYTHON_POETRY_VERSION }} + PYTHON_VERSION=${{ env.PYTHON_VERSION }} + SCANCODE_VERSION=${{ env.SCANCODE_VERSION }} + + rust_image: + name: Rust Image + needs: [base_image] + runs-on: ubuntu-22.04 + steps: + - name: Checkout default branch + uses: actions/checkout@v3 + - name: Set environment variables + run: | + cat .versions >> $GITHUB_ENV + - name: Build Rust Image + uses: ./.github/actions/ortdocker + with: + name: rust + token: ${{ secrets.GITHUB_TOKEN }} + version: "${{ env.RUST_VERSION }}" + build-args: | + RUST_VERSION=${{ env.RUST_VERSION }} + + ruby_image: + name: Ruby Image + needs: [base_image] + runs-on: ubuntu-22.04 + steps: + - name: Checkout default branch + uses: actions/checkout@v3 + - name: Set environment variables + run: | + cat .versions >> $GITHUB_ENV + - name: Build Ruby Image + uses: ./.github/actions/ortdocker + with: + name: ruby + token: ${{ secrets.GITHUB_TOKEN }} + version: "${{ env.RUBY_VERSION }}" + build-args: | + RUBY_VERSION=${{ env.RUBY_VERSION }} + COCOAPODS_VERSION=${{ env.COCOAPODS_VERSION }} + + golang_image: + name: Golang Image + needs: [base_image] + runs-on: ubuntu-22.04 + steps: + - name: Checkout default branch + uses: actions/checkout@v3 + - name: Set environment variables + run: | + cat .versions >> $GITHUB_ENV + - name: Build Golang Image + uses: ./.github/actions/ortdocker + with: + name: golang + token: ${{ secrets.GITHUB_TOKEN }} + version: "${{ env.GO_VERSION }}" + build-args: | + GO_DEP_VERSION=${{ env.GO_DEP_VERSION }} + GO_VERSION=${{ env.GO_VERSION }} + + runtime_image: + needs: [base_image, nodejs_image, python_image, rust_image, ruby_image, golang_image] name: Build ORT runtime Image runs-on: ubuntu-22.04 permissions: @@ -50,7 +175,7 @@ jobs: - name: Set environment variables run: | - cat .ortversions/nodejs.versions >> $GITHUB_ENV + cat .versions >> $GITHUB_ENV - name: Set up Docker Buildx uses: docker/setup-buildx-action@v2 @@ -72,11 +197,12 @@ jobs: type=raw,sha,enable=true,format=short - name: Build ORT runtime container + if: ${{ github.event_name == 'push' || github.event_name == 'workflow_dispatch' }} uses: docker/build-push-action@v4 with: context: . target: run - push: ${{ github.event_name == 'push' || github.event_name == 'workflow_dispatch' }} + push: true load: false build-args: | NODEJS_VERSION=${{ env.NODEJS_VERSION }} diff --git a/.github/workflows/docker-ort-rust.yml b/.github/workflows/docker-ort-rust.yml deleted file mode 100644 index 654c122e36594..0000000000000 --- a/.github/workflows/docker-ort-rust.yml +++ /dev/null @@ -1,91 +0,0 @@ -# Copyright (C) 2023 The ORT Project Authors (see ) -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# SPDX-License-Identifier: Apache-2.0 -# License-Filename: LICENSE - -name: Rust Image - -on: - workflow_dispatch: - push: - branches: - - main - pull_request: - paths: - - '.ortversions/rust.versions' - - '.github/workflows/docker-ort-base.yml' - workflow_run: - workflows: ["NodeJS Image"] - types: - - completed - -env: - REGISTRY: ghcr.io - -permissions: write-all - -jobs: - build: - name: Build ORT Rust Image - runs-on: ubuntu-22.04 - permissions: - contents: read - packages: write - - steps: - - name: Checkout main repository - uses: actions/checkout@v3 - - - name: Set environment variables - run: | - cat .ortversions/rust.versions >> $GITHUB_ENV - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 - - - name: Login to GitHub Container Registry - uses: docker/login-action@v2 - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Extract components metadata (tags, labels) for base image - id: meta - uses: docker/metadata-action@v4 - with: - images: | - ${{ env.REGISTRY }}/${{ github.repository }}/rust - tags: | - type=raw,value=${{ env.RUST_VERSION }} - - - - name: Build ORT Rust container - uses: docker/build-push-action@v4 - with: - context: . - target: rust - push: ${{ github.event_name == 'push' || github.event_name == 'workflow_dispatch' }} - load: false - build-args: | - RUST_VERSION=${{ env.RUST_VERSION }} - tags: | - ${{ steps.meta.outputs.tags }} - ${{ env.REGISTRY }}/${{ github.repository }}/rust:latest - labels: ${{ steps.meta.outputs.labels }} - build-contexts: | - ort-base-image=docker-image://${{ env.REGISTRY }}/${{ github.repository }}/base:latest - cache-from: type=gha,scope=ortrust - cache-to: type=gha,scope=ortrust,mode=max diff --git a/.github/workflows/docker-ort-sbt.yml b/.github/workflows/docker-ort-sbt.yml deleted file mode 100644 index b4395ce1ee800..0000000000000 --- a/.github/workflows/docker-ort-sbt.yml +++ /dev/null @@ -1,91 +0,0 @@ -# Copyright (C) 2023 The ORT Project Authors (see ) -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# SPDX-License-Identifier: Apache-2.0 -# License-Filename: LICENSE - -name: Scala Image - -on: - workflow_dispatch: - push: - branches: - - main - pull_request: - paths: - - '.ortversions/sbt.versions' - - '.github/workflows/docker-ort-base.yml' - workflow_run: - workflows: ["Base Image"] - types: - - completed - -env: - REGISTRY: ghcr.io - -permissions: write-all - -jobs: - build: - name: Build ORT sbt Image - runs-on: ubuntu-22.04 - permissions: - contents: read - packages: write - - steps: - - name: Checkout main repository - uses: actions/checkout@v3 - - - name: Set environment variables - run: | - cat .ortversions/sbt.versions >> $GITHUB_ENV - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 - - - name: Login to GitHub Container Registry - uses: docker/login-action@v2 - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Extract components metadata (tags, labels) for base image - id: meta - uses: docker/metadata-action@v4 - with: - images: | - ${{ env.REGISTRY }}/${{ github.repository }}/sbt - tags: | - type=semver,pattern={{version}},value=${{ env.SBT_VERSION }} - - - - name: Build ORT sbt container - uses: docker/build-push-action@v4 - with: - context: . - target: sbt - push: ${{ github.event_name == 'push' || github.event_name == 'workflow_dispatch' }} - load: false - build-args: | - SBT_VERSION=${{ env.SBT_VERSION }} - tags: | - ${{ steps.meta.outputs.tags }} - ${{ env.REGISTRY }}/${{ github.repository }}/sbt:latest - labels: ${{ steps.meta.outputs.labels }} - build-contexts: | - ort-base-image=docker-image://${{ env.REGISTRY }}/${{ github.repository }}/base:latest - cache-from: type=gha,scope=ortsbt - cache-to: type=gha,scope=ortsbt,mode=max diff --git a/.github/workflows/docker-ort-scala.yml b/.github/workflows/docker-ort-scala.yml deleted file mode 100644 index f091a60eb3c34..0000000000000 --- a/.github/workflows/docker-ort-scala.yml +++ /dev/null @@ -1,91 +0,0 @@ -# Copyright (C) 2023 The ORT Project Authors (see ) -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# SPDX-License-Identifier: Apache-2.0 -# License-Filename: LICENSE - -name: Scala Image - -on: - workflow_dispatch: - push: - branches: - - main - pull_request: - paths: - - '.ortversions/sbt.versions' - - '.github/workflows/docker-ort-base.yml' - workflow_run: - workflows: ["Swift Image"] - types: - - completed - -env: - REGISTRY: ghcr.io - -permissions: write-all - -jobs: - build: - name: Build ORT sbt Image - runs-on: ubuntu-22.04 - permissions: - contents: read - packages: write - - steps: - - name: Checkout main repository - uses: actions/checkout@v3 - - - name: Set environment variables - run: | - cat .ortversions/sbt.versions >> $GITHUB_ENV - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 - - - name: Login to GitHub Container Registry - uses: docker/login-action@v2 - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Extract components metadata (tags, labels) for base image - id: meta - uses: docker/metadata-action@v4 - with: - images: | - ${{ env.REGISTRY }}/${{ github.repository }}/sbt - tags: | - type=semver,pattern={{version}},value=${{ env.SBT_VERSION }} - - - - name: Build ORT sbt container - uses: docker/build-push-action@v4 - with: - context: . - target: sbt - push: ${{ github.event_name == 'push' || github.event_name == 'workflow_dispatch' }} - load: false - build-args: | - SBT_VERSION=${{ env.SBT_VERSION }} - tags: | - ${{ steps.meta.outputs.tags }} - ${{ env.REGISTRY }}/${{ github.repository }}/sbt:latest - labels: ${{ steps.meta.outputs.labels }} - build-contexts: | - ort-base-image=docker-image://${{ env.REGISTRY }}/${{ github.repository }}/base:latest - cache-from: type=gha,scope=ortsbt - cache-to: type=gha,scope=ortsbt,mode=max diff --git a/.github/workflows/docker-ort-swift.yml b/.github/workflows/docker-ort-swift.yml deleted file mode 100644 index eeeac6d0e7c35..0000000000000 --- a/.github/workflows/docker-ort-swift.yml +++ /dev/null @@ -1,91 +0,0 @@ -# Copyright (C) 2023 The ORT Project Authors (see ) -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# SPDX-License-Identifier: Apache-2.0 -# License-Filename: LICENSE - -name: Swift Image - -on: - workflow_dispatch: - push: - branches: - - main - pull_request: - paths: - - '.ortversions/swift.versions' - - '.github/workflows/docker-ort-base.yml' - workflow_run: - workflows: ["Android Image"] - types: - - completed - -env: - REGISTRY: ghcr.io - -permissions: write-all - -jobs: - build: - name: Build ORT Swift Image - runs-on: ubuntu-22.04 - permissions: - contents: read - packages: write - - steps: - - name: Checkout main repository - uses: actions/checkout@v3 - - - name: Set environment variables - run: | - cat .ortversions/swift.versions >> $GITHUB_ENV - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 - - - name: Login to GitHub Container Registry - uses: docker/login-action@v2 - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Extract components metadata (tags, labels) for base image - id: meta - uses: docker/metadata-action@v4 - with: - images: | - ${{ env.REGISTRY }}/${{ github.repository }}/swift - tags: | - type=raw,value=${{ env.SWIFT_VERSION }} - - - - name: Build ORT Swift container - uses: docker/build-push-action@v4 - with: - context: . - target: swift - push: ${{ github.event_name == 'push' || github.event_name == 'workflow_dispatch' }} - load: false - build-args: | - SWIFT_VERSION=${{ env.SWIFT_VERSION }} - tags: | - ${{ steps.meta.outputs.tags }} - ${{ env.REGISTRY }}/${{ github.repository }}/swift:latest - labels: ${{ steps.meta.outputs.labels }} - build-contexts: | - ort-base-image=docker-image://${{ env.REGISTRY }}/${{ github.repository }}/base:latest - cache-from: type=gha,scope=ortswift - cache-to: type=gha,scope=ortswift,mode=max diff --git a/.ortversions/android.versions b/.ortversions/android.versions deleted file mode 100644 index 6a895889da72a..0000000000000 --- a/.ortversions/android.versions +++ /dev/null @@ -1 +0,0 @@ -ANDROID_CMD_VERSION=9477386 \ No newline at end of file diff --git a/.ortversions/base.versions b/.ortversions/base.versions deleted file mode 100644 index abc7cc3fac010..0000000000000 --- a/.ortversions/base.versions +++ /dev/null @@ -1,2 +0,0 @@ -JAVA_VERSION=17 -UBUNTU_VERSION=jammy diff --git a/.ortversions/dart.versions b/.ortversions/dart.versions deleted file mode 100644 index 31191f8ff697a..0000000000000 --- a/.ortversions/dart.versions +++ /dev/null @@ -1 +0,0 @@ -DART_VERSION=2.18.4 \ No newline at end of file diff --git a/.ortversions/dotnet.versions b/.ortversions/dotnet.versions deleted file mode 100644 index 29159cfc2b228..0000000000000 --- a/.ortversions/dotnet.versions +++ /dev/null @@ -1,2 +0,0 @@ -DOTNET_VERSION=6.0 -NUGET_INSPECTOR_VERSION=0.9.12 diff --git a/.ortversions/golang.versions b/.ortversions/golang.versions deleted file mode 100644 index 8625a6c84e258..0000000000000 --- a/.ortversions/golang.versions +++ /dev/null @@ -1,2 +0,0 @@ -GO_DEP_VERSION=0.5.4 -GO_VERSION=1.20.5 diff --git a/.ortversions/haskell.versions b/.ortversions/haskell.versions deleted file mode 100644 index 9e794390d449f..0000000000000 --- a/.ortversions/haskell.versions +++ /dev/null @@ -1 +0,0 @@ -HASKELL_STACK_VERSION=2.7.5 \ No newline at end of file diff --git a/.ortversions/nodejs.versions b/.ortversions/nodejs.versions deleted file mode 100644 index a278ab79d394e..0000000000000 --- a/.ortversions/nodejs.versions +++ /dev/null @@ -1,5 +0,0 @@ -BOWER_VERSION=1.8.12 -NODEJS_VERSION=18.17.1 -NPM_VERSION=8.15.1 -PNPM_VERSION=7.8.0 -YARN_VERSION=1.22.17 diff --git a/.ortversions/php.versions b/.ortversions/php.versions deleted file mode 100644 index 8114d4c2b7b06..0000000000000 --- a/.ortversions/php.versions +++ /dev/null @@ -1,2 +0,0 @@ -PHP_VERSION=8.1 -COMPOSER_VERSION=2.2 \ No newline at end of file diff --git a/.ortversions/python.versions b/.ortversions/python.versions deleted file mode 100644 index 35b83ed1ce6b0..0000000000000 --- a/.ortversions/python.versions +++ /dev/null @@ -1,8 +0,0 @@ -CONAN_VERSION=1.57.0 -PIPTOOL_VERSION=22.2.2 -PYENV_GIT_TAG=v2.3.25 -PYTHON_INSPECTOR_VERSION=0.9.8 -PYTHON_PIPENV_VERSION=2022.9.24 -PYTHON_POETRY_VERSION=1.6.1 -PYTHON_VERSION=3.10.13 -SCANCODE_VERSION=32.0.6 diff --git a/.ortversions/ruby.versions b/.ortversions/ruby.versions deleted file mode 100644 index 71b8026d9091e..0000000000000 --- a/.ortversions/ruby.versions +++ /dev/null @@ -1,2 +0,0 @@ -COCOAPODS_VERSION=1.11.2 -RUBY_VERSION=3.1.2 diff --git a/.ortversions/rust.versions b/.ortversions/rust.versions deleted file mode 100644 index 2e59451ab4622..0000000000000 --- a/.ortversions/rust.versions +++ /dev/null @@ -1 +0,0 @@ -RUST_VERSION=1.72.0 diff --git a/.ortversions/sbt.versions b/.ortversions/sbt.versions deleted file mode 100644 index 9e0c411cab528..0000000000000 --- a/.ortversions/sbt.versions +++ /dev/null @@ -1 +0,0 @@ -SBT_VERSION=1.6.1 diff --git a/.ortversions/swift.versions b/.ortversions/swift.versions deleted file mode 100644 index 46c2142b6fdb0..0000000000000 --- a/.ortversions/swift.versions +++ /dev/null @@ -1 +0,0 @@ -SWIFT_VERSION=5.8.1 diff --git a/.versions b/.versions new file mode 100644 index 0000000000000..41c33d6495d3e --- /dev/null +++ b/.versions @@ -0,0 +1,29 @@ +ANDROID_CMD_VERSION=9477386 +BOWER_VERSION=1.8.12 +COCOAPODS_VERSION=1.11.2 +COMPOSER_VERSION=2.2 +CONAN_VERSION=1.57.0 +DART_VERSION=2.18.4 +DOTNET_VERSION=6.0 +GO_DEP_VERSION=0.5.4 +GO_VERSION=1.20.5 +HASKELL_STACK_VERSION=2.7.5 +JAVA_VERSION=17 +NODEJS_VERSION=18.17.1 +NPM_VERSION=8.15.1 +NUGET_INSPECTOR_VERSION=0.9.12 +PHP_VERSION=8.1 +PIPTOOL_VERSION=22.2.2 +PNPM_VERSION=7.8.0 +PYENV_GIT_TAG=v2.3.25 +PYTHON_INSPECTOR_VERSION=0.9.8 +PYTHON_PIPENV_VERSION=2022.9.24 +PYTHON_POETRY_VERSION=1.6.1 +PYTHON_VERSION=3.10.13 +RUBY_VERSION=3.1.2 +RUST_VERSION=1.72.0 +SBT_VERSION=1.6.1 +SCANCODE_VERSION=32.0.6 +SWIFT_VERSION=5.8.1 +UBUNTU_VERSION=jammy +YARN_VERSION=1.22.17 diff --git a/Dockerfile b/Dockerfile index 9234dc5b3a4b7..a98de4635e182 100644 --- a/Dockerfile +++ b/Dockerfile @@ -22,7 +22,7 @@ ARG JAVA_VERSION=17 ARG UBUNTU_VERSION=jammy # Use OpenJDK Eclipe Temurin Ubuntu LTS -FROM eclipse-temurin:$JAVA_VERSION-jdk-$UBUNTU_VERSION as ort-base-image +FROM eclipse-temurin:$JAVA_VERSION-jdk-$UBUNTU_VERSION as base ENV LANG=en_US.UTF-8 ENV LANGUAGE=en_US:en @@ -116,7 +116,7 @@ ENTRYPOINT [ "/bin/bash" ] #------------------------------------------------------------------------ # PYTHON - Build Python as a separate component with pyenv -FROM ort-base-image AS pythonbuild +FROM base AS pythonbuild SHELL ["/bin/bash", "-o", "pipefail", "-c"] @@ -174,7 +174,7 @@ COPY --from=pythonbuild /opt/python /opt/python #------------------------------------------------------------------------ # NODEJS - Build NodeJS as a separate component with nvm -FROM ort-base-image AS nodejsbuild +FROM base AS nodejsbuild ARG BOWER_VERSION=1.8.12 ARG NODEJS_VERSION=18.14.2 @@ -197,7 +197,7 @@ COPY --from=nodejsbuild /opt/nvm /opt/nvm #------------------------------------------------------------------------ # RUBY - Build Ruby as a separate component with rbenv -FROM ort-base-image AS rubybuild +FROM base AS rubybuild # hadolint ignore=DL3004 RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ @@ -232,7 +232,7 @@ COPY --from=rubybuild /opt/rbenv /opt/rbenv #------------------------------------------------------------------------ # RUST - Build as a separate component -FROM ort-base-image AS rustbuild +FROM base AS rustbuild ARG RUST_VERSION=1.72.0 @@ -246,7 +246,7 @@ COPY --from=rustbuild /opt/rust /opt/rust #------------------------------------------------------------------------ # GOLANG - Build as a separate component -FROM ort-base-image AS gobuild +FROM base AS gobuild ARG GO_DEP_VERSION=0.5.4 ARG GO_VERSION=1.20.5 @@ -263,7 +263,7 @@ COPY --from=gobuild /opt/go /opt/go #------------------------------------------------------------------------ # HASKELL STACK -FROM ort-base-image AS haskellbuild +FROM base AS haskellbuild RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ --mount=type=cache,target=/var/lib/apt,sharing=locked \ @@ -284,7 +284,7 @@ COPY --from=haskellbuild /opt/haskell /opt/haskell #------------------------------------------------------------------------ # REPO / ANDROID SDK -FROM ort-base-image AS androidbuild +FROM base AS androidbuild RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ --mount=type=cache,target=/var/lib/apt,sharing=locked \ @@ -316,7 +316,7 @@ COPY --from=androidbuild /opt/android-sdk /opt/android-sdk #------------------------------------------------------------------------ # Dart -FROM ort-base-image AS dartbuild +FROM base AS dartbuild ARG DART_VERSION=2.18.4 WORKDIR /opt/ @@ -336,7 +336,7 @@ COPY --from=dartbuild /opt/dart-sdk /opt/dart-sdk #------------------------------------------------------------------------ # SBT -FROM ort-base-image AS sbtbuild +FROM base AS scalabuild ARG SBT_VERSION=1.6.1 @@ -345,12 +345,12 @@ ENV PATH=$PATH:$SBT_HOME/bin RUN curl -L https://github.com/sbt/sbt/releases/download/v$SBT_VERSION/sbt-$SBT_VERSION.tgz | tar -C /opt -xz -FROM scratch AS sbt -COPY --from=sbtbuild /opt/sbt /opt/sbt +FROM scratch AS scala +COPY --from=scalabuild /opt/sbt /opt/sbt #------------------------------------------------------------------------ # SWIFT -FROM ort-base-image AS swiftbuild +FROM base AS swiftbuild ARG SWIFT_VERSION=5.8.1 @@ -372,7 +372,7 @@ COPY --from=swiftbuild /opt/swift /opt/swift #------------------------------------------------------------------------ # DOTNET -FROM ort-base-image AS dotnetbuild +FROM base AS dotnetbuild ARG DOTNET_VERSION=6.0 ARG NUGET_INSPECTOR_VERSION=0.9.12 @@ -401,7 +401,7 @@ COPY --from=dotnetbuild /opt/dotnet /opt/dotnet #------------------------------------------------------------------------ # ORT -FROM ort-base-image as ortbuild +FROM base as ortbuild # Set this to the version ORT should report. ARG ORT_VERSION="DOCKER-SNAPSHOT" @@ -429,7 +429,7 @@ COPY --from=ortbuild /opt/ort /opt/ort #------------------------------------------------------------------------ # Main Minimal Runtime container -FROM ort-base-image as run +FROM base as run # Remove ort build scripts RUN [ -d /etc/scripts ] && sudo rm -rf /etc/scripts @@ -486,6 +486,6 @@ USER $USER WORKDIR $HOME # Ensure that the ORT data directory exists to be able to mount the config into it with correct permissions. -RUN mkdir -p "$HOMEDIR/.ort" +RUN mkdir -p "$HOME/.ort" ENTRYPOINT ["/opt/ort/bin/ort"] diff --git a/Dockerfile-extended b/Dockerfile-extended index 3dce2517060ce..182b146b6b05c 100644 --- a/Dockerfile-extended +++ b/Dockerfile-extended @@ -1,4 +1,4 @@ -FROM ghcr.io/oss-review-toolkit/ort +FROM ort # Copyright (C) 2023 The ORT Project Authors (see ) # @@ -38,7 +38,7 @@ RUN syft $SWIFT_HOME -o spdx-json --file /usr/share/doc/ort/ort-swift.spdx.json # Scala ENV SBT_HOME=/opt/sbt ENV PATH=$PATH:$SBT_HOME/bin -COPY --from=sbt --chown=$USER:$USER $SBT_HOME $SBT_HOME +COPY --from=scala --chown=$USER:$USER $SBT_HOME $SBT_HOME RUN syft $SBT_HOME -o spdx-json --file /usr/share/doc/ort/ort-sbt.spdx.json diff --git a/scripts/docker_build.sh b/scripts/docker_build.sh index 3bf5622a4b4a1..0c357107c8e7c 100755 --- a/scripts/docker_build.sh +++ b/scripts/docker_build.sh @@ -25,6 +25,9 @@ DOCKER_IMAGE_ROOT="${DOCKER_IMAGE_ROOT:-ghcr.io/oss-review-toolkit}" echo "Setting ORT_VERSION to $GIT_REVISION." +# shellcheck disable=SC1091 +. .versions + # --------------------------- # image_build function # Usage ( position paramenters): @@ -46,21 +49,16 @@ image_build() { --target "$target" \ --tag "${DOCKER_IMAGE_ROOT}/$name:$version" \ --tag "${DOCKER_IMAGE_ROOT}/$name:latest" \ - --build-context "base=docker-image://${DOCKER_IMAGE_ROOT}/base:latest" \ "$@" . } # Base -# shellcheck disable=SC1091 -. .ortversions/base.versions -image_build ort-base-image base "${JAVA_VERSION}-jdk-${UBUNTU_VERSION}" \ +image_build base base "${JAVA_VERSION}-jdk-${UBUNTU_VERSION}" \ --build-arg UBUNTU_VERSION="$UBUNTU_VERSION" \ --build-arg JAVA_VERSION="$JAVA_VERSION" \ "$@" # Python -# shellcheck disable=SC1091 -. .ortversions/python.versions image_build python python "$PYTHON_VERSION" \ --build-arg PYTHON_VERSION="$PYTHON_VERSION" \ --build-arg CONAN_VERSION="$CONAN_VERSION" \ @@ -69,45 +67,37 @@ image_build python python "$PYTHON_VERSION" \ --build-arg PYTHON_POETRY_VERSION="$PYTHON_POETRY_VERSION" \ --build-arg PIPTOOL_VERSION="$PIPTOOL_VERSION" \ --build-arg SCANCODE_VERSION="$SCANCODE_VERSION" \ + --build-context "base=docker-image://${DOCKER_IMAGE_ROOT}/base:latest" \ "$@" # Nodejs -# shellcheck disable=SC1091 -. .ortversions/nodejs.versions image_build nodejs nodejs "$NODEJS_VERSION" \ --build-arg NODEJS_VERSION="$NODEJS_VERSION" \ --build-arg BOWER_VERSION="$BOWER_VERSION" \ --build-arg NPM_VERSION="$NPM_VERSION" \ --build-arg PNPM_VERSION="$PNPM_VERSION" \ --build-arg YARN_VERSION="$YARN_VERSION" \ + --build-context "base=docker-image://${DOCKER_IMAGE_ROOT}/base:latest" \ "$@" # Rust -# shellcheck disable=SC1091 -. .ortversions/rust.versions image_build rust rust "$RUST_VERSION" \ --build-arg RUST_VERSION="$RUST_VERSION" \ + --build-context "base=docker-image://${DOCKER_IMAGE_ROOT}/base:latest" \ "$@" # Ruby -# shellcheck disable=SC1091 -. .ortversions/ruby.versions image_build ruby ruby "$RUBY_VERSION" \ --build-arg RUBY_VERSION="$RUBY_VERSION" \ --build-arg COCOAPODS_VERSION="$COCOAPODS_VERSION" \ + --build-context "base=docker-image://${DOCKER_IMAGE_ROOT}/base:latest" \ "$@" # Golang -# shellcheck disable=SC1091 -. .ortversions/golang.versions image_build golang golang "$GO_VERSION" \ --build-arg GO_VERSION="$GO_VERSION" \ --build-arg GO_DEP_VERSION="$GO_DEP_VERSION" \ - "$@" - -# Ort -image_build ortbin ortbin "$GIT_REVISION" \ - --build-arg ORT_VERSION="$GIT_REVISION" \ + --build-context "base=docker-image://${DOCKER_IMAGE_ROOT}/base:latest" \ "$@" # Runtime ORT image @@ -118,7 +108,7 @@ image_build run ort "$GIT_REVISION" \ --build-context "rust=docker-image://${DOCKER_IMAGE_ROOT}/rust:latest" \ --build-context "golang=docker-image://${DOCKER_IMAGE_ROOT}/golang:latest" \ --build-context "ruby=docker-image://${DOCKER_IMAGE_ROOT}/ruby:latest" \ - --build-context "ortbin=docker-image://${DOCKER_IMAGE_ROOT}/ortbin:latest" \ + --build-context "base=docker-image://${DOCKER_IMAGE_ROOT}/base:latest" \ "$@" # Build adjacent language containers if ALL_LANGUAGES is set. @@ -126,45 +116,40 @@ image_build run ort "$GIT_REVISION" \ # Android # shellcheck disable=SC1091 -. .ortversions/android.versions image_build android android "$ANDROID_CMD_VERSION" \ --build-arg ANDROID_CMD_VERSION="$ANDROID_CMD_VERSION" \ + --build-context "base=docker-image://${DOCKER_IMAGE_ROOT}/base:latest" \ "$@" # Swift -# shellcheck disable=SC1091 -. .ortversions/swift.versions image_build swift swift "$SWIFT_VERSION" \ --build-arg SWIFT_VERSION="$SWIFT_VERSION" \ + --build-context "base=docker-image://${DOCKER_IMAGE_ROOT}/base:latest" \ "$@" # SBT -# shellcheck disable=SC1091 -. .ortversions/sbt.versions image_build sbt sbt "$SBT_VERSION" \ --build-arg SBT_VERSION="$SBT_VERSION" \ + --build-context "base=docker-image://${DOCKER_IMAGE_ROOT}/base:latest" \ "$@" # Dart -# shellcheck disable=SC1091 -. .ortversions/dart.versions image_build dart dart "$DART_VERSION" \ --build-arg DART_VERSION="$DART_VERSION" \ + --build-context "base=docker-image://${DOCKER_IMAGE_ROOT}/base:latest" \ "$@" # Dotnet -# shellcheck disable=SC1091 -. .ortversions/dotnet.versions image_build dotnet dotnet "$DOTNET_VERSION" \ --build-arg DOTNET_VERSION="$DOTNET_VERSION" \ --build-arg NUGET_INSPECTOR_VERSION="$NUGET_INSPECTOR_VERSION" \ + --build-context "base=docker-image://${DOCKER_IMAGE_ROOT}/base:latest" \ "$@" # Haskell -# shellcheck disable=SC1091 -. .ortversions/haskell.versions image_build haskell haskell "$HASKELL_STACK_VERSION" \ --build-arg HASKELL_STACK_VERSION="$HASKELL_STACK_VERSION" \ + --build-context "base=docker-image://${DOCKER_IMAGE_ROOT}/base:latest" \ "$@" # Runtime Extended ORT image From 8e6fd16ed53ac7a499ddf86c8c64c793c583eaa0 Mon Sep 17 00:00:00 2001 From: Helio Chissini de Castro Date: Tue, 26 Sep 2023 17:47:03 +0200 Subject: [PATCH 13/47] chore(docker): Use ort namespace Language images lives under the ort namespace to avoid clash on main organization image naming, e.g. ghcr.io/oss-review-toolkit/python versus ghcr.io/oss-review-toolkit/ort/python. Signed-off-by: Helio Chissini de Castro --- Dockerfile-extended | 4 +-- scripts/docker_build.sh | 74 +++++++++++++++++++++-------------------- 2 files changed, 40 insertions(+), 38 deletions(-) diff --git a/Dockerfile-extended b/Dockerfile-extended index 182b146b6b05c..3a878f1bb1e4f 100644 --- a/Dockerfile-extended +++ b/Dockerfile-extended @@ -1,5 +1,3 @@ -FROM ort - # Copyright (C) 2023 The ORT Project Authors (see ) # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -17,6 +15,8 @@ FROM ort # SPDX-License-Identifier: Apache-2.0 # License-Filename: LICENSE +FROM ort + # Repo and Android ENV ANDROID_HOME=/opt/android-sdk ENV ANDROID_USER_HOME=$HOME/.android diff --git a/scripts/docker_build.sh b/scripts/docker_build.sh index 0c357107c8e7c..9a8ba59a44620 100755 --- a/scripts/docker_build.sh +++ b/scripts/docker_build.sh @@ -53,13 +53,13 @@ image_build() { } # Base -image_build base base "${JAVA_VERSION}-jdk-${UBUNTU_VERSION}" \ +image_build base ort/base "${JAVA_VERSION}-jdk-${UBUNTU_VERSION}" \ --build-arg UBUNTU_VERSION="$UBUNTU_VERSION" \ --build-arg JAVA_VERSION="$JAVA_VERSION" \ "$@" # Python -image_build python python "$PYTHON_VERSION" \ +image_build python ort/python "$PYTHON_VERSION" \ --build-arg PYTHON_VERSION="$PYTHON_VERSION" \ --build-arg CONAN_VERSION="$CONAN_VERSION" \ --build-arg PYTHON_INSPECTOR_VERSION="$PYTHON_INSPECTOR_VERSION" \ @@ -67,48 +67,48 @@ image_build python python "$PYTHON_VERSION" \ --build-arg PYTHON_POETRY_VERSION="$PYTHON_POETRY_VERSION" \ --build-arg PIPTOOL_VERSION="$PIPTOOL_VERSION" \ --build-arg SCANCODE_VERSION="$SCANCODE_VERSION" \ - --build-context "base=docker-image://${DOCKER_IMAGE_ROOT}/base:latest" \ + --build-context "base=docker-image://${DOCKER_IMAGE_ROOT}/ort/base:latest" \ "$@" # Nodejs -image_build nodejs nodejs "$NODEJS_VERSION" \ +image_build nodejs ort/nodejs "$NODEJS_VERSION" \ --build-arg NODEJS_VERSION="$NODEJS_VERSION" \ --build-arg BOWER_VERSION="$BOWER_VERSION" \ --build-arg NPM_VERSION="$NPM_VERSION" \ --build-arg PNPM_VERSION="$PNPM_VERSION" \ --build-arg YARN_VERSION="$YARN_VERSION" \ - --build-context "base=docker-image://${DOCKER_IMAGE_ROOT}/base:latest" \ + --build-context "base=docker-image://${DOCKER_IMAGE_ROOT}/ort/base:latest" \ "$@" # Rust -image_build rust rust "$RUST_VERSION" \ +image_build rust ort/rust "$RUST_VERSION" \ --build-arg RUST_VERSION="$RUST_VERSION" \ - --build-context "base=docker-image://${DOCKER_IMAGE_ROOT}/base:latest" \ + --build-context "base=docker-image://${DOCKER_IMAGE_ROOT}/ort/base:latest" \ "$@" # Ruby -image_build ruby ruby "$RUBY_VERSION" \ +image_build ruby ort/ruby "$RUBY_VERSION" \ --build-arg RUBY_VERSION="$RUBY_VERSION" \ --build-arg COCOAPODS_VERSION="$COCOAPODS_VERSION" \ - --build-context "base=docker-image://${DOCKER_IMAGE_ROOT}/base:latest" \ + --build-context "base=docker-image://${DOCKER_IMAGE_ROOT}/ort/base:latest" \ "$@" # Golang -image_build golang golang "$GO_VERSION" \ +image_build golang ort/golang "$GO_VERSION" \ --build-arg GO_VERSION="$GO_VERSION" \ --build-arg GO_DEP_VERSION="$GO_DEP_VERSION" \ - --build-context "base=docker-image://${DOCKER_IMAGE_ROOT}/base:latest" \ + --build-context "base=docker-image://${DOCKER_IMAGE_ROOT}/ort/base:latest" \ "$@" # Runtime ORT image image_build run ort "$GIT_REVISION" \ - --build-context "python=docker-image://${DOCKER_IMAGE_ROOT}/python:latest" \ --build-arg NODEJS_VERSION="$NODEJS_VERSION" \ - --build-context "nodejs=docker-image://${DOCKER_IMAGE_ROOT}/nodejs:latest" \ - --build-context "rust=docker-image://${DOCKER_IMAGE_ROOT}/rust:latest" \ - --build-context "golang=docker-image://${DOCKER_IMAGE_ROOT}/golang:latest" \ - --build-context "ruby=docker-image://${DOCKER_IMAGE_ROOT}/ruby:latest" \ - --build-context "base=docker-image://${DOCKER_IMAGE_ROOT}/base:latest" \ + --build-context "base=docker-image://${DOCKER_IMAGE_ROOT}/ort/base:latest" \ + --build-context "python=docker-image://${DOCKER_IMAGE_ROOT}/ort/python:latest" \ + --build-context "nodejs=docker-image://${DOCKER_IMAGE_ROOT}/ort/nodejs:latest" \ + --build-context "rust=docker-image://${DOCKER_IMAGE_ROOT}/ort/rust:latest" \ + --build-context "golang=docker-image://${DOCKER_IMAGE_ROOT}/ort/golang:latest" \ + --build-context "ruby=docker-image://${DOCKER_IMAGE_ROOT}/ort/ruby:latest" \ "$@" # Build adjacent language containers if ALL_LANGUAGES is set. @@ -116,52 +116,54 @@ image_build run ort "$GIT_REVISION" \ # Android # shellcheck disable=SC1091 -image_build android android "$ANDROID_CMD_VERSION" \ +image_build android ort/android "$ANDROID_CMD_VERSION" \ --build-arg ANDROID_CMD_VERSION="$ANDROID_CMD_VERSION" \ - --build-context "base=docker-image://${DOCKER_IMAGE_ROOT}/base:latest" \ + --build-context "base=docker-image://${DOCKER_IMAGE_ROOT}/ort/base:latest" \ "$@" # Swift -image_build swift swift "$SWIFT_VERSION" \ +image_build swift ort/swift "$SWIFT_VERSION" \ --build-arg SWIFT_VERSION="$SWIFT_VERSION" \ - --build-context "base=docker-image://${DOCKER_IMAGE_ROOT}/base:latest" \ + --build-context "base=docker-image://${DOCKER_IMAGE_ROOT}/ort/base:latest" \ "$@" # SBT -image_build sbt sbt "$SBT_VERSION" \ +image_build scala ort/scala "$SBT_VERSION" \ --build-arg SBT_VERSION="$SBT_VERSION" \ - --build-context "base=docker-image://${DOCKER_IMAGE_ROOT}/base:latest" \ + --build-context "base=docker-image://${DOCKER_IMAGE_ROOT}/ort/base:latest" \ "$@" # Dart -image_build dart dart "$DART_VERSION" \ +image_build dart ort/dart "$DART_VERSION" \ --build-arg DART_VERSION="$DART_VERSION" \ - --build-context "base=docker-image://${DOCKER_IMAGE_ROOT}/base:latest" \ + --build-context "base=docker-image://${DOCKER_IMAGE_ROOT}/ort/base:latest" \ "$@" # Dotnet -image_build dotnet dotnet "$DOTNET_VERSION" \ +image_build dotnet ort/dotnet "$DOTNET_VERSION" \ --build-arg DOTNET_VERSION="$DOTNET_VERSION" \ --build-arg NUGET_INSPECTOR_VERSION="$NUGET_INSPECTOR_VERSION" \ - --build-context "base=docker-image://${DOCKER_IMAGE_ROOT}/base:latest" \ + --build-context "base=docker-image://${DOCKER_IMAGE_ROOT}/ort/base:latest" \ "$@" # Haskell -image_build haskell haskell "$HASKELL_STACK_VERSION" \ +image_build haskell ort/haskell "$HASKELL_STACK_VERSION" \ --build-arg HASKELL_STACK_VERSION="$HASKELL_STACK_VERSION" \ - --build-context "base=docker-image://${DOCKER_IMAGE_ROOT}/base:latest" \ + --build-context "base=docker-image://${DOCKER_IMAGE_ROOT}/ort/base:latest" \ "$@" -# Runtime Extended ORT image +# Runtime extended ORT image docker buildx build \ --file Dockerfile-extended \ --tag "${DOCKER_IMAGE_ROOT}/ort-extended:$GIT_REVISION" \ --tag "${DOCKER_IMAGE_ROOT}/ort-extended:latest" \ - --build-context "sbt=docker-image://${DOCKER_IMAGE_ROOT}/sbt:latest" \ - --build-context "dotnet=docker-image://${DOCKER_IMAGE_ROOT}/dotnet:latest" \ - --build-context "swift=docker-image://${DOCKER_IMAGE_ROOT}/swift:latest" \ - --build-context "android=docker-image://${DOCKER_IMAGE_ROOT}/android:latest" \ - --build-context "dart=docker-image://${DOCKER_IMAGE_ROOT}/dart:latest" \ - --build-context "haskell=docker-image://${DOCKER_IMAGE_ROOT}/haskell:latest" \ + --build-context "ort=docker-image://${DOCKER_IMAGE_ROOT}/ort:latest" \ + --build-context "sbt=docker-image://${DOCKER_IMAGE_ROOT}/ort/sbt:latest" \ + --build-context "dotnet=docker-image://${DOCKER_IMAGE_ROOT}/ort/dotnet:latest" \ + --build-context "swift=docker-image://${DOCKER_IMAGE_ROOT}/ort/swift:latest" \ + --build-context "android=docker-image://${DOCKER_IMAGE_ROOT}/ort/android:latest" \ + --build-context "dart=docker-image://${DOCKER_IMAGE_ROOT}/ort/dart:latest" \ + --build-context "haskell=docker-image://${DOCKER_IMAGE_ROOT}/ort/haskell:latest" \ + --build-context "scala=docker-image://${DOCKER_IMAGE_ROOT}/ort/scala:latest" \ "$@" . From 029f1cabe2849cc4a1cec7a6bdc9be8aa2284cc1 Mon Sep 17 00:00:00 2001 From: Helio Chissini de Castro Date: Tue, 26 Sep 2023 17:47:28 +0200 Subject: [PATCH 14/47] chore(docker): Use consistent naming and descriptions Signed-off-by: Helio Chissini de Castro --- .github/actions/ortdocker/action.yml | 16 ++++----- .github/actions/ortdocker/check_image.py | 4 +++ .github/workflows/docker-ort-runtime-ext.yml | 38 ++++++++++---------- .github/workflows/docker-ort-runtime.yml | 36 +++++++++---------- Dockerfile | 6 ++-- 5 files changed, 52 insertions(+), 48 deletions(-) diff --git a/.github/actions/ortdocker/action.yml b/.github/actions/ortdocker/action.yml index fe9380c71f029..74d9a2f48d9a5 100644 --- a/.github/actions/ortdocker/action.yml +++ b/.github/actions/ortdocker/action.yml @@ -15,13 +15,13 @@ # SPDX-License-Identifier: Apache-2.0 # License-Filename: LICENSE -name: "ORT Docker Image" -description: "Check and create docker image for Ort components" -author: "Helio Chissini de Castro " +name: "ORT Docker image" +description: "Check and create Docker image for ORT components" +author: "The ORT Project Authors" inputs: registry: - description: "GitHub Container Registry" + description: "GitHub container registry" default: "ghcr.io" token: description: "GitHub token" @@ -61,11 +61,11 @@ runs: echo $result echo "result=$result" >> $GITHUB_OUTPUT - - name: Set up Docker Buildx + - name: Set up Docker build if: steps.check_image.outputs.result != 'found' uses: docker/setup-buildx-action@v2 - - name: Login to GitHub Container Registry + - name: Login to GitHub container registry if: steps.check_image.outputs.result != 'found' uses: docker/login-action@v2 with: @@ -73,7 +73,7 @@ runs: username: ${{ github.actor }} password: ${{ inputs.token }} - - name: Extract components metadata (tags, labels) for base image + - name: Extract components metadata (tags, labels) if: steps.check_image.outputs.result != 'found' id: meta uses: docker/metadata-action@v4 @@ -83,7 +83,7 @@ runs: tags: type=raw,value=${{ inputs.version }} - - name: Build ORT Base container + - name: Build image if: steps.check_image.outputs.result != 'found' uses: docker/build-push-action@v4 with: diff --git a/.github/actions/ortdocker/check_image.py b/.github/actions/ortdocker/check_image.py index 98846b0671d93..31af107f11752 100644 --- a/.github/actions/ortdocker/check_image.py +++ b/.github/actions/ortdocker/check_image.py @@ -19,6 +19,10 @@ import requests +""" Use current GitHub API to check if a container image with the + given name and version exists. +""" + token = os.getenv("INPUT_TOKEN") org = os.getenv("GITHUB_REPOSITORY_OWNER") name = os.getenv("INPUT_NAME") diff --git a/.github/workflows/docker-ort-runtime-ext.yml b/.github/workflows/docker-ort-runtime-ext.yml index 1b8e68567af52..c78711cd8e765 100644 --- a/.github/workflows/docker-ort-runtime-ext.yml +++ b/.github/workflows/docker-ort-runtime-ext.yml @@ -15,7 +15,7 @@ # SPDX-License-Identifier: Apache-2.0 # License-Filename: LICENSE -name: MegaDocker Extended Image +name: Mega Docker extended image on: workflow_dispatch: @@ -31,7 +31,7 @@ on: - '*' workflow_run: workflows: - - 'MegaDocker Runtime Image' + - 'Mega Docker runtime image' types: - completed @@ -42,7 +42,7 @@ permissions: write-all jobs: android_image: - name: Android Image + name: Android image runs-on: ubuntu-22.04 steps: - name: Checkout default branch @@ -50,7 +50,7 @@ jobs: - name: Set environment variables run: | cat .versions >> $GITHUB_ENV - - name: Build Android Image + - name: Build Android image uses: ./.github/actions/ortdocker with: name: android @@ -60,7 +60,7 @@ jobs: ANDROID_CMD_VERSION=${{ env.ANDROID_CMD_VERSION }} dart_image: - name: Dart Image + name: Dart image runs-on: ubuntu-22.04 steps: - name: Checkout default branch @@ -68,7 +68,7 @@ jobs: - name: Set environment variables run: | cat .versions >> $GITHUB_ENV - - name: Build Dart Image + - name: Build Dart image uses: ./.github/actions/ortdocker with: name: dart @@ -78,7 +78,7 @@ jobs: DART_VERSION=${{ env.DART_VERSION }} dotnet_image: - name: Dotnet Image + name: Dotnet image runs-on: ubuntu-22.04 steps: - name: Checkout default branch @@ -86,7 +86,7 @@ jobs: - name: Set environment variables run: | cat .versions >> $GITHUB_ENV - - name: Build Dotnet Image + - name: Build Dotnet image uses: ./.github/actions/ortdocker with: name: dotnet @@ -97,7 +97,7 @@ jobs: NUGET_INSPECTOR_VERSION=${{ env.NUGET_INSPECTOR_VERSION }} haskell_image: - name: Haskell Image + name: Haskell image runs-on: ubuntu-22.04 steps: - name: Checkout default branch @@ -105,7 +105,7 @@ jobs: - name: Set environment variables run: | cat .versions >> $GITHUB_ENV - - name: Build Haskell Image + - name: Build Haskell image uses: ./.github/actions/ortdocker with: name: haskell @@ -115,7 +115,7 @@ jobs: HASKELL_STACK_VERSION=${{ env.HASKELL_STACK_VERSION }} scala_image: - name: Scala Image + name: Scala image runs-on: ubuntu-22.04 steps: - name: Checkout default branch @@ -123,7 +123,7 @@ jobs: - name: Set environment variables run: | cat .versions >> $GITHUB_ENV - - name: Build Scala Image + - name: Build Scala image uses: ./.github/actions/ortdocker with: name: scala @@ -133,7 +133,7 @@ jobs: SBT_VERSION=${{ env.SBT_VERSION }} swift_image: - name: Swift Image + name: Swift image runs-on: ubuntu-22.04 steps: - name: Checkout default branch @@ -141,7 +141,7 @@ jobs: - name: Set environment variables run: | cat .versions >> $GITHUB_ENV - - name: Build Swift Image + - name: Build Swift image uses: ./.github/actions/ortdocker with: name: swift @@ -152,7 +152,7 @@ jobs: runtime_extended_image: if: ${{ github.event_name == 'push' || github.event_name == 'workflow_dispatch' }} - name: Build ORT Extended Image + name: Build ORT extended image needs: [ android_image, dart_image, dotnet_image, haskell_image, scala_image, swift_image ] runs-on: ubuntu-22.04 permissions: @@ -163,17 +163,17 @@ jobs: - name: Checkout default branch uses: actions/checkout@v3 - - name: Set up Docker Buildx + - name: Set up Docker build uses: docker/setup-buildx-action@v2 - - name: Login to GitHub Container Registry + - name: Login to GitHub container registry uses: docker/login-action@v2 with: registry: ${{ env.REGISTRY }} username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - - name: Extract components metadata (tags, labels) for Ort runtime image + - name: Extract components metadata (tags, labels) id: meta-ort uses: docker/metadata-action@v4 with: @@ -182,7 +182,7 @@ jobs: tags: | type=raw,sha,enable=true,format=short - - name: Build ORT runtime container + - name: Build ORT extended runtime image uses: docker/build-push-action@v4 with: context: . diff --git a/.github/workflows/docker-ort-runtime.yml b/.github/workflows/docker-ort-runtime.yml index f467e7d46dd01..ef800a2a64f89 100644 --- a/.github/workflows/docker-ort-runtime.yml +++ b/.github/workflows/docker-ort-runtime.yml @@ -15,7 +15,7 @@ # SPDX-License-Identifier: Apache-2.0 # License-Filename: LICENSE -name: MegaDocker Runtime Image +name: Mega Docker runtime image on: workflow_dispatch: @@ -36,7 +36,7 @@ permissions: write-all jobs: base_image: - name: Base Image + name: Base image runs-on: ubuntu-22.04 steps: - name: Checkout default branch @@ -44,7 +44,7 @@ jobs: - name: Set environment variables run: | cat .versions >> $GITHUB_ENV - - name: Build Base Image + - name: Build base image uses: ./.github/actions/ortdocker with: name: base @@ -55,7 +55,7 @@ jobs: UBUNTU_VERSION=${{ env.UBUNTU_VERSION }} nodejs_image: - name: NodeJS Image + name: NodeJS image needs: [base_image] runs-on: ubuntu-22.04 steps: @@ -64,7 +64,7 @@ jobs: - name: Set environment variables run: | cat .versions >> $GITHUB_ENV - - name: Build NodeJS Image + - name: Build NodeJS image uses: ./.github/actions/ortdocker with: name: nodejs @@ -77,7 +77,7 @@ jobs: YARN_VERSION=${{ env.YARN_VERSION }} python_image: - name: Python Image + name: Python image needs: [base_image] runs-on: ubuntu-22.04 steps: @@ -86,7 +86,7 @@ jobs: - name: Set environment variables run: | cat .versions >> $GITHUB_ENV - - name: Build Python Image + - name: Build Python image uses: ./.github/actions/ortdocker with: name: python @@ -103,7 +103,7 @@ jobs: SCANCODE_VERSION=${{ env.SCANCODE_VERSION }} rust_image: - name: Rust Image + name: Rust image needs: [base_image] runs-on: ubuntu-22.04 steps: @@ -112,7 +112,7 @@ jobs: - name: Set environment variables run: | cat .versions >> $GITHUB_ENV - - name: Build Rust Image + - name: Build Rust image uses: ./.github/actions/ortdocker with: name: rust @@ -122,7 +122,7 @@ jobs: RUST_VERSION=${{ env.RUST_VERSION }} ruby_image: - name: Ruby Image + name: Ruby image needs: [base_image] runs-on: ubuntu-22.04 steps: @@ -131,7 +131,7 @@ jobs: - name: Set environment variables run: | cat .versions >> $GITHUB_ENV - - name: Build Ruby Image + - name: Build Ruby image uses: ./.github/actions/ortdocker with: name: ruby @@ -142,7 +142,7 @@ jobs: COCOAPODS_VERSION=${{ env.COCOAPODS_VERSION }} golang_image: - name: Golang Image + name: Golang image needs: [base_image] runs-on: ubuntu-22.04 steps: @@ -151,7 +151,7 @@ jobs: - name: Set environment variables run: | cat .versions >> $GITHUB_ENV - - name: Build Golang Image + - name: Build Golang image uses: ./.github/actions/ortdocker with: name: golang @@ -163,7 +163,7 @@ jobs: runtime_image: needs: [base_image, nodejs_image, python_image, rust_image, ruby_image, golang_image] - name: Build ORT runtime Image + name: Build ORT runtime image runs-on: ubuntu-22.04 permissions: contents: read @@ -177,17 +177,17 @@ jobs: run: | cat .versions >> $GITHUB_ENV - - name: Set up Docker Buildx + - name: Set up Docker build uses: docker/setup-buildx-action@v2 - - name: Login to GitHub Container Registry + - name: Login to GitHub container registry uses: docker/login-action@v2 with: registry: ${{ env.REGISTRY }} username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - - name: Extract components metadata (tags, labels) for Ort runtime image + - name: Extract components metadata (tags, labels) id: meta-ort uses: docker/metadata-action@v4 with: @@ -196,7 +196,7 @@ jobs: tags: | type=raw,sha,enable=true,format=short - - name: Build ORT runtime container + - name: Build ORT runtime image if: ${{ github.event_name == 'push' || github.event_name == 'workflow_dispatch' }} uses: docker/build-push-action@v4 with: diff --git a/Dockerfile b/Dockerfile index a98de4635e182..92b323de6e21f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -103,8 +103,8 @@ COPY "$CRT_FILES" /tmp/certificates/ RUN /etc/scripts/export_proxy_certificates.sh /tmp/certificates/ \ && /etc/scripts/import_certificates.sh /tmp/certificates/ -# Add syft to use as primary spdx docker scanner -# Create docs dir to store future spdxs +# Add Syft to use as primary SPDX Docker scanner +# Create docs dir to store future SPDX files RUN curl -sSfL https://raw.githubusercontent.com/anchore/syft/main/install.sh | sudo sh -s -- -b /usr/local/bin \ && mkdir -p /usr/share/doc/ort \ && chown $USER:$USER /usr/share/doc/ort @@ -434,7 +434,7 @@ FROM base as run # Remove ort build scripts RUN [ -d /etc/scripts ] && sudo rm -rf /etc/scripts -# Minor requirements +# Install optional tool subversion for ORT analyzer RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ --mount=type=cache,target=/var/lib/apt,sharing=locked \ sudo apt-get update && \ From 58fd9d4ff358c6d2425f4c17ef0ef0bdccf383cc Mon Sep 17 00:00:00 2001 From: Martin Nonnenmacher Date: Tue, 26 Sep 2023 21:12:50 +0200 Subject: [PATCH 15/47] chore: Fix formatting of workflow files Fix mainly indentation but also some other formatting issues. Signed-off-by: Martin Nonnenmacher --- .github/actions/ortdocker/action.yml | 106 +++-- .github/workflows/docker-ort-runtime-ext.yml | 356 ++++++++--------- .github/workflows/docker-ort-runtime.yml | 386 +++++++++---------- 3 files changed, 423 insertions(+), 425 deletions(-) diff --git a/.github/actions/ortdocker/action.yml b/.github/actions/ortdocker/action.yml index 74d9a2f48d9a5..e851c2c4990b5 100644 --- a/.github/actions/ortdocker/action.yml +++ b/.github/actions/ortdocker/action.yml @@ -40,63 +40,61 @@ runs: using: "composite" steps: - - name: Install Python - uses: actions/setup-python@v4 - with: - python-version: '3.10' - cache: 'pip' + - name: Install Python + uses: actions/setup-python@v4 + with: + python-version: '3.10' + cache: 'pip' - - name: Check if Docker image tag exists - id: check_image - shell: bash - env: - INPUT_REGISTRY: ${{ inputs.registry }} - INPUT_TOKEN: ${{ inputs.token }} - INPUT_NAME: ${{ inputs.name }} - INPUT_VERSION: ${{ inputs.version }} - run: | - pip install -q -U pip requests + - name: Check if Docker image tag exists + id: check_image + shell: bash + env: + INPUT_REGISTRY: ${{ inputs.registry }} + INPUT_TOKEN: ${{ inputs.token }} + INPUT_NAME: ${{ inputs.name }} + INPUT_VERSION: ${{ inputs.version }} + run: | + pip install -q -U pip requests - result=$(python ./.github/actions/ortdocker/check_image.py) - echo $result - echo "result=$result" >> $GITHUB_OUTPUT + result=$(python ./.github/actions/ortdocker/check_image.py) + echo $result + echo "result=$result" >> $GITHUB_OUTPUT - - name: Set up Docker build - if: steps.check_image.outputs.result != 'found' - uses: docker/setup-buildx-action@v2 + - name: Set up Docker build + if: steps.check_image.outputs.result != 'found' + uses: docker/setup-buildx-action@v2 - - name: Login to GitHub container registry - if: steps.check_image.outputs.result != 'found' - uses: docker/login-action@v2 - with: - registry: ${{ inputs.registry }} - username: ${{ github.actor }} - password: ${{ inputs.token }} - - - name: Extract components metadata (tags, labels) - if: steps.check_image.outputs.result != 'found' - id: meta - uses: docker/metadata-action@v4 - with: - images: | - ${{ env.REGISTRY }}/${{ github.repository }}/${{ inputs.name }} - tags: - type=raw,value=${{ inputs.version }} - - - name: Build image - if: steps.check_image.outputs.result != 'found' - uses: docker/build-push-action@v4 - with: - context: . - target: ${{ inputs.name }} - push: ${{ github.event_name == 'push' || github.event_name == 'workflow_dispatch' }} - load: false - build-args: ${{ inputs.build-args }} - tags: | - ${{ steps.meta.outputs.tags }} - ${{ env.REGISTRY }}/${{ github.repository }}/${{ inputs.name }}:latest - labels: ${{ steps.meta.outputs.labels }} - build-contexts: | - base=docker-image://${{ inputs.registry }}/${{ github.repository }}/base:latest + - name: Login to GitHub container registry + if: steps.check_image.outputs.result != 'found' + uses: docker/login-action@v2 + with: + registry: ${{ inputs.registry }} + username: ${{ github.actor }} + password: ${{ inputs.token }} + - name: Extract components metadata (tags, labels) + if: steps.check_image.outputs.result != 'found' + id: meta + uses: docker/metadata-action@v4 + with: + images: | + ${{ env.REGISTRY }}/${{ github.repository }}/${{ inputs.name }} + tags: + type=raw,value=${{ inputs.version }} + - name: Build image + if: steps.check_image.outputs.result != 'found' + uses: docker/build-push-action@v4 + with: + context: . + target: ${{ inputs.name }} + push: ${{ github.event_name == 'push' || github.event_name == 'workflow_dispatch' }} + load: false + build-args: ${{ inputs.build-args }} + tags: | + ${{ steps.meta.outputs.tags }} + ${{ env.REGISTRY }}/${{ github.repository }}/${{ inputs.name }}:latest + labels: ${{ steps.meta.outputs.labels }} + build-contexts: | + base=docker-image://${{ inputs.registry }}/${{ github.repository }}/base:latest diff --git a/.github/workflows/docker-ort-runtime-ext.yml b/.github/workflows/docker-ort-runtime-ext.yml index c78711cd8e765..c664f7d2c4e99 100644 --- a/.github/workflows/docker-ort-runtime-ext.yml +++ b/.github/workflows/docker-ort-runtime-ext.yml @@ -1,4 +1,4 @@ - # Copyright (C) 2023 The ORT Project Authors (see ) +# Copyright (C) 2023 The ORT Project Authors (see ) # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -18,186 +18,186 @@ name: Mega Docker extended image on: - workflow_dispatch: - pull_request: - paths: - - '.versions' - - 'Dockerfile' - - 'Dockerfile-extended' - push: - branches: - - main - tags: - - '*' - workflow_run: - workflows: - - 'Mega Docker runtime image' - types: - - completed + workflow_dispatch: + pull_request: + paths: + - '.versions' + - 'Dockerfile' + - 'Dockerfile-extended' + push: + branches: + - main + tags: + - '*' + workflow_run: + workflows: + - 'Mega Docker runtime image' + types: + - completed env: - REGISTRY: ghcr.io + REGISTRY: ghcr.io permissions: write-all jobs: - android_image: - name: Android image - runs-on: ubuntu-22.04 - steps: - - name: Checkout default branch - uses: actions/checkout@v3 - - name: Set environment variables - run: | - cat .versions >> $GITHUB_ENV - - name: Build Android image - uses: ./.github/actions/ortdocker - with: - name: android - token: ${{ secrets.GITHUB_TOKEN }} - version: "${{ env.ANDROID_CMD_VERSION }}" - build-args: | - ANDROID_CMD_VERSION=${{ env.ANDROID_CMD_VERSION }} - - dart_image: - name: Dart image - runs-on: ubuntu-22.04 - steps: - - name: Checkout default branch - uses: actions/checkout@v3 - - name: Set environment variables - run: | - cat .versions >> $GITHUB_ENV - - name: Build Dart image - uses: ./.github/actions/ortdocker - with: - name: dart - token: ${{ secrets.GITHUB_TOKEN }} - version: "${{ env.DART_VERSION }}" - build-args: | - DART_VERSION=${{ env.DART_VERSION }} - - dotnet_image: - name: Dotnet image - runs-on: ubuntu-22.04 - steps: - - name: Checkout default branch - uses: actions/checkout@v3 - - name: Set environment variables - run: | - cat .versions >> $GITHUB_ENV - - name: Build Dotnet image - uses: ./.github/actions/ortdocker - with: - name: dotnet - token: ${{ secrets.GITHUB_TOKEN }} - version: "${{ env.DOTNET_VERSION }}" - build-args: | - DOTNET_VERSION=${{ env.DOTNET_VERSION }} - NUGET_INSPECTOR_VERSION=${{ env.NUGET_INSPECTOR_VERSION }} - - haskell_image: - name: Haskell image - runs-on: ubuntu-22.04 - steps: - - name: Checkout default branch - uses: actions/checkout@v3 - - name: Set environment variables - run: | - cat .versions >> $GITHUB_ENV - - name: Build Haskell image - uses: ./.github/actions/ortdocker - with: - name: haskell - token: ${{ secrets.GITHUB_TOKEN }} - version: "${{ env.HASKELL_STACK_VERSION }}" - build-args: | - HASKELL_STACK_VERSION=${{ env.HASKELL_STACK_VERSION }} - - scala_image: - name: Scala image - runs-on: ubuntu-22.04 - steps: - - name: Checkout default branch - uses: actions/checkout@v3 - - name: Set environment variables - run: | - cat .versions >> $GITHUB_ENV - - name: Build Scala image - uses: ./.github/actions/ortdocker - with: - name: scala - token: ${{ secrets.GITHUB_TOKEN }} - version: "${{ env.SBT_VERSION }}" - build-args: | - SBT_VERSION=${{ env.SBT_VERSION }} - - swift_image: - name: Swift image - runs-on: ubuntu-22.04 - steps: - - name: Checkout default branch - uses: actions/checkout@v3 - - name: Set environment variables - run: | - cat .versions >> $GITHUB_ENV - - name: Build Swift image - uses: ./.github/actions/ortdocker - with: - name: swift - token: ${{ secrets.GITHUB_TOKEN }} - version: "${{ env.SWIFT_VERSION }}" - build-args: | - SWIFT_VERSION=${{ env.SWIFT_VERSION }} - - runtime_extended_image: - if: ${{ github.event_name == 'push' || github.event_name == 'workflow_dispatch' }} - name: Build ORT extended image - needs: [ android_image, dart_image, dotnet_image, haskell_image, scala_image, swift_image ] - runs-on: ubuntu-22.04 - permissions: - contents: read - packages: write - - steps: - - name: Checkout default branch - uses: actions/checkout@v3 - - - name: Set up Docker build - uses: docker/setup-buildx-action@v2 - - - name: Login to GitHub container registry - uses: docker/login-action@v2 - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Extract components metadata (tags, labels) - id: meta-ort - uses: docker/metadata-action@v4 - with: - images: | - ${{ env.REGISTRY }}/${{ github.repository_owner }}/ort-extended - tags: | - type=raw,sha,enable=true,format=short - - - name: Build ORT extended runtime image - uses: docker/build-push-action@v4 - with: - context: . - file: Dockerfile-extended - push: ${{ github.event_name == 'push' || github.event_name == 'workflow_dispatch' }} - load: false - tags: | - ${{ steps.meta-ort.outputs.tags }} - ${{ env.REGISTRY }}/${{ github.repository_owner }}/ort-extended:latest - labels: ${{ steps.meta.outputs.labels }} - build-contexts: | - ort=docker-image://${{ env.REGISTRY }}/${{ github.repository_owner }}/ort:latest - android=docker-image://${{ env.REGISTRY }}/${{ github.repository }}/android:latest - swift=docker-image://${{ env.REGISTRY }}/${{ github.repository }}/swift:latest - scala=docker-image://${{ env.REGISTRY }}/${{ github.repository }}/scala:latest - dart=docker-image://${{ env.REGISTRY }}/${{ github.repository }}/dart:latest - dotnet=docker-image://${{ env.REGISTRY }}/${{ github.repository }}/dotnet:latest - haskell=docker-image://${{ env.REGISTRY }}/${{ github.repository }}/haskell:latest + android_image: + name: Android image + runs-on: ubuntu-22.04 + steps: + - name: Checkout default branch + uses: actions/checkout@v3 + - name: Set environment variables + run: | + cat .versions >> $GITHUB_ENV + - name: Build Android image + uses: ./.github/actions/ortdocker + with: + name: android + token: ${{ secrets.GITHUB_TOKEN }} + version: "${{ env.ANDROID_CMD_VERSION }}" + build-args: | + ANDROID_CMD_VERSION=${{ env.ANDROID_CMD_VERSION }} + + dart_image: + name: Dart image + runs-on: ubuntu-22.04 + steps: + - name: Checkout default branch + uses: actions/checkout@v3 + - name: Set environment variables + run: | + cat .versions >> $GITHUB_ENV + - name: Build Dart image + uses: ./.github/actions/ortdocker + with: + name: dart + token: ${{ secrets.GITHUB_TOKEN }} + version: "${{ env.DART_VERSION }}" + build-args: | + DART_VERSION=${{ env.DART_VERSION }} + + dotnet_image: + name: Dotnet image + runs-on: ubuntu-22.04 + steps: + - name: Checkout default branch + uses: actions/checkout@v3 + - name: Set environment variables + run: | + cat .versions >> $GITHUB_ENV + - name: Build Dotnet image + uses: ./.github/actions/ortdocker + with: + name: dotnet + token: ${{ secrets.GITHUB_TOKEN }} + version: "${{ env.DOTNET_VERSION }}" + build-args: | + DOTNET_VERSION=${{ env.DOTNET_VERSION }} + NUGET_INSPECTOR_VERSION=${{ env.NUGET_INSPECTOR_VERSION }} + + haskell_image: + name: Haskell image + runs-on: ubuntu-22.04 + steps: + - name: Checkout default branch + uses: actions/checkout@v3 + - name: Set environment variables + run: | + cat .versions >> $GITHUB_ENV + - name: Build Haskell image + uses: ./.github/actions/ortdocker + with: + name: haskell + token: ${{ secrets.GITHUB_TOKEN }} + version: "${{ env.HASKELL_STACK_VERSION }}" + build-args: | + HASKELL_STACK_VERSION=${{ env.HASKELL_STACK_VERSION }} + + scala_image: + name: Scala image + runs-on: ubuntu-22.04 + steps: + - name: Checkout default branch + uses: actions/checkout@v3 + - name: Set environment variables + run: | + cat .versions >> $GITHUB_ENV + - name: Build Scala image + uses: ./.github/actions/ortdocker + with: + name: scala + token: ${{ secrets.GITHUB_TOKEN }} + version: "${{ env.SBT_VERSION }}" + build-args: | + SBT_VERSION=${{ env.SBT_VERSION }} + + swift_image: + name: Swift image + runs-on: ubuntu-22.04 + steps: + - name: Checkout default branch + uses: actions/checkout@v3 + - name: Set environment variables + run: | + cat .versions >> $GITHUB_ENV + - name: Build Swift image + uses: ./.github/actions/ortdocker + with: + name: swift + token: ${{ secrets.GITHUB_TOKEN }} + version: "${{ env.SWIFT_VERSION }}" + build-args: | + SWIFT_VERSION=${{ env.SWIFT_VERSION }} + + runtime_extended_image: + if: ${{ github.event_name == 'push' || github.event_name == 'workflow_dispatch' }} + name: Build ORT extended image + needs: [ android_image, dart_image, dotnet_image, haskell_image, scala_image, swift_image ] + runs-on: ubuntu-22.04 + permissions: + contents: read + packages: write + + steps: + - name: Checkout default branch + uses: actions/checkout@v3 + + - name: Set up Docker build + uses: docker/setup-buildx-action@v2 + + - name: Login to GitHub container registry + uses: docker/login-action@v2 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract components metadata (tags, labels) + id: meta-ort + uses: docker/metadata-action@v4 + with: + images: | + ${{ env.REGISTRY }}/${{ github.repository_owner }}/ort-extended + tags: | + type=raw,sha,enable=true,format=short + + - name: Build ORT extended runtime image + uses: docker/build-push-action@v4 + with: + context: . + file: Dockerfile-extended + push: ${{ github.event_name == 'push' || github.event_name == 'workflow_dispatch' }} + load: false + tags: | + ${{ steps.meta-ort.outputs.tags }} + ${{ env.REGISTRY }}/${{ github.repository_owner }}/ort-extended:latest + labels: ${{ steps.meta.outputs.labels }} + build-contexts: | + ort=docker-image://${{ env.REGISTRY }}/${{ github.repository_owner }}/ort:latest + android=docker-image://${{ env.REGISTRY }}/${{ github.repository }}/android:latest + swift=docker-image://${{ env.REGISTRY }}/${{ github.repository }}/swift:latest + scala=docker-image://${{ env.REGISTRY }}/${{ github.repository }}/scala:latest + dart=docker-image://${{ env.REGISTRY }}/${{ github.repository }}/dart:latest + dotnet=docker-image://${{ env.REGISTRY }}/${{ github.repository }}/dotnet:latest + haskell=docker-image://${{ env.REGISTRY }}/${{ github.repository }}/haskell:latest diff --git a/.github/workflows/docker-ort-runtime.yml b/.github/workflows/docker-ort-runtime.yml index ef800a2a64f89..6c27b2b9c2833 100644 --- a/.github/workflows/docker-ort-runtime.yml +++ b/.github/workflows/docker-ort-runtime.yml @@ -1,4 +1,4 @@ - # Copyright (C) 2023 The ORT Project Authors (see ) +# Copyright (C) 2023 The ORT Project Authors (see ) # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -18,201 +18,201 @@ name: Mega Docker runtime image on: - workflow_dispatch: - pull_request: - paths: - - '.versions' - - 'Dockerfile' - push: - branches: - - main - tags: - - '*' + workflow_dispatch: + pull_request: + paths: + - '.versions' + - 'Dockerfile' + push: + branches: + - main + tags: + - '*' env: - REGISTRY: ghcr.io + REGISTRY: ghcr.io permissions: write-all jobs: - base_image: - name: Base image - runs-on: ubuntu-22.04 - steps: - - name: Checkout default branch - uses: actions/checkout@v3 - - name: Set environment variables - run: | - cat .versions >> $GITHUB_ENV - - name: Build base image - uses: ./.github/actions/ortdocker - with: - name: base - token: ${{ secrets.GITHUB_TOKEN }} - version: "${{ env.JAVA_VERSION }}-jdk-${{ env.UBUNTU_VERSION }}" - build-args: | - JAVA_VERSION=${{ env.JAVA_VERSION }} - UBUNTU_VERSION=${{ env.UBUNTU_VERSION }} - - nodejs_image: - name: NodeJS image - needs: [base_image] - runs-on: ubuntu-22.04 - steps: - - name: Checkout default branch - uses: actions/checkout@v3 - - name: Set environment variables - run: | - cat .versions >> $GITHUB_ENV - - name: Build NodeJS image - uses: ./.github/actions/ortdocker - with: - name: nodejs - token: ${{ secrets.GITHUB_TOKEN }} - version: "${{ env.NODEJS_VERSION }}" - build-args: | - NODEJS_VERSION=${{ env.NODEJS_VERSION }} - NPM_VERSION=${{ env.NPM_VERSION }} - PNPM_VERSION=${{ env.PNPM_VERSION }} - YARN_VERSION=${{ env.YARN_VERSION }} - - python_image: - name: Python image - needs: [base_image] - runs-on: ubuntu-22.04 - steps: - - name: Checkout default branch - uses: actions/checkout@v3 - - name: Set environment variables - run: | - cat .versions >> $GITHUB_ENV - - name: Build Python image - uses: ./.github/actions/ortdocker - with: - name: python - token: ${{ secrets.GITHUB_TOKEN }} - version: "${{ env.PYTHON_VERSION }}" - build-args: | - CONAN_VERSION=${{ env.CONAN_VERSION }} - PIPTOOL_VERSION=${{ env.PIPTOOL_VERSION }} - PYENV_GIT_TAG=${{ env.PYENV_GIT_TAG }} - PYTHON_INSPECTOR_VERSION=${{ env.PYTHON_INSPECTOR_VERSION }} - PYTHON_PIPENV_VERSION=${{ env.PYTHON_PIPENV_VERSION }} - PYTHON_POETRY_VERSION=${{ env.PYTHON_POETRY_VERSION }} - PYTHON_VERSION=${{ env.PYTHON_VERSION }} - SCANCODE_VERSION=${{ env.SCANCODE_VERSION }} - - rust_image: - name: Rust image - needs: [base_image] - runs-on: ubuntu-22.04 - steps: - - name: Checkout default branch - uses: actions/checkout@v3 - - name: Set environment variables - run: | - cat .versions >> $GITHUB_ENV - - name: Build Rust image - uses: ./.github/actions/ortdocker - with: - name: rust - token: ${{ secrets.GITHUB_TOKEN }} - version: "${{ env.RUST_VERSION }}" - build-args: | - RUST_VERSION=${{ env.RUST_VERSION }} - - ruby_image: - name: Ruby image - needs: [base_image] - runs-on: ubuntu-22.04 - steps: - - name: Checkout default branch - uses: actions/checkout@v3 - - name: Set environment variables - run: | - cat .versions >> $GITHUB_ENV - - name: Build Ruby image - uses: ./.github/actions/ortdocker - with: - name: ruby - token: ${{ secrets.GITHUB_TOKEN }} - version: "${{ env.RUBY_VERSION }}" - build-args: | - RUBY_VERSION=${{ env.RUBY_VERSION }} - COCOAPODS_VERSION=${{ env.COCOAPODS_VERSION }} - - golang_image: - name: Golang image - needs: [base_image] - runs-on: ubuntu-22.04 - steps: - - name: Checkout default branch - uses: actions/checkout@v3 - - name: Set environment variables - run: | - cat .versions >> $GITHUB_ENV - - name: Build Golang image - uses: ./.github/actions/ortdocker - with: - name: golang - token: ${{ secrets.GITHUB_TOKEN }} - version: "${{ env.GO_VERSION }}" - build-args: | - GO_DEP_VERSION=${{ env.GO_DEP_VERSION }} - GO_VERSION=${{ env.GO_VERSION }} - - runtime_image: - needs: [base_image, nodejs_image, python_image, rust_image, ruby_image, golang_image] - name: Build ORT runtime image - runs-on: ubuntu-22.04 - permissions: - contents: read - packages: write - - steps: - - name: Checkout default branch - uses: actions/checkout@v3 - - - name: Set environment variables - run: | - cat .versions >> $GITHUB_ENV - - - name: Set up Docker build - uses: docker/setup-buildx-action@v2 - - - name: Login to GitHub container registry - uses: docker/login-action@v2 - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Extract components metadata (tags, labels) - id: meta-ort - uses: docker/metadata-action@v4 - with: - images: | - ${{ env.REGISTRY }}/${{ github.repository_owner }}/ort - tags: | - type=raw,sha,enable=true,format=short - - - name: Build ORT runtime image - if: ${{ github.event_name == 'push' || github.event_name == 'workflow_dispatch' }} - uses: docker/build-push-action@v4 - with: - context: . - target: run - push: true - load: false - build-args: | - NODEJS_VERSION=${{ env.NODEJS_VERSION }} - tags: | - ${{ steps.meta-ort.outputs.tags }} - ${{ env.REGISTRY }}/${{ github.repository_owner }}/ort:latest - labels: ${{ steps.meta.outputs.labels }} - build-contexts: | - nodejs=docker-image://${{ env.REGISTRY }}/${{ github.repository }}/nodejs:latest - python=docker-image://${{ env.REGISTRY }}/${{ github.repository }}/python:latest - rust=docker-image://${{ env.REGISTRY }}/${{ github.repository }}/rust:latest - ruby=docker-image://${{ env.REGISTRY }}/${{ github.repository }}/ruby:latest - golang=docker-image://${{ env.REGISTRY }}/${{ github.repository }}/golang:latest + base_image: + name: Base image + runs-on: ubuntu-22.04 + steps: + - name: Checkout default branch + uses: actions/checkout@v3 + - name: Set environment variables + run: | + cat .versions >> $GITHUB_ENV + - name: Build base image + uses: ./.github/actions/ortdocker + with: + name: base + token: ${{ secrets.GITHUB_TOKEN }} + version: "${{ env.JAVA_VERSION }}-jdk-${{ env.UBUNTU_VERSION }}" + build-args: | + JAVA_VERSION=${{ env.JAVA_VERSION }} + UBUNTU_VERSION=${{ env.UBUNTU_VERSION }} + + nodejs_image: + name: NodeJS image + needs: [ base_image ] + runs-on: ubuntu-22.04 + steps: + - name: Checkout default branch + uses: actions/checkout@v3 + - name: Set environment variables + run: | + cat .versions >> $GITHUB_ENV + - name: Build NodeJS image + uses: ./.github/actions/ortdocker + with: + name: nodejs + token: ${{ secrets.GITHUB_TOKEN }} + version: "${{ env.NODEJS_VERSION }}" + build-args: | + NODEJS_VERSION=${{ env.NODEJS_VERSION }} + NPM_VERSION=${{ env.NPM_VERSION }} + PNPM_VERSION=${{ env.PNPM_VERSION }} + YARN_VERSION=${{ env.YARN_VERSION }} + + python_image: + name: Python image + needs: [ base_image ] + runs-on: ubuntu-22.04 + steps: + - name: Checkout default branch + uses: actions/checkout@v3 + - name: Set environment variables + run: | + cat .versions >> $GITHUB_ENV + - name: Build Python image + uses: ./.github/actions/ortdocker + with: + name: python + token: ${{ secrets.GITHUB_TOKEN }} + version: "${{ env.PYTHON_VERSION }}" + build-args: | + CONAN_VERSION=${{ env.CONAN_VERSION }} + PIPTOOL_VERSION=${{ env.PIPTOOL_VERSION }} + PYENV_GIT_TAG=${{ env.PYENV_GIT_TAG }} + PYTHON_INSPECTOR_VERSION=${{ env.PYTHON_INSPECTOR_VERSION }} + PYTHON_PIPENV_VERSION=${{ env.PYTHON_PIPENV_VERSION }} + PYTHON_POETRY_VERSION=${{ env.PYTHON_POETRY_VERSION }} + PYTHON_VERSION=${{ env.PYTHON_VERSION }} + SCANCODE_VERSION=${{ env.SCANCODE_VERSION }} + + rust_image: + name: Rust image + needs: [ base_image ] + runs-on: ubuntu-22.04 + steps: + - name: Checkout default branch + uses: actions/checkout@v3 + - name: Set environment variables + run: | + cat .versions >> $GITHUB_ENV + - name: Build Rust image + uses: ./.github/actions/ortdocker + with: + name: rust + token: ${{ secrets.GITHUB_TOKEN }} + version: "${{ env.RUST_VERSION }}" + build-args: | + RUST_VERSION=${{ env.RUST_VERSION }} + + ruby_image: + name: Ruby image + needs: [ base_image ] + runs-on: ubuntu-22.04 + steps: + - name: Checkout default branch + uses: actions/checkout@v3 + - name: Set environment variables + run: | + cat .versions >> $GITHUB_ENV + - name: Build Ruby image + uses: ./.github/actions/ortdocker + with: + name: ruby + token: ${{ secrets.GITHUB_TOKEN }} + version: "${{ env.RUBY_VERSION }}" + build-args: | + RUBY_VERSION=${{ env.RUBY_VERSION }} + COCOAPODS_VERSION=${{ env.COCOAPODS_VERSION }} + + golang_image: + name: Golang image + needs: [ base_image ] + runs-on: ubuntu-22.04 + steps: + - name: Checkout default branch + uses: actions/checkout@v3 + - name: Set environment variables + run: | + cat .versions >> $GITHUB_ENV + - name: Build Golang image + uses: ./.github/actions/ortdocker + with: + name: golang + token: ${{ secrets.GITHUB_TOKEN }} + version: "${{ env.GO_VERSION }}" + build-args: | + GO_DEP_VERSION=${{ env.GO_DEP_VERSION }} + GO_VERSION=${{ env.GO_VERSION }} + + runtime_image: + needs: [ base_image, nodejs_image, python_image, rust_image, ruby_image, golang_image ] + name: Build ORT runtime image + runs-on: ubuntu-22.04 + permissions: + contents: read + packages: write + + steps: + - name: Checkout default branch + uses: actions/checkout@v3 + + - name: Set environment variables + run: | + cat .versions >> $GITHUB_ENV + + - name: Set up Docker build + uses: docker/setup-buildx-action@v2 + + - name: Login to GitHub container registry + uses: docker/login-action@v2 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract components metadata (tags, labels) + id: meta-ort + uses: docker/metadata-action@v4 + with: + images: | + ${{ env.REGISTRY }}/${{ github.repository_owner }}/ort + tags: | + type=raw,sha,enable=true,format=short + + - name: Build ORT runtime image + if: ${{ github.event_name == 'push' || github.event_name == 'workflow_dispatch' }} + uses: docker/build-push-action@v4 + with: + context: . + target: run + push: true + load: false + build-args: | + NODEJS_VERSION=${{ env.NODEJS_VERSION }} + tags: | + ${{ steps.meta-ort.outputs.tags }} + ${{ env.REGISTRY }}/${{ github.repository_owner }}/ort:latest + labels: ${{ steps.meta.outputs.labels }} + build-contexts: | + nodejs=docker-image://${{ env.REGISTRY }}/${{ github.repository }}/nodejs:latest + python=docker-image://${{ env.REGISTRY }}/${{ github.repository }}/python:latest + rust=docker-image://${{ env.REGISTRY }}/${{ github.repository }}/rust:latest + ruby=docker-image://${{ env.REGISTRY }}/${{ github.repository }}/ruby:latest + golang=docker-image://${{ env.REGISTRY }}/${{ github.repository }}/golang:latest From 9794da6a66bb9c73417e721884c2e8adca198fb5 Mon Sep 17 00:00:00 2001 From: Martin Nonnenmacher Date: Mon, 25 Sep 2023 23:27:45 +0200 Subject: [PATCH 16/47] feat(scanner)!: Remove unused `downloaderConfig` parameter Remove the unused `downloaderConfig` parameter from the `create` function of the `ScannerWrapperFactory`. The `downloaderConfig` was required by the old scanner API which was removed in c71628d, because there the code to download the source code was not separated from the scanner plugins. Signed-off-by: Martin Nonnenmacher --- plugins/commands/scanner/src/main/kotlin/ScannerCommand.kt | 4 ++-- plugins/scanners/askalono/src/main/kotlin/Askalono.kt | 4 +--- plugins/scanners/boyterlc/src/main/kotlin/BoyterLc.kt | 4 +--- plugins/scanners/fossid/src/main/kotlin/FossId.kt | 3 +-- plugins/scanners/licensee/src/main/kotlin/Licensee.kt | 4 +--- plugins/scanners/scancode/src/main/kotlin/ScanCode.kt | 4 +--- plugins/scanners/scanoss/src/main/kotlin/ScanOss.kt | 4 +--- .../scanoss/src/test/kotlin/ScanOssScannerDirectoryTest.kt | 3 +-- .../scanoss/src/test/kotlin/ScanOssScannerFileTest.kt | 3 +-- scanner/src/main/kotlin/ScannerWrapperFactory.kt | 7 +++---- scanner/src/main/kotlin/storages/ClearlyDefinedStorage.kt | 3 +-- 11 files changed, 14 insertions(+), 29 deletions(-) diff --git a/plugins/commands/scanner/src/main/kotlin/ScannerCommand.kt b/plugins/commands/scanner/src/main/kotlin/ScannerCommand.kt index b48d564e06e8b..73793f281ee49 100644 --- a/plugins/commands/scanner/src/main/kotlin/ScannerCommand.kt +++ b/plugins/commands/scanner/src/main/kotlin/ScannerCommand.kt @@ -179,10 +179,10 @@ class ScannerCommand : OrtCommand( ): OrtResult { val packageScannerWrappers = scannerWrapperFactories .takeIf { PackageType.PACKAGE in packageTypes }.orEmpty() - .map { it.create(ortConfig.scanner, ortConfig.downloader) } + .map { it.create(ortConfig.scanner) } val projectScannerWrappers = projectScannerWrapperFactories .takeIf { PackageType.PROJECT in packageTypes }.orEmpty() - .map { it.create(ortConfig.scanner, ortConfig.downloader) } + .map { it.create(ortConfig.scanner) } if (projectScannerWrappers.isNotEmpty()) { echo("Scanning projects with:") diff --git a/plugins/scanners/askalono/src/main/kotlin/Askalono.kt b/plugins/scanners/askalono/src/main/kotlin/Askalono.kt index f6681b9aa971d..f49f6ea08839a 100644 --- a/plugins/scanners/askalono/src/main/kotlin/Askalono.kt +++ b/plugins/scanners/askalono/src/main/kotlin/Askalono.kt @@ -32,7 +32,6 @@ import org.ossreviewtoolkit.model.LicenseFinding import org.ossreviewtoolkit.model.ScanSummary import org.ossreviewtoolkit.model.Severity import org.ossreviewtoolkit.model.TextLocation -import org.ossreviewtoolkit.model.config.DownloaderConfiguration import org.ossreviewtoolkit.model.config.ScannerConfiguration import org.ossreviewtoolkit.scanner.AbstractScannerWrapperFactory import org.ossreviewtoolkit.scanner.CommandLinePathScannerWrapper @@ -52,8 +51,7 @@ class Askalono internal constructor( private companion object : Logging class Factory : AbstractScannerWrapperFactory("Askalono") { - override fun create(scannerConfig: ScannerConfiguration, downloaderConfig: DownloaderConfiguration) = - Askalono(type, scannerConfig) + override fun create(scannerConfig: ScannerConfiguration) = Askalono(type, scannerConfig) } override val configuration = "" diff --git a/plugins/scanners/boyterlc/src/main/kotlin/BoyterLc.kt b/plugins/scanners/boyterlc/src/main/kotlin/BoyterLc.kt index 86e969b704e00..25e3a7b509a4b 100644 --- a/plugins/scanners/boyterlc/src/main/kotlin/BoyterLc.kt +++ b/plugins/scanners/boyterlc/src/main/kotlin/BoyterLc.kt @@ -31,7 +31,6 @@ import org.ossreviewtoolkit.model.LicenseFinding import org.ossreviewtoolkit.model.ScanSummary import org.ossreviewtoolkit.model.Severity import org.ossreviewtoolkit.model.TextLocation -import org.ossreviewtoolkit.model.config.DownloaderConfiguration import org.ossreviewtoolkit.model.config.ScannerConfiguration import org.ossreviewtoolkit.scanner.AbstractScannerWrapperFactory import org.ossreviewtoolkit.scanner.CommandLinePathScannerWrapper @@ -56,8 +55,7 @@ class BoyterLc internal constructor( } class Factory : AbstractScannerWrapperFactory("BoyterLc") { - override fun create(scannerConfig: ScannerConfiguration, downloaderConfig: DownloaderConfiguration) = - BoyterLc(type, scannerConfig) + override fun create(scannerConfig: ScannerConfiguration) = BoyterLc(type, scannerConfig) } override val configuration = CONFIGURATION_OPTIONS.joinToString(" ") diff --git a/plugins/scanners/fossid/src/main/kotlin/FossId.kt b/plugins/scanners/fossid/src/main/kotlin/FossId.kt index 00f3d4d17b686..019f411eaf24f 100644 --- a/plugins/scanners/fossid/src/main/kotlin/FossId.kt +++ b/plugins/scanners/fossid/src/main/kotlin/FossId.kt @@ -72,7 +72,6 @@ import org.ossreviewtoolkit.model.ScanSummary import org.ossreviewtoolkit.model.Severity import org.ossreviewtoolkit.model.UnknownProvenance import org.ossreviewtoolkit.model.VcsType -import org.ossreviewtoolkit.model.config.DownloaderConfiguration import org.ossreviewtoolkit.model.config.Options import org.ossreviewtoolkit.model.config.ScannerConfiguration import org.ossreviewtoolkit.model.createAndLogIssue @@ -171,7 +170,7 @@ class FossId internal constructor( } class Factory : AbstractScannerWrapperFactory("FossId") { - override fun create(scannerConfig: ScannerConfiguration, downloaderConfig: DownloaderConfiguration) = + override fun create(scannerConfig: ScannerConfiguration) = FossId(type, scannerConfig, FossIdConfig.create(scannerConfig)) } diff --git a/plugins/scanners/licensee/src/main/kotlin/Licensee.kt b/plugins/scanners/licensee/src/main/kotlin/Licensee.kt index 661a73e0fd0da..d0007e927d380 100644 --- a/plugins/scanners/licensee/src/main/kotlin/Licensee.kt +++ b/plugins/scanners/licensee/src/main/kotlin/Licensee.kt @@ -32,7 +32,6 @@ import org.ossreviewtoolkit.model.LicenseFinding import org.ossreviewtoolkit.model.ScanSummary import org.ossreviewtoolkit.model.Severity import org.ossreviewtoolkit.model.TextLocation -import org.ossreviewtoolkit.model.config.DownloaderConfiguration import org.ossreviewtoolkit.model.config.ScannerConfiguration import org.ossreviewtoolkit.scanner.AbstractScannerWrapperFactory import org.ossreviewtoolkit.scanner.CommandLinePathScannerWrapper @@ -55,8 +54,7 @@ class Licensee internal constructor( } class Factory : AbstractScannerWrapperFactory("Licensee") { - override fun create(scannerConfig: ScannerConfiguration, downloaderConfig: DownloaderConfiguration) = - Licensee(type, scannerConfig) + override fun create(scannerConfig: ScannerConfiguration) = Licensee(type, scannerConfig) } override val configuration = CONFIGURATION_OPTIONS.joinToString(" ") diff --git a/plugins/scanners/scancode/src/main/kotlin/ScanCode.kt b/plugins/scanners/scancode/src/main/kotlin/ScanCode.kt index 159880f432787..3420928f93dc7 100644 --- a/plugins/scanners/scancode/src/main/kotlin/ScanCode.kt +++ b/plugins/scanners/scancode/src/main/kotlin/ScanCode.kt @@ -28,7 +28,6 @@ import org.apache.logging.log4j.kotlin.Logging import org.ossreviewtoolkit.model.ScanSummary import org.ossreviewtoolkit.model.ScannerDetails -import org.ossreviewtoolkit.model.config.DownloaderConfiguration import org.ossreviewtoolkit.model.config.ScannerConfiguration import org.ossreviewtoolkit.scanner.AbstractScannerWrapperFactory import org.ossreviewtoolkit.scanner.CommandLinePathScannerWrapper @@ -95,8 +94,7 @@ class ScanCode internal constructor( } class Factory : AbstractScannerWrapperFactory(SCANNER_NAME) { - override fun create(scannerConfig: ScannerConfiguration, downloaderConfig: DownloaderConfiguration) = - ScanCode(type, scannerConfig) + override fun create(scannerConfig: ScannerConfiguration) = ScanCode(type, scannerConfig) } override val criteria by lazy { ScannerCriteria.fromConfig(details, scannerConfig) } diff --git a/plugins/scanners/scanoss/src/main/kotlin/ScanOss.kt b/plugins/scanners/scanoss/src/main/kotlin/ScanOss.kt index 80173f9d6754e..54cb63a0b7565 100644 --- a/plugins/scanners/scanoss/src/main/kotlin/ScanOss.kt +++ b/plugins/scanners/scanoss/src/main/kotlin/ScanOss.kt @@ -38,7 +38,6 @@ import org.apache.logging.log4j.kotlin.Logging import org.ossreviewtoolkit.clients.scanoss.ScanOssService import org.ossreviewtoolkit.model.ScanSummary -import org.ossreviewtoolkit.model.config.DownloaderConfiguration import org.ossreviewtoolkit.model.config.ScannerConfiguration import org.ossreviewtoolkit.scanner.AbstractScannerWrapperFactory import org.ossreviewtoolkit.scanner.PathScannerWrapper @@ -57,8 +56,7 @@ class ScanOss internal constructor( private companion object : Logging class Factory : AbstractScannerWrapperFactory("SCANOSS") { - override fun create(scannerConfig: ScannerConfiguration, downloaderConfig: DownloaderConfiguration) = - ScanOss(type, scannerConfig) + override fun create(scannerConfig: ScannerConfiguration) = ScanOss(type, scannerConfig) } private val config = ScanOssConfig.create(scannerConfig).also { diff --git a/plugins/scanners/scanoss/src/test/kotlin/ScanOssScannerDirectoryTest.kt b/plugins/scanners/scanoss/src/test/kotlin/ScanOssScannerDirectoryTest.kt index c67ec3c339552..26782d764f1c9 100644 --- a/plugins/scanners/scanoss/src/test/kotlin/ScanOssScannerDirectoryTest.kt +++ b/plugins/scanners/scanoss/src/test/kotlin/ScanOssScannerDirectoryTest.kt @@ -42,7 +42,6 @@ import org.ossreviewtoolkit.model.SnippetFinding import org.ossreviewtoolkit.model.TextLocation import org.ossreviewtoolkit.model.VcsInfo import org.ossreviewtoolkit.model.VcsType -import org.ossreviewtoolkit.model.config.DownloaderConfiguration import org.ossreviewtoolkit.model.config.ScannerConfiguration import org.ossreviewtoolkit.scanner.ScanContext import org.ossreviewtoolkit.utils.spdx.SpdxExpression @@ -65,7 +64,7 @@ class ScanOssScannerDirectoryTest : StringSpec({ server.start() val scannerOptions = mapOf(ScanOssConfig.API_URL_PROPERTY to "http://localhost:${server.port()}") val configuration = ScannerConfiguration(options = mapOf("ScanOss" to scannerOptions)) - scanner = spyk(ScanOss.Factory().create(configuration, DownloaderConfiguration())) + scanner = spyk(ScanOss.Factory().create(configuration)) } afterSpec { diff --git a/plugins/scanners/scanoss/src/test/kotlin/ScanOssScannerFileTest.kt b/plugins/scanners/scanoss/src/test/kotlin/ScanOssScannerFileTest.kt index 1e1ecffe67d23..2ca262979bedc 100644 --- a/plugins/scanners/scanoss/src/test/kotlin/ScanOssScannerFileTest.kt +++ b/plugins/scanners/scanoss/src/test/kotlin/ScanOssScannerFileTest.kt @@ -36,7 +36,6 @@ import java.util.UUID import org.ossreviewtoolkit.model.LicenseFinding import org.ossreviewtoolkit.model.PackageType import org.ossreviewtoolkit.model.TextLocation -import org.ossreviewtoolkit.model.config.DownloaderConfiguration import org.ossreviewtoolkit.model.config.ScannerConfiguration import org.ossreviewtoolkit.scanner.ScanContext @@ -58,7 +57,7 @@ class ScanOssScannerFileTest : StringSpec({ server.start() val scannerOptions = mapOf(ScanOssConfig.API_URL_PROPERTY to "http://localhost:${server.port()}") val configuration = ScannerConfiguration(options = mapOf("ScanOss" to scannerOptions)) - scanner = spyk(ScanOss.Factory().create(configuration, DownloaderConfiguration())) + scanner = spyk(ScanOss.Factory().create(configuration)) } afterSpec { diff --git a/scanner/src/main/kotlin/ScannerWrapperFactory.kt b/scanner/src/main/kotlin/ScannerWrapperFactory.kt index 732dd72a2d8e8..451f7aa4b6060 100644 --- a/scanner/src/main/kotlin/ScannerWrapperFactory.kt +++ b/scanner/src/main/kotlin/ScannerWrapperFactory.kt @@ -21,7 +21,6 @@ package org.ossreviewtoolkit.scanner import java.util.ServiceLoader -import org.ossreviewtoolkit.model.config.DownloaderConfiguration import org.ossreviewtoolkit.model.config.ScannerConfiguration import org.ossreviewtoolkit.utils.common.Plugin @@ -30,9 +29,9 @@ import org.ossreviewtoolkit.utils.common.Plugin */ interface ScannerWrapperFactory : Plugin { /** - * Create a [ScannerWrapper] using the specified [scannerConfig] and [downloaderConfig]. + * Create a [ScannerWrapper] using the specified [scannerConfig]. */ - fun create(scannerConfig: ScannerConfiguration, downloaderConfig: DownloaderConfiguration): ScannerWrapper + fun create(scannerConfig: ScannerConfiguration): ScannerWrapper } /** @@ -41,7 +40,7 @@ interface ScannerWrapperFactory : Plugin { abstract class AbstractScannerWrapperFactory( override val type: String ) : ScannerWrapperFactory { - abstract override fun create(scannerConfig: ScannerConfiguration, downloaderConfig: DownloaderConfiguration): T + abstract override fun create(scannerConfig: ScannerConfiguration): T /** * Return the scanner wrapper's name here to allow Clikt to display something meaningful when listing the scanners diff --git a/scanner/src/main/kotlin/storages/ClearlyDefinedStorage.kt b/scanner/src/main/kotlin/storages/ClearlyDefinedStorage.kt index dae4ebb1fd402..e4f425c61f92a 100644 --- a/scanner/src/main/kotlin/storages/ClearlyDefinedStorage.kt +++ b/scanner/src/main/kotlin/storages/ClearlyDefinedStorage.kt @@ -47,7 +47,6 @@ import org.ossreviewtoolkit.model.UnknownProvenance import org.ossreviewtoolkit.model.VcsInfo import org.ossreviewtoolkit.model.VcsType import org.ossreviewtoolkit.model.config.ClearlyDefinedStorageConfiguration -import org.ossreviewtoolkit.model.config.DownloaderConfiguration import org.ossreviewtoolkit.model.config.ScannerConfiguration import org.ossreviewtoolkit.model.jsonMapper import org.ossreviewtoolkit.model.utils.toClearlyDefinedCoordinates @@ -126,7 +125,7 @@ class ClearlyDefinedStorage( val supportedScanners = toolVersionsByName.mapNotNull { (name, versions) -> // For the ClearlyDefined tool names see https://github.com/clearlydefined/service#tool-name-registry. ScannerWrapper.ALL[name]?.let { factory -> - val scanner = factory.create(ScannerConfiguration(), DownloaderConfiguration()) + val scanner = factory.create(ScannerConfiguration()) (scanner as? CommandLinePathScannerWrapper)?.let { cliScanner -> cliScanner to versions.last() } }.also { if (it == null) logger.debug { "Unsupported tool '$name' for coordinates '$coordinates'." } From 464363834e3420b650e0e497365e748b969a1ec7 Mon Sep 17 00:00:00 2001 From: Martin Nonnenmacher Date: Tue, 26 Sep 2023 00:29:58 +0200 Subject: [PATCH 17/47] refactor(scanner)!: Rename `fromConfig` in `ScannerCriteria` to `create` Use a generic name for the factory function as the previous name will not match anymore after an upcoming change. Signed-off-by: Martin Nonnenmacher --- plugins/scanners/askalono/src/main/kotlin/Askalono.kt | 2 +- plugins/scanners/boyterlc/src/main/kotlin/BoyterLc.kt | 2 +- plugins/scanners/licensee/src/main/kotlin/Licensee.kt | 2 +- plugins/scanners/scancode/src/main/kotlin/ScanCode.kt | 2 +- plugins/scanners/scanoss/src/main/kotlin/ScanOss.kt | 2 +- scanner/src/main/kotlin/ScannerCriteria.kt | 2 +- scanner/src/test/kotlin/ScannerCriteriaTest.kt | 10 +++++----- 7 files changed, 11 insertions(+), 11 deletions(-) diff --git a/plugins/scanners/askalono/src/main/kotlin/Askalono.kt b/plugins/scanners/askalono/src/main/kotlin/Askalono.kt index f49f6ea08839a..e2614af0ac6fd 100644 --- a/plugins/scanners/askalono/src/main/kotlin/Askalono.kt +++ b/plugins/scanners/askalono/src/main/kotlin/Askalono.kt @@ -56,7 +56,7 @@ class Askalono internal constructor( override val configuration = "" - override val criteria by lazy { ScannerCriteria.fromConfig(details, scannerConfig) } + override val criteria by lazy { ScannerCriteria.create(details, scannerConfig) } override fun command(workingDir: File?) = listOfNotNull(workingDir, if (Os.isWindows) "askalono.exe" else "askalono").joinToString(File.separator) diff --git a/plugins/scanners/boyterlc/src/main/kotlin/BoyterLc.kt b/plugins/scanners/boyterlc/src/main/kotlin/BoyterLc.kt index 25e3a7b509a4b..393c9ae529bee 100644 --- a/plugins/scanners/boyterlc/src/main/kotlin/BoyterLc.kt +++ b/plugins/scanners/boyterlc/src/main/kotlin/BoyterLc.kt @@ -60,7 +60,7 @@ class BoyterLc internal constructor( override val configuration = CONFIGURATION_OPTIONS.joinToString(" ") - override val criteria by lazy { ScannerCriteria.fromConfig(details, scannerConfig) } + override val criteria by lazy { ScannerCriteria.create(details, scannerConfig) } override fun command(workingDir: File?) = listOfNotNull(workingDir, if (Os.isWindows) "lc.exe" else "lc").joinToString(File.separator) diff --git a/plugins/scanners/licensee/src/main/kotlin/Licensee.kt b/plugins/scanners/licensee/src/main/kotlin/Licensee.kt index d0007e927d380..1114f0a94c52e 100644 --- a/plugins/scanners/licensee/src/main/kotlin/Licensee.kt +++ b/plugins/scanners/licensee/src/main/kotlin/Licensee.kt @@ -59,7 +59,7 @@ class Licensee internal constructor( override val configuration = CONFIGURATION_OPTIONS.joinToString(" ") - override val criteria by lazy { ScannerCriteria.fromConfig(details, scannerConfig) } + override val criteria by lazy { ScannerCriteria.create(details, scannerConfig) } override fun command(workingDir: File?) = listOfNotNull(workingDir, if (Os.isWindows) "licensee.bat" else "licensee").joinToString(File.separator) diff --git a/plugins/scanners/scancode/src/main/kotlin/ScanCode.kt b/plugins/scanners/scancode/src/main/kotlin/ScanCode.kt index 3420928f93dc7..d5ea0376862c6 100644 --- a/plugins/scanners/scancode/src/main/kotlin/ScanCode.kt +++ b/plugins/scanners/scancode/src/main/kotlin/ScanCode.kt @@ -97,7 +97,7 @@ class ScanCode internal constructor( override fun create(scannerConfig: ScannerConfiguration) = ScanCode(type, scannerConfig) } - override val criteria by lazy { ScannerCriteria.fromConfig(details, scannerConfig) } + override val criteria by lazy { ScannerCriteria.create(details, scannerConfig) } override val configuration by lazy { buildList { diff --git a/plugins/scanners/scanoss/src/main/kotlin/ScanOss.kt b/plugins/scanners/scanoss/src/main/kotlin/ScanOss.kt index 54cb63a0b7565..ec7ae793e2bee 100644 --- a/plugins/scanners/scanoss/src/main/kotlin/ScanOss.kt +++ b/plugins/scanners/scanoss/src/main/kotlin/ScanOss.kt @@ -74,7 +74,7 @@ class ScanOss internal constructor( override val configuration = "" - override val criteria by lazy { ScannerCriteria.fromConfig(details, scannerConfig) } + override val criteria by lazy { ScannerCriteria.create(details, scannerConfig) } /** * The name of the file corresponding to the fingerprints can be sent to SCANOSS for more precise matches. diff --git a/scanner/src/main/kotlin/ScannerCriteria.kt b/scanner/src/main/kotlin/ScannerCriteria.kt index b8f6dcc2d3380..f6e54d0c19a94 100644 --- a/scanner/src/main/kotlin/ScannerCriteria.kt +++ b/scanner/src/main/kotlin/ScannerCriteria.kt @@ -135,7 +135,7 @@ data class ScannerCriteria( * instance, to specify that a specific minimum version of ScanCode is allowed, set this property: * `options.ScanCode.minVersion=3.0.2`. */ - fun fromConfig(details: ScannerDetails, config: ScannerConfiguration): ScannerCriteria { + fun create(details: ScannerDetails, config: ScannerConfiguration): ScannerCriteria { val options = config.options?.get(details.name).orEmpty() val scannerVersion = Semver(normalizeVersion(details.version)) val minVersion = parseVersion(options[PROP_CRITERIA_MIN_VERSION]) ?: scannerVersion diff --git a/scanner/src/test/kotlin/ScannerCriteriaTest.kt b/scanner/src/test/kotlin/ScannerCriteriaTest.kt index 2fdbbb6238ac7..a5f76144779f0 100644 --- a/scanner/src/test/kotlin/ScannerCriteriaTest.kt +++ b/scanner/src/test/kotlin/ScannerCriteriaTest.kt @@ -68,11 +68,11 @@ class ScannerCriteriaTest : WordSpec({ } } - "ScannerCriteria.fromConfig()" should { + "ScannerCriteria.create()" should { "obtain default values from the scanner details" { val config = createScannerConfig(emptyMap()) - val criteria = ScannerCriteria.fromConfig(testDetails, config) + val criteria = ScannerCriteria.create(testDetails, config) criteria.regScannerName shouldBe SCANNER_NAME criteria.minVersion.version shouldBe SCANNER_VERSION @@ -88,7 +88,7 @@ class ScannerCriteriaTest : WordSpec({ ) ) - val criteria = ScannerCriteria.fromConfig(testDetails, config) + val criteria = ScannerCriteria.create(testDetails, config) criteria.regScannerName shouldBe "foo" criteria.minVersion.version shouldBe "1.2.3" @@ -103,7 +103,7 @@ class ScannerCriteriaTest : WordSpec({ ) ) - val criteria = ScannerCriteria.fromConfig(testDetails, config) + val criteria = ScannerCriteria.create(testDetails, config) criteria.minVersion.version shouldBe "1.0.0" criteria.maxVersion.version shouldBe "3.7.0" @@ -112,7 +112,7 @@ class ScannerCriteriaTest : WordSpec({ "use an exact configuration matcher" { val config = createScannerConfig(emptyMap()) - val criteria = ScannerCriteria.fromConfig(testDetails, config) + val criteria = ScannerCriteria.create(testDetails, config) criteria.configMatcher(testDetails.configuration) shouldBe true criteria.configMatcher(testDetails.configuration + "_other") shouldBe false From 6e245cf79520be313a3b994555b7ebfa827640a7 Mon Sep 17 00:00:00 2001 From: Martin Nonnenmacher Date: Tue, 26 Sep 2023 00:33:19 +0200 Subject: [PATCH 18/47] docs(scanner): Fix a typo Signed-off-by: Martin Nonnenmacher --- scanner/src/main/kotlin/ScannerCriteria.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scanner/src/main/kotlin/ScannerCriteria.kt b/scanner/src/main/kotlin/ScannerCriteria.kt index f6e54d0c19a94..764171cca52f9 100644 --- a/scanner/src/main/kotlin/ScannerCriteria.kt +++ b/scanner/src/main/kotlin/ScannerCriteria.kt @@ -129,7 +129,7 @@ data class ScannerCriteria( /** * Return a [ScannerCriteria] instance that is to be used when looking up existing scan results from a * [ScanResultsStorage]. By default, the properties of this instance are initialized to match the scanner - * [details]. These default can be overridden by the [ScannerConfiguration.options] property in the provided + * [details]. These defaults can be overridden by the [ScannerConfiguration.options] property in the provided * [config]: Use properties of the form `scannerName.property`, where `scannerName` is the name of the scanner * the configuration applies to, and `property` is the name of a property of the [ScannerCriteria] class. For * instance, to specify that a specific minimum version of ScanCode is allowed, set this property: From a84a1f43e66f2862b4acefd40ef7b760d23edbc5 Mon Sep 17 00:00:00 2001 From: Martin Nonnenmacher Date: Tue, 26 Sep 2023 00:41:24 +0200 Subject: [PATCH 19/47] refactor(scanner)!: Use only scanner specific options for criteria Provide only the scanner specific options to the `ScannerCriteria` factory function. This improves the separation of concerns as the function only has to deal with the relevant options. Provide an empty map as default value for the options to simplify usage in tests. Signed-off-by: Martin Nonnenmacher --- .../askalono/src/main/kotlin/Askalono.kt | 4 +- .../boyterlc/src/main/kotlin/BoyterLc.kt | 4 +- .../licensee/src/main/kotlin/Licensee.kt | 4 +- .../scancode/src/main/kotlin/ScanCode.kt | 4 +- .../scanoss/src/main/kotlin/ScanOss.kt | 4 +- scanner/src/main/kotlin/ScannerCriteria.kt | 13 +++--- .../src/test/kotlin/ScannerCriteriaTest.kt | 40 +++++-------------- 7 files changed, 31 insertions(+), 42 deletions(-) diff --git a/plugins/scanners/askalono/src/main/kotlin/Askalono.kt b/plugins/scanners/askalono/src/main/kotlin/Askalono.kt index e2614af0ac6fd..489b51edebddc 100644 --- a/plugins/scanners/askalono/src/main/kotlin/Askalono.kt +++ b/plugins/scanners/askalono/src/main/kotlin/Askalono.kt @@ -56,7 +56,9 @@ class Askalono internal constructor( override val configuration = "" - override val criteria by lazy { ScannerCriteria.create(details, scannerConfig) } + override val criteria by lazy { + ScannerCriteria.create(details, scannerConfig.options?.get(details.name).orEmpty()) + } override fun command(workingDir: File?) = listOfNotNull(workingDir, if (Os.isWindows) "askalono.exe" else "askalono").joinToString(File.separator) diff --git a/plugins/scanners/boyterlc/src/main/kotlin/BoyterLc.kt b/plugins/scanners/boyterlc/src/main/kotlin/BoyterLc.kt index 393c9ae529bee..d66e29990aa55 100644 --- a/plugins/scanners/boyterlc/src/main/kotlin/BoyterLc.kt +++ b/plugins/scanners/boyterlc/src/main/kotlin/BoyterLc.kt @@ -60,7 +60,9 @@ class BoyterLc internal constructor( override val configuration = CONFIGURATION_OPTIONS.joinToString(" ") - override val criteria by lazy { ScannerCriteria.create(details, scannerConfig) } + override val criteria by lazy { + ScannerCriteria.create(details, scannerConfig.options?.get(details.name).orEmpty()) + } override fun command(workingDir: File?) = listOfNotNull(workingDir, if (Os.isWindows) "lc.exe" else "lc").joinToString(File.separator) diff --git a/plugins/scanners/licensee/src/main/kotlin/Licensee.kt b/plugins/scanners/licensee/src/main/kotlin/Licensee.kt index 1114f0a94c52e..09846cd4b3b2e 100644 --- a/plugins/scanners/licensee/src/main/kotlin/Licensee.kt +++ b/plugins/scanners/licensee/src/main/kotlin/Licensee.kt @@ -59,7 +59,9 @@ class Licensee internal constructor( override val configuration = CONFIGURATION_OPTIONS.joinToString(" ") - override val criteria by lazy { ScannerCriteria.create(details, scannerConfig) } + override val criteria by lazy { + ScannerCriteria.create(details, scannerConfig.options?.get(details.name).orEmpty()) + } override fun command(workingDir: File?) = listOfNotNull(workingDir, if (Os.isWindows) "licensee.bat" else "licensee").joinToString(File.separator) diff --git a/plugins/scanners/scancode/src/main/kotlin/ScanCode.kt b/plugins/scanners/scancode/src/main/kotlin/ScanCode.kt index d5ea0376862c6..e975659150129 100644 --- a/plugins/scanners/scancode/src/main/kotlin/ScanCode.kt +++ b/plugins/scanners/scancode/src/main/kotlin/ScanCode.kt @@ -97,7 +97,9 @@ class ScanCode internal constructor( override fun create(scannerConfig: ScannerConfiguration) = ScanCode(type, scannerConfig) } - override val criteria by lazy { ScannerCriteria.create(details, scannerConfig) } + override val criteria by lazy { + ScannerCriteria.create(details, scannerConfig.options?.get(details.name).orEmpty()) + } override val configuration by lazy { buildList { diff --git a/plugins/scanners/scanoss/src/main/kotlin/ScanOss.kt b/plugins/scanners/scanoss/src/main/kotlin/ScanOss.kt index ec7ae793e2bee..59849ea207cb8 100644 --- a/plugins/scanners/scanoss/src/main/kotlin/ScanOss.kt +++ b/plugins/scanners/scanoss/src/main/kotlin/ScanOss.kt @@ -74,7 +74,9 @@ class ScanOss internal constructor( override val configuration = "" - override val criteria by lazy { ScannerCriteria.create(details, scannerConfig) } + override val criteria by lazy { + ScannerCriteria.create(details, scannerConfig.options?.get(details.name).orEmpty()) + } /** * The name of the file corresponding to the fingerprints can be sent to SCANOSS for more precise matches. diff --git a/scanner/src/main/kotlin/ScannerCriteria.kt b/scanner/src/main/kotlin/ScannerCriteria.kt index 764171cca52f9..c583848d65bc7 100644 --- a/scanner/src/main/kotlin/ScannerCriteria.kt +++ b/scanner/src/main/kotlin/ScannerCriteria.kt @@ -20,7 +20,7 @@ package org.ossreviewtoolkit.scanner import org.ossreviewtoolkit.model.ScannerDetails -import org.ossreviewtoolkit.model.config.ScannerConfiguration +import org.ossreviewtoolkit.model.config.Options import org.semver4j.Semver import org.semver4j.Semver.VersionDiff @@ -129,14 +129,11 @@ data class ScannerCriteria( /** * Return a [ScannerCriteria] instance that is to be used when looking up existing scan results from a * [ScanResultsStorage]. By default, the properties of this instance are initialized to match the scanner - * [details]. These defaults can be overridden by the [ScannerConfiguration.options] property in the provided - * [config]: Use properties of the form `scannerName.property`, where `scannerName` is the name of the scanner - * the configuration applies to, and `property` is the name of a property of the [ScannerCriteria] class. For - * instance, to specify that a specific minimum version of ScanCode is allowed, set this property: - * `options.ScanCode.minVersion=3.0.2`. + * [details]. These defaults can be overridden by the provided [options]. The keys of the option map must match + * names of the [ScannerCriteria] class. For example, to specify that a specific minimum version of the scanner + * is allowed, set this option: `minVersion=3.0.2`. */ - fun create(details: ScannerDetails, config: ScannerConfiguration): ScannerCriteria { - val options = config.options?.get(details.name).orEmpty() + fun create(details: ScannerDetails, options: Options = emptyMap()): ScannerCriteria { val scannerVersion = Semver(normalizeVersion(details.version)) val minVersion = parseVersion(options[PROP_CRITERIA_MIN_VERSION]) ?: scannerVersion val maxVersion = parseVersion(options[PROP_CRITERIA_MAX_VERSION]) ?: minVersion.nextMinor() diff --git a/scanner/src/test/kotlin/ScannerCriteriaTest.kt b/scanner/src/test/kotlin/ScannerCriteriaTest.kt index a5f76144779f0..6efba51c99a20 100644 --- a/scanner/src/test/kotlin/ScannerCriteriaTest.kt +++ b/scanner/src/test/kotlin/ScannerCriteriaTest.kt @@ -23,8 +23,6 @@ import io.kotest.core.spec.style.WordSpec import io.kotest.matchers.shouldBe import org.ossreviewtoolkit.model.ScannerDetails -import org.ossreviewtoolkit.model.config.Options -import org.ossreviewtoolkit.model.config.ScannerConfiguration import org.semver4j.Semver @@ -70,9 +68,7 @@ class ScannerCriteriaTest : WordSpec({ "ScannerCriteria.create()" should { "obtain default values from the scanner details" { - val config = createScannerConfig(emptyMap()) - - val criteria = ScannerCriteria.create(testDetails, config) + val criteria = ScannerCriteria.create(testDetails) criteria.regScannerName shouldBe SCANNER_NAME criteria.minVersion.version shouldBe SCANNER_VERSION @@ -80,15 +76,13 @@ class ScannerCriteriaTest : WordSpec({ } "obtain values from the configuration" { - val config = createScannerConfig( - mapOf( - ScannerCriteria.PROP_CRITERIA_NAME to "foo", - ScannerCriteria.PROP_CRITERIA_MIN_VERSION to "1.2.3", - ScannerCriteria.PROP_CRITERIA_MAX_VERSION to "4.5.6" - ) + val options = mapOf( + ScannerCriteria.PROP_CRITERIA_NAME to "foo", + ScannerCriteria.PROP_CRITERIA_MIN_VERSION to "1.2.3", + ScannerCriteria.PROP_CRITERIA_MAX_VERSION to "4.5.6" ) - val criteria = ScannerCriteria.create(testDetails, config) + val criteria = ScannerCriteria.create(testDetails, options) criteria.regScannerName shouldBe "foo" criteria.minVersion.version shouldBe "1.2.3" @@ -96,23 +90,19 @@ class ScannerCriteriaTest : WordSpec({ } "parse versions in a lenient way" { - val config = createScannerConfig( - mapOf( - ScannerCriteria.PROP_CRITERIA_MIN_VERSION to "1", - ScannerCriteria.PROP_CRITERIA_MAX_VERSION to "3.7" - ) + val options = mapOf( + ScannerCriteria.PROP_CRITERIA_MIN_VERSION to "1", + ScannerCriteria.PROP_CRITERIA_MAX_VERSION to "3.7" ) - val criteria = ScannerCriteria.create(testDetails, config) + val criteria = ScannerCriteria.create(testDetails, options) criteria.minVersion.version shouldBe "1.0.0" criteria.maxVersion.version shouldBe "3.7.0" } "use an exact configuration matcher" { - val config = createScannerConfig(emptyMap()) - - val criteria = ScannerCriteria.create(testDetails, config) + val criteria = ScannerCriteria.create(testDetails) criteria.configMatcher(testDetails.configuration) shouldBe true criteria.configMatcher(testDetails.configuration + "_other") shouldBe false @@ -177,11 +167,3 @@ private val matchingCriteria = ScannerCriteria( maxVersion = Semver(testDetails.version).nextPatch(), configMatcher = ScannerCriteria.exactConfigMatcher(testDetails.configuration) ) - -/** - * Creates a [ScannerConfiguration] with the given properties for the test scanner. - */ -private fun createScannerConfig(properties: Options): ScannerConfiguration { - val options = mapOf(SCANNER_NAME to properties) - return ScannerConfiguration(options = options) -} From fd71440289264fa6245fbe85fcf07b8588e8c0f4 Mon Sep 17 00:00:00 2001 From: Martin Nonnenmacher Date: Tue, 26 Sep 2023 01:07:03 +0200 Subject: [PATCH 20/47] refactor(scanner)!: Remove the unused `ScannerConfigMatcher` The `ScannerConfigMatcher` was intended to support cases where the `ScannerDetails.configuration` must not be matched exactly. As the feature was not taken into use since its introduction three years ago, replace it with a simpler approach to match either the whole configuration or ignore the configuration to simplify code. Signed-off-by: Martin Nonnenmacher --- scanner/src/main/kotlin/ScannerCriteria.kt | 37 +++++---------- .../src/test/kotlin/ScannerCriteriaTest.kt | 45 +++++++------------ .../storages/ClearlyDefinedStorageTest.kt | 6 +-- 3 files changed, 27 insertions(+), 61 deletions(-) diff --git a/scanner/src/main/kotlin/ScannerCriteria.kt b/scanner/src/main/kotlin/ScannerCriteria.kt index c583848d65bc7..b6ad608fe1133 100644 --- a/scanner/src/main/kotlin/ScannerCriteria.kt +++ b/scanner/src/main/kotlin/ScannerCriteria.kt @@ -25,18 +25,6 @@ import org.ossreviewtoolkit.model.config.Options import org.semver4j.Semver import org.semver4j.Semver.VersionDiff -/** - * Definition of a predicate to check whether the configuration of a scanner is compatible with the requirements - * specified by a [ScannerCriteria]. - * - * When testing whether a scan result is compatible with specific criteria this function is invoked on the - * scanner configuration data stored in the result. By having different, scanner-specific matcher functions, this - * compatibility check can be made very flexible. - * - * TODO: Switch to a more advanced type than String to represent the scanner configuration. - */ -typealias ScannerConfigMatcher = (String) -> Boolean - /** * A data class defining selection criteria for scanners. * @@ -69,17 +57,12 @@ data class ScannerCriteria( val maxVersion: Semver, /** - * A function to check whether the configuration of a scanner is compatible with this [ScannerCriteria]. + * Criterion to match the [configuration][ScannerDetails.configuration] of the scanner. If `null`, all + * configurations are matched. */ - val configMatcher: ScannerConfigMatcher + val configuration: String? ) { companion object { - /** - * A matcher for scanner configurations that accepts all configurations passed in. This function can be - * used if the concrete configuration of a scanner is irrelevant. - */ - val ALL_CONFIG_MATCHER: ScannerConfigMatcher = { true } - /** * The name of the property defining the regular expression for the scanner name as part of [ScannerCriteria]. * The [scanner details][ScannerDetails] of the corresponding scanner must match the criteria. @@ -99,10 +82,10 @@ data class ScannerCriteria( const val PROP_CRITERIA_MAX_VERSION = "maxVersion" /** - * A matcher for scanner configurations that accepts only exact matches of the [originalConfig]. This - * function can be used by scanners that are extremely sensitive about their configuration. + * The name of the property defining the configuration of the scanner as part of [ScannerCriteria]. The + * [scanner details][ScannerDetails] of the corresponding scanner must match the criteria. */ - fun exactConfigMatcher(originalConfig: String): ScannerConfigMatcher = { config -> originalConfig == config } + const val PROP_CRITERIA_CONFIGURATION = "configuration" /** * Generate a [ScannerCriteria] instance that is compatible with the given [details] and versions that differ @@ -122,7 +105,7 @@ data class ScannerCriteria( regScannerName = details.name, minVersion = minVersion, maxVersion = maxVersion, - configMatcher = exactConfigMatcher(details.configuration) + configuration = details.configuration ) } @@ -138,8 +121,9 @@ data class ScannerCriteria( val minVersion = parseVersion(options[PROP_CRITERIA_MIN_VERSION]) ?: scannerVersion val maxVersion = parseVersion(options[PROP_CRITERIA_MAX_VERSION]) ?: minVersion.nextMinor() val name = options[PROP_CRITERIA_NAME] ?: details.name + val configuration = options[PROP_CRITERIA_CONFIGURATION] ?: details.configuration - return ScannerCriteria(name, minVersion, maxVersion, exactConfigMatcher(details.configuration)) + return ScannerCriteria(name, minVersion, maxVersion, configuration) } } @@ -160,7 +144,8 @@ data class ScannerCriteria( if (!nameRegex.matches(details.name)) return false val version = Semver(details.version) - return minVersion <= version && version < maxVersion && configMatcher(details.configuration) + return minVersion <= version && version < maxVersion && + (configuration == null || configuration == details.configuration) } } diff --git a/scanner/src/test/kotlin/ScannerCriteriaTest.kt b/scanner/src/test/kotlin/ScannerCriteriaTest.kt index 6efba51c99a20..14fb848d1b0e5 100644 --- a/scanner/src/test/kotlin/ScannerCriteriaTest.kt +++ b/scanner/src/test/kotlin/ScannerCriteriaTest.kt @@ -27,22 +27,6 @@ import org.ossreviewtoolkit.model.ScannerDetails import org.semver4j.Semver class ScannerCriteriaTest : WordSpec({ - "ScannerCriteria" should { - "provide a config matcher that accepts every configuration" { - ScannerCriteria.ALL_CONFIG_MATCHER("") shouldBe true - ScannerCriteria.ALL_CONFIG_MATCHER("foo") shouldBe true - ScannerCriteria.ALL_CONFIG_MATCHER("Supercalifragilisticexpialidocious") shouldBe true - } - - "provide a config matcher that accepts only exact configuration matches" { - val orgConfig = "--info --copyright --licenses" - val matcher = ScannerCriteria.exactConfigMatcher(orgConfig) - - matcher(orgConfig) shouldBe true - matcher("$orgConfig --more") shouldBe false - } - } - "ScannerCriteria.forDetails()" should { "create criteria that only match the passed details by default" { val criteria = ScannerCriteria.forDetails(testDetails) @@ -73,13 +57,15 @@ class ScannerCriteriaTest : WordSpec({ criteria.regScannerName shouldBe SCANNER_NAME criteria.minVersion.version shouldBe SCANNER_VERSION criteria.maxVersion shouldBe Semver(SCANNER_VERSION).nextMinor() + criteria.configuration shouldBe SCANNER_CONFIGURATION } "obtain values from the configuration" { val options = mapOf( ScannerCriteria.PROP_CRITERIA_NAME to "foo", ScannerCriteria.PROP_CRITERIA_MIN_VERSION to "1.2.3", - ScannerCriteria.PROP_CRITERIA_MAX_VERSION to "4.5.6" + ScannerCriteria.PROP_CRITERIA_MAX_VERSION to "4.5.6", + ScannerCriteria.PROP_CRITERIA_CONFIGURATION to "config" ) val criteria = ScannerCriteria.create(testDetails, options) @@ -87,6 +73,7 @@ class ScannerCriteriaTest : WordSpec({ criteria.regScannerName shouldBe "foo" criteria.minVersion.version shouldBe "1.2.3" criteria.maxVersion.version shouldBe "4.5.6" + criteria.configuration shouldBe "config" } "parse versions in a lenient way" { @@ -100,13 +87,6 @@ class ScannerCriteriaTest : WordSpec({ criteria.minVersion.version shouldBe "1.0.0" criteria.maxVersion.version shouldBe "3.7.0" } - - "use an exact configuration matcher" { - val criteria = ScannerCriteria.create(testDetails) - - criteria.configMatcher(testDetails.configuration) shouldBe true - criteria.configMatcher(testDetails.configuration + "_other") shouldBe false - } } "ScannerCriteria.matches()" should { @@ -144,26 +124,31 @@ class ScannerCriteriaTest : WordSpec({ criteria.matches(testDetails) shouldBe false } - "detect a difference reported by the config matcher" { - val criteria = matchingCriteria.copy( - configMatcher = ScannerCriteria.exactConfigMatcher(testDetails.configuration + "_other") - ) + "detect a scanner configuration that does not match" { + val criteria = matchingCriteria.copy(configuration = "${testDetails.configuration}_other") criteria.matches(testDetails) shouldBe false } + + "ignore the scanner configuration if it is null" { + val criteria = matchingCriteria.copy(configuration = null) + + criteria.matches(testDetails) shouldBe true + } } }) private const val SCANNER_NAME = "ScannerCriteriaTest" private const val SCANNER_VERSION = "3.2.1-rc2" +private const val SCANNER_CONFIGURATION = "--command-line-option" /** Test details to match against. */ -private val testDetails = ScannerDetails(SCANNER_NAME, SCANNER_VERSION, "--command-line-option") +private val testDetails = ScannerDetails(SCANNER_NAME, SCANNER_VERSION, SCANNER_CONFIGURATION) /** A test instance which should accept the test details. */ private val matchingCriteria = ScannerCriteria( regScannerName = testDetails.name, minVersion = Semver(testDetails.version), maxVersion = Semver(testDetails.version).nextPatch(), - configMatcher = ScannerCriteria.exactConfigMatcher(testDetails.configuration) + configuration = testDetails.configuration ) diff --git a/scanner/src/test/kotlin/storages/ClearlyDefinedStorageTest.kt b/scanner/src/test/kotlin/storages/ClearlyDefinedStorageTest.kt index 8c3ebc07389b6..42ae3585dd9a8 100644 --- a/scanner/src/test/kotlin/storages/ClearlyDefinedStorageTest.kt +++ b/scanner/src/test/kotlin/storages/ClearlyDefinedStorageTest.kt @@ -106,11 +106,7 @@ private val TEST_PACKAGE = ) /** The scanner details used by tests. */ -private val SCANNER_CRITERIA = - ScannerCriteria( - "aScanner", Semver("1.0.0"), Semver("2.0.0"), - ScannerCriteria.exactConfigMatcher("aConfig") - ) +private val SCANNER_CRITERIA = ScannerCriteria("aScanner", Semver("1.0.0"), Semver("2.0.0"), "aConfig") /** The template for a ClearlyDefined definitions request. */ private val DEFINITIONS_TEMPLATE = readDefinitionsTemplate() From ffce6dc9256afd5ffbefa0d5b9ec53fac88de80a Mon Sep 17 00:00:00 2001 From: Martin Nonnenmacher Date: Tue, 26 Sep 2023 01:22:20 +0200 Subject: [PATCH 21/47] refactor(scanner)!: Remove `ScannerCriteria.forDetails()` Remove the `ScannerCriteria.forDetails()` function as it was only used in tests and usage can be replaced by calling the `create()` function instead. The functionality to create the `maxVersion` based on a `VersionDiff` was never made configurable and was always exclusively used by tests. Signed-off-by: Martin Nonnenmacher --- .../scanners/ScannerIntegrationFunTest.kt | 2 +- .../AbstractProvenanceBasedStorageFunTest.kt | 10 ++++---- .../kotlin/storages/AbstractStorageFunTest.kt | 2 +- scanner/src/main/kotlin/ScannerCriteria.kt | 23 ------------------- .../src/test/kotlin/ScannerCriteriaTest.kt | 23 ------------------- scanner/src/test/kotlin/ScannerTest.kt | 6 ++--- 6 files changed, 9 insertions(+), 57 deletions(-) diff --git a/scanner/src/funTest/kotlin/scanners/ScannerIntegrationFunTest.kt b/scanner/src/funTest/kotlin/scanners/ScannerIntegrationFunTest.kt index 83726fb2cc7c1..ed9b9700112c4 100644 --- a/scanner/src/funTest/kotlin/scanners/ScannerIntegrationFunTest.kt +++ b/scanner/src/funTest/kotlin/scanners/ScannerIntegrationFunTest.kt @@ -215,7 +215,7 @@ internal class DummyScanner(override val name: String = "Dummy") : PathScannerWr override val version = "1.0.0" override val configuration = "" - override val criteria = ScannerCriteria.forDetails(details) + override val criteria = ScannerCriteria.create(details) override fun scanPath(path: File, context: ScanContext): ScanSummary { val relevantFiles = path.walk() diff --git a/scanner/src/funTest/kotlin/storages/AbstractProvenanceBasedStorageFunTest.kt b/scanner/src/funTest/kotlin/storages/AbstractProvenanceBasedStorageFunTest.kt index fb73241044d39..caf0f095063bc 100644 --- a/scanner/src/funTest/kotlin/storages/AbstractProvenanceBasedStorageFunTest.kt +++ b/scanner/src/funTest/kotlin/storages/AbstractProvenanceBasedStorageFunTest.kt @@ -45,8 +45,6 @@ import org.ossreviewtoolkit.scanner.ProvenanceBasedScanStorage import org.ossreviewtoolkit.scanner.ScanStorageException import org.ossreviewtoolkit.scanner.ScannerCriteria -import org.semver4j.Semver - abstract class AbstractProvenanceBasedStorageFunTest(vararg listeners: TestListener) : WordSpec() { private lateinit var storage: ProvenanceBasedScanStorage @@ -133,7 +131,7 @@ abstract class AbstractProvenanceBasedStorageFunTest(vararg listeners: TestListe "fail if the provenance contains a VCS path" { val provenance = createRepositoryProvenance(vcsInfo = VcsInfo.valid().copy(path = "path")) - val criteria = ScannerCriteria.forDetails(createScannerDetails()) + val criteria = ScannerCriteria.create(createScannerDetails()) shouldThrow { storage.read(provenance) } shouldThrow { storage.read(provenance, criteria) } @@ -148,7 +146,7 @@ abstract class AbstractProvenanceBasedStorageFunTest(vararg listeners: TestListe val readResult = storage.read( scanResult1.provenance as KnownProvenance, - ScannerCriteria.forDetails(scanResult1.scanner) + ScannerCriteria.create(scanResult1.scanner) ) readResult should containExactly(scanResult1) @@ -158,7 +156,7 @@ abstract class AbstractProvenanceBasedStorageFunTest(vararg listeners: TestListe val scanResult1 = createScanResult(scannerDetails = createScannerDetails(name = "name1")) val scanResult2 = createScanResult(scannerDetails = createScannerDetails(name = "name2")) val scanResult3 = createScanResult(scannerDetails = createScannerDetails(name = "other name")) - val criteria = ScannerCriteria.forDetails(scanResult1.scanner).copy(regScannerName = "name.+") + val criteria = ScannerCriteria.create(scanResult1.scanner).copy(regScannerName = "name.+") storage.write(scanResult1) storage.write(scanResult2) @@ -175,7 +173,7 @@ abstract class AbstractProvenanceBasedStorageFunTest(vararg listeners: TestListe val scanResultCompatible2 = createScanResult(scannerDetails = createScannerDetails(version = "1.0.1-alpha.1")) val scanResultIncompatible = createScanResult(scannerDetails = createScannerDetails(version = "2.0.0")) - val criteria = ScannerCriteria.forDetails(scanResult.scanner, Semver.VersionDiff.PATCH) + val criteria = ScannerCriteria.create(scanResult.scanner) storage.write(scanResult) storage.write(scanResultCompatible1) diff --git a/scanner/src/funTest/kotlin/storages/AbstractStorageFunTest.kt b/scanner/src/funTest/kotlin/storages/AbstractStorageFunTest.kt index c7df1a68f9902..6e02a414a43f1 100644 --- a/scanner/src/funTest/kotlin/storages/AbstractStorageFunTest.kt +++ b/scanner/src/funTest/kotlin/storages/AbstractStorageFunTest.kt @@ -97,7 +97,7 @@ abstract class AbstractStorageFunTest(vararg listeners: TestListener) : WordSpec private val scannerDetailsCompatibleVersion2 = ScannerDetails("name 1", "1.0.1-alpha.1", "config 1") private val scannerDetailsIncompatibleVersion = ScannerDetails("name 1", "1.1.0", "config 1") - private val scannerCriteriaForDetails1 = ScannerCriteria.forDetails(scannerDetails1, Semver.VersionDiff.PATCH) + private val scannerCriteriaForDetails1 = ScannerCriteria.create(scannerDetails1) private val scanSummaryWithFiles = ScanSummary.EMPTY.copy( startTime = Instant.EPOCH + Duration.ofMinutes(1), diff --git a/scanner/src/main/kotlin/ScannerCriteria.kt b/scanner/src/main/kotlin/ScannerCriteria.kt index b6ad608fe1133..b064de3d49336 100644 --- a/scanner/src/main/kotlin/ScannerCriteria.kt +++ b/scanner/src/main/kotlin/ScannerCriteria.kt @@ -23,7 +23,6 @@ import org.ossreviewtoolkit.model.ScannerDetails import org.ossreviewtoolkit.model.config.Options import org.semver4j.Semver -import org.semver4j.Semver.VersionDiff /** * A data class defining selection criteria for scanners. @@ -87,28 +86,6 @@ data class ScannerCriteria( */ const val PROP_CRITERIA_CONFIGURATION = "configuration" - /** - * Generate a [ScannerCriteria] instance that is compatible with the given [details] and versions that differ - * only in the provided [versionDiff]. - */ - fun forDetails(details: ScannerDetails, versionDiff: VersionDiff = VersionDiff.NONE): ScannerCriteria { - val minVersion = Semver(details.version) - - val maxVersion = when (versionDiff) { - VersionDiff.NONE, VersionDiff.PRE_RELEASE, VersionDiff.BUILD -> minVersion.nextPatch() - VersionDiff.PATCH -> minVersion.nextMinor() - VersionDiff.MINOR -> minVersion.nextMajor() - else -> throw IllegalArgumentException("Invalid version difference $versionDiff") - } - - return ScannerCriteria( - regScannerName = details.name, - minVersion = minVersion, - maxVersion = maxVersion, - configuration = details.configuration - ) - } - /** * Return a [ScannerCriteria] instance that is to be used when looking up existing scan results from a * [ScanResultsStorage]. By default, the properties of this instance are initialized to match the scanner diff --git a/scanner/src/test/kotlin/ScannerCriteriaTest.kt b/scanner/src/test/kotlin/ScannerCriteriaTest.kt index 14fb848d1b0e5..4ddfb24401de1 100644 --- a/scanner/src/test/kotlin/ScannerCriteriaTest.kt +++ b/scanner/src/test/kotlin/ScannerCriteriaTest.kt @@ -27,29 +27,6 @@ import org.ossreviewtoolkit.model.ScannerDetails import org.semver4j.Semver class ScannerCriteriaTest : WordSpec({ - "ScannerCriteria.forDetails()" should { - "create criteria that only match the passed details by default" { - val criteria = ScannerCriteria.forDetails(testDetails) - val nextPatchVersion = Semver(testDetails.version).nextPatch().toString() - val testDetailsForNextPatchVersion = testDetails.copy(version = nextPatchVersion) - - criteria.matches(testDetails) shouldBe true - criteria.matches(testDetailsForNextPatchVersion) shouldBe false - } - - "can create criteria that match details with respect to a version difference" { - val criteria = ScannerCriteria.forDetails(testDetails, Semver.VersionDiff.PATCH) - val nextPatchVersion = Semver(testDetails.version).nextPatch().toString() - val testDetailsForNextPatchVersion = testDetails.copy(version = nextPatchVersion) - val nextMinorVersion = Semver(testDetails.version).nextMinor().toString() - val testDetailsForNextMinorVersion = testDetails.copy(version = nextMinorVersion) - - criteria.matches(testDetails) shouldBe true - criteria.matches(testDetailsForNextPatchVersion) shouldBe true - criteria.matches(testDetailsForNextMinorVersion) shouldBe false - } - } - "ScannerCriteria.create()" should { "obtain default values from the scanner details" { val criteria = ScannerCriteria.create(testDetails) diff --git a/scanner/src/test/kotlin/ScannerTest.kt b/scanner/src/test/kotlin/ScannerTest.kt index 3189de38944fc..4f499e4c86205 100644 --- a/scanner/src/test/kotlin/ScannerTest.kt +++ b/scanner/src/test/kotlin/ScannerTest.kt @@ -808,7 +808,7 @@ private class FakePackageScannerWrapper( override val configuration = "config" // Explicit nullability is required here for a mock response. - override val criteria: ScannerCriteria? = ScannerCriteria.forDetails(details) + override val criteria: ScannerCriteria? = ScannerCriteria.create(details) override fun scanPackage(pkg: Package, context: ScanContext): ScanResult = createScanResult(packageProvenanceResolver.resolveProvenance(pkg, sourceCodeOriginPriority), details) @@ -822,7 +822,7 @@ private class FakeProvenanceScannerWrapper : ProvenanceScannerWrapper { override val version = "1.0.0" override val configuration = "config" - override val criteria = ScannerCriteria.forDetails(details) + override val criteria = ScannerCriteria.create(details) override fun scanProvenance(provenance: KnownProvenance, context: ScanContext): ScanResult = createScanResult(provenance, details) @@ -836,7 +836,7 @@ private class FakePathScannerWrapper : PathScannerWrapper { override val version = "1.0.0" override val configuration = "config" - override val criteria = ScannerCriteria.forDetails(details) + override val criteria = ScannerCriteria.create(details) override fun scanPath(path: File, context: ScanContext): ScanSummary { val licenseFindings = path.walk().filter { it.isFile }.mapTo(mutableSetOf()) { file -> From d1492bb92034bce940a8460d166966135dc53bdb Mon Sep 17 00:00:00 2001 From: Martin Nonnenmacher Date: Tue, 26 Sep 2023 14:13:42 +0200 Subject: [PATCH 22/47] feat(scanner): Add detected license mapping to `ScanContext` Add the detected license mapping from the `ScannerConfiguration` to the `ScanContext` to make it available to `ScannerWrapper`s. This is currently only required by the `FossId` implementation because FossID can return arbitrary strings as licenses. These strings can sometimes not be parsed as SPDX expressions and can therefore not be returned as part of a `LicenseFinding` in the `ScanSummary`. A better solution could be to automatically convert all license strings returned by FossID to a form that can be parsed as SPDX expression, then the mapping could be applied globally. However, as this would be a breaking configuration change it is not implemented now and only added as a TODO comment instead. Signed-off-by: Martin Nonnenmacher --- plugins/scanners/fossid/src/main/kotlin/FossId.kt | 15 ++++++++++++--- .../fossid/src/main/kotlin/FossIdScanResults.kt | 4 +++- scanner/src/main/kotlin/ScanContext.kt | 15 ++++++++++++++- scanner/src/main/kotlin/Scanner.kt | 13 +++++++++++-- 4 files changed, 40 insertions(+), 7 deletions(-) diff --git a/plugins/scanners/fossid/src/main/kotlin/FossId.kt b/plugins/scanners/fossid/src/main/kotlin/FossId.kt index 019f411eaf24f..edba2c776cf10 100644 --- a/plugins/scanners/fossid/src/main/kotlin/FossId.kt +++ b/plugins/scanners/fossid/src/main/kotlin/FossId.kt @@ -310,7 +310,15 @@ class FossId internal constructor( if (config.waitForResult) { val rawResults = getRawResults(scanCode) - createResultSummary(startTime, provenance, rawResults, scanCode, scanId, issues) + createResultSummary( + startTime, + provenance, + rawResults, + scanCode, + scanId, + issues, + context.detectedLicenseMapping + ) } else { val issue = createAndLogIssue( source = name, @@ -857,7 +865,8 @@ class FossId internal constructor( rawResults: RawResults, scanCode: String, scanId: String, - additionalIssues: MutableList + additionalIssues: MutableList, + detectedLicenseMapping: Map ): ScanResult { // TODO: Maybe get issues from FossID (see has_failed_scan_files, get_failed_files and maybe get_scan_log). @@ -875,7 +884,7 @@ class FossId internal constructor( val (licenseFindings, copyrightFindings) = rawResults.markedAsIdentifiedFiles.ifEmpty { rawResults.identifiedFiles - }.mapSummary(ignoredFiles, issues, scannerConfig.detectedLicenseMapping) + }.mapSummary(ignoredFiles, issues, detectedLicenseMapping) val summary = ScanSummary( startTime = startTime, diff --git a/plugins/scanners/fossid/src/main/kotlin/FossIdScanResults.kt b/plugins/scanners/fossid/src/main/kotlin/FossIdScanResults.kt index 2a25181de7437..e949b064a791d 100644 --- a/plugins/scanners/fossid/src/main/kotlin/FossIdScanResults.kt +++ b/plugins/scanners/fossid/src/main/kotlin/FossIdScanResults.kt @@ -85,7 +85,9 @@ internal fun List.mapSummary( summary.licences.forEach { runCatching { - // TODO: The license mapping should be moved to a central place. + // TODO: The detected license mapping must be applied here, because FossID can return license strings + // which cannot be parsed to an SpdxExpression. A better solution could be to automatically + // convert the strings into a form that can be parsed, then the mapping could be applied globally. LicenseFinding(it.identifier.mapLicense(detectedLicenseMapping), location) }.onSuccess { licenseFinding -> licenseFindings += licenseFinding.copy(license = licenseFinding.license.normalize()) diff --git a/scanner/src/main/kotlin/ScanContext.kt b/scanner/src/main/kotlin/ScanContext.kt index 2ab658cd6c92c..0c597c842ef44 100644 --- a/scanner/src/main/kotlin/ScanContext.kt +++ b/scanner/src/main/kotlin/ScanContext.kt @@ -19,9 +19,12 @@ package org.ossreviewtoolkit.scanner +import org.ossreviewtoolkit.model.LicenseFinding import org.ossreviewtoolkit.model.OrtResult import org.ossreviewtoolkit.model.PackageType import org.ossreviewtoolkit.model.config.Excludes +import org.ossreviewtoolkit.model.config.ScannerConfiguration +import org.ossreviewtoolkit.utils.spdx.SpdxExpression /** * Additional context information that can be used by a [ScannerWrapper] to alter its behavior. @@ -40,5 +43,15 @@ data class ScanContext( /** * The [Excludes] of the project to scan. */ - val excludes: Excludes? = null + val excludes: Excludes? = null, + + /** + * The detected license mappings configured in the + * [scanner configuration][ScannerConfiguration.detectedLicenseMapping]. Can be used by [ScannerWrapper] + * implementations where the scanner can return arbitrary license strings which cannot be parsed as + * [SpdxExpression]s and can therefore not be returned as a [LicenseFinding] without being mapped first. Should not + * be used by scanners where scan results are stored, because then changes in the mapping would not be applied to + * stored results. + */ + val detectedLicenseMapping: Map = emptyMap() ) diff --git a/scanner/src/main/kotlin/Scanner.kt b/scanner/src/main/kotlin/Scanner.kt index 3a2ccfdc6fdf9..3492bc5bfc96d 100644 --- a/scanner/src/main/kotlin/Scanner.kt +++ b/scanner/src/main/kotlin/Scanner.kt @@ -124,7 +124,8 @@ class Scanner( ScanContext( ortResult.labels + labels, PackageType.PROJECT, - ortResult.repository.config.excludes + ortResult.repository.config.excludes, + scannerConfig.detectedLicenseMapping ) ) } else { @@ -139,7 +140,15 @@ class Scanner( logger.info { "Scanning ${packages.size} package(s) with ${packageScannerWrappers.size} scanner(s)." } - scan(packages, ScanContext(ortResult.labels, PackageType.PACKAGE, ortResult.repository.config.excludes)) + scan( + packages, + ScanContext( + ortResult.labels, + PackageType.PACKAGE, + ortResult.repository.config.excludes, + scannerConfig.detectedLicenseMapping + ) + ) } else { logger.info { "Skipping package scan as no package scanner is configured." } From dd70b7285726aa6043883ecd21ab4c3f3864739f Mon Sep 17 00:00:00 2001 From: Martin Nonnenmacher Date: Tue, 26 Sep 2023 09:33:24 +0200 Subject: [PATCH 23/47] refactor(scanner)!: Provide only scanner specific options in factory Provide only the scanner specific `Options` instead of the `ScannerConfiguration` to the factory method in `ScannerWrapperFactory`. This simplifies the implementation of the scanners as they do not have to find the relevant configuration on their own, and improves separation of concerns as scanners do not have access to the configuration of other scanners and the global scanner configuration anymore. Signed-off-by: Martin Nonnenmacher --- .../scanner/src/main/kotlin/ScannerCommand.kt | 4 +- .../src/funTest/kotlin/AskalonoFunTest.kt | 2 +- .../askalono/src/main/kotlin/Askalono.kt | 13 ++-- .../src/funTest/kotlin/BoyterLcFunTest.kt | 2 +- .../boyterlc/src/main/kotlin/BoyterLc.kt | 13 ++-- .../scanners/fossid/src/main/kotlin/FossId.kt | 5 +- .../fossid/src/main/kotlin/FossIdConfig.kt | 35 +++++----- .../src/test/kotlin/FossIdConfigTest.kt | 67 ++++++++----------- .../fossid/src/test/kotlin/FossIdTest.kt | 3 +- .../src/funTest/kotlin/LicenseeFunTest.kt | 2 +- .../licensee/src/main/kotlin/Licensee.kt | 13 ++-- .../funTest/kotlin/ScanCodeScannerFunTest.kt | 2 +- .../scancode/src/main/kotlin/ScanCode.kt | 19 ++---- .../scancode/src/test/kotlin/ScanCodeTest.kt | 42 ++++-------- .../scanoss/src/main/kotlin/ScanOss.kt | 15 ++--- .../scanoss/src/main/kotlin/ScanOssConfig.kt | 9 ++- .../src/test/kotlin/ScanOssConfigTest.kt | 9 +-- .../kotlin/ScanOssScannerDirectoryTest.kt | 6 +- .../src/test/kotlin/ScanOssScannerFileTest.kt | 6 +- .../src/main/kotlin/ScannerWrapperFactory.kt | 8 +-- .../kotlin/storages/ClearlyDefinedStorage.kt | 3 +- 21 files changed, 103 insertions(+), 175 deletions(-) diff --git a/plugins/commands/scanner/src/main/kotlin/ScannerCommand.kt b/plugins/commands/scanner/src/main/kotlin/ScannerCommand.kt index 73793f281ee49..3d7a18fb6361a 100644 --- a/plugins/commands/scanner/src/main/kotlin/ScannerCommand.kt +++ b/plugins/commands/scanner/src/main/kotlin/ScannerCommand.kt @@ -179,10 +179,10 @@ class ScannerCommand : OrtCommand( ): OrtResult { val packageScannerWrappers = scannerWrapperFactories .takeIf { PackageType.PACKAGE in packageTypes }.orEmpty() - .map { it.create(ortConfig.scanner) } + .map { it.create(ortConfig.scanner.options?.get(it.type).orEmpty()) } val projectScannerWrappers = projectScannerWrapperFactories .takeIf { PackageType.PROJECT in packageTypes }.orEmpty() - .map { it.create(ortConfig.scanner) } + .map { it.create(ortConfig.scanner.options?.get(it.type).orEmpty()) } if (projectScannerWrappers.isNotEmpty()) { echo("Scanning projects with:") diff --git a/plugins/scanners/askalono/src/funTest/kotlin/AskalonoFunTest.kt b/plugins/scanners/askalono/src/funTest/kotlin/AskalonoFunTest.kt index ebf493a77c615..9dd4aae581824 100644 --- a/plugins/scanners/askalono/src/funTest/kotlin/AskalonoFunTest.kt +++ b/plugins/scanners/askalono/src/funTest/kotlin/AskalonoFunTest.kt @@ -24,7 +24,7 @@ import org.ossreviewtoolkit.model.TextLocation import org.ossreviewtoolkit.scanner.scanners.AbstractPathScannerWrapperFunTest class AskalonoFunTest : AbstractPathScannerWrapperFunTest() { - override val scanner = Askalono("Askalono", scannerConfig) + override val scanner = Askalono("Askalono", emptyMap()) override val expectedFileLicenses = listOf( LicenseFinding("Apache-2.0", TextLocation("LICENSE", TextLocation.UNKNOWN_LINE), 1.0f) diff --git a/plugins/scanners/askalono/src/main/kotlin/Askalono.kt b/plugins/scanners/askalono/src/main/kotlin/Askalono.kt index 489b51edebddc..0c214fbff981a 100644 --- a/plugins/scanners/askalono/src/main/kotlin/Askalono.kt +++ b/plugins/scanners/askalono/src/main/kotlin/Askalono.kt @@ -32,7 +32,7 @@ import org.ossreviewtoolkit.model.LicenseFinding import org.ossreviewtoolkit.model.ScanSummary import org.ossreviewtoolkit.model.Severity import org.ossreviewtoolkit.model.TextLocation -import org.ossreviewtoolkit.model.config.ScannerConfiguration +import org.ossreviewtoolkit.model.config.Options import org.ossreviewtoolkit.scanner.AbstractScannerWrapperFactory import org.ossreviewtoolkit.scanner.CommandLinePathScannerWrapper import org.ossreviewtoolkit.scanner.ScanContext @@ -44,21 +44,16 @@ private const val CONFIDENCE_NOTICE = "Confidence threshold not high enough for private val JSON = Json { ignoreUnknownKeys = true } -class Askalono internal constructor( - name: String, - private val scannerConfig: ScannerConfiguration -) : CommandLinePathScannerWrapper(name) { +class Askalono internal constructor(name: String, private val options: Options) : CommandLinePathScannerWrapper(name) { private companion object : Logging class Factory : AbstractScannerWrapperFactory("Askalono") { - override fun create(scannerConfig: ScannerConfiguration) = Askalono(type, scannerConfig) + override fun create(options: Options) = Askalono(type, options) } override val configuration = "" - override val criteria by lazy { - ScannerCriteria.create(details, scannerConfig.options?.get(details.name).orEmpty()) - } + override val criteria by lazy { ScannerCriteria.create(details, options) } override fun command(workingDir: File?) = listOfNotNull(workingDir, if (Os.isWindows) "askalono.exe" else "askalono").joinToString(File.separator) diff --git a/plugins/scanners/boyterlc/src/funTest/kotlin/BoyterLcFunTest.kt b/plugins/scanners/boyterlc/src/funTest/kotlin/BoyterLcFunTest.kt index 8e2acc7606811..ba26fe9cee604 100644 --- a/plugins/scanners/boyterlc/src/funTest/kotlin/BoyterLcFunTest.kt +++ b/plugins/scanners/boyterlc/src/funTest/kotlin/BoyterLcFunTest.kt @@ -24,7 +24,7 @@ import org.ossreviewtoolkit.model.TextLocation import org.ossreviewtoolkit.scanner.scanners.AbstractPathScannerWrapperFunTest class BoyterLcFunTest : AbstractPathScannerWrapperFunTest() { - override val scanner = BoyterLc("BoyterLc", scannerConfig) + override val scanner = BoyterLc("BoyterLc", emptyMap()) override val expectedFileLicenses = listOf( LicenseFinding("Apache-2.0", TextLocation("LICENSE", TextLocation.UNKNOWN_LINE), 0.98388565f), diff --git a/plugins/scanners/boyterlc/src/main/kotlin/BoyterLc.kt b/plugins/scanners/boyterlc/src/main/kotlin/BoyterLc.kt index d66e29990aa55..f944194d2ef9c 100644 --- a/plugins/scanners/boyterlc/src/main/kotlin/BoyterLc.kt +++ b/plugins/scanners/boyterlc/src/main/kotlin/BoyterLc.kt @@ -31,7 +31,7 @@ import org.ossreviewtoolkit.model.LicenseFinding import org.ossreviewtoolkit.model.ScanSummary import org.ossreviewtoolkit.model.Severity import org.ossreviewtoolkit.model.TextLocation -import org.ossreviewtoolkit.model.config.ScannerConfiguration +import org.ossreviewtoolkit.model.config.Options import org.ossreviewtoolkit.scanner.AbstractScannerWrapperFactory import org.ossreviewtoolkit.scanner.CommandLinePathScannerWrapper import org.ossreviewtoolkit.scanner.ScanContext @@ -43,10 +43,7 @@ import org.ossreviewtoolkit.utils.ort.createOrtTempDir private val JSON = Json { ignoreUnknownKeys = true } -class BoyterLc internal constructor( - name: String, - private val scannerConfig: ScannerConfiguration -) : CommandLinePathScannerWrapper(name) { +class BoyterLc internal constructor(name: String, private val options: Options) : CommandLinePathScannerWrapper(name) { companion object : Logging { val CONFIGURATION_OPTIONS = listOf( "--confidence", "0.95", // Cut-off value to only get most relevant matches. @@ -55,14 +52,12 @@ class BoyterLc internal constructor( } class Factory : AbstractScannerWrapperFactory("BoyterLc") { - override fun create(scannerConfig: ScannerConfiguration) = BoyterLc(type, scannerConfig) + override fun create(options: Options) = BoyterLc(type, options) } override val configuration = CONFIGURATION_OPTIONS.joinToString(" ") - override val criteria by lazy { - ScannerCriteria.create(details, scannerConfig.options?.get(details.name).orEmpty()) - } + override val criteria by lazy { ScannerCriteria.create(details, options) } override fun command(workingDir: File?) = listOfNotNull(workingDir, if (Os.isWindows) "lc.exe" else "lc").joinToString(File.separator) diff --git a/plugins/scanners/fossid/src/main/kotlin/FossId.kt b/plugins/scanners/fossid/src/main/kotlin/FossId.kt index edba2c776cf10..0615e41fd858f 100644 --- a/plugins/scanners/fossid/src/main/kotlin/FossId.kt +++ b/plugins/scanners/fossid/src/main/kotlin/FossId.kt @@ -96,7 +96,7 @@ import org.ossreviewtoolkit.utils.ort.showStackTrace */ class FossId internal constructor( override val name: String, - private val scannerConfig: ScannerConfiguration, + private val options: Options, private val config: FossIdConfig ) : PackageScannerWrapper { companion object : Logging { @@ -170,8 +170,7 @@ class FossId internal constructor( } class Factory : AbstractScannerWrapperFactory("FossId") { - override fun create(scannerConfig: ScannerConfiguration) = - FossId(type, scannerConfig, FossIdConfig.create(scannerConfig)) + override fun create(options: Options) = FossId(type, options, FossIdConfig.create(options)) } /** diff --git a/plugins/scanners/fossid/src/main/kotlin/FossIdConfig.kt b/plugins/scanners/fossid/src/main/kotlin/FossIdConfig.kt index 76a02a533ed6d..e72d5a413c211 100644 --- a/plugins/scanners/fossid/src/main/kotlin/FossIdConfig.kt +++ b/plugins/scanners/fossid/src/main/kotlin/FossIdConfig.kt @@ -21,11 +21,12 @@ package org.ossreviewtoolkit.plugins.scanners.fossid import org.apache.logging.log4j.kotlin.Logging +import org.ossreviewtoolkit.model.config.Options import org.ossreviewtoolkit.model.config.ScannerConfiguration /** * A data class that holds the configuration options supported by the [FossId] scanner. An instance of this class is - * created from the options contained in a [ScannerConfiguration] object under the key _FossId_. It offers the + * created from the [Options] contained in a [ScannerConfiguration] object under the key _FossId_. It offers the * following configuration options: * * * **"serverUrl":** The URL of the FossID server. @@ -160,32 +161,28 @@ internal data class FossIdConfig( @JvmStatic private val DEFAULT_TIMEOUT = 60 - fun create(scannerConfig: ScannerConfiguration): FossIdConfig { - val fossIdScannerOptions = scannerConfig.options?.get("FossId") + fun create(options: Options): FossIdConfig { + require(options.isNotEmpty()) { "No FossID Scanner configuration found." } - requireNotNull(fossIdScannerOptions) { "No FossID Scanner configuration found." } - - val serverUrl = fossIdScannerOptions[SERVER_URL_PROPERTY] + val serverUrl = options[SERVER_URL_PROPERTY] ?: throw IllegalArgumentException("No FossID server URL configuration found.") - val user = fossIdScannerOptions[USER_PROPERTY] + val user = options[USER_PROPERTY] ?: throw IllegalArgumentException("No FossID User configuration found.") - val apiKey = fossIdScannerOptions[API_KEY_PROPERTY] + val apiKey = options[API_KEY_PROPERTY] ?: throw IllegalArgumentException("No FossID API Key configuration found.") - val waitForResult = fossIdScannerOptions[WAIT_FOR_RESULT_PROPERTY]?.toBoolean() ?: true + val waitForResult = options[WAIT_FOR_RESULT_PROPERTY]?.toBoolean() ?: true - val keepFailedScans = fossIdScannerOptions[KEEP_FAILED_SCANS_PROPERTY]?.toBoolean() ?: false - val deltaScans = fossIdScannerOptions[DELTA_SCAN_PROPERTY]?.toBoolean() ?: false - val deltaScanLimit = fossIdScannerOptions[DELTA_SCAN_LIMIT_PROPERTY]?.toInt() ?: Int.MAX_VALUE + val keepFailedScans = options[KEEP_FAILED_SCANS_PROPERTY]?.toBoolean() ?: false + val deltaScans = options[DELTA_SCAN_PROPERTY]?.toBoolean() ?: false + val deltaScanLimit = options[DELTA_SCAN_LIMIT_PROPERTY]?.toInt() ?: Int.MAX_VALUE - val detectLicenseDeclarations = - fossIdScannerOptions[DETECT_LICENSE_DECLARATIONS_PROPERTY]?.toBoolean() ?: false - val detectCopyrightStatements = - fossIdScannerOptions[DETECT_COPYRIGHT_STATEMENTS_PROPERTY]?.toBoolean() ?: false + val detectLicenseDeclarations = options[DETECT_LICENSE_DECLARATIONS_PROPERTY]?.toBoolean() ?: false + val detectCopyrightStatements = options[DETECT_COPYRIGHT_STATEMENTS_PROPERTY]?.toBoolean() ?: false - val timeout = fossIdScannerOptions[TIMEOUT]?.toInt() ?: DEFAULT_TIMEOUT + val timeout = options[TIMEOUT]?.toInt() ?: DEFAULT_TIMEOUT - val fetchSnippetMatchedLines = fossIdScannerOptions[FETCH_SNIPPET_MATCHED_LINES]?.toBoolean() ?: false + val fetchSnippetMatchedLines = options[FETCH_SNIPPET_MATCHED_LINES]?.toBoolean() ?: false require(deltaScanLimit > 0) { "deltaScanLimit must be > 0, current value is $deltaScanLimit." @@ -205,7 +202,7 @@ internal data class FossIdConfig( detectCopyrightStatements = detectCopyrightStatements, timeout = timeout, fetchSnippetMatchedLines = fetchSnippetMatchedLines, - options = fossIdScannerOptions + options = options ) } } diff --git a/plugins/scanners/fossid/src/test/kotlin/FossIdConfigTest.kt b/plugins/scanners/fossid/src/test/kotlin/FossIdConfigTest.kt index 2951469d6b631..b7f9f9088911a 100644 --- a/plugins/scanners/fossid/src/test/kotlin/FossIdConfigTest.kt +++ b/plugins/scanners/fossid/src/test/kotlin/FossIdConfigTest.kt @@ -33,14 +33,11 @@ import io.kotest.matchers.shouldBe import kotlin.IllegalArgumentException import org.ossreviewtoolkit.clients.fossid.FossIdRestService -import org.ossreviewtoolkit.model.config.ScannerConfiguration class FossIdConfigTest : WordSpec({ "create" should { "throw if no options for FossID are provided in the scanner configuration" { - val scannerConfig = ScannerConfiguration() - - shouldThrow { FossIdConfig.create(scannerConfig) } + shouldThrow { FossIdConfig.create(emptyMap()) } } "read all properties from the scanner configuration" { @@ -57,9 +54,8 @@ class FossIdConfigTest : WordSpec({ "timeout" to "300", "fetchSnippetMatchedLines" to "true" ) - val scannerConfig = options.toScannerConfig() - val fossIdConfig = FossIdConfig.create(scannerConfig) + val fossIdConfig = FossIdConfig.create(options) fossIdConfig shouldBe FossIdConfig( serverUrl = SERVER_URL, @@ -83,9 +79,8 @@ class FossIdConfigTest : WordSpec({ "user" to USER, "apiKey" to API_KEY ) - val scannerConfig = options.toScannerConfig() - val fossIdConfig = FossIdConfig.create(scannerConfig) + val fossIdConfig = FossIdConfig.create(options) fossIdConfig shouldBe FossIdConfig( serverUrl = SERVER_URL, @@ -104,56 +99,56 @@ class FossIdConfigTest : WordSpec({ } "throw if the server URL is missing" { - val scannerConfig = mapOf( + val options = mapOf( "user" to USER, "apiKey" to API_KEY - ).toScannerConfig() + ) - shouldThrow { FossIdConfig.create(scannerConfig) } + shouldThrow { FossIdConfig.create(options) } } "throw if the API key is missing" { - val scannerConfig = mapOf( + val options = mapOf( "serverUrl" to SERVER_URL, "user" to USER - ).toScannerConfig() + ) - shouldThrow { FossIdConfig.create(scannerConfig) } + shouldThrow { FossIdConfig.create(options) } } "throw if the user name is missing" { - val scannerConfig = mapOf( + val options = mapOf( "serverUrl" to SERVER_URL, "apiKey" to API_KEY - ).toScannerConfig() + ) - shouldThrow { FossIdConfig.create(scannerConfig) } + shouldThrow { FossIdConfig.create(options) } } "throw if the deltaScanLimit is invalid" { - val scannerConfig = mapOf( + val options = mapOf( "serverUrl" to SERVER_URL, "user" to USER, "apiKey" to API_KEY, "deltaScanLimit" to "0" - ).toScannerConfig() + ) - shouldThrow { FossIdConfig.create(scannerConfig) } + shouldThrow { FossIdConfig.create(options) } } } "createNamingProvider" should { "create a naming provider with a correct project naming convention" { - val scannerConfig = mapOf( + val options = mapOf( "serverUrl" to SERVER_URL, "user" to USER, "apiKey" to API_KEY, "namingProjectPattern" to "#projectName_\$Org_\$Unit", "namingVariableOrg" to "TestOrganization", "namingVariableUnit" to "TestUnit" - ).toScannerConfig() + ) - val fossIdConfig = FossIdConfig.create(scannerConfig) + val fossIdConfig = FossIdConfig.create(options) val namingProvider = fossIdConfig.createNamingProvider() val projectName = namingProvider.createProjectCode("TestProject") @@ -162,16 +157,16 @@ class FossIdConfigTest : WordSpec({ } "create a naming provider with a correct scan naming convention" { - val scannerConfig = mapOf( + val options = mapOf( "serverUrl" to SERVER_URL, "user" to USER, "apiKey" to API_KEY, "namingScanPattern" to "#projectName_\$Org_\$Unit_#deltaTag", "namingVariableOrg" to "TestOrganization", "namingVariableUnit" to "TestUnit" - ).toScannerConfig() + ) - val fossIdConfig = FossIdConfig.create(scannerConfig) + val fossIdConfig = FossIdConfig.create(options) val namingProvider = fossIdConfig.createNamingProvider() val scanCode = namingProvider.createScanCode("TestProject", FossId.DeltaTag.DELTA) @@ -183,14 +178,14 @@ class FossIdConfigTest : WordSpec({ "createUrlProvider" should { "initialize correct URL mappings" { val url = "https://changeit.example.org/foo" - val scannerConfig = mapOf( + val options = mapOf( "serverUrl" to SERVER_URL, "user" to USER, "apiKey" to API_KEY, "urlMappingChangeHost" to "$url -> $SERVER_URL" - ).toScannerConfig() + ) - val fossIdConfig = FossIdConfig.create(scannerConfig) + val fossIdConfig = FossIdConfig.create(options) val urlProvider = fossIdConfig.createUrlProvider() urlProvider.getUrl(url) shouldBe SERVER_URL @@ -206,11 +201,11 @@ class FossIdConfigTest : WordSpec({ try { val serverUrl = "http://localhost:${server.port()}" - val scannerConfig = mapOf( + val options = mapOf( "serverUrl" to serverUrl, "user" to USER, "apiKey" to API_KEY - ).toScannerConfig() + ) server.stubFor( get(urlPathEqualTo("/index.php")) @@ -221,7 +216,7 @@ class FossIdConfigTest : WordSpec({ ) ) - val fossIdConfig = FossIdConfig.create(scannerConfig) + val fossIdConfig = FossIdConfig.create(options) val service = FossIdRestService.create(fossIdConfig.serverUrl) service.getLoginPage().string() shouldBe loginPage @@ -235,11 +230,3 @@ class FossIdConfigTest : WordSpec({ private const val SERVER_URL = "https://www.example.org/fossid" private const val USER = "fossIdTestUser" private const val API_KEY = "test_api_key" - -/** - * Return a [ScannerConfiguration] with this map as options for the FossID scanner. - */ -private fun Map.toScannerConfig(): ScannerConfiguration { - val options = mapOf("FossId" to this) - return ScannerConfiguration(options = options) -} diff --git a/plugins/scanners/fossid/src/test/kotlin/FossIdTest.kt b/plugins/scanners/fossid/src/test/kotlin/FossIdTest.kt index ea53898302419..77e6856378321 100644 --- a/plugins/scanners/fossid/src/test/kotlin/FossIdTest.kt +++ b/plugins/scanners/fossid/src/test/kotlin/FossIdTest.kt @@ -102,7 +102,6 @@ import org.ossreviewtoolkit.model.VcsType import org.ossreviewtoolkit.model.config.Excludes import org.ossreviewtoolkit.model.config.PathExclude import org.ossreviewtoolkit.model.config.PathExcludeReason -import org.ossreviewtoolkit.model.config.ScannerConfiguration import org.ossreviewtoolkit.plugins.scanners.fossid.FossId.Companion.SCAN_CODE_KEY import org.ossreviewtoolkit.plugins.scanners.fossid.FossId.Companion.SCAN_ID_KEY import org.ossreviewtoolkit.plugins.scanners.fossid.FossId.Companion.SERVER_URL_KEY @@ -1153,7 +1152,7 @@ private val DEFAULT_IGNORE_RULE_SCOPE = RuleScope.SCAN /** * Create a new [FossId] instance with the specified [config]. */ -private fun createFossId(config: FossIdConfig): FossId = FossId("FossId", ScannerConfiguration(), config) +private fun createFossId(config: FossIdConfig): FossId = FossId("FossId", emptyMap(), config) /** * Create a standard [FossIdConfig] whose properties can be partly specified. diff --git a/plugins/scanners/licensee/src/funTest/kotlin/LicenseeFunTest.kt b/plugins/scanners/licensee/src/funTest/kotlin/LicenseeFunTest.kt index d9f428aeeaa93..d852b8d54b498 100644 --- a/plugins/scanners/licensee/src/funTest/kotlin/LicenseeFunTest.kt +++ b/plugins/scanners/licensee/src/funTest/kotlin/LicenseeFunTest.kt @@ -25,7 +25,7 @@ import org.ossreviewtoolkit.scanner.scanners.AbstractPathScannerWrapperFunTest import org.ossreviewtoolkit.utils.test.ExpensiveTag class LicenseeFunTest : AbstractPathScannerWrapperFunTest(setOf(ExpensiveTag)) { - override val scanner = Licensee("Licensee", scannerConfig) + override val scanner = Licensee("Licensee", emptyMap()) override val expectedFileLicenses = listOf( LicenseFinding("Apache-2.0", TextLocation("LICENSE", TextLocation.UNKNOWN_LINE), 100.0f) diff --git a/plugins/scanners/licensee/src/main/kotlin/Licensee.kt b/plugins/scanners/licensee/src/main/kotlin/Licensee.kt index 09846cd4b3b2e..ddca072b15b27 100644 --- a/plugins/scanners/licensee/src/main/kotlin/Licensee.kt +++ b/plugins/scanners/licensee/src/main/kotlin/Licensee.kt @@ -32,7 +32,7 @@ import org.ossreviewtoolkit.model.LicenseFinding import org.ossreviewtoolkit.model.ScanSummary import org.ossreviewtoolkit.model.Severity import org.ossreviewtoolkit.model.TextLocation -import org.ossreviewtoolkit.model.config.ScannerConfiguration +import org.ossreviewtoolkit.model.config.Options import org.ossreviewtoolkit.scanner.AbstractScannerWrapperFactory import org.ossreviewtoolkit.scanner.CommandLinePathScannerWrapper import org.ossreviewtoolkit.scanner.ScanContext @@ -45,23 +45,18 @@ private val JSON = Json { namingStrategy = JsonNamingStrategy.SnakeCase } -class Licensee internal constructor( - name: String, - private val scannerConfig: ScannerConfiguration -) : CommandLinePathScannerWrapper(name) { +class Licensee internal constructor(name: String, private val options: Options) : CommandLinePathScannerWrapper(name) { companion object : Logging { val CONFIGURATION_OPTIONS = listOf("--json") } class Factory : AbstractScannerWrapperFactory("Licensee") { - override fun create(scannerConfig: ScannerConfiguration) = Licensee(type, scannerConfig) + override fun create(options: Options) = Licensee(type, options) } override val configuration = CONFIGURATION_OPTIONS.joinToString(" ") - override val criteria by lazy { - ScannerCriteria.create(details, scannerConfig.options?.get(details.name).orEmpty()) - } + override val criteria by lazy { ScannerCriteria.create(details, options) } override fun command(workingDir: File?) = listOfNotNull(workingDir, if (Os.isWindows) "licensee.bat" else "licensee").joinToString(File.separator) diff --git a/plugins/scanners/scancode/src/funTest/kotlin/ScanCodeScannerFunTest.kt b/plugins/scanners/scancode/src/funTest/kotlin/ScanCodeScannerFunTest.kt index 8549350a92fc6..8db3e3430ddb7 100644 --- a/plugins/scanners/scancode/src/funTest/kotlin/ScanCodeScannerFunTest.kt +++ b/plugins/scanners/scancode/src/funTest/kotlin/ScanCodeScannerFunTest.kt @@ -32,7 +32,7 @@ import org.ossreviewtoolkit.utils.spdx.getLicenseText import org.ossreviewtoolkit.utils.test.ExpensiveTag class ScanCodeScannerFunTest : AbstractPathScannerWrapperFunTest(setOf(ExpensiveTag)) { - override val scanner = ScanCode("ScanCode", scannerConfig) + override val scanner = ScanCode("ScanCode", emptyMap()) override val expectedFileLicenses = listOf( LicenseFinding("Apache-2.0", TextLocation("LICENSE", 1, 187), 100.0f), diff --git a/plugins/scanners/scancode/src/main/kotlin/ScanCode.kt b/plugins/scanners/scancode/src/main/kotlin/ScanCode.kt index e975659150129..66465f5f1895b 100644 --- a/plugins/scanners/scancode/src/main/kotlin/ScanCode.kt +++ b/plugins/scanners/scancode/src/main/kotlin/ScanCode.kt @@ -28,6 +28,7 @@ import org.apache.logging.log4j.kotlin.Logging import org.ossreviewtoolkit.model.ScanSummary import org.ossreviewtoolkit.model.ScannerDetails +import org.ossreviewtoolkit.model.config.Options import org.ossreviewtoolkit.model.config.ScannerConfiguration import org.ossreviewtoolkit.scanner.AbstractScannerWrapperFactory import org.ossreviewtoolkit.scanner.CommandLinePathScannerWrapper @@ -56,10 +57,7 @@ import org.semver4j.Semver * * **"commandLineNonConfig":** Command line options that do not modify the result and should therefore not be * considered in [configuration], like "--processes". Defaults to [DEFAULT_NON_CONFIGURATION_OPTIONS]. */ -class ScanCode internal constructor( - name: String, - private val scannerConfig: ScannerConfiguration -) : CommandLinePathScannerWrapper(name) { +class ScanCode internal constructor(name: String, private val options: Options) : CommandLinePathScannerWrapper(name) { companion object : Logging { const val SCANNER_NAME = "ScanCode" @@ -94,12 +92,10 @@ class ScanCode internal constructor( } class Factory : AbstractScannerWrapperFactory(SCANNER_NAME) { - override fun create(scannerConfig: ScannerConfiguration) = ScanCode(type, scannerConfig) + override fun create(options: Options) = ScanCode(type, options) } - override val criteria by lazy { - ScannerCriteria.create(details, scannerConfig.options?.get(details.name).orEmpty()) - } + override val criteria by lazy { ScannerCriteria.create(details, options) } override val configuration by lazy { buildList { @@ -108,11 +104,8 @@ class ScanCode internal constructor( }.joinToString(" ") } - private val scanCodeConfiguration = scannerConfig.options?.get("ScanCode").orEmpty() - - private val configurationOptions = scanCodeConfiguration["commandLine"]?.splitOnWhitespace() - ?: DEFAULT_CONFIGURATION_OPTIONS - private val nonConfigurationOptions = scanCodeConfiguration["commandLineNonConfig"]?.splitOnWhitespace() + private val configurationOptions = options["commandLine"]?.splitOnWhitespace() ?: DEFAULT_CONFIGURATION_OPTIONS + private val nonConfigurationOptions = options["commandLineNonConfig"]?.splitOnWhitespace() ?: DEFAULT_NON_CONFIGURATION_OPTIONS internal fun getCommandLineOptions(version: String) = diff --git a/plugins/scanners/scancode/src/test/kotlin/ScanCodeTest.kt b/plugins/scanners/scancode/src/test/kotlin/ScanCodeTest.kt index 3c7fff1c0e61b..3c1bbf3c92abd 100644 --- a/plugins/scanners/scancode/src/test/kotlin/ScanCodeTest.kt +++ b/plugins/scanners/scancode/src/test/kotlin/ScanCodeTest.kt @@ -34,12 +34,11 @@ import io.mockk.spyk import java.io.File import org.ossreviewtoolkit.model.PackageType -import org.ossreviewtoolkit.model.config.ScannerConfiguration import org.ossreviewtoolkit.scanner.ScanContext import org.ossreviewtoolkit.utils.common.ProcessCapture class ScanCodeTest : WordSpec({ - val scanner = ScanCode("ScanCode", ScannerConfiguration()) + val scanner = ScanCode("ScanCode", emptyMap()) "configuration" should { "return the default values if the scanner configuration is empty" { @@ -49,13 +48,9 @@ class ScanCodeTest : WordSpec({ "return the non-config values from the scanner configuration" { val scannerWithConfig = ScanCode( "ScanCode", - ScannerConfiguration( - options = mapOf( - "ScanCode" to mapOf( - "commandLine" to "--command --line", - "commandLineNonConfig" to "--commandLineNonConfig" - ) - ) + mapOf( + "commandLine" to "--command --line", + "commandLineNonConfig" to "--commandLineNonConfig" ) ) @@ -74,13 +69,9 @@ class ScanCodeTest : WordSpec({ "contain the values from the scanner configuration" { val scannerWithConfig = ScanCode( "ScanCode", - ScannerConfiguration( - options = mapOf( - "ScanCode" to mapOf( - "commandLine" to "--command --line", - "commandLineNonConfig" to "--commandLineNonConfig" - ) - ) + mapOf( + "commandLine" to "--command --line", + "commandLineNonConfig" to "--commandLineNonConfig" ) ) @@ -91,13 +82,9 @@ class ScanCodeTest : WordSpec({ "be handled correctly when containing multiple spaces" { val scannerWithConfig = ScanCode( "ScanCode", - ScannerConfiguration( - options = mapOf( - "ScanCode" to mapOf( - "commandLine" to " --command --line ", - "commandLineNonConfig" to " -n -c " - ) - ) + mapOf( + "commandLine" to " --command --line ", + "commandLineNonConfig" to " -n -c " ) ) @@ -132,13 +119,8 @@ class ScanCodeTest : WordSpec({ } "transformVersion()" should { - val scanCode = ScanCode( - "ScanCode", - ScannerConfiguration() - ) - "work with a version output without a colon" { - scanCode.transformVersion( + scanner.transformVersion( """ ScanCode version 30.0.1 ScanCode Output Format version 1.0.0 @@ -148,7 +130,7 @@ class ScanCodeTest : WordSpec({ } "work with a version output with a colon" { - scanCode.transformVersion( + scanner.transformVersion( """ ScanCode version: 31.0.0b4 ScanCode Output Format version: 2.0.0 diff --git a/plugins/scanners/scanoss/src/main/kotlin/ScanOss.kt b/plugins/scanners/scanoss/src/main/kotlin/ScanOss.kt index 59849ea207cb8..868a4bfc8a50c 100644 --- a/plugins/scanners/scanoss/src/main/kotlin/ScanOss.kt +++ b/plugins/scanners/scanoss/src/main/kotlin/ScanOss.kt @@ -38,7 +38,7 @@ import org.apache.logging.log4j.kotlin.Logging import org.ossreviewtoolkit.clients.scanoss.ScanOssService import org.ossreviewtoolkit.model.ScanSummary -import org.ossreviewtoolkit.model.config.ScannerConfiguration +import org.ossreviewtoolkit.model.config.Options import org.ossreviewtoolkit.scanner.AbstractScannerWrapperFactory import org.ossreviewtoolkit.scanner.PathScannerWrapper import org.ossreviewtoolkit.scanner.ScanContext @@ -49,17 +49,14 @@ import org.ossreviewtoolkit.utils.common.VCS_DIRECTORIES private const val FAKE_WFP_FILE_NAME = "fake.wfp" private const val ARG_FIELD_NAME = "file" -class ScanOss internal constructor( - override val name: String, - private val scannerConfig: ScannerConfiguration -) : PathScannerWrapper { +class ScanOss internal constructor(override val name: String, private val options: Options) : PathScannerWrapper { private companion object : Logging class Factory : AbstractScannerWrapperFactory("SCANOSS") { - override fun create(scannerConfig: ScannerConfiguration) = ScanOss(type, scannerConfig) + override fun create(options: Options) = ScanOss(type, options) } - private val config = ScanOssConfig.create(scannerConfig).also { + private val config = ScanOssConfig.create(options).also { logger.info { "The $name API URL is ${it.apiUrl}." } } @@ -74,9 +71,7 @@ class ScanOss internal constructor( override val configuration = "" - override val criteria by lazy { - ScannerCriteria.create(details, scannerConfig.options?.get(details.name).orEmpty()) - } + override val criteria by lazy { ScannerCriteria.create(details, options) } /** * The name of the file corresponding to the fingerprints can be sent to SCANOSS for more precise matches. diff --git a/plugins/scanners/scanoss/src/main/kotlin/ScanOssConfig.kt b/plugins/scanners/scanoss/src/main/kotlin/ScanOssConfig.kt index 0c2818fd110fa..46317ed7b3951 100644 --- a/plugins/scanners/scanoss/src/main/kotlin/ScanOssConfig.kt +++ b/plugins/scanners/scanoss/src/main/kotlin/ScanOssConfig.kt @@ -20,6 +20,7 @@ package org.ossreviewtoolkit.plugins.scanners.scanoss import org.ossreviewtoolkit.clients.scanoss.ScanOssService +import org.ossreviewtoolkit.model.config.Options import org.ossreviewtoolkit.model.config.ScannerConfiguration /** @@ -41,11 +42,9 @@ internal data class ScanOssConfig( /** Name of the configuration property for the API key. */ const val API_KEY_PROPERTY = "apiKey" - fun create(scannerConfig: ScannerConfiguration): ScanOssConfig { - val scanOssOptions = scannerConfig.options?.get("ScanOss") - - val apiUrl = scanOssOptions?.get(API_URL_PROPERTY) ?: ScanOssService.DEFAULT_API_URL - val apiKey = scanOssOptions?.get(API_KEY_PROPERTY).orEmpty() + fun create(options: Options): ScanOssConfig { + val apiUrl = options[API_URL_PROPERTY] ?: ScanOssService.DEFAULT_API_URL + val apiKey = options[API_KEY_PROPERTY].orEmpty() return ScanOssConfig(apiUrl, apiKey) } diff --git a/plugins/scanners/scanoss/src/test/kotlin/ScanOssConfigTest.kt b/plugins/scanners/scanoss/src/test/kotlin/ScanOssConfigTest.kt index ddbd02f617e15..386e87fa74f64 100644 --- a/plugins/scanners/scanoss/src/test/kotlin/ScanOssConfigTest.kt +++ b/plugins/scanners/scanoss/src/test/kotlin/ScanOssConfigTest.kt @@ -25,25 +25,22 @@ import io.kotest.matchers.shouldBe import io.kotest.matchers.string.beEmpty import org.ossreviewtoolkit.clients.scanoss.ScanOssService -import org.ossreviewtoolkit.model.config.ScannerConfiguration class ScanOssConfigTest : StringSpec({ "Default values are used" { - with(ScanOssConfig.create(ScannerConfiguration())) { + with(ScanOssConfig.create(emptyMap())) { apiUrl shouldBe ScanOssService.DEFAULT_API_URL apiKey should beEmpty() } } "Default values can be overridden" { - val scanOssOptions = mapOf( + val options = mapOf( ScanOssConfig.API_URL_PROPERTY to "url", ScanOssConfig.API_KEY_PROPERTY to "key" ) - val config = ScannerConfiguration(options = mapOf("ScanOss" to scanOssOptions)) - - with(ScanOssConfig.create(config)) { + with(ScanOssConfig.create(options)) { apiUrl shouldBe "url" apiKey shouldBe "key" } diff --git a/plugins/scanners/scanoss/src/test/kotlin/ScanOssScannerDirectoryTest.kt b/plugins/scanners/scanoss/src/test/kotlin/ScanOssScannerDirectoryTest.kt index 26782d764f1c9..f08cde25ad378 100644 --- a/plugins/scanners/scanoss/src/test/kotlin/ScanOssScannerDirectoryTest.kt +++ b/plugins/scanners/scanoss/src/test/kotlin/ScanOssScannerDirectoryTest.kt @@ -42,7 +42,6 @@ import org.ossreviewtoolkit.model.SnippetFinding import org.ossreviewtoolkit.model.TextLocation import org.ossreviewtoolkit.model.VcsInfo import org.ossreviewtoolkit.model.VcsType -import org.ossreviewtoolkit.model.config.ScannerConfiguration import org.ossreviewtoolkit.scanner.ScanContext import org.ossreviewtoolkit.utils.spdx.SpdxExpression @@ -62,9 +61,8 @@ class ScanOssScannerDirectoryTest : StringSpec({ beforeSpec { server.start() - val scannerOptions = mapOf(ScanOssConfig.API_URL_PROPERTY to "http://localhost:${server.port()}") - val configuration = ScannerConfiguration(options = mapOf("ScanOss" to scannerOptions)) - scanner = spyk(ScanOss.Factory().create(configuration)) + val options = mapOf(ScanOssConfig.API_URL_PROPERTY to "http://localhost:${server.port()}") + scanner = spyk(ScanOss.Factory().create(options)) } afterSpec { diff --git a/plugins/scanners/scanoss/src/test/kotlin/ScanOssScannerFileTest.kt b/plugins/scanners/scanoss/src/test/kotlin/ScanOssScannerFileTest.kt index 2ca262979bedc..aecfc7a46fd44 100644 --- a/plugins/scanners/scanoss/src/test/kotlin/ScanOssScannerFileTest.kt +++ b/plugins/scanners/scanoss/src/test/kotlin/ScanOssScannerFileTest.kt @@ -36,7 +36,6 @@ import java.util.UUID import org.ossreviewtoolkit.model.LicenseFinding import org.ossreviewtoolkit.model.PackageType import org.ossreviewtoolkit.model.TextLocation -import org.ossreviewtoolkit.model.config.ScannerConfiguration import org.ossreviewtoolkit.scanner.ScanContext private val TEST_FILE_TO_SCAN = File("src/test/assets/filesToScan/ScannerFactory.kt") @@ -55,9 +54,8 @@ class ScanOssScannerFileTest : StringSpec({ beforeSpec { server.start() - val scannerOptions = mapOf(ScanOssConfig.API_URL_PROPERTY to "http://localhost:${server.port()}") - val configuration = ScannerConfiguration(options = mapOf("ScanOss" to scannerOptions)) - scanner = spyk(ScanOss.Factory().create(configuration)) + val options = mapOf(ScanOssConfig.API_URL_PROPERTY to "http://localhost:${server.port()}") + scanner = spyk(ScanOss.Factory().create(options)) } afterSpec { diff --git a/scanner/src/main/kotlin/ScannerWrapperFactory.kt b/scanner/src/main/kotlin/ScannerWrapperFactory.kt index 451f7aa4b6060..8c3682d302012 100644 --- a/scanner/src/main/kotlin/ScannerWrapperFactory.kt +++ b/scanner/src/main/kotlin/ScannerWrapperFactory.kt @@ -21,7 +21,7 @@ package org.ossreviewtoolkit.scanner import java.util.ServiceLoader -import org.ossreviewtoolkit.model.config.ScannerConfiguration +import org.ossreviewtoolkit.model.config.Options import org.ossreviewtoolkit.utils.common.Plugin /** @@ -29,9 +29,9 @@ import org.ossreviewtoolkit.utils.common.Plugin */ interface ScannerWrapperFactory : Plugin { /** - * Create a [ScannerWrapper] using the specified [scannerConfig]. + * Create a [ScannerWrapper] using the provided [options]. */ - fun create(scannerConfig: ScannerConfiguration): ScannerWrapper + fun create(options: Options): ScannerWrapper } /** @@ -40,7 +40,7 @@ interface ScannerWrapperFactory : Plugin { abstract class AbstractScannerWrapperFactory( override val type: String ) : ScannerWrapperFactory { - abstract override fun create(scannerConfig: ScannerConfiguration): T + abstract override fun create(options: Options): T /** * Return the scanner wrapper's name here to allow Clikt to display something meaningful when listing the scanners diff --git a/scanner/src/main/kotlin/storages/ClearlyDefinedStorage.kt b/scanner/src/main/kotlin/storages/ClearlyDefinedStorage.kt index e4f425c61f92a..3dc6b18fa0f9b 100644 --- a/scanner/src/main/kotlin/storages/ClearlyDefinedStorage.kt +++ b/scanner/src/main/kotlin/storages/ClearlyDefinedStorage.kt @@ -47,7 +47,6 @@ import org.ossreviewtoolkit.model.UnknownProvenance import org.ossreviewtoolkit.model.VcsInfo import org.ossreviewtoolkit.model.VcsType import org.ossreviewtoolkit.model.config.ClearlyDefinedStorageConfiguration -import org.ossreviewtoolkit.model.config.ScannerConfiguration import org.ossreviewtoolkit.model.jsonMapper import org.ossreviewtoolkit.model.utils.toClearlyDefinedCoordinates import org.ossreviewtoolkit.model.utils.toClearlyDefinedSourceLocation @@ -125,7 +124,7 @@ class ClearlyDefinedStorage( val supportedScanners = toolVersionsByName.mapNotNull { (name, versions) -> // For the ClearlyDefined tool names see https://github.com/clearlydefined/service#tool-name-registry. ScannerWrapper.ALL[name]?.let { factory -> - val scanner = factory.create(ScannerConfiguration()) + val scanner = factory.create(emptyMap()) (scanner as? CommandLinePathScannerWrapper)?.let { cliScanner -> cliScanner to versions.last() } }.also { if (it == null) logger.debug { "Unsupported tool '$name' for coordinates '$coordinates'." } From eb2efd6b99660ed9d43a3d32aba893c5c6427be4 Mon Sep 17 00:00:00 2001 From: Martin Nonnenmacher Date: Tue, 26 Sep 2023 15:10:01 +0200 Subject: [PATCH 24/47] feat(plugins): Add the `TypedConfigurablePluginFactory` Extract an interface from the duplicated code in `PackageCurationProviderFactory` and `PackageConfigurationProviderFactory` to make it reusable by other plugins. Signed-off-by: Martin Nonnenmacher --- .../PackageConfigurationProviderFactory.kt | 17 +++-------------- .../kotlin/PackageCurationProviderFactory.kt | 16 ++-------------- utils/common/src/main/kotlin/PluginManager.kt | 19 +++++++++++++++++++ 3 files changed, 24 insertions(+), 28 deletions(-) diff --git a/plugins/package-configuration-providers/api/src/main/kotlin/PackageConfigurationProviderFactory.kt b/plugins/package-configuration-providers/api/src/main/kotlin/PackageConfigurationProviderFactory.kt index 0de148922d352..71ad26fc436a6 100644 --- a/plugins/package-configuration-providers/api/src/main/kotlin/PackageConfigurationProviderFactory.kt +++ b/plugins/package-configuration-providers/api/src/main/kotlin/PackageConfigurationProviderFactory.kt @@ -21,14 +21,15 @@ package org.ossreviewtoolkit.plugins.packageconfigurationproviders.api import org.ossreviewtoolkit.model.config.ProviderPluginConfiguration import org.ossreviewtoolkit.model.utils.PackageConfigurationProvider -import org.ossreviewtoolkit.utils.common.ConfigurablePluginFactory import org.ossreviewtoolkit.utils.common.Plugin +import org.ossreviewtoolkit.utils.common.TypedConfigurablePluginFactory import org.ossreviewtoolkit.utils.common.getDuplicates /** * The extension point for [PackageConfigurationProvider]s. */ -interface PackageConfigurationProviderFactory : ConfigurablePluginFactory { +interface PackageConfigurationProviderFactory : + TypedConfigurablePluginFactory { companion object { val ALL = Plugin.getAll>() @@ -56,16 +57,4 @@ interface PackageConfigurationProviderFactory : ConfigurablePluginFactor } } } - - override fun create(config: Map): PackageConfigurationProvider = create(parseConfig(config)) - - /** - * Create a new [PackageConfigurationProvider] with [config]. - */ - fun create(config: CONFIG): PackageConfigurationProvider - - /** - * Parse the [config] map into a [CONFIG] instance. - */ - fun parseConfig(config: Map): CONFIG } diff --git a/plugins/package-curation-providers/api/src/main/kotlin/PackageCurationProviderFactory.kt b/plugins/package-curation-providers/api/src/main/kotlin/PackageCurationProviderFactory.kt index 00683bdf15d8c..0e25fc079a0ce 100644 --- a/plugins/package-curation-providers/api/src/main/kotlin/PackageCurationProviderFactory.kt +++ b/plugins/package-curation-providers/api/src/main/kotlin/PackageCurationProviderFactory.kt @@ -22,14 +22,14 @@ package org.ossreviewtoolkit.plugins.packagecurationproviders.api import org.ossreviewtoolkit.model.ResolvedPackageCurations.Companion.REPOSITORY_CONFIGURATION_PROVIDER_ID import org.ossreviewtoolkit.model.config.ProviderPluginConfiguration import org.ossreviewtoolkit.model.utils.PackageCurationProvider -import org.ossreviewtoolkit.utils.common.ConfigurablePluginFactory import org.ossreviewtoolkit.utils.common.Plugin +import org.ossreviewtoolkit.utils.common.TypedConfigurablePluginFactory import org.ossreviewtoolkit.utils.common.getDuplicates /** * The extension point for [PackageCurationProvider]s. */ -interface PackageCurationProviderFactory : ConfigurablePluginFactory { +interface PackageCurationProviderFactory : TypedConfigurablePluginFactory { companion object { val ALL = Plugin.getAll>() @@ -60,16 +60,4 @@ interface PackageCurationProviderFactory : ConfigurablePluginFactory): PackageCurationProvider = create(parseConfig(config)) - - /** - * Create a new [PackageCurationProvider] with [config]. - */ - fun create(config: CONFIG): PackageCurationProvider - - /** - * Parse the [config] map into an object. - */ - fun parseConfig(config: Map): CONFIG } diff --git a/utils/common/src/main/kotlin/PluginManager.kt b/utils/common/src/main/kotlin/PluginManager.kt index 7a42574b08286..8f05623d08762 100644 --- a/utils/common/src/main/kotlin/PluginManager.kt +++ b/utils/common/src/main/kotlin/PluginManager.kt @@ -65,3 +65,22 @@ interface ConfigurablePluginFactory : Plugin { */ fun create(config: Map): PLUGIN } + +/** + * An interface to be implemented by [configurable plugin factories][ConfigurablePluginFactory] that provide a + * [typed configuration class][CONFIG]. The benefit of implementing this interface over [ConfigurablePluginFactory] is + * that it enforces the separation of parsing the config map and creating the plugin. + */ +interface TypedConfigurablePluginFactory : ConfigurablePluginFactory { + override fun create(config: Map): PLUGIN = create(parseConfig(config)) + + /** + * Create a new instance of [PLUGIN] from [config]. + */ + fun create(config: CONFIG): PLUGIN + + /** + * Parse the [config] map into a [CONFIG] object. + */ + fun parseConfig(config: Map): CONFIG +} From f5e00467be97a438ec27036ce8819bb76f265b76 Mon Sep 17 00:00:00 2001 From: Martin Nonnenmacher Date: Tue, 26 Sep 2023 15:41:58 +0200 Subject: [PATCH 25/47] refactor(scanner): Consolidate the API for scanner wrapper factories Merge the `ScannerWrapperFactory` interface and the `AbstractScannerWrapperFactory` abstract class into a single abstract class to simplify code. All implementations already inherited from the abstract class, and the `toString` implementation from the abstract class should also be supported by all scanner wrapper factories. Signed-off-by: Martin Nonnenmacher --- .../scanner/src/main/kotlin/ScannerCommand.kt | 4 ++-- .../askalono/src/main/kotlin/Askalono.kt | 4 ++-- .../boyterlc/src/main/kotlin/BoyterLc.kt | 4 ++-- .../scanners/fossid/src/main/kotlin/FossId.kt | 4 ++-- .../licensee/src/main/kotlin/Licensee.kt | 4 ++-- .../scancode/src/main/kotlin/ScanCode.kt | 4 ++-- .../scanners/scanoss/src/main/kotlin/ScanOss.kt | 4 ++-- scanner/src/main/kotlin/ScannerWrapper.kt | 2 +- .../src/main/kotlin/ScannerWrapperFactory.kt | 17 ++++------------- 9 files changed, 19 insertions(+), 28 deletions(-) diff --git a/plugins/commands/scanner/src/main/kotlin/ScannerCommand.kt b/plugins/commands/scanner/src/main/kotlin/ScannerCommand.kt index 3d7a18fb6361a..b64b5d1ebd445 100644 --- a/plugins/commands/scanner/src/main/kotlin/ScannerCommand.kt +++ b/plugins/commands/scanner/src/main/kotlin/ScannerCommand.kt @@ -173,8 +173,8 @@ class ScannerCommand : OrtCommand( } private fun runScanners( - scannerWrapperFactories: List, - projectScannerWrapperFactories: List, + scannerWrapperFactories: List>, + projectScannerWrapperFactories: List>, ortConfig: OrtConfiguration ): OrtResult { val packageScannerWrappers = scannerWrapperFactories diff --git a/plugins/scanners/askalono/src/main/kotlin/Askalono.kt b/plugins/scanners/askalono/src/main/kotlin/Askalono.kt index 0c214fbff981a..1a3f904e5d81c 100644 --- a/plugins/scanners/askalono/src/main/kotlin/Askalono.kt +++ b/plugins/scanners/askalono/src/main/kotlin/Askalono.kt @@ -33,11 +33,11 @@ import org.ossreviewtoolkit.model.ScanSummary import org.ossreviewtoolkit.model.Severity import org.ossreviewtoolkit.model.TextLocation import org.ossreviewtoolkit.model.config.Options -import org.ossreviewtoolkit.scanner.AbstractScannerWrapperFactory import org.ossreviewtoolkit.scanner.CommandLinePathScannerWrapper import org.ossreviewtoolkit.scanner.ScanContext import org.ossreviewtoolkit.scanner.ScanException import org.ossreviewtoolkit.scanner.ScannerCriteria +import org.ossreviewtoolkit.scanner.ScannerWrapperFactory import org.ossreviewtoolkit.utils.common.Os private const val CONFIDENCE_NOTICE = "Confidence threshold not high enough for any known license" @@ -47,7 +47,7 @@ private val JSON = Json { ignoreUnknownKeys = true } class Askalono internal constructor(name: String, private val options: Options) : CommandLinePathScannerWrapper(name) { private companion object : Logging - class Factory : AbstractScannerWrapperFactory("Askalono") { + class Factory : ScannerWrapperFactory("Askalono") { override fun create(options: Options) = Askalono(type, options) } diff --git a/plugins/scanners/boyterlc/src/main/kotlin/BoyterLc.kt b/plugins/scanners/boyterlc/src/main/kotlin/BoyterLc.kt index f944194d2ef9c..22e6151932a93 100644 --- a/plugins/scanners/boyterlc/src/main/kotlin/BoyterLc.kt +++ b/plugins/scanners/boyterlc/src/main/kotlin/BoyterLc.kt @@ -32,11 +32,11 @@ import org.ossreviewtoolkit.model.ScanSummary import org.ossreviewtoolkit.model.Severity import org.ossreviewtoolkit.model.TextLocation import org.ossreviewtoolkit.model.config.Options -import org.ossreviewtoolkit.scanner.AbstractScannerWrapperFactory import org.ossreviewtoolkit.scanner.CommandLinePathScannerWrapper import org.ossreviewtoolkit.scanner.ScanContext import org.ossreviewtoolkit.scanner.ScanException import org.ossreviewtoolkit.scanner.ScannerCriteria +import org.ossreviewtoolkit.scanner.ScannerWrapperFactory import org.ossreviewtoolkit.utils.common.Os import org.ossreviewtoolkit.utils.common.safeDeleteRecursively import org.ossreviewtoolkit.utils.ort.createOrtTempDir @@ -51,7 +51,7 @@ class BoyterLc internal constructor(name: String, private val options: Options) ) } - class Factory : AbstractScannerWrapperFactory("BoyterLc") { + class Factory : ScannerWrapperFactory("BoyterLc") { override fun create(options: Options) = BoyterLc(type, options) } diff --git a/plugins/scanners/fossid/src/main/kotlin/FossId.kt b/plugins/scanners/fossid/src/main/kotlin/FossId.kt index 0615e41fd858f..84d52ea93eeeb 100644 --- a/plugins/scanners/fossid/src/main/kotlin/FossId.kt +++ b/plugins/scanners/fossid/src/main/kotlin/FossId.kt @@ -75,11 +75,11 @@ import org.ossreviewtoolkit.model.VcsType import org.ossreviewtoolkit.model.config.Options import org.ossreviewtoolkit.model.config.ScannerConfiguration import org.ossreviewtoolkit.model.createAndLogIssue -import org.ossreviewtoolkit.scanner.AbstractScannerWrapperFactory import org.ossreviewtoolkit.scanner.PackageScannerWrapper import org.ossreviewtoolkit.scanner.ProvenanceScannerWrapper import org.ossreviewtoolkit.scanner.ScanContext import org.ossreviewtoolkit.scanner.ScannerCriteria +import org.ossreviewtoolkit.scanner.ScannerWrapperFactory import org.ossreviewtoolkit.utils.common.enumSetOf import org.ossreviewtoolkit.utils.common.replaceCredentialsInUri import org.ossreviewtoolkit.utils.ort.showStackTrace @@ -169,7 +169,7 @@ class FossId internal constructor( ) } - class Factory : AbstractScannerWrapperFactory("FossId") { + class Factory : ScannerWrapperFactory("FossId") { override fun create(options: Options) = FossId(type, options, FossIdConfig.create(options)) } diff --git a/plugins/scanners/licensee/src/main/kotlin/Licensee.kt b/plugins/scanners/licensee/src/main/kotlin/Licensee.kt index ddca072b15b27..b0cefefdc4a09 100644 --- a/plugins/scanners/licensee/src/main/kotlin/Licensee.kt +++ b/plugins/scanners/licensee/src/main/kotlin/Licensee.kt @@ -33,11 +33,11 @@ import org.ossreviewtoolkit.model.ScanSummary import org.ossreviewtoolkit.model.Severity import org.ossreviewtoolkit.model.TextLocation import org.ossreviewtoolkit.model.config.Options -import org.ossreviewtoolkit.scanner.AbstractScannerWrapperFactory import org.ossreviewtoolkit.scanner.CommandLinePathScannerWrapper import org.ossreviewtoolkit.scanner.ScanContext import org.ossreviewtoolkit.scanner.ScanException import org.ossreviewtoolkit.scanner.ScannerCriteria +import org.ossreviewtoolkit.scanner.ScannerWrapperFactory import org.ossreviewtoolkit.utils.common.Os private val JSON = Json { @@ -50,7 +50,7 @@ class Licensee internal constructor(name: String, private val options: Options) val CONFIGURATION_OPTIONS = listOf("--json") } - class Factory : AbstractScannerWrapperFactory("Licensee") { + class Factory : ScannerWrapperFactory("Licensee") { override fun create(options: Options) = Licensee(type, options) } diff --git a/plugins/scanners/scancode/src/main/kotlin/ScanCode.kt b/plugins/scanners/scancode/src/main/kotlin/ScanCode.kt index 66465f5f1895b..1c873a22ea6ec 100644 --- a/plugins/scanners/scancode/src/main/kotlin/ScanCode.kt +++ b/plugins/scanners/scancode/src/main/kotlin/ScanCode.kt @@ -30,11 +30,11 @@ import org.ossreviewtoolkit.model.ScanSummary import org.ossreviewtoolkit.model.ScannerDetails import org.ossreviewtoolkit.model.config.Options import org.ossreviewtoolkit.model.config.ScannerConfiguration -import org.ossreviewtoolkit.scanner.AbstractScannerWrapperFactory import org.ossreviewtoolkit.scanner.CommandLinePathScannerWrapper import org.ossreviewtoolkit.scanner.ScanContext import org.ossreviewtoolkit.scanner.ScanResultsStorage import org.ossreviewtoolkit.scanner.ScannerCriteria +import org.ossreviewtoolkit.scanner.ScannerWrapperFactory import org.ossreviewtoolkit.utils.common.Os import org.ossreviewtoolkit.utils.common.ProcessCapture import org.ossreviewtoolkit.utils.common.safeDeleteRecursively @@ -91,7 +91,7 @@ class ScanCode internal constructor(name: String, private val options: Options) } } - class Factory : AbstractScannerWrapperFactory(SCANNER_NAME) { + class Factory : ScannerWrapperFactory(SCANNER_NAME) { override fun create(options: Options) = ScanCode(type, options) } diff --git a/plugins/scanners/scanoss/src/main/kotlin/ScanOss.kt b/plugins/scanners/scanoss/src/main/kotlin/ScanOss.kt index 868a4bfc8a50c..2d51001a7aac0 100644 --- a/plugins/scanners/scanoss/src/main/kotlin/ScanOss.kt +++ b/plugins/scanners/scanoss/src/main/kotlin/ScanOss.kt @@ -39,10 +39,10 @@ import org.apache.logging.log4j.kotlin.Logging import org.ossreviewtoolkit.clients.scanoss.ScanOssService import org.ossreviewtoolkit.model.ScanSummary import org.ossreviewtoolkit.model.config.Options -import org.ossreviewtoolkit.scanner.AbstractScannerWrapperFactory import org.ossreviewtoolkit.scanner.PathScannerWrapper import org.ossreviewtoolkit.scanner.ScanContext import org.ossreviewtoolkit.scanner.ScannerCriteria +import org.ossreviewtoolkit.scanner.ScannerWrapperFactory import org.ossreviewtoolkit.utils.common.VCS_DIRECTORIES // An arbitrary name to use for the multipart body being sent. @@ -52,7 +52,7 @@ private const val ARG_FIELD_NAME = "file" class ScanOss internal constructor(override val name: String, private val options: Options) : PathScannerWrapper { private companion object : Logging - class Factory : AbstractScannerWrapperFactory("SCANOSS") { + class Factory : ScannerWrapperFactory("SCANOSS") { override fun create(options: Options) = ScanOss(type, options) } diff --git a/scanner/src/main/kotlin/ScannerWrapper.kt b/scanner/src/main/kotlin/ScannerWrapper.kt index ae02cfa866980..5985745248129 100644 --- a/scanner/src/main/kotlin/ScannerWrapper.kt +++ b/scanner/src/main/kotlin/ScannerWrapper.kt @@ -39,7 +39,7 @@ sealed interface ScannerWrapper { /** * All [scanner wrapper factories][ScannerWrapperFactory] available in the classpath, associated by their names. */ - val ALL by lazy { Plugin.getAll() } + val ALL by lazy { Plugin.getAll>() } } /** diff --git a/scanner/src/main/kotlin/ScannerWrapperFactory.kt b/scanner/src/main/kotlin/ScannerWrapperFactory.kt index 8c3682d302012..9e85dbfeb1e0c 100644 --- a/scanner/src/main/kotlin/ScannerWrapperFactory.kt +++ b/scanner/src/main/kotlin/ScannerWrapperFactory.kt @@ -25,26 +25,17 @@ import org.ossreviewtoolkit.model.config.Options import org.ossreviewtoolkit.utils.common.Plugin /** - * A common interface for use with [ServiceLoader] that all [ScannerWrapperFactory] classes need to implement. + * A common abstract class for use with [ServiceLoader] that all [ScannerWrapperFactory] classes need to implement. */ -interface ScannerWrapperFactory : Plugin { +abstract class ScannerWrapperFactory(override val type: String) : Plugin { /** * Create a [ScannerWrapper] using the provided [options]. */ - fun create(options: Options): ScannerWrapper -} - -/** - * A generic factory class for a [ScannerWrapper]. - */ -abstract class AbstractScannerWrapperFactory( - override val type: String -) : ScannerWrapperFactory { - abstract override fun create(options: Options): T + abstract fun create(options: Options): T /** * Return the scanner wrapper's name here to allow Clikt to display something meaningful when listing the scanners - * which are enabled by default via their factories. + * wrapper factories which are enabled by default. */ override fun toString() = type } From 9a5d8059e5f8a3c3cf916c6f2c0cbe90b69b5e62 Mon Sep 17 00:00:00 2001 From: Martin Nonnenmacher Date: Tue, 26 Sep 2023 16:05:42 +0200 Subject: [PATCH 26/47] refactor: Move `Options` from model to common-utils Move the `Options` typealias from model to common-utils to make it available to the plugin API. This allows to increase consistency as many plugin configurations already use the typealias. Signed-off-by: Martin Nonnenmacher --- .../src/main/kotlin/AdviceProviderFactory.kt | 2 +- analyzer/src/main/kotlin/PackageManager.kt | 2 +- .../kotlin/config/AdvisorConfiguration.kt | 2 ++ .../main/kotlin/config/OrtConfiguration.kt | 5 ---- .../config/PackageManagerConfiguration.kt | 2 ++ .../kotlin/config/ReporterConfiguration.kt | 2 ++ .../kotlin/config/ScannerConfiguration.kt | 1 + .../askalono/src/main/kotlin/Askalono.kt | 2 +- .../boyterlc/src/main/kotlin/BoyterLc.kt | 2 +- .../scanners/fossid/src/main/kotlin/FossId.kt | 2 +- .../fossid/src/main/kotlin/FossIdConfig.kt | 2 +- .../licensee/src/main/kotlin/Licensee.kt | 2 +- .../scancode/src/main/kotlin/ScanCode.kt | 2 +- .../scanoss/src/main/kotlin/ScanOss.kt | 2 +- .../scanoss/src/main/kotlin/ScanOssConfig.kt | 2 +- scanner/src/main/kotlin/Scanner.kt | 2 +- scanner/src/main/kotlin/ScannerCriteria.kt | 2 +- scanner/src/main/kotlin/ScannerWrapper.kt | 2 +- .../src/main/kotlin/ScannerWrapperFactory.kt | 2 +- utils/common/src/main/kotlin/Options.kt | 25 +++++++++++++++++++ 20 files changed, 46 insertions(+), 19 deletions(-) create mode 100644 utils/common/src/main/kotlin/Options.kt diff --git a/advisor/src/main/kotlin/AdviceProviderFactory.kt b/advisor/src/main/kotlin/AdviceProviderFactory.kt index 30c28f723ccc6..62093ff3f05be 100644 --- a/advisor/src/main/kotlin/AdviceProviderFactory.kt +++ b/advisor/src/main/kotlin/AdviceProviderFactory.kt @@ -22,7 +22,7 @@ package org.ossreviewtoolkit.advisor import java.util.ServiceLoader import org.ossreviewtoolkit.model.config.AdvisorConfiguration -import org.ossreviewtoolkit.model.config.Options +import org.ossreviewtoolkit.utils.common.Options import org.ossreviewtoolkit.utils.common.Plugin import org.ossreviewtoolkit.utils.ort.ORT_CONFIG_FILENAME import org.ossreviewtoolkit.utils.ort.ortConfigDirectory diff --git a/analyzer/src/main/kotlin/PackageManager.kt b/analyzer/src/main/kotlin/PackageManager.kt index 70d87a126ac07..5607a0f620272 100644 --- a/analyzer/src/main/kotlin/PackageManager.kt +++ b/analyzer/src/main/kotlin/PackageManager.kt @@ -43,10 +43,10 @@ import org.ossreviewtoolkit.model.VcsInfo import org.ossreviewtoolkit.model.VcsType import org.ossreviewtoolkit.model.config.AnalyzerConfiguration import org.ossreviewtoolkit.model.config.Excludes -import org.ossreviewtoolkit.model.config.Options import org.ossreviewtoolkit.model.config.PackageManagerConfiguration import org.ossreviewtoolkit.model.config.RepositoryConfiguration import org.ossreviewtoolkit.model.createAndLogIssue +import org.ossreviewtoolkit.utils.common.Options import org.ossreviewtoolkit.utils.common.Plugin import org.ossreviewtoolkit.utils.common.VCS_DIRECTORIES import org.ossreviewtoolkit.utils.common.collectMessages diff --git a/model/src/main/kotlin/config/AdvisorConfiguration.kt b/model/src/main/kotlin/config/AdvisorConfiguration.kt index 54e70af5abca5..d9634c6fb412c 100644 --- a/model/src/main/kotlin/config/AdvisorConfiguration.kt +++ b/model/src/main/kotlin/config/AdvisorConfiguration.kt @@ -22,6 +22,8 @@ package org.ossreviewtoolkit.model.config import com.fasterxml.jackson.annotation.JsonInclude import com.fasterxml.jackson.annotation.JsonProperty +import org.ossreviewtoolkit.utils.common.Options + /** * The base configuration model of the advisor. */ diff --git a/model/src/main/kotlin/config/OrtConfiguration.kt b/model/src/main/kotlin/config/OrtConfiguration.kt index 1ef8a2d921702..a2e4efada8cdc 100644 --- a/model/src/main/kotlin/config/OrtConfiguration.kt +++ b/model/src/main/kotlin/config/OrtConfiguration.kt @@ -193,11 +193,6 @@ data class OrtConfigurationWrapper( val ort: OrtConfiguration ) -/** - * A typealias for key-value pairs, used in several configuration classes. - */ -typealias Options = Map - /** * The filename of the reference configuration file. */ diff --git a/model/src/main/kotlin/config/PackageManagerConfiguration.kt b/model/src/main/kotlin/config/PackageManagerConfiguration.kt index 43c5775dff850..54b691092125e 100644 --- a/model/src/main/kotlin/config/PackageManagerConfiguration.kt +++ b/model/src/main/kotlin/config/PackageManagerConfiguration.kt @@ -21,6 +21,8 @@ package org.ossreviewtoolkit.model.config import com.fasterxml.jackson.annotation.JsonInclude +import org.ossreviewtoolkit.utils.common.Options + /** * The configuration for a package manager. */ diff --git a/model/src/main/kotlin/config/ReporterConfiguration.kt b/model/src/main/kotlin/config/ReporterConfiguration.kt index 6a456ddd2ce6d..fe4ed27d46ce0 100644 --- a/model/src/main/kotlin/config/ReporterConfiguration.kt +++ b/model/src/main/kotlin/config/ReporterConfiguration.kt @@ -21,6 +21,8 @@ package org.ossreviewtoolkit.model.config import com.fasterxml.jackson.annotation.JsonProperty +import org.ossreviewtoolkit.utils.common.Options + /** * The base configuration model of the reporter. */ diff --git a/model/src/main/kotlin/config/ScannerConfiguration.kt b/model/src/main/kotlin/config/ScannerConfiguration.kt index 9271af1246ff1..b21ea49caad04 100644 --- a/model/src/main/kotlin/config/ScannerConfiguration.kt +++ b/model/src/main/kotlin/config/ScannerConfiguration.kt @@ -22,6 +22,7 @@ package org.ossreviewtoolkit.model.config import com.fasterxml.jackson.annotation.JsonInclude import org.ossreviewtoolkit.model.utils.FileArchiver +import org.ossreviewtoolkit.utils.common.Options import org.ossreviewtoolkit.utils.ort.ORT_REPO_CONFIG_FILENAME import org.ossreviewtoolkit.utils.ort.storage.FileStorage import org.ossreviewtoolkit.utils.spdx.SpdxConstants diff --git a/plugins/scanners/askalono/src/main/kotlin/Askalono.kt b/plugins/scanners/askalono/src/main/kotlin/Askalono.kt index 1a3f904e5d81c..f8f2a3ab7e99d 100644 --- a/plugins/scanners/askalono/src/main/kotlin/Askalono.kt +++ b/plugins/scanners/askalono/src/main/kotlin/Askalono.kt @@ -32,12 +32,12 @@ import org.ossreviewtoolkit.model.LicenseFinding import org.ossreviewtoolkit.model.ScanSummary import org.ossreviewtoolkit.model.Severity import org.ossreviewtoolkit.model.TextLocation -import org.ossreviewtoolkit.model.config.Options import org.ossreviewtoolkit.scanner.CommandLinePathScannerWrapper import org.ossreviewtoolkit.scanner.ScanContext import org.ossreviewtoolkit.scanner.ScanException import org.ossreviewtoolkit.scanner.ScannerCriteria import org.ossreviewtoolkit.scanner.ScannerWrapperFactory +import org.ossreviewtoolkit.utils.common.Options import org.ossreviewtoolkit.utils.common.Os private const val CONFIDENCE_NOTICE = "Confidence threshold not high enough for any known license" diff --git a/plugins/scanners/boyterlc/src/main/kotlin/BoyterLc.kt b/plugins/scanners/boyterlc/src/main/kotlin/BoyterLc.kt index 22e6151932a93..5e510d1167f66 100644 --- a/plugins/scanners/boyterlc/src/main/kotlin/BoyterLc.kt +++ b/plugins/scanners/boyterlc/src/main/kotlin/BoyterLc.kt @@ -31,12 +31,12 @@ import org.ossreviewtoolkit.model.LicenseFinding import org.ossreviewtoolkit.model.ScanSummary import org.ossreviewtoolkit.model.Severity import org.ossreviewtoolkit.model.TextLocation -import org.ossreviewtoolkit.model.config.Options import org.ossreviewtoolkit.scanner.CommandLinePathScannerWrapper import org.ossreviewtoolkit.scanner.ScanContext import org.ossreviewtoolkit.scanner.ScanException import org.ossreviewtoolkit.scanner.ScannerCriteria import org.ossreviewtoolkit.scanner.ScannerWrapperFactory +import org.ossreviewtoolkit.utils.common.Options import org.ossreviewtoolkit.utils.common.Os import org.ossreviewtoolkit.utils.common.safeDeleteRecursively import org.ossreviewtoolkit.utils.ort.createOrtTempDir diff --git a/plugins/scanners/fossid/src/main/kotlin/FossId.kt b/plugins/scanners/fossid/src/main/kotlin/FossId.kt index 84d52ea93eeeb..2709cbdef457b 100644 --- a/plugins/scanners/fossid/src/main/kotlin/FossId.kt +++ b/plugins/scanners/fossid/src/main/kotlin/FossId.kt @@ -72,7 +72,6 @@ import org.ossreviewtoolkit.model.ScanSummary import org.ossreviewtoolkit.model.Severity import org.ossreviewtoolkit.model.UnknownProvenance import org.ossreviewtoolkit.model.VcsType -import org.ossreviewtoolkit.model.config.Options import org.ossreviewtoolkit.model.config.ScannerConfiguration import org.ossreviewtoolkit.model.createAndLogIssue import org.ossreviewtoolkit.scanner.PackageScannerWrapper @@ -80,6 +79,7 @@ import org.ossreviewtoolkit.scanner.ProvenanceScannerWrapper import org.ossreviewtoolkit.scanner.ScanContext import org.ossreviewtoolkit.scanner.ScannerCriteria import org.ossreviewtoolkit.scanner.ScannerWrapperFactory +import org.ossreviewtoolkit.utils.common.Options import org.ossreviewtoolkit.utils.common.enumSetOf import org.ossreviewtoolkit.utils.common.replaceCredentialsInUri import org.ossreviewtoolkit.utils.ort.showStackTrace diff --git a/plugins/scanners/fossid/src/main/kotlin/FossIdConfig.kt b/plugins/scanners/fossid/src/main/kotlin/FossIdConfig.kt index e72d5a413c211..066029c52d777 100644 --- a/plugins/scanners/fossid/src/main/kotlin/FossIdConfig.kt +++ b/plugins/scanners/fossid/src/main/kotlin/FossIdConfig.kt @@ -21,8 +21,8 @@ package org.ossreviewtoolkit.plugins.scanners.fossid import org.apache.logging.log4j.kotlin.Logging -import org.ossreviewtoolkit.model.config.Options import org.ossreviewtoolkit.model.config.ScannerConfiguration +import org.ossreviewtoolkit.utils.common.Options /** * A data class that holds the configuration options supported by the [FossId] scanner. An instance of this class is diff --git a/plugins/scanners/licensee/src/main/kotlin/Licensee.kt b/plugins/scanners/licensee/src/main/kotlin/Licensee.kt index b0cefefdc4a09..9ef6cfbb03b07 100644 --- a/plugins/scanners/licensee/src/main/kotlin/Licensee.kt +++ b/plugins/scanners/licensee/src/main/kotlin/Licensee.kt @@ -32,12 +32,12 @@ import org.ossreviewtoolkit.model.LicenseFinding import org.ossreviewtoolkit.model.ScanSummary import org.ossreviewtoolkit.model.Severity import org.ossreviewtoolkit.model.TextLocation -import org.ossreviewtoolkit.model.config.Options import org.ossreviewtoolkit.scanner.CommandLinePathScannerWrapper import org.ossreviewtoolkit.scanner.ScanContext import org.ossreviewtoolkit.scanner.ScanException import org.ossreviewtoolkit.scanner.ScannerCriteria import org.ossreviewtoolkit.scanner.ScannerWrapperFactory +import org.ossreviewtoolkit.utils.common.Options import org.ossreviewtoolkit.utils.common.Os private val JSON = Json { diff --git a/plugins/scanners/scancode/src/main/kotlin/ScanCode.kt b/plugins/scanners/scancode/src/main/kotlin/ScanCode.kt index 1c873a22ea6ec..a69ac7ee9214b 100644 --- a/plugins/scanners/scancode/src/main/kotlin/ScanCode.kt +++ b/plugins/scanners/scancode/src/main/kotlin/ScanCode.kt @@ -28,13 +28,13 @@ import org.apache.logging.log4j.kotlin.Logging import org.ossreviewtoolkit.model.ScanSummary import org.ossreviewtoolkit.model.ScannerDetails -import org.ossreviewtoolkit.model.config.Options import org.ossreviewtoolkit.model.config.ScannerConfiguration import org.ossreviewtoolkit.scanner.CommandLinePathScannerWrapper import org.ossreviewtoolkit.scanner.ScanContext import org.ossreviewtoolkit.scanner.ScanResultsStorage import org.ossreviewtoolkit.scanner.ScannerCriteria import org.ossreviewtoolkit.scanner.ScannerWrapperFactory +import org.ossreviewtoolkit.utils.common.Options import org.ossreviewtoolkit.utils.common.Os import org.ossreviewtoolkit.utils.common.ProcessCapture import org.ossreviewtoolkit.utils.common.safeDeleteRecursively diff --git a/plugins/scanners/scanoss/src/main/kotlin/ScanOss.kt b/plugins/scanners/scanoss/src/main/kotlin/ScanOss.kt index 2d51001a7aac0..88cd2d5962384 100644 --- a/plugins/scanners/scanoss/src/main/kotlin/ScanOss.kt +++ b/plugins/scanners/scanoss/src/main/kotlin/ScanOss.kt @@ -38,11 +38,11 @@ import org.apache.logging.log4j.kotlin.Logging import org.ossreviewtoolkit.clients.scanoss.ScanOssService import org.ossreviewtoolkit.model.ScanSummary -import org.ossreviewtoolkit.model.config.Options import org.ossreviewtoolkit.scanner.PathScannerWrapper import org.ossreviewtoolkit.scanner.ScanContext import org.ossreviewtoolkit.scanner.ScannerCriteria import org.ossreviewtoolkit.scanner.ScannerWrapperFactory +import org.ossreviewtoolkit.utils.common.Options import org.ossreviewtoolkit.utils.common.VCS_DIRECTORIES // An arbitrary name to use for the multipart body being sent. diff --git a/plugins/scanners/scanoss/src/main/kotlin/ScanOssConfig.kt b/plugins/scanners/scanoss/src/main/kotlin/ScanOssConfig.kt index 46317ed7b3951..749d9c6ca057c 100644 --- a/plugins/scanners/scanoss/src/main/kotlin/ScanOssConfig.kt +++ b/plugins/scanners/scanoss/src/main/kotlin/ScanOssConfig.kt @@ -20,8 +20,8 @@ package org.ossreviewtoolkit.plugins.scanners.scanoss import org.ossreviewtoolkit.clients.scanoss.ScanOssService -import org.ossreviewtoolkit.model.config.Options import org.ossreviewtoolkit.model.config.ScannerConfiguration +import org.ossreviewtoolkit.utils.common.Options /** * A data class that holds the configuration options supported by the [ScanOss] scanner. An instance of this class is diff --git a/scanner/src/main/kotlin/Scanner.kt b/scanner/src/main/kotlin/Scanner.kt index 3492bc5bfc96d..186ac6ea16264 100644 --- a/scanner/src/main/kotlin/Scanner.kt +++ b/scanner/src/main/kotlin/Scanner.kt @@ -49,7 +49,6 @@ import org.ossreviewtoolkit.model.ScanSummary import org.ossreviewtoolkit.model.ScannerRun import org.ossreviewtoolkit.model.VcsInfo import org.ossreviewtoolkit.model.config.DownloaderConfiguration -import org.ossreviewtoolkit.model.config.Options import org.ossreviewtoolkit.model.config.ScannerConfiguration import org.ossreviewtoolkit.model.config.createFileArchiver import org.ossreviewtoolkit.model.config.createStorage @@ -65,6 +64,7 @@ import org.ossreviewtoolkit.scanner.provenance.NestedProvenanceScanResult import org.ossreviewtoolkit.scanner.provenance.PackageProvenanceResolver import org.ossreviewtoolkit.scanner.provenance.ProvenanceDownloader import org.ossreviewtoolkit.scanner.utils.FileListResolver +import org.ossreviewtoolkit.utils.common.Options import org.ossreviewtoolkit.utils.common.collectMessages import org.ossreviewtoolkit.utils.common.safeDeleteRecursively import org.ossreviewtoolkit.utils.ort.Environment diff --git a/scanner/src/main/kotlin/ScannerCriteria.kt b/scanner/src/main/kotlin/ScannerCriteria.kt index b064de3d49336..3b659eb50dacb 100644 --- a/scanner/src/main/kotlin/ScannerCriteria.kt +++ b/scanner/src/main/kotlin/ScannerCriteria.kt @@ -20,7 +20,7 @@ package org.ossreviewtoolkit.scanner import org.ossreviewtoolkit.model.ScannerDetails -import org.ossreviewtoolkit.model.config.Options +import org.ossreviewtoolkit.utils.common.Options import org.semver4j.Semver diff --git a/scanner/src/main/kotlin/ScannerWrapper.kt b/scanner/src/main/kotlin/ScannerWrapper.kt index 5985745248129..f3fd9c3622e83 100644 --- a/scanner/src/main/kotlin/ScannerWrapper.kt +++ b/scanner/src/main/kotlin/ScannerWrapper.kt @@ -27,8 +27,8 @@ import org.ossreviewtoolkit.model.Provenance import org.ossreviewtoolkit.model.ScanResult import org.ossreviewtoolkit.model.ScanSummary import org.ossreviewtoolkit.model.ScannerDetails -import org.ossreviewtoolkit.model.config.Options import org.ossreviewtoolkit.model.config.ScannerConfiguration +import org.ossreviewtoolkit.utils.common.Options import org.ossreviewtoolkit.utils.common.Plugin /** diff --git a/scanner/src/main/kotlin/ScannerWrapperFactory.kt b/scanner/src/main/kotlin/ScannerWrapperFactory.kt index 9e85dbfeb1e0c..2d18341630406 100644 --- a/scanner/src/main/kotlin/ScannerWrapperFactory.kt +++ b/scanner/src/main/kotlin/ScannerWrapperFactory.kt @@ -21,7 +21,7 @@ package org.ossreviewtoolkit.scanner import java.util.ServiceLoader -import org.ossreviewtoolkit.model.config.Options +import org.ossreviewtoolkit.utils.common.Options import org.ossreviewtoolkit.utils.common.Plugin /** diff --git a/utils/common/src/main/kotlin/Options.kt b/utils/common/src/main/kotlin/Options.kt new file mode 100644 index 0000000000000..bac8d909489c7 --- /dev/null +++ b/utils/common/src/main/kotlin/Options.kt @@ -0,0 +1,25 @@ +/* + * Copyright (C) 2022 The ORT Project Authors (see ) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * License-Filename: LICENSE + */ + +package org.ossreviewtoolkit.utils.common + +/** + * A typealias for key-value pairs, used in several configuration classes. + */ +typealias Options = Map From b5fdb7974d042ec92942f6a5af8be8175d044499 Mon Sep 17 00:00:00 2001 From: Martin Nonnenmacher Date: Tue, 26 Sep 2023 16:14:19 +0200 Subject: [PATCH 27/47] refactor(utils)!: Use the `Options` typealias in `PluginManager` Use the `Options` typealias in the `PluginManager` and rename the `parseConfig` function of `TypedConfigurablePluginFactory` to `parseOptions` accordingly. Signed-off-by: Martin Nonnenmacher --- .../kotlin/DirPackageConfigurationProvider.kt | 9 +++++---- .../OrtConfigPackageConfigurationProvider.kt | 3 ++- .../ClearlyDefinedPackageCurationProvider.kt | 7 ++++--- .../main/kotlin/FilePackageCurationProvider.kt | 11 ++++++----- .../kotlin/OrtConfigPackageCurationProvider.kt | 3 ++- .../main/kotlin/Sw360PackageCurationProvider.kt | 17 +++++++++-------- utils/common/src/main/kotlin/PluginManager.kt | 10 +++++----- 7 files changed, 33 insertions(+), 27 deletions(-) diff --git a/plugins/package-configuration-providers/dir/src/main/kotlin/DirPackageConfigurationProvider.kt b/plugins/package-configuration-providers/dir/src/main/kotlin/DirPackageConfigurationProvider.kt index 862cae6c19ed3..07a594c5b2152 100644 --- a/plugins/package-configuration-providers/dir/src/main/kotlin/DirPackageConfigurationProvider.kt +++ b/plugins/package-configuration-providers/dir/src/main/kotlin/DirPackageConfigurationProvider.kt @@ -30,6 +30,7 @@ import org.ossreviewtoolkit.model.readValue import org.ossreviewtoolkit.model.utils.PackageConfigurationProvider import org.ossreviewtoolkit.plugins.packageconfigurationproviders.api.PackageConfigurationProviderFactory import org.ossreviewtoolkit.plugins.packageconfigurationproviders.api.SimplePackageConfigurationProvider +import org.ossreviewtoolkit.utils.common.Options import org.ossreviewtoolkit.utils.common.getDuplicates import org.ossreviewtoolkit.utils.ort.ORT_PACKAGE_CONFIGURATIONS_DIRNAME import org.ossreviewtoolkit.utils.ort.ortConfigDirectory @@ -52,17 +53,17 @@ open class DirPackageConfigurationProviderFactory : override fun create(config: DirPackageConfigurationProviderConfig) = DirPackageConfigurationProvider(config) - override fun parseConfig(config: Map) = + override fun parseOptions(options: Options) = DirPackageConfigurationProviderConfig( - path = File(config.getValue("path")), - mustExist = config["mustExist"]?.toBooleanStrict() ?: true + path = File(options.getValue("path")), + mustExist = options["mustExist"]?.toBooleanStrict() ?: true ) } class DefaultDirPackageConfigurationProviderFactory : DirPackageConfigurationProviderFactory() { override val type = "DefaultDir" - override fun parseConfig(config: Map) = + override fun parseOptions(options: Options) = DirPackageConfigurationProviderConfig( path = ortConfigDirectory.resolve(ORT_PACKAGE_CONFIGURATIONS_DIRNAME), mustExist = false diff --git a/plugins/package-configuration-providers/ort-config/src/main/kotlin/OrtConfigPackageConfigurationProvider.kt b/plugins/package-configuration-providers/ort-config/src/main/kotlin/OrtConfigPackageConfigurationProvider.kt index 332b722c846c0..d43545b2c003a 100644 --- a/plugins/package-configuration-providers/ort-config/src/main/kotlin/OrtConfigPackageConfigurationProvider.kt +++ b/plugins/package-configuration-providers/ort-config/src/main/kotlin/OrtConfigPackageConfigurationProvider.kt @@ -32,6 +32,7 @@ import org.ossreviewtoolkit.model.config.PackageConfiguration import org.ossreviewtoolkit.model.utils.PackageConfigurationProvider import org.ossreviewtoolkit.plugins.packageconfigurationproviders.api.PackageConfigurationProviderFactory import org.ossreviewtoolkit.plugins.packageconfigurationproviders.dir.DirPackageConfigurationProvider +import org.ossreviewtoolkit.utils.common.Options import org.ossreviewtoolkit.utils.common.safeMkdirs import org.ossreviewtoolkit.utils.ort.ortDataDirectory @@ -44,7 +45,7 @@ class OrtConfigPackageConfigurationProviderFactory : PackageConfigurationProvide override fun create(config: Unit): PackageConfigurationProvider = OrtConfigPackageConfigurationProvider() - override fun parseConfig(config: Map) = Unit + override fun parseOptions(options: Options) = Unit } /** diff --git a/plugins/package-curation-providers/clearly-defined/src/main/kotlin/ClearlyDefinedPackageCurationProvider.kt b/plugins/package-curation-providers/clearly-defined/src/main/kotlin/ClearlyDefinedPackageCurationProvider.kt index d837b4d533416..30c1d229755cd 100644 --- a/plugins/package-curation-providers/clearly-defined/src/main/kotlin/ClearlyDefinedPackageCurationProvider.kt +++ b/plugins/package-curation-providers/clearly-defined/src/main/kotlin/ClearlyDefinedPackageCurationProvider.kt @@ -43,6 +43,7 @@ import org.ossreviewtoolkit.model.VcsType import org.ossreviewtoolkit.model.utils.PackageCurationProvider import org.ossreviewtoolkit.model.utils.toClearlyDefinedCoordinates import org.ossreviewtoolkit.plugins.packagecurationproviders.api.PackageCurationProviderFactory +import org.ossreviewtoolkit.utils.common.Options import org.ossreviewtoolkit.utils.common.collectMessages import org.ossreviewtoolkit.utils.ort.OkHttpClientHelper import org.ossreviewtoolkit.utils.ort.showStackTrace @@ -70,10 +71,10 @@ class ClearlyDefinedPackageCurationProviderFactory : override fun create(config: ClearlyDefinedPackageCurationProviderConfig) = ClearlyDefinedPackageCurationProvider(config) - override fun parseConfig(config: Map) = + override fun parseOptions(options: Options) = ClearlyDefinedPackageCurationProviderConfig( - serverUrl = config["serverUrl"] ?: Server.PRODUCTION.apiUrl, - minTotalLicenseScore = config["minTotalLicenseScore"]?.toInt() ?: 0 + serverUrl = options["serverUrl"] ?: Server.PRODUCTION.apiUrl, + minTotalLicenseScore = options["minTotalLicenseScore"]?.toInt() ?: 0 ) } diff --git a/plugins/package-curation-providers/file/src/main/kotlin/FilePackageCurationProvider.kt b/plugins/package-curation-providers/file/src/main/kotlin/FilePackageCurationProvider.kt index b9386c62067af..96419a608262e 100644 --- a/plugins/package-curation-providers/file/src/main/kotlin/FilePackageCurationProvider.kt +++ b/plugins/package-curation-providers/file/src/main/kotlin/FilePackageCurationProvider.kt @@ -30,6 +30,7 @@ import org.ossreviewtoolkit.model.readValue import org.ossreviewtoolkit.model.utils.PackageCurationProvider import org.ossreviewtoolkit.plugins.packagecurationproviders.api.PackageCurationProviderFactory import org.ossreviewtoolkit.plugins.packagecurationproviders.api.SimplePackageCurationProvider +import org.ossreviewtoolkit.utils.common.Options import org.ossreviewtoolkit.utils.common.getDuplicates import org.ossreviewtoolkit.utils.ort.ORT_PACKAGE_CURATIONS_DIRNAME import org.ossreviewtoolkit.utils.ort.ORT_PACKAGE_CURATIONS_FILENAME @@ -52,17 +53,17 @@ open class FilePackageCurationProviderFactory : PackageCurationProviderFactory) = + override fun parseOptions(options: Options) = FilePackageCurationProviderConfig( - path = File(config.getValue("path")), - mustExist = config["mustExist"]?.toBooleanStrict() ?: true + path = File(options.getValue("path")), + mustExist = options["mustExist"]?.toBooleanStrict() ?: true ) } class DefaultFilePackageCurationProviderFactory : FilePackageCurationProviderFactory() { override val type = "DefaultFile" - override fun parseConfig(config: Map) = + override fun parseOptions(options: Options) = FilePackageCurationProviderConfig( path = ortConfigDirectory.resolve(ORT_PACKAGE_CURATIONS_FILENAME), mustExist = false @@ -72,7 +73,7 @@ class DefaultFilePackageCurationProviderFactory : FilePackageCurationProviderFac class DefaultDirPackageCurationProviderFactory : FilePackageCurationProviderFactory() { override val type = "DefaultDir" - override fun parseConfig(config: Map) = + override fun parseOptions(options: Options) = FilePackageCurationProviderConfig( path = ortConfigDirectory.resolve(ORT_PACKAGE_CURATIONS_DIRNAME), mustExist = false diff --git a/plugins/package-curation-providers/ort-config/src/main/kotlin/OrtConfigPackageCurationProvider.kt b/plugins/package-curation-providers/ort-config/src/main/kotlin/OrtConfigPackageCurationProvider.kt index efddc4f508cbc..94b4c66a864f0 100644 --- a/plugins/package-curation-providers/ort-config/src/main/kotlin/OrtConfigPackageCurationProvider.kt +++ b/plugins/package-curation-providers/ort-config/src/main/kotlin/OrtConfigPackageCurationProvider.kt @@ -33,6 +33,7 @@ import org.ossreviewtoolkit.model.VcsType import org.ossreviewtoolkit.model.readValue import org.ossreviewtoolkit.model.utils.PackageCurationProvider import org.ossreviewtoolkit.plugins.packagecurationproviders.api.PackageCurationProviderFactory +import org.ossreviewtoolkit.utils.common.Options import org.ossreviewtoolkit.utils.common.encodeOr import org.ossreviewtoolkit.utils.common.safeMkdirs import org.ossreviewtoolkit.utils.ort.ortDataDirectory @@ -45,7 +46,7 @@ class OrtConfigPackageCurationProviderFactory : PackageCurationProviderFactory) = Unit + override fun parseOptions(options: Options) = Unit } /** diff --git a/plugins/package-curation-providers/sw360/src/main/kotlin/Sw360PackageCurationProvider.kt b/plugins/package-curation-providers/sw360/src/main/kotlin/Sw360PackageCurationProvider.kt index 1923dd2152199..794d7dd25d7e4 100644 --- a/plugins/package-curation-providers/sw360/src/main/kotlin/Sw360PackageCurationProvider.kt +++ b/plugins/package-curation-providers/sw360/src/main/kotlin/Sw360PackageCurationProvider.kt @@ -44,6 +44,7 @@ import org.ossreviewtoolkit.model.jsonMapper import org.ossreviewtoolkit.model.orEmpty import org.ossreviewtoolkit.model.utils.PackageCurationProvider import org.ossreviewtoolkit.plugins.packagecurationproviders.api.PackageCurationProviderFactory +import org.ossreviewtoolkit.utils.common.Options import org.ossreviewtoolkit.utils.ort.DeclaredLicenseProcessor import org.ossreviewtoolkit.utils.spdx.SpdxExpression @@ -52,15 +53,15 @@ class Sw360PackageCurationProviderFactory : PackageCurationProviderFactory) = + override fun parseOptions(options: Options) = Sw360StorageConfiguration( - restUrl = config.getValue("restUrl"), - authUrl = config.getValue("authUrl"), - username = config.getValue("username"), - password = config["password"].orEmpty(), - clientId = config.getValue("clientId"), - clientPassword = config["clientPassword"].orEmpty(), - token = config["token"].orEmpty() + restUrl = options.getValue("restUrl"), + authUrl = options.getValue("authUrl"), + username = options.getValue("username"), + password = options["password"].orEmpty(), + clientId = options.getValue("clientId"), + clientPassword = options["clientPassword"].orEmpty(), + token = options["token"].orEmpty() ) } diff --git a/utils/common/src/main/kotlin/PluginManager.kt b/utils/common/src/main/kotlin/PluginManager.kt index 8f05623d08762..859790f4adc71 100644 --- a/utils/common/src/main/kotlin/PluginManager.kt +++ b/utils/common/src/main/kotlin/PluginManager.kt @@ -61,9 +61,9 @@ interface Plugin { */ interface ConfigurablePluginFactory : Plugin { /** - * Create a new instance of [PLUGIN] from [config]. + * Create a new instance of [PLUGIN] from [options]. */ - fun create(config: Map): PLUGIN + fun create(options: Options): PLUGIN } /** @@ -72,7 +72,7 @@ interface ConfigurablePluginFactory : Plugin { * that it enforces the separation of parsing the config map and creating the plugin. */ interface TypedConfigurablePluginFactory : ConfigurablePluginFactory { - override fun create(config: Map): PLUGIN = create(parseConfig(config)) + override fun create(options: Options): PLUGIN = create(parseOptions(options)) /** * Create a new instance of [PLUGIN] from [config]. @@ -80,7 +80,7 @@ interface TypedConfigurablePluginFactory : ConfigurablePlugi fun create(config: CONFIG): PLUGIN /** - * Parse the [config] map into a [CONFIG] object. + * Parse the [options] map into a [CONFIG] object. */ - fun parseConfig(config: Map): CONFIG + fun parseOptions(options: Options): CONFIG } From 6f1976c6427c815b9f62b84a6f706b9128ea01e5 Mon Sep 17 00:00:00 2001 From: Martin Nonnenmacher Date: Tue, 26 Sep 2023 16:17:13 +0200 Subject: [PATCH 28/47] refactor(fossid)!: Remove the unused `options` from the constructor Signed-off-by: Martin Nonnenmacher --- plugins/scanners/fossid/src/main/kotlin/FossId.kt | 3 +-- plugins/scanners/fossid/src/test/kotlin/FossIdTest.kt | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/plugins/scanners/fossid/src/main/kotlin/FossId.kt b/plugins/scanners/fossid/src/main/kotlin/FossId.kt index 2709cbdef457b..f7975cf115f9f 100644 --- a/plugins/scanners/fossid/src/main/kotlin/FossId.kt +++ b/plugins/scanners/fossid/src/main/kotlin/FossId.kt @@ -96,7 +96,6 @@ import org.ossreviewtoolkit.utils.ort.showStackTrace */ class FossId internal constructor( override val name: String, - private val options: Options, private val config: FossIdConfig ) : PackageScannerWrapper { companion object : Logging { @@ -170,7 +169,7 @@ class FossId internal constructor( } class Factory : ScannerWrapperFactory("FossId") { - override fun create(options: Options) = FossId(type, options, FossIdConfig.create(options)) + override fun create(options: Options) = FossId(type, FossIdConfig.create(options)) } /** diff --git a/plugins/scanners/fossid/src/test/kotlin/FossIdTest.kt b/plugins/scanners/fossid/src/test/kotlin/FossIdTest.kt index 77e6856378321..a2ee291cf127b 100644 --- a/plugins/scanners/fossid/src/test/kotlin/FossIdTest.kt +++ b/plugins/scanners/fossid/src/test/kotlin/FossIdTest.kt @@ -1152,7 +1152,7 @@ private val DEFAULT_IGNORE_RULE_SCOPE = RuleScope.SCAN /** * Create a new [FossId] instance with the specified [config]. */ -private fun createFossId(config: FossIdConfig): FossId = FossId("FossId", emptyMap(), config) +private fun createFossId(config: FossIdConfig): FossId = FossId("FossId", config) /** * Create a standard [FossIdConfig] whose properties can be partly specified. From 5b518559142d2dd4e71dd9aaeaa2d0101e035045 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Tue, 26 Sep 2023 19:34:16 +0000 Subject: [PATCH 29/47] deps: update actions/checkout action to v4 --- .github/workflows/docker-ort-runtime-ext.yml | 14 +++++++------- .github/workflows/docker-ort-runtime.yml | 14 +++++++------- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/.github/workflows/docker-ort-runtime-ext.yml b/.github/workflows/docker-ort-runtime-ext.yml index c664f7d2c4e99..fa12760e03929 100644 --- a/.github/workflows/docker-ort-runtime-ext.yml +++ b/.github/workflows/docker-ort-runtime-ext.yml @@ -46,7 +46,7 @@ jobs: runs-on: ubuntu-22.04 steps: - name: Checkout default branch - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Set environment variables run: | cat .versions >> $GITHUB_ENV @@ -64,7 +64,7 @@ jobs: runs-on: ubuntu-22.04 steps: - name: Checkout default branch - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Set environment variables run: | cat .versions >> $GITHUB_ENV @@ -82,7 +82,7 @@ jobs: runs-on: ubuntu-22.04 steps: - name: Checkout default branch - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Set environment variables run: | cat .versions >> $GITHUB_ENV @@ -101,7 +101,7 @@ jobs: runs-on: ubuntu-22.04 steps: - name: Checkout default branch - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Set environment variables run: | cat .versions >> $GITHUB_ENV @@ -119,7 +119,7 @@ jobs: runs-on: ubuntu-22.04 steps: - name: Checkout default branch - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Set environment variables run: | cat .versions >> $GITHUB_ENV @@ -137,7 +137,7 @@ jobs: runs-on: ubuntu-22.04 steps: - name: Checkout default branch - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Set environment variables run: | cat .versions >> $GITHUB_ENV @@ -161,7 +161,7 @@ jobs: steps: - name: Checkout default branch - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Set up Docker build uses: docker/setup-buildx-action@v2 diff --git a/.github/workflows/docker-ort-runtime.yml b/.github/workflows/docker-ort-runtime.yml index 6c27b2b9c2833..edc081d0619e4 100644 --- a/.github/workflows/docker-ort-runtime.yml +++ b/.github/workflows/docker-ort-runtime.yml @@ -40,7 +40,7 @@ jobs: runs-on: ubuntu-22.04 steps: - name: Checkout default branch - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Set environment variables run: | cat .versions >> $GITHUB_ENV @@ -60,7 +60,7 @@ jobs: runs-on: ubuntu-22.04 steps: - name: Checkout default branch - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Set environment variables run: | cat .versions >> $GITHUB_ENV @@ -82,7 +82,7 @@ jobs: runs-on: ubuntu-22.04 steps: - name: Checkout default branch - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Set environment variables run: | cat .versions >> $GITHUB_ENV @@ -108,7 +108,7 @@ jobs: runs-on: ubuntu-22.04 steps: - name: Checkout default branch - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Set environment variables run: | cat .versions >> $GITHUB_ENV @@ -127,7 +127,7 @@ jobs: runs-on: ubuntu-22.04 steps: - name: Checkout default branch - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Set environment variables run: | cat .versions >> $GITHUB_ENV @@ -147,7 +147,7 @@ jobs: runs-on: ubuntu-22.04 steps: - name: Checkout default branch - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Set environment variables run: | cat .versions >> $GITHUB_ENV @@ -171,7 +171,7 @@ jobs: steps: - name: Checkout default branch - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Set environment variables run: | From 114c1539a9be10ead521ee46b3e8a5986efe863a Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Tue, 26 Sep 2023 19:34:21 +0000 Subject: [PATCH 30/47] deps: update docker/build-push-action action to v5 --- .github/actions/ortdocker/action.yml | 2 +- .github/workflows/docker-ort-runtime-ext.yml | 2 +- .github/workflows/docker-ort-runtime.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/actions/ortdocker/action.yml b/.github/actions/ortdocker/action.yml index e851c2c4990b5..38b6060f6c1a0 100644 --- a/.github/actions/ortdocker/action.yml +++ b/.github/actions/ortdocker/action.yml @@ -85,7 +85,7 @@ runs: - name: Build image if: steps.check_image.outputs.result != 'found' - uses: docker/build-push-action@v4 + uses: docker/build-push-action@v5 with: context: . target: ${{ inputs.name }} diff --git a/.github/workflows/docker-ort-runtime-ext.yml b/.github/workflows/docker-ort-runtime-ext.yml index fa12760e03929..8ed0ebcf35981 100644 --- a/.github/workflows/docker-ort-runtime-ext.yml +++ b/.github/workflows/docker-ort-runtime-ext.yml @@ -183,7 +183,7 @@ jobs: type=raw,sha,enable=true,format=short - name: Build ORT extended runtime image - uses: docker/build-push-action@v4 + uses: docker/build-push-action@v5 with: context: . file: Dockerfile-extended diff --git a/.github/workflows/docker-ort-runtime.yml b/.github/workflows/docker-ort-runtime.yml index edc081d0619e4..8d7d4df83a57a 100644 --- a/.github/workflows/docker-ort-runtime.yml +++ b/.github/workflows/docker-ort-runtime.yml @@ -198,7 +198,7 @@ jobs: - name: Build ORT runtime image if: ${{ github.event_name == 'push' || github.event_name == 'workflow_dispatch' }} - uses: docker/build-push-action@v4 + uses: docker/build-push-action@v5 with: context: . target: run From 63d89b41dc8a10a969a383783d6dc7cabc20ba28 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Tue, 26 Sep 2023 19:34:25 +0000 Subject: [PATCH 31/47] deps: update docker/login-action action to v3 --- .github/actions/ortdocker/action.yml | 2 +- .github/workflows/docker-ort-runtime-ext.yml | 2 +- .github/workflows/docker-ort-runtime.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/actions/ortdocker/action.yml b/.github/actions/ortdocker/action.yml index 38b6060f6c1a0..ba35bdb310b89 100644 --- a/.github/actions/ortdocker/action.yml +++ b/.github/actions/ortdocker/action.yml @@ -67,7 +67,7 @@ runs: - name: Login to GitHub container registry if: steps.check_image.outputs.result != 'found' - uses: docker/login-action@v2 + uses: docker/login-action@v3 with: registry: ${{ inputs.registry }} username: ${{ github.actor }} diff --git a/.github/workflows/docker-ort-runtime-ext.yml b/.github/workflows/docker-ort-runtime-ext.yml index 8ed0ebcf35981..f2df786c0e55f 100644 --- a/.github/workflows/docker-ort-runtime-ext.yml +++ b/.github/workflows/docker-ort-runtime-ext.yml @@ -167,7 +167,7 @@ jobs: uses: docker/setup-buildx-action@v2 - name: Login to GitHub container registry - uses: docker/login-action@v2 + uses: docker/login-action@v3 with: registry: ${{ env.REGISTRY }} username: ${{ github.actor }} diff --git a/.github/workflows/docker-ort-runtime.yml b/.github/workflows/docker-ort-runtime.yml index 8d7d4df83a57a..9f13d4410601c 100644 --- a/.github/workflows/docker-ort-runtime.yml +++ b/.github/workflows/docker-ort-runtime.yml @@ -181,7 +181,7 @@ jobs: uses: docker/setup-buildx-action@v2 - name: Login to GitHub container registry - uses: docker/login-action@v2 + uses: docker/login-action@v3 with: registry: ${{ env.REGISTRY }} username: ${{ github.actor }} From cb982b5cc8a2b1fc83843a37567b0b5ce3d5ac83 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Tue, 26 Sep 2023 19:34:30 +0000 Subject: [PATCH 32/47] deps: update docker/metadata-action action to v5 --- .github/actions/ortdocker/action.yml | 2 +- .github/workflows/docker-ort-runtime-ext.yml | 2 +- .github/workflows/docker-ort-runtime.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/actions/ortdocker/action.yml b/.github/actions/ortdocker/action.yml index ba35bdb310b89..660c6e458d957 100644 --- a/.github/actions/ortdocker/action.yml +++ b/.github/actions/ortdocker/action.yml @@ -76,7 +76,7 @@ runs: - name: Extract components metadata (tags, labels) if: steps.check_image.outputs.result != 'found' id: meta - uses: docker/metadata-action@v4 + uses: docker/metadata-action@v5 with: images: | ${{ env.REGISTRY }}/${{ github.repository }}/${{ inputs.name }} diff --git a/.github/workflows/docker-ort-runtime-ext.yml b/.github/workflows/docker-ort-runtime-ext.yml index f2df786c0e55f..6d35472bb8e54 100644 --- a/.github/workflows/docker-ort-runtime-ext.yml +++ b/.github/workflows/docker-ort-runtime-ext.yml @@ -175,7 +175,7 @@ jobs: - name: Extract components metadata (tags, labels) id: meta-ort - uses: docker/metadata-action@v4 + uses: docker/metadata-action@v5 with: images: | ${{ env.REGISTRY }}/${{ github.repository_owner }}/ort-extended diff --git a/.github/workflows/docker-ort-runtime.yml b/.github/workflows/docker-ort-runtime.yml index 9f13d4410601c..8a4ed65b2b8de 100644 --- a/.github/workflows/docker-ort-runtime.yml +++ b/.github/workflows/docker-ort-runtime.yml @@ -189,7 +189,7 @@ jobs: - name: Extract components metadata (tags, labels) id: meta-ort - uses: docker/metadata-action@v4 + uses: docker/metadata-action@v5 with: images: | ${{ env.REGISTRY }}/${{ github.repository_owner }}/ort From 33d5fd936e6bea7144225e9eefc51f1063cbf5bd Mon Sep 17 00:00:00 2001 From: Frank Viernau Date: Tue, 26 Sep 2023 22:57:42 +0200 Subject: [PATCH 33/47] fix(docker): Correct a typo Signed-off-by: Frank Viernau --- scripts/docker_build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/docker_build.sh b/scripts/docker_build.sh index 9a8ba59a44620..7aa091653f2eb 100755 --- a/scripts/docker_build.sh +++ b/scripts/docker_build.sh @@ -30,7 +30,7 @@ echo "Setting ORT_VERSION to $GIT_REVISION." # --------------------------- # image_build function -# Usage ( position paramenters): +# Usage ( position parameters): # image_build image_build() { From 902bfef1dc2fbfed35a2ce156c1f2007e124d9bd Mon Sep 17 00:00:00 2001 From: Frank Viernau Date: Tue, 26 Sep 2023 22:49:27 +0200 Subject: [PATCH 34/47] docs(cyclonedx): Fix a typo Signed-off-by: Frank Viernau --- .../reporters/cyclonedx/src/main/kotlin/CycloneDxReporter.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/reporters/cyclonedx/src/main/kotlin/CycloneDxReporter.kt b/plugins/reporters/cyclonedx/src/main/kotlin/CycloneDxReporter.kt index 7cf9677272283..808b03654afe0 100644 --- a/plugins/reporters/cyclonedx/src/main/kotlin/CycloneDxReporter.kt +++ b/plugins/reporters/cyclonedx/src/main/kotlin/CycloneDxReporter.kt @@ -57,7 +57,7 @@ import org.ossreviewtoolkit.utils.spdx.SpdxLicense * each [Project] contained in the ORT result a separate SBOM is created. * * This reporter supports the following options: - * - *data.license*: The license for the data contained in the report. Deafults to [DEFAULT_DATA_LICENSE]. + * - *data.license*: The license for the data contained in the report. Defaults to [DEFAULT_DATA_LICENSE]. * - *schema.version*: The CycloneDX schema version to use. Defaults to [DEFAULT_SCHEMA_VERSION]. * - *single.bom*: If true (the default), a single SBOM for all projects is created; if set to false, separate SBOMs are * created for each project. From ab0f931a0879309eb1d9ab773e7dd3a3276a61ac Mon Sep 17 00:00:00 2001 From: Sebastian Schuberth Date: Wed, 27 Sep 2023 12:52:57 +0200 Subject: [PATCH 35/47] test(ClearlyDefinedStorageFunTest): Update expected results A manual harvest was triggered for `semver4j` on ClearlyDefined today by myself for testing, so the expected ScanCode result data needs to be updated. Signed-off-by: Sebastian Schuberth --- .../storages/ClearlyDefinedStorageFunTest.kt | 51 ++++++++++--------- 1 file changed, 27 insertions(+), 24 deletions(-) diff --git a/scanner/src/funTest/kotlin/storages/ClearlyDefinedStorageFunTest.kt b/scanner/src/funTest/kotlin/storages/ClearlyDefinedStorageFunTest.kt index 335f93a82d117..35dc0c82274bc 100644 --- a/scanner/src/funTest/kotlin/storages/ClearlyDefinedStorageFunTest.kt +++ b/scanner/src/funTest/kotlin/storages/ClearlyDefinedStorageFunTest.kt @@ -29,12 +29,8 @@ import java.time.Instant import kotlin.time.Duration.Companion.seconds import org.ossreviewtoolkit.clients.clearlydefined.ClearlyDefinedService.Server -import org.ossreviewtoolkit.model.ArtifactProvenance -import org.ossreviewtoolkit.model.Hash -import org.ossreviewtoolkit.model.HashAlgorithm import org.ossreviewtoolkit.model.Identifier import org.ossreviewtoolkit.model.LicenseFinding -import org.ossreviewtoolkit.model.RemoteArtifact import org.ossreviewtoolkit.model.RepositoryProvenance import org.ossreviewtoolkit.model.ScanResult import org.ossreviewtoolkit.model.ScanSummary @@ -47,7 +43,7 @@ import org.ossreviewtoolkit.model.config.ClearlyDefinedStorageConfiguration class ClearlyDefinedStorageFunTest : StringSpec({ val storage = ClearlyDefinedStorage(ClearlyDefinedStorageConfiguration(Server.PRODUCTION.apiUrl)) - "Scan results for ScanCode 3.0.2 should be read correctly" { + "Scan results for 'semver4j' from ScanCode should be read correctly" { val id = Identifier("Maven:com.vdurmont:semver4j:3.1.0") withRetry { @@ -55,29 +51,36 @@ class ClearlyDefinedStorageFunTest : StringSpec({ results.shouldBeSuccess { it shouldContain ScanResult( - provenance = ArtifactProvenance( - sourceArtifact = RemoteArtifact( - url = "https://search.maven.org/remotecontent" + - "?filepath=com/vdurmont/semver4j/3.1.0/semver4j-3.1.0-sources.jar", - hash = Hash( - value = "0de1248f09dfe8df3b021c84e0642ee222cceb13", - algorithm = HashAlgorithm.SHA1 - ) - ) + provenance = RepositoryProvenance( + vcsInfo = VcsInfo( + type = VcsType.GIT, + url = "https://github.com/vdurmont/semver4j/tree/88912638db3f6112a2b345f1638ced33a0a606e1", + revision = "88912638db3f6112a2b345f1638ced33a0a606e1" + ), + resolvedRevision = "88912638db3f6112a2b345f1638ced33a0a606e1" ), scanner = ScannerDetails( name = "ScanCode", - version = "3.0.2", - configuration = "input /tmp/cd-2rGiCR --classify true --copyright true --email true " + - "--generated true --info true --is-license-text true --json-pp /tmp/cd-0EjTZ7 " + - "--license true --license-clarity-score true --license-diag true --license-text true " + - "--package true --processes 2 --strip-root true --summary true --summary-key-files " + - "true --timeout 1000.0 --url true" + version = "30.1.0", + configuration = "input /tmp/cd-BxlI4n --classify true --copyright true --email true " + + "--generated true --info true --is-license-text true --json-pp /tmp/cd-d8WH1p " + + "--license true --license-clarity-score true --license-text true " + + "--license-text-diagnostics true --package true --processes 2 --strip-root true " + + "--summary true --summary-key-files true --timeout 1000.0 --url true" ), summary = ScanSummary.EMPTY.copy( - startTime = Instant.parse("2020-02-14T00:36:14.000335513Z"), - endTime = Instant.parse("2020-02-14T00:36:37.000492119Z"), + startTime = Instant.parse("2023-09-27T08:28:44.000244665Z"), + endTime = Instant.parse("2023-09-27T08:29:22.000369368Z"), licenseFindings = setOf( + LicenseFinding( + license = "BSD-3-Clause", + location = TextLocation( + path = "META-INF/maven/com.vdurmont/semver4j/pom.xml", + startLine = 28, + endLine = 34 + ), + score = 75.0f + ), LicenseFinding( license = "MIT", location = TextLocation( @@ -85,7 +88,7 @@ class ClearlyDefinedStorageFunTest : StringSpec({ startLine = 30, endLine = 31 ), - score = 60.87f + score = 83.33f ) ) ) @@ -94,7 +97,7 @@ class ClearlyDefinedStorageFunTest : StringSpec({ } } - "Scan results for ScanCode 30.1.0 should be read correctly" { + "Scan results for 'hoplite-core' from ScanCode should be read correctly" { val id = Identifier("Maven:com.sksamuel.hoplite:hoplite-core:2.1.3") withRetry { From b7878c0bdc5ff6c297eda50a6b7b3ab45e1f53e5 Mon Sep 17 00:00:00 2001 From: Sebastian Schuberth Date: Wed, 27 Sep 2023 14:32:16 +0200 Subject: [PATCH 36/47] fix(clearly-defined): Ignore new `InnerError` fields New "ok", "errmsg", "code", and "codeName" fields were added recently, see e.g. "ok": 0, "errmsg": "Command find not supported prior to authentication.", "code": 115, "codeName": "CommandNotSupported", All of them sound pretty much internal. Also, using them in ORT does not really add any value to the existing fields. So ignore those fields by using a separate serializer instance, because kotlinx-serialization does not support per-class annotations for that yet (see [1]). [1]: https://github.com/Kotlin/kotlinx.serialization/issues/1420 Signed-off-by: Sebastian Schuberth --- .../src/main/kotlin/ClearlyDefinedService.kt | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/clients/clearly-defined/src/main/kotlin/ClearlyDefinedService.kt b/clients/clearly-defined/src/main/kotlin/ClearlyDefinedService.kt index 7901ca06e77bf..46e96f932db82 100644 --- a/clients/clearly-defined/src/main/kotlin/ClearlyDefinedService.kt +++ b/clients/clearly-defined/src/main/kotlin/ClearlyDefinedService.kt @@ -62,6 +62,11 @@ interface ClearlyDefinedService { */ val JSON = Json { encodeDefaults = false } + /** + * The JSON (de-)serialization object used by this service for errors. + */ + val JSON_FOR_ERRORS = Json(JSON) { ignoreUnknownKeys = true } + /** * Create a ClearlyDefined service instance for communicating with the given [server], optionally using a * pre-built OkHttp [client]. @@ -287,7 +292,7 @@ suspend fun ClearlyDefinedService.call(block: suspend ClearlyDefinedService. block() } catch (e: HttpException) { val errorMessage = e.response()?.errorBody()?.let { - val errorResponse = ClearlyDefinedService.JSON.decodeFromString(it.string()) + val errorResponse = ClearlyDefinedService.JSON_FOR_ERRORS.decodeFromString(it.string()) val innerError = errorResponse.error.innererror "The ClearlyDefined service call failed with: ${innerError.message}" From 2f3708f483bb9925e2b922383efaf2c829c2a174 Mon Sep 17 00:00:00 2001 From: Sebastian Schuberth Date: Wed, 27 Sep 2023 14:37:43 +0200 Subject: [PATCH 37/47] chore(clearly-defined): Also show the inner error name This gives additional info about the scope of the error, like "MongoError". Signed-off-by: Sebastian Schuberth --- .../clearly-defined/src/main/kotlin/ClearlyDefinedService.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clients/clearly-defined/src/main/kotlin/ClearlyDefinedService.kt b/clients/clearly-defined/src/main/kotlin/ClearlyDefinedService.kt index 46e96f932db82..820fb16b11e8b 100644 --- a/clients/clearly-defined/src/main/kotlin/ClearlyDefinedService.kt +++ b/clients/clearly-defined/src/main/kotlin/ClearlyDefinedService.kt @@ -295,7 +295,7 @@ suspend fun ClearlyDefinedService.call(block: suspend ClearlyDefinedService. val errorResponse = ClearlyDefinedService.JSON_FOR_ERRORS.decodeFromString(it.string()) val innerError = errorResponse.error.innererror - "The ClearlyDefined service call failed with: ${innerError.message}" + "The ClearlyDefined service call failed. ${innerError.name}: ${innerError.message}" } ?: "The ClearlyDefined service call failed with code ${e.code()}: ${e.message()}" throw IOException(errorMessage, e) From 9f7debdbb538480d5ee1fd24d524082b5c183a59 Mon Sep 17 00:00:00 2001 From: Sebastian Schuberth Date: Wed, 27 Sep 2023 14:40:03 +0200 Subject: [PATCH 38/47] test(clearly-defined): Do not test against the development server The development server is less stable and might suffer from occasional configuration errors or disappearing curations, so do not rely on it in tests. Signed-off-by: Sebastian Schuberth --- ...lyDefinedPackageCurationProviderFunTest.kt | 24 ------------------- 1 file changed, 24 deletions(-) diff --git a/plugins/package-curation-providers/clearly-defined/src/funTest/kotlin/ClearlyDefinedPackageCurationProviderFunTest.kt b/plugins/package-curation-providers/clearly-defined/src/funTest/kotlin/ClearlyDefinedPackageCurationProviderFunTest.kt index 951d3b23f2ca5..2d2f263bf9b42 100644 --- a/plugins/package-curation-providers/clearly-defined/src/funTest/kotlin/ClearlyDefinedPackageCurationProviderFunTest.kt +++ b/plugins/package-curation-providers/clearly-defined/src/funTest/kotlin/ClearlyDefinedPackageCurationProviderFunTest.kt @@ -70,30 +70,6 @@ class ClearlyDefinedPackageCurationProviderFunTest : WordSpec({ } } - "The development server" should { - val provider = ClearlyDefinedPackageCurationProvider(Server.DEVELOPMENT) - - "return an existing curation for the platform-express NPM package" { - val packages = createPackagesFromIds("NPM:@nestjs:platform-express:6.2.3") - - withRetry { - val curations = provider.getCurationsFor(packages) - - curations.map { it.data.concludedLicense } shouldHaveSingleElement "Apache-1.0".toSpdx() - } - } - - "return no curation for a non-existing dummy Maven package" { - val packages = createPackagesFromIds("Maven:group:name:1.2.3") - - withRetry { - val curations = provider.getCurationsFor(packages) - - curations should beEmpty() - } - } - } - "Curations" should { "get filtered by score" { val config = ClearlyDefinedPackageCurationProviderConfig( From 0de789481b0017b5eff7ef0a4d2985bd467a9b0d Mon Sep 17 00:00:00 2001 From: Frank Viernau Date: Wed, 27 Sep 2023 15:25:18 +0200 Subject: [PATCH 39/47] test(conan): Update expected results Signed-off-by: Frank Viernau --- .../projects/synthetic/conan-expected-output-txt.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/plugins/package-managers/conan/src/funTest/assets/projects/synthetic/conan-expected-output-txt.yml b/plugins/package-managers/conan/src/funTest/assets/projects/synthetic/conan-expected-output-txt.yml index f20b7beb11417..bd841f5001a0f 100644 --- a/plugins/package-managers/conan/src/funTest/assets/projects/synthetic/conan-expected-output-txt.yml +++ b/plugins/package-managers/conan/src/funTest/assets/projects/synthetic/conan-expected-output-txt.yml @@ -28,7 +28,7 @@ project: dependencies: - id: "Conan::libcurl:7.85.0" dependencies: - - id: "Conan::openssl:3.1.2" + - id: "Conan::openssl:3.1.3" dependencies: - id: "Conan::zlib:1.2.13" - id: "Conan::zlib:1.2.13" @@ -148,8 +148,8 @@ packages: revision: "" path: "" is_modified: true -- id: "Conan::openssl:3.1.2" - purl: "pkg:conan/openssl@3.1.2" +- id: "Conan::openssl:3.1.3" + purl: "pkg:conan/openssl@3.1.3" declared_licenses: - "Apache-2.0" declared_licenses_processed: @@ -163,9 +163,9 @@ packages: value: "" algorithm: "" source_artifact: - url: "https://www.openssl.org/source/openssl-3.1.2.tar.gz" + url: "https://www.openssl.org/source/openssl-3.1.3.tar.gz" hash: - value: "a0ce69b8b97ea6a35b96875235aa453b966ba3cba8af2de23657d8b6767d6539" + value: "f0316a2ebd89e7f2352976445458689f80302093788c466692fb2a188b2eacf6" algorithm: "SHA-256" vcs: type: "Git" From a4b12df0acc6329cc164712e189d2c865c4280b2 Mon Sep 17 00:00:00 2001 From: Frank Viernau Date: Wed, 27 Sep 2023 14:59:37 +0200 Subject: [PATCH 40/47] fix(docker): Install Git LFS There are projects which use Git LFS for `gradle/wrapper/gradle-wrapper.jar`. Without Git LFS installed, an analysis of the project yields an empty project without dependencies. Fix that issue by installing Git LFS. Signed-off-by: Frank Viernau --- Dockerfile | 4 +++- Dockerfile-legacy | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 92b323de6e21f..6258224245823 100644 --- a/Dockerfile +++ b/Dockerfile @@ -43,6 +43,7 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ dirmngr \ gcc \ git \ + git-lfs \ g++ \ gnupg2 \ iproute2 \ @@ -64,7 +65,8 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ unzip \ wget \ xz-utils \ - && rm -rf /var/lib/apt/lists/* + && rm -rf /var/lib/apt/lists/* \ + && git lfs install RUN echo $LANG > /etc/locale.gen \ && locale-gen $LANG \ diff --git a/Dockerfile-legacy b/Dockerfile-legacy index dc56bc6fd5e3f..621fa402f78d2 100644 --- a/Dockerfile-legacy +++ b/Dockerfile-legacy @@ -119,6 +119,7 @@ RUN --mount=type=cache,target=/var/cache/apt --mount=type=cache,target=/var/lib/ zlib1g-dev \ # Install VCS tools (no specific versions required here). git \ + git-lfs \ mercurial \ subversion \ # Install package managers (in versions known to work). @@ -142,7 +143,8 @@ RUN --mount=type=cache,target=/var/cache/apt --mount=type=cache,target=/var/lib/ libstdc++6 \ libunwind8 \ && \ - rm -rf /var/lib/apt/lists/* + rm -rf /var/lib/apt/lists/* && \ + git lfs install COPY scripts/*.sh /opt/ort/bin/ COPY "$CRT_FILES" /tmp/certificates/ From 55c1b94c26cf7539ef0355354cd0dc38c1d2b37d Mon Sep 17 00:00:00 2001 From: Sebastian Schuberth Date: Wed, 27 Sep 2023 10:05:00 +0200 Subject: [PATCH 41/47] chore(ScannerCommand): Log configured scan storages at info level Esp. when passing scan storage configuration via command line options while other storage may be configured in the global `config.yml` it can be beneficial to know what exactly takes effect. Signed-off-by: Sebastian Schuberth --- .../commands/scanner/src/main/kotlin/ScannerCommand.kt | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/plugins/commands/scanner/src/main/kotlin/ScannerCommand.kt b/plugins/commands/scanner/src/main/kotlin/ScannerCommand.kt index b64b5d1ebd445..2061767d756fc 100644 --- a/plugins/commands/scanner/src/main/kotlin/ScannerCommand.kt +++ b/plugins/commands/scanner/src/main/kotlin/ScannerCommand.kt @@ -201,6 +201,16 @@ class ScannerCommand : OrtCommand( val scanStorages = ScanStorages.createFromConfig(ortConfig.scanner) val workingTreeCache = DefaultWorkingTreeCache() + logger.info { + val readers = scanStorages.readers.map { it.javaClass.simpleName } + "Using the following scan storages for reading results: $readers" + } + + logger.info { + val writers = scanStorages.writers.map { it.javaClass.simpleName } + "Using the following scan storages for writing results: $writers" + } + try { val scanner = Scanner( scannerConfig = ortConfig.scanner, From 2282526d23d7ed45f3c6e6fc1c5d04ab465427f5 Mon Sep 17 00:00:00 2001 From: Sebastian Schuberth Date: Wed, 27 Sep 2023 10:25:43 +0200 Subject: [PATCH 42/47] chore(ScanResultsStorage): Clarify log output about read results Emphasize that this is about matching results only, not all results. Signed-off-by: Sebastian Schuberth --- scanner/src/main/kotlin/ScanResultsStorage.kt | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/scanner/src/main/kotlin/ScanResultsStorage.kt b/scanner/src/main/kotlin/ScanResultsStorage.kt index 920b8e5ae6259..9c5da5ce64c15 100644 --- a/scanner/src/main/kotlin/ScanResultsStorage.kt +++ b/scanner/src/main/kotlin/ScanResultsStorage.kt @@ -84,8 +84,8 @@ abstract class ScanResultsStorage : PackageBasedScanStorage { result.onSuccess { results -> logger.info { - "Read ${results.size} scan result(s) for '${pkg.id.toCoordinates()}' from ${javaClass.simpleName} in " + - "$duration." + "Read ${results.size} matching scan result(s) for '${pkg.id.toCoordinates()}' from " + + "${javaClass.simpleName} in $duration." } } @@ -108,7 +108,8 @@ abstract class ScanResultsStorage : PackageBasedScanStorage { result.onSuccess { results -> logger.info { - "Read ${results.values.sumOf { it.size }} scan result(s) from ${javaClass.simpleName} in $duration." + val count = results.values.sumOf { it.size } + "Read $count matching scan result(s) from ${javaClass.simpleName} in $duration." } } From fd4ed1bb0c1f6432cfa9d6b227ef7b74e0f93af1 Mon Sep 17 00:00:00 2001 From: Sebastian Schuberth Date: Wed, 27 Sep 2023 11:49:39 +0200 Subject: [PATCH 43/47] fix(ScanResultsStorage): Correct debug log output about mismatches Previously, the empty `scanResults` were used in log output. Refactor code to avoid that error. Signed-off-by: Sebastian Schuberth --- scanner/src/main/kotlin/ScanResultsStorage.kt | 27 ++++++++++--------- 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/scanner/src/main/kotlin/ScanResultsStorage.kt b/scanner/src/main/kotlin/ScanResultsStorage.kt index 9c5da5ce64c15..462d45d821193 100644 --- a/scanner/src/main/kotlin/ScanResultsStorage.kt +++ b/scanner/src/main/kotlin/ScanResultsStorage.kt @@ -154,27 +154,30 @@ abstract class ScanResultsStorage : PackageBasedScanStorage { if (results.isEmpty()) { results } else { - val scanResults = results.toMutableList() + val (matchingProvenance, nonMatchingProvenance) = results.partition { it.provenance.matches(pkg) } - // Only keep scan results whose provenance information matches the package information. - scanResults.retainAll { it.provenance.matches(pkg) } - if (scanResults.isEmpty()) { + if (matchingProvenance.isEmpty()) { logger.debug { "No stored scan results found for $pkg. The following entries with non-matching provenance " + - "have been ignored: ${scanResults.map { it.provenance }}" + "have been ignored: ${nonMatchingProvenance.map { it.provenance }}" } + + matchingProvenance } else { - // Only keep scan results from compatible scanners. - scanResults.retainAll { scannerCriteria.matches(it.scanner) } - if (scanResults.isEmpty()) { + val (matchingCriteria, nonMatchingCriteria) = matchingProvenance.partition { + scannerCriteria.matches(it.scanner) + } + + if (matchingCriteria.isEmpty()) { logger.debug { - "No stored scan results found for $scannerCriteria. The following entries with " + - "incompatible scanners have been ignored: ${scanResults.map { it.scanner }}" + "No stored scan results for '${pkg.id.toCoordinates()}' match $scannerCriteria. The " + + "following entries with non-matching criteria have been ignored: " + + nonMatchingCriteria.map { it.scanner } } } - } - scanResults + matchingCriteria + } } } From 0d4b1f75932d3a6666d599dbf6359f7e8fcebdff Mon Sep 17 00:00:00 2001 From: Sebastian Schuberth Date: Wed, 27 Sep 2023 11:34:04 +0200 Subject: [PATCH 44/47] fix(ClearlyDefinedStorage): Properly parse returned VCS URLs ClearlyDefined returns VCS URLs like [1] which makes the provenance matcher fail as it checks for [2] (plus the revision independently). Properly parse the VCS URL returned by ClearlyDefined to fix this. [1]: https://github.com/vdurmont/semver4j/tree/88912638db3f6112a2b345f1638ced33a0a606e1 [2]: https://github.com/vdurmont/semver4j.git Signed-off-by: Sebastian Schuberth --- .../kotlin/storages/ClearlyDefinedStorageFunTest.kt | 6 +++--- .../src/main/kotlin/storages/ClearlyDefinedStorage.kt | 10 ++-------- 2 files changed, 5 insertions(+), 11 deletions(-) diff --git a/scanner/src/funTest/kotlin/storages/ClearlyDefinedStorageFunTest.kt b/scanner/src/funTest/kotlin/storages/ClearlyDefinedStorageFunTest.kt index 35dc0c82274bc..c452853cb88d6 100644 --- a/scanner/src/funTest/kotlin/storages/ClearlyDefinedStorageFunTest.kt +++ b/scanner/src/funTest/kotlin/storages/ClearlyDefinedStorageFunTest.kt @@ -54,7 +54,7 @@ class ClearlyDefinedStorageFunTest : StringSpec({ provenance = RepositoryProvenance( vcsInfo = VcsInfo( type = VcsType.GIT, - url = "https://github.com/vdurmont/semver4j/tree/88912638db3f6112a2b345f1638ced33a0a606e1", + url = "https://github.com/vdurmont/semver4j.git", revision = "88912638db3f6112a2b345f1638ced33a0a606e1" ), resolvedRevision = "88912638db3f6112a2b345f1638ced33a0a606e1" @@ -108,7 +108,7 @@ class ClearlyDefinedStorageFunTest : StringSpec({ provenance = RepositoryProvenance( vcsInfo = VcsInfo( type = VcsType.GIT, - url = "https://github.com/sksamuel/hoplite/tree/b3bf5d7bd3814cb7576091acfecd097cb3a79e72", + url = "https://github.com/sksamuel/hoplite.git", revision = "b3bf5d7bd3814cb7576091acfecd097cb3a79e72" ), resolvedRevision = "b3bf5d7bd3814cb7576091acfecd097cb3a79e72" @@ -141,7 +141,7 @@ class ClearlyDefinedStorageFunTest : StringSpec({ result.map { it.provenance } shouldContain RepositoryProvenance( vcsInfo = VcsInfo( type = VcsType.GIT, - url = "https://github.com/bropat/ioBroker.eusec/tree/327b125548c9b806490085a2dacfdfc6e7776803", + url = "https://github.com/bropat/ioBroker.eusec.git", revision = "327b125548c9b806490085a2dacfdfc6e7776803" ), resolvedRevision = "327b125548c9b806490085a2dacfdfc6e7776803" diff --git a/scanner/src/main/kotlin/storages/ClearlyDefinedStorage.kt b/scanner/src/main/kotlin/storages/ClearlyDefinedStorage.kt index 3dc6b18fa0f9b..82a8a852dab0d 100644 --- a/scanner/src/main/kotlin/storages/ClearlyDefinedStorage.kt +++ b/scanner/src/main/kotlin/storages/ClearlyDefinedStorage.kt @@ -34,6 +34,7 @@ import org.ossreviewtoolkit.clients.clearlydefined.ClearlyDefinedService import org.ossreviewtoolkit.clients.clearlydefined.ComponentType import org.ossreviewtoolkit.clients.clearlydefined.Coordinates import org.ossreviewtoolkit.clients.clearlydefined.toCoordinates +import org.ossreviewtoolkit.downloader.VcsHost import org.ossreviewtoolkit.model.ArtifactProvenance import org.ossreviewtoolkit.model.Hash import org.ossreviewtoolkit.model.Identifier @@ -44,8 +45,6 @@ import org.ossreviewtoolkit.model.RepositoryProvenance import org.ossreviewtoolkit.model.ScanResult import org.ossreviewtoolkit.model.ScannerDetails import org.ossreviewtoolkit.model.UnknownProvenance -import org.ossreviewtoolkit.model.VcsInfo -import org.ossreviewtoolkit.model.VcsType import org.ossreviewtoolkit.model.config.ClearlyDefinedStorageConfiguration import org.ossreviewtoolkit.model.jsonMapper import org.ossreviewtoolkit.model.utils.toClearlyDefinedCoordinates @@ -214,12 +213,7 @@ class ClearlyDefinedStorage( sourceLocation.type == ComponentType.GIT -> { RepositoryProvenance( - vcsInfo = VcsInfo( - type = VcsType.GIT, - url = sourceLocation.url.orEmpty(), - revision = sourceLocation.revision, - path = sourceLocation.path.orEmpty() - ), + vcsInfo = VcsHost.parseUrl(sourceLocation.url.orEmpty()), resolvedRevision = sourceLocation.revision ) } From bc3404ca95cd44b3db9640e9b9a638bf55e3d649 Mon Sep 17 00:00:00 2001 From: Helio Chissini de Castro Date: Wed, 27 Sep 2023 17:24:19 +0200 Subject: [PATCH 45/47] chore(SW360): Use Maven Central and update version Signed-off-by: Helio Chissini de Castro --- .../src/main/kotlin/ort-base-conventions.gradle.kts | 10 ---------- gradle/libs.versions.toml | 2 +- 2 files changed, 1 insertion(+), 11 deletions(-) diff --git a/buildSrc/src/main/kotlin/ort-base-conventions.gradle.kts b/buildSrc/src/main/kotlin/ort-base-conventions.gradle.kts index d528c164f72e0..1a0f8c7c483b5 100644 --- a/buildSrc/src/main/kotlin/ort-base-conventions.gradle.kts +++ b/buildSrc/src/main/kotlin/ort-base-conventions.gradle.kts @@ -31,16 +31,6 @@ repositories { } } - exclusiveContent { - forRepository { - maven("https://repo.eclipse.org/content/repositories/sw360-releases/") - } - - filter { - includeGroup("org.eclipse.sw360") - } - } - exclusiveContent { forRepository { maven("https://repo.gradle.org/gradle/libs-releases/") diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 6899433c4f837..4bdc6e3aeec0c 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -54,7 +54,7 @@ scanoss = "1.1.6" semver4j = "5.2.1" springCore = "5.3.30" svnkit = "1.10.11" -sw360Client = "16.0.0-m1" +sw360Client = "17.0.0-m1" toml4j = "0.7.2" wiremock = "3.0.1" xz = "1.9" From 5893bd7840585f0a8d1ad6d5ec7f4d1ae20e1e05 Mon Sep 17 00:00:00 2001 From: Sebastian Schuberth Date: Wed, 27 Sep 2023 16:02:44 +0200 Subject: [PATCH 46/47] chore(ClearlyDefinedStorageTest): Move private data below the test This eases running the tests from the IDE by clicking on the play icons in the gutter without the need to scroll down. Signed-off-by: Sebastian Schuberth --- .../storages/ClearlyDefinedStorageTest.kt | 270 +++++++++--------- 1 file changed, 135 insertions(+), 135 deletions(-) diff --git a/scanner/src/test/kotlin/storages/ClearlyDefinedStorageTest.kt b/scanner/src/test/kotlin/storages/ClearlyDefinedStorageTest.kt index 42ae3585dd9a8..93dfd46d027eb 100644 --- a/scanner/src/test/kotlin/storages/ClearlyDefinedStorageTest.kt +++ b/scanner/src/test/kotlin/storages/ClearlyDefinedStorageTest.kt @@ -66,141 +66,6 @@ import org.ossreviewtoolkit.utils.ort.OkHttpClientHelper import org.semver4j.Semver -private const val PACKAGE_TYPE = "Maven" -private const val NAMESPACE = "someNamespace" -private const val NAME = "somePackage" -private const val VERSION = "0.1.8" -private const val COMMIT = "02b7f3d06fcbbedb44563aaa88ab62db3669946e" -private const val SCANCODE_VERSION = "3.2.2" - -private const val TEST_FILES_ROOT = "src/test/assets" -private const val TEST_FILES_DIRECTORY = "clearly-defined" - -/** The name of the file with the test response from ClearlyDefined. */ -private const val RESPONSE_FILE = "scancode-$SCANCODE_VERSION.json" - -/** The ClearlyDefined coordinates referencing the test package. */ -private val COORDINATES = Coordinates(ComponentType.MAVEN, Provider.MAVEN_CENTRAL, NAMESPACE, NAME, VERSION) - -/** Path to a file contained in the test ClearlyDefined result. */ -private const val TEST_PATH = - "src/main/java/org/apache/commons/configuration2/tree/DefaultExpressionEngine.java" - -private val TEST_IDENTIFIER = - Identifier( - type = PACKAGE_TYPE, - namespace = NAMESPACE, - name = NAME, - version = VERSION - ) - -private val TEST_PACKAGE = - Package( - id = TEST_IDENTIFIER, - declaredLicenses = emptySet(), - description = "test package description", - homepageUrl = "https://www.test-package.com", - vcs = VcsInfo.EMPTY, - sourceArtifact = RemoteArtifact.EMPTY, - binaryArtifact = RemoteArtifact.EMPTY - ) - -/** The scanner details used by tests. */ -private val SCANNER_CRITERIA = ScannerCriteria("aScanner", Semver("1.0.0"), Semver("2.0.0"), "aConfig") - -/** The template for a ClearlyDefined definitions request. */ -private val DEFINITIONS_TEMPLATE = readDefinitionsTemplate() - -/** The template variable with the coordinates of the package that is requested. */ -private const val PACKAGE_VARIABLE = "<>" - -/** - * Return a storage configuration that points to the mock [server]. - */ -private fun storageConfiguration(server: WireMockServer): ClearlyDefinedStorageConfiguration { - val url = "http://localhost:${server.port()}" - return ClearlyDefinedStorageConfiguration(url) -} - -/** - * Generate the URL used by ClearlyDefined to reference the results for a package with the given [coordinates] - * produced by the tool with the [toolName] and [toolVersion]. - */ -private fun toolUrl(coordinates: Coordinates, toolName: String, toolVersion: String): String = - "$coordinates/$toolName/$toolVersion" - -/** - * Stub a request for the available harvest tools on the [server] server for the package with the given [coordinates] - * to return the specified [tools]. - */ -private fun stubHarvestTools(server: WireMockServer, coordinates: Coordinates, tools: List) { - val urlPath = "/harvest/$coordinates" - val response = tools.joinToString(separator = ",", prefix = "[", postfix = "]") { "\"$it\"" } - server.stubFor( - get(urlPathEqualTo(urlPath)) - .withQueryParam("form", equalTo("list")) - .willReturn( - aResponse().withStatus(200) - .withBody(response) - ) - ) -} - -/** - * Stub a request for the harvested data from ScanCode for the given [coordinates] on the [server] server. - */ -private fun stubHarvestToolResponse(server: WireMockServer, coordinates: Coordinates) { - val urlPath = "/harvest/${toolUrl(coordinates, "scancode", SCANCODE_VERSION)}" - server.stubFor( - get(urlPathEqualTo(urlPath)) - .withQueryParam("form", equalTo("streamed")) - .willReturn( - aResponse().withStatus(200) - .withBodyFile("$TEST_FILES_DIRECTORY/$RESPONSE_FILE") - ) - ) -} - -/** - * Stub a request for the definition's endpoint for the given [coordinates] on the [server] server. - */ -private fun stubDefinitions(server: WireMockServer, coordinates: Coordinates = COORDINATES) { - val coordinatesList = listOf(coordinates) - val expectedBody = ClearlyDefinedService.JSON.encodeToString(coordinatesList) - server.stubFor( - post(urlPathEqualTo("/definitions")) - .withRequestBody(equalToJson(expectedBody)) - .willReturn( - aResponse().withStatus(200) - .withBody(DEFINITIONS_TEMPLATE.replace(PACKAGE_VARIABLE, coordinates.toString())) - ) - ) -} - -/** - * Check that this [Result] contains the expected data and return the first scan result from the list on success. - */ -private fun Result>.shouldBeValid(block: (ScanResult.() -> Unit)? = null) { - shouldBeSuccess { - it shouldHaveSize 1 - - val scanResult = it.first() - scanResult.summary.licenseFindings.find { finding -> - finding.location.path == TEST_PATH && "Apache-2.0" in finding.license.licenses() - } shouldNot beNull() - - if (block != null) scanResult.block() - } -} - -/** - * Read the template for a ClearlyDefines definitions request from the test file. - */ -private fun readDefinitionsTemplate(): String { - val templateFile = File("$TEST_FILES_ROOT/cd_definitions.json") - return templateFile.readText() -} - class ClearlyDefinedStorageTest : WordSpec({ val server = WireMockServer( WireMockConfiguration.options() @@ -421,3 +286,138 @@ class ClearlyDefinedStorageTest : WordSpec({ } } }) + +private const val PACKAGE_TYPE = "Maven" +private const val NAMESPACE = "someNamespace" +private const val NAME = "somePackage" +private const val VERSION = "0.1.8" +private const val COMMIT = "02b7f3d06fcbbedb44563aaa88ab62db3669946e" +private const val SCANCODE_VERSION = "3.2.2" + +private const val TEST_FILES_ROOT = "src/test/assets" +private const val TEST_FILES_DIRECTORY = "clearly-defined" + +/** The name of the file with the test response from ClearlyDefined. */ +private const val RESPONSE_FILE = "scancode-$SCANCODE_VERSION.json" + +/** The ClearlyDefined coordinates referencing the test package. */ +private val COORDINATES = Coordinates(ComponentType.MAVEN, Provider.MAVEN_CENTRAL, NAMESPACE, NAME, VERSION) + +/** Path to a file contained in the test ClearlyDefined result. */ +private const val TEST_PATH = + "src/main/java/org/apache/commons/configuration2/tree/DefaultExpressionEngine.java" + +private val TEST_IDENTIFIER = + Identifier( + type = PACKAGE_TYPE, + namespace = NAMESPACE, + name = NAME, + version = VERSION + ) + +private val TEST_PACKAGE = + Package( + id = TEST_IDENTIFIER, + declaredLicenses = emptySet(), + description = "test package description", + homepageUrl = "https://www.test-package.com", + vcs = VcsInfo.EMPTY, + sourceArtifact = RemoteArtifact.EMPTY, + binaryArtifact = RemoteArtifact.EMPTY + ) + +/** The scanner details used by tests. */ +private val SCANNER_CRITERIA = ScannerCriteria("aScanner", Semver("1.0.0"), Semver("2.0.0"), "aConfig") + +/** The template for a ClearlyDefined definitions request. */ +private val DEFINITIONS_TEMPLATE = readDefinitionsTemplate() + +/** The template variable with the coordinates of the package that is requested. */ +private const val PACKAGE_VARIABLE = "<>" + +/** + * Return a storage configuration that points to the mock [server]. + */ +private fun storageConfiguration(server: WireMockServer): ClearlyDefinedStorageConfiguration { + val url = "http://localhost:${server.port()}" + return ClearlyDefinedStorageConfiguration(url) +} + +/** + * Generate the URL used by ClearlyDefined to reference the results for a package with the given [coordinates] + * produced by the tool with the [toolName] and [toolVersion]. + */ +private fun toolUrl(coordinates: Coordinates, toolName: String, toolVersion: String): String = + "$coordinates/$toolName/$toolVersion" + +/** + * Stub a request for the available harvest tools on the [server] server for the package with the given [coordinates] + * to return the specified [tools]. + */ +private fun stubHarvestTools(server: WireMockServer, coordinates: Coordinates, tools: List) { + val urlPath = "/harvest/$coordinates" + val response = tools.joinToString(separator = ",", prefix = "[", postfix = "]") { "\"$it\"" } + server.stubFor( + get(urlPathEqualTo(urlPath)) + .withQueryParam("form", equalTo("list")) + .willReturn( + aResponse().withStatus(200) + .withBody(response) + ) + ) +} + +/** + * Stub a request for the harvested data from ScanCode for the given [coordinates] on the [server] server. + */ +private fun stubHarvestToolResponse(server: WireMockServer, coordinates: Coordinates) { + val urlPath = "/harvest/${toolUrl(coordinates, "scancode", SCANCODE_VERSION)}" + server.stubFor( + get(urlPathEqualTo(urlPath)) + .withQueryParam("form", equalTo("streamed")) + .willReturn( + aResponse().withStatus(200) + .withBodyFile("$TEST_FILES_DIRECTORY/$RESPONSE_FILE") + ) + ) +} + +/** + * Stub a request for the definition's endpoint for the given [coordinates] on the [server] server. + */ +private fun stubDefinitions(server: WireMockServer, coordinates: Coordinates = COORDINATES) { + val coordinatesList = listOf(coordinates) + val expectedBody = ClearlyDefinedService.JSON.encodeToString(coordinatesList) + server.stubFor( + post(urlPathEqualTo("/definitions")) + .withRequestBody(equalToJson(expectedBody)) + .willReturn( + aResponse().withStatus(200) + .withBody(DEFINITIONS_TEMPLATE.replace(PACKAGE_VARIABLE, coordinates.toString())) + ) + ) +} + +/** + * Check that this [Result] contains the expected data and return the first scan result from the list on success. + */ +private fun Result>.shouldBeValid(block: (ScanResult.() -> Unit)? = null) { + shouldBeSuccess { + it shouldHaveSize 1 + + val scanResult = it.first() + scanResult.summary.licenseFindings.find { finding -> + finding.location.path == TEST_PATH && "Apache-2.0" in finding.license.licenses() + } shouldNot beNull() + + if (block != null) scanResult.block() + } +} + +/** + * Read the template for a ClearlyDefines definitions request from the test file. + */ +private fun readDefinitionsTemplate(): String { + val templateFile = File("$TEST_FILES_ROOT/cd_definitions.json") + return templateFile.readText() +} From 49bf674e6db1ead79e13d4669fca1bdea92770e0 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Thu, 28 Sep 2023 06:01:11 +0000 Subject: [PATCH 47/47] deps: update docker/setup-buildx-action action to v3 --- .github/actions/ortdocker/action.yml | 2 +- .github/workflows/docker-ort-runtime-ext.yml | 2 +- .github/workflows/docker-ort-runtime.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/actions/ortdocker/action.yml b/.github/actions/ortdocker/action.yml index 660c6e458d957..d4ff3ac196f47 100644 --- a/.github/actions/ortdocker/action.yml +++ b/.github/actions/ortdocker/action.yml @@ -63,7 +63,7 @@ runs: - name: Set up Docker build if: steps.check_image.outputs.result != 'found' - uses: docker/setup-buildx-action@v2 + uses: docker/setup-buildx-action@v3 - name: Login to GitHub container registry if: steps.check_image.outputs.result != 'found' diff --git a/.github/workflows/docker-ort-runtime-ext.yml b/.github/workflows/docker-ort-runtime-ext.yml index 6d35472bb8e54..c1a9efbfc9a2b 100644 --- a/.github/workflows/docker-ort-runtime-ext.yml +++ b/.github/workflows/docker-ort-runtime-ext.yml @@ -164,7 +164,7 @@ jobs: uses: actions/checkout@v4 - name: Set up Docker build - uses: docker/setup-buildx-action@v2 + uses: docker/setup-buildx-action@v3 - name: Login to GitHub container registry uses: docker/login-action@v3 diff --git a/.github/workflows/docker-ort-runtime.yml b/.github/workflows/docker-ort-runtime.yml index 8a4ed65b2b8de..63fb83751d121 100644 --- a/.github/workflows/docker-ort-runtime.yml +++ b/.github/workflows/docker-ort-runtime.yml @@ -178,7 +178,7 @@ jobs: cat .versions >> $GITHUB_ENV - name: Set up Docker build - uses: docker/setup-buildx-action@v2 + uses: docker/setup-buildx-action@v3 - name: Login to GitHub container registry uses: docker/login-action@v3