feat(Repository): Use KnownProvenance instead of VcsInfo

In order to allow source artifacts as well as local source code to be scanned, we require a more generallized Repository, which allows any type of `KnownProvenance`. While we still set the signature of `Repository` to allow `KnownProvenance`s, this commit focuses on accomidateing `RepositoryProvenance` as then main input for now. Therefore we wrap `VcsInfo` with `RepositoryProvenance`, whenever it is used as input, and unwrap it, when it is produced as output. This gives us a quick update of the now depreacted code, without the necessity to support all `KnownProvenance` types from the get go. We also update any realted tests, mostly the expected `OrtResults`, but also some `Repository` definitions and calls. To avoid having `vcs` and `vcsProcessed` appear in the `OrtResult` output, we change them to be a method (fun) instead of a variable (val). This is still a soft refactor, to keep the widely used `vcsProcessed` available for now. We might still remove it later. Signed-off-by: Jens Keim <[email protected]>
oss-review-toolkit · Jun 26, 2024 · b6d59b2 · b6d59b2
1 parent 8de3e7c
commit b6d59b2
Show file tree

Hide file tree

Showing 53 changed files with 386 additions and 419 deletions.
diff --git a/analyzer/src/main/kotlin/Analyzer.kt b/analyzer/src/main/kotlin/Analyzer.kt
@@ -42,6 +42,7 @@ import org.ossreviewtoolkit.model.AnalyzerResult
 import org.ossreviewtoolkit.model.AnalyzerRun
 import org.ossreviewtoolkit.model.OrtResult
 import org.ossreviewtoolkit.model.Repository
+import org.ossreviewtoolkit.model.RepositoryProvenance
 import org.ossreviewtoolkit.model.config.AnalyzerConfiguration
 import org.ossreviewtoolkit.model.config.Excludes
 import org.ossreviewtoolkit.model.config.RepositoryConfiguration
@@ -135,11 +136,18 @@ class Analyzer(private val config: AnalyzerConfiguration, private val labels: Ma
 
         val workingTree = VersionControlSystem.forDirectory(info.absoluteProjectPath)
         val vcs = workingTree?.getInfo().orEmpty()
+        val revision = workingTree?.getRevision().orEmpty()
         val nestedVcs = workingTree?.getNested()?.filter { (path, _) ->
             // Only include nested VCS if they are part of the analyzed directory.
             workingTree.getRootPath().resolve(path).startsWith(info.absoluteProjectPath)
         }.orEmpty()
-        val repository = Repository(vcs = vcs, nestedRepositories = nestedVcs, config = info.repositoryConfiguration)
+        val repository = Repository(
+            provenance = RepositoryProvenance(vcs, revision),
+            nestedRepositories = nestedVcs.map {
+                it.key to RepositoryProvenance(it.value, it.value.revision)
+            }.toMap(),
+            config = info.repositoryConfiguration
+        )
 
         val endTime = Instant.now()
 

diff --git a/cli/src/funTest/assets/git-repo-expected-output.yml b/cli/src/funTest/assets/git-repo-expected-output.yml
@@ -1,46 +1,55 @@
 ---
 repository:
-  vcs:
-    type: "GitRepo"
-    url: "https://github.com/oss-review-toolkit/ort-test-data-git-repo?manifest=manifest.xml"
-    revision: "31588aa8f8555474e1c3c66a359ec99e4cd4b1fa"
-    path: ""
-  vcs_processed:
-    type: "GitRepo"
-    url: "https://github.com/oss-review-toolkit/ort-test-data-git-repo.git?manifest=manifest.xml"
-    revision: "31588aa8f8555474e1c3c66a359ec99e4cd4b1fa"
-    path: ""
+  provenance:
+    vcs_info:
+      type: "GitRepo"
+      url: "https://github.com/oss-review-toolkit/ort-test-data-git-repo?manifest=manifest.xml"
+      revision: "31588aa8f8555474e1c3c66a359ec99e4cd4b1fa"
+      path: ""
+    resolved_revision: "31588aa8f8555474e1c3c66a359ec99e4cd4b1fa"
   nested_repositories:
     spdx-tools:
-      type: "Git"
-      url: "https://github.com/spdx/tools"
-      revision: "e179fae47590eccedc46186ea0ce20cbade5fda7"
-      path: ""
+      vcs_info:
+        type: "Git"
+        url: "https://github.com/spdx/tools"
+        revision: "e179fae47590eccedc46186ea0ce20cbade5fda7"
+        path: ""
+      resolved_revision: "e179fae47590eccedc46186ea0ce20cbade5fda7"
     submodules:
-      type: "Git"
-      url: "https://github.com/oss-review-toolkit/ort-test-data-git-submodules"
-      revision: "fcea94bab5835172e826afddb9f6427274c983b9"
-      path: ""
+      vcs_info:
+        type: "Git"
+        url: "https://github.com/oss-review-toolkit/ort-test-data-git-submodules"
+        revision: "fcea94bab5835172e826afddb9f6427274c983b9"
+        path: ""
+      resolved_revision: "fcea94bab5835172e826afddb9f6427274c983b9"
     submodules/commons-text:
-      type: "Git"
-      url: "https://github.com/apache/commons-text.git"
-      revision: "7643b12421100d29fd2b78053e77bcb04a251b2e"
-      path: ""
+      vcs_info:
+        type: "Git"
+        url: "https://github.com/apache/commons-text.git"
+        revision: "7643b12421100d29fd2b78053e77bcb04a251b2e"
+        path: ""
+      resolved_revision: "7643b12421100d29fd2b78053e77bcb04a251b2e"
     submodules/test-data-npm:
-      type: "Git"
-      url: "https://github.com/oss-review-toolkit/ort-test-data-npm.git"
-      revision: "ad0367b7b9920144a47b8d30cc0c84cea102b821"
-      path: ""
+      vcs_info:
+        type: "Git"
+        url: "https://github.com/oss-review-toolkit/ort-test-data-npm.git"
+        revision: "ad0367b7b9920144a47b8d30cc0c84cea102b821"
+        path: ""
+      resolved_revision: "ad0367b7b9920144a47b8d30cc0c84cea102b821"
     submodules/test-data-npm/isarray:
-      type: "Git"
-      url: "https://github.com/juliangruber/isarray.git"
-      revision: "63ea4ca0a0d6b0574d6a470ebd26880c3026db4a"
-      path: ""
+      vcs_info:
+        type: "Git"
+        url: "https://github.com/juliangruber/isarray.git"
+        revision: "63ea4ca0a0d6b0574d6a470ebd26880c3026db4a"
+        path: ""
+      resolved_revision: "63ea4ca0a0d6b0574d6a470ebd26880c3026db4a"
     submodules/test-data-npm/long.js:
-      type: "Git"
-      url: "https://github.com/dcodeIO/long.js.git"
-      revision: "941c5c62471168b5d18153755c2a7b38d2560e58"
-      path: ""
+      vcs_info:
+        type: "Git"
+        url: "https://github.com/dcodeIO/long.js.git"
+        revision: "941c5c62471168b5d18153755c2a7b38d2560e58"
+        path: ""
+      resolved_revision: "941c5c62471168b5d18153755c2a7b38d2560e58"
   config: {}
 analyzer:
   start_time: "1970-01-01T00:00:00Z"

diff --git a/cli/src/funTest/assets/gradle-all-dependencies-expected-result-with-curations.yml b/cli/src/funTest/assets/gradle-all-dependencies-expected-result-with-curations.yml
@@ -1,15 +1,12 @@
 ---
 repository:
-  vcs:
-    type: "Git"
-    url: "<REPLACE_URL>"
-    revision: "<REPLACE_REVISION>"
-    path: "plugins/package-managers/gradle/src/funTest/assets/projects/synthetic/gradle"
-  vcs_processed:
-    type: "Git"
-    url: "<REPLACE_URL_PROCESSED>"
-    revision: "<REPLACE_REVISION>"
-    path: "plugins/package-managers/gradle/src/funTest/assets/projects/synthetic/gradle"
+  provenance:
+    vcs_info:
+      type: "Git"
+      url: "<REPLACE_URL>"
+      revision: "<REPLACE_REVISION>"
+      path: "plugins/package-managers/gradle/src/funTest/assets/projects/synthetic/gradle"
+    resolved_revision: "<REPLACE_REVISION>"
   config:
     excludes:
       paths:

diff --git a/cli/src/funTest/assets/semver4j-ort-result.yml b/cli/src/funTest/assets/semver4j-ort-result.yml
@@ -1,15 +1,12 @@
 ---
 repository:
-  vcs:
-    type: "Git"
-    url: "https://github.com/vdurmont/semver4j.git"
-    revision: "7653e418d610ffcd2811bcb55fd72d00d420950b"
-    path: ""
-  vcs_processed:
-    type: "Git"
-    url: "https://github.com/vdurmont/semver4j.git"
-    revision: "7653e418d610ffcd2811bcb55fd72d00d420950b"
-    path: ""
+  provenance:
+    vcs_info:
+      type: "Git"
+      url: "https://github.com/vdurmont/semver4j.git"
+      revision: "7653e418d610ffcd2811bcb55fd72d00d420950b"
+      path: ""
+    resolved_revision: "<REPLACE_REVISION>"
   config:
     excludes:
       scopes:

diff --git a/evaluator/src/main/kotlin/ProjectSourceRule.kt b/evaluator/src/main/kotlin/ProjectSourceRule.kt
@@ -91,7 +91,7 @@ open class ProjectSourceRule(
     /**
      * Return the [VcsType] of the project's code repository.
      */
-    fun projectSourceGetVcsType(): VcsType = ortResult.repository.vcsProcessed.type
+    fun projectSourceGetVcsType(): VcsType = ortResult.repository.vcsProcessed().type
 
     /**
      * Return the file paths matching any of the given [glob expressions][patterns] with its file content matching

diff --git a/evaluator/src/main/kotlin/RuleSet.kt b/evaluator/src/main/kotlin/RuleSet.kt
@@ -159,7 +159,7 @@ fun ruleSet(
     licenseInfoResolver: LicenseInfoResolver = ortResult.createLicenseInfoResolver(),
     resolutionProvider: ResolutionProvider = DefaultResolutionProvider.create(),
     projectSourceResolver: SourceTreeResolver = SourceTreeResolver.forRemoteRepository(
-        ortResult.repository.vcsProcessed
+        ortResult.repository.vcsProcessed()
     ),
     configure: RuleSet.() -> Unit = { }
 ) = RuleSet(ortResult, licenseInfoResolver, resolutionProvider, projectSourceResolver).apply(configure)
diff --git a/evaluator/src/test/kotlin/ProjectSourceRuleTest.kt b/evaluator/src/test/kotlin/ProjectSourceRuleTest.kt
@@ -208,7 +208,7 @@ private fun createOrtResult(
     }
 
     return OrtResult.EMPTY.copy(
-        repository = Repository(vcsInfo),
+        repository = Repository(RepositoryProvenance(vcsInfo, vcsInfo.revision)),
         analyzer = AnalyzerRun.EMPTY.copy(
             result = AnalyzerResult.EMPTY.copy(
                 projects = setOf(

diff --git a/evaluator/src/test/kotlin/TestData.kt b/evaluator/src/test/kotlin/TestData.kt
@@ -37,6 +37,7 @@ import org.ossreviewtoolkit.model.Package
 import org.ossreviewtoolkit.model.PackageLinkage
 import org.ossreviewtoolkit.model.Project
 import org.ossreviewtoolkit.model.Repository
+import org.ossreviewtoolkit.model.RepositoryProvenance
 import org.ossreviewtoolkit.model.ScanResult
 import org.ossreviewtoolkit.model.ScanSummary
 import org.ossreviewtoolkit.model.ScannerDetails
@@ -180,7 +181,7 @@ val allProjects = setOf(
 
 val ortResult = OrtResult(
     repository = Repository(
-        vcs = VcsInfo.EMPTY,
+        provenance = RepositoryProvenance(VcsInfo.EMPTY, ""),
         config = RepositoryConfiguration(
             excludes = Excludes(
                 paths = listOf(

diff --git a/helper-cli/src/funTest/assets/create-analyzer-result-from-pkg-list-expected-output.yml b/helper-cli/src/funTest/assets/create-analyzer-result-from-pkg-list-expected-output.yml
@@ -1,15 +1,12 @@
 ---
 repository:
-  vcs:
-    type: "Git"
-    url: "https://github.com/example/project.git"
-    revision: "2222222222222222222222222222222222222222"
-    path: "vcs-path/project"
-  vcs_processed:
-    type: "Git"
-    url: "https://github.com/example/project.git"
-    revision: "2222222222222222222222222222222222222222"
-    path: "vcs-path/project"
+  provenance:
+    vcs_info:
+      type: "Git"
+      url: "https://github.com/example/project.git"
+      revision: "2222222222222222222222222222222222222222"
+      path: "vcs-path/project"
+    resolved_revision: "2222222222222222222222222222222222222222"
   config:
     excludes:
       scopes:

diff --git a/helper-cli/src/main/kotlin/commands/CreateAnalyzerResultFromPackageListCommand.kt b/helper-cli/src/main/kotlin/commands/CreateAnalyzerResultFromPackageListCommand.kt
@@ -41,6 +41,7 @@ import org.ossreviewtoolkit.model.PackageReference
 import org.ossreviewtoolkit.model.Project
 import org.ossreviewtoolkit.model.RemoteArtifact
 import org.ossreviewtoolkit.model.Repository
+import org.ossreviewtoolkit.model.RepositoryProvenance
 import org.ossreviewtoolkit.model.Scope
 import org.ossreviewtoolkit.model.VcsInfo
 import org.ossreviewtoolkit.model.VcsType
@@ -118,7 +119,7 @@ internal class CreateAnalyzerResultFromPackageListCommand : CliktCommand(
                 environment = Environment()
             ),
             repository = Repository(
-                vcs = projectVcs.normalize(),
+                provenance = RepositoryProvenance(projectVcs.normalize(), projectVcs.revision),
                 config = RepositoryConfiguration(
                     excludes = Excludes(
                         scopes = listOf(

diff --git a/helper-cli/src/main/kotlin/utils/Extensions.kt b/helper-cli/src/main/kotlin/utils/Extensions.kt
@@ -38,9 +38,7 @@ import org.ossreviewtoolkit.model.OrtResult
 import org.ossreviewtoolkit.model.PackageCuration
 import org.ossreviewtoolkit.model.Project
 import org.ossreviewtoolkit.model.Provenance
-import org.ossreviewtoolkit.model.Repository
 import org.ossreviewtoolkit.model.RepositoryProvenance
-import org.ossreviewtoolkit.model.RuleViolation
 import org.ossreviewtoolkit.model.ScanResult
 import org.ossreviewtoolkit.model.Severity
 import org.ossreviewtoolkit.model.SourceCodeOrigin
@@ -691,10 +689,12 @@ internal fun OrtResult.getScanResultFor(packageConfiguration: PackageConfigurati
  * tree.
  */
 internal fun OrtResult.getRepositoryPaths(): Map<String, Set<String>> {
-    val result = mutableMapOf(repository.vcsProcessed.url to mutableSetOf(""))
+    val result = mutableMapOf(repository.vcsProcessed().url to mutableSetOf(""))
 
-    repository.nestedRepositories.mapValues { (path, vcsInfo) ->
-        result.getOrPut(vcsInfo.url) { mutableSetOf() } += path
+    repository.nestedRepositories.mapValues { (path, provenance) ->
+        if (provenance is RepositoryProvenance) {
+            result.getOrPut(provenance.vcsInfo.url) { mutableSetOf() } += path
+        }
     }
 
     // For some Git-repo projects `OrtResult.repository.nestedRepositories´ misses some nested repositories for Git

diff --git a/helper-cli/src/main/kotlin/utils/Utils.kt b/helper-cli/src/main/kotlin/utils/Utils.kt
@@ -25,8 +25,10 @@ import java.io.File
 
 import org.ossreviewtoolkit.downloader.VersionControlSystem
 import org.ossreviewtoolkit.model.Identifier
+import org.ossreviewtoolkit.model.KnownProvenance
 import org.ossreviewtoolkit.model.OrtResult
 import org.ossreviewtoolkit.model.PackageCuration
+import org.ossreviewtoolkit.model.RepositoryProvenance
 import org.ossreviewtoolkit.model.VcsInfo
 import org.ossreviewtoolkit.model.config.LicenseFindingCuration
 import org.ossreviewtoolkit.model.config.PathExclude
@@ -68,8 +70,10 @@ internal fun findRepositoryPaths(directory: File): Map<String, Set<String>> {
 
     val result = mutableMapOf<String, MutableSet<String>>()
 
-    findRepositories(directory).forEach { (path, vcs) ->
-        result.getOrPut(vcs.url.replaceCredentialsInUri()) { mutableSetOf() } += path
+    findRepositories(directory).forEach { (path, provenance) ->
+        if (provenance is RepositoryProvenance) {
+            result.getOrPut(provenance.vcsInfo.url.replaceCredentialsInUri()) { mutableSetOf() } += path
+        }
     }
 
     return result
@@ -79,14 +83,17 @@ internal fun findRepositoryPaths(directory: File): Map<String, Set<String>> {
  * Search the given [directory] for repositories and return a mapping from paths where each respective repository was
  * found to the corresponding [VcsInfo].
  */
-internal fun findRepositories(directory: File): Map<String, VcsInfo> {
+internal fun findRepositories(directory: File): Map<String, KnownProvenance> {
     require(directory.isDirectory)
 
     val workingTree = VersionControlSystem.forDirectory(directory)
-    return workingTree?.getNested()?.filter { (path, _) ->
+    val nestedVcs = workingTree?.getNested()?.filter { (path, _) ->
         // Only include nested VCS if they are part of the analyzed directory.
         workingTree.getRootPath().resolve(path).startsWith(directory)
     }.orEmpty()
+    return nestedVcs.map {
+        it.key to RepositoryProvenance(it.value, it.value.revision)
+    }.toMap()
 }
 
 /**
@@ -166,15 +173,17 @@ internal data class ProcessedCopyrightStatement(
  */
 internal fun getLicenseFindingCurationsByRepository(
     curations: Collection<LicenseFindingCuration>,
-    nestedRepositories: Map<String, VcsInfo>
+    nestedRepositories: Map<String, KnownProvenance>
 ): RepositoryLicenseFindingCurations {
     val result = mutableMapOf<String, MutableList<LicenseFindingCuration>>()
 
-    nestedRepositories.forEach { (path, vcs) ->
-        val pathExcludesForRepository = result.getOrPut(vcs.url) { mutableListOf() }
-        curations.forEach { curation ->
-            curation.path.withoutPrefix("$path/")?.let {
-                pathExcludesForRepository += curation.copy(path = it)
+    nestedRepositories.forEach { (path, provenance) ->
+        if (provenance is RepositoryProvenance) {
+            val pathExcludesForRepository = result.getOrPut(provenance.vcsInfo.url) { mutableListOf() }
+            curations.forEach { curation ->
+                curation.path.withoutPrefix("$path/")?.let {
+                    pathExcludesForRepository += curation.copy(path = it)
+                }
             }
         }
     }
@@ -187,15 +196,17 @@ internal fun getLicenseFindingCurationsByRepository(
  */
 internal fun getPathExcludesByRepository(
     pathExcludes: Collection<PathExclude>,
-    nestedRepositories: Map<String, VcsInfo>
+    nestedRepositories: Map<String, KnownProvenance>
 ): RepositoryPathExcludes {
     val result = mutableMapOf<String, MutableList<PathExclude>>()
 
-    nestedRepositories.forEach { (path, vcs) ->
-        val pathExcludesForRepository = result.getOrPut(vcs.url) { mutableListOf() }
-        pathExcludes.forEach { pathExclude ->
-            pathExclude.pattern.withoutPrefix("$path/")?.let {
-                pathExcludesForRepository += pathExclude.copy(pattern = it)
+    nestedRepositories.forEach { (path, provenance) ->
+        if (provenance is RepositoryProvenance) {
+            val pathExcludesForRepository = result.getOrPut(provenance.vcsInfo.url) { mutableListOf() }
+            pathExcludes.forEach { pathExclude ->
+                pathExclude.pattern.withoutPrefix("$path/")?.let {
+                    pathExcludesForRepository += pathExclude.copy(pattern = it)
+                }
             }
         }
     }

diff --git a/model/src/main/kotlin/OrtResult.kt b/model/src/main/kotlin/OrtResult.kt
@@ -194,7 +194,9 @@ data class OrtResult(
     }
 
     private val relativeProjectVcsPath: Map<Identifier, String?> by lazy {
-        getProjects().associateBy({ it.id }, { repository.getRelativePath(it.vcsProcessed) })
+        getProjects().associateBy({ it.id }, {
+            repository.getRelativePath(RepositoryProvenance(it.vcsProcessed, it.vcsProcessed.revision))
+        })
     }
 
     /**