From b778c251667ab37008ac0fccba0a83689b65e40d Mon Sep 17 00:00:00 2001 From: Frank Viernau Date: Thu, 19 Oct 2023 21:59:58 +0200 Subject: [PATCH] analyzer: Support references to local modules with `GoMod` The only way of referencing a module on the local file system from another is by specifying a relative or absolute path via a replace directive. This is currently not supported by ORT's Go Modules integration. Implement that support, so that a reference inbetween two modules on the local file system gets represent as a `PackageReference` between the corresponding `Project`s (in ORT speak). However, keep on failing in case a referenced module is not located within the analysis root, because that would violate ORT's package manager API. Signed-off-by: Frank Viernau --- .../funTest/kotlin/managers/GoModFunTest.kt | 32 +++++++++++++++---- analyzer/src/main/kotlin/managers/GoMod.kt | 26 ++++++++++++--- .../src/testFixtures/kotlin/Extensions.kt | 17 ++++++---- 3 files changed, 57 insertions(+), 18 deletions(-) diff --git a/analyzer/src/funTest/kotlin/managers/GoModFunTest.kt b/analyzer/src/funTest/kotlin/managers/GoModFunTest.kt index fc980cbcb083a..2c194eb3d7e03 100644 --- a/analyzer/src/funTest/kotlin/managers/GoModFunTest.kt +++ b/analyzer/src/funTest/kotlin/managers/GoModFunTest.kt @@ -22,10 +22,12 @@ package org.ossreviewtoolkit.analyzer.managers import io.kotest.core.spec.style.StringSpec import io.kotest.matchers.collections.shouldHaveSize import io.kotest.matchers.should +import org.ossreviewtoolkit.downloader.VersionControlSystem import org.ossreviewtoolkit.model.toYaml import org.ossreviewtoolkit.utils.test.getAssetFile import org.ossreviewtoolkit.utils.test.matchExpectedResult +import java.io.File class GoModFunTest : StringSpec({ val testDir = getAssetFile("projects/synthetic") @@ -93,12 +95,28 @@ class GoModFunTest : StringSpec({ result.toYaml() should matchExpectedResult(expectedResultFile, definitionFile) } - "Local module dependencies make the analysis fail" { - // TODO: Implement support for local dependencies, see https://github.com/oss-review-toolkit/ort/issues/7649. - val definitionFile = testDir.resolve("gomod-submodules/app/go.mod") - - val result = create("GoMod").resolveSingleProject(definitionFile) - - result.issues shouldHaveSize 1 + "Project dependencies with a (relative) local module dependency are detected correctly" { + val definitionFileApp = testDir.resolve("gomod-submodules/app/go.mod") + val definitionFileUtils = testDir.resolve("gomod-submodules/utils/go.mod") + val expectedResultFile = testDir.resolve("gomod-submodules-embed-expected-output.yml") + val expectedDefinitionFilePathUtils = getDefinitionFilePath(definitionFileUtils) + + val result = create("GoMod").collateMultipleProjects(definitionFileApp, definitionFileUtils) + + result.withResolvedScopes().toYaml() should matchExpectedResult( + expectedResultFile, + definitionFileApp, + custom = mapOf( + "" to expectedDefinitionFilePathUtils, + "" to expectedDefinitionFilePathUtils.substringBeforeLast('/') + ) + ) } }) + +private fun getDefinitionFilePath(definitionFile: File): String { + val projectDir = definitionFile.parentFile + val vcsDir = VersionControlSystem.forDirectory(projectDir)!! + val path = vcsDir.getPathToRoot(projectDir) + return "$path/${definitionFile.name}" +} diff --git a/analyzer/src/main/kotlin/managers/GoMod.kt b/analyzer/src/main/kotlin/managers/GoMod.kt index 6cdf3ab5aaf68..12f372323d095 100644 --- a/analyzer/src/main/kotlin/managers/GoMod.kt +++ b/analyzer/src/main/kotlin/managers/GoMod.kt @@ -220,15 +220,20 @@ class GoMod( return if (version.isBlank()) { // If the version is blank, it is a project in ORT speak. - check(main) { "Found a local module dependency which is not supported." } - checkNotNull(dir) { "For projects, the directory is expected to not be null." } + val projectDir = File(dir) + + require(projectDir.absoluteFile.startsWith(analysisRoot.absoluteFile)) { + "A replace directive references a module in '${projectDir.absolutePath}' outside of analysis root " + + "which is not supported." + } + Identifier( type = managerName, namespace = "", - name = path, - version = processProjectVcs(File(dir)).revision + name = getProjectName(projectDir), + version = processProjectVcs(projectDir).revision ) } else { // If the version is not blank, it is a package in ORT speak. @@ -241,6 +246,19 @@ class GoMod( } } + private fun getProjectName(projectDir: File): String { + projectDir.resolve("go.mod").let { goModFile -> + require(goModFile.isFile) { + "Expected file '$goModFile.' which does not exist." + } + } + + val list = runGo("list", "-m", "-json", "-buildvcs=false", workingDir = projectDir) + + return list.stdout.byteInputStream().use { JSON.decodeToSequence(it) }.single().path + } + + /** * Return the list of all modules contained in the dependency tree with resolved versions and the 'replace' * directive applied. diff --git a/analyzer/src/testFixtures/kotlin/Extensions.kt b/analyzer/src/testFixtures/kotlin/Extensions.kt index 1a871d1d22c7f..1a7f81a9bcb69 100644 --- a/analyzer/src/testFixtures/kotlin/Extensions.kt +++ b/analyzer/src/testFixtures/kotlin/Extensions.kt @@ -20,6 +20,7 @@ package org.ossreviewtoolkit.analyzer.managers import io.kotest.core.spec.Spec +import io.kotest.inspectors.forAll import io.kotest.matchers.collections.haveSize import io.kotest.matchers.collections.shouldHaveAtLeastSize import io.kotest.matchers.nulls.shouldNotBeNull @@ -63,19 +64,21 @@ fun PackageManager.resolveSingleProject(definitionFile: File, resolveScopes: Boo } /** - * Resolve the dependencies of a [definitionFile] which should create at least one project. All created projects will be - * collated in an [AnalyzerResult] with their dependency graph. + * Resolve the dependencies of all [definitionFiles] which should create at least one project. All created projects will + * be collated in an [AnalyzerResult] with their dependency graph. */ -fun PackageManager.collateMultipleProjects(definitionFile: File): AnalyzerResult { - val managerResult = resolveDependencies(listOf(definitionFile), emptyMap()) +fun PackageManager.collateMultipleProjects(vararg definitionFiles: File): AnalyzerResult { + val managerResult = resolveDependencies(definitionFiles.toList(), emptyMap()) val builder = AnalyzerResultBuilder() managerResult.dependencyGraph?.let { builder.addDependencyGraph(managerName, it).addPackages(managerResult.sharedPackages) } - managerResult.projectResults[definitionFile].shouldNotBeNull { - this shouldHaveAtLeastSize 1 - forEach { builder.addResult(it) } + definitionFiles.forAll { definitionFile -> + managerResult.projectResults[definitionFile].shouldNotBeNull { + this shouldHaveAtLeastSize 1 + forEach { builder.addResult(it) } + } } return builder.build()