diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 373c05a7a8025..2ed6cea933dd6 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -11,7 +11,6 @@ kotlinPlugin = "1.9.22" mavenPublishPlugin = "0.27.0" versionsPlugin = "0.51.0" -antlr = "4.13.1" asciidoctorj = "2.5.11" asciidoctorjPdf = "2.3.10" clikt = "4.2.2" @@ -80,7 +79,6 @@ plugin-graalVmNativeImage = { module = "org.graalvm.buildtools:native-gradle-plu plugin-kotlin = { module = "org.jetbrains.kotlin:kotlin-gradle-plugin", version.ref = "kotlinPlugin" } plugin-mavenPublish = { module = "com.vanniktech:gradle-maven-publish-plugin", version.ref = "mavenPublishPlugin" } -antlr = { module = "org.antlr:antlr4", version.ref = "antlr" } asciidoctorj = { module = "org.asciidoctor:asciidoctorj", version.ref = "asciidoctorj" } asciidoctorj-pdf = { module = "org.asciidoctor:asciidoctorj-pdf", version.ref = "asciidoctorjPdf" } awsS3 = { module = "software.amazon.awssdk:s3", version.ref = "s3" } diff --git a/utils/spdx/build.gradle.kts b/utils/spdx/build.gradle.kts index 29b8f825e44d9..2a19c420990b7 100644 --- a/utils/spdx/build.gradle.kts +++ b/utils/spdx/build.gradle.kts @@ -28,9 +28,6 @@ import java.net.URI val spdxLicenseListVersion: String by project plugins { - // Apply core plugins. - antlr - // Apply precompiled plugins. id("ort-library-conventions") @@ -38,40 +35,7 @@ plugins { alias(libs.plugins.download) } -tasks.withType().configureEach { - arguments = arguments + listOf("-visitor") - - doLast { - // Work around https://github.com/antlr/antlr4/issues/4128. - outputDirectory.walk() - .filter { it.isFile && it.extension == "java" } - .forEach { javaFile -> - val lines = javaFile.readLines() - - val text = buildString { - lines.mapIndexed { index, line -> - val patchedLine = when { - index == 0 && line.startsWith("// Generated from ") -> line.replace('\\', '/') - else -> line - } - - appendLine(patchedLine) - } - } - - javaFile.writeText(text) - } - } -} - -sourceSets.configureEach { - val generateGrammarSource = tasks.named(getTaskName("generate", "GrammarSource")) - java.srcDir(generateGrammarSource.map { files() }) -} - dependencies { - antlr(libs.antlr) - api(libs.jackson.databind) implementation(projects.utils.commonUtils) diff --git a/utils/spdx/src/main/antlr/org/ossreviewtoolkit/utils/spdx/SpdxExpression.g4 b/utils/spdx/src/main/antlr/org/ossreviewtoolkit/utils/spdx/SpdxExpression.g4 deleted file mode 100644 index beb1aa01c6ff8..0000000000000 --- a/utils/spdx/src/main/antlr/org/ossreviewtoolkit/utils/spdx/SpdxExpression.g4 +++ /dev/null @@ -1,85 +0,0 @@ -/* - * Copyright (C) 2017 The ORT Project Authors (see ) - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - * License-Filename: LICENSE - */ - -/* - * All definitions are based on the specification for SPDX expressions: - * https://spdx.org/spdx-specification-21-web-version#h.jxpfx0ykyb60 - */ -grammar SpdxExpression; - -@header { -package org.ossreviewtoolkit.utils.spdx; -} - -/* - * Parser Rules - */ - -licenseReferenceExpression - : - DOCUMENTREFERENCE | LICENSEREFERENCE - ; - -licenseIdExpression - : - IDSTRING - PLUS? - ; - -simpleExpression - : - licenseReferenceExpression - | licenseIdExpression - ; - -compoundExpression - : - simpleExpression - | simpleExpression WITH (IDSTRING | LICENSEREFERENCE) - | compoundExpression AND compoundExpression - | compoundExpression OR compoundExpression - | OPEN compoundExpression CLOSE - ; - -licenseExpression - : - (simpleExpression | compoundExpression) - EOF - ; - -/* - * Lexer Rules - */ - -fragment ALPHA : [A-Za-z] ; -fragment DIGIT : [0-9] ; - -AND : ('AND' | 'and') ; -OR : ('OR' | 'or') ; -WITH : ('WITH' | 'with') ; - -OPEN : '(' ; -CLOSE : ')' ; -PLUS : '+' ; - -DOCUMENTREFERENCE : 'DocumentRef-' IDSTRING ':' LICENSEREFERENCE ; -LICENSEREFERENCE : 'LicenseRef-' IDSTRING ; -IDSTRING : (ALPHA | DIGIT)(ALPHA | DIGIT | '-' | '.')* ; - -WHITESPACE : ' ' -> skip ; diff --git a/utils/spdx/src/main/kotlin/SpdxErrorListener.kt b/utils/spdx/src/main/kotlin/SpdxErrorListener.kt deleted file mode 100644 index bb60e9458de8c..0000000000000 --- a/utils/spdx/src/main/kotlin/SpdxErrorListener.kt +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Copyright (C) 2017 The ORT Project Authors (see ) - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - * License-Filename: LICENSE - */ - -package org.ossreviewtoolkit.utils.spdx - -import org.antlr.v4.runtime.BaseErrorListener -import org.antlr.v4.runtime.RecognitionException -import org.antlr.v4.runtime.Recognizer - -/** - * An ANTLR error listener that throws an [SpdxException]. - */ -class SpdxErrorListener : BaseErrorListener() { - override fun syntaxError( - recognizer: Recognizer<*, *>?, - offendingSymbol: Any?, - line: Int, - charPositionInLine: Int, - msg: String?, - e: RecognitionException? - ) { - throw SpdxException(msg) - } -} diff --git a/utils/spdx/src/main/kotlin/SpdxExpressionDefaultVisitor.kt b/utils/spdx/src/main/kotlin/SpdxExpressionDefaultVisitor.kt deleted file mode 100644 index 1088fdba4a011..0000000000000 --- a/utils/spdx/src/main/kotlin/SpdxExpressionDefaultVisitor.kt +++ /dev/null @@ -1,89 +0,0 @@ -/* - * Copyright (C) 2017 The ORT Project Authors (see ) - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - * License-Filename: LICENSE - */ - -package org.ossreviewtoolkit.utils.spdx - -import org.ossreviewtoolkit.utils.spdx.SpdxExpression.Strictness -import org.ossreviewtoolkit.utils.spdx.SpdxExpressionParser.CompoundExpressionContext -import org.ossreviewtoolkit.utils.spdx.SpdxExpressionParser.LicenseExpressionContext -import org.ossreviewtoolkit.utils.spdx.SpdxExpressionParser.LicenseIdExpressionContext -import org.ossreviewtoolkit.utils.spdx.SpdxExpressionParser.LicenseReferenceExpressionContext -import org.ossreviewtoolkit.utils.spdx.SpdxExpressionParser.SimpleExpressionContext - -class SpdxExpressionDefaultVisitor(private val strictness: Strictness) : - SpdxExpressionBaseVisitor() { - override fun visitLicenseExpression(ctx: LicenseExpressionContext): SpdxExpression = - when (ctx.childCount) { - 2 -> visit(ctx.getChild(0)) - else -> throw SpdxException("SpdxExpression has invalid amount of children: '${ctx.childCount}'") - } - - override fun visitCompoundExpression(ctx: CompoundExpressionContext): SpdxExpression = - when (ctx.childCount) { - 1 -> visit(ctx.getChild(0)) - 3 -> { - if (ctx.getChild(0).text == "(" && ctx.getChild(2).text == ")") { - visit(ctx.getChild(1)) - } else { - val left = visit(ctx.getChild(0)) - val operator = ctx.getChild(1).text - - when (val uppercaseOperator = operator.uppercase()) { - SpdxExpression.WITH -> { - val right = ctx.getChild(2).text - SpdxLicenseWithExceptionExpression(left as SpdxSimpleExpression, right) - .apply { validate(strictness) } - } - else -> { - val right = visit(ctx.getChild(2)) - val spdxOperator = try { - SpdxOperator.valueOf(uppercaseOperator) - } catch (e: IllegalArgumentException) { - throw SpdxException("Illegal operator '$operator' in expression '${ctx.text}'.", e) - } - - SpdxCompoundExpression(left, spdxOperator, right) - } - } - } - } - else -> throw SpdxException("SpdxCompoundExpression has invalid amount of children: '${ctx.childCount}'") - } - - override fun visitSimpleExpression(ctx: SimpleExpressionContext): SpdxExpression = - when (ctx.childCount) { - 1 -> visit(ctx.getChild(0)) - else -> throw SpdxException("SpdxSimpleExpression has invalid amount of children: '${ctx.childCount}'") - } - - override fun visitLicenseIdExpression(ctx: LicenseIdExpressionContext): SpdxExpression = - when (ctx.childCount) { - 1 -> SpdxLicenseIdExpression(ctx.text, ctx.text.endsWith("-or-later")) - 2 -> SpdxLicenseIdExpression(ctx.text.dropLast(1) /* drop the trailing "+" */, true) - else -> throw SpdxException("SpdxLicenseIdExpression has invalid amount of children: '${ctx.childCount}'") - }.apply { validate(strictness) } - - override fun visitLicenseReferenceExpression(ctx: LicenseReferenceExpressionContext): SpdxExpression = - when (ctx.childCount) { - 1 -> SpdxLicenseReferenceExpression(ctx.text) - else -> throw SpdxException( - "SpdxLicenseReferenceExpression has invalid amount of children: '${ctx.childCount}'" - ) - }.apply { validate(strictness) } -} diff --git a/utils/spdx/src/test/kotlin/SpdxDeclaredLicenseMappingTest.kt b/utils/spdx/src/test/kotlin/SpdxDeclaredLicenseMappingTest.kt index 1b973aca416a5..55e7c0ea5d8df 100644 --- a/utils/spdx/src/test/kotlin/SpdxDeclaredLicenseMappingTest.kt +++ b/utils/spdx/src/test/kotlin/SpdxDeclaredLicenseMappingTest.kt @@ -30,6 +30,7 @@ import io.kotest.matchers.string.containADigit import io.kotest.matchers.string.shouldContain import org.ossreviewtoolkit.utils.common.titlecase +import org.ossreviewtoolkit.utils.spdx.parser.SpdxExpressionLexer class SpdxDeclaredLicenseMappingTest : WordSpec({ "The list" should { @@ -76,7 +77,7 @@ class SpdxDeclaredLicenseMappingTest : WordSpec({ licenseIdMapping.keys.forAll { declaredLicense -> @Suppress("SwallowedException") try { - val tokens = getTokensByTypeForExpression(declaredLicense) + val tokens = SpdxExpressionLexer(declaredLicense).tokens().toList() tokens.size shouldBeGreaterThanOrEqual 2 diff --git a/utils/spdx/src/test/kotlin/SpdxExpressionLexerTest.kt b/utils/spdx/src/test/kotlin/SpdxExpressionLexerTest.kt deleted file mode 100644 index 59535e3fc3c4a..0000000000000 --- a/utils/spdx/src/test/kotlin/SpdxExpressionLexerTest.kt +++ /dev/null @@ -1,78 +0,0 @@ -/* - * Copyright (C) 2017 The ORT Project Authors (see ) - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - * License-Filename: LICENSE - */ - -package org.ossreviewtoolkit.utils.spdx - -import io.kotest.assertions.throwables.shouldThrow -import io.kotest.core.spec.style.WordSpec -import io.kotest.matchers.collections.containExactly -import io.kotest.matchers.should -import io.kotest.matchers.shouldBe - -import org.antlr.v4.runtime.CharStreams - -fun getTokensByTypeForExpression(expression: String): List> { - val charStream = CharStreams.fromString(expression) - val lexer = SpdxExpressionLexer(charStream) - - lexer.removeErrorListeners() - lexer.addErrorListener(SpdxErrorListener()) - - return lexer.allTokens.map { it.type to it.text } -} - -class SpdxExpressionLexerTest : WordSpec({ - "SpdxExpressionLexer" should { - "create the correct tokens for a valid expression" { - val expression = "(license-1 AND OR license.2 WITH 0LiCeNsE-3+) and (or with license-4 license-.5((" - - val tokens = getTokensByTypeForExpression(expression) - - tokens should containExactly( - SpdxExpressionLexer.OPEN to "(", - SpdxExpressionLexer.IDSTRING to "license-1", - SpdxExpressionLexer.AND to "AND", - SpdxExpressionLexer.OR to "OR", - SpdxExpressionLexer.IDSTRING to "license.2", - SpdxExpressionLexer.WITH to "WITH", - SpdxExpressionLexer.IDSTRING to "0LiCeNsE-3", - SpdxExpressionLexer.PLUS to "+", - SpdxExpressionLexer.CLOSE to ")", - SpdxExpressionLexer.AND to "and", - SpdxExpressionLexer.OPEN to "(", - SpdxExpressionLexer.OR to "or", - SpdxExpressionLexer.WITH to "with", - SpdxExpressionLexer.IDSTRING to "license-4", - SpdxExpressionLexer.IDSTRING to "license-.5", - SpdxExpressionLexer.OPEN to "(", - SpdxExpressionLexer.OPEN to "(" - ) - } - - "fail for an invalid expression" { - val expression = "/" - - val exception = shouldThrow { - getTokensByTypeForExpression(expression) - } - - exception.message shouldBe "token recognition error at: '/'" - } - } -}) diff --git a/utils/spdx/src/test/kotlin/SpdxSimpleLicenseMappingTest.kt b/utils/spdx/src/test/kotlin/SpdxSimpleLicenseMappingTest.kt index 078d4b13fe7ea..a3cf1466c4e73 100644 --- a/utils/spdx/src/test/kotlin/SpdxSimpleLicenseMappingTest.kt +++ b/utils/spdx/src/test/kotlin/SpdxSimpleLicenseMappingTest.kt @@ -28,8 +28,12 @@ import io.kotest.matchers.nulls.beNull import io.kotest.matchers.should import io.kotest.matchers.shouldBe import io.kotest.matchers.string.containADigit +import io.kotest.matchers.types.beOfType import org.ossreviewtoolkit.utils.common.titlecase +import org.ossreviewtoolkit.utils.spdx.parser.SpdxExpressionLexer +import org.ossreviewtoolkit.utils.spdx.parser.SpdxExpressionParser +import org.ossreviewtoolkit.utils.spdx.parser.Token class SpdxSimpleLicenseMappingTest : WordSpec({ "The raw map" should { @@ -59,14 +63,15 @@ class SpdxSimpleLicenseMappingTest : WordSpec({ "The mapping" should { "contain only single ID strings" { SpdxSimpleLicenseMapping.mapping.keys.forAll { declaredLicense -> - val tokens = getTokensByTypeForExpression(declaredLicense) - val types = tokens.map { (type, _) -> type } + val tokens = SpdxExpressionLexer(declaredLicense).tokens().toList() tokens shouldHaveAtLeastSize 1 tokens shouldHaveAtMostSize 2 - tokens.joinToString("") { (_, text) -> text } shouldBe declaredLicense - types.first() shouldBe SpdxExpressionLexer.IDSTRING - types.getOrElse(1) { SpdxExpressionLexer.PLUS } shouldBe SpdxExpressionLexer.PLUS + + tokens.first() should beOfType() + tokens.getOrNull(1)?.let { it should beOfType() } + + SpdxExpressionParser(tokens.asSequence()).parse().toString() shouldBe declaredLicense } }