From 66970c78e86b909574c9b1c12ca4a4e5c5afe0a5 Mon Sep 17 00:00:00 2001 From: Roman Chernyatchik Date: Fri, 11 Oct 2024 17:50:55 +0200 Subject: [PATCH] refactor: 1) Force SnakemakeAPI to load data from snakemake_api.yaml instead of hardcoded stings 2) Change SmkWeapperCrawler to work w/o ability to load data from SnakemakeFrameworkAPIProvider service 3) avoid cyclic dependency from SnakemakeFrameworkAPIProvider --- build.gradle.kts | 2 + snakemake_api.yaml | 26 +++++- .../snakecharm/codeInsight/SnakemakeAPI.kt | 55 +++++------- .../SmkKeywordCompletionContributor.kt | 16 +--- .../completion/wrapper/SmkWrapperCrawler.kt | 52 +++++++++--- .../completion/wrapper/SmkWrapperStorage.kt | 6 +- .../wrapper/SnakemakeStartupActivity.kt | 5 +- .../framework/SmkFrameworkProjectSettings.kt | 2 +- ...er.kt => SnakemakeFrameworkAPIProvider.kt} | 85 ++++++++++++++----- .../SmkDepreciatedKeywordsInspection.kt | 8 +- .../snakecharm/lang/SnakemakeNames.kt | 2 +- src/test/kotlin/features/glue/ActionsSteps.kt | 3 +- src/test/kotlin/features/glue/FilesSteps.kt | 4 +- src/test/kotlin/features/glue/StepDefs.kt | 5 +- .../completion/keywords_completion.feature | 7 +- 15 files changed, 183 insertions(+), 95 deletions(-) rename src/main/kotlin/com/jetbrains/snakecharm/framework/{SmkFrameworkAPIProvider.kt => SnakemakeFrameworkAPIProvider.kt} (78%) diff --git a/build.gradle.kts b/build.gradle.kts index b2e3cea4..7525e3b3 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -275,6 +275,7 @@ tasks { gradleProperty("snakemakeWrappersRepoPath").get(), gradleProperty("snakemakeWrappersRepoVersion").get(), layout.buildDirectory.file("bundledWrappers/smk-wrapper-storage-bundled.cbor").get(), + layout.projectDirectory.file("snakemake_api.yaml") ) maxHeapSize = "1024m" // Not much RAM is available on TC agents } @@ -297,6 +298,7 @@ tasks { layout.projectDirectory.file("testData/wrappers_storage"), "test", layout.buildDirectory.file("bundledWrappers/smk-wrapper-storage.test.cbor").get(), + layout.projectDirectory.file("snakemake_api.yaml") ) maxHeapSize = "1024m" // Not much RAM is available on TC agents } diff --git a/snakemake_api.yaml b/snakemake_api.yaml index 81545019..b821bfb3 100644 --- a/snakemake_api.yaml +++ b/snakemake_api.yaml @@ -1,3 +1,5 @@ +annotationsFormatVersion: 1 + # Default Language Level used for all projects: defaultVersion: "7.32.4" @@ -31,9 +33,13 @@ changelog: introduced: - name: "retries" type: "rule-like" + - version: "7.0.0" + introduced: + - name: "template_engine" + type: "rule-like" - version: "6.15.0" introduced: - - name: "default_rule" + - name: "default_target" type: "rule-like" - version: "6.2.0" introduced: @@ -45,6 +51,10 @@ changelog: type: "top-level" - name: "module" type: "top-level" + - name: "containerized" + type: "top-level" + - name: "containerized" + type: "rule-like" deprecated: - name: "subworkflow" type: "top-level" @@ -53,6 +63,10 @@ changelog: introduced: - name: "name" type: "rule-like" + - version: "5.12.0" + introduced: + - name: "cache" + type: "rule-like" - version: "5.11.0" deprecated: - name: "singularity" @@ -68,6 +82,10 @@ changelog: type: "top-level" - name: "container" type: "rule-like" + - version: "5.9.0" + introduced: + - name: "envmodules" + type: "rule-like" - version: "5.4.0" deprecated: - name: "dynamic" @@ -77,6 +95,10 @@ changelog: introduced: - name: "cwl" type: "rule-like" + - name: "conda" + type: "rule-like" + - name: "singularity" + type: "rule-like" - name: "singularity" type: "top-level" - version: "3.8.0" @@ -94,4 +116,6 @@ changelog: - version: "3.5.2" introduced: - name: "script" + type: "rule-like" + - name: "notebook" type: "rule-like" \ No newline at end of file diff --git a/src/main/kotlin/com/jetbrains/snakecharm/codeInsight/SnakemakeAPI.kt b/src/main/kotlin/com/jetbrains/snakecharm/codeInsight/SnakemakeAPI.kt index 7c3fe6b3..dacb222e 100644 --- a/src/main/kotlin/com/jetbrains/snakecharm/codeInsight/SnakemakeAPI.kt +++ b/src/main/kotlin/com/jetbrains/snakecharm/codeInsight/SnakemakeAPI.kt @@ -1,7 +1,8 @@ package com.jetbrains.snakecharm.codeInsight +import com.jetbrains.snakecharm.codeInsight.SnakemakeAPICompanion.RULE_OR_CHECKPOINT_ARGS_SECTION_KEYWORDS_HARDCODED +import com.jetbrains.snakecharm.framework.SnakemakeFrameworkAPIProvider import com.jetbrains.snakecharm.lang.SnakemakeNames -import com.jetbrains.snakecharm.lang.SnakemakeNames.CHECKPOINT_KEYWORD import com.jetbrains.snakecharm.lang.SnakemakeNames.MODULE_CONFIG_KEYWORD import com.jetbrains.snakecharm.lang.SnakemakeNames.MODULE_META_WRAPPER_KEYWORD import com.jetbrains.snakecharm.lang.SnakemakeNames.MODULE_REPLACE_PREFIX_KEYWORD @@ -53,7 +54,6 @@ import com.jetbrains.snakecharm.lang.SnakemakeNames.SNAKEMAKE_IO_METHOD_TEMP import com.jetbrains.snakecharm.lang.SnakemakeNames.SNAKEMAKE_IO_METHOD_TOUCH import com.jetbrains.snakecharm.lang.SnakemakeNames.SNAKEMAKE_IO_METHOD_UNPACK import com.jetbrains.snakecharm.lang.SnakemakeNames.USE_EXCLUDE_KEYWORD -import com.jetbrains.snakecharm.lang.SnakemakeNames.USE_KEYWORD import com.jetbrains.snakecharm.lang.SnakemakeNames.WORKFLOW_CONFIGFILE_KEYWORD import com.jetbrains.snakecharm.lang.SnakemakeNames.WORKFLOW_CONTAINERIZED_KEYWORD import com.jetbrains.snakecharm.lang.SnakemakeNames.WORKFLOW_CONTAINER_KEYWORD @@ -159,29 +159,13 @@ object SnakemakeAPI { WORKFLOW_PEPFILE_KEYWORD ) - /** - * For rules parsing - */ - val RULE_OR_CHECKPOINT_ARGS_SECTION_KEYWORDS = setOf( - SECTION_OUTPUT, SECTION_INPUT, SECTION_PARAMS, SECTION_LOG, SECTION_RESOURCES, - SECTION_BENCHMARK, SECTION_VERSION, SECTION_MESSAGE, SECTION_SHELL, SECTION_THREADS, SECTION_SINGULARITY, - SECTION_PRIORITY, SECTION_WILDCARD_CONSTRAINTS, SECTION_GROUP, SECTION_SHADOW, - SECTION_CONDA, - SECTION_SCRIPT, SECTION_WRAPPER, SECTION_CWL, SECTION_NOTEBOOK, SECTION_TEMPLATE_ENGINE, - SECTION_CACHE, - SECTION_CONTAINER, - SECTION_CONTAINERIZED, - SECTION_ENVMODULES, - SECTION_NAME, - SECTION_HANDOVER, - SECTION_DEFAULT_TARGET, - SECTION_RETRIES - ) + val RULE_OR_CHECKPOINT_ARGS_SECTION_KEYWORDS = RULE_OR_CHECKPOINT_ARGS_SECTION_KEYWORDS_HARDCODED + + SnakemakeFrameworkAPIProvider.getInstance().collectAllPossibleRuleOrCheckpointSubsectionKeywords() val RULE_OR_CHECKPOINT_SECTION_KEYWORDS = (RULE_OR_CHECKPOINT_ARGS_SECTION_KEYWORDS + setOf(SECTION_RUN)) /** - * For subworkflows parsing + * For codeInsight */ val SUBWORKFLOW_SECTIONS_KEYWORDS = setOf( SnakemakeNames.SUBWORKFLOW_WORKDIR_KEYWORD, @@ -190,7 +174,7 @@ object SnakemakeAPI { ) /** - * For modules parsing + * For modules codeInsight */ val MODULE_SECTIONS_KEYWORDS = setOf( MODULE_SNAKEFILE_KEYWORD, @@ -198,12 +182,14 @@ object SnakemakeAPI { MODULE_SKIP_VALIDATION_KEYWORD, MODULE_META_WRAPPER_KEYWORD, MODULE_REPLACE_PREFIX_KEYWORD - ) + ) + SnakemakeFrameworkAPIProvider.getInstance() + .collectAllPossibleModuleSubsectionKeywords() /** - * For uses parsing + * For uses codeInsight */ - val USE_SECTIONS_KEYWORDS = RULE_OR_CHECKPOINT_SECTION_KEYWORDS - EXECUTION_SECTIONS_KEYWORDS - SECTION_RUN + val USE_SECTIONS_KEYWORDS = RULE_OR_CHECKPOINT_SECTION_KEYWORDS + SnakemakeFrameworkAPIProvider.getInstance() + .collectAllPossibleUseSubsectionKeywords() - EXECUTION_SECTIONS_KEYWORDS - SECTION_RUN val USE_DECLARATION_KEYWORDS = setOf( RULE_KEYWORD, @@ -213,13 +199,6 @@ object SnakemakeAPI { USE_EXCLUDE_KEYWORD ) - /** - * For Snakemake YAML api descriptor - */ - val RULE_LIKE_KEYWORDS = setOf( - RULE_KEYWORD, CHECKPOINT_KEYWORD, USE_KEYWORD - ) - /** * For type inference: * Some sections in snakemake are inaccessible after `rules.NAME.
`, so this set is required @@ -376,4 +355,16 @@ object SnakemakeAPI { val SMK_API_PKG_NAME_SMK = "snakemake" val SMK_API_PKG_NAME_SMK_MINIMAL = "snakemake-minimal" val SMK_API_VERS_6_1 = "6.1" +} + +object SnakemakeAPICompanion { + /** + * For codeInsight + */ + val RULE_OR_CHECKPOINT_ARGS_SECTION_KEYWORDS_HARDCODED = setOf( + // hardcoded list missing in 'snakmake_api.yaml' + SECTION_OUTPUT, SECTION_INPUT, SECTION_PARAMS, SECTION_LOG, SECTION_RESOURCES, + SECTION_BENCHMARK, SECTION_MESSAGE, SECTION_SHELL, SECTION_THREADS, + SECTION_PRIORITY, SECTION_GROUP, SECTION_SHADOW, + ) } \ No newline at end of file diff --git a/src/main/kotlin/com/jetbrains/snakecharm/codeInsight/completion/SmkKeywordCompletionContributor.kt b/src/main/kotlin/com/jetbrains/snakecharm/codeInsight/completion/SmkKeywordCompletionContributor.kt index 16e94abe..8862458a 100644 --- a/src/main/kotlin/com/jetbrains/snakecharm/codeInsight/completion/SmkKeywordCompletionContributor.kt +++ b/src/main/kotlin/com/jetbrains/snakecharm/codeInsight/completion/SmkKeywordCompletionContributor.kt @@ -25,8 +25,8 @@ import com.jetbrains.snakecharm.codeInsight.SnakemakeAPI.TOPLEVEL_ARGS_SECTION_K import com.jetbrains.snakecharm.codeInsight.SnakemakeAPI.USE_DECLARATION_KEYWORDS import com.jetbrains.snakecharm.codeInsight.SnakemakeAPI.USE_SECTIONS_KEYWORDS import com.jetbrains.snakecharm.framework.SmkAPIKeywordContextType -import com.jetbrains.snakecharm.framework.SmkFrameworkAPIProvider import com.jetbrains.snakecharm.framework.SmkSupportProjectSettings +import com.jetbrains.snakecharm.framework.SnakemakeFrameworkAPIProvider import com.jetbrains.snakecharm.framework.UpdateType import com.jetbrains.snakecharm.lang.SmkLanguageVersion import com.jetbrains.snakecharm.lang.SnakemakeNames @@ -299,7 +299,7 @@ private fun filterByDeprecationAndAddLookupItems( defaultTailType: TailType = ColonAndWhiteSpaceTail, parentContextProvider: () -> String? ) { - val deprecationProvider = SmkFrameworkDeprecationProvider.getInstance() + val deprecationProvider = SnakemakeFrameworkAPIProvider.getInstance() val settings = SmkSupportProjectSettings.getInstance(project) val contextName = parentContextProvider() @@ -349,20 +349,12 @@ private fun filterByDeprecationAndAddLookupItems( } buf.append("removed $removedVersion") } -// if (currentVersion != null) { -// if (issue?.updateType == UpdateType.DEPRECATED) { -// if (buf.isNotEmpty()) { -// buf.append(", ") -// } -// buf.append("deprecated in $currentVersion") -// } -// } - versionInfo = buf.toString() + versionInfo = if (buf.isEmpty()) null else buf.toString() } else { versionInfo = null } - var tailType = defaultTailType; + var tailType = defaultTailType if (customTailTypes != null && s in customTailTypes.keys) { tailType = customTailTypes[s]!! } diff --git a/src/main/kotlin/com/jetbrains/snakecharm/codeInsight/completion/wrapper/SmkWrapperCrawler.kt b/src/main/kotlin/com/jetbrains/snakecharm/codeInsight/completion/wrapper/SmkWrapperCrawler.kt index d71ba801..1e8ba59e 100644 --- a/src/main/kotlin/com/jetbrains/snakecharm/codeInsight/completion/wrapper/SmkWrapperCrawler.kt +++ b/src/main/kotlin/com/jetbrains/snakecharm/codeInsight/completion/wrapper/SmkWrapperCrawler.kt @@ -2,7 +2,8 @@ package com.jetbrains.snakecharm.codeInsight.completion.wrapper import com.intellij.openapi.util.io.FileUtil import com.intellij.util.io.write -import com.jetbrains.snakecharm.codeInsight.SnakemakeAPI +import com.jetbrains.snakecharm.codeInsight.SnakemakeAPICompanion.RULE_OR_CHECKPOINT_ARGS_SECTION_KEYWORDS_HARDCODED +import com.jetbrains.snakecharm.framework.SnakemakeFrameworkAPIProvider import kotlinx.serialization.ExperimentalSerializationApi import kotlinx.serialization.cbor.Cbor import kotlinx.serialization.encodeToByteArray @@ -10,8 +11,12 @@ import org.yaml.snakeyaml.Yaml import java.io.File import java.nio.file.Paths import kotlin.io.path.exists +import kotlin.io.path.inputStream import kotlin.io.path.isDirectory +/** + * Should be used ouside + */ object SmkWrapperCrawler { private val VERBOSE = false private val YAML_WRAPPER_DETAILS_KEYS = listOf("name", "description", "authors", "url", "notes") @@ -19,10 +24,10 @@ object SmkWrapperCrawler { @ExperimentalSerializationApi @JvmStatic fun main(args: Array) { - println("Usage: SmkWrapperCrawler {WRAPPERS_SRC_ROOT_FOLDER} {WRAPPERS_REPO_VERSION} {WRAPPERS_INFO_CBOR_OUTPUT}") + println("Usage: SmkWrapperCrawler {WRAPPERS_SRC_ROOT_FOLDER} {WRAPPERS_REPO_VERSION} {WRAPPERS_INFO_CBOR_OUTPUT} {SNAKEMAKE_API_YAML}") println("Smk wrapper crawler args: ${args.joinToString()}") - require(args.size == 3) { - "3 input args expected, but was: ${args.size}" + require(args.size == 4) { + "4 input args expected, but was: ${args.size}" } val wrappersFolder = args[0] @@ -42,8 +47,16 @@ object SmkWrapperCrawler { val outputFile = args[2] + val snakemakeAPIYaml = args[3] + val snakemakeAPIYamlPath = Paths.get(snakemakeAPIYaml) + require(snakemakeAPIYamlPath.exists()) { + "Snakemake API YAML file doesn't exist: [$snakemakeAPIYamlPath]" + } + println("Launching smk wrappers crawler...") - val wrappers = localWrapperParser(wrappersFolder, true) + val provider = SnakemakeFrameworkAPIProvider(null) + provider.reinitializeInTests(snakemakeAPIYamlPath.inputStream()) + val wrappers = localWrapperParser(wrappersFolder, provider=provider) wrappers.forEach { wrapper -> println(wrapper.path) } @@ -99,10 +112,17 @@ object SmkWrapperCrawler { */ - fun localWrapperParser(folder: String, relativePath: Boolean = false): List { + fun localWrapperParser( + folder: String, + provider: SnakemakeFrameworkAPIProvider + ): List { val wrappers = mutableListOf() val mainFolder = File(folder) + // could be launched also outside IDE process, so, we need to init manually: + val allowedKeywords = RULE_OR_CHECKPOINT_ARGS_SECTION_KEYWORDS_HARDCODED + + provider.collectAllPossibleRuleOrCheckpointSubsectionKeywords() + mainFolder.walkTopDown() .filter { it.isFile && it.name.startsWith("wrapper") } .forEach { wrapperFile -> @@ -123,7 +143,7 @@ object SmkWrapperCrawler { val metaYamlContent = metaYaml.readText() wrappers.add( - collectWrapperInfo(path, wrapperFileContent, wrapperFileExt, metaYamlContent) + collectWrapperInfo(path, wrapperFileContent, wrapperFileExt, metaYamlContent, allowedKeywords) ) } @@ -134,7 +154,8 @@ object SmkWrapperCrawler { wrapperFullName: String, wrapperFileContent: String, wrapperFileExt: String, - metaYamlContent: String + metaYamlContent: String, + allowedKeywords: Set ): SmkWrapperStorage.WrapperInfo { val wrapperArgs: List> = when (wrapperFileExt.lowercase()) { "py" -> parseArgsPython(wrapperFileContent) @@ -146,7 +167,7 @@ object SmkWrapperCrawler { return SmkWrapperStorage.WrapperInfo( path = FileUtil.toSystemIndependentName(wrapperFullName), - args = toParamsMapping(wrapperArgs + yamlArgs), + args = toParamsMapping(wrapperArgs + yamlArgs, allowedKeywords), description = metaYamlContent ) } @@ -238,11 +259,18 @@ object SmkWrapperCrawler { return sectionAndArgPairs } - private fun toParamsMapping(sectionAndArgPairs: List>): Map> { + private fun toParamsMapping( + sectionAndArgPairs: List>, + allowedKeywords: Set + ): Map> { val map = HashMap>() sectionAndArgPairs - // XXX: parse not sections here, e.g. 'notes:' or 'url:' should be ignored - .filter { (section, _) -> section in SnakemakeAPI.RULE_OR_CHECKPOINT_ARGS_SECTION_KEYWORDS } + .filter { + // XXX: alg could find a lot of garbage, especialy from PY and R files + // + some sections form YAML description that not needed (e.g. notes, url) + // Lets do filtering: + (section, _) -> section in allowedKeywords + } .forEach { (section, arg) -> val sectionKeywords = map.getOrPut(section) { arrayListOf() } if (arg.isNotEmpty() && arg !in sectionKeywords) { diff --git a/src/main/kotlin/com/jetbrains/snakecharm/codeInsight/completion/wrapper/SmkWrapperStorage.kt b/src/main/kotlin/com/jetbrains/snakecharm/codeInsight/completion/wrapper/SmkWrapperStorage.kt index b177329a..f684a255 100644 --- a/src/main/kotlin/com/jetbrains/snakecharm/codeInsight/completion/wrapper/SmkWrapperStorage.kt +++ b/src/main/kotlin/com/jetbrains/snakecharm/codeInsight/completion/wrapper/SmkWrapperStorage.kt @@ -10,6 +10,7 @@ import com.intellij.openapi.util.Disposer import com.jetbrains.snakecharm.SnakemakeBundle import com.jetbrains.snakecharm.SnakemakePluginUtil import com.jetbrains.snakecharm.SnakemakeTestUtil +import com.jetbrains.snakecharm.framework.SnakemakeFrameworkAPIProvider import com.jetbrains.snakecharm.framework.SmkSupportProjectSettings import com.jetbrains.snakecharm.framework.SmkSupportProjectSettingsListener import kotlinx.serialization.ExperimentalSerializationApi @@ -170,7 +171,10 @@ class SmkWrapperStorage(val project: Project) : Disposable { } storage.initFrom( "file://${config.wrappersCustomSourcesFolder}", - SmkWrapperCrawler.localWrapperParser(config.wrappersCustomSourcesFolder) + SmkWrapperCrawler.localWrapperParser( + config.wrappersCustomSourcesFolder, + SnakemakeFrameworkAPIProvider.getInstance() + ) ) } } diff --git a/src/main/kotlin/com/jetbrains/snakecharm/codeInsight/completion/wrapper/SnakemakeStartupActivity.kt b/src/main/kotlin/com/jetbrains/snakecharm/codeInsight/completion/wrapper/SnakemakeStartupActivity.kt index 1b8b34ed..48f34b10 100644 --- a/src/main/kotlin/com/jetbrains/snakecharm/codeInsight/completion/wrapper/SnakemakeStartupActivity.kt +++ b/src/main/kotlin/com/jetbrains/snakecharm/codeInsight/completion/wrapper/SnakemakeStartupActivity.kt @@ -4,13 +4,15 @@ import com.intellij.openapi.components.service import com.intellij.openapi.project.Project import com.intellij.openapi.startup.ProjectActivity import com.jetbrains.snakecharm.codeInsight.ImplicitPySymbolsProvider -import com.jetbrains.snakecharm.framework.SmkFrameworkDeprecationProvider import com.jetbrains.snakecharm.framework.SmkSupportProjectSettings +import com.jetbrains.snakecharm.framework.SnakemakeFrameworkAPIProvider import kotlinx.serialization.ExperimentalSerializationApi @ExperimentalSerializationApi class SnakemakeStartupActivity : ProjectActivity { override suspend fun execute(project: Project) { + SnakemakeFrameworkAPIProvider.getInstance() // other classes could use it implicitly + val smkSettings = project.service() smkSettings.initOnStartup() val smkWrapperStorage = project.service() @@ -19,6 +21,5 @@ class SnakemakeStartupActivity : ProjectActivity { val implicitPySymbolsProvider = project.service() implicitPySymbolsProvider.initOnStartup() - SmkFrameworkDeprecationProvider.getInstance() } } \ No newline at end of file diff --git a/src/main/kotlin/com/jetbrains/snakecharm/framework/SmkFrameworkProjectSettings.kt b/src/main/kotlin/com/jetbrains/snakecharm/framework/SmkFrameworkProjectSettings.kt index 572f0e95..65a37d5f 100644 --- a/src/main/kotlin/com/jetbrains/snakecharm/framework/SmkFrameworkProjectSettings.kt +++ b/src/main/kotlin/com/jetbrains/snakecharm/framework/SmkFrameworkProjectSettings.kt @@ -146,7 +146,7 @@ class SmkSupportProjectSettings(val project: Project) : PersistentStateComponent var snakemakeSupportBannerEnabled by property(true) @get:Attribute("smk_language_version") - var snakemakeLanguageVersion by string(SmkFrameworkDeprecationProvider.getInstance().getDefaultVersion()) + var snakemakeLanguageVersion by string(SnakemakeFrameworkAPIProvider.getInstance().getDefaultVersion()) } companion object { diff --git a/src/main/kotlin/com/jetbrains/snakecharm/framework/SmkFrameworkAPIProvider.kt b/src/main/kotlin/com/jetbrains/snakecharm/framework/SnakemakeFrameworkAPIProvider.kt similarity index 78% rename from src/main/kotlin/com/jetbrains/snakecharm/framework/SmkFrameworkAPIProvider.kt rename to src/main/kotlin/com/jetbrains/snakecharm/framework/SnakemakeFrameworkAPIProvider.kt index 220e5e0f..fe2730d2 100644 --- a/src/main/kotlin/com/jetbrains/snakecharm/framework/SmkFrameworkAPIProvider.kt +++ b/src/main/kotlin/com/jetbrains/snakecharm/framework/SnakemakeFrameworkAPIProvider.kt @@ -1,11 +1,16 @@ package com.jetbrains.snakecharm.framework +import com.intellij.openapi.application.Application import com.intellij.openapi.application.ApplicationManager import com.intellij.openapi.components.Service import com.jetbrains.snakecharm.SnakemakePluginUtil import com.jetbrains.snakecharm.SnakemakeTestUtil -import com.jetbrains.snakecharm.codeInsight.SnakemakeAPI import com.jetbrains.snakecharm.lang.SmkLanguageVersion +import com.jetbrains.snakecharm.lang.SnakemakeNames +import com.jetbrains.snakecharm.lang.SnakemakeNames.CHECKPOINT_KEYWORD +import com.jetbrains.snakecharm.lang.SnakemakeNames.RULE_KEYWORD +import com.jetbrains.snakecharm.lang.SnakemakeNames.USE_KEYWORD +import io.ktor.util.* import org.jetbrains.annotations.TestOnly import org.yaml.snakeyaml.LoaderOptions import org.yaml.snakeyaml.Yaml @@ -18,8 +23,9 @@ import kotlin.io.path.exists import kotlin.io.path.inputStream @Service -class SmkFrameworkDeprecationProvider { - +class SnakemakeFrameworkAPIProvider( + application: Application? // required to launch outside of IDEA process +) { private lateinit var topLevelCorrection: Map> private lateinit var functionCorrection: Map> private lateinit var subsectionCorrection: Map, TreeMap> @@ -34,16 +40,25 @@ class SmkFrameworkDeprecationProvider { private lateinit var defaultVersion: String init { - if (ApplicationManager.getApplication().isUnitTestMode) { - reinitializeInTests() - } else { - val pluginSandboxPath = SnakemakePluginUtil.getPluginSandboxPath( - SmkFrameworkDeprecationProvider::class.java - ) - initialize(pluginSandboxPath.resolve("extra/snakemake_api.yaml")) + if (application != null) { + if (application.isUnitTestMode) { + reinitializeInTests() + } else { + val pluginSandboxPath = SnakemakePluginUtil.getPluginSandboxPath( + SnakemakeFrameworkAPIProvider::class.java + ) + initialize(pluginSandboxPath.resolve("extra/snakemake_api.yaml")) + } } } + /** + * Constructor for service creation in IDEA (default usage) + */ + constructor() : this(ApplicationManager.getApplication()) + + + @TestOnly fun reinitializeInTests() { require(ApplicationManager.getApplication().isUnitTestMode) @@ -92,7 +107,7 @@ class SmkFrameworkDeprecationProvider { // else: subsection type: else -> { if (srcType == SmkAPIKeywordContextType.RULE_LIKE.typeStr) { - SnakemakeAPI.RULE_LIKE_KEYWORDS.forEach { directive -> + RULE_LIKE_KEYWORDS.forEach { directive -> subsectionData.getOrPut(src.name to directive) { TreeMap() }[version] = newValue } } @@ -107,9 +122,9 @@ class SmkFrameworkDeprecationProvider { putKeywordDataIntoMap(data.deprecated, UpdateType.DEPRECATED, mapGetter) putKeywordDataIntoMap(data.removed, UpdateType.REMOVED, mapGetter) - foo(topLevelIntroductionsMap, version, subsectionIntroductionsMap, data.introduced) - foo(topLevelDeprecationsMap, version, subsectionDeprecationsMap, data.deprecated) - foo(topLevelRemovalMap, version, subsectionRemovalMap, data.removed) + fillEventsMapWithKeywordsAndVersions(topLevelIntroductionsMap, version, subsectionIntroductionsMap, data.introduced) + fillEventsMapWithKeywordsAndVersions(topLevelDeprecationsMap, version, subsectionDeprecationsMap, data.deprecated) + fillEventsMapWithKeywordsAndVersions(topLevelRemovalMap, version, subsectionRemovalMap, data.removed) } topLevelCorrection = topLevelData @@ -124,7 +139,7 @@ class SmkFrameworkDeprecationProvider { topLevelDeprecation = topLevelDeprecationsMap } - private fun foo( + private fun fillEventsMapWithKeywordsAndVersions( topLevelEventsMap: MutableMap, version: SmkLanguageVersion, subsectionEventsMap: MutableMap, SmkLanguageVersion>, @@ -138,7 +153,7 @@ class SmkFrameworkDeprecationProvider { // TODO do nothing, functions processed isn't implemented, issue: ...... } SmkAPIKeywordContextType.RULE_LIKE.typeStr -> { - SnakemakeAPI.RULE_LIKE_KEYWORDS.forEach { directive -> + RULE_LIKE_KEYWORDS.forEach { directive -> subsectionEventsMap[event.name to directive] = version } } @@ -208,6 +223,31 @@ class SmkFrameworkDeprecationProvider { return subsectionRemoval[name to contextSectionKeyword] } + fun collectAllPossibleUseSubsectionKeywords() = collectAllPossibleSubsectionKeywords { type -> + type == USE_KEYWORD + } + + fun collectAllPossibleModuleSubsectionKeywords() = collectAllPossibleSubsectionKeywords { type -> + type == SnakemakeNames.MODULE_KEYWORD + } + + fun collectAllPossibleRuleOrCheckpointSubsectionKeywords() = collectAllPossibleSubsectionKeywords { type -> + (type == RULE_KEYWORD) || (type == CHECKPOINT_KEYWORD) + } + + private fun collectAllPossibleSubsectionKeywords(ctxTypeFilter: (String) -> Boolean): Set { + val mutableSet = mutableSetOf() + + // Collect all sections ignoring deprecation/removal/introduction marks: + listOf(subsectionIntroduction.keys, subsectionDeprecation.keys, subsectionRemoval.keys).forEach { keys -> + mutableSet.addAll( + keys.filter { (_, type) -> ctxTypeFilter(type) } + .map { (name, _) -> name } + ) + } + return mutableSet.unmodifiable() + } + private fun getKeywordCorrection( keywords: TreeMap?, version: SmkLanguageVersion, @@ -217,7 +257,6 @@ class SmkFrameworkDeprecationProvider { return SmkCorrectionInfo(entry.value.type, entry.value.advice, entry.key, isGlobalChange) } - private fun putKeywordDataIntoMap( keywords: List, updateType: UpdateType, @@ -228,8 +267,15 @@ class SmkFrameworkDeprecationProvider { } } companion object { + /** + * For Snakemake YAML api descriptor + */ + private val RULE_LIKE_KEYWORDS = setOf( + RULE_KEYWORD, CHECKPOINT_KEYWORD, USE_KEYWORD + ) + fun getInstance() = - ApplicationManager.getApplication().getService(SmkFrameworkDeprecationProvider::class.java)!! + ApplicationManager.getApplication().getService(SnakemakeFrameworkAPIProvider::class.java)!! } } enum class SmkAPIKeywordContextType(val typeStr: String) { @@ -263,7 +309,8 @@ data class SmkDeprecationVersionData( data class SmkDeprecationData( val changelog: List = emptyList(), - val defaultVersion: String = "7.32.4" + val defaultVersion: String = "0.0.0", + val annotationsFormatVersion: Int = 0 ) data class SmkCorrectionInfo( diff --git a/src/main/kotlin/com/jetbrains/snakecharm/inspections/SmkDepreciatedKeywordsInspection.kt b/src/main/kotlin/com/jetbrains/snakecharm/inspections/SmkDepreciatedKeywordsInspection.kt index 14b64f7c..182f19c3 100644 --- a/src/main/kotlin/com/jetbrains/snakecharm/inspections/SmkDepreciatedKeywordsInspection.kt +++ b/src/main/kotlin/com/jetbrains/snakecharm/inspections/SmkDepreciatedKeywordsInspection.kt @@ -8,7 +8,7 @@ import com.jetbrains.python.psi.PyFunction import com.jetbrains.python.psi.PyReferenceExpression import com.jetbrains.snakecharm.SnakemakeBundle import com.jetbrains.snakecharm.codeInsight.SnakemakeAPI -import com.jetbrains.snakecharm.framework.SmkFrameworkDeprecationProvider +import com.jetbrains.snakecharm.framework.SnakemakeFrameworkAPIProvider import com.jetbrains.snakecharm.framework.SmkSupportProjectSettings import com.jetbrains.snakecharm.framework.UpdateType import com.jetbrains.snakecharm.lang.SmkLanguageVersion @@ -99,7 +99,7 @@ class SmkDepreciatedKeywordsInspection : SnakemakeInspection() { declaration is PyFunction && declaration.containingFile.name == SnakemakeAPI.SNAKEMAKE_MODULE_NAME_IO_PY ) { val settings = SmkSupportProjectSettings.getInstance(holder.project) - val deprecationProvider = SmkFrameworkDeprecationProvider.getInstance() + val deprecationProvider = SnakemakeFrameworkAPIProvider.getInstance() val currentVersionString = settings.snakemakeLanguageVersion val currentVersion = if (currentVersionString == null) null else SmkLanguageVersion(currentVersionString) val name = node.name @@ -143,7 +143,7 @@ class SmkDepreciatedKeywordsInspection : SnakemakeInspection() { private fun checkTopLevelDefinition(psiElement: PsiElement, name: String) { val settings = SmkSupportProjectSettings.getInstance(holder.project) - val deprecationProvider = SmkFrameworkDeprecationProvider.getInstance() + val deprecationProvider = SnakemakeFrameworkAPIProvider.getInstance() val lowestVersion = deprecationProvider.getTopLevelIntroductionVersion(name) val currentVersionString = settings.snakemakeLanguageVersion val currentVersion = if (currentVersionString == null) null else SmkLanguageVersion(currentVersionString) @@ -207,7 +207,7 @@ class SmkDepreciatedKeywordsInspection : SnakemakeInspection() { private fun checkSubSectionDefinition(psiElement: PsiElement, name: String, parentName: String) { val settings = SmkSupportProjectSettings.getInstance(holder.project) - val deprecationProvider = SmkFrameworkDeprecationProvider.getInstance() + val deprecationProvider = SnakemakeFrameworkAPIProvider.getInstance() val lowestVersion = deprecationProvider.getSubSectionIntroductionVersion(name, parentName) val currentVersionString = settings.snakemakeLanguageVersion val currentVersion = if (currentVersionString == null) null else SmkLanguageVersion(currentVersionString) diff --git a/src/main/kotlin/com/jetbrains/snakecharm/lang/SnakemakeNames.kt b/src/main/kotlin/com/jetbrains/snakecharm/lang/SnakemakeNames.kt index 166f7f22..f93d2f38 100644 --- a/src/main/kotlin/com/jetbrains/snakecharm/lang/SnakemakeNames.kt +++ b/src/main/kotlin/com/jetbrains/snakecharm/lang/SnakemakeNames.kt @@ -70,7 +70,7 @@ object SnakemakeNames { const val SECTION_ENVMODULES = "envmodules" // >= 5.9 const val SECTION_NAME = "name" // >= 5.31 const val SECTION_HANDOVER = "handover" // >= 6.2 - const val SECTION_DEFAULT_TARGET = "default_target" // >= 5.12.0 + const val SECTION_DEFAULT_TARGET = "default_target" // >= 6.15.0 const val SECTION_TEMPLATE_ENGINE = "template_engine" // >= 7.0.0 const val SECTION_RETRIES = "retries" // >= 7.7.0 diff --git a/src/test/kotlin/features/glue/ActionsSteps.kt b/src/test/kotlin/features/glue/ActionsSteps.kt index 73f7f9b6..dd36b388 100644 --- a/src/test/kotlin/features/glue/ActionsSteps.kt +++ b/src/test/kotlin/features/glue/ActionsSteps.kt @@ -27,6 +27,7 @@ import com.intellij.psi.PsiFile import com.intellij.util.IncorrectOperationException import com.intellij.util.containers.ContainerUtil import com.jetbrains.snakecharm.FakeSnakemakeInjector +import com.jetbrains.snakecharm.codeInsight.SnakemakeAPI import com.jetbrains.snakecharm.codeInsight.completion.wrapper.SmkWrapperCrawler import com.jetbrains.snakecharm.inspections.SmkUnrecognizedSectionInspection import com.jetbrains.snakecharm.lang.highlighter.SmkColorSettingsPage @@ -624,7 +625,7 @@ class ActionsSteps { } ?: "" val info = SmkWrapperCrawler.collectWrapperInfo( - "wrapper", wrapperFileContent, ext, metaYamlContent + "wrapper", wrapperFileContent, ext, metaYamlContent, SnakemakeAPI.RULE_OR_CHECKPOINT_ARGS_SECTION_KEYWORDS ) val args = info.args diff --git a/src/test/kotlin/features/glue/FilesSteps.kt b/src/test/kotlin/features/glue/FilesSteps.kt index 45bd9197..bb796975 100644 --- a/src/test/kotlin/features/glue/FilesSteps.kt +++ b/src/test/kotlin/features/glue/FilesSteps.kt @@ -4,7 +4,7 @@ import com.intellij.openapi.application.ApplicationManager import com.intellij.openapi.application.ModalityState import com.intellij.openapi.fileEditor.FileDocumentManager import com.intellij.openapi.util.text.StringUtil -import com.jetbrains.snakecharm.framework.SmkFrameworkDeprecationProvider +import com.jetbrains.snakecharm.framework.SnakemakeFrameworkAPIProvider import com.jetbrains.snakecharm.lang.highlighter.SmkColorSettingsPage import io.cucumber.java.en.Given import io.cucumber.java.en.Then @@ -38,7 +38,7 @@ class FilesSteps { @Given("snakemake framework api yaml descriptor is$") fun snakemakeFrameworkApiInfoContent(text: String) { ApplicationManager.getApplication().invokeAndWait({ - SmkFrameworkDeprecationProvider.getInstance().reinitializeInTests(text.byteInputStream()) + SnakemakeFrameworkAPIProvider.getInstance().reinitializeInTests(text.byteInputStream()) }, ModalityState.nonModal()) } diff --git a/src/test/kotlin/features/glue/StepDefs.kt b/src/test/kotlin/features/glue/StepDefs.kt index a49586e9..2a15b0e9 100644 --- a/src/test/kotlin/features/glue/StepDefs.kt +++ b/src/test/kotlin/features/glue/StepDefs.kt @@ -17,11 +17,10 @@ import com.jetbrains.python.fixtures.PyLightProjectDescriptor import com.jetbrains.python.psi.LanguageLevel import com.jetbrains.python.psi.PyFile import com.jetbrains.snakecharm.SnakemakeTestUtil -import com.jetbrains.snakecharm.framework.SmkFrameworkDeprecationProvider +import com.jetbrains.snakecharm.framework.SnakemakeFrameworkAPIProvider import com.jetbrains.snakecharm.framework.SmkSupportProjectSettings import io.cucumber.java.en.Given import javax.swing.SwingUtilities -import kotlin.Throws import kotlin.test.fail @@ -136,7 +135,7 @@ class StepDefs { // XXX: reset Snakemake API settings if smth was overridden in other tests ApplicationManager.getApplication().invokeAndWait { ApplicationManager.getApplication().runWriteAction { - SmkFrameworkDeprecationProvider.getInstance().reinitializeInTests() + SnakemakeFrameworkAPIProvider.getInstance().reinitializeInTests() } } setProjectSdk("python with snakemake") diff --git a/src/test/resources/features/completion/keywords_completion.feature b/src/test/resources/features/completion/keywords_completion.feature index cf95cde2..fe3e989c 100644 --- a/src/test/resources/features/completion/keywords_completion.feature +++ b/src/test/resources/features/completion/keywords_completion.feature @@ -263,8 +263,6 @@ Feature: Completion for snakemake keyword-like things | checkpoint | out | output | | rule | par | params | | checkpoint | par | params | - | rule | lo | log | - | checkpoint | lo | log | | rule | be | benchmark | | checkpoint | be | benchmark | | rule | vers | version | @@ -296,7 +294,7 @@ Feature: Completion for snakemake keyword-like things | rule | retr | retries | | rule | temp | template_engine | - Scenario Outline: Complete at rule/checkpoint/module level + Scenario Outline: Complete at rule/checkpoint/module level (multiple variants) Given a snakemake project Given I open a file "foo.smk" with text """ @@ -325,7 +323,8 @@ Feature: Completion for snakemake keyword-like things | checkpoint | co | conda | | rule | sh | shell | | checkpoint | sh | shell | - | module | s | snakefile | + | rule | lo | log | + | checkpoint | lo | log | | module | c | config | | module | m | meta_wrapper | | module | s | skip_validation |