refactor: 1) Force SnakemakeAPI to load data from snakemake_api.yaml …

…instead of hardcoded stings 2) Change SmkWeapperCrawler to work w/o ability to load data from SnakemakeFrameworkAPIProvider service 3) avoid cyclic dependency from SnakemakeFrameworkAPIProvider

build.gradle.kts

@@ -275,6 +275,7 @@ tasks {
             gradleProperty("snakemakeWrappersRepoPath").get(),
             gradleProperty("snakemakeWrappersRepoVersion").get(),
             layout.buildDirectory.file("bundledWrappers/smk-wrapper-storage-bundled.cbor").get(),
+            layout.projectDirectory.file("snakemake_api.yaml")
         )
         maxHeapSize = "1024m" // Not much RAM is available on TC agents
     }
@@ -297,6 +298,7 @@ tasks {
             layout.projectDirectory.file("testData/wrappers_storage"),
             "test",
             layout.buildDirectory.file("bundledWrappers/smk-wrapper-storage.test.cbor").get(),
+            layout.projectDirectory.file("snakemake_api.yaml")
         )
         maxHeapSize = "1024m" // Not much RAM is available on TC agents
     }

snakemake_api.yaml

@@ -1,3 +1,5 @@
+annotationsFormatVersion: 1
+
 # Default Language Level used for all projects:
 defaultVersion: "7.32.4"
 
@@ -31,9 +33,13 @@ changelog:
     introduced:
       - name: "retries"
         type: "rule-like"
+  - version: "7.0.0"
+    introduced:
+      - name: "template_engine"
+        type: "rule-like"
   - version: "6.15.0"
     introduced:
-      - name: "default_rule"
+      - name: "default_target"
         type: "rule-like"
   - version: "6.2.0"
     introduced:
@@ -45,6 +51,10 @@ changelog:
         type: "top-level"
       - name: "module"
         type: "top-level"
+      - name: "containerized"
+        type: "top-level"
+      - name: "containerized"
+        type: "rule-like"
     deprecated:
       - name: "subworkflow"
         type: "top-level"
@@ -53,6 +63,10 @@ changelog:
     introduced:
       - name: "name"
         type: "rule-like"
+  - version: "5.12.0"
+    introduced:
+      - name: "cache"
+        type: "rule-like"
   - version: "5.11.0"
     deprecated:
       - name: "singularity"
@@ -68,6 +82,10 @@ changelog:
         type: "top-level"
       - name: "container"
         type: "rule-like"
+  - version: "5.9.0"
+    introduced:
+      - name: "envmodules"
+        type: "rule-like"
   - version: "5.4.0"
     deprecated:
       - name: "dynamic"
@@ -77,6 +95,10 @@ changelog:
     introduced:
       - name: "cwl"
         type: "rule-like"
+      - name: "conda"
+        type: "rule-like"
+      - name: "singularity"
+        type: "rule-like"
       - name: "singularity"
         type: "top-level"
   - version: "3.8.0"
@@ -94,4 +116,6 @@ changelog:
   - version: "3.5.2"
     introduced:
       - name: "script"
+        type: "rule-like"
+      - name: "notebook"
         type: "rule-like"

src/main/kotlin/com/jetbrains/snakecharm/codeInsight/SnakemakeAPI.kt

@@ -1,7 +1,8 @@
 package com.jetbrains.snakecharm.codeInsight
 
+import com.jetbrains.snakecharm.codeInsight.SnakemakeAPICompanion.RULE_OR_CHECKPOINT_ARGS_SECTION_KEYWORDS_HARDCODED
+import com.jetbrains.snakecharm.framework.SnakemakeFrameworkAPIProvider
 import com.jetbrains.snakecharm.lang.SnakemakeNames
-import com.jetbrains.snakecharm.lang.SnakemakeNames.CHECKPOINT_KEYWORD
 import com.jetbrains.snakecharm.lang.SnakemakeNames.MODULE_CONFIG_KEYWORD
 import com.jetbrains.snakecharm.lang.SnakemakeNames.MODULE_META_WRAPPER_KEYWORD
 import com.jetbrains.snakecharm.lang.SnakemakeNames.MODULE_REPLACE_PREFIX_KEYWORD
@@ -53,7 +54,6 @@ import com.jetbrains.snakecharm.lang.SnakemakeNames.SNAKEMAKE_IO_METHOD_TEMP
 import com.jetbrains.snakecharm.lang.SnakemakeNames.SNAKEMAKE_IO_METHOD_TOUCH
 import com.jetbrains.snakecharm.lang.SnakemakeNames.SNAKEMAKE_IO_METHOD_UNPACK
 import com.jetbrains.snakecharm.lang.SnakemakeNames.USE_EXCLUDE_KEYWORD
-import com.jetbrains.snakecharm.lang.SnakemakeNames.USE_KEYWORD
 import com.jetbrains.snakecharm.lang.SnakemakeNames.WORKFLOW_CONFIGFILE_KEYWORD
 import com.jetbrains.snakecharm.lang.SnakemakeNames.WORKFLOW_CONTAINERIZED_KEYWORD
 import com.jetbrains.snakecharm.lang.SnakemakeNames.WORKFLOW_CONTAINER_KEYWORD
@@ -159,29 +159,13 @@ object SnakemakeAPI {
         WORKFLOW_PEPFILE_KEYWORD
     )
 
-    /**
-     * For rules parsing
-     */
-    val RULE_OR_CHECKPOINT_ARGS_SECTION_KEYWORDS = setOf(
-        SECTION_OUTPUT, SECTION_INPUT, SECTION_PARAMS, SECTION_LOG, SECTION_RESOURCES,
-        SECTION_BENCHMARK, SECTION_VERSION, SECTION_MESSAGE, SECTION_SHELL, SECTION_THREADS, SECTION_SINGULARITY,
-        SECTION_PRIORITY, SECTION_WILDCARD_CONSTRAINTS, SECTION_GROUP, SECTION_SHADOW,
-        SECTION_CONDA,
-        SECTION_SCRIPT, SECTION_WRAPPER, SECTION_CWL, SECTION_NOTEBOOK, SECTION_TEMPLATE_ENGINE,
-        SECTION_CACHE,
-        SECTION_CONTAINER,
-        SECTION_CONTAINERIZED,
-        SECTION_ENVMODULES,
-        SECTION_NAME,
-        SECTION_HANDOVER,
-        SECTION_DEFAULT_TARGET,
-        SECTION_RETRIES
-    )
+    val RULE_OR_CHECKPOINT_ARGS_SECTION_KEYWORDS = RULE_OR_CHECKPOINT_ARGS_SECTION_KEYWORDS_HARDCODED +
+            SnakemakeFrameworkAPIProvider.getInstance().collectAllPossibleRuleOrCheckpointSubsectionKeywords()
 
     val RULE_OR_CHECKPOINT_SECTION_KEYWORDS = (RULE_OR_CHECKPOINT_ARGS_SECTION_KEYWORDS + setOf(SECTION_RUN))
 
     /**
-     * For subworkflows parsing
+     * For codeInsight
      */
     val SUBWORKFLOW_SECTIONS_KEYWORDS = setOf(
         SnakemakeNames.SUBWORKFLOW_WORKDIR_KEYWORD,
@@ -190,20 +174,22 @@ object SnakemakeAPI {
     )
 
     /**
-     * For modules parsing
+     * For modules codeInsight
      */
     val MODULE_SECTIONS_KEYWORDS = setOf(
         MODULE_SNAKEFILE_KEYWORD,
         MODULE_CONFIG_KEYWORD,
         MODULE_SKIP_VALIDATION_KEYWORD,
         MODULE_META_WRAPPER_KEYWORD,
         MODULE_REPLACE_PREFIX_KEYWORD
-    )
+    ) + SnakemakeFrameworkAPIProvider.getInstance()
+        .collectAllPossibleModuleSubsectionKeywords()
 
     /**
-     * For uses parsing
+     * For uses codeInsight
      */
-    val USE_SECTIONS_KEYWORDS = RULE_OR_CHECKPOINT_SECTION_KEYWORDS - EXECUTION_SECTIONS_KEYWORDS - SECTION_RUN
+    val USE_SECTIONS_KEYWORDS = RULE_OR_CHECKPOINT_SECTION_KEYWORDS + SnakemakeFrameworkAPIProvider.getInstance()
+        .collectAllPossibleUseSubsectionKeywords() - EXECUTION_SECTIONS_KEYWORDS - SECTION_RUN
 
     val USE_DECLARATION_KEYWORDS = setOf(
         RULE_KEYWORD,
@@ -213,13 +199,6 @@ object SnakemakeAPI {
         USE_EXCLUDE_KEYWORD
     )
 
-    /**
-     * For Snakemake YAML api descriptor
-     */
-    val RULE_LIKE_KEYWORDS = setOf(
-        RULE_KEYWORD, CHECKPOINT_KEYWORD, USE_KEYWORD
-    )
-
     /**
      * For type inference:
      * Some sections in snakemake are inaccessible after `rules.NAME.<section>`, so this set is required
@@ -376,4 +355,16 @@ object SnakemakeAPI {
     val SMK_API_PKG_NAME_SMK = "snakemake"
     val SMK_API_PKG_NAME_SMK_MINIMAL = "snakemake-minimal"
     val SMK_API_VERS_6_1 = "6.1"
+}
+
+object SnakemakeAPICompanion {
+    /**
+     * For codeInsight
+     */
+    val RULE_OR_CHECKPOINT_ARGS_SECTION_KEYWORDS_HARDCODED = setOf(
+        // hardcoded list missing in 'snakmake_api.yaml'
+        SECTION_OUTPUT, SECTION_INPUT, SECTION_PARAMS, SECTION_LOG, SECTION_RESOURCES,
+        SECTION_BENCHMARK, SECTION_MESSAGE, SECTION_SHELL, SECTION_THREADS,
+        SECTION_PRIORITY, SECTION_GROUP, SECTION_SHADOW,
+    )
 }

src/main/kotlin/com/jetbrains/snakecharm/codeInsight/completion/SmkKeywordCompletionContributor.kt

@@ -25,8 +25,8 @@ import com.jetbrains.snakecharm.codeInsight.SnakemakeAPI.TOPLEVEL_ARGS_SECTION_K
 import com.jetbrains.snakecharm.codeInsight.SnakemakeAPI.USE_DECLARATION_KEYWORDS
 import com.jetbrains.snakecharm.codeInsight.SnakemakeAPI.USE_SECTIONS_KEYWORDS
 import com.jetbrains.snakecharm.framework.SmkAPIKeywordContextType
-import com.jetbrains.snakecharm.framework.SmkFrameworkAPIProvider
 import com.jetbrains.snakecharm.framework.SmkSupportProjectSettings
+import com.jetbrains.snakecharm.framework.SnakemakeFrameworkAPIProvider
 import com.jetbrains.snakecharm.framework.UpdateType
 import com.jetbrains.snakecharm.lang.SmkLanguageVersion
 import com.jetbrains.snakecharm.lang.SnakemakeNames
@@ -299,7 +299,7 @@ private fun filterByDeprecationAndAddLookupItems(
     defaultTailType: TailType = ColonAndWhiteSpaceTail,
     parentContextProvider: () -> String?
 ) {
-    val deprecationProvider = SmkFrameworkDeprecationProvider.getInstance()
+    val deprecationProvider = SnakemakeFrameworkAPIProvider.getInstance()
     val settings = SmkSupportProjectSettings.getInstance(project)
     val contextName = parentContextProvider()
 
@@ -349,20 +349,12 @@ private fun filterByDeprecationAndAddLookupItems(
                 }
                 buf.append("removed $removedVersion")
             }
-//            if (currentVersion != null) {
-//                if (issue?.updateType == UpdateType.DEPRECATED) {
-//                    if (buf.isNotEmpty()) {
-//                        buf.append(", ")
-//                    }
-//                    buf.append("deprecated in $currentVersion")
-//                }
-//            }
-            versionInfo = buf.toString()
+            versionInfo = if (buf.isEmpty()) null else buf.toString()
         } else {
             versionInfo = null
         }
 
-        var tailType = defaultTailType;
+        var tailType = defaultTailType
         if (customTailTypes != null && s in customTailTypes.keys) {
             tailType = customTailTypes[s]!!
         }

src/main/kotlin/com/jetbrains/snakecharm/codeInsight/completion/wrapper/SmkWrapperCrawler.kt

@@ -2,27 +2,32 @@ package com.jetbrains.snakecharm.codeInsight.completion.wrapper
 
 import com.intellij.openapi.util.io.FileUtil
 import com.intellij.util.io.write
-import com.jetbrains.snakecharm.codeInsight.SnakemakeAPI
+import com.jetbrains.snakecharm.codeInsight.SnakemakeAPICompanion.RULE_OR_CHECKPOINT_ARGS_SECTION_KEYWORDS_HARDCODED
+import com.jetbrains.snakecharm.framework.SnakemakeFrameworkAPIProvider
 import kotlinx.serialization.ExperimentalSerializationApi
 import kotlinx.serialization.cbor.Cbor
 import kotlinx.serialization.encodeToByteArray
 import org.yaml.snakeyaml.Yaml
 import java.io.File
 import java.nio.file.Paths
 import kotlin.io.path.exists
+import kotlin.io.path.inputStream
 import kotlin.io.path.isDirectory
 
+/**
+ * Should be used ouside
+ */
 object SmkWrapperCrawler {
     private val VERBOSE = false
     private val YAML_WRAPPER_DETAILS_KEYS = listOf("name", "description", "authors", "url", "notes")
 
     @ExperimentalSerializationApi
     @JvmStatic
     fun main(args: Array<String>) {
-        println("Usage: SmkWrapperCrawler {WRAPPERS_SRC_ROOT_FOLDER} {WRAPPERS_REPO_VERSION} {WRAPPERS_INFO_CBOR_OUTPUT}")
+        println("Usage: SmkWrapperCrawler {WRAPPERS_SRC_ROOT_FOLDER} {WRAPPERS_REPO_VERSION} {WRAPPERS_INFO_CBOR_OUTPUT} {SNAKEMAKE_API_YAML}")
         println("Smk wrapper crawler args: ${args.joinToString()}")
-        require(args.size == 3) {
-            "3 input args expected, but was: ${args.size}"
+        require(args.size == 4) {
+            "4 input args expected, but was: ${args.size}"
         }
 
         val wrappersFolder = args[0]
@@ -42,8 +47,16 @@ object SmkWrapperCrawler {
 
         val outputFile = args[2]
 
+        val snakemakeAPIYaml = args[3]
+        val snakemakeAPIYamlPath = Paths.get(snakemakeAPIYaml)
+        require(snakemakeAPIYamlPath.exists()) {
+            "Snakemake API YAML file doesn't exist: [$snakemakeAPIYamlPath]"
+        }
+
         println("Launching smk wrappers crawler...")
-        val wrappers = localWrapperParser(wrappersFolder, true)
+        val provider = SnakemakeFrameworkAPIProvider(null)
+        provider.reinitializeInTests(snakemakeAPIYamlPath.inputStream())
+        val wrappers = localWrapperParser(wrappersFolder, provider=provider)
         wrappers.forEach { wrapper ->
             println(wrapper.path)
         }
@@ -99,10 +112,17 @@ object SmkWrapperCrawler {
 
      */
 
-    fun localWrapperParser(folder: String, relativePath: Boolean = false): List<SmkWrapperStorage.WrapperInfo> {
+    fun localWrapperParser(
+        folder: String,
+        provider: SnakemakeFrameworkAPIProvider
+    ): List<SmkWrapperStorage.WrapperInfo> {
         val wrappers = mutableListOf<SmkWrapperStorage.WrapperInfo>()
         val mainFolder = File(folder)
 
+        // could be launched also outside IDE process, so, we need to init manually:
+        val allowedKeywords = RULE_OR_CHECKPOINT_ARGS_SECTION_KEYWORDS_HARDCODED +
+                provider.collectAllPossibleRuleOrCheckpointSubsectionKeywords()
+
         mainFolder.walkTopDown()
             .filter { it.isFile && it.name.startsWith("wrapper") }
             .forEach { wrapperFile ->
@@ -123,7 +143,7 @@ object SmkWrapperCrawler {
                 val metaYamlContent = metaYaml.readText()
 
                 wrappers.add(
-                    collectWrapperInfo(path, wrapperFileContent, wrapperFileExt, metaYamlContent)
+                    collectWrapperInfo(path, wrapperFileContent, wrapperFileExt, metaYamlContent, allowedKeywords)
                 )
             }
 
@@ -134,7 +154,8 @@ object SmkWrapperCrawler {
         wrapperFullName: String,
         wrapperFileContent: String,
         wrapperFileExt: String,
-        metaYamlContent: String
+        metaYamlContent: String,
+        allowedKeywords: Set<String>
     ): SmkWrapperStorage.WrapperInfo {
         val wrapperArgs: List<Pair<String, String>> = when (wrapperFileExt.lowercase()) {
             "py" -> parseArgsPython(wrapperFileContent)
@@ -146,7 +167,7 @@ object SmkWrapperCrawler {
 
         return SmkWrapperStorage.WrapperInfo(
             path = FileUtil.toSystemIndependentName(wrapperFullName),
-            args = toParamsMapping(wrapperArgs + yamlArgs),
+            args = toParamsMapping(wrapperArgs + yamlArgs, allowedKeywords),
             description = metaYamlContent
         )
     }
@@ -238,11 +259,18 @@ object SmkWrapperCrawler {
         return sectionAndArgPairs
     }
 
-    private fun toParamsMapping(sectionAndArgPairs: List<Pair<String, String>>): Map<String, List<String>> {
+    private fun toParamsMapping(
+        sectionAndArgPairs: List<Pair<String, String>>,
+        allowedKeywords: Set<String>
+    ): Map<String, List<String>> {
         val map = HashMap<String, ArrayList<String>>()
         sectionAndArgPairs
-            // XXX: parse not sections here, e.g. 'notes:' or 'url:' should be ignored
-            .filter { (section, _) -> section in SnakemakeAPI.RULE_OR_CHECKPOINT_ARGS_SECTION_KEYWORDS }
+            .filter {
+                // XXX: alg could find a lot of garbage, especialy from PY and R files
+                // + some sections form YAML description that not needed (e.g. notes, url)
+                // Lets do filtering:
+                (section, _) -> section in allowedKeywords
+            }
             .forEach { (section, arg) ->
                 val sectionKeywords = map.getOrPut(section) { arrayListOf() }
                 if (arg.isNotEmpty() && arg !in sectionKeywords) {

src/main/kotlin/com/jetbrains/snakecharm/codeInsight/completion/wrapper/SmkWrapperStorage.kt

@@ -10,6 +10,7 @@ import com.intellij.openapi.util.Disposer
 import com.jetbrains.snakecharm.SnakemakeBundle
 import com.jetbrains.snakecharm.SnakemakePluginUtil
 import com.jetbrains.snakecharm.SnakemakeTestUtil
+import com.jetbrains.snakecharm.framework.SnakemakeFrameworkAPIProvider
 import com.jetbrains.snakecharm.framework.SmkSupportProjectSettings
 import com.jetbrains.snakecharm.framework.SmkSupportProjectSettingsListener
 import kotlinx.serialization.ExperimentalSerializationApi
@@ -170,7 +171,10 @@ class SmkWrapperStorage(val project: Project) : Disposable {
                 }
                 storage.initFrom(
                     "file://${config.wrappersCustomSourcesFolder}",
-                    SmkWrapperCrawler.localWrapperParser(config.wrappersCustomSourcesFolder)
+                    SmkWrapperCrawler.localWrapperParser(
+                        config.wrappersCustomSourcesFolder,
+                        SnakemakeFrameworkAPIProvider.getInstance()
+                    )
                 )
             }
         }