From 9895687b600414a4e5d02d07eaeb8391b889ba87 Mon Sep 17 00:00:00 2001 From: wojtekzyla <108660584+wojtekzyla@users.noreply.github.com> Date: Mon, 1 Jul 2024 13:43:41 +0200 Subject: [PATCH] fix: add yamllint (#1032) fix: add validation of profiles and groups configuration --- CHANGELOG.md | 2 + charts/splunk-connect-for-snmp/values.yaml | 6 +- docs/bestpractices.md | 11 + docs/gettingstarted/sc4snmp-installation.md | 16 ++ examples/custom-config.yamllint | 6 + examples/lightweight_installation.yaml | 34 +-- examples/o11y_values.yaml | 2 +- examples/polling_and_traps_v3.yaml | 2 +- examples/polling_groups_values.yaml | 2 +- examples/polling_values.yaml | 2 +- integration_tests/test_poller_integration.py | 142 ++++++++++++ poetry.lock | 202 +++++++++++++++++- pyproject.toml | 1 + .../common/collection_manager.py | 91 +++++++- .../common/collections_schemas.py | 124 +++++++++++ 15 files changed, 602 insertions(+), 41 deletions(-) create mode 100644 examples/custom-config.yamllint create mode 100644 splunk_connect_for_snmp/common/collections_schemas.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 8379605bd..47fd7140c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,8 @@ - while mounting volumes in docker compose explicitly mount them as read only or read write - added `values.yaml` schema validation using `values.schema.json` - released beta version of improved polling performance +- added `yamllint` validation for the `values.yaml` formatting +- added "in code" validation of groups and profiles ### Fixed - fixed a bug with configuration from values.yaml not being transferred to the UI while migrating to SC4SNMP-UI diff --git a/charts/splunk-connect-for-snmp/values.yaml b/charts/splunk-connect-for-snmp/values.yaml index 8f6f0c5ae..3dce450f8 100644 --- a/charts/splunk-connect-for-snmp/values.yaml +++ b/charts/splunk-connect-for-snmp/values.yaml @@ -86,7 +86,7 @@ splunk: # name of the metrics index metricsIndex: "netmetrics" - + ################################################################################ # Splunk Observability configuration ################################################################################ @@ -97,7 +97,7 @@ sim: # Splunk Observability realm to send telemetry data to. signalfxToken: "" - # Required for Splunk Observability (if `realm` is specified). + # Required for Splunk Observability (if `realm` is specified). # Splunk Observability org access token. signalfxRealm: "" @@ -383,7 +383,7 @@ inventory: name: "" service: - annotations: { } + annotations: {} # set CPU and Memory limits for an inventory pod resources: {} diff --git a/docs/bestpractices.md b/docs/bestpractices.md index 335bdc608..aa0c97786 100644 --- a/docs/bestpractices.md +++ b/docs/bestpractices.md @@ -153,6 +153,17 @@ microk8s kubectl delete job/snmp-splunk-connect-for-snmp-inventory -n sc4snmp ``` The upgrade command can be executed again. +### "The following profiles have invalid configuration" or "The following groups have invalid configuration" errors +Following errors are examples of wrong configuration: +``` +The following groups have invalid configuration and won't be used: ['group1']. Please check indentation and keywords spelling inside mentioned groups configuration. +``` +``` +The following profiles have invalid configuration and won't be used: ['standard_profile', 'walk_profile']. Please check indentation and keywords spelling inside mentioned profiles configuration. +``` +Errors above indicate, that the mentioned groups or profiles might have wrong indentation or some keywords were omitted or misspelled. Refer to [Configuring profiles](./configuration/configuring-profiles.md) +or [Configuring Groups](./configuration/configuring-groups.md) sections to check how the correct configuration should look like. + ### Identifying Traps issues #### Wrong IP or port diff --git a/docs/gettingstarted/sc4snmp-installation.md b/docs/gettingstarted/sc4snmp-installation.md index 827b6a459..ca8e9488c 100644 --- a/docs/gettingstarted/sc4snmp-installation.md +++ b/docs/gettingstarted/sc4snmp-installation.md @@ -42,6 +42,22 @@ or download it directly from Helm using the command `microk8s helm3 show values It is recommended to start by completing the base template and gradually add additional configurations as needed. + +The `values.yaml` file is validated using `JSON schema` built into `helm chart` and inside the code. +To ensure that your `values.yaml` follows formatting standards, you can use `yamllint`. In order to download +`yamllint` refer to the [installation instructions](https://yamllint.readthedocs.io/en/stable/quickstart.html#installing-yamllint). +Then create `custom-config.yamllint` file and add the following configuration: +```yaml +extends: default + +rules: + line-length: + max: 80 + level: warning +``` +Configuration above can be found in the `examples` directory in SC4SNMP [GitHub repository](https://github.com/splunk/splunk-connect-for-snmp). +Next run `yamllint -c ` command. Warnings can be ignored. + #### Install SC4SNMP After the `values.yaml` creation, you can proceed with the SC4SNMP installation: diff --git a/examples/custom-config.yamllint b/examples/custom-config.yamllint new file mode 100644 index 000000000..ae4843569 --- /dev/null +++ b/examples/custom-config.yamllint @@ -0,0 +1,6 @@ +extends: default + +rules: + line-length: + max: 80 + level: warning diff --git a/examples/lightweight_installation.yaml b/examples/lightweight_installation.yaml index 9742eb01e..e72f4c823 100644 --- a/examples/lightweight_installation.yaml +++ b/examples/lightweight_installation.yaml @@ -14,12 +14,12 @@ traps: - homelab replicaCount: 1 resources: - limits: - cpu: 100m - memory: 300Mi - requests: - cpu: 40m - memory: 256Mi + limits: + cpu: 100m + memory: 300Mi + requests: + cpu: 40m + memory: 256Mi #loadBalancerIP: The IP address in the metallb pool loadBalancerIP: ###TRAP_RECEIVER_IP### worker: @@ -54,12 +54,12 @@ worker: scheduler: logLevel: "INFO" resources: - limits: - cpu: 40m - memory: 260Mi - requests: - cpu: 20m - memory: 180Mi + limits: + cpu: 40m + memory: 260Mi + requests: + cpu: 20m + memory: 180Mi profiles: | generic_switch: frequency: 300 @@ -80,8 +80,8 @@ poller: 10.0.0.100,,3,,sc4snmp-hlab-sha-des,,1800,generic_switch,, inventory: resources: - limits: - cpu: 60m - memory: 300Mi - requests: - cpu: 20m \ No newline at end of file + limits: + cpu: 60m + memory: 300Mi + requests: + cpu: 20m diff --git a/examples/o11y_values.yaml b/examples/o11y_values.yaml index 88d5a6e4d..1712b89b3 100644 --- a/examples/o11y_values.yaml +++ b/examples/o11y_values.yaml @@ -42,4 +42,4 @@ poller: inventory: | address,port,version,community,secret,security_engine,walk_interval,profiles,smart_profiles,delete 54.82.4.248,,2c,public,,,4000,small_walk;IF_profile;ICMP_profile,, - 54.82.4.249,,2c,public,,,1800,small_walk;IF_profile,, \ No newline at end of file + 54.82.4.249,,2c,public,,,1800,small_walk;IF_profile,, diff --git a/examples/polling_and_traps_v3.yaml b/examples/polling_and_traps_v3.yaml index 5f5d47ecd..b91ca9424 100644 --- a/examples/polling_and_traps_v3.yaml +++ b/examples/polling_and_traps_v3.yaml @@ -28,4 +28,4 @@ poller: - sc4snmp-hlab-sha-aes inventory: | address,port,version,community,secret,security_engine,walk_interval,profiles,smart_profiles,delete - 54.82.4.248,,3,public,sc4snmp-hlab-sha-aes,,2000,switch_profile,, \ No newline at end of file + 54.82.4.248,,3,public,sc4snmp-hlab-sha-aes,,2000,switch_profile,, diff --git a/examples/polling_groups_values.yaml b/examples/polling_groups_values.yaml index a2fe73746..995dd6870 100644 --- a/examples/polling_groups_values.yaml +++ b/examples/polling_groups_values.yaml @@ -22,4 +22,4 @@ scheduler: poller: inventory: | address,port,version,community,secret,security_engine,walk_interval,profiles,smart_profiles,delete - switch_group,,2c,public,,,2000,switch_profile,, \ No newline at end of file + switch_group,,2c,public,,,2000,switch_profile,, diff --git a/examples/polling_values.yaml b/examples/polling_values.yaml index dcf193ef6..f0de6e5c7 100644 --- a/examples/polling_values.yaml +++ b/examples/polling_values.yaml @@ -49,4 +49,4 @@ poller: inventory: | address,port,version,community,secret,security_engine,walk_interval,profiles,smart_profiles,delete 54.82.4.248,,2c,public,,,4000,small_walk;IF_profile;ICMP_profile,, - 54.82.4.249,,2c,public,,,1800,small_walk;IF_profile,, \ No newline at end of file + 54.82.4.249,,2c,public,,,1800,small_walk;IF_profile,, diff --git a/integration_tests/test_poller_integration.py b/integration_tests/test_poller_integration.py index f74492e1b..04816fb0b 100644 --- a/integration_tests/test_poller_integration.py +++ b/integration_tests/test_poller_integration.py @@ -1268,6 +1268,148 @@ def test_wrong_profiles(self, request, setup_splunk): assert metric_count == 0 +@pytest.fixture(scope="class") +def setup_misconfigured_profiles(request): + """ + None of the profiles below should poll anything. + """ + trap_external_ip = request.config.getoption("trap_external_ip") + profiles = { + "no_varbinds_profile": { + "frequency": 7, + "varBinds": [], + "conditions": [ + {"field": "IF-MIB.ifIndex", "operation": dq("gt"), "value": 20}, + { + "field": "IF-MIB.ifDescr", + "operation": dq("equals"), + "value": dq("eth0"), + }, + ], + }, + "no_operation_key_in_condition_profile": { + "frequency": 7, + "varBinds": [yaml_escape_list(sq("IF-MIB"), sq("ifOutDiscards"))], + "conditions": [ + {"field": "IF-MIB.ifIndex", "operation": dq("lt"), "value": 20}, + { + "field": "IF-MIB.ifDescr", + "value": [dq("test value 1"), dq("test value 2")], + }, + ], + }, + "no_frequency_profile": { + "varBinds": [yaml_escape_list(sq("IF-MIB"), sq("ifOutDiscards"))], + "conditions": [ + {"field": "IF-MIB.ifIndex", "operation": dq("equals"), "value": 200} + ], + }, + "no_patterns_profile": { + "frequency": 3, + "condition": { + "type": "field", + "field": "SNMPv2-MIB.sysDescr", + }, + "varBinds": [ + yaml_escape_list(sq("IP-MIB"), sq("icmpOutDestUnreachs"), 0), + yaml_escape_list(sq("IP-MIB"), sq("icmpOutEchoReps"), 0), + ], + }, + } + + update_profiles(profiles) + update_file( + [ + f"{trap_external_ip},1165,2c,public,,,600,no_varbinds_profile;no_operation_key_in_condition_profile;no_frequency_profile;no_patterns_profile,,", + ], + "inventory.yaml", + ) + upgrade_helm(["inventory.yaml", "profiles.yaml"]) + time.sleep(120) + yield + update_file( + [ + f"{trap_external_ip},1165,2c,public,,,600,no_varbinds_profile;no_operation_key_in_condition_profile;no_frequency_profile;no_patterns_profile,,t", + ], + "inventory.yaml", + ) + upgrade_helm(["inventory.yaml"]) + time.sleep(120) + + +@pytest.mark.usefixtures("setup_misconfigured_profiles") +class TestMisconfiguredProfiles: + def test_wrong_profiles(self, request, setup_splunk): + time.sleep(20) + search_string = """| mpreview index=netmetrics | search profiles=no_varbinds_profile OR profiles=no_operation_key_in_condition_profile OR profiles=no_frequency_profile OR profiles=no_patterns_profile """ + result_count, metric_count = run_retried_single_search( + setup_splunk, search_string, 2 + ) + assert result_count == 0 + assert metric_count == 0 + + +@pytest.fixture(scope="class") +def setup_misconfigured_groups(request): + trap_external_ip = request.config.getoption("trap_external_ip") + profiles = { + "routers_wrong_group_profile": { + "frequency": 7, + "varBinds": [yaml_escape_list(sq("IP-MIB"))], + }, + "switches_wrong_group_profile": { + "frequency": 7, + "varBinds": [yaml_escape_list(sq("IP-MIB"))], + }, + } + groups = { + "routers": [ + {"address": trap_external_ip, "port": 1163}, + {"address": trap_external_ip, "port": 1164, "wrong_key": 1}, + ], + "switches": [ + {"addre": trap_external_ip}, + {"address": trap_external_ip, "port": 1162}, + ], + } + + update_profiles(profiles) + update_groups(groups) + update_file( + [ + f"{trap_external_ip},1165,2c,public,,,600,single_profile,,", + f"routers,,2c,public,,,600,routers_wrong_group_profile,,", + f"switches,,2c,public,,,600,switches_wrong_group_profile,,", + ], + "inventory.yaml", + ) + upgrade_helm(["inventory.yaml", "profiles.yaml", "groups.yaml"]) + time.sleep(120) + yield + update_file( + [ + f"{trap_external_ip},1165,2c,public,,,600,single_profile,,t", + f"routers,,2c,public,,,600,routers_wrong_group_profile,,t", + f"switches,,2c,public,,,600,switches_wrong_group_profile,,t", + ], + "inventory.yaml", + ) + upgrade_helm(["inventory.yaml"]) + time.sleep(100) + + +@pytest.mark.usefixtures("setup_misconfigured_groups") +class TestMisconfiguredGroups: + def test_wrong_groups(self, request, setup_splunk): + time.sleep(20) + search_string = """| mpreview index=netmetrics | search profiles=routers_wrong_group_profile OR profiles=routers_wrong_group_profile """ + result_count, metric_count = run_retried_single_search( + setup_splunk, search_string, 2 + ) + assert result_count == 0 + assert metric_count == 0 + + def run_retried_single_search(setup_splunk, search_string, retries): for _ in range(retries): result_count, metric_count = splunk_single_search(setup_splunk, search_string) diff --git a/poetry.lock b/poetry.lock index 3d5da3de1..3a8ce7509 100644 --- a/poetry.lock +++ b/poetry.lock @@ -27,20 +27,22 @@ files = [ [[package]] name = "attrs" -version = "22.1.0" +version = "23.2.0" description = "Classes Without Boilerplate" optional = false -python-versions = ">=3.5" +python-versions = ">=3.7" files = [ - {file = "attrs-22.1.0-py2.py3-none-any.whl", hash = "sha256:86efa402f67bf2df34f51a335487cf46b1ec130d02b8d39fd248abfd30da551c"}, - {file = "attrs-22.1.0.tar.gz", hash = "sha256:29adc2665447e5191d0e7c568fde78b21f9672d344281d0c6e1ab085429b22b6"}, + {file = "attrs-23.2.0-py3-none-any.whl", hash = "sha256:99b87a485a5820b23b879f04c2305b44b951b502fd64be915879d77a7e8fc6f1"}, + {file = "attrs-23.2.0.tar.gz", hash = "sha256:935dc3b529c262f6cf76e50877d35a4bd3c1de194fd41f47a2b7ae8f19971f30"}, ] [package.extras] -dev = ["cloudpickle", "coverage[toml] (>=5.0.2)", "furo", "hypothesis", "mypy (>=0.900,!=0.940)", "pre-commit", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "sphinx", "sphinx-notfound-page", "zope.interface"] -docs = ["furo", "sphinx", "sphinx-notfound-page", "zope.interface"] -tests = ["cloudpickle", "coverage[toml] (>=5.0.2)", "hypothesis", "mypy (>=0.900,!=0.940)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "zope.interface"] -tests-no-zope = ["cloudpickle", "coverage[toml] (>=5.0.2)", "hypothesis", "mypy (>=0.900,!=0.940)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins"] +cov = ["attrs[tests]", "coverage[toml] (>=5.3)"] +dev = ["attrs[tests]", "pre-commit"] +docs = ["furo", "myst-parser", "sphinx", "sphinx-notfound-page", "sphinxcontrib-towncrier", "towncrier", "zope-interface"] +tests = ["attrs[tests-no-zope]", "zope-interface"] +tests-mypy = ["mypy (>=1.6)", "pytest-mypy-plugins"] +tests-no-zope = ["attrs[tests-mypy]", "cloudpickle", "hypothesis", "pympler", "pytest (>=4.3.0)", "pytest-xdist[psutil]"] [[package]] name = "babel" @@ -517,6 +519,44 @@ files = [ {file = "JSON-log-formatter-0.5.2.tar.gz", hash = "sha256:7b191ae4056468baf2b7445c5ce651bd9ee6b76b4162a479a7d85db6ae029f2d"}, ] +[[package]] +name = "jsonschema" +version = "4.22.0" +description = "An implementation of JSON Schema validation for Python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "jsonschema-4.22.0-py3-none-any.whl", hash = "sha256:ff4cfd6b1367a40e7bc6411caec72effadd3db0bbe5017de188f2d6108335802"}, + {file = "jsonschema-4.22.0.tar.gz", hash = "sha256:5b22d434a45935119af990552c862e5d6d564e8f6601206b305a61fdf661a2b7"}, +] + +[package.dependencies] +attrs = ">=22.2.0" +importlib-resources = {version = ">=1.4.0", markers = "python_version < \"3.9\""} +jsonschema-specifications = ">=2023.03.6" +pkgutil-resolve-name = {version = ">=1.3.10", markers = "python_version < \"3.9\""} +referencing = ">=0.28.4" +rpds-py = ">=0.7.1" + +[package.extras] +format = ["fqdn", "idna", "isoduration", "jsonpointer (>1.13)", "rfc3339-validator", "rfc3987", "uri-template", "webcolors (>=1.11)"] +format-nongpl = ["fqdn", "idna", "isoduration", "jsonpointer (>1.13)", "rfc3339-validator", "rfc3986-validator (>0.1.0)", "uri-template", "webcolors (>=1.11)"] + +[[package]] +name = "jsonschema-specifications" +version = "2023.12.1" +description = "The JSON Schema meta-schemas and vocabularies, exposed as a Registry" +optional = false +python-versions = ">=3.8" +files = [ + {file = "jsonschema_specifications-2023.12.1-py3-none-any.whl", hash = "sha256:87e4fdf3a94858b8a2ba2778d9ba57d8a9cafca7c7489c46ba0d30a8bc6a9c3c"}, + {file = "jsonschema_specifications-2023.12.1.tar.gz", hash = "sha256:48a76787b3e70f5ed53f1160d2b81f586e4ca6d1548c5de7085d1682674764cc"}, +] + +[package.dependencies] +importlib-resources = {version = ">=1.4.0", markers = "python_version < \"3.9\""} +referencing = ">=0.31.0" + [[package]] name = "kombu" version = "5.3.4" @@ -1032,6 +1072,17 @@ gevent = ["gevent"] tornado = ["tornado"] twisted = ["twisted"] +[[package]] +name = "pkgutil-resolve-name" +version = "1.3.10" +description = "Resolve a name to an object." +optional = false +python-versions = ">=3.6" +files = [ + {file = "pkgutil_resolve_name-1.3.10-py3-none-any.whl", hash = "sha256:ca27cc078d25c5ad71a9de0a7a330146c4e014c2462d9af19c6b828280649c5e"}, + {file = "pkgutil_resolve_name-1.3.10.tar.gz", hash = "sha256:357d6c9e6a755653cfd78893817c0853af365dd51ec97f3d358a819373bbd174"}, +] + [[package]] name = "platformdirs" version = "3.10.0" @@ -1481,6 +1532,7 @@ files = [ {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938"}, {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d"}, {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515"}, + {file = "PyYAML-6.0.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:326c013efe8048858a6d312ddd31d56e468118ad4cdeda36c719bf5bb6192290"}, {file = "PyYAML-6.0.1-cp310-cp310-win32.whl", hash = "sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924"}, {file = "PyYAML-6.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d"}, {file = "PyYAML-6.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007"}, @@ -1488,8 +1540,15 @@ files = [ {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d"}, {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc"}, {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673"}, + {file = "PyYAML-6.0.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e7d73685e87afe9f3b36c799222440d6cf362062f78be1013661b00c5c6f678b"}, {file = "PyYAML-6.0.1-cp311-cp311-win32.whl", hash = "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741"}, {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, + {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, + {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, + {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, + {file = "PyYAML-6.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df"}, {file = "PyYAML-6.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47"}, {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98"}, {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c"}, @@ -1506,6 +1565,7 @@ files = [ {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5"}, {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696"}, {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735"}, + {file = "PyYAML-6.0.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:49a183be227561de579b4a36efbb21b3eab9651dd81b1858589f796549873dd6"}, {file = "PyYAML-6.0.1-cp38-cp38-win32.whl", hash = "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206"}, {file = "PyYAML-6.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62"}, {file = "PyYAML-6.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8"}, @@ -1513,6 +1573,7 @@ files = [ {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6"}, {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0"}, {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c"}, + {file = "PyYAML-6.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5"}, {file = "PyYAML-6.0.1-cp39-cp39-win32.whl", hash = "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c"}, {file = "PyYAML-6.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486"}, {file = "PyYAML-6.0.1.tar.gz", hash = "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43"}, @@ -1550,6 +1611,21 @@ async-timeout = ">=4.0.2" hiredis = ["hiredis (>=1.0.0)"] ocsp = ["cryptography (>=36.0.1)", "pyopenssl (==20.0.1)", "requests (>=2.26.0)"] +[[package]] +name = "referencing" +version = "0.35.1" +description = "JSON Referencing + Python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "referencing-0.35.1-py3-none-any.whl", hash = "sha256:eda6d3234d62814d1c64e305c1331c9a3a6132da475ab6382eaa997b21ee75de"}, + {file = "referencing-0.35.1.tar.gz", hash = "sha256:25b42124a6c8b632a425174f24087783efb348a6f1e0008e63cd4466fedf703c"}, +] + +[package.dependencies] +attrs = ">=22.2.0" +rpds-py = ">=0.7.0" + [[package]] name = "regex" version = "2023.8.8" @@ -1716,6 +1792,114 @@ requests = ">=2.20" [package.extras] docs = ["furo (>=2023.3,<2024.0)", "myst-parser (>=1.0)", "sphinx (>=5.2,<6.0)", "sphinx-autodoc-typehints (>=1.22,<2.0)", "sphinx-copybutton (>=0.5)"] +[[package]] +name = "rpds-py" +version = "0.18.1" +description = "Python bindings to Rust's persistent data structures (rpds)" +optional = false +python-versions = ">=3.8" +files = [ + {file = "rpds_py-0.18.1-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:d31dea506d718693b6b2cffc0648a8929bdc51c70a311b2770f09611caa10d53"}, + {file = "rpds_py-0.18.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:732672fbc449bab754e0b15356c077cc31566df874964d4801ab14f71951ea80"}, + {file = "rpds_py-0.18.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4a98a1f0552b5f227a3d6422dbd61bc6f30db170939bd87ed14f3c339aa6c7c9"}, + {file = "rpds_py-0.18.1-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:7f1944ce16401aad1e3f7d312247b3d5de7981f634dc9dfe90da72b87d37887d"}, + {file = "rpds_py-0.18.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:38e14fb4e370885c4ecd734f093a2225ee52dc384b86fa55fe3f74638b2cfb09"}, + {file = "rpds_py-0.18.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:08d74b184f9ab6289b87b19fe6a6d1a97fbfea84b8a3e745e87a5de3029bf944"}, + {file = "rpds_py-0.18.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d70129cef4a8d979caa37e7fe957202e7eee8ea02c5e16455bc9808a59c6b2f0"}, + {file = "rpds_py-0.18.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:ce0bb20e3a11bd04461324a6a798af34d503f8d6f1aa3d2aa8901ceaf039176d"}, + {file = "rpds_py-0.18.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:81c5196a790032e0fc2464c0b4ab95f8610f96f1f2fa3d4deacce6a79852da60"}, + {file = "rpds_py-0.18.1-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:f3027be483868c99b4985fda802a57a67fdf30c5d9a50338d9db646d590198da"}, + {file = "rpds_py-0.18.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:d44607f98caa2961bab4fa3c4309724b185b464cdc3ba6f3d7340bac3ec97cc1"}, + {file = "rpds_py-0.18.1-cp310-none-win32.whl", hash = "sha256:c273e795e7a0f1fddd46e1e3cb8be15634c29ae8ff31c196debb620e1edb9333"}, + {file = "rpds_py-0.18.1-cp310-none-win_amd64.whl", hash = "sha256:8352f48d511de5f973e4f2f9412736d7dea76c69faa6d36bcf885b50c758ab9a"}, + {file = "rpds_py-0.18.1-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:6b5ff7e1d63a8281654b5e2896d7f08799378e594f09cf3674e832ecaf396ce8"}, + {file = "rpds_py-0.18.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:8927638a4d4137a289e41d0fd631551e89fa346d6dbcfc31ad627557d03ceb6d"}, + {file = "rpds_py-0.18.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:154bf5c93d79558b44e5b50cc354aa0459e518e83677791e6adb0b039b7aa6a7"}, + {file = "rpds_py-0.18.1-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:07f2139741e5deb2c5154a7b9629bc5aa48c766b643c1a6750d16f865a82c5fc"}, + {file = "rpds_py-0.18.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8c7672e9fba7425f79019db9945b16e308ed8bc89348c23d955c8c0540da0a07"}, + {file = "rpds_py-0.18.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:489bdfe1abd0406eba6b3bb4fdc87c7fa40f1031de073d0cfb744634cc8fa261"}, + {file = "rpds_py-0.18.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3c20f05e8e3d4fc76875fc9cb8cf24b90a63f5a1b4c5b9273f0e8225e169b100"}, + {file = "rpds_py-0.18.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:967342e045564cef76dfcf1edb700b1e20838d83b1aa02ab313e6a497cf923b8"}, + {file = "rpds_py-0.18.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:2cc7c1a47f3a63282ab0f422d90ddac4aa3034e39fc66a559ab93041e6505da7"}, + {file = "rpds_py-0.18.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:f7afbfee1157e0f9376c00bb232e80a60e59ed716e3211a80cb8506550671e6e"}, + {file = "rpds_py-0.18.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:9e6934d70dc50f9f8ea47081ceafdec09245fd9f6032669c3b45705dea096b88"}, + {file = "rpds_py-0.18.1-cp311-none-win32.whl", hash = "sha256:c69882964516dc143083d3795cb508e806b09fc3800fd0d4cddc1df6c36e76bb"}, + {file = "rpds_py-0.18.1-cp311-none-win_amd64.whl", hash = "sha256:70a838f7754483bcdc830444952fd89645569e7452e3226de4a613a4c1793fb2"}, + {file = "rpds_py-0.18.1-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:3dd3cd86e1db5aadd334e011eba4e29d37a104b403e8ca24dcd6703c68ca55b3"}, + {file = "rpds_py-0.18.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:05f3d615099bd9b13ecf2fc9cf2d839ad3f20239c678f461c753e93755d629ee"}, + {file = "rpds_py-0.18.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:35b2b771b13eee8729a5049c976197ff58a27a3829c018a04341bcf1ae409b2b"}, + {file = "rpds_py-0.18.1-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:ee17cd26b97d537af8f33635ef38be873073d516fd425e80559f4585a7b90c43"}, + {file = "rpds_py-0.18.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b646bf655b135ccf4522ed43d6902af37d3f5dbcf0da66c769a2b3938b9d8184"}, + {file = "rpds_py-0.18.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:19ba472b9606c36716062c023afa2484d1e4220548751bda14f725a7de17b4f6"}, + {file = "rpds_py-0.18.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6e30ac5e329098903262dc5bdd7e2086e0256aa762cc8b744f9e7bf2a427d3f8"}, + {file = "rpds_py-0.18.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:d58ad6317d188c43750cb76e9deacf6051d0f884d87dc6518e0280438648a9ac"}, + {file = "rpds_py-0.18.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:e1735502458621921cee039c47318cb90b51d532c2766593be6207eec53e5c4c"}, + {file = "rpds_py-0.18.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:f5bab211605d91db0e2995a17b5c6ee5edec1270e46223e513eaa20da20076ac"}, + {file = "rpds_py-0.18.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:2fc24a329a717f9e2448f8cd1f960f9dac4e45b6224d60734edeb67499bab03a"}, + {file = "rpds_py-0.18.1-cp312-none-win32.whl", hash = "sha256:1805d5901779662d599d0e2e4159d8a82c0b05faa86ef9222bf974572286b2b6"}, + {file = "rpds_py-0.18.1-cp312-none-win_amd64.whl", hash = "sha256:720edcb916df872d80f80a1cc5ea9058300b97721efda8651efcd938a9c70a72"}, + {file = "rpds_py-0.18.1-cp38-cp38-macosx_10_12_x86_64.whl", hash = "sha256:c827576e2fa017a081346dce87d532a5310241648eb3700af9a571a6e9fc7e74"}, + {file = "rpds_py-0.18.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:aa3679e751408d75a0b4d8d26d6647b6d9326f5e35c00a7ccd82b78ef64f65f8"}, + {file = "rpds_py-0.18.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0abeee75434e2ee2d142d650d1e54ac1f8b01e6e6abdde8ffd6eeac6e9c38e20"}, + {file = "rpds_py-0.18.1-cp38-cp38-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:ed402d6153c5d519a0faf1bb69898e97fb31613b49da27a84a13935ea9164dfc"}, + {file = "rpds_py-0.18.1-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:338dee44b0cef8b70fd2ef54b4e09bb1b97fc6c3a58fea5db6cc083fd9fc2724"}, + {file = "rpds_py-0.18.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7750569d9526199c5b97e5a9f8d96a13300950d910cf04a861d96f4273d5b104"}, + {file = "rpds_py-0.18.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:607345bd5912aacc0c5a63d45a1f73fef29e697884f7e861094e443187c02be5"}, + {file = "rpds_py-0.18.1-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:207c82978115baa1fd8d706d720b4a4d2b0913df1c78c85ba73fe6c5804505f0"}, + {file = "rpds_py-0.18.1-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:6d1e42d2735d437e7e80bab4d78eb2e459af48c0a46e686ea35f690b93db792d"}, + {file = "rpds_py-0.18.1-cp38-cp38-musllinux_1_2_i686.whl", hash = "sha256:5463c47c08630007dc0fe99fb480ea4f34a89712410592380425a9b4e1611d8e"}, + {file = "rpds_py-0.18.1-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:06d218939e1bf2ca50e6b0ec700ffe755e5216a8230ab3e87c059ebb4ea06afc"}, + {file = "rpds_py-0.18.1-cp38-none-win32.whl", hash = "sha256:312fe69b4fe1ffbe76520a7676b1e5ac06ddf7826d764cc10265c3b53f96dbe9"}, + {file = "rpds_py-0.18.1-cp38-none-win_amd64.whl", hash = "sha256:9437ca26784120a279f3137ee080b0e717012c42921eb07861b412340f85bae2"}, + {file = "rpds_py-0.18.1-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:19e515b78c3fc1039dd7da0a33c28c3154458f947f4dc198d3c72db2b6b5dc93"}, + {file = "rpds_py-0.18.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:a7b28c5b066bca9a4eb4e2f2663012debe680f097979d880657f00e1c30875a0"}, + {file = "rpds_py-0.18.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:673fdbbf668dd958eff750e500495ef3f611e2ecc209464f661bc82e9838991e"}, + {file = "rpds_py-0.18.1-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:d960de62227635d2e61068f42a6cb6aae91a7fe00fca0e3aeed17667c8a34611"}, + {file = "rpds_py-0.18.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:352a88dc7892f1da66b6027af06a2e7e5d53fe05924cc2cfc56495b586a10b72"}, + {file = "rpds_py-0.18.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4e0ee01ad8260184db21468a6e1c37afa0529acc12c3a697ee498d3c2c4dcaf3"}, + {file = "rpds_py-0.18.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e4c39ad2f512b4041343ea3c7894339e4ca7839ac38ca83d68a832fc8b3748ab"}, + {file = "rpds_py-0.18.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:aaa71ee43a703c321906813bb252f69524f02aa05bf4eec85f0c41d5d62d0f4c"}, + {file = "rpds_py-0.18.1-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:6cd8098517c64a85e790657e7b1e509b9fe07487fd358e19431cb120f7d96338"}, + {file = "rpds_py-0.18.1-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:4adec039b8e2928983f885c53b7cc4cda8965b62b6596501a0308d2703f8af1b"}, + {file = "rpds_py-0.18.1-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:32b7daaa3e9389db3695964ce8e566e3413b0c43e3394c05e4b243a4cd7bef26"}, + {file = "rpds_py-0.18.1-cp39-none-win32.whl", hash = "sha256:2625f03b105328729f9450c8badda34d5243231eef6535f80064d57035738360"}, + {file = "rpds_py-0.18.1-cp39-none-win_amd64.whl", hash = "sha256:bf18932d0003c8c4d51a39f244231986ab23ee057d235a12b2684ea26a353590"}, + {file = "rpds_py-0.18.1-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:cbfbea39ba64f5e53ae2915de36f130588bba71245b418060ec3330ebf85678e"}, + {file = "rpds_py-0.18.1-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:a3d456ff2a6a4d2adcdf3c1c960a36f4fd2fec6e3b4902a42a384d17cf4e7a65"}, + {file = "rpds_py-0.18.1-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7700936ef9d006b7ef605dc53aa364da2de5a3aa65516a1f3ce73bf82ecfc7ae"}, + {file = "rpds_py-0.18.1-pp310-pypy310_pp73-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:51584acc5916212e1bf45edd17f3a6b05fe0cbb40482d25e619f824dccb679de"}, + {file = "rpds_py-0.18.1-pp310-pypy310_pp73-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:942695a206a58d2575033ff1e42b12b2aece98d6003c6bc739fbf33d1773b12f"}, + {file = "rpds_py-0.18.1-pp310-pypy310_pp73-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b906b5f58892813e5ba5c6056d6a5ad08f358ba49f046d910ad992196ea61397"}, + {file = "rpds_py-0.18.1-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f6f8e3fecca256fefc91bb6765a693d96692459d7d4c644660a9fff32e517843"}, + {file = "rpds_py-0.18.1-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:7732770412bab81c5a9f6d20aeb60ae943a9b36dcd990d876a773526468e7163"}, + {file = "rpds_py-0.18.1-pp310-pypy310_pp73-musllinux_1_2_aarch64.whl", hash = "sha256:bd1105b50ede37461c1d51b9698c4f4be6e13e69a908ab7751e3807985fc0346"}, + {file = "rpds_py-0.18.1-pp310-pypy310_pp73-musllinux_1_2_i686.whl", hash = "sha256:618916f5535784960f3ecf8111581f4ad31d347c3de66d02e728de460a46303c"}, + {file = "rpds_py-0.18.1-pp310-pypy310_pp73-musllinux_1_2_x86_64.whl", hash = "sha256:17c6d2155e2423f7e79e3bb18151c686d40db42d8645e7977442170c360194d4"}, + {file = "rpds_py-0.18.1-pp38-pypy38_pp73-macosx_10_12_x86_64.whl", hash = "sha256:6c4c4c3f878df21faf5fac86eda32671c27889e13570645a9eea0a1abdd50922"}, + {file = "rpds_py-0.18.1-pp38-pypy38_pp73-macosx_11_0_arm64.whl", hash = "sha256:fab6ce90574645a0d6c58890e9bcaac8d94dff54fb51c69e5522a7358b80ab64"}, + {file = "rpds_py-0.18.1-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:531796fb842b53f2695e94dc338929e9f9dbf473b64710c28af5a160b2a8927d"}, + {file = "rpds_py-0.18.1-pp38-pypy38_pp73-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:740884bc62a5e2bbb31e584f5d23b32320fd75d79f916f15a788d527a5e83644"}, + {file = "rpds_py-0.18.1-pp38-pypy38_pp73-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:998125738de0158f088aef3cb264a34251908dd2e5d9966774fdab7402edfab7"}, + {file = "rpds_py-0.18.1-pp38-pypy38_pp73-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e2be6e9dd4111d5b31ba3b74d17da54a8319d8168890fbaea4b9e5c3de630ae5"}, + {file = "rpds_py-0.18.1-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d0cee71bc618cd93716f3c1bf56653740d2d13ddbd47673efa8bf41435a60daa"}, + {file = "rpds_py-0.18.1-pp38-pypy38_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:2c3caec4ec5cd1d18e5dd6ae5194d24ed12785212a90b37f5f7f06b8bedd7139"}, + {file = "rpds_py-0.18.1-pp38-pypy38_pp73-musllinux_1_2_aarch64.whl", hash = "sha256:27bba383e8c5231cd559affe169ca0b96ec78d39909ffd817f28b166d7ddd4d8"}, + {file = "rpds_py-0.18.1-pp38-pypy38_pp73-musllinux_1_2_i686.whl", hash = "sha256:a888e8bdb45916234b99da2d859566f1e8a1d2275a801bb8e4a9644e3c7e7909"}, + {file = "rpds_py-0.18.1-pp38-pypy38_pp73-musllinux_1_2_x86_64.whl", hash = "sha256:6031b25fb1b06327b43d841f33842b383beba399884f8228a6bb3df3088485ff"}, + {file = "rpds_py-0.18.1-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:48c2faaa8adfacefcbfdb5f2e2e7bdad081e5ace8d182e5f4ade971f128e6bb3"}, + {file = "rpds_py-0.18.1-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:d85164315bd68c0806768dc6bb0429c6f95c354f87485ee3593c4f6b14def2bd"}, + {file = "rpds_py-0.18.1-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6afd80f6c79893cfc0574956f78a0add8c76e3696f2d6a15bca2c66c415cf2d4"}, + {file = "rpds_py-0.18.1-pp39-pypy39_pp73-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:fa242ac1ff583e4ec7771141606aafc92b361cd90a05c30d93e343a0c2d82a89"}, + {file = "rpds_py-0.18.1-pp39-pypy39_pp73-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d21be4770ff4e08698e1e8e0bce06edb6ea0626e7c8f560bc08222880aca6a6f"}, + {file = "rpds_py-0.18.1-pp39-pypy39_pp73-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5c45a639e93a0c5d4b788b2613bd637468edd62f8f95ebc6fcc303d58ab3f0a8"}, + {file = "rpds_py-0.18.1-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:910e71711d1055b2768181efa0a17537b2622afeb0424116619817007f8a2b10"}, + {file = "rpds_py-0.18.1-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:b9bb1f182a97880f6078283b3505a707057c42bf55d8fca604f70dedfdc0772a"}, + {file = "rpds_py-0.18.1-pp39-pypy39_pp73-musllinux_1_2_aarch64.whl", hash = "sha256:1d54f74f40b1f7aaa595a02ff42ef38ca654b1469bef7d52867da474243cc633"}, + {file = "rpds_py-0.18.1-pp39-pypy39_pp73-musllinux_1_2_i686.whl", hash = "sha256:8d2e182c9ee01135e11e9676e9a62dfad791a7a467738f06726872374a83db49"}, + {file = "rpds_py-0.18.1-pp39-pypy39_pp73-musllinux_1_2_x86_64.whl", hash = "sha256:636a15acc588f70fda1661234761f9ed9ad79ebed3f2125d44be0862708b666e"}, + {file = "rpds_py-0.18.1.tar.gz", hash = "sha256:dc48b479d540770c811fbd1eb9ba2bb66951863e448efec2e2c102625328e92f"}, +] + [[package]] name = "ruamel-yaml" version = "0.18.6" @@ -2118,4 +2302,4 @@ testing = ["func-timeout", "jaraco.itertools", "pytest (>=6)", "pytest-black (>= [metadata] lock-version = "2.0" python-versions = "^3.8" -content-hash = "e292c76c15cb1f537bdd38d763f4c809c54e59f07496b422e8bb796f9a0d981f" +content-hash = "91fd8d543aa315511bfe32d7bd16607b8f79dafd761985a75f786fac2bd5a82f" diff --git a/pyproject.toml b/pyproject.toml index ad93f9ff4..6f32b4627 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -37,6 +37,7 @@ JSON-log-formatter ="^0.5.1" "ruamel.yaml" = "^0.18.0" pysnmplib = {git = "https://github.com/pysnmp/pysnmp.git", branch = "main"} urllib3 = "^1.26.17" +jsonschema = "4.22.0" [tool.poetry.group.dev.dependencies] pytest = "^8.0.0" diff --git a/splunk_connect_for_snmp/common/collection_manager.py b/splunk_connect_for_snmp/common/collection_manager.py index f921d6e46..6c92ce71b 100644 --- a/splunk_connect_for_snmp/common/collection_manager.py +++ b/splunk_connect_for_snmp/common/collection_manager.py @@ -1,10 +1,20 @@ +import logging import os +import sys from abc import abstractmethod from contextlib import suppress import yaml from celery.utils.log import get_task_logger - +from jsonschema import ValidationError, validate + +from splunk_connect_for_snmp.common.collections_schemas import ( + get_all_group_schemas, + get_all_profile_schemas, +) +from splunk_connect_for_snmp.common.customised_json_formatter import ( + CustomisedJSONFormatter, +) from splunk_connect_for_snmp.common.hummanbool import human_bool with suppress(ImportError, OSError): @@ -14,7 +24,18 @@ CONFIG_PATH = os.getenv("CONFIG_PATH", "/app/config/config.yaml") CONFIG_FROM_MONGO = human_bool(os.getenv("CONFIG_FROM_MONGO", "false").lower()) -logger = get_task_logger(__name__) +celery_logger = get_task_logger(__name__) + +log_level = "INFO" +formatter = CustomisedJSONFormatter() +logger = logging.getLogger(__name__) +logger.setLevel(log_level) + +# writing to stdout +handler = logging.StreamHandler(sys.stdout) +handler.setLevel(log_level) +handler.setFormatter(formatter) +logger.addHandler(handler) class CollectionManager: @@ -27,6 +48,11 @@ def __init__(self, mongo, collection_name): def gather_elements(): pass + @staticmethod + @abstractmethod + def validate_elements(elements: dict): + pass + def return_collection_once(self): collection_elements = {} collection_cursor = self.collection.find({}, {"_id": 0}) @@ -62,6 +88,7 @@ def update_all(self): # check in case only header is present if all_elements is None: all_elements = {} + self.validate_elements(all_elements) self.update_collection(all_elements) @@ -69,6 +96,30 @@ class GroupsManager(CollectionManager): def __init__(self, mongo): super().__init__(mongo, "groups") + @staticmethod + @abstractmethod + def validate_elements(elements: dict): + schemas = get_all_group_schemas() + invalid_groups = [] + for group_name, group_body in elements.items(): + valid = False + for schema in schemas: + try: + validate(group_body, schema) + valid = True + break + except ValidationError: + continue + if not valid: + invalid_groups.append(group_name) + for group in invalid_groups: + del elements[group] + if invalid_groups: + logger.error( + f"The following groups have invalid configuration and won't be used: {invalid_groups}. Please check " + f"indentation and keywords spelling inside inside mentioned groups configuration." + ) + def gather_elements(self): groups = {} if CONFIG_FROM_MONGO: @@ -82,7 +133,7 @@ def gather_elements(self): if "groups" in config_runtime: groups = config_runtime.get("groups", {}) except FileNotFoundError: - logger.info(f"File: {CONFIG_PATH} not found") + celery_logger.info(f"File: {CONFIG_PATH} not found") return groups @@ -90,6 +141,30 @@ class ProfilesManager(CollectionManager): def __init__(self, mongo): super().__init__(mongo, "profiles") + @staticmethod + @abstractmethod + def validate_elements(elements: dict): + schemas = get_all_profile_schemas() + invalid_profiles = [] + for profile_name, profile_body in elements.items(): + valid = False + for schema in schemas: + try: + validate(profile_body, schema) + valid = True + break + except ValidationError: + continue + if not valid: + invalid_profiles.append(profile_name) + for profile in invalid_profiles: + del elements[profile] + if invalid_profiles: + logger.error( + f"The following profiles have invalid configuration and won't be used: {invalid_profiles}. Please check " + f"indentation and keywords spelling inside inside mentioned profiles configuration." + ) + def gather_elements(self): active_profiles = {} @@ -100,7 +175,7 @@ def gather_elements(self): if file.endswith("yaml"): with open(os.path.join(pkg_path, file), encoding="utf-8") as of: profiles = yaml.safe_load(of) - logger.info( + celery_logger.info( f"loading {len(profiles.keys())} profiles from shared profile group {file}" ) for key, profile in profiles.items(): @@ -112,7 +187,7 @@ def gather_elements(self): profile = pr[key] if key in active_profiles: if not profile.get("enabled", True): - logger.info(f"disabling profile {key}") + celery_logger.info(f"disabling profile {key}") del active_profiles[key] else: active_profiles[key] = profile @@ -124,18 +199,18 @@ def gather_elements(self): config_runtime = yaml.safe_load(file) if "profiles" in config_runtime: profiles = config_runtime.get("profiles", {}) - logger.info( + celery_logger.info( f"loading {len(profiles.keys())} profiles from runtime profile group" ) for key, profile in profiles.items(): if key in active_profiles: if not profile.get("enabled", True): - logger.info(f"disabling profile {key}") + celery_logger.info(f"disabling profile {key}") del active_profiles[key] else: active_profiles[key] = profile else: active_profiles[key] = profile except FileNotFoundError: - logger.info(f"File: {CONFIG_PATH} not found") + celery_logger.info(f"File: {CONFIG_PATH} not found") return active_profiles diff --git a/splunk_connect_for_snmp/common/collections_schemas.py b/splunk_connect_for_snmp/common/collections_schemas.py new file mode 100644 index 000000000..9028fd3fd --- /dev/null +++ b/splunk_connect_for_snmp/common/collections_schemas.py @@ -0,0 +1,124 @@ +varbinds_type = { + "type": "array", + "items": {"type": "array", "items": {"type": ["integer", "string"]}}, +} + +standard_profile_schema = { + "type": "object", + "properties": { + "frequency": {"type": "integer"}, + "varBinds": varbinds_type, + }, + "required": ["frequency", "varBinds"], + "additionalProperties": False, +} + +walk_profile_schema = { + "type": "object", + "properties": { + "varBinds": varbinds_type, + "condition": { + "type": "object", + "properties": {"type": {"type": "string", "enum": ["walk"]}}, + "required": ["type"], + "additionalProperties": False, + }, + }, + "required": ["condition", "varBinds"], + "additionalProperties": False, +} + +base_profile_schema = { + "type": "object", + "properties": { + "frequency": {"type": "integer"}, + "varBinds": varbinds_type, + "condition": { + "type": "object", + "properties": {"type": {"type": "string", "enum": ["base", "mandatory"]}}, + "required": ["type"], + "additionalProperties": False, + }, + }, + "required": ["condition", "varBinds", "frequency"], + "additionalProperties": False, +} + +smart_profile_schema = { + "type": "object", + "properties": { + "frequency": {"type": "integer"}, + "varBinds": varbinds_type, + "condition": { + "type": "object", + "properties": { + "type": {"type": "string", "enum": ["field"]}, + "field": {"type": "string"}, + "patterns": {"type": "array", "items": {"type": "string"}}, + }, + "required": ["type", "field", "patterns"], + "additionalProperties": False, + }, + }, + "required": ["condition", "varBinds", "frequency"], + "additionalProperties": False, +} + +conditional_profile_schema = { + "type": "object", + "properties": { + "frequency": {"type": "integer"}, + "varBinds": varbinds_type, + "conditions": { + "type": "array", + "items": { + "type": "object", + "properties": { + "field": {"type": "string"}, + "operation": { + "type": "string", + "enum": ["lt", "gt", "regex", "in", "equals"], + }, + "value": {"type": ["number", "string", "array"]}, + "negate_operation": {"type": ["boolean", "string"]}, + }, + "required": ["field", "operation", "value"], + "additionalProperties": False, + }, + }, + }, + "required": ["conditions", "varBinds", "frequency"], + "additionalProperties": False, +} + + +def get_all_profile_schemas(): + return [ + standard_profile_schema, + walk_profile_schema, + base_profile_schema, + smart_profile_schema, + conditional_profile_schema, + ] + + +group_schema = { + "type": "array", + "items": { + "type": "object", + "properties": { + "address": {"type": "string"}, + "port": {"type": "integer"}, + "community": {"type": "string"}, + "secret": {"type": "string"}, + "version": {"type": "string"}, + "security_engine": {"type": "string"}, + }, + "required": ["address"], + "additionalProperties": False, + }, +} + + +def get_all_group_schemas(): + return [group_schema]