From 14cefe46fb62f1992a92d7b4949fff389a76fa29 Mon Sep 17 00:00:00 2001 From: Matthias Veit Date: Wed, 27 Sep 2023 15:35:13 +0200 Subject: [PATCH] [resotocore][chore] Remove unused plugins (#1786) --- .../check_pr_plugin_cleanup_aws_alarms.yml | 73 --------- ...ck_pr_plugin_cleanup_aws_loadbalancers.yml | 73 --------- .../check_pr_plugin_cleanup_aws_vpcs.yml | 73 --------- .../check_pr_plugin_cleanup_expired.yml | 71 --------- .../check_pr_plugin_cleanup_untagged.yml | 71 --------- .../check_pr_plugin_cleanup_volumes.yml | 71 --------- .../workflows/check_pr_plugin_protector.yml | 71 --------- .../check_pr_plugin_tagvalidator.yml | 71 --------- plugins/cleanup_aws_alarms/MANIFEST.in | 1 - plugins/cleanup_aws_alarms/README.md | 29 ---- plugins/cleanup_aws_alarms/pyproject.toml | 42 ----- .../__init__.py | 70 -------- .../config.py | 35 ---- plugins/cleanup_aws_alarms/setup.cfg | 7 - .../cleanup_aws_alarms/test/test_config.py | 10 -- plugins/cleanup_aws_alarms/tox.ini | 30 ---- plugins/cleanup_aws_loadbalancers/MANIFEST.in | 1 - plugins/cleanup_aws_loadbalancers/README.md | 29 ---- .../cleanup_aws_loadbalancers/pyproject.toml | 45 ------ .../__init__.py | 149 ------------------ .../config.py | 24 --- plugins/cleanup_aws_loadbalancers/setup.cfg | 7 - .../test/test_config.py | 10 -- plugins/cleanup_aws_loadbalancers/tox.ini | 30 ---- plugins/cleanup_aws_vpcs/MANIFEST.in | 1 - plugins/cleanup_aws_vpcs/README.md | 39 ----- plugins/cleanup_aws_vpcs/pyproject.toml | 45 ------ .../__init__.py | 134 ---------------- .../resoto_plugin_cleanup_aws_vpcs/config.py | 42 ----- plugins/cleanup_aws_vpcs/setup.cfg | 7 - plugins/cleanup_aws_vpcs/test/test_config.py | 10 -- plugins/cleanup_aws_vpcs/tox.ini | 30 ---- plugins/cleanup_expired/MANIFEST.in | 1 - plugins/cleanup_expired/README.md | 38 ----- plugins/cleanup_expired/pyproject.toml | 44 ------ .../resoto_plugin_cleanup_expired/__init__.py | 41 ----- .../resoto_plugin_cleanup_expired/config.py | 11 -- plugins/cleanup_expired/setup.cfg | 7 - plugins/cleanup_expired/test/test_config.py | 9 -- plugins/cleanup_expired/tox.ini | 29 ---- plugins/cleanup_untagged/MANIFEST.in | 1 - plugins/cleanup_untagged/README.md | 68 -------- plugins/cleanup_untagged/example.yaml | 31 ---- plugins/cleanup_untagged/pyproject.toml | 44 ------ .../__init__.py | 55 ------- .../resoto_plugin_cleanup_untagged/config.py | 85 ---------- plugins/cleanup_untagged/setup.cfg | 7 - plugins/cleanup_untagged/test/test_config.py | 35 ---- plugins/cleanup_untagged/tox.ini | 31 ---- plugins/cleanup_volumes/MANIFEST.in | 1 - plugins/cleanup_volumes/README.md | 37 ----- plugins/cleanup_volumes/pyproject.toml | 44 ------ .../resoto_plugin_cleanup_volumes/__init__.py | 67 -------- .../resoto_plugin_cleanup_volumes/config.py | 24 --- plugins/cleanup_volumes/setup.cfg | 7 - plugins/cleanup_volumes/test/test_config.py | 10 -- plugins/cleanup_volumes/tox.ini | 29 ---- plugins/protector/MANIFEST.in | 1 - plugins/protector/README.md | 42 ----- plugins/protector/example.yaml | 16 -- plugins/protector/pyproject.toml | 44 ------ .../resoto_plugin_protector/__init__.py | 48 ------ .../resoto_plugin_protector/config.py | 69 -------- plugins/protector/setup.cfg | 7 - plugins/protector/test/test_config.py | 10 -- plugins/protector/tox.ini | 29 ---- plugins/tagvalidator/MANIFEST.in | 1 - plugins/tagvalidator/README.md | 53 ------- plugins/tagvalidator/example.yaml | 26 --- plugins/tagvalidator/pyproject.toml | 44 ------ .../resoto_plugin_tagvalidator/__init__.py | 99 ------------ .../resoto_plugin_tagvalidator/config.py | 77 --------- plugins/tagvalidator/setup.cfg | 7 - plugins/tagvalidator/test/test_config.py | 60 ------- plugins/tagvalidator/tox.ini | 31 ---- setup_venv.sh | 2 +- 76 files changed, 1 insertion(+), 2822 deletions(-) delete mode 100644 .github/workflows/check_pr_plugin_cleanup_aws_alarms.yml delete mode 100644 .github/workflows/check_pr_plugin_cleanup_aws_loadbalancers.yml delete mode 100644 .github/workflows/check_pr_plugin_cleanup_aws_vpcs.yml delete mode 100644 .github/workflows/check_pr_plugin_cleanup_expired.yml delete mode 100644 .github/workflows/check_pr_plugin_cleanup_untagged.yml delete mode 100644 .github/workflows/check_pr_plugin_cleanup_volumes.yml delete mode 100644 .github/workflows/check_pr_plugin_protector.yml delete mode 100644 .github/workflows/check_pr_plugin_tagvalidator.yml delete mode 100644 plugins/cleanup_aws_alarms/MANIFEST.in delete mode 100644 plugins/cleanup_aws_alarms/README.md delete mode 100644 plugins/cleanup_aws_alarms/pyproject.toml delete mode 100644 plugins/cleanup_aws_alarms/resoto_plugin_cleanup_aws_alarms/__init__.py delete mode 100644 plugins/cleanup_aws_alarms/resoto_plugin_cleanup_aws_alarms/config.py delete mode 100644 plugins/cleanup_aws_alarms/setup.cfg delete mode 100644 plugins/cleanup_aws_alarms/test/test_config.py delete mode 100644 plugins/cleanup_aws_alarms/tox.ini delete mode 100644 plugins/cleanup_aws_loadbalancers/MANIFEST.in delete mode 100644 plugins/cleanup_aws_loadbalancers/README.md delete mode 100644 plugins/cleanup_aws_loadbalancers/pyproject.toml delete mode 100644 plugins/cleanup_aws_loadbalancers/resoto_plugin_cleanup_aws_loadbalancers/__init__.py delete mode 100644 plugins/cleanup_aws_loadbalancers/resoto_plugin_cleanup_aws_loadbalancers/config.py delete mode 100644 plugins/cleanup_aws_loadbalancers/setup.cfg delete mode 100644 plugins/cleanup_aws_loadbalancers/test/test_config.py delete mode 100644 plugins/cleanup_aws_loadbalancers/tox.ini delete mode 100644 plugins/cleanup_aws_vpcs/MANIFEST.in delete mode 100644 plugins/cleanup_aws_vpcs/README.md delete mode 100644 plugins/cleanup_aws_vpcs/pyproject.toml delete mode 100644 plugins/cleanup_aws_vpcs/resoto_plugin_cleanup_aws_vpcs/__init__.py delete mode 100644 plugins/cleanup_aws_vpcs/resoto_plugin_cleanup_aws_vpcs/config.py delete mode 100644 plugins/cleanup_aws_vpcs/setup.cfg delete mode 100644 plugins/cleanup_aws_vpcs/test/test_config.py delete mode 100644 plugins/cleanup_aws_vpcs/tox.ini delete mode 100644 plugins/cleanup_expired/MANIFEST.in delete mode 100644 plugins/cleanup_expired/README.md delete mode 100644 plugins/cleanup_expired/pyproject.toml delete mode 100644 plugins/cleanup_expired/resoto_plugin_cleanup_expired/__init__.py delete mode 100644 plugins/cleanup_expired/resoto_plugin_cleanup_expired/config.py delete mode 100644 plugins/cleanup_expired/setup.cfg delete mode 100644 plugins/cleanup_expired/test/test_config.py delete mode 100644 plugins/cleanup_expired/tox.ini delete mode 100644 plugins/cleanup_untagged/MANIFEST.in delete mode 100644 plugins/cleanup_untagged/README.md delete mode 100644 plugins/cleanup_untagged/example.yaml delete mode 100644 plugins/cleanup_untagged/pyproject.toml delete mode 100644 plugins/cleanup_untagged/resoto_plugin_cleanup_untagged/__init__.py delete mode 100644 plugins/cleanup_untagged/resoto_plugin_cleanup_untagged/config.py delete mode 100644 plugins/cleanup_untagged/setup.cfg delete mode 100644 plugins/cleanup_untagged/test/test_config.py delete mode 100644 plugins/cleanup_untagged/tox.ini delete mode 100644 plugins/cleanup_volumes/MANIFEST.in delete mode 100644 plugins/cleanup_volumes/README.md delete mode 100644 plugins/cleanup_volumes/pyproject.toml delete mode 100644 plugins/cleanup_volumes/resoto_plugin_cleanup_volumes/__init__.py delete mode 100644 plugins/cleanup_volumes/resoto_plugin_cleanup_volumes/config.py delete mode 100644 plugins/cleanup_volumes/setup.cfg delete mode 100644 plugins/cleanup_volumes/test/test_config.py delete mode 100644 plugins/cleanup_volumes/tox.ini delete mode 100644 plugins/protector/MANIFEST.in delete mode 100644 plugins/protector/README.md delete mode 100644 plugins/protector/example.yaml delete mode 100644 plugins/protector/pyproject.toml delete mode 100644 plugins/protector/resoto_plugin_protector/__init__.py delete mode 100644 plugins/protector/resoto_plugin_protector/config.py delete mode 100644 plugins/protector/setup.cfg delete mode 100644 plugins/protector/test/test_config.py delete mode 100644 plugins/protector/tox.ini delete mode 100644 plugins/tagvalidator/MANIFEST.in delete mode 100644 plugins/tagvalidator/README.md delete mode 100644 plugins/tagvalidator/example.yaml delete mode 100644 plugins/tagvalidator/pyproject.toml delete mode 100644 plugins/tagvalidator/resoto_plugin_tagvalidator/__init__.py delete mode 100644 plugins/tagvalidator/resoto_plugin_tagvalidator/config.py delete mode 100644 plugins/tagvalidator/setup.cfg delete mode 100644 plugins/tagvalidator/test/test_config.py delete mode 100644 plugins/tagvalidator/tox.ini diff --git a/.github/workflows/check_pr_plugin_cleanup_aws_alarms.yml b/.github/workflows/check_pr_plugin_cleanup_aws_alarms.yml deleted file mode 100644 index 72c9bf4467..0000000000 --- a/.github/workflows/check_pr_plugin_cleanup_aws_alarms.yml +++ /dev/null @@ -1,73 +0,0 @@ -# Note: this workflow is automatically generated via the `create_pr` script in the same folder. -# Please do not change the file, but the script! - -name: Check PR (Plugin cleanup_aws_alarms) -on: - push: - tags: - - "*.*.*" - branches: - - main - pull_request: - paths: - - 'resotolib/**' - - 'plugins/cleanup_aws_alarms/**' - - '.github/**' - - 'requirements-all.txt' - -jobs: - cleanup_aws_alarms: - name: "cleanup_aws_alarms" - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v2 - - - name: Setup Python - uses: actions/setup-python@v2 - with: - python-version: '3.10' - architecture: 'x64' - - - name: Restore dependency cache - uses: actions/cache@v3 - with: - path: ~/.cache/pip - key: ${{runner.os}}-pip-${{hashFiles('./plugins/cleanup_aws_alarms/pyproject.toml')}} - restore-keys: | - ${{runner.os}}-pip- - - - name: Install Dependencies - run: | - python -m pip install --upgrade pip - pip install --upgrade --editable resotolib/ - pip install tox wheel flake8 build - - pip install --upgrade --editable plugins/aws/ - - - name: Run tests - working-directory: ./plugins/cleanup_aws_alarms - run: tox - - - name: Archive code coverage results - uses: actions/upload-artifact@v2 - with: - name: plugin-cleanup_aws_alarms-code-coverage-report - path: ./plugins/cleanup_aws_alarms/htmlcov/ - - - name: Build a binary wheel and a source tarball - working-directory: ./plugins/cleanup_aws_alarms - run: >- - python -m - build - --sdist - --wheel - --outdir dist/ - - - name: Publish distribution to PyPI - if: github.ref_type == 'tag' - uses: pypa/gh-action-pypi-publish@release/v1 - with: - user: __token__ - password: ${{ secrets.PYPI_RESOTO_PLUGIN_CLEANUP_AWS_ALARMS }} - packages_dir: ./plugins/cleanup_aws_alarms/dist/ diff --git a/.github/workflows/check_pr_plugin_cleanup_aws_loadbalancers.yml b/.github/workflows/check_pr_plugin_cleanup_aws_loadbalancers.yml deleted file mode 100644 index e2aaeac6c1..0000000000 --- a/.github/workflows/check_pr_plugin_cleanup_aws_loadbalancers.yml +++ /dev/null @@ -1,73 +0,0 @@ -# Note: this workflow is automatically generated via the `create_pr` script in the same folder. -# Please do not change the file, but the script! - -name: Check PR (Plugin cleanup_aws_loadbalancers) -on: - push: - tags: - - "*.*.*" - branches: - - main - pull_request: - paths: - - 'resotolib/**' - - 'plugins/cleanup_aws_loadbalancers/**' - - '.github/**' - - 'requirements-all.txt' - -jobs: - cleanup_aws_loadbalancers: - name: "cleanup_aws_loadbalancers" - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v2 - - - name: Setup Python - uses: actions/setup-python@v2 - with: - python-version: '3.10' - architecture: 'x64' - - - name: Restore dependency cache - uses: actions/cache@v3 - with: - path: ~/.cache/pip - key: ${{runner.os}}-pip-${{hashFiles('./plugins/cleanup_aws_loadbalancers/pyproject.toml')}} - restore-keys: | - ${{runner.os}}-pip- - - - name: Install Dependencies - run: | - python -m pip install --upgrade pip - pip install --upgrade --editable resotolib/ - pip install tox wheel flake8 build - - pip install --upgrade --editable plugins/aws/ - - - name: Run tests - working-directory: ./plugins/cleanup_aws_loadbalancers - run: tox - - - name: Archive code coverage results - uses: actions/upload-artifact@v2 - with: - name: plugin-cleanup_aws_loadbalancers-code-coverage-report - path: ./plugins/cleanup_aws_loadbalancers/htmlcov/ - - - name: Build a binary wheel and a source tarball - working-directory: ./plugins/cleanup_aws_loadbalancers - run: >- - python -m - build - --sdist - --wheel - --outdir dist/ - - - name: Publish distribution to PyPI - if: github.ref_type == 'tag' - uses: pypa/gh-action-pypi-publish@release/v1 - with: - user: __token__ - password: ${{ secrets.PYPI_RESOTO_PLUGIN_CLEANUP_AWS_LOADBALANCERS }} - packages_dir: ./plugins/cleanup_aws_loadbalancers/dist/ diff --git a/.github/workflows/check_pr_plugin_cleanup_aws_vpcs.yml b/.github/workflows/check_pr_plugin_cleanup_aws_vpcs.yml deleted file mode 100644 index a0723feea9..0000000000 --- a/.github/workflows/check_pr_plugin_cleanup_aws_vpcs.yml +++ /dev/null @@ -1,73 +0,0 @@ -# Note: this workflow is automatically generated via the `create_pr` script in the same folder. -# Please do not change the file, but the script! - -name: Check PR (Plugin cleanup_aws_vpcs) -on: - push: - tags: - - "*.*.*" - branches: - - main - pull_request: - paths: - - 'resotolib/**' - - 'plugins/cleanup_aws_vpcs/**' - - '.github/**' - - 'requirements-all.txt' - -jobs: - cleanup_aws_vpcs: - name: "cleanup_aws_vpcs" - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v2 - - - name: Setup Python - uses: actions/setup-python@v2 - with: - python-version: '3.10' - architecture: 'x64' - - - name: Restore dependency cache - uses: actions/cache@v3 - with: - path: ~/.cache/pip - key: ${{runner.os}}-pip-${{hashFiles('./plugins/cleanup_aws_vpcs/pyproject.toml')}} - restore-keys: | - ${{runner.os}}-pip- - - - name: Install Dependencies - run: | - python -m pip install --upgrade pip - pip install --upgrade --editable resotolib/ - pip install tox wheel flake8 build - - pip install --upgrade --editable plugins/aws/ - - - name: Run tests - working-directory: ./plugins/cleanup_aws_vpcs - run: tox - - - name: Archive code coverage results - uses: actions/upload-artifact@v2 - with: - name: plugin-cleanup_aws_vpcs-code-coverage-report - path: ./plugins/cleanup_aws_vpcs/htmlcov/ - - - name: Build a binary wheel and a source tarball - working-directory: ./plugins/cleanup_aws_vpcs - run: >- - python -m - build - --sdist - --wheel - --outdir dist/ - - - name: Publish distribution to PyPI - if: github.ref_type == 'tag' - uses: pypa/gh-action-pypi-publish@release/v1 - with: - user: __token__ - password: ${{ secrets.PYPI_RESOTO_PLUGIN_CLEANUP_AWS_VPCS }} - packages_dir: ./plugins/cleanup_aws_vpcs/dist/ diff --git a/.github/workflows/check_pr_plugin_cleanup_expired.yml b/.github/workflows/check_pr_plugin_cleanup_expired.yml deleted file mode 100644 index 655a865060..0000000000 --- a/.github/workflows/check_pr_plugin_cleanup_expired.yml +++ /dev/null @@ -1,71 +0,0 @@ -# Note: this workflow is automatically generated via the `create_pr` script in the same folder. -# Please do not change the file, but the script! - -name: Check PR (Plugin cleanup_expired) -on: - push: - tags: - - "*.*.*" - branches: - - main - pull_request: - paths: - - 'resotolib/**' - - 'plugins/cleanup_expired/**' - - '.github/**' - - 'requirements-all.txt' - -jobs: - cleanup_expired: - name: "cleanup_expired" - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v2 - - - name: Setup Python - uses: actions/setup-python@v2 - with: - python-version: '3.10' - architecture: 'x64' - - - name: Restore dependency cache - uses: actions/cache@v3 - with: - path: ~/.cache/pip - key: ${{runner.os}}-pip-${{hashFiles('./plugins/cleanup_expired/pyproject.toml')}} - restore-keys: | - ${{runner.os}}-pip- - - - name: Install Dependencies - run: | - python -m pip install --upgrade pip - pip install --upgrade --editable resotolib/ - pip install tox wheel flake8 build - - - name: Run tests - working-directory: ./plugins/cleanup_expired - run: tox - - - name: Archive code coverage results - uses: actions/upload-artifact@v2 - with: - name: plugin-cleanup_expired-code-coverage-report - path: ./plugins/cleanup_expired/htmlcov/ - - - name: Build a binary wheel and a source tarball - working-directory: ./plugins/cleanup_expired - run: >- - python -m - build - --sdist - --wheel - --outdir dist/ - - - name: Publish distribution to PyPI - if: github.ref_type == 'tag' - uses: pypa/gh-action-pypi-publish@release/v1 - with: - user: __token__ - password: ${{ secrets.PYPI_RESOTO_PLUGIN_CLEANUP_EXPIRED }} - packages_dir: ./plugins/cleanup_expired/dist/ diff --git a/.github/workflows/check_pr_plugin_cleanup_untagged.yml b/.github/workflows/check_pr_plugin_cleanup_untagged.yml deleted file mode 100644 index 4acdf0b9f6..0000000000 --- a/.github/workflows/check_pr_plugin_cleanup_untagged.yml +++ /dev/null @@ -1,71 +0,0 @@ -# Note: this workflow is automatically generated via the `create_pr` script in the same folder. -# Please do not change the file, but the script! - -name: Check PR (Plugin cleanup_untagged) -on: - push: - tags: - - "*.*.*" - branches: - - main - pull_request: - paths: - - 'resotolib/**' - - 'plugins/cleanup_untagged/**' - - '.github/**' - - 'requirements-all.txt' - -jobs: - cleanup_untagged: - name: "cleanup_untagged" - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v2 - - - name: Setup Python - uses: actions/setup-python@v2 - with: - python-version: '3.10' - architecture: 'x64' - - - name: Restore dependency cache - uses: actions/cache@v3 - with: - path: ~/.cache/pip - key: ${{runner.os}}-pip-${{hashFiles('./plugins/cleanup_untagged/pyproject.toml')}} - restore-keys: | - ${{runner.os}}-pip- - - - name: Install Dependencies - run: | - python -m pip install --upgrade pip - pip install --upgrade --editable resotolib/ - pip install tox wheel flake8 build - - - name: Run tests - working-directory: ./plugins/cleanup_untagged - run: tox - - - name: Archive code coverage results - uses: actions/upload-artifact@v2 - with: - name: plugin-cleanup_untagged-code-coverage-report - path: ./plugins/cleanup_untagged/htmlcov/ - - - name: Build a binary wheel and a source tarball - working-directory: ./plugins/cleanup_untagged - run: >- - python -m - build - --sdist - --wheel - --outdir dist/ - - - name: Publish distribution to PyPI - if: github.ref_type == 'tag' - uses: pypa/gh-action-pypi-publish@release/v1 - with: - user: __token__ - password: ${{ secrets.PYPI_RESOTO_PLUGIN_CLEANUP_UNTAGGED }} - packages_dir: ./plugins/cleanup_untagged/dist/ diff --git a/.github/workflows/check_pr_plugin_cleanup_volumes.yml b/.github/workflows/check_pr_plugin_cleanup_volumes.yml deleted file mode 100644 index 22ffdc5e2d..0000000000 --- a/.github/workflows/check_pr_plugin_cleanup_volumes.yml +++ /dev/null @@ -1,71 +0,0 @@ -# Note: this workflow is automatically generated via the `create_pr` script in the same folder. -# Please do not change the file, but the script! - -name: Check PR (Plugin cleanup_volumes) -on: - push: - tags: - - "*.*.*" - branches: - - main - pull_request: - paths: - - 'resotolib/**' - - 'plugins/cleanup_volumes/**' - - '.github/**' - - 'requirements-all.txt' - -jobs: - cleanup_volumes: - name: "cleanup_volumes" - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v2 - - - name: Setup Python - uses: actions/setup-python@v2 - with: - python-version: '3.10' - architecture: 'x64' - - - name: Restore dependency cache - uses: actions/cache@v3 - with: - path: ~/.cache/pip - key: ${{runner.os}}-pip-${{hashFiles('./plugins/cleanup_volumes/pyproject.toml')}} - restore-keys: | - ${{runner.os}}-pip- - - - name: Install Dependencies - run: | - python -m pip install --upgrade pip - pip install --upgrade --editable resotolib/ - pip install tox wheel flake8 build - - - name: Run tests - working-directory: ./plugins/cleanup_volumes - run: tox - - - name: Archive code coverage results - uses: actions/upload-artifact@v2 - with: - name: plugin-cleanup_volumes-code-coverage-report - path: ./plugins/cleanup_volumes/htmlcov/ - - - name: Build a binary wheel and a source tarball - working-directory: ./plugins/cleanup_volumes - run: >- - python -m - build - --sdist - --wheel - --outdir dist/ - - - name: Publish distribution to PyPI - if: github.ref_type == 'tag' - uses: pypa/gh-action-pypi-publish@release/v1 - with: - user: __token__ - password: ${{ secrets.PYPI_RESOTO_PLUGIN_CLEANUP_VOLUMES }} - packages_dir: ./plugins/cleanup_volumes/dist/ diff --git a/.github/workflows/check_pr_plugin_protector.yml b/.github/workflows/check_pr_plugin_protector.yml deleted file mode 100644 index 08be9376bf..0000000000 --- a/.github/workflows/check_pr_plugin_protector.yml +++ /dev/null @@ -1,71 +0,0 @@ -# Note: this workflow is automatically generated via the `create_pr` script in the same folder. -# Please do not change the file, but the script! - -name: Check PR (Plugin protector) -on: - push: - tags: - - "*.*.*" - branches: - - main - pull_request: - paths: - - 'resotolib/**' - - 'plugins/protector/**' - - '.github/**' - - 'requirements-all.txt' - -jobs: - protector: - name: "protector" - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v2 - - - name: Setup Python - uses: actions/setup-python@v2 - with: - python-version: '3.10' - architecture: 'x64' - - - name: Restore dependency cache - uses: actions/cache@v3 - with: - path: ~/.cache/pip - key: ${{runner.os}}-pip-${{hashFiles('./plugins/protector/pyproject.toml')}} - restore-keys: | - ${{runner.os}}-pip- - - - name: Install Dependencies - run: | - python -m pip install --upgrade pip - pip install --upgrade --editable resotolib/ - pip install tox wheel flake8 build - - - name: Run tests - working-directory: ./plugins/protector - run: tox - - - name: Archive code coverage results - uses: actions/upload-artifact@v2 - with: - name: plugin-protector-code-coverage-report - path: ./plugins/protector/htmlcov/ - - - name: Build a binary wheel and a source tarball - working-directory: ./plugins/protector - run: >- - python -m - build - --sdist - --wheel - --outdir dist/ - - - name: Publish distribution to PyPI - if: github.ref_type == 'tag' - uses: pypa/gh-action-pypi-publish@release/v1 - with: - user: __token__ - password: ${{ secrets.PYPI_RESOTO_PLUGIN_PROTECTOR }} - packages_dir: ./plugins/protector/dist/ diff --git a/.github/workflows/check_pr_plugin_tagvalidator.yml b/.github/workflows/check_pr_plugin_tagvalidator.yml deleted file mode 100644 index 8bfc28662f..0000000000 --- a/.github/workflows/check_pr_plugin_tagvalidator.yml +++ /dev/null @@ -1,71 +0,0 @@ -# Note: this workflow is automatically generated via the `create_pr` script in the same folder. -# Please do not change the file, but the script! - -name: Check PR (Plugin tagvalidator) -on: - push: - tags: - - "*.*.*" - branches: - - main - pull_request: - paths: - - 'resotolib/**' - - 'plugins/tagvalidator/**' - - '.github/**' - - 'requirements-all.txt' - -jobs: - tagvalidator: - name: "tagvalidator" - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v2 - - - name: Setup Python - uses: actions/setup-python@v2 - with: - python-version: '3.10' - architecture: 'x64' - - - name: Restore dependency cache - uses: actions/cache@v3 - with: - path: ~/.cache/pip - key: ${{runner.os}}-pip-${{hashFiles('./plugins/tagvalidator/pyproject.toml')}} - restore-keys: | - ${{runner.os}}-pip- - - - name: Install Dependencies - run: | - python -m pip install --upgrade pip - pip install --upgrade --editable resotolib/ - pip install tox wheel flake8 build - - - name: Run tests - working-directory: ./plugins/tagvalidator - run: tox - - - name: Archive code coverage results - uses: actions/upload-artifact@v2 - with: - name: plugin-tagvalidator-code-coverage-report - path: ./plugins/tagvalidator/htmlcov/ - - - name: Build a binary wheel and a source tarball - working-directory: ./plugins/tagvalidator - run: >- - python -m - build - --sdist - --wheel - --outdir dist/ - - - name: Publish distribution to PyPI - if: github.ref_type == 'tag' - uses: pypa/gh-action-pypi-publish@release/v1 - with: - user: __token__ - password: ${{ secrets.PYPI_RESOTO_PLUGIN_TAGVALIDATOR }} - packages_dir: ./plugins/tagvalidator/dist/ diff --git a/plugins/cleanup_aws_alarms/MANIFEST.in b/plugins/cleanup_aws_alarms/MANIFEST.in deleted file mode 100644 index bb3ec5f0d4..0000000000 --- a/plugins/cleanup_aws_alarms/MANIFEST.in +++ /dev/null @@ -1 +0,0 @@ -include README.md diff --git a/plugins/cleanup_aws_alarms/README.md b/plugins/cleanup_aws_alarms/README.md deleted file mode 100644 index 7078381e75..0000000000 --- a/plugins/cleanup_aws_alarms/README.md +++ /dev/null @@ -1,29 +0,0 @@ -# resoto-plugin-cleanup-aws-alarms -AWS Cloudwatch Alarms Cleanup Plugin for Resoto - -This plugin marks all orphaned AWS CloudWatch Instance Alarms for cleanup. - -The following resources are currently being marked for cleanup - -- Instance Alarms - -## Usage - -In `resh` execute - -``` -> config edit resoto.worker -``` - -and find the following section - -``` -plugin_cleanup_aws_alarms: - # Dictionary of key cloud with list of account IDs for which the plugin should be active as value - config: - aws: - - '1234567' - - '567890' - # Enable plugin? - enabled: false -``` diff --git a/plugins/cleanup_aws_alarms/pyproject.toml b/plugins/cleanup_aws_alarms/pyproject.toml deleted file mode 100644 index ea23480503..0000000000 --- a/plugins/cleanup_aws_alarms/pyproject.toml +++ /dev/null @@ -1,42 +0,0 @@ -[project] -name = "resoto-plugin-cleanup-aws-alarms" -version = "3.7.0" -authors = [{name="Some Engineering Inc."}] -description = "AWS Cloudwatch Alarms Cleaner Plugin" -license = {file="LICENSE"} -requires-python = ">=3.9" -classifiers = [ - # Current project status - "Development Status :: 4 - Beta", - # Audience - "Intended Audience :: System Administrators", - "Intended Audience :: Information Technology", - # License information - "License :: OSI Approved :: Apache Software License", - # Supported python versions - "Programming Language :: Python :: 3.9", - # Supported OS's - "Operating System :: POSIX :: Linux", - "Operating System :: Unix", - # Extra metadata - "Environment :: Console", - "Natural Language :: English", - "Topic :: Security", - "Topic :: Utilities", -] -readme = {file="README.md", content-type="text/markdown"} - -dependencies = [ "resotolib==3.7.0" ] - -[project.entry-points."resoto.plugins"] -cleanup_aws_alarms = "resoto_plugin_cleanup_aws_alarms:CleanupAWSAlarmsPlugin" - -[project.urls] -Documentation = "https://resoto.com" -Source = "https://github.com/someengineering/resoto/tree/main/plugins/cleanup_aws_alarms" - -[build-system] -requires = ["setuptools>=67.8.0", "wheel>=0.40.0", "build>=0.10.0"] -build-backend = "setuptools.build_meta" - - diff --git a/plugins/cleanup_aws_alarms/resoto_plugin_cleanup_aws_alarms/__init__.py b/plugins/cleanup_aws_alarms/resoto_plugin_cleanup_aws_alarms/__init__.py deleted file mode 100644 index 9255840431..0000000000 --- a/plugins/cleanup_aws_alarms/resoto_plugin_cleanup_aws_alarms/__init__.py +++ /dev/null @@ -1,70 +0,0 @@ -from resotolib.baseplugin import BaseActionPlugin -from resotolib.core.search import CoreGraph -from resotolib.graph import Graph -from resoto_plugin_aws.resource.cloudwatch import AwsCloudwatchAlarm -from resoto_plugin_aws.resource.ec2 import AwsEc2Instance -from resotolib.logger import log -from resotolib.config import Config -from .config import CleanupAWSAlarmsConfig -from typing import Dict - - -class CleanupAWSAlarmsPlugin(BaseActionPlugin): - action = "cleanup_plan" - - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - - self.config = {} - if Config.plugin_cleanup_aws_alarms.enabled: - self.config = Config.plugin_cleanup_aws_alarms.config - - def bootstrap(self) -> bool: - return Config.plugin_cleanup_aws_alarms.enabled - - def do_action(self, data: Dict) -> None: - cg = CoreGraph(tls_data=self.tls_data) - - query = "is(aws_cloudwatch_alarm) <-default,delete[0:]delete->" - graph = cg.graph(query) - self.alarm_cleanup(graph) - cg.patch_nodes(graph) - - def alarm_cleanup(self, graph: Graph): - log.info("AWS Cloudwatch Alarms cleanup called") - for node in graph.nodes: - if node.protected or not isinstance(node, AwsCloudwatchAlarm): - continue - - cloud = node.cloud(graph) - account = node.account(graph) - region = node.region(graph) - log_prefix = f"Found {node.rtdname} in cloud {cloud.name} account {account.dname} " f"region {region.name}." - - if len(self.config) > 0: - if cloud.id not in self.config or account.id not in self.config[cloud.id]: - log.debug((f"{log_prefix} Account not found in config - ignoring.")) - continue - - should_clean = False - i = None - log_msg = log_prefix - for dimension in node.dimensions: - if dimension.get("Name") == "InstanceId": - instance_id = dimension.get("Value") - i = graph.search_first_all({"kind": "aws_ec2_instance", "id": instance_id}) - if isinstance(i, AwsEc2Instance) and i.instance_status not in ("terminated"): - should_clean = False - break - else: - should_clean = True - log_msg += f" Referenced EC2 instance {instance_id} not found." - - if not should_clean: - continue - log.debug(f"{log_msg} - cleaning alarm") - node.clean = True - - @staticmethod - def add_config(config: Config) -> None: - config.add_config(CleanupAWSAlarmsConfig) diff --git a/plugins/cleanup_aws_alarms/resoto_plugin_cleanup_aws_alarms/config.py b/plugins/cleanup_aws_alarms/resoto_plugin_cleanup_aws_alarms/config.py deleted file mode 100644 index 1eda1dced1..0000000000 --- a/plugins/cleanup_aws_alarms/resoto_plugin_cleanup_aws_alarms/config.py +++ /dev/null @@ -1,35 +0,0 @@ -from attrs import define, field -from typing import ClassVar, Dict, List - - -@define -class CleanupAWSAlarmsConfig: - kind: ClassVar[str] = "plugin_cleanup_aws_alarms" - enabled: bool = field( - default=False, - metadata={"description": "Enable plugin?", "restart_required": True}, - ) - config: Dict[str, List[str]] = field( - factory=lambda: {"aws": ["1234567", "567890"]}, - metadata={ - "description": ( - "Dictionary of key cloud with list of account IDs" " for which the plugin should be active as value" - ) - }, - ) - - @staticmethod - def validate(config: Dict) -> bool: - if not isinstance(config, dict): - raise ValueError("Config is no dict") - - for cloud_id, account_ids in config.items(): - if not isinstance(cloud_id, str): - raise ValueError(f"Cloud ID {cloud_id} is no string") - if not isinstance(account_ids, list): - raise ValueError(f"Account IDs {account_ids} is no list") - - for account_id in account_ids: - if not isinstance(account_id, str): - raise ValueError(f"Account ID {account_id} is no string") - return True diff --git a/plugins/cleanup_aws_alarms/setup.cfg b/plugins/cleanup_aws_alarms/setup.cfg deleted file mode 100644 index 7ca6537935..0000000000 --- a/plugins/cleanup_aws_alarms/setup.cfg +++ /dev/null @@ -1,7 +0,0 @@ -[options] -packages = find: -include_package_data = True -zip_safe = False - -[aliases] -test=pytest diff --git a/plugins/cleanup_aws_alarms/test/test_config.py b/plugins/cleanup_aws_alarms/test/test_config.py deleted file mode 100644 index 67057883a7..0000000000 --- a/plugins/cleanup_aws_alarms/test/test_config.py +++ /dev/null @@ -1,10 +0,0 @@ -from resotolib.config import Config -from resoto_plugin_cleanup_aws_alarms import CleanupAWSAlarmsPlugin - - -def test_config(): - config = Config("dummy", "dummy") - CleanupAWSAlarmsPlugin.add_config(config) - Config.init_default_config() - assert Config.plugin_cleanup_aws_alarms.enabled is False - assert Config.plugin_cleanup_aws_alarms.validate(Config.plugin_cleanup_aws_alarms.config) is True diff --git a/plugins/cleanup_aws_alarms/tox.ini b/plugins/cleanup_aws_alarms/tox.ini deleted file mode 100644 index 907b45766b..0000000000 --- a/plugins/cleanup_aws_alarms/tox.ini +++ /dev/null @@ -1,30 +0,0 @@ -[tox] -env_list = syntax, tests, black - -[flake8] -max-line-length=120 -exclude = .git,.tox,__pycache__,.idea,.pytest_cache -ignore=F403, F405, E722, N806, N813, E266, W503, E203 - -[pytest] -addopts= --cov=resoto_plugin_cleanup_aws_alarms -rs -vv --cov-report html -testpaths= test - -[testenv] -usedevelop = true -deps = - --editable=file:///{toxinidir}/../../resotolib - --editable=file:///{toxinidir}/../aws - -r../../requirements-all.txt -# until this is fixed: https://github.com/pypa/setuptools/issues/3518 -setenv = - SETUPTOOLS_ENABLE_FEATURES = legacy-editable - -[testenv:syntax] -commands = flake8 --verbose - -[testenv:tests] -commands= pytest - -[testenv:black] -commands = black --line-length 120 --check --diff --target-version py39 . diff --git a/plugins/cleanup_aws_loadbalancers/MANIFEST.in b/plugins/cleanup_aws_loadbalancers/MANIFEST.in deleted file mode 100644 index bb3ec5f0d4..0000000000 --- a/plugins/cleanup_aws_loadbalancers/MANIFEST.in +++ /dev/null @@ -1 +0,0 @@ -include README.md diff --git a/plugins/cleanup_aws_loadbalancers/README.md b/plugins/cleanup_aws_loadbalancers/README.md deleted file mode 100644 index 0d0fee4602..0000000000 --- a/plugins/cleanup_aws_loadbalancers/README.md +++ /dev/null @@ -1,29 +0,0 @@ -# resoto-plugin-cleanup-aws-loadbalancers -AWS Loadbalancers Cleanup Plugin for Resoto - -This plugin cleans up AWS ALB/ELB load balancers with no instances attached to them. - -## Usage - -``` -plugin_cleanup_aws_loadbalancers: - # Enable plugin? - enabled: false - # Minimum age of unused load balancers to cleanup - min_age: '7 days' -``` - -The default load balancer age is 7 days. Meaning if a load balancer is more than 7 days old and does not have any instances/backends attached it will be flagged for cleanup. - -Optionally change the age cutoff value using the `min_age` option. - -Example of valid age units: - -``` -weeks -days -hours -minutes -``` - -Each of them can be abbreviated down to one letter. E.g. `7d`, `24h`, `60m`, etc. A space in between the numeric and the unit is optional, meaning `7d` and `7 days` are equivalent. diff --git a/plugins/cleanup_aws_loadbalancers/pyproject.toml b/plugins/cleanup_aws_loadbalancers/pyproject.toml deleted file mode 100644 index 5712ddea45..0000000000 --- a/plugins/cleanup_aws_loadbalancers/pyproject.toml +++ /dev/null @@ -1,45 +0,0 @@ -[project] -name = "resoto-plugin-cleanup-aws-loadbalancers" -version = "3.7.0" -authors = [{name="Some Engineering Inc."}] -description = "AWS Loadbalancers Cleaner Plugin" -license = {file="LICENSE"} -requires-python = ">=3.9" -classifiers = [ - # Current project status - "Development Status :: 4 - Beta", - # Audience - "Intended Audience :: System Administrators", - "Intended Audience :: Information Technology", - # License information - "License :: OSI Approved :: Apache Software License", - # Supported python versions - "Programming Language :: Python :: 3.9", - # Supported OS's - "Operating System :: POSIX :: Linux", - "Operating System :: Unix", - # Extra metadata - "Environment :: Console", - "Natural Language :: English", - "Topic :: Security", - "Topic :: Utilities", -] -readme = {file="README.md", content-type="text/markdown"} - -dependencies = [ - "resotolib==3.7.0", - "resoto-plugin-aws==3.7.0" -] - -[project.entry-points."resoto.plugins"] -cleanup_aws_loadbalancers = "resoto_plugin_cleanup_aws_loadbalancers:CleanupAWSLoadbalancersPlugin" - -[project.urls] -Documentation = "https://resoto.com" -Source = "https://github.com/someengineering/resoto/tree/main/plugins/cleanup_aws_loadbalancers" - -[build-system] -requires = ["setuptools>=67.8.0", "wheel>=0.40.0", "build>=0.10.0"] -build-backend = "setuptools.build_meta" - - diff --git a/plugins/cleanup_aws_loadbalancers/resoto_plugin_cleanup_aws_loadbalancers/__init__.py b/plugins/cleanup_aws_loadbalancers/resoto_plugin_cleanup_aws_loadbalancers/__init__.py deleted file mode 100644 index fa9e8b45d7..0000000000 --- a/plugins/cleanup_aws_loadbalancers/resoto_plugin_cleanup_aws_loadbalancers/__init__.py +++ /dev/null @@ -1,149 +0,0 @@ -from resotolib.baseplugin import BaseActionPlugin -from resotolib.baseresources import EdgeType -from resotolib.logger import log -from resotolib.core.search import CoreGraph -from resotolib.graph import Graph -from resoto_plugin_aws.resource.elb import AwsElb -from resoto_plugin_aws.resource.elbv2 import AwsAlb, AwsAlbTargetGroup -from resoto_plugin_aws.resource.ec2 import AwsEc2Instance -from resotolib.config import Config -from .config import CleanupAWSLoadbalancersConfig -from resotolib.durations import parse_duration -from typing import Dict - - -class CleanupAWSLoadbalancersPlugin(BaseActionPlugin): - action = "cleanup_plan" - - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self.age = None - if Config.plugin_cleanup_aws_loadbalancers.enabled: - self.update_age() - - def bootstrap(self) -> bool: - return Config.plugin_cleanup_aws_loadbalancers.enabled - - def do_action(self, data: Dict) -> None: - self.update_age() - cg = CoreGraph(tls_data=self.tls_data) - query = 'is(["aws_elb", "aws_alb", "aws_alb_target_group"]) <-default,delete[0:]delete->' - graph = cg.graph(query) - self.loadbalancer_cleanup(graph) - cg.patch_nodes(graph) - - def update_age(self) -> None: - try: - self.age = parse_duration(Config.plugin_cleanup_aws_loadbalancers.min_age) - log.debug(f"Cleanup AWS Load balancers minimum age is {self.age}") - except ValueError: - log.error( - "Error while parsing Cleanup AWS Load balancers minimum age" - f" {Config.plugin_cleanup_aws_loadbalancers.min_age}" - ) - raise - - def loadbalancer_cleanup(self, graph: Graph): - log.info("AWS Loadbalancers Cleanup called") - for node in graph.nodes: - if ( - not isinstance(node, AwsElb) - and not isinstance(node, AwsAlb) - and not isinstance(node, AwsAlbTargetGroup) - ): - continue - - if node.age < self.age: - continue - - if node.tags.get("expiration") == "never": - continue - - cloud = node.cloud(graph) - account = node.account(graph) - region = node.region(graph) - - if ( - isinstance(node, AwsElb) - and len( - [ - i - for i in node.predecessors(graph, edge_type=EdgeType.delete) - if isinstance(i, AwsEc2Instance) and i.instance_status != "terminated" - ] - ) - == 0 - and len(node.backends) == 0 - ): - log.debug( - ( - f"Found orphaned AWS ELB {node.dname} in cloud {cloud.name} account {account.dname} " - f"region {region.name} with age {node.age} and no EC2 instances attached to it." - ) - ) - node.clean = True - elif ( - isinstance(node, AwsAlb) - and len( - [n for n in node.predecessors(graph, edge_type=EdgeType.delete) if isinstance(n, AwsAlbTargetGroup)] - ) - == 0 - and len(node.backends) == 0 - ): - log.debug( - ( - f"Found orphaned AWS ALB {node.dname} in cloud {cloud.name} account {account.dname} " - f"region {region.name} with age {node.age} and no Target Groups attached to it." - ) - ) - node.clean = True - elif ( - isinstance(node, AwsAlbTargetGroup) - and len(list(node.successors(graph, edge_type=EdgeType.delete))) == 0 - ): - log.debug( - ( - f"Found orphaned AWS ALB Target Group {node.dname} in cloud {cloud.name} " - f"account {account.dname} region {region.name} with age {node.age}" - ) - ) - node.clean = True - elif isinstance(node, AwsAlb): - cleanup_alb = True - target_groups = [ - n for n in node.predecessors(graph, edge_type=EdgeType.delete) if isinstance(n, AwsAlbTargetGroup) - ] - - if len(node.backends) > 0: - cleanup_alb = False - - for tg in target_groups: - if ( - tg.target_type != "instance" - or tg.age < self.age - or len( - [ - i - for i in tg.predecessors(graph, edge_type=EdgeType.delete) - if isinstance(i, AwsEc2Instance) and i.instance_status != "terminated" - ] - ) - > 0 - ): - cleanup_alb = False - - if cleanup_alb: - log.debug( - ( - f"Found AWS ALB {node.dname} in cloud {cloud.name} account {account.dname} " - f"region {region.name} with age {node.age} and no EC2 instances attached " - f"to its {len(target_groups)} target groups." - ) - ) - for tg in target_groups: - tg.clean = True - node.clean = True - - @staticmethod - def add_config(config: Config) -> None: - config.add_config(CleanupAWSLoadbalancersConfig) diff --git a/plugins/cleanup_aws_loadbalancers/resoto_plugin_cleanup_aws_loadbalancers/config.py b/plugins/cleanup_aws_loadbalancers/resoto_plugin_cleanup_aws_loadbalancers/config.py deleted file mode 100644 index c42f83fb23..0000000000 --- a/plugins/cleanup_aws_loadbalancers/resoto_plugin_cleanup_aws_loadbalancers/config.py +++ /dev/null @@ -1,24 +0,0 @@ -from attrs import define, field -from typing import ClassVar -from resotolib.durations import parse_duration - - -@define -class CleanupAWSLoadbalancersConfig: - kind: ClassVar[str] = "plugin_cleanup_aws_loadbalancers" - enabled: bool = field( - default=False, - metadata={"description": "Enable plugin?", "restart_required": True}, - ) - min_age: str = field( - default="7 days", - metadata={ - "description": "Minimum age of unused load balancers to cleanup", - "type_hint": "duration", - }, - ) - - @staticmethod - def validate(config: "CleanupAWSLoadbalancersConfig") -> bool: - parse_duration(config.min_age) - return True diff --git a/plugins/cleanup_aws_loadbalancers/setup.cfg b/plugins/cleanup_aws_loadbalancers/setup.cfg deleted file mode 100644 index 7ca6537935..0000000000 --- a/plugins/cleanup_aws_loadbalancers/setup.cfg +++ /dev/null @@ -1,7 +0,0 @@ -[options] -packages = find: -include_package_data = True -zip_safe = False - -[aliases] -test=pytest diff --git a/plugins/cleanup_aws_loadbalancers/test/test_config.py b/plugins/cleanup_aws_loadbalancers/test/test_config.py deleted file mode 100644 index e3ac4a25c3..0000000000 --- a/plugins/cleanup_aws_loadbalancers/test/test_config.py +++ /dev/null @@ -1,10 +0,0 @@ -from resotolib.config import Config -from resoto_plugin_cleanup_aws_loadbalancers import CleanupAWSLoadbalancersPlugin - - -def test_config(): - config = Config("dummy", "dummy") - CleanupAWSLoadbalancersPlugin.add_config(config) - Config.init_default_config() - assert Config.plugin_cleanup_aws_loadbalancers.enabled is False - assert Config.plugin_cleanup_aws_loadbalancers.min_age == "7 days" diff --git a/plugins/cleanup_aws_loadbalancers/tox.ini b/plugins/cleanup_aws_loadbalancers/tox.ini deleted file mode 100644 index eff9305300..0000000000 --- a/plugins/cleanup_aws_loadbalancers/tox.ini +++ /dev/null @@ -1,30 +0,0 @@ -[tox] -env_list = syntax, tests, black - -[flake8] -max-line-length=120 -exclude = .git,.tox,__pycache__,.idea,.pytest_cache -ignore=F403, F405, E722, N806, N813, E266, W503, E203 - -[pytest] -addopts= --cov=resoto_plugin_cleanup_aws_loadbalancers -rs -vv --cov-report html -testpaths= test - -[testenv] -usedevelop = true -deps = - --editable=file:///{toxinidir}/../../resotolib - --editable=file:///{toxinidir}/../aws - -r../../requirements-all.txt -# until this is fixed: https://github.com/pypa/setuptools/issues/3518 -setenv = - SETUPTOOLS_ENABLE_FEATURES = legacy-editable - -[testenv:syntax] -commands = flake8 --verbose - -[testenv:tests] -commands= pytest - -[testenv:black] -commands = black --line-length 120 --check --diff --target-version py39 . diff --git a/plugins/cleanup_aws_vpcs/MANIFEST.in b/plugins/cleanup_aws_vpcs/MANIFEST.in deleted file mode 100644 index bb3ec5f0d4..0000000000 --- a/plugins/cleanup_aws_vpcs/MANIFEST.in +++ /dev/null @@ -1 +0,0 @@ -include README.md diff --git a/plugins/cleanup_aws_vpcs/README.md b/plugins/cleanup_aws_vpcs/README.md deleted file mode 100644 index e9e058c3b9..0000000000 --- a/plugins/cleanup_aws_vpcs/README.md +++ /dev/null @@ -1,39 +0,0 @@ -# resoto-plugin-cleanup-aws-vpcs -AWS VPC Cleanup Plugin for Resoto - -This plugin marks all VPC dependencies for cleanup. The VPC must have been previously marked for cleanup by another cleanup plugin. - -The following resources are currently being marked for cleanup - -- AWS VPC Peering Connections -- AWS EC2 Network ACLs -- AWS EC2 Network Interfaces -- AWS ELB -- AWS ALB -- AWS ALB Target Groups -- AWS EC2 Subnets -- AWS EC2 Security Groups -- AWS EC2 Internet Gateways -- AWS EC2 NAT Gateways -- AWS EC2 Route Tables - -## Usage - -In `resh` execute - -``` -> config edit resoto.worker -``` - -and find the following section - -``` -plugin_cleanup_aws_vpcs: - # Dictionary of key cloud with list of account IDs for which the plugin should be active as value - config: - aws: - - '1234567' - - '567890' - # Enable plugin? - enabled: false -``` diff --git a/plugins/cleanup_aws_vpcs/pyproject.toml b/plugins/cleanup_aws_vpcs/pyproject.toml deleted file mode 100644 index fd1f812ba3..0000000000 --- a/plugins/cleanup_aws_vpcs/pyproject.toml +++ /dev/null @@ -1,45 +0,0 @@ -[project] -name = "resoto-plugin-cleanup-aws-vpcs" -description = "AWS VPC Cleaner Plugin" -version = "3.7.0" -authors = [{name="Some Engineering Inc."}] -license = {file="LICENSE"} -requires-python = ">=3.9" -classifiers = [ - # Current project status - "Development Status :: 4 - Beta", - # Audience - "Intended Audience :: System Administrators", - "Intended Audience :: Information Technology", - # License information - "License :: OSI Approved :: Apache Software License", - # Supported python versions - "Programming Language :: Python :: 3.9", - # Supported OS's - "Operating System :: POSIX :: Linux", - "Operating System :: Unix", - # Extra metadata - "Environment :: Console", - "Natural Language :: English", - "Topic :: Security", - "Topic :: Utilities", -] -readme = {file="README.md", content-type="text/markdown"} - -dependencies = [ - "resotolib==3.7.0", - "resoto-plugin-aws==3.7.0" -] - -[project.entry-points."resoto.plugins"] -cleanup_aws_vpcs = "resoto_plugin_cleanup_aws_vpcs:CleanupAWSVPCsPlugin" - -[project.urls] -Documentation = "https://resoto.com" -Source = "https://github.com/someengineering/resoto/tree/main/plugins/cleanup_aws_vpcs" - -[build-system] -requires = ["setuptools>=67.8.0", "wheel>=0.40.0", "build>=0.10.0"] -build-backend = "setuptools.build_meta" - - diff --git a/plugins/cleanup_aws_vpcs/resoto_plugin_cleanup_aws_vpcs/__init__.py b/plugins/cleanup_aws_vpcs/resoto_plugin_cleanup_aws_vpcs/__init__.py deleted file mode 100644 index c29d0162c9..0000000000 --- a/plugins/cleanup_aws_vpcs/resoto_plugin_cleanup_aws_vpcs/__init__.py +++ /dev/null @@ -1,134 +0,0 @@ -from copy import deepcopy -from resotolib.baseresources import EdgeType -from resotolib.baseplugin import BaseActionPlugin -from resotolib.logger import log -from resotolib.core.search import CoreGraph -from resotolib.graph import Graph -from resoto_plugin_aws.resource.ec2 import ( - AwsEc2Vpc, - AwsEc2VpcPeeringConnection, - AwsEc2NetworkAcl, - AwsEc2NetworkInterface, - AwsEc2Subnet, - AwsEc2SecurityGroup, - AwsEc2InternetGateway, - AwsEc2NatGateway, - AwsEc2RouteTable, - AwsEc2VpcEndpoint, - AwsEc2Instance, - AwsEc2ElasticIp, -) -from resoto_plugin_aws.resource.elb import AwsElb -from resoto_plugin_aws.resource.elbv2 import AwsAlb, AwsAlbTargetGroup -from resotolib.config import Config -from .config import CleanupAWSVPCsConfig -from typing import Dict - - -class CleanupAWSVPCsPlugin(BaseActionPlugin): - action = "post_cleanup_plan" - - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self.config = {} - - def bootstrap(self) -> bool: - return Config.plugin_cleanup_aws_vpcs.enabled - - def do_action(self, data: Dict) -> None: - Config.plugin_cleanup_aws_vpcs.validate(Config.plugin_cleanup_aws_vpcs) - self.config = deepcopy(Config.plugin_cleanup_aws_vpcs.config) - cg = CoreGraph(tls_data=self.tls_data) - query = "is(aws_vpc) and /desired.clean == true and /metadata.cleaned == false <-default,delete[0:]delete->" - graph = cg.graph(query) - self.vpc_cleanup(graph) - cg.patch_nodes(graph) - - def vpc_cleanup(self, graph: Graph): - log.info("AWS VPC cleanup called") - for node in graph.nodes: - if node.protected or not node.clean or not isinstance(node, AwsEc2Vpc): - continue - - cloud = node.cloud(graph) - account = node.account(graph) - region = node.region(graph) - log_prefix = ( - f"Found AWS VPC {node.dname} in cloud {cloud.name} account {account.dname} " - f"region {region.name} marked for cleanup." - ) - - if self.config and len(self.config) > 0: - if cloud.id not in self.config or account.id not in self.config[cloud.id]: - log.debug((f"{log_prefix} Account not found in config - ignoring dependent resources.")) - continue - - vpc_instances = [ - i - for i in node.descendants(graph, edge_type=EdgeType.delete) - if isinstance(i, AwsEc2Instance) - and i.instance_status not in ("shutting-down", "terminated") - and not i.clean - ] - if len(vpc_instances) > 0: - log_msg = "VPC contains active EC2 instances - not cleaning VPC." - log.debug(f"{log_prefix} {log_msg}") - node.log(log_msg) - node.clean = False - continue - - log.debug(f"{log_prefix} Marking dependent resources for cleanup as well.") - - for descendant in node.descendants(graph, edge_type=EdgeType.delete): - log.debug(f"Found descendant {descendant.rtdname} of VPC {node.dname}") - if isinstance( - descendant, - ( - AwsEc2VpcPeeringConnection, - AwsEc2NetworkAcl, - AwsEc2NetworkInterface, - AwsElb, - AwsAlb, - AwsAlbTargetGroup, - AwsEc2Subnet, - AwsEc2SecurityGroup, - AwsEc2InternetGateway, - AwsEc2NatGateway, - AwsEc2RouteTable, - AwsEc2VpcEndpoint, - AwsEc2ElasticIp, - ), - ): - descendant.log( - ( - f"Marking for cleanup because resource is a descendant of VPC {node.dname} " - f"which is set to be cleaned" - ) - ) - node.log(f"Marking {descendant.rtdname} for cleanup because resource is a descendant") - descendant.clean = True - else: - if descendant.clean: - log.debug( - ( - f"Descendant {descendant.rtdname} of VPC {node.dname} is not targeted but " - f"already marked for cleaning" - ) - ) - else: - log.error( - ( - f"Descendant {descendant.rtdname} of VPC {node.dname} is not targeted and " - f"not marked for cleaning - VPC cleanup will likely fail" - ) - ) - node.log( - ( - f"Descendant {descendant.rtdname} is not targeted and not marked for cleaning " - f"- cleanup will likely fail" - ) - ) - - @staticmethod - def add_config(config: Config) -> None: - config.add_config(CleanupAWSVPCsConfig) diff --git a/plugins/cleanup_aws_vpcs/resoto_plugin_cleanup_aws_vpcs/config.py b/plugins/cleanup_aws_vpcs/resoto_plugin_cleanup_aws_vpcs/config.py deleted file mode 100644 index 470d5b15df..0000000000 --- a/plugins/cleanup_aws_vpcs/resoto_plugin_cleanup_aws_vpcs/config.py +++ /dev/null @@ -1,42 +0,0 @@ -from attrs import define, field -from typing import ClassVar, Dict, List - - -@define -class CleanupAWSVPCsConfig: - kind: ClassVar[str] = "plugin_cleanup_aws_vpcs" - enabled: bool = field( - default=False, - metadata={"description": "Enable plugin?", "restart_required": True}, - ) - config: Dict[str, List[str]] = field( - factory=lambda: {"aws": ["1234567", "567890"]}, - metadata={ - "description": ( - "Dictionary of key cloud with list of account IDs" " for which the plugin should be active as value" - ) - }, - ) - - @staticmethod - def validate(cfg: "CleanupAWSVPCsConfig") -> bool: - if not isinstance(cfg, CleanupAWSVPCsConfig): - raise ValueError("Config is no CleanupAWSVPCsConfig") - - config = cfg.config - if config is None: - return True - - if not isinstance(config, dict): - raise ValueError("Config is no dict") - - for cloud_id, account_ids in config.items(): - if not isinstance(cloud_id, str): - raise ValueError(f"Cloud ID {cloud_id} is no string") - if not isinstance(account_ids, list): - raise ValueError(f"Account IDs {account_ids} is no list") - - for account_id in account_ids: - if not isinstance(account_id, str): - raise ValueError(f"Account ID {account_id} is no string") - return True diff --git a/plugins/cleanup_aws_vpcs/setup.cfg b/plugins/cleanup_aws_vpcs/setup.cfg deleted file mode 100644 index 7ca6537935..0000000000 --- a/plugins/cleanup_aws_vpcs/setup.cfg +++ /dev/null @@ -1,7 +0,0 @@ -[options] -packages = find: -include_package_data = True -zip_safe = False - -[aliases] -test=pytest diff --git a/plugins/cleanup_aws_vpcs/test/test_config.py b/plugins/cleanup_aws_vpcs/test/test_config.py deleted file mode 100644 index 7bfc6e8bdc..0000000000 --- a/plugins/cleanup_aws_vpcs/test/test_config.py +++ /dev/null @@ -1,10 +0,0 @@ -from resotolib.config import Config -from resoto_plugin_cleanup_aws_vpcs import CleanupAWSVPCsPlugin - - -def test_config(): - config = Config("dummy", "dummy") - CleanupAWSVPCsPlugin.add_config(config) - Config.init_default_config() - assert Config.plugin_cleanup_aws_vpcs.enabled is False - assert Config.plugin_cleanup_aws_vpcs.validate(Config.plugin_cleanup_aws_vpcs) is True diff --git a/plugins/cleanup_aws_vpcs/tox.ini b/plugins/cleanup_aws_vpcs/tox.ini deleted file mode 100644 index b698ad543b..0000000000 --- a/plugins/cleanup_aws_vpcs/tox.ini +++ /dev/null @@ -1,30 +0,0 @@ -[tox] -env_list = syntax, tests, black - -[flake8] -max-line-length=120 -exclude = .git,.tox,__pycache__,.idea,.pytest_cache -ignore=F403, F405, E722, N806, N813, E266, W503, E203 - -[pytest] -addopts= --cov=resoto_plugin_cleanup_aws_vpcs -rs -vv --cov-report html -testpaths= test - -[testenv] -usedevelop = true -deps = - --editable=file:///{toxinidir}/../../resotolib - --editable=file:///{toxinidir}/../aws - -r../../requirements-all.txt -# until this is fixed: https://github.com/pypa/setuptools/issues/3518 -setenv = - SETUPTOOLS_ENABLE_FEATURES = legacy-editable - -[testenv:syntax] -commands = flake8 --verbose - -[testenv:tests] -commands= pytest - -[testenv:black] -commands = black --line-length 120 --check --diff --target-version py39 . diff --git a/plugins/cleanup_expired/MANIFEST.in b/plugins/cleanup_expired/MANIFEST.in deleted file mode 100644 index bb3ec5f0d4..0000000000 --- a/plugins/cleanup_expired/MANIFEST.in +++ /dev/null @@ -1 +0,0 @@ -include README.md diff --git a/plugins/cleanup_expired/README.md b/plugins/cleanup_expired/README.md deleted file mode 100644 index ee24ec7c49..0000000000 --- a/plugins/cleanup_expired/README.md +++ /dev/null @@ -1,38 +0,0 @@ -# resoto-plugin-cleanup-expired -Resoto plugin for cleanup of expired resources - -This plugin looks for resources with the tags `expiration` or `resoto:expires` and flags them for cleanup if they are expired. - -## Tag format - -| Tag | Format | Description | -| ---------------- | --------------------------- | ---------------------------------------------------- | -| `resoto:expires` | `2019-09-05T10:40:11+00:00` | ISO 8601 Timestamp | -| `expiration` | `24h` | A timedelta relative to the resource's creation time | - -Example of valid units for the `expiration` tag: - -``` -weeks -days -hours -minutes -``` - -Each of them can be abbreviated down to one letter. E.g. `7d`, `24h`, `60m`, etc. A space in between the numeric and the unit is optional, meaning `7d` and `7 days` are equivalent. - -## Usage - -In `resh` execute - -``` -> config edit resoto.worker -``` - -and find the following section - -``` -plugin_cleanup_expired: - # Enable plugin? - enabled: false -``` diff --git a/plugins/cleanup_expired/pyproject.toml b/plugins/cleanup_expired/pyproject.toml deleted file mode 100644 index 8af83bac0e..0000000000 --- a/plugins/cleanup_expired/pyproject.toml +++ /dev/null @@ -1,44 +0,0 @@ -[project] -name = "resoto-plugin-cleanup-expired" -description = "Resoto Expired Resource Cleanup Plugin" -version = "3.7.0" -authors = [{name="Some Engineering Inc."}] -license = {file="LICENSE"} -requires-python = ">=3.9" -classifiers = [ - # Current project status - "Development Status :: 4 - Beta", - # Audience - "Intended Audience :: System Administrators", - "Intended Audience :: Information Technology", - # License information - "License :: OSI Approved :: Apache Software License", - # Supported python versions - "Programming Language :: Python :: 3.9", - # Supported OS's - "Operating System :: POSIX :: Linux", - "Operating System :: Unix", - # Extra metadata - "Environment :: Console", - "Natural Language :: English", - "Topic :: Security", - "Topic :: Utilities", -] -readme = {file="README.md", content-type="text/markdown"} - -dependencies = [ - "resotolib==3.7.0", -] - -[project.entry-points."resoto.plugins"] -cleanup_expired = "resoto_plugin_cleanup_expired:CleanupExpiredPlugin" - -[project.urls] -Documentation = "https://resoto.com" -Source = "https://github.com/someengineering/resoto/tree/main/plugins/cleanup_expired" - -[build-system] -requires = ["setuptools>=67.8.0", "wheel>=0.40.0", "build>=0.10.0"] -build-backend = "setuptools.build_meta" - - diff --git a/plugins/cleanup_expired/resoto_plugin_cleanup_expired/__init__.py b/plugins/cleanup_expired/resoto_plugin_cleanup_expired/__init__.py deleted file mode 100644 index 45b52f3401..0000000000 --- a/plugins/cleanup_expired/resoto_plugin_cleanup_expired/__init__.py +++ /dev/null @@ -1,41 +0,0 @@ -from resotolib.baseplugin import BaseActionPlugin -from resotolib.logger import log -from resotolib.core.search import CoreGraph -from resotolib.config import Config -from .config import CleanupExpiredConfig -from typing import Dict - - -class CleanupExpiredPlugin(BaseActionPlugin): - action = "cleanup_plan" - - def bootstrap(self): - return Config.plugin_cleanup_expired.enabled - - def do_action(self, data: Dict) -> None: - log.debug("Cleanup Expired called") - cg = CoreGraph(tls_data=self.tls_data) - command = 'query /metadata.expires < "@NOW@" | clean "Resource is expired"' - for response in cg.execute(command): - if isinstance(response, Dict) and "type" in response and response["type"] == "node": - reported = response.get("reported", {}) - kind = reported.get("kind") - node_id = reported.get("id") - node_name = reported.get("name") - age = reported.get("age") - ancestors = response.get("ancestors", {}) - cloud = ancestors.get("cloud", {}).get("id") - account = ancestors.get("account", {}).get("id") - region = ancestors.get("region", {}).get("id") - dname = node_id if node_id == node_name else f"{node_name} ({node_id})" - log.debug( - f"Marking expired {kind} {dname} with age {age}" - f" in cloud {cloud} account {account} region {region}" - " for cleanup" - ) - else: - log.debug(f"Response: {response}") - - @staticmethod - def add_config(config: Config) -> None: - config.add_config(CleanupExpiredConfig) diff --git a/plugins/cleanup_expired/resoto_plugin_cleanup_expired/config.py b/plugins/cleanup_expired/resoto_plugin_cleanup_expired/config.py deleted file mode 100644 index 8d63ebda61..0000000000 --- a/plugins/cleanup_expired/resoto_plugin_cleanup_expired/config.py +++ /dev/null @@ -1,11 +0,0 @@ -from attrs import define, field -from typing import ClassVar - - -@define -class CleanupExpiredConfig: - kind: ClassVar[str] = "plugin_cleanup_expired" - enabled: bool = field( - default=False, - metadata={"description": "Enable plugin?", "restart_required": True}, - ) diff --git a/plugins/cleanup_expired/setup.cfg b/plugins/cleanup_expired/setup.cfg deleted file mode 100644 index 7ca6537935..0000000000 --- a/plugins/cleanup_expired/setup.cfg +++ /dev/null @@ -1,7 +0,0 @@ -[options] -packages = find: -include_package_data = True -zip_safe = False - -[aliases] -test=pytest diff --git a/plugins/cleanup_expired/test/test_config.py b/plugins/cleanup_expired/test/test_config.py deleted file mode 100644 index c593a5cc63..0000000000 --- a/plugins/cleanup_expired/test/test_config.py +++ /dev/null @@ -1,9 +0,0 @@ -from resotolib.config import Config -from resoto_plugin_cleanup_expired import CleanupExpiredPlugin - - -def test_config(): - config = Config("dummy", "dummy") - CleanupExpiredPlugin.add_config(config) - Config.init_default_config() - assert Config.plugin_cleanup_expired.enabled is False diff --git a/plugins/cleanup_expired/tox.ini b/plugins/cleanup_expired/tox.ini deleted file mode 100644 index 902f2964cd..0000000000 --- a/plugins/cleanup_expired/tox.ini +++ /dev/null @@ -1,29 +0,0 @@ -[tox] -env_list = syntax, tests, black - -[flake8] -max-line-length=120 -exclude = .git,.tox,__pycache__,.idea,.pytest_cache -ignore=F403, F405, E722, N806, N813, E266, W503, E203 - -[pytest] -addopts= --cov=resoto_plugin_cleanup_expired -rs -vv --cov-report html -testpaths= test - -[testenv] -usedevelop = true -deps = - --editable=file:///{toxinidir}/../../resotolib - -r../../requirements-all.txt -# until this is fixed: https://github.com/pypa/setuptools/issues/3518 -setenv = - SETUPTOOLS_ENABLE_FEATURES = legacy-editable - -[testenv:syntax] -commands = flake8 --verbose - -[testenv:tests] -commands= pytest - -[testenv:black] -commands = black --line-length 120 --check --diff --target-version py39 . diff --git a/plugins/cleanup_untagged/MANIFEST.in b/plugins/cleanup_untagged/MANIFEST.in deleted file mode 100644 index bb3ec5f0d4..0000000000 --- a/plugins/cleanup_untagged/MANIFEST.in +++ /dev/null @@ -1 +0,0 @@ -include README.md diff --git a/plugins/cleanup_untagged/README.md b/plugins/cleanup_untagged/README.md deleted file mode 100644 index af0c119a6a..0000000000 --- a/plugins/cleanup_untagged/README.md +++ /dev/null @@ -1,68 +0,0 @@ -# resoto-plugin-cleanup-untagged -Cleanup Untagged Plugin for Resoto - -This plugin deletes cloud resources that are missing mandatory tags after a certain amount of time has passed since their creation. - -## Usage - -In `resh` execute - -``` -> config edit resoto.worker -``` - -and find the following section - -``` -plugin_cleanup_untagged: - # Configuration for the plugin - config: - default: - age: '2h' - tags: - - 'owner' - - 'expiration' - kinds: - - 'aws_ec2_instance' - - 'aws_ec2_volume' - - 'aws_vpc' - - 'aws_cloudformation_stack' - - 'aws_elb' - - 'aws_alb' - - 'aws_alb_target_group' - - 'aws_eks_cluster' - - 'aws_eks_nodegroup' - - 'example_instance' - - 'example_network' - accounts: - aws: - 068564737731: - name: 'playground' - age: '7d' - '575584959047': - name: 'eng-sre' - example: - Example Account: - name: 'Example Account' - # Enable plugin? - enabled: false -``` - -### Config section format - -The config section consists of four sub-sections. `default`, `tags`, `classes` and `accounts`. The `default` section specifies the default `age` a resource must have before we enforce mandatory tags on it. For instance if `age` is set to `2h` this means that whatever mechanism creates a resource has two hours to add those mandatory tags. - -The `tags` section is a list of tag names that MUST exist on every resource class specified in `classes`. The `classes` section is a list of resource class names for which tags specified in the `tags` list must exist. - -The `accounts` section contains a dictionary with cloud IDs as keys (e.g. `aws`) and account IDs for which tags will be enforced as values (e.g. `068564737731`). Those in turn contain a `name` and optionally an `age` override. - -The following age units are valid: - -``` -weeks -days -hours -minutes -``` - -Each of them can be abbreviated down to one letter. E.g. `7d`, `24h`, `60m`, etc. A space in between the numeric and the unit is optional, meaning `7d` and `7 days` are equivalent. diff --git a/plugins/cleanup_untagged/example.yaml b/plugins/cleanup_untagged/example.yaml deleted file mode 100644 index 57e412cfa6..0000000000 --- a/plugins/cleanup_untagged/example.yaml +++ /dev/null @@ -1,31 +0,0 @@ -default: - age: 2h - -tags: - - owner - - expiration - -kinds: - - aws_ec2_instance - - aws_ec2_volume - - aws_vpc - - aws_cloudformation_stack - - aws_elb - - aws_alb - - aws_alb_target_group - - aws_eks_cluster - - aws_eks_nodegroup - - example_instance - - example_network - -accounts: - aws: - 068564737731: - name: playground - age: 7d - 575584959047: - name: eng-sre - - example: - Example Account: - name: example_account diff --git a/plugins/cleanup_untagged/pyproject.toml b/plugins/cleanup_untagged/pyproject.toml deleted file mode 100644 index 2e6b463869..0000000000 --- a/plugins/cleanup_untagged/pyproject.toml +++ /dev/null @@ -1,44 +0,0 @@ -[project] -name = "resoto-plugin-cleanup-untagged" -description = "Resoto Cleanup Untagged Plugin" -version = "3.7.0" -authors = [{name="Some Engineering Inc."}] -license = {file="LICENSE"} -requires-python = ">=3.9" -classifiers = [ - # Current project status - "Development Status :: 4 - Beta", - # Audience - "Intended Audience :: System Administrators", - "Intended Audience :: Information Technology", - # License information - "License :: OSI Approved :: Apache Software License", - # Supported python versions - "Programming Language :: Python :: 3.9", - # Supported OS's - "Operating System :: POSIX :: Linux", - "Operating System :: Unix", - # Extra metadata - "Environment :: Console", - "Natural Language :: English", - "Topic :: Security", - "Topic :: Utilities", -] -readme = {file="README.md", content-type="text/markdown"} - -dependencies = [ - "resotolib==3.7.0", -] - -[project.entry-points."resoto.plugins"] -cleanup_untagged = "resoto_plugin_cleanup_untagged:CleanupUntaggedPlugin" - -[project.urls] -Documentation = "https://resoto.com" -Source = "https://github.com/someengineering/resoto/tree/main/plugins/cleanup_untagged" - -[build-system] -requires = ["setuptools>=67.8.0", "wheel>=0.40.0", "build>=0.10.0"] -build-backend = "setuptools.build_meta" - - diff --git a/plugins/cleanup_untagged/resoto_plugin_cleanup_untagged/__init__.py b/plugins/cleanup_untagged/resoto_plugin_cleanup_untagged/__init__.py deleted file mode 100644 index 8cdd5e1212..0000000000 --- a/plugins/cleanup_untagged/resoto_plugin_cleanup_untagged/__init__.py +++ /dev/null @@ -1,55 +0,0 @@ -from copy import deepcopy - -from resotolib.baseplugin import BaseActionPlugin -from resotolib.config import Config -from resotolib.core.model_export import node_from_dict -from resotolib.core.search import CoreGraph -from resotolib.json import value_in_path -from resotolib.logger import log -from resotolib.types import Json -from .config import CleanupUntaggedConfig - - -class CleanupUntaggedPlugin(BaseActionPlugin): - action = "cleanup_plan" - - def bootstrap(self) -> bool: - return Config.plugin_cleanup_untagged.enabled is True - - @staticmethod - def create_command(cfg: Json) -> str: - tags_part = 'not(has_key(tags, ["' + '", "'.join(cfg["tags"]) + '"]))' - kinds_part = 'is(["' + '", "'.join(cfg["kinds"]) + '"])' - account_parts = [] - default_age: str = value_in_path(cfg, ["default", "age"]) or "2h" - for cloud_id, account in cfg["accounts"].items(): - for account_id, account_data in account.items(): - age = account_data.get("age", default_age) - account_part = ( - f'(/ancestors.cloud.id == "{cloud_id}" and ' - f'/ancestors.account.id == "{account_id}" and ' - f"age > {age})" - ) - account_parts.append(account_part) - accounts_part = "(" + " or ".join(account_parts) + ")" - exclusion_part = "/metadata.protected == false and /metadata.phantom == false and /metadata.cleaned == false" - required_tags = ", ".join(cfg["tags"]) - reason = f"Missing one or more of required tags {required_tags}" " and age more than threshold" - return f'search {exclusion_part} and {kinds_part} and {tags_part} and {accounts_part} | clean "{reason}"' - - def do_action(self, data: Json) -> None: - log.debug("Cleanup Untagged called") - cg = CoreGraph(tls_data=self.tls_data) - cfg: Json = deepcopy(Config.plugin_cleanup_untagged.config) - command = self.create_command(cfg) - required_tags = ", ".join(cfg.get("tags", {})) - for node_data in cg.execute(command): - node = node_from_dict(node_data) - log.debug( - f"Marking {node.rtdname} with age {node.age} for cleanup for" - f" missing one or more of tags: {required_tags}" - ) - - @staticmethod - def add_config(config: Config) -> None: - config.add_config(CleanupUntaggedConfig) diff --git a/plugins/cleanup_untagged/resoto_plugin_cleanup_untagged/config.py b/plugins/cleanup_untagged/resoto_plugin_cleanup_untagged/config.py deleted file mode 100644 index 28801121e7..0000000000 --- a/plugins/cleanup_untagged/resoto_plugin_cleanup_untagged/config.py +++ /dev/null @@ -1,85 +0,0 @@ -from typing import ClassVar - -from attrs import define, field - -from resotolib.json import value_in_path -from resotolib.types import Json - -default_config = { - "default": {"age": "2h"}, - "tags": ["owner", "expiration"], - "kinds": [ - "aws_ec2_instance", - "aws_ec2_volume", - "aws_vpc", - "aws_cloudformation_stack", - "aws_elb", - "aws_alb", - "aws_alb_target_group", - "aws_eks_cluster", - "aws_eks_nodegroup", - "example_instance", - "example_network", - ], - "accounts": { - "aws": { - "068564737731": {"name": "playground", "age": "7d"}, - "575584959047": { - "name": "eng-sre", - }, - }, - "example": { - "Example Account": { - "name": "Example Account", - } - }, - }, -} - - -@define -class CleanupUntaggedConfig: - kind: ClassVar[str] = "plugin_cleanup_untagged" - enabled: bool = field( - default=False, - metadata={"description": "Enable plugin?", "restart_required": True}, - ) - config: Json = field( - factory=lambda: default_config, - metadata={ - "description": ( - "Configuration for the plugin\n" - "See https://github.com/someengineering/resoto/tree/main/plugins/cleanup_untagged for syntax details" - ) - }, - ) - - @staticmethod - def validate(cfg: "CleanupUntaggedConfig") -> bool: - config = cfg.config - required_sections = ["tags", "kinds", "accounts"] - for section in required_sections: - if section not in config: - raise ValueError(f"Section '{section}' not found in config") - - if not isinstance(config["tags"], list) or len(config["tags"]) == 0: - raise ValueError("Error in 'tags' section") - - if not isinstance(config["kinds"], list) or len(config["kinds"]) == 0: - raise ValueError("Error in 'kinds' section") - - if not isinstance(config["accounts"], dict) or len(config["accounts"]) == 0: - raise ValueError("Error in 'accounts' section") - - maybe_default_age = value_in_path(config, ["default", "age"]) - for cloud_id, account in config["accounts"].items(): - for account_id, account_data in account.items(): - if "name" not in account_data: - raise ValueError(f"Missing 'name' for account '{cloud_id}/{account_id}") - if "age" in account_data: - account_data["age"] = account_data.get("age") - elif maybe_default_age is None: - raise ValueError(f"Missing 'age' for account '{cloud_id}/{account_id}' and no default age defined'") - else: - account_data["age"] = maybe_default_age - return True diff --git a/plugins/cleanup_untagged/setup.cfg b/plugins/cleanup_untagged/setup.cfg deleted file mode 100644 index 7ca6537935..0000000000 --- a/plugins/cleanup_untagged/setup.cfg +++ /dev/null @@ -1,7 +0,0 @@ -[options] -packages = find: -include_package_data = True -zip_safe = False - -[aliases] -test=pytest diff --git a/plugins/cleanup_untagged/test/test_config.py b/plugins/cleanup_untagged/test/test_config.py deleted file mode 100644 index a57b5c8e4e..0000000000 --- a/plugins/cleanup_untagged/test/test_config.py +++ /dev/null @@ -1,35 +0,0 @@ -from datetime import timedelta - -from resotolib.config import Config -from resoto_plugin_cleanup_untagged import CleanupUntaggedPlugin - - -def test_config(): - config = Config("dummy", "dummy") - CleanupUntaggedPlugin.add_config(config) - Config.init_default_config() - assert Config.plugin_cleanup_untagged.enabled is False - assert Config.plugin_cleanup_untagged.validate(Config.plugin_cleanup_untagged) is True - - -class WorkerConfig: - def __init__(self) -> None: - self.timeout = timedelta(seconds=60) - - -def test_create_command(): - cfg = Config("dummy", "dummy") - CleanupUntaggedPlugin.add_config(cfg) - Config.running_config.data["resotoworker"] = WorkerConfig() - Config.init_default_config() - assert CleanupUntaggedPlugin().create_command(Config.plugin_cleanup_untagged.config) == ( - "search /metadata.protected == false and /metadata.phantom == false and /metadata.cleaned == false and " - 'is(["aws_ec2_instance", "aws_ec2_volume", "aws_vpc", "aws_cloudformation_stack", "aws_elb", ' - '"aws_alb", "aws_alb_target_group", "aws_eks_cluster", "aws_eks_nodegroup", ' - '"example_instance", "example_network"]) ' - 'and not(has_key(tags, ["owner", "expiration"])) ' - 'and ((/ancestors.cloud.id == "aws" and /ancestors.account.id == "068564737731" and age > 7d) ' - 'or (/ancestors.cloud.id == "aws" and /ancestors.account.id == "575584959047" and age > 2h) ' - 'or (/ancestors.cloud.id == "example" and /ancestors.account.id == "Example Account" and age > 2h)) | ' - 'clean "Missing one or more of required tags owner, expiration and age more than threshold"' - ) diff --git a/plugins/cleanup_untagged/tox.ini b/plugins/cleanup_untagged/tox.ini deleted file mode 100644 index db4eeac4e4..0000000000 --- a/plugins/cleanup_untagged/tox.ini +++ /dev/null @@ -1,31 +0,0 @@ -[tox] -env_list = syntax, tests, black, mypy - -[flake8] -max-line-length=120 -exclude = .git,.tox,__pycache__,.idea,.pytest_cache -ignore=F403, F405, E722, N806, N813, E266, W503, E203 - -[pytest] -testpaths= test - -[testenv] -usedevelop = true -deps = - --editable=file:///{toxinidir}/../../resotolib - -r../../requirements-all.txt -# until this is fixed: https://github.com/pypa/setuptools/issues/3518 -setenv = - SETUPTOOLS_ENABLE_FEATURES = legacy-editable - -[testenv:syntax] -commands = flake8 --verbose - -[testenv:tests] -commands= pytest - -[testenv:black] -commands = black --line-length 120 --check --diff --target-version py39 . - -[testenv:mypy] -commands= python -m mypy --install-types --non-interactive --python-version 3.9 --strict resoto_plugin_cleanup_untagged diff --git a/plugins/cleanup_volumes/MANIFEST.in b/plugins/cleanup_volumes/MANIFEST.in deleted file mode 100644 index bb3ec5f0d4..0000000000 --- a/plugins/cleanup_volumes/MANIFEST.in +++ /dev/null @@ -1 +0,0 @@ -include README.md diff --git a/plugins/cleanup_volumes/README.md b/plugins/cleanup_volumes/README.md deleted file mode 100644 index 51d2e29405..0000000000 --- a/plugins/cleanup_volumes/README.md +++ /dev/null @@ -1,37 +0,0 @@ -# resoto-plugin-cleanup-volumes -Volume cleanup plugin for Resoto - -This plugin cleans up storage volumes. - -## Usage - -In `resh` execute - -``` -> config edit resoto.worker -``` - -and find the following section - -``` -plugin_cleanup_volumes: - # Enable plugin? - enabled: false - # Minimum age of unused volumes to cleanup - min_age: '14 days' -``` - -The default volume age is 14 days. Meaning if a volume is not in use and has not had any read or write IOPS within the last 14 days it will be deleted. - -Optionally change the age cutoff value using the `min_age` option. - -Example of valid age units: - -``` -weeks -days -hours -minutes -``` - -Each of them can be abbreviated down to one letter. E.g. `7d`, `24h`, `60m`, etc. A space in between the numeric and the unit is optional, meaning `7d` and `7 days` are equivalent. diff --git a/plugins/cleanup_volumes/pyproject.toml b/plugins/cleanup_volumes/pyproject.toml deleted file mode 100644 index a51d79d4e0..0000000000 --- a/plugins/cleanup_volumes/pyproject.toml +++ /dev/null @@ -1,44 +0,0 @@ -[project] -name = "resoto-plugin-cleanup-volumes" -description = "Volume Cleaner Plugin" -version = "3.7.0" -authors = [{name="Some Engineering Inc."}] -license = {file="LICENSE"} -requires-python = ">=3.9" -classifiers = [ - # Current project status - "Development Status :: 4 - Beta", - # Audience - "Intended Audience :: System Administrators", - "Intended Audience :: Information Technology", - # License information - "License :: OSI Approved :: Apache Software License", - # Supported python versions - "Programming Language :: Python :: 3.9", - # Supported OS's - "Operating System :: POSIX :: Linux", - "Operating System :: Unix", - # Extra metadata - "Environment :: Console", - "Natural Language :: English", - "Topic :: Security", - "Topic :: Utilities", -] -readme = {file="README.md", content-type="text/markdown"} - -dependencies = [ - "resotolib==3.7.0", -] - -[project.entry-points."resoto.plugins"] -cleanup_volumes = "resoto_plugin_cleanup_volumes:CleanupVolumesPlugin" - -[project.urls] -Documentation = "https://resoto.com" -Source = "https://github.com/someengineering/resoto/tree/main/plugins/cleanup_volumes" - -[build-system] -requires = ["setuptools>=67.8.0", "wheel>=0.40.0", "build>=0.10.0"] -build-backend = "setuptools.build_meta" - - diff --git a/plugins/cleanup_volumes/resoto_plugin_cleanup_volumes/__init__.py b/plugins/cleanup_volumes/resoto_plugin_cleanup_volumes/__init__.py deleted file mode 100644 index 036ee4122a..0000000000 --- a/plugins/cleanup_volumes/resoto_plugin_cleanup_volumes/__init__.py +++ /dev/null @@ -1,67 +0,0 @@ -from resotolib.baseplugin import BaseActionPlugin -from resotolib.logger import log -from resotolib.core.search import CoreGraph -from resotolib.graph import Graph -from resotolib.baseresources import BaseVolume -from resotolib.config import Config -from resotolib.durations import parse_duration -from .config import CleanupVolumesConfig -from typing import Dict - - -class CleanupVolumesPlugin(BaseActionPlugin): - action = "cleanup_plan" - - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self.age = None - if Config.plugin_cleanup_volumes.enabled: - self.update_age() - - def bootstrap(self) -> bool: - return Config.plugin_cleanup_volumes.enabled - - def do_action(self, data: Dict) -> None: - self.update_age() - cg = CoreGraph(tls_data=self.tls_data) - query = "is(volume) and reported.volume_status == available <-[0:]->" - graph = cg.graph(query) - self.volumes_cleanup(graph) - cg.patch_nodes(graph) - - def update_age(self) -> None: - try: - self.age = parse_duration(Config.plugin_cleanup_volumes.min_age) - log.debug(f"Volume Cleanup Plugin Age {self.age}") - except ValueError: - log.error(f"Error while parsing Volume Cleanup Age {Config.plugin_cleanup_volumes.min_age}") - raise - - def volumes_cleanup(self, graph: Graph): - log.info("Volume Cleanup called") - for node in graph.nodes: - if ( - isinstance(node, BaseVolume) - and node.volume_status == "available" - and node.age > self.age - and node.last_access is not None - and node.last_update is not None - and node.last_access > self.age - and node.last_update > self.age - ): - cloud = node.cloud(graph) - account = node.account(graph) - region = node.region(graph) - log.debug( - ( - f"Found available volume {node.dname} in cloud {cloud.name} account {account.dname} " - f"region {region.name} with age {node.age}. Last update was {node.last_update} ago " - f"and last access {node.last_access} ago both of which is longer than {self.age} " - f"- setting to be cleaned" - ) - ) - node.clean = True - - @staticmethod - def add_config(config: Config) -> None: - config.add_config(CleanupVolumesConfig) diff --git a/plugins/cleanup_volumes/resoto_plugin_cleanup_volumes/config.py b/plugins/cleanup_volumes/resoto_plugin_cleanup_volumes/config.py deleted file mode 100644 index 0a127ffbb8..0000000000 --- a/plugins/cleanup_volumes/resoto_plugin_cleanup_volumes/config.py +++ /dev/null @@ -1,24 +0,0 @@ -from attrs import define, field -from typing import ClassVar -from resotolib.durations import parse_duration - - -@define -class CleanupVolumesConfig: - kind: ClassVar[str] = "plugin_cleanup_volumes" - enabled: bool = field( - default=False, - metadata={"description": "Enable plugin?", "restart_required": True}, - ) - min_age: str = field( - default="14 days", - metadata={ - "description": "Minimum age of unused volumes to cleanup", - "type_hint": "duration", - }, - ) - - @staticmethod - def validate(config: "CleanupVolumesConfig") -> bool: - parse_duration(config.min_age) - return True diff --git a/plugins/cleanup_volumes/setup.cfg b/plugins/cleanup_volumes/setup.cfg deleted file mode 100644 index 7ca6537935..0000000000 --- a/plugins/cleanup_volumes/setup.cfg +++ /dev/null @@ -1,7 +0,0 @@ -[options] -packages = find: -include_package_data = True -zip_safe = False - -[aliases] -test=pytest diff --git a/plugins/cleanup_volumes/test/test_config.py b/plugins/cleanup_volumes/test/test_config.py deleted file mode 100644 index 340898087a..0000000000 --- a/plugins/cleanup_volumes/test/test_config.py +++ /dev/null @@ -1,10 +0,0 @@ -from resotolib.config import Config -from resoto_plugin_cleanup_volumes import CleanupVolumesPlugin - - -def test_config(): - config = Config("dummy", "dummy") - CleanupVolumesPlugin.add_config(config) - Config.init_default_config() - assert Config.plugin_cleanup_volumes.enabled is False - assert Config.plugin_cleanup_volumes.min_age == "14 days" diff --git a/plugins/cleanup_volumes/tox.ini b/plugins/cleanup_volumes/tox.ini deleted file mode 100644 index 6fafa4b6e7..0000000000 --- a/plugins/cleanup_volumes/tox.ini +++ /dev/null @@ -1,29 +0,0 @@ -[tox] -env_list = syntax, tests, black - -[flake8] -max-line-length=120 -exclude = .git,.tox,__pycache__,.idea,.pytest_cache -ignore=F403, F405, E722, N806, N813, E266, W503, E203 - -[pytest] -addopts= --cov=resoto_plugin_cleanup_volumes -rs -vv --cov-report html -testpaths= test - -[testenv] -usedevelop = true -deps = - --editable=file:///{toxinidir}/../../resotolib - -r../../requirements-all.txt -# until this is fixed: https://github.com/pypa/setuptools/issues/3518 -setenv = - SETUPTOOLS_ENABLE_FEATURES = legacy-editable - -[testenv:syntax] -commands = flake8 --verbose - -[testenv:tests] -commands= pytest - -[testenv:black] -commands = black --line-length 120 --check --diff --target-version py39 . diff --git a/plugins/protector/MANIFEST.in b/plugins/protector/MANIFEST.in deleted file mode 100644 index bb3ec5f0d4..0000000000 --- a/plugins/protector/MANIFEST.in +++ /dev/null @@ -1 +0,0 @@ -include README.md diff --git a/plugins/protector/README.md b/plugins/protector/README.md deleted file mode 100644 index ea823dfa47..0000000000 --- a/plugins/protector/README.md +++ /dev/null @@ -1,42 +0,0 @@ -# resoto-plugin-protector -Protector Plugin for Resoto - -This plugin protects important resources from deletion by Resoto. - -## Usage - -In `resh` execute - -``` -> config edit resoto.worker -``` - -and find the following section - -``` -plugin_protector: - # Configuration for the plugin - # See https://github.com/someengineering/resoto/tree/main/plugins/protector for syntax details - config: - aws: - '110465657741': - us-east-1: - aws_ec2_instance: - - 'i-0fcbe8974615bfd37' - # Enable plugin? - enabled: false -``` - -The format of the `config` section is as follows: - -``` -cloud.id: - account.id: - region.id: - kind: - - resource.id -``` - -### Implementation details - -Each Resoto resource has an attributed `/metadata.protected` which takes a boolean value. By default it is set to `false`. Each Resoto resource inherits BaseResource which contains two methods for cleaning up a resource, `cleanup()` and `delete()`. Both those methods will refuse to manipulate a resource once the `protected` attribute has been set to `true`. Meaning if a resource is marked as protected but has also been flagged for cleanup the cleanup will fail because protected resources cannot be deleted. diff --git a/plugins/protector/example.yaml b/plugins/protector/example.yaml deleted file mode 100644 index 5acbbac9c6..0000000000 --- a/plugins/protector/example.yaml +++ /dev/null @@ -1,16 +0,0 @@ -# Format: -# cloud.id: -# account.id: -# region.id: -# kind: -# - resource.id -'aws': - '110465657741': - 'us-east-1': - 'aws_ec2_instance': - - 'i-0fcbe8974615bfd37' -'example': - 'Example Account': - 'us-west': - 'example_network': - - 'someNetwork' diff --git a/plugins/protector/pyproject.toml b/plugins/protector/pyproject.toml deleted file mode 100644 index 74c8264ec4..0000000000 --- a/plugins/protector/pyproject.toml +++ /dev/null @@ -1,44 +0,0 @@ -[project] -name = "resoto-plugin-protector" -description = "Resoto Protector Plugin" -version = "3.7.0" -authors = [{name="Some Engineering Inc."}] -license = {file="LICENSE"} -requires-python = ">=3.9" -classifiers = [ - # Current project status - "Development Status :: 4 - Beta", - # Audience - "Intended Audience :: System Administrators", - "Intended Audience :: Information Technology", - # License information - "License :: OSI Approved :: Apache Software License", - # Supported python versions - "Programming Language :: Python :: 3.9", - # Supported OS's - "Operating System :: POSIX :: Linux", - "Operating System :: Unix", - # Extra metadata - "Environment :: Console", - "Natural Language :: English", - "Topic :: Security", - "Topic :: Utilities", -] -readme = {file="README.md", content-type="text/markdown"} - -dependencies = [ - "resotolib==3.7.0", -] - -[project.entry-points."resoto.plugins"] -protector = "resoto_plugin_protector:ProtectorPlugin" - -[project.urls] -Documentation = "https://resoto.com" -Source = "https://github.com/someengineering/resoto/tree/main/plugins/protector" - -[build-system] -requires = ["setuptools>=67.8.0", "wheel>=0.40.0", "build>=0.10.0"] -build-backend = "setuptools.build_meta" - - diff --git a/plugins/protector/resoto_plugin_protector/__init__.py b/plugins/protector/resoto_plugin_protector/__init__.py deleted file mode 100644 index 44770e4d9d..0000000000 --- a/plugins/protector/resoto_plugin_protector/__init__.py +++ /dev/null @@ -1,48 +0,0 @@ -from copy import deepcopy -from resotolib.logger import log -from resotolib.core.search import CoreGraph -from resotolib.baseplugin import BaseActionPlugin -from resotolib.core.model_export import node_from_dict -from resotolib.config import Config -from .config import ProtectorConfig -from typing import Dict - - -class ProtectorPlugin(BaseActionPlugin): - action = "post_collect" - - def bootstrap(self) -> bool: - return Config.plugin_protector.enabled - - def do_action(self, data: Dict) -> None: - log.info("Protector called") - Config.plugin_protector.validate(Config.plugin_protector) - self.config = deepcopy(Config.plugin_protector.config) - - cg = CoreGraph(tls_data=self.tls_data) - resource_parts = [] - for cloud_id, accounts in self.config.items(): - for account_id, regions in accounts.items(): - for region_id, kinds in regions.items(): - for kind, resources in kinds.items(): - for resource_id in resources: - log.debug( - f"Protecting {resource_id} of kind {kind} in" - f" region {region_id} account {account_id}" - f" cloud {cloud_id}" - ) - resource_parts.append( - f'(/reported.id == "{resource_id}"' - f' and /reported.kind == "{kind}"' - f' and /ancestors.region.reported.id == "{region_id}"' - f' and /ancestors.cloud.reported.id == "{cloud_id}")' - ) - resource_part = " or ".join(resource_parts) - command = f"search {resource_part} | protect" - for node_data in cg.execute(command): - node = node_from_dict(node_data) - log.debug(f"Protected {node.rtdname}") - - @staticmethod - def add_config(config: Config) -> None: - config.add_config(ProtectorConfig) diff --git a/plugins/protector/resoto_plugin_protector/config.py b/plugins/protector/resoto_plugin_protector/config.py deleted file mode 100644 index 0c372f65f0..0000000000 --- a/plugins/protector/resoto_plugin_protector/config.py +++ /dev/null @@ -1,69 +0,0 @@ -from attrs import define, field -from typing import ClassVar, Dict, List - - -default_config = { - "example": { - "Example Account": { - "us-west": {"example_instance": ["someInstance1"]}, - }, - }, -} - - -@define -class ProtectorConfig: - kind: ClassVar[str] = "plugin_protector" - enabled: bool = field( - default=False, - metadata={"description": "Enable plugin?", "restart_required": True}, - ) - config: Dict[str, Dict[str, Dict[str, Dict[str, List[str]]]]] = field( - factory=lambda: default_config, - metadata={ - "description": ( - "Configuration for the plugin\n" - "Format:\n" - " cloud.id:\n" - " account.id:\n" - " region.id:\n" - " kind:\n" - " - resource.id" - ) - }, - ) - - @staticmethod - def validate(cfg: "ProtectorConfig") -> bool: - config = cfg.config - if not isinstance(config, dict): - raise ValueError("Config is no dict") - - for cloud_id, account_data in config.items(): - if not isinstance(cloud_id, str): - raise ValueError(f"Cloud ID {cloud_id} is no string") - if not isinstance(account_data, dict): - raise ValueError(f"Account Data {account_data} is no dict") - - for account_id, region_data in account_data.items(): - if not isinstance(account_id, str): - raise ValueError(f"Account ID {account_id} is no string") - if not isinstance(region_data, dict): - raise ValueError(f"Region Data {region_data} is no dict") - - for region_id, resource_data in region_data.items(): - if not isinstance(region_id, str): - raise ValueError(f"Region ID {region_id} is no string") - if not isinstance(resource_data, dict): - raise ValueError(f"Resource Data {resource_data} is no dict") - - for kind, resource_list in resource_data.items(): - if not isinstance(kind, str): - raise ValueError(f"Resource Kind {kind} is no string") - if not isinstance(resource_list, list): - raise ValueError(f"Resource List {resource_list} is no list") - - for resource_id in resource_list: - if not isinstance(resource_id, str): - raise ValueError(f"Resource ID {resource_id} is no string") - return True diff --git a/plugins/protector/setup.cfg b/plugins/protector/setup.cfg deleted file mode 100644 index 7ca6537935..0000000000 --- a/plugins/protector/setup.cfg +++ /dev/null @@ -1,7 +0,0 @@ -[options] -packages = find: -include_package_data = True -zip_safe = False - -[aliases] -test=pytest diff --git a/plugins/protector/test/test_config.py b/plugins/protector/test/test_config.py deleted file mode 100644 index 4639e00d43..0000000000 --- a/plugins/protector/test/test_config.py +++ /dev/null @@ -1,10 +0,0 @@ -from resotolib.config import Config -from resoto_plugin_protector import ProtectorPlugin - - -def test_config(): - config = Config("dummy", "dummy") - ProtectorPlugin.add_config(config) - Config.init_default_config() - assert Config.plugin_protector.enabled is False - assert Config.plugin_protector.validate(Config.plugin_protector) is True diff --git a/plugins/protector/tox.ini b/plugins/protector/tox.ini deleted file mode 100644 index 8a8e381cef..0000000000 --- a/plugins/protector/tox.ini +++ /dev/null @@ -1,29 +0,0 @@ -[tox] -env_list = syntax, tests, black - -[flake8] -max-line-length=120 -exclude = .git,.tox,__pycache__,.idea,.pytest_cache -ignore=F403, F405, E722, N806, N813, E266, W503, E203 - -[pytest] -addopts= --cov=resoto_plugin_protect_snowflakes -rs -vv --cov-report html -testpaths= test - -[testenv] -usedevelop = true -deps = - --editable=file:///{toxinidir}/../../resotolib - -r../../requirements-all.txt -# until this is fixed: https://github.com/pypa/setuptools/issues/3518 -setenv = - SETUPTOOLS_ENABLE_FEATURES = legacy-editable - -[testenv:syntax] -commands = flake8 --verbose - -[testenv:tests] -commands= pytest - -[testenv:black] -commands = black --line-length 120 --check --diff --target-version py39 . diff --git a/plugins/tagvalidator/MANIFEST.in b/plugins/tagvalidator/MANIFEST.in deleted file mode 100644 index bb3ec5f0d4..0000000000 --- a/plugins/tagvalidator/MANIFEST.in +++ /dev/null @@ -1 +0,0 @@ -include README.md diff --git a/plugins/tagvalidator/README.md b/plugins/tagvalidator/README.md deleted file mode 100644 index 7a605217f8..0000000000 --- a/plugins/tagvalidator/README.md +++ /dev/null @@ -1,53 +0,0 @@ -# resoto-plugin-tagvalidator -Tag Validator plugin for Resoto - -This plugin validates the contents of expiration tags. With it you can enforce a max. expiration length for certain resources in an account. For instance you could have an org policy that says in our "dev" account compute instances are only allowed to exist for 2 days max. Then this plugin can ensure that the expiration tag on those instances is set to no more than 2 days. If it is set to e.g. 50h it would be corrected down to 48h. - -## Usage - -In `resh` execute - -``` -> config edit resoto.worker -``` - -and find the following section - -``` -plugin_tagvalidator: - # Configuration for the plugin - # See https://github.com/someengineering/resoto/tree/main/plugins/tagvalidator for syntax details - config: - default: - expiration: '24h' - kinds: - - 'aws_ec2_instance' - - 'aws_vpc' - - 'aws_cloudformation_stack' - - 'aws_elb' - - 'aws_alb' - - 'aws_alb_target_group' - - 'aws_eks_cluster' - - 'aws_eks_nodegroup' - - 'aws_ec2_nat_gateway' - accounts: - aws: - '123465706934': - name: 'eng-audit' - '123479172032': - name: 'eng-devprod' - '123453451782': - name: 'sales-lead-gen' - expiration: '12h' - '123415487488': - name: 'sales-hosted-lead-gen' - expiration: '8d' - # Dry run - dry_run: false - # Enable plugin? - enabled: false -``` - -## Structure of the config section - -The config contains a default section with the expiration that should be used for all accounts by default. The kinds section contains the list of kinds that these expiration tag rules apply to. The accounts section contain the cloud ids followed by the account ids. Each account id must contain a `name` and optionally an `expiration` that overwrites the global default. diff --git a/plugins/tagvalidator/example.yaml b/plugins/tagvalidator/example.yaml deleted file mode 100644 index 7a4bab7483..0000000000 --- a/plugins/tagvalidator/example.yaml +++ /dev/null @@ -1,26 +0,0 @@ -default: - expiration: 24h - -kinds: - - aws_ec2_instance - - aws_vpc - - aws_cloudformation_stack - - aws_elb - - aws_alb - - aws_alb_target_group - - aws_eks_cluster - - aws_eks_nodegroup - - aws_ec2_nat_gateway - -accounts: - aws: - '123465706934': - name: 'eng-audit' - '123479172032': - name: 'eng-devprod' - '123453451782': - name: 'sales-lead-gen' - expiration: 12h - '123415487488': - name: 'sales-hosted-lead-gen' - expiration: 8d diff --git a/plugins/tagvalidator/pyproject.toml b/plugins/tagvalidator/pyproject.toml deleted file mode 100644 index 0b3211f447..0000000000 --- a/plugins/tagvalidator/pyproject.toml +++ /dev/null @@ -1,44 +0,0 @@ -[project] -name = "resoto-plugin-tagvalidator" -description = "Resoto Tag Validator Plugin" -version = "3.7.0" -authors = [{name="Some Engineering Inc."}] -license = {file="LICENSE"} -requires-python = ">=3.9" -classifiers = [ - # Current project status - "Development Status :: 4 - Beta", - # Audience - "Intended Audience :: System Administrators", - "Intended Audience :: Information Technology", - # License information - "License :: OSI Approved :: Apache Software License", - # Supported python versions - "Programming Language :: Python :: 3.9", - # Supported OS's - "Operating System :: POSIX :: Linux", - "Operating System :: Unix", - # Extra metadata - "Environment :: Console", - "Natural Language :: English", - "Topic :: Security", - "Topic :: Utilities", -] -readme = {file="README.md", content-type="text/markdown"} - -dependencies = [ - "resotolib==3.7.0", -] - -[project.entry-points."resoto.plugins"] -tagvalidator = "resoto_plugin_tagvalidator:TagValidatorPlugin" - -[project.urls] -Documentation = "https://resoto.com" -Source = "https://github.com/someengineering/resoto/tree/main/plugins/tagvalidator" - -[build-system] -requires = ["setuptools>=67.8.0", "wheel>=0.40.0", "build>=0.10.0"] -build-backend = "setuptools.build_meta" - - diff --git a/plugins/tagvalidator/resoto_plugin_tagvalidator/__init__.py b/plugins/tagvalidator/resoto_plugin_tagvalidator/__init__.py deleted file mode 100644 index 225177213a..0000000000 --- a/plugins/tagvalidator/resoto_plugin_tagvalidator/__init__.py +++ /dev/null @@ -1,99 +0,0 @@ -from copy import deepcopy -from typing import Optional, Any - -from resotolib.baseplugin import BaseActionPlugin -from resotolib.baseresources import BaseResource -from resotolib.config import Config -from resotolib.core.search import CoreGraph -from resotolib.durations import parse_duration -from resotolib.graph import Graph -from resotolib.json import value_in_path -from resotolib.logger import log -from resotolib.types import Json -from .config import TagValidatorConfig - - -class TagValidatorPlugin(BaseActionPlugin): - action = "pre_cleanup_plan" - - def __init__(self, *args: Any, **kwargs: Any) -> None: - super().__init__(*args, **kwargs) - - def bootstrap(self) -> bool: - return Config.plugin_tagvalidator.enabled is True - - @staticmethod - def invalid_expiration(cfg: Json, graph: Graph, node: BaseResource, default_expiration: str) -> Optional[str]: - """ - Return the expiration value to set on the node if it is invalid, or None if it is valid. - """ - cloud = node.cloud(graph) - account = node.account(graph) - region = node.region(graph) - max_expiration_str: str = ( - value_in_path(cfg, ["accounts", cloud.id, account.id, "expiration"]) or default_expiration - ) - - max_expiration = parse_duration(max_expiration_str) - node_expiration_str = node.tags.get("expiration") - # If the node has no expiration tag - it is fine - if node_expiration_str is None: - return None - try: - node_expiration = parse_duration(node_expiration_str) - except Exception: - log_msg = f"Invalid expiration tag value {node_expiration_str}" f" - updating tag to {max_expiration_str}" - node.log(log_msg) - log.error(f"{log_msg} on {node.rtdname} in {cloud.rtdname}" f" {account.rtdname} {region.rtdname}") - return max_expiration_str - else: - if max_expiration < node_expiration: - log_msg = ( - f"Current expiration tag value {node_expiration_str} is larger" - f" than {max_expiration_str} - updating tag" - ) - node.log(log_msg) - log.error(f"{log_msg} on {node.rtdname}") - return max_expiration_str - return None - - def do_action(self, data: Json) -> None: - log.info("Tag Validator called") - Config.plugin_tagvalidator.validate(Config.plugin_tagvalidator) - cfg: Json = deepcopy(Config.plugin_tagvalidator.config) - cg = CoreGraph(tls_data=self.tls_data) - query_tag = "tagvalidate" - exclusion_part = "metadata.protected == false and metadata.phantom == false and metadata.cleaned == false" - tags_part = "has_key(reported.tags, expiration)" - kinds_part = 'reported.kind in ["' + '", "'.join(cfg["kinds"]) + '"]' - - account_parts = [] - for cloud_id, account in cfg["accounts"].items(): - for account_id in account.keys(): - account_part = ( - f'(metadata.ancestors.cloud.id == "{cloud_id}" and ' - f'metadata.ancestors.account.id == "{account_id}")' - ) - account_parts.append(account_part) - accounts_part = "(" + " or ".join(account_parts) + ")" - query = f"{exclusion_part} and {kinds_part} and {tags_part} and {accounts_part} #{query_tag} <-[0:]-" - - default_expiration: str = value_in_path(cfg, ["default", "expiration"]) or "24h" - graph = cg.graph(query) - commands = [] - for node in graph.nodes: - if node.protected or node._resotocore_query_tag != query_tag: - continue - if max_expiration := self.invalid_expiration(cfg, graph, node, default_expiration): - commands.append(f"query id({node._resotocore_id}) | tag update --nowait expiration {max_expiration}") - cg.patch_nodes(graph) - for command in commands: - if Config.plugin_tagvalidator.dry_run: - log.debug(f"Tag validator dry run - not executing: {command}") - continue - for response in cg.execute(command): - log.debug(f"Response: {response}") - - @staticmethod - def add_config(config: Config) -> None: - config.add_config(TagValidatorConfig) diff --git a/plugins/tagvalidator/resoto_plugin_tagvalidator/config.py b/plugins/tagvalidator/resoto_plugin_tagvalidator/config.py deleted file mode 100644 index 055921dd7d..0000000000 --- a/plugins/tagvalidator/resoto_plugin_tagvalidator/config.py +++ /dev/null @@ -1,77 +0,0 @@ -from typing import ClassVar - -from attrs import define, field - -from resotolib.json import value_in_path -from resotolib.types import Json - -default_config: Json = { - "default": {"expiration": "24h"}, - "kinds": [ - "aws_ec2_instance", - "aws_vpc", - "aws_cloudformation_stack", - "aws_elb", - "aws_alb", - "aws_alb_target_group", - "aws_eks_cluster", - "aws_eks_nodegroup", - "aws_ec2_nat_gateway", - ], - "accounts": { - "aws": { - "123465706934": {"name": "eng-audit"}, - "123479172032": {"name": "eng-devprod"}, - "123453451782": {"name": "sales-lead-gen", "expiration": "12h"}, - "123415487488": {"name": "sales-hosted-lead-gen", "expiration": "8d"}, - }, - }, -} - - -@define -class TagValidatorConfig: - kind: ClassVar[str] = "plugin_tagvalidator" - enabled: bool = field( - default=False, - metadata={"description": "Enable plugin?", "restart_required": True}, - ) - dry_run: bool = field( - default=False, - metadata={"description": "Dry run"}, - ) - config: Json = field( - factory=lambda: default_config, - metadata={ - "description": ( - "Configuration for the plugin\n" - "See https://github.com/someengineering/resoto/tree/main/plugins/tagvalidator for syntax details" - ) - }, - ) - - @staticmethod - def validate(cfg: "TagValidatorConfig") -> bool: - config = cfg.config - required_sections = ["kinds", "accounts"] - for section in required_sections: - if section not in config: - raise ValueError(f"Section '{section}' not found in config") - - if not isinstance(config["kinds"], list) or len(config["kinds"]) == 0: - raise ValueError("Error in 'kinds' section") - - if not isinstance(config["accounts"], dict) or len(config["accounts"]) == 0: - raise ValueError("Error in 'accounts' section") - - maybe_default_expiration = value_in_path(config, ["default", "expiration"]) - for cloud_id, account in config["accounts"].items(): - for account_id, account_data in account.items(): - if "name" not in account_data: - raise ValueError(f"Missing 'name' for account '{cloud_id}/{account_id}") - if account_data.get("expiration") is None and maybe_default_expiration is None: - raise ValueError( - f"Missing 'expiration' for account '{cloud_id}/{account_id}'" - "and no default expiration defined" - ) - return True diff --git a/plugins/tagvalidator/setup.cfg b/plugins/tagvalidator/setup.cfg deleted file mode 100644 index 7ca6537935..0000000000 --- a/plugins/tagvalidator/setup.cfg +++ /dev/null @@ -1,7 +0,0 @@ -[options] -packages = find: -include_package_data = True -zip_safe = False - -[aliases] -test=pytest diff --git a/plugins/tagvalidator/test/test_config.py b/plugins/tagvalidator/test/test_config.py deleted file mode 100644 index 84d605c8f8..0000000000 --- a/plugins/tagvalidator/test/test_config.py +++ /dev/null @@ -1,60 +0,0 @@ -from datetime import timedelta - -from resotolib.baseresources import Cloud, BaseRegion, BaseAccount, BaseInstance -from resotolib.config import Config -from resoto_plugin_tagvalidator import TagValidatorPlugin -from resotolib.graph import Graph -from resotolib.types import Json - - -def test_config() -> None: - config = Config("dummy", "dummy") - TagValidatorPlugin.add_config(config) - Config.init_default_config() - assert Config.plugin_tagvalidator.enabled is False - assert Config.plugin_tagvalidator.dry_run is False - assert Config.plugin_tagvalidator.validate(Config.plugin_tagvalidator) is True - - -class TestAccount(BaseAccount): - def delete(self, graph: Graph) -> bool: - return False - - -class TestRegion(BaseRegion): - def delete(self, graph: Graph) -> bool: - return False - - -class TestInstance(BaseInstance): - def delete(self, graph: Graph) -> bool: - return False - - -class WorkerConfig: - def __init__(self) -> None: - self.timeout = timedelta(seconds=60) - - -def test_invalid() -> None: - worker_config = Config("dummy", "dummy") - Config.running_config.data["resotoworker"] = WorkerConfig() - TagValidatorPlugin.add_config(worker_config) - Config.init_default_config() - cfg: Json = Config.plugin_tagvalidator.config - plugin = TagValidatorPlugin() - - graph = Graph() - cloud = Cloud(id="aws") - region = TestRegion(id="eu-central-1", cloud=cloud) - account_eng = TestAccount(id="123465706934") - account_sales = TestAccount(id="123453451782") - - ok = TestInstance(id="i-1", cloud=cloud, account=account_sales, region=region, tags={"expiration": "4h"}) - si = TestInstance(id="i-1", cloud=cloud, account=account_sales, region=region, tags={"expiration": "4d"}) - ei = TestInstance(id="i-1", cloud=cloud, account=account_eng, region=region) - wr = TestInstance(id="i-1", cloud=cloud, account=account_sales, region=region, tags={"expiration": "smthg"}) - assert plugin.invalid_expiration(cfg, graph, ok, "24h") is None # expiration is ok - assert plugin.invalid_expiration(cfg, graph, ei, "24h") is None # no expiration tag - assert plugin.invalid_expiration(cfg, graph, si, "24h") == "12h" # expiration is too long - assert plugin.invalid_expiration(cfg, graph, wr, "24h") == "12h" # expiration can not be parsed diff --git a/plugins/tagvalidator/tox.ini b/plugins/tagvalidator/tox.ini deleted file mode 100644 index 6d61548573..0000000000 --- a/plugins/tagvalidator/tox.ini +++ /dev/null @@ -1,31 +0,0 @@ -[tox] -env_list = syntax, tests, black, mypy - -[flake8] -max-line-length=120 -exclude = .git,.tox,__pycache__,.idea,.pytest_cache -ignore=F403, F405, E722, N806, N813, E266, W503, E203 - -[pytest] -testpaths= test - -[testenv] -usedevelop = true -deps = - --editable=file:///{toxinidir}/../../resotolib - -r../../requirements-all.txt -# until this is fixed: https://github.com/pypa/setuptools/issues/3518 -setenv = - SETUPTOOLS_ENABLE_FEATURES = legacy-editable - -[testenv:syntax] -commands = flake8 --verbose - -[testenv:tests] -commands= pytest - -[testenv:black] -commands = black --line-length 120 --check --diff --target-version py39 . - -[testenv:mypy] -commands= python -m mypy --install-types --non-interactive --python-version 3.9 --strict resoto_plugin_tagvalidator test diff --git a/setup_venv.sh b/setup_venv.sh index 627e6f10c3..f8e7355479 100755 --- a/setup_venv.sh +++ b/setup_venv.sh @@ -180,7 +180,7 @@ install_resoto() { } install_plugins() { - local collector_plugins=(aws azure cleanup_aws_alarms cleanup_aws_loadbalancers cleanup_aws_vpcs cleanup_expired cleanup_untagged cleanup_volumes digitalocean dockerhub example_collector gcp github k8s onelogin onprem posthog protector random scarf slack tagvalidator vsphere) + local collector_plugins=(aws azure digitalocean dockerhub example_collector gcp github k8s onelogin onprem posthog random scarf slack vsphere) for plugin in "${collector_plugins[@]}"; do pip_install "$plugin" true done