From a00832557f93bf686b9ef354156a742cf436da25 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lukas=20L=C3=B6sche?= Date: Thu, 17 Oct 2024 09:21:32 +0200 Subject: [PATCH] [plugins/onprem][feat] Move onprem plugin to dedicated repository (#2251) --- .github/workflows/check_pr_plugin_onprem.yml | 75 ---------- .github/workflows/model_check.yml | 4 +- Dockerfile.fixinventorybase | 1 + plugins/onprem/MANIFEST.in | 1 - plugins/onprem/README.md | 5 - plugins/onprem/fix_plugin_onprem/__init__.py | 133 ----------------- plugins/onprem/fix_plugin_onprem/config.py | 18 --- plugins/onprem/fix_plugin_onprem/resources.py | 62 -------- plugins/onprem/fix_plugin_onprem/ssh.py | 140 ------------------ plugins/onprem/pyproject.toml | 45 ------ plugins/onprem/setup.cfg | 7 - plugins/onprem/test/test_config.py | 15 -- plugins/onprem/tox.ini | 29 ---- requirements-all.txt | 1 - requirements-extra.txt | 1 - requirements.txt | 1 - setup_venv.sh | 2 +- 17 files changed, 4 insertions(+), 536 deletions(-) delete mode 100644 .github/workflows/check_pr_plugin_onprem.yml delete mode 100644 plugins/onprem/MANIFEST.in delete mode 100644 plugins/onprem/README.md delete mode 100644 plugins/onprem/fix_plugin_onprem/__init__.py delete mode 100644 plugins/onprem/fix_plugin_onprem/config.py delete mode 100644 plugins/onprem/fix_plugin_onprem/resources.py delete mode 100644 plugins/onprem/fix_plugin_onprem/ssh.py delete mode 100644 plugins/onprem/pyproject.toml delete mode 100644 plugins/onprem/setup.cfg delete mode 100644 plugins/onprem/test/test_config.py delete mode 100644 plugins/onprem/tox.ini diff --git a/.github/workflows/check_pr_plugin_onprem.yml b/.github/workflows/check_pr_plugin_onprem.yml deleted file mode 100644 index 2715e4782d..0000000000 --- a/.github/workflows/check_pr_plugin_onprem.yml +++ /dev/null @@ -1,75 +0,0 @@ -# Note: this workflow is automatically generated via the `create_pr` script in the same folder. -# Please do not change the file, but the script! - -name: Check PR (Plugin onprem) -on: - push: - tags: - - "*.*.*" - branches: - - main - pull_request: - paths: - - 'fixlib/**' - - 'plugins/onprem/**' - - '.github/**' - - 'requirements-all.txt' - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }} - cancel-in-progress: true - -jobs: - onprem: - name: "onprem" - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Setup Python - uses: actions/setup-python@v5 - with: - python-version: '3.12' - architecture: 'x64' - - - name: Restore dependency cache - uses: actions/cache@v4 - with: - path: ~/.cache/pip - key: ${{runner.os}}-pip-${{hashFiles('./plugins/onprem/pyproject.toml')}} - restore-keys: | - ${{runner.os}}-pip- - - - name: Install Dependencies - run: | - python -m pip install --upgrade pip - pip install --upgrade --editable fixlib/ - pip install tox wheel flake8 build - - - name: Run tests - working-directory: ./plugins/onprem - run: tox - - - name: Archive code coverage results - uses: actions/upload-artifact@v4 - with: - name: plugin-onprem-code-coverage-report - path: ./plugins/onprem/htmlcov/ - - - name: Build a binary wheel and a source tarball - working-directory: ./plugins/onprem - run: >- - python -m - build - --sdist - --wheel - --outdir dist/ - - - name: Publish distribution to PyPI - if: github.ref_type == 'tag' - uses: pypa/gh-action-pypi-publish@release/v1 - with: - user: __token__ - password: ${{ secrets.PYPI_FIXINVENTORY_PLUGIN_ONPREM }} - packages_dir: ./plugins/onprem/dist/ diff --git a/.github/workflows/model_check.yml b/.github/workflows/model_check.yml index 3fbcdbece9..650f62dcf5 100644 --- a/.github/workflows/model_check.yml +++ b/.github/workflows/model_check.yml @@ -15,7 +15,6 @@ on: - 'plugins/github/**' - 'plugins/k8s/**' - 'plugins/onelogin/**' - - 'plugins/onprem/**' - 'plugins/slack/**' - '.github/**' - 'requirements-all.txt' @@ -52,8 +51,9 @@ jobs: run: | python -m pip install --upgrade pip python -m pip install -r requirements-all.txt - pip install fixlib/ plugins/aws/ plugins/azure/ plugins/digitalocean/ plugins/dockerhub/ plugins/example_collector/ plugins/gcp/ plugins/github/ plugins/k8s/ plugins/onelogin/ plugins/onprem/ plugins/posthog/ plugins/random/ plugins/scarf/ plugins/slack/ + pip install fixlib/ plugins/aws/ plugins/azure/ plugins/digitalocean/ plugins/dockerhub/ plugins/example_collector/ plugins/gcp/ plugins/github/ plugins/k8s/ plugins/onelogin/ plugins/posthog/ plugins/random/ plugins/scarf/ plugins/slack/ pip install fixinventory-plugin-vsphere + pip install fixinventory-plugin-onprem - name: Run tests working-directory: ./fixlib diff --git a/Dockerfile.fixinventorybase b/Dockerfile.fixinventorybase index d091226128..a86deaba8b 100644 --- a/Dockerfile.fixinventorybase +++ b/Dockerfile.fixinventorybase @@ -57,6 +57,7 @@ WORKDIR /usr/src RUN . /usr/local/fix-venv-python3/bin/activate && pip install -r requirements-extra.txt RUN . /usr/local/fix-venv-python3/bin/activate && find plugins/ -maxdepth 1 -mindepth 1 -type d -print0 | xargs -0 python -m pip install ./fixlib ./fixcore ./fixworker ./fixmetrics ./fixshell RUN . /usr/local/fix-venv-python3/bin/activate && python -m pip install fixinventory-plugin-vsphere +RUN . /usr/local/fix-venv-python3/bin/activate && python -m pip install fixinventory-plugin-onprem # Install AWS CLI WORKDIR /usr/src diff --git a/plugins/onprem/MANIFEST.in b/plugins/onprem/MANIFEST.in deleted file mode 100644 index bb3ec5f0d4..0000000000 --- a/plugins/onprem/MANIFEST.in +++ /dev/null @@ -1 +0,0 @@ -include README.md diff --git a/plugins/onprem/README.md b/plugins/onprem/README.md deleted file mode 100644 index dcaeec824f..0000000000 --- a/plugins/onprem/README.md +++ /dev/null @@ -1,5 +0,0 @@ -# fix-plugin-onprem (WIP) -On-Premises Collector Plugin for Fix - -## License -See [LICENSE](../../LICENSE) for details. diff --git a/plugins/onprem/fix_plugin_onprem/__init__.py b/plugins/onprem/fix_plugin_onprem/__init__.py deleted file mode 100644 index 8ab47b5e78..0000000000 --- a/plugins/onprem/fix_plugin_onprem/__init__.py +++ /dev/null @@ -1,133 +0,0 @@ -from fixlib.baseresources import BaseResource -import socket -import multiprocessing -import fixlib.proc -from concurrent import futures -from fixlib.baseplugin import BaseCollectorPlugin -from argparse import Namespace -from fixlib.args import ArgumentParser -from fixlib.config import Config, RunningConfig -from .resources import OnpremLocation, OnpremRegion, OnpremNetwork -from .ssh import instance_from_ssh -from .config import OnpremConfig -from paramiko import ssh_exception -from typing import Dict -from fixlib.logger import log, setup_logger - - -class OnpremCollectorPlugin(BaseCollectorPlugin): - cloud = "onprem" - - def collect(self) -> None: - log.debug("plugin: collecting on-prem resources") - - if len(Config.onprem.server) == 0: - log.debug("No On-Prem servers specified") - return - - default_location = OnpremLocation(id=Config.onprem.location) - self.graph.add_resource(self.graph.root, default_location) - - default_region = OnpremRegion(id=Config.onprem.region) - self.graph.add_resource(default_location, default_region) - - servers = [] - for server in Config.onprem.server: - location = region = network = None - srv = {} - if "%" in server: - server_location, server = server.split("%", 1) - location = self.graph.search_first_all({"id": server_location, "kind": "onprem_location"}) - if location is None: - location = OnpremLocation(id=server_location, tags={}) - self.graph.add_resource(self.graph.root, location) - srv.update({"location": location}) - log.debug(f"Location for {server} is {location.rtdname}") - if "%" in server: - server_region, server = server.split("%", 1) - region = self.graph.search_first_all({"id": server_region, "kind": "onprem_region"}) - if region is None: - region = OnpremRegion(id=server_region, tags={}) - self.graph.add_resource(location, region) - srv.update({"region": region}) - log.debug(f"Region for {server} is {region.rtdname}") - if "%" in server: - server_network, server = server.split("%", 1) - network = self.graph.search_first_all({"id": server_network, "kind": "onprem_network"}) - if network is None: - network = OnpremNetwork(id=server_network, tags={}) - self.graph.add_resource(region, network) - srv.update({"network": network}) - log.debug(f"Network for {server} is {network.rtdname}") - srv.update({"hostname": server}) - servers.append(srv) - - max_workers = len(servers) if len(servers) < Config.onprem.pool_size else Config.onprem.pool_size - pool_args = {"max_workers": max_workers} - if Config.onprem.fork_process: - pool_args["mp_context"] = multiprocessing.get_context("spawn") - pool_args["initializer"] = fixlib.proc.collector_initializer - pool_executor = futures.ProcessPoolExecutor - collect_args = { - "args": ArgumentParser.args, - "running_config": Config.running_config, - } - else: - pool_executor = futures.ThreadPoolExecutor - collect_args = {} - - with pool_executor(**pool_args) as executor: - wait_for = [ - executor.submit( - collect_server, - srv, - **collect_args, - ) - for srv in servers - ] - for future in futures.as_completed(wait_for): - (src, s) = future.result() - if src is None: - src = default_region - if not isinstance(src, BaseResource) or not isinstance(s, BaseResource): - log.error(f"Skipping invalid server {type(s)}") - continue - self.graph.add_resource(src, s) - - @staticmethod - def add_config(config: Config) -> None: - config.add_config(OnpremConfig) - - -def collect_server(srv: Dict, args: Namespace = None, running_config: RunningConfig = None) -> Dict: - if args is not None: - ArgumentParser.args = args - setup_logger("fixworker-onprem", force=True, level=getattr(args, "log_level", None)) - - if running_config is not None: - Config.running_config.apply(running_config) - - hostname: str = srv.get("hostname") - username = None - port = 22 - if "@" in hostname: - username, hostname = hostname.split("@", 1) - if ":" in hostname: - hostname, port = hostname.split(":", 1) - - collector_name = f"onprem_{hostname}" - fixlib.proc.set_thread_name(collector_name) - try: - s = instance_from_ssh( - hostname, - username=username, - port=port, - key_filename=Config.onprem.ssh_key, - passphrase=Config.onprem.ssh_key_pass, - ) - src = srv.get("network", srv.get("region", srv.get("location", None))) - except (socket.timeout, ssh_exception.PasswordRequiredException): - log.exception(f"Failed to collect {hostname}") - else: - log.debug(f"onprem: collected {s.rtdname}") - return (src, s) diff --git a/plugins/onprem/fix_plugin_onprem/config.py b/plugins/onprem/fix_plugin_onprem/config.py deleted file mode 100644 index 244a25a7d9..0000000000 --- a/plugins/onprem/fix_plugin_onprem/config.py +++ /dev/null @@ -1,18 +0,0 @@ -from attrs import define, field -from typing import List, ClassVar, Optional - - -@define -class OnpremConfig: - kind: ClassVar[str] = "onprem" - location: str = field(default="Default location", metadata={"description": "Default location"}) - region: str = field(default="Default region", metadata={"description": "Default region"}) - ssh_user: str = field(default="root", metadata={"description": "SSH user"}) - ssh_key: Optional[str] = field(default=None, metadata={"description": "SSH key"}) - ssh_key_pass: Optional[str] = field(default=None, metadata={"description": "SSH key passphrase"}) - server: List[str] = field(factory=list, metadata={"description": "Server(s)"}) - pool_size: int = field(default=5, metadata={"description": "Thread/process pool size"}) - fork_process: bool = field( - default=True, - metadata={"description": "Fork collector process instead of using threads"}, - ) diff --git a/plugins/onprem/fix_plugin_onprem/resources.py b/plugins/onprem/fix_plugin_onprem/resources.py deleted file mode 100644 index fc2354567d..0000000000 --- a/plugins/onprem/fix_plugin_onprem/resources.py +++ /dev/null @@ -1,62 +0,0 @@ -from attrs import define -from typing import Optional, ClassVar -from fixlib.graph import Graph -from fixlib.baseresources import ( - BaseAccount, - BaseRegion, - BaseInstance, - BaseNetwork, -) - - -@define(eq=False, slots=False) -class OnpremLocation(BaseAccount): - kind: ClassVar[str] = "onprem_location" - _kind_display: ClassVar[str] = "Onprem Location" - _kind_description: ClassVar[str] = "An Onprem Location." - - def delete(self, graph: Graph) -> bool: - return False - - -@define(eq=False, slots=False) -class OnpremRegion(BaseRegion): - kind: ClassVar[str] = "onprem_region" - _kind_display: ClassVar[str] = "Onprem Region" - _kind_description: ClassVar[str] = "An Onprem Region." - - def delete(self, graph: Graph) -> bool: - return False - - -@define(eq=False, slots=False) -class OnpremResource: - kind: ClassVar[str] = "onprem_resource" - kind_display: ClassVar[str] = "Onprem Resource" - kind_description: ClassVar[str] = "An Onprem Resource." - - def delete(self, graph: Graph) -> bool: - return False - - def update_tag(self, key, value) -> bool: - return False - - def delete_tag(self, key) -> bool: - return False - - -@define(eq=False, slots=False) -class OnpremInstance(OnpremResource, BaseInstance): - kind: ClassVar[str] = "onprem_instance" - _kind_display: ClassVar[str] = "Onprem Instance" - _kind_description: ClassVar[str] = "An Onprem Instance." - network_device: Optional[str] = None - network_ip4: Optional[str] = None - network_ip6: Optional[str] = None - - -@define(eq=False, slots=False) -class OnpremNetwork(OnpremResource, BaseNetwork): - kind: ClassVar[str] = "onprem_network" - _kind_display: ClassVar[str] = "Onprem Network" - _kind_description: ClassVar[str] = "An Onprem Network." diff --git a/plugins/onprem/fix_plugin_onprem/ssh.py b/plugins/onprem/fix_plugin_onprem/ssh.py deleted file mode 100644 index 132954c269..0000000000 --- a/plugins/onprem/fix_plugin_onprem/ssh.py +++ /dev/null @@ -1,140 +0,0 @@ -from collections import defaultdict -from paramiko import SSHClient -from .resources import OnpremInstance -from fixlib.baseresources import InstanceStatus -from fixlib.logger import log - - -instance_status_map = { - "running": InstanceStatus.RUNNING, -} - - -def instance_from_ssh( - hostname: str, - port: int = 22, - username: str = None, - password: str = None, - pkey: str = None, - key_filename: str = None, - auth_timeout: float = 10, - timeout: float = 10, - allow_agent: bool = True, - look_for_keys: bool = True, - passphrase: str = None, -): - log.debug(f"Establishing SSH connection to {hostname}") - client = SSHClient() - client.load_system_host_keys() - client.connect( - hostname, - port=port, - username=username, - password=password, - pkey=pkey, - passphrase=passphrase, - key_filename=key_filename, - timeout=timeout, - auth_timeout=auth_timeout, - allow_agent=allow_agent, - look_for_keys=look_for_keys, - ) - meminfo = get_proc_meminfo(client) - cpuinfo = get_proc_cpuinfo(client) - netdev, ip4, ip6 = get_net_info(client) - client.close() - - s = OnpremInstance( - id=hostname, - instance_cores=len(cpuinfo), - instance_memory=round(meminfo.get("MemTotal", 0) / 1024**2), - instance_status=instance_status_map.get("running", InstanceStatus.UNKNOWN), - instance_type=cpuinfo.get("0", {}).get("model name"), - network_device=netdev, - network_ip4=ip4, - network_ip6=ip6, - ) - return s - - -def get_proc_meminfo(client: SSHClient): - log.debug("Getting memory information") - cmd = "cat /proc/meminfo" - out, err = client_exec(client, cmd) - if err: - raise RuntimeError(f"Error while executing {cmd}: {err}") - meminfo = {i[0].rstrip(":"): int(i[1]) for i in [line.split() for line in str(out).splitlines()]} - return meminfo - - -def get_proc_cpuinfo(client: SSHClient): - log.debug("Getting CPU information") - cmd = "cat /proc/cpuinfo" - out, err = client_exec(client, cmd) - if err: - raise RuntimeError(f"Error while executing {cmd}: {err}") - cpuinfo = defaultdict(dict) - num_core = "0" - for line in str(out).splitlines(): - if len(line) == 0: - continue - k, v = line.split(":", 1) - k = k.strip() - v = v.strip() - if k == "processor": - num_core = v - else: - cpuinfo[num_core][k] = v - return dict(cpuinfo) - - -def get_net_info(client: SSHClient): - log.debug("Getting network information") - dst4 = "8.8.8.8" - dst6 = "2001:4860:4860::8888" - ip4 = None - ip6 = None - dev = None - for dst in [dst4, dst6]: - cmd = f"ip r g {dst}" - out, err = client_exec(client, cmd) - if err: - log.error(f"Error while executing {cmd}: {err}") - continue - src = None - for line in str(out).splitlines(): - line = line.strip() - if line.startswith(dst) and "dev" in line and "src" in line: - line = line.split() - dev = line[line.index("dev") + 1] - src = line[line.index("src") + 1] - break - if dev is None or src is None: - raise RuntimeError("Unable to determine IP interface") - cmd = f"ip a s dev {dev}" - out, err = client_exec(client, cmd) - if err: - raise RuntimeError(f"Error while executing {cmd}: {err}") - ip = None - for line in str(out).splitlines(): - line = line.strip() - if line.startswith("inet") and src in line: - line = line.split() - ip = line[1] - break - if ip is None: - raise RuntimeError("Unable to determine IP address") - if "." in ip: - ip4 = ip - elif ":" in ip: - ip6 = ip - else: - raise RuntimeError(f"Unable to parse IP {ip}") - return (dev, ip4, ip6) - - -def client_exec(client: SSHClient, command: str, timeout: float = None): - _, stdout, stderr = client.exec_command(command, timeout=timeout) - out = stdout.read().decode().strip() - err = stderr.read().decode().strip() - return (out, err) diff --git a/plugins/onprem/pyproject.toml b/plugins/onprem/pyproject.toml deleted file mode 100644 index 6c37f716aa..0000000000 --- a/plugins/onprem/pyproject.toml +++ /dev/null @@ -1,45 +0,0 @@ -[project] -name = "fixinventory-plugin-onprem" -description = "Fix On-Premises Collector Plugin" -version = "4.1.0" -authors = [{name="Some Engineering Inc."}] -license = { text="AGPLv3" } -requires-python = ">=3.12" -classifiers = [ - # Current project status - "Development Status :: 4 - Beta", - # Audience - "Intended Audience :: System Administrators", - "Intended Audience :: Information Technology", - # License information - "License :: OSI Approved :: GNU Affero General Public License v3 or later (AGPLv3+)", - # Supported python versions - "Programming Language :: Python :: 3.12", - # Supported OS's - "Operating System :: POSIX :: Linux", - "Operating System :: Unix", - # Extra metadata - "Environment :: Console", - "Natural Language :: English", - "Topic :: Security", - "Topic :: Utilities", -] -readme = {file="README.md", content-type="text/markdown"} - -dependencies = [ - "fixinventorylib==4.1.0", - "paramiko", -] - -[project.entry-points."fix.plugins"] -onprem = "fix_plugin_onprem:OnpremCollectorPlugin" - -[project.urls] -Documentation = "https://inventory.fix.security" -Source = "https://github.com/someengineering/fix/tree/main/plugins/onelogin" - -[build-system] -requires = ["setuptools>=67.8.0", "wheel>=0.40.0", "build>=0.10.0"] -build-backend = "setuptools.build_meta" - - diff --git a/plugins/onprem/setup.cfg b/plugins/onprem/setup.cfg deleted file mode 100644 index 7ca6537935..0000000000 --- a/plugins/onprem/setup.cfg +++ /dev/null @@ -1,7 +0,0 @@ -[options] -packages = find: -include_package_data = True -zip_safe = False - -[aliases] -test=pytest diff --git a/plugins/onprem/test/test_config.py b/plugins/onprem/test/test_config.py deleted file mode 100644 index 47446e59c2..0000000000 --- a/plugins/onprem/test/test_config.py +++ /dev/null @@ -1,15 +0,0 @@ -from fixlib.config import Config -from fix_plugin_onprem import OnpremCollectorPlugin - - -def test_config(): - config = Config("dummy", "dummy") - OnpremCollectorPlugin.add_config(config) - config.init_default_config() - assert Config.onprem.location == "Default location" - assert Config.onprem.region == "Default region" - assert Config.onprem.ssh_user == "root" - assert Config.onprem.ssh_key is None - assert len(Config.onprem.server) == 0 - assert Config.onprem.pool_size == 5 - assert Config.onprem.fork_process is True diff --git a/plugins/onprem/tox.ini b/plugins/onprem/tox.ini deleted file mode 100644 index 40ef7754d5..0000000000 --- a/plugins/onprem/tox.ini +++ /dev/null @@ -1,29 +0,0 @@ -[tox] -env_list = syntax, tests, black - -[flake8] -max-line-length=120 -exclude = .git,.tox,__pycache__,.idea,.pytest_cache -ignore=F403, F405, E722, N806, N813, E266, W503, E203 - -[pytest] -addopts= --cov=fix_plugin_onprem -rs -vv --cov-report html -testpaths= test - -[testenv] -usedevelop = true -deps = - --editable=file:///{toxinidir}/../../fixlib - -r../../requirements-all.txt -# until this is fixed: https://github.com/pypa/setuptools/issues/3518 -setenv = - SETUPTOOLS_ENABLE_FEATURES = legacy-editable - -[testenv:syntax] -commands = flake8 --verbose - -[testenv:tests] -commands= pytest - -[testenv:black] -commands = black --line-length 120 --check --diff --target-version py39 . diff --git a/requirements-all.txt b/requirements-all.txt index 816998fd8f..2fa88676c1 100644 --- a/requirements-all.txt +++ b/requirements-all.txt @@ -106,7 +106,6 @@ onelogin==2.0.4 orderly-set==5.2.2 orjson==3.10.7 packaging==24.1 -paramiko==3.5.0 parsy==2.1 pathspec==0.12.1 pep8-naming==0.14.1 diff --git a/requirements-extra.txt b/requirements-extra.txt index 7c465a1a0c..9bc0650895 100644 --- a/requirements-extra.txt +++ b/requirements-extra.txt @@ -91,7 +91,6 @@ onelogin==2.0.4 orderly-set==5.2.2 orjson==3.10.7 packaging==24.1 -paramiko==3.5.0 parsy==2.1 pint==0.24.3 plantuml==0.3.0 diff --git a/requirements.txt b/requirements.txt index 265b09fa3e..c50cf492ff 100644 --- a/requirements.txt +++ b/requirements.txt @@ -84,7 +84,6 @@ onelogin==2.0.4 orderly-set==5.2.2 orjson==3.10.7 packaging==24.1 -paramiko==3.5.0 parsy==2.1 pint==0.24.3 plantuml==0.3.0 diff --git a/setup_venv.sh b/setup_venv.sh index 94cf0cf9ac..b00df7853c 100755 --- a/setup_venv.sh +++ b/setup_venv.sh @@ -180,7 +180,7 @@ install_fixinventory() { } install_plugins() { - local collector_plugins=(aws azure digitalocean dockerhub example_collector gcp github k8s onelogin onprem posthog random scarf slack) + local collector_plugins=(aws azure digitalocean dockerhub example_collector gcp github k8s onelogin posthog random scarf slack) for plugin in "${collector_plugins[@]}"; do pip_install "$plugin" true done