From 2bb57ae7eb7a8550ae35f70f8f8b5e03a892a773 Mon Sep 17 00:00:00 2001 From: JesseBot Date: Fri, 25 Nov 2022 17:03:28 +0100 Subject: [PATCH] add linux gaming pkg mngr func, fix logging to file, fix packages installed, and apt list cmd (#94) * adding gaming specific workflow * use sudo to edit /etc/apt/sources.list and only catch end of line with sed * remove wine, because lutris installs wine now * adding new script for running sed with sudo, and removing old include files from poetry * make subproc prettier in debug * adding comments for python and adding back ruff for auto linting * comments for auto linting * changing where gaming commands get called to avoid duplicate apt updates * fix required_pkg_groups variable to be available_pkg_groups to be more clear * fix comments to be more readable for console * turn off spinner so that we get a sudo prompt * fixing debug output to always use markup * fixing logging to file * fix apt list command to be apt-cache pkgnames * ticking up version because a few things changed and got updated * fix rich output for when we do log directly to console * remove broken release action --- .github/workflows/python-publish.yml | 32 -------------- onboardme/__init__.py | 57 +++++++++++++------------ onboardme/config/packages.yml | 14 +++--- onboardme/env_config.py | 10 ++--- onboardme/pkg_management.py | 44 +++++++++++++------ onboardme/scripts/update_apt_sources.sh | 5 +++ onboardme/subproc.py | 5 ++- pyproject.toml | 9 ++-- 8 files changed, 85 insertions(+), 91 deletions(-) create mode 100755 onboardme/scripts/update_apt_sources.sh diff --git a/.github/workflows/python-publish.yml b/.github/workflows/python-publish.yml index 72e8e3df..db4efd80 100644 --- a/.github/workflows/python-publish.yml +++ b/.github/workflows/python-publish.yml @@ -14,35 +14,3 @@ jobs: with: python_version: "3.11.0" pypi_token: ${{ secrets.DEPLOY_FROM_GITHUB_TO_PYPI_ONBOARDME }} - release: - name: Create Release - runs-on: ubuntu-latest - steps: - - name: Checkout code - uses: actions/checkout@v2 - with: - fetch-depth: 0 - - - name: Get Previous Tag - run: | - PREV_TAG=$(git describe --abbrev=0 --tags "${{ github.ref }}^") - echo "::set-env name=baseRef::$PREV_TAG" - - - name: Generate Changelog - id: generate_changelog - uses: nblagoev/pull-release-notes-action@v1.0.3 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # This token is provided by Actions, you do not need to create your own token - with: - base-ref: ${{ env.baseRef }} - head-ref: ${{ github.ref }} - - - name: Create Release - uses: actions/create-release@v1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - tag_name: ${{ github.ref }} - release_name: Release ${{ github.ref }} - body: ${{steps.generate_changelog.outputs.result}} - draft: false diff --git a/onboardme/__init__.py b/onboardme/__init__.py index 615968d8..696eea6b 100755 --- a/onboardme/__init__.py +++ b/onboardme/__init__.py @@ -2,7 +2,7 @@ NAME: onboardme DESCRIPTION: Program to take care of a bunch of onboarding tasks for new machines running macOS and/or Debian, including: - dot_files, packages, ide_setup, and group management + dot_files, managing packages, ide_setup, group management AUTHOR: Jesse Hitch LICENSE: GNU AFFERO GENERAL PUBLIC LICENSE """ @@ -23,14 +23,6 @@ from .firewall import configure_firewall -# for importing modules by str names -# for getting the version of onboardme - -# rich helps pretty print everything - -# custom libs - - HELP = options_help() @@ -39,35 +31,44 @@ def setup_logger(level="", log_file=""): Sets up rich logger and stores the values for it in a db for future import in other files. Returns logging.getLogger("rich") """ + # determine logging level if not level: if USR_CONFIG_FILE and 'log' in USR_CONFIG_FILE: level = USR_CONFIG_FILE['log']['level'] else: level = 'warn' + log_level = getattr(logging, level.upper(), None) + + # these are params to be passed into logging.basicConfig + opts = {'level': log_level, 'format': "%(message)s", 'datefmt': "[%X]"} + + # we only log to a file if one was passed into config.yaml or the cli if not log_file: if USR_CONFIG_FILE: log_file = USR_CONFIG_FILE['log'].get('file', None) - log_level = getattr(logging, level.upper(), None) - # these are params to be passed into logging.basicConfig - log_opts = {'level': log_level, - 'format': "%(message)s", - 'datefmt': "[%X]", - 'handlers': [RichHandler(rich_tracebacks=True)]} + # rich typically handles much of this but we don't use rich with files + if log_file: + opts['filename'] = log_file + opts['format'] = "%(asctime)s %(levelname)s %(funcName)s: %(message)s" + else: + rich_handler_opts = {'rich_tracebacks': True} + # 10 is the DEBUG logging level int value + if log_level == 10: + # log the name of the function if we're in debug mode :) + opts['format'] = "[bold]%(funcName)s()[/bold]: %(message)s" + rich_handler_opts['markup'] = True - if log_level == 10: - # log the name of the function if we're in debug mode :) - log_opts['format'] = "[bold]%(funcName)s()[/bold]: %(message)s" + opts['handlers'] = [RichHandler(**rich_handler_opts)] - # we only log to a file if one was passed into config.yaml or the cli - if log_file: - log_opts['handlers'] = [RichHandler(console=Console(file=log_file), - rich_tracebacks=True)] + # this uses the opts dictionary as parameters to logging.basicConfig() + logging.basicConfig(**opts) - # this uses the log_opts dictionary as parameters to logging.basicConfig() - logging.basicConfig(**log_opts) - return logging.getLogger("rich") + if log_file: + return None + else: + return logging.getLogger("rich") # @option('--quiet', '-q', is_flag=True, help=HELP['quiet']) @@ -126,8 +127,10 @@ def main(log_level: str = "", usr_pref = process_configs(overwrite, git_url, git_branch, pkg_managers, pkg_groups, firewall, remote_host, steps) - log.debug(f"User passed in the following preferences:\n{usr_pref}\n", - extra={"markup": True}) + if log: + log.debug(f"User passed in the following preferences:\n{usr_pref}\n") + else: + logging.debug(f"User passed in the following preferences:\n{usr_pref}") # actual heavy lifting of onboardme happens in these for step in usr_pref['steps'][OS[0]]: diff --git a/onboardme/config/packages.yml b/onboardme/config/packages.yml index ae049dab..d180c809 100644 --- a/onboardme/config/packages.yml +++ b/onboardme/config/packages.yml @@ -58,7 +58,7 @@ brew: - gh # gitlab cli - glab - # making things + # for setup scripts and compiling things (e.g. YouCompleteMe) - cmake # programming languages and their package managers - python@3.11 @@ -151,7 +151,7 @@ brew: apt: emoji: "🙃" commands: - list: "apt list --installed" + list: "apt-cache pkgnames" update: "sudo apt-get update -y" upgrade: "sudo apt-get upgrade -y" install: "sudo apt-get install -y " @@ -192,8 +192,8 @@ apt: gaming: # helpful for gaming on linux - lutris - - winehq-staging - - steam + - "steam:i386" + - "nvidia-driver-libs:i386" # to format disks to exFAT; FAT is too thin for modern windows 10 ISOs # - exfat-utils @@ -236,17 +236,19 @@ pip3.11: install: "pip3.11 install --upgrade " packages: default: - # this is for python development + # this is for python development - specifically linting/auto-linting - flake8 - pyflakes + - ruff - autoflake + # for building and installing packages - pip - build - twine - poetry # this does some magic with imports when developing - autoimport - # this is all powerline + # this is all powerline (status line for tmux/BASH) - "powerline-status" - "powerline-gitstatus" - "powerline-kubernetes" diff --git a/onboardme/env_config.py b/onboardme/env_config.py index 6bd0e41b..14d18fff 100755 --- a/onboardme/env_config.py +++ b/onboardme/env_config.py @@ -152,21 +152,19 @@ def process_configs(overwrite=False, repo="", git_branch="", pkg_mngrs=[], 'dot_files': {'overwrite': overwrite, 'git_url': repo, 'git_branch': git_branch}} - log.debug(f"cli_dict is:\n{cli_dict}\n", extra={"markup": True}) + log.debug(f"cli_dict is:\n{cli_dict}\n") if USR_CONFIG_FILE: - log.debug(f"🗂 ⚙️ user_config_file: \n{USR_CONFIG_FILE}\n", - extra={"markup": True}) + log.debug(f"🗂 ⚙️ user_config_file: \n{USR_CONFIG_FILE}\n") usr_cfgs = fill_in_defaults(DEFAULTS, USR_CONFIG_FILE) - log.debug("after user_config_file filled in with defaults: " + - f"{usr_cfgs}", extra={"markup": True}) + log.debug(f"after usr config file filled in with defaults: {usr_cfgs}") final_defaults = fill_in_defaults(cli_dict, usr_cfgs, True) else: final_defaults = fill_in_defaults(DEFAULTS, cli_dict) log.debug(" final config after filling cli_dict in with defaults:\n" - f"{final_defaults}\n", extra={"markup": True}) + f"{final_defaults}\n") valid_steps = process_steps(final_defaults['steps'][OS[0]], final_defaults['remote_hosts']) diff --git a/onboardme/pkg_management.py b/onboardme/pkg_management.py index 83d8f7cd..8db8cc1e 100755 --- a/onboardme/pkg_management.py +++ b/onboardme/pkg_management.py @@ -24,33 +24,38 @@ def run_pkg_mngrs(pkg_mngrs=[], pkg_groups=[]): default_config = path.join(PWD, 'config/packages.yml') pkg_mngrs_list_of_dicts = load_cfg(default_config) - log.debug(f"pkg_mngrs: {pkg_mngrs}", extra={"markup": True}) - log.debug(f"pkg_groups: {pkg_groups}", extra={"markup": True}) + log.debug(f"pkg_mngrs: {pkg_mngrs}") + log.debug(f"pkg_groups: {pkg_groups}") # we iterate through pkg_mngrs which should already be sorted for pkg_mngr in pkg_mngrs: pkg_mngr_dict = pkg_mngrs_list_of_dicts[pkg_mngr] - required_pkg_groups = pkg_mngr_dict['packages'] + available_pkg_groups = pkg_mngr_dict['packages'] # brew has a special flow because it works on both linux and mac if pkg_mngr == 'brew': if 'Darwin' in OS: pkg_groups.append("macOS") - debug_line = f"pkg groups for {pkg_mngr} are {required_pkg_groups}" - log.debug(debug_line, extra={"markup": True}) + debug_line = f"pkg groups for {pkg_mngr} are {available_pkg_groups}" + log.debug(debug_line) # make sure that the package manage has any groups that were passed in - if any(check in pkg_groups for check in required_pkg_groups): + if any(check in pkg_groups for check in available_pkg_groups): pkg_emoji = pkg_mngr_dict['emoji'] msg = f'{pkg_emoji} [green][b]{pkg_mngr}[/b][/] app Installs' print_header(msg) - # run package manager specific setup if needed, & updates/upgrades pkg_cmds = pkg_mngr_dict['commands'] - log.debug(f"package manager commands are: {pkg_cmds}") + log.debug(f"{pkg_mngr} pre-install commands are: {pkg_cmds}") + + # gaming has a special flow that needs to be done before updates + if "gaming" in pkg_groups and pkg_mngr == "apt": + run_gaming_specific_cmds() + + # run package manager specific setup if needed, & updates/upgrades for pre_cmd in ['setup', 'update', 'upgrade']: if pre_cmd in pkg_cmds: SPINNER = True @@ -60,12 +65,13 @@ def run_pkg_mngrs(pkg_mngrs=[], pkg_groups=[]): sub_header(f"[b]{pre_cmd.title()}[/b] completed.") # list of actually installed packages - installed_pkgs = subproc([pkg_cmds['list']], quiet=True) + installed_pkgs = subproc([pkg_cmds['list']]) for pkg_group in pkg_groups: - if pkg_group in required_pkg_groups: + if pkg_group in available_pkg_groups: + install_pkg_group(installed_pkgs, - required_pkg_groups[pkg_group], + available_pkg_groups[pkg_group], pkg_cmds['install']) sub_header(f'{pkg_group.title()} packages installed.') @@ -89,12 +95,24 @@ def install_pkg_group(installed_pkgs=[], pkgs_to_install=[], install_cmd=""): if 'upgrade' in install_cmd or not installed_pkgs: install_pkg = True - log.debug(f"pkgs_to_install are {pkgs_to_install}", - extra={"markup": True}) + log.debug(f"pkgs_to_install are {pkgs_to_install}") + for pkg in pkgs_to_install: if installed_pkgs: if pkg not in installed_pkgs: + log.info(f"{pkg} not in installed packages. Installing...") install_pkg = True if install_pkg: subproc([install_cmd + pkg], quiet=True, spinner=SPINNER) return True + + +def run_gaming_specific_cmds(): + """ + run commands specific to gaming package group: + add i386 architecture, add contrib/non-free to sources.list, and update + """ + cmds = ["sudo dpkg --add-architecture i386", + f"sudo {PWD}/scripts/update_apt_sources.sh"] + subproc(cmds, spinner=False) + return True diff --git a/onboardme/scripts/update_apt_sources.sh b/onboardme/scripts/update_apt_sources.sh new file mode 100755 index 00000000..d600f39a --- /dev/null +++ b/onboardme/scripts/update_apt_sources.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +# enables contribution and non-free apt package manager sources + +# append "contrib non-free" to lines that end with "bookworm main" +sed -i 's/bookworm main$/bookworm main contrib non-free/g' /etc/apt/sources.list diff --git a/onboardme/subproc.py b/onboardme/subproc.py index 14546d93..c7ff57ed 100644 --- a/onboardme/subproc.py +++ b/onboardme/subproc.py @@ -28,7 +28,7 @@ def subproc(commands=[], **kwargs): quiet = kwargs.pop('quiet', False) if spinner: - # we don't actually need this if we're not doing a progress spinner + # only need this if we're doing a progress spinner console = Console() status_line = "[bold green]♥ Running command[/bold green]" @@ -63,6 +63,7 @@ def run_subprocess(command, **kwargs): env - environment variables you'd like to pass in """ # subprocess expects a list if there are spaces in the command + log.debug(command) cmd = command.split() error_ok = False @@ -81,7 +82,7 @@ def run_subprocess(command, **kwargs): # check return code, raise error if failure if not ret_code: if res_stderr: - log.debug(res_stderr, extra={"markup": True}) + log.debug(res_stderr) else: if ret_code != 0: # also scan both stdout and stdin for weird errors diff --git a/pyproject.toml b/pyproject.toml index 687605e3..46c0ed2f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "onboardme" -version = "0.15.4" +version = "0.15.6" description = "An onboarding tool to install dot files and packages including a default mode with sensible defaults to run on most Debian/macOS machines." authors = ["Jesse Hitch "] license = "AGPL-3.0-or-later" @@ -16,10 +16,9 @@ classifiers = ["Development Status :: 3 - Alpha", "Topic :: System :: Installation/Setup", "License :: OSI Approved :: GNU Affero General Public License v3 or later (AGPLv3+)"] packages = [{include = "onboardme"}] -include = ["onboardme/config/onboardme_config.yml", - "onboardme/config/packages.yml", - "onboardme/config/brew/Brewfile_Darwin", - "onboardme/config/brew/Brewfile_devops"] +include = ["onboardme/scripts/update_apt_sources.sh", + "onboardme/config/onboardme_config.yml", + "onboardme/config/packages.yml"] [tool.poetry.dependencies] python = "^3.11"