add linux gaming pkg mngr func, fix logging to file, fix packages ins…

…talled, and apt list cmd (#94)

* adding gaming specific workflow

* use sudo to edit /etc/apt/sources.list and only catch end of line with sed

* remove wine, because lutris installs wine now

* adding new script for running sed with sudo, and removing old include files from poetry

* make subproc prettier in debug

* adding comments for python and adding back ruff for auto linting

* comments for auto linting

* changing where gaming commands get called to avoid duplicate apt updates

* fix required_pkg_groups variable to be available_pkg_groups to be more clear

* fix comments to be more readable for console

* turn off spinner so that we get a sudo prompt

* fixing debug output to always use markup

* fixing logging to file

* fix apt list command to be apt-cache pkgnames

* ticking up version because a few things changed and got updated

* fix rich output for when we do log directly to console

* remove broken release action

.github/workflows/python-publish.yml

@@ -14,35 +14,3 @@ jobs:
         with:
           python_version: "3.11.0"
           pypi_token: ${{ secrets.DEPLOY_FROM_GITHUB_TO_PYPI_ONBOARDME }}
-  release:
-    name: Create Release
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v2
-        with:
-          fetch-depth: 0
-
-      - name: Get Previous Tag
-        run: |
-          PREV_TAG=$(git describe --abbrev=0 --tags "${{ github.ref }}^")
-          echo "::set-env name=baseRef::$PREV_TAG"
-
-      - name: Generate Changelog
-        id: generate_changelog
-        uses: nblagoev/[email protected]
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # This token is provided by Actions, you do not need to create your own token
-        with:
-          base-ref: ${{ env.baseRef }}
-          head-ref: ${{ github.ref }}
-
-      - name: Create Release
-        uses: actions/create-release@v1
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        with:
-          tag_name: ${{ github.ref }}
-          release_name: Release ${{ github.ref }}
-          body: ${{steps.generate_changelog.outputs.result}}
-          draft: false

onboardme/__init__.py

@@ -2,7 +2,7 @@
     NAME:           onboardme
     DESCRIPTION:    Program to take care of a bunch of onboarding tasks for new
                     machines running macOS and/or Debian, including:
-                      dot_files, packages, ide_setup, and group management
+                      dot_files, managing packages, ide_setup, group management
     AUTHOR:         Jesse Hitch
     LICENSE:        GNU AFFERO GENERAL PUBLIC LICENSE
 """
@@ -23,14 +23,6 @@
 from .firewall import configure_firewall
 
 
-# for importing modules by str names
-# for getting the version of onboardme
-
-# rich helps pretty print everything
-
-# custom libs
-
-
 HELP = options_help()
 
 
@@ -39,35 +31,44 @@ def setup_logger(level="", log_file=""):
     Sets up rich logger and stores the values for it in a db for future import
     in other files. Returns logging.getLogger("rich")
     """
+    # determine logging level
     if not level:
         if USR_CONFIG_FILE and 'log' in USR_CONFIG_FILE:
             level = USR_CONFIG_FILE['log']['level']
         else:
             level = 'warn'
 
+    log_level = getattr(logging, level.upper(), None)
+
+    # these are params to be passed into logging.basicConfig
+    opts = {'level': log_level, 'format': "%(message)s", 'datefmt': "[%X]"}
+
+    # we only log to a file if one was passed into config.yaml or the cli
     if not log_file:
         if USR_CONFIG_FILE:
             log_file = USR_CONFIG_FILE['log'].get('file', None)
 
-    log_level = getattr(logging, level.upper(), None)
-    # these are params to be passed into logging.basicConfig
-    log_opts = {'level': log_level,
-                'format': "%(message)s",
-                'datefmt': "[%X]",
-                'handlers': [RichHandler(rich_tracebacks=True)]}
+    # rich typically handles much of this but we don't use rich with files
+    if log_file:
+        opts['filename'] = log_file
+        opts['format'] = "%(asctime)s %(levelname)s %(funcName)s: %(message)s"
+    else:
+        rich_handler_opts = {'rich_tracebacks': True}
+        # 10 is the DEBUG logging level int value
+        if log_level == 10:
+            # log the name of the function if we're in debug mode :)
+            opts['format'] = "[bold]%(funcName)s()[/bold]: %(message)s"
+            rich_handler_opts['markup'] = True
 
-    if log_level == 10:
-        # log the name of the function if we're in debug mode :)
-        log_opts['format'] = "[bold]%(funcName)s()[/bold]: %(message)s"
+        opts['handlers'] = [RichHandler(**rich_handler_opts)]
 
-    # we only log to a file if one was passed into config.yaml or the cli
-    if log_file:
-        log_opts['handlers'] = [RichHandler(console=Console(file=log_file),
-                                            rich_tracebacks=True)]
+    # this uses the opts dictionary as parameters to logging.basicConfig()
+    logging.basicConfig(**opts)
 
-    # this uses the log_opts dictionary as parameters to logging.basicConfig()
-    logging.basicConfig(**log_opts)
-    return logging.getLogger("rich")
+    if log_file:
+        return None
+    else:
+        return logging.getLogger("rich")
 
 
 # @option('--quiet', '-q', is_flag=True, help=HELP['quiet'])
@@ -126,8 +127,10 @@ def main(log_level: str = "",
     usr_pref = process_configs(overwrite, git_url, git_branch, pkg_managers,
                                pkg_groups, firewall, remote_host, steps)
 
-    log.debug(f"User passed in the following preferences:\n{usr_pref}\n",
-              extra={"markup": True})
+    if log:
+        log.debug(f"User passed in the following preferences:\n{usr_pref}\n")
+    else:
+        logging.debug(f"User passed in the following preferences:\n{usr_pref}")
 
     # actual heavy lifting of onboardme happens in these
     for step in usr_pref['steps'][OS[0]]:

onboardme/config/packages.yml

@@ -58,7 +58,7 @@ brew:
       - gh
       # gitlab cli
       - glab
-      # making things
+      # for setup scripts and compiling things (e.g. YouCompleteMe)
       - cmake
       # programming languages and their package managers
       - [email protected]
@@ -151,7 +151,7 @@ brew:
 apt:
   emoji: "🙃"
   commands:
-    list: "apt list --installed"
+    list: "apt-cache pkgnames"
     update: "sudo apt-get update -y"
     upgrade: "sudo apt-get upgrade -y"
     install: "sudo apt-get install -y "
@@ -192,8 +192,8 @@ apt:
     gaming:
       # helpful for gaming on linux
       - lutris
-      - winehq-staging
-      - steam
+      - "steam:i386"
+      - "nvidia-driver-libs:i386"
       # to format disks to exFAT; FAT is too thin for modern windows 10 ISOs
       # - exfat-utils
 
@@ -236,17 +236,19 @@ pip3.11:
     install: "pip3.11 install --upgrade "
   packages:
     default:
-      # this is for python development
+      # this is for python development - specifically linting/auto-linting
       - flake8
       - pyflakes
+      - ruff
       - autoflake
+      # for building and installing packages
       - pip
       - build
       - twine
       - poetry
       # this does some magic with imports when developing
       - autoimport
-      # this is all powerline
+      # this is all powerline (status line for tmux/BASH)
       - "powerline-status"
       - "powerline-gitstatus"
       - "powerline-kubernetes"

onboardme/env_config.py

@@ -152,21 +152,19 @@ def process_configs(overwrite=False, repo="", git_branch="", pkg_mngrs=[],
                 'dot_files': {'overwrite': overwrite,
                               'git_url': repo, 'git_branch': git_branch}}
 
-    log.debug(f"cli_dict is:\n{cli_dict}\n", extra={"markup": True})
+    log.debug(f"cli_dict is:\n{cli_dict}\n")
     if USR_CONFIG_FILE:
-        log.debug(f"🗂 ⚙️  user_config_file: \n{USR_CONFIG_FILE}\n",
-                  extra={"markup": True})
+        log.debug(f"🗂 ⚙️  user_config_file: \n{USR_CONFIG_FILE}\n")
 
         usr_cfgs = fill_in_defaults(DEFAULTS, USR_CONFIG_FILE)
-        log.debug("after user_config_file filled in with defaults: " +
-                  f"{usr_cfgs}", extra={"markup": True})
+        log.debug(f"after usr config file filled in with defaults: {usr_cfgs}")
 
         final_defaults = fill_in_defaults(cli_dict, usr_cfgs, True)
     else:
         final_defaults = fill_in_defaults(DEFAULTS, cli_dict)
 
     log.debug(" final config after filling cli_dict in with defaults:\n"
-              f"{final_defaults}\n", extra={"markup": True})
+              f"{final_defaults}\n")
 
     valid_steps = process_steps(final_defaults['steps'][OS[0]],
                                 final_defaults['remote_hosts'])

onboardme/pkg_management.py

@@ -24,33 +24,38 @@ def run_pkg_mngrs(pkg_mngrs=[], pkg_groups=[]):
         default_config = path.join(PWD, 'config/packages.yml')
         pkg_mngrs_list_of_dicts = load_cfg(default_config)
 
-    log.debug(f"pkg_mngrs: {pkg_mngrs}", extra={"markup": True})
-    log.debug(f"pkg_groups: {pkg_groups}", extra={"markup": True})
+    log.debug(f"pkg_mngrs: {pkg_mngrs}")
+    log.debug(f"pkg_groups: {pkg_groups}")
 
     # we iterate through pkg_mngrs which should already be sorted
     for pkg_mngr in pkg_mngrs:
 
         pkg_mngr_dict = pkg_mngrs_list_of_dicts[pkg_mngr]
-        required_pkg_groups = pkg_mngr_dict['packages']
+        available_pkg_groups = pkg_mngr_dict['packages']
 
         # brew has a special flow because it works on both linux and mac
         if pkg_mngr == 'brew':
             if 'Darwin' in OS:
                 pkg_groups.append("macOS")
 
-        debug_line = f"pkg groups for {pkg_mngr} are {required_pkg_groups}"
-        log.debug(debug_line, extra={"markup": True})
+        debug_line = f"pkg groups for {pkg_mngr} are {available_pkg_groups}"
+        log.debug(debug_line)
 
         # make sure that the package manage has any groups that were passed in
-        if any(check in pkg_groups for check in required_pkg_groups):
+        if any(check in pkg_groups for check in available_pkg_groups):
 
             pkg_emoji = pkg_mngr_dict['emoji']
             msg = f'{pkg_emoji} [green][b]{pkg_mngr}[/b][/] app Installs'
             print_header(msg)
 
-            # run package manager specific setup if needed, & updates/upgrades
             pkg_cmds = pkg_mngr_dict['commands']
-            log.debug(f"package manager commands are: {pkg_cmds}")
+            log.debug(f"{pkg_mngr} pre-install commands are: {pkg_cmds}")
+
+            # gaming has a special flow that needs to be done before updates
+            if "gaming" in pkg_groups and pkg_mngr == "apt":
+                run_gaming_specific_cmds()
+
+            # run package manager specific setup if needed, & updates/upgrades
             for pre_cmd in ['setup', 'update', 'upgrade']:
                 if pre_cmd in pkg_cmds:
                     SPINNER = True
@@ -60,12 +65,13 @@ def run_pkg_mngrs(pkg_mngrs=[], pkg_groups=[]):
                     sub_header(f"[b]{pre_cmd.title()}[/b] completed.")
 
             # list of actually installed packages
-            installed_pkgs = subproc([pkg_cmds['list']], quiet=True)
+            installed_pkgs = subproc([pkg_cmds['list']])
 
             for pkg_group in pkg_groups:
-                if pkg_group in required_pkg_groups:
+                if pkg_group in available_pkg_groups:
+
                     install_pkg_group(installed_pkgs,
-                                      required_pkg_groups[pkg_group],
+                                      available_pkg_groups[pkg_group],
                                       pkg_cmds['install'])
                     sub_header(f'{pkg_group.title()} packages installed.')
 
@@ -89,12 +95,24 @@ def install_pkg_group(installed_pkgs=[], pkgs_to_install=[], install_cmd=""):
     if 'upgrade' in install_cmd or not installed_pkgs:
         install_pkg = True
 
-    log.debug(f"pkgs_to_install are {pkgs_to_install}",
-              extra={"markup": True})
+    log.debug(f"pkgs_to_install are {pkgs_to_install}")
+
     for pkg in pkgs_to_install:
         if installed_pkgs:
             if pkg not in installed_pkgs:
+                log.info(f"{pkg} not in installed packages. Installing...")
                 install_pkg = True
         if install_pkg:
             subproc([install_cmd + pkg], quiet=True, spinner=SPINNER)
     return True
+
+
+def run_gaming_specific_cmds():
+    """
+    run commands specific to gaming package group:
+      add i386 architecture, add contrib/non-free to sources.list, and update
+    """
+    cmds = ["sudo dpkg --add-architecture i386",
+            f"sudo {PWD}/scripts/update_apt_sources.sh"]
+    subproc(cmds, spinner=False)
+    return True

onboardme/scripts/update_apt_sources.sh

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+# enables contribution and non-free apt package manager sources
+
+# append "contrib non-free" to lines that end with "bookworm main"
+sed -i 's/bookworm main$/bookworm main contrib non-free/g' /etc/apt/sources.list

onboardme/subproc.py

@@ -28,7 +28,7 @@ def subproc(commands=[], **kwargs):
     quiet = kwargs.pop('quiet', False)
 
     if spinner:
-        # we don't actually need this if we're not doing a progress spinner
+        # only need this if we're doing a progress spinner
         console = Console()
 
     status_line = "[bold green]♥ Running command[/bold green]"
@@ -63,6 +63,7 @@ def run_subprocess(command, **kwargs):
         env       - environment variables you'd like to pass in
     """
     # subprocess expects a list if there are spaces in the command
+    log.debug(command)
     cmd = command.split()
 
     error_ok = False
@@ -81,7 +82,7 @@ def run_subprocess(command, **kwargs):
         # check return code, raise error if failure
         if not ret_code:
             if res_stderr:
-                log.debug(res_stderr, extra={"markup": True})
+                log.debug(res_stderr)
         else:
             if ret_code != 0:
                 # also scan both stdout and stdin for weird errors

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name          = "onboardme"
-version       = "0.15.4"
+version       = "0.15.6"
 description   = "An onboarding tool to install dot files and packages including a default mode with sensible defaults to run on most Debian/macOS machines."
 authors       = ["Jesse Hitch <[email protected]>"]
 license       = "AGPL-3.0-or-later"
@@ -16,10 +16,9 @@ classifiers   = ["Development Status :: 3 - Alpha",
                  "Topic :: System :: Installation/Setup",
                  "License :: OSI Approved :: GNU Affero General Public License v3 or later (AGPLv3+)"]
 packages      = [{include = "onboardme"}]
-include       = ["onboardme/config/onboardme_config.yml",
-                 "onboardme/config/packages.yml",
-                 "onboardme/config/brew/Brewfile_Darwin",
-                 "onboardme/config/brew/Brewfile_devops"]
+include       = ["onboardme/scripts/update_apt_sources.sh",
+                 "onboardme/config/onboardme_config.yml",
+                 "onboardme/config/packages.yml"]
 
 [tool.poetry.dependencies]
 python    = "^3.11"