Merge pull request #3 from shaneknapp/fixing-some-bugs

resolving some issues/etc unearthed in the demo/walkthrough

.github/workflows/linters.yaml

@@ -0,0 +1,23 @@
+name: Lint and spell check
+on:
+  pull_request:
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+        - uses: actions/checkout@v4
+
+        - name: Install Python
+          uses: actions/setup-python@v5
+          with:
+            python-version: "3.11"
+        - name: Install dependencies
+          run: |
+            python -m pip install --upgrade pip
+            pip install -r requirements.txt
+        # Update output format to enable automatic inline annotations.
+        - name: Ruff Check
+          run: ruff check --output-format=github .
+
+        - uses: codespell-project/actions-codespell@v2

README.md

@@ -1,4 +1,4 @@
-# Tooling to help mange user image repos
+# Tooling to help manage user image repos
 
 To install, clone this repo and from inside the directory run:
 

manage_repos/__main__.py

@@ -1,7 +1,30 @@
 import argparse
+import re
+import sys
+
 from . import manage_repos
 
 
+def check_config(config):
+    """
+    Check all entries in the config file to confirm they are in the proper
+    format:
+
+    [email protected]:<user or org>/<repo name>.git
+    """
+    with open(config) as f:
+        for line in f:
+            line = line.strip()
+            if not line or line.startswith("#"):
+                continue
+
+            if not re.match(
+                "^[email protected]:[a-zA-Z0-9\.\-\_]+/[a-zA-Z0-9\.\-\_]+\.git$", line
+            ):
+                print(f"Malformed entry in {config}: {line}. Exiting.")
+                sys.exit(1)
+
+
 def main():
     argparser = argparse.ArgumentParser()
     subparsers = argparser.add_subparsers(
@@ -13,14 +36,15 @@ def main():
         "-c",
         "--config",
         default="repos.txt",
-        help="Path to file containing list of repositories to operate on.",
+        help="Path to the file containing list of repositories to operate on. "
+        + "Defaults to repos.txt located in the current working directory.",
     )
     argparser.add_argument(
         "-d",
         "--destination",
         default=".",
-        help="Location on the filesystem of the managed repositories. If "
-        + "the directory does not exist, it will be created.",
+        help="Location on the filesystem of the directory containing the "
+        + "managed repositories. Defaults to the current working directory.",
     )
 
     branch_parser = subparsers.add_parser(
@@ -36,25 +60,21 @@ def main():
     clone_parser = subparsers.add_parser(
         "clone",
         description="Clone repositories in the config file and "
-        + "optionally set a remote for a fork.",
+        + "optionally set a remote for a fork. If a repository sub-directory "
+        + "does not exist, it will be created.",
     )
     clone_parser.add_argument(
         "-s",
         "--set-remote",
-        action="store_true",
-        help="Set the user's GitHub fork as a remote.",
-    )
-    clone_parser.add_argument(
-        "-r",
-        "--remote",
-        default="origin",
-        help="If --set-remote is used, override the name of the remote to "
-        + "set for the fork. This is optional and defaults to 'origin'.",
+        const="origin",
+        nargs="?",
+        help="Set the user's GitHub fork as a remote. Defaults to 'origin'.",
     )
     clone_parser.add_argument(
         "-g",
         "--github-user",
-        help="The GitHub username of the fork to set in the remote.",
+        help="The GitHub username of the fork to set in the remote. Required "
+        + "if --set-remote is used.",
     )
 
     patch_parser = subparsers.add_parser(
@@ -86,7 +106,8 @@ def main():
         "--files",
         nargs="+",
         default=["."],
-        help="List of files to stage in the repositories.",
+        help="Space-delimited list of files to stage in the repositories. "
+        + "Optional, and if left blank will default to all modified files.",
     )
     stage_parser.add_argument(
         "-m",
@@ -128,8 +149,10 @@ def main():
     )
 
     args = argparser.parse_args()
-
     print(args)
+
+    check_config(args.config)
+
     errors = []
     if args.command == "branch":
         errors.append(manage_repos.branch(args))
@@ -147,8 +170,7 @@ def main():
         argparser.print_help()
 
     if any(errors):
-        print("The following errors occurred during execution:")
-        print(errors)
+        print("\nThe following errors occurred during execution:")
         for error in errors:
             for e in error:
                 print(e)

manage_repos/manage_repos.py

@@ -1,10 +1,5 @@
-"""
-General tool for mass cloning and syncing of user image repositories.
-
-To use this tool, copy it to a directory in your PATH or run it directly from
-this directory.
-"""
-
+import re
+import shutil
 import subprocess
 import sys
 import os
@@ -23,9 +18,16 @@ def _iter_repos(args):
                 continue
             name = line.split("/")[-1].replace(".git", "")
             path = os.path.join(args.destination, name)
-            if not os.path.exists(path):
-                if args.command != "clone":
-                    print(f"Skipping {name} as it doesn't exist in {args.destination}")
+
+            # clone behavior is a little different than other commands when
+            # checking for existing repos
+            if args.command == "clone":
+                if os.path.exists(path):
+                    print(f"Skipping {name} as it already exists in {path}")
+                    continue
+            else:
+                if not os.path.exists(path):
+                    print(f"Skipping {name} as it doesn't exist at {path}")
                     continue
             yield name, path, line
 
@@ -60,23 +62,19 @@ def clone(args):
     Optionally set the the user's GitHub fork as a remote, which defaults to
     'origin'.
     """
+    if args.set_remote and args.github_user is None:
+        print(
+            "Remote cannot be updated, please specify a GitHub username "
+            + "for the fork to continue."
+        )
+        sys.exit(1)
+
     if not os.path.exists(args.destination):
         os.makedirs(args.destination)
         print(f"Created destination directory {args.destination}")
 
     errors = []
     for name, path, repo in _iter_repos(args):
-        if os.path.exists(path):
-            print(f"Skipping {name} as it already exists.")
-            next
-
-        if args.remote and not args.github_user:
-            print(
-                "Remote cannot be updated, please specify a GitHub username "
-                + "for the fork to continue."
-            )
-            sys.exit(1)
-
         print(f"Cloning {name} from {repo} to {path}.")
         try:
             subprocess.check_call(["git", "clone", repo, path])
@@ -102,11 +100,12 @@ def clone(args):
                 print()
                 continue
 
-            remote = repo.replace("berkeley-dsep-infra", args.github_user)
-            print(f"Setting remote of fork to: {remote}")
+            original_remote = re.search(".+:(.+?)/.*$", repo).group(1)
+            repo = repo.replace(original_remote, args.github_user)
+            print(f"Setting remote of fork to: {repo}")
             try:
                 subprocess.check_call(
-                    ["git", "remote", "add", args.remote, remote], cwd=path
+                    ["git", "remote", "add", args.set_remote, repo], cwd=path
                 )
             except subprocess.CalledProcessError as e:
                 error = f"Error setting remote in {name} in {path}: {e}"
@@ -134,17 +133,21 @@ def patch(args):
         print(f"Patch file {args.patch} does not exist.")
         sys.exit(1)
 
+    type(args.patch)
     errors = []
     for name, path, _ in _iter_repos(args):
         print(f"Applying patch to {name} in {path}")
         try:
+            shutil.copy(args.patch, path)
             subprocess.check_call(["git", "apply", args.patch], cwd=path)
         except subprocess.CalledProcessError as e:
-            error = f"Error applying patch {patch} to {name} in {path}: {e}"
+            error = f"Error applying patch {args.patch} in {path}: {e}"
             print(error)
             errors.append(error)
             print()
             continue
+
+        os.remove(os.path.join(path, args.patch))
         print()
 
     if errors is not None:
@@ -182,6 +185,11 @@ def stage(args):
     committing them.
     """
     errors = []
+
+    if not args.message:
+        print("Please provide a commit message via the --message argument.")
+        sys.exit(1)
+
     for name, path, repo in _iter_repos(args):
         for file in args.files:
             if file == ".":
@@ -234,10 +242,20 @@ def sync(args):
     for name, path, repo in _iter_repos(args):
         print(f"Syncing {name} from {repo} to {path}.")
         subprocess.check_call(["git", "switch", args.branch_default], cwd=path)
-        subprocess.check_call(["git", "fetch", "--all", "--prune"], cwd=path)
+
+        try:
+            subprocess.check_call(["git", "fetch", "--all", "--prune"], cwd=path)
+        except subprocess.CalledProcessError as e:
+            error = f"Error fetching {name}: {e}"
+            error += f"\nPlease check to see if your fork of of {name} exists."
+            print(error)
+            errors.append(error)
+            print()
+            continue
+
         try:
             subprocess.check_call(
-                ["git", "rebase", "upstream/" + args.branch_default], cwd=path
+                ["git", "rebase", "--stat", "upstream/" + args.branch_default], cwd=path
             )
         except subprocess.CalledProcessError as e:
             error = f"Error rebasing {name} to {path}: {e}"

pyproject.toml

@@ -20,3 +20,6 @@ Repository = "https://github.com/berkeley-dsep-infra/manage-repos.git"
 
 [project.scripts]
 manage-repos = "manage_repos.__main__:main"
+
+[tool.codespell]
+skip = '.git,*.pdf,*.svg,*.egg-info*,*/__pycache__/*'

requirements.txt

@@ -0,0 +1,2 @@
+# these are currently only used for github actions and linting
+ruff==0.7.0