From 2354e598b475ba96088336e4e3f9569dab4e1e0f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lien=20Rainone?= Date: Mon, 7 Nov 2016 15:46:22 +0100 Subject: [PATCH 1/7] Remove references to gittify in setup --- setup.py | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/setup.py b/setup.py index d12884d..e6c0bed 100644 --- a/setup.py +++ b/setup.py @@ -26,20 +26,15 @@ setup( name='git-externals', version=__version__, - description='utility to manage svn externals', - long_description='', + description='Use git to handle your svn:externals', + long_description='ease the migration from Git to SVN by handling svn externals through a cli tool', packages=['git_externals'], install_requires=['click', - 'pathlib', - 'python-gitlab'], + 'git-svn-clone-externals'], entry_points={ 'console_scripts': [ - 'git-externals = git_externals.git_externals:cli', - 'gittify-cleanup = git_externals.cleanup_repo:main', + 'git-externals = git_externals.cli:cli', 'svn-externals-info = git_externals.process_externals:main', - 'gittify = git_externals.gittify:main', - 'gittify-gen = git_externals.makefiles:cli', - 'gittify-gitlab = git_externals.gitlab_utils:main', ], }, author=__author__, From 1b6f0e66ed83384c126966901464bd34e6d754f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lien=20Rainone?= Date: Mon, 7 Nov 2016 15:49:41 +0100 Subject: [PATCH 2/7] remove pycharm additions --- git_externals/pycharm.additions.patch | 38 --------------------------- 1 file changed, 38 deletions(-) delete mode 100644 git_externals/pycharm.additions.patch diff --git a/git_externals/pycharm.additions.patch b/git_externals/pycharm.additions.patch deleted file mode 100644 index 240b691..0000000 --- a/git_externals/pycharm.additions.patch +++ /dev/null @@ -1,38 +0,0 @@ -diff --git a/git_externals/git_externals.py b/git_externals/git_externals.py -index 16225fc..fe06aa1 100755 ---- a/git_externals/git_externals.py -+++ b/git_externals/git_externals.py -@@ -1,9 +1,19 @@ - #!/usr/bin/env python - - from __future__ import print_function, unicode_literals -- --from . import __version__ --from .utils import chdir, mkdir_p, link, rm_link, git, GitError -+# -+# from . import __version__ -+# from .utils import chdir, mkdir_p, link, rm_link, git, GitError -+ -+if __package__ is None: -+ import sys -+ from os import path -+ sys.path.append( path.dirname( path.dirname( path.abspath(__file__) ) ) ) -+ from __init__ import __version__ -+ from utils import chdir, mkdir_p, link, rm_link, git, GitError -+else: -+ from . import __version__ -+ from .utils import chdir, mkdir_p, link, rm_link, git, GitError - - import click - import json -@@ -450,3 +460,10 @@ def enable_colored_output(): - for entry in get_entries(): - with chdir(os.path.join(DEFAULT_DIR, entry)): - git('config', 'color.ui', 'always') -+ -+ -+if __name__ == '__main__': -+ from click.testing import CliRunner -+ runner = CliRunner() -+ result = runner.invoke(cli, ['update']) -+ assert result.exit_code == 0 From d565d1ec55fd6c0904c611cd2557af342121fdd6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lien=20Rainone?= Date: Mon, 7 Nov 2016 15:52:48 +0100 Subject: [PATCH 3/7] Remove gittify only files --- git_externals/cleanup_repo.py | 123 ----- git_externals/gitlab_utils.py | 160 ------ git_externals/gittify.py | 754 ----------------------------- git_externals/process_externals.py | 201 -------- 4 files changed, 1238 deletions(-) delete mode 100755 git_externals/cleanup_repo.py delete mode 100644 git_externals/gitlab_utils.py delete mode 100755 git_externals/gittify.py delete mode 100755 git_externals/process_externals.py diff --git a/git_externals/cleanup_repo.py b/git_externals/cleanup_repo.py deleted file mode 100755 index bd36d2b..0000000 --- a/git_externals/cleanup_repo.py +++ /dev/null @@ -1,123 +0,0 @@ -#!/usr/bin/env python - -import posixpath -import re -import argparse - -import logging -from .utils import svn, git, SVNError, checkout, chdir, tags, git_remote_branches_and_tags - - -def name_of(remote): - return posixpath.basename(remote.strip('/')) - - -def branchtag_to_tag(tag_name, remote_tag): - with checkout(tag_name, remote_tag): - pass - - if tag_name in tags(): - git('tag', '-d', tag_name) - - git('tag', tag_name, tag_name) - git('branch', '-D', tag_name) - - -def svn_remote_branches(repo): - try: - entries = svn('ls', posixpath.join(repo, 'branches')).splitlines() - except SVNError: - return set() - return set([b[:-1] for b in entries]) - - -def svn_remote_tags(repo): - try: - entries = svn('ls', posixpath.join(repo, 'tags')).splitlines() - except SVNError: - return set() - return set([t[:-1] for t in entries]) - - -def cleanup(repo, with_revbound=False, remote=None, log=None): - if log is None: - logging.basicConfig() - log = logging.getLogger(__name__) - log.setLevel(logging.INFO) - - with chdir(repo): - if remote is not None: - remote_branches = svn_remote_branches(remote) - remote_tags = svn_remote_tags(remote) - else: - remote_branches = [] - remote_tags = [] - - repo_name, _ = posixpath.splitext(name_of(repo)) - - branches, tags = git_remote_branches_and_tags() - - revbound_re = re.compile(r'.+@(\d+)') - - log.info('Cleaning up branches...') - for branch in branches: - branch_name = name_of(branch) - - match = revbound_re.match(branch_name) - is_revbound = match is not None - rev = match.group(1) if is_revbound else None - - # trunk is automatically remapped to master by git svn - if branch_name in ('trunk', 'git-svn'): - continue - - if not with_revbound and is_revbound: - log.warning('Skipping cleanup of {} because it is bound to rev {}' - .format(branch_name, rev)) - continue - - if branch_name not in remote_branches and not branch_name.startswith(repo_name): - log.warning('Skipping cleanup of {} because it has been deleted (maybe after a merge?)' - .format(branch_name)) - continue - - log.info('Cleaning up branch {}'.format(branch_name)) - with checkout(branch_name, branch): - pass - - log.info('Cleaning up tags') - for tag in tags: - match = revbound_re.match(tag) - is_revbound = match is not None - rev = match.group(1) if is_revbound else None - - tag_name = name_of(tag) - - if tag_name not in remote_tags: - log.warning('Skipping tag {} because it has been deleted' - .format(tag)) - continue - - if not with_revbound and is_revbound: - log.warning('Skipping tag {} because it is bound to rev {}' - .format(tag, rev)) - continue - - log.info('Cleaning up tag {}'.format(tag)) - branchtag_to_tag(tag_name, tag) - - -def main(): - parser = argparse.ArgumentParser() - parser.add_argument('repo', help='path to local repo to cleanup') - parser.add_argument('--remote', default=None, - help='remote repo containing all the branches and tags') - parser.add_argument('--with-revbound', default=False, action='store_true', - help='keep both revision bounded branches and tags') - - args = parser.parse_args() - - cleanup(args.repo, args.with_revbound, args.remote) - -if __name__ == '__main__': - main() diff --git a/git_externals/gitlab_utils.py b/git_externals/gitlab_utils.py deleted file mode 100644 index c8c8aae..0000000 --- a/git_externals/gitlab_utils.py +++ /dev/null @@ -1,160 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -import os -import time -import posixpath - -import click -import gitlab -import requests - - -def _iter_paginated(source, per_page=10): - page = 0 - while True: - projects = source(page=page, per_page=per_page) - if len(projects) == 0: - break - else: - page = page + 1 - for project in projects: - yield project - - -def iter_projects(gl): - for prj in _iter_paginated(gl.projects.list): - yield prj - - -def search_projects(gl, name): - def source(page, per_page): - return gl.projects.search(name, page=page, per_page=per_page) - for prj in _iter_paginated(source): - yield prj - - -@click.group() -@click.option('--gitlab-id', default=None) -@click.option('--config', default=None) -@click.pass_context -def cli(ctx, gitlab_id, config): - if config is not None: - config = [config] - gl = gitlab.Gitlab.from_config(gitlab_id=gitlab_id, config_files=config) - ctx.obj['api'] = gl - - -@cli.group() -@click.pass_context -def project(ctx): - pass - - -def get_project_by_path(gl, path): - name = posixpath.basename(path) - with click.progressbar(search_projects(gl, name), label="Searching project '%s' ..." % path) as projects: - for prj in projects: - if prj.path_with_namespace == path: - return prj - click.echo("Unable to find a matching project for path '%s'" % path, err=True) - - -@project.command() -@click.argument('path') -@click.option('--sync', is_flag=True) -@click.pass_context -def delete(ctx, path, sync): - gl = ctx.obj['api'] - prj = get_project_by_path(gl, path) - if prj is None: - return - try: - if not gl.delete(prj): - raise click.UsegeError("Unable to delete project for path '%s'" % path) - except gitlab.GitlabGetError: - click.echo("The project '%s' seems to be already deleted (or queued for deletion)" % path, err=True) - return - if sync: - with click.progressbar(range(6*4), label='Waiting for deletion...') as waiting: - for step in waiting: - time.sleep(6) - click.echo("Project '%s' hopefully deleted" % path) - else: - click.echo("Project '%s' submitted for deletion" % path) - - -@project.command() -@click.argument('path') -@click.pass_context -def archive(ctx, path): - gl = ctx.obj['api'] - prj = get_project_by_path(gl, path) - if prj is None: - click.echo("Unable to find a matching project for path '%s'" % path, err=True) - return - try: - if not prj.archive(): - raise click.UsegeError("Unable to archive project for path '%s'" % path) - except gitlab.GitlabGetError: - click.echo("The project '%s' seems to be already archived" % path, err=True) - return - click.echo("Project '%s' archived" % path) - - -@project.command() -@click.argument('path') -@click.argument('branch') -@click.pass_context -def protect(ctx, path, branch): - gl = ctx.obj['api'] - prj = get_project_by_path(gl, path) - if prj is None: - click.echo("Unable to find a matching project for path '%s'" % path, err=True) - return - try: - brc = prj.branches.get(branch) - except gitlab.GitlabGetError: - raise click.UsageError("Unable to find a matching branch '%s' of project '%s'" % (branch, path)) - if brc.protected: - click.echo("Branch '%s' of project '%s' already protected" % (branch, path)) - else: - brc.protect() - click.echo("Branch '%s' of project '%s' protected" % (branch, path)) - - -@project.command() -@click.argument('path') -@click.argument('branch') -@click.pass_context -def unprotect(ctx, path, branch): - gl = ctx.obj['api'] - prj = get_project_by_path(gl, path) - if prj is None: - click.echo("Unable to find a matching project for path '%s'" % path, err=True) - return - try: - brc = prj.branches.get(branch) - except gitlab.GitlabGetError: - raise click.UsageError("Unable to find a matching branch '%s' of project '%s'" % (branch, path)) - if not brc.protected: - click.echo("Branch '%s' of project '%s' already unprotected" % (branch, path)) - else: - brc.unprotect() - click.echo("Branch '%s' of project '%s' unprotected" % (branch, path)) - - -@project.command() -@click.pass_context -def list(ctx): - gl = ctx.obj['api'] - for prj in iter_projects(gl): - click.echo(prj.path_with_namespace) - - -def main(): - cli(obj={}) - - -if __name__ == '__main__': - main() diff --git a/git_externals/gittify.py b/git_externals/gittify.py deleted file mode 100755 index 6b1b22b..0000000 --- a/git_externals/gittify.py +++ /dev/null @@ -1,754 +0,0 @@ -#!/usr/bin/env python - -from __future__ import division, print_function, absolute_import - -import os -import os.path -import posixpath -import shutil -import json -import glob -import re -import sys -import argparse -import logging -from subprocess import check_call, check_output, call -from collections import defaultdict -from contextlib import contextmanager - -try: - from urllib.parse import urlparse -except ImportError: - from urlparse import urlparse - -try: - from lxml import etree as ET -except ImportError: - from xml.etree import ElementTree as ET - -try: - from pathlib2 import Path -except ImportError: - from pathlib import Path - -import click - -from . import cleanup_repo -from .utils import git, svn, chdir, checkout, current_branch, SVNError, branches, \ - tags, IndentedLoggerAdapter, git_remote_branches_and_tags, GitError -from .process_externals import parsed_externals - -GITSVN_EXT = '.gitsvn' -GIT_EXT = '.git' - -logging.basicConfig(format='%(levelname)8s %(asctime)s: %(message)s', - level=logging.INFO, datefmt='%Y-%m-%d %I:%m:%S') -log = IndentedLoggerAdapter(logging.getLogger(__name__)) - - -def echo(*args): - click.echo(' '.join(args)) - - -def info(*args): - click.secho(' '.join(args), fg='blue') - - -def warn(*args): - click.secho(' '.join(args), fg='red') - - -def error(*args, **kwargs): - click.secho(' '.join(args), fg='red') - exitcode = kwargs.get('exitcode', 1) - if exitcode is not None: - sys.exit(exitcode) - -# Regexs -BRANCH_RE = re.compile(r'branches/([^/]+)') -TAG_RE = re.compile(r'tags/([^/]+)') - - -@click.group() -@click.option('--gitsvn-dir', type=click.Path(resolve_path=True), default='gitsvn') -@click.option('--finalize-dir', type=click.Path(resolve_path=True), default='finalize') -@click.pass_context -def cli(ctx, gitsvn_dir, finalize_dir): - ctx.obj['gitsvn_dir'] = gitsvn_dir = Path(gitsvn_dir) - if not gitsvn_dir.exists(): - gitsvn_dir.mkdir() - ctx.obj['finalize_dir'] = finalize_dir = Path(finalize_dir) - if not finalize_dir.exists(): - finalize_dir.mkdir() - - -def get_externals(repo, skip_relative=False): - data = svn('propget', '--xml', '-R', 'svn:externals', repo, universal_newlines=False) - - targets = ET.fromstring(data).findall('target') - - externals = parsed_externals(targets) - if skip_relative: - externals = (e for e in externals if not e['location'].startswith('..')) - return list(externals) - - -def svn_path_type(repo, revision=None, cache={}): - if (repo, revision) in cache: - return cache[(repo, revision)] - args = ['info', '--xml'] - if revision is not None: - args += ['-r', str(revision)] - data = svn(*args + [repo], universal_newlines=False) - cache[(repo, revision)] = res = ET.fromstring(data).find('./entry').get('kind') - return res - - -def last_changed_rev_from(rev, repo): - data = svn('info', '--xml', '-{}'.format(rev), repo, universal_newlines=False) - - rev = ET.fromstring(data).find('./entry/commit').get('revision') - return 'r{}'.format(rev) - - -def svnrev_to_gitsha(repo, branch, rev, cache={}): - - def _svnrev_to_gitsha(rev): - gitsvn_id_re = re.compile(r'(\w+) .*git-svn-id: [^@]+@{}'.format(rev[1:]), re.S) - data = git('log', '--format="%H %b"') - - match = gitsvn_id_re.search(data) - if match is not None: - return match.group(1) - else: - log.error("Unable to find a valid sha for revision: %s in %s", rev, os.path.basename(os.getcwd())) - return None - - if (repo, branch, rev) in cache: - return cache[(repo, branch, rev)] - else: - cache[(repo, branch, rev)] = sha = _svnrev_to_gitsha(rev) - return sha - - -def gitsvn_url(): - return git('svn', 'info', '--url').strip() - - -def svnext_to_gitext(ext, config, cache={}): - key = '{}{}'.format(ext, config) - if key in cache: - return cache[key] - - gitext = {} - - path = urlparse(ext['target']).path - - repo_name = extract_repo_name(path, config['super_repos']) - remote_dir = extract_repo_path(path, repo_name) - - gitext['destination'] = posixpath.join(remote_dir, ext['path']) - - loc = ext['location'].lstrip('/') - - complete_ext_repo = extract_repo_name(ext['location'], config['super_repos']) - ext_repo = posixpath.basename(complete_ext_repo) - if ext_repo.startswith('.'): - source = posixpath.join(remote_dir, ext['location']) - gitext['source'] = source - - # assume for simplicity it's the current repo(e.g. multiple .. not allowed) - ext_repo = repo_name - else: - source = extract_repo_path(ext['location'], complete_ext_repo) - try: - is_dir = svn_path_type(posixpath.join(config['svn_server'], loc), revision=ext['rev']) == 'dir' - except SVNError as err: - warn(str(err)) - is_dir = True - gitext['source'] = source + '/' if is_dir else source - - gitext['repo'] = posixpath.join(config['git_server'], ext_repo) + GIT_EXT - - match = TAG_RE.search(ext['location']) - if match is not None: - gitext['tag'] = match.group(1) - - else: - gitext['branch'] = 'master' - match = BRANCH_RE.search(ext['location']) - if match is not None: - gitext['branch'] = match.group(1).split('/')[0] - - gitext['ref'] = None - if ext['rev'] is not None: - gitext['ref'] = 'svn:' + ext['rev'].strip('r') - - cache[key] = gitext - return gitext - - -def group_gitexternals(exts): - ret = {} - mismatched_refs = {} - for ext in exts: - repo = ext['repo'] - src = ext['source'] - dst = ext['destination'] - - if repo not in ret: - ret[repo] = {'targets': {src: [dst]}} - if 'branch' in ext: - ret[repo]['branch'] = ext['branch'] - ret[repo]['ref'] = ext['ref'] - else: - ret[repo]['tag'] = ext['tag'] - else: - def equal(lhs, rhs, field): - return lhs.get(field, None) == rhs.get(field, None) - - if not equal(ret[repo], ext, 'branch') or \ - not equal(ret[repo], ext, 'ref') or \ - not equal(ret[repo], ext, 'tag'): - mismatched_refs.setdefault(repo, [ret[repo]]).append(ext) - log.critical('Branch or ref mismatch across different dirs of git ext') - - if dst not in ret[repo]['targets'].setdefault(src, []): - ret[repo]['targets'][src].append(dst) - - return ret, mismatched_refs - - -def write_extfile(gitexts, externals_filename, mismatched_refs_filename): - gitexts, mismatched = group_gitexternals(gitexts) - - with open(externals_filename, 'wt') as fd: - json.dump(gitexts, fd, indent=4, sort_keys=True) - - if len(mismatched) > 0: - with open(mismatched_refs_filename, 'wt') as fp: - json.dump(mismatched, fp, indent=4, sort_keys=True) - - -def extract_repo_name(remote_name, super_repos=None): - super_repos = super_repos or [] - - if remote_name[0] == '/': - remote_name = remote_name[1:] - - if remote_name.startswith('svn/'): - remote_name = remote_name[len('svn/'):] - - for super_repo in super_repos: - if remote_name.startswith(super_repo): - i = len(super_repo) - 1 - return super_repo + extract_repo_name(remote_name[i:], super_repos) - - j = remote_name.find('/') - if j < 0: - return remote_name - return remote_name[:j] - - -def extract_repo_root(repo, cache={}): - if repo in cache: - return cache[repo] - - output = svn('info', '--xml', repo) - - rootnode = ET.fromstring(output) - cache[repo] = root = rootnode.find('./entry/repository/root').text - return root - - -def extract_repo_path(path, repo_name): - if path[0] == '/': - path = path[1:] - - path = path[len(repo_name)+1:] - - if len(path) == 0: - return '.' - - remote_dir = path.split('/') - - def index(x): - try: - return remote_dir.index(x) - except ValueError: - return len(remote_dir) - - trunk_idx = index('trunk') - branches_idx = index('branches') - tags_idx = index('tags') - - first = min(trunk_idx, branches_idx, tags_idx) - - if first < len(remote_dir): - if first == trunk_idx: - remote_dir = remote_dir[first+1:] - else: - remote_dir = remote_dir[first+2:] - - if len(remote_dir) > 0: - remote_dir = '/'.join(remote_dir) - else: - remote_dir = '.' - - return remote_dir - - -def get_layout_opts(repo=None, entries=None): - if entries is None: - entries = set(svn('ls', repo).splitlines()) - - opts = [] - - if 'trunk/' in entries: - opts.append( - '--trunk=trunk' - ) - - if 'branches/' in entries: - opts.append( - '--branches=branches' - ) - - if 'tags/' in entries: - opts.append( - '--tags=tags' - ) - - return opts - - -def remote_rm(remote): - remotes = set(git('remote').splitlines()) - if remote in remotes: - git('remote', 'rm', remote) - - -def gittify_branch(gitsvn_repo, branch_name, obj, config, mode='branch', finalize=False): - log.info('Gittifying {} {} [{}]'.format(mode, branch_name, 'finalizze' if finalize else 'prepare')) - - with chdir(os.path.join('..', gitsvn_repo)): - with checkout(branch_name): - git_ignore = git('svn', 'show-ignore') - repo = gitsvn_url() - - with checkout(branch_name, obj): - if finalize: - with open('.gitignore', 'wt') as fp: - fp.write(git_ignore) - git('add', '.gitignore') - git('commit', '-m', 'gittify: convert svn:ignore to .gitignore', - '--author="gittify <>"') - - externals = get_externals(repo) - - if len(externals) > 0: - with log.indent(): - ext_to_write = [] - with chdir('..'): - for ext in externals: - # assume no multiple .., so it's a reference to the current repo - if not ext['location'].startswith('..'): - repo_name = extract_repo_name(ext['location'], - config['super_repos']) - - gittified_externals = [os.path.splitext(os.path.basename(e))[0] - for e in gitsvn_cloned()] - - gittified_name = posixpath.basename(repo_name) - if gittified_name not in gittified_externals: - log.info('{} is a SVN external, however it has not been found!'.format(gittified_name)) - continue - try: - gitext = svnext_to_gitext(ext, config) - except SVNError: - if not config['ignore_not_found']: - raise - ext_to_write.append(gitext) - - if finalize: - # If tagging, not ready to write - write_extfile(ext_to_write, config) - git('add', config['externals_filename']) - if os.path.exists(config['mismatched_refs_filename']): - git('add', config['mismatched_refs_filename']) - git('commit', '-m', - 'gittify: create {} file'.format(config['externals_filename']), - '--author="gittify <>"') - - -def gittify(repo, config, checkout_branches=False, finalize=False): - repo_name = os.path.splitext(repo)[0] - - git_repo = repo_name + GIT_EXT - gittified = set([repo_name]) - gitsvn_repo = repo_name + GITSVN_EXT - - # clone first in a working copy, because we need to perform checkouts, commit, etc... - if not os.path.exists(git_repo): - log.info('Cloning into final git repo {}'.format(git_repo)) - git('clone', gitsvn_repo, git_repo) - - with chdir(git_repo): - if checkout_branches: - for branch in git_remote_branches_and_tags()[0]: - with checkout(cleanup_repo.name_of(branch), branch): - pass - - log.info('Gittifying branches...') - for branch in branches(): - gittify_branch(gitsvn_repo, branch, branch, config, finalize=finalize) - - log.info('Gittifying tags...') - for tag in tags(): - gittify_branch(gitsvn_repo, tag, tag, config, mode='tag', finalize=finalize) - - if finalize: - # retag in case the dump was committed - git('tag', '-d', tag) - git('tag', tag, tag) - - git('branch', '-D', tag) - - if finalize: - remote_rm('origin') - - -@cli.command('clone') -@click.argument('root', metavar='SVNROOT') -@click.argument('path', metavar='REPOPATH') -@click.argument('authors-file', type=click.Path(exists=True, resolve_path=True), - metavar='USERMAP') -@click.option('--same-level-branches', default=None) -@click.option('--dry-run', is_flag=True) -@click.pass_context -def clone(ctx, root, path, authors_file, same_level_branches, dry_run): - remote_repo = posixpath.join(root, path) - info('Cloning {}'.format(remote_repo)) - repo_name = posixpath.basename(remote_repo) - gitsvn_repo = ctx.obj['gitsvn_dir'] / (path + GITSVN_EXT) - - if not gitsvn_repo.exists(): - gitsvn_repo.mkdir(parents=True) - else: - echo('{} already cloned'.format(repo_name)) - return - - with chdir(str(gitsvn_repo)): - args = ['git', 'svn', 'init', '--prefix=origin/'] - - try: - auto_layout_opts = get_layout_opts(remote_repo) - except SVNError as err: - warn('Error {} in {}'.format(err, remote_repo)) - auto_layout_opts = [] - - if auto_layout_opts and same_level_branches: - error('Incompatible layout options! auto layout = %s vs --same-level-branches %s' %\ - (' '.join(auto_layout_opts), same_level_branches)) - - if same_level_branches: - trunk = posixpath.basename(remote_repo) - auto_layout_opts = ['--branches=/', '--trunk=%s' % trunk] - checkout_path = posixpath.dirname(remote_repo) - else: - checkout_path = remote_repo - - args += auto_layout_opts + [checkout_path] - - echo(' '.join(map(str, args))) - check_call(args) - - if same_level_branches: - config = open('.git/config').read() - with open('.git/config', 'w') as f: - f.write(config.replace('/*:refs', '/{%s}:refs' % same_level_branches)) - - args = ['git', 'svn', 'fetch'] - if authors_file is not None: - args += ['-A', authors_file] - - echo(' '.join(map(str, args))) - if not dry_run: - check_call(args) - - -@cli.command('fetch') -@click.argument('root', metavar='SVNROOT') -@click.argument('path', metavar='REPOPATH') -@click.argument('authors-file', type=click.Path(exists=True, resolve_path=True), - metavar='USERMAP') -@click.option('--dry-run', is_flag=True) -@click.pass_context -def fetch(ctx, root, path, authors_file, dry_run, git=git, checkout=checkout, check_call=check_call): - remote_repo = posixpath.join(root, path) - info('Fetching {}'.format(remote_repo)) - repo_name = posixpath.basename(remote_repo) - gitsvn_repo = ctx.obj['gitsvn_dir'] / (path + GITSVN_EXT) - with chdir(str(gitsvn_repo)): - if dry_run: - git = echo - check_call = echo - @contextmanager - def checkout(x): - echo('checkout', x) - yield - check_call(['git','svn', 'fetch', '--all', '-A', authors_file]) - for branch in branches(): - with checkout(branch): - check_call(['git', 'svn', 'rebase', '-A', authors_file]) - - -@cli.command('cleanup') -@click.argument('repo', metavar='REPOPATH', default='.', type=click.Path(exists=True, resolve_path=True)) -@click.option('--dry-run', is_flag=True) -@click.pass_context -def cleanup(ctx, repo, dry_run, check_call=check_call): - gitsvn_repo = Path(repo) - info('Cleaning {}'.format(gitsvn_repo)) - - if dry_run: - def check_call(command): - click.secho('check_call({})'.format(command), fg='red') - - with chdir(str(gitsvn_repo)): - tags = check_output(['git', 'for-each-ref', - '--format', '%(refname:short) %(objectname)', - 'refs/remotes/origin/tags'], - universal_newlines=True).strip().splitlines() or [] - branches = check_output(['git', 'for-each-ref', - '--format', '%(refname:short) %(objectname)', - 'refs/remotes/origin'], - universal_newlines=True).strip().splitlines() or [] - for branch, ref in (tag.strip().split() for tag in tags): - tag_name = os.path.basename(branch) - body = check_output(['git', 'log', '-1', '--format=format:%B', ref], universal_newlines=True) - parent = check_output(['git', 'rev-parse', '{}^'.format(ref)], universal_newlines=True) - click.secho('ref={ref} parent={parent} tag_name={tag_name} body={body}'.format(**locals()), fg='blue') - call(['git', 'tag', '-a', '-m', body, tag_name, '{}^'.format(ref)]) - check_call(['git', 'branch', '-r', '-d', branch]) - - for branch, ref in (branch.strip().split() for branch in branches if '/tags/' not in branch and '/trunk' not in branch): - branch_name = os.path.basename(branch) - call(['git', 'branch', '-D', branch_name]) - call(['git', 'checkout', '-b', branch_name, branch]) - check_call(['git', 'checkout', 'master']) - check_call(['git', 'tag', 'trunk-snapshot']) - - -@cli.command('finalize') -@click.argument('root', metavar='SVNROOT') -@click.argument('path', metavar='REPOPATH') -@click.option('--ignore-not-found', is_flag=True) -@click.option('--externals-filename', default='git_externals.json') -@click.option('--mismatched-refs-filename', default='mismatched_ext.json') -@click.option('--dry-run', is_flag=True) -@click.option('--git-server', default=None) -@click.pass_context -def finalize(ctx, root, path, ignore_not_found, externals_filename, mismatched_refs_filename, - dry_run, git_server, git=git, checkout=checkout, check_call=check_call): - - if git_server is None: - error('git-server is required') - - gitsvn_repo = ctx.obj['gitsvn_dir'] / (path + GITSVN_EXT) - git_repo = ctx.obj['finalize_dir'] / (path + GIT_EXT) - info('Finalize {}'.format(git_repo)) - - if dry_run: - git = echo - check_call = echo - @contextmanager - def checkout(x, target=''): - echo('checkout', x, target) - yield - def add_extfile(*args): - echo('add_extfile') - def add_ignores(*args): - echo('add_ignores') - - config = {'super_repos': ['packages/', 'vendor/'], - 'svn_server': root, - 'git_server': git_server, - 'basedir': str(ctx.obj['gitsvn_dir']), - } - - def add_ignores(git_ignore): - with open('.gitignore', 'wt') as fp: - fp.write(git_ignore) - check_call(['git', 'add', '.gitignore']) - call(['git', 'commit', '-m', 'gittify: convert svn:ignore to .gitignore', - '--author="gittify <>"']) - - def add_extfile(ext_to_write): - write_extfile(ext_to_write, externals_filename, mismatched_refs_filename) - check_call(['git', 'add', externals_filename]) - if os.path.exists(mismatched_refs_filename): - check_call(['git', 'add', mismatched_refs_filename]) - call(['git', 'commit', '-m', 'gittify: create {} file'.format(externals_filename), - '--author="gittify <>"']) - - echo('Searching externals in branches...') - with chdir(str(git_repo)): - for branch in branches(): - echo('.. searching in branch %s ...' % branch) - with chdir(str(gitsvn_repo)): - with checkout(posixpath.join('origin', branch) if branch != 'master' else branch, force=True): - try: - git_ignore = git('svn', 'show-ignore') - except GitError as err: - warn('Error processing %s branch in %s:\n\t%s' % (branch, git_repo, err)) - git_ignore = None - svn_url = gitsvn_url() - - externals = [] - check_call(['git', 'stash']) - with checkout(branch, force=True): - try: - for ext in get_externals(svn_url, skip_relative=True): - echo('... processing external %s ...' % ext['location']) - externals += [svnext_to_gitext(ext, prefix(config, ext['location']))] - except SVNError as err: - warn('Error processing %s branch in %s:\n\t%s' % (branch, git_repo, err)) - if git_ignore is not None: - add_ignores(git_ignore) - if len(externals) > 0: - add_extfile(externals) - - -def clone_branch(branch_name, obj, config): - with checkout(branch_name, obj): - externals = get_externals(gitsvn_url()) - - if len(externals) > 0: - log.info('Cloning externals...') - with log.indent(): - with chdir('..'): - for ext in externals: - # assume no multiple .., so it's a reference to the current repo - if not ext['location'].startswith('..'): - repo_name = extract_repo_name(ext['location'], config['super_repos']) - clone(repo_name, config) - - -def gitsvn_cloned(): - return glob.iglob('*' + GITSVN_EXT) - - -def remove_gitsvn_repos(): - for tmp_repo in gitsvn_cloned(): - shutil.rmtree(tmp_repo) - - -def gitsvn_fetch_all(config): - authors_file = config['authors_file'] - authors_file_opt = [] if authors_file is None else ['-A', authors_file] - - check_call(['git', 'svn', 'fetch', '--all'] + authors_file_opt) - - for branch in branches(): - log.info('Rebasing branch {}'.format(branch)) - with checkout(branch): - check_call(['git', 'svn', 'rebase'] + authors_file_opt) - - url = gitsvn_url() - root = extract_repo_root(url) - repo_name = extract_repo_name(url[len(root):], config['super_repos']) - - remote_repo = posixpath.join(root, repo_name) - - cleanup_repo.cleanup('.', False, remote_repo, log=log) - - -def parse_args(): - parser = argparse.ArgumentParser() - parser.add_argument('repos', nargs='*', help='SVN repos to migrate to Git') - parser.add_argument('--git-server', default='/', - help='Url to use as base url for the git server') - parser.add_argument('--filename', default='git_externals.json', - help='Filename of the json dump of the svn externals') - parser.add_argument('--mismatched-filename', default='mismatched_ext.json', - help='Filename of the json dump of the svn externals those point to different revisions') - parser.add_argument('--crash-on-404', default=False, action='store_true', - help='Give error if an external has not been found') - parser.add_argument('--rm-gitsvn', default=False, action='store_true', - help='Remove gitsvn temporary repos') - parser.add_argument('--finalize', default=False, action='store_true', - help='Clone into final git repos') - parser.add_argument('--fetch', default=False, action='store_true', - help='Fetch from the svn repos') - parser.add_argument('--super-repos', type=set, nargs='+', - default=set(['packages/', 'vendor/']), - help='A SVN super repo is not a real repo but it is a container of repos') - parser.add_argument('-A', '--authors-file', default=None, - help='Authors file to map svn users to git') - parser.add_argument('-n', '--no-authors-file', default=False, action='store_true', - help='Continue without users mapping file') - - args = parser.parse_args() - - return args.repos, { - 'basedir': os.getcwd(), - 'git_server': args.git_server, - 'externals_filename': args.filename, - 'mismatched_refs_filename': args.mismatched_filename, - 'ignore_not_found': not args.crash_on_404, - 'rm_gitsvn': args.rm_gitsvn, - 'finalize': args.finalize, - 'fetch': args.fetch, - 'super_repos': args.super_repos, - 'authors_file': None if args.authors_file is None else os.path.abspath(args.authors_file), - 'no_authors_file': args.no_authors_file, - } - - -def old_main(): - repos, config = parse_args() - - for r in repos: - log.info("---- Cloning %s ----", r) - root = extract_repo_root(r) - repo = r[len(root) + 1:] - config['svn_server'] = root - clone(repo, config) - - if config['fetch']: - if config['authors_file'] is None: - log.warn('Fetching without authors-file') - if not config['no_authors_file']: - log.error('Provide --no-authors-file to continue without user mapping') - sys.exit(2) - for gitsvn_repo in gitsvn_cloned(): - with chdir(gitsvn_repo): - log.info('---- Fetching all in {} ----'.format(gitsvn_repo)) - gitsvn_fetch_all(config) - - if config['finalize']: - for gitsvn_repo in gitsvn_cloned(): - log.info("---- Pre-Finalize %s ----", gitsvn_repo) - with chdir(gitsvn_repo): - config['svn_server'] = extract_repo_root(gitsvn_url()) - - gittify(gitsvn_repo, config) - - for gitsvn_repo in gitsvn_cloned(): - log.info("---- Finalize %s ----", gitsvn_repo) - with chdir(gitsvn_repo): - config['svn_server'] = extract_repo_root(gitsvn_url()) - - gittify(gitsvn_repo, config, finalize=True) - - if config['rm_gitsvn']: - remove_gitsvn_repos() - - -def main(): - cli(obj={}) - - -if __name__ == '__main__': - main() diff --git a/git_externals/process_externals.py b/git_externals/process_externals.py deleted file mode 100755 index d760be3..0000000 --- a/git_externals/process_externals.py +++ /dev/null @@ -1,201 +0,0 @@ -#!/usr/bin/env python - -from __future__ import unicode_literals, print_function - -import collections -import os -import os.path -import re - -try: - from lxml import etree as ET -except ImportError: - from xml.etree import ElementTree as ET - -from .utils import header -from argparse import ArgumentParser, FileType - -RE_REVISION = re.compile(r'(-r\s*|@)(\d+)') -SVNROOT = "file:///var/lib/svn" - - -def get_args(): - parser = ArgumentParser() - parser.add_argument('ext', type=FileType('r'), - help='XML file containing the SVN externals to parse') - - parser.add_argument('--to-print', nargs='+', - choices=['all', 'dir', 'file', 'locked', 'unique'], default='all', - help='print only specified externals') - - parser.add_argument('--tags', action='store_true', default=False, - help='analyze externals only in tags') - - return parser.parse_args() - - -def main(): - args = get_args() - - # Essentials - doc = ET.parse(args.ext) - targets = doc.findall("target") - - filterfn = notags if not args.tags else withtags - ranks = ranked_externals(targets, filterfn) - - # View data - support methods - def print_ranked(externals): - print('') - print("Number of externals", len(externals)) - - for rank, external in reversed(externals): - print('') - print('External {} referenced {} times by:'.format(external["location"], rank)) - - for ref in sorted(ranks[external["location"]]): - print(" " + ref) - - # View data - def print_externals_to_dir(): - to_dir = sorted_by_rank(unique_externals(targets, onlydir), ranks) - header("Externals to a directory") - print_ranked(to_dir) - - def print_externals_to_file(): - to_file = sorted_by_rank(unique_externals(targets, onlyfile), ranks) - header("Externals to a file") - print_ranked(to_file) - - def print_locked_externals(): - to_rev = unique_externals(targets, onlylocked) - header("Externals locked to a certain revision") - for external in to_rev: - print("r{0} {1}".format(external["rev"], external["location"])) - - def print_unique_externals(): - externals = [e['location'] for e in unique_externals(targets)] - externals.sort() - header("Unique externals") - - print('') - print('Found {} unique externals'.format(len(externals))) - print('') - for external in externals: - print(external) - - - printers = [ - ('dir', print_externals_to_dir), - ('file', print_externals_to_file), - ('locked', print_locked_externals), - ('unique', print_unique_externals), - ] - - if 'all' in args.to_print: - funcs = [p[1] for p in printers] - else: - funcs = [] - for tp in sorted(set(args.to_print)): - fn = [p[1] for p in printers if p[0] == tp][0] - funcs.append(fn) - - for fn in funcs: - fn() - - -def ranked_externals(targets, filterf=lambda x: True): - ret = collections.defaultdict(list) - - for external in parsed_externals(targets): - if filterf(external): - ret[external["location"]].append(external["target"]) - - return ret - - -def sorted_by_rank(externals, ranks): - ret = [] - - for external in externals: - ret.append((len(ranks[external["location"]]), external)) - - return sorted(ret, key=lambda x: (x[0], x[1]['location'])) - - -def unique_externals(targets, filterf=lambda x: True): - return {v["location"]: v for v in parsed_externals(targets) if filterf(v)}.values() - - -def parsed_externals(targets, root=SVNROOT): - for target in targets: - for item in get_externals_from_target(target, root): - yield item - - -def get_externals_from_target(target, root=SVNROOT): - prop = target.find("property") - - for external in [l for l in prop.text.split("\n") if l.strip()]: - ret = parse_external(external) - ret["target"] = target.get("path").replace(root, "") - - yield ret - - -def parse_external(external): - normalized = RE_REVISION.sub("", external).strip() - - revision = RE_REVISION.search(external) - revision = revision.group(2) if revision else None - - pieces = normalized.split(" ", 1) - pieces = [p.strip() for p in pieces] - - # Determine whether the first or the second piece is a reference to another SVN repo/location. - if pieces[0].startswith("^/") or pieces[0].startswith("/") or pieces[0].startswith("../"): - location = pieces[0] - localpath = pieces[1] - else: - location = pieces[1] - localpath = pieces[0] - - # If the location starts with the "relative to svnroot" operator, normalize it. - if location.startswith("^/"): - location = location[1:] - - return { - "location": location, - "path": localpath, - "rev": revision, - } - - -# -# Filters -# - - -def notags(external): - return "/tags/" not in external["target"] - -def withtags(external): - return '/tags/' in external['target'] - -def onlyfile(external): - return os.path.splitext(external["location"])[1] != "" - - -def onlydir(external): - return os.path.splitext(external["location"])[1] == "" - - -def onlylocked(external): - return external["rev"] is not None - -# -# Entry point -# - -if __name__ == "__main__": - main() From dd432142700a8074ca3d4d50c2bf24f131c20831 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lien=20Rainone?= Date: Mon, 7 Nov 2016 15:54:02 +0100 Subject: [PATCH 4/7] Add latest script versions from gitlab --- git_externals/cli.py | 285 ++++++++++++++++++++++++ git_externals/git_externals.py | 391 ++++++++++----------------------- git_externals/utils.py | 126 +++++++---- 3 files changed, 480 insertions(+), 322 deletions(-) create mode 100644 git_externals/cli.py diff --git a/git_externals/cli.py b/git_externals/cli.py new file mode 100644 index 0000000..a7cfc48 --- /dev/null +++ b/git_externals/cli.py @@ -0,0 +1,285 @@ +#!/usr/bin/env python + +from __future__ import print_function, unicode_literals + +if __package__ is None: + from __init__ import __version__ + from utils import command, CommandError, chdir, git, ProgError +else: + from . import __version__ + from .utils import command, CommandError, chdir, git, ProgError, decode_utf8 + +import click +import os +import sys + +click.disable_unicode_literals_warning = True + + +def echo(*args): + click.echo(u' '.join(args)) + + +def info(*args): + click.secho(u' '.join(args), fg='blue') + + +def error(*args, **kwargs): + click.secho(u' '.join(args), fg='red') + exitcode = kwargs.get('exitcode', 1) + if exitcode is not None: + sys.exit(exitcode) + + +@click.group(context_settings={ + 'allow_extra_args': True, + 'ignore_unknown_options': True, + 'help_option_names':['-h','--help'], +}) +@click.version_option(__version__) +@click.option('--with-color/--no-color', + default=True, + help='Enable/disable colored output') +@click.pass_context +def cli(ctx, with_color): + """Utility to manage git externals, meant to be used as a drop-in + replacement to svn externals + + This script works by cloning externals found in the `git_externals.json` + file into `.git/externals` and symlinks them to recreate the wanted + directory layout. + """ + from git_externals import is_git_repo, externals_json_path, externals_root_path + + if not is_git_repo(): + error("{} is not a git repository!".format(os.getcwd()), exitcode=2) + + if ctx.invoked_subcommand != 'add' and not os.path.exists(externals_json_path()): + error("Unable to find", externals_json_path(), exitcode=1) + + if not os.path.exists(externals_root_path()): + if ctx.invoked_subcommand not in set(['update', 'add']): + error('You must first run git-externals update/add', exitcode=2) + else: + if with_color: + enable_colored_output() + + if ctx.invoked_subcommand is None: + gitext_st(()) + + +@cli.command('foreach') +@click.option('--recursive/--no-recursive', help='If --recursive is specified, this command will recurse into nested externals', default=True) +@click.argument('subcommand', nargs=-1, required=True) +def gitext_foreach(recursive, subcommand): + """Evaluates an arbitrary shell command in each checked out external + """ + from git_externals import externals_sanity_check, get_repo_name, foreach_externals_dir, root_path + + externals_sanity_check() + + def run_command(rel_url, ext_path, targets): + try: + info("External {}".format(get_repo_name(rel_url))) + output = decode_utf8(command(*subcommand)) + echo(output) + except CommandError as err: + error(str(err), exitcode=err.errcode) + + foreach_externals_dir(root_path(), run_command, recursive=recursive) + + +@cli.command('update') +@click.option('--recursive/--no-recursive', help='Do not call git-externals update recursively', default=True) +@click.option('--gitsvn/--no-gitsvn', help='Do not call git-externals update recursively (used only for the first checkout)', default=False) +@click.option('--reset', help='Reset repo, overwrite local modifications', is_flag=True) +def gitext_update(recursive, gitsvn, reset): + """Update the working copy cloning externals if needed and create the desired layout using symlinks + """ + from git_externals import externals_sanity_check, root_path, is_workingtree_clean, foreach_externals, gitext_up + + externals_sanity_check() + root = root_path() + + if reset: + git('reset', '--hard') + + # Aggregate in a list the `clean flags` of all working trees (root + externals) + clean = [is_workingtree_clean(root, fail_on_empty=False)] + foreach_externals(root, + lambda u, p, r: clean.append(is_workingtree_clean(p, fail_on_empty=False)), + recursive=recursive) + + if reset or all(clean): + # Proceed with update if everything is clean + try: + gitext_up(recursive, reset=reset, use_gitsvn=gitsvn) + except ProgError as e: + error(str(e), exitcode=e.errcode) + else: + echo("Cannot perform git externals update because one or more repositories contain some local modifications") + echo("Run:\tgit externals status\tto have more information") + + +@cli.command('status') +@click.option( + '--porcelain', + is_flag=True, + help='Print output using the porcelain format, useful mostly for scripts') +@click.option( + '--verbose/--no-verbose', + is_flag=True, + help='Show the full output of git status, instead of showing only the modifications regarding tracked file') +@click.argument('externals', nargs=-1) +def gitext_st(porcelain, verbose, externals): + """Call git status on the given externals""" + from git_externals import foreach_externals_dir, root_path, \ + is_workingtree_clean, get_repo_name + + def get_status(rel_url, ext_path, targets): + try: + if porcelain: + echo(rel_url) + click.echo(git('status', '--porcelain')) + elif verbose or not is_workingtree_clean(ext_path): + info("External {}".format(get_repo_name(rel_url))) + echo(git('status', '--untracked-files=no' if not verbose else '')) + except CommandError as err: + error(str(err), exitcode=err.errcode) + + foreach_externals_dir(root_path(), get_status, recursive=True, only=externals) + + +@cli.command('diff') +@click.argument('external', nargs=-1) +def gitext_diff(external): + """Call git diff on the given externals""" + from git_externals import iter_externals + for _ in iter_externals(external): + click.echo(git('diff')) + + +@cli.command('list') +def gitext_ls(): + """Print just a list of all externals used""" + from git_externals import iter_externals + for entry in iter_externals([], verbose=False): + info(entry) + + +@cli.command('add') +@click.argument('external', + metavar='URL') +@click.argument('src', metavar='PATH') +@click.argument('dst', metavar='PATH') +@click.option('--branch', '-b', default=None, help='Checkout the given branch') +@click.option('--tag', '-t', default=None, help='Checkout the given tag') +@click.option( + '--ref', + '-r', + default=None, + help='Checkout the given commit sha, it requires that a branch is given') +def gitext_add(external, src, dst, branch, tag, ref): + """Add a git external to the current repo. + + Be sure to add '/' to `src` if it's a directory! + It's possible to add multiple `dst` to the same `src`, however you cannot mix different branches, tags or refs + for the same external. + + It's safe to use this command to add `src` to an already present external, as well as adding + `dst` to an already present `src`. + + It requires one of --branch or --tag. + """ + from git_externals import load_gitexts, dump_gitexts + + git_externals = load_gitexts() + + if branch is None and tag is None: + error('Please specifiy at least a branch or a tag', exitcode=3) + + if external not in git_externals: + git_externals[external] = {'targets': {src: [dst]}} + if branch is not None: + git_externals[external]['branch'] = branch + git_externals[external]['ref'] = ref + else: + git_externals[external]['tag'] = tag + + else: + if branch is not None: + if 'branch' not in git_externals[external]: + error( + '{} is bound to tag {}, cannot set it to branch {}'.format( + external, git_externals[external]['tag'], branch), + exitcode=4) + + if ref != git_externals[external]['ref']: + error('{} is bound to ref {}, cannot set it to ref {}'.format( + external, git_externals[external]['ref'], ref), + exitcode=4) + + elif 'tag' not in git_externals[external]: + error('{} is bound to branch {}, cannot set it to tag {}'.format( + external, git_externals[external]['branch'], tag), + exitcode=4) + + if dst not in git_externals[external]['targets'].setdefault(src, []): + git_externals[external]['targets'][src].append(dst) + + dump_gitexts(git_externals) + + +@cli.command('remove') +@click.argument('external', nargs=-1, metavar='URL') +def gitext_remove(external): + """Remove the externals at the given repository URLs """ + from git_externals import load_gitexts, dump_gitexts + + git_externals = load_gitexts() + + for ext in external: + if ext in git_externals: + del git_externals[ext] + + dump_gitexts(git_externals) + + +@cli.command('info') +@click.argument('external', nargs=-1) +@click.option('--recursive/--no-recursive', default=True, + help='Print info only for top level externals') +def gitext_info(external, recursive): + """Print some info about the externals.""" + from git_externals import print_gitext_info, get_repo_name, gitext_recursive_info, load_gitexts + + if recursive: + gitext_recursive_info('.') + return + + external = set(external) + git_externals = load_gitexts() + + filtered = [(ext_repo, ext) + for (ext_repo, ext) in git_externals.items() + if get_repo_name(ext_repo) in external] + filtered = filtered or git_externals.items() + + for ext_repo, ext in filtered: + print_gitext_info(ext_repo, ext, root_dir='.') + + +def enable_colored_output(): + from git_externals import externals_root_path, get_entries + + for entry in get_entries(): + with chdir(os.path.join(externals_root_path(), entry)): + git('config', 'color.ui', 'always') + + +def main(): + cli() + + +if __name__ == '__main__': + main() diff --git a/git_externals/git_externals.py b/git_externals/git_externals.py index 100cbb5..654b771 100755 --- a/git_externals/git_externals.py +++ b/git_externals/git_externals.py @@ -2,8 +2,13 @@ from __future__ import print_function, unicode_literals -from . import __version__ -from .utils import chdir, mkdir_p, link, rm_link, git, GitError, command, CommandError +if __package__ is None: + import sys + from os import path + sys.path.append(path.dirname(path.dirname(path.abspath(__file__)))) + +from .utils import (chdir, mkdir_p, link, rm_link, git, GitError, svn, gitsvn, gitsvnrebase) +from .cli import echo, info, error import click import json @@ -14,35 +19,23 @@ from collections import defaultdict, namedtuple try: - from urllib.parse import urlparse, urlunparse, urlsplit, urlunsplit + from urllib.parse import urlparse, urlsplit, urlunsplit except ImportError: - from urlparse import urlparse, urlunparse, urlsplit, urlunsplit + from urlparse import urlparse, urlsplit, urlunsplit + EXTERNALS_ROOT = os.path.join('.git', 'externals') EXTERNALS_JSON = 'git_externals.json' -click.disable_unicode_literals_warning = True - - -def echo(*args): - click.echo(u' '.join(args)) - - -def info(*args): - click.secho(u' '.join(args), fg='blue') - - -def error(*args, **kwargs): - click.secho(u' '.join(args), fg='red') - exitcode = kwargs.get('exitcode', 1) - if exitcode is not None: - sys.exit(exitcode) - def get_repo_name(repo): + if repo[-1] == '/': + repo = repo[:-1] name = repo.split('/')[-1] if name.endswith('.git'): name = name[:-len('.git')] + if not name: + error("Invalid repository name: \"{}\"".format(repo), exitcode=1) return name @@ -92,13 +85,6 @@ def get_entries(): if os.path.exists(os.path.join(externals_root_path(), get_repo_name(e)))] -def load_gitexts(): - if os.path.exists(externals_json_path()): - with open(externals_json_path()) as fp: - return json.load(fp) - return {} - - def load_gitexts(pwd=None): """Load the *externals definition file* present in given directory, or cwd @@ -107,13 +93,26 @@ def load_gitexts(pwd=None): fn = os.path.join(d, EXTERNALS_JSON) if os.path.exists(fn): with open(fn) as f: - return json.load(f) + gitext = json.load(f) + for url, _ in gitext.items(): + # svn external url must be absolute + gitext[url]['vcs'] = 'svn' if 'svn' in urlparse(url).scheme else 'git' + return gitext return {} def dump_gitexts(externals): - with open(externals_json_path(), 'wt') as fp: - json.dump(externals, fp, sort_keys=True, indent=4) + """ + Dump externals dictionary as json in current working directory + git_externals.json. Remove 'vcs' key that is only used at runtime. + """ + with open(externals_json_path(), 'w') as f: + # copy the externals dict (we want to keep the 'vcs' + dump = {k:v for k,v in externals.items()} + for v in dump.values(): + if 'vcs' in v: + del v['vcs'] + json.dump(dump, f, sort_keys=True, indent=4, separators=(',', ': ')) def foreach_externals(pwd, callback, recursive=True, only=()): @@ -178,7 +177,7 @@ def is_workingtree_clean(path, fail_on_empty=True): Returns true if and only if there are no modifications to tracked files. By modifications it is intended additions, deletions, file removal or conflicts. If True is returned, that means that performing a - `git reset --hard` would result in no local modifications lost because: + `git reset --hard` would result in no loss of local modifications because: - tracked files are unchanged - untracked files are not modified anyway """ @@ -190,6 +189,7 @@ def is_workingtree_clean(path, fail_on_empty=True): try: return len([line.strip for line in git('status', '--untracked-files=no', '--porcelain').splitlines(True)]) == 0 except GitError as err: + echo('Couldn\'t retrieve Git status of', path) error(str(err), exitcode=err.errcode) @@ -221,78 +221,6 @@ def untrack(paths): fp.write(p + '\n') -@click.group(invoke_without_command=True) -@click.version_option(__version__) -@click.option('--with-color/--no-color', - default=True, - help='Enable/disable colored output') -@click.pass_context -def cli(ctx, with_color): - """Utility to manage git externals, meant to be used as a drop-in replacement to svn externals - - This script works cloning externals found in the `git_externals.json` file into `.git/externals` and - then it uses symlinks to create the wanted directory layout. - """ - - if not is_git_repo(): - error("{} is not a git repository!".format(os.getcwd()), exitcode=2) - - if ctx.invoked_subcommand != 'add' and not os.path.exists(externals_json_path()): - error("Unable to find", externals_json_path(), exitcode=1) - - if not os.path.exists(externals_root_path()): - if ctx.invoked_subcommand not in set(['update', 'add']): - error('You must first run git-externals update/add', exitcode=2) - else: - if with_color: - enable_colored_output() - - if ctx.invoked_subcommand is None: - gitext_st(()) - - -@cli.command('foreach') -@click.option('--recursive/--no-recursive', help='If --recursive is specified, this command will recurse into nested externals', default=True) -@click.argument('subcommand', nargs=-1, required=True) -def gitext_foreach(recursive, subcommand): - """Evaluates an arbitrary shell command in each checked out external - """ - - externals_sanity_check() - - def run_command(rel_url, ext_path, targets): - try: - info("External {}".format(get_repo_name(rel_url))) - output = command(*subcommand) - echo(output) - except CommandError as err: - error(str(err), exitcode=err.errcode) - - foreach_externals_dir(root_path(), run_command, recursive=recursive) - - -@cli.command('update') -@click.option('--recursive/--no-recursive', help='Do not call git-externals update recursively', default=True) -def gitext_update(recursive): - """Update the working copy cloning externals if needed and create the desired layout using symlinks - """ - - externals_sanity_check() - root = root_path() - - # Aggregate in a list the `clean flags` of all working trees (root + externals) - clean = [is_workingtree_clean(root, fail_on_empty=False)] - foreach_externals(root, - lambda u,p,r: clean.append(is_workingtree_clean(p, fail_on_empty=False)), - recursive=recursive) - - if all(clean): - # Proceed with update if everything is clean - gitext_up(recursive) - else: - echo("Cannot perform git externals update because one or more repositories contain some local modifications") - echo("Run:\tgit externals status\tto have more information") - def externals_sanity_check(): """Check that we are not trying to track various refs of the same external repo""" @@ -342,7 +270,7 @@ def filter_externals_not_needed(all_externals, entries): return git_externals -def gitext_up(recursive, entries=None): +def gitext_up(recursive, entries=None, reset=False, use_gitsvn=False): if not os.path.exists(externals_json_path()): return @@ -350,9 +278,84 @@ def gitext_up(recursive, entries=None): all_externals = load_gitexts() git_externals = all_externals if entries is None else filter_externals_not_needed(all_externals, entries) + def egit(command, *args): + if command == 'checkout' and reset: + args = ('--force',) + args + git(command, *args, capture=False) + + def git_initial_checkout(repo_name, repo_url): + """Perform the initial git clone (or sparse checkout)""" + dirs = git_externals[ext_repo]['targets'].keys() + if './' not in dirs: + echo('Doing a sparse checkout of:', ', '.join(dirs)) + sparse_checkout(repo_name, repo_url, dirs) + else: + egit('clone', repo_url, repo_name) + + def git_update_checkout(reset): + """Update an already existing git working tree""" + if reset: + egit('reset', '--hard') + egit('clean', '-df') + egit('fetch', '--all') + egit('fetch', '--tags') + if 'tag' in git_externals[ext_repo]: + echo('Checking out tag', git_externals[ext_repo]['tag']) + egit('checkout', git_externals[ext_repo]['tag']) + else: + echo('Checking out branch', git_externals[ext_repo]['branch']) + egit('checkout', git_externals[ext_repo]['branch']) + egit('pull', 'origin', git_externals[ext_repo]['branch']) + + if git_externals[ext_repo]['ref'] is not None: + echo('Checking out commit', git_externals[ext_repo]['ref']) + ref = git_externals[ext_repo]['ref'] + if git_externals[ext_repo]['ref'].startswith('svn:'): + ref = egit('log', '--grep', 'git-svn-id:.*@%s' % ref.strip('svn:r'), '--format=%H').strip() + egit('checkout', ref) + + def gitsvn_initial_checkout(repo_name, repo_url): + """Perform the initial git-svn clone (or sparse checkout)""" + gitsvn('clone', normalized_ext_repo, repo_name, '-rHEAD', capture=False) + + def gitsvn_update_checkout(reset): + """Update an already existing git-svn working tree""" + # FIXME: seems this might be necessary sometimes (happened with + # 'vectorfonts' for example that the following error: "Unable to + # determine upstream SVN information from HEAD history" was fixed by + # adding that, but breaks sometimes. (investigate) + # git('rebase', '--onto', 'git-svn', '--root', 'master') + gitsvnrebase('.', capture=False) + + def svn_initial_checkout(repo_name, repo_url): + """Perform the initial svn checkout""" + svn('checkout', normalized_ext_repo, repo_name, capture=False) + + def svn_update_checkout(reset): + """Update an already existing svn working tree""" + if reset: + svn('revert', '-r', '.') + svn('up', capture=False) + + def autosvn_update_checkout(reset): + if os.path.exists('.git'): + gitsvn_update_checkout(reset) + else: + svn_update_checkout(reset) + for ext_repo in git_externals.keys(): normalized_ext_repo = normalize_gitext_url(ext_repo) + if all_externals[ext_repo]['vcs'] == 'git': + _initial_checkout = git_initial_checkout + _update_checkout = git_update_checkout + else: + if use_gitsvn: + _initial_checkout = gitsvn_initial_checkout + else: + _initial_checkout = svn_initial_checkout + _update_checkout = autosvn_update_checkout + mkdir_p(externals_root_path()) with chdir(externals_root_path()): repo_name = get_repo_name(normalized_ext_repo) @@ -360,33 +363,11 @@ def gitext_up(recursive, entries=None): info('External', repo_name) if not os.path.exists(repo_name): echo('Cloning external', repo_name) - - dirs = git_externals[ext_repo]['targets'].keys() - if './' not in dirs: - echo('Doing a sparse checkout of:', ', '.join(dirs)) - sparse_checkout(repo_name, normalized_ext_repo, dirs) - else: - git('clone', normalized_ext_repo, repo_name, capture=False) + _initial_checkout(repo_name, normalized_ext_repo) with chdir(repo_name): - echo('Fetching data of', repo_name) - git('fetch', '--all', capture=False) - git('fetch', '--tags', capture=False) - - if 'tag' in git_externals[ext_repo]: - echo('Checking out tag', git_externals[ext_repo]['tag']) - git('checkout', git_externals[ext_repo]['tag'], capture=False) - else: - echo('Checking out branch', git_externals[ext_repo]['branch']) - git('checkout', git_externals[ext_repo]['branch'], capture=False) - git('pull', 'origin', git_externals[ext_repo]['branch'], capture=False) - - if git_externals[ext_repo]['ref'] is not None: - echo('Checking out commit', git_externals[ext_repo]['ref']) - ref = git_externals[ext_repo]['ref'] - if git_externals[ext_repo]['ref'].startswith('svn:'): - ref = git('log', '--grep', 'git-svn-id:.*@%s' % ref.strip('svn:r'), '--format=%H').strip() - git('checkout', ref, capture=False) + echo('Retrieving changes from server: ', repo_name) + _update_checkout(reset) link_entries(git_externals) @@ -396,7 +377,7 @@ def gitext_up(recursive, entries=None): for t in git_externals[ext_repo]['targets'].values() for d in t] with chdir(os.path.join(externals_root_path(), get_repo_name(ext_repo))): - gitext_up(recursive, entries) + gitext_up(recursive, entries, reset=reset) to_untrack = [] for ext in git_externals.values(): @@ -404,148 +385,6 @@ def gitext_up(recursive, entries=None): untrack(to_untrack) -@cli.command('status') -@click.option( - '--porcelain', - is_flag=True, - help='Print output using the porcelain format, useful mostly for scripts') -@click.option( - '--verbose/--no-verbose', - is_flag=True, - help='Show the full output of git status, instead of showing only the modifications regarding tracked file') -@click.argument('externals', nargs=-1) -def gitext_st(porcelain, verbose, externals): - """Call git status on the given externals""" - - def get_status(rel_url, ext_path, targets): - try: - if porcelain: - echo(rel_url) - click.echo(git('status', '--porcelain')) - elif verbose or not is_workingtree_clean(ext_path): - info("External {}".format(get_repo_name(rel_url))) - echo(git('status', '--untracked-files=no' if not verbose else '')) - except CommandError as err: - error(str(err), exitcode=err.errcode) - - foreach_externals_dir(root_path(), get_status, recursive=True, only=externals) - - -@cli.command('diff') -@click.argument('external', nargs=-1) -def gitext_diff(external): - """Call git diff on the given externals""" - - for _ in iter_externals(external): - click.echo(git('diff')) - - -@cli.command('list') -def gitext_ls(): - """Print just a list of all externals used""" - - for entry in iter_externals([], verbose=False): - info(entry) - - -@cli.command('add') -@click.argument('external', - metavar='URL') -@click.argument('src', metavar='PATH') -@click.argument('dst', metavar='PATH') -@click.option('--branch', '-b', default=None, help='Checkout the given branch') -@click.option('--tag', '-t', default=None, help='Checkout the given tag') -@click.option( - '--ref', - '-r', - default=None, - help='Checkout the given commit sha, it requires that a branch is given') -def gitext_add(external, src, dst, branch, tag, ref): - """Add a git external to the current repo. - - Be sure to add '/' to `src` if it's a directory! - It's possible to add multiple `dst` to the same `src`, however you cannot mix different branches, tags or refs - for the same external. - - It's safe to use this command to add `src` to an already present external, as well as adding - `dst` to an already present `src`. - - It requires one of --branch or --tag. - """ - - git_externals = load_gitexts() - - if branch is None and tag is None: - error('Please specifiy at least a branch or a tag', exitcode=3) - - if external not in git_externals: - git_externals[external] = {'targets': {src: [dst]}} - if branch is not None: - git_externals[external]['branch'] = branch - git_externals[external]['ref'] = ref - else: - git_externals[external]['tag'] = tag - - else: - if branch is not None: - if 'branch' not in git_externals[external]: - error( - '{} is bound to tag {}, cannot set it to branch {}'.format( - external, git_externals[external]['tag'], branch), - exitcode=4) - - if ref != git_externals[external]['ref']: - error('{} is bound to ref {}, cannot set it to ref {}'.format( - external, git_externals[external]['ref'], ref), - exitcode=4) - - elif 'tag' not in git_externals[external]: - error('{} is bound to branch {}, cannot set it to tag {}'.format( - external, git_externals[external]['branch'], tag), - exitcode=4) - - if dst not in git_externals[external]['targets'].setdefault(src, []): - git_externals[external]['targets'][src].append(dst) - - dump_gitexts(git_externals) - - -@cli.command('remove') -@click.argument('external', nargs=-1, metavar='URL') -def gitext_remove(external): - """Remove the externals at the given repository URLs """ - - git_externals = load_gitexts() - - for ext in external: - if ext in git_externals: - del git_externals[ext] - - dump_gitexts(git_externals) - - -@cli.command('info') -@click.argument('external', nargs=-1) -@click.option('--recursive/--no-recursive', default=True, help='Print info only for top level externals') -def gitext_info(external, recursive): - """Print some info about the externals.""" - - if recursive: - gitext_recursive_info('.') - return - - external = set(external) - git_externals = load_gitexts() - - filtered = [(ext_repo, ext) - for (ext_repo, ext) in git_externals.items() - if get_repo_name(ext_repo) in external] - filtered = filtered or git_externals.items() - - for ext_repo, ext in filtered: - print_gitext_info(ext_repo, ext, root_dir='.') - - def gitext_recursive_info(root_dir): git_exts = load_gitexts() git_exts = {ext_repo: ext for ext_repo, ext in git_exts.items() @@ -608,9 +447,3 @@ def iter_externals(externals, verbose=True): if verbose: info('External {}'.format(entry)) yield entry - - -def enable_colored_output(): - for entry in get_entries(): - with chdir(os.path.join(externals_root_path(), entry)): - git('config', 'color.ui', 'always') diff --git a/git_externals/utils.py b/git_externals/utils.py index a1fe4b1..bea3cac 100644 --- a/git_externals/utils.py +++ b/git_externals/utils.py @@ -13,8 +13,10 @@ class ProgError(Exception): - def __init__(self, prog='', errcode=1, errmsg=''): - super(ProgError, self).__init__(u'{} {}'.format(prog, errmsg)) + def __init__(self, prog='', errcode=1, errmsg='', args=''): + if isinstance(args, tuple): + args = u' '.join(args) + super(ProgError, self).__init__(u'\"{} {}\" {}'.format(prog, args, errmsg)) self.prog = prog self.errcode = errcode @@ -29,9 +31,15 @@ def __init__(self, **kwargs): super(GitError, self).__init__(prog='git', **kwargs) -class SVNError(ProgError): +class SvnError(ProgError): def __init__(self, **kwargs): - super(SVNError, self).__init__(prog='svn', **kwargs) + super(SvnError, self).__init__(prog='svn', **kwargs) + + +class GitSvnError(ProgError): + def __init__(self, **kwargs): + super(GitSvnError, self).__init__(prog='git-svn', **kwargs) + class CommandError(ProgError): def __init__(self, cmd, **kwargs): @@ -40,51 +48,61 @@ def __init__(self, cmd, **kwargs): def svn(*args, **kwargs): universal_newlines = kwargs.get('universal_newlines', True) - p = subprocess.Popen(['svn'] + list(args), - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - universal_newlines=universal_newlines) - output, err = p.communicate() - - if p.returncode != 0: - raise SVNError(errcode=p.returncode, errmsg=err) - + output, err, errcode = _command('svn', *args, capture=True, universal_newlines=universal_newlines) + if errcode != 0: + raise SvnError(errcode=errcode, errmsg=err) return output def git(*args, **kwargs): capture = kwargs.get('capture', True) - if capture: - stdout = subprocess.PIPE - stderr = subprocess.PIPE - else: - stdout = None - stderr = None - p = subprocess.Popen(['git'] + list(args), - stdout=stdout, - stderr=stderr, - universal_newlines=True) - output, err = p.communicate() + output, err, errcode = _command('git', *args, capture=capture, universal_newlines=True) + if errcode != 0: + raise GitError(errcode=errcode, errmsg=err, args=args) + return output - if p.returncode != 0: - raise GitError(errcode=p.returncode, errmsg=err) +def gitsvn(*args, **kwargs): + capture = kwargs.get('capture', True) + output, err, errcode = _command('git', 'svn', *args, capture=capture, universal_newlines=True) + if errcode != 0: + raise GitSvnError(errcode=errcode, errmsg=err, args=args) return output -def command(cmd, *args): - p = subprocess.Popen([cmd] + list(args), - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - universal_newlines=True) - output, err = p.communicate() +def gitsvnrebase(*args, **kwargs): + capture = kwargs.get('capture', True) + output, err, errcode = _command('git-svn-rebase', *args, capture=capture, universal_newlines=True) + if errcode != 0: + raise GitSvnError(errcode=errcode, errmsg=err, args=args) + return output - if p.returncode != 0: - raise CommandError(cmd, errcode=p.returncode, errmsg=err) +def command(cmd, *args, **kwargs): + universal_newlines = kwargs.get('universal_newlines', True) + capture = kwargs.get('capture', True) + output, err, errcode = _command(cmd, *args, universal_newlines=universal_newlines, capture=capture) + if errcode != 0: + raise CommandError(cmd, errcode=errcode, errmsg=err, args=args) return output +def _command(cmd, *args, **kwargs): + universal_newlines = kwargs.get('universal_newlines', True) + capture = kwargs.get('capture', True) + if capture: + stdout, stderr = subprocess.PIPE, subprocess.PIPE + else: + stdout, stderr = None, None + + p = subprocess.Popen([cmd] + list(args), + stdout=stdout, + stderr=stderr, + universal_newlines=universal_newlines) + output, err = p.communicate() + return output, err, p.returncode + + def current_branch(): return git('name-rev', '--name-only', 'HEAD').strip() @@ -161,20 +179,42 @@ def print_msg(msg): print(u' {}'.format(msg)) +def decode_utf8(msg): + """ + Py2 / Py3 decode + """ + try: + return msg.decode('utf8') + except AttributeError: + return msg + + if not sys.platform.startswith('win32'): link = os.symlink rm_link = os.remove # following works but it requires admin privileges else: - - def link(src, dst): - import ctypes - csl = ctypes.windll.kernel32.CreateSymbolicLinkW - csl.argtypes = (ctypes.c_wchar_p, ctypes.c_wchar_p, ctypes.c_uint32) - csl.restype = ctypes.c_ubyte - if csl(dst, src, 0 if os.path.isfile(src) else 1) == 0: - raise ctypes.WinError() + if sys.getwindowsversion()[0] >= 6: + def link(src, dst): + import ctypes + csl = ctypes.windll.kernel32.CreateSymbolicLinkW + csl.argtypes = (ctypes.c_wchar_p, ctypes.c_wchar_p, ctypes.c_uint32) + csl.restype = ctypes.c_ubyte + if csl(dst, src, 0 if os.path.isfile(src) else 1) == 0: + raise ctypes.WinError("Error in CreateSymbolicLinkW(%s, %s)" % (dst, src)) + else: + import shutil + def link(src, dst): + if os.path.isfile(src): + print_msg("WARNING: Unsupported SymLink on Windows before Vista, single files will be copied") + shutil.copy2(src, dst) + else: + try: + subprocess.check_call(['junction', dst, src], shell=True) + except: + print_msg("ERROR: Is http://live.sysinternals.com/junction.exe in your PATH?") + raise def rm_link(path): if os.path.isfile(path): @@ -187,7 +227,7 @@ class IndentedLoggerAdapter(logging.LoggerAdapter): def __init__(self, logger, indent_val=4): super(IndentedLoggerAdapter, self).__init__(logger, {}) self.indent_level = 0 - self.indent_val = 4 + self.indent_val = indent_val def process(self, msg, kwargs): return (' ' * self.indent_level + msg, kwargs) From c936cead028dca8d6d115be82f774cf4817f0026 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lien=20Rainone?= Date: Mon, 7 Nov 2016 15:54:25 +0100 Subject: [PATCH 5/7] Add bash completion --- git_externals/gitext.completion.bash | 197 +++++++++++++++++++++++++++ 1 file changed, 197 insertions(+) create mode 100644 git_externals/gitext.completion.bash diff --git a/git_externals/gitext.completion.bash b/git_externals/gitext.completion.bash new file mode 100644 index 0000000..f6ef40b --- /dev/null +++ b/git_externals/gitext.completion.bash @@ -0,0 +1,197 @@ +#! /usr/bin/env bash +# +# Bash completion for git-externals +# +# ============ +# Installation +# ============ +# +# Completion after typing "git-externals" +# =========================== +# Put: "source gitext.completion.bash" in your .bashrc +# or place it in "/etc/bash_completion.d" folder, it will +# be sourced automatically. +# +# completion after typing "git externals" (through git-completion) +# =========================== +# Put: Ensure git-completion is installed (normally it comes +# with Git on Debian systems). In case it isn't, see: +# https://github.com/git/git/blob/master/contrib/completion/git-completion.bash + +_git_ext_cmds=" \ +add \ +diff \ +foreach \ +info \ +list \ +remove \ +status \ +update +" + +_git_externals () +{ + local subcommands="add diff foreach info list remove status update" + local subcommand="$(__git_find_on_cmdline "$subcommands")" + + if [ -z "$subcommand" ]; then + __gitcomp "$subcommands" + return + fi + + case "$subcommand" in + add) + __git_ext_add + return + ;; + diff) + __git_ext_diff + return + ;; + info) + __git_ext_info + return + ;; + update|foreach) + __git_ext_update_foreach + return + ;; + list) + __git_ext_list + return + ;; + remove) + __git_ext_remove + return + ;; + status) + __git_ext_status + return + ;; + list) + __git_ext_list + return + ;; + *) + COMPREPLY=() + ;; + esac +} + +__git_ext_info () +{ + local cur + local opts="" + COMPREPLY=() + cur="${COMP_WORDS[COMP_CWORD]}" + + case "$cur" in + -*) opts="--recursive --no-recursive" ;; + *) __gitext_complete_externals "${cur}" ;; + esac + + if [[ -n "${opts}" ]]; then + COMPREPLY=( ${COMPREPLY[@]:-} $(compgen -W "${opts}" -- "${cur}") ) + fi +} + +__git_ext_status () +{ + local cur + local opts="" + COMPREPLY=() + cur="${COMP_WORDS[COMP_CWORD]}" + + case "$cur" in + -*) opts="--porcelain --verbose --no-verbose" ;; + *) __gitext_complete_externals "${cur}" ;; + esac + + if [[ -n "${opts}" ]]; then + COMPREPLY=( ${COMPREPLY[@]:-} $(compgen -W "${opts}" -- "${cur}") ) + fi +} + +__git_ext_diff () +{ + local cur + COMPREPLY=() + cur="${COMP_WORDS[COMP_CWORD]}" + + __gitext_complete_externals "${cur}" +} + +__git_ext_update_foreach () +{ + local opts="" + opts="--recursive --no-recursive" + COMPREPLY=( ${COMPREPLY[@]:-} $(compgen -W "${opts}" -- "${cur}") ) +} + +__git_ext_add () +{ + local opts="" + opts="--branch --tag" + COMPREPLY=( ${COMPREPLY[@]:-} $(compgen -W "${opts}" -- "${cur}") ) +} + +__git_externals () +{ + local cur prev + local i cmd cmd_index option option_index + local opts="" + COMPREPLY=() + cur="${COMP_WORDS[COMP_CWORD]}" + prev="${COMP_WORDS[COMP_CWORD-1]}" + + # Search for the subcommand + local skip_next=0 + for ((i=1; $i<=$COMP_CWORD; i++)); do + if [[ ${skip_next} -eq 1 ]]; then + skip_next=0; + elif [[ ${COMP_WORDS[i]} != -* ]]; then + cmd="${COMP_WORDS[i]}" + cmd_index=${i} + break + elif [[ ${COMP_WORDS[i]} == -f ]]; then + skip_next=1 + fi + done + + options="" + if [[ $COMP_CWORD -le $cmd_index ]]; then + # The user has not specified a subcommand yet + COMPREPLY=( ${COMPREPLY[@]:-} $(compgen -W "${_git_ext_cmds}" -- "${cur}") ) + else + case ${cmd} in + diff) + __git_ext_diff ;; + info) + __git_ext_info ;; + status) + __git_ext_status ;; + update|foreach) + __git_ext_update_foreach ;; + add) + __git_ext_update_add ;; + esac # case ${cmd} + fi # command specified + + if [[ -n "${options}" ]]; then + COMPREPLY=( ${COMPREPLY[@]:-} $(compgen -W "${options}" -- "${cur}") ) + fi +} + +__gitext_complete_externals () +{ + local IFS=$'\n' + local cur="${1}" + COMPREPLY=( ${COMPREPLY[@]:-} $(compgen -W "$(git-externals list 2>/dev/null)" -- "${cur}") ) +} + +# alias __git_find_on_cmdline for backwards compatibility +if [ -z "`type -t __git_find_on_cmdline`" ]; then + alias __git_find_on_cmdline=__git_find_subcommand +fi + +complete -F __git_externals git-externals From 08cf38ae3efd907cc6fafb050c2d0f92889e4dc8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lien=20Rainone?= Date: Mon, 7 Nov 2016 16:46:08 +0100 Subject: [PATCH 6/7] Update README: remove Gittify section, add basic example --- README.md | 147 +++++++++++++++++++++++++++++++++++------------------- 1 file changed, 96 insertions(+), 51 deletions(-) diff --git a/README.md b/README.md index 7221ad3..207bb06 100644 --- a/README.md +++ b/README.md @@ -1,72 +1,115 @@ -# Git Externals -Couple of scripts to throw away SVN and migrate to Git. These scripts are particularly -useful when the SVN repos make heavy use of externals. In some cases it's just impossible -to use submodules or subtrees to emulate SVN externals, because they aren't as flexible as -SVN externals. For instance it's impossible to use submodules to include just a single file in an -arbitrary directory. +Git Externals +------------- -Basically what you need to do is launch gittify to slowly migrate all the given SVN repos -and related externals and then you can use git-externals to manage them. +`git-externals` is a command line tool that helps you throw **SVN** away and +migrate to **Git** for projects that make heavy use of *svn:externals*. In some +cases it's just impossible to use *Git submodules* or *subtrees* to emulate *SVN +externals*, because they aren't as flexible as SVN externals. For example **SVN** +lets you handle a *single file dependency* through *SVN external*, whereas **Git** +doesn't. -On Windows this requires **ADMIN PRIVILEGES**, because under the hood it uses symlinks. Moreover for the same -reason this script is not meant to be used with the old Windows XP. +On Windows this requires **ADMIN PRIVILEGES**, because under the hood it uses +symlinks. For the same reason this script is not meant to be used with the old +Windows XP. ## How to Install -```bash + +```sh $ pip install https://github.com/develersrl/git-externals/archive/master.zip ``` -## Gittify -This tool tries to do the best to convert SVN repos to Git repos. Under the hood it uses -`git-svn` to clone repos and does some extra work to handle externals and removed tags and branches. - -If (hopefully) doesn't crash, a dump named "git\_externals.json" will be created in every branch -and tag containing various info about the externals. -In case different revisions, branches or tags are used for the same external "mismatched\_externals.json" is -created with a bit of info. - -This script can work in 3 modes: -- Clone: clone the desired svn repos and all their externals into .gitsvn repos; -- Fetch: it's possible to simply fetch new revisions from svn without cloning everythin again; -- Finalization: when everything is ready convert all the git-svn repos into bare git repos; - -Usage: -```bash -$ gittify -A authors-file.txt clone file:///var/lib/svn/foo file:///var/lib/svn/bar -$ gittify -A authors-file.txt fetch -$ gittify finalize --git-server=https://git.foo.com/ +## Usage: + +### Content of `git_externals.json` + +Once your main project repository is handled by Git, `git-externals` expects to +find a file called `git_externals.json` at the project root. Here is how to fill +it: + +Let's take an hypothetical project **A**, under Subversion, having 2 +dependencies, **B** and **C**, declared as `svn:externals` as +follows. + +```sh +$ svn propget svn:externals . +^/svn/libraries/B lib/B +^/svn/libraries/C src/C +``` + +``` +A +├── lib +│   └── B +└── src + └── C +``` + +Once **A**, **B** and **C** have all been migrated over different Git +repositories, fill `git_externals.json` by running the following commands. +They describe, for each dependency, its remote location, and the destination +directory, relative to the project root. Check out all the possibilities by +running `git externals add --help`. + +```sh +$ git externals add --branch=master git@github.com:username/libB.git . lib/B +$ git externals add --branch=master git@github.com:username/libC.git . src/C ``` -However ```gittify --help``` is useful as well. +This is now the content of `git_externals.json`: + +```json +{ + "git@github.com:username/libB.git": { + "branch": "master", + "ref": null, + "targets": { + "./": [ + "lib/B" + ] + } + }, + "git@github.com:username/libC.git": { + "branch": "master", + "ref": null, + "targets": { + "./": [ + "src/C" + ] + } + } +} +``` -## Git-Externals ### Git externals update If you want to: -* download the externals of a freshly cloned Git repository and creates their -symlinks, in order to have the wanted directory layout. -* checkout the latest version of all externals (as defined in `git_externals.json` -file) + +- download the externals of a freshly cloned Git repository and creates their + symlinks, in order to have the wanted directory layout. +- checkout the latest version of all externals (as defined in + `git_externals.json` file) Run: -```bash + +```sh $ git externals update ``` ### Git externals status -```bash +```sh $ git externals status [--porcelain|--verbose] $ git externals status [--porcelain|--verbose] [external1 [external2] ...] ``` Shows the working tree status of one, multiple, or all externals: + - add `--verbose` if you are also interested to see the externals that haven't -been modified + been modified - add `--porcelain` if you want the output easily parsable (for non-humans). -```bash +```sh $ git externals status $ git externals status deploy $ git externals status deploy qtwidgets @@ -74,12 +117,12 @@ $ git externals status deploy qtwidgets ### Git externals foreach -```bash +```sh $ git externals foreach [--] cmd [arg1 [arg2] ...] ``` Evaluates an arbitrary shell command in each checked out external. -```bash +```sh $ git externals foreach git fetch ``` @@ -87,26 +130,28 @@ $ git externals foreach git fetch `git rev-parse --all`, you must pass `--` after `foreach` in order to stop git externals argument processing, example: -```bash +```sh $ git externals foreach -- git rev-parse --all ``` - ### Example usage -```bash -$ git externals add --branch=master https://gitlab.com/gitlab-org/gitlab-ce.git shared/ foo -$ git externals add --branch=master https://gitlab.com/gitlab-org/gitlab-ce.git shared/ bar -$ git externals add --branch=master https://gitlab.com/gitlab-org/gitlab-ce.git README.md baz/README.md + +```sh +$ git externals add --branch=master https://github.com/username/projectA.git shared/ foo +$ git externals add --branch=master https://github.com/username/projectB.git shared/ bar +$ git externals add --branch=master https://github.com/username/projectC.git README.md baz/README.md $ git externals add --tag=v4.4 https://github.com/torvalds/linux.git Makefile Makefile $ git add git_externals.json -$ git commit -m "DO NOT FORGET TO COMMIT THE EXTERNALS!!!" +$ git commit -m "Let git-externals handle our externals ;-)" $ git externals update $ git externals diff $ git externals info $ git externals list +$ git externals foreach -- git diff HEAD~1 ``` +**Note**: Append `/` to the source path if it represents a directory. +### Bash command-line completion -Please be careful to include / if the source is a directory! - +See installation instructions in [gitext.completion.bash](./git_externals/gitext.completion.bash). From 77bc842e8b5baf4864a55414f58fb6e4fd28c793 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lien=20Rainone?= Date: Mon, 7 Nov 2016 17:32:55 +0100 Subject: [PATCH 7/7] Update setup.py --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index e6c0bed..1163c36 100644 --- a/setup.py +++ b/setup.py @@ -26,8 +26,8 @@ setup( name='git-externals', version=__version__, - description='Use git to handle your svn:externals', - long_description='ease the migration from Git to SVN by handling svn externals through a cli tool', + description='cli tool to manage git externals', + long_description='Ease the migration from Git to SVN by handling svn externals through a cli tool', packages=['git_externals'], install_requires=['click', 'git-svn-clone-externals'],