diff --git a/edx_repo_tools/data.py b/edx_repo_tools/data.py index 87a051fa..9c21cd22 100644 --- a/edx_repo_tools/data.py +++ b/edx_repo_tools/data.py @@ -6,7 +6,6 @@ logging.basicConfig() LOGGER = logging.getLogger(__name__) -OPEN_EDX_YAML = 'openedx.yaml' def iter_nonforks(hub, orgs): @@ -28,9 +27,9 @@ def iter_nonforks(hub, orgs): yield repo -def iter_openedx_yaml(hub, orgs, branches=None): +def iter_openedx_yaml(file_name, hub, orgs, branches=None): """ - Yield the data from all openedx.yaml files found in repositories in ``orgs`` + Yield the data from all catalog-info.yaml or openedx.yaml files found in repositories in ``orgs`` on any of ``branches``. Arguments: @@ -45,19 +44,20 @@ def iter_openedx_yaml(hub, orgs, branches=None): Repositories (:class:`~github3.Repository) """ + for repo in iter_nonforks(hub, orgs): for branch in (branches or [repo.default_branch]): try: - contents = repo.file_contents(OPEN_EDX_YAML, ref=branch) + contents = repo.file_contents(file_name, ref=branch) except NotFoundError: contents = None if contents is not None: - LOGGER.debug("Found openedx.yaml at %s:%s", repo.full_name, branch) + LOGGER.debug("Found %s at %s:%s", file_name, repo.full_name, branch) try: data = yaml.safe_load(contents.decoded) except Exception as exc: - LOGGER.error("Couldn't parse openedx.yaml from %s:%s, skipping repo", repo.full_name, branch, exc_info=True) + LOGGER.error("Couldn't parse %s from %s:%s, skipping repo", file_name, repo.full_name, branch, exc_info=True) else: if data is not None: yield repo, data diff --git a/edx_repo_tools/oep2/explode_repos_yaml.py b/edx_repo_tools/oep2/explode_repos_yaml.py index f83b1ad5..c96dac99 100644 --- a/edx_repo_tools/oep2/explode_repos_yaml.py +++ b/edx_repo_tools/oep2/explode_repos_yaml.py @@ -27,7 +27,7 @@ ) def implode(hub, org, branch): """ - Implode all openedx.yaml files, and print the results as formatted output. + Implode all catalog-info.yaml or openedx.yaml files, and print the results as formatted output. """ data = { repo.full_name: openedx_yaml diff --git a/edx_repo_tools/release/tag_release.py b/edx_repo_tools/release/tag_release.py index 9c1fcf1b..53f6fb58 100644 --- a/edx_repo_tools/release/tag_release.py +++ b/edx_repo_tools/release/tag_release.py @@ -52,6 +52,57 @@ def nice_tqdm(iterable, desc): return tqdm(iterable, desc=desc.ljust(27)) +def filter_repos(openedx_repo, catalog_repo): + """ + Return the subset of the repos with catalog-info.yaml file if they have 'openedx.org/release' section otherwise + with openedx.yaml file. + + Arguments: + openedx_repo (list of dictionaries): list of repos with data of openedx.yaml with `openedx-release` section. + catalog_repo (list of dictionaries): list of repos with data of catalog.yaml with 'openedx.org/release' section. + """ + result_dict = {} + for repo_key, openedx_data in openedx_repo.items(): + if repo_key in catalog_repo: + result_dict[repo_key] = catalog_repo[repo_key] + else: + result_dict[repo_key] = openedx_data + + return result_dict + + +def openedx_repos_with_catalog_info(hub, orgs=None, branches=None): + """ + Return a subset of the repos with catalog-info.yaml files: the repos + with an annotation `openedx.org/release`. + + Arguments: + hub (:class:`~github3.GitHub`): an authenticated GitHub instance. + orgs (list of str): The GitHub organizations to scan. Defaults to + OPENEDX_ORGS. + branches (list of str): The branches to scan in all repos in the selected + orgs, defaulting to the repo's default branch. + + Returns: + A dict from :class:`~github3.Repository` objects to openedx.yaml data for all of the + repos with an ``openedx-release`` key specified. + + """ + orgs = orgs or OPENEDX_ORGS + repos = {} + for repo, data in tqdm(iter_openedx_yaml('catalog-info.yaml', hub, orgs=orgs, branches=branches), desc='Find repos'): + + if 'metadata' in data: + annotations = data['metadata'].get('annotations') + if annotations: + # Check if 'openedx.org/release' is present in annotations + if 'openedx.org/release' in annotations: + repo = repo.refresh() + repos[repo] = data + + return repos + + def openedx_release_repos(hub, orgs=None, branches=None): """ Return a subset of the repos with openedx.yaml files: the repos @@ -71,9 +122,8 @@ def openedx_release_repos(hub, orgs=None, branches=None): """ orgs = orgs or OPENEDX_ORGS repos = {} - - for repo, data in tqdm(iter_openedx_yaml(hub, orgs=orgs, branches=branches), desc='Find repos'): - if data.get('openedx-release'): + for repo, data in tqdm(iter_openedx_yaml('openedx.yaml', hub, orgs=orgs, branches=branches), desc='Find repos'): + if data.get('openedx-release'): repo = repo.refresh() repos[repo] = data @@ -180,7 +230,10 @@ def override_repo_refs(repos, override_ref=None, overrides=None): for repo, repo_data in repos.items(): local_override = overrides.get(repo.full_name, override_ref) if local_override: - repo_data["openedx-release"]["ref"] = local_override + if "metadata" in repo_data: + repo_data["metadata"]["annotations"]["openedx.org/release"] = local_override + elif "openedx-release" in repo_data: + repo_data["openedx-release"]["ref"] = local_override return repos @@ -219,11 +272,19 @@ def commit_ref_info(repos, skip_invalid=False): } """ - ref_info = {} for repo, repo_data in nice_tqdm(repos.items(), desc='Find commits'): # are we specifying a ref? - ref = repo_data["openedx-release"].get("ref") + + if 'metadata' in repo_data: + annotations = repo_data['metadata'].get('annotations', {}) + ref = annotations.get('openedx.org/release') + + # Check if 'openedx-release' is present in repo_data and get the ref + # This check will be remove once we will just support catalog_info.yaml and remove openedx.yaml + elif 'openedx-release' in repo_data: + ref = repo_data['openedx-release'].get('ref') + if ref: try: ref_info[repo] = get_latest_commit_for_ref(repo, ref) @@ -742,19 +803,21 @@ def main(hub, ref, use_tag, override_ref, overrides, interactive, quiet, for r in json.load(f) } else: - repos = openedx_release_repos(hub, orgs, branches) + repos = filter_repos(openedx_release_repos(hub, orgs, branches), openedx_repos_with_catalog_info(hub, orgs, branches)) + if output_repos: with open(output_repos, "w") as f: dumped = [{"repo": repo.as_dict(), "data": data} for repo, data in repos.items()] json.dump(dumped, f, indent=2, sort_keys=True) if not repos: - raise ValueError("No repos marked for openedx-release in their openedx.yaml files!") + raise ValueError("No repos marked for openedx-release neither in openedx.yaml nor catalog_info.yaml files!") if included_repos: repos = include_only_repos(repos, included_repos) - repos = trim_skipped_repos(repos, skip_repos) - repos = trim_dependent_repos(repos) - repos = trim_indecisive_repos(repos) + if 'openedx-release' in repos: + repos = trim_skipped_repos(repos, skip_repos) + repos = trim_dependent_repos(repos) + repos = trim_indecisive_repos(repos) repos = override_repo_refs( repos, override_ref=override_ref, diff --git a/tests/test_tag_release.py b/tests/test_tag_release.py index 5562062b..50df71ca 100644 --- a/tests/test_tag_release.py +++ b/tests/test_tag_release.py @@ -88,6 +88,13 @@ def expected_repos(): 'ref': 'master', } }, + mock_repository('edx/edx-package'): { + 'metadata': { + 'annotations': { + 'openedx.org/release': 'master', + } + } + }, mock_repository('edx/XBlock'): { 'openedx-release': { 'parent-repo': 'edx/edx-platform', @@ -122,6 +129,14 @@ def expected_commits(): 'ref': 'master', 'message': 'commit message for configuration master commit', }, + mock_repository('edx/edx-package'): { + 'committer': {'name': 'Dev 2'}, + 'author': {'name': 'Dev 2'}, + 'sha': '12345deadbeef', + 'ref_type': 'branch', + 'ref': 'master', + 'message': 'commit message for configuration master commit', + }, mock_repository('edx/XBlock'): { 'committer': {'name': 'Dev 1'}, 'author': {'name': 'Dev 1'}, @@ -159,6 +174,7 @@ def test_get_ref_for_repos_not_exist(): repos = [ mock_repository('edx/edx-platform'), mock_repository('edx/configuration'), + mock_repository('edx/edx-package'), mock_repository('edx/XBlock'), ] for repo in repos: @@ -172,16 +188,20 @@ def test_commit_ref_info(expected_repos): edx_edx_platform = find_repo(expected_repos, 'edx/edx-platform') edx_configuration = find_repo(expected_repos, 'edx/configuration') + edx_edx_package = find_repo(expected_repos, 'edx/edx-package') edx_edx_platform.branch.assert_called_once_with('release') assert not edx_edx_platform.ref.called edx_configuration.branch.assert_called_once_with('master') assert not edx_configuration.ref.called + edx_edx_package.branch.assert_called_once_with('master') + assert not edx_edx_package.ref.called # The XBlock repo shouldn't have been examined at all. assert not find_repo(expected_repos, 'edx/XBlock').branch.called edx_platform_commit = edx_edx_platform.git_commit().refresh() configuration_commit = edx_configuration.git_commit().refresh() + edx_edx_package_commit = edx_edx_package.git_commit().refresh() expected_commits = { edx_edx_platform: { @@ -200,6 +220,14 @@ def test_commit_ref_info(expected_repos): 'ref': 'master', 'message': configuration_commit.message, }, + edx_edx_package: { + 'committer': edx_edx_package_commit.committer, + 'author': edx_edx_package_commit.author, + 'sha': edx_edx_package_commit.sha, + 'ref_type': 'branch', + 'ref': 'master', + 'message': edx_edx_package_commit.message, + }, } assert result == expected_commits @@ -214,6 +242,7 @@ def test_overrides_global_ref(expected_repos): result = override_repo_refs(expected_repos, override_ref="abcdef") assert find_repo_data(result, "edx/edx-platform")["openedx-release"]["ref"] == "abcdef" assert find_repo_data(result, "edx/configuration")["openedx-release"]["ref"] == "abcdef" + assert find_repo_data(result, "edx/edx-package")["metadata"]["annotations"]["openedx.org/release"] == "abcdef" def test_overrides_dict(expected_repos): @@ -221,10 +250,12 @@ def test_overrides_dict(expected_repos): "edx/edx-platform": "xyz", "edx/configuration": "refs/branch/no-way", "edx/does-not-exist": "does-not-matter", + "edx/edx-package": "release" } result = override_repo_refs(expected_repos, overrides=overrides) assert find_repo_data(result, "edx/edx-platform")["openedx-release"]["ref"] == "xyz" assert find_repo_data(result, "edx/configuration")["openedx-release"]["ref"] == "refs/branch/no-way" + assert find_repo_data(result, "edx/edx-package")["metadata"]["annotations"]["openedx.org/release"] == "release" def test_overrides_global_ref_and_dict(expected_repos): @@ -240,6 +271,7 @@ def test_overrides_global_ref_and_dict(expected_repos): ) assert find_repo_data(result, "edx/edx-platform")["openedx-release"]["ref"] == "xyz" assert find_repo_data(result, "edx/configuration")["openedx-release"]["ref"] == "fakie-mcfakerson" + assert find_repo_data(result, "edx/edx-package")["metadata"]["annotations"]["openedx.org/release"] == "fakie-mcfakerson" def test_create_happy_path(expected_commits):