diff --git a/Makefile b/Makefile index e81e5c1..4be9c8c 100644 --- a/Makefile +++ b/Makefile @@ -51,3 +51,7 @@ dist: clean ## create and check packages $(IN_VENV) python -m build $(IN_VENV) twine check dist/* ls -l dist + +format: ## Format Python code base + $(IN_VENV) isort . + $(IN_VENV) black . diff --git a/dev-requirements.txt b/dev-requirements.txt index 158bc4d..8390272 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -5,6 +5,8 @@ mypy ruff black +flake8 +isort # For release build diff --git a/galaxy_release_util/bootstrap_history.py b/galaxy_release_util/bootstrap_history.py index 6ee16b2..07db6ad 100644 --- a/galaxy_release_util/bootstrap_history.py +++ b/galaxy_release_util/bootstrap_history.py @@ -1,21 +1,26 @@ -# Little script to make HISTORY.rst more easy to format properly, lots TODO -# pull message down and embed, handle multiple, etc... - import calendar import datetime +import logging import os import re import string import sys import textwrap from pathlib import Path -from typing import Optional +from typing import ( + List, + Optional, + Set, +) import click +from github import GithubException +from github.Issue import Issue from github.PullRequest import PullRequest from packaging.version import Version from .cli.options import ( + ClickDate, ClickVersion, galaxy_root_option, group_options, @@ -32,8 +37,7 @@ strip_release, ) -RELEASE_DELTA_MONTHS = 4 # Number of months between releases. -MINOR_TO_MONTH = {0: 2, 1: 6, 2: 10} +OLDER_RELEASES_FILENAME = "older_releases.rst" TEMPLATE = """ @@ -74,7 +78,7 @@ ANNOUNCE_TEMPLATE = string.Template( """ =========================================================== -${month_name} 20${year} Galaxy Release (v ${release}) +${release} Galaxy Release (${month_name} ${year}) =========================================================== .. include:: _header.rst @@ -143,7 +147,7 @@ ANNOUNCE_USER_TEMPLATE = string.Template( """ =========================================================== -${month_name} 20${year} Galaxy Release (v ${release}) +${release} Galaxy Release (${month_name} ${year}) =========================================================== .. include:: _header.rst @@ -205,14 +209,8 @@ :orphan: =========================================================== -${month_name} 20${year} Galaxy Release (v ${version}) +${release} Galaxy Release =========================================================== - - -Schedule -=========================================================== - * Planned Freeze Date: ${freeze_date} - * Planned Release Date: ${release_date} """ ) @@ -223,17 +221,12 @@ RELEASE_ISSUE_TEMPLATE = string.Template( """ -- [ ] **Prep** - - - [X] ~~Create this release issue ``make release-issue``.~~ - - [X] ~~Set freeze date (${freeze_date}).~~ - - [ ] Verify that your installed version of `galaxy-release-util` is up-to-date. - - [ ] **Branch Release (on or around ${freeze_date})** - - [ ] Ensure all [blocking milestone pull requests](https://github.com/galaxyproject/galaxy/pulls?q=is%3Aopen+is%3Apr+milestone%3A${version}) have been merged, delayed, or closed. + - [ ] Verify that your installed version of `galaxy-release-util` is up-to-date. + - [ ] Ensure all [blocking milestone pull requests](https://github.com/galaxyproject/galaxy/pulls?q=is%3Aopen+is%3Apr+milestone%3A${version}) have been merged, closed, or postponed until the next release. - make release-check-blocking-prs + galaxy-release-util check-blocking-prs ${version} --release-date ${release_date} - [ ] Add latest database revision identifier (for ``release_${version}`` and ``${version}``) to ``REVISION_TAGS`` in ``galaxy/model/migrations/dbscript.py``. @@ -247,7 +240,8 @@ make release-create-rc - [ ] Open pull requests from your fork of branch ``version-${version}`` to upstream ``release_${version}`` and of ``version-${next_version}.dev`` to ``dev``. - - [ ] Update ``MILESTONE_NUMBER`` in the [maintenance bot](https://github.com/galaxyproject/galaxy/blob/dev/.github/workflows/maintenance_bot.yaml) to `${next_version}` so it properly tags new pull requests. + - [ ] [Create milestone](https://github.com/galaxyproject/galaxy/milestones) `${next_version}` for next release. + - [ ] Update ``MILESTONE_NUMBER`` in the [maintenance bot](https://github.com/galaxyproject/galaxy/blob/dev/.github/workflows/maintenance_bot.yaml) to reference `${next_version}` so it properly tags new pull requests. - [ ] **Issue Review Timeline Notes** @@ -258,10 +252,10 @@ - [ ] Update test.galaxyproject.org to ensure it is running the ``release_${version}`` branch. - [ ] Update testtoolshed.g2.bx.psu.edu to ensure it is running a dev at or past branch point (${freeze_date} + 1 day). - - [ ] Conduct release testing on test.galaxyproject.org. - - [ ] Deploy to usegalaxy.org (${freeze_date} + 1 week). - - [ ] Deploy to toolshed.g2.bx.psu.edu (${freeze_date} + 1 week). - - [ ] Conduct release testing on usegalaxy.org. + - [ ] Conduct first stage of release testing on test.galaxyproject.org. + - [ ] Upon completing release testing and fixing all critical bugs, deploy to usegalaxy.org. + - [ ] Deploy to toolshed.g2.bx.psu.edu. + - [ ] Conduct second stage of release testing on usegalaxy.org. - [ ] [Update BioBlend CI testing](https://github.com/galaxyproject/bioblend/blob/main/.github/workflows/test.yaml) to include a ``release_${version}`` target: add ``- release_${version}`` to the ``galaxy_version`` list in ``.github/workflows/test.yaml`` . - [ ] Update GALAXY_RELEASE in IUC and devteam github workflows - [ ] https://github.com/galaxyproject/tools-iuc/blob/master/.github/workflows/ @@ -269,13 +263,13 @@ - [ ] **Create Release Notes** - - [ ] Review merged pull requests and ensure they all have a milestone attached. [Link](https://github.com/galaxyproject/galaxy/pulls?utf8=%E2%9C%93&q=is%3Apr+is%3Amerged+no%3Amilestone+-label%3Amerge+) + - [ ] Review pull requests merged since `release_${previous_version}`, ensure their titles are properly formatted and they all have a `${version}` or `${next_version}` milestone attached. [Link](https://github.com/galaxyproject/galaxy/pulls?utf8=%E2%9C%93&q=is%3Apr+is%3Amerged+no%3Amilestone+-label%3Amerge+) - [ ] Switch to release branch and create a new branch for release notes git checkout release_${version} -b ${version}_release_notes - [ ] Bootstrap the release notes - make release-bootstrap-history RELEASE_CURR=${version} + galaxy-release-util create-changelog ${version} --release-date ${release_date} --next-version ${next_version} - [ ] Open newly created files and manually curate major topics and release notes. - [ ] Run ``python scripts/release-diff.py release_${previous_version}`` and add configuration changes to release notes. - [ ] Add new release to doc/source/releases/index.rst @@ -286,14 +280,16 @@ - [ ] Ensure all [blocking milestone issues](https://github.com/galaxyproject/galaxy/issues?q=is%3Aopen+is%3Aissue+milestone%3A${version}) have been resolved. - make release-check-blocking-issues RELEASE_CURR=${version} - - [ ] Ensure all [blocking milestone pull requests](https://github.com/galaxyproject/galaxy/pulls?q=is%3Aopen+is%3Apr+milestone%3A${version}) have been merged or closed. + galaxy-release-util check-blocking-issues ${version} + - [ ] Ensure all [blocking milestone pull requests](https://github.com/galaxyproject/galaxy/pulls?q=is%3Aopen+is%3Apr+milestone%3A${version}) have been merged, closed, or postponed until the next release. - make release-check-blocking-prs RELEASE_CURR=${version} - - [ ] Ensure all pull requests merged into the pre-release branch during the freeze have [milestones attached](https://github.com/galaxyproject/galaxy/pulls?q=is%3Apr+is%3Aclosed+base%3Arelease_${version}+is%3Amerged+no%3Amilestone) and that they are the not [${next_version} milestones](https://github.com/galaxyproject/galaxy/pulls?q=is%3Apr+is%3Aclosed+base%3Arelease_${version}+is%3Amerged+milestone%3A${next_version}) + galaxy-release-util check-blocking-prs ${version} --release-date ${release_date} + - [ ] Ensure all pull requests merged into the pre-release branch during the freeze have [milestones attached](https://github.com/galaxyproject/galaxy/pulls?q=is%3Apr+is%3Aclosed+base%3Arelease_${version}+is%3Amerged+no%3Amilestone) + - [ ] Ensure all pull requests merged into the pre-release branch during the freeze are the not [${next_version} milestones](https://github.com/galaxyproject/galaxy/pulls?q=is%3Apr+is%3Aclosed+base%3Arelease_${version}+is%3Amerged+milestone%3A${next_version}) + - [ ] Ensure there are no blocking pull requests that target the `release_${version}` branch but [do not have the `${version}` milestone attached](https://github.com/galaxyproject/galaxy/pulls?q=is%3Apr+base%3Arelease_${version}+-label%3Akind%2Fbug+-milestone%3A${version}). - [ ] Ensure release notes include all pull requests added during the freeze by re-running the release note bootstrapping: - make release-bootstrap-history + galaxy-release-util create-changelog ${version} --release-date ${release_date} --next-version ${next_version} - [ ] Ensure previous release is merged into current. [GitHub branch comparison](https://github.com/galaxyproject/galaxy/compare/release_${version}...release_${previous_version}) - [ ] Create and push release tag: @@ -316,17 +312,39 @@ - [ ] Email announcement to [galaxy-dev](http://dev.list.galaxyproject.org/) and [galaxy-announce](http://announce.list.galaxyproject.org/) @lists.galaxyproject.org. [An example](https://lists.galaxyproject.org/archives/list/galaxy-announce@lists.galaxyproject.org/thread/ISB7ZNBDY3LQMC2KALGPVQ3DEJTH657Q/). - [ ] Adjust http://getgalaxy.org text and links to match current master branch by opening a PR at https://github.com/galaxyproject/galaxy-hub/ -- [ ] **Prepare for next release** +- [ ] **Complete release** - [ ] Close milestone ``${version}`` and ensure milestone ``${next_version}`` exists. - - [ ] Create release issue for next version ``make release-issue``. - - [ ] Schedule committer meeting to discuss re-alignment of priorities. - [ ] Close this issue. """ # noqa: E501 ) release_version_argument = click.argument("release-version", type=ClickVersion()) +next_version_option = click.option( + "--next-version", + type=ClickVersion(), + help="Next release version", +) + +freeze_date_option = click.option( + "--freeze-date", + type=ClickDate(), + required=True, +) + +release_date_option = click.option( + "--release-date", + type=ClickDate(), + required=True, +) + +dry_run_option = click.option( + "--dry-run", type=bool, default=False, help="Do not connect to GitHub's API, print out output" +) + +log = logging.getLogger(__name__) + @click.group(help="Subcommands of this script can perform various tasks around creating Galaxy releases") def cli(): @@ -334,93 +352,108 @@ def cli(): @cli.command(help="Create release checklist issue on GitHub") -@group_options(release_version_argument, galaxy_root_option) -def create_release_issue(release_version: Version, galaxy_root: Path): - previous_release = _previous_release(galaxy_root, release_version) - new_version_params = _next_version_params(release_version) - next_version = new_version_params["version"] - freeze_date, _ = _release_dates(release_version) - release_issue_template_params = dict( +@group_options( + release_version_argument, + next_version_option, + freeze_date_option, + galaxy_root_option, + release_date_option, + dry_run_option, +) +def create_release_issue( + release_version: Version, + next_version: Version, + freeze_date: datetime.date, + galaxy_root: Path, + release_date: datetime.date, + dry_run: bool, +): + previous_version = _get_previous_release_version(galaxy_root, release_version) + next_version = next_version or _get_next_release_version(release_version) + assert next_version > release_version, "Next release version should be greater than release version" + + issue_template_params = dict( version=release_version, next_version=next_version, - previous_version=previous_release, + previous_version=previous_version, freeze_date=freeze_date, + release_date=release_date, ) - release_issue_contents = RELEASE_ISSUE_TEMPLATE.safe_substitute(**release_issue_template_params) - github = github_client() - repo = github.get_repo(f"{PROJECT_OWNER}/{PROJECT_NAME}") - release_issue = repo.create_issue( - title=f"Publication of Galaxy Release v {release_version}", - body=release_issue_contents, - ) - return release_issue - + issue_contents = RELEASE_ISSUE_TEMPLATE.substitute(**issue_template_params) + issue_title = f"Publication of Galaxy Release v {release_version}" -@cli.command(help="Create or update release changelog") -@group_options(release_version_argument, galaxy_root_option) -def create_changelog(release_version: Version, galaxy_root: Path): - release_file = _release_file(galaxy_root, str(release_version) + ".rst") - enhancement_targets = "\n\n".join(f".. enhancement_tag_{a}" for a in GROUPED_TAGS.values()) - bug_targets = "\n\n".join(f".. bug_tag_{a}" for a in GROUPED_TAGS.values()) - template = TEMPLATE - template = template.replace(".. enhancement", f"{enhancement_targets}\n\n.. enhancement") - template = template.replace(".. bug", f"{bug_targets}\n\n.. bug") - release_info = string.Template(template).safe_substitute(release=release_version) - _write_file(release_file, release_info, skip_if_exists=True) - month = MINOR_TO_MONTH[release_version.minor] - month_name = calendar.month_name[month] - year = release_version.major - - announce_info = ANNOUNCE_TEMPLATE.substitute(month_name=month_name, year=year, release=release_version) - announce_file = _release_file(galaxy_root, str(release_version) + "_announce.rst") - _write_file(announce_file, announce_info, skip_if_exists=True) - - announce_user_info = ANNOUNCE_USER_TEMPLATE.substitute(month_name=month_name, year=year, release=release_version) - announce_user_file = _release_file(galaxy_root, str(release_version) + "_announce_user.rst") - _write_file(announce_user_file, announce_user_info, skip_if_exists=True) - - prs_file = _release_file(galaxy_root, str(release_version) + "_prs.rst") - seen_prs = set() + if dry_run: + print(issue_title) + print(issue_contents) + return None try: - with open(prs_file) as fh: - seen_prs = set(map(int, re.findall(r"\.\. _Pull Request (\d+): https", fh.read()))) - except FileNotFoundError: - pass - _write_file(prs_file, PRS_TEMPLATE, skip_if_exists=True) - - next_version_params = _next_version_params(release_version) - next_version = next_version_params["version"] - next_release_file = _release_file(galaxy_root, str(next_version) + "_announce.rst") - - next_announce = NEXT_TEMPLATE.substitute(**next_version_params) - with open(next_release_file, "w") as fh: - fh.write(next_announce) - releases_index = _release_file(galaxy_root, "index.rst") - releases_index_contents = _read_file(releases_index) - releases_index_contents = releases_index_contents.replace( - ".. announcements\n", - ".. announcements\n " + str(next_version) + "_announce\n", - ) - _write_file(releases_index, releases_index_contents, skip_if_exists=True) + github = github_client() + repo = github.get_repo(f"{PROJECT_OWNER}/{PROJECT_NAME}") + release_issue = repo.create_issue( + title=issue_title, + body=issue_contents, + ) + return release_issue + except GithubException: + log.exception( + "Failed to create an issue on GitHub. You need to be authenticated to use GitHub API." + "\nSee galaxy_release_util/github_client.py" + ) - for pr in _get_prs(str(release_version)): - if pr.number not in seen_prs: - pr_to_doc( - galaxy_root=galaxy_root, - release_version=release_version, - pr=pr, - ) + +@cli.command(help="Create or update release changelog") +@group_options(release_version_argument, next_version_option, galaxy_root_option, release_date_option) +def create_changelog(release_version: Version, next_version: Version, galaxy_root: Path, release_date: datetime.date): + + def create_release_file() -> None: + enhancement_targets = "\n\n".join(f".. enhancement_tag_{value}" for value in GROUPED_TAGS.values()) + bug_targets = "\n\n".join(f".. bug_tag_{value}" for value in GROUPED_TAGS.values()) + + content = string.Template(TEMPLATE).substitute(release=release_version) + content = content.replace(".. enhancement", f"{enhancement_targets}\n\n.. enhancement") + content = content.replace(".. bug", f"{bug_targets}\n\n.. bug") + filename = _release_file(galaxy_root, f"{release_version}.rst") + _write_file(filename, content, skip_if_exists=True) + + def create_announcement_file() -> None: + month = calendar.month_name[release_date.month] + year = release_date.year + content = ANNOUNCE_TEMPLATE.substitute(month_name=month, year=year, release=release_version) + filename = _release_file(galaxy_root, f"{release_version}_announce.rst") + _write_file(filename, content, skip_if_exists=False) + + def create_user_announcement_file() -> None: + month = calendar.month_name[release_date.month] + year = release_date.year + content = ANNOUNCE_USER_TEMPLATE.substitute(month_name=month, year=year, release=release_version) + filename = _release_file(galaxy_root, f"{release_version}_announce_user.rst") + _write_file(filename, content, skip_if_exists=True) + + def create_prs_file() -> None: + _write_file(_get_prs_file(galaxy_root, release_version), PRS_TEMPLATE, skip_if_exists=True) + + def create_next_release_announcement_file() -> None: + content = NEXT_TEMPLATE.substitute(release=next_version) + filename = _release_file(galaxy_root, f"{next_version}_announce.rst") + _write_file(filename, content, skip_if_exists=True) + + next_version = next_version or _get_next_release_version(release_version) + create_release_file() + create_announcement_file() + create_user_announcement_file() + create_prs_file() + create_next_release_announcement_file() + _load_prs(galaxy_root, release_version, release_date) @cli.command(help="List release blocking PRs") -@group_options(release_version_argument) -def check_blocking_prs(release_version: Version): - block = False - for pr in _get_prs(str(release_version), state="open"): +@group_options(release_version_argument, release_date_option) +def check_blocking_prs(release_version: Version, release_date: datetime.date): + block = 0 + for pr in _get_prs(release_version, release_date, state="open"): click.echo(f"Blocking PR| {_pr_to_str(pr)}", err=True) - block = True - if block: - sys.exit(1) + block = 1 + sys.exit(block) @cli.command(help="List release blocking issues") @@ -431,162 +464,179 @@ def check_blocking_issues(release_version: Version): repo = github.get_repo(f"{PROJECT_OWNER}/{PROJECT_NAME}") issues = repo.get_issues(state="open") for issue in issues: - # issue can also be a pull request, which could be filtered out with `not issue.pull_request` if ( issue.milestone and issue.milestone.title == str(release_version) and "Publication of Galaxy Release" not in issue.title + and not issue.pull_request ): click.echo(f"Blocking issue| {_issue_to_str(issue)}", err=True) block = 1 - sys.exit(block) -def _issue_to_str(pr): - if isinstance(pr, str): - return pr - return f"Issue #{pr.number} ({pr.title}) {pr.html_url}" - - -def _next_version_params(release_version: Version): - # we'll just hardcode this to 3 "minor" versions per year - if release_version.minor < 2: - next_major = release_version.major - next_minor = release_version.minor + 1 - else: - next_major = release_version.major + 1 - next_minor = 0 - next_month_name = calendar.month_name[MINOR_TO_MONTH[next_minor]] - next_version = Version(f"{next_major}.{next_minor}") - freeze_date, release_date = _release_dates(next_version) - return dict( - version=next_version, - year=next_major, - month_name=next_month_name, - freeze_date=freeze_date, - release_date=release_date, - ) +def _get_prs_file(galaxy_root: Path, release_version: Version) -> Path: + return _release_file(galaxy_root, f"{release_version}_prs.rst") -def _release_dates(version: Version): - # hardcoded to 3 releases a year, freeze dates kind of random - year = version.major - month = MINOR_TO_MONTH[version.minor] - first_of_month = datetime.date(year + 2000, month, 1) - freeze_date = next_weekday(first_of_month, 0) - release_date = next_weekday(first_of_month, 0) + datetime.timedelta(21) - return freeze_date, release_date +def _load_prs(galaxy_root: Path, release_version: Version, release_date: datetime.date) -> None: + def get_prs_from_prs_file() -> Set[int]: + with open(_get_prs_file(galaxy_root, release_version)) as fh: + return set(map(int, re.findall(r"\.\. _Pull Request (\d+): https", fh.read()))) -def _get_prs(release_version: str, state="closed"): + seen_prs = get_prs_from_prs_file() + prs = _get_prs(release_version, release_date) + n_prs = len(prs) + for i, pr in enumerate(prs): + if pr.number not in seen_prs: + print(f"Processing PR {i + 1} of {n_prs}") + _pr_to_doc( + galaxy_root=galaxy_root, + release_version=release_version, + pr=pr, + ) + else: + print(f"Skipping PR {i + 1} of {n_prs} (previously processed)") + + +def _get_prs(release_version: Version, release_date: datetime.date, state: str = "closed") -> List[PullRequest]: github = github_client() repo = github.get_repo(f"{PROJECT_OWNER}/{PROJECT_NAME}") - pull_requests = repo.get_pulls(state=state) - reached_old_prs = False - for pr in pull_requests: - if reached_old_prs: + # A pull request that was last updated before the previous release branch was created cannot be part of this release: + # the value of `updated_at` is updated on merge, so it had to be merged before the branch existed and, therefore, included in the previous release. + # Given that we can't reliably determine when the previous release branch was created, we subtract a year from + # the planned release date of the current release and use that as a cutoff date. For example, if the planned release date is August 1, 2025, + # we will not consider pull requests that were last updated before August 1, 2024. This is based on the assumption that + # there have to be at least two release branches created within a year: the previous release + the current release. + _cutoff_date = release_date.replace(year=release_date.year - 1) + cutoff_time = datetime.datetime.combine(_cutoff_date, datetime.time.min) + + prs: List[PullRequest] = [] + counter = 0 + print("Collecting relevant pull requests...") + for pr in repo.get_pulls(state=state, sort="updated", direction="desc"): + assert pr.updated_at + if pr.updated_at.replace(tzinfo=None) < cutoff_time: break + counter += 1 + if counter % 100 == 0: + print( + f"Examined {counter} PRs; collected {len(prs)} (currently on #{pr.number} updated on {pr.updated_at.date()})" + ) + # Select PRs that are merged + have correct milestone + have not been previously collected and added to the prs file + proper_state = state != "closed" or pr.merged_at # open PRs or PRs that have been merged + if proper_state and pr.milestone and pr.milestone.title == str(release_version): + prs.append(pr) - if pr.created_at.replace(tzinfo=None) < datetime.datetime(2020, 5, 1, 0, 0): - reached_old_prs = True - pass - merged_at = pr.merged_at - milestone = pr.milestone - proper_state = state != "closed" or merged_at - if not proper_state or not milestone or milestone.title != release_version: - continue - yield pr - + print(f"Collected {len(prs)} pull requests") + return prs -def pr_to_doc(galaxy_root: Path, release_version: Version, pr: PullRequest): - history_path = _release_file(galaxy_root, str(release_version)) + ".rst" - user_announce_path = history_path[0 : -len(".rst")] + "_announce_user.rst" - prs_path = history_path[0 : -len(".rst")] + "_prs.rst" - history = _read_file(history_path) - user_announce = _read_file(user_announce_path) - prs_content = _read_file(prs_path) +def _pr_to_doc(galaxy_root: Path, release_version: Version, pr: PullRequest) -> None: - def extend_target(target, line, source=history): + def extend_target(target: str, line: str, source: str) -> str: from_str = f".. {target}\n" if target not in source: raise Exception(f"Failed to find target [{target}] in source [{source}]") - return source.replace(from_str, from_str + line + "\n") - - text_target = "to_doc" - to_doc = pr.title.rstrip(".") + " " - - owner = None - user = pr.user - owner = user.login - text = f".. _Pull Request {pr.number}: {PROJECT_URL}/pull/{pr.number}" - prs_content = extend_target("github_links", text, prs_content) - to_doc += f"\n(thanks to `@{owner} `__)." - to_doc += f"\n`Pull Request {pr.number}`_" - labels = _pr_to_labels(pr) - text_target = _text_target(pr) - - to_doc = wrap(to_doc) - if text_target is not None: - history = extend_target(text_target, to_doc, history) - if "area/datatypes" in labels: - user_announce = extend_target("datatypes", to_doc, user_announce) - if "area/visualizations" in labels: - user_announce = extend_target("visualizations", to_doc, user_announce) - if "area/tools" in labels: - user_announce = extend_target("tools", to_doc, user_announce) - _write_file(history_path, history) - _write_file(prs_path, prs_content) - _write_file(user_announce_path, user_announce) - - -def _read_file(path): + return source.replace(from_str, f"{from_str}{line}\n") + + def extend_prs_file_content(filename: Path) -> None: + content = _read_file(filename) + text = f".. _Pull Request {pr.number}: {PROJECT_URL}/pull/{pr.number}" + content = extend_target("github_links", text, content) + _write_file(filename, content) + + def extend_release_file_content(filename: Path) -> None: + content = _read_file(filename) + text_target = _text_target(pr) + if text_target is not None: + content = extend_target(text_target, to_doc, content) + _write_file(filename, content) + + def extend_user_announce_file_content(filename: Path) -> None: + content = _read_file(filename) + labels = _pr_to_labels(pr) + if "area/datatypes" in labels: + content = extend_target("datatypes", to_doc, content) + if "area/visualizations" in labels: + content = extend_target("visualizations", to_doc, content) + if "area/tools" in labels: + content = extend_target("tools", to_doc, content) + _write_file(filename, content) + + def make_pr_to_doc() -> str: + to_doc = pr.title.rstrip(".") + " " + to_doc += f"\n(thanks to `@{pr.user.login} `__)." + to_doc += f"\n`Pull Request {pr.number}`_" + return wrap(to_doc) + + to_doc = make_pr_to_doc() + + filename = _release_file(galaxy_root, f"{release_version}.rst") + extend_release_file_content(filename) + + filename = _release_file(galaxy_root, f"{release_version}_announce_user.rst") + extend_user_announce_file_content(filename) + + filename = _release_file(galaxy_root, f"{release_version}_prs.rst") + extend_prs_file_content(filename) + + +def _read_file(path: Path) -> str: with open(path) as f: return f.read() -def _write_file(path, contents, skip_if_exists=False): +def _write_file(path: Path, contents: str, skip_if_exists: bool = False) -> None: if skip_if_exists and os.path.exists(path): return with open(path, "w") as f: f.write(contents) -def _previous_release(galaxy_root, to: Version): - previous_release = None - for release in _releases(galaxy_root): - if release == str(to): - break +def _get_next_release_version(version: Version) -> Version: + return Version(f"{version.major}.{version.minor + 1}") - previous_release = release - return previous_release +def _get_previous_release_version(galaxy_root: Path, version: Version) -> Optional[Version]: + """Return previous release version if it exists.""" + # NOTE: We convert strings to Version objects to compare apples to apples: + # str(Version(foo)) is not the same as the string foo: str(Version("22.05")) == "22.5" + prev = None + for release in _get_release_version_strings(galaxy_root): + release_version = Version(release) + if release_version >= version: + return prev + prev = release_version + return prev -def _releases(galaxy_root): - releases_path = galaxy_root / "doc" / "source" / "releases" - all_files = sorted(os.listdir(releases_path)) - release_note_file_pattern = re.compile(r"\d+\.\d+.rst") - release_note_files = [f for f in all_files if release_note_file_pattern.match(f)] - return sorted(f.rstrip(".rst") for f in release_note_files) +def _get_release_version_strings(galaxy_root: Path) -> List[str]: + """Return sorted list of release version strings.""" + all_files = _get_release_documentation_filenames(galaxy_root) + release_notes_file_pattern = re.compile(r"\d+\.\d+.rst") + filenames = [f.rstrip(".rst") for f in all_files if release_notes_file_pattern.match(f)] + return sorted(filenames) -def _release_file(galaxy_root: Path, release: Optional[str]) -> str: +def _get_release_documentation_filenames(galaxy_root: Path) -> List[str]: + """Return contents of release documentation directory.""" releases_path = galaxy_root / "doc" / "source" / "releases" - if release is None: - release = sorted(os.listdir(releases_path))[-1] - history_path = os.path.join(releases_path, release) - return history_path + if not os.path.exists(releases_path): + msg = f"Path to releases documentation not found: {releases_path}. If you are running this script outside of galaxy root directory, you should specify the '--galaxy-root' argument" + raise Exception(msg) + return sorted(os.listdir(releases_path)) -def get_first_sentence(message: str) -> str: - first_line = message.split("\n")[0] - return first_line +def _release_file(galaxy_root: Path, filename: Optional[str]) -> Path: + """Construct and return path to a release documentation file.""" + filename = filename or OLDER_RELEASES_FILENAME + return galaxy_root / "doc" / "source" / "releases" / filename -def process_sentence(message): +def _process_sentence(message: str) -> str: # Strip tags like [15.07]. message = strip_release(message=message) # Link issues and pull requests... @@ -595,8 +645,8 @@ def process_sentence(message): return message -def wrap(message): - message = process_sentence(message) +def wrap(message: str) -> str: + message = _process_sentence(message) wrapper = textwrap.TextWrapper(initial_indent="* ") wrapper.subsequent_indent = " " wrapper.width = 160 @@ -607,9 +657,7 @@ def wrap(message): return first_lines + ("\n" + rest_lines if rest_lines else "") -def next_weekday(d, weekday): - """Return the next week day (0 for Monday, 6 for Sunday) starting from ``d``.""" - days_ahead = weekday - d.weekday() - if days_ahead <= 0: # Target day already happened this week - days_ahead += 7 - return d + datetime.timedelta(days_ahead) +def _issue_to_str(issue: Issue) -> str: + if isinstance(issue, str): + return issue + return f"Issue #{issue.number} ({issue.title}) {issue.html_url}" diff --git a/galaxy_release_util/cli/options.py b/galaxy_release_util/cli/options.py index 044b0b8..964e8ec 100644 --- a/galaxy_release_util/cli/options.py +++ b/galaxy_release_util/cli/options.py @@ -1,3 +1,4 @@ +import datetime import pathlib from typing import ( Any, @@ -15,6 +16,7 @@ "--galaxy-root", type=click.Path(exists=True, file_okay=False, resolve_path=True, path_type=pathlib.Path), default=".", + help="Path to galaxy root.", ) @@ -33,5 +35,15 @@ class ClickVersion(click.ParamType): def convert(self, value: Any, param: Optional[Parameter], ctx: Optional[Context]) -> Version: try: return Version(value) - except Exception as e: + except ValueError as e: self.fail(f"{value!r} is not a valid PEP440 version number: {str(e)}", param, ctx) + + +class ClickDate(click.ParamType): + name = "date" + + def convert(self, value: Any, param: Optional[Parameter], ctx: Optional[Context]) -> datetime.date: + try: + return datetime.datetime.strptime(value, "%Y-%m-%d").date() + except ValueError as e: + self.fail(f"{value!r} is not a valid date: {str(e)}", param, ctx) diff --git a/pyproject.toml b/pyproject.toml index 417e723..d30adfd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,10 +3,24 @@ requires = ["setuptools"] build-backend = "setuptools.build_meta" [tool.black] -target-version = ['py37'] +target-version = ['py38'] line-length = 120 [tool.ruff] +target-version = "py38" line-length = 120 # Never enforce `E501`, enforced by black lint.ignore = ["E501"] + +[tool.isort] +combine_as_imports = true +force_alphabetical_sort_within_sections = true +# Override force_grid_wrap value from profile=black, but black is still happy +force_grid_wrap = 2 +# Same line length as for black +line_length = 120 +no_lines_before = "LOCALFOLDER" +profile = "black" +reverse_relative = true +skip_gitignore = true +src_paths = ["galaxy_release_util", "tests"] diff --git a/setup.cfg b/setup.cfg index c365a96..22d87bb 100644 --- a/setup.cfg +++ b/setup.cfg @@ -9,11 +9,11 @@ classifiers = Natural Language :: English Operating System :: POSIX Programming Language :: Python :: 3 - Programming Language :: Python :: 3.7 Programming Language :: Python :: 3.8 Programming Language :: Python :: 3.9 Programming Language :: Python :: 3.10 Programming Language :: Python :: 3.11 + Programming Language :: Python :: 3.12 Topic :: Software Development Topic :: Software Development :: Code Generators Topic :: Software Development :: Testing @@ -34,13 +34,14 @@ include_package_data = True install_requires = build click + python-dateutil docutils packaging PyGithub requests twine packages = find: -python_requires = >=3.7 +python_requires = >=3.8 [options.packages.find] exclude = diff --git a/tests/test_bootstrap_history.py b/tests/test_bootstrap_history.py new file mode 100644 index 0000000..743a79f --- /dev/null +++ b/tests/test_bootstrap_history.py @@ -0,0 +1,115 @@ +import os +from pathlib import Path + +import pytest +from click.testing import CliRunner +from packaging.version import Version + +from galaxy_release_util import bootstrap_history +from galaxy_release_util.bootstrap_history import ( # _get_release_date, + _get_next_release_version, + _get_previous_release_version, + _get_release_version_strings, + create_changelog, +) + + +@pytest.fixture +def release_files_dir(): + return Path(".") / "tests" / "test_data" + + +@pytest.fixture +def release_file(release_files_dir): + with open(release_files_dir / "98.2.rst") as f: + return f.read() + + +@pytest.fixture +def announcement_file(release_files_dir): + with open(release_files_dir / "98.2_announce.rst") as f: + return f.read() + + +@pytest.fixture +def user_announcement_file(release_files_dir): + with open(release_files_dir / "98.2_announce_user.rst") as f: + return f.read() + + +@pytest.fixture +def next_release_announcement_file(release_files_dir): + with open(release_files_dir / "99.0_announce.rst") as f: + return f.read() + + +@pytest.fixture +def prs_file(release_files_dir): + with open(release_files_dir / "98.2_prs.rst") as f: + return f.read() + + +def test_get_previous_release_version(monkeypatch): + monkeypatch.setattr( + bootstrap_history, "_get_release_version_strings", lambda x: sorted(["22.01", "22.05", "23.0", "23.1"]) + ) + + assert _get_previous_release_version(None, Version("15.1")) is None + assert _get_previous_release_version(None, Version("22.01")) is None + assert _get_previous_release_version(None, Version("22.05")) == Version("22.01") + assert _get_previous_release_version(None, Version("23.0")) == Version("22.05") + assert _get_previous_release_version(None, Version("23.1")) == Version("23.0") + assert _get_previous_release_version(None, Version("23.2")) == Version("23.1") + assert _get_previous_release_version(None, Version("99.99")) == Version("23.1") + + +def test_get_next_release_version(): + assert _get_next_release_version(Version("25.0")) == Version("25.1") + assert _get_next_release_version(Version("26.1")) == Version("26.2") + + +def test_get_release_version_strings(monkeypatch): + filenames = [ + "15.0.not_rst", + "22.01.rst", + "22.05.rst", + "23.0.rst", + "23.1.rst", + "23.not_a_release.rst", + "not_a_release.23.rst", + ] + monkeypatch.setattr(bootstrap_history, "_get_release_documentation_filenames", lambda x: sorted(filenames)) + assert _get_release_version_strings(None) == ["22.01", "22.05", "23.0", "23.1"] + + +def test_create_changelog( + monkeypatch, + release_file, + announcement_file, + user_announcement_file, + prs_file, + next_release_announcement_file, +): + monkeypatch.setattr( + bootstrap_history, "_load_prs", lambda x, y, z: None + ) # We don't want to call github's API on test data. + runner = CliRunner() + with runner.isolated_filesystem(): + os.makedirs("doc/source/releases") + result = runner.invoke( + create_changelog, ["98.2", "--galaxy-root", ".", "--release-date", "2099-1-15", "--next-version", "99.0"] + ) # version 98.2 to be released on January 15, 2099 + assert result.exit_code == 0 + + releases_path = Path("doc") / "source" / "releases" + + with open(releases_path / "98.2.rst") as f: + assert f.read() == release_file + with open(releases_path / "98.2_announce.rst") as f: + assert f.read() == announcement_file + with open(releases_path / "98.2_announce_user.rst") as f: + assert f.read() == user_announcement_file + with open(releases_path / "98.2_prs.rst") as f: + assert f.read() == prs_file + with open(releases_path / "99.0_announce.rst") as f: + assert f.read() == next_release_announcement_file diff --git a/tests/test_data/98.2.rst b/tests/test_data/98.2.rst new file mode 100644 index 0000000..eb9cfcd --- /dev/null +++ b/tests/test_data/98.2.rst @@ -0,0 +1,61 @@ + +.. to_doc + +98.2 +=============================== + +.. announce_start + +Enhancements +------------------------------- + +.. major_feature + + +.. feature + +.. enhancement_tag_viz + +.. enhancement_tag_datatypes + +.. enhancement_tag_tools + +.. enhancement_tag_workflows + +.. enhancement_tag_ui + +.. enhancement_tag_jobs + +.. enhancement_tag_admin + +.. enhancement + +.. small_enhancement + + + +Fixes +------------------------------- + +.. major_bug + + +.. bug_tag_viz + +.. bug_tag_datatypes + +.. bug_tag_tools + +.. bug_tag_workflows + +.. bug_tag_ui + +.. bug_tag_jobs + +.. bug_tag_admin + +.. bug + + +.. include:: 98.2_prs.rst + diff --git a/tests/test_data/98.2_announce.rst b/tests/test_data/98.2_announce.rst new file mode 100644 index 0000000..4a5afcf --- /dev/null +++ b/tests/test_data/98.2_announce.rst @@ -0,0 +1,65 @@ + +=========================================================== +98.2 Galaxy Release (January 2099) +=========================================================== + +.. include:: _header.rst + +Highlights +=========================================================== + +Feature1 +-------- + +Feature description. + +Feature2 +-------- + +Feature description. + +Feature3 +-------- + +Feature description. + +Also check out the `98.2 user release notes <98.2_announce_user.html>`__. +Are you an admin? Check out `some admin relevant PRs `__. + +Get Galaxy +=========================================================== + +The code lives at `GitHub `__ and you should have `Git `__ to obtain it. + +To get a new Galaxy repository run: + .. code-block:: shell + + $ git clone -b release_98.2 https://github.com/galaxyproject/galaxy.git + +To update an existing Galaxy repository run: + .. code-block:: shell + + $ git fetch origin && git checkout release_98.2 && git pull --ff-only origin release_98.2 + +See the `community hub `__ for additional details on source code locations. + + +Administration Notes +=========================================================== +Add content or drop section. + +Configuration Changes +=========================================================== +Add content or drop section. + +Deprecation Notices +=========================================================== +Add content or drop section. + +Release Notes +=========================================================== + +.. include:: 98.2.rst + :start-after: announce_start + +.. include:: _thanks.rst diff --git a/tests/test_data/98.2_announce_user.rst b/tests/test_data/98.2_announce_user.rst new file mode 100644 index 0000000..e26e647 --- /dev/null +++ b/tests/test_data/98.2_announce_user.rst @@ -0,0 +1,56 @@ + +=========================================================== +98.2 Galaxy Release (January 2099) +=========================================================== + +.. include:: _header.rst + +Highlights +=========================================================== + +Feature1 +-------- + +Feature description. + +Feature2 +-------- + +Feature description. + +Feature3 +-------- + +Feature description. + + +Visualizations +=========================================================== + +.. visualizations + +Datatypes +=========================================================== + +.. datatypes + +Builtin Tool Updates +=========================================================== + +.. tools + +Release Testing Team +=========================================================== + +A special thanks to the release testing team for testing many of the new features and reporting many bugs: + + + +Release Notes +=========================================================== + +Please see the :doc:`full release notes <98.2_announce>` for more details. + +.. include:: 98.2_prs.rst + +.. include:: _thanks.rst diff --git a/tests/test_data/98.2_prs.rst b/tests/test_data/98.2_prs.rst new file mode 100644 index 0000000..7b86841 --- /dev/null +++ b/tests/test_data/98.2_prs.rst @@ -0,0 +1,2 @@ + +.. github_links diff --git a/tests/test_data/99.0_announce.rst b/tests/test_data/99.0_announce.rst new file mode 100644 index 0000000..7abfe1f --- /dev/null +++ b/tests/test_data/99.0_announce.rst @@ -0,0 +1,6 @@ + +:orphan: + +=========================================================== +99.0 Galaxy Release +=========================================================== diff --git a/tox.ini b/tox.ini index 51b79c8..404d82c 100644 --- a/tox.ini +++ b/tox.ini @@ -2,7 +2,7 @@ source_dir = galaxy_release_util test_dir = tests isolated_build = True -envlist = py37, py38, py39, py310, py311 +envlist = py38, py39, py310, py311, py312 [gh] python = @@ -16,7 +16,7 @@ python = commands = lint: flake8 --ignore E203,E501,D,W503 --max-line-length 120 --exclude .tox,.venv,build,dist,.git . lint: black --check --diff . - lint: ruff . + lint: ruff check . unit: pytest {env:PYTEST_FAIL_FAIL:} {env:PYTEST_CAPTURE:} -m {env:PYTEST_MARK:""} {env:PYTEST_TARGET:{[tox]test_dir}} {posargs} mypy: mypy --install-types --non-interactive {[tox]source_dir} tests/ {posargs}