From 7960fe5e2f95e3977f3d2625818eed336435b45e Mon Sep 17 00:00:00 2001 From: Rahul Arya Date: Tue, 17 Aug 2021 22:42:35 -0700 Subject: [PATCH 1/4] Build from berkeley-cs61a directly --- buildserver/app_config.py | 75 +++++++++++++--------------- buildserver/build.py | 51 +++++++++++-------- buildserver/conf.py | 3 +- buildserver/env.py | 1 - buildserver/external_build_worker.py | 22 -------- buildserver/external_repo_utils.py | 7 --- buildserver/github_utils.py | 4 +- buildserver/main.py | 19 +++---- buildserver/target_determinator.py | 69 +++++++++++++++---------- buildserver/worker.py | 65 ++++++++++++------------ 10 files changed, 150 insertions(+), 166 deletions(-) delete mode 100644 buildserver/env.py delete mode 100644 buildserver/external_build_worker.py diff --git a/buildserver/app_config.py b/buildserver/app_config.py index 8064f945..a5755477 100644 --- a/buildserver/app_config.py +++ b/buildserver/app_config.py @@ -29,14 +29,18 @@ class Config(TypedDict): - build_type: Literal[ - "create_react_app", - "oh_queue", - "webpack", - "61a_website", - "hugo", - "jekyll", - "none", + target: str + match: List[str] + build_type: Optional[ + Literal[ + "create_react_app", + "oh_queue", + "webpack", + "61a_website", + "hugo", + "jekyll", + "none", + ] ] deploy_type: Literal[ "flask", @@ -90,39 +94,28 @@ class App: # updated by deploy.py, since PyPI takes a while to update deployed_pypi_version: Optional[str] - def __init__(self, name: str): - self.name = name - self.deployed_pypi_version = None - with tmp_directory(): - try: - with open(f"{name}/deploy.yaml") as f: - self.config = Config(**yaml.safe_load(f)) - self.config["build_image"] = self.config.get("build_image", None) - self.config["cpus"] = self.config.get("cpus", 1) - self.config["memory_limit"] = self.config.get( - "memory_limit", "256M" - ) - self.config["first_party_domains"] = self.config.get( - "first_party_domains", [f"{name}.cs61a.org"] - ) - self.config["concurrency"] = self.config.get("concurrency", 80) - self.config["tasks"] = self.config.get("tasks", []) - self.config["dependencies"] = self.config.get("dependencies", []) - self.config["package_name"] = self.config.get("package_name", name) - self.config["static_consumers"] = self.config.get( - "static_consumers", [] - ) - self.config["repo"] = self.config.get("repo") - self.config["service"] = self.config.get("service") - self.config["pr_consumers"] = self.config.get( - "pr_consumers", [name] - ) - self.config["permissions"] = self.config.get( - "permissions", ["rpc", "database"] - ) - except FileNotFoundError: - # app has been deleted in PR - self.config = None + def __init__(self, name: str, data: Optional[dict]): + if data is None: + self.config = None + return + + self.config = Config(**data) + self.config["build_type"] = self.config.get("build_type", None) + self.config["build_image"] = self.config.get("build_image", None) + self.config["cpus"] = self.config.get("cpus", 1) + self.config["memory_limit"] = self.config.get("memory_limit", "256M") + self.config["first_party_domains"] = self.config.get( + "first_party_domains", [f"{name}.cs61a.org"] + ) + self.config["concurrency"] = self.config.get("concurrency", 80) + self.config["tasks"] = self.config.get("tasks", []) + self.config["dependencies"] = self.config.get("dependencies", []) + self.config["package_name"] = self.config.get("package_name", name) + self.config["static_consumers"] = self.config.get("static_consumers", []) + self.config["repo"] = self.config.get("repo") + self.config["service"] = self.config.get("service") + self.config["pr_consumers"] = self.config.get("pr_consumers", [name]) + self.config["permissions"] = self.config.get("permissions", ["rpc", "database"]) def __str__(self): return self.name diff --git a/buildserver/build.py b/buildserver/build.py index 3c44e4de..65c08163 100644 --- a/buildserver/build.py +++ b/buildserver/build.py @@ -1,4 +1,5 @@ import os +from os.path import join from shutil import copytree, rmtree from urllib.parse import urlparse @@ -34,29 +35,39 @@ def clone(): def build(app: App): with tmp_directory(): - os.chdir(app.name) - - app_dir = app.name - - os.chdir("..") working_dir = gen_working_dir(app) - - copytree(app_dir, working_dir, dirs_exist_ok=True, symlinks=False) - os.chdir(working_dir) - { - "oh_queue": run_oh_queue_build, - "create_react_app": run_create_react_app_build, - "webpack": run_webpack_build, - "61a_website": run_61a_website_build, - "hugo": run_hugo_build, - "sphinx": run_sphinx_build, - "jekyll": run_jekyll_build, - "none": run_noop_build, - }[app.config["build_type"]]() - - os.chdir("..") + if app.config["build_type"]: + { + "oh_queue": run_oh_queue_build, + "create_react_app": run_create_react_app_build, + "webpack": run_webpack_build, + "61a_website": run_61a_website_build, + "hugo": run_hugo_build, + "sphinx": run_sphinx_build, + "jekyll": run_jekyll_build, + "none": run_noop_build, + }[app.config["build_type"]]() + else: + # bt will place the built output here: + # // -> working_dir -> _destination -> target + destination_name = "__destination" + output_directory = join(working_dir, destination_name) + target = app.config["target"] + sh( + "bt", + target, + "--quiet", + "--profile", + *("--num-threads", 4), + *("--flag", output_directory), + ) + + # We will move _destination/target into working_dir, then clear _destination + clean_all_except([destination_name]) + copytree(join(destination_name, target), ".", dirs_exist_ok=True) + rmtree(destination_name) def run_oh_queue_build(): diff --git a/buildserver/conf.py b/buildserver/conf.py index ed538a25..0f9a070f 100644 --- a/buildserver/conf.py +++ b/buildserver/conf.py @@ -1,6 +1,7 @@ -GITHUB_REPO = "Cal-CS-61A-Staff/cs61a-apps" +GITHUB_REPO = "Cal-CS-61A-Staff/berkeley-cs61a" PROJECT_ID = "cs61a-140900" DB_SHORT_INSTANCE_NAME = "cs61a-apps-us-west1" DB_INSTANCE_NAME = f"{PROJECT_ID}:us-west2:{DB_SHORT_INSTANCE_NAME}" DB_IP_ADDRESS = "35.236.93.51" STATIC_SERVER = "static-server" +GITHUB_BOT_USER = "pusherbot" diff --git a/buildserver/env.py b/buildserver/env.py deleted file mode 100644 index 76f29694..00000000 --- a/buildserver/env.py +++ /dev/null @@ -1 +0,0 @@ -GITHUB_BOT_USER = "pusherbot" diff --git a/buildserver/external_build_worker.py b/buildserver/external_build_worker.py deleted file mode 100644 index 0a59d078..00000000 --- a/buildserver/external_build_worker.py +++ /dev/null @@ -1,22 +0,0 @@ -# this is used to trigger the worker via Cloud Build -import sys - -from github import Github - -from app_config import App -from build import clone_commit -from common.rpc.secrets import get_secret -from conf import GITHUB_REPO -from worker import land_app_worker - -if __name__ == "__main__": - g = Github(get_secret(secret_name="GITHUB_ACCESS_TOKEN")) - _, app_name, pr_number, sha, repo_id = sys.argv - base_repo = g.get_repo(GITHUB_REPO) - clone_commit( - base_repo.clone_url, - sha - if repo_id == base_repo.full_name - else base_repo.get_branch(base_repo.default_branch).commit.sha, - ) - land_app_worker(App(app_name), int(pr_number), sha, g.get_repo(repo_id)) diff --git a/buildserver/external_repo_utils.py b/buildserver/external_repo_utils.py index d5d8e039..e350cfef 100644 --- a/buildserver/external_repo_utils.py +++ b/buildserver/external_repo_utils.py @@ -12,10 +12,3 @@ def update_config(app: App, pr_number: int): "INSERT INTO services VALUES (%s, %s, %s, %s)", [app.name, pr_number, False, app.config["deploy_type"] in WEB_DEPLOY_TYPES], ) - if pr_number == 0: - db("DELETE FROM apps WHERE app=%s", [app.name]) - if app.config["repo"]: - db( - "INSERT INTO apps (app, repo, autobuild) VALUES (%s, %s, %s)", - [app.name, app.config["repo"], True], - ) diff --git a/buildserver/github_utils.py b/buildserver/github_utils.py index 7e1acfe4..ff39f590 100644 --- a/buildserver/github_utils.py +++ b/buildserver/github_utils.py @@ -8,7 +8,7 @@ from common.db import connect_db from common.rpc.secrets import get_secret from common.url_for import url_for -from env import GITHUB_BOT_USER +from conf import GITHUB_BOT_USER from target_determinator import determine_targets @@ -70,7 +70,7 @@ def update_status(packed_ref: str, pr_number: int): pr = repo.get_pull(pr_number) # Now we will update the PR comment, looking at builds for all packed_refs in the PR - apps = determine_targets(repo, pr.get_files()) + apps = determine_targets(repo, sha, pr.get_files()) success = [] failure = [] running = [] diff --git a/buildserver/main.py b/buildserver/main.py index a4e66922..48e4d083 100644 --- a/buildserver/main.py +++ b/buildserver/main.py @@ -43,12 +43,6 @@ ) """ ) - db( - """CREATE TABLE IF NOT EXISTS apps ( - app varchar(128), - repo varchar(128), - autobuild boolean -)""" ) db( """CREATE TABLE IF NOT EXISTS mysql_users ( @@ -124,7 +118,8 @@ def handle_deploy_prod_app_sync(app, is_staging, target_app): repo, repo, None, - [f"{target_app}/main.py"], + [], + target_app=target_app, ) @@ -228,10 +223,10 @@ def webhook(): pr = repo.get_pull(payload["pull_request"]["number"]) if payload["action"] in ("opened", "synchronize", "reopened"): - if repo.full_name != GITHUB_REPO: - land_commit(pr.head.sha, repo, g.get_repo(GITHUB_REPO), pr, []) - else: - for target in determine_targets(repo, pr.get_files()): + if repo.full_name == GITHUB_REPO: + # todo: remove hardcoded website-base + land_commit(pr.head.sha, repo, repo, pr, [], target_app="website-base") + for target in determine_targets(repo, pr.head.sha, pr.get_files()): report_build_status( target, pr.number, @@ -241,6 +236,8 @@ def webhook(): None, private=True, ) + else: + land_commit(pr.head.sha, repo, g.get_repo(GITHUB_REPO), pr, []) elif payload["action"] == "closed": set_pr_comment("PR closed, shutting down PR builds...", pr) diff --git a/buildserver/target_determinator.py b/buildserver/target_determinator.py index ebf8176f..efc07c8e 100644 --- a/buildserver/target_determinator.py +++ b/buildserver/target_determinator.py @@ -1,41 +1,56 @@ -import os -from typing import Iterable, Optional, Set, Union +import fnmatch +from collections import defaultdict +from typing import Dict, Iterable, Set, Union +import yaml from github.File import File from github.Repository import Repository -from common.db import connect_db +from buildserver.app_config import App +from buildserver.github_utils import get_github from conf import GITHUB_REPO -def get_app(path: Optional[str]): - if path is None: - return None - path = os.path.normpath(path) - folders = path.split(os.sep) - if len(folders) <= 1: - # not in any app folder - return None +def get_all_apps(repo: Repository, sha: str) -> Dict[str, App]: + base_repo = get_github().get_repo(GITHUB_REPO) + base_sha = ( + sha + if repo.full_name == GITHUB_REPO + else base_repo.get_branch(base_repo.default_branch).commit.sha + ) + return { + target: App(target, config) + for target, config in yaml.safe_load( + base_repo.get_contents("targets.yaml", base_sha).decoded_content + ).items() + } - return str(folders[0]) +def determine_targets( + repo: Repository, sha: str, files: Iterable[Union[File, str]] +) -> Set[str]: + apps = get_all_apps(repo, sha) + + # todo: target detection when we modify targets.yaml itself + modified_targets = [] -def determine_targets(repo: Repository, files: Iterable[Union[File, str]]) -> Set[str]: - modified_apps = [] if repo.full_name == GITHUB_REPO: + modified_paths = [] for file in files: if isinstance(file, str): - modified_apps.append(get_app(file)) + modified_paths.append(file) else: - modified_apps.append(get_app(file.filename)) - modified_apps.append(get_app(file.previous_filename)) - - with connect_db() as db: - modified_apps.extend( - app - for [app] in db( - "SELECT app FROM apps WHERE repo=(%s)", [repo.full_name] - ).fetchall() - ) - - return set([app for app in modified_apps if app is not None]) + modified_paths.append(file.filename) + modified_paths.append(file.previous_filename) + + for app_name, app in apps.items(): + if any( + fnmatch.filter(modified_paths, match) for match in app.config["match"] + ): + modified_targets.append(app_name) + + for app_name, app in apps.items(): + if app.config["repo"] == repo.full_name: + modified_targets.append(app_name) + + return set(modified_targets) diff --git a/buildserver/worker.py b/buildserver/worker.py index 432b81d6..46858dd5 100644 --- a/buildserver/worker.py +++ b/buildserver/worker.py @@ -1,7 +1,7 @@ import tempfile import traceback from sys import stderr, stdout -from typing import Iterable, Optional, Union +from typing import Callable, Iterable, Optional, Union from github import Github from github.File import File @@ -13,7 +13,6 @@ from common.db import connect_db from common.rpc.buildserver import clear_queue from common.rpc.buildserver_hosted_worker import build_worker_build -from common.rpc.secrets import get_secret from common.shell_utils import redirect_descriptor from dependency_loader import load_dependencies from deploy import deploy_commit @@ -33,32 +32,16 @@ report_build_status, ) from service_management import get_pr_subdomains, update_service_routes -from target_determinator import determine_targets +from target_determinator import determine_targets, get_all_apps -def land_app( - app: App, - pr_number: int, - sha: str, - repo: Repository, -): +def land_app(app: App, pr_number: int, sha: str, repo: Repository, clone: Callable): if app.config is None: delete_app(app, pr_number) return update_config(app, pr_number) - if app.config["build_image"]: - run_highcpu_build(app, pr_number, sha, repo) - else: - land_app_worker(app, pr_number, sha, repo) - -def land_app_worker( - app: App, - pr_number: int, - sha: str, - repo: Repository, -): if app.name in TARGETS_BUILT_ON_WORKER: if repo.full_name != app.config.get("repo", repo.full_name): # the worker does not do dependency resolution, so we must @@ -74,7 +57,13 @@ def land_app_worker( if not success: raise Exception("Build failed") else: - load_dependencies(app, sha, repo) + # We defer cloning to here, so that if we're building on the worker / not building at all, + # we don't have to do a slow clone + if app.config["repo"]: + load_dependencies(app, sha, repo) + else: + clone() + build(app) deploy_commit(app, pr_number) @@ -85,8 +74,6 @@ def delete_app(app: App, pr_number: int): "DELETE FROM services WHERE app=%s AND pr_number=%s", [app.name, pr_number], ) - if pr_number == 0: - db("DELETE FROM apps WHERE app=%s", [app.name]) def land_commit( @@ -111,7 +98,7 @@ def land_commit( targets = [target_app] else: targets = determine_targets( - repo, files if repo.full_name == base_repo.full_name else [] + repo, sha, files if repo.full_name == base_repo.full_name else [] ) pr_number = pr.number if pr else 0 enqueue_builds(targets, pr_number, pack(repo.clone_url, sha)) @@ -124,22 +111,32 @@ def dequeue_and_build(base_repo: Repository): repo_clone_url, sha = unpack(packed_ref) repo_name = repo_name_from_packed_ref(packed_ref) repo = get_github().get_repo(repo_name) - # If the commit is made on the base repo, take the config from the current commit. - # Otherwise, retrieve it from master - clone_commit( - base_repo.clone_url, - sha - if repo_clone_url == base_repo.clone_url - else base_repo.get_branch(base_repo.default_branch).commit.sha, - ) + cloned = False + + def clone(): + nonlocal cloned + if cloned: + return + + cloned = True + # If the commit is made on the base repo, take the config from the current commit. + # Otherwise, retrieve it from master + clone_commit( + base_repo.clone_url, + sha + if repo_clone_url == base_repo.clone_url + else base_repo.get_branch(base_repo.default_branch).commit.sha, + ) + + all_apps = get_all_apps(repo, sha) for app_name, pr_number in targets: - app = App(app_name) + app = all_apps.get(app_name, App(app_name, None)) with tempfile.TemporaryFile("w+") as logs: try: with redirect_descriptor(stdout, logs), redirect_descriptor( stderr, logs ): - land_app(app, pr_number, sha, repo) + land_app(app, pr_number, sha, repo, clone) if app.config is not None: update_service_routes([app], pr_number) except: From 66eaba0c1a864c5487dcdfc153ceb83cf0812427 Mon Sep 17 00:00:00 2001 From: Rahul Arya Date: Tue, 17 Aug 2021 22:47:45 -0700 Subject: [PATCH 2/4] fmt --- buildserver/main.py | 1 - 1 file changed, 1 deletion(-) diff --git a/buildserver/main.py b/buildserver/main.py index 48e4d083..23b964b3 100644 --- a/buildserver/main.py +++ b/buildserver/main.py @@ -42,7 +42,6 @@ is_web_service boolean ) """ - ) ) db( """CREATE TABLE IF NOT EXISTS mysql_users ( From 05fb5737e75830facc91df93cda403d160a25e71 Mon Sep 17 00:00:00 2001 From: Rahul Arya Date: Tue, 17 Aug 2021 23:06:21 -0700 Subject: [PATCH 3/4] fix imports --- buildserver/target_determinator.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/buildserver/target_determinator.py b/buildserver/target_determinator.py index efc07c8e..e7baa3d9 100644 --- a/buildserver/target_determinator.py +++ b/buildserver/target_determinator.py @@ -1,13 +1,12 @@ import fnmatch -from collections import defaultdict from typing import Dict, Iterable, Set, Union import yaml from github.File import File from github.Repository import Repository -from buildserver.app_config import App -from buildserver.github_utils import get_github +from app_config import App +from github_utils import get_github from conf import GITHUB_REPO From 9920bccbbfe80b10b7b6a7c8c162757b81417396 Mon Sep 17 00:00:00 2001 From: Rahul Arya Date: Tue, 17 Aug 2021 23:12:33 -0700 Subject: [PATCH 4/4] fix import cycle --- buildserver/github_utils.py | 178 ------------------------------ buildserver/main.py | 3 +- buildserver/scheduling.py | 2 +- buildserver/status_reporting.py | 186 ++++++++++++++++++++++++++++++++ buildserver/worker.py | 3 +- 5 files changed, 190 insertions(+), 182 deletions(-) create mode 100644 buildserver/status_reporting.py diff --git a/buildserver/github_utils.py b/buildserver/github_utils.py index ff39f590..304f03ee 100644 --- a/buildserver/github_utils.py +++ b/buildserver/github_utils.py @@ -1,15 +1,11 @@ -from enum import Enum, unique from typing import Optional from urllib.parse import urlparse from github import Github from github.PullRequest import PullRequest -from common.db import connect_db from common.rpc.secrets import get_secret -from common.url_for import url_for from conf import GITHUB_BOT_USER -from target_determinator import determine_targets def get_github(): @@ -21,171 +17,6 @@ def repo_name_from_packed_ref(packed_ref): return urlparse(repo_url).path.split(".")[0][1:] # This is awful ... but it works -def update_status(packed_ref: str, pr_number: int): - g = get_github() - repo_url, sha = unpack(packed_ref) - repo_name = repo_name_from_packed_ref(packed_ref) - repo = g.get_repo(repo_name) - - # First we will update the commit-specific status indicator - with connect_db() as db: - statuses = db( - "SELECT app, status FROM builds WHERE packed_ref=%s", [packed_ref] - ).fetchall() - statuses = [(app, BuildStatus(status)) for app, status in statuses] - if all(status == BuildStatus.success for _, status in statuses): - repo.get_commit(sha).create_status( - "success", - "https://logs.cs61a.org/service/buildserver", - "All modified services built!", - "Pusher", - ) - elif any(status == BuildStatus.failure for _, status in statuses): - repo.get_commit(sha).create_status( - "failure", - "https://logs.cs61a.org/service/buildserver", - "Pusher failed to build a modified service", - "Pusher", - ) - elif all( - status in (BuildStatus.building, BuildStatus.queued) for _, status in statuses - ): - repo.get_commit(sha).create_status( - "pending", - "https://logs.cs61a.org/service/buildserver", - "Pusher is building all modified services", - "Pusher", - ) - else: - # There are no failures, but not everything is building / built - repo.get_commit(sha).create_status( - "pending", - "https://logs.cs61a.org/service/buildserver", - "You must build all modified apps before merging", - "Pusher", - ) - - if pr_number == 0: - return - - pr = repo.get_pull(pr_number) - # Now we will update the PR comment, looking at builds for all packed_refs in the PR - apps = determine_targets(repo, sha, pr.get_files()) - success = [] - failure = [] - running = [] - queued = [] - triggerable = [] - with connect_db() as db: - for app in apps: - successful_build = db( - "SELECT url, log_url, unix, packed_ref FROM builds WHERE app=%s AND pr_number=%s AND status='success' ORDER BY unix DESC LIMIT 1", - [app, pr_number], - ).fetchone() - if successful_build: - url, log_url, success_unix, packed_ref = successful_build - _, sha = unpack(packed_ref) - if url: - for link in url.split(","): - success.append((app, link, sha, log_url)) - else: - success.append((app, None, sha, log_url)) - - failed_build = db( - "SELECT unix, log_url, packed_ref FROM builds WHERE app=%s AND pr_number=%s AND status='failure' ORDER BY unix DESC LIMIT 1", - [app, pr_number], - ).fetchone() - if failed_build: - unix, log_url, packed_ref = failed_build - if not successful_build or success_unix < unix: - _, sha = unpack(packed_ref) - failure.append((app, sha, log_url)) - - running_build = db( - "SELECT packed_ref FROM builds WHERE app=%s AND pr_number=%s AND status='building'", - [app, pr_number], - ).fetchone() - if running_build: - [packed_ref] = running_build - _, sha = unpack(packed_ref) - running.append((app, sha)) - - queued_build = db( - "SELECT packed_ref FROM builds WHERE app=%s AND pr_number=%s AND status='queued'", - [app, pr_number], - ).fetchone() - if queued_build: - [packed_ref] = queued_build - _, sha = unpack(packed_ref) - queued.append((app, sha)) - - latest_commit_build = db( - "SELECT * FROM builds WHERE app=%s AND pr_number=%s AND packed_ref=%s AND status!='pushed'", - [app, pr_number, pack(repo_url, pr.head.sha)], - ).fetchone() - if not latest_commit_build: - triggerable.append(app) - - if repo.name == "berkeley-cs61a": - message = f"## Build Status ([pr/{pr_number}]({pr.html_url}))\n\n" - elif repo.name == "cs61a-apps": - message = f"## Build Status ([apps/{pr_number}]({pr.html_url}))\n\n" - else: - message = f"## Build Status (#{pr_number})\n\n" - - if success: - message += ( - "**Successful Builds**\n" - + "\n".join( - f" - [{host}](https://{host}) ({sha}) [[logs]({log_url})]" - if host - else f" - `{app}` ({sha}) [[logs]({log_url})]" - for app, host, sha, log_url in success - ) - + "\n\n" - ) - - if failure: - message += ( - "**Failed Builds**\n" - + "\n".join( - f" - `{app}` ({sha}) [[logs]({log_url})]" - for app, sha, log_url in failure - ) - + "\n\n" - ) - - if running: - message += ( - "**Running Builds**\n" - + "\n".join(f" - `{app}` ({sha})" for app, sha in running) - + "\n\n" - ) - - if queued: - message += ( - "**Queued Builds**\n" - + "\n".join(f" - `{app}` ({sha})" for app, sha in queued) - + "\n\n" - ) - - if (success or failure or running or queued) and triggerable: - message += "-----\n" - - if triggerable: - message += ( - f"**[Click here]({url_for('trigger_build', pr_number=pr.number)})** to trigger all builds " - f"for the most recent commit ({pr.head.sha})\n\n" - "Or trigger builds individually:\n" - ) + "\n".join( - f" - [Click here]({url_for('trigger_build', pr_number=pr.number, app=app)}) " - f"to build `{app}` at the most recent commit ({pr.head.sha})" - for app in triggerable - ) - - set_pr_comment(message, pr) - - def set_pr_comment(text: str, pr: Optional[PullRequest]): if pr is None: return @@ -208,12 +39,3 @@ def pack(clone_url: str, sha: str) -> str: def unpack(packed_ref: str): return packed_ref.split("|") - - -@unique -class BuildStatus(Enum): - pushed = "pushed" - queued = "queued" - building = "building" - failure = "failure" - success = "success" diff --git a/buildserver/main.py b/buildserver/main.py index 23b964b3..629a2d3f 100644 --- a/buildserver/main.py +++ b/buildserver/main.py @@ -17,10 +17,11 @@ from common.rpc.secrets import get_secret, only, validates_master_secret from common.url_for import url_for from conf import GITHUB_REPO -from github_utils import BuildStatus, get_github, pack, set_pr_comment +from github_utils import get_github, pack, set_pr_comment from rebuilder import create_rebuilder from scheduling import report_build_status from service_management import delete_unused_services +from status_reporting import BuildStatus from target_determinator import determine_targets from worker import dequeue_and_build, land_commit diff --git a/buildserver/scheduling.py b/buildserver/scheduling.py index cb96348e..5a8490ae 100644 --- a/buildserver/scheduling.py +++ b/buildserver/scheduling.py @@ -6,7 +6,7 @@ from common.db import connect_db from common.rpc.auth import post_slack_message from common.rpc.paste import get_paste_url, paste_text -from github_utils import BuildStatus, update_status +from status_reporting import BuildStatus, update_status BUILD_TIME = timedelta(minutes=20).total_seconds() diff --git a/buildserver/status_reporting.py b/buildserver/status_reporting.py new file mode 100644 index 00000000..af65a5c9 --- /dev/null +++ b/buildserver/status_reporting.py @@ -0,0 +1,186 @@ +from enum import Enum, unique + +from common.db import connect_db +from common.url_for import url_for +from github_utils import ( + get_github, + pack, + repo_name_from_packed_ref, + set_pr_comment, + unpack, +) +from target_determinator import determine_targets + + +@unique +class BuildStatus(Enum): + pushed = "pushed" + queued = "queued" + building = "building" + failure = "failure" + success = "success" + + +def update_status(packed_ref: str, pr_number: int): + g = get_github() + repo_url, sha = unpack(packed_ref) + repo_name = repo_name_from_packed_ref(packed_ref) + repo = g.get_repo(repo_name) + + # First we will update the commit-specific status indicator + with connect_db() as db: + statuses = db( + "SELECT app, status FROM builds WHERE packed_ref=%s", [packed_ref] + ).fetchall() + statuses = [(app, BuildStatus(status)) for app, status in statuses] + if all(status == BuildStatus.success for _, status in statuses): + repo.get_commit(sha).create_status( + "success", + "https://logs.cs61a.org/service/buildserver", + "All modified services built!", + "Pusher", + ) + elif any(status == BuildStatus.failure for _, status in statuses): + repo.get_commit(sha).create_status( + "failure", + "https://logs.cs61a.org/service/buildserver", + "Pusher failed to build a modified service", + "Pusher", + ) + elif all( + status in (BuildStatus.building, BuildStatus.queued) for _, status in statuses + ): + repo.get_commit(sha).create_status( + "pending", + "https://logs.cs61a.org/service/buildserver", + "Pusher is building all modified services", + "Pusher", + ) + else: + # There are no failures, but not everything is building / built + repo.get_commit(sha).create_status( + "pending", + "https://logs.cs61a.org/service/buildserver", + "You must build all modified apps before merging", + "Pusher", + ) + + if pr_number == 0: + return + + pr = repo.get_pull(pr_number) + # Now we will update the PR comment, looking at builds for all packed_refs in the PR + apps = determine_targets(repo, sha, pr.get_files()) + success = [] + failure = [] + running = [] + queued = [] + triggerable = [] + with connect_db() as db: + for app in apps: + successful_build = db( + "SELECT url, log_url, unix, packed_ref FROM builds WHERE app=%s AND pr_number=%s AND status='success' ORDER BY unix DESC LIMIT 1", + [app, pr_number], + ).fetchone() + if successful_build: + url, log_url, success_unix, packed_ref = successful_build + _, sha = unpack(packed_ref) + if url: + for link in url.split(","): + success.append((app, link, sha, log_url)) + else: + success.append((app, None, sha, log_url)) + + failed_build = db( + "SELECT unix, log_url, packed_ref FROM builds WHERE app=%s AND pr_number=%s AND status='failure' ORDER BY unix DESC LIMIT 1", + [app, pr_number], + ).fetchone() + if failed_build: + unix, log_url, packed_ref = failed_build + if not successful_build or success_unix < unix: + _, sha = unpack(packed_ref) + failure.append((app, sha, log_url)) + + running_build = db( + "SELECT packed_ref FROM builds WHERE app=%s AND pr_number=%s AND status='building'", + [app, pr_number], + ).fetchone() + if running_build: + [packed_ref] = running_build + _, sha = unpack(packed_ref) + running.append((app, sha)) + + queued_build = db( + "SELECT packed_ref FROM builds WHERE app=%s AND pr_number=%s AND status='queued'", + [app, pr_number], + ).fetchone() + if queued_build: + [packed_ref] = queued_build + _, sha = unpack(packed_ref) + queued.append((app, sha)) + + latest_commit_build = db( + "SELECT * FROM builds WHERE app=%s AND pr_number=%s AND packed_ref=%s AND status!='pushed'", + [app, pr_number, pack(repo_url, pr.head.sha)], + ).fetchone() + if not latest_commit_build: + triggerable.append(app) + + if repo.name == "berkeley-cs61a": + message = f"## Build Status ([pr/{pr_number}]({pr.html_url}))\n\n" + elif repo.name == "cs61a-apps": + message = f"## Build Status ([apps/{pr_number}]({pr.html_url}))\n\n" + else: + message = f"## Build Status (#{pr_number})\n\n" + + if success: + message += ( + "**Successful Builds**\n" + + "\n".join( + f" - [{host}](https://{host}) ({sha}) [[logs]({log_url})]" + if host + else f" - `{app}` ({sha}) [[logs]({log_url})]" + for app, host, sha, log_url in success + ) + + "\n\n" + ) + + if failure: + message += ( + "**Failed Builds**\n" + + "\n".join( + f" - `{app}` ({sha}) [[logs]({log_url})]" + for app, sha, log_url in failure + ) + + "\n\n" + ) + + if running: + message += ( + "**Running Builds**\n" + + "\n".join(f" - `{app}` ({sha})" for app, sha in running) + + "\n\n" + ) + + if queued: + message += ( + "**Queued Builds**\n" + + "\n".join(f" - `{app}` ({sha})" for app, sha in queued) + + "\n\n" + ) + + if (success or failure or running or queued) and triggerable: + message += "-----\n" + + if triggerable: + message += ( + f"**[Click here]({url_for('trigger_build', pr_number=pr.number)})** to trigger all builds " + f"for the most recent commit ({pr.head.sha})\n\n" + "Or trigger builds individually:\n" + ) + "\n".join( + f" - [Click here]({url_for('trigger_build', pr_number=pr.number, app=app)}) " + f"to build `{app}` at the most recent commit ({pr.head.sha})" + for app in triggerable + ) + + set_pr_comment(message, pr) diff --git a/buildserver/worker.py b/buildserver/worker.py index 46858dd5..ede8ad9b 100644 --- a/buildserver/worker.py +++ b/buildserver/worker.py @@ -16,10 +16,8 @@ from common.shell_utils import redirect_descriptor from dependency_loader import load_dependencies from deploy import deploy_commit -from external_build import run_highcpu_build from external_repo_utils import update_config from github_utils import ( - BuildStatus, get_github, pack, repo_name_from_packed_ref, @@ -32,6 +30,7 @@ report_build_status, ) from service_management import get_pr_subdomains, update_service_routes +from status_reporting import BuildStatus from target_determinator import determine_targets, get_all_apps