From d1b329fd209d156014b9b7e73a423a1f30961680 Mon Sep 17 00:00:00 2001 From: Tony Le Date: Wed, 6 Nov 2024 17:56:25 -0500 Subject: [PATCH] Single endpoint coverage command initial implementation --- codecov_cli/commands/combined_upload.py | 288 ++++++++++++++++++ codecov_cli/main.py | 3 +- .../services/combined_upload/__init__.py | 173 +++++++++++ codecov_cli/services/upload/upload_sender.py | 15 +- 4 files changed, 477 insertions(+), 2 deletions(-) create mode 100644 codecov_cli/commands/combined_upload.py create mode 100644 codecov_cli/services/combined_upload/__init__.py diff --git a/codecov_cli/commands/combined_upload.py b/codecov_cli/commands/combined_upload.py new file mode 100644 index 00000000..aae2930d --- /dev/null +++ b/codecov_cli/commands/combined_upload.py @@ -0,0 +1,288 @@ +import logging +import os +import pathlib +import typing + +import click + +from codecov_cli.fallbacks import CodecovOption, FallbackFieldEnum +from codecov_cli.helpers.args import get_cli_args +from codecov_cli.helpers.git import GitService +from codecov_cli.helpers.options import global_options +from codecov_cli.services.combined_upload import combined_upload_logic +from codecov_cli.types import CommandContext + +logger = logging.getLogger("codecovcli") + + +def _turn_env_vars_into_dict(ctx, params, value): + return dict((v, os.getenv(v, None)) for v in value) + +@click.command() +@click.option( + "--parent-sha", + help="SHA (with 40 chars) of what should be the parent of this commit", +) +@click.option( + "-P", + "--pr", + "--pull-request-number", + "pull_request_number", + help="Specify the pull request number mannually. Used to override pre-existing CI environment variables", + cls=CodecovOption, + fallback_field=FallbackFieldEnum.pull_request_number, +) +@click.option( + "-B", + "--branch", + help="Branch to which this commit belongs to", + cls=CodecovOption, + fallback_field=FallbackFieldEnum.branch, +) +@click.option( + "--code", + "--report-code", + "report_code", + help="The code of the report. If unsure, leave default", + default="default", +) +@click.option( + "--network-root-folder", + help="Root folder from which to consider paths on the network section", + type=click.Path(path_type=pathlib.Path), + default=pathlib.Path.cwd, + show_default="Current working directory", +) +@click.option( + "-s", + "--dir", + "--coverage-files-search-root-folder", + "--files-search-root-folder", + "files_search_root_folder", + help="Folder where to search for coverage files", + type=click.Path(path_type=pathlib.Path), + default=pathlib.Path.cwd, + show_default="Current Working Directory", +) +@click.option( + "--exclude", + "--coverage-files-search-exclude-folder", + "--files-search-exclude-folder", + "files_search_exclude_folders", + help="Folders to exclude from search", + type=click.Path(path_type=pathlib.Path), + multiple=True, + default=[], +) +@click.option( + "-f", + "--file", + "--coverage-files-search-direct-file", + "--files-search-direct-file", + "files_search_explicitly_listed_files", + help="Explicit files to upload. These will be added to the coverage files found for upload. If you wish to only upload the specified files, please consider using --disable-search to disable uploading other files.", + type=click.Path(path_type=pathlib.Path), + multiple=True, + default=[], +) +@click.option( + "--disable-search", + help="Disable search for coverage files. This is helpful when specifying what files you want to upload with the --file option.", + is_flag=True, + default=False, +) +@click.option( + "--disable-file-fixes", + help="Disable file fixes to ignore common lines from coverage (e.g. blank lines or empty brackets)", + is_flag=True, + default=False, +) +@click.option( + "-b", + "--build", + "--build-code", + "build_code", + cls=CodecovOption, + help="Specify the build number manually", + fallback_field=FallbackFieldEnum.build_code, +) +@click.option( + "--build-url", + "build_url", + cls=CodecovOption, + help="The URL of the build where this is running", + fallback_field=FallbackFieldEnum.build_url, +) +@click.option( + "--job-code", + cls=CodecovOption, + fallback_field=FallbackFieldEnum.job_code, +) +@click.option( + "-n", + "--name", + help="Custom defined name of the upload. Visible in Codecov UI", + cls=CodecovOption, + fallback_field=FallbackFieldEnum.build_code, +) +@click.option( + "-e", + "--env", + "--env-var", + "env_vars", + multiple=True, + callback=_turn_env_vars_into_dict, + help="Specify environment variables to be included with this build.", +) +@click.option( + "-F", + "--flag", + "flags", + multiple=True, + default=[], + help="Flag the upload to group coverage metrics. Multiple flags allowed.", +) +@click.option( + "--plugin", + "plugin_names", + multiple=True, + default=["xcode", "gcov", "pycoverage"], +) +@click.option( + "-d", + "--dry-run", + "dry_run", + is_flag=True, + help="Don't upload files to Codecov", +) +@click.option( + "--legacy", + "--use-legacy-uploader", + "use_legacy_uploader", + is_flag=True, + help="Use the legacy upload endpoint", +) +@click.option( + "--handle-no-reports-found", + "handle_no_reports_found", + is_flag=True, + help="Raise no excpetions when no coverage reports found.", +) +@click.option( + "--report-type", + help="The type of the file to upload, coverage by default. Possible values are: testing, coverage.", + default="coverage", + type=click.Choice(["coverage", "test_results"]), +) +@click.option( + "--network-filter", + help="Specify a filter on the files listed in the network section of the Codecov report. This will only add files whose path begin with the specified filter. Useful for upload-specific path fixing", +) +@click.option( + "--network-prefix", + help="Specify a prefix on files listed in the network section of the Codecov report. Useful to help resolve path fixing", +) +@click.option( + "--gcov-args", + help="Extra arguments to pass to gcov", +) +@click.option( + "--gcov-ignore", + help="Paths to ignore during gcov gathering", +) +@click.option( + "--gcov-include", + help="Paths to include during gcov gathering", +) +@click.option( + "--gcov-executable", + help="gcov executable to run. Defaults to 'gcov'", +) +@click.option( + "--swift-project", + help="Specify the swift project", +) +@global_options +@click.pass_context +def combined_upload( + ctx: CommandContext, + branch: typing.Optional[str], + build_code: typing.Optional[str], + build_url: typing.Optional[str], + commit_sha: str, + disable_file_fixes: bool, + disable_search: bool, + dry_run: bool, + env_vars: typing.Dict[str, str], + fail_on_error: bool, + files_search_exclude_folders: typing.List[pathlib.Path], + files_search_explicitly_listed_files: typing.List[pathlib.Path], + files_search_root_folder: pathlib.Path, + flags: typing.List[str], + gcov_args: typing.Optional[str], + gcov_executable: typing.Optional[str], + gcov_ignore: typing.Optional[str], + gcov_include: typing.Optional[str], + git_service: typing.Optional[str], + handle_no_reports_found: bool, + job_code: typing.Optional[str], + name: typing.Optional[str], + network_filter: typing.Optional[str], + network_prefix: typing.Optional[str], + network_root_folder: pathlib.Path, + parent_sha: typing.Optional[str], + plugin_names: typing.List[str], + pull_request_number: typing.Optional[str], + report_code: str, + report_type: str, + slug: typing.Optional[str], + swift_project: typing.Optional[str], + token: typing.Optional[str], + use_legacy_uploader: bool, +): + versioning_system = ctx.obj["versioning_system"] + codecov_yaml = ctx.obj["codecov_yaml"] or {} + cli_config = codecov_yaml.get("cli", {}) + ci_adapter = ctx.obj.get("ci_adapter") + enterprise_url = ctx.obj.get("enterprise_url") + args = get_cli_args(ctx) + combined_upload_logic( + cli_config, + versioning_system, + ci_adapter, + branch=branch, + build_code=build_code, + build_url=build_url, + commit_sha=commit_sha, + disable_file_fixes=disable_file_fixes, + disable_search=disable_search, + dry_run=dry_run, + enterprise_url=enterprise_url, + env_vars=env_vars, + fail_on_error=fail_on_error, + files_search_exclude_folders=files_search_exclude_folders, + files_search_explicitly_listed_files=files_search_explicitly_listed_files, + files_search_root_folder=files_search_root_folder, + flags=flags, + gcov_args=gcov_args, + gcov_executable=gcov_executable, + gcov_ignore=gcov_ignore, + gcov_include=gcov_include, + git_service=git_service, + handle_no_reports_found=handle_no_reports_found, + job_code=job_code, + name=name, + network_filter=network_filter, + network_prefix=network_prefix, + network_root_folder=network_root_folder, + parent_sha=parent_sha, + plugin_names=plugin_names, + pull_request_number=pull_request_number, + report_code=report_code, + slug=slug, + swift_project=swift_project, + token=token, + upload_file_type=report_type, + use_legacy_uploader=use_legacy_uploader, + args=args, + ) diff --git a/codecov_cli/main.py b/codecov_cli/main.py index 9505aaa6..00400ed9 100644 --- a/codecov_cli/main.py +++ b/codecov_cli/main.py @@ -6,6 +6,7 @@ from codecov_cli import __version__ from codecov_cli.commands.base_picking import pr_base_picking +from codecov_cli.commands.combined_upload import combined_upload from codecov_cli.commands.commit import create_commit from codecov_cli.commands.create_report_result import create_report_results from codecov_cli.commands.empty_upload import empty_upload @@ -64,7 +65,7 @@ def cli( ctx.default_map = {ctx.invoked_subcommand: {"token": token}} ctx.obj["enterprise_url"] = enterprise_url - +cli.add_command(combined_upload) cli.add_command(do_upload) cli.add_command(create_commit) cli.add_command(create_report) diff --git a/codecov_cli/services/combined_upload/__init__.py b/codecov_cli/services/combined_upload/__init__.py new file mode 100644 index 00000000..4461b984 --- /dev/null +++ b/codecov_cli/services/combined_upload/__init__.py @@ -0,0 +1,173 @@ +import logging +import os +import pathlib +import typing + +import click + +from codecov_cli.fallbacks import FallbackFieldEnum +from codecov_cli.helpers.ci_adapters.base import CIAdapterBase +from codecov_cli.helpers.config import CODECOV_INGEST_URL +from codecov_cli.helpers.encoder import decode_slug, encode_slug +from codecov_cli.helpers.request import ( + get_token_header_or_fail, + log_warnings_and_errors_if_any, +) +from codecov_cli.helpers.versioning_systems import VersioningSystemInterface +from codecov_cli.plugins import select_preparation_plugins +from codecov_cli.services.upload.file_finder import select_file_finder +from codecov_cli.services.upload.legacy_upload_sender import LegacyUploadSender +from codecov_cli.services.upload.network_finder import select_network_finder +from codecov_cli.services.upload.upload_collector import UploadCollector +from codecov_cli.services.upload.upload_sender import UploadSender +from codecov_cli.services.upload_completion import upload_completion_logic +from codecov_cli.types import RequestResult + +logger = logging.getLogger("codecovcli") +MAX_NUMBER_TRIES = 3 + + +def combined_upload_logic( + cli_config: typing.Dict, + versioning_system: VersioningSystemInterface, + ci_adapter: CIAdapterBase, + *, + branch: typing.Optional[str], + build_code: typing.Optional[str], + build_url: typing.Optional[str], + commit_sha: str, + disable_file_fixes: bool, + disable_search: bool, + dry_run: bool, + enterprise_url: typing.Optional[str], + env_vars: typing.Dict[str, str], + fail_on_error: bool, + files_search_exclude_folders: typing.List[pathlib.Path], + files_search_explicitly_listed_files: typing.List[pathlib.Path], + files_search_root_folder: pathlib.Path, + flags: typing.List[str], + gcov_args: typing.Optional[str], + gcov_executable: typing.Optional[str], + gcov_ignore: typing.Optional[str], + gcov_include: typing.Optional[str], + git_service: typing.Optional[str], + handle_no_reports_found: bool, + job_code: typing.Optional[str], + name: typing.Optional[str], + network_filter: typing.Optional[str], + network_prefix: typing.Optional[str], + network_root_folder: pathlib.Path, + parent_sha: typing.Optional[str], + plugin_names: typing.List[str], + pull_request_number: typing.Optional[str], + report_code: str, + slug: typing.Optional[str], + swift_project: typing.Optional[str], + token: typing.Optional[str], + use_legacy_uploader: bool, + upload_file_type: str = "coverage", + args: dict = None, +): + plugin_config = { + "folders_to_ignore": files_search_exclude_folders, + "gcov_args": gcov_args, + "gcov_executable": gcov_executable, + "gcov_ignore": gcov_ignore, + "gcov_include": gcov_include, + "project_root": files_search_root_folder, + "swift_project": swift_project, + } + if upload_file_type == "coverage": + preparation_plugins = select_preparation_plugins( + cli_config, plugin_names, plugin_config + ) + elif upload_file_type == "test_results": + preparation_plugins = [] + file_selector = select_file_finder( + files_search_root_folder, + files_search_exclude_folders, + files_search_explicitly_listed_files, + disable_search, + upload_file_type, + ) + network_finder = select_network_finder( + versioning_system, + network_filter=network_filter, + network_prefix=network_prefix, + network_root_folder=network_root_folder, + ) + collector = UploadCollector( + preparation_plugins, + network_finder, + file_selector, + disable_file_fixes, + plugin_config, + ) + try: + upload_data = collector.generate_upload_data(upload_file_type) + except click.ClickException as exp: + if handle_no_reports_found: + logger.info( + "No coverage reports found. Triggering notificaions without uploading." + ) + upload_completion_logic( + commit_sha=commit_sha, + slug=slug, + token=token, + git_service=git_service, + enterprise_url=enterprise_url, + fail_on_error=fail_on_error, + ) + return RequestResult( + error=None, + warnings=None, + status_code=200, + text="No coverage reports found. Triggering notificaions without uploading.", + ) + else: + raise exp + if use_legacy_uploader: + # sender = LegacyUploadSender() + raise NotImplementedError("Legacy uploader is not implemented") + else: + sender = UploadSender() + logger.debug(f"Selected uploader to use: {type(sender)}") + ci_service = ( + ci_adapter.get_fallback_value(FallbackFieldEnum.service) + if ci_adapter is not None + else None + ) + + if not dry_run: + sending_result = sender.send_upload_data( + upload_data, + branch=branch, + build_code=build_code, + build_url=build_url, + ci_service=ci_service, + commit_sha=commit_sha, + enterprise_url=enterprise_url, + env_vars=env_vars, + flags=flags, + git_service=git_service, + job_code=job_code, + name=name, + parent_sha=parent_sha, + pull_request_number=pull_request_number, + report_code=report_code, + slug=slug, + token=token, + upload_file_type=upload_file_type, + combined_upload=True, + args=args, + ) + else: + logger.info("dry-run option activated. NOT sending data to Codecov.") + sending_result = RequestResult( + error=None, + warnings=None, + status_code=200, + text="Data NOT sent to Codecov because of dry-run option", + ) + log_warnings_and_errors_if_any(sending_result, "Upload", fail_on_error) + return sending_result diff --git a/codecov_cli/services/upload/upload_sender.py b/codecov_cli/services/upload/upload_sender.py index bfd5a07f..2ec49a7d 100644 --- a/codecov_cli/services/upload/upload_sender.py +++ b/codecov_cli/services/upload/upload_sender.py @@ -42,6 +42,8 @@ def send_upload_data( ci_service: typing.Optional[str] = None, git_service: typing.Optional[str] = None, enterprise_url: typing.Optional[str] = None, + parent_sha: typing.Optional[str] = None, + combined_upload: bool = False, args: dict = None, ) -> RequestResult: data = { @@ -54,6 +56,12 @@ def send_upload_data( "name": name, "version": codecov_cli_version, } + if combined_upload: + data["branch"] = branch + data["commit_sha"] = commit_sha + data["parent_commit_id"] = parent_sha + data["code"] = report_code + data["pullid"] = pull_request_number headers = get_token_header(token) encoded_slug = encode_slug(slug) upload_url = enterprise_url or CODECOV_INGEST_URL @@ -66,6 +74,7 @@ def send_upload_data( encoded_slug, commit_sha, report_code, + combined_upload, ) # Data that goes to storage reports_payload = self._generate_payload( @@ -176,9 +185,13 @@ def get_url_and_possibly_update_data( encoded_slug, commit_sha, report_code, + combined_upload=False, ): if report_type == "coverage": - url = f"{upload_url}/upload/{git_service}/{encoded_slug}/commits/{commit_sha}/reports/{report_code}/uploads" + if combined_upload: + url = f"{upload_url}/upload/{git_service}/{encoded_slug}/combined-upload" + else: + url = f"{upload_url}/upload/{git_service}/{encoded_slug}/commits/{commit_sha}/reports/{report_code}/uploads" elif report_type == "test_results": data["slug"] = encoded_slug data["branch"] = branch