diff --git a/codecov_cli/commands/upload.py b/codecov_cli/commands/upload.py index 77c52a19..aa9e7f75 100644 --- a/codecov_cli/commands/upload.py +++ b/codecov_cli/commands/upload.py @@ -164,6 +164,14 @@ def _turn_env_vars_into_dict(ctx, params, value): default="coverage", type=click.Choice(["coverage", "test_results"]), ), + click.option( + "--network-filter", + help="Specify a filter on the files listed in the network section of the Codecov report. This will only add files whose path begin with the specified filter. Useful for upload-specific path fixing", + ), + click.option( + "--network-prefix", + help="Specify a prefix on files listed in the network section of the Codecov report. Useful to help resolve path fixing", + ), ] @@ -181,29 +189,31 @@ def do_upload( ctx: click.Context, commit_sha: str, report_code: str, + branch: typing.Optional[str], build_code: typing.Optional[str], build_url: typing.Optional[str], - job_code: typing.Optional[str], + disable_file_fixes: bool, + disable_search: bool, + dry_run: bool, env_vars: typing.Dict[str, str], + fail_on_error: bool, + files_search_exclude_folders: typing.List[pathlib.Path], + files_search_explicitly_listed_files: typing.List[pathlib.Path], + files_search_root_folder: pathlib.Path, flags: typing.List[str], + git_service: typing.Optional[str], + handle_no_reports_found: bool, + job_code: typing.Optional[str], name: typing.Optional[str], + network_filter: typing.Optional[str], + network_prefix: typing.Optional[str], network_root_folder: pathlib.Path, - files_search_root_folder: pathlib.Path, - files_search_exclude_folders: typing.List[pathlib.Path], - files_search_explicitly_listed_files: typing.List[pathlib.Path], - disable_search: bool, - disable_file_fixes: bool, - token: typing.Optional[str], plugin_names: typing.List[str], - branch: typing.Optional[str], - slug: typing.Optional[str], pull_request_number: typing.Optional[str], - use_legacy_uploader: bool, - fail_on_error: bool, - dry_run: bool, - git_service: typing.Optional[str], - handle_no_reports_found: bool, report_type: str, + slug: typing.Optional[str], + token: typing.Optional[str], + use_legacy_uploader: bool, ): versioning_system = ctx.obj["versioning_system"] codecov_yaml = ctx.obj["codecov_yaml"] or {} @@ -214,29 +224,31 @@ def do_upload( "Starting upload processing", extra=dict( extra_log_attributes=dict( - upload_file_type=report_type, - commit_sha=commit_sha, - report_code=report_code, + branch=branch, build_code=build_code, build_url=build_url, - job_code=job_code, + commit_sha=commit_sha, + disable_file_fixes=disable_file_fixes, + disable_search=disable_search, + enterprise_url=enterprise_url, env_vars=env_vars, + files_search_exclude_folders=files_search_exclude_folders, + files_search_explicitly_listed_files=files_search_explicitly_listed_files, + files_search_root_folder=files_search_root_folder, flags=flags, + git_service=git_service, + handle_no_reports_found=handle_no_reports_found, + job_code=job_code, name=name, + network_filter=network_filter, + network_prefix=network_prefix, network_root_folder=network_root_folder, - files_search_root_folder=files_search_root_folder, - files_search_exclude_folders=files_search_exclude_folders, - files_search_explicitly_listed_files=files_search_explicitly_listed_files, plugin_names=plugin_names, - token=token, - branch=branch, - slug=slug, pull_request_number=pull_request_number, - git_service=git_service, - enterprise_url=enterprise_url, - disable_search=disable_search, - disable_file_fixes=disable_file_fixes, - handle_no_reports_found=handle_no_reports_found, + report_code=report_code, + slug=slug, + token=token, + upload_file_type=report_type, ) ), ) @@ -244,30 +256,32 @@ def do_upload( cli_config, versioning_system, ci_adapter, - upload_file_type=report_type, - commit_sha=commit_sha, - report_code=report_code, + branch=branch, build_code=build_code, build_url=build_url, - job_code=job_code, + commit_sha=commit_sha, + disable_file_fixes=disable_file_fixes, + disable_search=disable_search, + dry_run=dry_run, + enterprise_url=enterprise_url, env_vars=env_vars, + fail_on_error=fail_on_error, + files_search_exclude_folders=list(files_search_exclude_folders), + files_search_explicitly_listed_files=list(files_search_explicitly_listed_files), + files_search_root_folder=files_search_root_folder, flags=flags, + git_service=git_service, + handle_no_reports_found=handle_no_reports_found, + job_code=job_code, name=name, + network_filter=network_filter, + network_prefix=network_prefix, network_root_folder=network_root_folder, - files_search_root_folder=files_search_root_folder, - files_search_exclude_folders=list(files_search_exclude_folders), - files_search_explicitly_listed_files=list(files_search_explicitly_listed_files), plugin_names=plugin_names, - token=token, - branch=branch, - slug=slug, pull_request_number=pull_request_number, + report_code=report_code, + slug=slug, + token=token, + upload_file_type=report_type, use_legacy_uploader=use_legacy_uploader, - fail_on_error=fail_on_error, - dry_run=dry_run, - git_service=git_service, - enterprise_url=enterprise_url, - disable_search=disable_search, - handle_no_reports_found=handle_no_reports_found, - disable_file_fixes=disable_file_fixes, ) diff --git a/codecov_cli/commands/upload_process.py b/codecov_cli/commands/upload_process.py index 20b67352..2f5acd3e 100644 --- a/codecov_cli/commands/upload_process.py +++ b/codecov_cli/commands/upload_process.py @@ -31,6 +31,8 @@ def upload_process( env_vars: typing.Dict[str, str], flags: typing.List[str], name: typing.Optional[str], + network_filter: typing.Optional[str], + network_prefix: typing.Optional[str], network_root_folder: pathlib.Path, files_search_root_folder: pathlib.Path, files_search_exclude_folders: typing.List[pathlib.Path], diff --git a/codecov_cli/services/upload/__init__.py b/codecov_cli/services/upload/__init__.py index 30960ae6..a8fc64d7 100644 --- a/codecov_cli/services/upload/__init__.py +++ b/codecov_cli/services/upload/__init__.py @@ -25,32 +25,34 @@ def do_upload_logic( versioning_system: VersioningSystemInterface, ci_adapter: CIAdapterBase, *, - commit_sha: str, - report_code: str, + branch: typing.Optional[str], build_code: typing.Optional[str], build_url: typing.Optional[str], - job_code: typing.Optional[str], + commit_sha: str, + disable_file_fixes: bool = False, + disable_search: bool = False, + dry_run: bool = False, + enterprise_url: typing.Optional[str], env_vars: typing.Dict[str, str], + fail_on_error: bool = False, + files_search_exclude_folders: typing.List[Path], + files_search_explicitly_listed_files: typing.List[Path], + files_search_root_folder: Path, flags: typing.List[str], + git_service: typing.Optional[str], + handle_no_reports_found: bool = False, + job_code: typing.Optional[str], name: typing.Optional[str], + network_filter: typing.Optional[str], + network_prefix: typing.Optional[str], network_root_folder: Path, - files_search_root_folder: Path, - files_search_exclude_folders: typing.List[Path], - files_search_explicitly_listed_files: typing.List[Path], plugin_names: typing.List[str], - token: str, - branch: typing.Optional[str], - slug: typing.Optional[str], pull_request_number: typing.Optional[str], + report_code: str, + slug: typing.Optional[str], + token: str, upload_file_type: str = "coverage", use_legacy_uploader: bool = False, - fail_on_error: bool = False, - dry_run: bool = False, - git_service: typing.Optional[str], - enterprise_url: typing.Optional[str], - disable_search: bool = False, - handle_no_reports_found: bool = False, - disable_file_fixes: bool = False, ): if upload_file_type == "coverage": preparation_plugins = select_preparation_plugins(cli_config, plugin_names) @@ -63,7 +65,12 @@ def do_upload_logic( disable_search, upload_file_type, ) - network_finder = select_network_finder(versioning_system) + network_finder = select_network_finder( + versioning_system, + network_filter=network_filter, + network_prefix=network_prefix, + network_root_folder=network_root_folder, + ) collector = UploadCollector( preparation_plugins, network_finder, file_selector, disable_file_fixes ) diff --git a/codecov_cli/services/upload/network_finder.py b/codecov_cli/services/upload/network_finder.py index 3ccfb463..8da568eb 100644 --- a/codecov_cli/services/upload/network_finder.py +++ b/codecov_cli/services/upload/network_finder.py @@ -5,17 +5,39 @@ class NetworkFinder(object): - def __init__(self, versioning_system: VersioningSystemInterface): + def __init__( + self, + versioning_system: VersioningSystemInterface, + network_filter: typing.Optional[str], + network_prefix: typing.Optional[str], + network_root_folder: pathlib.Path, + ): self.versioning_system = versioning_system + self.network_filter = network_filter + self.network_prefix = network_prefix + self.network_root_folder = network_root_folder - def find_files( - self, - network_root: typing.Optional[pathlib.Path] = None, - network_filter=None, - network_adjuster=None, - ) -> typing.List[str]: - return self.versioning_system.list_relevant_files(network_root) + def find_files(self, ignore_filters=False) -> typing.List[str]: + files = self.versioning_system.list_relevant_files(self.network_root_folder) + + if not ignore_filters: + if self.network_filter: + files = [file for file in files if file.startswith(self.network_filter)] + if self.network_prefix: + files = [self.network_prefix + file for file in files] + + return files -def select_network_finder(versioning_system: VersioningSystemInterface): - return NetworkFinder(versioning_system) +def select_network_finder( + versioning_system: VersioningSystemInterface, + network_filter: typing.Optional[str], + network_prefix: typing.Optional[str], + network_root_folder: pathlib.Path, +): + return NetworkFinder( + versioning_system, + network_filter, + network_prefix, + network_root_folder, + ) diff --git a/codecov_cli/services/upload/upload_collector.py b/codecov_cli/services/upload/upload_collector.py index d687ab98..9ba8564f 100644 --- a/codecov_cli/services/upload/upload_collector.py +++ b/codecov_cli/services/upload/upload_collector.py @@ -36,10 +36,10 @@ def __init__( self.file_finder = file_finder self.disable_file_fixes = disable_file_fixes - def _produce_file_fixes_for_network( - self, network: typing.List[str] + def _produce_file_fixes( + self, files: typing.List[str] ) -> typing.List[UploadCollectionResultFileFixer]: - if not network or self.disable_file_fixes: + if not files or self.disable_file_fixes: return [] # patterns that we don't need to specify a reason for empty_line_regex = re.compile(r"^\s*$") @@ -94,7 +94,7 @@ def _produce_file_fixes_for_network( } result = [] - for filename in network: + for filename in files: for glob, fix_patterns in file_regex_patterns.items(): if fnmatch(filename, glob): result.append(self._get_file_fixes(filename, fix_patterns)) @@ -150,9 +150,9 @@ def generate_upload_data(self, report_type="coverage") -> UploadCollectionResult prep.run_preparation(self) logger.debug("Collecting relevant files") network = self.network_finder.find_files() - files = self.file_finder.find_files() - logger.info(f"Found {len(files)} {report_type} files to upload") - if not files: + report_files = self.file_finder.find_files() + logger.info(f"Found {len(report_files)} {report_type} files to report") + if not report_files: if report_type == "test_results": error_message = "No JUnit XML reports found. Please review our documentation (https://docs.codecov.com/docs/test-result-ingestion-beta) to generate and upload the file." else: @@ -163,13 +163,13 @@ def generate_upload_data(self, report_type="coverage") -> UploadCollectionResult fg="red", ) ) - for file in files: + for file in report_files: logger.info(f"> {file}") return UploadCollectionResult( network=network, - files=files, + files=report_files, file_fixes=( - self._produce_file_fixes_for_network(network) + self._produce_file_fixes(self.network_finder.find_files(True)) if report_type == "coverage" else [] ), diff --git a/tests/commands/test_invoke_upload_process.py b/tests/commands/test_invoke_upload_process.py index cac7a116..10d2a52d 100644 --- a/tests/commands/test_invoke_upload_process.py +++ b/tests/commands/test_invoke_upload_process.py @@ -118,6 +118,14 @@ def test_upload_process_options(mocker): " The type of the file to upload, coverage by", " default. Possible values are: testing,", " coverage.", + " --network-filter TEXT Specify a filter on the files listed in the", + " network section of the Codecov report. This", + " will only add files whose path begin with the", + " specified filter. Useful for upload-specific", + " path fixing", + " --network-prefix TEXT Specify a prefix on files listed in the", + " network section of the Codecov report. Useful", + " to help resolve path fixing", " --parent-sha TEXT SHA (with 40 chars) of what should be the", " parent of this commit", " -h, --help Show this message and exit.", diff --git a/tests/helpers/test_network_finder.py b/tests/helpers/test_network_finder.py index b7881812..859abc7d 100644 --- a/tests/helpers/test_network_finder.py +++ b/tests/helpers/test_network_finder.py @@ -6,11 +6,46 @@ def test_find_files(mocker, tmp_path): + filenames = ["a.txt", "b.txt"] + filtered_filenames = [] - expected_filenames = ["a.txt", "b.txt"] + mocked_vs = MagicMock() + mocked_vs.list_relevant_files.return_value = filenames + + assert NetworkFinder(versioning_system=mocked_vs, network_filter=None, network_prefix=None, network_root_folder=tmp_path).find_files() == filenames + assert NetworkFinder(versioning_system=mocked_vs, network_filter="hello", network_prefix="bello", network_root_folder=tmp_path).find_files(False) == filtered_filenames + assert NetworkFinder(versioning_system=mocked_vs, network_filter="hello", network_prefix="bello", network_root_folder=tmp_path).find_files(True) == filenames + mocked_vs.list_relevant_files.assert_called_with(tmp_path) + +def test_find_files_with_filter(mocker, tmp_path): + filenames = ["hello/a.txt", "hello/c.txt", "bello/b.txt"] + filtered_filenames = ["hello/a.txt", "hello/c.txt"] + + mocked_vs = MagicMock() + mocked_vs.list_relevant_files.return_value = filenames + + assert NetworkFinder(versioning_system=mocked_vs, network_filter="hello", network_prefix=None, network_root_folder=tmp_path).find_files() == filtered_filenames + assert NetworkFinder(versioning_system=mocked_vs, network_filter="hello", network_prefix="bello", network_root_folder=tmp_path).find_files(True) == filenames + mocked_vs.list_relevant_files.assert_called_with(tmp_path) + +def test_find_files_with_prefix(mocker, tmp_path): + filenames = ["hello/a.txt", "hello/c.txt", "bello/b.txt"] + filtered_filenames = ["hellohello/a.txt", "hellohello/c.txt", "hellobello/b.txt"] + + mocked_vs = MagicMock() + mocked_vs.list_relevant_files.return_value = filenames + + assert NetworkFinder(versioning_system=mocked_vs, network_filter=None, network_prefix="hello", network_root_folder=tmp_path).find_files() == filtered_filenames + assert NetworkFinder(versioning_system=mocked_vs, network_filter="hello", network_prefix="bello", network_root_folder=tmp_path).find_files(True) == filenames + mocked_vs.list_relevant_files.assert_called_with(tmp_path) + +def test_find_files_with_filter_and_prefix(mocker, tmp_path): + filenames = ["hello/a.txt", "hello/c.txt", "bello/b.txt"] + filtered_filenames = ["bellohello/a.txt", "bellohello/c.txt"] mocked_vs = MagicMock() - mocked_vs.list_relevant_files.return_value = expected_filenames + mocked_vs.list_relevant_files.return_value = filenames - assert NetworkFinder(mocked_vs).find_files(tmp_path) == expected_filenames + assert NetworkFinder(versioning_system=mocked_vs, network_filter="hello", network_prefix="bello", network_root_folder=tmp_path).find_files() == filtered_filenames + assert NetworkFinder(versioning_system=mocked_vs, network_filter="hello", network_prefix="bello", network_root_folder=tmp_path).find_files(True) == filenames mocked_vs.list_relevant_files.assert_called_with(tmp_path) diff --git a/tests/services/upload/test_upload_collector.py b/tests/services/upload/test_upload_collector.py index 0df1c0bc..183ff8db 100644 --- a/tests/services/upload/test_upload_collector.py +++ b/tests/services/upload/test_upload_collector.py @@ -13,7 +13,7 @@ def test_fix_kt_files(): col = UploadCollector(None, None, None) - fixes = col._produce_file_fixes_for_network([str(kt_file)]) + fixes = col._produce_file_fixes([kt_file]) assert len(fixes) == 1 fixes_for_kt_file = fixes[0] @@ -33,7 +33,7 @@ def test_fix_go_files(): col = UploadCollector(None, None, None) - fixes = col._produce_file_fixes_for_network([str(go_file)]) + fixes = col._produce_file_fixes([go_file]) assert len(fixes) == 1 fixes_for_go_file = fixes[0] @@ -59,7 +59,7 @@ def test_fix_bad_encoding_files(mock_open): col = UploadCollector(None, None, None) - fixes = col._produce_file_fixes_for_network([str(go_file)]) + fixes = col._produce_file_fixes([go_file]) assert len(fixes) == 1 fixes_for_go_file = fixes[0] assert fixes_for_go_file.eof is None @@ -72,7 +72,7 @@ def test_fix_php_files(): col = UploadCollector(None, None, None) - fixes = col._produce_file_fixes_for_network([str(php_file)]) + fixes = col._produce_file_fixes([php_file]) assert len(fixes) == 1 fixes_for_php_file = fixes[0] @@ -87,7 +87,7 @@ def test_fix_for_cpp_swift_vala(tmp_path): col = UploadCollector(None, None, None) - fixes = col._produce_file_fixes_for_network([str(cpp_file)]) + fixes = col._produce_file_fixes([cpp_file]) assert len(fixes) == 1 fixes_for_cpp_file = fixes[0] @@ -109,7 +109,7 @@ def test_fix_when_disabled_fixes(tmp_path): col = UploadCollector(None, None, None, True) - fixes = col._produce_file_fixes_for_network([str(cpp_file)]) + fixes = col._produce_file_fixes([cpp_file]) assert len(fixes) == 0 assert fixes == [] @@ -164,7 +164,7 @@ def test_generate_upload_data(tmp_path): file_finder = FileFinder(tmp_path) - network_finder = NetworkFinder(GitVersioningSystem()) + network_finder = NetworkFinder(GitVersioningSystem(), None, None, None) collector = UploadCollector([], network_finder, file_finder) diff --git a/tests/services/upload/test_upload_service.py b/tests/services/upload/test_upload_service.py index 9f7dbf61..8f7ad3e4 100644 --- a/tests/services/upload/test_upload_service.py +++ b/tests/services/upload/test_upload_service.py @@ -56,6 +56,8 @@ def test_do_upload_logic_happy_path_legacy_uploader(mocker): env_vars=None, flags=None, name="name", + network_filter=None, + network_prefix=None, network_root_folder=None, files_search_root_folder=None, files_search_exclude_folders=None, @@ -81,7 +83,7 @@ def test_do_upload_logic_happy_path_legacy_uploader(mocker): cli_config, ["first_plugin", "another", "forth"] ) mock_select_file_finder.assert_called_with(None, None, None, False, "coverage") - mock_select_network_finder.assert_called_with(versioning_system) + mock_select_network_finder.assert_called_with(versioning_system, network_filter=None, network_prefix=None, network_root_folder=None) mock_generate_upload_data.assert_called_with("coverage") mock_send_upload_data.assert_called_with( mock_generate_upload_data.return_value, @@ -144,6 +146,8 @@ def test_do_upload_logic_happy_path(mocker): env_vars=None, flags=None, name="name", + network_filter=None, + network_prefix=None, network_root_folder=None, files_search_root_folder=None, files_search_exclude_folders=None, @@ -168,7 +172,7 @@ def test_do_upload_logic_happy_path(mocker): cli_config, ["first_plugin", "another", "forth"] ) mock_select_file_finder.assert_called_with(None, None, None, False, "coverage") - mock_select_network_finder.assert_called_with(versioning_system) + mock_select_network_finder.assert_called_with(versioning_system, network_filter=None, network_prefix=None, network_root_folder=None) mock_generate_upload_data.assert_called_with("coverage") mock_send_upload_data.assert_called_with( mock_generate_upload_data.return_value, @@ -227,6 +231,8 @@ def test_do_upload_logic_dry_run(mocker): env_vars=None, flags=None, name="name", + network_filter=None, + network_prefix=None, network_root_folder=None, files_search_root_folder=None, files_search_exclude_folders=None, @@ -242,7 +248,7 @@ def test_do_upload_logic_dry_run(mocker): ) out_bytes = parse_outstreams_into_log_lines(outstreams[0].getvalue()) mock_select_file_finder.assert_called_with(None, None, None, False, "coverage") - mock_select_network_finder.assert_called_with(versioning_system) + mock_select_network_finder.assert_called_with(versioning_system, network_filter=None, network_prefix=None, network_root_folder=None) assert mock_generate_upload_data.call_count == 1 assert mock_send_upload_data.call_count == 0 mock_select_preparation_plugins.assert_called_with( @@ -288,6 +294,8 @@ def test_do_upload_logic_verbose(mocker, use_verbose_option): env_vars=None, flags=None, name="name", + network_filter=None, + network_prefix=None, network_root_folder=None, files_search_root_folder=None, files_search_exclude_folders=None, @@ -363,6 +371,8 @@ def side_effect(*args, **kwargs): env_vars=None, flags=None, name="name", + network_filter=None, + network_prefix=None, network_root_folder=None, files_search_root_folder=None, files_search_exclude_folders=None, @@ -393,7 +403,7 @@ def side_effect(*args, **kwargs): cli_config, ["first_plugin", "another", "forth"] ) mock_select_file_finder.assert_called_with(None, None, None, False, "coverage") - mock_select_network_finder.assert_called_with(versioning_system) + mock_select_network_finder.assert_called_with(versioning_system, network_filter=None, network_prefix=None, network_root_folder=None) mock_generate_upload_data.assert_called_with("coverage") mock_upload_completion_call.assert_called_with( commit_sha="commit_sha", @@ -444,6 +454,8 @@ def side_effect(*args, **kwargs): env_vars=None, flags=None, name="name", + network_filter=None, + network_prefix=None, network_root_folder=None, files_search_root_folder=None, files_search_exclude_folders=None, @@ -465,7 +477,7 @@ def side_effect(*args, **kwargs): cli_config, ["first_plugin", "another", "forth"] ) mock_select_file_finder.assert_called_with(None, None, None, False, "coverage") - mock_select_network_finder.assert_called_with(versioning_system) + mock_select_network_finder.assert_called_with(versioning_system, network_filter=None, network_prefix=None, network_root_folder=None) mock_generate_upload_data.assert_called_with("coverage") @@ -509,7 +521,9 @@ def test_do_upload_logic_happy_path_test_results(mocker): env_vars=None, flags=None, name="name", - network_root_folder=None, + network_filter="some_dir", + network_prefix="hello/", + network_root_folder="root/", files_search_root_folder=None, files_search_exclude_folders=None, files_search_explicitly_listed_files=None, @@ -531,7 +545,7 @@ def test_do_upload_logic_happy_path_test_results(mocker): assert res == UploadSender.send_upload_data.return_value mock_select_preparation_plugins.assert_not_called mock_select_file_finder.assert_called_with(None, None, None, False, "test_results") - mock_select_network_finder.assert_called_with(versioning_system) + mock_select_network_finder.assert_called_with(versioning_system, network_filter="some_dir", network_prefix="hello/", network_root_folder="root/") mock_generate_upload_data.assert_called_with("test_results") mock_send_upload_data.assert_called_with( mock_generate_upload_data.return_value,