diff --git a/src/clusterfuzz/_internal/bot/tasks/utasks/corpus_pruning_task.py b/src/clusterfuzz/_internal/bot/tasks/utasks/corpus_pruning_task.py index 4ed00611df..641718f71f 100644 --- a/src/clusterfuzz/_internal/bot/tasks/utasks/corpus_pruning_task.py +++ b/src/clusterfuzz/_internal/bot/tasks/utasks/corpus_pruning_task.py @@ -595,16 +595,12 @@ def _record_cross_pollination_stats(output): client.insert([big_query.Insert(row=bigquery_row, insert_id=None)]) -def do_corpus_pruning(uworker_input, context, revision) -> CorpusPruningResult: +def do_corpus_pruning(context, revision) -> CorpusPruningResult: """Run corpus pruning.""" # Set |FUZZ_TARGET| environment variable to help with unarchiving only fuzz # target and its related files. environment.set_value('FUZZ_TARGET', context.fuzz_target.binary) - if environment.is_trusted_host(): - from clusterfuzz._internal.bot.untrusted_runner import tasks_host - return tasks_host.do_corpus_pruning(uworker_input, context, revision) - if not build_manager.setup_build( revision=revision, fuzz_target=context.fuzz_target.binary): raise CorpusPruningError('Failed to setup build.') @@ -729,23 +725,7 @@ def do_corpus_pruning(uworker_input, context, revision) -> CorpusPruningResult: cross_pollination_stats=cross_pollination_stats) -def _update_crash_unit_path(context, crash): - """If running on a trusted host, updates the crash unit_path after copying - the file locally.""" - if not environment.is_trusted_host(): - return - from clusterfuzz._internal.bot.untrusted_runner import file_host - unit_path = os.path.join(context.bad_units_path, - os.path.basename(crash.unit_path)) - # Prevent the worker from escaping out of |context.bad_units_path|. - if not file_host.is_directory_parent(unit_path, context.bad_units_path): - raise CorpusPruningError('Invalid units path from worker.') - - file_host.copy_file_from_worker(crash.unit_path, unit_path) - crash.unit_path = unit_path - - -def _upload_corpus_crashes_zip(context: Context, result: CorpusPruningResult, +def _upload_corpus_crashes_zip(result: CorpusPruningResult, corpus_crashes_blob_name, corpus_crashes_upload_url): """Packs the corpus crashes in a zip file. The file is then uploaded @@ -754,7 +734,6 @@ def _upload_corpus_crashes_zip(context: Context, result: CorpusPruningResult, zip_filename = os.path.join(temp_dir, corpus_crashes_blob_name) with zipfile.ZipFile(zip_filename, 'w') as zip_file: for crash in result.crashes: - _update_crash_unit_path(context, crash) unit_name = os.path.basename(crash.unit_path) zip_file.write(crash.unit_path, unit_name, zipfile.ZIP_DEFLATED) @@ -1007,11 +986,11 @@ def utask_main(uworker_input): uworker_output = None try: - result = do_corpus_pruning(uworker_input, context, revision) + result = do_corpus_pruning(context, revision) issue_metadata = engine_common.get_fuzz_target_issue_metadata(fuzz_target) issue_metadata = issue_metadata or {} _upload_corpus_crashes_zip( - context, result, + result, uworker_input.corpus_pruning_task_input.corpus_crashes_blob_name, uworker_input.corpus_pruning_task_input.corpus_crashes_upload_url) uworker_output = uworker_msg_pb2.Output( # pylint: disable=no-member diff --git a/src/clusterfuzz/_internal/bot/untrusted_runner/tasks_impl.py b/src/clusterfuzz/_internal/bot/untrusted_runner/tasks_impl.py index 8a71ade599..8b0ff1b74c 100644 --- a/src/clusterfuzz/_internal/bot/untrusted_runner/tasks_impl.py +++ b/src/clusterfuzz/_internal/bot/untrusted_runner/tasks_impl.py @@ -17,86 +17,14 @@ from google.protobuf.any_pb2 import Any # pylint: disable=no-name-in-module from clusterfuzz._internal.bot import testcase_manager -from clusterfuzz._internal.bot.tasks.utasks import corpus_pruning_task from clusterfuzz._internal.bot.tasks.utasks import fuzz_task from clusterfuzz._internal.bot.tasks.utasks import minimize_task -from clusterfuzz._internal.datastore import data_types from clusterfuzz._internal.protos import untrusted_runner_pb2 from clusterfuzz.fuzz import engine # pylint:disable=no-member -def _proto_to_fuzz_target(proto): - """Convert protobuf to FuzzTarget.""" - return data_types.FuzzTarget( - engine=proto.engine, project=proto.project, binary=proto.binary) - - -def _proto_to_cross_pollinate_fuzzer(proto): - """Convert protobuf to CrossPollinateFuzzer.""" - return corpus_pruning_task.CrossPollinateFuzzer( - fuzz_target=_proto_to_fuzz_target(proto.fuzz_target), - backup_bucket_name=proto.backup_bucket_name, - corpus_engine_name=proto.corpus_engine_name) - - -def prune_corpus(request, _): - """Prune corpus.""" - context = corpus_pruning_task.Context( - request.uworker_input, _proto_to_fuzz_target(request.fuzz_target), [ - _proto_to_cross_pollinate_fuzzer(proto) - for proto in request.cross_pollinate_fuzzers - ]) - - result = corpus_pruning_task.do_corpus_pruning(request.uworker_input, context, - request.revision) - - cross_pollination_stats = None - if result.cross_pollination_stats: - cross_pollination_stats = untrusted_runner_pb2.CrossPollinationStats( - project_qualified_name=result.cross_pollination_stats. - project_qualified_name, - sources=result.cross_pollination_stats.sources, - initial_corpus_size=result.cross_pollination_stats.initial_corpus_size, - corpus_size=result.cross_pollination_stats.corpus_size, - initial_edge_coverage=result.cross_pollination_stats. - initial_edge_coverage, - edge_coverage=result.cross_pollination_stats.edge_coverage, - initial_feature_coverage=result.cross_pollination_stats. - initial_feature_coverage, - feature_coverage=result.cross_pollination_stats.feature_coverage) - - # Intentionally skip edge and function coverage values as those would come - # from fuzzer coverage cron task (see src/go/server/cron/coverage.go). - coverage_info = untrusted_runner_pb2.CoverageInfo( - corpus_size_units=result.coverage_info.corpus_size_units, - corpus_size_bytes=result.coverage_info.corpus_size_bytes, - corpus_location=result.coverage_info.corpus_location, - corpus_backup_location=result.coverage_info.corpus_backup_location, - quarantine_size_units=result.coverage_info.quarantine_size_units, - quarantine_size_bytes=result.coverage_info.quarantine_size_bytes, - quarantine_location=result.coverage_info.quarantine_location) - - crashes = [ - untrusted_runner_pb2.CorpusCrash( - crash_state=crash.crash_state, - crash_type=crash.crash_type, - crash_address=crash.crash_address, - crash_stacktrace=crash.crash_stacktrace, - unit_path=crash.unit_path, - security_flag=crash.security_flag, - ) for crash in result.crashes - ] - - return untrusted_runner_pb2.PruneCorpusResponse( - coverage_info=coverage_info, - crashes=crashes, - fuzzer_binary_name=result.fuzzer_binary_name, - revision=result.revision, - cross_pollination_stats=cross_pollination_stats) - - def process_testcase(request, _): """Process testcase.""" tool_name_map = { diff --git a/src/clusterfuzz/_internal/tests/core/bot/tasks/utasks/corpus_pruning_task_test.py b/src/clusterfuzz/_internal/tests/core/bot/tasks/utasks/corpus_pruning_task_test.py index 7130b2bb19..d6a2186046 100644 --- a/src/clusterfuzz/_internal/tests/core/bot/tasks/utasks/corpus_pruning_task_test.py +++ b/src/clusterfuzz/_internal/tests/core/bot/tasks/utasks/corpus_pruning_task_test.py @@ -644,8 +644,7 @@ def test_upload_corpus_crashes_zip(self): cross_pollination_stats=None) corpus_pruning_task._upload_corpus_crashes_zip( - None, result, self.corpus_crashes_blob_name, - self.corpus_crashes_upload_url) + result, self.corpus_crashes_blob_name, self.corpus_crashes_upload_url) corpus_crashes_zip_local_path = os.path.join( self.temp_dir, f'{self.corpus_crashes_blob_name}.zip')