Sync updater files to align with 0.234.0 (#852)

server/Tingle.Dependabot/Controllers/UpdateJobsController.cs

@@ -79,6 +79,22 @@ public async Task<IActionResult> RecordErrorAsync([FromRoute, Required] string i
         return Ok();
     }
 
+    [HttpPost("{id}/record_update_job_unknown_error")]
+    public async Task<IActionResult> RecordUpdateJobUnknownErrorAsync([FromRoute, Required] string id, [FromBody] PayloadWithData<DependabotRecordUpdateJobErrorModel> model)
+    {
+        var job = await dbContext.UpdateJobs.SingleAsync(j => j.Id == id);
+
+        job.Error = new UpdateJobError
+        {
+            Type = model.Data!.ErrorType,
+            Detail = model.Data.ErrorDetail,
+        };
+
+        await dbContext.SaveChangesAsync();
+
+        return Ok();
+    }
+
     [HttpPatch("{id}/mark_as_processed")]
     public async Task<IActionResult> MarkAsProcessedAsync([FromRoute, Required] string id, [FromBody] PayloadWithData<DependabotMarkAsProcessedModel> model)
     {

server/Tingle.Dependabot/Workflow/UpdateRunner.cs

@@ -121,7 +121,12 @@ public async Task CreateAsync(Project project, Repository repository, Repository
         };
 
         // write job definition file
-        var jobDefinitionPath = await WriteJobDefinitionAsync(project, update, job, directory, credentials, cancellationToken);
+        var experiments = new Dictionary<string, bool>
+        {
+            // ["record-ecosystem-versions"] = await featureManager.IsEnabledAsync(FeatureNames.RecordEcosystemVersions, fmc),
+            // ["record-update-job-unknown-error"] = await featureManager.IsEnabledAsync(FeatureNames.RecordUpdateJobUnknownError, fmc),
+        };
+        var jobDefinitionPath = await WriteJobDefinitionAsync(project, update, job, experiments, directory, credentials, cancellationToken);
         logger.WrittenJobDefinitionFile(job.Id, jobDefinitionPath);
 
         // create the ContainerApp Job
@@ -291,6 +296,7 @@ internal async Task<IDictionary<string, string>> CreateEnvironmentVariables(Proj
     internal async Task<string> WriteJobDefinitionAsync(Project project,
                                                         RepositoryUpdate update,
                                                         UpdateJob job,
+                                                        IDictionary<string, bool> experiments,
                                                         string directory,
                                                         IList<Dictionary<string, string>> credentials,
                                                         CancellationToken cancellationToken = default) // TODO: unit test this
@@ -314,6 +320,7 @@ internal async Task<string> WriteJobDefinitionAsync(Project project,
                 // ["dependencies"] = null, // object array
                 ["directory"] = job.Directory,
                 // ["existing-pull-requests"] = null, // object array
+                ["experiments"] = ToJsonNode(experiments),
                 ["ignore-conditions"] = ToJsonNode(update.Ignore ?? new()),
                 // ["security-advisories"] = null, // object array
                 ["package_manager"] = ConvertEcosystemToPackageManager(job.PackageEcosystem!),

update-files.ps1

@@ -23,6 +23,7 @@ $files = @(
     "updater/config/.yarnrc"
 
     "updater/lib/dependabot/logger/formats.rb"
+    "updater/lib/dependabot/updater/operations/create_group_security_update_pull_request.rb"
     "updater/lib/dependabot/updater/operations/create_group_update_pull_request.rb"
     "updater/lib/dependabot/updater/operations/create_security_update_pull_request.rb"
     "updater/lib/dependabot/updater/operations/group_update_all_versions.rb"

updater/lib/dependabot/api_client.rb

@@ -89,6 +89,24 @@ def record_update_job_error(error_type:, error_details:)
       sleep(rand(3.0..10.0)) && retry
     end
 
+    def record_update_job_unknown_error(error_type: "unknown_error", error_details:)
+      api_url = "#{base_url}/update_jobs/#{job_id}/record_update_job_unknown_error"
+      body = {
+        data: {
+          "error-type": error_type,
+          "error-details": error_details
+        }
+      }
+      response = http_client.post(api_url, json: body)
+      raise ApiError, response.body if response.code >= 400
+    rescue HTTP::ConnectionError, OpenSSL::SSL::SSLError
+      retry_count ||= 0
+      retry_count += 1
+      raise if retry_count > 3
+
+      sleep(rand(3.0..10.0)) && retry
+    end
+
     def mark_job_as_processed(base_commit_sha)
       api_url = "#{base_url}/update_jobs/#{job_id}/mark_as_processed"
       body = { data: { "base-commit-sha": base_commit_sha } }

updater/lib/dependabot/base_command.rb

@@ -64,9 +64,27 @@ def run
     def handle_exception(err)
       Dependabot.logger.error(err.message)
       err.backtrace.each { |line| Dependabot.logger.error(line) }
-
       service.capture_exception(error: err, job: job)
       service.record_update_job_error(error_type: "unknown_error", error_details: { message: err.message })
+      # We don't set this flag in GHES because there older GHES version does not support reporting unknown errors.
+      handle_unknown_error(err) if Experiments.enabled?(:record_update_job_unknown_error)
+    end
+
+    def handle_unknown_error(err)
+      error_details = {
+        "error-class" => err.class.to_s,
+        "error-message" => err.message,
+        "error-backtrace" => err.backtrace.join("\n"),
+        "package-manager" => job.package_manager,
+        "job-id" => job.id,
+        "job-dependencies" => job.dependencies,
+        "job-dependency_group" => job.dependency_groups
+      }.compact
+      service.record_update_job_unknown_error(error_type: "updater_error", error_details: error_details)
+      service.increment_metric("updater.update_job_unknown_error", tags: {
+        package_manager: job.package_manager,
+        class_name: err.class.name
+      })
     end
 
     def job_id

updater/lib/dependabot/dependency_snapshot.rb

@@ -26,7 +26,11 @@ def self.create_from_job_definition(job:, job_definition:)
       )
     end
 
-    attr_reader :base_commit_sha, :dependency_files, :dependencies
+    attr_reader :base_commit_sha, :dependency_files, :dependencies, :handled_dependencies
+
+    def add_handled_dependencies(dependency_names)
+      @handled_dependencies += Array(dependency_names)
+    end
 
     # Returns the subset of all project dependencies which are permitted
     # by the project configuration.
@@ -57,16 +61,13 @@ def job_dependencies
     # Returns just the group that is specifically requested to be updated by
     # the job definition
     def job_group
-      return nil unless Dependabot::Experiments.enabled?(:grouped_updates_prototype)
       return nil unless job.dependency_group_to_refresh
       return @job_group if defined?(@job_group)
 
       @job_group = @dependency_group_engine.find_group(name: job.dependency_group_to_refresh)
     end
 
     def groups
-      return [] unless Dependabot::Experiments.enabled?(:grouped_updates_prototype)
-
       @dependency_group_engine.dependency_groups
     end
 
@@ -75,8 +76,7 @@ def ungrouped_dependencies
       return allowed_dependencies unless groups.any?
 
       # Otherwise return dependencies that haven't been handled during the group update portion.
-      all_handled_dependencies = Set.new(groups.map { |g| g.handled_dependencies.to_a }.flatten)
-      allowed_dependencies.reject { |dep| all_handled_dependencies.include?(dep.name) }
+      allowed_dependencies.reject { |dep| @handled_dependencies.include?(dep.name) }
     end
 
     private
@@ -85,11 +85,10 @@ def initialize(job:, base_commit_sha:, dependency_files:)
       @job = job
       @base_commit_sha = base_commit_sha
       @dependency_files = dependency_files
+      @handled_dependencies = Set.new
 
       @dependencies = parse_files!
 
-      return unless Dependabot::Experiments.enabled?(:grouped_updates_prototype)
-
       @dependency_group_engine = DependencyGroupEngine.from_job_config(job: job)
       @dependency_group_engine.assign_to_groups!(dependencies: allowed_dependencies)
     end

updater/lib/dependabot/file_fetcher_command.rb

@@ -120,6 +120,15 @@ def already_cloned?
     def handle_file_fetcher_error(error)
       error_details =
         case error
+        when Dependabot::ToolVersionNotSupported
+          {
+            "error-type": "tool_version_not_supported",
+            "error-detail": {
+              "tool-name": error.tool_name,
+              "detected-version": error.detected_version,
+              "supported-versions": error.supported_versions
+            }
+          }
         when Dependabot::BranchNotFound
           {
             "error-type": "branch_not_found",
@@ -161,7 +170,7 @@ def handle_file_fetcher_error(error)
           # If we get a 500 from GitHub there's very little we can do about it,
           # and responsibility for fixing it is on them, not us. As a result we
           # quietly log these as errors
-          { "error-type": "unknown_error" }
+          { "error-type": "server_error" }
         when *Octokit::RATE_LIMITED_ERRORS
           # If we get a rate-limited error we let dependabot-api handle the
           # retry by re-enqueing the update job after the reset
@@ -172,11 +181,23 @@ def handle_file_fetcher_error(error)
             }
           }
         else
-          Dependabot.logger.error(error.message)
-          error.backtrace.each { |line| Dependabot.logger.error line }
+          log_error(error)
+
+          unknown_error_details = {
+            "error-class" => error.class.to_s,
+            "error-message" => error.message,
+            "error-backtrace" => error.backtrace.join("\n"),
+            "package-manager" => job.package_manager,
+            "job-id" => job.id,
+            "job-dependencies" => job.dependencies,
+            "job-dependency_group" => job.dependency_groups
+          }.compact
 
           service.capture_exception(error: error, job: job)
-          { "error-type": "unknown_error" }
+          {
+            "error-type": "file_fetcher_error",
+            "error-detail": unknown_error_details
+          }
         end
 
       record_error(error_details) if error_details
@@ -190,11 +211,25 @@ def rate_limit_error_remaining(error)
       remaining.positive? ? remaining : 0
     end
 
+    def log_error(error)
+      Dependabot.logger.error(error.message)
+      error.backtrace.each { |line| Dependabot.logger.error line }
+    end
+
     def record_error(error_details)
       service.record_update_job_error(
         error_type: error_details.fetch(:"error-type"),
         error_details: error_details[:"error-detail"]
       )
+
+      # We don't set this flag in GHES because there older GHES version does not support reporting unknown errors.
+      return unless Experiments.enabled?(:record_update_job_unknown_error)
+      return unless error_details.fetch(:"error-type") == "file_fetcher_error"
+
+      service.record_update_job_unknown_error(
+        error_type: error_details.fetch(:"error-type"),
+        error_details: error_details[:"error-detail"]
+      )
     end
 
     # Perform a debug check of connectivity to GitHub/GHES. This also ensures

updater/lib/dependabot/job.rb

@@ -171,7 +171,14 @@ def reject_external_code?
     # separately, if required.
     #
     # rubocop:disable Metrics/PerceivedComplexity
+    # rubocop:disable Metrics/CyclomaticComplexity
     def allowed_update?(dependency)
+      # Ignoring all versions is another way to say no updates allowed
+      if completely_ignored?(dependency)
+        Dependabot.logger.info("All versions of #{dependency.name} ignored, no update allowed")
+        return false
+      end
+
       allowed_updates.any? do |update|
         # Check the update-type (defaulting to all)
         update_type = update.fetch("update-type", "all")
@@ -200,6 +207,7 @@ def allowed_update?(dependency)
       end
     end
     # rubocop:enable Metrics/PerceivedComplexity
+    # rubocop:enable Metrics/CyclomaticComplexity
 
     def vulnerable?(dependency)
       security_advisories = security_advisories_for(dependency)
@@ -296,6 +304,10 @@ def log_ignore_conditions_for(dependency)
 
     private
 
+    def completely_ignored?(dependency)
+      ignore_conditions_for(dependency).any?(Dependabot::Config::IgnoreCondition::ALL_VERSIONS)
+    end
+
     def register_experiments
       experiments.each do |name, value|
         Dependabot::Experiments.register(name, value)

updater/lib/dependabot/service.rb

@@ -49,6 +49,10 @@ def record_update_job_error(error_type:, error_details:, dependency: nil)
       client.record_update_job_error(error_type: error_type, error_details: error_details)
     end
 
+    def record_update_job_unknown_error(error_type:, error_details:)
+      client.record_update_job_unknown_error(error_type: error_type, error_details: error_details)
+    end
+
     def update_dependency_list(dependency_snapshot:)
       dependency_payload = dependency_snapshot.dependencies.map do |dep|
         {

updater/lib/dependabot/update_files_command.rb

@@ -58,6 +58,8 @@ def base_commit_sha
       Environment.job_definition["base_commit_sha"]
     end
 
+    # rubocop:disable Metrics/AbcSize
+    # rubocop:disable Metrics/CyclomaticComplexity
     # rubocop:disable Metrics/MethodLength
     def handle_parser_error(error)
       # This happens if the repo gets removed after a job gets kicked off.
@@ -121,7 +123,7 @@ def handle_parser_error(error)
           # If we get a 500 from GitHub there's very little we can do about it,
           # and responsibility for fixing it is on them, not us. As a result we
           # quietly log these as errors
-          { "error-type": "unknown_error" }
+          { "error-type": "server_error" }
         else
           # Check if the error is a known "run halting" state we should handle
           if (error_type = Updater::ErrorHandler::RUN_HALTING_ERRORS[error.class])
@@ -131,19 +133,41 @@ def handle_parser_error(error)
             # tracker know about it
             Dependabot.logger.error error.message
             error.backtrace.each { |line| Dependabot.logger.error line }
+            unknown_error_details = {
+              "error-class" => error.class.to_s,
+              "error-message" => error.message,
+              "error-backtrace" => error.backtrace.join("\n"),
+              "package-manager" => job.package_manager,
+              "job-id" => job.id,
+              "job-dependencies" => job.dependencies,
+              "job-dependency_group" => job.dependency_groups
+            }.compact
 
             service.capture_exception(error: error, job: job)
 
-            # Set an unknown error type to be added to the job
-            { "error-type": "unknown_error" }
+            # Set an unknown error type as update_files_error to be added to the job
+            {
+              "error-type": "update_files_error",
+              "error-detail": unknown_error_details
+            }
           end
         end
 
       service.record_update_job_error(
         error_type: error_details.fetch(:"error-type"),
         error_details: error_details[:"error-detail"]
       )
+      # We don't set this flag in GHES because there older GHES version does not support reporting unknown errors.
+      return unless Experiments.enabled?(:record_update_job_unknown_error)
+      return unless error_details.fetch(:"error-type") == "update_files_error"
+
+      service.record_update_job_unknown_error(
+        error_type: error_details.fetch(:"error-type"),
+        error_details: error_details[:"error-detail"]
+      )
     end
+    # rubocop:enable Metrics/AbcSize
+    # rubocop:enable Metrics/CyclomaticComplexity
     # rubocop:enable Metrics/MethodLength
   end
 end