Merge pull request #5611 from psiinon/auto/continueOnFailure

Auto: Added env/continueOnFailure
zaproxy · Jul 29, 2024 · 97ff9af · 97ff9af
2 parents bb1e4eb + 9bbd42b
commit 97ff9af
Show file tree

Hide file tree

Showing 13 changed files with 239 additions and 4 deletions.
diff --git a/addOns/automation/CHANGELOG.md b/addOns/automation/CHANGELOG.md
@@ -4,6 +4,8 @@ All notable changes to this add-on will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 
 ## Unreleased
+### Added
+- Env / continueOnFailure option.
 ### Changed
 - Maintenance changes.
 

diff --git a/addOns/automation/src/main/java/org/zaproxy/addon/automation/AutomationEnvironment.java b/addOns/automation/src/main/java/org/zaproxy/addon/automation/AutomationEnvironment.java
@@ -414,9 +414,14 @@ public boolean isFailOnWarning() {
         return this.getData().getParameters().getFailOnWarning();
     }
 
+    public boolean isContinueOnFailure() {
+        return this.getData().getParameters().getContinueOnFailure();
+    }
+
     public boolean isTimeToQuit() {
-        return (isFailOnError() && progress.hasErrors())
-                || (isFailOnWarning() && progress.hasWarnings());
+        return !isContinueOnFailure()
+                && ((isFailOnError() && progress.hasErrors())
+                        || (isFailOnWarning() && progress.hasWarnings()));
     }
 
     public void showDialog() {
@@ -487,14 +492,20 @@ public static class Parameters extends AutomationData {
         private boolean failOnError = true;
         private boolean failOnWarning;
         private boolean progressToStdout = true;
+        private boolean continueOnFailure = false;
 
         public Parameters() {}
 
-        public Parameters(boolean failOnError, boolean failOnWarning, boolean progressToStdout) {
+        public Parameters(
+                boolean failOnError,
+                boolean failOnWarning,
+                boolean progressToStdout,
+                boolean continueOnFailure) {
             super();
             this.failOnError = failOnError;
             this.failOnWarning = failOnWarning;
             this.progressToStdout = progressToStdout;
+            this.continueOnFailure = continueOnFailure;
         }
 
         public boolean getFailOnError() {
@@ -505,6 +516,10 @@ public boolean getFailOnWarning() {
             return failOnWarning;
         }
 
+        public boolean getContinueOnFailure() {
+            return continueOnFailure;
+        }
+
         public boolean getProgressToStdout() {
             return progressToStdout;
         }
@@ -517,6 +532,10 @@ public void setFailOnWarning(boolean failOnWarning) {
             this.failOnWarning = failOnWarning;
         }
 
+        public void setContinueOnFailure(boolean continueOnFailure) {
+            this.continueOnFailure = continueOnFailure;
+        }
+
         public void setProgressToStdout(boolean progressToStdout) {
             this.progressToStdout = progressToStdout;
         }

diff --git a/addOns/automation/src/main/java/org/zaproxy/addon/automation/gui/EnvironmentDialog.java b/addOns/automation/src/main/java/org/zaproxy/addon/automation/gui/EnvironmentDialog.java
@@ -51,6 +51,7 @@ public class EnvironmentDialog extends StandardFieldsDialog {
     private static final String TITLE = "automation.dialog.env.title";
     private static final String FAIL_ON_ERROR_PARAM = "automation.dialog.env.failonerror";
     private static final String FAIL_ON_WARNING_PARAM = "automation.dialog.env.failonwarning";
+    private static final String CONTINUE_ON_FAILURE_PARAM = "automation.dialog.env.continueonfail";
     private static final String PROGRESS_TO_STDOUT_PARAM = "automation.dialog.env.progresstostdout";
 
     private static final String PROXY_HOSTNAME = "automation.dialog.env.proxyhost";
@@ -94,6 +95,8 @@ public EnvironmentDialog(AutomationEnvironment env) {
                 1, FAIL_ON_ERROR_PARAM, env.getData().getParameters().getFailOnError());
         this.addCheckBoxField(
                 1, FAIL_ON_WARNING_PARAM, env.getData().getParameters().getFailOnWarning());
+        this.addCheckBoxField(
+                1, CONTINUE_ON_FAILURE_PARAM, env.getData().getParameters().getContinueOnFailure());
         this.addCheckBoxField(
                 1, PROGRESS_TO_STDOUT_PARAM, env.getData().getParameters().getProgressToStdout());
         this.addPadding(1);
@@ -120,6 +123,10 @@ public void save() {
                 .getData()
                 .getParameters()
                 .setFailOnWarning(this.getBoolValue(FAIL_ON_WARNING_PARAM));
+        this.env
+                .getData()
+                .getParameters()
+                .setContinueOnFailure(this.getBoolValue(CONTINUE_ON_FAILURE_PARAM));
         this.env
                 .getData()
                 .getParameters()

diff --git a/...n/src/main/javahelp/org/zaproxy/addon/automation/resources/help/contents/environment.html b/...n/src/main/javahelp/org/zaproxy/addon/automation/resources/help/contents/environment.html
@@ -67,6 +67,7 @@ <H1>Automation Framework - Environment</H1>
   parameters:
     failOnError: true                  # If set exit on an error         
     failOnWarning: false               # If set exit on a warning
+    continueOnFailure: false           # Continue running all jobs, even if one fails
     progressToStdout: true             # If set will write job progress to stdout
   proxy:                               # Optional upstream proxy settings
     hostname:                          # String, the proxy host

diff --git a/.../automation/src/main/resources/org/zaproxy/addon/automation/resources/Messages.properties b/.../automation/src/main/resources/org/zaproxy/addon/automation/resources/Messages.properties
@@ -112,11 +112,12 @@ automation.dialog.delay.fileName = File Name:
 automation.dialog.delay.summary = Time: {0} file name: {1}
 automation.dialog.delay.time = Time:
 automation.dialog.delay.title = Delay Job
+automation.dialog.env.continueonfail = Continue On Failure:
 
 automation.dialog.env.error.nocontext = You must define at least one Context
 automation.dialog.env.failonerror = Fail On Error:
 automation.dialog.env.failonwarning = Fail On Warning:
-automation.dialog.env.progresstostdout = Progresss To Stdout:
+automation.dialog.env.progresstostdout = Progress To Stdout:
 automation.dialog.env.proxyhost = Hostname:
 automation.dialog.env.proxyport = Port:
 automation.dialog.env.proxypwd = Password: 

diff --git a/addOns/automation/src/main/resources/org/zaproxy/addon/automation/resources/env-max.yaml b/addOns/automation/src/main/resources/org/zaproxy/addon/automation/resources/env-max.yaml
@@ -44,6 +44,7 @@ env:                                   # The environment, mandatory
   parameters:
     failOnError: true                  # If set exit on an error         
     failOnWarning: false               # If set exit on a warning
+    continueOnFailure: false           # Continue running all jobs, even if one fails
     progressToStdout: true             # If set will write job progress to stdout
   proxy:                               # Optional upstream proxy settings
     hostname:                          # String, the proxy host

diff --git a/addOns/automation/src/main/resources/org/zaproxy/addon/automation/resources/env-min.yaml b/addOns/automation/src/main/resources/org/zaproxy/addon/automation/resources/env-min.yaml
@@ -9,6 +9,7 @@ env:                                   # The environment, mandatory
   parameters:
     failOnError: true                  # If set exit on an error         
     failOnWarning: false               # If set exit on a warning
+    continueOnFailure: false           # Continue running all jobs, even if one fails
     progressToStdout: true             # If set will write job progress to stdout
 
 jobs:
diff --git a/...ns/automation/src/test/java/org/zaproxy/addon/automation/ExtentionAutomationUnitTest.java b/...ns/automation/src/test/java/org/zaproxy/addon/automation/ExtentionAutomationUnitTest.java
@@ -489,6 +489,158 @@ public Order getOrder() {
         assertThat(stats.getStat(ExtensionAutomation.TOTAL_JOBS_RUN_STATS), is(nullValue()));
     }
 
+    @Test
+    void shouldContinuePlanOnError() {
+        // Given
+        ExtensionAutomation extAuto = new ExtensionAutomation();
+        String job1Name = "job1";
+        String job2Name = "job2";
+        String job3Name = "job3";
+
+        AutomationJobImpl job1 =
+                new AutomationJobImpl(true) {
+                    @Override
+                    public String getType() {
+                        return job1Name;
+                    }
+
+                    @Override
+                    public Order getOrder() {
+                        return Order.REPORT;
+                    }
+
+                    @Override
+                    public void runJob(AutomationEnvironment env, AutomationProgress progress) {
+                        super.runJob(env, progress);
+                        progress.error("Test Error");
+                    }
+                };
+        AutomationJobImpl job2 =
+                new AutomationJobImpl(true) {
+                    @Override
+                    public String getType() {
+                        return job2Name;
+                    }
+
+                    @Override
+                    public Order getOrder() {
+                        return Order.REPORT;
+                    }
+                };
+        AutomationJobImpl job3 =
+                new AutomationJobImpl(true) {
+                    @Override
+                    public String getType() {
+                        return job3Name;
+                    }
+
+                    @Override
+                    public Order getOrder() {
+                        return Order.REPORT;
+                    }
+                };
+        Path filePath = getResourcePath("resources/testplan-continueonerror.yaml");
+        InMemoryStats stats = new InMemoryStats();
+        Stats.addListener(stats);
+
+        // When
+        extAuto.registerAutomationJob(job1);
+        extAuto.registerAutomationJob(job2);
+        extAuto.registerAutomationJob(job3);
+        AutomationProgress progress =
+                extAuto.runAutomationFile(filePath.toAbsolutePath().toString());
+
+        // Then
+        assertThat(progress.hasWarnings(), is(equalTo(false)));
+        assertThat(progress.hasErrors(), is(equalTo(true)));
+        assertThat(progress.getErrors().size(), is(equalTo(1)));
+        assertThat(progress.getErrors().get(0), is(equalTo("Test Error")));
+        assertThat(job1.wasRun(), is(equalTo(true)));
+        assertThat(job2.wasRun(), is(equalTo(true)));
+        assertThat(job3.wasRun(), is(equalTo(true)));
+
+        assertThat(stats.getStat(ExtensionAutomation.WARNING_COUNT_STATS), is(equalTo(0L)));
+        assertThat(stats.getStat(ExtensionAutomation.ERROR_COUNT_STATS), is(equalTo(1L)));
+        assertThat(stats.getStat(ExtensionAutomation.PLANS_RUN_STATS), is(equalTo(1L)));
+        assertThat(stats.getStat(ExtensionAutomation.TOTAL_JOBS_RUN_STATS), is(3L));
+    }
+
+    @Test
+    void shouldContinuePlanOnWarning() {
+        // Given
+        ExtensionAutomation extAuto = new ExtensionAutomation();
+        String job1Name = "job1";
+        String job2Name = "job2";
+        String job3Name = "job3";
+
+        AutomationJobImpl job1 =
+                new AutomationJobImpl(true) {
+                    @Override
+                    public String getType() {
+                        return job1Name;
+                    }
+
+                    @Override
+                    public Order getOrder() {
+                        return Order.REPORT;
+                    }
+                };
+        AutomationJobImpl job2 =
+                new AutomationJobImpl(true) {
+                    @Override
+                    public String getType() {
+                        return job2Name;
+                    }
+
+                    @Override
+                    public Order getOrder() {
+                        return Order.REPORT;
+                    }
+
+                    @Override
+                    public void runJob(AutomationEnvironment env, AutomationProgress progress) {
+                        super.runJob(env, progress);
+                        progress.warn("Test Warning");
+                    }
+                };
+        AutomationJobImpl job3 =
+                new AutomationJobImpl(true) {
+                    @Override
+                    public String getType() {
+                        return job3Name;
+                    }
+
+                    @Override
+                    public Order getOrder() {
+                        return Order.REPORT;
+                    }
+                };
+        Path filePath = getResourcePath("resources/testplan-continueonwarning.yaml");
+        InMemoryStats stats = new InMemoryStats();
+        Stats.addListener(stats);
+
+        // When
+        extAuto.registerAutomationJob(job1);
+        extAuto.registerAutomationJob(job2);
+        extAuto.registerAutomationJob(job3);
+        AutomationProgress progress =
+                extAuto.runAutomationFile(filePath.toAbsolutePath().toString());
+
+        // Then
+        assertThat(progress.hasWarnings(), is(equalTo(true)));
+        assertThat(progress.hasErrors(), is(equalTo(false)));
+        assertThat(progress.getWarnings().size(), is(equalTo(1)));
+        assertThat(progress.getWarnings().get(0), is(equalTo("Test Warning")));
+        assertThat(job1.wasRun(), is(equalTo(true)));
+        assertThat(job2.wasRun(), is(equalTo(true)));
+        assertThat(job3.wasRun(), is(equalTo(true)));
+
+        assertThat(stats.getStat(ExtensionAutomation.WARNING_COUNT_STATS), is(equalTo(1L)));
+        assertThat(stats.getStat(ExtensionAutomation.ERROR_COUNT_STATS), is(equalTo(0L)));
+        assertThat(stats.getStat(ExtensionAutomation.PLANS_RUN_STATS), is(equalTo(1L)));
+        assertThat(stats.getStat(ExtensionAutomation.TOTAL_JOBS_RUN_STATS), is(3L));
+    }
+
     @Test
     void shouldRunPlanWithJobsWithSameType() {
         // Given
@@ -882,9 +1034,14 @@ private static class AutomationJobImpl extends AutomationJob {
         private boolean testsAdded = false;
         private String testsLoggedString;
         private boolean testsLogError = false;
+        private boolean returnOriginal = false;
 
         public AutomationJobImpl() {}
 
+        public AutomationJobImpl(boolean returnOriginal) {
+            this.returnOriginal = returnOriginal;
+        }
+
         public AutomationJobImpl(String type) {
             this.type = type;
         }
@@ -964,6 +1121,9 @@ public String getOptional() {
 
         @Override
         public AutomationJob newJob() {
+            if (returnOriginal) {
+                return this;
+            }
             AutomationJobImpl job = new AutomationJobImpl();
             job.paramMethodObject = this.paramMethodObject;
             job.type = this.getType();

diff --git a/...automation/src/test/resources/org/zaproxy/addon/automation/resources/template-config.yaml b/...automation/src/test/resources/org/zaproxy/addon/automation/resources/template-config.yaml
@@ -9,6 +9,7 @@ env:                                   # The environment, mandatory
   parameters:
     failOnError: true                  # If set exit on an error         
     failOnWarning: false               # If set exit on a warning
+    continueOnFailure: false           # Continue running all jobs, even if one fails
     progressToStdout: true             # If set will write job progress to stdout
 
 jobs:

diff --git a/...ns/automation/src/test/resources/org/zaproxy/addon/automation/resources/template-max.yaml b/...ns/automation/src/test/resources/org/zaproxy/addon/automation/resources/template-max.yaml
@@ -44,6 +44,7 @@ env:                                   # The environment, mandatory
   parameters:
     failOnError: true                  # If set exit on an error         
     failOnWarning: false               # If set exit on a warning
+    continueOnFailure: false           # Continue running all jobs, even if one fails
     progressToStdout: true             # If set will write job progress to stdout
   proxy:                               # Optional upstream proxy settings
     hostname:                          # String, the proxy host

diff --git a/...ns/automation/src/test/resources/org/zaproxy/addon/automation/resources/template-min.yaml b/...ns/automation/src/test/resources/org/zaproxy/addon/automation/resources/template-min.yaml
@@ -9,6 +9,7 @@ env:                                   # The environment, mandatory
   parameters:
     failOnError: true                  # If set exit on an error         
     failOnWarning: false               # If set exit on a warning
+    continueOnFailure: false           # Continue running all jobs, even if one fails
     progressToStdout: true             # If set will write job progress to stdout
 
 jobs:

diff --git a/...n/src/test/resources/org/zaproxy/addon/automation/resources/testplan-continueonerror.yaml b/...n/src/test/resources/org/zaproxy/addon/automation/resources/testplan-continueonerror.yaml
@@ -0,0 +1,20 @@
+env:
+  contexts:
+    - name: example
+      urls: 
+      - https://www.example.com/
+  parameters:
+    failOnError: true                  
+    failOnWarning: false               
+    progressToStdout: false         
+    continueOnFailure: true   
+
+jobs:
+  - type: job1
+    parameters:
+
+  - type: job2
+    parameters:
+
+  - type: job3
+    parameters:
diff --git a/...src/test/resources/org/zaproxy/addon/automation/resources/testplan-continueonwarning.yaml b/...src/test/resources/org/zaproxy/addon/automation/resources/testplan-continueonwarning.yaml
@@ -0,0 +1,20 @@
+env:
+  contexts:
+    - name: example
+      urls: 
+      - https://www.example.com/
+  parameters:
+    failOnError: true                  
+    failOnWarning: true               
+    progressToStdout: false            
+    continueOnFailure: true   
+
+jobs:
+  - type: job1
+    parameters:
+
+  - type: job2
+    parameters:
+
+  - type: job3
+    parameters: