Enable 'CatchSystemErrors' only during debug sessions

BoostTestAdapter/Boost/Runner/BoostTestRunnerCommandLineArgs.cs

@@ -220,7 +220,7 @@ public BoostTestRunnerCommandLineArgs()
             this.ShowProgress = false;
             this.BuildInfo = false;
             this.AutoStartDebug = "no";
-            this.CatchSystemErrors = true;
+            this.CatchSystemErrors = null;
             this.ColorOutput = false;
             this.ResultCode = true;
             this.Random = 0;
@@ -358,7 +358,13 @@ public string ReportFile
         /// <summary>
         /// Determines whether system errors should be caught.
         /// </summary>
-        public bool CatchSystemErrors { get; set; }
+        /// <remarks>
+        /// Since the default value of '--catch_system_errors' is dependent on Boost.Test's 
+        /// '#define BOOST_TEST_DEFAULTS_TO_CORE_DUMP', this value has been set to a optional type
+        /// 
+        /// Reference: http://www.boost.org/doc/libs/1_60_0/libs/test/doc/html/boost_test/utf_reference/rt_param_reference/catch_system.html
+        /// </remarks>
+        public bool? CatchSystemErrors { get; set; }
 
         /// <summary>
         /// States whether standard output text is colour coded.
@@ -485,10 +491,10 @@ public override string ToString()
                 AddArgument(AutoStartDebugArg, this.AutoStartDebug, args);
             }
 
-            // --catch_system_errors=no
-            if (!this.CatchSystemErrors)
+            // --catch_system_errors=yes
+            if (this.CatchSystemErrors.HasValue)
             {
-                AddArgument(CatchSystemErrorsArg, No, args);
+                AddArgument(CatchSystemErrorsArg, (this.CatchSystemErrors.Value ? Yes : No), args);
             }
 
             // --color_output=yes

BoostTestAdapter/BoostTestExecutor.cs

@@ -187,7 +187,7 @@ public void RunTests(IEnumerable<string> sources,
             Logger.Debug("IRunContext.RunSettings.SettingsXml: {0}", runContext.RunSettings.SettingsXml);
 
             BoostTestAdapterSettings settings = BoostTestAdapterSettingsProvider.GetSettings(runContext);
-
+            
             foreach (string source in sources)
             {
                 if (_cancelled)
@@ -219,7 +219,7 @@ public void RunTests(IEnumerable<string> sources,
                         // NOTE For code-coverage speed is given preference over adapter responsiveness.
                         TestBatch.Strategy strategy = ((runContext.IsDataCollectionEnabled) ? TestBatch.Strategy.Source : settings.TestBatchStrategy);
 
-                        ITestBatchingStrategy batchStrategy = GetBatchStrategy(strategy, settings);
+                        ITestBatchingStrategy batchStrategy = GetBatchStrategy(strategy, settings, runContext);
                         if (batchStrategy == null)
                         {
                             Logger.Error(Resources.BatchStrategyNotFoundFor, source);
@@ -274,7 +274,7 @@ public void RunTests(IEnumerable<VSTestCase> tests, IRunContext runContext, IFra
                 strategy = Strategy.TestSuite;
             }
 
-            ITestBatchingStrategy batchStrategy = GetBatchStrategy(strategy, settings);
+            ITestBatchingStrategy batchStrategy = GetBatchStrategy(strategy, settings, runContext);
             if (batchStrategy == null)
             {
                 Logger.Error(Resources.BatchStrategyNotFound);
@@ -306,13 +306,32 @@ public void Cancel()
         /// </summary>
         /// <param name="strategy">The base strategy to provide</param>
         /// <param name="settings">Adapter settings currently in use</param>
+        /// <param name="runContext">The RunContext for this TestCase. Determines whether the test should be debugged or not.</param>
         /// <returns>An ITestBatchingStrategy instance or null if one cannot be provided</returns>
-        private ITestBatchingStrategy GetBatchStrategy(TestBatch.Strategy strategy, BoostTestAdapterSettings settings)
+        private ITestBatchingStrategy GetBatchStrategy(TestBatch.Strategy strategy, BoostTestAdapterSettings settings, IRunContext runContext)
         {
-            TestBatch.CommandLineArgsBuilder argsBuilder = GetDefaultArguments;
+            TestBatch.CommandLineArgsBuilder argsBuilder = (string _source, BoostTestAdapterSettings _settings) =>
+            {
+                return GetDefaultArguments(_source, _settings, runContext.IsBeingDebugged);
+            };
+
             if (strategy != Strategy.TestCase)
             {
-                argsBuilder = GetBatchedTestRunsArguments;
+                // Disable stdout, stderr and memory leak detection since it is difficult
+                // to distinguish from which test does portions of the output map to
+                argsBuilder = (string _source, BoostTestAdapterSettings _settings) =>
+                {
+                    var args = GetDefaultArguments(_source, _settings, runContext.IsBeingDebugged);
+
+                    // Disable standard error/standard output capture
+                    args.StandardOutFile = null;
+                    args.StandardErrorFile = null;
+
+                    // Disable memory leak detection
+                    args.DetectMemoryLeaks = 0;
+
+                    return args;
+                };
             }
 
             switch (strategy)
@@ -449,11 +468,12 @@ private void GetDebugConfigurationProperties(string source, BoostTestAdapterSett
         /// </summary>
         /// <param name="source">The TestCases source</param>
         /// <param name="settings">The Boost Test adapter settings currently in use</param>
+        /// <param name="debugMode">Determines whether the test should be debugged or not.</param>
         /// <returns>A BoostTestRunnerCommandLineArgs structure for the provided source</returns>
-        private BoostTestRunnerCommandLineArgs GetDefaultArguments(string source, BoostTestAdapterSettings settings)
+        private BoostTestRunnerCommandLineArgs GetDefaultArguments(string source, BoostTestAdapterSettings settings, bool debugMode)
         {
             BoostTestRunnerCommandLineArgs args = settings.CommandLineArgs.Clone();
-
+            
             GetDebugConfigurationProperties(source, settings, args);
 
             // Specify log and report file information
@@ -468,26 +488,10 @@ private BoostTestRunnerCommandLineArgs GetDefaultArguments(string source, BoostT
             args.StandardOutFile = ((settings.EnableStdOutRedirection) ? TestPathGenerator.Generate(source, FileExtensions.StdOutFile) : null);
             args.StandardErrorFile = ((settings.EnableStdErrRedirection) ? TestPathGenerator.Generate(source, FileExtensions.StdErrFile) : null);
 
-            return args;
-        }
-
-        /// <summary>
-        /// Factory function which returns an appropriate BoostTestRunnerCommandLineArgs structure for batched test runs
-        /// </summary>
-        /// <param name="source">The TestCases source</param>
-        /// <param name="settings">The Boost Test adapter settings currently in use</param>
-        /// <returns>A BoostTestRunnerCommandLineArgs structure for the provided source</returns>
-        private BoostTestRunnerCommandLineArgs GetBatchedTestRunsArguments(string source, BoostTestAdapterSettings settings)
-        {
-            BoostTestRunnerCommandLineArgs args = GetDefaultArguments(source, settings);
-
-            // Disable standard error/standard output capture
-            args.StandardOutFile = null;
-            args.StandardErrorFile = null;
-
-            // Disable memory leak detection
-            args.DetectMemoryLeaks = 0;
-
+            // Set '--catch_system_errors' to 'yes' if the test is not being debugged
+            // or if this value was not overridden via configuration before-hand
+            args.CatchSystemErrors = args.CatchSystemErrors.GetValueOrDefault(false) || !debugMode;
+
             return args;
         }
 

BoostTestAdapter/Settings/BoostTestAdapterSettings.cs

@@ -40,7 +40,7 @@ public BoostTestAdapterSettings() :
 
             this.LogLevel = LogLevel.TestSuite;
 
-            this.CatchSystemErrors = true;
+            this.CatchSystemErrors = null;
 
             this.DetectFloatingPointExceptions = false;
 
@@ -107,8 +107,8 @@ public int ExecutionTimeoutMilliseconds
         /// <summary>
         /// Set to 'true'|'1' to enable Boost Test's catch_system_errors.
         /// </summary>
-        [DefaultValue(true)]
-        public bool CatchSystemErrors
+        [DefaultValue(null)]
+        public bool? CatchSystemErrors
         {
             get
             {

BoostTestAdapterNunit/BoostTestExecutorTest.cs

@@ -543,6 +543,11 @@ public void RunTestsFromSource()
             );
 
             AssertDefaultTestResultProperties(this.FrameworkHandle.Results);
+
+            MockBoostTestRunner runner = this.RunnerFactory.LastTestRunner as MockBoostTestRunner;
+
+            Assert.That(runner, Is.Not.Null);
+            Assert.That(runner.ExecutionArgs.All(args => (!args.Arguments.CatchSystemErrors.HasValue || args.Arguments.CatchSystemErrors.Value)), Is.True);
         }
 
         /// <summary>
@@ -579,6 +584,10 @@ public void RunTestSelection()
             );
 
             AssertDefaultTestResultProperties(this.FrameworkHandle.Results);
+            MockBoostTestRunner runner = this.RunnerFactory.LastTestRunner as MockBoostTestRunner;
+
+            Assert.That(runner, Is.Not.Null);
+            Assert.That(runner.ExecutionArgs.All(args => (!args.Arguments.CatchSystemErrors.HasValue || args.Arguments.CatchSystemErrors.Value)), Is.True);
         }
 
         /// <summary>
@@ -602,6 +611,7 @@ public void DebugTestSelection()
 
             Assert.That(runner, Is.Not.Null);
             Assert.That(runner.ExecutionArgs.First().Context, Is.TypeOf<DebugFrameworkExecutionContext>());
+            Assert.That(runner.ExecutionArgs.All(args => (args.Arguments.CatchSystemErrors.HasValue && !args.Arguments.CatchSystemErrors.Value)), Is.True);
 
             AssertDefaultTestResultProperties(this.FrameworkHandle.Results);
         }

BoostTestAdapterNunit/BoostTestRunnerCommandLineArgsTest.cs

@@ -5,7 +5,6 @@
 
 using BoostTestAdapter.Boost.Runner;
 using NUnit.Framework;
-using System.IO;
 
 namespace BoostTestAdapterNunit
 {
@@ -45,7 +44,7 @@ private static BoostTestRunnerCommandLineArgs GenerateCommandLineArgs()
 
             args.DetectMemoryLeaks = 0;
 
-            args.CatchSystemErrors = false;
+            args.CatchSystemErrors = true;
             args.DetectFPExceptions = true;
 
             args.StandardOutFile = GenerateFullyQualifiedPath("stdout.log");
@@ -82,7 +81,7 @@ public void SampleCommandLineArgs()
         {
             BoostTestRunnerCommandLineArgs args = GenerateCommandLineArgs();
             // serge: boost 1.60 requires uppercase input
-            Assert.That(args.ToString(), Is.EqualTo("\"--run_test=test,suite/*\" \"--catch_system_errors=no\" \"--log_format=XML\" \"--log_level=test_suite\" \"--log_sink="
+            Assert.That(args.ToString(), Is.EqualTo("\"--run_test=test,suite/*\" \"--catch_system_errors=yes\" \"--log_format=XML\" \"--log_level=test_suite\" \"--log_sink="
                 + GenerateFullyQualifiedPath("log.xml") + "\" \"--report_format=XML\" \"--report_level=detailed\" \"--report_sink="
                 + GenerateFullyQualifiedPath("report.xml") + "\" \"--detect_memory_leak=0\" \"--detect_fp_exceptions=yes\" > \"" 
                 + GenerateFullyQualifiedPath("stdout.log") + "\" 2> \""

BoostTestAdapterNunit/BoostTestSettingsTest.cs

@@ -62,7 +62,7 @@ private void AssertDefaultSettings(BoostTestAdapterSettings settings)
             Assert.That(settings.LogLevel, Is.EqualTo(LogLevel.TestSuite));
             Assert.That(settings.ExternalTestRunner, Is.Null);
             Assert.That(settings.DetectFloatingPointExceptions, Is.False);
-            Assert.That(settings.CatchSystemErrors, Is.True);
+            Assert.That(settings.CatchSystemErrors, Is.Null);
             Assert.That(settings.TestBatchStrategy, Is.EqualTo(Strategy.TestCase));
             Assert.That(settings.ForceListContent, Is.False);
             Assert.That(settings.WorkingDirectory, Is.Null);
@@ -171,7 +171,9 @@ public void ParseComplexSettings()
         ///     - Assert that: the Boost 1.62 workaround option can be parsed
         /// </summary>
         [TestCase("<?xml version=\"1.0\" encoding=\"utf-8\"?><RunSettings><BoostTest><UseBoost162Workaround>true</UseBoost162Workaround></BoostTest></RunSettings>", Result = true)]
+        [TestCase("<?xml version=\"1.0\" encoding=\"utf-8\"?><RunSettings><BoostTest><UseBoost162Workaround>1</UseBoost162Workaround></BoostTest></RunSettings>", Result = true)]
         [TestCase("<?xml version=\"1.0\" encoding=\"utf-8\"?><RunSettings><BoostTest><UseBoost162Workaround>false</UseBoost162Workaround></BoostTest></RunSettings>", Result = false)]
+        [TestCase("<?xml version=\"1.0\" encoding=\"utf-8\"?><RunSettings><BoostTest><UseBoost162Workaround>0</UseBoost162Workaround></BoostTest></RunSettings>", Result = false)]
         [TestCase("<?xml version=\"1.0\" encoding=\"utf-8\"?><RunSettings><BoostTest /></RunSettings>", Result = false)]
         public bool ParseWorkaroundOption(string settingsXml)
         {
@@ -195,6 +197,23 @@ public int ParsePostTestDelayOption(string settingsXml)
             return settings.PostTestDelay;
         }
 
+        /// <summary>
+        /// The 'CatchSystemErrors' option can be properly parsed
+        /// 
+        /// Test aims:
+        ///     - Assert that: the 'CatchSystemErrors' option can be properly parsed
+        /// </summary>
+        [TestCase("<?xml version=\"1.0\" encoding=\"utf-8\"?><RunSettings><BoostTest><CatchSystemErrors>true</CatchSystemErrors></BoostTest></RunSettings>", Result = true)]
+        [TestCase("<?xml version=\"1.0\" encoding=\"utf-8\"?><RunSettings><BoostTest><CatchSystemErrors>1</CatchSystemErrors></BoostTest></RunSettings>", Result = true)]
+        [TestCase("<?xml version=\"1.0\" encoding=\"utf-8\"?><RunSettings><BoostTest><CatchSystemErrors>false</CatchSystemErrors></BoostTest></RunSettings>", Result = false)]
+        [TestCase("<?xml version=\"1.0\" encoding=\"utf-8\"?><RunSettings><BoostTest><CatchSystemErrors>0</CatchSystemErrors></BoostTest></RunSettings>", Result = false)]
+        [TestCase("<?xml version=\"1.0\" encoding=\"utf-8\"?><RunSettings><BoostTest /></RunSettings>", Result = null)]
+        public bool? ParseCatchSystemErrorsOption(string settingsXml)
+        {
+            BoostTestAdapterSettings settings = ParseXml(settingsXml);
+            return settings.CatchSystemErrors;
+        }
+
         #endregion Tests
     }
 }