improve test suite (#48)

- test mono v5.16.0 ( visual studio channel )
- add `--known-failure` and `--flaky` arguments to test runner, to filter these kind of tests

.travis.yml

@@ -1,6 +1,5 @@
 language: csharp
 
-mono: 5.18.0
 dotnet: 2.1.401
 
 install:
@@ -9,12 +8,37 @@ install:
 
 script:
   - dotnet pack -v n
-  - dotnet test -v n
+  - dotnet test -v n -p:UnstableTests=$UnstableTests
 
 matrix:
   include:
     - os: linux
       dist: trusty
       sudo: required
+      mono: 5.16.0
+      env: UnstableTests=false
     - os: osx
       osx_image: xcode9
+      mono: 5.16.0
+      env: UnstableTests=false
+    - os: linux
+      dist: trusty
+      sudo: required
+      mono: 5.16.0
+      env: UnstableTests=true
+    - os: osx
+      osx_image: xcode9
+      mono: 5.16.0
+      env: UnstableTests=true
+    - os: linux
+      dist: trusty
+      sudo: required
+      mono: 5.18.0
+      env: UnstableTests=false
+    - os: osx
+      osx_image: xcode9
+      mono: 5.18.0
+      env: UnstableTests=false
+  allow_failures:
+    # unstable tests may fail
+    - env: UnstableTests=true

build.proj

@@ -2,27 +2,39 @@
 
   <Import Project="Directory.Build.props" />
 
+  <PropertyGroup>
+    <Configuration>Release</Configuration>
+    <UnstableTests>false</UnstableTests>
+  </PropertyGroup>
+
   <Target Name="Restore">
     <Exec Command="$(MonoOrEmpty)paket.exe restore" WorkingDirectory=".paket" ConsoleToMSBuild="true" />
-    <Exec Command='dotnet restore src/dotnet-proj.sln /p:Version=$(Version)' />
+    <Exec Command='dotnet restore src/dotnet-proj.sln -p:Version=$(Version)' />
   </Target>
 
   <Target Name="Build">
-    <Exec Command='dotnet build src/dotnet-proj.sln -c Release /p:Version=$(Version)' />
+    <Exec Command='dotnet build src/dotnet-proj.sln -c $(Configuration) -p:Version=$(Version)' />
   </Target>
 
   <Target Name="Pack">
     <RemoveDir Directories="$(NupkgsDir)" />
-    <Exec Command='dotnet pack src/dotnet-proj -c Release -o "$(NupkgsDir)" /p:Version=$(Version)' />
-    <Exec Command='dotnet pack src/Dotnet.ProjInfo -c Release -o "$(NupkgsDir)" /p:Version=$(Version)' />
-    <Exec Command='dotnet pack src/Dotnet.ProjInfo.Workspace -c Release -o "$(NupkgsDir)" /p:Version=$(Version)' />
-    <Exec Command='dotnet pack src/Dotnet.ProjInfo.Workspace.FCS -c Release -o "$(NupkgsDir)" /p:Version=$(Version)' />
+    <Exec Command='dotnet pack src/dotnet-proj -c $(Configuration) -o "$(NupkgsDir)" -p:Version=$(Version)' />
+    <Exec Command='dotnet pack src/Dotnet.ProjInfo -c $(Configuration) -o "$(NupkgsDir)" -p:Version=$(Version)' />
+    <Exec Command='dotnet pack src/Dotnet.ProjInfo.Workspace -c $(Configuration) -o "$(NupkgsDir)" -p:Version=$(Version)' />
+    <Exec Command='dotnet pack src/Dotnet.ProjInfo.Workspace.FCS -c $(Configuration) -o "$(NupkgsDir)" -p:Version=$(Version)' />
   </Target>
 
   <Target Name="Test">
-    <Exec Command='dotnet run -c Release -- "$(Version)" --fail-on-focused-tests --summary' WorkingDirectory="$(RepoRootDir)/test/dotnet-proj.Tests" IgnoreStandardErrorWarningFormat="true" />
-    <Exec Command='dotnet run -c Release -- --fail-on-focused-tests --summary' WorkingDirectory="$(RepoRootDir)/test/Dotnet.ProjInfo.Workspace.Tests" IgnoreStandardErrorWarningFormat="true" />
-    <Exec Command='dotnet run -c Release -- --fail-on-focused-tests --summary' WorkingDirectory="$(RepoRootDir)/test/Dotnet.ProjInfo.Workspace.FCS.Tests" IgnoreStandardErrorWarningFormat="true" />
+    <Exec Command='dotnet run -c $(Configuration) -- "$(Version)" --fail-on-focused-tests --summary' WorkingDirectory="$(RepoRootDir)/test/dotnet-proj.Tests" IgnoreStandardErrorWarningFormat="true" Condition=" '$(UnstableTests)' != 'true' " />
+
+    <Exec Command='dotnet run -c $(Configuration) -- --fail-on-focused-tests --summary' WorkingDirectory="$(RepoRootDir)/test/Dotnet.ProjInfo.Workspace.Tests" IgnoreStandardErrorWarningFormat="true" Condition=" '$(UnstableTests)' != 'true' " />
+    <Exec Command='dotnet run -c $(Configuration) -- --fail-on-focused-tests --summary' WorkingDirectory="$(RepoRootDir)/test/Dotnet.ProjInfo.Workspace.FCS.Tests" IgnoreStandardErrorWarningFormat="true" Condition=" '$(UnstableTests)' != 'true' " />
+
+    <Exec Command='dotnet run -c $(Configuration) -- --known-failure --fail-on-focused-tests --summary' WorkingDirectory="$(RepoRootDir)/test/Dotnet.ProjInfo.Workspace.Tests" IgnoreExitCode="true" IgnoreStandardErrorWarningFormat="true" Condition=" '$(UnstableTests)' == 'true' " />
+    <Exec Command='dotnet run -c $(Configuration) -- --known-failure --fail-on-focused-tests --summary' WorkingDirectory="$(RepoRootDir)/test/Dotnet.ProjInfo.Workspace.FCS.Tests" IgnoreExitCode="true" IgnoreStandardErrorWarningFormat="true" Condition=" '$(UnstableTests)' == 'true' " />
+
+    <Exec Command='dotnet run -c $(Configuration) -- --flaky --fail-on-focused-tests --summary' WorkingDirectory="$(RepoRootDir)/test/Dotnet.ProjInfo.Workspace.Tests" IgnoreExitCode="true" IgnoreStandardErrorWarningFormat="true" Condition=" '$(UnstableTests)' == 'true' " />
+    <Exec Command='dotnet run -c $(Configuration) -- --flaky --fail-on-focused-tests --summary' WorkingDirectory="$(RepoRootDir)/test/Dotnet.ProjInfo.Workspace.FCS.Tests" IgnoreExitCode="true" IgnoreStandardErrorWarningFormat="true" Condition=" '$(UnstableTests)' == 'true' " />
   </Target>
 
   <Target Name="VSTest" DependsOnTargets="Test" />

test/Dotnet.ProjInfo.Workspace.FCS.Tests/Dotnet.ProjInfo.Workspace.FCS.Tests.fsproj

@@ -7,6 +7,7 @@
   <ItemGroup>
     <Compile Include="..\dotnet-proj.Tests\FileUtils.fs" />
     <Compile Include="..\dotnet-proj.Tests\TestAssets.fs" />
+    <Compile Include="..\Dotnet.ProjInfo.Workspace.Tests\TestsConfig.fs" />
     <Compile Include="Tests.fs" />
     <Compile Include="Program.fs" />
   </ItemGroup>

test/Dotnet.ProjInfo.Workspace.FCS.Tests/Program.fs

@@ -4,9 +4,29 @@ open Expecto
 open System
 open System.IO
 
+open TestsConfig
+
+type Args =
+    { RunOnlyFlaky: bool
+      RunOnlyKnownFailure: bool }
+
+let rec parseArgs (config, runArgs, args) =
+    match args with
+    | "--flaky" :: xs ->
+        parseArgs ({ config with SkipFlaky = false }, { runArgs with RunOnlyFlaky = true }, xs)
+    | "--known-failure" :: xs ->
+        parseArgs ({ config with SkipKnownFailure = false }, { runArgs with RunOnlyKnownFailure = true }, xs)
+    | xs ->
+        config, runArgs, xs
+
 [<EntryPoint>]
 let main argv =
 
+    let suiteConfig, runArgs, otherArgs =
+        let defaultConfig = { SkipFlaky = true; SkipKnownFailure = true }
+        let defaultRunArgs = { Args.RunOnlyFlaky = false; RunOnlyKnownFailure = false }
+        parseArgs (defaultConfig, defaultRunArgs, List.ofArray argv)
+
     Environment.SetEnvironmentVariable("DOTNET_PROJ_INFO_MSBUILD_BL", "1")
     Environment.SetEnvironmentVariable("MSBuildExtensionsPath", null)
 
@@ -15,4 +35,9 @@ let main argv =
     let writeResults = TestResults.writeNUnitSummary (resultsPath, "Dotnet.ProjInfo.Workspace.FCS.Tests")
     let config = defaultConfig.appendSummaryHandler writeResults
 
-    Tests.runTestsWithArgs config argv (Tests.tests ())
+    let tests =
+        Tests.tests suiteConfig
+        |> Test.filter (fun s -> if runArgs.RunOnlyFlaky then s.Contains "[flaky]" else true)
+        |> Test.filter (fun s -> if runArgs.RunOnlyKnownFailure then s.Contains "[known-failure]" else true)
+
+    Tests.runTestsWithArgs config (otherArgs |> Array.ofList) tests

test/Dotnet.ProjInfo.Workspace.FCS.Tests/Tests.fs

@@ -138,7 +138,9 @@ let expectP2PKeyIsTargetPath po =
   for (tar, fcsPO, poDPW) in allP2P po do
     Expect.equal tar poDPW.ExtraProjectInfo.TargetPath (sprintf "p2p key is TargetPath, fsc projet options was '%A'" fcsPO)
 
-let tests () =
+open TestsConfig
+
+let tests (suiteConfig: TestSuiteConfig) =
 
   let prepareTestsAssets = lazy(
       let logger = Log.create "Tests Assets"
@@ -364,7 +366,7 @@ let tests () =
 
       )
 
-      testCase |> withLog "can load sample3" (fun logger fs ->
+      testCase |> withLog ("can load sample3" |> knownFailure) (fun logger fs ->
         let testDir = inDir fs "load_sample3"
         copyDirFromAssets fs ``sample3 Netsdk projs``.ProjDir testDir
 
@@ -409,7 +411,7 @@ let tests () =
           fcs.ParseAndCheckProject(fcsPo)
           |> Async.RunSynchronously
 
-        if (isOSX ()) then
+        if (isOSX () && suiteConfig.SkipKnownFailure) then
           let errorOnOsx =
             """
 no errors but was: [|commandLineArgs (0,1)-(0,1) parameter error No inputs specified;

test/Dotnet.ProjInfo.Workspace.Tests/Dotnet.ProjInfo.Workspace.Tests.fsproj

@@ -7,6 +7,7 @@
   <ItemGroup>
     <Compile Include="..\dotnet-proj.Tests\FileUtils.fs" />
     <Compile Include="..\dotnet-proj.Tests\TestAssets.fs" />
+    <Compile Include="TestsConfig.fs" />
     <Compile Include="Tests.fs" />
     <Compile Include="Program.fs" />
   </ItemGroup>

test/Dotnet.ProjInfo.Workspace.Tests/Program.fs

@@ -4,9 +4,29 @@ open Expecto
 open System
 open System.IO
 
+open TestsConfig
+
+type Args =
+    { RunOnlyFlaky: bool
+      RunOnlyKnownFailure: bool }
+
+let rec parseArgs (config, runArgs, args) =
+    match args with
+    | "--flaky" :: xs ->
+        parseArgs ({ config with SkipFlaky = false }, { runArgs with RunOnlyFlaky = true }, xs)
+    | "--known-failure" :: xs ->
+        parseArgs ({ config with SkipKnownFailure = false }, { runArgs with RunOnlyKnownFailure = true }, xs)
+    | xs ->
+        config, runArgs, xs
+
 [<EntryPoint>]
 let main argv =
 
+    let suiteConfig, runArgs, otherArgs =
+        let defaultConfig = { SkipFlaky = true; SkipKnownFailure = true }
+        let defaultRunArgs = { Args.RunOnlyFlaky = false; RunOnlyKnownFailure = false }
+        parseArgs (defaultConfig, defaultRunArgs, List.ofArray argv)
+
     Environment.SetEnvironmentVariable("DOTNET_PROJ_INFO_MSBUILD_BL", "1")
     Environment.SetEnvironmentVariable("MSBuildExtensionsPath", null)
 
@@ -15,4 +35,9 @@ let main argv =
     let writeResults = TestResults.writeNUnitSummary (resultsPath, "Dotnet.ProjInfo.Workspace.Tests")
     let config = defaultConfig.appendSummaryHandler writeResults
 
-    Tests.runTestsWithArgs config argv (Tests.tests ())
+    let tests =
+        Tests.tests suiteConfig
+        |> Test.filter (fun s -> if runArgs.RunOnlyFlaky then s.Contains "[flaky]" else true)
+        |> Test.filter (fun s -> if runArgs.RunOnlyKnownFailure then s.Contains "[known-failure]" else true)
+
+    Tests.runTestsWithArgs config (otherArgs |> Array.ofList) tests

test/Dotnet.ProjInfo.Workspace.Tests/Tests.fs

@@ -148,7 +148,9 @@ let isOSX () =
   System.Runtime.InteropServices.RuntimeInformation.IsOSPlatform(
       System.Runtime.InteropServices.OSPlatform.OSX)
 
-let tests () =
+open TestsConfig
+
+let tests (suiteConfig: TestSuiteConfig) =
 
   let prepareTestsAssets = lazy(
       let logger = Log.create "Tests Assets"
@@ -292,7 +294,7 @@ let tests () =
         Expect.equal n1Parsed.SourceFiles expectedSources "check sources"
       )
 
-      testCase |> withLog "can load sample3" (fun logger fs ->
+      testCase |> withLog ("can load sample3" |> knownFailure) (fun logger fs ->
         let testDir = inDir fs "load_sample3"
         copyDirFromAssets fs ``sample3 Netsdk projs``.ProjDir testDir
 
@@ -349,7 +351,7 @@ let tests () =
           parsed
           |> expectFind projPath { ProjectKey.ProjectPath = projPath; TargetFramework = "netcoreapp2.1" } "the F# console"
 
-        if (isOSX ()) then
+        if (isOSX () && suiteConfig.SkipKnownFailure) then
           let errorOnOsx =
             """
          check sources.

test/Dotnet.ProjInfo.Workspace.Tests/TestsConfig.fs

@@ -0,0 +1,8 @@
+module TestsConfig
+
+type TestSuiteConfig =
+    { SkipFlaky: bool
+      SkipKnownFailure: bool }
+
+let flaky name = sprintf "%s [flaky]" name
+let knownFailure name = sprintf "%s [known-failure]" name