From ec90868392ae8f8ef3a12fde2dfc147dae58f2c7 Mon Sep 17 00:00:00 2001 From: Enrico Sada Date: Mon, 11 Mar 2019 23:21:56 +0100 Subject: [PATCH] improve test suite (#48) - test mono v5.16.0 ( visual studio channel ) - add `--known-failure` and `--flaky` arguments to test runner, to filter these kind of tests --- .travis.yml | 28 +++++++++++++++-- build.proj | 30 +++++++++++++------ ...Dotnet.ProjInfo.Workspace.FCS.Tests.fsproj | 1 + .../Program.fs | 27 ++++++++++++++++- .../Tests.fs | 8 +++-- .../Dotnet.ProjInfo.Workspace.Tests.fsproj | 1 + .../Program.fs | 27 ++++++++++++++++- test/Dotnet.ProjInfo.Workspace.Tests/Tests.fs | 8 +++-- .../TestsConfig.fs | 8 +++++ 9 files changed, 119 insertions(+), 19 deletions(-) create mode 100644 test/Dotnet.ProjInfo.Workspace.Tests/TestsConfig.fs diff --git a/.travis.yml b/.travis.yml index 1bbc1eb5..cecf6a48 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,5 @@ language: csharp -mono: 5.18.0 dotnet: 2.1.401 install: @@ -9,12 +8,37 @@ install: script: - dotnet pack -v n - - dotnet test -v n + - dotnet test -v n -p:UnstableTests=$UnstableTests matrix: include: - os: linux dist: trusty sudo: required + mono: 5.16.0 + env: UnstableTests=false - os: osx osx_image: xcode9 + mono: 5.16.0 + env: UnstableTests=false + - os: linux + dist: trusty + sudo: required + mono: 5.16.0 + env: UnstableTests=true + - os: osx + osx_image: xcode9 + mono: 5.16.0 + env: UnstableTests=true + - os: linux + dist: trusty + sudo: required + mono: 5.18.0 + env: UnstableTests=false + - os: osx + osx_image: xcode9 + mono: 5.18.0 + env: UnstableTests=false + allow_failures: + # unstable tests may fail + - env: UnstableTests=true diff --git a/build.proj b/build.proj index fa6c115e..38e55c6d 100644 --- a/build.proj +++ b/build.proj @@ -2,27 +2,39 @@ + + Release + false + + - + - + - - - - + + + + - - - + + + + + + + + + + diff --git a/test/Dotnet.ProjInfo.Workspace.FCS.Tests/Dotnet.ProjInfo.Workspace.FCS.Tests.fsproj b/test/Dotnet.ProjInfo.Workspace.FCS.Tests/Dotnet.ProjInfo.Workspace.FCS.Tests.fsproj index 7a0da761..04a68c0d 100644 --- a/test/Dotnet.ProjInfo.Workspace.FCS.Tests/Dotnet.ProjInfo.Workspace.FCS.Tests.fsproj +++ b/test/Dotnet.ProjInfo.Workspace.FCS.Tests/Dotnet.ProjInfo.Workspace.FCS.Tests.fsproj @@ -7,6 +7,7 @@ + diff --git a/test/Dotnet.ProjInfo.Workspace.FCS.Tests/Program.fs b/test/Dotnet.ProjInfo.Workspace.FCS.Tests/Program.fs index 0770b2b8..ff535fa0 100644 --- a/test/Dotnet.ProjInfo.Workspace.FCS.Tests/Program.fs +++ b/test/Dotnet.ProjInfo.Workspace.FCS.Tests/Program.fs @@ -4,9 +4,29 @@ open Expecto open System open System.IO +open TestsConfig + +type Args = + { RunOnlyFlaky: bool + RunOnlyKnownFailure: bool } + +let rec parseArgs (config, runArgs, args) = + match args with + | "--flaky" :: xs -> + parseArgs ({ config with SkipFlaky = false }, { runArgs with RunOnlyFlaky = true }, xs) + | "--known-failure" :: xs -> + parseArgs ({ config with SkipKnownFailure = false }, { runArgs with RunOnlyKnownFailure = true }, xs) + | xs -> + config, runArgs, xs + [] let main argv = + let suiteConfig, runArgs, otherArgs = + let defaultConfig = { SkipFlaky = true; SkipKnownFailure = true } + let defaultRunArgs = { Args.RunOnlyFlaky = false; RunOnlyKnownFailure = false } + parseArgs (defaultConfig, defaultRunArgs, List.ofArray argv) + Environment.SetEnvironmentVariable("DOTNET_PROJ_INFO_MSBUILD_BL", "1") Environment.SetEnvironmentVariable("MSBuildExtensionsPath", null) @@ -15,4 +35,9 @@ let main argv = let writeResults = TestResults.writeNUnitSummary (resultsPath, "Dotnet.ProjInfo.Workspace.FCS.Tests") let config = defaultConfig.appendSummaryHandler writeResults - Tests.runTestsWithArgs config argv (Tests.tests ()) + let tests = + Tests.tests suiteConfig + |> Test.filter (fun s -> if runArgs.RunOnlyFlaky then s.Contains "[flaky]" else true) + |> Test.filter (fun s -> if runArgs.RunOnlyKnownFailure then s.Contains "[known-failure]" else true) + + Tests.runTestsWithArgs config (otherArgs |> Array.ofList) tests diff --git a/test/Dotnet.ProjInfo.Workspace.FCS.Tests/Tests.fs b/test/Dotnet.ProjInfo.Workspace.FCS.Tests/Tests.fs index 9cafb99a..67b223a4 100644 --- a/test/Dotnet.ProjInfo.Workspace.FCS.Tests/Tests.fs +++ b/test/Dotnet.ProjInfo.Workspace.FCS.Tests/Tests.fs @@ -138,7 +138,9 @@ let expectP2PKeyIsTargetPath po = for (tar, fcsPO, poDPW) in allP2P po do Expect.equal tar poDPW.ExtraProjectInfo.TargetPath (sprintf "p2p key is TargetPath, fsc projet options was '%A'" fcsPO) -let tests () = +open TestsConfig + +let tests (suiteConfig: TestSuiteConfig) = let prepareTestsAssets = lazy( let logger = Log.create "Tests Assets" @@ -364,7 +366,7 @@ let tests () = ) - testCase |> withLog "can load sample3" (fun logger fs -> + testCase |> withLog ("can load sample3" |> knownFailure) (fun logger fs -> let testDir = inDir fs "load_sample3" copyDirFromAssets fs ``sample3 Netsdk projs``.ProjDir testDir @@ -409,7 +411,7 @@ let tests () = fcs.ParseAndCheckProject(fcsPo) |> Async.RunSynchronously - if (isOSX ()) then + if (isOSX () && suiteConfig.SkipKnownFailure) then let errorOnOsx = """ no errors but was: [|commandLineArgs (0,1)-(0,1) parameter error No inputs specified; diff --git a/test/Dotnet.ProjInfo.Workspace.Tests/Dotnet.ProjInfo.Workspace.Tests.fsproj b/test/Dotnet.ProjInfo.Workspace.Tests/Dotnet.ProjInfo.Workspace.Tests.fsproj index ef764c76..71cdac8c 100644 --- a/test/Dotnet.ProjInfo.Workspace.Tests/Dotnet.ProjInfo.Workspace.Tests.fsproj +++ b/test/Dotnet.ProjInfo.Workspace.Tests/Dotnet.ProjInfo.Workspace.Tests.fsproj @@ -7,6 +7,7 @@ + diff --git a/test/Dotnet.ProjInfo.Workspace.Tests/Program.fs b/test/Dotnet.ProjInfo.Workspace.Tests/Program.fs index 210afe9f..ec93917d 100644 --- a/test/Dotnet.ProjInfo.Workspace.Tests/Program.fs +++ b/test/Dotnet.ProjInfo.Workspace.Tests/Program.fs @@ -4,9 +4,29 @@ open Expecto open System open System.IO +open TestsConfig + +type Args = + { RunOnlyFlaky: bool + RunOnlyKnownFailure: bool } + +let rec parseArgs (config, runArgs, args) = + match args with + | "--flaky" :: xs -> + parseArgs ({ config with SkipFlaky = false }, { runArgs with RunOnlyFlaky = true }, xs) + | "--known-failure" :: xs -> + parseArgs ({ config with SkipKnownFailure = false }, { runArgs with RunOnlyKnownFailure = true }, xs) + | xs -> + config, runArgs, xs + [] let main argv = + let suiteConfig, runArgs, otherArgs = + let defaultConfig = { SkipFlaky = true; SkipKnownFailure = true } + let defaultRunArgs = { Args.RunOnlyFlaky = false; RunOnlyKnownFailure = false } + parseArgs (defaultConfig, defaultRunArgs, List.ofArray argv) + Environment.SetEnvironmentVariable("DOTNET_PROJ_INFO_MSBUILD_BL", "1") Environment.SetEnvironmentVariable("MSBuildExtensionsPath", null) @@ -15,4 +35,9 @@ let main argv = let writeResults = TestResults.writeNUnitSummary (resultsPath, "Dotnet.ProjInfo.Workspace.Tests") let config = defaultConfig.appendSummaryHandler writeResults - Tests.runTestsWithArgs config argv (Tests.tests ()) + let tests = + Tests.tests suiteConfig + |> Test.filter (fun s -> if runArgs.RunOnlyFlaky then s.Contains "[flaky]" else true) + |> Test.filter (fun s -> if runArgs.RunOnlyKnownFailure then s.Contains "[known-failure]" else true) + + Tests.runTestsWithArgs config (otherArgs |> Array.ofList) tests diff --git a/test/Dotnet.ProjInfo.Workspace.Tests/Tests.fs b/test/Dotnet.ProjInfo.Workspace.Tests/Tests.fs index 91a46a0b..592160ed 100644 --- a/test/Dotnet.ProjInfo.Workspace.Tests/Tests.fs +++ b/test/Dotnet.ProjInfo.Workspace.Tests/Tests.fs @@ -148,7 +148,9 @@ let isOSX () = System.Runtime.InteropServices.RuntimeInformation.IsOSPlatform( System.Runtime.InteropServices.OSPlatform.OSX) -let tests () = +open TestsConfig + +let tests (suiteConfig: TestSuiteConfig) = let prepareTestsAssets = lazy( let logger = Log.create "Tests Assets" @@ -292,7 +294,7 @@ let tests () = Expect.equal n1Parsed.SourceFiles expectedSources "check sources" ) - testCase |> withLog "can load sample3" (fun logger fs -> + testCase |> withLog ("can load sample3" |> knownFailure) (fun logger fs -> let testDir = inDir fs "load_sample3" copyDirFromAssets fs ``sample3 Netsdk projs``.ProjDir testDir @@ -349,7 +351,7 @@ let tests () = parsed |> expectFind projPath { ProjectKey.ProjectPath = projPath; TargetFramework = "netcoreapp2.1" } "the F# console" - if (isOSX ()) then + if (isOSX () && suiteConfig.SkipKnownFailure) then let errorOnOsx = """ check sources. diff --git a/test/Dotnet.ProjInfo.Workspace.Tests/TestsConfig.fs b/test/Dotnet.ProjInfo.Workspace.Tests/TestsConfig.fs new file mode 100644 index 00000000..d38637fa --- /dev/null +++ b/test/Dotnet.ProjInfo.Workspace.Tests/TestsConfig.fs @@ -0,0 +1,8 @@ +module TestsConfig + +type TestSuiteConfig = + { SkipFlaky: bool + SkipKnownFailure: bool } + +let flaky name = sprintf "%s [flaky]" name +let knownFailure name = sprintf "%s [known-failure]" name