diff --git a/.travis.yml b/.travis.yml
index 1bbc1eb5..cecf6a48 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -1,6 +1,5 @@
language: csharp
-mono: 5.18.0
dotnet: 2.1.401
install:
@@ -9,12 +8,37 @@ install:
script:
- dotnet pack -v n
- - dotnet test -v n
+ - dotnet test -v n -p:UnstableTests=$UnstableTests
matrix:
include:
- os: linux
dist: trusty
sudo: required
+ mono: 5.16.0
+ env: UnstableTests=false
- os: osx
osx_image: xcode9
+ mono: 5.16.0
+ env: UnstableTests=false
+ - os: linux
+ dist: trusty
+ sudo: required
+ mono: 5.16.0
+ env: UnstableTests=true
+ - os: osx
+ osx_image: xcode9
+ mono: 5.16.0
+ env: UnstableTests=true
+ - os: linux
+ dist: trusty
+ sudo: required
+ mono: 5.18.0
+ env: UnstableTests=false
+ - os: osx
+ osx_image: xcode9
+ mono: 5.18.0
+ env: UnstableTests=false
+ allow_failures:
+ # unstable tests may fail
+ - env: UnstableTests=true
diff --git a/build.proj b/build.proj
index fa6c115e..38e55c6d 100644
--- a/build.proj
+++ b/build.proj
@@ -2,27 +2,39 @@
+
+ Release
+ false
+
+
-
+
-
+
-
-
-
-
+
+
+
+
-
-
-
+
+
+
+
+
+
+
+
+
+
diff --git a/test/Dotnet.ProjInfo.Workspace.FCS.Tests/Dotnet.ProjInfo.Workspace.FCS.Tests.fsproj b/test/Dotnet.ProjInfo.Workspace.FCS.Tests/Dotnet.ProjInfo.Workspace.FCS.Tests.fsproj
index 7a0da761..04a68c0d 100644
--- a/test/Dotnet.ProjInfo.Workspace.FCS.Tests/Dotnet.ProjInfo.Workspace.FCS.Tests.fsproj
+++ b/test/Dotnet.ProjInfo.Workspace.FCS.Tests/Dotnet.ProjInfo.Workspace.FCS.Tests.fsproj
@@ -7,6 +7,7 @@
+
diff --git a/test/Dotnet.ProjInfo.Workspace.FCS.Tests/Program.fs b/test/Dotnet.ProjInfo.Workspace.FCS.Tests/Program.fs
index 0770b2b8..ff535fa0 100644
--- a/test/Dotnet.ProjInfo.Workspace.FCS.Tests/Program.fs
+++ b/test/Dotnet.ProjInfo.Workspace.FCS.Tests/Program.fs
@@ -4,9 +4,29 @@ open Expecto
open System
open System.IO
+open TestsConfig
+
+type Args =
+ { RunOnlyFlaky: bool
+ RunOnlyKnownFailure: bool }
+
+let rec parseArgs (config, runArgs, args) =
+ match args with
+ | "--flaky" :: xs ->
+ parseArgs ({ config with SkipFlaky = false }, { runArgs with RunOnlyFlaky = true }, xs)
+ | "--known-failure" :: xs ->
+ parseArgs ({ config with SkipKnownFailure = false }, { runArgs with RunOnlyKnownFailure = true }, xs)
+ | xs ->
+ config, runArgs, xs
+
[]
let main argv =
+ let suiteConfig, runArgs, otherArgs =
+ let defaultConfig = { SkipFlaky = true; SkipKnownFailure = true }
+ let defaultRunArgs = { Args.RunOnlyFlaky = false; RunOnlyKnownFailure = false }
+ parseArgs (defaultConfig, defaultRunArgs, List.ofArray argv)
+
Environment.SetEnvironmentVariable("DOTNET_PROJ_INFO_MSBUILD_BL", "1")
Environment.SetEnvironmentVariable("MSBuildExtensionsPath", null)
@@ -15,4 +35,9 @@ let main argv =
let writeResults = TestResults.writeNUnitSummary (resultsPath, "Dotnet.ProjInfo.Workspace.FCS.Tests")
let config = defaultConfig.appendSummaryHandler writeResults
- Tests.runTestsWithArgs config argv (Tests.tests ())
+ let tests =
+ Tests.tests suiteConfig
+ |> Test.filter (fun s -> if runArgs.RunOnlyFlaky then s.Contains "[flaky]" else true)
+ |> Test.filter (fun s -> if runArgs.RunOnlyKnownFailure then s.Contains "[known-failure]" else true)
+
+ Tests.runTestsWithArgs config (otherArgs |> Array.ofList) tests
diff --git a/test/Dotnet.ProjInfo.Workspace.FCS.Tests/Tests.fs b/test/Dotnet.ProjInfo.Workspace.FCS.Tests/Tests.fs
index 9cafb99a..67b223a4 100644
--- a/test/Dotnet.ProjInfo.Workspace.FCS.Tests/Tests.fs
+++ b/test/Dotnet.ProjInfo.Workspace.FCS.Tests/Tests.fs
@@ -138,7 +138,9 @@ let expectP2PKeyIsTargetPath po =
for (tar, fcsPO, poDPW) in allP2P po do
Expect.equal tar poDPW.ExtraProjectInfo.TargetPath (sprintf "p2p key is TargetPath, fsc projet options was '%A'" fcsPO)
-let tests () =
+open TestsConfig
+
+let tests (suiteConfig: TestSuiteConfig) =
let prepareTestsAssets = lazy(
let logger = Log.create "Tests Assets"
@@ -364,7 +366,7 @@ let tests () =
)
- testCase |> withLog "can load sample3" (fun logger fs ->
+ testCase |> withLog ("can load sample3" |> knownFailure) (fun logger fs ->
let testDir = inDir fs "load_sample3"
copyDirFromAssets fs ``sample3 Netsdk projs``.ProjDir testDir
@@ -409,7 +411,7 @@ let tests () =
fcs.ParseAndCheckProject(fcsPo)
|> Async.RunSynchronously
- if (isOSX ()) then
+ if (isOSX () && suiteConfig.SkipKnownFailure) then
let errorOnOsx =
"""
no errors but was: [|commandLineArgs (0,1)-(0,1) parameter error No inputs specified;
diff --git a/test/Dotnet.ProjInfo.Workspace.Tests/Dotnet.ProjInfo.Workspace.Tests.fsproj b/test/Dotnet.ProjInfo.Workspace.Tests/Dotnet.ProjInfo.Workspace.Tests.fsproj
index ef764c76..71cdac8c 100644
--- a/test/Dotnet.ProjInfo.Workspace.Tests/Dotnet.ProjInfo.Workspace.Tests.fsproj
+++ b/test/Dotnet.ProjInfo.Workspace.Tests/Dotnet.ProjInfo.Workspace.Tests.fsproj
@@ -7,6 +7,7 @@
+
diff --git a/test/Dotnet.ProjInfo.Workspace.Tests/Program.fs b/test/Dotnet.ProjInfo.Workspace.Tests/Program.fs
index 210afe9f..ec93917d 100644
--- a/test/Dotnet.ProjInfo.Workspace.Tests/Program.fs
+++ b/test/Dotnet.ProjInfo.Workspace.Tests/Program.fs
@@ -4,9 +4,29 @@ open Expecto
open System
open System.IO
+open TestsConfig
+
+type Args =
+ { RunOnlyFlaky: bool
+ RunOnlyKnownFailure: bool }
+
+let rec parseArgs (config, runArgs, args) =
+ match args with
+ | "--flaky" :: xs ->
+ parseArgs ({ config with SkipFlaky = false }, { runArgs with RunOnlyFlaky = true }, xs)
+ | "--known-failure" :: xs ->
+ parseArgs ({ config with SkipKnownFailure = false }, { runArgs with RunOnlyKnownFailure = true }, xs)
+ | xs ->
+ config, runArgs, xs
+
[]
let main argv =
+ let suiteConfig, runArgs, otherArgs =
+ let defaultConfig = { SkipFlaky = true; SkipKnownFailure = true }
+ let defaultRunArgs = { Args.RunOnlyFlaky = false; RunOnlyKnownFailure = false }
+ parseArgs (defaultConfig, defaultRunArgs, List.ofArray argv)
+
Environment.SetEnvironmentVariable("DOTNET_PROJ_INFO_MSBUILD_BL", "1")
Environment.SetEnvironmentVariable("MSBuildExtensionsPath", null)
@@ -15,4 +35,9 @@ let main argv =
let writeResults = TestResults.writeNUnitSummary (resultsPath, "Dotnet.ProjInfo.Workspace.Tests")
let config = defaultConfig.appendSummaryHandler writeResults
- Tests.runTestsWithArgs config argv (Tests.tests ())
+ let tests =
+ Tests.tests suiteConfig
+ |> Test.filter (fun s -> if runArgs.RunOnlyFlaky then s.Contains "[flaky]" else true)
+ |> Test.filter (fun s -> if runArgs.RunOnlyKnownFailure then s.Contains "[known-failure]" else true)
+
+ Tests.runTestsWithArgs config (otherArgs |> Array.ofList) tests
diff --git a/test/Dotnet.ProjInfo.Workspace.Tests/Tests.fs b/test/Dotnet.ProjInfo.Workspace.Tests/Tests.fs
index 91a46a0b..592160ed 100644
--- a/test/Dotnet.ProjInfo.Workspace.Tests/Tests.fs
+++ b/test/Dotnet.ProjInfo.Workspace.Tests/Tests.fs
@@ -148,7 +148,9 @@ let isOSX () =
System.Runtime.InteropServices.RuntimeInformation.IsOSPlatform(
System.Runtime.InteropServices.OSPlatform.OSX)
-let tests () =
+open TestsConfig
+
+let tests (suiteConfig: TestSuiteConfig) =
let prepareTestsAssets = lazy(
let logger = Log.create "Tests Assets"
@@ -292,7 +294,7 @@ let tests () =
Expect.equal n1Parsed.SourceFiles expectedSources "check sources"
)
- testCase |> withLog "can load sample3" (fun logger fs ->
+ testCase |> withLog ("can load sample3" |> knownFailure) (fun logger fs ->
let testDir = inDir fs "load_sample3"
copyDirFromAssets fs ``sample3 Netsdk projs``.ProjDir testDir
@@ -349,7 +351,7 @@ let tests () =
parsed
|> expectFind projPath { ProjectKey.ProjectPath = projPath; TargetFramework = "netcoreapp2.1" } "the F# console"
- if (isOSX ()) then
+ if (isOSX () && suiteConfig.SkipKnownFailure) then
let errorOnOsx =
"""
check sources.
diff --git a/test/Dotnet.ProjInfo.Workspace.Tests/TestsConfig.fs b/test/Dotnet.ProjInfo.Workspace.Tests/TestsConfig.fs
new file mode 100644
index 00000000..d38637fa
--- /dev/null
+++ b/test/Dotnet.ProjInfo.Workspace.Tests/TestsConfig.fs
@@ -0,0 +1,8 @@
+module TestsConfig
+
+type TestSuiteConfig =
+ { SkipFlaky: bool
+ SkipKnownFailure: bool }
+
+let flaky name = sprintf "%s [flaky]" name
+let knownFailure name = sprintf "%s [known-failure]" name