Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[WIP] Implement .NET conformance testing #314

Draft
wants to merge 3 commits into
base: main
Choose a base branch
from
+16,623 −27
Draft

[WIP] Implement .NET conformance testing #314

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
2effa48
Implement .NET conformance testing
Zaid-Ajaj Aug 7, 2024
771239d
Return the nuget file path from Pack(...)
Zaid-Ajaj Aug 7, 2024
fff1093
No need to set env vars for command
Zaid-Ajaj Aug 7, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 12 additions & 0 deletions build/Program.fs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -197,6 +197,17 @@ let syncProtoFiles() = GitSync.repository {
]
}

let syncPulumiTestLanguage() = GitSync.repository {
remoteRepository = "https://github.com/pulumi/pulumi.git"
localRepositoryPath = repositoryRoot
contents = [
GitSync.folder {
sourcePath = [ "cmd"; "pulumi-test-language" ]
destinationPath = [ "cmd"; "pulumi-test-language" ]
}
]
}

let runSpecificIntegrationTest(testName: string) =
buildLanguagePlugin()
cleanSdk()
Expand Down Expand Up @@ -226,6 +237,7 @@ let main(args: string[]) : int =
| [| "test-automation-sdk"; "coverage" |] -> testPulumiAutomationSdk true
| [| "publish-sdks" |] -> publishSdks()
| [| "sync-proto-files" |] -> syncProtoFiles()
| [| "sync-pulumi-test-language" |] -> syncPulumiTestLanguage()
| [| "list-integration-tests" |] -> listIntegrationTests()
| [| "integration"; "test"; testName |] -> runSpecificIntegrationTest testName
| [| "all-integration-tests" |] -> runAllIntegrationTests()
Expand Down
63 changes: 63 additions & 0 deletions cmd/pulumi-test-language/README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,63 @@
pulumi-language-test runs a gRPC interface that language plugins can use to run a suite of standard tests.

# Architecture

pulumi-language-test is used to run a standard suite of tests against
any compliant language plugin.

The diagram below shows the main interactions and data flows for how this system works.

There are three main actors involved. Firstly `test` which is a test function coordinating the language plugin and pulumi-language-test. Secondly `ptl` which is the pulumi-language-test process. Finally `uut` which is the language plugin actually being tested. This will generally be a grpc server running in the same process as the test method.

```mermaid

sequenceDiagram
test->>ptl: Start ptl process
ptl-->>test: Read stdout for ptl address to connect to

Note right of test: All future calls to ptl are via grpc

test->>+ptl: GetLanguageTests()
ptl-->>-test: Returns list of test names

test->>+uut: Serve
uut-->>-test: Returns uut server address

test->>+ptl: PrepareLanguageTests(uut)
ptl->>+uut: Pack(core)
uut-->>-ptl: Returns name of core artifact
ptl-->>-test: Returns `token`

loop for each test
test->>+ptl: RunLanguageTest(token, test)
loop for each sdk used in test
ptl->>+uut: GeneratePackage(sdk)
uut-->>-ptl: Write package code to temporary directory
ptl->>ptl: Verify sdk snapshot
ptl->>+uut: Pack(sdk)
uut-->>-ptl: Returns name of sdk artifact
end

ptl->>+uut: GenerateProject(test)
uut-->>-ptl: Write project code to temporary directory
ptl->>ptl: Verify project snapshot

ptl->>+uut: InstallDependencies(project)
uut-->>-ptl:

note right of ptl: Execute test with engine
activate ptl
ptl->>+uut: Run(project)
uut-->>-ptl: Return run result
ptl->>ptl: Run test asserts against run result and snapshot
deactivate ptl

ptl-->>-test: Returns test result from asserts
end

test->>ptl: sigkill
```

## Meta tests

This module contains a number of `_test.go` files. These are tests of the conformance test system itself. The actual conformance tests are all defined in `tests.go`.
204 changes: 204 additions & 0 deletions cmd/pulumi-test-language/go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,204 @@
module github.com/pulumi/pulumi/cmd/pulumi-test-language

go 1.20

replace github.com/pulumi/pulumi/sdk/v3 => ../../../pulumi/sdk

replace github.com/pulumi/pulumi/pkg/v3 => ../../../pulumi/pkg

require (
github.com/blang/semver v3.5.1+incompatible
github.com/deckarep/golang-set/v2 v2.5.0
github.com/hexops/gotextdiff v1.0.3
github.com/pulumi/pulumi/pkg/v3 v3.98.0
github.com/pulumi/pulumi/sdk/v3 v3.127.0
github.com/segmentio/encoding v0.3.6
github.com/stretchr/testify v1.9.0
google.golang.org/grpc v1.63.2
google.golang.org/protobuf v1.33.0
gopkg.in/yaml.v2 v2.4.0
)

require (
cloud.google.com/go v0.112.1 // indirect
cloud.google.com/go/compute v1.25.0 // indirect
cloud.google.com/go/compute/metadata v0.2.3 // indirect
cloud.google.com/go/iam v1.1.6 // indirect
cloud.google.com/go/kms v1.15.7 // indirect
cloud.google.com/go/logging v1.9.0 // indirect
cloud.google.com/go/longrunning v0.5.5 // indirect
cloud.google.com/go/storage v1.39.1 // indirect
dario.cat/mergo v1.0.0 // indirect
github.com/AlecAivazis/survey/v2 v2.3.7 // indirect
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.11.1 // indirect
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.6.0 // indirect
github.com/Azure/azure-sdk-for-go/sdk/internal v1.8.0 // indirect
github.com/Azure/azure-sdk-for-go/sdk/keyvault/azkeys v0.10.0 // indirect
github.com/Azure/azure-sdk-for-go/sdk/keyvault/internal v0.7.1 // indirect
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.3.1 // indirect
github.com/Azure/go-autorest v14.2.0+incompatible // indirect
github.com/Azure/go-autorest/autorest/to v0.4.0 // indirect
github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2 // indirect
github.com/BurntSushi/toml v1.2.1 // indirect
github.com/Microsoft/go-winio v0.6.1 // indirect
github.com/ProtonMail/go-crypto v1.0.0 // indirect
github.com/aead/chacha20 v0.0.0-20180709150244-8b13a72661da // indirect
github.com/agext/levenshtein v1.2.3 // indirect
github.com/apparentlymart/go-textseg/v13 v13.0.0 // indirect
github.com/atotto/clipboard v0.1.4 // indirect
github.com/aws/aws-sdk-go v1.50.36 // indirect
github.com/aws/aws-sdk-go-v2 v1.26.1 // indirect
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.2 // indirect
github.com/aws/aws-sdk-go-v2/config v1.27.11 // indirect
github.com/aws/aws-sdk-go-v2/credentials v1.17.11 // indirect
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.1 // indirect
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.16.15 // indirect
github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.5 // indirect
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.5 // indirect
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 // indirect
github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.5 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.2 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.7 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.7 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.5 // indirect
github.com/aws/aws-sdk-go-v2/service/kms v1.30.1 // indirect
github.com/aws/aws-sdk-go-v2/service/s3 v1.53.1 // indirect
github.com/aws/aws-sdk-go-v2/service/sso v1.20.5 // indirect
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.4 // indirect
github.com/aws/aws-sdk-go-v2/service/sts v1.28.6 // indirect
github.com/aws/smithy-go v1.20.2 // indirect
github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect
github.com/cenkalti/backoff/v3 v3.2.2 // indirect
github.com/charmbracelet/bubbles v0.16.1 // indirect
github.com/charmbracelet/bubbletea v0.25.0 // indirect
github.com/charmbracelet/lipgloss v0.7.1 // indirect
github.com/cheggaaa/pb v1.0.29 // indirect
github.com/cloudflare/circl v1.3.7 // indirect
github.com/containerd/console v1.0.4-0.20230313162750-1ae8d489ac81 // indirect
github.com/cyphar/filepath-securejoin v0.2.4 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/djherbis/times v1.5.0 // indirect
github.com/dustin/go-humanize v1.0.1 // indirect
github.com/edsrzf/mmap-go v1.1.0 // indirect
github.com/emirpasic/gods v1.18.1 // indirect
github.com/felixge/httpsnoop v1.0.4 // indirect
github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect
github.com/go-git/go-billy/v5 v5.5.0 // indirect
github.com/go-git/go-git/v5 v5.12.0 // indirect
github.com/go-jose/go-jose/v3 v3.0.3 // indirect
github.com/go-logr/logr v1.4.1 // indirect
github.com/go-logr/stdr v1.2.2 // indirect
github.com/gofrs/uuid v4.2.0+incompatible // indirect
github.com/gogo/protobuf v1.3.2 // indirect
github.com/golang-jwt/jwt/v5 v5.2.1 // indirect
github.com/golang/glog v1.2.0 // indirect
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
github.com/golang/protobuf v1.5.4 // indirect
github.com/google/go-querystring v1.1.0 // indirect
github.com/google/s2a-go v0.1.7 // indirect
github.com/google/uuid v1.6.0 // indirect
github.com/google/wire v0.6.0 // indirect
github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect
github.com/googleapis/gax-go/v2 v2.12.2 // indirect
github.com/gorilla/mux v1.8.0 // indirect
github.com/grpc-ecosystem/grpc-opentracing v0.0.0-20180507213350-8e809c8a8645 // indirect
github.com/hashicorp/errwrap v1.1.0 // indirect
github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
github.com/hashicorp/go-multierror v1.1.1 // indirect
github.com/hashicorp/go-retryablehttp v0.7.7 // indirect
github.com/hashicorp/go-rootcerts v1.0.2 // indirect
github.com/hashicorp/go-secure-stdlib/parseutil v0.1.8 // indirect
github.com/hashicorp/go-secure-stdlib/strutil v0.1.2 // indirect
github.com/hashicorp/go-sockaddr v1.0.6 // indirect
github.com/hashicorp/hcl v1.0.0 // indirect
github.com/hashicorp/hcl/v2 v2.17.0 // indirect
github.com/hashicorp/vault/api v1.12.0 // indirect
github.com/ijc/Gotty v0.0.0-20170406111628-a8b993ba6abd // indirect
github.com/inconshreveable/mousetrap v1.1.0 // indirect
github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect
github.com/jmespath/go-jmespath v0.4.0 // indirect
github.com/json-iterator/go v1.1.12 // indirect
github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect
github.com/kevinburke/ssh_config v1.2.0 // indirect
github.com/kylelemons/godebug v1.1.0 // indirect
github.com/lucasb-eyer/go-colorful v1.2.0 // indirect
github.com/mattn/go-colorable v0.1.13 // indirect
github.com/mattn/go-isatty v0.0.20 // indirect
github.com/mattn/go-localereader v0.0.1 // indirect
github.com/mattn/go-runewidth v0.0.15 // indirect
github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b // indirect
github.com/mitchellh/copystructure v1.2.0 // indirect
github.com/mitchellh/go-homedir v1.1.0 // indirect
github.com/mitchellh/go-ps v1.0.0 // indirect
github.com/mitchellh/go-wordwrap v1.0.1 // indirect
github.com/mitchellh/mapstructure v1.5.0 // indirect
github.com/mitchellh/reflectwalk v1.0.2 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/muesli/ansi v0.0.0-20230316100256-276c6243b2f6 // indirect
github.com/muesli/cancelreader v0.2.2 // indirect
github.com/muesli/reflow v0.3.0 // indirect
github.com/muesli/termenv v0.15.2 // indirect
github.com/natefinch/atomic v1.0.1 // indirect
github.com/opentracing/basictracer-go v1.1.0 // indirect
github.com/opentracing/opentracing-go v1.2.0 // indirect
github.com/pgavlin/fx v0.1.6 // indirect
github.com/pgavlin/goldmark v1.1.33-0.20200616210433-b5eb04559386 // indirect
github.com/pjbgf/sha1cd v0.3.0 // indirect
github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/pkg/term v1.1.0 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/pulumi/appdash v0.0.0-20231130102222-75f619a67231 // indirect
github.com/pulumi/esc v0.9.1 // indirect
github.com/pulumi/inflector v0.1.1 // indirect
github.com/rivo/uniseg v0.4.4 // indirect
github.com/rogpeppe/go-internal v1.12.0 // indirect
github.com/ryanuber/go-glob v1.0.0 // indirect
github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06 // indirect
github.com/santhosh-tekuri/jsonschema/v5 v5.0.0 // indirect
github.com/segmentio/asm v1.1.3 // indirect
github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 // indirect
github.com/skeema/knownhosts v1.2.2 // indirect
github.com/spf13/afero v1.9.5 // indirect
github.com/spf13/cobra v1.8.0 // indirect
github.com/spf13/pflag v1.0.5 // indirect
github.com/texttheater/golang-levenshtein v1.0.1 // indirect
github.com/tweekmonster/luser v0.0.0-20161003172636-3fa38070dbd7 // indirect
github.com/uber/jaeger-client-go v2.30.0+incompatible // indirect
github.com/uber/jaeger-lib v2.4.1+incompatible // indirect
github.com/xanzy/ssh-agent v0.3.3 // indirect
github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f // indirect
github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect
github.com/xeipuuv/gojsonschema v1.2.0 // indirect
github.com/zclconf/go-cty v1.13.2 // indirect
go.opencensus.io v0.24.0 // indirect
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 // indirect
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 // indirect
go.opentelemetry.io/otel v1.24.0 // indirect
go.opentelemetry.io/otel/metric v1.24.0 // indirect
go.opentelemetry.io/otel/trace v1.24.0 // indirect
go.uber.org/atomic v1.9.0 // indirect
gocloud.dev v0.37.0 // indirect
gocloud.dev/secrets/hashivault v0.37.0 // indirect
golang.org/x/crypto v0.24.0 // indirect
golang.org/x/exp v0.0.0-20240604190554-fc45aab8b7f8 // indirect
golang.org/x/mod v0.18.0 // indirect
golang.org/x/net v0.26.0 // indirect
golang.org/x/oauth2 v0.18.0 // indirect
golang.org/x/sync v0.7.0 // indirect
golang.org/x/sys v0.21.0 // indirect
golang.org/x/term v0.21.0 // indirect
golang.org/x/text v0.16.0 // indirect
golang.org/x/time v0.5.0 // indirect
golang.org/x/tools v0.22.0 // indirect
golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect
google.golang.org/api v0.169.0 // indirect
google.golang.org/appengine v1.6.8 // indirect
google.golang.org/genproto v0.0.0-20240311173647-c811ad7063a7 // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20240311173647-c811ad7063a7 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20240311173647-c811ad7063a7 // indirect
gopkg.in/warnings.v0 v0.1.2 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
lukechampine.com/frand v1.4.2 // indirect
)
Loading
Loading