Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add temporary repository flag #3453

Merged
merged 3 commits into from
Dec 17, 2024
Merged

Add temporary repository flag #3453

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
a892bb2
Add temporary repository flag
S7evinK Dec 17, 2024
93a82a9
Fetch versions from the correct repo as well
S7evinK Dec 17, 2024
4fe14c5
Test SQLite if we say so
S7evinK Dec 17, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 4 additions & 4 deletions .github/workflows/dendrite.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -294,9 +294,9 @@ jobs:
- name: Build upgrade-tests
run: go build ./cmd/dendrite-upgrade-tests
- name: Test upgrade (PostgreSQL)
run: ./dendrite-upgrade-tests --head .
run: ./dendrite-upgrade-tests -repository=matrix-org/dendrite --head .
- name: Test upgrade (SQLite)
run: ./dendrite-upgrade-tests --sqlite --head .
run: ./dendrite-upgrade-tests --sqlite -repository=matrix-org/dendrite --head .

# run database upgrade tests, skipping over one version
upgrade_test_direct:
Expand Down Expand Up @@ -324,9 +324,9 @@ jobs:
- name: Build upgrade-tests
run: go build ./cmd/dendrite-upgrade-tests
- name: Test upgrade (PostgreSQL)
run: ./dendrite-upgrade-tests -direct -from HEAD-2 --head .
run: ./dendrite-upgrade-tests -direct -from HEAD-2 -repository=matrix-org/dendrite --head .
- name: Test upgrade (SQLite)
run: ./dendrite-upgrade-tests -direct -from HEAD-2 --head .
run: ./dendrite-upgrade-tests --sqlite -direct -from HEAD-2 -repository=matrix-org/dendrite --head .

# run Sytest in different variations
sytest:
Expand Down
21 changes: 11 additions & 10 deletions cmd/dendrite-upgrade-tests/main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@ var (
flagDockerHost = flag.String("docker-host", "localhost", "The hostname of the docker client. 'localhost' if running locally, 'host.docker.internal' if running in Docker.")
flagDirect = flag.Bool("direct", false, "If a direct upgrade from the defined FROM version to TO should be done")
flagSqlite = flag.Bool("sqlite", false, "Test SQLite instead of PostgreSQL")
flagRepository = flag.String("repository", "element-hq/dendrite", "The base repository to use when running upgrade tests.")
alphaNumerics = regexp.MustCompile("[^a-zA-Z0-9]+")
)

Expand Down Expand Up @@ -187,7 +188,7 @@ func downloadArchive(cli *http.Client, tmpDir, archiveURL string, dockerfile []b
}

// buildDendrite builds Dendrite on the branchOrTagName given. Returns the image ID or an error
func buildDendrite(httpClient *http.Client, dockerClient *client.Client, tmpDir string, branchOrTagName, binary string) (string, error) {
func buildDendrite(httpClient *http.Client, dockerClient *client.Client, tmpDir string, branchOrTagName, binary, repository string) (string, error) {
var tarball *bytes.Buffer
var err error
// If a custom HEAD location is given, use that, else pull from github. Mostly useful for CI
Expand All @@ -210,7 +211,7 @@ func buildDendrite(httpClient *http.Client, dockerClient *client.Client, tmpDir
log.Printf("%s: Downloading version %s to %s\n", branchOrTagName, branchOrTagName, tmpDir)
// pull an archive, this contains a top-level directory which screws with the build context
// which we need to fix up post download
u := fmt.Sprintf("https://github.com/element-hq/dendrite/archive/%s.tar.gz", branchOrTagName)
u := fmt.Sprintf("https://github.com/%s/archive/%s.tar.gz", repository, branchOrTagName)
tarball, err = downloadArchive(httpClient, tmpDir, u, dockerfile())
if err != nil {
return "", fmt.Errorf("failed to download archive %s: %w", u, err)
Expand Down Expand Up @@ -254,8 +255,8 @@ func buildDendrite(httpClient *http.Client, dockerClient *client.Client, tmpDir
return imageID, nil
}

func getAndSortVersionsFromGithub(httpClient *http.Client) (semVers []*semver.Version, err error) {
u := "https://api.github.com/repos/element-hq/dendrite/tags"
func getAndSortVersionsFromGithub(httpClient *http.Client, repository string) (semVers []*semver.Version, err error) {
u := fmt.Sprintf("https://api.github.com/repos/%s/tags", repository)

var res *http.Response
for i := 0; i < 3; i++ {
Expand Down Expand Up @@ -290,8 +291,8 @@ func getAndSortVersionsFromGithub(httpClient *http.Client) (semVers []*semver.Ve
return semVers, nil
}

func calculateVersions(cli *http.Client, from, to string, direct bool) []*semver.Version {
semvers, err := getAndSortVersionsFromGithub(cli)
func calculateVersions(cli *http.Client, from, to, repository string, direct bool) []*semver.Version {
semvers, err := getAndSortVersionsFromGithub(cli, repository)
if err != nil {
log.Fatalf("failed to collect semvers from github: %s", err)
}
Expand Down Expand Up @@ -348,7 +349,7 @@ func calculateVersions(cli *http.Client, from, to string, direct bool) []*semver
return semvers
}

func buildDendriteImages(httpClient *http.Client, dockerClient *client.Client, baseTempDir string, concurrency int, versions []*semver.Version) map[string]string {
func buildDendriteImages(httpClient *http.Client, dockerClient *client.Client, baseTempDir, repository string, concurrency int, versions []*semver.Version) map[string]string {
// concurrently build all versions, this can be done in any order. The mutex protects the map
branchToImageID := make(map[string]string)
var mu sync.Mutex
Expand All @@ -368,7 +369,7 @@ func buildDendriteImages(httpClient *http.Client, dockerClient *client.Client, b
branchName, binary := versionToBranchAndBinary(version)
log.Printf("Building version %s with binary %s", branchName, binary)
tmpDir := baseTempDir + alphaNumerics.ReplaceAllString(branchName, "")
imgID, err := buildDendrite(httpClient, dockerClient, tmpDir, branchName, binary)
imgID, err := buildDendrite(httpClient, dockerClient, tmpDir, branchName, binary, repository)
if err != nil {
log.Fatalf("%s: failed to build dendrite image: %s", version, err)
}
Expand Down Expand Up @@ -583,10 +584,10 @@ func main() {
os.Exit(1)
}
cleanup(dockerClient)
versions := calculateVersions(httpClient, *flagFrom, *flagTo, *flagDirect)
versions := calculateVersions(httpClient, *flagFrom, *flagTo, *flagRepository, *flagDirect)
log.Printf("Testing dendrite versions: %v\n", versions)

branchToImageID := buildDendriteImages(httpClient, dockerClient, *flagTempDir, *flagBuildConcurrency, versions)
branchToImageID := buildDendriteImages(httpClient, dockerClient, *flagTempDir, *flagRepository, *flagBuildConcurrency, versions)

// make a shared postgres volume
volume, err := dockerClient.VolumeCreate(context.Background(), volume.CreateOptions{
Expand Down
Loading