Merge pull request #70 from globekeeper/daniel/ldap-integration

Merge remote-tracking branch 'ldap/syncloud' into daniel/ldap-integra…

.github/workflows/helm.yml

@@ -6,6 +6,7 @@ on:
       - main
     paths:
       - 'helm/**' # only execute if we have helm chart changes
+  workflow_dispatch:
 
 jobs:
   release:

.github/workflows/k8s.yml

@@ -84,6 +84,7 @@ jobs:
           kubectl get pods -A
           kubectl get services
           kubectl get ingress
+          kubectl logs -l app.kubernetes.io/name=dendrite
       - name: Run create account
         run: |
           podName=$(kubectl get pods -l app.kubernetes.io/name=dendrite -o name)

.github/workflows/schedules.yaml

@@ -65,10 +65,11 @@ jobs:
         uses: actions/upload-artifact@v2
         if: ${{ always() }}
         with:
-          name: Sytest Logs - ${{ job.status }} - (Dendrite, ${{ join(matrix.*, ', ') }})
+          name: Sytest Logs - ${{ job.status }} - (Dendrite ${{ join(matrix.*, ' ') }})
           path: |
             /logs/results.tap
             /logs/**/*.log*
+            /logs/**/covdatafiles/**
 
   sytest-coverage:
     timeout-minutes: 5
@@ -85,16 +86,15 @@ jobs:
           cache: true
       - name: Download all artifacts
         uses: actions/download-artifact@v3
-      - name: Install gocovmerge
-        run: go install github.com/wadey/gocovmerge@latest
-      - name: Run gocovmerge
+      - name: Collect coverage
         run: |
-          find -name 'integrationcover.log' -printf '"%p"\n' | xargs gocovmerge | grep -Ev 'relayapi|setup/mscs|api_trace' > sytest.cov
-          go tool cover -func=sytest.cov
+          go tool covdata textfmt -i="$(find Sytest* -name 'covmeta*' -type f -exec dirname {} \; | uniq | paste -s -d ',' -)" -o sytest.cov
+          grep -Ev 'relayapi|setup/mscs|api_trace' sytest.cov > final.cov
+          go tool covdata func -i="$(find Sytest* -name 'covmeta*' -type f -exec dirname {} \; | uniq | paste -s -d ',' -)"
       - name: Upload coverage to Codecov
         uses: codecov/codecov-action@v3
         with:
-          files: ./sytest.cov
+          files: ./final.cov
           flags: sytest
           fail_ci_if_error: true
 
@@ -167,7 +167,7 @@ jobs:
           cat <<EOF > /tmp/posttest.sh
           #!/bin/bash
           mkdir -p /tmp/Complement/logs/\$2/\$1/
-          docker cp \$1:/dendrite/complementcover.log /tmp/Complement/logs/\$2/\$1/
+          docker cp \$1:/tmp/covdatafiles/. /tmp/Complement/logs/\$2/\$1/
           EOF
           
           chmod +x /tmp/posttest.sh
@@ -188,9 +188,9 @@ jobs:
         uses: actions/upload-artifact@v2
         if: ${{ always() }}
         with:
-          name: Complement Logs - (Dendrite, ${{ join(matrix.*, ', ') }})
+          name: Complement Logs - (Dendrite ${{ join(matrix.*, ' ') }})
           path: |
-            /tmp/Complement/**/complementcover.log
+            /tmp/Complement/logs/**
 
   complement-coverage:
     timeout-minutes: 5
@@ -207,20 +207,19 @@ jobs:
           cache: true
       - name: Download all artifacts
         uses: actions/download-artifact@v3
-      - name: Install gocovmerge
-        run: go install github.com/wadey/gocovmerge@latest
-      - name: Run gocovmerge
+      - name: Collect coverage
         run: |
-          find -name 'complementcover.log' -printf '"%p"\n' | xargs gocovmerge | grep -Ev 'relayapi|setup/mscs|api_trace' > complement.cov
-          go tool cover -func=complement.cov
+          go tool covdata textfmt -i="$(find Complement* -name 'covmeta*' -type f -exec dirname {} \; | uniq | paste -s -d ',' -)" -o complement.cov
+          grep -Ev 'relayapi|setup/mscs|api_trace' complement.cov > final.cov
+          go tool covdata func -i="$(find Complement* -name 'covmeta*' -type f -exec dirname {} \; | uniq | paste -s -d ',' -)"
       - name: Upload coverage to Codecov
         uses: codecov/codecov-action@v3
         with:
-          files: ./complement.cov
+          files: ./final.cov
           flags: complement
           fail_ci_if_error: true
 
-  element_web:
+  element-web:
     timeout-minutes: 120
     runs-on: ubuntu-latest
     steps:
@@ -258,3 +257,42 @@ jobs:
         env:
           PUPPETEER_SKIP_CHROMIUM_DOWNLOAD: true
           TMPDIR: ${{ runner.temp }}
+
+  element-web-pinecone:
+    timeout-minutes: 120
+    runs-on: ubuntu-latest
+    steps:
+      - uses: tecolicom/actions-use-apt-tools@v1
+        with:
+          # Our test suite includes some screenshot tests with unusual diacritics, which are
+          # supposed to be covered by STIXGeneral.
+          tools: fonts-stix
+      - uses: actions/checkout@v2
+        with:
+          repository: matrix-org/matrix-react-sdk
+      - uses: actions/setup-node@v3
+        with:
+          cache: 'yarn'
+      - name: Fetch layered build
+        run: scripts/ci/layered.sh
+      - name: Copy config
+        run: cp element.io/develop/config.json config.json
+        working-directory: ./element-web
+      - name: Build
+        env:
+          CI_PACKAGE: true
+          NODE_OPTIONS: "--openssl-legacy-provider"
+        run: yarn build
+        working-directory: ./element-web
+      - name: Edit Test Config
+        run: |
+          sed -i '/HOMESERVER/c\        HOMESERVER: "dendritePinecone",' cypress.config.ts
+      - name: "Run cypress tests"
+        uses: cypress-io/[email protected]
+        with:
+          browser: chrome
+          start: npx serve -p 8080 ./element-web/webapp
+          wait-on: 'http://localhost:8080'
+        env:
+          PUPPETEER_SKIP_CHROMIUM_DOWNLOAD: true
+          TMPDIR: ${{ runner.temp }}

.gitignore

@@ -79,4 +79,5 @@ media_store/
 
 __debug_bin
 
-cmd/dendrite-monolith-server/dendrite-monolith-server
+cmd/dendrite-monolith-server/dendrite-monolith-server
+build

appservice/api/query.go

@@ -22,8 +22,6 @@ import (
 	"encoding/json"
 	"errors"
 
-	"github.com/matrix-org/gomatrixserverlib"
-
 	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
 	userapi "github.com/matrix-org/dendrite/userapi/api"
 )
@@ -150,32 +148,22 @@ type ASLocationResponse struct {
 	Fields   json.RawMessage `json:"fields"`
 }
 
+// ErrProfileNotExists is returned when trying to lookup a user's profile that
+// doesn't exist locally.
+var ErrProfileNotExists = errors.New("no known profile for given user ID")
+
 // RetrieveUserProfile is a wrapper that queries both the local database and
 // application services for a given user's profile
 // TODO: Remove this, it's called from federationapi and clientapi but is a pure function
 func RetrieveUserProfile(
 	ctx context.Context,
 	userID string,
 	asAPI AppServiceInternalAPI,
-	profileAPI userapi.ClientUserAPI,
+	profileAPI userapi.ProfileAPI,
 ) (*authtypes.Profile, error) {
-	localpart, _, err := gomatrixserverlib.SplitID('@', userID)
-	if err != nil {
-		return nil, err
-	}
-
 	// Try to query the user from the local database
-	res := &userapi.QueryProfileResponse{}
-	err = profileAPI.QueryProfile(ctx, &userapi.QueryProfileRequest{UserID: userID}, res)
-	if err != nil {
-		return nil, err
-	}
-	profile := &authtypes.Profile{
-		Localpart:   localpart,
-		DisplayName: res.DisplayName,
-		AvatarURL:   res.AvatarURL,
-	}
-	if res.UserExists {
+	profile, err := profileAPI.QueryProfile(ctx, userID)
+	if err == nil {
 		return profile, nil
 	}
 
@@ -188,19 +176,15 @@ func RetrieveUserProfile(
 
 	// If no user exists, return
 	if !userResp.UserIDExists {
-		return nil, errors.New("no known profile for given user ID")
+		return nil, ErrProfileNotExists
 	}
 
 	// Try to query the user from the local database again
-	err = profileAPI.QueryProfile(ctx, &userapi.QueryProfileRequest{UserID: userID}, res)
+	profile, err = profileAPI.QueryProfile(ctx, userID)
 	if err != nil {
 		return nil, err
 	}
 
 	// profile should not be nil at this point
-	return &authtypes.Profile{
-		Localpart:   localpart,
-		DisplayName: res.DisplayName,
-		AvatarURL:   res.AvatarURL,
-	}, nil
+	return profile, nil
 }

appservice/appservice.go

@@ -16,11 +16,10 @@ package appservice
 
 import (
 	"context"
-	"crypto/tls"
-	"net/http"
 	"sync"
-	"time"
 
+	"github.com/matrix-org/dendrite/setup/jetstream"
+	"github.com/matrix-org/dendrite/setup/process"
 	"github.com/sirupsen/logrus"
 
 	"github.com/matrix-org/gomatrixserverlib"
@@ -29,47 +28,38 @@ import (
 	"github.com/matrix-org/dendrite/appservice/consumers"
 	"github.com/matrix-org/dendrite/appservice/query"
 	roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
-	"github.com/matrix-org/dendrite/setup/base"
 	"github.com/matrix-org/dendrite/setup/config"
 	userapi "github.com/matrix-org/dendrite/userapi/api"
 )
 
 // NewInternalAPI returns a concerete implementation of the internal API. Callers
 // can call functions directly on the returned API or via an HTTP interface using AddInternalRoutes.
 func NewInternalAPI(
-	base *base.BaseDendrite,
+	processContext *process.ProcessContext,
+	cfg *config.Dendrite,
+	natsInstance *jetstream.NATSInstance,
 	userAPI userapi.AppserviceUserAPI,
 	rsAPI roomserverAPI.RoomserverInternalAPI,
 ) appserviceAPI.AppServiceInternalAPI {
-	client := &http.Client{
-		Timeout: time.Second * 30,
-		Transport: &http.Transport{
-			DisableKeepAlives: true,
-			TLSClientConfig: &tls.Config{
-				InsecureSkipVerify: base.Cfg.AppServiceAPI.DisableTLSValidation,
-			},
-			Proxy: http.ProxyFromEnvironment,
-		},
-	}
+
 	// Create appserivce query API with an HTTP client that will be used for all
 	// outbound and inbound requests (inbound only for the internal API)
 	appserviceQueryAPI := &query.AppServiceQueryAPI{
-		HTTPClient:    client,
-		Cfg:           &base.Cfg.AppServiceAPI,
+		Cfg:           &cfg.AppServiceAPI,
 		ProtocolCache: map[string]appserviceAPI.ASProtocolResponse{},
 		CacheMu:       sync.Mutex{},
 	}
 
-	if len(base.Cfg.Derived.ApplicationServices) == 0 {
+	if len(cfg.Derived.ApplicationServices) == 0 {
 		return appserviceQueryAPI
 	}
 
 	// Wrap application services in a type that relates the application service and
 	// a sync.Cond object that can be used to notify workers when there are new
 	// events to be sent out.
-	for _, appservice := range base.Cfg.Derived.ApplicationServices {
+	for _, appservice := range cfg.Derived.ApplicationServices {
 		// Create bot account for this AS if it doesn't already exist
-		if err := generateAppServiceAccount(userAPI, appservice, base.Cfg.Global.ServerName); err != nil {
+		if err := generateAppServiceAccount(userAPI, appservice, cfg.Global.ServerName); err != nil {
 			logrus.WithFields(logrus.Fields{
 				"appservice": appservice.ID,
 			}).WithError(err).Panicf("failed to generate bot account for appservice")
@@ -78,10 +68,10 @@ func NewInternalAPI(
 
 	// Only consume if we actually have ASes to track, else we'll just chew cycles needlessly.
 	// We can't add ASes at runtime so this is safe to do.
-	js, _ := base.NATS.Prepare(base.ProcessContext, &base.Cfg.Global.JetStream)
+	js, _ := natsInstance.Prepare(processContext, &cfg.Global.JetStream)
 	consumer := consumers.NewOutputRoomEventConsumer(
-		base.ProcessContext, &base.Cfg.AppServiceAPI,
-		client, js, rsAPI,
+		processContext, &cfg.AppServiceAPI,
+		js, rsAPI,
 	)
 	if err := consumer.Start(); err != nil {
 		logrus.WithError(err).Panicf("failed to start appservice roomserver consumer")