From 465fc11f80b0661e8e83575286cc270e4f2cddd6 Mon Sep 17 00:00:00 2001 From: Christoph Pirkl Date: Wed, 27 Mar 2024 15:37:57 +0100 Subject: [PATCH] #169: Add caching for HTTP registry (#170) --- .github/workflows/ci-build-next-java.yml | 10 - .github/workflows/ci-build.yml | 13 +- ...elease_droid_prepare_original_checksum.yml | 10 - .../release_droid_print_quick_checksum.yml | 10 - ...release_droid_release_on_maven_central.yml | 10 - ...ase_droid_upload_github_release_assets.yml | 10 - .gitignore | 2 + cmd/main.go | 8 +- cmd/manual_i_test.go | 186 ++++++++++++++++++ doc/changes/changes_0.5.9.md | 5 +- doc/design.md | 14 ++ doc/developer_guide.md | 39 +++- .../exasol/extensionmanager/ExampleIT.java | 2 +- .../itest/IntegrationTestCommon.java | 2 +- pkg/extensionAPI/extensionApi.go | 4 +- pkg/extensionController/bfs/BucketFsApi.go | 11 +- pkg/extensionController/controller.go | 7 +- pkg/extensionController/registry/http.go | 37 +++- pkg/extensionController/registry/http_test.go | 20 +- pkg/extensionController/registry/localDir.go | 2 + .../transactionController.go | 12 +- pkg/integrationTesting/dbTestSetup.go | 2 +- sonar-project.properties | 1 + 23 files changed, 315 insertions(+), 102 deletions(-) create mode 100644 cmd/manual_i_test.go diff --git a/.github/workflows/ci-build-next-java.yml b/.github/workflows/ci-build-next-java.yml index 754ec4cf..a413155b 100644 --- a/.github/workflows/ci-build-next-java.yml +++ b/.github/workflows/ci-build-next-java.yml @@ -27,16 +27,6 @@ jobs: distribution: "temurin" java-version: 17 cache: "maven" - - name: Cache go modules - uses: actions/cache@v4 - with: - path: | - ~/.cache/go-build - ~/go/pkg/mod - key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} - restore-keys: | - ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} - ${{ runner.os }}-go- - name: Run tests and build with Maven run: | diff --git a/.github/workflows/ci-build.yml b/.github/workflows/ci-build.yml index 527b7446..5814764a 100644 --- a/.github/workflows/ci-build.yml +++ b/.github/workflows/ci-build.yml @@ -13,7 +13,7 @@ jobs: group: ${{ github.workflow }}-${{ github.ref }}-build cancel-in-progress: true env: - EXASOL_VERSION: "8.23.1" + EXASOL_VERSION: "8.24.0" steps: - name: Check out code uses: actions/checkout@v4 @@ -35,17 +35,6 @@ jobs: 17 cache: "maven" - - name: Cache go modules - uses: actions/cache@v4 - with: - path: | - ~/.cache/go-build - ~/go/pkg/mod - key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} - restore-keys: | - ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} - ${{ runner.os }}-go- - - name: Get npm cache directory id: npm-cache-dir run: echo "dir=$(npm config get cache)" >> $GITHUB_OUTPUT diff --git a/.github/workflows/release_droid_prepare_original_checksum.yml b/.github/workflows/release_droid_prepare_original_checksum.yml index 824f696a..4dff1eab 100644 --- a/.github/workflows/release_droid_prepare_original_checksum.yml +++ b/.github/workflows/release_droid_prepare_original_checksum.yml @@ -24,16 +24,6 @@ jobs: with: go-version: "1.21" id: go - - name: Cache go modules - uses: actions/cache@v4 - with: - path: | - ~/.cache/go-build - ~/go/pkg/mod - key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} - restore-keys: | - ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} - ${{ runner.os }}-go- - name: Prepare testing extension run: cd ./extension-manager-integration-test-java/testing-extension && npm ci diff --git a/.github/workflows/release_droid_print_quick_checksum.yml b/.github/workflows/release_droid_print_quick_checksum.yml index 408fe4f4..774f8d8b 100644 --- a/.github/workflows/release_droid_print_quick_checksum.yml +++ b/.github/workflows/release_droid_print_quick_checksum.yml @@ -24,16 +24,6 @@ jobs: with: go-version: "1.21" id: go - - name: Cache go modules - uses: actions/cache@v4 - with: - path: | - ~/.cache/go-build - ~/go/pkg/mod - key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} - restore-keys: | - ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} - ${{ runner.os }}-go- - name: Build with Maven skipping tests run: mvn --batch-mode clean verify -DskipTests diff --git a/.github/workflows/release_droid_release_on_maven_central.yml b/.github/workflows/release_droid_release_on_maven_central.yml index 5870ba89..f4cd8ef7 100644 --- a/.github/workflows/release_droid_release_on_maven_central.yml +++ b/.github/workflows/release_droid_release_on_maven_central.yml @@ -30,16 +30,6 @@ jobs: with: go-version: "1.21" id: go - - name: Cache go modules - uses: actions/cache@v4 - with: - path: | - ~/.cache/go-build - ~/go/pkg/mod - key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} - restore-keys: | - ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} - ${{ runner.os }}-go- - name: Publish to Central Repository run: mvn --batch-mode -Dgpg.skip=false -DskipTests clean deploy diff --git a/.github/workflows/release_droid_upload_github_release_assets.yml b/.github/workflows/release_droid_upload_github_release_assets.yml index ef603f98..f1024dae 100644 --- a/.github/workflows/release_droid_upload_github_release_assets.yml +++ b/.github/workflows/release_droid_upload_github_release_assets.yml @@ -28,16 +28,6 @@ jobs: with: go-version: "1.21" id: go - - name: Cache go modules - uses: actions/cache@v4 - with: - path: | - ~/.cache/go-build - ~/go/pkg/mod - key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} - restore-keys: | - ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} - ${{ runner.os }}-go- - name: Prepare testing extension run: | diff --git a/.gitignore b/.gitignore index c71f5ea1..765ee4a6 100644 --- a/.gitignore +++ b/.gitignore @@ -39,6 +39,8 @@ vendor/ /doc/tracing-report.md /doc/tracing-report.html +/manual-test.properties + !/pkg/extensionController/bfs/udf/poetry.lock /pkg/extensionController/bfs/udf/__pycache__ /pkg/extensionController/bfs/udf/.pytest_cache diff --git a/cmd/main.go b/cmd/main.go index c2916af5..cd03c69e 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -1,7 +1,6 @@ package main import ( - "bytes" "errors" "flag" "fmt" @@ -99,10 +98,5 @@ type simpleFormatter struct { } func (f *simpleFormatter) Format(entry *log.Entry) ([]byte, error) { - b := &bytes.Buffer{} - b.WriteString(strings.ToUpper(entry.Level.String())) - b.WriteByte(' ') - b.WriteString(entry.Message) - b.WriteByte('\n') - return b.Bytes(), nil + return []byte(fmt.Sprintf("%-7s %s\n", strings.ToUpper(entry.Level.String()), entry.Message)), nil } diff --git a/cmd/manual_i_test.go b/cmd/manual_i_test.go new file mode 100644 index 00000000..53aa3381 --- /dev/null +++ b/cmd/manual_i_test.go @@ -0,0 +1,186 @@ +package main + +import ( + "bufio" + "context" + "database/sql" + "fmt" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/exasol/exasol-driver-go" + "github.com/exasol/extension-manager/pkg/extensionAPI" + "github.com/exasol/extension-manager/pkg/extensionController" + + log "github.com/sirupsen/logrus" + + "github.com/stretchr/testify/suite" +) + +// Create file manual-test.properties, add configuration and +// run this test with `go test -v ./cmd/...` + +type ManualITestSuite struct { + suite.Suite + config configProperties + db *sql.DB + ctrl extensionController.TransactionController // Add this line +} + +func TestManualITestSuite(t *testing.T) { + suite.Run(t, new(ManualITestSuite)) +} + +func (suite *ManualITestSuite) SetupSuite() { + log.SetLevel(log.DebugLevel) + log.SetFormatter(new(simpleFormatter)) + config, err := readPropertiesFile("../manual-test.properties") + if err != nil { + suite.T().Skip("Skipping manual integration tests: " + err.Error()) + } + suite.config = config + suite.db = suite.createDBConnection() + suite.ctrl = suite.createController() +} + +func (suite *ManualITestSuite) TearDownSuite() { + suite.NoError(suite.db.Close()) +} + +func (suite *ManualITestSuite) createController() extensionController.TransactionController { + ctrl, err := extensionController.CreateWithValidatedConfig(suite.createExtensionManagerConfig()) + if err != nil { + suite.FailNow("Error creating controller: " + err.Error()) + } + return ctrl +} + +func (suite *ManualITestSuite) createExtensionManagerConfig() extensionController.ExtensionManagerConfig { + return extensionController.ExtensionManagerConfig{ + ExtensionRegistryURL: suite.getConfigValue("extensionRegistryURL"), + BucketFSBasePath: suite.getConfigValue("bucketFSBasePath"), + ExtensionSchema: suite.getConfigValue("extensionSchema"), + } +} + +func (suite *ManualITestSuite) TestListInstalledExtensions() { + t0 := time.Now() + extensions := suite.getAllExtensions() + suite.GreaterOrEqual(len(extensions), 6, "Expected at least six extension") + log.Infof("Found %d extensions, listing installed extensions...", len(extensions)) + installed := suite.getInstalledExtensions() + suite.GreaterOrEqual(len(installed), 1, "Expected at least one installations") + duration := time.Since(t0) + log.Infof("Total duration %dms: %v", duration.Milliseconds(), installed) + suite.LessOrEqual(duration.Milliseconds(), int64(2000), "Process took too long") +} + +func (suite *ManualITestSuite) getAllExtensions() []*extensionController.Extension { + extensions, err := suite.ctrl.GetAllExtensions(context.Background(), suite.db) + if err != nil { + suite.FailNow("Error getting extensions: " + err.Error()) + } + return extensions +} + +func (suite *ManualITestSuite) getInstalledExtensions() []*extensionAPI.JsExtInstallation { + installed, err := suite.ctrl.GetInstalledExtensions(context.Background(), suite.db) + if err != nil { + suite.FailNow("Error getting installed extensions: " + err.Error()) + } + return installed +} + +func (suite *ManualITestSuite) TestInstallAndUninstallExtension() { + t0 := time.Now() + extensions := suite.getAllExtensions() + ext := extensions[0] + log.Infof("Installing extension %q...", ext.Id) + err := suite.ctrl.InstallExtension(context.Background(), suite.db, ext.Id, ext.InstallableVersions[0].Name) + if err != nil { + suite.FailNow("Error installing extension: " + err.Error()) + } + + err = suite.ctrl.UninstallExtension(context.Background(), suite.db, ext.Id, ext.InstallableVersions[0].Name) + if err != nil { + suite.FailNow("Error uninstalling extension: " + err.Error()) + } + duration := time.Since(t0) + log.Infof("Total duration %dms", duration.Milliseconds()) + suite.LessOrEqual(duration.Milliseconds(), int64(2500), "Process took too long") +} + +func (suite *ManualITestSuite) createDBConnection() *sql.DB { + dbHost := suite.getConfigValue("databaseHost") + log.Debugf("Connecting to database at %q...", dbHost) + db, err := sql.Open("exasol", exasol. + NewConfigWithRefreshToken(suite.getConfigValue("databaseToken")). + Host(dbHost). + Port(8563). + Autocommit(false). + String()) + if err != nil { + suite.FailNow("Error connecting to database: " + err.Error()) + } + suite.testConnection(db) + return db +} + +func (suite *ManualITestSuite) testConnection(db *sql.DB) { + row := db.QueryRow("SELECT 'a'") + var value sql.NullString + err := row.Scan(&value) + if err != nil { + suite.FailNow("Error scanning row: " + err.Error()) + } +} + +func (suite *ManualITestSuite) getConfigValue(key string) string { + value := suite.config[key] + if value == "" { + suite.FailNow(fmt.Sprintf("Key %q not found in config file", key)) + } + return value +} + +type configProperties map[string]string + +// readPropertiesFile reads a Java properties file and returns a map of key-value pairs. +// Based on https://stackoverflow.com/a/46860900 +func readPropertiesFile(filename string) (configProperties, error) { + path, err := filepath.Abs(filename) + if err != nil { + return nil, err + } + config := configProperties{} + file, err := os.Open(path) + if err != nil { + return nil, fmt.Errorf("Error opening file %q: %w", path, err) + } + defer file.Close() + + scanner := bufio.NewScanner(file) + for scanner.Scan() { + line := scanner.Text() + line = strings.TrimSpace(line) + if strings.HasPrefix(line, "#") { + continue + } + if equal := strings.Index(line, "="); equal >= 0 { + if key := strings.TrimSpace(line[:equal]); len(key) > 0 { + value := "" + if len(line) > equal { + value = strings.TrimSpace(line[equal+1:]) + } + config[key] = value + } + } + } + if err := scanner.Err(); err != nil { + return nil, err + } + return config, nil +} diff --git a/doc/changes/changes_0.5.9.md b/doc/changes/changes_0.5.9.md index d1c2ce0e..f77b2632 100644 --- a/doc/changes/changes_0.5.9.md +++ b/doc/changes/changes_0.5.9.md @@ -1,11 +1,14 @@ # Extension Manager 0.5.9, released 2024-??-?? -Code name: +Code name: Speedup listing extensions ## Summary +This release speeds up listing extensions and installations by caching the extension registry. + ## Features +* #169: Enabled caching for http registry * #167: Added script for automatically generating extension registry ## Dependency Updates diff --git a/doc/design.md b/doc/design.md index dc04bdb3..5bc92127 100644 --- a/doc/design.md +++ b/doc/design.md @@ -106,6 +106,20 @@ Covers: Needs: impl, utest, itest +#### Extension Registry Caches Registry Content +`dsn~extension-registry.cache~1` + +EM caches the content returned by the registry. + +Rationale: +* Caching the registry content avoids fetching the same data multiple times and speeds up the process. +* Cache expiration is not necessary because a new instance of the EM controller is created for each request. + +Covers: +* [`req~finding-available-extensions~1`](system_requirements.md#em-finds-available-extensions) + +Needs: impl, itest + ### Extensions The Extension Manager has an extension mechanism. diff --git a/doc/developer_guide.md b/doc/developer_guide.md index 595d9be2..99502fd3 100644 --- a/doc/developer_guide.md +++ b/doc/developer_guide.md @@ -4,7 +4,9 @@ This guide describes how to develop, test and build Extension Manager (EM). If you want to embed EM into another application, see the [embedding guide](./embedding_extension_manager.md). If you want to develop an extension for EM, see the [extension developer guide](./extension_developer_guide.md). -## Building +## Extension Manager + +### Building To build the binary, run @@ -28,7 +30,7 @@ After starting the server you can get the OpenApi definition by executing curl "http://localhost:8080/openapi.json" -o extension-manager-api.json ``` -## Requirement Tracing +### Requirement Tracing You can run requirements tracing by executing: @@ -38,7 +40,7 @@ You can run requirements tracing by executing: If tracing fails with a `org.xml.sax.SAXParseException` you might need to run `mvn clean` before to delete temporary files like `target/site/jacoco/jacoco.xml`. -## Testing +### Testing The extension-manager project contains unit and integration tests that verify * Loading and executing of JavaScript extensions @@ -48,7 +50,7 @@ The extension-manager project contains unit and integration tests that verify Tests use dummy extensions, no real extensions. -### Non-Parallel Tests +#### Non-Parallel Tests The tests of this project use [`exasol-test-setup-abstraction-server`](https://github.com/exasol/exasol-test-setup-abstraction-server/). There the tests connect to an Exasol database running in a docker container. For performance reasons the test-setup-abstraction reuses that container. This feature is not compatible with running tests in parallel. @@ -74,9 +76,29 @@ go test -p 1 -short ./... Please note that also `-short` tests need `-p 1` because extension integration tests share a directory for building a test extension. Tests will fail randomly without `-p 1`. -## Static Code Analysis +#### Manual Integration Tests With SaaS + +Normal integration test EM against an Exasol DB running in a Docker container. There might be differences to a real Exasol SaaS DB. Tests against SaaS are not automated, you need to run them manually: + +1. Create a Personal Access Token in SaaS with scope `databases:use` +2. Create a new schema you want to use for testing +3. Upload required Adapter JARs to BucketFS +4. Create file `manual-test.properties` with the following content: + ```properties + databaseHost = + databaseToken = + extensionRegistryURL= + extensionSchema = + bucketFSBasePath = + ``` +5. Run test with + ```sh + go test -v ./cmd/... + ``` -### Go Linter +### Static Code Analysis + +#### Go Linter To install golangci-lint on your machine, follow [these instruction](https://golangci-lint.run/usage/install/#local-installation). @@ -88,7 +110,7 @@ golangci-lint run File `.golangci.yml` contains configuration like enabled or disabled linters. -### Sonar +#### Sonar Download sonar-scanner as a zip file from [sonarqube.org](https://docs.sonarqube.org/latest/analysis/scan/sonarscanner/) and unpack it. @@ -105,7 +127,7 @@ Then run Sonar with the following command in the project root: sonar-scanner -Dsonar.token=$SONAR_TOKEN ``` -## Using a Local Extension Interface +### Using a Local Extension Interface To use a local, non-published version of the extension interface for testing EM follow these steps: @@ -187,6 +209,7 @@ cd pkg/extensionController/bfs/udf poetry env use 3.8 poetry install ``` + ### Run tests ```sh diff --git a/extension-manager-integration-test-java/src/test/java/com/exasol/extensionmanager/ExampleIT.java b/extension-manager-integration-test-java/src/test/java/com/exasol/extensionmanager/ExampleIT.java index df8e14fe..3bf672a2 100644 --- a/extension-manager-integration-test-java/src/test/java/com/exasol/extensionmanager/ExampleIT.java +++ b/extension-manager-integration-test-java/src/test/java/com/exasol/extensionmanager/ExampleIT.java @@ -36,7 +36,7 @@ class ExampleIT { @BeforeAll static void setup() { // Overwrite default Exasol version - System.setProperty("com.exasol.dockerdb.image", "8.23.1"); + System.setProperty("com.exasol.dockerdb.image", "8.24.0"); exasolTestSetup = new ExasolTestSetupFactory(Path.of("cloud-setup")).getTestSetup(); diff --git a/extension-manager-integration-test-java/src/test/java/com/exasol/extensionmanager/itest/IntegrationTestCommon.java b/extension-manager-integration-test-java/src/test/java/com/exasol/extensionmanager/itest/IntegrationTestCommon.java index aa578f48..f9b28a5d 100644 --- a/extension-manager-integration-test-java/src/test/java/com/exasol/extensionmanager/itest/IntegrationTestCommon.java +++ b/extension-manager-integration-test-java/src/test/java/com/exasol/extensionmanager/itest/IntegrationTestCommon.java @@ -18,7 +18,7 @@ private IntegrationTestCommon() { static ExasolTestSetup createExasolTestSetup() { if (System.getProperty("com.exasol.dockerdb.image") == null) { - System.setProperty("com.exasol.dockerdb.image", "8.23.1"); + System.setProperty("com.exasol.dockerdb.image", "8.24.0"); } return new ExasolTestSetupFactory(Path.of("dummy-config")).getTestSetup(); } diff --git a/pkg/extensionAPI/extensionApi.go b/pkg/extensionAPI/extensionApi.go index dc802a64..141bdcba 100644 --- a/pkg/extensionAPI/extensionApi.go +++ b/pkg/extensionAPI/extensionApi.go @@ -2,6 +2,7 @@ package extensionAPI import ( "fmt" + "time" "github.com/exasol/extension-manager/pkg/extensionAPI/context" "github.com/exasol/extension-manager/pkg/extensionAPI/exaMetadata" @@ -15,6 +16,7 @@ import ( // LoadExtension loads an extension from the given file content. /* [impl -> dsn~extension-definition~1]. */ func LoadExtension(id, content string) (*JsExtension, error) { + t0 := time.Now() logPrefix := fmt.Sprintf("JS:%s>", id) vm := newJavaScriptVm(logPrefix) extensionJs, err := loadExtension(vm, id, content) @@ -26,7 +28,7 @@ func LoadExtension(id, content string) (*JsExtension, error) { return nil, err } wrappedExtension := wrapExtension(&extensionJs.Extension, id, vm) - log.Debugf("Extension %q with id %q using API version %q loaded successfully", wrappedExtension.Name, wrappedExtension.Id, extensionJs.APIVersion) + log.Tracef("Extension %q with id %q using API version %q loaded in %dms", wrappedExtension.Name, wrappedExtension.Id, extensionJs.APIVersion, time.Since(t0).Milliseconds()) return wrappedExtension, nil } diff --git a/pkg/extensionController/bfs/BucketFsApi.go b/pkg/extensionController/bfs/BucketFsApi.go index 3b2144e7..3e19f64e 100644 --- a/pkg/extensionController/bfs/BucketFsApi.go +++ b/pkg/extensionController/bfs/BucketFsApi.go @@ -74,7 +74,12 @@ func (bfs bucketFsAPIImpl) ListFiles() ([]BfsFile, error) { } defer result.Close() files, err := readQueryResult(result) - logrus.Tracef("Listed %d files in %.2fs", len(files), time.Since(t0).Seconds()) + if logrus.IsLevelEnabled(logrus.TraceLevel) { + for _, file := range files { + logrus.Tracef("- Found file %q with size %d", file.Path, file.Size) + } + } + logrus.Debugf("Listed %d files under %q in %dms", len(files), bfs.bucketFsBasePath, time.Since(t0).Milliseconds()) return files, err } @@ -104,7 +109,7 @@ func (bfs bucketFsAPIImpl) FindAbsolutePath(fileName string) (string, error) { if err != nil { return "", fmt.Errorf("failed reading absolute path. Cause: %w", err) } - logrus.Tracef("Found absolute path %q for file %q in %.2fs", absolutePath, fileName, time.Since(t0).Seconds()) + logrus.Tracef("Found absolute path %q for file %q in %dms", absolutePath, fileName, time.Since(t0)) return absolutePath, nil } @@ -127,7 +132,7 @@ func createUdfScript(transaction *sql.Tx) (string, error) { if err != nil { return "", fmt.Errorf("failed to create UDF script for listing bucket. Cause: %w", err) } - logrus.Tracef("Created UDF script %s in %.2f", udfScriptName, time.Since(t0).Seconds()) + logrus.Debugf("Created UDF script %s in %dms", udfScriptName, time.Since(t0).Milliseconds()) return udfScriptName, nil } diff --git a/pkg/extensionController/controller.go b/pkg/extensionController/controller.go index 0b36cbe6..401f705e 100644 --- a/pkg/extensionController/controller.go +++ b/pkg/extensionController/controller.go @@ -3,6 +3,7 @@ package extensionController import ( "fmt" "strings" + "time" log "github.com/sirupsen/logrus" @@ -94,7 +95,7 @@ func convertExtension(jsExtension *extensionAPI.JsExtension) *Extension { func (c *controllerImpl) requiredFilesAvailable(extension *extensionAPI.JsExtension, bfsFiles []bfs.BfsFile) bool { for _, requiredFile := range extension.BucketFsUploads { if !existsFileInBfs(bfsFiles, requiredFile) { - log.Debugf("Ignoring extension %q since the required file %q (%q) does not exist or has a wrong file size.\n", extension.Name, requiredFile.Name, requiredFile.BucketFsFilename) + log.Debugf("Ignoring extension %q since the required file %q does not exist or has a wrong file size.\n", extension.Name, requiredFile.BucketFsFilename) return false } } @@ -107,7 +108,7 @@ func existsFileInBfs(bfsFiles []bfs.BfsFile, requiredFile extensionAPI.BucketFsU return true } } - log.Debugf("Required file %q of size %db not found", requiredFile.Name, requiredFile.FileSize) + log.Tracef("Required file %q of size %db not found", requiredFile.Name, requiredFile.FileSize) return false } @@ -128,6 +129,7 @@ func fileMatches(requiredFile extensionAPI.BucketFsUpload, existingFile bfs.BfsF } func (c *controllerImpl) getAllExtensions() ([]*extensionAPI.JsExtension, error) { + t0 := time.Now() extensionIds, err := c.registry.FindExtensions() if err != nil { return nil, err @@ -140,6 +142,7 @@ func (c *controllerImpl) getAllExtensions() ([]*extensionAPI.JsExtension, error) } extensions = append(extensions, extension) } + log.Debugf("Loaded %d extensions JS files in %dms", len(extensions), time.Since(t0).Milliseconds()) return extensions, nil } diff --git a/pkg/extensionController/registry/http.go b/pkg/extensionController/registry/http.go index 4aea6d80..059aad55 100644 --- a/pkg/extensionController/registry/http.go +++ b/pkg/extensionController/registry/http.go @@ -6,12 +6,15 @@ import ( "io" "net/http" "strings" + "time" "github.com/exasol/extension-manager/pkg/apiErrors" "github.com/exasol/extension-manager/pkg/extensionController/registry/index" + log "github.com/sirupsen/logrus" ) func newHttpRegistry(url string) Registry { + log.Debugf("Creating HTTP registry for %q", url) return &httpRegistry{url: url, index: nil} } @@ -23,25 +26,38 @@ type httpRegistry struct { /* [impl -> dsn~extension-registry~1] */ /* [impl -> dsn~extension-definitions-storage~1]. */ func (h *httpRegistry) FindExtensions() ([]string, error) { - err := h.loadIndex() + index, err := h.getIndex() if err != nil { return nil, err } - return h.index.GetExtensionIDs(), nil + return index.GetExtensionIDs(), nil } -func (h *httpRegistry) loadIndex() error { - response, err := getResponse(h.url) +/* [impl -> dsn~extension-registry.cache~1]. */ +func (h *httpRegistry) getIndex() (*index.RegistryIndex, error) { + if h.index == nil { + index, err := loadIndex(h.url) + if err != nil { + return nil, err + } + h.index = index + } + return h.index, nil +} + +func loadIndex(url string) (*index.RegistryIndex, error) { + t0 := time.Now() + response, err := getResponse(url) if err != nil { - return err + return nil, fmt.Errorf("failed to load index from %q: %w", url, err) } defer response.Body.Close() index, err := index.Decode(response.Body) if err != nil { - return err + return nil, fmt.Errorf("failed to decode index from %q: %w", url, err) } - h.index = &index - return nil + log.Debugf("Loaded registry index with %d extensions from %q in %dms", len(index.Extensions), url, time.Since(t0).Milliseconds()) + return &index, nil } func getResponse(url string) (*http.Response, error) { @@ -61,14 +77,15 @@ func getResponse(url string) (*http.Response, error) { } func (h *httpRegistry) ReadExtension(id string) (string, error) { - err := h.loadIndex() + index, err := h.getIndex() if err != nil { return "", err } - ext, ok := h.index.GetExtension(id) + ext, ok := index.GetExtension(id) if !ok { return "", apiErrors.NewNotFoundErrorF("extension %q not found", id) } + extContent, err := getUrlContent(ext.URL) if err != nil { return "", fmt.Errorf("failed to load extension %q: %w", id, err) diff --git a/pkg/extensionController/registry/http_test.go b/pkg/extensionController/registry/http_test.go index f1bba653..3c305aa6 100644 --- a/pkg/extensionController/registry/http_test.go +++ b/pkg/extensionController/registry/http_test.go @@ -1,6 +1,7 @@ package registry import ( + "fmt" "testing" "github.com/exasol/extension-manager/pkg/integrationTesting" @@ -31,7 +32,7 @@ func (suite *HttpRegistrySuite) SetupTest() { suite.server.Reset() } -func (suite *HttpRegistrySuite) TestFindExtensions_noExtensionsAvailable() { +func (suite *HttpRegistrySuite) TestFindExtensionsNoExtensionsAvailable() { suite.server.SetRegistryContent(`{}`) extensions, err := suite.registry.FindExtensions() suite.Require().NoError(err) @@ -42,15 +43,28 @@ func (suite *HttpRegistrySuite) TestFindExtensions_noExtensionsAvailable() { /* [itest -> dsn~extension-definitions-storage~1]. */ func (suite *HttpRegistrySuite) TestFindExtensions() { suite.server.SetRegistryContent(`{"extensions":[{"id": "ext1"},{"id": "ext2"},{"id": "ext3"}]}`) + suite.assertExtensions([]string{"ext1", "ext2", "ext3"}) +} + +/* [itest -> dsn~extension-registry.cache~1]. */ +func (suite *HttpRegistrySuite) TestFindExtensionsCachesContent() { + suite.server.SetRegistryContent(`{"extensions":[{"id": "ext1"},{"id": "ext2"},{"id": "ext3"}]}`) + suite.assertExtensions([]string{"ext1", "ext2", "ext3"}) + + suite.server.SetRegistryContent(`invalid content`) + suite.assertExtensions([]string{"ext1", "ext2", "ext3"}) +} + +func (suite *HttpRegistrySuite) assertExtensions(expectedExtensions []string) { extensions, err := suite.registry.FindExtensions() suite.Require().NoError(err) - suite.Equal([]string{"ext1", "ext2", "ext3"}, extensions) + suite.Equal(expectedExtensions, extensions) } func (suite *HttpRegistrySuite) TestReadExtensionFailsWhenLoadingIndex() { suite.server.SetRegistryContent(`invalid`) content, err := suite.registry.ReadExtension("unknown-ext-id") - suite.Require().EqualError(err, "failed to decode registry content: invalid character 'i' looking for beginning of value") + suite.Require().EqualError(err, fmt.Sprintf(`failed to decode index from "%s": failed to decode registry content: invalid character 'i' looking for beginning of value`, suite.server.IndexUrl())) suite.Equal("", content) } diff --git a/pkg/extensionController/registry/localDir.go b/pkg/extensionController/registry/localDir.go index f638115a..75c3af43 100644 --- a/pkg/extensionController/registry/localDir.go +++ b/pkg/extensionController/registry/localDir.go @@ -9,9 +9,11 @@ import ( "strings" "github.com/exasol/extension-manager/pkg/apiErrors" + log "github.com/sirupsen/logrus" ) func newLocalDirRegistry(dir string) Registry { + log.Debugf("Creating local directory registry using dir %q", dir) return &localDirRegistry{dir: dir} } diff --git a/pkg/extensionController/transactionController.go b/pkg/extensionController/transactionController.go index 31c1b1e6..be39d9f8 100644 --- a/pkg/extensionController/transactionController.go +++ b/pkg/extensionController/transactionController.go @@ -5,11 +5,13 @@ import ( "database/sql" "errors" "fmt" + "time" "github.com/exasol/extension-manager/pkg/extensionAPI" "github.com/exasol/extension-manager/pkg/extensionController/bfs" "github.com/exasol/extension-manager/pkg/extensionController/transaction" "github.com/exasol/extension-manager/pkg/parameterValidator" + log "github.com/sirupsen/logrus" ) // TransactionController is the core part of the extension-manager that provides the extension handling functionality. @@ -138,11 +140,14 @@ type transactionControllerImpl struct { } func (c *transactionControllerImpl) GetAllExtensions(ctx context.Context, db *sql.DB) ([]*Extension, error) { + t0 := time.Now() bfsFiles, err := c.listBfsFiles(ctx, db) if err != nil { return nil, err } - return c.controller.GetAllExtensions(bfsFiles) + extensions, err := c.controller.GetAllExtensions(bfsFiles) + log.Debugf("Found %d extensions in %dms (%d files in BucketFS)", len(extensions), time.Since(t0).Milliseconds(), len(bfsFiles)) + return extensions, err } func (c *transactionControllerImpl) listBfsFiles(ctx context.Context, db *sql.DB) ([]bfs.BfsFile, error) { @@ -212,12 +217,15 @@ func (c *transactionControllerImpl) UpgradeExtension(ctx context.Context, db *sq } func (c *transactionControllerImpl) GetInstalledExtensions(ctx context.Context, db *sql.DB) ([]*extensionAPI.JsExtInstallation, error) { + t0 := time.Now() tx, err := c.beginTransaction(ctx, db) if err != nil { return nil, err } defer tx.Rollback() - return c.controller.GetAllInstallations(tx) + installations, err := c.controller.GetAllInstallations(tx) + log.Debugf("Found %d installed extensions in %dms", len(installations), time.Since(t0).Milliseconds()) + return installations, err } func (c *transactionControllerImpl) GetParameterDefinitions(ctx context.Context, db *sql.DB, extensionId string, extensionVersion string) ([]parameterValidator.ParameterDefinition, error) { diff --git a/pkg/integrationTesting/dbTestSetup.go b/pkg/integrationTesting/dbTestSetup.go index af868405..6c01c463 100644 --- a/pkg/integrationTesting/dbTestSetup.go +++ b/pkg/integrationTesting/dbTestSetup.go @@ -9,7 +9,7 @@ import ( "github.com/stretchr/testify/suite" ) -const defaultExasolDbVersion = "8.23.1" +const defaultExasolDbVersion = "8.24.0" type DbTestSetup struct { suite *suite.Suite diff --git a/sonar-project.properties b/sonar-project.properties index ec4792db..bb1223f7 100644 --- a/sonar-project.properties +++ b/sonar-project.properties @@ -14,3 +14,4 @@ sonar.coverage.jacoco.xmlReportPaths=extension-manager-integration-test-java/tar sonar.javascript.lcov.reportPaths=registry/coverage/lcov.info sonar.python.coverage.reportPaths=pkg/extensionController/bfs/udf/coverage.xml sonar.java.source=11 +sonar.python.version=3.8