Skip to content

Commit

Permalink
#169: Add caching for HTTP registry (#170)
Browse files Browse the repository at this point in the history
  • Loading branch information
kaklakariada authored Mar 27, 2024
1 parent c8d7629 commit 465fc11
Show file tree
Hide file tree
Showing 23 changed files with 315 additions and 102 deletions.
10 changes: 0 additions & 10 deletions .github/workflows/ci-build-next-java.yml
Original file line number Diff line number Diff line change
Expand Up @@ -27,16 +27,6 @@ jobs:
distribution: "temurin"
java-version: 17
cache: "maven"
- name: Cache go modules
uses: actions/cache@v4
with:
path: |
~/.cache/go-build
~/go/pkg/mod
key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
restore-keys: |
${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
${{ runner.os }}-go-

- name: Run tests and build with Maven
run: |
Expand Down
13 changes: 1 addition & 12 deletions .github/workflows/ci-build.yml
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ jobs:
group: ${{ github.workflow }}-${{ github.ref }}-build
cancel-in-progress: true
env:
EXASOL_VERSION: "8.23.1"
EXASOL_VERSION: "8.24.0"
steps:
- name: Check out code
uses: actions/checkout@v4
Expand All @@ -35,17 +35,6 @@ jobs:
17
cache: "maven"

- name: Cache go modules
uses: actions/cache@v4
with:
path: |
~/.cache/go-build
~/go/pkg/mod
key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
restore-keys: |
${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
${{ runner.os }}-go-
- name: Get npm cache directory
id: npm-cache-dir
run: echo "dir=$(npm config get cache)" >> $GITHUB_OUTPUT
Expand Down
10 changes: 0 additions & 10 deletions .github/workflows/release_droid_prepare_original_checksum.yml
Original file line number Diff line number Diff line change
Expand Up @@ -24,16 +24,6 @@ jobs:
with:
go-version: "1.21"
id: go
- name: Cache go modules
uses: actions/cache@v4
with:
path: |
~/.cache/go-build
~/go/pkg/mod
key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
restore-keys: |
${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
${{ runner.os }}-go-

- name: Prepare testing extension
run: cd ./extension-manager-integration-test-java/testing-extension && npm ci
Expand Down
10 changes: 0 additions & 10 deletions .github/workflows/release_droid_print_quick_checksum.yml
Original file line number Diff line number Diff line change
Expand Up @@ -24,16 +24,6 @@ jobs:
with:
go-version: "1.21"
id: go
- name: Cache go modules
uses: actions/cache@v4
with:
path: |
~/.cache/go-build
~/go/pkg/mod
key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
restore-keys: |
${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
${{ runner.os }}-go-

- name: Build with Maven skipping tests
run: mvn --batch-mode clean verify -DskipTests
Expand Down
10 changes: 0 additions & 10 deletions .github/workflows/release_droid_release_on_maven_central.yml
Original file line number Diff line number Diff line change
Expand Up @@ -30,16 +30,6 @@ jobs:
with:
go-version: "1.21"
id: go
- name: Cache go modules
uses: actions/cache@v4
with:
path: |
~/.cache/go-build
~/go/pkg/mod
key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
restore-keys: |
${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
${{ runner.os }}-go-

- name: Publish to Central Repository
run: mvn --batch-mode -Dgpg.skip=false -DskipTests clean deploy
Expand Down
10 changes: 0 additions & 10 deletions .github/workflows/release_droid_upload_github_release_assets.yml
Original file line number Diff line number Diff line change
Expand Up @@ -28,16 +28,6 @@ jobs:
with:
go-version: "1.21"
id: go
- name: Cache go modules
uses: actions/cache@v4
with:
path: |
~/.cache/go-build
~/go/pkg/mod
key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
restore-keys: |
${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
${{ runner.os }}-go-

- name: Prepare testing extension
run: |
Expand Down
2 changes: 2 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,8 @@ vendor/
/doc/tracing-report.md
/doc/tracing-report.html

/manual-test.properties

!/pkg/extensionController/bfs/udf/poetry.lock
/pkg/extensionController/bfs/udf/__pycache__
/pkg/extensionController/bfs/udf/.pytest_cache
Expand Down
8 changes: 1 addition & 7 deletions cmd/main.go
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
package main

import (
"bytes"
"errors"
"flag"
"fmt"
Expand Down Expand Up @@ -99,10 +98,5 @@ type simpleFormatter struct {
}

func (f *simpleFormatter) Format(entry *log.Entry) ([]byte, error) {
b := &bytes.Buffer{}
b.WriteString(strings.ToUpper(entry.Level.String()))
b.WriteByte(' ')
b.WriteString(entry.Message)
b.WriteByte('\n')
return b.Bytes(), nil
return []byte(fmt.Sprintf("%-7s %s\n", strings.ToUpper(entry.Level.String()), entry.Message)), nil
}
186 changes: 186 additions & 0 deletions cmd/manual_i_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,186 @@
package main

import (
"bufio"
"context"
"database/sql"
"fmt"
"os"
"path/filepath"
"strings"
"testing"
"time"

"github.com/exasol/exasol-driver-go"
"github.com/exasol/extension-manager/pkg/extensionAPI"
"github.com/exasol/extension-manager/pkg/extensionController"

log "github.com/sirupsen/logrus"

"github.com/stretchr/testify/suite"
)

// Create file manual-test.properties, add configuration and
// run this test with `go test -v ./cmd/...`

type ManualITestSuite struct {
suite.Suite
config configProperties
db *sql.DB
ctrl extensionController.TransactionController // Add this line
}

func TestManualITestSuite(t *testing.T) {
suite.Run(t, new(ManualITestSuite))
}

func (suite *ManualITestSuite) SetupSuite() {
log.SetLevel(log.DebugLevel)
log.SetFormatter(new(simpleFormatter))
config, err := readPropertiesFile("../manual-test.properties")
if err != nil {
suite.T().Skip("Skipping manual integration tests: " + err.Error())
}
suite.config = config
suite.db = suite.createDBConnection()
suite.ctrl = suite.createController()
}

func (suite *ManualITestSuite) TearDownSuite() {
suite.NoError(suite.db.Close())
}

func (suite *ManualITestSuite) createController() extensionController.TransactionController {
ctrl, err := extensionController.CreateWithValidatedConfig(suite.createExtensionManagerConfig())
if err != nil {
suite.FailNow("Error creating controller: " + err.Error())
}
return ctrl
}

func (suite *ManualITestSuite) createExtensionManagerConfig() extensionController.ExtensionManagerConfig {
return extensionController.ExtensionManagerConfig{
ExtensionRegistryURL: suite.getConfigValue("extensionRegistryURL"),
BucketFSBasePath: suite.getConfigValue("bucketFSBasePath"),
ExtensionSchema: suite.getConfigValue("extensionSchema"),
}
}

func (suite *ManualITestSuite) TestListInstalledExtensions() {
t0 := time.Now()
extensions := suite.getAllExtensions()
suite.GreaterOrEqual(len(extensions), 6, "Expected at least six extension")
log.Infof("Found %d extensions, listing installed extensions...", len(extensions))
installed := suite.getInstalledExtensions()
suite.GreaterOrEqual(len(installed), 1, "Expected at least one installations")
duration := time.Since(t0)
log.Infof("Total duration %dms: %v", duration.Milliseconds(), installed)
suite.LessOrEqual(duration.Milliseconds(), int64(2000), "Process took too long")
}

func (suite *ManualITestSuite) getAllExtensions() []*extensionController.Extension {
extensions, err := suite.ctrl.GetAllExtensions(context.Background(), suite.db)
if err != nil {
suite.FailNow("Error getting extensions: " + err.Error())
}
return extensions
}

func (suite *ManualITestSuite) getInstalledExtensions() []*extensionAPI.JsExtInstallation {
installed, err := suite.ctrl.GetInstalledExtensions(context.Background(), suite.db)
if err != nil {
suite.FailNow("Error getting installed extensions: " + err.Error())
}
return installed
}

func (suite *ManualITestSuite) TestInstallAndUninstallExtension() {
t0 := time.Now()
extensions := suite.getAllExtensions()
ext := extensions[0]
log.Infof("Installing extension %q...", ext.Id)
err := suite.ctrl.InstallExtension(context.Background(), suite.db, ext.Id, ext.InstallableVersions[0].Name)
if err != nil {
suite.FailNow("Error installing extension: " + err.Error())
}

err = suite.ctrl.UninstallExtension(context.Background(), suite.db, ext.Id, ext.InstallableVersions[0].Name)
if err != nil {
suite.FailNow("Error uninstalling extension: " + err.Error())
}
duration := time.Since(t0)
log.Infof("Total duration %dms", duration.Milliseconds())
suite.LessOrEqual(duration.Milliseconds(), int64(2500), "Process took too long")
}

func (suite *ManualITestSuite) createDBConnection() *sql.DB {
dbHost := suite.getConfigValue("databaseHost")
log.Debugf("Connecting to database at %q...", dbHost)
db, err := sql.Open("exasol", exasol.
NewConfigWithRefreshToken(suite.getConfigValue("databaseToken")).
Host(dbHost).
Port(8563).
Autocommit(false).
String())
if err != nil {
suite.FailNow("Error connecting to database: " + err.Error())
}
suite.testConnection(db)
return db
}

func (suite *ManualITestSuite) testConnection(db *sql.DB) {
row := db.QueryRow("SELECT 'a'")
var value sql.NullString
err := row.Scan(&value)
if err != nil {
suite.FailNow("Error scanning row: " + err.Error())
}
}

func (suite *ManualITestSuite) getConfigValue(key string) string {
value := suite.config[key]
if value == "" {
suite.FailNow(fmt.Sprintf("Key %q not found in config file", key))
}
return value
}

type configProperties map[string]string

// readPropertiesFile reads a Java properties file and returns a map of key-value pairs.
// Based on https://stackoverflow.com/a/46860900
func readPropertiesFile(filename string) (configProperties, error) {
path, err := filepath.Abs(filename)
if err != nil {
return nil, err
}
config := configProperties{}
file, err := os.Open(path)
if err != nil {
return nil, fmt.Errorf("Error opening file %q: %w", path, err)
}
defer file.Close()

scanner := bufio.NewScanner(file)
for scanner.Scan() {
line := scanner.Text()
line = strings.TrimSpace(line)
if strings.HasPrefix(line, "#") {
continue
}
if equal := strings.Index(line, "="); equal >= 0 {
if key := strings.TrimSpace(line[:equal]); len(key) > 0 {
value := ""
if len(line) > equal {
value = strings.TrimSpace(line[equal+1:])
}
config[key] = value
}
}
}
if err := scanner.Err(); err != nil {
return nil, err
}
return config, nil
}
5 changes: 4 additions & 1 deletion doc/changes/changes_0.5.9.md
Original file line number Diff line number Diff line change
@@ -1,11 +1,14 @@
# Extension Manager 0.5.9, released 2024-??-??

Code name:
Code name: Speedup listing extensions

## Summary

This release speeds up listing extensions and installations by caching the extension registry.

## Features

* #169: Enabled caching for http registry
* #167: Added script for automatically generating extension registry

## Dependency Updates
Expand Down
14 changes: 14 additions & 0 deletions doc/design.md
Original file line number Diff line number Diff line change
Expand Up @@ -106,6 +106,20 @@ Covers:

Needs: impl, utest, itest

#### Extension Registry Caches Registry Content
`dsn~extension-registry.cache~1`

EM caches the content returned by the registry.

Rationale:
* Caching the registry content avoids fetching the same data multiple times and speeds up the process.
* Cache expiration is not necessary because a new instance of the EM controller is created for each request.

Covers:
* [`req~finding-available-extensions~1`](system_requirements.md#em-finds-available-extensions)

Needs: impl, itest

### Extensions

The Extension Manager has an extension mechanism.
Expand Down
Loading

0 comments on commit 465fc11

Please sign in to comment.