#161: Adapt for JDBC based extensions

.project-keeper.yml

@@ -26,7 +26,7 @@ sources:
     path: registry-upload/package.json
   - type: npm
     path: pkg/parameterValidator/package.json
-version: 0.5.5
+version: 0.5.6
 linkReplacements:
   - "Unknown|https://github.com/DATA-DOG/go-sqlmock/blob/master/LICENSE"
   - "https://github.com/swagger-api/swagger-core/modules/swagger-annotations|https://github.com/swagger-api/swagger-core/tree/master/modules/swagger-annotations"

doc/changes/changes_0.5.6.md

@@ -0,0 +1,23 @@
+# Extension Manager 0.5.6, released 2023-11-09
+
+Code name: Adapt for JDBC based virtual schema extensions
+
+## Summary
+
+This release adapts Extension Manager integration test framework for JDBC based extensions:
+* Skip `upgradeFromPreviousVersion()` when no previous version is available
+* Update `createInstanceWithSingleQuote()` to accept any adapter name, not just the hard-coded S3 adapter
+
+The release also allows extensions to ignore the file size of required BucketFS files by setting a negative `fileSize`. This is useful for JDBC drivers where we don't know the version and file size beforehand and want to allow arbitrary versions and file sizes.
+
+## Features
+
+* #161: Adapted shared tests for JDBC based extensions
+
+## Dependency Updates
+
+### Extension Integration Tests Library
+
+#### Compile Dependency Updates
+
+* Updated `com.exasol:extension-manager-client-java:0.5.5` to `0.5.6`

extension-manager-integration-test-java/src/main/java/com/exasol/extensionmanager/itest/base/AbstractScriptExtensionIT.java

@@ -5,6 +5,7 @@
 import static org.hamcrest.Matchers.*;
 import static org.junit.jupiter.api.Assertions.assertAll;
 import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
+import static org.junit.jupiter.api.Assumptions.assumeTrue;
 
 import java.util.List;
 import java.util.logging.Logger;
@@ -232,6 +233,7 @@ public void upgradeFailsWhenAlreadyUpToDate() {
      */
     @Test
     void upgradeFromPreviousVersion() {
+        assumeTrue(config.getPreviousVersion() != null, "No previous version available for testing");
         final PreviousExtensionVersion previousVersion = createPreviousVersion();
         previousVersion.prepare();
         previousVersion.install();

extension-manager-integration-test-java/src/main/java/com/exasol/extensionmanager/itest/base/AbstractVirtualSchemaExtensionIT.java

@@ -6,6 +6,7 @@
 import static org.hamcrest.Matchers.*;
 import static org.junit.jupiter.api.Assertions.assertAll;
 import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
+import static org.junit.jupiter.api.Assumptions.assumeTrue;
 
 import java.util.*;
 import java.util.logging.Logger;
@@ -247,6 +248,7 @@ public void upgradeFailsWhenAlreadyUpToDate() {
      */
     @Test
     public void upgradeFromPreviousVersion() {
+        assumeTrue(config.getPreviousVersion() != null, "No previous version available for testing");
         final PreviousExtensionVersion previousVersion = createPreviousVersion();
         previousVersion.prepare();
         previousVersion.install();
@@ -374,9 +376,8 @@ public void createInstanceWithSingleQuote() {
         assertAll(
                 () -> getSetup().exasolMetadata().assertConnection(
                         table().row("QUOTED'SCHEMA_CONNECTION", getInstanceComment(virtualSchemaName)).matches()),
-                () -> getSetup().exasolMetadata().assertVirtualSchema(table()
-                        .row(virtualSchemaName, "SYS", EXTENSION_SCHEMA, "S3_FILES_ADAPTER", not(emptyOrNullString()))
-                        .matches()));
+                () -> getSetup().exasolMetadata().assertVirtualSchema(table().row(virtualSchemaName, "SYS",
+                        EXTENSION_SCHEMA, not(emptyOrNullString()), not(emptyOrNullString())).matches()));
     }
 
     /**

pkg/extensionAPI/extensionApi.go

@@ -101,11 +101,11 @@ type rawJsExtensionVersion struct {
 }
 
 type BucketFsUpload struct {
-	Name             string `json:"name"`
-	DownloadURL      string `json:"downloadUrl"`
-	LicenseURL       string `json:"licenseUrl"`
-	FileSize         int    `json:"fileSize"`
-	BucketFsFilename string `json:"bucketFsFilename"`
+	Name             string `json:"name"`             // Human-readable name or short description of the file
+	DownloadURL      string `json:"downloadUrl"`      // Optional
+	LicenseURL       string `json:"licenseUrl"`       // Optional
+	FileSize         int    `json:"fileSize"`         // File size in bytes. Negative if EM should ignore the file size
+	BucketFsFilename string `json:"bucketFsFilename"` // File name in BucketFS
 }
 
 type JsExtInstallation struct {

pkg/extensionAPI/extensionApi_test.go

@@ -47,6 +47,41 @@ func (suite *ExtensionApiSuite) TestLoadExtension() {
 	extension := suite.loadExtension(extensionContent)
 	suite.Equal("MyDemoExtension", extension.Name)
 	suite.Equal("Demo category", extension.Category)
+	suite.Equal([]JsExtensionVersion{{Name: "0.1.0", Latest: true, Deprecated: false}}, extension.InstallableVersions)
+	suite.Empty(extension.BucketFsUploads)
+}
+
+func (suite *ExtensionApiSuite) TestNoBucketFsUploads() {
+	extensionContent := integrationTesting.CreateTestExtensionBuilder(suite.T()).
+		WithRawBucketFsUpload("[]").
+		Build().AsString()
+	extension := suite.loadExtension(extensionContent)
+	suite.Empty(extension.BucketFsUploads)
+}
+
+func (suite *ExtensionApiSuite) TestSingleBucketFsUploads() {
+	extensionContent := integrationTesting.CreateTestExtensionBuilder(suite.T()).
+		WithRawBucketFsUpload(`[{name:"name",downloadUrl:"url",licenseUrl:"license",fileSize:123,bucketFsFilename:"filename"}]`).
+		Build().AsString()
+	extension := suite.loadExtension(extensionContent)
+	suite.Equal([]BucketFsUpload{{Name: "name", DownloadURL: "url", LicenseURL: "license", FileSize: 123, BucketFsFilename: "filename"}}, extension.BucketFsUploads)
+}
+
+func (suite *ExtensionApiSuite) TestBucketFsUploadsWithMissingValues() {
+	extensionContent := integrationTesting.CreateTestExtensionBuilder(suite.T()).
+		WithRawBucketFsUpload(`[{}]`).
+		Build().AsString()
+	extension := suite.loadExtension(extensionContent)
+	suite.Equal([]BucketFsUpload{{Name: "", DownloadURL: "", LicenseURL: "", FileSize: 0, BucketFsFilename: ""}}, extension.BucketFsUploads)
+}
+
+func (suite *ExtensionApiSuite) TestTwoBucketFsUploadsWithMissingValues() {
+	extensionContent := integrationTesting.CreateTestExtensionBuilder(suite.T()).
+		WithRawBucketFsUpload(`[{}, {}]`).
+		Build().AsString()
+	extension := suite.loadExtension(extensionContent)
+	suite.Equal([]BucketFsUpload{{Name: "", DownloadURL: "", LicenseURL: "", FileSize: 0, BucketFsFilename: ""},
+		{Name: "", DownloadURL: "", LicenseURL: "", FileSize: 0, BucketFsFilename: ""}}, extension.BucketFsUploads)
 }
 
 func (suite *ExtensionApiSuite) TestGetParameterDefinitionsEmptyResult() {

pkg/extensionController/controller.go

@@ -103,13 +103,27 @@ func (c *controllerImpl) requiredFilesAvailable(extension *extensionAPI.JsExtens
 
 func existsFileInBfs(bfsFiles []bfs.BfsFile, requiredFile extensionAPI.BucketFsUpload) bool {
 	for _, existingFile := range bfsFiles {
-		if requiredFile.BucketFsFilename == existingFile.Name && requiredFile.FileSize == existingFile.Size {
+		if fileMatches(requiredFile, existingFile) {
 			return true
 		}
-		if requiredFile.BucketFsFilename == existingFile.Name {
-			log.Debugf("File %q exists but has wrong size %d, expected %d bytes", existingFile.Name, existingFile.Size, requiredFile.FileSize)
-		}
 	}
+	log.Debugf("Required file %q of size %db not found", requiredFile.Name, requiredFile.FileSize)
+	return false
+}
+
+func fileMatches(requiredFile extensionAPI.BucketFsUpload, existingFile bfs.BfsFile) bool {
+	if requiredFile.BucketFsFilename != existingFile.Name {
+		return false
+	}
+	if requiredFile.FileSize < 0 {
+		log.Tracef("Found required file %q of size %db ignoring file size", existingFile.Name, existingFile.Size)
+		return true
+	}
+	if requiredFile.FileSize == existingFile.Size {
+		log.Tracef("Found required file %q of size %db", existingFile.Name, existingFile.Size)
+		return true
+	}
+	log.Debugf("File %q exists but has wrong size %d, expected %d bytes", existingFile.Name, existingFile.Size, requiredFile.FileSize)
 	return false
 }
 

pkg/extensionController/controller_test.go

@@ -102,6 +102,7 @@ func (suite *ControllerUTestSuite) TestGetAllExtensionsAndBucketFSContainsJar()
 	suite.Equal([]*Extension{{Name: "MyDemoExtension", Id: "testing-extension.js", Category: "Demo category", Description: "An extension for testing.",
 		InstallableVersions: []extensionAPI.JsExtensionVersion{{Name: "0.1.0", Latest: true, Deprecated: false}}}}, extensions)
 }
+
 func (suite *ControllerUTestSuite) TestGetAllExtensionsFailsStartingTransaction() {
 	suite.simulateTransactionBeginFails(mockError)
 	extensions, err := suite.controller.GetAllExtensions(mockContext(), suite.db)
@@ -121,6 +122,43 @@ func (suite *ControllerUTestSuite) TestGetAllExtensionsButNoJarInBucketFS() {
 	suite.Empty(extensions)
 }
 
+func (suite *ControllerUTestSuite) TestGetAllExtensionsWrongFileSize() {
+	suite.registerDefaultExtensionDefinition()
+	suite.dbMock.ExpectBegin()
+	suite.bucketFsMock.SimulateFiles([]bfs.BfsFile{{Name: "my-extension.1.2.3.jar", Size: 5, Path: "path"}})
+	suite.bucketFsMock.SimulateCloseSuccess()
+	suite.dbMock.ExpectRollback()
+	extensions, err := suite.controller.GetAllExtensions(mockContext(), suite.db)
+	suite.NoError(err)
+	suite.Empty(extensions)
+}
+
+func (suite *ControllerUTestSuite) TestGetAllExtensionsWrongFileName() {
+	suite.registerDefaultExtensionDefinition()
+	suite.dbMock.ExpectBegin()
+	suite.bucketFsMock.SimulateFiles([]bfs.BfsFile{{Name: "wrong.jar", Size: 3, Path: "path"}})
+	suite.bucketFsMock.SimulateCloseSuccess()
+	suite.dbMock.ExpectRollback()
+	extensions, err := suite.controller.GetAllExtensions(mockContext(), suite.db)
+	suite.NoError(err)
+	suite.Empty(extensions)
+}
+
+func (suite *ControllerUTestSuite) TestGetAllExtensionsIgnoresFileSizeForNegativeValue() {
+	integrationTesting.CreateTestExtensionBuilder(suite.T()).
+		WithBucketFsUpload(integrationTesting.BucketFsUploadParams{Name: "extension jar", BucketFsFilename: "my-extension.1.2.3.jar", FileSize: -1, DownloadUrl: "", LicenseUrl: "", LicenseAgreementRequired: false}).
+		Build().
+		WriteToFile(path.Join(suite.tempExtensionRepo, EXTENSION_ID))
+	suite.dbMock.ExpectBegin()
+	suite.bucketFsMock.SimulateFiles([]bfs.BfsFile{{Name: "my-extension.1.2.3.jar", Size: 3, Path: "path"}})
+	suite.bucketFsMock.SimulateCloseSuccess()
+	suite.dbMock.ExpectRollback()
+	extensions, err := suite.controller.GetAllExtensions(mockContext(), suite.db)
+	suite.NoError(err)
+	suite.Equal([]*Extension{{Name: "MyDemoExtension", Id: "testing-extension.js", Category: "Demo category", Description: "An extension for testing.",
+		InstallableVersions: []extensionAPI.JsExtensionVersion{{Name: "0.1.0", Latest: true, Deprecated: false}}}}, extensions)
+}
+
 func (suite *ControllerUTestSuite) TestGetAllExtensionsFailsForInvalidExtension() {
 	suite.writeFile("broken-extension.js", "invalid javascript")
 	suite.dbMock.ExpectBegin()

pkg/integrationTesting/testExtensionBuilder.go

@@ -30,13 +30,15 @@ func CreateTestExtensionBuilder(t *testing.T) *TestExtensionBuilder {
 		deleteInstanceFunc:                  "context.sqlClient.execute(`drop instance ${instanceId}`)",
 		getInstanceParameterDefinitionsFunc: "return []",
 		bucketFsUploads:                     []BucketFsUploadParams{},
+		rawBucketFsUploads:                  "",
 	}
 	return &builder
 }
 
 type TestExtensionBuilder struct {
 	testing                             *testing.T
 	extensionApiVersion                 string
+	rawBucketFsUploads                  string
 	bucketFsUploads                     []BucketFsUploadParams
 	findInstallationsFunc               string
 	installFunc                         string
@@ -110,6 +112,11 @@ func (builder *TestExtensionBuilder) WithBucketFsUpload(upload BucketFsUploadPar
 	return builder
 }
 
+func (builder *TestExtensionBuilder) WithRawBucketFsUpload(uploads string) *TestExtensionBuilder {
+	builder.rawBucketFsUploads = uploads
+	return builder
+}
+
 func (builder TestExtensionBuilder) Build() *BuiltExtension {
 	workDir := builder.createWorkDir()
 	err := os.WriteFile(path.Join(workDir, "package.json"), []byte(builder.createPackageJsonContent()), 0600)
@@ -146,11 +153,7 @@ func (builder TestExtensionBuilder) createPackageJsonContent() string {
 var extensionTemplate string
 
 func (builder TestExtensionBuilder) createExtensionTsContent() string {
-	bfsUploadsJson, err := json.Marshal(builder.bucketFsUploads)
-	if err != nil {
-		panic(err)
-	}
-	content := strings.Replace(extensionTemplate, "$UPLOADS$", string(bfsUploadsJson), 1)
+	content := strings.Replace(extensionTemplate, "$UPLOADS$", builder.getBucketFsUpload(), 1)
 	content = strings.Replace(content, "$FIND_INSTALLATIONS$", builder.findInstallationsFunc, 1)
 	content = strings.Replace(content, "$INSTALL_EXTENSION$", builder.installFunc, 1)
 	content = strings.Replace(content, "$UNINSTALL_EXTENSION$", builder.uninstallFunc, 1)
@@ -162,6 +165,17 @@ func (builder TestExtensionBuilder) createExtensionTsContent() string {
 	return content
 }
 
+func (builder TestExtensionBuilder) getBucketFsUpload() string {
+	if builder.rawBucketFsUploads != "" {
+		return builder.rawBucketFsUploads
+	}
+	json, err := json.Marshal(builder.bucketFsUploads)
+	if err != nil {
+		panic(err)
+	}
+	return string(json)
+}
+
 type BuiltExtension struct {
 	testing *testing.T
 	content []byte

pom.xml

@@ -9,7 +9,7 @@
     <packaging>pom</packaging>
     <url>https://github.com/exasol/extension-manager/</url>
     <properties>
-        <revision>0.5.5</revision>
+        <revision>0.5.6</revision>
         <junit.version>5.10.1</junit.version>
     </properties>
     <modules>