#174 Fix reading NULL values from metadata tables

.github/workflows/ci-build.yml

@@ -13,7 +13,7 @@ jobs:
       group: ${{ github.workflow }}-${{ github.ref }}-build
       cancel-in-progress: true
     env:
-      EXASOL_VERSION: "8.24.0"
+      EXASOL_VERSION: "8.26.0"
     steps:
       - name: Check out code
         uses: actions/checkout@v4

.project-keeper.yml

@@ -26,7 +26,7 @@ sources:
     path: registry-upload/package.json
   - type: npm
     path: pkg/parameterValidator/package.json
-version: 0.5.9
+version: 0.5.10
 linkReplacements:
   - "Unknown|https://github.com/DATA-DOG/go-sqlmock/blob/master/LICENSE"
   - "https://github.com/swagger-api/swagger-core/modules/swagger-annotations|https://github.com/swagger-api/swagger-core/tree/master/modules/swagger-annotations"

doc/changes/changes_0.5.10.md

@@ -0,0 +1,36 @@
+# Extension Manager 0.5.10, released 2024-04-25
+
+Code name: Fix reading NULL values from metadata tables
+
+## Summary
+
+This release fixes an error reading `NULL` values from metadata tables:
+
+```
+Could not get database installed extension list failed to read metadata tables. Cause: failed to read row of SYS.EXA_ALL_VIRTUAL_SCHEMAS: sql: Scan error on column index 4, name \"ADAPTER_NOTES\": converting NULL to string is unsupported
+```
+
+## Bugfixes
+
+* #174: Fixed reading `NULL` values from metadata tables
+
+## Dependency Updates
+
+### Extension-manager
+
+#### Compile Dependency Updates
+
+* Updated `github.com/dop251/goja_nodejs:v0.0.0-20240221231712-27eeffc9c235` to `v0.0.0-20240418154818-2aae10d4cbcf`
+* Updated `github.com/exasol/exasol-test-setup-abstraction-server/go-client:v0.3.6` to `v0.3.8`
+
+### Extension Integration Tests Library
+
+#### Compile Dependency Updates
+
+* Updated `com.exasol:exasol-test-setup-abstraction-java:2.1.2` to `2.1.3`
+* Updated `com.exasol:extension-manager-client-java:0.5.9` to `0.5.10`
+
+#### Test Dependency Updates
+
+* Updated `com.exasol:udf-debugging-java:0.6.12` to `0.6.13`
+* Updated `org.slf4j:slf4j-jdk14:2.0.12` to `2.0.13`

extension-manager-integration-test-java/pom.xml

@@ -20,7 +20,7 @@
         <dependency>
             <groupId>com.exasol</groupId>
             <artifactId>exasol-test-setup-abstraction-java</artifactId>
-            <version>2.1.2</version>
+            <version>2.1.3</version>
             <exclusions>
                 <exclusion>
                     <!-- org.hamcrest:hamcrest-core:1.3 contains same classes as org.hamcrest:hamcrest:2.2 -->
@@ -65,7 +65,7 @@
         <dependency>
             <groupId>com.exasol</groupId>
             <artifactId>udf-debugging-java</artifactId>
-            <version>0.6.12</version>
+            <version>0.6.13</version>
             <scope>test</scope>
         </dependency>
         <dependency>
@@ -78,7 +78,7 @@
         <dependency>
             <groupId>org.slf4j</groupId>
             <artifactId>slf4j-jdk14</artifactId>
-            <version>2.0.12</version>
+            <version>2.0.13</version>
             <scope>test</scope>
         </dependency>
     </dependencies>

extension-manager-integration-test-java/src/test/java/com/exasol/extensionmanager/ExampleIT.java

@@ -36,7 +36,7 @@ class ExampleIT {
     @BeforeAll
     static void setup() {
         // Overwrite default Exasol version
-        System.setProperty("com.exasol.dockerdb.image", "8.24.0");
+        System.setProperty("com.exasol.dockerdb.image", "8.26.0");
 
         exasolTestSetup = new ExasolTestSetupFactory(Path.of("cloud-setup")).getTestSetup();
 

extension-manager-integration-test-java/src/test/java/com/exasol/extensionmanager/itest/IntegrationTestCommon.java

@@ -18,7 +18,7 @@ private IntegrationTestCommon() {
 
     static ExasolTestSetup createExasolTestSetup() {
         if (System.getProperty("com.exasol.dockerdb.image") == null) {
-            System.setProperty("com.exasol.dockerdb.image", "8.24.0");
+            System.setProperty("com.exasol.dockerdb.image", "8.26.0");
         }
         return new ExasolTestSetupFactory(Path.of("dummy-config")).getTestSetup();
     }

go.mod

@@ -4,15 +4,15 @@ go 1.21
 
 require (
 	github.com/dop251/goja v0.0.0-20240220182346-e401ed450204
-	github.com/dop251/goja_nodejs v0.0.0-20240221231712-27eeffc9c235
+	github.com/dop251/goja_nodejs v0.0.0-20240418154818-2aae10d4cbcf
 	github.com/stretchr/testify v1.9.0
 	golang.org/x/mod v0.17.0
 )
 
 require (
 	github.com/DATA-DOG/go-sqlmock v1.5.2
 	github.com/exasol/exasol-driver-go v1.0.6
-	github.com/exasol/exasol-test-setup-abstraction-server/go-client v0.3.6
+	github.com/exasol/exasol-test-setup-abstraction-server/go-client v0.3.8
 	github.com/go-chi/chi/v5 v5.0.12
 	github.com/kinbiko/jsonassert v1.1.1
 	github.com/swaggo/http-swagger v1.3.4
@@ -24,7 +24,7 @@ require (
 	github.com/go-openapi/jsonpointer v0.21.0 // indirect
 	github.com/go-openapi/jsonreference v0.21.0 // indirect
 	github.com/go-openapi/spec v0.21.0 // indirect
-	github.com/google/pprof v0.0.0-20240402174815-29b9bb013b0f // indirect
+	github.com/google/pprof v0.0.0-20240424215950-a892ee059fd6 // indirect
 	github.com/iancoleman/orderedmap v0.3.0 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
 	github.com/mailru/easyjson v0.7.7 // indirect

pkg/extensionAPI/exaMetadata/ExaMetadata.go

@@ -21,21 +21,29 @@ type ExaMetadata struct {
 	AllVirtualSchemas ExaVirtualSchemasTable `json:"allVirtualSchemas"`
 }
 
+// CreateExaMetaDataReader creates a new ExaMetadataReader for the Exasol meta data schema SYS.
 func CreateExaMetaDataReader() ExaMetadataReader {
-	return &metaDataReaderImpl{}
+	return CreateExaMetaDataReaderForCustomMetadataSchema("SYS")
+}
+
+// CreateExaMetaDataReaderForCustomMetadataSchema creates a new ExaMetadataReader for the given Exasol meta data schema.
+// This is only used for integration tests.
+func CreateExaMetaDataReaderForCustomMetadataSchema(metaDataSchema string) ExaMetadataReader {
+	return &metaDataReaderImpl{metaDataSchema: metaDataSchema}
 }
 
 type metaDataReaderImpl struct {
+	metaDataSchema string
 }
 
 // ReadMetadataTables reads the metadata tables of the given schema.
 /* [impl -> dsn~extension-components~1]. */
 func (r *metaDataReaderImpl) ReadMetadataTables(tx *sql.Tx, schemaName string) (*ExaMetadata, error) {
-	allScripts, err := readExaAllScriptTable(tx, schemaName)
+	allScripts, err := r.readExaAllScriptTable(tx, schemaName)
 	if err != nil {
 		return nil, err
 	}
-	allVirtualSchemas, err := readExaAllVirtualSchemasTable(tx)
+	allVirtualSchemas, err := r.readExaAllVirtualSchemasTable(tx)
 	if err != nil {
 		return nil, err
 	}
@@ -44,12 +52,14 @@ func (r *metaDataReaderImpl) ReadMetadataTables(tx *sql.Tx, schemaName string) (
 
 /* [impl -> dsn~extension-context-metadata~1]. */
 func (r *metaDataReaderImpl) GetScriptByName(tx *sql.Tx, schemaName, scriptName string) (*ExaScriptRow, error) {
-	result, err := tx.Query(`
+	// #nosec G201 Using schema as query parameter is not possible
+	query := fmt.Sprintf(`
 SELECT SCRIPT_SCHEMA, SCRIPT_NAME, SCRIPT_TYPE, SCRIPT_INPUT_TYPE, SCRIPT_RESULT_TYPE, SCRIPT_TEXT, SCRIPT_COMMENT
-FROM SYS.EXA_ALL_SCRIPTS
-WHERE SCRIPT_SCHEMA=? AND SCRIPT_NAME=?`, schemaName, scriptName)
+FROM %s.EXA_ALL_SCRIPTS
+WHERE SCRIPT_SCHEMA=? AND SCRIPT_NAME=?`, r.metaDataSchema)
+	result, err := tx.Query(query, schemaName, scriptName)
 	if err != nil {
-		return nil, fmt.Errorf("failed to read SYS.EXA_ALL_SCRIPTS: %w", err)
+		return nil, fmt.Errorf("failed to read %s.EXA_ALL_SCRIPTS: %w", r.metaDataSchema, err)
 	}
 	defer result.Close()
 	if !result.Next() {
@@ -62,19 +72,21 @@ WHERE SCRIPT_SCHEMA=? AND SCRIPT_NAME=?`, schemaName, scriptName)
 	return row, nil
 }
 
-func readExaAllScriptTable(tx *sql.Tx, schemaName string) (*ExaScriptTable, error) {
-	result, err := tx.Query(`
+func (r *metaDataReaderImpl) readExaAllScriptTable(tx *sql.Tx, schemaName string) (*ExaScriptTable, error) {
+	// #nosec G201 Using schema as query parameter is not possible
+	query := fmt.Sprintf(`
 SELECT SCRIPT_SCHEMA, SCRIPT_NAME, SCRIPT_TYPE, SCRIPT_INPUT_TYPE, SCRIPT_RESULT_TYPE, SCRIPT_TEXT, SCRIPT_COMMENT
-FROM SYS.EXA_ALL_SCRIPTS
-WHERE SCRIPT_SCHEMA=?`, schemaName)
+FROM %s.EXA_ALL_SCRIPTS
+WHERE SCRIPT_SCHEMA=?`, r.metaDataSchema)
+	result, err := tx.Query(query, schemaName)
 	if err != nil {
-		return nil, fmt.Errorf("failed to read SYS.EXA_ALL_SCRIPTS: %w", err)
+		return nil, fmt.Errorf("failed to read %s.EXA_ALL_SCRIPTS: %w", r.metaDataSchema, err)
 	}
 	defer result.Close()
 	rows := make([]ExaScriptRow, 0)
 	for result.Next() {
 		if result.Err() != nil {
-			return nil, fmt.Errorf("failed to iterate SYS.EXA_ALL_SCRIPTS: %w", result.Err())
+			return nil, fmt.Errorf("failed to iterate %s.EXA_ALL_SCRIPTS: %w", r.metaDataSchema, result.Err())
 		}
 		row, err := readScriptRow(result)
 		if err != nil {
@@ -86,43 +98,73 @@ WHERE SCRIPT_SCHEMA=?`, schemaName)
 }
 
 func readScriptRow(result *sql.Rows) (*ExaScriptRow, error) {
-	var row ExaScriptRow
+	var schema sql.NullString
+	var name sql.NullString
+	var scriptType sql.NullString
 	var inputType sql.NullString
 	var resultType sql.NullString
+	var text sql.NullString
 	var comment sql.NullString
-	err := result.Scan(&row.Schema, &row.Name, &row.Type, &inputType, &resultType, &row.Text, &comment)
+	err := result.Scan(&schema, &name, &scriptType, &inputType, &resultType, &text, &comment)
 	if err != nil {
-		return nil, fmt.Errorf("failed to read row of SYS.EXA_ALL_SCRIPTS: %w", err)
+		return nil, fmt.Errorf("failed to read row of EXA_ALL_SCRIPTS: %w", err)
+	}
+	row := ExaScriptRow{
+		Schema:     schema.String,
+		Name:       name.String,
+		Type:       scriptType.String,
+		InputType:  inputType.String,
+		ResultType: resultType.String,
+		Text:       text.String,
+		Comment:    comment.String,
 	}
-	row.InputType = inputType.String
-	row.ResultType = resultType.String
-	row.Comment = comment.String
 	return &row, nil
 }
 
-func readExaAllVirtualSchemasTable(tx *sql.Tx) (*ExaVirtualSchemasTable, error) {
-	result, err := tx.Query(`
+func (r *metaDataReaderImpl) readExaAllVirtualSchemasTable(tx *sql.Tx) (*ExaVirtualSchemasTable, error) {
+	// #nosec G201 Using schema as query parameter is not possible
+	query := fmt.Sprintf(`
 SELECT SCHEMA_NAME, SCHEMA_OWNER, ADAPTER_SCRIPT_SCHEMA, ADAPTER_SCRIPT_NAME, ADAPTER_NOTES
-FROM SYS.EXA_ALL_VIRTUAL_SCHEMAS`)
+FROM %s.EXA_ALL_VIRTUAL_SCHEMAS`, r.metaDataSchema)
+	result, err := tx.Query(query)
 	if err != nil {
-		return nil, fmt.Errorf("failed to read SYS.EXA_ALL_VIRTUAL_SCHEMAS: %w", err)
+		return nil, fmt.Errorf("failed to read %s.EXA_ALL_VIRTUAL_SCHEMAS: %w", r.metaDataSchema, err)
 	}
 	defer result.Close()
 	rows := make([]ExaVirtualSchemaRow, 0)
 	for result.Next() {
 		if result.Err() != nil {
-			return nil, fmt.Errorf("failed to iterate SYS.EXA_ALL_VIRTUAL_SCHEMAS: %w", result.Err())
+			return nil, fmt.Errorf("failed to iterate %s.EXA_ALL_VIRTUAL_SCHEMAS: %w", r.metaDataSchema, result.Err())
 		}
-		var row ExaVirtualSchemaRow
-		err := result.Scan(&row.Name, &row.Owner, &row.AdapterScriptSchema, &row.AdapterScriptName, &row.AdapterNotes)
+		row, err := readVirtualSchemaRow(result)
 		if err != nil {
-			return nil, fmt.Errorf("failed to read row of SYS.EXA_ALL_VIRTUAL_SCHEMAS: %w", err)
+			return nil, err
 		}
-		rows = append(rows, row)
+		rows = append(rows, *row)
 	}
 	return &ExaVirtualSchemasTable{Rows: rows}, nil
 }
 
+func readVirtualSchemaRow(result *sql.Rows) (*ExaVirtualSchemaRow, error) {
+	var name sql.NullString
+	var owner sql.NullString
+	var adapterScriptSchema sql.NullString
+	var adapterScriptName sql.NullString
+	var adapterNotes sql.NullString
+	err := result.Scan(&name, &owner, &adapterScriptSchema, &adapterScriptName, &adapterNotes)
+	if err != nil {
+		return nil, fmt.Errorf("failed to read row of EXA_ALL_VIRTUAL_SCHEMAS: %w", err)
+	}
+	row := ExaVirtualSchemaRow{
+		Name:                name.String,
+		Owner:               owner.String,
+		AdapterScriptSchema: adapterScriptSchema.String,
+		AdapterScriptName:   adapterScriptName.String,
+		AdapterNotes:        adapterNotes.String,
+	}
+	return &row, nil
+}
+
 type ExaScriptTable struct {
 	Rows []ExaScriptRow `json:"rows"`
 }