From 3e4e2a0e858a5530cc2c7194540481c1ee60fc93 Mon Sep 17 00:00:00 2001 From: Jason McIntosh Date: Wed, 13 Nov 2024 16:43:25 -0600 Subject: [PATCH 01/15] chore(java): Full java 17 only --- Dockerfile.compile | 4 ++-- Dockerfile.slim | 4 ++-- Dockerfile.ubuntu | 4 ++-- gradle.properties | 1 + keel-sql/src/test/resources/testcontainers.properties | 2 +- 5 files changed, 8 insertions(+), 7 deletions(-) diff --git a/Dockerfile.compile b/Dockerfile.compile index 5609ffa51a..f47e1a778d 100644 --- a/Dockerfile.compile +++ b/Dockerfile.compile @@ -1,9 +1,9 @@ -FROM ubuntu:bionic +FROM ubuntu:jammy LABEL maintainer="sig-platform@spinnaker.io" RUN echo "mysql-server mysql-server/root_password password sa" | debconf-set-selections RUN echo "mysql-server mysql-server/root_password_again password sa" | debconf-set-selections RUN apt-get update && apt-get install -y \ - openjdk-11-jdk \ + openjdk-17-jdk \ mysql-server \ && rm -rf /var/lib/apt/lists/* ENV GRADLE_USER_HOME /workspace/.gradle diff --git a/Dockerfile.slim b/Dockerfile.slim index 0e2429c98d..3f1e011c21 100644 --- a/Dockerfile.slim +++ b/Dockerfile.slim @@ -1,6 +1,6 @@ -FROM alpine:3.11 +FROM alpine:3.20 LABEL maintainer="sig-platform@spinnaker.io" -RUN apk --no-cache add --update bash openjdk11-jre +RUN apk --no-cache add --update bash openjdk17-jre RUN addgroup -S -g 10111 spinnaker RUN adduser -S -G spinnaker -u 10111 spinnaker COPY keel-web/build/install/keel /opt/keel diff --git a/Dockerfile.ubuntu b/Dockerfile.ubuntu index 4a492faf9d..6d5f3cc612 100644 --- a/Dockerfile.ubuntu +++ b/Dockerfile.ubuntu @@ -1,6 +1,6 @@ -FROM ubuntu:bionic +FROM ubuntu:jammy LABEL maintainer="sig-platform@spinnaker.io" -RUN apt-get update && apt-get -y install openjdk-11-jre-headless wget +RUN apt-get update && apt-get -y install openjdk-17-jre-headless wget RUN adduser --system --uid 10111 --group spinnaker COPY keel-web/build/install/keel /opt/keel RUN mkdir -p /opt/keel/plugins && chown -R spinnaker:nogroup /opt/keel/plugins diff --git a/gradle.properties b/gradle.properties index 275ab41834..12e961f2bc 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,3 +1,4 @@ +targetJava17=true kotlinVersion=1.6.21 fiatVersion=1.50.0 korkVersion=7.243.0 diff --git a/keel-sql/src/test/resources/testcontainers.properties b/keel-sql/src/test/resources/testcontainers.properties index 5ecaa452ec..06137e2be6 100644 --- a/keel-sql/src/test/resources/testcontainers.properties +++ b/keel-sql/src/test/resources/testcontainers.properties @@ -1 +1 @@ -ryuk.container.image = public.ecr.aws/s4w6t4b6/testcontainers/ryuk:0.3.4 +ryuk.container.image = testcontainers/ryuk:0.11.0 From 83990c6b477f1ed6272b0101f260bfe56b0d0fde Mon Sep 17 00:00:00 2001 From: Jason McIntosh Date: Wed, 13 Nov 2024 16:44:50 -0600 Subject: [PATCH 02/15] chore(java): Full java 17 only --- .github/workflows/build.yml | 2 +- .github/workflows/pr.yml | 2 +- .github/workflows/release.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 521010b58a..5af569f7c7 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -25,7 +25,7 @@ jobs: uses: docker/setup-buildx-action@v3 - uses: actions/setup-java@v4 with: - java-version: 11 + java-version: 17 distribution: 'zulu' cache: 'gradle' - name: Prepare build variables diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 036ca190a2..6cce09965f 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -19,7 +19,7 @@ jobs: uses: docker/setup-buildx-action@v3 - uses: actions/setup-java@v4 with: - java-version: 11 + java-version: 17 distribution: 'zulu' cache: 'gradle' - name: Prepare build variables diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 5fa8fcc1aa..544178928f 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -23,7 +23,7 @@ jobs: uses: docker/setup-buildx-action@v3 - uses: actions/setup-java@v4 with: - java-version: 11 + java-version: 17 distribution: 'zulu' cache: 'gradle' - name: Assemble release info From b3638281694695788bcf54bab17f82e9accede9a Mon Sep 17 00:00:00 2001 From: Jason McIntosh Date: Wed, 13 Nov 2024 16:47:25 -0600 Subject: [PATCH 03/15] chore(java): Full java 17 only --- gradle/kotlin.gradle | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gradle/kotlin.gradle b/gradle/kotlin.gradle index 2d790a97bb..aef735ac5e 100644 --- a/gradle/kotlin.gradle +++ b/gradle/kotlin.gradle @@ -5,7 +5,7 @@ apply plugin: "kotlin-allopen" compileKotlin { kotlinOptions { languageVersion = "1.6" - jvmTarget = "11" + jvmTarget = "17" // see https://kotlinlang.org/docs/java-to-kotlin-interop.html#compatibility-mode-for-default-methods freeCompilerArgs += "-Xjvm-default=enable" } @@ -14,7 +14,7 @@ compileKotlin { compileTestKotlin { kotlinOptions { languageVersion = "1.6" - jvmTarget = "11" + jvmTarget = "17" freeCompilerArgs += "-Xjvm-default=enable" } } From b592c180c698af4d1011ece37f574dec7ac8f13d Mon Sep 17 00:00:00 2001 From: Jason McIntosh Date: Wed, 13 Nov 2024 16:58:29 -0600 Subject: [PATCH 04/15] chore(java): Let gradle plugin compat set the target and source versions --- build.gradle | 4 ---- 1 file changed, 4 deletions(-) diff --git a/build.gradle b/build.gradle index 9b015eb7ff..5f74024e9f 100644 --- a/build.gradle +++ b/build.gradle @@ -30,10 +30,6 @@ allprojects { apply(plugin: "com.github.ben-manes.versions") apply(plugin: "com.diffplug.spotless") - java { - sourceCompatibility = JavaVersion.VERSION_11 - targetCompatibility = JavaVersion.VERSION_11 - } dependencies { annotationProcessor(platform("io.spinnaker.kork:kork-bom:${korkVersion}")) From 812f141c6a0cb432bb6805a8e6d189c078bc9ddf Mon Sep 17 00:00:00 2001 From: Jason McIntosh Date: Wed, 13 Nov 2024 19:50:41 -0600 Subject: [PATCH 05/15] chore(java): Attempt to fix the serialization behavior on linux when using Instant clocks and sql conversions of the JSON string from that conversion --- gradle.properties | 2 ++ .../keel/serialization/ObjectMappers.kt | 29 ++++++++++++++++--- 2 files changed, 27 insertions(+), 4 deletions(-) diff --git a/gradle.properties b/gradle.properties index 12e961f2bc..7f16895cca 100644 --- a/gradle.properties +++ b/gradle.properties @@ -10,6 +10,8 @@ okHttpVersion=4.5.0 resilience4jVersion=1.5.0 spinnakerGradleVersion=8.32.1 +org.gradle.jvmargs=-Xmx4g -Xms4g + # Used to control whether to spin up docker to run liquibase before jooq buildingInDocker=false diff --git a/keel-core/src/main/kotlin/com/netflix/spinnaker/keel/serialization/ObjectMappers.kt b/keel-core/src/main/kotlin/com/netflix/spinnaker/keel/serialization/ObjectMappers.kt index 9f786634d5..aade2fbe54 100644 --- a/keel-core/src/main/kotlin/com/netflix/spinnaker/keel/serialization/ObjectMappers.kt +++ b/keel-core/src/main/kotlin/com/netflix/spinnaker/keel/serialization/ObjectMappers.kt @@ -1,9 +1,11 @@ package com.netflix.spinnaker.keel.serialization import com.fasterxml.jackson.annotation.JsonInclude.Include.NON_NULL +import com.fasterxml.jackson.core.JsonGenerator +import com.fasterxml.jackson.core.JsonParser +import com.fasterxml.jackson.databind.* import com.fasterxml.jackson.databind.DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES import com.fasterxml.jackson.databind.MapperFeature.ACCEPT_CASE_INSENSITIVE_ENUMS -import com.fasterxml.jackson.databind.ObjectMapper import com.fasterxml.jackson.databind.SerializationFeature.WRITE_DATES_AS_TIMESTAMPS import com.fasterxml.jackson.databind.SerializationFeature.WRITE_DATES_WITH_ZONE_ID import com.fasterxml.jackson.databind.SerializationFeature.WRITE_DATE_KEYS_AS_TIMESTAMPS @@ -13,11 +15,14 @@ import com.fasterxml.jackson.databind.ser.std.ToStringSerializer import com.fasterxml.jackson.dataformat.yaml.YAMLGenerator.Feature.USE_NATIVE_TYPE_ID import com.fasterxml.jackson.dataformat.yaml.YAMLMapper import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule +import com.fasterxml.jackson.datatype.jsr310.ser.InstantSerializer import com.fasterxml.jackson.module.kotlin.registerKotlinModule import com.netflix.spinnaker.keel.jackson.registerKeelApiModule import de.huxhorn.sulky.ulid.ULID import org.springframework.boot.jackson.JsonComponentModule import java.text.SimpleDateFormat +import java.time.Instant +import java.time.format.DateTimeFormatter import java.util.TimeZone /** @@ -29,18 +34,34 @@ fun configuredObjectMapper(): ObjectMapper = ObjectMapper().configureForKeel() * Factory method for [YAMLMapper]s configured how we like 'em. */ fun configuredYamlMapper(): YAMLMapper = YAMLMapper().configureForKeel().disable(USE_NATIVE_TYPE_ID) +class CustomInstantSerializer : JsonSerializer() { + override fun serialize(p0: Instant?, p1: JsonGenerator?, p2: SerializerProvider?) { + p1?.writeObject(DateTimeFormatter.ISO_INSTANT.format(p0)) + } +} -fun T.configureForKeel(): T = - apply { +class CustomInstantDeserializer : JsonDeserializer() { + //KEY here is for the DB to correctly serialize/deserialize date from strings, and so MUST match the supported formatter for str_to_date + // (str_to_date(json->>'$.triggeredAt', '%Y-%m-%dT%T.%fZ')) + override fun deserialize(p0: JsonParser?, p1: DeserializationContext?): Instant { + return Instant.from(DateTimeFormatter.ISO_INSTANT.parse(p0?.getValueAsString())) + } +} +fun T.configureForKeel(): T { + val javaTimeModule = JavaTimeModule() + javaTimeModule.addSerializer(Instant::class.java, CustomInstantSerializer()) + javaTimeModule.addDeserializer(Instant::class.java, CustomInstantDeserializer()) + return apply { registerKeelApiModule() .registerKotlinModule() .registerULIDModule() - .registerModule(JavaTimeModule()) + .registerModule(javaTimeModule) .configureSaneDateTimeRepresentation() .disable(FAIL_ON_UNKNOWN_PROPERTIES) .enable(ACCEPT_CASE_INSENSITIVE_ENUMS) .setSerializationInclusion(NON_NULL) } +} private fun ObjectMapper.registerULIDModule(): ObjectMapper = registerModule( From f92b438799a0deb82e2bdb8422222cbf2e56e5e4 Mon Sep 17 00:00:00 2001 From: Jason McIntosh Date: Thu, 14 Nov 2024 01:02:42 -0600 Subject: [PATCH 06/15] fix(sql): Attempting to fix the precision error in a slightly different manner --- .../keel/serialization/ObjectMappers.kt | 30 +++++++++---------- .../changelog/202410303-long-column-sizes.yml | 20 +++++++++++++ 2 files changed, 35 insertions(+), 15 deletions(-) create mode 100644 keel-sql/src/main/resources/db/changelog/202410303-long-column-sizes.yml diff --git a/keel-core/src/main/kotlin/com/netflix/spinnaker/keel/serialization/ObjectMappers.kt b/keel-core/src/main/kotlin/com/netflix/spinnaker/keel/serialization/ObjectMappers.kt index aade2fbe54..6b5a54bd37 100644 --- a/keel-core/src/main/kotlin/com/netflix/spinnaker/keel/serialization/ObjectMappers.kt +++ b/keel-core/src/main/kotlin/com/netflix/spinnaker/keel/serialization/ObjectMappers.kt @@ -34,23 +34,23 @@ fun configuredObjectMapper(): ObjectMapper = ObjectMapper().configureForKeel() * Factory method for [YAMLMapper]s configured how we like 'em. */ fun configuredYamlMapper(): YAMLMapper = YAMLMapper().configureForKeel().disable(USE_NATIVE_TYPE_ID) -class CustomInstantSerializer : JsonSerializer() { - override fun serialize(p0: Instant?, p1: JsonGenerator?, p2: SerializerProvider?) { - p1?.writeObject(DateTimeFormatter.ISO_INSTANT.format(p0)) - } -} - -class CustomInstantDeserializer : JsonDeserializer() { - //KEY here is for the DB to correctly serialize/deserialize date from strings, and so MUST match the supported formatter for str_to_date - // (str_to_date(json->>'$.triggeredAt', '%Y-%m-%dT%T.%fZ')) - override fun deserialize(p0: JsonParser?, p1: DeserializationContext?): Instant { - return Instant.from(DateTimeFormatter.ISO_INSTANT.parse(p0?.getValueAsString())) - } -} +//class CustomInstantSerializer : JsonSerializer() { +// override fun serialize(p0: Instant?, p1: JsonGenerator?, p2: SerializerProvider?) { +// p1?.writeObject(DateTimeFormatter.ISO_INSTANT.format(p0)) +// } +//} +// +//class CustomInstantDeserializer : JsonDeserializer() { +// //KEY here is for the DB to correctly serialize/deserialize date from strings, and so MUST match the supported formatter for str_to_date +// // (str_to_date(json->>'$.triggeredAt', '%Y-%m-%dT%T.%fZ')) +// override fun deserialize(p0: JsonParser?, p1: DeserializationContext?): Instant { +// return Instant.from(DateTimeFormatter.ISO_INSTANT.parse(p0?.getValueAsString())) +// } +//} fun T.configureForKeel(): T { val javaTimeModule = JavaTimeModule() - javaTimeModule.addSerializer(Instant::class.java, CustomInstantSerializer()) - javaTimeModule.addDeserializer(Instant::class.java, CustomInstantDeserializer()) +// javaTimeModule.addSerializer(Instant::class.java, CustomInstantSerializer()) +// javaTimeModule.addDeserializer(Instant::class.java, CustomInstantDeserializer()) return apply { registerKeelApiModule() .registerKotlinModule() diff --git a/keel-sql/src/main/resources/db/changelog/202410303-long-column-sizes.yml b/keel-sql/src/main/resources/db/changelog/202410303-long-column-sizes.yml new file mode 100644 index 0000000000..5b2f8f8baa --- /dev/null +++ b/keel-sql/src/main/resources/db/changelog/202410303-long-column-sizes.yml @@ -0,0 +1,20 @@ +databaseChangeLog: + - changeSet: + id: long-date-sizes-linux-java-17 + author: jasonmcintosh + changes: + - sql: + sql: | + alter table event + change timestamp_gen datetime(6) generated always as (str_to_date(json->>'$.timestamp', '%Y-%m-%dT%T.%fZ')) + after application; + - sql: + sql: | + alter table dismissible_notification + change triggered_at datetime(6) generated always as (str_to_date(json->>'$.triggeredAt', '%Y-%m-%dT%T.%fZ')) + after uid; + - sql: + sql: | + alter table dismissible_notification + change dismissed_at datetime(6) generated always as (str_to_date(json->>'$.dismissedAt', '%Y-%m-%dT%T.%fZ')) + after dismissed_by; From eaa4d7efdd6536c9924eda977abe8e153e3f4512 Mon Sep 17 00:00:00 2001 From: Jason McIntosh Date: Thu, 14 Nov 2024 11:29:01 -0600 Subject: [PATCH 07/15] fix(sql): Attempting to fix the precision error in a slightly different manner --- .../db/changelog/202410303-long-column-sizes.yml | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/keel-sql/src/main/resources/db/changelog/202410303-long-column-sizes.yml b/keel-sql/src/main/resources/db/changelog/202410303-long-column-sizes.yml index 5b2f8f8baa..5b7a67f67f 100644 --- a/keel-sql/src/main/resources/db/changelog/202410303-long-column-sizes.yml +++ b/keel-sql/src/main/resources/db/changelog/202410303-long-column-sizes.yml @@ -3,18 +3,19 @@ databaseChangeLog: id: long-date-sizes-linux-java-17 author: jasonmcintosh changes: - - sql: - sql: | - alter table event - change timestamp_gen datetime(6) generated always as (str_to_date(json->>'$.timestamp', '%Y-%m-%dT%T.%fZ')) - after application; + - dropColumn: + tableName: dismissible_notification + columnName: triggered_at - sql: sql: | alter table dismissible_notification - change triggered_at datetime(6) generated always as (str_to_date(json->>'$.triggeredAt', '%Y-%m-%dT%T.%fZ')) + add column triggered_at datetime(6) generated always as (str_to_date(json->>'$.triggeredAt', '%Y-%m-%dT%T.%fZ')) after uid; + - dropColumn: + tableName: dismissible_notification + columnName: dismissed_at - sql: sql: | alter table dismissible_notification - change dismissed_at datetime(6) generated always as (str_to_date(json->>'$.dismissedAt', '%Y-%m-%dT%T.%fZ')) + add column dismissed_at datetime(6) generated always as (str_to_date(json->>'$.dismissedAt', '%Y-%m-%dT%T.%fZ')) after dismissed_by; From 0c590bb3e7cc0f5ba079f8cdc2f4e407c3b6359c Mon Sep 17 00:00:00 2001 From: Jason McIntosh Date: Thu, 14 Nov 2024 11:53:24 -0600 Subject: [PATCH 08/15] fix(sql): Attempting to fix the precision error by force setting mapping of instants --- .../keel/serialization/ObjectMappers.kt | 26 ++++++------ .../spinnaker/config/SqlConfiguration.kt | 40 ++++++++++++++----- .../changelog/202410303-long-column-sizes.yml | 21 ---------- 3 files changed, 42 insertions(+), 45 deletions(-) delete mode 100644 keel-sql/src/main/resources/db/changelog/202410303-long-column-sizes.yml diff --git a/keel-core/src/main/kotlin/com/netflix/spinnaker/keel/serialization/ObjectMappers.kt b/keel-core/src/main/kotlin/com/netflix/spinnaker/keel/serialization/ObjectMappers.kt index 6b5a54bd37..b8a9db3877 100644 --- a/keel-core/src/main/kotlin/com/netflix/spinnaker/keel/serialization/ObjectMappers.kt +++ b/keel-core/src/main/kotlin/com/netflix/spinnaker/keel/serialization/ObjectMappers.kt @@ -34,19 +34,19 @@ fun configuredObjectMapper(): ObjectMapper = ObjectMapper().configureForKeel() * Factory method for [YAMLMapper]s configured how we like 'em. */ fun configuredYamlMapper(): YAMLMapper = YAMLMapper().configureForKeel().disable(USE_NATIVE_TYPE_ID) -//class CustomInstantSerializer : JsonSerializer() { -// override fun serialize(p0: Instant?, p1: JsonGenerator?, p2: SerializerProvider?) { -// p1?.writeObject(DateTimeFormatter.ISO_INSTANT.format(p0)) -// } -//} -// -//class CustomInstantDeserializer : JsonDeserializer() { -// //KEY here is for the DB to correctly serialize/deserialize date from strings, and so MUST match the supported formatter for str_to_date -// // (str_to_date(json->>'$.triggeredAt', '%Y-%m-%dT%T.%fZ')) -// override fun deserialize(p0: JsonParser?, p1: DeserializationContext?): Instant { -// return Instant.from(DateTimeFormatter.ISO_INSTANT.parse(p0?.getValueAsString())) -// } -//} +class CustomInstantSerializer : JsonSerializer() { + override fun serialize(p0: Instant?, p1: JsonGenerator?, p2: SerializerProvider?) { + p1?.writeObject(DateTimeFormatter.ISO_INSTANT.format(p0)) + } +} + + class CustomInstantDeserializer : JsonDeserializer() { + //KEY here is for the DB to correctly serialize/deserialize date from strings, and so MUST match the supported formatter for str_to_date + // (str_to_date(json->>'$.triggeredAt', '%Y-%m-%dT%T.%fZ')) + override fun deserialize(p0: JsonParser?, p1: DeserializationContext?): Instant { + return Instant.from(DateTimeFormatter.ISO_INSTANT.parse(p0?.getValueAsString())) + } +} fun T.configureForKeel(): T { val javaTimeModule = JavaTimeModule() // javaTimeModule.addSerializer(Instant::class.java, CustomInstantSerializer()) diff --git a/keel-sql/src/main/kotlin/com/netflix/spinnaker/config/SqlConfiguration.kt b/keel-sql/src/main/kotlin/com/netflix/spinnaker/config/SqlConfiguration.kt index 63c7713546..0a79132661 100644 --- a/keel-sql/src/main/kotlin/com/netflix/spinnaker/config/SqlConfiguration.kt +++ b/keel-sql/src/main/kotlin/com/netflix/spinnaker/config/SqlConfiguration.kt @@ -1,11 +1,16 @@ package com.netflix.spinnaker.config -import com.fasterxml.jackson.databind.ObjectMapper +import com.fasterxml.jackson.core.JsonGenerator +import com.fasterxml.jackson.core.JsonParser +import com.fasterxml.jackson.databind.* +import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule import com.netflix.spectator.api.Registry import com.netflix.spinnaker.keel.api.plugins.ArtifactSupplier import com.netflix.spinnaker.keel.events.PersistentEvent.Companion.clock import com.netflix.spinnaker.keel.resources.ResourceFactory import com.netflix.spinnaker.keel.scheduled.ScheduledAgent +import com.netflix.spinnaker.keel.serialization.CustomInstantDeserializer +import com.netflix.spinnaker.keel.serialization.CustomInstantSerializer import com.netflix.spinnaker.keel.sql.SqlActionRepository import com.netflix.spinnaker.keel.sql.SqlAgentLockRepository import com.netflix.spinnaker.keel.sql.SqlArtifactRepository @@ -40,14 +45,18 @@ import org.springframework.context.annotation.Configuration import org.springframework.context.annotation.Import import org.springframework.core.env.Environment import java.time.Clock +import java.time.Instant +import java.time.format.DateTimeFormatter import javax.annotation.PostConstruct @Configuration @ConditionalOnProperty("sql.enabled") @EnableConfigurationProperties(RetentionProperties::class) @Import(DefaultSqlConfiguration::class, SqlRetryProperties::class, EnvironmentExclusionConfig::class) -class SqlConfiguration -{ +class SqlConfiguration { + + + @Autowired lateinit var jooqConfiguration: DefaultConfiguration @@ -67,6 +76,15 @@ class SqlConfiguration jooqConfiguration.settings().isRenderSchema = false } + fun overloadMapper(mapper: ObjectMapper): ObjectMapper { + val copy = mapper.copy() + val timeModule = JavaTimeModule() + timeModule.addSerializer(Instant::class.java, CustomInstantSerializer()) + timeModule.addDeserializer(Instant::class.java, CustomInstantDeserializer()) + copy.registerModule(timeModule) + return copy + } + @Bean fun resourceRepository( jooq: DSLContext, @@ -80,7 +98,7 @@ class SqlConfiguration SqlResourceRepository( jooq, clock, - objectMapper, + overloadMapper(objectMapper), resourceFactory, SqlRetry(sqlRetryProperties), publisher, @@ -99,7 +117,7 @@ class SqlConfiguration SqlArtifactRepository( jooq, clock, - objectMapper, + overloadMapper(objectMapper), SqlRetry(sqlRetryProperties), artifactSuppliers, publisher @@ -118,7 +136,7 @@ class SqlConfiguration jooq = jooq, clock = clock, resourceFactory = resourceFactory, - objectMapper = objectMapper, + objectMapper = overloadMapper(objectMapper), sqlRetry = SqlRetry(sqlRetryProperties), artifactSuppliers = artifactSuppliers, publisher = publisher @@ -181,7 +199,7 @@ class SqlConfiguration ) = SqlActionRepository( jooq, clock, - objectMapper, + overloadMapper(objectMapper), resourceFactory, SqlRetry(sqlRetryProperties), artifactSuppliers, @@ -205,7 +223,7 @@ class SqlConfiguration clock: Clock, properties: SqlProperties, objectMapper: ObjectMapper - ) = SqlLifecycleMonitorRepository(jooq, clock, objectMapper, SqlRetry(sqlRetryProperties)) + ) = SqlLifecycleMonitorRepository(jooq, clock, overloadMapper(objectMapper), SqlRetry(sqlRetryProperties)) @Bean fun bakedImageRepository( @@ -213,7 +231,7 @@ class SqlConfiguration clock: Clock, properties: SqlProperties, objectMapper: ObjectMapper - ) = SqlBakedImageRepository(jooq, clock, objectMapper, SqlRetry(sqlRetryProperties)) + ) = SqlBakedImageRepository(jooq, clock, overloadMapper(objectMapper), SqlRetry(sqlRetryProperties)) @Bean fun environmentLeaseRepository( @@ -239,7 +257,7 @@ class SqlConfiguration ) = SqlEnvironmentDeletionRepository( jooq, clock, - objectMapper, + overloadMapper(objectMapper), SqlRetry(sqlRetryProperties), resourceFactory, artifactSuppliers @@ -250,7 +268,7 @@ class SqlConfiguration jooq: DSLContext, clock: Clock, objectMapper: ObjectMapper - ) = SqlWorkQueueRepository(jooq, clock, objectMapper, SqlRetry(sqlRetryProperties)) + ) = SqlWorkQueueRepository(jooq, clock, overloadMapper(objectMapper), SqlRetry(sqlRetryProperties)) @Bean fun featureRolloutRepository( diff --git a/keel-sql/src/main/resources/db/changelog/202410303-long-column-sizes.yml b/keel-sql/src/main/resources/db/changelog/202410303-long-column-sizes.yml deleted file mode 100644 index 5b7a67f67f..0000000000 --- a/keel-sql/src/main/resources/db/changelog/202410303-long-column-sizes.yml +++ /dev/null @@ -1,21 +0,0 @@ -databaseChangeLog: - - changeSet: - id: long-date-sizes-linux-java-17 - author: jasonmcintosh - changes: - - dropColumn: - tableName: dismissible_notification - columnName: triggered_at - - sql: - sql: | - alter table dismissible_notification - add column triggered_at datetime(6) generated always as (str_to_date(json->>'$.triggeredAt', '%Y-%m-%dT%T.%fZ')) - after uid; - - dropColumn: - tableName: dismissible_notification - columnName: dismissed_at - - sql: - sql: | - alter table dismissible_notification - add column dismissed_at datetime(6) generated always as (str_to_date(json->>'$.dismissedAt', '%Y-%m-%dT%T.%fZ')) - after dismissed_by; From 9747b89f2a8ff99e8390fa5dd686e4b877050409 Mon Sep 17 00:00:00 2001 From: Jason McIntosh Date: Thu, 14 Nov 2024 14:52:09 -0600 Subject: [PATCH 09/15] fix(sql): Use base date time serializer with some mods --- .../PrecisionSqlDeserializer.java | 18 +++++++++ .../serialization/PrecisionSqlSerializer.java | 24 +++++++++++ .../keel/serialization/ObjectMappers.kt | 24 ++--------- .../spinnaker/config/SqlConfiguration.kt | 40 +++++-------------- 4 files changed, 56 insertions(+), 50 deletions(-) create mode 100644 keel-core/src/main/java/com/netflix/spinnaker/keel/serialization/PrecisionSqlDeserializer.java create mode 100644 keel-core/src/main/java/com/netflix/spinnaker/keel/serialization/PrecisionSqlSerializer.java diff --git a/keel-core/src/main/java/com/netflix/spinnaker/keel/serialization/PrecisionSqlDeserializer.java b/keel-core/src/main/java/com/netflix/spinnaker/keel/serialization/PrecisionSqlDeserializer.java new file mode 100644 index 0000000000..10dfd495ba --- /dev/null +++ b/keel-core/src/main/java/com/netflix/spinnaker/keel/serialization/PrecisionSqlDeserializer.java @@ -0,0 +1,18 @@ +package com.netflix.spinnaker.keel.serialization; + +import com.fasterxml.jackson.datatype.jsr310.deser.InstantDeserializer; +import java.time.Instant; +import java.time.format.DateTimeFormatter; + +public class PrecisionSqlDeserializer extends InstantDeserializer { + public PrecisionSqlDeserializer() { + super( + Instant.class, + DateTimeFormatter.ISO_INSTANT, + Instant::from, + (a) -> Instant.ofEpochMilli(a.value), + (a) -> Instant.ofEpochSecond(a.integer, a.fraction), + null, + true); + } +} diff --git a/keel-core/src/main/java/com/netflix/spinnaker/keel/serialization/PrecisionSqlSerializer.java b/keel-core/src/main/java/com/netflix/spinnaker/keel/serialization/PrecisionSqlSerializer.java new file mode 100644 index 0000000000..07772b413d --- /dev/null +++ b/keel-core/src/main/java/com/netflix/spinnaker/keel/serialization/PrecisionSqlSerializer.java @@ -0,0 +1,24 @@ +package com.netflix.spinnaker.keel.serialization; + +import com.fasterxml.jackson.annotation.JsonFormat; +import com.fasterxml.jackson.datatype.jsr310.ser.InstantSerializerBase; +import java.time.Instant; +import java.time.format.DateTimeFormatter; +import java.time.format.DateTimeFormatterBuilder; + +public class PrecisionSqlSerializer extends InstantSerializerBase { + public PrecisionSqlSerializer() { + super( + Instant.class, + Instant::toEpochMilli, + Instant::getEpochSecond, + Instant::getNano, + new DateTimeFormatterBuilder().appendInstant(3).toFormatter()); + } + + @Override + protected InstantSerializerBase withFormat( + Boolean aBoolean, DateTimeFormatter dateTimeFormatter, JsonFormat.Shape shape) { + return this; + } +} diff --git a/keel-core/src/main/kotlin/com/netflix/spinnaker/keel/serialization/ObjectMappers.kt b/keel-core/src/main/kotlin/com/netflix/spinnaker/keel/serialization/ObjectMappers.kt index b8a9db3877..e041f2caf0 100644 --- a/keel-core/src/main/kotlin/com/netflix/spinnaker/keel/serialization/ObjectMappers.kt +++ b/keel-core/src/main/kotlin/com/netflix/spinnaker/keel/serialization/ObjectMappers.kt @@ -1,11 +1,9 @@ package com.netflix.spinnaker.keel.serialization import com.fasterxml.jackson.annotation.JsonInclude.Include.NON_NULL -import com.fasterxml.jackson.core.JsonGenerator -import com.fasterxml.jackson.core.JsonParser -import com.fasterxml.jackson.databind.* import com.fasterxml.jackson.databind.DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES import com.fasterxml.jackson.databind.MapperFeature.ACCEPT_CASE_INSENSITIVE_ENUMS +import com.fasterxml.jackson.databind.ObjectMapper import com.fasterxml.jackson.databind.SerializationFeature.WRITE_DATES_AS_TIMESTAMPS import com.fasterxml.jackson.databind.SerializationFeature.WRITE_DATES_WITH_ZONE_ID import com.fasterxml.jackson.databind.SerializationFeature.WRITE_DATE_KEYS_AS_TIMESTAMPS @@ -15,14 +13,11 @@ import com.fasterxml.jackson.databind.ser.std.ToStringSerializer import com.fasterxml.jackson.dataformat.yaml.YAMLGenerator.Feature.USE_NATIVE_TYPE_ID import com.fasterxml.jackson.dataformat.yaml.YAMLMapper import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule -import com.fasterxml.jackson.datatype.jsr310.ser.InstantSerializer import com.fasterxml.jackson.module.kotlin.registerKotlinModule import com.netflix.spinnaker.keel.jackson.registerKeelApiModule import de.huxhorn.sulky.ulid.ULID -import org.springframework.boot.jackson.JsonComponentModule import java.text.SimpleDateFormat import java.time.Instant -import java.time.format.DateTimeFormatter import java.util.TimeZone /** @@ -34,23 +29,10 @@ fun configuredObjectMapper(): ObjectMapper = ObjectMapper().configureForKeel() * Factory method for [YAMLMapper]s configured how we like 'em. */ fun configuredYamlMapper(): YAMLMapper = YAMLMapper().configureForKeel().disable(USE_NATIVE_TYPE_ID) -class CustomInstantSerializer : JsonSerializer() { - override fun serialize(p0: Instant?, p1: JsonGenerator?, p2: SerializerProvider?) { - p1?.writeObject(DateTimeFormatter.ISO_INSTANT.format(p0)) - } -} - - class CustomInstantDeserializer : JsonDeserializer() { - //KEY here is for the DB to correctly serialize/deserialize date from strings, and so MUST match the supported formatter for str_to_date - // (str_to_date(json->>'$.triggeredAt', '%Y-%m-%dT%T.%fZ')) - override fun deserialize(p0: JsonParser?, p1: DeserializationContext?): Instant { - return Instant.from(DateTimeFormatter.ISO_INSTANT.parse(p0?.getValueAsString())) - } -} fun T.configureForKeel(): T { val javaTimeModule = JavaTimeModule() -// javaTimeModule.addSerializer(Instant::class.java, CustomInstantSerializer()) -// javaTimeModule.addDeserializer(Instant::class.java, CustomInstantDeserializer()) + javaTimeModule.addDeserializer(Instant::class.java, PrecisionSqlDeserializer()) + javaTimeModule.addSerializer(Instant::class.java,PrecisionSqlSerializer()) return apply { registerKeelApiModule() .registerKotlinModule() diff --git a/keel-sql/src/main/kotlin/com/netflix/spinnaker/config/SqlConfiguration.kt b/keel-sql/src/main/kotlin/com/netflix/spinnaker/config/SqlConfiguration.kt index 0a79132661..63c7713546 100644 --- a/keel-sql/src/main/kotlin/com/netflix/spinnaker/config/SqlConfiguration.kt +++ b/keel-sql/src/main/kotlin/com/netflix/spinnaker/config/SqlConfiguration.kt @@ -1,16 +1,11 @@ package com.netflix.spinnaker.config -import com.fasterxml.jackson.core.JsonGenerator -import com.fasterxml.jackson.core.JsonParser -import com.fasterxml.jackson.databind.* -import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule +import com.fasterxml.jackson.databind.ObjectMapper import com.netflix.spectator.api.Registry import com.netflix.spinnaker.keel.api.plugins.ArtifactSupplier import com.netflix.spinnaker.keel.events.PersistentEvent.Companion.clock import com.netflix.spinnaker.keel.resources.ResourceFactory import com.netflix.spinnaker.keel.scheduled.ScheduledAgent -import com.netflix.spinnaker.keel.serialization.CustomInstantDeserializer -import com.netflix.spinnaker.keel.serialization.CustomInstantSerializer import com.netflix.spinnaker.keel.sql.SqlActionRepository import com.netflix.spinnaker.keel.sql.SqlAgentLockRepository import com.netflix.spinnaker.keel.sql.SqlArtifactRepository @@ -45,18 +40,14 @@ import org.springframework.context.annotation.Configuration import org.springframework.context.annotation.Import import org.springframework.core.env.Environment import java.time.Clock -import java.time.Instant -import java.time.format.DateTimeFormatter import javax.annotation.PostConstruct @Configuration @ConditionalOnProperty("sql.enabled") @EnableConfigurationProperties(RetentionProperties::class) @Import(DefaultSqlConfiguration::class, SqlRetryProperties::class, EnvironmentExclusionConfig::class) -class SqlConfiguration { - - - +class SqlConfiguration +{ @Autowired lateinit var jooqConfiguration: DefaultConfiguration @@ -76,15 +67,6 @@ class SqlConfiguration { jooqConfiguration.settings().isRenderSchema = false } - fun overloadMapper(mapper: ObjectMapper): ObjectMapper { - val copy = mapper.copy() - val timeModule = JavaTimeModule() - timeModule.addSerializer(Instant::class.java, CustomInstantSerializer()) - timeModule.addDeserializer(Instant::class.java, CustomInstantDeserializer()) - copy.registerModule(timeModule) - return copy - } - @Bean fun resourceRepository( jooq: DSLContext, @@ -98,7 +80,7 @@ class SqlConfiguration { SqlResourceRepository( jooq, clock, - overloadMapper(objectMapper), + objectMapper, resourceFactory, SqlRetry(sqlRetryProperties), publisher, @@ -117,7 +99,7 @@ class SqlConfiguration { SqlArtifactRepository( jooq, clock, - overloadMapper(objectMapper), + objectMapper, SqlRetry(sqlRetryProperties), artifactSuppliers, publisher @@ -136,7 +118,7 @@ class SqlConfiguration { jooq = jooq, clock = clock, resourceFactory = resourceFactory, - objectMapper = overloadMapper(objectMapper), + objectMapper = objectMapper, sqlRetry = SqlRetry(sqlRetryProperties), artifactSuppliers = artifactSuppliers, publisher = publisher @@ -199,7 +181,7 @@ class SqlConfiguration { ) = SqlActionRepository( jooq, clock, - overloadMapper(objectMapper), + objectMapper, resourceFactory, SqlRetry(sqlRetryProperties), artifactSuppliers, @@ -223,7 +205,7 @@ class SqlConfiguration { clock: Clock, properties: SqlProperties, objectMapper: ObjectMapper - ) = SqlLifecycleMonitorRepository(jooq, clock, overloadMapper(objectMapper), SqlRetry(sqlRetryProperties)) + ) = SqlLifecycleMonitorRepository(jooq, clock, objectMapper, SqlRetry(sqlRetryProperties)) @Bean fun bakedImageRepository( @@ -231,7 +213,7 @@ class SqlConfiguration { clock: Clock, properties: SqlProperties, objectMapper: ObjectMapper - ) = SqlBakedImageRepository(jooq, clock, overloadMapper(objectMapper), SqlRetry(sqlRetryProperties)) + ) = SqlBakedImageRepository(jooq, clock, objectMapper, SqlRetry(sqlRetryProperties)) @Bean fun environmentLeaseRepository( @@ -257,7 +239,7 @@ class SqlConfiguration { ) = SqlEnvironmentDeletionRepository( jooq, clock, - overloadMapper(objectMapper), + objectMapper, SqlRetry(sqlRetryProperties), resourceFactory, artifactSuppliers @@ -268,7 +250,7 @@ class SqlConfiguration { jooq: DSLContext, clock: Clock, objectMapper: ObjectMapper - ) = SqlWorkQueueRepository(jooq, clock, overloadMapper(objectMapper), SqlRetry(sqlRetryProperties)) + ) = SqlWorkQueueRepository(jooq, clock, objectMapper, SqlRetry(sqlRetryProperties)) @Bean fun featureRolloutRepository( From 2d003a989a1982938e712481fb754213406a9942 Mon Sep 17 00:00:00 2001 From: Jason McIntosh Date: Thu, 14 Nov 2024 15:12:48 -0600 Subject: [PATCH 10/15] fix(sql): Use less precision on comparison to an instant now --- .../spinnaker/keel/persistence/ArtifactRepositoryTests.kt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/keel-core-test/src/main/kotlin/com/netflix/spinnaker/keel/persistence/ArtifactRepositoryTests.kt b/keel-core-test/src/main/kotlin/com/netflix/spinnaker/keel/persistence/ArtifactRepositoryTests.kt index fce0c44a3c..897ee3de7f 100644 --- a/keel-core-test/src/main/kotlin/com/netflix/spinnaker/keel/persistence/ArtifactRepositoryTests.kt +++ b/keel-core-test/src/main/kotlin/com/netflix/spinnaker/keel/persistence/ArtifactRepositoryTests.kt @@ -54,6 +54,7 @@ import strikt.assertions.isNull import strikt.assertions.isTrue import java.time.Clock import java.time.Instant +import java.time.temporal.ChronoUnit abstract class ArtifactRepositoryTests : JUnit5Minutests { val publisher: ApplicationEventPublisher = mockk(relaxed = true) @@ -649,7 +650,7 @@ abstract class ArtifactRepositoryTests : JUnit5Minutests } context("artifact creation timestamp exists") { - val createdAt = Instant.now() + val createdAt = Instant.now().truncatedTo(ChronoUnit.MILLIS) before { subject.register(versionedSnapshotDebian) From 25f5b51c29dae9cdbc549db5765dc12c7b914d41 Mon Sep 17 00:00:00 2001 From: Jason McIntosh Date: Thu, 14 Nov 2024 15:29:59 -0600 Subject: [PATCH 11/15] fix(sql): Test ONLY with serializer vs serial and deserializer --- .../com/netflix/spinnaker/keel/serialization/ObjectMappers.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/keel-core/src/main/kotlin/com/netflix/spinnaker/keel/serialization/ObjectMappers.kt b/keel-core/src/main/kotlin/com/netflix/spinnaker/keel/serialization/ObjectMappers.kt index e041f2caf0..57b9f5db25 100644 --- a/keel-core/src/main/kotlin/com/netflix/spinnaker/keel/serialization/ObjectMappers.kt +++ b/keel-core/src/main/kotlin/com/netflix/spinnaker/keel/serialization/ObjectMappers.kt @@ -31,7 +31,7 @@ fun configuredObjectMapper(): ObjectMapper = ObjectMapper().configureForKeel() fun configuredYamlMapper(): YAMLMapper = YAMLMapper().configureForKeel().disable(USE_NATIVE_TYPE_ID) fun T.configureForKeel(): T { val javaTimeModule = JavaTimeModule() - javaTimeModule.addDeserializer(Instant::class.java, PrecisionSqlDeserializer()) +// javaTimeModule.addDeserializer(Instant::class.java, PrecisionSqlDeserializer()) javaTimeModule.addSerializer(Instant::class.java,PrecisionSqlSerializer()) return apply { registerKeelApiModule() From 5a56da2eb06f4d980686d5c719c7dd8a23751481 Mon Sep 17 00:00:00 2001 From: Jason McIntosh Date: Thu, 14 Nov 2024 15:44:14 -0600 Subject: [PATCH 12/15] fix(sql): Now with tests passing remove the extraneous code --- .../PrecisionSqlDeserializer.java | 18 ------------------ .../keel/serialization/ObjectMappers.kt | 1 - 2 files changed, 19 deletions(-) delete mode 100644 keel-core/src/main/java/com/netflix/spinnaker/keel/serialization/PrecisionSqlDeserializer.java diff --git a/keel-core/src/main/java/com/netflix/spinnaker/keel/serialization/PrecisionSqlDeserializer.java b/keel-core/src/main/java/com/netflix/spinnaker/keel/serialization/PrecisionSqlDeserializer.java deleted file mode 100644 index 10dfd495ba..0000000000 --- a/keel-core/src/main/java/com/netflix/spinnaker/keel/serialization/PrecisionSqlDeserializer.java +++ /dev/null @@ -1,18 +0,0 @@ -package com.netflix.spinnaker.keel.serialization; - -import com.fasterxml.jackson.datatype.jsr310.deser.InstantDeserializer; -import java.time.Instant; -import java.time.format.DateTimeFormatter; - -public class PrecisionSqlDeserializer extends InstantDeserializer { - public PrecisionSqlDeserializer() { - super( - Instant.class, - DateTimeFormatter.ISO_INSTANT, - Instant::from, - (a) -> Instant.ofEpochMilli(a.value), - (a) -> Instant.ofEpochSecond(a.integer, a.fraction), - null, - true); - } -} diff --git a/keel-core/src/main/kotlin/com/netflix/spinnaker/keel/serialization/ObjectMappers.kt b/keel-core/src/main/kotlin/com/netflix/spinnaker/keel/serialization/ObjectMappers.kt index 57b9f5db25..da9b069c1b 100644 --- a/keel-core/src/main/kotlin/com/netflix/spinnaker/keel/serialization/ObjectMappers.kt +++ b/keel-core/src/main/kotlin/com/netflix/spinnaker/keel/serialization/ObjectMappers.kt @@ -31,7 +31,6 @@ fun configuredObjectMapper(): ObjectMapper = ObjectMapper().configureForKeel() fun configuredYamlMapper(): YAMLMapper = YAMLMapper().configureForKeel().disable(USE_NATIVE_TYPE_ID) fun T.configureForKeel(): T { val javaTimeModule = JavaTimeModule() -// javaTimeModule.addDeserializer(Instant::class.java, PrecisionSqlDeserializer()) javaTimeModule.addSerializer(Instant::class.java,PrecisionSqlSerializer()) return apply { registerKeelApiModule() From 4a823d5c91c4f1714797d055be4538c80ad36b8d Mon Sep 17 00:00:00 2001 From: Jason McIntosh Date: Thu, 14 Nov 2024 20:53:52 -0600 Subject: [PATCH 13/15] fix(sql): Document the changes --- .../persistence/ArtifactRepositoryTests.kt | 3 +++ .../serialization/PrecisionSqlSerializer.java | 22 +++++++++++++++++++ 2 files changed, 25 insertions(+) diff --git a/keel-core-test/src/main/kotlin/com/netflix/spinnaker/keel/persistence/ArtifactRepositoryTests.kt b/keel-core-test/src/main/kotlin/com/netflix/spinnaker/keel/persistence/ArtifactRepositoryTests.kt index 897ee3de7f..0ed764fcee 100644 --- a/keel-core-test/src/main/kotlin/com/netflix/spinnaker/keel/persistence/ArtifactRepositoryTests.kt +++ b/keel-core-test/src/main/kotlin/com/netflix/spinnaker/keel/persistence/ArtifactRepositoryTests.kt @@ -650,6 +650,9 @@ abstract class ArtifactRepositoryTests : JUnit5Minutests } context("artifact creation timestamp exists") { + // We truncate this since we're using a serialization to java that reduces the level of precision + // and later comparisons break otherwise. This is needed to work with generated columns in + // certain databases. See the PrecisionSqlSerializer class for more info val createdAt = Instant.now().truncatedTo(ChronoUnit.MILLIS) before { diff --git a/keel-core/src/main/java/com/netflix/spinnaker/keel/serialization/PrecisionSqlSerializer.java b/keel-core/src/main/java/com/netflix/spinnaker/keel/serialization/PrecisionSqlSerializer.java index 07772b413d..839320652d 100644 --- a/keel-core/src/main/java/com/netflix/spinnaker/keel/serialization/PrecisionSqlSerializer.java +++ b/keel-core/src/main/java/com/netflix/spinnaker/keel/serialization/PrecisionSqlSerializer.java @@ -6,6 +6,28 @@ import java.time.format.DateTimeFormatter; import java.time.format.DateTimeFormatterBuilder; +/** + * This class overloads the default Serialization instance in the JavaTimeModule to a precision + * level that the database can process via str_to_date string conversion It's needed because we do + * JSON serialization to a column and then the column gets used to generate another field via string + * conversion. Specifically * triggered_at * dismissed_at are "generated" columns. NOTE this should + * ONLY impact running on Linux as the system clock on Linux systems returns a higher precision in + * Java 17+. It's also impactful as ISO_INSTANT has NO precision limitation giving inconsistent + * string output as a result + * + *

See + * + *

    + *
  • https://stackoverflow.com/questions/74781495/changes-in-instant-now-between-java-11-and-java-17-in-aws-ubuntu-standard-6-0 + *
  • https://stackoverflow.com/a/38042457 for more information. + *
+ * + *

NOTE we're going to 3 digits, but could go to 6 if but NO larger than 6 due to Mysql str_to_date function + * limits on microseconds parsing. This is used in the SQL definition: + * add column triggered_at datetime(3) generated always as (str_to_date(json->>'$.triggeredAt', '%Y-%m-%dT%T.%fZ')) + * column in the 20210616-create-dismissible-notifications.yml liquibase change set. + */ public class PrecisionSqlSerializer extends InstantSerializerBase { public PrecisionSqlSerializer() { super( From dbe8e0bc8ef073238d2bc38173f58f8f6609fa29 Mon Sep 17 00:00:00 2001 From: Jason McIntosh Date: Fri, 15 Nov 2024 10:00:00 -0600 Subject: [PATCH 14/15] fix(sql): Add test verifying precision and adjust precision to 6 to match mysql --- .../serialization/PrecisionSqlSerializer.java | 2 +- .../ObjectMapperSerializationTests.kt | 34 +++++++++++++++++++ 2 files changed, 35 insertions(+), 1 deletion(-) create mode 100644 keel-core/src/test/kotlin/com/netflix/spinnaker/keel/serialization/ObjectMapperSerializationTests.kt diff --git a/keel-core/src/main/java/com/netflix/spinnaker/keel/serialization/PrecisionSqlSerializer.java b/keel-core/src/main/java/com/netflix/spinnaker/keel/serialization/PrecisionSqlSerializer.java index 839320652d..79dcd5c5dc 100644 --- a/keel-core/src/main/java/com/netflix/spinnaker/keel/serialization/PrecisionSqlSerializer.java +++ b/keel-core/src/main/java/com/netflix/spinnaker/keel/serialization/PrecisionSqlSerializer.java @@ -35,7 +35,7 @@ public PrecisionSqlSerializer() { Instant::toEpochMilli, Instant::getEpochSecond, Instant::getNano, - new DateTimeFormatterBuilder().appendInstant(3).toFormatter()); + new DateTimeFormatterBuilder().appendInstant(6).toFormatter()); } @Override diff --git a/keel-core/src/test/kotlin/com/netflix/spinnaker/keel/serialization/ObjectMapperSerializationTests.kt b/keel-core/src/test/kotlin/com/netflix/spinnaker/keel/serialization/ObjectMapperSerializationTests.kt new file mode 100644 index 0000000000..4d75665a4e --- /dev/null +++ b/keel-core/src/test/kotlin/com/netflix/spinnaker/keel/serialization/ObjectMapperSerializationTests.kt @@ -0,0 +1,34 @@ +package com.netflix.spinnaker.keel.serialization + +import dev.minutest.junit.JUnit5Minutests +import dev.minutest.rootContext +import org.assertj.core.api.Assertions.assertThat +import java.time.Instant +import java.time.LocalDateTime +import java.time.OffsetDateTime +import java.time.ZoneOffset +import java.time.format.DateTimeFormatter +import java.time.format.DateTimeFormatterBuilder + +class ObjectMapperSerializationTests : JUnit5Minutests { + + class ObjectMapperSerializationTestsObject { + var date = Instant.from(DateTimeFormatter.ISO_INSTANT.parse("2024-03-10T12:28:19.228816801Z")) + } + + fun tests() = rootContext { + test("Verify instant precision is no more than 6 characters") { + val mapper = configuredObjectMapper() + val sampleObject = ObjectMapperSerializationTestsObject() + val readBack = mapper.readValue( + mapper.writeValueAsString(sampleObject), + ObjectMapperSerializationTestsObject::class.java + ) + // 6 precision is the max allowed via str_to_date in SQL. See + // https://dev.mysql.com/doc/refman/8.4/en/date-and-time-type-syntax.html#:~:text=MySQL%20permits%20fractional%20seconds%20for,microseconds%20(6%20digits)%20precision. + // SO we want to make sure waht we store matches this + val foramt = DateTimeFormatterBuilder().parseCaseInsensitive().appendInstant(6).parseStrict().toFormatter() + assertThat(readBack.date).isEqualTo(foramt.format(sampleObject.date)) + } + } +} From f379420c996dc43ee0e1552bcaeb525933d6d551 Mon Sep 17 00:00:00 2001 From: Jason McIntosh Date: Fri, 15 Nov 2024 10:58:19 -0600 Subject: [PATCH 15/15] fix(sql): Adjust comment and change to MICROS for tests --- .../spinnaker/keel/persistence/ArtifactRepositoryTests.kt | 2 +- .../spinnaker/keel/serialization/PrecisionSqlSerializer.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/keel-core-test/src/main/kotlin/com/netflix/spinnaker/keel/persistence/ArtifactRepositoryTests.kt b/keel-core-test/src/main/kotlin/com/netflix/spinnaker/keel/persistence/ArtifactRepositoryTests.kt index 0ed764fcee..ef920c0821 100644 --- a/keel-core-test/src/main/kotlin/com/netflix/spinnaker/keel/persistence/ArtifactRepositoryTests.kt +++ b/keel-core-test/src/main/kotlin/com/netflix/spinnaker/keel/persistence/ArtifactRepositoryTests.kt @@ -653,7 +653,7 @@ abstract class ArtifactRepositoryTests : JUnit5Minutests // We truncate this since we're using a serialization to java that reduces the level of precision // and later comparisons break otherwise. This is needed to work with generated columns in // certain databases. See the PrecisionSqlSerializer class for more info - val createdAt = Instant.now().truncatedTo(ChronoUnit.MILLIS) + val createdAt = Instant.now().truncatedTo(ChronoUnit.MICROS) before { subject.register(versionedSnapshotDebian) diff --git a/keel-core/src/main/java/com/netflix/spinnaker/keel/serialization/PrecisionSqlSerializer.java b/keel-core/src/main/java/com/netflix/spinnaker/keel/serialization/PrecisionSqlSerializer.java index 79dcd5c5dc..cdfaeaaae4 100644 --- a/keel-core/src/main/java/com/netflix/spinnaker/keel/serialization/PrecisionSqlSerializer.java +++ b/keel-core/src/main/java/com/netflix/spinnaker/keel/serialization/PrecisionSqlSerializer.java @@ -22,7 +22,7 @@ *

  • https://stackoverflow.com/a/38042457 for more information. * * - *

    NOTE we're going to 3 digits, but could go to 6 if but NO larger than 6 due to NOTE we're going to 6 digits to match Mysql str_to_date function * limits on microseconds parsing. This is used in the SQL definition: * add column triggered_at datetime(3) generated always as (str_to_date(json->>'$.triggeredAt', '%Y-%m-%dT%T.%fZ'))