Additional format support and fixes to serializer selection

documentation.md

@@ -427,8 +427,8 @@ server. You can add it to your server like any other path:
 
     val auth = AuthEndpoints(
         path = path("auth"),
-        userSerializer = Serialization.module.serializer(),
-        idSerializer = Serialization.module.serializer(),
+        userSerializer = Serialization.module.contextualSerializerIfHandled(),
+        idSerializer = Serialization.module.contextualSerializerIfHandled(),
         authRequirement = AuthInfo<User>(),
         jwtSigner = userSigner,
         email = email,

gradle/serverlibs.versions.toml

@@ -3,10 +3,10 @@ agp="8.2.2"
 androidDesugaring="2.0.4"
 angusMail="2.0.3"
 awsVersion="2.25.24"
-awssdk="2.17.232"
+awssdk="2.25.24"
 azureFunctions="3.1.0"
 azureStorage="12.25.3"
-bouncyCastle="1.77"
+bouncyCastle="1.78.1"
 bson="5.1.0"
 coroutines="1.8.1"
 dokka="1.9.20"
@@ -20,10 +20,10 @@ graalVmNative="0.9.24"
 guava="33.0.0-jre"
 hierynomusSshj="0.38.0"
 javaJwt="4.4.0"
-kaml="0.58.0"
+kaml="0.61.0"
 kbson="0.5.0"
 kotlin="2.0.0"
-kotlinXSerialization="1.6.3"
+kotlinXSerialization="1.7.1"
 kotlinXDatetime="0.6.0"
 kotlinHtmlJvm="0.11.0"
 kotlinerCli="1.0.3"
@@ -40,12 +40,13 @@ oneTimePass="2.4.0"
 orgCrac="0.1.3"
 postgresql="42.7.3"
 proguard="7.3.2"
+#noinspection GradleDependency,OutdatedLibrary
 sentry9="1.7.30"
 sentry="7.9.0"
-serializationLibs="1.6.3"
+serializationLibs="1.7.1"
 serializationXmlUtil="0.86.3"
-shadow="7.1.0"
-xmlUtilJvm="0.86.3"
+shadow="8.1.1"
+xmlUtilJvm="0.90.1"
 clamAv="2.1.2"
 apacheTika="2.9.2"
 
@@ -120,6 +121,7 @@ sentry9Logback={module="io.sentry:sentry-logback", version.ref="sentry9"}
 sentry={module="io.sentry:sentry", version.ref="sentry"}
 serialization={module="org.jetbrains.kotlin:kotlin-serialization", version.ref="kotlin"}
 serializationCbor={module="org.jetbrains.kotlinx:kotlinx-serialization-cbor", version.ref="serializationLibs"}
+serializationProtobuf={module="org.jetbrains.kotlinx:kotlinx-serialization-protobuf", version.ref="serializationLibs"}
 serializationProperties={module="org.jetbrains.kotlinx:kotlinx-serialization-properties", version.ref="serializationLibs"}
 xmlUtilJvm={module="io.github.pdvrieze.xmlutil:serialization-jvm", version.ref="xmlUtilJvm"}
 

server-core/build.gradle.kts

@@ -30,6 +30,7 @@ dependencies {
     api(serverlibs.mongoBson)
     api(serverlibs.kBson)
     api(serverlibs.kaml)
+    api(serverlibs.serializationProtobuf)
     api(serverlibs.kotlinReflect)
     implementation(serverlibs.bouncyCastleBcprov)
     implementation(serverlibs.bouncyCastleBcpkix)

server-core/src/main/kotlin/com/lightningkite/lightningdb/Mask.kt

@@ -1,5 +1,6 @@
 package com.lightningkite.lightningdb
 
+import com.lightningkite.lightningserver.serialization.Serialization
 import kotlinx.serialization.KSerializer
 import kotlinx.serialization.Serializable
 import org.slf4j.LoggerFactory
@@ -83,7 +84,7 @@ data class Mask<T>(
 }
 
 inline fun <reified T> mask(builder: Mask.Builder<T>.(DataClassPath<T, T>)->Unit): Mask<T> {
-    return Mask.Builder<T>(serializerOrContextual()).apply { builder(path()) }.build()
+    return Mask.Builder<T>(Serialization.module.contextualSerializerIfHandled()).apply { builder(path()) }.build()
 }
 
 operator fun <T> Condition<T>.invoke(map: Partial<T>): Boolean? {

server-core/src/main/kotlin/com/lightningkite/lightningdb/UpdateRestrictions.kt

@@ -1,5 +1,6 @@
 package com.lightningkite.lightningdb
 
+import com.lightningkite.lightningserver.serialization.Serialization
 import kotlinx.serialization.KSerializer
 import kotlinx.serialization.Serializable
 
@@ -72,5 +73,5 @@ data class UpdateRestrictions<T>(
  * DSL for defining [UpdateRestrictions]
  */
 inline fun <reified T> updateRestrictions(builder: UpdateRestrictions.Builder<T>.(DataClassPath<T, T>)->Unit): UpdateRestrictions<T> {
-    return UpdateRestrictions.Builder<T>(serializerOrContextual()).apply { builder(path<T>()) }.build()
+    return UpdateRestrictions.Builder<T>(Serialization.module.contextualSerializerIfHandled()).apply { builder(path<T>()) }.build()
 }

server-core/src/main/kotlin/com/lightningkite/lightningserver/auth/oauth/OauthCallbackEndpoint.kt

@@ -1,10 +1,10 @@
 package com.lightningkite.lightningserver.auth.oauth
 
+import com.lightningkite.lightningdb.contextualSerializerIfHandled
 import com.lightningkite.lightningserver.core.ServerPath
 import com.lightningkite.lightningserver.http.*
 import com.lightningkite.lightningserver.serialization.*
 import kotlinx.serialization.KSerializer
-import kotlinx.serialization.serializer
 
 class OauthCallbackEndpoint<STATE>(
     path: ServerPath,
@@ -67,7 +67,7 @@ inline fun <reified STATE> ServerPath.oauthCallback(
     },
     noinline onAccess: suspend (OauthResponse, STATE) -> HttpResponse
 ) = OauthCallbackEndpoint(
-    stateSerializer = Serialization.module.serializer<STATE>(),
+    stateSerializer = Serialization.module.contextualSerializerIfHandled<STATE>(),
     path = this,
     oauthProviderInfo = oauthProviderInfo,
     credentials = credentials,

server-core/src/main/kotlin/com/lightningkite/lightningserver/auth/oauth/OauthClientEndpoints.kt

@@ -4,15 +4,13 @@ import com.lightningkite.lightningdb.*
 import com.lightningkite.lightningserver.auth.*
 import com.lightningkite.lightningserver.core.ServerPath
 import com.lightningkite.lightningserver.core.ServerPathGroup
-import com.lightningkite.lightningserver.db.ModelInfoWithDefault
 import com.lightningkite.lightningserver.db.ModelRestEndpoints
 import com.lightningkite.lightningserver.db.ModelSerializationInfo
 import com.lightningkite.lightningserver.db.modelInfoWithDefault
 import com.lightningkite.lightningserver.encryption.secureHash
 import com.lightningkite.lightningserver.exceptions.NotFoundException
 import com.lightningkite.lightningserver.serialization.Serialization
 import com.lightningkite.lightningserver.typed.*
-import kotlinx.serialization.serializer
 import java.util.Base64
 import kotlin.random.Random
 
@@ -32,8 +30,8 @@ class OauthClientEndpoints(
 
     val modelInfo = modelInfoWithDefault(
         serialization = ModelSerializationInfo<OauthClient, String>(
-            serializer = Serialization.module.serializer(),
-            idSerializer = Serialization.module.serializer()
+            serializer = Serialization.module.contextualSerializerIfHandled(),
+            idSerializer = Serialization.module.contextualSerializerIfHandled()
         ),
         authOptions = @Suppress("UNCHECKED_CAST") (maintainPermissions as AuthOptions<HasId<*>>) + noAuth,
         getBaseCollection = { database().collection<OauthClient>() },

server-core/src/main/kotlin/com/lightningkite/lightningserver/auth/old/JwtSigner.kt

@@ -2,6 +2,7 @@
 
 package com.lightningkite.lightningserver.auth
 
+import com.lightningkite.lightningdb.contextualSerializerIfHandled
 import com.lightningkite.lightningserver.encryption.*
 import com.lightningkite.lightningserver.encryption.SecureHasher.*
 import com.lightningkite.lightningserver.exceptions.UnauthorizedException
@@ -96,7 +97,7 @@ data class JwtSigner(
         ReplaceWith("token(subject, Duration.ofMillis(expireDuration))", "java.time.Duration")
     )
     inline fun <reified T> token(subject: T, expireDuration: Long): String =
-        token(Serialization.module.serializer(), subject, expireDuration.milliseconds)
+        token(Serialization.module.contextualSerializerIfHandled(), subject, expireDuration.milliseconds)
 
     @Deprecated(
         "Use the version with duration instead",
@@ -106,7 +107,7 @@ data class JwtSigner(
         token(serializer, subject, expireDuration.milliseconds)
 
     inline fun <reified T> token(subject: T, expireDuration: Duration = expiration): String =
-        token(Serialization.module.serializer(), subject, expireDuration)
+        token(Serialization.module.contextualSerializerIfHandled(), subject, expireDuration)
 
     fun <T> token(serializer: KSerializer<T>, subject: T, expireDuration: Duration = expiration): String {
         return hasher.signJwt(JwtClaims(
@@ -119,7 +120,7 @@ data class JwtSigner(
         ))
     }
 
-    inline fun <reified T> verify(token: String): T = verify(Serialization.module.serializer(), token)
+    inline fun <reified T> verify(token: String): T = verify(Serialization.module.contextualSerializerIfHandled(), token)
     fun <T> verify(serializer: KSerializer<T>, token: String): T {
         return try {
             hasher.verifyJwt(token, audience)!!.sub!!.let {

server-core/src/main/kotlin/com/lightningkite/lightningserver/cache/CacheHandle.kt

@@ -1,8 +1,8 @@
 package com.lightningkite.lightningserver.cache
 
+import com.lightningkite.lightningdb.contextualSerializerIfHandled
 import com.lightningkite.lightningserver.serialization.Serialization
 import kotlinx.serialization.KSerializer
-import kotlinx.serialization.serializer
 import kotlin.time.Duration
 
 /**
@@ -25,4 +25,4 @@ class CacheHandle<T>(val cache: () -> Cache, val key: String, val serializer: KS
 }
 
 inline operator fun <reified T> (() -> Cache).get(key: String) =
-    CacheHandle<T>(this, key, Serialization.module.serializer())
+    CacheHandle<T>(this, key, Serialization.module.contextualSerializerIfHandled())

server-core/src/main/kotlin/com/lightningkite/lightningserver/core/ContentType.kt

@@ -51,6 +51,7 @@ class ContentType(val type: String, val subtype: String, val parameters: Map<Str
         public val HalJson: ContentType = ContentType("application", "hal+json")
         public val JavaScript: ContentType = ContentType("application", "javascript")
         public val OctetStream: ContentType = ContentType("application", "octet-stream")
+        public val StructuredBytes: ContentType = ContentType("application", "x-structured-bytes")
         public val FontWoff: ContentType = ContentType("application", "font-woff")
         public val Rss: ContentType = ContentType("application", "rss+xml")
         public val Xml: ContentType = ContentType("application", "xml")
@@ -75,6 +76,8 @@ class ContentType(val type: String, val subtype: String, val parameters: Map<Str
             "vnd.openxmlformats-officedocument.presentationml.presentation"
         )
         public val ProtoBuf: ContentType = ContentType("application", "protobuf")
+        public val ProtoBufText: ContentType = ContentType("application", "x-protobuf-text")
+        public val ProtoBufDeclaration: ContentType = ContentType("application", "x-protobuf-declaration")
         public val Wasm: ContentType = ContentType("application", "wasm")
         public val ProblemJson: ContentType = ContentType("application", "problem+json")
         public val ProblemXml: ContentType = ContentType("application", "problem+xml")
@@ -148,6 +151,7 @@ class ContentType(val type: String, val subtype: String, val parameters: Map<Str
         public val Xml: ContentType = ContentType("text", "xml", mapOf("charset" to "UTF-8"))
         public val EventStream: ContentType = ContentType("text", "event-stream", mapOf("charset" to "UTF-8"))
         public val UriList: ContentType = ContentType("text", "uri-list", mapOf("charset" to "UTF-8"))
+        public val Yaml: ContentType = ContentType("text", "vnd.yaml", mapOf("charset" to "UTF-8"))
     }
 
     /**

server-core/src/main/kotlin/com/lightningkite/lightningserver/db/ModelInfo.kt

@@ -4,16 +4,14 @@ import com.lightningkite.lightningdb.*
 import com.lightningkite.lightningserver.auth.AuthOptions
 import com.lightningkite.lightningserver.serialization.Serialization
 import com.lightningkite.lightningserver.typed.AuthAccessor
-import kotlinx.serialization.KSerializer
-import kotlinx.serialization.serializer
 
 @Suppress("DEPRECATION")
 @Deprecated("User newer version with auth accessor instead, as it enables more potential optimizations.")
 inline fun <reified USER : HasId<*>, reified T : HasId<ID>, reified ID : Comparable<ID>> ModelInfo(
     noinline getCollection: () -> FieldCollection<T>,
     noinline getBaseCollection: () -> FieldCollection<T> = { getCollection() },
     noinline forUser: suspend FieldCollection<T>.(principal: USER) -> FieldCollection<T>,
-    modelName: String = Serialization.module.serializer<T>().descriptor.serialName.substringBefore('<')
+    modelName: String = Serialization.module.contextualSerializerIfHandled<T>().descriptor.serialName.substringBefore('<')
         .substringAfterLast('.'),
 ) = ModelInfo(
     serialization = ModelSerializationInfo<T, ID>(),

server-core/src/main/kotlin/com/lightningkite/lightningserver/db/ModelInfoWithDefault.kt

@@ -1,22 +1,18 @@
 package com.lightningkite.lightningserver.db
 
-import com.lightningkite.lightningdb.CollectionChanges
-import com.lightningkite.lightningdb.FieldCollection
-import com.lightningkite.lightningdb.HasId
-import com.lightningkite.lightningdb.withChangeListeners
+import com.lightningkite.lightningdb.*
 import com.lightningkite.lightningserver.auth.AuthOptions
 import com.lightningkite.lightningserver.auth.RequestAuth
 import com.lightningkite.lightningserver.serialization.Serialization
 import com.lightningkite.lightningserver.typed.AuthAccessor
-import kotlinx.serialization.serializer
 
 @Suppress("DEPRECATION")
 @Deprecated("User newer version with auth accessor instead, as it enables more potential optimizations.")
 inline fun <reified USER: HasId<*>?, reified T : HasId<ID>, reified ID : Comparable<ID>> ModelInfoWithDefault(
     noinline getCollection: () -> FieldCollection<T>,
     noinline getBaseCollection: () -> FieldCollection<T> = { getCollection() },
     noinline forUser: suspend FieldCollection<T>.(principal: USER) -> FieldCollection<T>,
-    modelName: String = Serialization.module.serializer<T>().descriptor.serialName.substringBefore('<').substringAfterLast('.'),
+    modelName: String = Serialization.module.contextualSerializerIfHandled<T>().descriptor.serialName.substringBefore('<').substringAfterLast('.'),
     noinline defaultItem: suspend (auth: USER) -> T,
     noinline exampleItem: ()->T? = { null },
 ): ModelInfoWithDefault<USER, T, ID> = ModelInfoWithDefault(

server-core/src/main/kotlin/com/lightningkite/lightningserver/db/ModelSerializationInfo.kt

@@ -1,15 +1,19 @@
 package com.lightningkite.lightningserver.db
 
 import com.lightningkite.lightningdb.HasId
+import com.lightningkite.lightningdb.serializableProperties
+import com.lightningkite.lightningdb.contextualSerializerIfHandled
 import com.lightningkite.lightningserver.serialization.Serialization
 import kotlinx.serialization.KSerializer
-import kotlinx.serialization.serializer
 
-inline fun <reified T : HasId<ID>, reified ID : Comparable<ID>> ModelSerializationInfo() =
-    ModelSerializationInfo<T, ID>(
-        serializer = Serialization.module.serializer(),
-        idSerializer = Serialization.module.serializer(),
+inline fun <reified T : HasId<ID>, reified ID : Comparable<ID>> ModelSerializationInfo(): ModelSerializationInfo<T, ID> {
+    val ser = Serialization.module.contextualSerializerIfHandled<T>()
+    @Suppress("UNCHECKED_CAST")
+    return ModelSerializationInfo<T, ID>(
+        serializer = ser,
+        idSerializer = (ser.serializableProperties?.find { it.name == "_id" }?.serializer as? KSerializer<ID>) ?: Serialization.module.contextualSerializerIfHandled(),
     )
+}
 
 data class ModelSerializationInfo<T : HasId<ID>, ID : Comparable<ID>>(
     val serializer: KSerializer<T>,

server-core/src/main/kotlin/com/lightningkite/lightningserver/exceptions/GroupedDatabaseExceptionReporter.kt

@@ -11,7 +11,6 @@ import com.lightningkite.lightningserver.serialization.Serialization
 import com.lightningkite.now
 import kotlinx.coroutines.Dispatchers
 import kotlinx.coroutines.withContext
-import kotlinx.serialization.serializer
 import java.net.NetworkInterface
 
 class GroupedDatabaseExceptionReporter(val packageName: String, val database: Database): ExceptionReporter {
@@ -65,8 +64,8 @@ class GroupedDatabaseExceptionReporter(val packageName: String, val database: Da
     @Suppress("UNCHECKED_CAST")
     val modelInfo = modelInfoWithDefault(
         serialization = ModelSerializationInfo<ReportedExceptionGroup, Int>(
-            serializer = Serialization.module.serializer(),
-            idSerializer = Serialization.module.serializer()
+            serializer = Serialization.module.contextualSerializerIfHandled(),
+            idSerializer = Serialization.module.contextualSerializerIfHandled()
         ),
         authOptions = Authentication.isDeveloper as AuthOptions<HasId<*>>,
         getBaseCollection = { database.collection<ReportedExceptionGroup>() },

server-core/src/main/kotlin/com/lightningkite/lightningserver/forms/FormSerialization.kt

@@ -1,6 +1,7 @@
 package com.lightningkite.lightningserver.forms
 
 
+import com.lightningkite.lightningdb.contextualSerializerIfHandled
 import com.lightningkite.lightningserver.serialization.Serialization
 import kotlinx.html.FlowContent
 import kotlinx.serialization.SerializationException
@@ -9,12 +10,11 @@ import kotlinx.serialization.descriptors.SerialDescriptor
 import kotlinx.serialization.encoding.AbstractEncoder
 import kotlinx.serialization.modules.EmptySerializersModule
 import kotlinx.serialization.modules.SerializersModule
-import kotlinx.serialization.serializer
 
 typealias HtmlRenderer<T> = FlowContent.(inputKey: String, value: T) -> Unit
 
 class HtmlSerializer(val serializersModule: SerializersModule = EmptySerializersModule(), val module: Module) {
-    inline fun <reified T> render(value: T, into: FlowContent) = render(Serialization.module.serializer(), value, into)
+    inline fun <reified T> render(value: T, into: FlowContent) = render(Serialization.module.contextualSerializerIfHandled(), value, into)
     fun <T> render(serializer: SerializationStrategy<T>, value: T, into: FlowContent) {
         HtmlEncoder(into).encodeSerializableValue(serializer, value)
     }