KTOR-1795 Fix serialization failure due to types mismatch

21771311 · rsinukov · Rustam · 3a4122a7 · 21771311 · 21771311
--- a/ktor-features/ktor-serialization/jvm/src/io/ktor/serialization/SerializationConverter.kt
+++ b/ktor-features/ktor-serialization/jvm/src/io/ktor/serialization/SerializationConverter.kt
@@ -110,16 +110,39 @@ private constructor(
        contentType: ContentType,
        value: Any
    ): Any? {
-        @Suppress("UNCHECKED_CAST")
-        val serializer = serializerForSending(context, value, format.serializersModule) as KSerializer<Any>
+        val result = try {
+            serializerFromResponseType(context, format.serializersModule)?.let {
+                serializeContent(it, format, value, contentType, context)
+            }
+        } catch (cause: SerializationException) {
+            // can fail due to
+            // 1. https://github.com/Kotlin/kotlinx.serialization/issues/1163)
+            // 2. mismatching between compile-time and runtime types of the response.
+            null
+        }
+        if (result != null) {
+            return result
+        }

+        val guessedSearchSerializer = guessSerializer(value, format.serializersModule)
+        return serializeContent(guessedSearchSerializer, format, value, contentType, context)
+    }
+
+    private fun serializeContent(
+        serializer: KSerializer<*>,
+        format: SerialFormat,
+        value: Any,
+        contentType: ContentType,
+        context: PipelineContext<Any, ApplicationCall>
+    ): OutgoingContent.ByteArrayContent {
+        @Suppress("UNCHECKED_CAST")
        return when (format) {
            is StringFormat -> {
-                val content = format.encodeToString(serializer, value)
+                val content = format.encodeToString(serializer as KSerializer<Any>, value)
                TextContent(content, contentType.withCharset(context.call.suitableCharset()))
            }
            is BinaryFormat -> {
-                val content = format.encodeToByteArray(serializer, value)
+                val content = format.encodeToByteArray(serializer as KSerializer<Any>, value)
                ByteArrayContent(content, contentType)
            }
            else -> error("Unsupported format $format")

--- a/ktor-features/ktor-serialization/jvm/src/io/ktor/serialization/SerializerLookup.kt
+++ b/ktor-features/ktor-serialization/jvm/src/io/ktor/serialization/SerializerLookup.kt
@@ -37,28 +37,29 @@ private fun arraySerializer(type: KType): KSerializer<*> {
    )
 }

-@OptIn(ExperimentalSerializationApi::class)
-internal fun serializerForSending(
+internal fun serializerFromResponseType(
    context: PipelineContext<Any, ApplicationCall>,
+    module: SerializersModule
+): KSerializer<*>? {
+    val responseType = context.call.response.responseType ?: return null
+    return module.serializer(responseType)
+}
+
+internal fun guessSerializer(
    value: Any,
    module: SerializersModule
 ): KSerializer<*> {
-    val responseType = context.call.response.responseType
-    if (responseType != null) try {
-        return module.serializer(responseType)
-    } catch (_: SerializationException) {
-        // right now there is no better way to check if serializer is available (https://github.com/Kotlin/kotlinx.serialization/issues/1163)
-        // if serializer for type was not found, fallback to manual search
-    }
-
    return when (value) {
        is JsonElement -> JsonElement.serializer()
-        is List<*> -> ListSerializer(value.elementSerializer(context, module))
-        is Set<*> -> SetSerializer(value.elementSerializer(context, module))
-        is Map<*, *> -> MapSerializer(value.keys.elementSerializer(context, module), value.values.elementSerializer(context, module))
+        is List<*> -> ListSerializer(value.elementSerializer(module))
+        is Set<*> -> SetSerializer(value.elementSerializer(module))
+        is Map<*, *> -> MapSerializer(
+            value.keys.elementSerializer(module),
+            value.values.elementSerializer(module)
+        )
        is Map.Entry<*, *> -> MapEntrySerializer(
-            serializerForSending(context, value.key ?: error("Map.Entry(null, ...) is not supported"), module),
-            serializerForSending(context, value.value ?: error("Map.Entry(..., null) is not supported)"), module)
+            guessSerializer(value.key ?: error("Map.Entry(null, ...) is not supported"), module),
+            guessSerializer(value.value ?: error("Map.Entry(..., null) is not supported)"), module)
        )
        is Array<*> -> {
            val componentType = value.javaClass.componentType.kotlin.starProjectedType
@@ -76,12 +77,9 @@ internal fun serializerForSending(
 }

 @OptIn(ExperimentalSerializationApi::class)
-private fun Collection<*>.elementSerializer(
-    context: PipelineContext<Any, ApplicationCall>,
-    module: SerializersModule
-): KSerializer<*> {
+private fun Collection<*>.elementSerializer(module: SerializersModule): KSerializer<*> {
    val serializers = mapNotNull { value ->
-        value?.let { serializerForSending(context, it, module) }
+        value?.let { guessSerializer(it, module) }
    }.distinctBy { it.descriptor.serialName }

    if (serializers.size > 1) {

--- a/ktor-features/ktor-serialization/jvm/test/io/ktor/tests/serialization/SerializationTest.kt
+++ b/ktor-features/ktor-serialization/jvm/test/io/ktor/tests/serialization/SerializationTest.kt
@@ -529,6 +529,24 @@ class SerializationTest {
            assertEquals("""{"id":1,"data":"asd"}""", it.response.content)
        }
    }
+
+    @Test
+    fun testRespondCollection(): Unit = withTestApplication {
+        application.install(ContentNegotiation) {
+            register(ContentType.Application.Json, SerializationConverter(Json))
+        }
+        application.routing {
+            get("/") {
+                val collection: Collection<String> = listOf("a", "b")
+                call.respond(collection)
+            }
+        }
+
+        handleRequest(HttpMethod.Get, "/").let { call ->
+            assertEquals(HttpStatusCode.OK, call.response.status())
+            assertEquals("""["a","b"]""", call.response.content)
+        }
+    }
 }

 @Serializable