Skip to content

Commit ab2e690

Browse files
pjBoomspjBooms
authored
Implement saving benchmark results to JSON for browser (#5327)
Fixes [CMP-8282](https://youtrack.jetbrains.com/issue/CMP-8282/Benchmarks-support-saveStatsToJSON-for-web) ## Release Notes N/A
1 parent 87dae28 commit ab2e690

File tree

12 files changed

+339
-15
lines changed

12 files changed

+339
-15
lines changed

‎benchmarks/multiplatform/benchmarks/build.gradle.kts

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -70,13 +70,29 @@ kotlin {
7070
implementation(libs.kotlinx.serialization.json)
7171
implementation(libs.kotlinx.io)
7272
implementation(libs.kotlinx.datetime)
73+
implementation(libs.ktor.client.core)
74+
implementation(libs.ktor.client.content.negotiation)
75+
implementation(libs.ktor.serialization.kotlinx.json)
7376
}
7477
}
7578

7679
val desktopMain by getting {
7780
dependencies {
7881
implementation(compose.desktop.currentOs)
7982
runtimeOnly(libs.kotlinx.coroutines.swing)
83+
implementation(libs.ktor.server.core)
84+
implementation(libs.ktor.server.netty)
85+
implementation(libs.ktor.server.content.negotiation)
86+
implementation(libs.ktor.client.java)
87+
implementation(libs.ktor.server.cors)
88+
}
89+
}
90+
91+
val wasmJsMain by getting {
92+
dependencies {
93+
implementation(libs.ktor.client.js)
94+
implementation(libs.ktor.client.content.negotiation)
95+
implementation(libs.ktor.serialization.kotlinx.json)
8096
}
8197
}
8298
}
@@ -142,6 +158,44 @@ tasks.register("buildD8Distribution", Zip::class.java) {
142158
destinationDirectory.set(rootProject.layout.buildDirectory.dir("distributions"))
143159
}
144160

161+
tasks.register("runBrowserAndSaveStats") {
162+
fun printProcessOutput(inputStream: java.io.InputStream) {
163+
Thread {
164+
inputStream.bufferedReader().use { reader ->
165+
reader.lines().forEach { line ->
166+
println(line)
167+
}
168+
}
169+
}.start()
170+
}
171+
172+
fun runCommand(vararg command: String): Process {
173+
return ProcessBuilder(*command).start().also {
174+
printProcessOutput(it.inputStream)
175+
printProcessOutput(it.errorStream)
176+
}
177+
}
178+
179+
doFirst {
180+
var serverProcess: Process? = null
181+
var clientProcess: Process? = null
182+
try {
183+
serverProcess = runCommand("./gradlew", "benchmarks:run",
184+
"-PrunArguments=runServer=true saveStatsToJSON=true")
185+
186+
clientProcess = runCommand("./gradlew", "benchmarks:wasmJsBrowserProductionRun",
187+
"-PrunArguments=$runArguments saveStatsToJSON=true")
188+
189+
serverProcess.waitFor()
190+
} catch (e: Throwable) {
191+
e.printStackTrace()
192+
} finally {
193+
serverProcess?.destroy()
194+
clientProcess?.destroy()
195+
}
196+
}
197+
}
198+
145199
tasks.withType<org.jetbrains.kotlin.gradle.targets.js.binaryen.BinaryenExec>().configureEach {
146200
binaryenArgs.add("-g") // keep the readable names
147201
}

‎benchmarks/multiplatform/benchmarks/src/appleMain/kotlin/BenchmarksSave.apple.kt

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
/*
2+
* Copyright 2020-2025 JetBrains s.r.o. and respective authors and developers.
3+
* Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE.txt file.
4+
*/
5+
6+
actual fun saveBenchmarkStats(name: String, stats: BenchmarkStats) = saveBenchmarkStatsOnDisk(name, stats)

‎benchmarks/multiplatform/benchmarks/src/commonMain/kotlin/Benchmarks.kt

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,6 @@ import benchmarks.multipleComponents.MultipleComponentsExample
55
import benchmarks.lazygrid.LazyGrid
66
import benchmarks.visualeffects.NYContent
77
import kotlinx.serialization.Serializable
8-
import kotlinx.serialization.Transient
98
import kotlinx.serialization.json.Json
109
import kotlin.math.roundToInt
1110
import kotlin.time.Duration
@@ -247,7 +246,9 @@ suspend fun runBenchmark(
247246
content = content
248247
).generateStats()
249248
stats.prettyPrint()
250-
saveBenchmarkStatsOnDisk(name = name, stats = stats)
249+
if (Config.saveStats()) {
250+
saveBenchmarkStats(name = name, stats = stats)
251+
}
251252
}
252253
}
253254

‎benchmarks/multiplatform/benchmarks/src/commonMain/kotlin/BenchmarksSave.kt

Lines changed: 23 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -17,10 +17,24 @@ import kotlinx.io.files.Path
1717
import kotlinx.io.files.SystemFileSystem
1818
import kotlinx.io.readByteArray
1919

20+
// port for the benchmarks save server
21+
val BENCHMARK_SERVER_PORT = 8090
22+
23+
private val BENCHMARKS_SAVE_DIR = "build/benchmarks"
24+
private fun pathToCsv(name: String) = Path("$BENCHMARKS_SAVE_DIR/$name.csv")
25+
private fun pathToJson(name: String) = Path("$BENCHMARKS_SAVE_DIR/json-reports/$name.json")
26+
27+
internal fun saveJson(benchmarkName: String, jsonString: String) {
28+
val jsonPath = pathToJson(benchmarkName)
29+
SystemFileSystem.createDirectories(jsonPath.parent!!)
30+
SystemFileSystem.sink(jsonPath).writeText(jsonString)
31+
println("JSON results saved to ${SystemFileSystem.resolve(jsonPath)}")
32+
}
33+
2034
fun saveBenchmarkStatsOnDisk(name: String, stats: BenchmarkStats) {
2135
try {
2236
if (Config.saveStatsToCSV) {
23-
val path = Path("build/benchmarks/$name.csv")
37+
val path = pathToCsv(name)
2438

2539
val keyToValue = mutableMapOf<String, String>()
2640
keyToValue.put("Date", currentFormattedDate)
@@ -40,12 +54,7 @@ fun saveBenchmarkStatsOnDisk(name: String, stats: BenchmarkStats) {
4054
println("CSV results saved to ${SystemFileSystem.resolve(path)}")
4155
println()
4256
} else if (Config.saveStatsToJSON) {
43-
val jsonString = stats.toJsonString()
44-
val jsonPath = Path("build/benchmarks/json-reports/$name.json")
45-
46-
SystemFileSystem.createDirectories(jsonPath.parent!!)
47-
SystemFileSystem.sink(jsonPath).writeText(jsonString)
48-
println("JSON results saved to ${SystemFileSystem.resolve(jsonPath)}")
57+
saveJson(name, stats.toJsonString())
4958
println()
5059
}
5160
} catch (_: IOException) {
@@ -55,11 +64,17 @@ fun saveBenchmarkStatsOnDisk(name: String, stats: BenchmarkStats) {
5564
}
5665
}
5766

67+
/**
68+
* Saves benchmark statistics to disk or sends them to a server.
69+
* This is an expect function with platform-specific implementations.
70+
*/
71+
expect fun saveBenchmarkStats(name: String, stats: BenchmarkStats)
72+
5873
private fun RawSource.readText() = use {
5974
it.buffered().readByteArray().decodeToString()
6075
}
6176

62-
private fun RawSink.writeText(text: String) = use {
77+
internal fun RawSink.writeText(text: String) = use {
6378
it.buffered().apply {
6479
write(text.encodeToByteArray())
6580
flush()

‎benchmarks/multiplatform/benchmarks/src/commonMain/kotlin/Config.kt

Lines changed: 13 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,7 @@ object Args {
3737
var versionInfo: String? = null
3838
var saveStatsToCSV: Boolean = false
3939
var saveStatsToJSON: Boolean = false
40+
var runServer: Boolean = false
4041

4142
for (arg in args) {
4243
if (arg.startsWith("modes=", ignoreCase = true)) {
@@ -51,6 +52,8 @@ object Args {
5152
saveStatsToJSON = arg.substringAfter("=").toBoolean()
5253
} else if (arg.startsWith("disabledBenchmarks=", ignoreCase = true)) {
5354
disabledBenchmarks += argToMap(arg.decodeArg()).keys
55+
} else if (arg.startsWith("runServer=", ignoreCase = true)) {
56+
runServer = arg.substringAfter("=").toBoolean()
5457
}
5558
}
5659

@@ -60,7 +63,8 @@ object Args {
6063
disabledBenchmarks = disabledBenchmarks,
6164
versionInfo = versionInfo,
6265
saveStatsToCSV = saveStatsToCSV,
63-
saveStatsToJSON = saveStatsToJSON
66+
saveStatsToJSON = saveStatsToJSON,
67+
runServer = runServer,
6468
)
6569
}
6670
}
@@ -83,7 +87,8 @@ data class Config(
8387
val disabledBenchmarks: Set<String> = emptySet(),
8488
val versionInfo: String? = null,
8589
val saveStatsToCSV: Boolean = false,
86-
val saveStatsToJSON: Boolean = false
90+
val saveStatsToJSON: Boolean = false,
91+
val runServer: Boolean = false,
8792
) {
8893
/**
8994
* Checks if a specific mode is enabled based on the configuration.
@@ -127,6 +132,9 @@ data class Config(
127132
val saveStatsToJSON: Boolean
128133
get() = global.saveStatsToJSON
129134

135+
val runServer: Boolean
136+
get() = global.runServer
137+
130138
fun setGlobal(global: Config) {
131139
this.global = global
132140
}
@@ -143,5 +151,7 @@ data class Config(
143151

144152
fun getBenchmarkProblemSize(benchmark: String, default: Int): Int =
145153
global.getBenchmarkProblemSize(benchmark, default)
146-
}
154+
155+
fun saveStats() = saveStatsToCSV || saveStatsToJSON
156+
}
147157
}

‎benchmarks/multiplatform/benchmarks/src/desktopMain/kotlin/BenchmarksSave.desktop.kt

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
/*
2+
* Copyright 2020-2025 JetBrains s.r.o. and respective authors and developers.
3+
* Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE.txt file.
4+
*/
5+
6+
actual fun saveBenchmarkStats(name: String, stats: BenchmarkStats) = saveBenchmarkStatsOnDisk(name, stats)

‎benchmarks/multiplatform/benchmarks/src/desktopMain/kotlin/BenchmarksSaveServer.kt

Lines changed: 92 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,92 @@
1+
/*
2+
* Copyright 2020-2025 JetBrains s.r.o. and respective authors and developers.
3+
* Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE.txt file.
4+
*/
5+
6+
import io.ktor.http.*
7+
import io.ktor.serialization.kotlinx.json.*
8+
import io.ktor.server.application.*
9+
import io.ktor.server.engine.*
10+
import io.ktor.server.netty.*
11+
import io.ktor.server.plugins.contentnegotiation.*
12+
import io.ktor.server.plugins.cors.routing.CORS
13+
import io.ktor.server.request.*
14+
import io.ktor.server.response.*
15+
import io.ktor.server.routing.*
16+
import kotlinx.coroutines.Dispatchers
17+
import kotlinx.coroutines.withContext
18+
import kotlinx.io.files.Path
19+
import kotlinx.io.files.SystemFileSystem
20+
import kotlinx.serialization.Serializable
21+
22+
/**
23+
* Data class for receiving benchmark results from client.
24+
*/
25+
@Serializable
26+
data class BenchmarkResultFromClient(
27+
val name: String,
28+
val stats: String // JSON string of BenchmarkStats
29+
)
30+
31+
/**
32+
* Starts a Ktor server to receive benchmark results from browsers
33+
* and save them to disk in the same format as the direct disk saving mechanism.
34+
*/
35+
object BenchmarksSaveServer {
36+
private var server: EmbeddedServer<NettyApplicationEngine, NettyApplicationEngine.Configuration>? = null
37+
38+
fun start(port: Int = BENCHMARK_SERVER_PORT) {
39+
if (server != null) {
40+
println("Benchmark server is already running")
41+
return
42+
}
43+
44+
server = embeddedServer(Netty, port = port) {
45+
install(ContentNegotiation) {
46+
json()
47+
}
48+
install(CORS) {
49+
allowMethod(HttpMethod.Get)
50+
allowMethod(HttpMethod.Post)
51+
allowHeader(HttpHeaders.ContentType)
52+
anyHost()
53+
}
54+
routing {
55+
post("/benchmark") {
56+
val result = call.receive<BenchmarkResultFromClient>()
57+
if (result.name.isEmpty()) {
58+
println("Stopping server! Received empty name from client")
59+
call.respond(HttpStatusCode.OK, "Server stopped.")
60+
stop()
61+
return@post
62+
}
63+
println("Received benchmark result for: ${result.name}")
64+
65+
withContext(Dispatchers.IO) {
66+
if (Config.saveStatsToJSON) {
67+
saveJson(result.name, result.stats)
68+
}
69+
70+
if (Config.saveStatsToCSV) {
71+
// TODO: for CSV, we would need to convert JSON to the values
72+
println("CSV results are not yet supported for the browser.")
73+
}
74+
}
75+
76+
call.respond(HttpStatusCode.OK, "Benchmark result saved")
77+
}
78+
79+
get("/") {
80+
call.respondText("Benchmark server is running", ContentType.Text.Plain)
81+
}
82+
}
83+
}.start(wait = true)
84+
}
85+
86+
fun stop() {
87+
server?.stop(1000, 2000)
88+
server = null
89+
println("Benchmark server stopped")
90+
System.exit(0)
91+
}
92+
}

‎benchmarks/multiplatform/benchmarks/src/desktopMain/kotlin/main.desktop.kt

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,5 +8,11 @@ import kotlinx.coroutines.runBlocking
88

99
fun main(args: Array<String>) {
1010
Config.setGlobalFromArgs(args)
11-
runBlocking(Dispatchers.Main) { runBenchmarks() }
11+
12+
if (Config.runServer) {
13+
// Start the benchmark server to receive results from browsers
14+
BenchmarksSaveServer.start()
15+
} else {
16+
runBlocking(Dispatchers.Main) { runBenchmarks() }
17+
}
1218
}

0 commit comments

Comments
 (0)