Merge branch 'main' into main

Author: ffp5

.changeset/big-sloths-dig.md

@@ -0,0 +1,74 @@
+name: Evals
+
+on:
+    pull_request:
+        types: [labeled]
+    workflow_dispatch:
+
+env:
+    DOCKER_BUILDKIT: 1
+    COMPOSE_DOCKER_CLI_BUILD: 1
+
+jobs:
+    evals:
+        # Run if triggered manually or if PR has 'evals' label.
+        if: github.event_name == 'workflow_dispatch' || contains(github.event.label.name, 'evals')
+        runs-on: blacksmith-16vcpu-ubuntu-2404
+        timeout-minutes: 45
+
+        defaults:
+            run:
+                working-directory: packages/evals
+
+        steps:
+            - name: Checkout repository
+              uses: actions/checkout@v4
+
+            - name: Set up Docker Buildx
+              uses: docker/setup-buildx-action@v3
+
+            - name: Create environment
+              run: |
+                  cat > .env.local << EOF
+                  OPENROUTER_API_KEY=${{ secrets.OPENROUTER_API_KEY || 'test-key-for-build' }}
+                  EOF
+
+                  cat > .env.development << EOF
+                  NODE_ENV=development
+                  DATABASE_URL=postgresql://postgres:password@db:5432/evals_development
+                  REDIS_URL=redis://redis:6379
+                  HOST_EXECUTION_METHOD=docker
+                  EOF
+
+            - name: Build image
+              uses: docker/build-push-action@v5
+              with:
+                  context: .
+                  file: packages/evals/Dockerfile.runner
+                  tags: evals-runner:latest
+                  cache-from: type=gha
+                  cache-to: type=gha,mode=max
+                  push: false
+                  load: true
+
+            - name: Tag image
+              run: docker tag evals-runner:latest evals-runner
+
+            - name: Start containers
+              run: |
+                  docker compose up -d db redis
+                  timeout 60 bash -c 'until docker compose exec -T db pg_isready -U postgres; do sleep 2; done'
+                  timeout 60 bash -c 'until docker compose exec -T redis redis-cli ping | grep -q PONG; do sleep 2; done'
+                  docker compose run --rm runner sh -c 'nc -z db 5432 && echo "✓ Runner -> Database connection successful"'
+                  docker compose run --rm runner sh -c 'nc -z redis 6379 && echo "✓ Runner -> Redis connection successful"'
+                  docker compose run --rm runner docker ps
+
+            - name: Run database migrations
+              run: docker compose run --rm runner pnpm --filter @roo-code/evals db:migrate
+
+            - name: Run evals
+              run: docker compose run --rm runner pnpm --filter @roo-code/evals cli --ci
+
+            - name: Cleanup
+              if: always()
+              run: docker compose down -v --remove-orphans

.github/workflows/evals.yml

@@ -43,3 +43,4 @@ logs
 .idea/
 .qodo/
 .vercel
+.roo/mcp.json

.gitignore

@@ -1,5 +1,47 @@
 # kilo-code
 
+## [v4.36.0]
+
+- [#690](https://github.com/Kilo-Org/kilocode/pull/690) [`9b1451a`](https://github.com/Kilo-Org/kilocode/commit/9b1451a47bd2bc567646a4a0c2a12b42826ab9d1) Thanks [@kevinvandijk](https://github.com/kevinvandijk)! - Include changes from Roo Code v3.19.7:
+
+    - Fix McpHub sidebar focus behavior to prevent unwanted focus grabbing
+    - Disable checkpoint functionality when nested git repositories are detected to prevent conflicts
+    - Remove unused Storybook components and dependencies to reduce bundle size
+    - Add data-testid ESLint rule for improved testing standards (thanks @elianiva!)
+    - Update development dependencies including eslint, knip, @types/node, i18next, fast-xml-parser, and @google/genai
+    - Improve CI infrastructure with GitHub Actions and Blacksmith runner migrations
+    - Replace explicit caching with implicit caching to reduce latency for Gemini models
+    - Clarify that the default concurrent file read limit is 15 files (thanks @olearycrew!)
+    - Fix copy button logic (thanks @samhvw8!)
+    - Fade buttons on history preview if no interaction in progress (thanks @sachasayan!)
+    - Allow MCP server refreshing, fix state changes in MCP server management UI view (thanks @taylorwilsdon!)
+    - Remove unnecessary npx usage in some npm scripts (thanks @user202729!)
+    - Bug fix for trailing slash error when using LiteLLM provider (thanks @kcwhite!)
+    - Fix Gemini 2.5 Pro Preview thinking budget bug
+    - Add Gemini Pro 06-05 model support (thanks @daniel-lxs and @shariqriazz!)
+    - Fix reading PDF, DOCX, and IPYNB files in read_file tool (thanks @samhvw8!)
+    - Fix Mermaid CSP errors with enhanced bundling strategy (thanks @KJ7LNW!)
+    - Improve model info detection for custom Bedrock ARNs (thanks @adamhill!)
+    - Add OpenAI Compatible embedder for codebase indexing (thanks @SannidhyaSah!)
+    - Fix multiple memory leaks in ChatView component (thanks @kiwina!)
+    - Fix WorkspaceTracker resource leaks by disposing FileSystemWatcher (thanks @kiwina!)
+    - Fix RooTips setTimeout cleanup to prevent state updates on unmounted components (thanks @kiwina!)
+    - Fix FileSystemWatcher leak in RooIgnoreController (thanks @kiwina!)
+    - Fix clipboard memory leak by clearing setTimeout in useCopyToClipboard (thanks @kiwina!)
+    - Fix ClineProvider instance cleanup (thanks @xyOz-dev!)
+    - Enforce codebase_search as primary tool for code understanding tasks (thanks @hannesrudolph!)
+    - Improve Docker setup for evals
+    - Move evals into pnpm workspace, switch from SQLite to Postgres
+    - Refactor MCP to use getDefaultEnvironment for stdio client transport (thanks @samhvw8!)
+    - Get rid of "partial" component in names referencing not necessarily partial messages (thanks @wkordalski!)
+    - Improve feature request template (thanks @elianiva!)
+
+- [#592](https://github.com/Kilo-Org/kilocode/pull/592) [`68c3d6e`](https://github.com/Kilo-Org/kilocode/commit/68c3d6e7a1250e08e2bd2b9cbbbd6b4312bad045) Thanks [@chrarnoldus](https://github.com/chrarnoldus)! - Workflow and rules configuration screen added
+
+### Patch Changes
+
+- [#697](https://github.com/Kilo-Org/kilocode/pull/697) [`9514f22`](https://github.com/Kilo-Org/kilocode/commit/9514f22a9d77b2d838ddcb97b5f2c5909aaea68a) Thanks [@kevinvandijk](https://github.com/kevinvandijk)! - Add correct path to walkthrough files to show walkthrough on first load (thanks for the report @adamhill!)
+
 ## [v4.35.1]
 
 - [#695](https://github.com/Kilo-Org/kilocode/pull/695) [`a7910eb`](https://github.com/Kilo-Org/kilocode/commit/a7910eba54a4ede296bfa82beddae71a1d9f77c5) Thanks [@kevinvandijk](https://github.com/kevinvandijk)! - Fix: Feedback button overlaps new mode creation dialog

CHANGELOG.md

@@ -5,7 +5,7 @@
 	"scripts": {
 		"lint": "next lint",
 		"check-types": "tsc -b",
-		"dev": "scripts/check-services.sh && next dev --turbopack",
+		"dev": "scripts/check-services.sh && next dev",
 		"format": "prettier --write src",
 		"build": "next build",
 		"start": "next start"

apps/web-evals/package.json

@@ -0,0 +1,22 @@
+"use server"
+
+import * as path from "path"
+import { fileURLToPath } from "url"
+
+import { exerciseLanguages, listDirectories } from "@roo-code/evals"
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url)) // <repo>/apps/web-evals/src/actions
+
+const EVALS_REPO_PATH = path.resolve(__dirname, "../../../../../evals")
+
+export const getExercises = async () => {
+	const result = await Promise.all(
+		exerciseLanguages.map(async (language) => {
+			const languagePath = path.join(EVALS_REPO_PATH, language)
+			const exercises = await listDirectories(__dirname, languagePath)
+			return exercises.map((exercise) => `${language}/${exercise}`)
+		}),
+	)
+
+	return result.flat()
+}

apps/web-evals/src/actions/exercises.ts

@@ -0,0 +1,8 @@
+"use server"
+
+import { redisClient } from "@/lib/server/redis"
+
+export const getHeartbeat = async (runId: number) => {
+	const redis = await redisClient()
+	return redis.get(`heartbeat:${runId}`)
+}

apps/web-evals/src/actions/heartbeat.ts

@@ -1,6 +1,6 @@
 "use server"
 
-import { redisClient } from "./redis"
+import { redisClient } from "@/lib/server/redis"
 
 export const getRunners = async (runId: number) => {
 	const redis = await redisClient()

apps/web-evals/src/lib/server/runners.ts renamed to apps/web-evals/src/actions/runners.ts

@@ -1,7 +1,9 @@
 "use server"
 
-import { spawn } from "child_process"
+import * as path from "path"
 import fs from "fs"
+import { fileURLToPath } from "url"
+import { spawn } from "child_process"
 
 import { revalidatePath } from "next/cache"
 import pMap from "p-map"
@@ -12,11 +14,12 @@ import {
 	createRun as _createRun,
 	deleteRun as _deleteRun,
 	createTask,
+	getExercisesForLanguage,
 } from "@roo-code/evals"
 
 import { CreateRun } from "@/lib/schemas"
 
-import { getExercisesForLanguage } from "./exercises"
+const EVALS_REPO_PATH = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../../../../../evals")
 
 // eslint-disable-next-line @typescript-eslint/no-unused-vars
 export async function createRun({ suite, exercises = [], systemPrompt, ...values }: CreateRun) {
@@ -37,9 +40,9 @@ export async function createRun({ suite, exercises = [], systemPrompt, ...values
 		}
 	} else {
 		for (const language of exerciseLanguages) {
-			const exercises = await getExercisesForLanguage(language)
+			const exercises = await getExercisesForLanguage(EVALS_REPO_PATH, language)
 
-			await pMap(exercises, (exercise) => createTask({ ...values, runId: run.id, language, exercise }), {
+			await pMap(exercises, (exercise) => createTask({ runId: run.id, language, exercise }), {
 				concurrency: 10,
 			})
 		}
@@ -52,9 +55,10 @@ export async function createRun({ suite, exercises = [], systemPrompt, ...values
 
 		const dockerArgs = [
 			`--name evals-controller-${run.id}`,
-			"--rm",
+			// "--rm",
 			"--network evals_default",
 			"-v /var/run/docker.sock:/var/run/docker.sock",
+			"-v /tmp/evals:/var/log/evals",
 			"-e HOST_EXECUTION_METHOD=docker",
 		]
 

apps/web-evals/src/lib/server/runs.ts renamed to apps/web-evals/src/actions/runs.ts

@@ -0,0 +1,24 @@
+import { NextResponse } from "next/server"
+
+export async function GET() {
+	try {
+		return NextResponse.json(
+			{
+				status: "healthy",
+				timestamp: new Date().toISOString(),
+				uptime: process.uptime(),
+				environment: process.env.NODE_ENV || "production",
+			},
+			{ status: 200 },
+		)
+	} catch (error) {
+		return NextResponse.json(
+			{
+				status: "unhealthy",
+				timestamp: new Date().toISOString(),
+				error: error instanceof Error ? error.message : "Unknown error",
+			},
+			{ status: 503 },
+		)
+	}
+}

apps/web-evals/src/lib/server/tasks.ts renamed to apps/web-evals/src/actions/tasks.ts

@@ -7,7 +7,7 @@ import { Ellipsis, Rocket } from "lucide-react"
 
 import type { Run, TaskMetrics } from "@roo-code/evals"
 
-import { deleteRun } from "@/lib/server/runs"
+import { deleteRun } from "@/actions/runs"
 import { formatCurrency, formatDuration, formatTokens, formatToolUsageSuccessRate } from "@/lib/formatters"
 import {
 	Button,

apps/web-evals/src/app/api/health/route.ts

@@ -0,0 +1,55 @@
+"use client"
+
+import type { RunStatus as _RunStatus } from "@/hooks/use-run-status"
+import { cn } from "@/lib/utils"
+
+export const RunStatus = ({ runStatus: { sseStatus, heartbeat, runners = [] } }: { runStatus: _RunStatus }) => (
+	<div>
+		<div className="flex items-center gap-2">
+			<div className="flex items-center gap-2">
+				<div>Task Stream:</div>
+				<div className="font-mono text-sm text-muted-foreground">{sseStatus}</div>
+			</div>
+			<div className="relative">
+				<div
+					className={cn("absolute size-2.5 rounded-full opacity-50 animate-ping", {
+						"bg-green-500": sseStatus === "connected",
+						"bg-amber-500": sseStatus === "waiting",
+						"bg-rose-500": sseStatus === "error",
+					})}
+				/>
+				<div
+					className={cn("size-2.5 rounded-full", {
+						"bg-green-500": sseStatus === "connected",
+						"bg-amber-500": sseStatus === "waiting",
+						"bg-rose-500": sseStatus === "error",
+					})}
+				/>
+			</div>
+		</div>
+		<div className="flex items-center gap-2">
+			<div className="flex items-center gap-2">
+				<div>Task Controller:</div>
+				<div className="font-mono text-sm text-muted-foreground">{heartbeat ?? "dead"}</div>
+			</div>
+			<div className="relative">
+				<div
+					className={cn("absolute size-2.5 rounded-full opacity-50 animate-ping", {
+						"bg-green-500": !!heartbeat,
+						"bg-rose-500": !heartbeat,
+					})}
+				/>
+				<div
+					className={cn("size-2.5 rounded-full", {
+						"bg-green-500": !!heartbeat,
+						"bg-rose-500": !heartbeat,
+					})}
+				/>
+			</div>
+		</div>
+		<div className="flex items-center gap-2">
+			<div>Task Runners:</div>
+			{runners.length > 0 && <div className="font-mono text-sm text-muted-foreground">{runners?.join(", ")}</div>}
+		</div>
+	</div>
+)