From 455d77ac6c2f15fe61800c411f0dd891818dea8d Mon Sep 17 00:00:00 2001 From: Tim Fish Date: Tue, 10 Jun 2025 14:06:46 +0200 Subject: [PATCH 01/16] new package --- package.json | 1 + packages/node-stalled/.eslintrc.js | 11 + packages/node-stalled/LICENSE | 21 ++ packages/node-stalled/README.md | 316 ++++++++++++++++++++ packages/node-stalled/package.json | 82 +++++ packages/node-stalled/rollup.npm.config.mjs | 19 ++ packages/node-stalled/src/common.ts | 43 +++ packages/node-stalled/src/index.ts | 201 +++++++++++++ packages/node-stalled/src/worker.ts | 292 ++++++++++++++++++ packages/node-stalled/tsconfig.json | 10 + packages/node-stalled/tsconfig.test.json | 12 + packages/node-stalled/tsconfig.types.json | 11 + packages/node-stalled/vite.config.ts | 8 + 13 files changed, 1027 insertions(+) create mode 100644 packages/node-stalled/.eslintrc.js create mode 100644 packages/node-stalled/LICENSE create mode 100644 packages/node-stalled/README.md create mode 100644 packages/node-stalled/package.json create mode 100644 packages/node-stalled/rollup.npm.config.mjs create mode 100644 packages/node-stalled/src/common.ts create mode 100644 packages/node-stalled/src/index.ts create mode 100644 packages/node-stalled/src/worker.ts create mode 100644 packages/node-stalled/tsconfig.json create mode 100644 packages/node-stalled/tsconfig.test.json create mode 100644 packages/node-stalled/tsconfig.types.json create mode 100644 packages/node-stalled/vite.config.ts diff --git a/package.json b/package.json index 13e1a600e83d..fd1fa2c0ee23 100644 --- a/package.json +++ b/package.json @@ -68,6 +68,7 @@ "packages/nestjs", "packages/nextjs", "packages/node", + "packages/node-stalled", "packages/nuxt", "packages/opentelemetry", "packages/profiling-node", diff --git a/packages/node-stalled/.eslintrc.js b/packages/node-stalled/.eslintrc.js new file mode 100644 index 000000000000..ab3515d9ad37 --- /dev/null +++ b/packages/node-stalled/.eslintrc.js @@ -0,0 +1,11 @@ +module.exports = { + env: { + node: true, + }, + extends: ['../../.eslintrc.js'], + + ignorePatterns: ['build/**/*', 'examples/**/*', 'vitest.config.ts'], + rules: { + '@sentry-internal/sdk/no-class-field-initializers': 'off', + }, +}; diff --git a/packages/node-stalled/LICENSE b/packages/node-stalled/LICENSE new file mode 100644 index 000000000000..293314012679 --- /dev/null +++ b/packages/node-stalled/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2022 Functional Software, Inc. dba Sentry + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +of the Software, and to permit persons to whom the Software is furnished to do +so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/packages/node-stalled/README.md b/packages/node-stalled/README.md new file mode 100644 index 000000000000..962bc8e6834f --- /dev/null +++ b/packages/node-stalled/README.md @@ -0,0 +1,316 @@ +

+ + Sentry + +

+ +# Official Sentry Profiling SDK for NodeJS + +[![npm version](https://img.shields.io/npm/v/@sentry/profiling-node.svg)](https://www.npmjs.com/package/@sentry/profiling-node) +[![npm dm](https://img.shields.io/npm/dm/@sentry/profiling-node.svg)](https://www.npmjs.com/package/@sentry/profiling-node) +[![npm dt](https://img.shields.io/npm/dt/@sentry/profiling-node.svg)](https://www.npmjs.com/package/@sentry/profiling-node) + +## Installation + +Profiling works as an extension of tracing so you will need both @sentry/node and @sentry/profiling-node installed. + +```bash +# Using yarn +yarn add @sentry/node @sentry/profiling-node + +# Using npm +npm install --save @sentry/node @sentry/profiling-node +``` + +## Usage + +```javascript +import * as Sentry from '@sentry/node'; +import { nodeProfilingIntegration } from '@sentry/profiling-node'; + +Sentry.init({ + dsn: 'https://7fa19397baaf433f919fbe02228d5470@o1137848.ingest.sentry.io/6625302', + debug: true, + tracesSampleRate: 1, + profilesSampleRate: 1, // Set profiling sampling rate. + integrations: [nodeProfilingIntegration()], +}); +``` + +Sentry SDK will now automatically profile all root spans, even the ones which may be started as a result of using an +automatic instrumentation integration. + +```javascript +Sentry.startSpan({ name: 'some workflow' }, () => { + // The code in here will be profiled +}); +``` + +### Building the package from source + +Profiling uses native modules to interop with the v8 javascript engine which means that you may be required to build it +from source. The libraries required to successfully build the package from source are often the same libraries that are +already required to build any other package which uses native modules and if your codebase uses any of those modules, +there is a fairly good chance this will work out of the box. The required packages are python, make and g++. + +**Windows:** If you are building on windows, you may need to install windows-build-tools + +**_Python:_** Python 3.12 is not supported yet so you will need a version of python that is lower than 3.12 + +```bash + +# using yarn package manager +yarn global add windows-build-tools +# or npm package manager +npm i -g windows-build-tools +``` + +After you have installed the toolchain, you should be able to build the binaries from source + +```bash +# configure node-gyp using yarn +yarn build:bindings:configure +# or using npm +npm run build:bindings:configure + +# compile the binaries using yarn +yarn build:bindings +# or using npm +npm run build:bindings +``` + +After the binaries are built, you should see them inside the profiling-node/lib folder. + +### Prebuilt binaries + +We currently ship prebuilt binaries for a few of the most common platforms and node versions (v18-24). + +- macOS x64 +- Linux ARM64 (musl) +- Linux x64 (glibc) +- Windows x64 + +For a more detailed list, see job_compile_bindings_profiling_node job in our build.yml github action workflow. + +### Bundling + +If you are looking to squeeze some extra performance or improve cold start in your application (especially true for +serverless environments where modules are often evaluates on a per request basis), then we recommend you look into +bundling your code. Modern JS engines are much faster at parsing and compiling JS than following long module resolution +chains and reading file contents from disk. Because @sentry/profiling-node is a package that uses native node modules, +bundling it is slightly different than just bundling javascript. In other words, the bundler needs to recognize that a +.node file is node native binding and move it to the correct location so that it can later be used. Failing to do so +will result in a MODULE_NOT_FOUND error. + +The easiest way to make bundling work with @sentry/profiling-node and other modules which use native nodejs bindings is +to mark the package as external - this will prevent the code from the package from being bundled, but it means that you +will now need to rely on the package to be installed in your production environment. + +To mark the package as external, use the following configuration: + +[Next.js 13+](https://nextjs.org/docs/app/api-reference/next-config-js/serverComponentsExternalPackages) + +```js +const { withSentryConfig } = require('@sentry/nextjs'); + +/** @type {import('next').NextConfig} */ +const nextConfig = { + experimental: { + // Add the "@sentry/profiling-node" to serverComponentsExternalPackages. + serverComponentsExternalPackages: ['@sentry/profiling-node'], + }, +}; + +module.exports = withSentryConfig(nextConfig, { + /* ... */ +}); +``` + +[webpack](https://webpack.js.org/configuration/externals/#externals) + +```js +externals: { + "@sentry/profiling-node": "commonjs @sentry/profiling-node", +}, +``` + +[esbuild](https://esbuild.github.io/api/#external) + +```js +{ + entryPoints: ['index.js'], + platform: 'node', + external: ['@sentry/profiling-node'], +} +``` + +[Rollup](https://rollupjs.org/configuration-options/#external) + +```js +{ + entry: 'index.js', + external: '@sentry/profiling-node' +} +``` + +[serverless-esbuild (serverless.yml)](https://www.serverless.com/plugins/serverless-esbuild#external-dependencies) + +```yml +custom: + esbuild: + external: + - @sentry/profiling-node + packagerOptions: + scripts: + - npm install @sentry/profiling-node +``` + +[vercel-ncc](https://github.com/vercel/ncc#programmatically-from-nodejs) + +```js +{ + externals: ["@sentry/profiling-node"], +} +``` + +[vite](https://vitejs.dev/config/ssr-options.html#ssr-external) + +```js +ssr: { + external: ['@sentry/profiling-node']; +} +``` + +Marking the package as external is the simplest and most future proof way of ensuring it will work, however if you want +to bundle it, it is possible to do so as well. Bundling has the benefit of improving your script startup time as all of +the code is (usually) inside a single executable .js file, which saves time on module resolution. + +In general, when attempting to bundle .node native file extensions, you will need to tell your bundler how to treat +these, as by default it does not know how to handle them. The required approach varies between build tools and you will +need to find which one will work for you. + +The result of bundling .node files correctly is that they are placed into your bundle output directory with their +require paths updated to reflect their final location. + +Example of bundling @sentry/profiling-node with esbuild and .copy loader + +```json +// package.json +{ + "scripts": "node esbuild.serverless.js" +} +``` + +```js +// esbuild.serverless.js +const { sentryEsbuildPlugin } = require('@sentry/esbuild-plugin'); + +require('esbuild').build({ + entryPoints: ['./index.js'], + outfile: './dist', + platform: 'node', + bundle: true, + minify: true, + sourcemap: true, + // This is no longer necessary + // external: ["@sentry/profiling-node"], + loader: { + // ensures .node binaries are copied to ./dist + '.node': 'copy', + }, + plugins: [ + // See https://docs.sentry.io/platforms/javascript/sourcemaps/uploading/esbuild/ + sentryEsbuildPlugin({ + project: '', + org: '', + authToken: '', + release: '', + sourcemaps: { + // Specify the directory containing build artifacts + assets: './dist/**', + }, + }), + ], +}); +``` + +Once you run `node esbuild.serverless.js` esbuild wil bundle and output the files to ./dist folder, but note that all of +the binaries will be copied. This is wasteful as you will likely only need one of these libraries to be available during +runtime. + +To prune the other libraries, profiling-node ships with a small utility script that helps you prune unused binaries. The +script can be invoked via `sentry-prune-profiler-binaries`, use `--help` to see a list of available options or +`--dry-run` if you want it to log the binaries that would have been deleted. + +Example of only preserving a binary to run node16 on linux x64 musl. + +```bash +sentry-prune-profiler-binaries --target_dir_path=./dist --target_platform=linux --target_node=16 --target_stdlib=musl --target_arch=x64 +``` + +Which will output something like + +``` +Sentry: pruned ./dist/sentry_cpu_profiler-darwin-x64-108-IFGH3SUR.node (90.41 KiB) +Sentry: pruned ./dist/sentry_cpu_profiler-darwin-x64-93-Q7KBVHSP.node (74.16 KiB) +Sentry: pruned ./dist/sentry_cpu_profiler-linux-arm64-glibc-108-NXSISRTB.node (52.17 KiB) +Sentry: pruned ./dist/sentry_cpu_profiler-linux-arm64-glibc-83-OEQT5HUK.node (52.08 KiB) +Sentry: pruned ./dist/sentry_cpu_profiler-linux-arm64-glibc-93-IIXXW2PN.node (52.06 KiB) +Sentry: pruned ./dist/sentry_cpu_profiler-linux-arm64-musl-108-DSILNYHA.node (48.46 KiB) +Sentry: pruned ./dist/sentry_cpu_profiler-linux-arm64-musl-83-4CNOBNC3.node (48.37 KiB) +Sentry: pruned ./dist/sentry_cpu_profiler-linux-arm64-musl-93-JA5PKNWQ.node (48.38 KiB) +Sentry: pruned ./dist/sentry_cpu_profiler-linux-x64-glibc-108-NXSISRTB.node (52.17 KiB) +Sentry: pruned ./dist/sentry_cpu_profiler-linux-x64-glibc-83-OEQT5HUK.node (52.08 KiB) +Sentry: pruned ./dist/sentry_cpu_profiler-linux-x64-glibc-93-IIXXW2PN.node (52.06 KiB) +Sentry: pruned ./dist/sentry_cpu_profiler-linux-x64-musl-108-CX7SL27U.node (51.50 KiB) +Sentry: pruned ./dist/sentry_cpu_profiler-linux-x64-musl-83-YD7ZQK2E.node (51.53 KiB) +Sentry: pruned ./dist/sentry_cpu_profiler-win32-x64-108-P7V3URQV.node (181.50 KiB) +Sentry: pruned ./dist/sentry_cpu_profiler-win32-x64-93-3PKQDSGE.node (181.50 KiB) +✅ Sentry: pruned 15 binaries, saved 1.06 MiB in total. +``` + +### Environment flags + +The default mode of the v8 CpuProfiler is kEagerLoggin which enables the profiler even when no profiles are active - +this is good because it makes calls to startProfiling fast at the tradeoff for constant CPU overhead. The behavior can +be controlled via the `SENTRY_PROFILER_LOGGING_MODE` environment variable with values of `eager|lazy`. If you opt to use +the lazy logging mode, calls to startProfiling may be slow (depending on environment and node version, it can be in the +order of a few hundred ms). + +Example of starting a server with lazy logging mode. + +```javascript +SENTRY_PROFILER_LOGGING_MODE=lazy node server.js +``` + +## FAQ 💭 + +### Can the profiler leak PII to Sentry? + +The profiler does not collect function arguments so leaking any PII is unlikely. We only collect a subset of the values +which may identify the device and os that the profiler is running on (if you are already using tracing, it is likely +that these values are already being collected by the SDK). + +There is one way a profiler could leak pii information, but this is unlikely and would only happen for cases where you +might be creating or naming functions which might contain pii information such as + +```js +eval('function scriptFor${PII_DATA}....'); +``` + +In that case it is possible that the function name may end up being reported to Sentry. + +### Are worker threads supported? + +No. All instances of the profiler are scoped per thread In practice, this means that starting a transaction on thread A +and delegating work to thread B will only result in sample stacks being collected from thread A. That said, nothing +should prevent you from starting a transaction on thread B concurrently which will result in two independent profiles +being sent to the Sentry backend. We currently do not do any correlation between such transactions, but we would be open +to exploring the possibilities. Please file an issue if you have suggestions or specific use-cases in mind. + +### How much overhead will this profiler add? + +The profiler uses the kEagerLogging option by default which trades off fast calls to startProfiling for a small amount +of constant CPU overhead. If you are using kEagerLogging then the tradeoff is reversed and there will be a small CPU +overhead while the profiler is not running, but calls to startProfiling could be slow (in our tests, this varies by +environments and node versions, but could be in the order of a couple 100ms). diff --git a/packages/node-stalled/package.json b/packages/node-stalled/package.json new file mode 100644 index 000000000000..de7c396f7687 --- /dev/null +++ b/packages/node-stalled/package.json @@ -0,0 +1,82 @@ +{ + "name": "@sentry/node-stalled", + "version": "9.27.0", + "description": "Official Sentry SDK for Node.js Profiling", + "repository": "git://github.com/getsentry/sentry-javascript.git", + "homepage": "https://github.com/getsentry/sentry-javascript/tree/master/packages/profiling-node", + "author": "Sentry", + "license": "MIT", + "main": "build/cjs/index.js", + "module": "build/esm/index.js", + "types": "build/types/index.d.ts", + "exports": { + "./package.json": "./package.json", + ".": { + "import": { + "types": "./build/types/index.d.ts", + "default": "./build/esm/index.js" + }, + "require": { + "types": "./build/types/index.d.ts", + "default": "./build/cjs/index.js" + } + } + }, + "typesVersions": { + "<5.0": { + "build/types/index.d.ts": [ + "build/types-ts3.8/index.d.ts" + ] + } + }, + "engines": { + "node": ">=18" + }, + "publishConfig": { + "access": "public" + }, + "files": [ + "/build", + "package.json" + ], + "scripts": { + "clean": "rm -rf build", + "lint": "eslint . --format stylish", + "lint:es-compatibility": "es-check es2022 ./build/cjs/*.js && es-check es2022 ./build/esm/*.js --module", + "fix": "eslint . --format stylish --fix", + "build": "yarn build:types && yarn build:transpile", + "build:transpile": "yarn rollup -c rollup.npm.config.mjs", + "build:types:downlevel": "yarn downlevel-dts build/types build/types-ts3.8 --to ts3.8", + "build:types": "tsc -p tsconfig.types.json && yarn build:types:downlevel", + "build:types:watch": "tsc -p tsconfig.types.json --watch", + "build:dev": "yarn clean && yarn build", + "build:transpile:watch": "rollup -c rollup.npm.config.mjs --watch", + "build:watch": "run-p build:transpile:watch build:types:watch", + "build:tarball": "npm pack", + "test:bundle": "node test-binaries.esbuild.js", + "test": "vitest run", + "test:watch": "vitest --watch" + }, + "dependencies": { + "@sentry-internal/node-native-stacktrace": "^0.1.0", + "@sentry/core": "9.27.0", + "@sentry/node": "9.27.0" + }, + "devDependencies": { + "@types/node": "^18.19.1" + }, + "volta": { + "extends": "../../package.json" + }, + "sideEffects": false, + "nx": { + "targets": { + "build:transpile": { + "dependsOn": [ + "^build:transpile", + "^build:types" + ] + } + } + } +} diff --git a/packages/node-stalled/rollup.npm.config.mjs b/packages/node-stalled/rollup.npm.config.mjs new file mode 100644 index 000000000000..80b26c2ac232 --- /dev/null +++ b/packages/node-stalled/rollup.npm.config.mjs @@ -0,0 +1,19 @@ +import { makeBaseNPMConfig, makeNPMConfigVariants } from '@sentry-internal/rollup-utils'; + +export default makeNPMConfigVariants( + makeBaseNPMConfig({ + packageSpecificConfig: { + output: { + dir: 'build', + // set exports to 'named' or 'auto' so that rollup doesn't warn + exports: 'named', + // set preserveModules to false because for profiling we actually want + // to bundle everything into one file. + preserveModules: + process.env.SENTRY_BUILD_PRESERVE_MODULES === undefined + ? false + : Boolean(process.env.SENTRY_BUILD_PRESERVE_MODULES), + }, + }, + }), +); diff --git a/packages/node-stalled/src/common.ts b/packages/node-stalled/src/common.ts new file mode 100644 index 000000000000..b9a5a6b85777 --- /dev/null +++ b/packages/node-stalled/src/common.ts @@ -0,0 +1,43 @@ +import type { Contexts, DsnComponents, Primitive, SdkMetadata } from '@sentry/core'; + +export interface AnrIntegrationOptions { + /** + * Interval to send heartbeat messages to the ANR worker. + * + * Defaults to 50ms. + */ + pollInterval: number; + /** + * Threshold in milliseconds to trigger an ANR event. + * + * Defaults to 5000ms. + */ + anrThreshold: number; + /** + * Maximum number of ANR events to send. + * + * Defaults to 1. + */ + maxAnrEvents: number; + /** + * Tags to include with ANR events. + */ + staticTags: { [key: string]: Primitive }; + /** + * @ignore Internal use only. + * + * If this is supplied, stack frame filenames will be rewritten to be relative to this path. + */ + appRootPath: string | undefined; +} + +export interface WorkerStartData extends AnrIntegrationOptions { + debug: boolean; + sdkMetadata: SdkMetadata; + dsn: DsnComponents; + tunnel: string | undefined; + release: string | undefined; + environment: string; + dist: string | undefined; + contexts: Contexts; +} diff --git a/packages/node-stalled/src/index.ts b/packages/node-stalled/src/index.ts new file mode 100644 index 000000000000..d65950ba99fe --- /dev/null +++ b/packages/node-stalled/src/index.ts @@ -0,0 +1,201 @@ +import { types } from 'node:util'; +import { Worker } from 'node:worker_threads'; +import type { Contexts, Event, EventHint, Integration, IntegrationFn } from '@sentry/core'; +import { + defineIntegration, + getClient, + getFilenameToDebugIdMap, + getIsolationScope, + logger, +} from '@sentry/core'; +import type { NodeClient } from '@sentry/node'; +import type { AnrIntegrationOptions, WorkerStartData } from './common'; + +const { isPromise } = types; + +const DEFAULT_INTERVAL = 50; +const DEFAULT_HANG_THRESHOLD = 5000; + +function log(message: string, ...args: unknown[]): void { + logger.log(`[Node Stalled] ${message}`, ...args); +} + +/** + * Gets contexts by calling all event processors. This shouldn't be called until all integrations are setup + */ +async function getContexts(client: NodeClient): Promise { + let event: Event | null = { message: 'ANR' }; + const eventHint: EventHint = {}; + + for (const processor of client.getEventProcessors()) { + if (event === null) break; + event = await processor(event, eventHint); + } + + return event?.contexts || {}; +} + +const INTEGRATION_NAME = 'NodeStalled'; + +type AnrInternal = { startWorker: () => void; stopWorker: () => void }; + +const _anrIntegration = ((options: Partial = {}) => { + let worker: Promise<() => void> | undefined; + let client: NodeClient | undefined; + + return { + name: INTEGRATION_NAME, + startWorker: () => { + if (worker) { + return; + } + + if (client) { + worker = _startWorker(client, options); + } + }, + stopWorker: () => { + if (worker) { + // eslint-disable-next-line @typescript-eslint/no-floating-promises + worker.then(stop => { + stop(); + worker = undefined; + }); + } + }, + async setup(initClient: NodeClient) { + client = initClient; + + // setImmediate is used to ensure that all other integrations have had their setup called first. + // This allows us to call into all integrations to fetch the full context + setImmediate(() => this.startWorker()); + }, + } as Integration & AnrInternal; +}) satisfies IntegrationFn; + +type AnrReturn = (options?: Partial) => Integration & AnrInternal; + +export const anrIntegration = defineIntegration(_anrIntegration) as AnrReturn; + +/** + * Starts the ANR worker thread + * + * @returns A function to stop the worker + */ +async function _startWorker( + client: NodeClient, + integrationOptions: Partial, +): Promise<() => void> { + const dsn = client.getDsn(); + + if (!dsn) { + return () => { + // + }; + } + + const contexts = await getContexts(client); + + // These will not be accurate if sent later from the worker thread + delete contexts.app?.app_memory; + delete contexts.device?.free_memory; + + const initOptions = client.getOptions(); + + const sdkMetadata = client.getSdkMetadata() || {}; + if (sdkMetadata.sdk) { + sdkMetadata.sdk.integrations = initOptions.integrations.map(i => i.name); + } + + const options: WorkerStartData = { + debug: logger.isEnabled(), + dsn, + tunnel: initOptions.tunnel, + environment: initOptions.environment || 'production', + release: initOptions.release, + dist: initOptions.dist, + sdkMetadata, + appRootPath: integrationOptions.appRootPath, + pollInterval: integrationOptions.pollInterval || DEFAULT_INTERVAL, + anrThreshold: integrationOptions.anrThreshold || DEFAULT_HANG_THRESHOLD, + maxAnrEvents: integrationOptions.maxAnrEvents || 1, + staticTags: integrationOptions.staticTags || {}, + contexts, + }; + + const worker = new Worker(new URL('./worker', import.meta.url), { + workerData: options, + // We don't want any Node args like --import to be passed to the worker + execArgv: [], + env: { ...process.env, NODE_OPTIONS: undefined }, + }); + + process.on('exit', () => { + // eslint-disable-next-line @typescript-eslint/no-floating-promises + worker.terminate(); + }); + + const timer = setInterval(() => { + try { + const currentSession = getIsolationScope().getSession(); + // We need to copy the session object and remove the toJSON method so it can be sent to the worker + // serialized without making it a SerializedSession + const session = currentSession ? { ...currentSession, toJSON: undefined } : undefined; + // message the worker to tell it the main event loop is still running + worker.postMessage({ session, debugImages: getFilenameToDebugIdMap(initOptions.stackParser) }); + } catch (_) { + // + } + }, options.pollInterval); + // Timer should not block exit + timer.unref(); + + worker.on('message', (msg: string) => { + if (msg === 'session-ended') { + log('ANR event sent from ANR worker. Clearing session in this thread.'); + getIsolationScope().setSession(undefined); + } + }); + + worker.once('error', (err: Error) => { + clearInterval(timer); + log('ANR worker error', err); + }); + + worker.once('exit', (code: number) => { + clearInterval(timer); + log('ANR worker exit', code); + }); + + // Ensure this thread can't block app exit + worker.unref(); + + return () => { + // eslint-disable-next-line @typescript-eslint/no-floating-promises + worker.terminate(); + clearInterval(timer); + }; +} + +export function disableAnrDetectionForCallback(callback: () => T): T; +export function disableAnrDetectionForCallback(callback: () => Promise): Promise; +/** + * Disables ANR detection for the duration of the callback + */ +export function disableAnrDetectionForCallback(callback: () => T | Promise): T | Promise { + const integration = getClient()?.getIntegrationByName(INTEGRATION_NAME) as AnrInternal | undefined; + + if (!integration) { + return callback(); + } + + integration.stopWorker(); + + const result = callback(); + if (isPromise(result)) { + return result.finally(() => integration.startWorker()); + } + + integration.startWorker(); + return result; +} diff --git a/packages/node-stalled/src/worker.ts b/packages/node-stalled/src/worker.ts new file mode 100644 index 000000000000..515633237d4f --- /dev/null +++ b/packages/node-stalled/src/worker.ts @@ -0,0 +1,292 @@ +import { parentPort, workerData } from 'node:worker_threads'; +import type { DebugImage, Event, ScopeData, Session, StackFrame } from '@sentry/core'; +import { + applyScopeDataToEvent, + callFrameToStackFrame, + createEventEnvelope, + createSessionEnvelope, + generateSpanId, + getEnvelopeEndpointWithUrlEncodedAuth, + makeSession, + normalizeUrlToBase, + stripSentryFramesAndReverse, + updateSession, + uuid4, +} from '@sentry/core'; +import { createGetModuleFromFilename, makeNodeTransport } from '@sentry/node'; +import type { WorkerStartData } from './common'; + +const options: WorkerStartData = workerData; +let session: Session | undefined; +let sentAnrEvents = 0; +let mainDebugImages: Record = {}; + +function log(msg: string): void { + if (options.debug) { + // eslint-disable-next-line no-console + console.log(`[Stalled Watchdog] ${msg}`); + } +} + +const url = getEnvelopeEndpointWithUrlEncodedAuth(options.dsn, options.tunnel, options.sdkMetadata.sdk); +const transport = makeNodeTransport({ + url, + recordDroppedEvent: () => { + // + }, +}); + +async function sendAbnormalSession(): Promise { + // of we have an existing session passed from the main thread, send it as abnormal + if (session) { + log('Sending abnormal session'); + + updateSession(session, { + status: 'abnormal', + abnormal_mechanism: 'anr_foreground', + release: options.release, + environment: options.environment, + }); + + const envelope = createSessionEnvelope(session, options.dsn, options.sdkMetadata, options.tunnel); + // Log the envelope so to aid in testing + log(JSON.stringify(envelope)); + + await transport.send(envelope); + + try { + // Notify the main process that the session has ended so the session can be cleared from the scope + parentPort?.postMessage('session-ended'); + } catch (_) { + // ignore + } + } +} + +log('Started'); + +function prepareStackFrames(stackFrames: StackFrame[] | undefined): StackFrame[] | undefined { + if (!stackFrames) { + return undefined; + } + + // Strip Sentry frames and reverse the stack frames so they are in the correct order + const strippedFrames = stripSentryFramesAndReverse(stackFrames); + + // If we have an app root path, rewrite the filenames to be relative to the app root + if (options.appRootPath) { + for (const frame of strippedFrames) { + if (!frame.filename) { + continue; + } + + frame.filename = normalizeUrlToBase(frame.filename, options.appRootPath); + } + } + + return strippedFrames; +} + +function applyDebugMeta(event: Event): void { + if (Object.keys(mainDebugImages).length === 0) { + return; + } + + const normalisedDebugImages = options.appRootPath ? {} : mainDebugImages; + if (options.appRootPath) { + for (const [path, debugId] of Object.entries(mainDebugImages)) { + normalisedDebugImages[normalizeUrlToBase(path, options.appRootPath)] = debugId; + } + } + + const filenameToDebugId = new Map(); + + for (const exception of event.exception?.values || []) { + for (const frame of exception.stacktrace?.frames || []) { + const filename = frame.abs_path || frame.filename; + if (filename && normalisedDebugImages[filename]) { + filenameToDebugId.set(filename, normalisedDebugImages[filename] as string); + } + } + } + + if (filenameToDebugId.size > 0) { + const images: DebugImage[] = []; + for (const [code_file, debug_id] of filenameToDebugId.entries()) { + images.push({ + type: 'sourcemap', + code_file, + debug_id, + }); + } + event.debug_meta = { images }; + } +} + +function applyScopeToEvent(event: Event, scope: ScopeData): void { + applyScopeDataToEvent(event, scope); + + if (!event.contexts?.trace) { + const { traceId, parentSpanId, propagationSpanId } = scope.propagationContext; + event.contexts = { + trace: { + trace_id: traceId, + span_id: propagationSpanId || generateSpanId(), + parent_span_id: parentSpanId, + }, + ...event.contexts, + }; + } +} + +async function sendAnrEvent(frames?: StackFrame[], scope?: ScopeData): Promise { + if (sentAnrEvents >= options.maxAnrEvents) { + return; + } + + sentAnrEvents += 1; + + await sendAbnormalSession(); + + log('Sending event'); + + const event: Event = { + event_id: uuid4(), + contexts: options.contexts, + release: options.release, + environment: options.environment, + dist: options.dist, + platform: 'node', + level: 'error', + exception: { + values: [ + { + type: 'ApplicationNotResponding', + value: `Application Not Responding for at least ${options.anrThreshold} ms`, + stacktrace: { frames: prepareStackFrames(frames) }, + // This ensures the UI doesn't say 'Crashed in' for the stack trace + mechanism: { type: 'ANR' }, + }, + ], + }, + tags: options.staticTags, + }; + + if (scope) { + applyScopeToEvent(event, scope); + } + + applyDebugMeta(event); + + const envelope = createEventEnvelope(event, options.dsn, options.sdkMetadata, options.tunnel); + // Log the envelope to aid in testing + log(JSON.stringify(envelope)); + + await transport.send(envelope); + await transport.flush(2000); + + if (sentAnrEvents >= options.maxAnrEvents) { + // Delay for 5 seconds so that stdio can flush if the main event loop ever restarts. + // This is mainly for the benefit of logging or debugging. + setTimeout(() => { + process.exit(0); + }, 5_000); + } +} + +if (options.captureStackTrace) { + log('Connecting to debugger'); + + const session = new InspectorSession(); + session.connectToMainThread(); + + log('Connected to debugger'); + + // Collect scriptId -> url map so we can look up the filenames later + const scripts = new Map(); + + session.on('Debugger.scriptParsed', event => { + scripts.set(event.params.scriptId, event.params.url); + }); + + session.on('Debugger.paused', event => { + if (event.params.reason !== 'other') { + return; + } + + try { + log('Debugger paused'); + + // copy the frames + const callFrames = [...event.params.callFrames]; + + const getModuleName = options.appRootPath ? createGetModuleFromFilename(options.appRootPath) : () => undefined; + const stackFrames = callFrames.map(frame => + callFrameToStackFrame(frame, scripts.get(frame.location.scriptId), getModuleName), + ); + + // Runtime.evaluate may never return if the event loop is blocked indefinitely + // In that case, we want to send the event anyway + const getScopeTimeout = setTimeout(() => { + sendAnrEvent(stackFrames).then(null, () => { + log('Sending ANR event failed.'); + }); + }, 5_000); + + // Evaluate a script in the currently paused context + session.post( + 'Runtime.evaluate', + { + // Grab the trace context from the current scope + expression: 'global.__SENTRY_GET_SCOPES__();', + // Don't re-trigger the debugger if this causes an error + silent: true, + // Serialize the result to json otherwise only primitives are supported + returnByValue: true, + }, + (err, param) => { + if (err) { + log(`Error executing script: '${err.message}'`); + } + + clearTimeout(getScopeTimeout); + + const scopes = param?.result ? (param.result.value as ScopeData) : undefined; + + session.post('Debugger.resume'); + session.post('Debugger.disable'); + + sendAnrEvent(stackFrames, scopes).then(null, () => { + log('Sending ANR event failed.'); + }); + }, + ); + } catch (e) { + session.post('Debugger.resume'); + session.post('Debugger.disable'); + throw e; + } + }); + + debuggerPause = () => { + try { + session.post('Debugger.enable', () => { + session.post('Debugger.pause'); + }); + } catch (_) { + // + } + }; +} + +parentPort?.on('message', (msg: { session: Session | undefined; debugImages?: Record }) => { + if (msg.session) { + session = makeSession(msg.session); + } + + if (msg.debugImages) { + mainDebugImages = msg.debugImages; + } + + poll(); +}); diff --git a/packages/node-stalled/tsconfig.json b/packages/node-stalled/tsconfig.json new file mode 100644 index 000000000000..29acbf3f36e9 --- /dev/null +++ b/packages/node-stalled/tsconfig.json @@ -0,0 +1,10 @@ +{ + "extends": "../../tsconfig.json", + "compilerOptions": { + "module": "esnext", + "lib": ["es2018"], + "outDir": "build", + "types": ["node"] + }, + "include": ["src/**/*"] +} diff --git a/packages/node-stalled/tsconfig.test.json b/packages/node-stalled/tsconfig.test.json new file mode 100644 index 000000000000..c401c76a5305 --- /dev/null +++ b/packages/node-stalled/tsconfig.test.json @@ -0,0 +1,12 @@ +{ + "extends": "./tsconfig.json", + + "include": ["test/**/*", "src/**/*.d.ts", "vite.config.ts"], + + "compilerOptions": { + // should include all types from `./tsconfig.json` plus types for all test frameworks used + "types": ["node"] + + // other package-specific, test-specific options + } +} diff --git a/packages/node-stalled/tsconfig.types.json b/packages/node-stalled/tsconfig.types.json new file mode 100644 index 000000000000..7a01535e9a4c --- /dev/null +++ b/packages/node-stalled/tsconfig.types.json @@ -0,0 +1,11 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "declaration": true, + "declarationMap": true, + "emitDeclarationOnly": true, + "outDir": "build/types", + "types": ["node"] + }, + "files": ["src/index.ts"] +} diff --git a/packages/node-stalled/vite.config.ts b/packages/node-stalled/vite.config.ts new file mode 100644 index 000000000000..f18ec92095bc --- /dev/null +++ b/packages/node-stalled/vite.config.ts @@ -0,0 +1,8 @@ +import baseConfig from '../../vite/vite.config'; + +export default { + ...baseConfig, + test: { + ...baseConfig.test, + }, +}; From 59b6bff782a39b04e1e9cee66226cbaa2a4a2c65 Mon Sep 17 00:00:00 2001 From: Tim Fish Date: Wed, 18 Jun 2025 10:16:25 +0200 Subject: [PATCH 02/16] Mostly working --- .../suites/thread-blocked-native/app-path.mjs | 23 ++ .../thread-blocked-native/basic-multiple.mjs | 23 ++ .../suites/thread-blocked-native/basic.js | 24 ++ .../suites/thread-blocked-native/basic.mjs | 24 ++ .../thread-blocked-native/indefinite.mjs | 27 ++ .../thread-blocked-native/instrument.mjs | 9 + .../suites/thread-blocked-native/long-work.js | 12 + .../should-exit-forced.js | 19 ++ .../thread-blocked-native/should-exit.js | 18 + .../thread-blocked-native/stop-and-start.js | 45 +++ .../suites/thread-blocked-native/test.ts | 201 +++++++++++ .../thread-blocked-native/worker-block.mjs | 5 + .../thread-blocked-native/worker-main.mjs | 14 + dev-packages/node-integration-tests/test.txt | 213 ------------ .../node-integration-tests/utils/runner.ts | 11 +- package.json | 2 +- packages/core/src/types-hoist/exception.ts | 2 +- packages/core/src/types-hoist/thread.ts | 3 +- .../.eslintrc.js | 0 .../{node-stalled => node-native}/LICENSE | 2 +- packages/node-native/README.md | 64 ++++ .../package.json | 23 +- .../rollup.npm.config.mjs | 8 +- .../src/common.ts | 23 +- packages/node-native/src/index.ts | 1 + .../src/thread-blocked-integration.ts} | 62 ++-- .../src/thread-blocked-watchdog.ts | 256 ++++++++++++++ .../tsconfig.json | 0 .../tsconfig.test.json | 0 .../tsconfig.types.json | 0 .../vite.config.ts | 0 packages/node-stalled/README.md | 316 ------------------ packages/node-stalled/src/worker.ts | 292 ---------------- yarn.lock | 11 +- 34 files changed, 845 insertions(+), 888 deletions(-) create mode 100644 dev-packages/node-integration-tests/suites/thread-blocked-native/app-path.mjs create mode 100644 dev-packages/node-integration-tests/suites/thread-blocked-native/basic-multiple.mjs create mode 100644 dev-packages/node-integration-tests/suites/thread-blocked-native/basic.js create mode 100644 dev-packages/node-integration-tests/suites/thread-blocked-native/basic.mjs create mode 100644 dev-packages/node-integration-tests/suites/thread-blocked-native/indefinite.mjs create mode 100644 dev-packages/node-integration-tests/suites/thread-blocked-native/instrument.mjs create mode 100644 dev-packages/node-integration-tests/suites/thread-blocked-native/long-work.js create mode 100644 dev-packages/node-integration-tests/suites/thread-blocked-native/should-exit-forced.js create mode 100644 dev-packages/node-integration-tests/suites/thread-blocked-native/should-exit.js create mode 100644 dev-packages/node-integration-tests/suites/thread-blocked-native/stop-and-start.js create mode 100644 dev-packages/node-integration-tests/suites/thread-blocked-native/test.ts create mode 100644 dev-packages/node-integration-tests/suites/thread-blocked-native/worker-block.mjs create mode 100644 dev-packages/node-integration-tests/suites/thread-blocked-native/worker-main.mjs delete mode 100644 dev-packages/node-integration-tests/test.txt rename packages/{node-stalled => node-native}/.eslintrc.js (100%) rename packages/{node-stalled => node-native}/LICENSE (94%) create mode 100644 packages/node-native/README.md rename packages/{node-stalled => node-native}/package.json (82%) rename packages/{node-stalled => node-native}/rollup.npm.config.mjs (52%) rename packages/{node-stalled => node-native}/src/common.ts (50%) create mode 100644 packages/node-native/src/index.ts rename packages/{node-stalled/src/index.ts => node-native/src/thread-blocked-integration.ts} (70%) create mode 100644 packages/node-native/src/thread-blocked-watchdog.ts rename packages/{node-stalled => node-native}/tsconfig.json (100%) rename packages/{node-stalled => node-native}/tsconfig.test.json (100%) rename packages/{node-stalled => node-native}/tsconfig.types.json (100%) rename packages/{node-stalled => node-native}/vite.config.ts (100%) delete mode 100644 packages/node-stalled/README.md delete mode 100644 packages/node-stalled/src/worker.ts diff --git a/dev-packages/node-integration-tests/suites/thread-blocked-native/app-path.mjs b/dev-packages/node-integration-tests/suites/thread-blocked-native/app-path.mjs new file mode 100644 index 000000000000..d3043df842ff --- /dev/null +++ b/dev-packages/node-integration-tests/suites/thread-blocked-native/app-path.mjs @@ -0,0 +1,23 @@ +import * as Sentry from '@sentry/node'; +import { threadBlockedIntegration } from '@sentry/node-native'; +import * as path from 'path'; +import * as url from 'url'; +import { longWork } from './long-work.js'; + +global._sentryDebugIds = { [new Error().stack]: 'aaaaaaaa-aaaa-4aaa-aaaa-aaaaaaaaaa' }; + +const __dirname = path.dirname(url.fileURLToPath(import.meta.url)); + +setTimeout(() => { + process.exit(); +}, 10000); + +Sentry.init({ + dsn: process.env.SENTRY_DSN, + release: '1.0', + integrations: [threadBlockedIntegration({ blockedThreshold: 100, appRootPath: __dirname })], +}); + +setTimeout(() => { + longWork(); +}, 1000); diff --git a/dev-packages/node-integration-tests/suites/thread-blocked-native/basic-multiple.mjs b/dev-packages/node-integration-tests/suites/thread-blocked-native/basic-multiple.mjs new file mode 100644 index 000000000000..62c024f81f45 --- /dev/null +++ b/dev-packages/node-integration-tests/suites/thread-blocked-native/basic-multiple.mjs @@ -0,0 +1,23 @@ +import * as Sentry from '@sentry/node'; +import { threadBlockedIntegration } from '@sentry/node-native'; +import { longWork } from './long-work.js'; + +global._sentryDebugIds = { [new Error().stack]: 'aaaaaaaa-aaaa-4aaa-aaaa-aaaaaaaaaa' }; + +setTimeout(() => { + process.exit(); +}, 10000); + +Sentry.init({ + dsn: process.env.SENTRY_DSN, + release: '1.0', + integrations: [threadBlockedIntegration({ blockedThreshold: 100, maxBlockedEvents: 2 })], +}); + +setTimeout(() => { + longWork(); +}, 1000); + +setTimeout(() => { + longWork(); +}, 4000); diff --git a/dev-packages/node-integration-tests/suites/thread-blocked-native/basic.js b/dev-packages/node-integration-tests/suites/thread-blocked-native/basic.js new file mode 100644 index 000000000000..2db4260f95bb --- /dev/null +++ b/dev-packages/node-integration-tests/suites/thread-blocked-native/basic.js @@ -0,0 +1,24 @@ +const Sentry = require('@sentry/node'); +const { threadBlockedIntegration } = require('@sentry/node-native'); +const { longWork } = require('./long-work.js'); + +global._sentryDebugIds = { [new Error().stack]: 'aaaaaaaa-aaaa-4aaa-aaaa-aaaaaaaaaa' }; + +setTimeout(() => { + process.exit(); +}, 10000); + +Sentry.init({ + dsn: process.env.SENTRY_DSN, + release: '1.0', + integrations: [threadBlockedIntegration({ blockedThreshold: 100 })], +}); + +setTimeout(() => { + longWork(); +}, 2000); + +// Ensure we only send one event even with multiple blocking events +setTimeout(() => { + longWork(); +}, 5000); diff --git a/dev-packages/node-integration-tests/suites/thread-blocked-native/basic.mjs b/dev-packages/node-integration-tests/suites/thread-blocked-native/basic.mjs new file mode 100644 index 000000000000..b18465e8655b --- /dev/null +++ b/dev-packages/node-integration-tests/suites/thread-blocked-native/basic.mjs @@ -0,0 +1,24 @@ +import * as Sentry from '@sentry/node'; +import { threadBlockedIntegration } from '@sentry/node-native'; +import { longWork } from './long-work.js'; + +global._sentryDebugIds = { [new Error().stack]: 'aaaaaaaa-aaaa-4aaa-aaaa-aaaaaaaaaa' }; + +setTimeout(() => { + process.exit(); +}, 12000); + +Sentry.init({ + dsn: process.env.SENTRY_DSN, + release: '1.0', + integrations: [threadBlockedIntegration({ blockedThreshold: 100 })], +}); + +setTimeout(() => { + longWork(); +}, 2000); + +// Ensure we only send one event even with multiple blocking events +setTimeout(() => { + longWork(); +}, 5000); diff --git a/dev-packages/node-integration-tests/suites/thread-blocked-native/indefinite.mjs b/dev-packages/node-integration-tests/suites/thread-blocked-native/indefinite.mjs new file mode 100644 index 000000000000..14a7b213c1b5 --- /dev/null +++ b/dev-packages/node-integration-tests/suites/thread-blocked-native/indefinite.mjs @@ -0,0 +1,27 @@ +import * as Sentry from '@sentry/node'; +import { threadBlockedIntegration } from '@sentry/node-native'; +import * as assert from 'assert'; +import * as crypto from 'crypto'; + +setTimeout(() => { + process.exit(); +}, 10000); + +Sentry.init({ + dsn: process.env.SENTRY_DSN, + release: '1.0', + integrations: [threadBlockedIntegration({ blockedThreshold: 100 })], +}); + +function longWork() { + // This loop will run almost indefinitely + for (let i = 0; i < 2000000000; i++) { + const salt = crypto.randomBytes(128).toString('base64'); + const hash = crypto.pbkdf2Sync('myPassword', salt, 10000, 512, 'sha512'); + assert.ok(hash); + } +} + +setTimeout(() => { + longWork(); +}, 1000); diff --git a/dev-packages/node-integration-tests/suites/thread-blocked-native/instrument.mjs b/dev-packages/node-integration-tests/suites/thread-blocked-native/instrument.mjs new file mode 100644 index 000000000000..4c3cae71c989 --- /dev/null +++ b/dev-packages/node-integration-tests/suites/thread-blocked-native/instrument.mjs @@ -0,0 +1,9 @@ +import * as Sentry from '@sentry/node'; +import { threadBlockedIntegration } from '@sentry/node-native'; + +Sentry.init({ + debug: true, + dsn: process.env.SENTRY_DSN, + release: '1.0', + integrations: [threadBlockedIntegration({ blockedThreshold: 100 })], +}); diff --git a/dev-packages/node-integration-tests/suites/thread-blocked-native/long-work.js b/dev-packages/node-integration-tests/suites/thread-blocked-native/long-work.js new file mode 100644 index 000000000000..f35e2268dcd6 --- /dev/null +++ b/dev-packages/node-integration-tests/suites/thread-blocked-native/long-work.js @@ -0,0 +1,12 @@ +const crypto = require('crypto'); +const assert = require('assert'); + +function longWork() { + for (let i = 0; i < 20; i++) { + const salt = crypto.randomBytes(128).toString('base64'); + const hash = crypto.pbkdf2Sync('myPassword', salt, 10000, 512, 'sha512'); + assert.ok(hash); + } +} + +exports.longWork = longWork; diff --git a/dev-packages/node-integration-tests/suites/thread-blocked-native/should-exit-forced.js b/dev-packages/node-integration-tests/suites/thread-blocked-native/should-exit-forced.js new file mode 100644 index 000000000000..5ffd83f48e24 --- /dev/null +++ b/dev-packages/node-integration-tests/suites/thread-blocked-native/should-exit-forced.js @@ -0,0 +1,19 @@ +const Sentry = require('@sentry/node'); +const { threadBlockedIntegration } = require('@sentry/node-native'); + +function configureSentry() { + Sentry.init({ + dsn: 'https://public@dsn.ingest.sentry.io/1337', + release: '1.0', + debug: true, + integrations: [threadBlockedIntegration()], + }); +} + +async function main() { + configureSentry(); + await new Promise(resolve => setTimeout(resolve, 1000)); + process.exit(0); +} + +main(); diff --git a/dev-packages/node-integration-tests/suites/thread-blocked-native/should-exit.js b/dev-packages/node-integration-tests/suites/thread-blocked-native/should-exit.js new file mode 100644 index 000000000000..16ca705fb236 --- /dev/null +++ b/dev-packages/node-integration-tests/suites/thread-blocked-native/should-exit.js @@ -0,0 +1,18 @@ +const Sentry = require('@sentry/node'); +const { threadBlockedIntegration } = require('@sentry/node-native'); + +function configureSentry() { + Sentry.init({ + dsn: 'https://public@dsn.ingest.sentry.io/1337', + release: '1.0', + debug: true, + integrations: [threadBlockedIntegration()], + }); +} + +async function main() { + configureSentry(); + await new Promise(resolve => setTimeout(resolve, 1000)); +} + +main(); diff --git a/dev-packages/node-integration-tests/suites/thread-blocked-native/stop-and-start.js b/dev-packages/node-integration-tests/suites/thread-blocked-native/stop-and-start.js new file mode 100644 index 000000000000..51402f633d8c --- /dev/null +++ b/dev-packages/node-integration-tests/suites/thread-blocked-native/stop-and-start.js @@ -0,0 +1,45 @@ +const Sentry = require('@sentry/node'); +const { threadBlockedIntegration } = require('@sentry/node-native'); +const { longWork } = require('./long-work.js'); +const crypto = require('crypto'); +const assert = require('assert'); + +setTimeout(() => { + process.exit(); +}, 10000); + +const threadBlocked = threadBlockedIntegration({ blockedThreshold: 100 }); + +Sentry.init({ + dsn: process.env.SENTRY_DSN, + release: '1.0', + debug: true, + integrations: [threadBlocked], +}); + +Sentry.setUser({ email: 'person@home.com' }); +Sentry.addBreadcrumb({ message: 'important message!' }); + +function longWorkIgnored() { + for (let i = 0; i < 20; i++) { + const salt = crypto.randomBytes(128).toString('base64'); + const hash = crypto.pbkdf2Sync('myPassword', salt, 10000, 512, 'sha512'); + assert.ok(hash); + } +} + +setTimeout(() => { + threadBlocked.stopWorker(); + + setTimeout(() => { + longWorkIgnored(); + + setTimeout(() => { + threadBlocked.startWorker(); + + setTimeout(() => { + longWork(); + }); + }, 2000); + }, 2000); +}, 2000); diff --git a/dev-packages/node-integration-tests/suites/thread-blocked-native/test.ts b/dev-packages/node-integration-tests/suites/thread-blocked-native/test.ts new file mode 100644 index 000000000000..2d0938fea306 --- /dev/null +++ b/dev-packages/node-integration-tests/suites/thread-blocked-native/test.ts @@ -0,0 +1,201 @@ +import { join } from 'node:path'; +import type { Event } from '@sentry/core'; +import { afterAll, describe, expect, test } from 'vitest'; +import { cleanupChildProcesses, createRunner } from '../../utils/runner'; + +function EXCEPTION(thread_id = '0') { + return { + values: [ + { + type: 'EventLoopBlocked', + value: 'Event Loop Blocked for at least 100 ms', + mechanism: { type: 'ANR' }, + thread_id, + stacktrace: { + frames: expect.arrayContaining([ + expect.objectContaining({ + colno: expect.any(Number), + lineno: expect.any(Number), + filename: expect.any(String), + function: '?', + in_app: true, + }), + expect.objectContaining({ + colno: expect.any(Number), + lineno: expect.any(Number), + filename: expect.any(String), + function: 'longWork', + in_app: true, + }), + ]), + }, + }, + ], + }; +} + +const ANR_EVENT = { + // Ensure we have context + contexts: { + device: { + arch: expect.any(String), + }, + app: { + app_start_time: expect.any(String), + }, + os: { + name: expect.any(String), + }, + culture: { + timezone: expect.any(String), + }, + }, + threads: { + values: [ + { + id: '0', + name: '0', + crashed: true, + current: true, + main: true, + }, + ], + }, + // and an exception that is our ANR + exception: EXCEPTION(), +}; + +const ANR_EVENT_WITH_DEBUG_META: Event = { + ...ANR_EVENT, + debug_meta: { + images: [ + { + type: 'sourcemap', + debug_id: 'aaaaaaaa-aaaa-4aaa-aaaa-aaaaaaaaaa', + code_file: expect.stringContaining('basic'), + }, + ], + }, +}; + +describe('Thread Blocked Native', { timeout: 30_000 }, () => { + afterAll(() => { + cleanupChildProcesses(); + }); + + test('CJS', async () => { + await createRunner(__dirname, 'basic.js') + .withMockSentryServer() + .expect({ event: ANR_EVENT_WITH_DEBUG_META }) + .start() + .completed(); + }); + + test('ESM', async () => { + await createRunner(__dirname, 'basic.mjs') + .withMockSentryServer() + .expect({ event: ANR_EVENT_WITH_DEBUG_META }) + .start() + .completed(); + }); + + test('Custom appRootPath', async () => { + const ANR_EVENT_WITH_SPECIFIC_DEBUG_META: Event = { + ...ANR_EVENT, + debug_meta: { + images: [ + { + type: 'sourcemap', + debug_id: 'aaaaaaaa-aaaa-4aaa-aaaa-aaaaaaaaaa', + code_file: 'app:///app-path.mjs', + }, + ], + }, + }; + + await createRunner(__dirname, 'app-path.mjs') + .withMockSentryServer() + .expect({ event: ANR_EVENT_WITH_SPECIFIC_DEBUG_META }) + .start() + .completed(); + }); + + test('multiple events via maxBlockedEvents', async () => { + await createRunner(__dirname, 'basic-multiple.mjs') + .withMockSentryServer() + .expect({ event: ANR_EVENT_WITH_DEBUG_META }) + .expect({ event: ANR_EVENT_WITH_DEBUG_META }) + .start() + .completed(); + }); + + test('blocked indefinitely', async () => { + await createRunner(__dirname, 'indefinite.mjs') + .withMockSentryServer() + .expect({ event: ANR_EVENT }) + .start() + .completed(); + }); + + test('should exit', async () => { + const runner = createRunner(__dirname, 'should-exit.js').start(); + + await new Promise(resolve => setTimeout(resolve, 5_000)); + + expect(runner.childHasExited()).toBe(true); + }); + + test('should exit forced', async () => { + const runner = createRunner(__dirname, 'should-exit-forced.js').start(); + + await new Promise(resolve => setTimeout(resolve, 5_000)); + + expect(runner.childHasExited()).toBe(true); + }); + + test('worker thread', async () => { + const ANR_EVENT_THREADS: Event = { + ...ANR_EVENT, + exception: { + ...EXCEPTION('2'), + }, + threads: { + values: [ + { + id: '0', + name: '0', + crashed: false, + current: true, + main: true, + stacktrace: { + frames: expect.any(Array), + }, + }, + { + id: '2', + name: '2', + crashed: true, + current: true, + main: false, + }, + ], + }, + }; + + const instrument = join(__dirname, 'instrument.mjs'); + await createRunner(__dirname, 'worker-main.mjs') + .withMockSentryServer() + .withFlags('--import', instrument) + .expect({ event: ANR_EVENT_THREADS }) + .start() + .completed(); + }); + + test('watchdog can be stopped and restarted', async () => { + await createRunner(__dirname, 'stop-and-start.js') + .withMockSentryServer() + .expect({ event: ANR_EVENT }) + .start() + .completed(); + }); +}); diff --git a/dev-packages/node-integration-tests/suites/thread-blocked-native/worker-block.mjs b/dev-packages/node-integration-tests/suites/thread-blocked-native/worker-block.mjs new file mode 100644 index 000000000000..274a4ce9e3a9 --- /dev/null +++ b/dev-packages/node-integration-tests/suites/thread-blocked-native/worker-block.mjs @@ -0,0 +1,5 @@ +import { longWork } from './long-work.js'; + +setTimeout(() => { + longWork(); +}, 2000); diff --git a/dev-packages/node-integration-tests/suites/thread-blocked-native/worker-main.mjs b/dev-packages/node-integration-tests/suites/thread-blocked-native/worker-main.mjs new file mode 100644 index 000000000000..8591be4197e3 --- /dev/null +++ b/dev-packages/node-integration-tests/suites/thread-blocked-native/worker-main.mjs @@ -0,0 +1,14 @@ +import { Worker } from 'node:worker_threads'; +import * as path from 'path'; +import * as url from 'url'; + +const __dirname = path.dirname(url.fileURLToPath(import.meta.url)); + +const workerPath = path.join(__dirname, 'worker-block.mjs'); + +const thread = new Worker(workerPath, { stdout: 'inherit' }); +thread.unref(); + +setInterval(() => { + // This keeps the main thread alive to allow the worker to run indefinitely +}, 1000); diff --git a/dev-packages/node-integration-tests/test.txt b/dev-packages/node-integration-tests/test.txt deleted file mode 100644 index 0a0fa7f94de9..000000000000 --- a/dev-packages/node-integration-tests/test.txt +++ /dev/null @@ -1,213 +0,0 @@ -yarn run v1.22.22 -$ /Users/abhijeetprasad/workspace/sentry-javascript/node_modules/.bin/jest contextLines/memory-leak - console.log - starting scenario /Users/abhijeetprasad/workspace/sentry-javascript/dev-packages/node-integration-tests/suites/contextLines/memory-leak/scenario.ts [ '-r', 'ts-node/register' ] undefined - - at log (utils/runner.ts:462:11) - - console.log - line COMMAND PID USER FD TYPE DEVICE SIZE/OFF NODE NAME - - at log (utils/runner.ts:462:11) - - console.log - line node 90932 abhijeetprasad cwd DIR 1,16 608 107673020 /Users/abhijeetprasad/workspace/sentry-javascript/dev-packages/node-integration-tests - - at log (utils/runner.ts:462:11) - - console.log - line node 90932 abhijeetprasad txt REG 1,16 88074480 114479727 /Users/abhijeetprasad/.volta/tools/image/node/18.20.5/bin/node - - at log (utils/runner.ts:462:11) - - console.log - line node 90932 abhijeetprasad 0u unix 0x6a083c8cc83ea8db 0t0 ->0xf2cacdd1d3a0ebec - - at log (utils/runner.ts:462:11) - - console.log - line node 90932 abhijeetprasad 1u unix 0xd99cc422a76ba47f 0t0 ->0x542148981a0b9ef2 - - at log (utils/runner.ts:462:11) - - console.log - line node 90932 abhijeetprasad 2u unix 0x97e70527ed5803f8 0t0 ->0xbafdaf00ef20de83 - - at log (utils/runner.ts:462:11) - - console.log - line node 90932 abhijeetprasad 3u KQUEUE count=0, state=0 - - at log (utils/runner.ts:462:11) - - console.log - line node 90932 abhijeetprasad 4 PIPE 0x271836c29e42bc67 16384 ->0x16ac23fcfd4fe1a3 - - at log (utils/runner.ts:462:11) - - console.log - line node 90932 abhijeetprasad 5 PIPE 0x16ac23fcfd4fe1a3 16384 ->0x271836c29e42bc67 - - at log (utils/runner.ts:462:11) - - console.log - line node 90932 abhijeetprasad 6 PIPE 0xd76fcd4ca2a35fcf 16384 ->0x30d26cd4f0e069b2 - - at log (utils/runner.ts:462:11) - - console.log - line node 90932 abhijeetprasad 7 PIPE 0x30d26cd4f0e069b2 16384 ->0xd76fcd4ca2a35fcf - - at log (utils/runner.ts:462:11) - - console.log - line node 90932 abhijeetprasad 8 PIPE 0x37691847717c3d6 16384 ->0x966eedd79d018252 - - at log (utils/runner.ts:462:11) - - console.log - line node 90932 abhijeetprasad 9 PIPE 0x966eedd79d018252 16384 ->0x37691847717c3d6 - - at log (utils/runner.ts:462:11) - - console.log - line node 90932 abhijeetprasad 10u KQUEUE count=0, state=0xa - - at log (utils/runner.ts:462:11) - - console.log - line node 90932 abhijeetprasad 11 PIPE 0x99c1186f14b865be 16384 ->0xe88675eb1eefb2b - - at log (utils/runner.ts:462:11) - - console.log - line node 90932 abhijeetprasad 12 PIPE 0xe88675eb1eefb2b 16384 ->0x99c1186f14b865be - - at log (utils/runner.ts:462:11) - - console.log - line node 90932 abhijeetprasad 13 PIPE 0x52173210451cdda9 16384 ->0x50bbc31a0f1cc1af - - at log (utils/runner.ts:462:11) - - console.log - line node 90932 abhijeetprasad 14 PIPE 0x50bbc31a0f1cc1af 16384 ->0x52173210451cdda9 - - at log (utils/runner.ts:462:11) - - console.log - line node 90932 abhijeetprasad 15u KQUEUE count=0, state=0 - - at log (utils/runner.ts:462:11) - - console.log - line node 90932 abhijeetprasad 16 PIPE 0xa115aa0653327e72 16384 ->0x100525c465ee1eb0 - - at log (utils/runner.ts:462:11) - - console.log - line node 90932 abhijeetprasad 17 PIPE 0x100525c465ee1eb0 16384 ->0xa115aa0653327e72 - - at log (utils/runner.ts:462:11) - - console.log - line node 90932 abhijeetprasad 18 PIPE 0x41945cf9fe740277 16384 ->0x8791d18eade5b1e0 - - at log (utils/runner.ts:462:11) - - console.log - line node 90932 abhijeetprasad 19 PIPE 0x8791d18eade5b1e0 16384 ->0x41945cf9fe740277 - - at log (utils/runner.ts:462:11) - - console.log - line node 90932 abhijeetprasad 20r CHR 3,2 0t0 333 /dev/null - - at log (utils/runner.ts:462:11) - - console.log - line node 90932 abhijeetprasad 21u KQUEUE count=0, state=0xa - - at log (utils/runner.ts:462:11) - - console.log - line node 90932 abhijeetprasad 22 PIPE 0xf4c6a2f47fb0bff5 16384 ->0xa00185e1c59cedbe - - at log (utils/runner.ts:462:11) - - console.log - line node 90932 abhijeetprasad 23 PIPE 0xa00185e1c59cedbe 16384 ->0xf4c6a2f47fb0bff5 - - at log (utils/runner.ts:462:11) - - console.log - line node 90932 abhijeetprasad 24 PIPE 0x4ac25a99f45f7ca4 16384 ->0x2032aef840c94700 - - at log (utils/runner.ts:462:11) - - console.log - line node 90932 abhijeetprasad 25 PIPE 0x2032aef840c94700 16384 ->0x4ac25a99f45f7ca4 - - at log (utils/runner.ts:462:11) - - console.log - line null - - at log (utils/runner.ts:462:11) - - console.log - line [{"sent_at":"2025-01-13T21:47:47.663Z","sdk":{"name":"sentry.javascript.node","version":"8.45.0"}},[[{"type":"session"},{"sid":"0ae9ef2ac2ba49dd92b6dab9d81444ac","init":true,"started":"2025-01-13T21:47:47.502Z","timestamp":"2025-01-13T21:47:47.663Z","status":"ok","errors":1,"duration":0.16146087646484375,"attrs":{"release":"1.0","environment":"production"}}]]] - - at log (utils/runner.ts:462:11) - - console.log - line [{"event_id":"2626269e3c634fc289338c441e76412c","sent_at":"2025-01-13T21:47:47.663Z","sdk":{"name":"sentry.javascript.node","version":"8.45.0"},"trace":{"environment":"production","release":"1.0","public_key":"public","trace_id":"efdb9350effb47959d48bd0aaf395824"}},[[{"type":"event"},{"exception":{"values":[{"type":"Error","value":"error in loop 0","stacktrace":{"frames":[{"filename":"node:internal/main/run_main_module","module":"run_main_module","function":"?","lineno":28,"colno":49,"in_app":false},{"filename":"node:internal/modules/run_main","module":"run_main","function":"Function.executeUserEntryPoint [as runMain]","lineno":128,"colno":12,"in_app":false},{"filename":"node:internal/modules/cjs/loader","module":"loader","function":"Module._load","lineno":1019,"colno":12,"in_app":false},{"filename":"node:internal/modules/cjs/loader","module":"loader","function":"Module.load","lineno":1203,"colno":32,"in_app":false},{"filename":"/Users/abhijeetprasad/workspace/sentry-javascript/node_modules/ts-node/src/index.ts","module":"ts-node.src:index.ts","function":"Object.require.extensions. [as .ts]","lineno":1621,"colno":12,"in_app":false},{"filename":"node:internal/modules/cjs/loader","module":"loader","function":"Module._extensions..js","lineno":1422,"colno":10,"in_app":false},{"filename":"/Users/abhijeetprasad/workspace/sentry-javascript/node_modules/ts-node/src/index.ts","module":"ts-node.src:index.ts","function":"Module.m._compile","lineno":1618,"colno":23,"in_app":false},{"filename":"node:internal/modules/cjs/loader","module":"loader","function":"Module._compile","lineno":1364,"colno":14,"in_app":false},{"filename":"/Users/abhijeetprasad/workspace/sentry-javascript/dev-packages/node-integration-tests/suites/contextLines/memory-leak/scenario.ts","module":"scenario.ts","function":"Object.?","lineno":14,"colno":10,"in_app":true,"pre_context":[" dsn: 'https://public@dsn.ingest.sentry.io/1337',"," release: '1.0',"," transport: loggingTransport,","});","","import { runSentry } from './other-file';",""],"context_line":"runSentry();","post_context":["","console.log(execSync(`lsof -p ${process.pid}`, { stdio: 'inherit', cwd: process.cwd() }));"]},{"filename":"/Users/abhijeetprasad/workspace/sentry-javascript/dev-packages/node-integration-tests/suites/contextLines/memory-leak/other-file.ts","module":"other-file.ts","function":"runSentry","lineno":5,"colno":29,"in_app":true,"pre_context":["import * as Sentry from '@sentry/node';","","export function runSentry(): void {"," for (let i = 0; i < 10; i++) {"],"context_line":" Sentry.captureException(new Error(`error in loop ${i}`));","post_context":[" }","}"]}]},"mechanism":{"type":"generic","handled":true}}]},"event_id":"2626269e3c634fc289338c441e76412c","level":"error","platform":"node","contexts":{"trace":{"trace_id":"efdb9350effb47959d48bd0aaf395824","span_id":"b1e1b8a0d410ef14"},"runtime":{"name":"node","version":"v18.20.5"},"app":{"app_start_time":"2025-01-13T21:47:46.327Z","app_memory":270073856},"os":{"kernel_version":"23.6.0","name":"macOS","version":"14.7","build":"23H124"},"device":{"boot_time":"2024-12-23T16:56:50.637Z","arch":"arm64","memory_size":34359738368,"free_memory":355794944,"processor_count":10,"cpu_description":"Apple M1 Pro","processor_frequency":24},"culture":{"locale":"en-CA","timezone":"America/Toronto"},"cloud_resource":{}},"server_name":"GT9RQ02WW5.local","timestamp":1736804867.528,"environment":"production","release":"1.0","sdk":{"integrations":["InboundFilters","FunctionToString","LinkedErrors","RequestData","Console","Http","NodeFetch","OnUncaughtException","OnUnhandledRejection","ContextLines","LocalVariables","Context","ChildProcess","Modules"],"name":"sentry.javascript.node","version":"8.45.0","packages":[{"name":"npm:@sentry/node","version":"8.45.0"}]},"modules":{"ts-node":"10.9.1","make-error":"1.3.6","yn":"3.1.1","arg":"4.1.3","v8-compile-cache-lib":"3.0.1","typescript":"5.0.4","tslib":"2.7.0","semver":"7.6.3","shimmer":"1.2.1","require-in-the-middle":"7.2.0","resolve":"1.22.1","is-core-module":"2.11.0","has":"1.0.3","function-bind":"1.1.1","debug":"4.3.4","supports-color":"7.2.0","has-flag":"4.0.0","module-details-from-path":"1.0.3","import-in-the-middle":"1.12.0","forwarded-parse":"2.1.2"}}]]] - - at log (utils/runner.ts:462:11) - - console.log - line [{"event_id":"f58236bf0a7f4a999f7daf5283f0400f","sent_at":"2025-01-13T21:47:47.664Z","sdk":{"name":"sentry.javascript.node","version":"8.45.0"},"trace":{"environment":"production","release":"1.0","public_key":"public","trace_id":"efdb9350effb47959d48bd0aaf395824"}},[[{"type":"event"},{"exception":{"values":[{"type":"Error","value":"error in loop 1","stacktrace":{"frames":[{"filename":"node:internal/main/run_main_module","module":"run_main_module","function":"?","lineno":28,"colno":49,"in_app":false},{"filename":"node:internal/modules/run_main","module":"run_main","function":"Function.executeUserEntryPoint [as runMain]","lineno":128,"colno":12,"in_app":false},{"filename":"node:internal/modules/cjs/loader","module":"loader","function":"Module._load","lineno":1019,"colno":12,"in_app":false},{"filename":"node:internal/modules/cjs/loader","module":"loader","function":"Module.load","lineno":1203,"colno":32,"in_app":false},{"filename":"/Users/abhijeetprasad/workspace/sentry-javascript/node_modules/ts-node/src/index.ts","module":"ts-node.src:index.ts","function":"Object.require.extensions. [as .ts]","lineno":1621,"colno":12,"in_app":false},{"filename":"node:internal/modules/cjs/loader","module":"loader","function":"Module._extensions..js","lineno":1422,"colno":10,"in_app":false},{"filename":"/Users/abhijeetprasad/workspace/sentry-javascript/node_modules/ts-node/src/index.ts","module":"ts-node.src:index.ts","function":"Module.m._compile","lineno":1618,"colno":23,"in_app":false},{"filename":"node:internal/modules/cjs/loader","module":"loader","function":"Module._compile","lineno":1364,"colno":14,"in_app":false},{"filename":"/Users/abhijeetprasad/workspace/sentry-javascript/dev-packages/node-integration-tests/suites/contextLines/memory-leak/scenario.ts","module":"scenario.ts","function":"Object.?","lineno":14,"colno":10,"in_app":true,"pre_context":[" dsn: 'https://public@dsn.ingest.sentry.io/1337',"," release: '1.0',"," transport: loggingTransport,","});","","import { runSentry } from './other-file';",""],"context_line":"runSentry();","post_context":["","console.log(execSync(`lsof -p ${process.pid}`, { stdio: 'inherit', cwd: process.cwd() }));"]},{"filename":"/Users/abhijeetprasad/workspace/sentry-javascript/dev-packages/node-integration-tests/suites/contextLines/memory-leak/other-file.ts","module":"other-file.ts","function":"runSentry","lineno":5,"colno":29,"in_app":true,"pre_context":["import * as Sentry from '@sentry/node';","","export function runSentry(): void {"," for (let i = 0; i < 10; i++) {"],"context_line":" Sentry.captureException(new Error(`error in loop ${i}`));","post_context":[" }","}"]}]},"mechanism":{"type":"generic","handled":true}}]},"event_id":"f58236bf0a7f4a999f7daf5283f0400f","level":"error","platform":"node","contexts":{"trace":{"trace_id":"efdb9350effb47959d48bd0aaf395824","span_id":"9b6ccaf59536bcb4"},"runtime":{"name":"node","version":"v18.20.5"},"app":{"app_start_time":"2025-01-13T21:47:46.327Z","app_memory":270073856},"os":{"kernel_version":"23.6.0","name":"macOS","version":"14.7","build":"23H124"},"device":{"boot_time":"2024-12-23T16:56:50.637Z","arch":"arm64","memory_size":34359738368,"free_memory":355794944,"processor_count":10,"cpu_description":"Apple M1 Pro","processor_frequency":24},"culture":{"locale":"en-CA","timezone":"America/Toronto"},"cloud_resource":{}},"server_name":"GT9RQ02WW5.local","timestamp":1736804867.531,"environment":"production","release":"1.0","sdk":{"integrations":["InboundFilters","FunctionToString","LinkedErrors","RequestData","Console","Http","NodeFetch","OnUncaughtException","OnUnhandledRejection","ContextLines","LocalVariables","Context","ChildProcess","Modules"],"name":"sentry.javascript.node","version":"8.45.0","packages":[{"name":"npm:@sentry/node","version":"8.45.0"}]},"modules":{"ts-node":"10.9.1","make-error":"1.3.6","yn":"3.1.1","arg":"4.1.3","v8-compile-cache-lib":"3.0.1","typescript":"5.0.4","tslib":"2.7.0","semver":"7.6.3","shimmer":"1.2.1","require-in-the-middle":"7.2.0","resolve":"1.22.1","is-core-module":"2.11.0","has":"1.0.3","function-bind":"1.1.1","debug":"4.3.4","supports-color":"7.2.0","has-flag":"4.0.0","module-details-from-path":"1.0.3","import-in-the-middle":"1.12.0","forwarded-parse":"2.1.2"}}]]] - - at log (utils/runner.ts:462:11) - - console.log - line [{"event_id":"d4d1b66dc41b44b98df2d2ff5d5370a2","sent_at":"2025-01-13T21:47:47.665Z","sdk":{"name":"sentry.javascript.node","version":"8.45.0"},"trace":{"environment":"production","release":"1.0","public_key":"public","trace_id":"efdb9350effb47959d48bd0aaf395824"}},[[{"type":"event"},{"exception":{"values":[{"type":"Error","value":"error in loop 2","stacktrace":{"frames":[{"filename":"node:internal/main/run_main_module","module":"run_main_module","function":"?","lineno":28,"colno":49,"in_app":false},{"filename":"node:internal/modules/run_main","module":"run_main","function":"Function.executeUserEntryPoint [as runMain]","lineno":128,"colno":12,"in_app":false},{"filename":"node:internal/modules/cjs/loader","module":"loader","function":"Module._load","lineno":1019,"colno":12,"in_app":false},{"filename":"node:internal/modules/cjs/loader","module":"loader","function":"Module.load","lineno":1203,"colno":32,"in_app":false},{"filename":"/Users/abhijeetprasad/workspace/sentry-javascript/node_modules/ts-node/src/index.ts","module":"ts-node.src:index.ts","function":"Object.require.extensions. [as .ts]","lineno":1621,"colno":12,"in_app":false},{"filename":"node:internal/modules/cjs/loader","module":"loader","function":"Module._extensions..js","lineno":1422,"colno":10,"in_app":false},{"filename":"/Users/abhijeetprasad/workspace/sentry-javascript/node_modules/ts-node/src/index.ts","module":"ts-node.src:index.ts","function":"Module.m._compile","lineno":1618,"colno":23,"in_app":false},{"filename":"node:internal/modules/cjs/loader","module":"loader","function":"Module._compile","lineno":1364,"colno":14,"in_app":false},{"filename":"/Users/abhijeetprasad/workspace/sentry-javascript/dev-packages/node-integration-tests/suites/contextLines/memory-leak/scenario.ts","module":"scenario.ts","function":"Object.?","lineno":14,"colno":10,"in_app":true,"pre_context":[" dsn: 'https://public@dsn.ingest.sentry.io/1337',"," release: '1.0',"," transport: loggingTransport,","});","","import { runSentry } from './other-file';",""],"context_line":"runSentry();","post_context":["","console.log(execSync(`lsof -p ${process.pid}`, { stdio: 'inherit', cwd: process.cwd() }));"]},{"filename":"/Users/abhijeetprasad/workspace/sentry-javascript/dev-packages/node-integration-tests/suites/contextLines/memory-leak/other-file.ts","module":"other-file.ts","function":"runSentry","lineno":5,"colno":29,"in_app":true,"pre_context":["import * as Sentry from '@sentry/node';","","export function runSentry(): void {"," for (let i = 0; i < 10; i++) {"],"context_line":" Sentry.captureException(new Error(`error in loop ${i}`));","post_context":[" }","}"]}]},"mechanism":{"type":"generic","handled":true}}]},"event_id":"d4d1b66dc41b44b98df2d2ff5d5370a2","level":"error","platform":"node","contexts":{"trace":{"trace_id":"efdb9350effb47959d48bd0aaf395824","span_id":"82d56f443d3f01f9"},"runtime":{"name":"node","version":"v18.20.5"},"app":{"app_start_time":"2025-01-13T21:47:46.327Z","app_memory":270073856},"os":{"kernel_version":"23.6.0","name":"macOS","version":"14.7","build":"23H124"},"device":{"boot_time":"2024-12-23T16:56:50.637Z","arch":"arm64","memory_size":34359738368,"free_memory":355794944,"processor_count":10,"cpu_description":"Apple M1 Pro","processor_frequency":24},"culture":{"locale":"en-CA","timezone":"America/Toronto"},"cloud_resource":{}},"server_name":"GT9RQ02WW5.local","timestamp":1736804867.532,"environment":"production","release":"1.0","sdk":{"integrations":["InboundFilters","FunctionToString","LinkedErrors","RequestData","Console","Http","NodeFetch","OnUncaughtException","OnUnhandledRejection","ContextLines","LocalVariables","Context","ChildProcess","Modules"],"name":"sentry.javascript.node","version":"8.45.0","packages":[{"name":"npm:@sentry/node","version":"8.45.0"}]},"modules":{"ts-node":"10.9.1","make-error":"1.3.6","yn":"3.1.1","arg":"4.1.3","v8-compile-cache-lib":"3.0.1","typescript":"5.0.4","tslib":"2.7.0","semver":"7.6.3","shimmer":"1.2.1","require-in-the-middle":"7.2.0","resolve":"1.22.1","is-core-module":"2.11.0","has":"1.0.3","function-bind":"1.1.1","debug":"4.3.4","supports-color":"7.2.0","has-flag":"4.0.0","module-details-from-path":"1.0.3","import-in-the-middle":"1.12.0","forwarded-parse":"2.1.2"}}]]] - - at log (utils/runner.ts:462:11) - - console.log - line [{"event_id":"293d7c8c731c48eca30735b41efd40ba","sent_at":"2025-01-13T21:47:47.665Z","sdk":{"name":"sentry.javascript.node","version":"8.45.0"},"trace":{"environment":"production","release":"1.0","public_key":"public","trace_id":"efdb9350effb47959d48bd0aaf395824"}},[[{"type":"event"},{"exception":{"values":[{"type":"Error","value":"error in loop 3","stacktrace":{"frames":[{"filename":"node:internal/main/run_main_module","module":"run_main_module","function":"?","lineno":28,"colno":49,"in_app":false},{"filename":"node:internal/modules/run_main","module":"run_main","function":"Function.executeUserEntryPoint [as runMain]","lineno":128,"colno":12,"in_app":false},{"filename":"node:internal/modules/cjs/loader","module":"loader","function":"Module._load","lineno":1019,"colno":12,"in_app":false},{"filename":"node:internal/modules/cjs/loader","module":"loader","function":"Module.load","lineno":1203,"colno":32,"in_app":false},{"filename":"/Users/abhijeetprasad/workspace/sentry-javascript/node_modules/ts-node/src/index.ts","module":"ts-node.src:index.ts","function":"Object.require.extensions. [as .ts]","lineno":1621,"colno":12,"in_app":false},{"filename":"node:internal/modules/cjs/loader","module":"loader","function":"Module._extensions..js","lineno":1422,"colno":10,"in_app":false},{"filename":"/Users/abhijeetprasad/workspace/sentry-javascript/node_modules/ts-node/src/index.ts","module":"ts-node.src:index.ts","function":"Module.m._compile","lineno":1618,"colno":23,"in_app":false},{"filename":"node:internal/modules/cjs/loader","module":"loader","function":"Module._compile","lineno":1364,"colno":14,"in_app":false},{"filename":"/Users/abhijeetprasad/workspace/sentry-javascript/dev-packages/node-integration-tests/suites/contextLines/memory-leak/scenario.ts","module":"scenario.ts","function":"Object.?","lineno":14,"colno":10,"in_app":true,"pre_context":[" dsn: 'https://public@dsn.ingest.sentry.io/1337',"," release: '1.0',"," transport: loggingTransport,","});","","import { runSentry } from './other-file';",""],"context_line":"runSentry();","post_context":["","console.log(execSync(`lsof -p ${process.pid}`, { stdio: 'inherit', cwd: process.cwd() }));"]},{"filename":"/Users/abhijeetprasad/workspace/sentry-javascript/dev-packages/node-integration-tests/suites/contextLines/memory-leak/other-file.ts","module":"other-file.ts","function":"runSentry","lineno":5,"colno":29,"in_app":true,"pre_context":["import * as Sentry from '@sentry/node';","","export function runSentry(): void {"," for (let i = 0; i < 10; i++) {"],"context_line":" Sentry.captureException(new Error(`error in loop ${i}`));","post_context":[" }","}"]}]},"mechanism":{"type":"generic","handled":true}}]},"event_id":"293d7c8c731c48eca30735b41efd40ba","level":"error","platform":"node","contexts":{"trace":{"trace_id":"efdb9350effb47959d48bd0aaf395824","span_id":"8be46494d3555ddb"},"runtime":{"name":"node","version":"v18.20.5"},"app":{"app_start_time":"2025-01-13T21:47:46.327Z","app_memory":270073856},"os":{"kernel_version":"23.6.0","name":"macOS","version":"14.7","build":"23H124"},"device":{"boot_time":"2024-12-23T16:56:50.637Z","arch":"arm64","memory_size":34359738368,"free_memory":355794944,"processor_count":10,"cpu_description":"Apple M1 Pro","processor_frequency":24},"culture":{"locale":"en-CA","timezone":"America/Toronto"},"cloud_resource":{}},"server_name":"GT9RQ02WW5.local","timestamp":1736804867.533,"environment":"production","release":"1.0","sdk":{"integrations":["InboundFilters","FunctionToString","LinkedErrors","RequestData","Console","Http","NodeFetch","OnUncaughtException","OnUnhandledRejection","ContextLines","LocalVariables","Context","ChildProcess","Modules"],"name":"sentry.javascript.node","version":"8.45.0","packages":[{"name":"npm:@sentry/node","version":"8.45.0"}]},"modules":{"ts-node":"10.9.1","make-error":"1.3.6","yn":"3.1.1","arg":"4.1.3","v8-compile-cache-lib":"3.0.1","typescript":"5.0.4","tslib":"2.7.0","semver":"7.6.3","shimmer":"1.2.1","require-in-the-middle":"7.2.0","resolve":"1.22.1","is-core-module":"2.11.0","has":"1.0.3","function-bind":"1.1.1","debug":"4.3.4","supports-color":"7.2.0","has-flag":"4.0.0","module-details-from-path":"1.0.3","import-in-the-middle":"1.12.0","forwarded-parse":"2.1.2"}}]]] - - at log (utils/runner.ts:462:11) - - console.log - line [{"event_id":"e9273b56624d4261b00f5431852da167","sent_at":"2025-01-13T21:47:47.666Z","sdk":{"name":"sentry.javascript.node","version":"8.45.0"},"trace":{"environment":"production","release":"1.0","public_key":"public","trace_id":"efdb9350effb47959d48bd0aaf395824"}},[[{"type":"event"},{"exception":{"values":[{"type":"Error","value":"error in loop 4","stacktrace":{"frames":[{"filename":"node:internal/main/run_main_module","module":"run_main_module","function":"?","lineno":28,"colno":49,"in_app":false},{"filename":"node:internal/modules/run_main","module":"run_main","function":"Function.executeUserEntryPoint [as runMain]","lineno":128,"colno":12,"in_app":false},{"filename":"node:internal/modules/cjs/loader","module":"loader","function":"Module._load","lineno":1019,"colno":12,"in_app":false},{"filename":"node:internal/modules/cjs/loader","module":"loader","function":"Module.load","lineno":1203,"colno":32,"in_app":false},{"filename":"/Users/abhijeetprasad/workspace/sentry-javascript/node_modules/ts-node/src/index.ts","module":"ts-node.src:index.ts","function":"Object.require.extensions. [as .ts]","lineno":1621,"colno":12,"in_app":false},{"filename":"node:internal/modules/cjs/loader","module":"loader","function":"Module._extensions..js","lineno":1422,"colno":10,"in_app":false},{"filename":"/Users/abhijeetprasad/workspace/sentry-javascript/node_modules/ts-node/src/index.ts","module":"ts-node.src:index.ts","function":"Module.m._compile","lineno":1618,"colno":23,"in_app":false},{"filename":"node:internal/modules/cjs/loader","module":"loader","function":"Module._compile","lineno":1364,"colno":14,"in_app":false},{"filename":"/Users/abhijeetprasad/workspace/sentry-javascript/dev-packages/node-integration-tests/suites/contextLines/memory-leak/scenario.ts","module":"scenario.ts","function":"Object.?","lineno":14,"colno":10,"in_app":true,"pre_context":[" dsn: 'https://public@dsn.ingest.sentry.io/1337',"," release: '1.0',"," transport: loggingTransport,","});","","import { runSentry } from './other-file';",""],"context_line":"runSentry();","post_context":["","console.log(execSync(`lsof -p ${process.pid}`, { stdio: 'inherit', cwd: process.cwd() }));"]},{"filename":"/Users/abhijeetprasad/workspace/sentry-javascript/dev-packages/node-integration-tests/suites/contextLines/memory-leak/other-file.ts","module":"other-file.ts","function":"runSentry","lineno":5,"colno":29,"in_app":true,"pre_context":["import * as Sentry from '@sentry/node';","","export function runSentry(): void {"," for (let i = 0; i < 10; i++) {"],"context_line":" Sentry.captureException(new Error(`error in loop ${i}`));","post_context":[" }","}"]}]},"mechanism":{"type":"generic","handled":true}}]},"event_id":"e9273b56624d4261b00f5431852da167","level":"error","platform":"node","contexts":{"trace":{"trace_id":"efdb9350effb47959d48bd0aaf395824","span_id":"9a067a8906c8c147"},"runtime":{"name":"node","version":"v18.20.5"},"app":{"app_start_time":"2025-01-13T21:47:46.327Z","app_memory":270073856},"os":{"kernel_version":"23.6.0","name":"macOS","version":"14.7","build":"23H124"},"device":{"boot_time":"2024-12-23T16:56:50.637Z","arch":"arm64","memory_size":34359738368,"free_memory":355794944,"processor_count":10,"cpu_description":"Apple M1 Pro","processor_frequency":24},"culture":{"locale":"en-CA","timezone":"America/Toronto"},"cloud_resource":{}},"server_name":"GT9RQ02WW5.local","timestamp":1736804867.533,"environment":"production","release":"1.0","sdk":{"integrations":["InboundFilters","FunctionToString","LinkedErrors","RequestData","Console","Http","NodeFetch","OnUncaughtException","OnUnhandledRejection","ContextLines","LocalVariables","Context","ChildProcess","Modules"],"name":"sentry.javascript.node","version":"8.45.0","packages":[{"name":"npm:@sentry/node","version":"8.45.0"}]},"modules":{"ts-node":"10.9.1","make-error":"1.3.6","yn":"3.1.1","arg":"4.1.3","v8-compile-cache-lib":"3.0.1","typescript":"5.0.4","tslib":"2.7.0","semver":"7.6.3","shimmer":"1.2.1","require-in-the-middle":"7.2.0","resolve":"1.22.1","is-core-module":"2.11.0","has":"1.0.3","function-bind":"1.1.1","debug":"4.3.4","supports-color":"7.2.0","has-flag":"4.0.0","module-details-from-path":"1.0.3","import-in-the-middle":"1.12.0","forwarded-parse":"2.1.2"}}]]] - - at log (utils/runner.ts:462:11) - - console.log - line [{"event_id":"cf92173285aa49b8bdb3fe31a5de6c90","sent_at":"2025-01-13T21:47:47.667Z","sdk":{"name":"sentry.javascript.node","version":"8.45.0"},"trace":{"environment":"production","release":"1.0","public_key":"public","trace_id":"efdb9350effb47959d48bd0aaf395824"}},[[{"type":"event"},{"exception":{"values":[{"type":"Error","value":"error in loop 5","stacktrace":{"frames":[{"filename":"node:internal/main/run_main_module","module":"run_main_module","function":"?","lineno":28,"colno":49,"in_app":false},{"filename":"node:internal/modules/run_main","module":"run_main","function":"Function.executeUserEntryPoint [as runMain]","lineno":128,"colno":12,"in_app":false},{"filename":"node:internal/modules/cjs/loader","module":"loader","function":"Module._load","lineno":1019,"colno":12,"in_app":false},{"filename":"node:internal/modules/cjs/loader","module":"loader","function":"Module.load","lineno":1203,"colno":32,"in_app":false},{"filename":"/Users/abhijeetprasad/workspace/sentry-javascript/node_modules/ts-node/src/index.ts","module":"ts-node.src:index.ts","function":"Object.require.extensions. [as .ts]","lineno":1621,"colno":12,"in_app":false},{"filename":"node:internal/modules/cjs/loader","module":"loader","function":"Module._extensions..js","lineno":1422,"colno":10,"in_app":false},{"filename":"/Users/abhijeetprasad/workspace/sentry-javascript/node_modules/ts-node/src/index.ts","module":"ts-node.src:index.ts","function":"Module.m._compile","lineno":1618,"colno":23,"in_app":false},{"filename":"node:internal/modules/cjs/loader","module":"loader","function":"Module._compile","lineno":1364,"colno":14,"in_app":false},{"filename":"/Users/abhijeetprasad/workspace/sentry-javascript/dev-packages/node-integration-tests/suites/contextLines/memory-leak/scenario.ts","module":"scenario.ts","function":"Object.?","lineno":14,"colno":10,"in_app":true,"pre_context":[" dsn: 'https://public@dsn.ingest.sentry.io/1337',"," release: '1.0',"," transport: loggingTransport,","});","","import { runSentry } from './other-file';",""],"context_line":"runSentry();","post_context":["","console.log(execSync(`lsof -p ${process.pid}`, { stdio: 'inherit', cwd: process.cwd() }));"]},{"filename":"/Users/abhijeetprasad/workspace/sentry-javascript/dev-packages/node-integration-tests/suites/contextLines/memory-leak/other-file.ts","module":"other-file.ts","function":"runSentry","lineno":5,"colno":29,"in_app":true,"pre_context":["import * as Sentry from '@sentry/node';","","export function runSentry(): void {"," for (let i = 0; i < 10; i++) {"],"context_line":" Sentry.captureException(new Error(`error in loop ${i}`));","post_context":[" }","}"]}]},"mechanism":{"type":"generic","handled":true}}]},"event_id":"cf92173285aa49b8bdb3fe31a5de6c90","level":"error","platform":"node","contexts":{"trace":{"trace_id":"efdb9350effb47959d48bd0aaf395824","span_id":"ac2ad9041812f9d9"},"runtime":{"name":"node","version":"v18.20.5"},"app":{"app_start_time":"2025-01-13T21:47:46.327Z","app_memory":270073856},"os":{"kernel_version":"23.6.0","name":"macOS","version":"14.7","build":"23H124"},"device":{"boot_time":"2024-12-23T16:56:50.637Z","arch":"arm64","memory_size":34359738368,"free_memory":355794944,"processor_count":10,"cpu_description":"Apple M1 Pro","processor_frequency":24},"culture":{"locale":"en-CA","timezone":"America/Toronto"},"cloud_resource":{}},"server_name":"GT9RQ02WW5.local","timestamp":1736804867.534,"environment":"production","release":"1.0","sdk":{"integrations":["InboundFilters","FunctionToString","LinkedErrors","RequestData","Console","Http","NodeFetch","OnUncaughtException","OnUnhandledRejection","ContextLines","LocalVariables","Context","ChildProcess","Modules"],"name":"sentry.javascript.node","version":"8.45.0","packages":[{"name":"npm:@sentry/node","version":"8.45.0"}]},"modules":{"ts-node":"10.9.1","make-error":"1.3.6","yn":"3.1.1","arg":"4.1.3","v8-compile-cache-lib":"3.0.1","typescript":"5.0.4","tslib":"2.7.0","semver":"7.6.3","shimmer":"1.2.1","require-in-the-middle":"7.2.0","resolve":"1.22.1","is-core-module":"2.11.0","has":"1.0.3","function-bind":"1.1.1","debug":"4.3.4","supports-color":"7.2.0","has-flag":"4.0.0","module-details-from-path":"1.0.3","import-in-the-middle":"1.12.0","forwarded-parse":"2.1.2"}}]]] - - at log (utils/runner.ts:462:11) - - console.log - line [{"event_id":"65224267e02049daadbc577de86960f3","sent_at":"2025-01-13T21:47:47.667Z","sdk":{"name":"sentry.javascript.node","version":"8.45.0"},"trace":{"environment":"production","release":"1.0","public_key":"public","trace_id":"efdb9350effb47959d48bd0aaf395824"}},[[{"type":"event"},{"exception":{"values":[{"type":"Error","value":"error in loop 6","stacktrace":{"frames":[{"filename":"node:internal/main/run_main_module","module":"run_main_module","function":"?","lineno":28,"colno":49,"in_app":false},{"filename":"node:internal/modules/run_main","module":"run_main","function":"Function.executeUserEntryPoint [as runMain]","lineno":128,"colno":12,"in_app":false},{"filename":"node:internal/modules/cjs/loader","module":"loader","function":"Module._load","lineno":1019,"colno":12,"in_app":false},{"filename":"node:internal/modules/cjs/loader","module":"loader","function":"Module.load","lineno":1203,"colno":32,"in_app":false},{"filename":"/Users/abhijeetprasad/workspace/sentry-javascript/node_modules/ts-node/src/index.ts","module":"ts-node.src:index.ts","function":"Object.require.extensions. [as .ts]","lineno":1621,"colno":12,"in_app":false},{"filename":"node:internal/modules/cjs/loader","module":"loader","function":"Module._extensions..js","lineno":1422,"colno":10,"in_app":false},{"filename":"/Users/abhijeetprasad/workspace/sentry-javascript/node_modules/ts-node/src/index.ts","module":"ts-node.src:index.ts","function":"Module.m._compile","lineno":1618,"colno":23,"in_app":false},{"filename":"node:internal/modules/cjs/loader","module":"loader","function":"Module._compile","lineno":1364,"colno":14,"in_app":false},{"filename":"/Users/abhijeetprasad/workspace/sentry-javascript/dev-packages/node-integration-tests/suites/contextLines/memory-leak/scenario.ts","module":"scenario.ts","function":"Object.?","lineno":14,"colno":10,"in_app":true,"pre_context":[" dsn: 'https://public@dsn.ingest.sentry.io/1337',"," release: '1.0',"," transport: loggingTransport,","});","","import { runSentry } from './other-file';",""],"context_line":"runSentry();","post_context":["","console.log(execSync(`lsof -p ${process.pid}`, { stdio: 'inherit', cwd: process.cwd() }));"]},{"filename":"/Users/abhijeetprasad/workspace/sentry-javascript/dev-packages/node-integration-tests/suites/contextLines/memory-leak/other-file.ts","module":"other-file.ts","function":"runSentry","lineno":5,"colno":29,"in_app":true,"pre_context":["import * as Sentry from '@sentry/node';","","export function runSentry(): void {"," for (let i = 0; i < 10; i++) {"],"context_line":" Sentry.captureException(new Error(`error in loop ${i}`));","post_context":[" }","}"]}]},"mechanism":{"type":"generic","handled":true}}]},"event_id":"65224267e02049daadbc577de86960f3","level":"error","platform":"node","contexts":{"trace":{"trace_id":"efdb9350effb47959d48bd0aaf395824","span_id":"b12818330e05cd2f"},"runtime":{"name":"node","version":"v18.20.5"},"app":{"app_start_time":"2025-01-13T21:47:46.327Z","app_memory":270073856},"os":{"kernel_version":"23.6.0","name":"macOS","version":"14.7","build":"23H124"},"device":{"boot_time":"2024-12-23T16:56:50.637Z","arch":"arm64","memory_size":34359738368,"free_memory":355794944,"processor_count":10,"cpu_description":"Apple M1 Pro","processor_frequency":24},"culture":{"locale":"en-CA","timezone":"America/Toronto"},"cloud_resource":{}},"server_name":"GT9RQ02WW5.local","timestamp":1736804867.535,"environment":"production","release":"1.0","sdk":{"integrations":["InboundFilters","FunctionToString","LinkedErrors","RequestData","Console","Http","NodeFetch","OnUncaughtException","OnUnhandledRejection","ContextLines","LocalVariables","Context","ChildProcess","Modules"],"name":"sentry.javascript.node","version":"8.45.0","packages":[{"name":"npm:@sentry/node","version":"8.45.0"}]},"modules":{"ts-node":"10.9.1","make-error":"1.3.6","yn":"3.1.1","arg":"4.1.3","v8-compile-cache-lib":"3.0.1","typescript":"5.0.4","tslib":"2.7.0","semver":"7.6.3","shimmer":"1.2.1","require-in-the-middle":"7.2.0","resolve":"1.22.1","is-core-module":"2.11.0","has":"1.0.3","function-bind":"1.1.1","debug":"4.3.4","supports-color":"7.2.0","has-flag":"4.0.0","module-details-from-path":"1.0.3","import-in-the-middle":"1.12.0","forwarded-parse":"2.1.2"}}]]] - - at log (utils/runner.ts:462:11) - - console.log - line [{"event_id":"b9e96b480e1a4e74a2ecebde9f0400a9","sent_at":"2025-01-13T21:47:47.668Z","sdk":{"name":"sentry.javascript.node","version":"8.45.0"},"trace":{"environment":"production","release":"1.0","public_key":"public","trace_id":"efdb9350effb47959d48bd0aaf395824"}},[[{"type":"event"},{"exception":{"values":[{"type":"Error","value":"error in loop 7","stacktrace":{"frames":[{"filename":"node:internal/main/run_main_module","module":"run_main_module","function":"?","lineno":28,"colno":49,"in_app":false},{"filename":"node:internal/modules/run_main","module":"run_main","function":"Function.executeUserEntryPoint [as runMain]","lineno":128,"colno":12,"in_app":false},{"filename":"node:internal/modules/cjs/loader","module":"loader","function":"Module._load","lineno":1019,"colno":12,"in_app":false},{"filename":"node:internal/modules/cjs/loader","module":"loader","function":"Module.load","lineno":1203,"colno":32,"in_app":false},{"filename":"/Users/abhijeetprasad/workspace/sentry-javascript/node_modules/ts-node/src/index.ts","module":"ts-node.src:index.ts","function":"Object.require.extensions. [as .ts]","lineno":1621,"colno":12,"in_app":false},{"filename":"node:internal/modules/cjs/loader","module":"loader","function":"Module._extensions..js","lineno":1422,"colno":10,"in_app":false},{"filename":"/Users/abhijeetprasad/workspace/sentry-javascript/node_modules/ts-node/src/index.ts","module":"ts-node.src:index.ts","function":"Module.m._compile","lineno":1618,"colno":23,"in_app":false},{"filename":"node:internal/modules/cjs/loader","module":"loader","function":"Module._compile","lineno":1364,"colno":14,"in_app":false},{"filename":"/Users/abhijeetprasad/workspace/sentry-javascript/dev-packages/node-integration-tests/suites/contextLines/memory-leak/scenario.ts","module":"scenario.ts","function":"Object.?","lineno":14,"colno":10,"in_app":true,"pre_context":[" dsn: 'https://public@dsn.ingest.sentry.io/1337',"," release: '1.0',"," transport: loggingTransport,","});","","import { runSentry } from './other-file';",""],"context_line":"runSentry();","post_context":["","console.log(execSync(`lsof -p ${process.pid}`, { stdio: 'inherit', cwd: process.cwd() }));"]},{"filename":"/Users/abhijeetprasad/workspace/sentry-javascript/dev-packages/node-integration-tests/suites/contextLines/memory-leak/other-file.ts","module":"other-file.ts","function":"runSentry","lineno":5,"colno":29,"in_app":true,"pre_context":["import * as Sentry from '@sentry/node';","","export function runSentry(): void {"," for (let i = 0; i < 10; i++) {"],"context_line":" Sentry.captureException(new Error(`error in loop ${i}`));","post_context":[" }","}"]}]},"mechanism":{"type":"generic","handled":true}}]},"event_id":"b9e96b480e1a4e74a2ecebde9f0400a9","level":"error","platform":"node","contexts":{"trace":{"trace_id":"efdb9350effb47959d48bd0aaf395824","span_id":"83cb86896d96bbf6"},"runtime":{"name":"node","version":"v18.20.5"},"app":{"app_start_time":"2025-01-13T21:47:46.327Z","app_memory":270073856},"os":{"kernel_version":"23.6.0","name":"macOS","version":"14.7","build":"23H124"},"device":{"boot_time":"2024-12-23T16:56:50.637Z","arch":"arm64","memory_size":34359738368,"free_memory":355794944,"processor_count":10,"cpu_description":"Apple M1 Pro","processor_frequency":24},"culture":{"locale":"en-CA","timezone":"America/Toronto"},"cloud_resource":{}},"server_name":"GT9RQ02WW5.local","timestamp":1736804867.536,"environment":"production","release":"1.0","sdk":{"integrations":["InboundFilters","FunctionToString","LinkedErrors","RequestData","Console","Http","NodeFetch","OnUncaughtException","OnUnhandledRejection","ContextLines","LocalVariables","Context","ChildProcess","Modules"],"name":"sentry.javascript.node","version":"8.45.0","packages":[{"name":"npm:@sentry/node","version":"8.45.0"}]},"modules":{"ts-node":"10.9.1","make-error":"1.3.6","yn":"3.1.1","arg":"4.1.3","v8-compile-cache-lib":"3.0.1","typescript":"5.0.4","tslib":"2.7.0","semver":"7.6.3","shimmer":"1.2.1","require-in-the-middle":"7.2.0","resolve":"1.22.1","is-core-module":"2.11.0","has":"1.0.3","function-bind":"1.1.1","debug":"4.3.4","supports-color":"7.2.0","has-flag":"4.0.0","module-details-from-path":"1.0.3","import-in-the-middle":"1.12.0","forwarded-parse":"2.1.2"}}]]] - - at log (utils/runner.ts:462:11) - - console.log - line [{"event_id":"c541f2c0a31345b78f93f69ffe5e0fc6","sent_at":"2025-01-13T21:47:47.668Z","sdk":{"name":"sentry.javascript.node","version":"8.45.0"},"trace":{"environment":"production","release":"1.0","public_key":"public","trace_id":"efdb9350effb47959d48bd0aaf395824"}},[[{"type":"event"},{"exception":{"values":[{"type":"Error","value":"error in loop 8","stacktrace":{"frames":[{"filename":"node:internal/main/run_main_module","module":"run_main_module","function":"?","lineno":28,"colno":49,"in_app":false},{"filename":"node:internal/modules/run_main","module":"run_main","function":"Function.executeUserEntryPoint [as runMain]","lineno":128,"colno":12,"in_app":false},{"filename":"node:internal/modules/cjs/loader","module":"loader","function":"Module._load","lineno":1019,"colno":12,"in_app":false},{"filename":"node:internal/modules/cjs/loader","module":"loader","function":"Module.load","lineno":1203,"colno":32,"in_app":false},{"filename":"/Users/abhijeetprasad/workspace/sentry-javascript/node_modules/ts-node/src/index.ts","module":"ts-node.src:index.ts","function":"Object.require.extensions. [as .ts]","lineno":1621,"colno":12,"in_app":false},{"filename":"node:internal/modules/cjs/loader","module":"loader","function":"Module._extensions..js","lineno":1422,"colno":10,"in_app":false},{"filename":"/Users/abhijeetprasad/workspace/sentry-javascript/node_modules/ts-node/src/index.ts","module":"ts-node.src:index.ts","function":"Module.m._compile","lineno":1618,"colno":23,"in_app":false},{"filename":"node:internal/modules/cjs/loader","module":"loader","function":"Module._compile","lineno":1364,"colno":14,"in_app":false},{"filename":"/Users/abhijeetprasad/workspace/sentry-javascript/dev-packages/node-integration-tests/suites/contextLines/memory-leak/scenario.ts","module":"scenario.ts","function":"Object.?","lineno":14,"colno":10,"in_app":true,"pre_context":[" dsn: 'https://public@dsn.ingest.sentry.io/1337',"," release: '1.0',"," transport: loggingTransport,","});","","import { runSentry } from './other-file';",""],"context_line":"runSentry();","post_context":["","console.log(execSync(`lsof -p ${process.pid}`, { stdio: 'inherit', cwd: process.cwd() }));"]},{"filename":"/Users/abhijeetprasad/workspace/sentry-javascript/dev-packages/node-integration-tests/suites/contextLines/memory-leak/other-file.ts","module":"other-file.ts","function":"runSentry","lineno":5,"colno":29,"in_app":true,"pre_context":["import * as Sentry from '@sentry/node';","","export function runSentry(): void {"," for (let i = 0; i < 10; i++) {"],"context_line":" Sentry.captureException(new Error(`error in loop ${i}`));","post_context":[" }","}"]}]},"mechanism":{"type":"generic","handled":true}}]},"event_id":"c541f2c0a31345b78f93f69ffe5e0fc6","level":"error","platform":"node","contexts":{"trace":{"trace_id":"efdb9350effb47959d48bd0aaf395824","span_id":"a0e8e199fcf05714"},"runtime":{"name":"node","version":"v18.20.5"},"app":{"app_start_time":"2025-01-13T21:47:46.327Z","app_memory":270073856},"os":{"kernel_version":"23.6.0","name":"macOS","version":"14.7","build":"23H124"},"device":{"boot_time":"2024-12-23T16:56:50.637Z","arch":"arm64","memory_size":34359738368,"free_memory":355794944,"processor_count":10,"cpu_description":"Apple M1 Pro","processor_frequency":24},"culture":{"locale":"en-CA","timezone":"America/Toronto"},"cloud_resource":{}},"server_name":"GT9RQ02WW5.local","timestamp":1736804867.536,"environment":"production","release":"1.0","sdk":{"integrations":["InboundFilters","FunctionToString","LinkedErrors","RequestData","Console","Http","NodeFetch","OnUncaughtException","OnUnhandledRejection","ContextLines","LocalVariables","Context","ChildProcess","Modules"],"name":"sentry.javascript.node","version":"8.45.0","packages":[{"name":"npm:@sentry/node","version":"8.45.0"}]},"modules":{"ts-node":"10.9.1","make-error":"1.3.6","yn":"3.1.1","arg":"4.1.3","v8-compile-cache-lib":"3.0.1","typescript":"5.0.4","tslib":"2.7.0","semver":"7.6.3","shimmer":"1.2.1","require-in-the-middle":"7.2.0","resolve":"1.22.1","is-core-module":"2.11.0","has":"1.0.3","function-bind":"1.1.1","debug":"4.3.4","supports-color":"7.2.0","has-flag":"4.0.0","module-details-from-path":"1.0.3","import-in-the-middle":"1.12.0","forwarded-parse":"2.1.2"}}]]] - - at log (utils/runner.ts:462:11) - - console.log - line [{"event_id":"dc08b3fe26e94759817c7b5e95469727","sent_at":"2025-01-13T21:47:47.669Z","sdk":{"name":"sentry.javascript.node","version":"8.45.0"},"trace":{"environment":"production","release":"1.0","public_key":"public","trace_id":"efdb9350effb47959d48bd0aaf395824"}},[[{"type":"event"},{"exception":{"values":[{"type":"Error","value":"error in loop 9","stacktrace":{"frames":[{"filename":"node:internal/main/run_main_module","module":"run_main_module","function":"?","lineno":28,"colno":49,"in_app":false},{"filename":"node:internal/modules/run_main","module":"run_main","function":"Function.executeUserEntryPoint [as runMain]","lineno":128,"colno":12,"in_app":false},{"filename":"node:internal/modules/cjs/loader","module":"loader","function":"Module._load","lineno":1019,"colno":12,"in_app":false},{"filename":"node:internal/modules/cjs/loader","module":"loader","function":"Module.load","lineno":1203,"colno":32,"in_app":false},{"filename":"/Users/abhijeetprasad/workspace/sentry-javascript/node_modules/ts-node/src/index.ts","module":"ts-node.src:index.ts","function":"Object.require.extensions. [as .ts]","lineno":1621,"colno":12,"in_app":false},{"filename":"node:internal/modules/cjs/loader","module":"loader","function":"Module._extensions..js","lineno":1422,"colno":10,"in_app":false},{"filename":"/Users/abhijeetprasad/workspace/sentry-javascript/node_modules/ts-node/src/index.ts","module":"ts-node.src:index.ts","function":"Module.m._compile","lineno":1618,"colno":23,"in_app":false},{"filename":"node:internal/modules/cjs/loader","module":"loader","function":"Module._compile","lineno":1364,"colno":14,"in_app":false},{"filename":"/Users/abhijeetprasad/workspace/sentry-javascript/dev-packages/node-integration-tests/suites/contextLines/memory-leak/scenario.ts","module":"scenario.ts","function":"Object.?","lineno":14,"colno":10,"in_app":true,"pre_context":[" dsn: 'https://public@dsn.ingest.sentry.io/1337',"," release: '1.0',"," transport: loggingTransport,","});","","import { runSentry } from './other-file';",""],"context_line":"runSentry();","post_context":["","console.log(execSync(`lsof -p ${process.pid}`, { stdio: 'inherit', cwd: process.cwd() }));"]},{"filename":"/Users/abhijeetprasad/workspace/sentry-javascript/dev-packages/node-integration-tests/suites/contextLines/memory-leak/other-file.ts","module":"other-file.ts","function":"runSentry","lineno":5,"colno":29,"in_app":true,"pre_context":["import * as Sentry from '@sentry/node';","","export function runSentry(): void {"," for (let i = 0; i < 10; i++) {"],"context_line":" Sentry.captureException(new Error(`error in loop ${i}`));","post_context":[" }","}"]}]},"mechanism":{"type":"generic","handled":true}}]},"event_id":"dc08b3fe26e94759817c7b5e95469727","level":"error","platform":"node","contexts":{"trace":{"trace_id":"efdb9350effb47959d48bd0aaf395824","span_id":"8ec7d145c5362df0"},"runtime":{"name":"node","version":"v18.20.5"},"app":{"app_start_time":"2025-01-13T21:47:46.327Z","app_memory":270106624},"os":{"kernel_version":"23.6.0","name":"macOS","version":"14.7","build":"23H124"},"device":{"boot_time":"2024-12-23T16:56:50.637Z","arch":"arm64","memory_size":34359738368,"free_memory":355794944,"processor_count":10,"cpu_description":"Apple M1 Pro","processor_frequency":24},"culture":{"locale":"en-CA","timezone":"America/Toronto"},"cloud_resource":{}},"server_name":"GT9RQ02WW5.local","timestamp":1736804867.537,"environment":"production","release":"1.0","sdk":{"integrations":["InboundFilters","FunctionToString","LinkedErrors","RequestData","Console","Http","NodeFetch","OnUncaughtException","OnUnhandledRejection","ContextLines","LocalVariables","Context","ChildProcess","Modules"],"name":"sentry.javascript.node","version":"8.45.0","packages":[{"name":"npm:@sentry/node","version":"8.45.0"}]},"modules":{"ts-node":"10.9.1","make-error":"1.3.6","yn":"3.1.1","arg":"4.1.3","v8-compile-cache-lib":"3.0.1","typescript":"5.0.4","tslib":"2.7.0","semver":"7.6.3","shimmer":"1.2.1","require-in-the-middle":"7.2.0","resolve":"1.22.1","is-core-module":"2.11.0","has":"1.0.3","function-bind":"1.1.1","debug":"4.3.4","supports-color":"7.2.0","has-flag":"4.0.0","module-details-from-path":"1.0.3","import-in-the-middle":"1.12.0","forwarded-parse":"2.1.2"}}]]] - - at log (utils/runner.ts:462:11) - -Done in 4.21s. diff --git a/dev-packages/node-integration-tests/utils/runner.ts b/dev-packages/node-integration-tests/utils/runner.ts index 97b1efa2dbb4..1f5b5508b12f 100644 --- a/dev-packages/node-integration-tests/utils/runner.ts +++ b/dev-packages/node-integration-tests/utils/runner.ts @@ -15,6 +15,7 @@ import { normalize } from '@sentry/core'; import { execSync, spawn, spawnSync } from 'child_process'; import { existsSync, readFileSync, unlinkSync, writeFileSync } from 'fs'; import { join } from 'path'; +import { inspect } from 'util'; import { afterAll, beforeAll, describe, test } from 'vitest'; import { assertEnvelopeHeader, @@ -338,6 +339,8 @@ export function createRunner(...paths: string[]) { } function newEnvelope(envelope: Envelope): void { + if (process.env.DEBUG) log('newEnvelope', inspect(envelope, false, null, true)); + for (const item of envelope[1]) { const envelopeItemType = item[0].type; @@ -447,7 +450,13 @@ export function createRunner(...paths: string[]) { if (process.env.DEBUG) log('starting scenario', testPath, flags, env.SENTRY_DSN); - child = spawn('node', [...flags, testPath], { env }); + child = spawn('node', [...flags, testPath], { env, stdio: process.env.DEBUG ? 'inherit' : 'ignore' }); + + child.on('error', e => { + // eslint-disable-next-line no-console + console.error('Error starting child process:', e); + complete(e); + }); CLEANUP_STEPS.add(() => { child?.kill(); diff --git a/package.json b/package.json index fd1fa2c0ee23..5be433b1e4c2 100644 --- a/package.json +++ b/package.json @@ -68,7 +68,7 @@ "packages/nestjs", "packages/nextjs", "packages/node", - "packages/node-stalled", + "packages/node-native", "packages/nuxt", "packages/opentelemetry", "packages/profiling-node", diff --git a/packages/core/src/types-hoist/exception.ts b/packages/core/src/types-hoist/exception.ts index a74adf6c1603..27b320363a82 100644 --- a/packages/core/src/types-hoist/exception.ts +++ b/packages/core/src/types-hoist/exception.ts @@ -7,6 +7,6 @@ export interface Exception { value?: string; mechanism?: Mechanism; module?: string; - thread_id?: number; + thread_id?: number | string; stacktrace?: Stacktrace; } diff --git a/packages/core/src/types-hoist/thread.ts b/packages/core/src/types-hoist/thread.ts index 1cfad253a299..76f5592ef401 100644 --- a/packages/core/src/types-hoist/thread.ts +++ b/packages/core/src/types-hoist/thread.ts @@ -2,8 +2,9 @@ import type { Stacktrace } from './stacktrace'; /** JSDoc */ export interface Thread { - id?: number; + id?: number | string; name?: string; + main?: boolean; stacktrace?: Stacktrace; crashed?: boolean; current?: boolean; diff --git a/packages/node-stalled/.eslintrc.js b/packages/node-native/.eslintrc.js similarity index 100% rename from packages/node-stalled/.eslintrc.js rename to packages/node-native/.eslintrc.js diff --git a/packages/node-stalled/LICENSE b/packages/node-native/LICENSE similarity index 94% rename from packages/node-stalled/LICENSE rename to packages/node-native/LICENSE index 293314012679..0da96cd2f885 100644 --- a/packages/node-stalled/LICENSE +++ b/packages/node-native/LICENSE @@ -1,6 +1,6 @@ MIT License -Copyright (c) 2022 Functional Software, Inc. dba Sentry +Copyright (c) 2025 Functional Software, Inc. dba Sentry Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in diff --git a/packages/node-native/README.md b/packages/node-native/README.md new file mode 100644 index 000000000000..e41389df2527 --- /dev/null +++ b/packages/node-native/README.md @@ -0,0 +1,64 @@ +

+ + Sentry + +

+ +# Native Tools for the Official Sentry Node.js SDK + +[![npm version](https://img.shields.io/npm/v/@sentry/node-native.svg)](https://www.npmjs.com/package/@sentry/node-native) +[![npm dm](https://img.shields.io/npm/dm/@sentry/node-native.svg)](https://www.npmjs.com/package/@sentry/node-native) +[![npm dt](https://img.shields.io/npm/dt/@sentry/node-native.svg)](https://www.npmjs.com/package/@sentry/node-native) + +## Installation + +```bash +# Using yarn +yarn add @sentry/node @sentry/node-native + +# Using npm +npm install --save @sentry/node @sentry/node-native +``` + +## `threadBlockedIntegration` + +The `threadBlockedIntegration` can be used to monitor for blocked event loops in +all threads of a Node.js application. + +If you instrument your application via the Node.js `--import` flag, this +instrumentation will be automatically applied to all worker threads. + +`instrument.mjs` +```javascript +import * as Sentry from '@sentry/node'; +import { threadBlockedIntegration } from '@sentry/node-native'; + +Sentry.init({ + dsn: '__YOUR_DSN__', + // Capture stack traces when the event loop is blocked for more than 500ms + integrations: [threadBlockedIntegration({ blockedThreshold: 500 })], +}); +``` + +`app.mjs` +```javascript +import { Worker } from 'worker_threads'; + +const worker = new Worker(new URL('./worker.mjs', import.meta.url)); + +// This main thread will be monitored for blocked event loops +``` + +`worker.mjs` +```javascript + +// This worker thread will also be monitored for blocked event loops too +``` +Start your application: + +```bash +node --import instrument.mjs app.mjs +``` + +If a thread is blocked for more than the configured threshold, stack traces will +be captured for all threads and sent to Sentry. diff --git a/packages/node-stalled/package.json b/packages/node-native/package.json similarity index 82% rename from packages/node-stalled/package.json rename to packages/node-native/package.json index de7c396f7687..525bbb4b63ba 100644 --- a/packages/node-stalled/package.json +++ b/packages/node-native/package.json @@ -1,9 +1,9 @@ { - "name": "@sentry/node-stalled", + "name": "@sentry/node-native", "version": "9.27.0", - "description": "Official Sentry SDK for Node.js Profiling", + "description": "Native Tools for the Official Sentry Node.js SDK", "repository": "git://github.com/getsentry/sentry-javascript.git", - "homepage": "https://github.com/getsentry/sentry-javascript/tree/master/packages/profiling-node", + "homepage": "https://github.com/getsentry/sentry-javascript/tree/master/packages/node-native", "author": "Sentry", "license": "MIT", "main": "build/cjs/index.js", @@ -20,6 +20,14 @@ "types": "./build/types/index.d.ts", "default": "./build/cjs/index.js" } + }, + "./worker": { + "import": { + "default": "./build/esm/thread-blocked-watchdog.js" + }, + "require": { + "default": "./build/cjs/thread-blocked-watchdog.js" + } } }, "typesVersions": { @@ -52,15 +60,12 @@ "build:dev": "yarn clean && yarn build", "build:transpile:watch": "rollup -c rollup.npm.config.mjs --watch", "build:watch": "run-p build:transpile:watch build:types:watch", - "build:tarball": "npm pack", - "test:bundle": "node test-binaries.esbuild.js", - "test": "vitest run", - "test:watch": "vitest --watch" + "build:tarball": "npm pack" }, "dependencies": { "@sentry-internal/node-native-stacktrace": "^0.1.0", - "@sentry/core": "9.27.0", - "@sentry/node": "9.27.0" + "@sentry/core": "9.29.0", + "@sentry/node": "9.29.0" }, "devDependencies": { "@types/node": "^18.19.1" diff --git a/packages/node-stalled/rollup.npm.config.mjs b/packages/node-native/rollup.npm.config.mjs similarity index 52% rename from packages/node-stalled/rollup.npm.config.mjs rename to packages/node-native/rollup.npm.config.mjs index 80b26c2ac232..501cdc7c4e4f 100644 --- a/packages/node-stalled/rollup.npm.config.mjs +++ b/packages/node-native/rollup.npm.config.mjs @@ -2,17 +2,13 @@ import { makeBaseNPMConfig, makeNPMConfigVariants } from '@sentry-internal/rollu export default makeNPMConfigVariants( makeBaseNPMConfig({ + entrypoints: ['src/index.ts', 'src/thread-blocked-watchdog.ts'], packageSpecificConfig: { output: { dir: 'build', // set exports to 'named' or 'auto' so that rollup doesn't warn exports: 'named', - // set preserveModules to false because for profiling we actually want - // to bundle everything into one file. - preserveModules: - process.env.SENTRY_BUILD_PRESERVE_MODULES === undefined - ? false - : Boolean(process.env.SENTRY_BUILD_PRESERVE_MODULES), + preserveModules: true }, }, }), diff --git a/packages/node-stalled/src/common.ts b/packages/node-native/src/common.ts similarity index 50% rename from packages/node-stalled/src/common.ts rename to packages/node-native/src/common.ts index b9a5a6b85777..4ed1a3fe1dc3 100644 --- a/packages/node-stalled/src/common.ts +++ b/packages/node-native/src/common.ts @@ -1,26 +1,26 @@ -import type { Contexts, DsnComponents, Primitive, SdkMetadata } from '@sentry/core'; +import type { Contexts, DsnComponents, Primitive, SdkMetadata, Session } from '@sentry/core'; -export interface AnrIntegrationOptions { +export interface ThreadBlockedIntegrationOptions { /** - * Interval to send heartbeat messages to the ANR worker. + * Interval to send heartbeat messages to the watchdog. * * Defaults to 50ms. */ pollInterval: number; /** - * Threshold in milliseconds to trigger an ANR event. + * Threshold in milliseconds to trigger a blocked event. * * Defaults to 5000ms. */ - anrThreshold: number; + blockedThreshold: number; /** - * Maximum number of ANR events to send. + * Maximum number of blocked events to send. * * Defaults to 1. */ - maxAnrEvents: number; + maxBlockedEvents: number; /** - * Tags to include with ANR events. + * Tags to include with blocked events. */ staticTags: { [key: string]: Primitive }; /** @@ -31,7 +31,7 @@ export interface AnrIntegrationOptions { appRootPath: string | undefined; } -export interface WorkerStartData extends AnrIntegrationOptions { +export interface WorkerStartData extends ThreadBlockedIntegrationOptions { debug: boolean; sdkMetadata: SdkMetadata; dsn: DsnComponents; @@ -41,3 +41,8 @@ export interface WorkerStartData extends AnrIntegrationOptions { dist: string | undefined; contexts: Contexts; } + +export interface ThreadState { + session: Session | undefined; + debugImages: Record; +} diff --git a/packages/node-native/src/index.ts b/packages/node-native/src/index.ts new file mode 100644 index 000000000000..c866b9a15c60 --- /dev/null +++ b/packages/node-native/src/index.ts @@ -0,0 +1 @@ +export { threadBlockedIntegration, disableBlockedDetectionForCallback } from './thread-blocked-integration'; diff --git a/packages/node-stalled/src/index.ts b/packages/node-native/src/thread-blocked-integration.ts similarity index 70% rename from packages/node-stalled/src/index.ts rename to packages/node-native/src/thread-blocked-integration.ts index d65950ba99fe..9e4394f20707 100644 --- a/packages/node-stalled/src/index.ts +++ b/packages/node-native/src/thread-blocked-integration.ts @@ -1,15 +1,10 @@ import { types } from 'node:util'; import { Worker } from 'node:worker_threads'; import type { Contexts, Event, EventHint, Integration, IntegrationFn } from '@sentry/core'; -import { - defineIntegration, - getClient, - getFilenameToDebugIdMap, - getIsolationScope, - logger, -} from '@sentry/core'; +import { defineIntegration, getClient, getFilenameToDebugIdMap, getIsolationScope, logger } from '@sentry/core'; import type { NodeClient } from '@sentry/node'; -import type { AnrIntegrationOptions, WorkerStartData } from './common'; +import { registerThread, threadPoll } from '@sentry-internal/node-native-stacktrace'; +import type { ThreadBlockedIntegrationOptions, WorkerStartData } from './common'; const { isPromise } = types; @@ -17,14 +12,14 @@ const DEFAULT_INTERVAL = 50; const DEFAULT_HANG_THRESHOLD = 5000; function log(message: string, ...args: unknown[]): void { - logger.log(`[Node Stalled] ${message}`, ...args); + logger.log(`[Thread Blocked] ${message}`, ...args); } /** * Gets contexts by calling all event processors. This shouldn't be called until all integrations are setup */ async function getContexts(client: NodeClient): Promise { - let event: Event | null = { message: 'ANR' }; + let event: Event | null = { message: INTEGRATION_NAME }; const eventHint: EventHint = {}; for (const processor of client.getEventProcessors()) { @@ -35,11 +30,11 @@ async function getContexts(client: NodeClient): Promise { return event?.contexts || {}; } -const INTEGRATION_NAME = 'NodeStalled'; +const INTEGRATION_NAME = 'ThreadBlocked'; -type AnrInternal = { startWorker: () => void; stopWorker: () => void }; +type ThreadBlockedInternal = { startWorker: () => void; stopWorker: () => void }; -const _anrIntegration = ((options: Partial = {}) => { +const _threadBlockedIntegration = ((options: Partial = {}) => { let worker: Promise<() => void> | undefined; let client: NodeClient | undefined; @@ -66,25 +61,27 @@ const _anrIntegration = ((options: Partial = {}) => { async setup(initClient: NodeClient) { client = initClient; + registerThread(); + // setImmediate is used to ensure that all other integrations have had their setup called first. // This allows us to call into all integrations to fetch the full context setImmediate(() => this.startWorker()); }, - } as Integration & AnrInternal; + } as Integration & ThreadBlockedInternal; }) satisfies IntegrationFn; -type AnrReturn = (options?: Partial) => Integration & AnrInternal; +type ThreadBlockedReturn = (options?: Partial) => Integration & ThreadBlockedInternal; -export const anrIntegration = defineIntegration(_anrIntegration) as AnrReturn; +export const threadBlockedIntegration = defineIntegration(_threadBlockedIntegration) as ThreadBlockedReturn; /** - * Starts the ANR worker thread + * Starts the worker thread * * @returns A function to stop the worker */ async function _startWorker( client: NodeClient, - integrationOptions: Partial, + integrationOptions: Partial, ): Promise<() => void> { const dsn = client.getDsn(); @@ -117,13 +114,13 @@ async function _startWorker( sdkMetadata, appRootPath: integrationOptions.appRootPath, pollInterval: integrationOptions.pollInterval || DEFAULT_INTERVAL, - anrThreshold: integrationOptions.anrThreshold || DEFAULT_HANG_THRESHOLD, - maxAnrEvents: integrationOptions.maxAnrEvents || 1, + blockedThreshold: integrationOptions.blockedThreshold || DEFAULT_HANG_THRESHOLD, + maxBlockedEvents: integrationOptions.maxBlockedEvents || 1, staticTags: integrationOptions.staticTags || {}, contexts, }; - const worker = new Worker(new URL('./worker', import.meta.url), { + const worker = new Worker(new URL('./thread-blocked-watchdog.js', import.meta.url), { workerData: options, // We don't want any Node args like --import to be passed to the worker execArgv: [], @@ -142,7 +139,7 @@ async function _startWorker( // serialized without making it a SerializedSession const session = currentSession ? { ...currentSession, toJSON: undefined } : undefined; // message the worker to tell it the main event loop is still running - worker.postMessage({ session, debugImages: getFilenameToDebugIdMap(initOptions.stackParser) }); + threadPoll({ session, debugImages: getFilenameToDebugIdMap(initOptions.stackParser) }); } catch (_) { // } @@ -150,21 +147,14 @@ async function _startWorker( // Timer should not block exit timer.unref(); - worker.on('message', (msg: string) => { - if (msg === 'session-ended') { - log('ANR event sent from ANR worker. Clearing session in this thread.'); - getIsolationScope().setSession(undefined); - } - }); - worker.once('error', (err: Error) => { clearInterval(timer); - log('ANR worker error', err); + log('watchdog worker error', err); }); worker.once('exit', (code: number) => { clearInterval(timer); - log('ANR worker exit', code); + log('watchdog worker exit', code); }); // Ensure this thread can't block app exit @@ -177,13 +167,13 @@ async function _startWorker( }; } -export function disableAnrDetectionForCallback(callback: () => T): T; -export function disableAnrDetectionForCallback(callback: () => Promise): Promise; +export function disableBlockedDetectionForCallback(callback: () => T): T; +export function disableBlockedDetectionForCallback(callback: () => Promise): Promise; /** - * Disables ANR detection for the duration of the callback + * Disables blocked detection for the duration of the callback */ -export function disableAnrDetectionForCallback(callback: () => T | Promise): T | Promise { - const integration = getClient()?.getIntegrationByName(INTEGRATION_NAME) as AnrInternal | undefined; +export function disableBlockedDetectionForCallback(callback: () => T | Promise): T | Promise { + const integration = getClient()?.getIntegrationByName(INTEGRATION_NAME) as ThreadBlockedInternal | undefined; if (!integration) { return callback(); diff --git a/packages/node-native/src/thread-blocked-watchdog.ts b/packages/node-native/src/thread-blocked-watchdog.ts new file mode 100644 index 000000000000..68e8dab2ff51 --- /dev/null +++ b/packages/node-native/src/thread-blocked-watchdog.ts @@ -0,0 +1,256 @@ +import { workerData } from 'node:worker_threads'; +import type { DebugImage, Event, Session, StackFrame, Thread } from '@sentry/core'; +import { + createEventEnvelope, + createSessionEnvelope, + filenameIsInApp, + getEnvelopeEndpointWithUrlEncodedAuth, + makeSession, + normalizeUrlToBase, + stripSentryFramesAndReverse, + updateSession, + uuid4, +} from '@sentry/core'; +import { makeNodeTransport } from '@sentry/node'; +import { captureStackTrace, getThreadsLastSeen } from '@sentry-internal/node-native-stacktrace'; +import type { ThreadState, WorkerStartData } from './common'; + +const { + blockedThreshold, + appRootPath, + contexts, + debug, + dist, + dsn, + environment, + maxBlockedEvents, + pollInterval, + release, + sdkMetadata, + staticTags: tags, + tunnel, +} = workerData as WorkerStartData; + +const triggeredThreads = new Set(); +let sentAnrEvents = 0; + +function log(...msg: unknown[]): void { + if (debug) { + // eslint-disable-next-line no-console + console.log('[Blocked Watchdog]', ...msg); + } +} + +const url = getEnvelopeEndpointWithUrlEncodedAuth(dsn, tunnel, sdkMetadata.sdk); +const transport = makeNodeTransport({ + url, + recordDroppedEvent: () => { + // + }, +}); + +async function sendAbnormalSession(serializedSession: Session | undefined): Promise { + if (!serializedSession) { + return; + } + + log('Sending abnormal session'); + const session = makeSession(serializedSession); + + updateSession(session, { + status: 'abnormal', + abnormal_mechanism: 'anr_foreground', + release, + environment, + }); + + const envelope = createSessionEnvelope(session, dsn, sdkMetadata, tunnel); + // Log the envelope so to aid in testing + log(JSON.stringify(envelope)); + + await transport.send(envelope); +} + +log('Started'); + +function prepareStackFrames(stackFrames: StackFrame[] | undefined): StackFrame[] | undefined { + if (!stackFrames) { + return undefined; + } + + // Strip Sentry frames and reverse the stack frames so they are in the correct order + const strippedFrames = stripSentryFramesAndReverse(stackFrames); + + for (const frame of strippedFrames) { + if (!frame.filename) { + continue; + } + + frame.in_app = filenameIsInApp(frame.filename); + + // If we have an app root path, rewrite the filenames to be relative to the app root + if (appRootPath) { + frame.filename = normalizeUrlToBase(frame.filename, appRootPath); + } + } + + return strippedFrames; +} + +function stripFileProtocol(filename: string | undefined): string | undefined { + if (!filename) { + return undefined; + } + return filename.replace(/^file:\/\//, ''); +} + +// eslint-disable-next-line complexity +function applyDebugMeta(event: Event, debugImages: Record): void { + if (Object.keys(debugImages).length === 0) { + return; + } + + const normalisedDebugImages = appRootPath ? {} : debugImages; + if (appRootPath) { + for (const [path, debugId] of Object.entries(debugImages)) { + normalisedDebugImages[normalizeUrlToBase(path, appRootPath)] = debugId; + } + } + + const filenameToDebugId = new Map(); + + for (const exception of event.exception?.values || []) { + for (const frame of exception.stacktrace?.frames || []) { + const filename = stripFileProtocol(frame.abs_path || frame.filename); + if (filename && normalisedDebugImages[filename]) { + filenameToDebugId.set(filename, normalisedDebugImages[filename] as string); + } + } + } + + for (const thread of event.threads?.values || []) { + for (const frame of thread.stacktrace?.frames || []) { + const filename = stripFileProtocol(frame.abs_path || frame.filename); + if (filename && normalisedDebugImages[filename]) { + filenameToDebugId.set(filename, normalisedDebugImages[filename] as string); + } + } + } + + if (filenameToDebugId.size > 0) { + const images: DebugImage[] = []; + for (const [code_file, debug_id] of filenameToDebugId.entries()) { + images.push({ + type: 'sourcemap', + code_file, + debug_id, + }); + } + event.debug_meta = { images }; + } +} + +function getExceptionAndThreads( + crashedThreadId: string, + threads: ReturnType>, +): Event { + const crashedThread = threads[crashedThreadId]; + + return { + exception: { + values: [ + { + type: 'EventLoopBlocked', + value: `Event Loop Blocked for at least ${blockedThreshold} ms`, + stacktrace: { frames: prepareStackFrames(crashedThread?.frames) }, + // This ensures the UI doesn't say 'Crashed in' for the stack trace + mechanism: { type: 'ANR' }, + thread_id: crashedThreadId, + }, + ], + }, + threads: { + values: Object.entries(threads).map(([threadId, threadState]) => { + const crashed = threadId === crashedThreadId; + + const thread: Thread = { id: threadId, name: threadId, crashed, current: true, main: threadId === '0' }; + + if (!crashed) { + thread.stacktrace = { frames: prepareStackFrames(threadState.frames) }; + } + + return thread; + }), + }, + }; +} + +async function sendAnrEvent(crashedThreadId: string): Promise { + if (sentAnrEvents >= maxBlockedEvents) { + return; + } + + sentAnrEvents += 1; + + const threads = captureStackTrace(); + const crashedThread = threads[crashedThreadId]; + + if (!crashedThread) { + log(`No thread found with ID '${crashedThreadId}'`); + return; + } + + await sendAbnormalSession(crashedThread.state?.session); + + log('Sending event'); + + const event: Event = { + event_id: uuid4(), + contexts, + release, + environment, + dist, + platform: 'node', + level: 'error', + tags, + ...getExceptionAndThreads(crashedThreadId, threads), + }; + + const allDebugImages: Record = Object.values(threads).reduce((acc, threadState) => { + return { ...acc, ...threadState.state?.debugImages }; + }, {}); + + applyDebugMeta(event, allDebugImages); + + const envelope = createEventEnvelope(event, dsn, sdkMetadata, tunnel); + // Log the envelope to aid in testing + log(JSON.stringify(envelope)); + + await transport.send(envelope); + await transport.flush(2000); + + if (sentAnrEvents >= maxBlockedEvents) { + // Delay for 5 seconds so that stdio can flush if the main event loop ever restarts. + // This is mainly for the benefit of logging or debugging. + setTimeout(() => { + process.exit(0); + }, 5_000); + } +} + +setInterval(async () => { + for (const [threadId, time] of Object.entries(getThreadsLastSeen())) { + if (time > blockedThreshold) { + if (triggeredThreads.has(threadId)) { + continue; + } + + log(`Detected ANR for thread '${threadId}' with last seen time ${time} ms`); + triggeredThreads.add(threadId); + + await sendAnrEvent(threadId); + } else { + triggeredThreads.delete(threadId); + } + } +}, pollInterval); diff --git a/packages/node-stalled/tsconfig.json b/packages/node-native/tsconfig.json similarity index 100% rename from packages/node-stalled/tsconfig.json rename to packages/node-native/tsconfig.json diff --git a/packages/node-stalled/tsconfig.test.json b/packages/node-native/tsconfig.test.json similarity index 100% rename from packages/node-stalled/tsconfig.test.json rename to packages/node-native/tsconfig.test.json diff --git a/packages/node-stalled/tsconfig.types.json b/packages/node-native/tsconfig.types.json similarity index 100% rename from packages/node-stalled/tsconfig.types.json rename to packages/node-native/tsconfig.types.json diff --git a/packages/node-stalled/vite.config.ts b/packages/node-native/vite.config.ts similarity index 100% rename from packages/node-stalled/vite.config.ts rename to packages/node-native/vite.config.ts diff --git a/packages/node-stalled/README.md b/packages/node-stalled/README.md deleted file mode 100644 index 962bc8e6834f..000000000000 --- a/packages/node-stalled/README.md +++ /dev/null @@ -1,316 +0,0 @@ -

- - Sentry - -

- -# Official Sentry Profiling SDK for NodeJS - -[![npm version](https://img.shields.io/npm/v/@sentry/profiling-node.svg)](https://www.npmjs.com/package/@sentry/profiling-node) -[![npm dm](https://img.shields.io/npm/dm/@sentry/profiling-node.svg)](https://www.npmjs.com/package/@sentry/profiling-node) -[![npm dt](https://img.shields.io/npm/dt/@sentry/profiling-node.svg)](https://www.npmjs.com/package/@sentry/profiling-node) - -## Installation - -Profiling works as an extension of tracing so you will need both @sentry/node and @sentry/profiling-node installed. - -```bash -# Using yarn -yarn add @sentry/node @sentry/profiling-node - -# Using npm -npm install --save @sentry/node @sentry/profiling-node -``` - -## Usage - -```javascript -import * as Sentry from '@sentry/node'; -import { nodeProfilingIntegration } from '@sentry/profiling-node'; - -Sentry.init({ - dsn: 'https://7fa19397baaf433f919fbe02228d5470@o1137848.ingest.sentry.io/6625302', - debug: true, - tracesSampleRate: 1, - profilesSampleRate: 1, // Set profiling sampling rate. - integrations: [nodeProfilingIntegration()], -}); -``` - -Sentry SDK will now automatically profile all root spans, even the ones which may be started as a result of using an -automatic instrumentation integration. - -```javascript -Sentry.startSpan({ name: 'some workflow' }, () => { - // The code in here will be profiled -}); -``` - -### Building the package from source - -Profiling uses native modules to interop with the v8 javascript engine which means that you may be required to build it -from source. The libraries required to successfully build the package from source are often the same libraries that are -already required to build any other package which uses native modules and if your codebase uses any of those modules, -there is a fairly good chance this will work out of the box. The required packages are python, make and g++. - -**Windows:** If you are building on windows, you may need to install windows-build-tools - -**_Python:_** Python 3.12 is not supported yet so you will need a version of python that is lower than 3.12 - -```bash - -# using yarn package manager -yarn global add windows-build-tools -# or npm package manager -npm i -g windows-build-tools -``` - -After you have installed the toolchain, you should be able to build the binaries from source - -```bash -# configure node-gyp using yarn -yarn build:bindings:configure -# or using npm -npm run build:bindings:configure - -# compile the binaries using yarn -yarn build:bindings -# or using npm -npm run build:bindings -``` - -After the binaries are built, you should see them inside the profiling-node/lib folder. - -### Prebuilt binaries - -We currently ship prebuilt binaries for a few of the most common platforms and node versions (v18-24). - -- macOS x64 -- Linux ARM64 (musl) -- Linux x64 (glibc) -- Windows x64 - -For a more detailed list, see job_compile_bindings_profiling_node job in our build.yml github action workflow. - -### Bundling - -If you are looking to squeeze some extra performance or improve cold start in your application (especially true for -serverless environments where modules are often evaluates on a per request basis), then we recommend you look into -bundling your code. Modern JS engines are much faster at parsing and compiling JS than following long module resolution -chains and reading file contents from disk. Because @sentry/profiling-node is a package that uses native node modules, -bundling it is slightly different than just bundling javascript. In other words, the bundler needs to recognize that a -.node file is node native binding and move it to the correct location so that it can later be used. Failing to do so -will result in a MODULE_NOT_FOUND error. - -The easiest way to make bundling work with @sentry/profiling-node and other modules which use native nodejs bindings is -to mark the package as external - this will prevent the code from the package from being bundled, but it means that you -will now need to rely on the package to be installed in your production environment. - -To mark the package as external, use the following configuration: - -[Next.js 13+](https://nextjs.org/docs/app/api-reference/next-config-js/serverComponentsExternalPackages) - -```js -const { withSentryConfig } = require('@sentry/nextjs'); - -/** @type {import('next').NextConfig} */ -const nextConfig = { - experimental: { - // Add the "@sentry/profiling-node" to serverComponentsExternalPackages. - serverComponentsExternalPackages: ['@sentry/profiling-node'], - }, -}; - -module.exports = withSentryConfig(nextConfig, { - /* ... */ -}); -``` - -[webpack](https://webpack.js.org/configuration/externals/#externals) - -```js -externals: { - "@sentry/profiling-node": "commonjs @sentry/profiling-node", -}, -``` - -[esbuild](https://esbuild.github.io/api/#external) - -```js -{ - entryPoints: ['index.js'], - platform: 'node', - external: ['@sentry/profiling-node'], -} -``` - -[Rollup](https://rollupjs.org/configuration-options/#external) - -```js -{ - entry: 'index.js', - external: '@sentry/profiling-node' -} -``` - -[serverless-esbuild (serverless.yml)](https://www.serverless.com/plugins/serverless-esbuild#external-dependencies) - -```yml -custom: - esbuild: - external: - - @sentry/profiling-node - packagerOptions: - scripts: - - npm install @sentry/profiling-node -``` - -[vercel-ncc](https://github.com/vercel/ncc#programmatically-from-nodejs) - -```js -{ - externals: ["@sentry/profiling-node"], -} -``` - -[vite](https://vitejs.dev/config/ssr-options.html#ssr-external) - -```js -ssr: { - external: ['@sentry/profiling-node']; -} -``` - -Marking the package as external is the simplest and most future proof way of ensuring it will work, however if you want -to bundle it, it is possible to do so as well. Bundling has the benefit of improving your script startup time as all of -the code is (usually) inside a single executable .js file, which saves time on module resolution. - -In general, when attempting to bundle .node native file extensions, you will need to tell your bundler how to treat -these, as by default it does not know how to handle them. The required approach varies between build tools and you will -need to find which one will work for you. - -The result of bundling .node files correctly is that they are placed into your bundle output directory with their -require paths updated to reflect their final location. - -Example of bundling @sentry/profiling-node with esbuild and .copy loader - -```json -// package.json -{ - "scripts": "node esbuild.serverless.js" -} -``` - -```js -// esbuild.serverless.js -const { sentryEsbuildPlugin } = require('@sentry/esbuild-plugin'); - -require('esbuild').build({ - entryPoints: ['./index.js'], - outfile: './dist', - platform: 'node', - bundle: true, - minify: true, - sourcemap: true, - // This is no longer necessary - // external: ["@sentry/profiling-node"], - loader: { - // ensures .node binaries are copied to ./dist - '.node': 'copy', - }, - plugins: [ - // See https://docs.sentry.io/platforms/javascript/sourcemaps/uploading/esbuild/ - sentryEsbuildPlugin({ - project: '', - org: '', - authToken: '', - release: '', - sourcemaps: { - // Specify the directory containing build artifacts - assets: './dist/**', - }, - }), - ], -}); -``` - -Once you run `node esbuild.serverless.js` esbuild wil bundle and output the files to ./dist folder, but note that all of -the binaries will be copied. This is wasteful as you will likely only need one of these libraries to be available during -runtime. - -To prune the other libraries, profiling-node ships with a small utility script that helps you prune unused binaries. The -script can be invoked via `sentry-prune-profiler-binaries`, use `--help` to see a list of available options or -`--dry-run` if you want it to log the binaries that would have been deleted. - -Example of only preserving a binary to run node16 on linux x64 musl. - -```bash -sentry-prune-profiler-binaries --target_dir_path=./dist --target_platform=linux --target_node=16 --target_stdlib=musl --target_arch=x64 -``` - -Which will output something like - -``` -Sentry: pruned ./dist/sentry_cpu_profiler-darwin-x64-108-IFGH3SUR.node (90.41 KiB) -Sentry: pruned ./dist/sentry_cpu_profiler-darwin-x64-93-Q7KBVHSP.node (74.16 KiB) -Sentry: pruned ./dist/sentry_cpu_profiler-linux-arm64-glibc-108-NXSISRTB.node (52.17 KiB) -Sentry: pruned ./dist/sentry_cpu_profiler-linux-arm64-glibc-83-OEQT5HUK.node (52.08 KiB) -Sentry: pruned ./dist/sentry_cpu_profiler-linux-arm64-glibc-93-IIXXW2PN.node (52.06 KiB) -Sentry: pruned ./dist/sentry_cpu_profiler-linux-arm64-musl-108-DSILNYHA.node (48.46 KiB) -Sentry: pruned ./dist/sentry_cpu_profiler-linux-arm64-musl-83-4CNOBNC3.node (48.37 KiB) -Sentry: pruned ./dist/sentry_cpu_profiler-linux-arm64-musl-93-JA5PKNWQ.node (48.38 KiB) -Sentry: pruned ./dist/sentry_cpu_profiler-linux-x64-glibc-108-NXSISRTB.node (52.17 KiB) -Sentry: pruned ./dist/sentry_cpu_profiler-linux-x64-glibc-83-OEQT5HUK.node (52.08 KiB) -Sentry: pruned ./dist/sentry_cpu_profiler-linux-x64-glibc-93-IIXXW2PN.node (52.06 KiB) -Sentry: pruned ./dist/sentry_cpu_profiler-linux-x64-musl-108-CX7SL27U.node (51.50 KiB) -Sentry: pruned ./dist/sentry_cpu_profiler-linux-x64-musl-83-YD7ZQK2E.node (51.53 KiB) -Sentry: pruned ./dist/sentry_cpu_profiler-win32-x64-108-P7V3URQV.node (181.50 KiB) -Sentry: pruned ./dist/sentry_cpu_profiler-win32-x64-93-3PKQDSGE.node (181.50 KiB) -✅ Sentry: pruned 15 binaries, saved 1.06 MiB in total. -``` - -### Environment flags - -The default mode of the v8 CpuProfiler is kEagerLoggin which enables the profiler even when no profiles are active - -this is good because it makes calls to startProfiling fast at the tradeoff for constant CPU overhead. The behavior can -be controlled via the `SENTRY_PROFILER_LOGGING_MODE` environment variable with values of `eager|lazy`. If you opt to use -the lazy logging mode, calls to startProfiling may be slow (depending on environment and node version, it can be in the -order of a few hundred ms). - -Example of starting a server with lazy logging mode. - -```javascript -SENTRY_PROFILER_LOGGING_MODE=lazy node server.js -``` - -## FAQ 💭 - -### Can the profiler leak PII to Sentry? - -The profiler does not collect function arguments so leaking any PII is unlikely. We only collect a subset of the values -which may identify the device and os that the profiler is running on (if you are already using tracing, it is likely -that these values are already being collected by the SDK). - -There is one way a profiler could leak pii information, but this is unlikely and would only happen for cases where you -might be creating or naming functions which might contain pii information such as - -```js -eval('function scriptFor${PII_DATA}....'); -``` - -In that case it is possible that the function name may end up being reported to Sentry. - -### Are worker threads supported? - -No. All instances of the profiler are scoped per thread In practice, this means that starting a transaction on thread A -and delegating work to thread B will only result in sample stacks being collected from thread A. That said, nothing -should prevent you from starting a transaction on thread B concurrently which will result in two independent profiles -being sent to the Sentry backend. We currently do not do any correlation between such transactions, but we would be open -to exploring the possibilities. Please file an issue if you have suggestions or specific use-cases in mind. - -### How much overhead will this profiler add? - -The profiler uses the kEagerLogging option by default which trades off fast calls to startProfiling for a small amount -of constant CPU overhead. If you are using kEagerLogging then the tradeoff is reversed and there will be a small CPU -overhead while the profiler is not running, but calls to startProfiling could be slow (in our tests, this varies by -environments and node versions, but could be in the order of a couple 100ms). diff --git a/packages/node-stalled/src/worker.ts b/packages/node-stalled/src/worker.ts deleted file mode 100644 index 515633237d4f..000000000000 --- a/packages/node-stalled/src/worker.ts +++ /dev/null @@ -1,292 +0,0 @@ -import { parentPort, workerData } from 'node:worker_threads'; -import type { DebugImage, Event, ScopeData, Session, StackFrame } from '@sentry/core'; -import { - applyScopeDataToEvent, - callFrameToStackFrame, - createEventEnvelope, - createSessionEnvelope, - generateSpanId, - getEnvelopeEndpointWithUrlEncodedAuth, - makeSession, - normalizeUrlToBase, - stripSentryFramesAndReverse, - updateSession, - uuid4, -} from '@sentry/core'; -import { createGetModuleFromFilename, makeNodeTransport } from '@sentry/node'; -import type { WorkerStartData } from './common'; - -const options: WorkerStartData = workerData; -let session: Session | undefined; -let sentAnrEvents = 0; -let mainDebugImages: Record = {}; - -function log(msg: string): void { - if (options.debug) { - // eslint-disable-next-line no-console - console.log(`[Stalled Watchdog] ${msg}`); - } -} - -const url = getEnvelopeEndpointWithUrlEncodedAuth(options.dsn, options.tunnel, options.sdkMetadata.sdk); -const transport = makeNodeTransport({ - url, - recordDroppedEvent: () => { - // - }, -}); - -async function sendAbnormalSession(): Promise { - // of we have an existing session passed from the main thread, send it as abnormal - if (session) { - log('Sending abnormal session'); - - updateSession(session, { - status: 'abnormal', - abnormal_mechanism: 'anr_foreground', - release: options.release, - environment: options.environment, - }); - - const envelope = createSessionEnvelope(session, options.dsn, options.sdkMetadata, options.tunnel); - // Log the envelope so to aid in testing - log(JSON.stringify(envelope)); - - await transport.send(envelope); - - try { - // Notify the main process that the session has ended so the session can be cleared from the scope - parentPort?.postMessage('session-ended'); - } catch (_) { - // ignore - } - } -} - -log('Started'); - -function prepareStackFrames(stackFrames: StackFrame[] | undefined): StackFrame[] | undefined { - if (!stackFrames) { - return undefined; - } - - // Strip Sentry frames and reverse the stack frames so they are in the correct order - const strippedFrames = stripSentryFramesAndReverse(stackFrames); - - // If we have an app root path, rewrite the filenames to be relative to the app root - if (options.appRootPath) { - for (const frame of strippedFrames) { - if (!frame.filename) { - continue; - } - - frame.filename = normalizeUrlToBase(frame.filename, options.appRootPath); - } - } - - return strippedFrames; -} - -function applyDebugMeta(event: Event): void { - if (Object.keys(mainDebugImages).length === 0) { - return; - } - - const normalisedDebugImages = options.appRootPath ? {} : mainDebugImages; - if (options.appRootPath) { - for (const [path, debugId] of Object.entries(mainDebugImages)) { - normalisedDebugImages[normalizeUrlToBase(path, options.appRootPath)] = debugId; - } - } - - const filenameToDebugId = new Map(); - - for (const exception of event.exception?.values || []) { - for (const frame of exception.stacktrace?.frames || []) { - const filename = frame.abs_path || frame.filename; - if (filename && normalisedDebugImages[filename]) { - filenameToDebugId.set(filename, normalisedDebugImages[filename] as string); - } - } - } - - if (filenameToDebugId.size > 0) { - const images: DebugImage[] = []; - for (const [code_file, debug_id] of filenameToDebugId.entries()) { - images.push({ - type: 'sourcemap', - code_file, - debug_id, - }); - } - event.debug_meta = { images }; - } -} - -function applyScopeToEvent(event: Event, scope: ScopeData): void { - applyScopeDataToEvent(event, scope); - - if (!event.contexts?.trace) { - const { traceId, parentSpanId, propagationSpanId } = scope.propagationContext; - event.contexts = { - trace: { - trace_id: traceId, - span_id: propagationSpanId || generateSpanId(), - parent_span_id: parentSpanId, - }, - ...event.contexts, - }; - } -} - -async function sendAnrEvent(frames?: StackFrame[], scope?: ScopeData): Promise { - if (sentAnrEvents >= options.maxAnrEvents) { - return; - } - - sentAnrEvents += 1; - - await sendAbnormalSession(); - - log('Sending event'); - - const event: Event = { - event_id: uuid4(), - contexts: options.contexts, - release: options.release, - environment: options.environment, - dist: options.dist, - platform: 'node', - level: 'error', - exception: { - values: [ - { - type: 'ApplicationNotResponding', - value: `Application Not Responding for at least ${options.anrThreshold} ms`, - stacktrace: { frames: prepareStackFrames(frames) }, - // This ensures the UI doesn't say 'Crashed in' for the stack trace - mechanism: { type: 'ANR' }, - }, - ], - }, - tags: options.staticTags, - }; - - if (scope) { - applyScopeToEvent(event, scope); - } - - applyDebugMeta(event); - - const envelope = createEventEnvelope(event, options.dsn, options.sdkMetadata, options.tunnel); - // Log the envelope to aid in testing - log(JSON.stringify(envelope)); - - await transport.send(envelope); - await transport.flush(2000); - - if (sentAnrEvents >= options.maxAnrEvents) { - // Delay for 5 seconds so that stdio can flush if the main event loop ever restarts. - // This is mainly for the benefit of logging or debugging. - setTimeout(() => { - process.exit(0); - }, 5_000); - } -} - -if (options.captureStackTrace) { - log('Connecting to debugger'); - - const session = new InspectorSession(); - session.connectToMainThread(); - - log('Connected to debugger'); - - // Collect scriptId -> url map so we can look up the filenames later - const scripts = new Map(); - - session.on('Debugger.scriptParsed', event => { - scripts.set(event.params.scriptId, event.params.url); - }); - - session.on('Debugger.paused', event => { - if (event.params.reason !== 'other') { - return; - } - - try { - log('Debugger paused'); - - // copy the frames - const callFrames = [...event.params.callFrames]; - - const getModuleName = options.appRootPath ? createGetModuleFromFilename(options.appRootPath) : () => undefined; - const stackFrames = callFrames.map(frame => - callFrameToStackFrame(frame, scripts.get(frame.location.scriptId), getModuleName), - ); - - // Runtime.evaluate may never return if the event loop is blocked indefinitely - // In that case, we want to send the event anyway - const getScopeTimeout = setTimeout(() => { - sendAnrEvent(stackFrames).then(null, () => { - log('Sending ANR event failed.'); - }); - }, 5_000); - - // Evaluate a script in the currently paused context - session.post( - 'Runtime.evaluate', - { - // Grab the trace context from the current scope - expression: 'global.__SENTRY_GET_SCOPES__();', - // Don't re-trigger the debugger if this causes an error - silent: true, - // Serialize the result to json otherwise only primitives are supported - returnByValue: true, - }, - (err, param) => { - if (err) { - log(`Error executing script: '${err.message}'`); - } - - clearTimeout(getScopeTimeout); - - const scopes = param?.result ? (param.result.value as ScopeData) : undefined; - - session.post('Debugger.resume'); - session.post('Debugger.disable'); - - sendAnrEvent(stackFrames, scopes).then(null, () => { - log('Sending ANR event failed.'); - }); - }, - ); - } catch (e) { - session.post('Debugger.resume'); - session.post('Debugger.disable'); - throw e; - } - }); - - debuggerPause = () => { - try { - session.post('Debugger.enable', () => { - session.post('Debugger.pause'); - }); - } catch (_) { - // - } - }; -} - -parentPort?.on('message', (msg: { session: Session | undefined; debugImages?: Record }) => { - if (msg.session) { - session = makeSession(msg.session); - } - - if (msg.debugImages) { - mainDebugImages = msg.debugImages; - } - - poll(); -}); diff --git a/yarn.lock b/yarn.lock index 0deb47ad0828..d81e278409a5 100644 --- a/yarn.lock +++ b/yarn.lock @@ -6493,6 +6493,14 @@ detect-libc "^2.0.3" node-abi "^3.73.0" +"@sentry-internal/node-native-stacktrace@^0.1.0": + version "0.1.0" + resolved "https://registry.yarnpkg.com/@sentry-internal/node-native-stacktrace/-/node-native-stacktrace-0.1.0.tgz#fa0eaf1e66245f463ca2294ff63da74c56d1a052" + integrity sha512-dWkxhDdjcRdEOTk1acrdBledqIroaYJrOSbecx5tJ/m9DiWZ1Oa4eNi/sI2SHLT+hKmsBBxrychf6+Iitz5Bzw== + dependencies: + detect-libc "^2.0.4" + node-abi "^3.73.0" + "@sentry-internal/rrdom@2.34.0": version "2.34.0" resolved "https://registry.yarnpkg.com/@sentry-internal/rrdom/-/rrdom-2.34.0.tgz#fccc9fe211c3995d4200abafbe8d75b671961ee9" @@ -13295,7 +13303,7 @@ detect-libc@^1.0.3: resolved "https://registry.yarnpkg.com/detect-libc/-/detect-libc-1.0.3.tgz#fa137c4bd698edf55cd5cd02ac559f91a4c4ba9b" integrity sha512-pGjwhsmsp4kL2RTz08wcOlGN83otlqHeD/Z5T8GXZB+/YcpQ/dgo+lbU8ZsGxV0HIvqqxo9l7mqYwyYMD9bKDg== -detect-libc@^2.0.0, detect-libc@^2.0.2, detect-libc@^2.0.3: +detect-libc@^2.0.0, detect-libc@^2.0.2, detect-libc@^2.0.3, detect-libc@^2.0.4: version "2.0.4" resolved "https://registry.yarnpkg.com/detect-libc/-/detect-libc-2.0.4.tgz#f04715b8ba815e53b4d8109655b6508a6865a7e8" integrity sha512-3UDv+G9CsCKO1WKMGw9fwq/SWJYbI0c5Y7LU1AXYoDdbhE2AHQ6N6Nb34sG8Fj7T5APy8qXDCKuuIHd1BR0tVA== @@ -27148,7 +27156,6 @@ stylus@0.59.0, stylus@^0.59.0: sucrase@^3.27.0, sucrase@^3.35.0, sucrase@getsentry/sucrase#es2020-polyfills: version "3.36.0" - uid fd682f6129e507c00bb4e6319cc5d6b767e36061 resolved "https://codeload.github.com/getsentry/sucrase/tar.gz/fd682f6129e507c00bb4e6319cc5d6b767e36061" dependencies: "@jridgewell/gen-mapping" "^0.3.2" From b49f3f5f2386b92bbf70d9549f5ada3f69ddde4c Mon Sep 17 00:00:00 2001 From: Tim Fish Date: Mon, 23 Jun 2025 23:09:39 +0200 Subject: [PATCH 03/16] =?UTF-8?q?Lint=20=F0=9F=A4=A6=F0=9F=8F=BB=E2=80=8D?= =?UTF-8?q?=E2=99=82=EF=B8=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/node-native/README.md | 5 ++++- packages/node-native/rollup.npm.config.mjs | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/packages/node-native/README.md b/packages/node-native/README.md index e41389df2527..de3b79194efe 100644 --- a/packages/node-native/README.md +++ b/packages/node-native/README.md @@ -29,6 +29,7 @@ If you instrument your application via the Node.js `--import` flag, this instrumentation will be automatically applied to all worker threads. `instrument.mjs` + ```javascript import * as Sentry from '@sentry/node'; import { threadBlockedIntegration } from '@sentry/node-native'; @@ -41,6 +42,7 @@ Sentry.init({ ``` `app.mjs` + ```javascript import { Worker } from 'worker_threads'; @@ -50,10 +52,11 @@ const worker = new Worker(new URL('./worker.mjs', import.meta.url)); ``` `worker.mjs` -```javascript +```javascript // This worker thread will also be monitored for blocked event loops too ``` + Start your application: ```bash diff --git a/packages/node-native/rollup.npm.config.mjs b/packages/node-native/rollup.npm.config.mjs index 501cdc7c4e4f..9412e4c2a255 100644 --- a/packages/node-native/rollup.npm.config.mjs +++ b/packages/node-native/rollup.npm.config.mjs @@ -8,7 +8,7 @@ export default makeNPMConfigVariants( dir: 'build', // set exports to 'named' or 'auto' so that rollup doesn't warn exports: 'named', - preserveModules: true + preserveModules: true, }, }, }), From aa6960031a7351514ea7e749eebc76ac816d3142 Mon Sep 17 00:00:00 2001 From: Tim Fish Date: Mon, 23 Jun 2025 23:15:35 +0200 Subject: [PATCH 04/16] Update Sentry deps --- packages/node-native/package.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/node-native/package.json b/packages/node-native/package.json index 525bbb4b63ba..d68b20b56afa 100644 --- a/packages/node-native/package.json +++ b/packages/node-native/package.json @@ -64,8 +64,8 @@ }, "dependencies": { "@sentry-internal/node-native-stacktrace": "^0.1.0", - "@sentry/core": "9.29.0", - "@sentry/node": "9.29.0" + "@sentry/core": "9.31.0", + "@sentry/node": "9.31.0" }, "devDependencies": { "@types/node": "^18.19.1" From 156f67b3c110183061b5157a85ab973672f13ea4 Mon Sep 17 00:00:00 2001 From: Tim Fish Date: Tue, 24 Jun 2025 12:18:03 +0200 Subject: [PATCH 05/16] Better thread names --- .../suites/thread-blocked-native/test.ts | 6 +++--- packages/node-native/src/thread-blocked-watchdog.ts | 8 +++++++- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/dev-packages/node-integration-tests/suites/thread-blocked-native/test.ts b/dev-packages/node-integration-tests/suites/thread-blocked-native/test.ts index 2d0938fea306..40541051267d 100644 --- a/dev-packages/node-integration-tests/suites/thread-blocked-native/test.ts +++ b/dev-packages/node-integration-tests/suites/thread-blocked-native/test.ts @@ -54,7 +54,7 @@ const ANR_EVENT = { values: [ { id: '0', - name: '0', + name: 'main', crashed: true, current: true, main: true, @@ -163,7 +163,7 @@ describe('Thread Blocked Native', { timeout: 30_000 }, () => { values: [ { id: '0', - name: '0', + name: 'main', crashed: false, current: true, main: true, @@ -173,7 +173,7 @@ describe('Thread Blocked Native', { timeout: 30_000 }, () => { }, { id: '2', - name: '2', + name: 'worker-2', crashed: true, current: true, main: false, diff --git a/packages/node-native/src/thread-blocked-watchdog.ts b/packages/node-native/src/thread-blocked-watchdog.ts index 68e8dab2ff51..a7ef0bed0944 100644 --- a/packages/node-native/src/thread-blocked-watchdog.ts +++ b/packages/node-native/src/thread-blocked-watchdog.ts @@ -173,7 +173,13 @@ function getExceptionAndThreads( values: Object.entries(threads).map(([threadId, threadState]) => { const crashed = threadId === crashedThreadId; - const thread: Thread = { id: threadId, name: threadId, crashed, current: true, main: threadId === '0' }; + const thread: Thread = { + id: threadId, + name: threadId === '0' ? 'main' : `worker-${threadId}`, + crashed, + current: true, + main: threadId === '0', + }; if (!crashed) { thread.stacktrace = { frames: prepareStackFrames(threadState.frames) }; From 327fa5540085ca3d5f2c6b0cbb26b2220e4f097e Mon Sep 17 00:00:00 2001 From: Tim Fish Date: Tue, 24 Jun 2025 12:43:55 +0200 Subject: [PATCH 06/16] yarn.lock changes --- yarn.lock | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/yarn.lock b/yarn.lock index cc1159f1f82d..66ef5e7ad52d 100644 --- a/yarn.lock +++ b/yarn.lock @@ -26928,7 +26928,7 @@ string-template@~0.2.1: is-fullwidth-code-point "^3.0.0" strip-ansi "^6.0.1" -"string-width@^1.0.2 || 2 || 3 || 4", string-width@^4.0.0, string-width@^4.1.0, string-width@^4.2.0, string-width@^4.2.2, string-width@^4.2.3: +string-width@4.2.3, "string-width@^1.0.2 || 2 || 3 || 4", string-width@^4.0.0, string-width@^4.1.0, string-width@^4.2.0, string-width@^4.2.2, string-width@^4.2.3: version "4.2.3" resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.3.tgz#269c7117d27b05ad2e536830a8ec895ef9c6d010" integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g== @@ -27038,6 +27038,13 @@ stringify-object@^3.2.1: dependencies: ansi-regex "^5.0.1" +strip-ansi@6.0.1, strip-ansi@^6.0.0, strip-ansi@^6.0.1: + version "6.0.1" + resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.1.tgz#9e26c63d30f53443e9489495b2105d37b67a85d9" + integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A== + dependencies: + ansi-regex "^5.0.1" + strip-ansi@^3.0.0: version "3.0.1" resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-3.0.1.tgz#6a385fb8853d952d5ff05d0e8aaf94278dc63dcf" @@ -27059,13 +27066,6 @@ strip-ansi@^5.1.0, strip-ansi@^5.2.0: dependencies: ansi-regex "^4.1.0" -strip-ansi@^6.0.0, strip-ansi@^6.0.1: - version "6.0.1" - resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.1.tgz#9e26c63d30f53443e9489495b2105d37b67a85d9" - integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A== - dependencies: - ansi-regex "^5.0.1" - strip-ansi@^7.0.1, strip-ansi@^7.1.0: version "7.1.0" resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-7.1.0.tgz#d5b6568ca689d8561370b0707685d22434faff45" @@ -27201,7 +27201,7 @@ stylus@0.59.0, stylus@^0.59.0: sax "~1.2.4" source-map "^0.7.3" -sucrase@^3.27.0, sucrase@^3.35.0: +sucrase@^3.27.0, sucrase@^3.35.0, sucrase@getsentry/sucrase#es2020-polyfills: version "3.36.0" resolved "https://codeload.github.com/getsentry/sucrase/tar.gz/fd682f6129e507c00bb4e6319cc5d6b767e36061" dependencies: @@ -29836,19 +29836,19 @@ wrangler@^3.67.1: string-width "^4.1.0" strip-ansi "^6.0.0" -wrap-ansi@^6.0.1: - version "6.2.0" - resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-6.2.0.tgz#e9393ba07102e6c91a3b221478f0257cd2856e53" - integrity sha512-r6lPcBGxZXlIcymEu7InxDMhdW0KDxpLgoFLcguasxCaJ/SOIZwINatK9KY/tf+ZrlywOKU0UDj3ATXUBfxJXA== +wrap-ansi@7.0.0, wrap-ansi@^7.0.0: + version "7.0.0" + resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-7.0.0.tgz#67e145cff510a6a6984bdf1152911d69d2eb9e43" + integrity sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q== dependencies: ansi-styles "^4.0.0" string-width "^4.1.0" strip-ansi "^6.0.0" -wrap-ansi@^7.0.0: - version "7.0.0" - resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-7.0.0.tgz#67e145cff510a6a6984bdf1152911d69d2eb9e43" - integrity sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q== +wrap-ansi@^6.0.1: + version "6.2.0" + resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-6.2.0.tgz#e9393ba07102e6c91a3b221478f0257cd2856e53" + integrity sha512-r6lPcBGxZXlIcymEu7InxDMhdW0KDxpLgoFLcguasxCaJ/SOIZwINatK9KY/tf+ZrlywOKU0UDj3ATXUBfxJXA== dependencies: ansi-styles "^4.0.0" string-width "^4.1.0" From 6fa63af0fb8040c2c05396ea235fd76a74d48e7d Mon Sep 17 00:00:00 2001 From: Tim Fish Date: Tue, 24 Jun 2025 13:20:23 +0200 Subject: [PATCH 07/16] Add to verdaccio config --- dev-packages/e2e-tests/verdaccio-config/config.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/dev-packages/e2e-tests/verdaccio-config/config.yaml b/dev-packages/e2e-tests/verdaccio-config/config.yaml index c5400118d12c..d96c3cc6905e 100644 --- a/dev-packages/e2e-tests/verdaccio-config/config.yaml +++ b/dev-packages/e2e-tests/verdaccio-config/config.yaml @@ -104,6 +104,12 @@ packages: unpublish: $all # proxy: npmjs # Don't proxy for E2E tests! + '@sentry/node-native': + access: $all + publish: $all + unpublish: $all + # proxy: npmjs # Don't proxy for E2E tests! + '@sentry/opentelemetry': access: $all publish: $all From 0745a2011542efdd81ffad647158f5084a814f10 Mon Sep 17 00:00:00 2001 From: Tim Fish Date: Tue, 24 Jun 2025 13:35:43 +0200 Subject: [PATCH 08/16] Fix tests --- .../suites/thread-blocked-native/test.ts | 34 ++++++++++--------- .../node-integration-tests/utils/runner.ts | 2 +- 2 files changed, 19 insertions(+), 17 deletions(-) diff --git a/dev-packages/node-integration-tests/suites/thread-blocked-native/test.ts b/dev-packages/node-integration-tests/suites/thread-blocked-native/test.ts index 40541051267d..37d2eed23d92 100644 --- a/dev-packages/node-integration-tests/suites/thread-blocked-native/test.ts +++ b/dev-packages/node-integration-tests/suites/thread-blocked-native/test.ts @@ -65,18 +65,20 @@ const ANR_EVENT = { exception: EXCEPTION(), }; -const ANR_EVENT_WITH_DEBUG_META: Event = { - ...ANR_EVENT, - debug_meta: { - images: [ - { - type: 'sourcemap', - debug_id: 'aaaaaaaa-aaaa-4aaa-aaaa-aaaaaaaaaa', - code_file: expect.stringContaining('basic'), - }, - ], - }, -}; +function ANR_EVENT_WITH_DEBUG_META(file: string): Event { + return { + ...ANR_EVENT, + debug_meta: { + images: [ + { + type: 'sourcemap', + debug_id: 'aaaaaaaa-aaaa-4aaa-aaaa-aaaaaaaaaa', + code_file: expect.stringContaining(file), + }, + ], + }, + }; +} describe('Thread Blocked Native', { timeout: 30_000 }, () => { afterAll(() => { @@ -86,7 +88,7 @@ describe('Thread Blocked Native', { timeout: 30_000 }, () => { test('CJS', async () => { await createRunner(__dirname, 'basic.js') .withMockSentryServer() - .expect({ event: ANR_EVENT_WITH_DEBUG_META }) + .expect({ event: ANR_EVENT_WITH_DEBUG_META('basic') }) .start() .completed(); }); @@ -94,7 +96,7 @@ describe('Thread Blocked Native', { timeout: 30_000 }, () => { test('ESM', async () => { await createRunner(__dirname, 'basic.mjs') .withMockSentryServer() - .expect({ event: ANR_EVENT_WITH_DEBUG_META }) + .expect({ event: ANR_EVENT_WITH_DEBUG_META('basic') }) .start() .completed(); }); @@ -123,8 +125,8 @@ describe('Thread Blocked Native', { timeout: 30_000 }, () => { test('multiple events via maxBlockedEvents', async () => { await createRunner(__dirname, 'basic-multiple.mjs') .withMockSentryServer() - .expect({ event: ANR_EVENT_WITH_DEBUG_META }) - .expect({ event: ANR_EVENT_WITH_DEBUG_META }) + .expect({ event: ANR_EVENT_WITH_DEBUG_META('basic-multiple') }) + .expect({ event: ANR_EVENT_WITH_DEBUG_META('basic-multiple') }) .start() .completed(); }); diff --git a/dev-packages/node-integration-tests/utils/runner.ts b/dev-packages/node-integration-tests/utils/runner.ts index 1f5b5508b12f..1006d71bf3f0 100644 --- a/dev-packages/node-integration-tests/utils/runner.ts +++ b/dev-packages/node-integration-tests/utils/runner.ts @@ -450,7 +450,7 @@ export function createRunner(...paths: string[]) { if (process.env.DEBUG) log('starting scenario', testPath, flags, env.SENTRY_DSN); - child = spawn('node', [...flags, testPath], { env, stdio: process.env.DEBUG ? 'inherit' : 'ignore' }); + child = spawn('node', [...flags, testPath], { env }); child.on('error', e => { // eslint-disable-next-line no-console From 3f83db22109766341ea50ea27cc5496b25d5f099 Mon Sep 17 00:00:00 2001 From: Tim Fish Date: Tue, 24 Jun 2025 14:05:46 +0200 Subject: [PATCH 09/16] rename package export --- packages/node-native/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/node-native/package.json b/packages/node-native/package.json index d68b20b56afa..2e495211952c 100644 --- a/packages/node-native/package.json +++ b/packages/node-native/package.json @@ -21,7 +21,7 @@ "default": "./build/cjs/index.js" } }, - "./worker": { + "./thread-blocked-watchdog": { "import": { "default": "./build/esm/thread-blocked-watchdog.js" }, From c4a718c3d54bbb7b81543733f73b63bb07b5e36e Mon Sep 17 00:00:00 2001 From: Tim Fish Date: Tue, 24 Jun 2025 14:18:44 +0200 Subject: [PATCH 10/16] Fix flakey worker thread test --- .../suites/thread-blocked-native/test.ts | 63 ++++++++++--------- 1 file changed, 34 insertions(+), 29 deletions(-) diff --git a/dev-packages/node-integration-tests/suites/thread-blocked-native/test.ts b/dev-packages/node-integration-tests/suites/thread-blocked-native/test.ts index 37d2eed23d92..d071afa23d54 100644 --- a/dev-packages/node-integration-tests/suites/thread-blocked-native/test.ts +++ b/dev-packages/node-integration-tests/suites/thread-blocked-native/test.ts @@ -156,39 +156,44 @@ describe('Thread Blocked Native', { timeout: 30_000 }, () => { }); test('worker thread', async () => { - const ANR_EVENT_THREADS: Event = { - ...ANR_EVENT, - exception: { - ...EXCEPTION('2'), - }, - threads: { - values: [ - { - id: '0', - name: 'main', - crashed: false, - current: true, - main: true, - stacktrace: { - frames: expect.any(Array), - }, - }, - { - id: '2', - name: 'worker-2', - crashed: true, - current: true, - main: false, - }, - ], - }, - }; - const instrument = join(__dirname, 'instrument.mjs'); await createRunner(__dirname, 'worker-main.mjs') .withMockSentryServer() .withFlags('--import', instrument) - .expect({ event: ANR_EVENT_THREADS }) + .expect({ + event: event => { + const crashedThread = event.threads?.values?.find(thread => thread.crashed)?.id as string; + expect(crashedThread).toBeDefined(); + + expect(event).toMatchObject({ + ...ANR_EVENT, + exception: { + ...EXCEPTION(crashedThread), + }, + threads: { + values: [ + { + id: '0', + name: 'main', + crashed: false, + current: true, + main: true, + stacktrace: { + frames: expect.any(Array), + }, + }, + { + id: crashedThread, + name: `worker-${crashedThread}`, + crashed: true, + current: true, + main: false, + }, + ], + }, + }); + }, + }) .start() .completed(); }); From ad4693a79a32308537562a0b0ab377f9d8c8b0f1 Mon Sep 17 00:00:00 2001 From: Tim Fish Date: Tue, 24 Jun 2025 22:04:48 +0200 Subject: [PATCH 11/16] Apply suggestions from code review Co-authored-by: Abhijeet Prasad --- packages/node-native/src/thread-blocked-integration.ts | 4 ++-- packages/node-native/src/thread-blocked-watchdog.ts | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/node-native/src/thread-blocked-integration.ts b/packages/node-native/src/thread-blocked-integration.ts index 9e4394f20707..8dfdfd7d7624 100644 --- a/packages/node-native/src/thread-blocked-integration.ts +++ b/packages/node-native/src/thread-blocked-integration.ts @@ -8,8 +8,8 @@ import type { ThreadBlockedIntegrationOptions, WorkerStartData } from './common' const { isPromise } = types; -const DEFAULT_INTERVAL = 50; -const DEFAULT_HANG_THRESHOLD = 5000; +const DEFAULT_INTERVAL_MS = 50; +const DEFAULT_HANG_THRESHOLD_MS = 5000; function log(message: string, ...args: unknown[]): void { logger.log(`[Thread Blocked] ${message}`, ...args); diff --git a/packages/node-native/src/thread-blocked-watchdog.ts b/packages/node-native/src/thread-blocked-watchdog.ts index a7ef0bed0944..5f47c3a8aed4 100644 --- a/packages/node-native/src/thread-blocked-watchdog.ts +++ b/packages/node-native/src/thread-blocked-watchdog.ts @@ -37,7 +37,7 @@ let sentAnrEvents = 0; function log(...msg: unknown[]): void { if (debug) { // eslint-disable-next-line no-console - console.log('[Blocked Watchdog]', ...msg); + console.log('[Sentry Blocked Watchdog]', ...msg); } } From 8443d3314e268fbf0fe7763d07f172f29eb114bf Mon Sep 17 00:00:00 2001 From: Tim Fish Date: Wed, 25 Jun 2025 00:08:09 +0200 Subject: [PATCH 12/16] PR review --- .../suites/thread-blocked-native/app-path.mjs | 4 +- .../thread-blocked-native/basic-multiple.mjs | 4 +- .../suites/thread-blocked-native/basic.js | 4 +- .../suites/thread-blocked-native/basic.mjs | 4 +- .../thread-blocked-native/indefinite.mjs | 4 +- .../thread-blocked-native/instrument.mjs | 4 +- .../suites/thread-blocked-native/long-work.js | 2 +- .../should-exit-forced.js | 4 +- .../thread-blocked-native/should-exit.js | 4 +- .../thread-blocked-native/stop-and-start.js | 4 +- .../suites/thread-blocked-native/test.ts | 2 +- packages/node-native/README.md | 13 ++--- packages/node-native/package.json | 6 +-- packages/node-native/rollup.npm.config.mjs | 2 +- packages/node-native/src/common.ts | 14 ++--- ...ion.ts => event-loop-block-integration.ts} | 52 ++++++++++++++----- ...tchdog.ts => event-loop-block-watchdog.ts} | 9 ++-- packages/node-native/src/index.ts | 2 +- 18 files changed, 82 insertions(+), 56 deletions(-) rename packages/node-native/src/{thread-blocked-integration.ts => event-loop-block-integration.ts} (78%) rename packages/node-native/src/{thread-blocked-watchdog.ts => event-loop-block-watchdog.ts} (97%) diff --git a/dev-packages/node-integration-tests/suites/thread-blocked-native/app-path.mjs b/dev-packages/node-integration-tests/suites/thread-blocked-native/app-path.mjs index d3043df842ff..c561b221d95f 100644 --- a/dev-packages/node-integration-tests/suites/thread-blocked-native/app-path.mjs +++ b/dev-packages/node-integration-tests/suites/thread-blocked-native/app-path.mjs @@ -1,5 +1,5 @@ import * as Sentry from '@sentry/node'; -import { threadBlockedIntegration } from '@sentry/node-native'; +import { eventLoopBlockIntegration } from '@sentry/node-native'; import * as path from 'path'; import * as url from 'url'; import { longWork } from './long-work.js'; @@ -15,7 +15,7 @@ setTimeout(() => { Sentry.init({ dsn: process.env.SENTRY_DSN, release: '1.0', - integrations: [threadBlockedIntegration({ blockedThreshold: 100, appRootPath: __dirname })], + integrations: [eventLoopBlockIntegration({ appRootPath: __dirname })], }); setTimeout(() => { diff --git a/dev-packages/node-integration-tests/suites/thread-blocked-native/basic-multiple.mjs b/dev-packages/node-integration-tests/suites/thread-blocked-native/basic-multiple.mjs index 62c024f81f45..3218f12bf4c6 100644 --- a/dev-packages/node-integration-tests/suites/thread-blocked-native/basic-multiple.mjs +++ b/dev-packages/node-integration-tests/suites/thread-blocked-native/basic-multiple.mjs @@ -1,5 +1,5 @@ import * as Sentry from '@sentry/node'; -import { threadBlockedIntegration } from '@sentry/node-native'; +import { eventLoopBlockIntegration } from '@sentry/node-native'; import { longWork } from './long-work.js'; global._sentryDebugIds = { [new Error().stack]: 'aaaaaaaa-aaaa-4aaa-aaaa-aaaaaaaaaa' }; @@ -11,7 +11,7 @@ setTimeout(() => { Sentry.init({ dsn: process.env.SENTRY_DSN, release: '1.0', - integrations: [threadBlockedIntegration({ blockedThreshold: 100, maxBlockedEvents: 2 })], + integrations: [eventLoopBlockIntegration({ maxBlockedEvents: 2 })], }); setTimeout(() => { diff --git a/dev-packages/node-integration-tests/suites/thread-blocked-native/basic.js b/dev-packages/node-integration-tests/suites/thread-blocked-native/basic.js index 2db4260f95bb..30740bbd031b 100644 --- a/dev-packages/node-integration-tests/suites/thread-blocked-native/basic.js +++ b/dev-packages/node-integration-tests/suites/thread-blocked-native/basic.js @@ -1,5 +1,5 @@ const Sentry = require('@sentry/node'); -const { threadBlockedIntegration } = require('@sentry/node-native'); +const { eventLoopBlockIntegration } = require('@sentry/node-native'); const { longWork } = require('./long-work.js'); global._sentryDebugIds = { [new Error().stack]: 'aaaaaaaa-aaaa-4aaa-aaaa-aaaaaaaaaa' }; @@ -11,7 +11,7 @@ setTimeout(() => { Sentry.init({ dsn: process.env.SENTRY_DSN, release: '1.0', - integrations: [threadBlockedIntegration({ blockedThreshold: 100 })], + integrations: [eventLoopBlockIntegration()], }); setTimeout(() => { diff --git a/dev-packages/node-integration-tests/suites/thread-blocked-native/basic.mjs b/dev-packages/node-integration-tests/suites/thread-blocked-native/basic.mjs index b18465e8655b..273760a6db39 100644 --- a/dev-packages/node-integration-tests/suites/thread-blocked-native/basic.mjs +++ b/dev-packages/node-integration-tests/suites/thread-blocked-native/basic.mjs @@ -1,5 +1,5 @@ import * as Sentry from '@sentry/node'; -import { threadBlockedIntegration } from '@sentry/node-native'; +import { eventLoopBlockIntegration } from '@sentry/node-native'; import { longWork } from './long-work.js'; global._sentryDebugIds = { [new Error().stack]: 'aaaaaaaa-aaaa-4aaa-aaaa-aaaaaaaaaa' }; @@ -11,7 +11,7 @@ setTimeout(() => { Sentry.init({ dsn: process.env.SENTRY_DSN, release: '1.0', - integrations: [threadBlockedIntegration({ blockedThreshold: 100 })], + integrations: [eventLoopBlockIntegration()], }); setTimeout(() => { diff --git a/dev-packages/node-integration-tests/suites/thread-blocked-native/indefinite.mjs b/dev-packages/node-integration-tests/suites/thread-blocked-native/indefinite.mjs index 14a7b213c1b5..55eecb5c23ec 100644 --- a/dev-packages/node-integration-tests/suites/thread-blocked-native/indefinite.mjs +++ b/dev-packages/node-integration-tests/suites/thread-blocked-native/indefinite.mjs @@ -1,5 +1,5 @@ import * as Sentry from '@sentry/node'; -import { threadBlockedIntegration } from '@sentry/node-native'; +import { eventLoopBlockIntegration } from '@sentry/node-native'; import * as assert from 'assert'; import * as crypto from 'crypto'; @@ -10,7 +10,7 @@ setTimeout(() => { Sentry.init({ dsn: process.env.SENTRY_DSN, release: '1.0', - integrations: [threadBlockedIntegration({ blockedThreshold: 100 })], + integrations: [eventLoopBlockIntegration()], }); function longWork() { diff --git a/dev-packages/node-integration-tests/suites/thread-blocked-native/instrument.mjs b/dev-packages/node-integration-tests/suites/thread-blocked-native/instrument.mjs index 4c3cae71c989..ee66bf82f8bf 100644 --- a/dev-packages/node-integration-tests/suites/thread-blocked-native/instrument.mjs +++ b/dev-packages/node-integration-tests/suites/thread-blocked-native/instrument.mjs @@ -1,9 +1,9 @@ import * as Sentry from '@sentry/node'; -import { threadBlockedIntegration } from '@sentry/node-native'; +import { eventLoopBlockIntegration } from '@sentry/node-native'; Sentry.init({ debug: true, dsn: process.env.SENTRY_DSN, release: '1.0', - integrations: [threadBlockedIntegration({ blockedThreshold: 100 })], + integrations: [eventLoopBlockIntegration()], }); diff --git a/dev-packages/node-integration-tests/suites/thread-blocked-native/long-work.js b/dev-packages/node-integration-tests/suites/thread-blocked-native/long-work.js index f35e2268dcd6..55f5358a10fe 100644 --- a/dev-packages/node-integration-tests/suites/thread-blocked-native/long-work.js +++ b/dev-packages/node-integration-tests/suites/thread-blocked-native/long-work.js @@ -2,7 +2,7 @@ const crypto = require('crypto'); const assert = require('assert'); function longWork() { - for (let i = 0; i < 20; i++) { + for (let i = 0; i < 200; i++) { const salt = crypto.randomBytes(128).toString('base64'); const hash = crypto.pbkdf2Sync('myPassword', salt, 10000, 512, 'sha512'); assert.ok(hash); diff --git a/dev-packages/node-integration-tests/suites/thread-blocked-native/should-exit-forced.js b/dev-packages/node-integration-tests/suites/thread-blocked-native/should-exit-forced.js index 5ffd83f48e24..71622bdbe083 100644 --- a/dev-packages/node-integration-tests/suites/thread-blocked-native/should-exit-forced.js +++ b/dev-packages/node-integration-tests/suites/thread-blocked-native/should-exit-forced.js @@ -1,12 +1,12 @@ const Sentry = require('@sentry/node'); -const { threadBlockedIntegration } = require('@sentry/node-native'); +const { eventLoopBlockIntegration } = require('@sentry/node-native'); function configureSentry() { Sentry.init({ dsn: 'https://public@dsn.ingest.sentry.io/1337', release: '1.0', debug: true, - integrations: [threadBlockedIntegration()], + integrations: [eventLoopBlockIntegration()], }); } diff --git a/dev-packages/node-integration-tests/suites/thread-blocked-native/should-exit.js b/dev-packages/node-integration-tests/suites/thread-blocked-native/should-exit.js index 16ca705fb236..cda4c0e10d3a 100644 --- a/dev-packages/node-integration-tests/suites/thread-blocked-native/should-exit.js +++ b/dev-packages/node-integration-tests/suites/thread-blocked-native/should-exit.js @@ -1,12 +1,12 @@ const Sentry = require('@sentry/node'); -const { threadBlockedIntegration } = require('@sentry/node-native'); +const { eventLoopBlockIntegration } = require('@sentry/node-native'); function configureSentry() { Sentry.init({ dsn: 'https://public@dsn.ingest.sentry.io/1337', release: '1.0', debug: true, - integrations: [threadBlockedIntegration()], + integrations: [eventLoopBlockIntegration()], }); } diff --git a/dev-packages/node-integration-tests/suites/thread-blocked-native/stop-and-start.js b/dev-packages/node-integration-tests/suites/thread-blocked-native/stop-and-start.js index 51402f633d8c..ec42ffd3c8e9 100644 --- a/dev-packages/node-integration-tests/suites/thread-blocked-native/stop-and-start.js +++ b/dev-packages/node-integration-tests/suites/thread-blocked-native/stop-and-start.js @@ -1,5 +1,5 @@ const Sentry = require('@sentry/node'); -const { threadBlockedIntegration } = require('@sentry/node-native'); +const { eventLoopBlockIntegration } = require('@sentry/node-native'); const { longWork } = require('./long-work.js'); const crypto = require('crypto'); const assert = require('assert'); @@ -8,7 +8,7 @@ setTimeout(() => { process.exit(); }, 10000); -const threadBlocked = threadBlockedIntegration({ blockedThreshold: 100 }); +const threadBlocked = eventLoopBlockIntegration(); Sentry.init({ dsn: process.env.SENTRY_DSN, diff --git a/dev-packages/node-integration-tests/suites/thread-blocked-native/test.ts b/dev-packages/node-integration-tests/suites/thread-blocked-native/test.ts index d071afa23d54..3b35fe8b3a4a 100644 --- a/dev-packages/node-integration-tests/suites/thread-blocked-native/test.ts +++ b/dev-packages/node-integration-tests/suites/thread-blocked-native/test.ts @@ -8,7 +8,7 @@ function EXCEPTION(thread_id = '0') { values: [ { type: 'EventLoopBlocked', - value: 'Event Loop Blocked for at least 100 ms', + value: 'Event Loop Blocked for at least 1000 ms', mechanism: { type: 'ANR' }, thread_id, stacktrace: { diff --git a/packages/node-native/README.md b/packages/node-native/README.md index de3b79194efe..4ff7b6fdab45 100644 --- a/packages/node-native/README.md +++ b/packages/node-native/README.md @@ -20,24 +20,25 @@ yarn add @sentry/node @sentry/node-native npm install --save @sentry/node @sentry/node-native ``` -## `threadBlockedIntegration` +## `eventLoopBlockIntegration` -The `threadBlockedIntegration` can be used to monitor for blocked event loops in +The `eventLoopBlockIntegration` can be used to monitor for blocked event loops in all threads of a Node.js application. -If you instrument your application via the Node.js `--import` flag, this -instrumentation will be automatically applied to all worker threads. +If you instrument your application via the Node.js `--import` flag, Sentry will +be started and this instrumentation will be automatically applied to all worker +threads. `instrument.mjs` ```javascript import * as Sentry from '@sentry/node'; -import { threadBlockedIntegration } from '@sentry/node-native'; +import { eventLoopBlockIntegration } from '@sentry/node-native'; Sentry.init({ dsn: '__YOUR_DSN__', // Capture stack traces when the event loop is blocked for more than 500ms - integrations: [threadBlockedIntegration({ blockedThreshold: 500 })], + integrations: [eventLoopBlockIntegration({ threshold: 500 })], }); ``` diff --git a/packages/node-native/package.json b/packages/node-native/package.json index 2e495211952c..844f85f4cc2b 100644 --- a/packages/node-native/package.json +++ b/packages/node-native/package.json @@ -21,12 +21,12 @@ "default": "./build/cjs/index.js" } }, - "./thread-blocked-watchdog": { + "./event-loop-block-watchdog": { "import": { - "default": "./build/esm/thread-blocked-watchdog.js" + "default": "./build/esm/event-loop-block-watchdog.js" }, "require": { - "default": "./build/cjs/thread-blocked-watchdog.js" + "default": "./build/cjs/event-loop-block-watchdog.js" } } }, diff --git a/packages/node-native/rollup.npm.config.mjs b/packages/node-native/rollup.npm.config.mjs index 9412e4c2a255..ce79b0ac9bbb 100644 --- a/packages/node-native/rollup.npm.config.mjs +++ b/packages/node-native/rollup.npm.config.mjs @@ -2,7 +2,7 @@ import { makeBaseNPMConfig, makeNPMConfigVariants } from '@sentry-internal/rollu export default makeNPMConfigVariants( makeBaseNPMConfig({ - entrypoints: ['src/index.ts', 'src/thread-blocked-watchdog.ts'], + entrypoints: ['src/index.ts', 'src/event-loop-block-watchdog.ts'], packageSpecificConfig: { output: { dir: 'build', diff --git a/packages/node-native/src/common.ts b/packages/node-native/src/common.ts index 4ed1a3fe1dc3..7ac9d62ed87d 100644 --- a/packages/node-native/src/common.ts +++ b/packages/node-native/src/common.ts @@ -1,18 +1,14 @@ import type { Contexts, DsnComponents, Primitive, SdkMetadata, Session } from '@sentry/core'; +export const POLL_RATIO = 2; + export interface ThreadBlockedIntegrationOptions { /** - * Interval to send heartbeat messages to the watchdog. - * - * Defaults to 50ms. - */ - pollInterval: number; - /** - * Threshold in milliseconds to trigger a blocked event. + * Threshold in milliseconds to trigger an event. * - * Defaults to 5000ms. + * Defaults to 1000ms. */ - blockedThreshold: number; + threshold: number; /** * Maximum number of blocked events to send. * diff --git a/packages/node-native/src/thread-blocked-integration.ts b/packages/node-native/src/event-loop-block-integration.ts similarity index 78% rename from packages/node-native/src/thread-blocked-integration.ts rename to packages/node-native/src/event-loop-block-integration.ts index 9e4394f20707..7a16e51c36c3 100644 --- a/packages/node-native/src/thread-blocked-integration.ts +++ b/packages/node-native/src/event-loop-block-integration.ts @@ -5,11 +5,11 @@ import { defineIntegration, getClient, getFilenameToDebugIdMap, getIsolationScop import type { NodeClient } from '@sentry/node'; import { registerThread, threadPoll } from '@sentry-internal/node-native-stacktrace'; import type { ThreadBlockedIntegrationOptions, WorkerStartData } from './common'; +import { POLL_RATIO } from './common'; const { isPromise } = types; -const DEFAULT_INTERVAL = 50; -const DEFAULT_HANG_THRESHOLD = 5000; +const DEFAULT_THRESHOLD = 1_000; function log(message: string, ...args: unknown[]): void { logger.log(`[Thread Blocked] ${message}`, ...args); @@ -34,7 +34,7 @@ const INTEGRATION_NAME = 'ThreadBlocked'; type ThreadBlockedInternal = { startWorker: () => void; stopWorker: () => void }; -const _threadBlockedIntegration = ((options: Partial = {}) => { +const _eventLoopBlockIntegration = ((options: Partial = {}) => { let worker: Promise<() => void> | undefined; let client: NodeClient | undefined; @@ -58,21 +58,48 @@ const _threadBlockedIntegration = ((options: Partial this.startWorker()); + this.startWorker(); }, } as Integration & ThreadBlockedInternal; }) satisfies IntegrationFn; type ThreadBlockedReturn = (options?: Partial) => Integration & ThreadBlockedInternal; -export const threadBlockedIntegration = defineIntegration(_threadBlockedIntegration) as ThreadBlockedReturn; +/** + * Monitors the Node.js event loop for blocking behavior and reports blocked events to Sentry. + * + * Uses a background worker thread to detect when the main thread is blocked for longer than + * the configured threshold (default: 5 seconds). + * + * When instrumenting via the `--import` flag, this integration will + * automatically monitor all worker threads as well. + * + * ```js + * // instrument.mjs + * import * as Sentry from '@sentry/node'; + * import { eventLoopBlockIntegration } from '@sentry/node-native'; + * + * Sentry.init({ + * dsn: '__YOUR_DSN__', + * integrations: [ + * eventLoopBlockIntegration({ + * threshold: 500, // Report blocks longer than 500ms + * }), + * ], + * }); + * ``` + * + * Start your application with: + * ```bash + * node --import instrument.mjs app.mjs + * ``` + */ +export const eventLoopBlockIntegration = defineIntegration(_eventLoopBlockIntegration) as ThreadBlockedReturn; /** * Starts the worker thread @@ -113,14 +140,15 @@ async function _startWorker( dist: initOptions.dist, sdkMetadata, appRootPath: integrationOptions.appRootPath, - pollInterval: integrationOptions.pollInterval || DEFAULT_INTERVAL, - blockedThreshold: integrationOptions.blockedThreshold || DEFAULT_HANG_THRESHOLD, + threshold: integrationOptions.threshold || DEFAULT_THRESHOLD, maxBlockedEvents: integrationOptions.maxBlockedEvents || 1, staticTags: integrationOptions.staticTags || {}, contexts, }; - const worker = new Worker(new URL('./thread-blocked-watchdog.js', import.meta.url), { + const pollInterval = options.threshold / POLL_RATIO + + const worker = new Worker(new URL('./event-loop-block-watchdog.js', import.meta.url), { workerData: options, // We don't want any Node args like --import to be passed to the worker execArgv: [], @@ -143,7 +171,7 @@ async function _startWorker( } catch (_) { // } - }, options.pollInterval); + }, pollInterval); // Timer should not block exit timer.unref(); diff --git a/packages/node-native/src/thread-blocked-watchdog.ts b/packages/node-native/src/event-loop-block-watchdog.ts similarity index 97% rename from packages/node-native/src/thread-blocked-watchdog.ts rename to packages/node-native/src/event-loop-block-watchdog.ts index a7ef0bed0944..1d628d7d5b1d 100644 --- a/packages/node-native/src/thread-blocked-watchdog.ts +++ b/packages/node-native/src/event-loop-block-watchdog.ts @@ -14,9 +14,10 @@ import { import { makeNodeTransport } from '@sentry/node'; import { captureStackTrace, getThreadsLastSeen } from '@sentry-internal/node-native-stacktrace'; import type { ThreadState, WorkerStartData } from './common'; +import { POLL_RATIO } from './common'; const { - blockedThreshold, + threshold, appRootPath, contexts, debug, @@ -24,13 +25,13 @@ const { dsn, environment, maxBlockedEvents, - pollInterval, release, sdkMetadata, staticTags: tags, tunnel, } = workerData as WorkerStartData; +const pollInterval = threshold / POLL_RATIO const triggeredThreads = new Set(); let sentAnrEvents = 0; @@ -161,7 +162,7 @@ function getExceptionAndThreads( values: [ { type: 'EventLoopBlocked', - value: `Event Loop Blocked for at least ${blockedThreshold} ms`, + value: `Event Loop Blocked for at least ${threshold} ms`, stacktrace: { frames: prepareStackFrames(crashedThread?.frames) }, // This ensures the UI doesn't say 'Crashed in' for the stack trace mechanism: { type: 'ANR' }, @@ -246,7 +247,7 @@ async function sendAnrEvent(crashedThreadId: string): Promise { setInterval(async () => { for (const [threadId, time] of Object.entries(getThreadsLastSeen())) { - if (time > blockedThreshold) { + if (time > threshold) { if (triggeredThreads.has(threadId)) { continue; } diff --git a/packages/node-native/src/index.ts b/packages/node-native/src/index.ts index c866b9a15c60..b992ec6e1e6c 100644 --- a/packages/node-native/src/index.ts +++ b/packages/node-native/src/index.ts @@ -1 +1 @@ -export { threadBlockedIntegration, disableBlockedDetectionForCallback } from './thread-blocked-integration'; +export { eventLoopBlockIntegration, disableBlockedDetectionForCallback } from './event-loop-block-integration'; From 66e6aa503fb2e38624a7900cab5b06dec471f1d5 Mon Sep 17 00:00:00 2001 From: Tim Fish Date: Wed, 25 Jun 2025 00:36:23 +0200 Subject: [PATCH 13/16] Add rate limits --- .../thread-blocked-native/basic-multiple.mjs | 2 +- .../thread-blocked-native/stop-and-start.js | 4 +- .../suites/thread-blocked-native/test.ts | 2 +- packages/node-native/src/common.ts | 4 +- .../src/event-loop-block-integration.ts | 27 ++++++----- .../src/event-loop-block-watchdog.ts | 45 ++++++++++++------- 6 files changed, 49 insertions(+), 35 deletions(-) diff --git a/dev-packages/node-integration-tests/suites/thread-blocked-native/basic-multiple.mjs b/dev-packages/node-integration-tests/suites/thread-blocked-native/basic-multiple.mjs index 3218f12bf4c6..32135d2246f2 100644 --- a/dev-packages/node-integration-tests/suites/thread-blocked-native/basic-multiple.mjs +++ b/dev-packages/node-integration-tests/suites/thread-blocked-native/basic-multiple.mjs @@ -11,7 +11,7 @@ setTimeout(() => { Sentry.init({ dsn: process.env.SENTRY_DSN, release: '1.0', - integrations: [eventLoopBlockIntegration({ maxBlockedEvents: 2 })], + integrations: [eventLoopBlockIntegration({ maxEventsPerHour: 2 })], }); setTimeout(() => { diff --git a/dev-packages/node-integration-tests/suites/thread-blocked-native/stop-and-start.js b/dev-packages/node-integration-tests/suites/thread-blocked-native/stop-and-start.js index ec42ffd3c8e9..5c797cd20c60 100644 --- a/dev-packages/node-integration-tests/suites/thread-blocked-native/stop-and-start.js +++ b/dev-packages/node-integration-tests/suites/thread-blocked-native/stop-and-start.js @@ -29,13 +29,13 @@ function longWorkIgnored() { } setTimeout(() => { - threadBlocked.stopWorker(); + threadBlocked.stop(); setTimeout(() => { longWorkIgnored(); setTimeout(() => { - threadBlocked.startWorker(); + threadBlocked.start(); setTimeout(() => { longWork(); diff --git a/dev-packages/node-integration-tests/suites/thread-blocked-native/test.ts b/dev-packages/node-integration-tests/suites/thread-blocked-native/test.ts index 3b35fe8b3a4a..a0c24894a19c 100644 --- a/dev-packages/node-integration-tests/suites/thread-blocked-native/test.ts +++ b/dev-packages/node-integration-tests/suites/thread-blocked-native/test.ts @@ -122,7 +122,7 @@ describe('Thread Blocked Native', { timeout: 30_000 }, () => { .completed(); }); - test('multiple events via maxBlockedEvents', async () => { + test('multiple events via maxEventsPerHour', async () => { await createRunner(__dirname, 'basic-multiple.mjs') .withMockSentryServer() .expect({ event: ANR_EVENT_WITH_DEBUG_META('basic-multiple') }) diff --git a/packages/node-native/src/common.ts b/packages/node-native/src/common.ts index 7ac9d62ed87d..2a96050dbc34 100644 --- a/packages/node-native/src/common.ts +++ b/packages/node-native/src/common.ts @@ -10,11 +10,11 @@ export interface ThreadBlockedIntegrationOptions { */ threshold: number; /** - * Maximum number of blocked events to send. + * Maximum number of blocked events to send per clock hour. * * Defaults to 1. */ - maxBlockedEvents: number; + maxEventsPerHour: number; /** * Tags to include with blocked events. */ diff --git a/packages/node-native/src/event-loop-block-integration.ts b/packages/node-native/src/event-loop-block-integration.ts index 7a16e51c36c3..32e30d30d3d5 100644 --- a/packages/node-native/src/event-loop-block-integration.ts +++ b/packages/node-native/src/event-loop-block-integration.ts @@ -9,10 +9,10 @@ import { POLL_RATIO } from './common'; const { isPromise } = types; -const DEFAULT_THRESHOLD = 1_000; +const DEFAULT_THRESHOLD_MS = 1_000; function log(message: string, ...args: unknown[]): void { - logger.log(`[Thread Blocked] ${message}`, ...args); + logger.log(`[Sentry Block Event Loop] ${message}`, ...args); } /** @@ -32,7 +32,7 @@ async function getContexts(client: NodeClient): Promise { const INTEGRATION_NAME = 'ThreadBlocked'; -type ThreadBlockedInternal = { startWorker: () => void; stopWorker: () => void }; +type ThreadBlockedInternal = { start: () => void; stop: () => void }; const _eventLoopBlockIntegration = ((options: Partial = {}) => { let worker: Promise<() => void> | undefined; @@ -40,7 +40,7 @@ const _eventLoopBlockIntegration = ((options: Partial { + start: () => { if (worker) { return; } @@ -49,7 +49,7 @@ const _eventLoopBlockIntegration = ((options: Partial { + stop: () => { if (worker) { // eslint-disable-next-line @typescript-eslint/no-floating-promises worker.then(stop => { @@ -58,12 +58,11 @@ const _eventLoopBlockIntegration = ((options: Partial(callback: () => T | Promis return callback(); } - integration.stopWorker(); + integration.stop(); const result = callback(); if (isPromise(result)) { - return result.finally(() => integration.startWorker()); + return result.finally(() => integration.start()); } - integration.startWorker(); + integration.start(); return result; } diff --git a/packages/node-native/src/event-loop-block-watchdog.ts b/packages/node-native/src/event-loop-block-watchdog.ts index 7298717cea1e..9d6e87814714 100644 --- a/packages/node-native/src/event-loop-block-watchdog.ts +++ b/packages/node-native/src/event-loop-block-watchdog.ts @@ -24,24 +24,48 @@ const { dist, dsn, environment, - maxBlockedEvents, + maxEventsPerHour, release, sdkMetadata, staticTags: tags, tunnel, } = workerData as WorkerStartData; -const pollInterval = threshold / POLL_RATIO +const pollInterval = threshold / POLL_RATIO; const triggeredThreads = new Set(); -let sentAnrEvents = 0; function log(...msg: unknown[]): void { if (debug) { // eslint-disable-next-line no-console - console.log('[Sentry Blocked Watchdog]', ...msg); + console.log('[Sentry Block Event Loop Watchdog]', ...msg); } } +function createRateLimiter(maxEventsPerHour: number): () => boolean { + let currentHour = 0; + let currentCount = 0; + + return function isRateLimited(): boolean { + const hour = new Date().getHours(); + + if (hour !== currentHour) { + currentHour = hour; + currentCount = 0; + } + + if (currentCount >= maxEventsPerHour) { + if (currentCount === maxEventsPerHour) { + currentCount += 1; + log(`Rate limit reached: ${currentCount} events in this hour`); + } + return true; + } + + currentCount += 1; + return false; + }; +} + const url = getEnvelopeEndpointWithUrlEncodedAuth(dsn, tunnel, sdkMetadata.sdk); const transport = makeNodeTransport({ url, @@ -49,6 +73,7 @@ const transport = makeNodeTransport({ // }, }); +const isRateLimited = createRateLimiter(maxEventsPerHour); async function sendAbnormalSession(serializedSession: Session | undefined): Promise { if (!serializedSession) { @@ -193,12 +218,10 @@ function getExceptionAndThreads( } async function sendAnrEvent(crashedThreadId: string): Promise { - if (sentAnrEvents >= maxBlockedEvents) { + if (isRateLimited()) { return; } - sentAnrEvents += 1; - const threads = captureStackTrace(); const crashedThread = threads[crashedThreadId]; @@ -235,14 +258,6 @@ async function sendAnrEvent(crashedThreadId: string): Promise { await transport.send(envelope); await transport.flush(2000); - - if (sentAnrEvents >= maxBlockedEvents) { - // Delay for 5 seconds so that stdio can flush if the main event loop ever restarts. - // This is mainly for the benefit of logging or debugging. - setTimeout(() => { - process.exit(0); - }, 5_000); - } } setInterval(async () => { From 487e6677c6655fa2c46398b3d4eea332cb2529fb Mon Sep 17 00:00:00 2001 From: Tim Fish Date: Wed, 25 Jun 2025 04:16:55 +0200 Subject: [PATCH 14/16] PR review --- .../src/event-loop-block-integration.ts | 9 ++++++--- .../node-native/src/event-loop-block-watchdog.ts | 16 ++++++++++++---- 2 files changed, 18 insertions(+), 7 deletions(-) diff --git a/packages/node-native/src/event-loop-block-integration.ts b/packages/node-native/src/event-loop-block-integration.ts index 32e30d30d3d5..b2569aa948c4 100644 --- a/packages/node-native/src/event-loop-block-integration.ts +++ b/packages/node-native/src/event-loop-block-integration.ts @@ -73,7 +73,7 @@ type ThreadBlockedReturn = (options?: Partial) * Monitors the Node.js event loop for blocking behavior and reports blocked events to Sentry. * * Uses a background worker thread to detect when the main thread is blocked for longer than - * the configured threshold (default: 5 seconds). + * the configured threshold (default: 1 second). * * When instrumenting via the `--import` flag, this integration will * automatically monitor all worker threads as well. @@ -213,6 +213,9 @@ export function disableBlockedDetectionForCallback(callback: () => T | Promis return result.finally(() => integration.start()); } - integration.start(); - return result; + try { + return result; + } finally { + integration.start(); + } } diff --git a/packages/node-native/src/event-loop-block-watchdog.ts b/packages/node-native/src/event-loop-block-watchdog.ts index 9d6e87814714..8909c00d1ea7 100644 --- a/packages/node-native/src/event-loop-block-watchdog.ts +++ b/packages/node-native/src/event-loop-block-watchdog.ts @@ -217,7 +217,7 @@ function getExceptionAndThreads( }; } -async function sendAnrEvent(crashedThreadId: string): Promise { +async function sendBlockEvent(crashedThreadId: string): Promise { if (isRateLimited()) { return; } @@ -230,7 +230,11 @@ async function sendAnrEvent(crashedThreadId: string): Promise { return; } - await sendAbnormalSession(crashedThread.state?.session); + try { + await sendAbnormalSession(crashedThread.state?.session); + } catch (error) { + log(`Failed to send abnormal session for thread '${crashedThreadId}':`, error); + } log('Sending event'); @@ -267,10 +271,14 @@ setInterval(async () => { continue; } - log(`Detected ANR for thread '${threadId}' with last seen time ${time} ms`); + log(`Blocked thread detected '${threadId}' last polled ${time} ms ago.`); triggeredThreads.add(threadId); - await sendAnrEvent(threadId); + try { + await sendBlockEvent(threadId); + } catch (error) { + log(`Failed to send event for thread '${threadId}':`, error); + } } else { triggeredThreads.delete(threadId); } From 7fb37dc1603def8e8d105b6a6af19dfda6f8f702 Mon Sep 17 00:00:00 2001 From: Tim Fish Date: Thu, 26 Jun 2025 17:23:56 +0200 Subject: [PATCH 15/16] Remove `disableBlockedDetectionForCallback` for now --- .../src/event-loop-block-integration.ts | 70 +++---------------- packages/node-native/src/index.ts | 2 +- 2 files changed, 9 insertions(+), 63 deletions(-) diff --git a/packages/node-native/src/event-loop-block-integration.ts b/packages/node-native/src/event-loop-block-integration.ts index b2569aa948c4..6b643e944adf 100644 --- a/packages/node-native/src/event-loop-block-integration.ts +++ b/packages/node-native/src/event-loop-block-integration.ts @@ -1,14 +1,11 @@ -import { types } from 'node:util'; import { Worker } from 'node:worker_threads'; -import type { Contexts, Event, EventHint, Integration, IntegrationFn } from '@sentry/core'; -import { defineIntegration, getClient, getFilenameToDebugIdMap, getIsolationScope, logger } from '@sentry/core'; +import type { Contexts, Event, EventHint, IntegrationFn } from '@sentry/core'; +import { defineIntegration, getFilenameToDebugIdMap, getIsolationScope, logger } from '@sentry/core'; import type { NodeClient } from '@sentry/node'; import { registerThread, threadPoll } from '@sentry-internal/node-native-stacktrace'; import type { ThreadBlockedIntegrationOptions, WorkerStartData } from './common'; import { POLL_RATIO } from './common'; -const { isPromise } = types; - const DEFAULT_THRESHOLD_MS = 1_000; function log(message: string, ...args: unknown[]): void { @@ -32,43 +29,18 @@ async function getContexts(client: NodeClient): Promise { const INTEGRATION_NAME = 'ThreadBlocked'; -type ThreadBlockedInternal = { start: () => void; stop: () => void }; - const _eventLoopBlockIntegration = ((options: Partial = {}) => { - let worker: Promise<() => void> | undefined; - let client: NodeClient | undefined; - return { name: INTEGRATION_NAME, - start: () => { - if (worker) { - return; - } - - if (client) { - worker = _startWorker(client, options); - } - }, - stop: () => { - if (worker) { - // eslint-disable-next-line @typescript-eslint/no-floating-promises - worker.then(stop => { - stop(); - worker = undefined; - }); - } - }, - afterAllSetup(initClient: NodeClient) { - client = initClient; - + afterAllSetup(client: NodeClient) { registerThread(); - this.start(); + _startWorker(client, options).catch(err => { + log('Failed to start event loop block worker', err); + }); }, - } as Integration & ThreadBlockedInternal; + }; }) satisfies IntegrationFn; -type ThreadBlockedReturn = (options?: Partial) => Integration & ThreadBlockedInternal; - /** * Monitors the Node.js event loop for blocking behavior and reports blocked events to Sentry. * @@ -98,7 +70,7 @@ type ThreadBlockedReturn = (options?: Partial) * node --import instrument.mjs app.mjs * ``` */ -export const eventLoopBlockIntegration = defineIntegration(_eventLoopBlockIntegration) as ThreadBlockedReturn; +export const eventLoopBlockIntegration = defineIntegration(_eventLoopBlockIntegration); /** * Starts the worker thread @@ -193,29 +165,3 @@ async function _startWorker( clearInterval(timer); }; } - -export function disableBlockedDetectionForCallback(callback: () => T): T; -export function disableBlockedDetectionForCallback(callback: () => Promise): Promise; -/** - * Disables blocked detection for the duration of the callback - */ -export function disableBlockedDetectionForCallback(callback: () => T | Promise): T | Promise { - const integration = getClient()?.getIntegrationByName(INTEGRATION_NAME) as ThreadBlockedInternal | undefined; - - if (!integration) { - return callback(); - } - - integration.stop(); - - const result = callback(); - if (isPromise(result)) { - return result.finally(() => integration.start()); - } - - try { - return result; - } finally { - integration.start(); - } -} diff --git a/packages/node-native/src/index.ts b/packages/node-native/src/index.ts index b992ec6e1e6c..454be4eb8ad2 100644 --- a/packages/node-native/src/index.ts +++ b/packages/node-native/src/index.ts @@ -1 +1 @@ -export { eventLoopBlockIntegration, disableBlockedDetectionForCallback } from './event-loop-block-integration'; +export { eventLoopBlockIntegration } from './event-loop-block-integration'; From c3b9780c2f17589764a12a0a3a3b52496967434a Mon Sep 17 00:00:00 2001 From: Tim Fish Date: Sat, 28 Jun 2025 16:52:01 +0200 Subject: [PATCH 16/16] remove unused tests --- .../thread-blocked-native/stop-and-start.js | 45 ------------------- .../suites/thread-blocked-native/test.ts | 8 ---- 2 files changed, 53 deletions(-) delete mode 100644 dev-packages/node-integration-tests/suites/thread-blocked-native/stop-and-start.js diff --git a/dev-packages/node-integration-tests/suites/thread-blocked-native/stop-and-start.js b/dev-packages/node-integration-tests/suites/thread-blocked-native/stop-and-start.js deleted file mode 100644 index 5c797cd20c60..000000000000 --- a/dev-packages/node-integration-tests/suites/thread-blocked-native/stop-and-start.js +++ /dev/null @@ -1,45 +0,0 @@ -const Sentry = require('@sentry/node'); -const { eventLoopBlockIntegration } = require('@sentry/node-native'); -const { longWork } = require('./long-work.js'); -const crypto = require('crypto'); -const assert = require('assert'); - -setTimeout(() => { - process.exit(); -}, 10000); - -const threadBlocked = eventLoopBlockIntegration(); - -Sentry.init({ - dsn: process.env.SENTRY_DSN, - release: '1.0', - debug: true, - integrations: [threadBlocked], -}); - -Sentry.setUser({ email: 'person@home.com' }); -Sentry.addBreadcrumb({ message: 'important message!' }); - -function longWorkIgnored() { - for (let i = 0; i < 20; i++) { - const salt = crypto.randomBytes(128).toString('base64'); - const hash = crypto.pbkdf2Sync('myPassword', salt, 10000, 512, 'sha512'); - assert.ok(hash); - } -} - -setTimeout(() => { - threadBlocked.stop(); - - setTimeout(() => { - longWorkIgnored(); - - setTimeout(() => { - threadBlocked.start(); - - setTimeout(() => { - longWork(); - }); - }, 2000); - }, 2000); -}, 2000); diff --git a/dev-packages/node-integration-tests/suites/thread-blocked-native/test.ts b/dev-packages/node-integration-tests/suites/thread-blocked-native/test.ts index a0c24894a19c..6798882015f1 100644 --- a/dev-packages/node-integration-tests/suites/thread-blocked-native/test.ts +++ b/dev-packages/node-integration-tests/suites/thread-blocked-native/test.ts @@ -197,12 +197,4 @@ describe('Thread Blocked Native', { timeout: 30_000 }, () => { .start() .completed(); }); - - test('watchdog can be stopped and restarted', async () => { - await createRunner(__dirname, 'stop-and-start.js') - .withMockSentryServer() - .expect({ event: ANR_EVENT }) - .start() - .completed(); - }); });