Merge pull request #4519 from PerBothner/clusters

New unicode-graphemes addon.

Author: Tyriar

.eslintrc.json

@@ -30,6 +30,9 @@
       "addons/xterm-addon-serialize/benchmark/tsconfig.json",
       "addons/xterm-addon-unicode11/src/tsconfig.json",
       "addons/xterm-addon-unicode11/test/tsconfig.json",
+      "addons/xterm-addon-unicode-graphemes/src/tsconfig.json",
+      "addons/xterm-addon-unicode-graphemes/test/tsconfig.json",
+      "addons/xterm-addon-unicode-graphemes/benchmark/tsconfig.json",
       "addons/xterm-addon-web-links/src/tsconfig.json",
       "addons/xterm-addon-web-links/test/tsconfig.json",
       "addons/xterm-addon-webgl/src/tsconfig.json",
@@ -38,6 +41,7 @@
     "sourceType": "module"
   },
   "ignorePatterns": [
+    "addons/*/src/third-party/*.ts",
     "**/inwasm-sdks/*",
     "**/typings/*.d.ts",
     "**/node_modules",

.github/workflows/ci.yml

@@ -42,6 +42,8 @@ jobs:
           ./addons/xterm-addon-serialize/out-test/* \
           ./addons/xterm-addon-unicode11/out/* \
           ./addons/xterm-addon-unicode11/out-test/* \
+          ./addons/xterm-addon-unicode-graphemes/out/* \
+          ./addons/xterm-addon-unicode-graphemes/out-test/* \
           ./addons/xterm-addon-web-links/out/* \
           ./addons/xterm-addon-web-links/out-test/* \
           ./addons/xterm-addon-webgl/out/* \
@@ -68,6 +70,8 @@ jobs:
         yarn --frozen-lockfile
         yarn install-addons
     - name: Lint code
+      env:
+        NODE_OPTIONS: --max_old_space_size=4096
       run: yarn lint
     - name: Lint API
       run: yarn lint-api

addons/xterm-addon-unicode-graphemes/.gitignore

@@ -0,0 +1,3 @@
+lib
+node_modules
+out-benchmark

addons/xterm-addon-unicode-graphemes/.npmignore

@@ -0,0 +1,29 @@
+# Blacklist - exclude everything except npm defaults such as LICENSE, etc
+*
+!*/
+
+# Whitelist - lib/
+!lib/**/*.d.ts
+
+!lib/**/*.js
+!lib/**/*.js.map
+
+!lib/**/*.css
+
+# Whitelist - src/
+!src/**/*.ts
+!src/**/*.d.ts
+
+!src/**/*.js
+!src/**/*.js.map
+
+!src/**/*.css
+
+# Blacklist - src/ test files
+src/**/*.test.ts
+src/**/*.test.d.ts
+src/**/*.test.js
+src/**/*.test.js.map
+
+# Whitelist - typings/
+!typings/*.d.ts

addons/xterm-addon-unicode-graphemes/LICENSE

@@ -0,0 +1,19 @@
+Copyright (c) 2023, The xterm.js authors (https://github.com/xtermjs/xterm.js)
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

addons/xterm-addon-unicode-graphemes/README.md

@@ -0,0 +1,24 @@
+## xterm-addon-unicode-graphemes
+
+⚠️ **This addon is currently experimental and may introduce unexpected and non-standard behavior**
+
+An addon providing enhanced Unicode support (include grapheme clustering) for xterm.js.
+
+The file `src/UnicodeProperties.ts` is generated and depends on the Unicode version. See [the unicode-properties project](https://github.com/PerBothner/unicode-properties) for credits and re-generation instructions.
+
+### Install
+
+```bash
+npm install --save xterm-addon-unicode-graphemes
+```
+
+### Usage
+
+```ts
+import { Terminal } from 'xterm';
+import { UnicodeGraphemeAddon } from 'xterm-addon-unicode-graphemes';
+
+const terminal = new Terminal();
+const unicodeGraphemeAddon = new UnicodeGraphemeAddon();
+terminal.loadAddon(unicodeGraphemeAddon);
+```

addons/xterm-addon-unicode-graphemes/benchmark/UnicodeGraphemeAddon.benchmark.ts

@@ -0,0 +1,78 @@
+/**
+ * Copyright (c) 2019 The xterm.js authors. All rights reserved.
+ * @license MIT
+ */
+
+import { perfContext, before, ThroughputRuntimeCase } from 'xterm-benchmark';
+
+import { spawn } from 'node-pty';
+import { Utf8ToUtf32, stringFromCodePoint } from 'common/input/TextDecoder';
+import { Terminal } from 'browser/Terminal';
+import { UnicodeGraphemeProvider } from 'UnicodeGraphemeProvider';
+
+
+function fakedAddonLoad(terminal: any): void {
+  // resembles what UnicodeGraphemesAddon.activate does under the hood
+  terminal.unicodeService.register(new UnicodeGraphemeProvider());
+  terminal.unicodeService.activeVersion = '15-graphemes';
+}
+
+
+perfContext('Terminal: ls -lR /usr/lib', () => {
+  let content = '';
+  let contentUtf8: Uint8Array;
+
+  before(async () => {
+    // grab output from "ls -lR /usr"
+    const p = spawn('ls', ['--color=auto', '-lR', '/usr/lib'], {
+      name: 'xterm-256color',
+      cols: 80,
+      rows: 25,
+      cwd: process.env.HOME,
+      env: process.env,
+      encoding: (null as unknown as string) // needs to be fixed in node-pty
+    });
+    const chunks: Buffer[] = [];
+    let length = 0;
+    p.on('data', data => {
+      chunks.push(data as unknown as Buffer);
+      length += data.length;
+    });
+    await new Promise<void>(resolve => p.on('exit', () => resolve()));
+    contentUtf8 = Buffer.concat(chunks, length);
+    // translate to content string
+    const buffer = new Uint32Array(contentUtf8.length);
+    const decoder = new Utf8ToUtf32();
+    const codepoints = decoder.decode(contentUtf8, buffer);
+    for (let i = 0; i < codepoints; ++i) {
+      content += stringFromCodePoint(buffer[i]);
+      // peek into content to force flat repr in v8
+      if (!(i % 10000000)) {
+        content[i];
+      }
+    }
+  });
+
+  perfContext('write/string/async', () => {
+    let terminal: Terminal;
+    before(() => {
+      terminal = new Terminal({ cols: 80, rows: 25, scrollback: 1000 });
+      fakedAddonLoad(terminal);
+    });
+    new ThroughputRuntimeCase('', async () => {
+      await new Promise<void>(res => terminal.write(content, res));
+      return { payloadSize: contentUtf8.length };
+    }, { fork: false }).showAverageThroughput();
+  });
+
+  perfContext('write/Utf8/async', () => {
+    let terminal: Terminal;
+    before(() => {
+      terminal = new Terminal({ cols: 80, rows: 25, scrollback: 1000 });
+    });
+    new ThroughputRuntimeCase('', async () => {
+      await new Promise<void>(res => terminal.write(content, res));
+      return { payloadSize: contentUtf8.length };
+    }, { fork: false }).showAverageThroughput();
+  });
+});

addons/xterm-addon-unicode-graphemes/benchmark/benchmark.json

@@ -0,0 +1,19 @@
+{
+  "APP_PATH": ".benchmark",
+  "evalConfig": {
+    "tolerance": {
+      "*": [0.75, 1.5],
+      "*.dev": [0.01, 1.5],
+      "*.cv": [0.01, 1.5],
+      "EscapeSequenceParser.benchmark.js.*.averageThroughput.mean": [0.9, 5]
+    },
+    "skip": [
+      "*.median",
+      "*.runs",
+      "*.dev",
+      "*.cv",
+      "EscapeSequenceParser.benchmark.js.*.averageRuntime",
+      "Terminal.benchmark.js.*.averageRuntime"
+    ]
+  }
+}

addons/xterm-addon-unicode-graphemes/benchmark/tsconfig.json

@@ -0,0 +1,23 @@
+{
+  "compilerOptions": {
+    "lib": ["dom", "es6"],
+    "outDir": "../out-benchmark",
+    "types": ["../../../node_modules/@types/node"],
+    "moduleResolution": "node",
+    "strict": false,
+    "target": "es2015",
+    "module": "commonjs",
+    "baseUrl": ".",
+    "paths": {
+      "common/*": ["../../../src/common/*"],
+      "browser/*": ["../../../src/browser/*"],
+      "UnicodeGraphemeProvider": ["../src/UnicodeGraphemeProvider"]
+    }
+  },
+  "include": ["../**/*", "../../../typings/xterm.d.ts"],
+  "exclude": ["../../../**/*test.ts", "../../**/*api.ts"],
+  "references": [
+    { "path": "../../../src/common" },
+    { "path": "../../../src/browser" }
+  ]
+}

addons/xterm-addon-unicode-graphemes/package.json

@@ -0,0 +1,29 @@
+{
+  "name": "xterm-addon-unicode-graphemes",
+  "version": "0.1.0",
+  "author": {
+    "name": "The xterm.js authors",
+    "url": "https://xtermjs.org/"
+  },
+  "main": "lib/xterm-addon-unicode-graphemes.js",
+  "types": "typings/xterm-addon-unicode-graphemes.d.ts",
+  "repository": "https://github.com/xtermjs/xterm.js/tree/master/addons/xterm-addon-unicode-graphemes",
+  "license": "MIT",
+  "keywords": [
+    "terminal",
+    "xterm",
+    "xterm.js"
+  ],
+  "scripts": {
+    "build": "../../node_modules/.bin/tsc -p .",
+    "prepackage": "npm run build",
+    "package": "../../node_modules/.bin/webpack",
+    "prepublishOnly": "npm run package",
+    "benchmark": "NODE_PATH=../../out:./out:./out-benchmark/ ../../node_modules/.bin/xterm-benchmark -r 5 -c benchmark/benchmark.json out-benchmark/benchmark/*benchmark.js",
+    "benchmark-baseline": "NODE_PATH=../../out:./out:./out-benchmark/ ../../node_modules/.bin/xterm-benchmark -r 5 -c benchmark/benchmark.json --baseline out-benchmark/benchmark/*benchmark.js",
+    "benchmark-eval": "NODE_PATH=../../out:./out:./out-benchmark/ ../../node_modules/.bin/xterm-benchmark -r 5 -c benchmark/benchmark.json --eval out-benchmark/benchmark/*benchmark.js"
+  },
+  "peerDependencies": {
+    "xterm": "^5.0.0"
+  }
+}

addons/xterm-addon-unicode-graphemes/src/UnicodeGraphemeProvider.ts

@@ -0,0 +1,72 @@
+/**
+ * Copyright (c) 2023 The xterm.js authors. All rights reserved.
+ * @license MIT
+ */
+
+import { IUnicodeVersionProvider } from 'xterm';
+import { UnicodeCharProperties, UnicodeCharWidth } from 'common/services/Services';
+import { UnicodeService } from 'common/services/UnicodeService';
+import * as UC from './third-party/UnicodeProperties';
+
+export class UnicodeGraphemeProvider implements IUnicodeVersionProvider {
+  public readonly version;
+  public ambiguousCharsAreWide: boolean = false;
+  public readonly handleGraphemes: boolean;
+
+  constructor(handleGraphemes: boolean = true) {
+    this.version = handleGraphemes ? '15-graphemes' : '15';
+    this.handleGraphemes = handleGraphemes;
+  }
+
+  private static readonly _plainNarrowProperties: UnicodeCharProperties
+    = UnicodeService.createPropertyValue(UC.GRAPHEME_BREAK_Other, 1, false);
+
+  public charProperties(codepoint: number, preceding: UnicodeCharProperties): UnicodeCharProperties {
+    // Optimize the simple ASCII case, under the condition that
+    // UnicodeService.extractCharKind(preceding) === GRAPHEME_BREAK_Other
+    // (which also covers the case that preceding === 0).
+    if ((codepoint >= 32 && codepoint < 127) && (preceding >> 3) === 0) {
+      return UnicodeGraphemeProvider._plainNarrowProperties;
+    }
+
+    let charInfo = UC.getInfo(codepoint);
+    let w = UC.infoToWidthInfo(charInfo);
+    let shouldJoin = false;
+    if (w >= 2) {
+      // Treat emoji_presentation_selector as WIDE.
+      w = w === 3 || this.ambiguousCharsAreWide || codepoint === 0xfe0f ? 2 : 1;
+    } else {
+      w = 1;
+    }
+    if (preceding !== 0) {
+      const oldWidth = UnicodeService.extractWidth(preceding);
+      if (this.handleGraphemes) {
+        charInfo = UC.shouldJoin(UnicodeService.extractCharKind(preceding), charInfo);
+      } else {
+        charInfo = w === 0 ? 1 : 0;
+      }
+      shouldJoin = charInfo > 0;
+      if (shouldJoin) {
+        if (oldWidth > w) {
+          w = oldWidth;
+        } else if (charInfo === 32) { // UC.GRAPHEME_BREAK_SAW_Regional_Pair)
+          w = 2;
+        }
+      }
+    }
+    return UnicodeService.createPropertyValue(charInfo, w, shouldJoin);
+  }
+
+  public wcwidth(codepoint: number): UnicodeCharWidth {
+    const charInfo = UC.getInfo(codepoint);
+    const w = UC.infoToWidthInfo(charInfo);
+    const kind = (charInfo & UC.GRAPHEME_BREAK_MASK) >> UC.GRAPHEME_BREAK_SHIFT;
+    if (kind === UC.GRAPHEME_BREAK_Extend || kind === UC.GRAPHEME_BREAK_Prepend) {
+      return 0;
+    }
+    if (w >= 2 && (w === 3 || this.ambiguousCharsAreWide)) {
+      return 2;
+    }
+    return 1;
+  }
+}

addons/xterm-addon-unicode-graphemes/src/UnicodeGraphemesAddon.ts

@@ -0,0 +1,38 @@
+/**
+ * Copyright (c) 2023 The xterm.js authors. All rights reserved.
+ * @license MIT
+ *
+ * UnicodeVersionProvider for V15 with grapeme cluster handleing.
+ */
+
+import { Terminal, ITerminalAddon, IUnicodeHandling } from 'xterm';
+import { UnicodeGraphemeProvider } from './UnicodeGraphemeProvider';
+
+
+export class UnicodeGraphemesAddon implements ITerminalAddon {
+  private _provider15Graphemes?: UnicodeGraphemeProvider;
+  private _provider15?: UnicodeGraphemeProvider;
+  private _unicode?: IUnicodeHandling;
+  private _oldVersion: string = '';
+
+  public activate(terminal: Terminal): void {
+    if (! this._provider15) {
+      this._provider15 = new UnicodeGraphemeProvider(false);
+    }
+    if (! this._provider15Graphemes) {
+      this._provider15Graphemes = new UnicodeGraphemeProvider(true);
+    }
+    const unicode = terminal.unicode;
+    this._unicode = unicode;
+    unicode.register(this._provider15);
+    unicode.register(this._provider15Graphemes);
+    this._oldVersion = unicode.activeVersion;
+    unicode.activeVersion = '15-graphemes';
+  }
+
+  public dispose(): void {
+    if (this._unicode) {
+      this._unicode.activeVersion = this._oldVersion;
+    }
+  }
+}