joh/alternative peacock

anycode/client/src/common/client.ts

@@ -9,6 +9,7 @@ import { CommonLanguageClient } from 'vscode-languageclient';
 import { SupportedLanguages } from './supportedLanguages';
 import TelemetryReporter from 'vscode-extension-telemetry';
 import type { InitOptions } from '../../../shared/common/initOptions';
+import { CustomMessages } from '../../../shared/common/messages';
 
 export interface LanguageClientFactory {
 	createLanguageClient(id: string, name: string, clientOptions: LanguageClientOptions): CommonLanguageClient;
@@ -181,7 +182,7 @@ async function _startServer(factory: LanguageClientFactory, context: vscode.Exte
 		log.appendLine(`[INDEX] using ${uris.length} of ${all.length} files for ${langPattern}`);
 
 		const t1 = performance.now();
-		await client.sendRequest('queue/init', uris.map(String));
+		await client.sendRequest(CustomMessages.QueueInit, uris.map(String));
 		/* __GDPR__
 			"init" : {
 				"numOfFiles" : { "classification": "SystemMetaData", "purpose": "FeatureInsight", "isMeasurement": true },
@@ -197,15 +198,39 @@ async function _startServer(factory: LanguageClientFactory, context: vscode.Exte
 			duration: performance.now() - t1,
 		});
 
+		// incremental indexing: per language we wait for the first document to appear
+		// and only then we starting indexing all files matching the language. this is 
+		// done with the "unleash" message
+		const suffixesByLangId = new Map<string, string[]>();
+		for (const [lang] of supportedLanguages) {
+			suffixesByLangId.set(lang.languageId, lang.suffixes);
+		}
+		const handleTextDocument = (doc: vscode.TextDocument) => {
+			const suffixes = suffixesByLangId.get(doc.languageId);
+			if (!suffixes) {
+				return;
+			}
+			suffixesByLangId.delete(doc.languageId);
+			const initLang = client.sendRequest(CustomMessages.QueueUnleash, suffixes);
+
+			const initCancel = new Promise<void>(resolve => disposables.push(new vscode.Disposable(resolve)));
+			vscode.window.withProgress({ location: vscode.ProgressLocation.Window, title: `Updating Index for '${doc.languageId}'...` }, () => Promise.race([initLang, initCancel]));
+
+			if (suffixesByLangId.size === 0) {
+				listener.dispose();
+			}
+		};
+		const listener = vscode.workspace.onDidOpenTextDocument(handleTextDocument);
+		disposables.push(listener);
+		vscode.workspace.textDocuments.forEach(handleTextDocument);
+
 		// show status/maybe notifications
 		_showStatusAndInfo(documentSelector, !hasWorkspaceContents && _isRemoteHubWorkspace(), disposables);
 	}));
-	// stop on server-end
-	const initCancel = new Promise<void>(resolve => disposables.push(new vscode.Disposable(resolve)));
-	vscode.window.withProgress({ location: vscode.ProgressLocation.Window, title: 'Building Index...' }, () => Promise.race([init, initCancel]));
+
 
 	// serve fileRead request
-	client.onRequest('file/read', async (raw: string): Promise<number[]> => {
+	client.onRequest(CustomMessages.FileRead, async (raw: string): Promise<number[]> => {
 		const uri = vscode.Uri.parse(raw);
 
 		if (uri.scheme === 'vscode-notebook-cell') {

anycode/server/src/common/documentStore.ts

@@ -6,6 +6,7 @@
 import * as lsp from 'vscode-languageserver';
 import { TextDocuments } from 'vscode-languageserver';
 import { TextDocument } from 'vscode-languageserver-textdocument';
+import { CustomMessages } from '../../../shared/common/messages';
 import Languages from './languages';
 import { LRUMap } from './util/lruMap';
 
@@ -78,7 +79,7 @@ export class DocumentStore extends TextDocuments<TextDocument> {
 	}
 
 	private async _requestDocument(uri: string): Promise<TextDocument> {
-		const reply = await this._connection.sendRequest<number[]>('file/read', uri);
+		const reply = await this._connection.sendRequest<number[]>(CustomMessages.FileRead, uri);
 		const bytes = new Uint8Array(reply);
 		return TextDocument.create(uri, Languages.getLanguageIdByUri(uri), 1, this._decoder.decode(bytes));
 	}

anycode/server/src/common/features/symbolIndex.ts

@@ -26,23 +26,19 @@ class Queue {
 		this._queue.delete(uri);
 	}
 
-	consume(n?: number): string[] {
+	consume(n: number | undefined, filter: (uri: string) => boolean): string[] {
 		if (n === undefined) {
-			const result = Array.from(this._queue.values());
-			this._queue.clear();
-			return result;
+			n = this._queue.size;
 		}
-
 		const result: string[] = [];
-		const iter = this._queue.values();
-		for (; n > 0; n--) {
-			const r = iter.next();
-			if (r.done) {
-				break;
+		for (const uri of this._queue) {
+			if (!filter(uri)) {
+				continue;
 			}
-			const uri = r.value;
-			result.push(uri);
 			this._queue.delete(uri);
+			if (result.push(uri) >= n) {
+				break;
+			}
 		}
 		return result;
 	}
@@ -134,12 +130,34 @@ class Index {
 	}
 }
 
+class SuffixFilter {
+
+	private _suffixes = new Set<string>();
+	private _regex?: RegExp;
+
+	accept(uri: string) {
+		return Boolean(this._regex?.test(uri));
+	}
+
+	update(suffixes: string[]) {
+		for (const item of suffixes) {
+			this._suffixes.add(item);
+		}
+		this._regex = new RegExp(`\\.(${Array.from(this._suffixes).map(SuffixFilter._escapeRegExpCharacters).join('|')})`, 'i');
+	}
+
+	private static _escapeRegExpCharacters(value: string): string {
+		return value.replace(/[\\\{\}\*\+\?\|\^\$\.\[\]\(\)]/g, '\\$&');
+	}
+}
+
 export class SymbolIndex {
 
 	readonly index = new Index();
 
 	private readonly _syncQueue = new Queue();
 	private readonly _asyncQueue = new Queue();
+	private readonly _suffixFilter = new SuffixFilter();
 
 	constructor(
 		private readonly _trees: Trees,
@@ -162,11 +180,12 @@ export class SymbolIndex {
 
 	async update(): Promise<void> {
 		await this._currentUpdate;
-		this._currentUpdate = this._doUpdate(this._syncQueue.consume());
+		const uris = this._syncQueue.consume(undefined, uri => this._suffixFilter.accept(uri));
+		this._currentUpdate = this._doUpdate(uris, false);
 		return this._currentUpdate;
 	}
 
-	private async _doUpdate(uris: string[], silent?: boolean): Promise<void> {
+	private async _doUpdate(uris: string[], async: boolean): Promise<void> {
 		if (uris.length !== 0) {
 
 			// schedule a new task to update the cache for changed uris
@@ -181,9 +200,7 @@ export class SymbolIndex {
 				totalIndex += stat.durationIndex;
 			}
 
-			if (!silent) {
-				console.log(`[index] added ${uris.length} files ${sw.elapsed()}ms\n\tretrieval: ${Math.round(totalRetrieve)}ms\n\tindexing: ${Math.round(totalIndex)}ms`);
-			}
+			console.log(`[index] (${async ? 'async' : 'sync'}) added ${uris.length} files ${sw.elapsed()}ms (retrieval: ${Math.round(totalRetrieve)}ms, indexing: ${Math.round(totalIndex)}ms) (files: ${uris.map(String)})`);
 		}
 	}
 
@@ -249,13 +266,13 @@ export class SymbolIndex {
 		const uris = new Set(_uris);
 		const sw = new StopWatch();
 
-		console.log(`[index] building index for ${uris.size} files.`);
+		console.log(`[index] initializing index for ${uris.size} files.`);
 		const persisted = await this._symbolInfoStorage.getAll();
 		const obsolete = new Set<string>();
 
 		for (const [uri, data] of persisted) {
 			if (!uris.delete(uri)) {
-				// this file isn't requested anymore, remove later
+				// this file isn't requested anymore -> remove later
 				obsolete.add(uri);
 
 			} else {
@@ -265,20 +282,30 @@ export class SymbolIndex {
 				this._asyncQueue.enqueue(uri);
 			}
 		}
-		console.log(`[index] added FROM CACHE ${persisted.size} files ${sw.elapsed()}ms\n\t${uris.size} files still need to be fetched\n\t${obsolete.size} files are obsolete in cache`);
 
-		// sync update all files that were not cached
-		uris.forEach(this.addFile, this);
-		await this.update();
+		for (const uri of uris) {
+			// this file wasn't seen yet -> add now
+			this.addFile(uri);
+		}
 
 		// remove from persisted cache files that aren't interesting anymore 
 		await this._symbolInfoStorage.delete(obsolete);
 
+		console.log(`[index] added FROM CACHE ${persisted.size} files ${sw.elapsed()}ms, all need revalidation, ${uris.size} files are NEW, ${obsolete.size} where OBSOLETE`);
+	}
+
+	async unleashFiles(suffixes: string[]) {
+
+		console.log(`[index] unleashed files matching: ${suffixes.join(',')}`);
+
+		this._suffixFilter.update(suffixes);
+
+		await this.update();
+
 		// async update all files that were taken from cache
 		const asyncUpdate = async () => {
-			const uris = this._asyncQueue.consume(70);
+			const uris = this._asyncQueue.consume(70, uri => this._suffixFilter.accept(uri));
 			if (uris.length === 0) {
-				console.log('[index] ASYNC update is done');
 				return;
 			}
 			const t1 = performance.now();

anycode/server/src/common/server.ts

@@ -6,6 +6,7 @@
 import { Connection, FileChangeType, InitializeParams, InitializeResult, TextDocumentSyncKind } from 'vscode-languageserver';
 import Parser from 'web-tree-sitter';
 import type { InitOptions } from '../../../shared/common/initOptions';
+import { CustomMessages } from '../../../shared/common/messages';
 import { DocumentStore } from './documentStore';
 import { CompletionItemProvider } from './features/completions';
 import { DefinitionProvider } from './features/definitions';
@@ -71,7 +72,14 @@ export function startServer(connection: Connection, factory: IStorageFactory) {
 		documents.all().forEach(doc => symbolIndex.addFile(doc.uri));
 		documents.onDidOpen(event => symbolIndex.addFile(event.document.uri));
 		documents.onDidChangeContent(event => symbolIndex.addFile(event.document.uri));
-		connection.onRequest('queue/init', uris => symbolIndex.initFiles(uris));
+
+		connection.onRequest(CustomMessages.QueueInit, uris => {
+			symbolIndex.initFiles(uris);
+		});
+
+		connection.onRequest(CustomMessages.QueueUnleash, suffixes => {
+			symbolIndex.unleashFiles(suffixes);
+		});
 
 		connection.onDidChangeWatchedFiles(e => {
 			for (const { type, uri } of e.changes) {
@@ -91,8 +99,6 @@ export function startServer(connection: Connection, factory: IStorageFactory) {
 			}
 		});
 
-		console.log('Tree-sitter, languages, and features are READY');
-
 		return {
 			capabilities: { textDocumentSync: TextDocumentSyncKind.Incremental }
 		};

anycode/shared/common/messages.ts

@@ -0,0 +1,10 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+export const enum CustomMessages {
+	QueueInit = 'queue/init',
+	QueueUnleash = 'queue/unleash',
+	FileRead = 'file/read',
+}