Provide encoding-related APIs for editor extensions

extensions/vscode-api-tests/package.json

@@ -44,6 +44,7 @@
     "terminalDataWriteEvent",
     "terminalDimensions",
     "testObserver",
+    "textDocumentEncoding",
     "textSearchProvider",
     "timeline",
     "tokenInformation",

extensions/vscode-api-tests/src/singlefolder-tests/workspace.test.ts

@@ -619,7 +619,6 @@ suite('vscode API - workspace', () => {
 
 	test('findFiles2, exclude', () => {
 		return vscode.workspace.findFiles2(['**/image.png'], { exclude: ['**/sub/**'] }).then((res) => {
-			res.forEach(r => console.log(r.toString()));
 			assert.strictEqual(res.length, 1);
 		});
 	});
@@ -1305,4 +1304,64 @@ suite('vscode API - workspace', () => {
 		disposeAll(disposables);
 		return deleteFile(file);
 	}
+
+	test('text document encodings', async () => {
+		const uri1 = await createRandomFile();
+		const uri2 = await createRandomFile();
+
+		// --- events
+		const disposables: vscode.Disposable[] = [];
+		const onDidChangeTextDocument = new Set<vscode.TextDocument>();
+		disposables.push(vscode.workspace.onDidChangeTextDocument(e => {
+			onDidChangeTextDocument.add(e.document);
+		}));
+
+		// --- create with encoding
+
+		const doc1 = await vscode.workspace.openTextDocument(uri1);
+		assert.strictEqual(doc1.encoding, 'utf8');
+
+		const doc2 = await vscode.workspace.openTextDocument(uri2, { encoding: 'utf16le' });
+		assert.strictEqual(doc2.encoding, 'utf16le');
+
+		const doc3 = await vscode.workspace.openTextDocument({ encoding: 'utf16be' });
+		assert.strictEqual(doc3.encoding, 'utf16be');
+
+		const doc4 = await vscode.workspace.openTextDocument(vscode.Uri.parse('untitled://foo/bar'), { encoding: 'cp1252' });
+		assert.strictEqual(doc4.encoding, 'cp1252');
+
+		// --- encode (save with encoding)
+		onDidChangeTextDocument.clear();
+		let res = await doc1.save({ encoding: 'utf16le' });
+		assert.strictEqual(res, true);
+		assert.ok(Array.from(onDidChangeTextDocument).find(e => e.uri.toString() === doc1.uri.toString()), 'did Change: ' + doc1.uri.toString());
+		res = await doc1.save({ encoding: 'utf16le' });
+		assert.strictEqual(res, true);
+		assert.strictEqual(doc1.encoding, 'utf16le');
+
+		// --- decode (reopen with encoding)
+
+		const textDocumentChangeEvent = function (uri: vscode.Uri) {
+			return new Promise(resolve => {
+				vscode.workspace.onDidChangeTextDocument(e => {
+					if (e.document.uri.toString() === uri.toString()) {
+						resolve(e);
+					}
+				});
+			});
+		};
+
+		let event = textDocumentChangeEvent(uri2);
+		const doc2Reopened = await vscode.workspace.openTextDocument(uri2, { encoding: 'cp1252' });
+		assert.strictEqual(doc2, doc2Reopened);
+		assert.strictEqual(doc2.encoding, 'cp1252');
+		await event;
+
+		event = textDocumentChangeEvent(uri2);
+		await vscode.workspace.openTextDocument(uri2, { encoding: 'utf8' });
+		assert.strictEqual(doc2.encoding, 'utf8');
+		await event;
+
+		disposeAll(disposables);
+	});
 });

src/vs/platform/extensions/common/extensionsApiProposals.ts

@@ -361,6 +361,9 @@ const _allApiProposals = {
 	testRelatedCode: {
 		proposal: 'https://raw.githubusercontent.com/microsoft/vscode/main/src/vscode-dts/vscode.proposed.testRelatedCode.d.ts',
 	},
+	textDocumentEncoding: {
+		proposal: 'https://raw.githubusercontent.com/microsoft/vscode/main/src/vscode-dts/vscode.proposed.textDocumentEncoding.d.ts',
+	},
 	textEditorDiffInformation: {
 		proposal: 'https://raw.githubusercontent.com/microsoft/vscode/main/src/vscode-dts/vscode.proposed.textEditorDiffInformation.d.ts',
 	},

src/vs/workbench/api/browser/mainThreadDocuments.ts

@@ -12,7 +12,8 @@ import { IModelService } from '../../../editor/common/services/model.js';
 import { ITextModelService } from '../../../editor/common/services/resolverService.js';
 import { IFileService, FileOperation } from '../../../platform/files/common/files.js';
 import { ExtHostContext, ExtHostDocumentsShape, MainThreadDocumentsShape } from '../common/extHost.protocol.js';
-import { ITextFileService } from '../../services/textfile/common/textfiles.js';
+import { EncodingMode, ITextFileEditorModel, ITextFileService } from '../../services/textfile/common/textfiles.js';
+import { IUntitledTextEditorModel } from '../../services/untitled/common/untitledTextEditorModel.js';
 import { IWorkbenchEnvironmentService } from '../../services/environment/common/environmentService.js';
 import { toLocalResource, extUri, IExtUri } from '../../../base/common/resources.js';
 import { IWorkingCopyFileService } from '../../services/workingCopy/common/workingCopyFileService.js';
@@ -145,6 +146,14 @@ export class MainThreadDocuments extends Disposable implements MainThreadDocumen
 				this._proxy.$acceptDirtyStateChanged(m.resource, m.isDirty());
 			}
 		}));
+		this._store.add(Event.any<ITextFileEditorModel | IUntitledTextEditorModel>(_textFileService.files.onDidChangeEncoding, _textFileService.untitled.onDidChangeEncoding)(m => {
+			if (this._shouldHandleFileEvent(m.resource)) {
+				const encoding = m.getEncoding();
+				if (encoding) {
+					this._proxy.$acceptEncodingChanged(m.resource, encoding);
+				}
+			}
+		}));
 
 		this._store.add(workingCopyFileService.onDidRunWorkingCopyFileOperation(e => {
 			const isMove = e.operation === FileOperation.MOVE;
@@ -205,12 +214,23 @@ export class MainThreadDocuments extends Disposable implements MainThreadDocumen
 
 	// --- from extension host process
 
-	async $trySaveDocument(uri: UriComponents): Promise<boolean> {
-		const target = await this._textFileService.save(URI.revive(uri));
+	async $trySaveDocument(uri: UriComponents, options?: { encoding?: string }): Promise<boolean> {
+		const inputUri = URI.revive(uri);
+
+		if (options?.encoding) {
+			const model = uri.scheme === Schemas.untitled ? this._textFileService.untitled.get(inputUri) : this._textFileService.files.get(inputUri);
+			const result = await model?.setEncoding(options.encoding, EncodingMode.Encode);
+
+			if (uri.scheme !== Schemas.untitled) {
+				return Boolean(result); // non untitled files get saved right away
+			}
+		}
+
+		const target = await this._textFileService.save(inputUri);
 		return Boolean(target);
 	}
 
-	async $tryOpenDocument(uriData: UriComponents): Promise<URI> {
+	async $tryOpenDocument(uriData: UriComponents, options?: { encoding?: string }): Promise<URI> {
 		const inputUri = URI.revive(uriData);
 		if (!inputUri.scheme || !(inputUri.fsPath || inputUri.authority)) {
 			throw new ErrorNoTelemetry(`Invalid uri. Scheme and authority or path must be set.`);
@@ -221,11 +241,11 @@ export class MainThreadDocuments extends Disposable implements MainThreadDocumen
 		let promise: Promise<URI>;
 		switch (canonicalUri.scheme) {
 			case Schemas.untitled:
-				promise = this._handleUntitledScheme(canonicalUri);
+				promise = this._handleUntitledScheme(canonicalUri, options);
 				break;
 			case Schemas.file:
 			default:
-				promise = this._handleAsResourceInput(canonicalUri);
+				promise = this._handleAsResourceInput(canonicalUri, options);
 				break;
 		}
 
@@ -246,31 +266,37 @@ export class MainThreadDocuments extends Disposable implements MainThreadDocumen
 		}
 	}
 
-	$tryCreateDocument(options?: { language?: string; content?: string }): Promise<URI> {
-		return this._doCreateUntitled(undefined, options ? options.language : undefined, options ? options.content : undefined);
+	$tryCreateDocument(options?: { language?: string; content?: string; encoding?: string }): Promise<URI> {
+		return this._doCreateUntitled(undefined, options);
 	}
 
-	private async _handleAsResourceInput(uri: URI): Promise<URI> {
+	private async _handleAsResourceInput(uri: URI, options?: { encoding?: string }): Promise<URI> {
+		if (options?.encoding) {
+			const model = await this._textFileService.files.resolve(uri);
+			await model.setEncoding(options.encoding, EncodingMode.Decode);
+		}
+
 		const ref = await this._textModelResolverService.createModelReference(uri);
 		this._modelReferenceCollection.add(uri, ref, ref.object.textEditorModel.getValueLength());
 		return ref.object.textEditorModel.uri;
 	}
 
-	private async _handleUntitledScheme(uri: URI): Promise<URI> {
+	private async _handleUntitledScheme(uri: URI, options?: { encoding?: string }): Promise<URI> {
 		const asLocalUri = toLocalResource(uri, this._environmentService.remoteAuthority, this._pathService.defaultUriScheme);
 		const exists = await this._fileService.exists(asLocalUri);
 		if (exists) {
 			// don't create a new file ontop of an existing file
 			return Promise.reject(new Error('file already exists'));
 		}
-		return await this._doCreateUntitled(Boolean(uri.path) ? uri : undefined);
+		return await this._doCreateUntitled(Boolean(uri.path) ? uri : undefined, options);
 	}
 
-	private async _doCreateUntitled(associatedResource?: URI, languageId?: string, initialValue?: string): Promise<URI> {
+	private async _doCreateUntitled(associatedResource?: URI, options?: { language?: string; content?: string; encoding?: string }): Promise<URI> {
 		const model = this._textFileService.untitled.create({
 			associatedResource,
-			languageId,
-			initialValue
+			languageId: options?.language,
+			initialValue: options?.content,
+			encoding: options?.encoding
 		});
 		const resource = model.resource;
 		const ref = await this._textModelResolverService.createModelReference(resource);

src/vs/workbench/api/browser/mainThreadDocumentsAndEditors.ts

@@ -386,7 +386,8 @@ export class MainThreadDocumentsAndEditors {
 			lines: model.getLinesContent(),
 			EOL: model.getEOL(),
 			languageId: model.getLanguageId(),
-			isDirty: this._textFileService.isDirty(model.uri)
+			isDirty: this._textFileService.isDirty(model.uri),
+			encoding: this._textFileService.getEncoding(model.uri)
 		};
 	}
 

src/vs/workbench/api/common/extHost.api.impl.ts

@@ -1024,10 +1024,14 @@ export function createApiFactoryAndRegisterActors(accessor: ServicesAccessor): I
 			set textDocuments(value) {
 				throw new errors.ReadonlyError('textDocuments');
 			},
-			openTextDocument(uriOrFileNameOrOptions?: vscode.Uri | string | { language?: string; content?: string }) {
+			openTextDocument(uriOrFileNameOrOptions?: vscode.Uri | string | { language?: string; content?: string; encoding?: string }, options?: { encoding?: string }) {
 				let uriPromise: Thenable<URI>;
 
-				const options = uriOrFileNameOrOptions as { language?: string; content?: string };
+				options = (options ?? uriOrFileNameOrOptions) as ({ language?: string; content?: string; encoding?: string } | undefined);
+				if (typeof options?.encoding === 'string') {
+					checkProposedApiEnabled(extension, 'textDocumentEncoding');
+				}
+
 				if (typeof uriOrFileNameOrOptions === 'string') {
 					uriPromise = Promise.resolve(URI.file(uriOrFileNameOrOptions));
 				} else if (URI.isUri(uriOrFileNameOrOptions)) {
@@ -1043,7 +1047,7 @@ export function createApiFactoryAndRegisterActors(accessor: ServicesAccessor): I
 					if (uri.scheme === Schemas.vscodeRemote && !uri.authority) {
 						extHostApiDeprecation.report('workspace.openTextDocument', extension, `A URI of 'vscode-remote' scheme requires an authority.`);
 					}
-					return extHostDocuments.ensureDocumentData(uri).then(documentData => {
+					return extHostDocuments.ensureDocumentData(uri, options).then(documentData => {
 						return documentData.document;
 					});
 				});

src/vs/workbench/api/common/extHost.protocol.ts

@@ -237,9 +237,9 @@ export interface MainThreadDocumentContentProvidersShape extends IDisposable {
 }
 
 export interface MainThreadDocumentsShape extends IDisposable {
-	$tryCreateDocument(options?: { language?: string; content?: string }): Promise<UriComponents>;
-	$tryOpenDocument(uri: UriComponents): Promise<UriComponents>;
-	$trySaveDocument(uri: UriComponents): Promise<boolean>;
+	$tryCreateDocument(options?: { language?: string; content?: string; encoding?: string }): Promise<UriComponents>;
+	$tryOpenDocument(uri: UriComponents, options?: { encoding?: string }): Promise<UriComponents>;
+	$trySaveDocument(uri: UriComponents, options?: { encoding?: string }): Promise<boolean>;
 }
 
 export interface ITextEditorConfigurationUpdate {
@@ -1811,11 +1811,13 @@ export interface IModelAddedData {
 	EOL: string;
 	languageId: string;
 	isDirty: boolean;
+	encoding: string;
 }
 export interface ExtHostDocumentsShape {
 	$acceptModelLanguageChanged(strURL: UriComponents, newLanguageId: string): void;
 	$acceptModelSaved(strURL: UriComponents): void;
 	$acceptDirtyStateChanged(strURL: UriComponents, isDirty: boolean): void;
+	$acceptEncodingChanged(strURL: UriComponents, encoding: string): void;
 	$acceptModelChanged(strURL: UriComponents, e: IModelChangedEvent, isDirty: boolean): void;
 }
 

src/vs/workbench/api/common/extHostDocumentData.ts

@@ -37,6 +37,7 @@ export class ExtHostDocumentData extends MirrorTextModel {
 		uri: URI, lines: string[], eol: string, versionId: number,
 		private _languageId: string,
 		private _isDirty: boolean,
+		private _encoding: string
 	) {
 		super(uri, lines, eol, versionId);
 	}
@@ -66,7 +67,8 @@ export class ExtHostDocumentData extends MirrorTextModel {
 				get version() { return that._versionId; },
 				get isClosed() { return that._isDisposed; },
 				get isDirty() { return that._isDirty; },
-				save() { return that._save(); },
+				get encoding() { return that._encoding; },
+				save(options?: { encoding?: string }) { return that._save(options); },
 				getText(range?) { return range ? that._getTextInRange(range) : that.getText(); },
 				get eol() { return that._eol === '\n' ? EndOfLine.LF : EndOfLine.CRLF; },
 				get lineCount() { return that._lines.length; },
@@ -94,11 +96,16 @@ export class ExtHostDocumentData extends MirrorTextModel {
 		this._isDirty = isDirty;
 	}
 
-	private _save(): Promise<boolean> {
+	_acceptEncoding(encoding: string): void {
+		ok(!this._isDisposed);
+		this._encoding = encoding;
+	}
+
+	private _save(options?: { encoding?: string }): Promise<boolean> {
 		if (this._isDisposed) {
 			return Promise.reject(new Error('Document has been closed'));
 		}
-		return this._proxy.$trySaveDocument(this._uri);
+		return this._proxy.$trySaveDocument(this._uri, options);
 	}
 
 	private _getTextInRange(_range: vscode.Range): string {

src/vs/workbench/api/common/extHostDocuments.ts

@@ -76,16 +76,16 @@ export class ExtHostDocuments implements ExtHostDocumentsShape {
 		return data.document;
 	}
 
-	public ensureDocumentData(uri: URI): Promise<ExtHostDocumentData> {
+	public ensureDocumentData(uri: URI, options?: { encoding?: string }): Promise<ExtHostDocumentData> {
 
 		const cached = this._documentsAndEditors.getDocument(uri);
-		if (cached) {
+		if (cached && (!options?.encoding || cached.document.encoding === options.encoding)) {
 			return Promise.resolve(cached);
 		}
 
 		let promise = this._documentLoader.get(uri.toString());
 		if (!promise) {
-			promise = this._proxy.$tryOpenDocument(uri).then(uriData => {
+			promise = this._proxy.$tryOpenDocument(uri, options).then(uriData => {
 				this._documentLoader.delete(uri.toString());
 				const canonicalUri = URI.revive(uriData);
 				return assertIsDefined(this._documentsAndEditors.getDocument(canonicalUri));
@@ -99,7 +99,7 @@ export class ExtHostDocuments implements ExtHostDocumentsShape {
 		return promise;
 	}
 
-	public createDocumentData(options?: { language?: string; content?: string }): Promise<URI> {
+	public createDocumentData(options?: { language?: string; content?: string; encoding?: string }): Promise<URI> {
 		return this._proxy.$tryCreateDocument(options).then(data => URI.revive(data));
 	}
 
@@ -140,6 +140,20 @@ export class ExtHostDocuments implements ExtHostDocumentsShape {
 		});
 	}
 
+	public $acceptEncodingChanged(uriComponents: UriComponents, encoding: string): void {
+		const uri = URI.revive(uriComponents);
+		const data = this._documentsAndEditors.getDocument(uri);
+		if (!data) {
+			throw new Error('unknown document');
+		}
+		data._acceptEncoding(encoding);
+		this._onDidChangeDocument.fire({
+			document: data.document,
+			contentChanges: [],
+			reason: undefined
+		});
+	}
+
 	public $acceptModelChanged(uriComponents: UriComponents, events: IModelChangedEvent, isDirty: boolean): void {
 		const uri = URI.revive(uriComponents);
 		const data = this._documentsAndEditors.getDocument(uri);

src/vs/workbench/api/common/extHostDocumentsAndEditors.ts

@@ -97,6 +97,7 @@ export class ExtHostDocumentsAndEditors implements ExtHostDocumentsAndEditorsSha
 						data.versionId,
 						data.languageId,
 						data.isDirty,
+						data.encoding
 					));
 					this._documents.set(resource, ref);
 					addedDocuments.push(ref.value);

src/vs/workbench/api/common/extHostInteractive.ts

@@ -52,6 +52,7 @@ export class ExtHostInteractive implements ExtHostInteractiveShape {
 				uri: uri,
 				isDirty: false,
 				versionId: 1,
+				encoding: 'utf8'
 			}]
 		});
 	}

src/vs/workbench/api/common/extHostNotebookDocument.ts

@@ -40,7 +40,8 @@ export class ExtHostCell {
 			languageId: cell.language,
 			uri: cell.uri,
 			isDirty: false,
-			versionId: 1
+			versionId: 1,
+			encoding: 'utf8'
 		};
 	}
 

src/vs/workbench/api/test/browser/extHostApiCommands.test.ts

@@ -167,6 +167,7 @@ suite('ExtHostLanguageFeatureCommands', function () {
 				uri: model.uri,
 				lines: model.getValue().split(model.getEOL()),
 				EOL: model.getEOL(),
+				encoding: 'utf8'
 			}]
 		});
 		const extHostDocuments = new ExtHostDocuments(rpcProtocol, extHostDocumentsAndEditors);

src/vs/workbench/api/test/browser/extHostBulkEdits.test.ts

@@ -40,6 +40,7 @@ suite('ExtHostBulkEdits.applyWorkspaceEdit', () => {
 				versionId: 1337,
 				lines: ['foo'],
 				EOL: '\n',
+				encoding: 'utf8'
 			}]
 		});
 		bulkEdits = new ExtHostBulkEdits(rpcProtocol, documentsAndEditors);