From 34583845a6e13018fe676211a117e2b5da2250f3 Mon Sep 17 00:00:00 2001 From: Erik Demaine Date: Mon, 30 Dec 2024 17:54:40 -0500 Subject: [PATCH] SourceMap class, preserve by Civet worker --- civet.dev/public/playground.worker.js | 4 +- integration/eslint/source/index.civet | 2 +- lsp/source/lib/typescript-service.mts | 12 +- lsp/source/lib/util.mts | 2 +- lsp/test/util.civet | 8 +- source/cli.civet | 2 +- source/generate.civet | 16 +-- source/main.civet | 4 +- source/sourcemap.civet | 163 +++++++++++++------------- source/ts-diagnostic.civet | 5 +- source/unplugin/unplugin.civet | 4 +- source/worker-pool.civet | 5 + test/sourcemap.civet | 2 +- test/util/locations.civet | 2 +- types/types.d.ts | 13 +- 15 files changed, 126 insertions(+), 118 deletions(-) diff --git a/civet.dev/public/playground.worker.js b/civet.dev/public/playground.worker.js index 4efc7fff..f506ac26 100644 --- a/civet.dev/public/playground.worker.js +++ b/civet.dev/public/playground.worker.js @@ -17,7 +17,7 @@ onmessage = async (e) => { if (errors.length) { // Rerun with SourceMap to get error location errors = [] - tsCode = Civet.generate(ast, { errors, sourceMap: Civet.SourceMap(code) }); + tsCode = Civet.generate(ast, { errors, sourceMap: new Civet.SourceMap(code) }); error = errors[0] } } catch (e) { @@ -61,7 +61,7 @@ onmessage = async (e) => { if (errors.length) { // Rerun with SourceMap to get error location errors = [] - jsCode = Civet.generate(ast, { js: true, errors, sourceMap: Civet.SourceMap(code) }); + jsCode = Civet.generate(ast, { js: true, errors, sourceMap: new Civet.SourceMap(code) }); // Don't postError(errors[0]) here so that we still display TypeScript // transpilation; only show error when trying to Run code throw errors[0] diff --git a/integration/eslint/source/index.civet b/integration/eslint/source/index.civet index c8ba0a65..88573343 100644 --- a/integration/eslint/source/index.civet +++ b/integration/eslint/source/index.civet @@ -27,7 +27,7 @@ export function civet(options: Options = {js: true}): ESLint.Plugin // Length 1 corresponding to the return value of `preprocess` [messages] := _messages if sourceMap := sourceMaps.get filename - sourceMapLines := sourceMap.data.lines + sourceMapLines := sourceMap.lines ?? sourceMap.data.lines // older Civet for each message of messages if message.line? [message.line, message.column] = remap message.line, message.column diff --git a/lsp/source/lib/typescript-service.mts b/lsp/source/lib/typescript-service.mts index 0b833f3a..d14e5dfd 100644 --- a/lsp/source/lib/typescript-service.mts +++ b/lsp/source/lib/typescript-service.mts @@ -46,9 +46,8 @@ try { } interface SourceMap { - data: { - lines: CivetSourceMap["data"]["lines"] - } + lines: CivetSourceMap["lines"] + data: CivetSourceMap["data"] } // ts doesn't have this key in the type @@ -57,7 +56,7 @@ interface ResolvedModuleWithFailedLookupLocations extends ts.ResolvedModuleWithF } export interface FileMeta { - sourcemapLines: SourceMap["data"]["lines"] | undefined + sourcemapLines: SourceMap["lines"] | undefined transpiledDoc: TextDocument | undefined parseErrors: (Error | ParseError)[] | undefined fatal: boolean // whether errors were fatal during compilation, so no doc @@ -384,7 +383,7 @@ function TSHost(compilationSettings: CompilerOptions, initialFileNames: string[] return snapshot } - function createOrUpdateMeta(path: string, transpiledDoc: TextDocument, sourcemapLines?: SourceMap["data"]["lines"], parseErrors?: (Error | ParseError)[], fatal?: boolean) { + function createOrUpdateMeta(path: string, transpiledDoc: TextDocument, sourcemapLines?: SourceMap["lines"], parseErrors?: (Error | ParseError)[], fatal?: boolean) { let meta = fileMetaData.get(path) if (!meta) { @@ -415,7 +414,8 @@ function TSHost(compilationSettings: CompilerOptions, initialFileNames: string[] if (result) { const { code: transpiledCode, sourceMap, errors } = result - createOrUpdateMeta(sourcePath, transpiledDoc, sourceMap?.data.lines, errors, false) + const sourceMapLines = sourceMap?.lines ?? sourceMap?.data.lines // older Civet + createOrUpdateMeta(sourcePath, transpiledDoc, sourceMapLines, errors, false) TextDocument.update(transpiledDoc, [{ text: transpiledCode }], version) return transpiledCode diff --git a/lsp/source/lib/util.mts b/lsp/source/lib/util.mts index 55d8175d..77e3a3a1 100644 --- a/lsp/source/lib/util.mts +++ b/lsp/source/lib/util.mts @@ -24,7 +24,7 @@ import { remapRange, } from '@danielx/civet/ts-diagnostic'; -export type SourcemapLines = SourceMap['data']['lines']; +export type SourcemapLines = SourceMap['lines']; export { flattenDiagnosticMessageText, diff --git a/lsp/test/util.civet b/lsp/test/util.civet index 058cd712..8bce0b54 100644 --- a/lsp/test/util.civet +++ b/lsp/test/util.civet @@ -1,5 +1,5 @@ // TODO: figure out the magic ts-note/TypeScript config to make this work without destructuring from default import -{ intersectRanges, containsRange, makeRange, remapPosition, forwardMap, convertCoffeeScriptSourceMap } from ../source/lib/util.mjs +{ intersectRanges, containsRange, makeRange, remapPosition, forwardMap } from ../source/lib/util.mjs assert from assert Civet from @danielx/civet @@ -25,7 +25,7 @@ describe "util", -> sourceMap: true }) - linesMap := sourceMap.data.lines + linesMap := sourceMap.lines // console.log code, linesMap assert.deepEqual remapPosition({ @@ -52,7 +52,7 @@ describe "util", -> [0..6].forEach (i) -> srcColumn := i + 13 - pos := forwardMap(sourceMap.data.lines, {line: 0, character: srcColumn}) + pos := forwardMap(sourceMap.lines, {line: 0, character: srcColumn}) srcStr := src.slice(srcColumn, srcColumn + 7) assert.equal srcStr.replace(" ", "("), generatedLines[pos.line].slice(pos.character, pos.character + 7) @@ -71,7 +71,7 @@ describe "util", -> generatedLines := code.split("\n") [2..6].forEach (srcColumn, i) -> - pos := forwardMap(sourceMap.data.lines, {line: 1, character: srcColumn}) + pos := forwardMap(sourceMap.lines, {line: 1, character: srcColumn}) srcStr := srcLines[1].slice(srcColumn, srcColumn + 5 - i) generatedStr := generatedLines[pos.line].slice(pos.character, pos.character + 5 - i) assert.equal srcStr, generatedStr diff --git a/source/cli.civet b/source/cli.civet index a573ef7c..58e133a1 100644 --- a/source/cli.civet +++ b/source/cli.civet @@ -339,7 +339,7 @@ export function repl(args: string[], options: Options) if errors# // Rerun with sourceMap errors = [] - generate ast, { ...options, errors, sourceMap: SourceMap(input) } + generate ast, { ...options, errors, sourceMap: new SourceMap input } showError errors[0] return callback null, undefined diff --git a/source/generate.civet b/source/generate.civet index be9fb9df..b8de342a 100644 --- a/source/generate.civet +++ b/source/generate.civet @@ -5,7 +5,9 @@ type { ASTNode } from './parser/types.civet' export type Options = sourceMap?: undefined | updateSourceMap: (token: string, pos?: number) => void - data: { srcLine: number, srcColumn: number, srcOffset: number } + srcLine: number + srcColumn: number + srcOffset: number js?: boolean filename?: string errors?: ParseError[] @@ -17,7 +19,7 @@ function stringify(node: ASTNode): string return `${node}` function gen(root: ASTNode, options: Options): string - updateSourceMap := options?.sourceMap?.updateSourceMap + updateSourceMap := options?.sourceMap?@updateSourceMap return recurse root function recurse(node: ASTNode): string @@ -26,7 +28,7 @@ function gen(root: ASTNode, options: Options): string if node (src: string, options?: if options.sourceMap or options.inlineMap //@ts-ignore sourceMap option for generate - options.sourceMap = SourceMap(src) + options.sourceMap = new SourceMap src code := generate ast, options checkErrors() @@ -211,7 +211,7 @@ export function compile(src: string, options?: if options!.errors?.length delete options.errors //@ts-ignore sourceMap option for generate - options.sourceMap = SourceMap(src) + options.sourceMap = new SourceMap src generate ast, options checkErrors() diff --git a/source/sourcemap.civet b/source/sourcemap.civet index e9bf38ed..66ee7047 100644 --- a/source/sourcemap.civet +++ b/source/sourcemap.civet @@ -1,11 +1,11 @@ -type SourceMapEntries = ( +export type SourceMapEntries = ( [number, number, number, number, number] | [number, number, number, number] | [number] )[][] // Utility function to create a line/column lookup table for an input string -export locationTable = (input: string) -> +export function locationTable(input: string): number[] linesRe := /([^\r\n]*)(\r\n|\r|\n|$)/y lines := [] line .= 0 @@ -19,7 +19,7 @@ export locationTable = (input: string) -> return lines -export lookupLineColumn = (table: number[], pos: number) -> +export function lookupLineColumn(table: number[], pos: number) l .= 0 prevEnd .= 0 @@ -29,83 +29,84 @@ export lookupLineColumn = (table: number[], pos: number) -> // [line, column]; zero based return [l, pos - prevEnd] -export SourceMap = (sourceString: string) -> - srcTable := locationTable sourceString - - sm := - lines: [[]] as SourceMapEntries - line: 0 - colOffset: 0 // relative to previous entry - srcLine: 0 - srcColumn: 0 - srcOffset: 0 - srcTable: srcTable - - EOL := /\r?\n|\r/ - - return - data: sm - source: -> - sourceString - renderMappings: -> - lastSourceLine .= 0 - lastSourceColumn .= 0 - - sm.lines.map (line) => - line.map (entry) => - if entry.length is 4 - [colDelta, sourceFileIndex, srcLine, srcCol] .= entry - lineDelta := srcLine - lastSourceLine - colDelta = srcCol - lastSourceColumn - lastSourceLine = srcLine - lastSourceColumn = srcCol - `${encodeVlq(entry[0])}${encodeVlq(sourceFileIndex)}${encodeVlq(lineDelta)}${encodeVlq(colDelta)}` - else - encodeVlq entry[0] - .join(",") - .join(";") - - json: (srcFileName: string, outFileName: string) -> - version: 3 - file: outFileName - sources: [srcFileName] - mappings: @renderMappings() - names: [] - sourcesContent: [sourceString] - toString: -> - JSON.stringify this - - updateSourceMap: (outputStr: string, inputPos?: number, colOffset=0) -> - outLines := outputStr.split(EOL) - - let srcLine: number, srcCol: number +EOL := /\r?\n|\r/ +export class SourceMap + lines: SourceMapEntries + line: number + colOffset: number // relative to previous entry + srcLine: number + srcColumn: number + srcOffset: number + srcTable: number[] + + @(@source: string) + @lines = [[]] + @line = 0 + @colOffset = 0 // relative to previous entry + @srcLine = 0 + @srcColumn = 0 + @srcOffset = 0 + @srcTable = locationTable @source + + renderMappings(): string + lastSourceLine .= 0 + lastSourceColumn .= 0 + + for each line of @lines + for each entry of line + if entry.length is 4 + [colDelta, sourceFileIndex, srcLine, srcCol] .= entry + lineDelta := srcLine - lastSourceLine + colDelta = srcCol - lastSourceColumn + lastSourceLine = srcLine + lastSourceColumn = srcCol + `${encodeVlq(entry[0])}${encodeVlq(sourceFileIndex)}${encodeVlq(lineDelta)}${encodeVlq(colDelta)}` + else + encodeVlq entry[0] + .join(",") + .join(";") + + json(srcFileName: string, outFileName: string) + version: 3 + file: outFileName + sources: [srcFileName] + mappings: @renderMappings() + names: [] + sourcesContent: [@source] + toString: -> + JSON.stringify this + + updateSourceMap(outputStr: string, inputPos?: number, colOffset=0) + outLines := outputStr.split(EOL) + + let srcLine: number, srcCol: number + + if inputPos? + [srcLine, srcCol] = lookupLineColumn @srcTable, inputPos + srcCol += colOffset + @srcLine = srcLine + @srcColumn = srcCol + @srcOffset = inputPos + outputStr# + + for each line, i of outLines + if i > 0 + @line++ + @srcLine++ + @colOffset = 0 + @lines[@line] = [] + @srcColumn = srcCol = colOffset + + l := @colOffset + @colOffset = line.length + @srcColumn += line.length if inputPos? - [srcLine, srcCol] = lookupLineColumn(srcTable, inputPos) - srcCol += colOffset - sm.srcLine = srcLine - sm.srcColumn = srcCol - sm.srcOffset = inputPos + outputStr# - - for each line, i of outLines - if i > 0 - sm.line++ - sm.srcLine++ - sm.colOffset = 0 - sm.lines[sm.line] = [] - sm.srcColumn = srcCol = colOffset - - l := sm.colOffset - sm.colOffset = line.length - sm.srcColumn += line.length - - if inputPos? - // srcLine and srcCol are absolute here - sm.lines[sm.line].push [l, 0, srcLine+i, srcCol] - else if l != 0 - sm.lines[sm.line].push [l] - - return + // srcLine and srcCol are absolute here + @lines[@line].push [l, 0, srcLine+i, srcCol] + else if l != 0 + @lines[@line].push [l] + + return smRegexp := /\n\/\/# sourceMappingURL=data:application\/json;charset=utf-8;base64,([+a-zA-Z0-9\/]*=?=?)$/ @@ -114,7 +115,7 @@ Remap a string with compiled code and a source map to use a new source map referencing upstream source files. */ /* c8 ignore start */ -remap := (codeWithSourceMap: string, upstreamMap: {data: {lines: SourceMapEntries}, json: any}, sourcePath: string, targetPath: string) -> +remap := (codeWithSourceMap: string, upstreamMap: {lines: SourceMapEntries, json: any}, sourcePath: string, targetPath: string) -> let sourceMapText?: string codeWithoutSourceMap := codeWithSourceMap.replace smRegexp, (match, sm) => sourceMapText = sm @@ -122,8 +123,8 @@ remap := (codeWithSourceMap: string, upstreamMap: {data: {lines: SourceMapEntrie if sourceMapText parsed := parseWithLines sourceMapText - composedLines := composeLines upstreamMap.data.lines, parsed.lines - upstreamMap.data.lines = composedLines + composedLines := composeLines upstreamMap.lines, parsed.lines + upstreamMap.lines = composedLines remappedSourceMapJSON := upstreamMap.json(sourcePath, targetPath) diff --git a/source/ts-diagnostic.civet b/source/ts-diagnostic.civet index b2019817..5ee426ca 100644 --- a/source/ts-diagnostic.civet +++ b/source/ts-diagnostic.civet @@ -13,10 +13,9 @@ type SourceMapping = [number] | [number, number, number, number] type SourceMap = updateSourceMap?(outputStr: string, inputPos: number): void json(srcFileName: string, outFileName: string): unknown - data: - lines: SourceMapping[][] + lines: SourceMapping[][] -export type SourcemapLines = SourceMap['data']['lines'] +export type SourcemapLines = SourceMap['lines'] /** * Take a position in generated code and map it into a position in source code. diff --git a/source/unplugin/unplugin.civet b/source/unplugin/unplugin.civet index 7d85e0af..87e61dfc 100644 --- a/source/unplugin/unplugin.civet +++ b/source/unplugin/unplugin.civet @@ -297,7 +297,7 @@ export const rawPlugin: Parameters>[0] = sourceMap := sourceMaps.get file.fileName if (!sourceMap) return diagnostic - sourcemapLines := sourceMap.data.lines + sourcemapLines := sourceMap.lines ?? sourceMap.data.lines range := remapRange( { start: diagnostic.start || 0, @@ -446,7 +446,7 @@ export const rawPlugin: Parameters>[0] = ...civetOptions ast: true } - civetSourceMap := SourceMap rawCivetSource + civetSourceMap := new SourceMap rawCivetSource if ts is "civet" compiled = await civet.generate ast, { diff --git a/source/worker-pool.civet b/source/worker-pool.civet index 579d20bc..14f8b210 100644 --- a/source/worker-pool.civet +++ b/source/worker-pool.civet @@ -1,4 +1,5 @@ type { Worker } from node:worker_threads +{ SourceMap } from './sourcemap.civet' type Job id: number @@ -53,6 +54,10 @@ export class WorkerPool try error.name = response.error.name callback.reject error else + if sourceMap := response.result?.sourceMap + // Restore SourceMap class wrapper not preserved by worker message + response.result.sourceMap = new SourceMap sourceMap.source + Object.assign response.result.sourceMap, sourceMap callback.resolve response.result // Worker is now available if @spawned > @threads // kill if now too many workers diff --git a/test/sourcemap.civet b/test/sourcemap.civet index 36e15366..8e1eed9e 100644 --- a/test/sourcemap.civet +++ b/test/sourcemap.civet @@ -51,4 +51,4 @@ describe "source map", -> //console.dir parsed, {depth: null} - assert.deepEqual parsed.lines, sourceMap.data.lines + assert.deepEqual parsed.lines, sourceMap.lines diff --git a/test/util/locations.civet b/test/util/locations.civet index b3a36152..a40bf1f5 100644 --- a/test/util/locations.civet +++ b/test/util/locations.civet @@ -39,6 +39,6 @@ describe "sourcemap", -> describe "SourceMap", -> it "should have `toString`", -> - sm := SourceMap("") + sm := new SourceMap "" assert sm.json("src.civet", "src.jsx").toString() !== "[object Object]" diff --git a/types/types.d.ts b/types/types.d.ts index 2155b9bf..ba0d8e5c 100644 --- a/types/types.d.ts +++ b/types/types.d.ts @@ -62,14 +62,15 @@ declare module "@danielx/civet" { export type SourceMapping = [number] | [number, number, number, number] - export interface SourceMap { + export class SourceMap { + constructor(source: string) updateSourceMap?(outputStr: string, inputPos: number): void json(srcFileName: string, outFileName: string): unknown - data: { - lines: SourceMapping[][] - } + source: string + lines: SourceMapping[][] + /** @deprecated */ + data: { lines: SourceMapping[][] } } - export function SourceMap(source: string): SourceMap // TODO: Import ParseError class from Hera export type ParseError = { @@ -121,7 +122,7 @@ declare module "@danielx/civet" { sourcemap: { locationTable(input: string): number[] lookupLineColumn(table: number[], pos: number): [number, number] - SourceMap(input: string): SourceMap + SourceMap: typeof SourceMap } }