From 28ad10a96b07fb2bedb9ff7d9ab3ae04e9cb4d9f Mon Sep 17 00:00:00 2001 From: Mihail Stoykov Date: Thu, 4 Jul 2024 18:23:03 +0300 Subject: [PATCH 1/2] Simplify tc39 - extracted from ESM branch --- js/tc39/tc39_test.go | 45 +++++++++++++++++--------------------------- 1 file changed, 17 insertions(+), 28 deletions(-) diff --git a/js/tc39/tc39_test.go b/js/tc39/tc39_test.go index 140b3ef9ed3..1bb494a2eac 100644 --- a/js/tc39/tc39_test.go +++ b/js/tc39/tc39_test.go @@ -412,11 +412,10 @@ func (ctx *tc39TestCtx) runTC39Test(t testing.TB, name, src string, meta *tc39Me _ = vm.Set("print", t.Log) } var early bool - var origErr error if meta.hasFlag("module") { - early, origErr, err = ctx.runTC39Module(name, src, meta.Includes, vm) + early, err = ctx.runTC39Module(name, src, meta.Includes, vm) } else { - early, origErr, err = ctx.runTC39Script(name, src, meta.Includes, vm, meta.Negative.Type != "") + early, err = ctx.runTC39Script(name, src, meta.Includes, vm, meta.Negative.Type != "") } if err == nil { @@ -447,9 +446,6 @@ func (ctx *tc39TestCtx) runTC39Test(t testing.TB, name, src string, meta *tc39Me errType := getErrType(name, err, failf) if errType != "" && errType != meta.Negative.Type { - if meta.Negative.Type == "SyntaxError" && origErr != nil && getErrType(name, origErr, failf) == meta.Negative.Type { - return - } // vm.vm.prg.dumpCode(t.Logf) failf("%s: unexpected error type (%s), expected (%s)", name, errType, meta.Negative.Type) return @@ -610,7 +606,7 @@ func (ctx *tc39TestCtx) compile(base, name string) (*sobek.Program, error) { str := string(b) comp := ctx.compilerPool.Get() defer ctx.compilerPool.Put(comp) - comp.Options = compiler.Options{Strict: false, CompatibilityMode: ctx.compatibilityMode} + comp.Options = compiler.Options{CompatibilityMode: ctx.compatibilityMode} prg, _, err = comp.Compile(str, name, true) if err != nil { return nil, err @@ -630,22 +626,22 @@ func (ctx *tc39TestCtx) runFile(base, name string, vm *sobek.Runtime) error { return err } -func (ctx *tc39TestCtx) runTC39Script(name, src string, includes []string, vm *sobek.Runtime, expectsError bool) (early bool, origErr, err error) { +func (ctx *tc39TestCtx) runTC39Script(name, src string, includes []string, vm *sobek.Runtime, expectsError bool) (early bool, err error) { early = true err = ctx.runFile(ctx.base, path.Join("harness", "assert.js"), vm) if err != nil { - return early, origErr, err + return early, err } err = ctx.runFile(ctx.base, path.Join("harness", "sta.js"), vm) if err != nil { - return early, origErr, err + return early, err } for _, include := range includes { err = ctx.runFile(ctx.base, path.Join("harness", include), vm) if err != nil { - return early, origErr, err + return early, err } } @@ -654,59 +650,52 @@ func (ctx *tc39TestCtx) runTC39Script(name, src string, includes []string, vm *s defer ctx.compilerPool.Put(comp) comp.Options = compiler.Options{Strict: false, CompatibilityMode: lib.CompatibilityModeBase} p, _, err = comp.Compile(src, name, true) - origErr = err if err != nil && !expectsError { comp.Options.CompatibilityMode = ctx.compatibilityMode p, _, err = comp.Compile(src, name, true) } - if err != nil { - return early, origErr, err + return early, err } early = false _, err = vm.RunProgram(p) - return early, origErr, err + return early, err } -func (ctx *tc39TestCtx) runTC39Module(name, src string, includes []string, vm *sobek.Runtime) (early bool, origErr, err error) { - currentFS := os.DirFS(".") - if err != nil { - panic(err) - } +func (ctx *tc39TestCtx) runTC39Module(name, src string, includes []string, vm *sobek.Runtime) (early bool, err error) { moduleRuntime := modulestest.NewRuntime(ctx.t) moduleRuntime.VU.RuntimeField = vm early = true err = ctx.runFile(ctx.base, path.Join("harness", "assert.js"), vm) if err != nil { - return early, origErr, err + return early, err } err = ctx.runFile(ctx.base, path.Join("harness", "sta.js"), vm) if err != nil { - return early, origErr, err + return early, err } for _, include := range includes { err = ctx.runFile(ctx.base, path.Join("harness", include), vm) if err != nil { - return early, origErr, err + return early, err } } comp := ctx.compilerPool.Get() defer ctx.compilerPool.Put(comp) comp.Options = compiler.Options{Strict: false, CompatibilityMode: ctx.compatibilityMode} - + u := &url.URL{Scheme: "file", Path: path.Join(ctx.base, name)} + base := u.JoinPath("..") mr := modules.NewModuleResolver(nil, func(specifier *url.URL, _ string) ([]byte, error) { - return fs.ReadFile(currentFS, specifier.Path[1:]) + return fs.ReadFile(os.DirFS("."), specifier.Path[1:]) }, comp) - u := &url.URL{Scheme: "file", Path: path.Join(ctx.base, name)} - base := u.JoinPath("..") ms := modules.NewModuleSystem(mr, moduleRuntime.VU) impl := modules.NewLegacyRequireImpl(moduleRuntime.VU, ms, *base) require.NoError(ctx.t, vm.Set("require", impl.Require)) @@ -718,7 +707,7 @@ func (ctx *tc39TestCtx) runTC39Module(name, src string, includes []string, vm *s URL: u, }) - return early, origErr, err + return early, err } func (ctx *tc39TestCtx) runTC39Tests(name string) { From 1745a501c2b861c49f324d4a20f1a41e170029d8 Mon Sep 17 00:00:00 2001 From: Mihail Stoykov Date: Thu, 4 Jul 2024 19:28:42 +0300 Subject: [PATCH 2/2] Backport changes to js/compiler from the ESM branch The changes revolve around moving away from compiling to mostly parsing the provided code. This will be relevant in ESM branch when only the parsing needs to be done for native ESM modules. For now it mostly adds some noise, but will make that PR a lot better. The changes to tc39 are mostly to make it nicer as well. The tc39 changes are quite strange as they definitely have *never* worked and do not work in k6 currently, so I have no idea what is happening there. It is very likely some error shadowing has been exposed. --- js/compiler/compiler.go | 213 ++++++++++-------- js/compiler/compiler_test.go | 73 +++--- js/compiler/enhanced_test.go | 8 +- js/modules/cjsmodule.go | 7 +- ...ing_test_errors-experimental_enhanced.json | 25 +- js/tc39/tc39_test.go | 26 ++- 6 files changed, 203 insertions(+), 149 deletions(-) diff --git a/js/compiler/compiler.go b/js/compiler/compiler.go index 1fed51109a4..105354034fa 100644 --- a/js/compiler/compiler.go +++ b/js/compiler/compiler.go @@ -16,6 +16,7 @@ import ( "github.com/go-sourcemap/sourcemap" "github.com/grafana/sobek" + "github.com/grafana/sobek/ast" "github.com/grafana/sobek/parser" "github.com/sirupsen/logrus" @@ -159,102 +160,99 @@ type Options struct { Strict bool } -// compilationState is helper struct to keep the state of a compilation -type compilationState struct { +// parsingState is helper struct to keep the state of a compilation +type parsingState struct { // set when we couldn't load external source map so we can try parsing without loading it couldntLoadSourceMap bool // srcMap is the current full sourceMap that has been generated read so far - srcMap []byte - srcMapError error - wrapped bool // whether the original source is wrapped in a function to make it a commonjs module + srcMap []byte + srcMapError error + wrapped bool // whether the original source is wrapped in a function to make it a commonjs module + compatibilityMode lib.CompatibilityMode + logger logrus.FieldLogger + + loader func(string) ([]byte, error) compiler *Compiler } -// Compile the program in the given CompatibilityMode, wrapping it between pre and post code -// TODO isESM will be used once Sobek support ESM modules natively -func (c *Compiler) Compile(src, filename string, isESM bool) (*sobek.Program, string, error) { - return c.compileImpl(src, filename, !isESM, c.Options.CompatibilityMode, nil) +// Parse parses the provided source. It wraps as the same as CommonJS support. +// The returned program can be compiled directly by Sobek. +// Additionally, it returns the end code that has been parsed including any required transformations. +func (c *Compiler) Parse( + src, filename string, wrap bool, +) (prg *ast.Program, finalCode string, err error) { + state := &parsingState{ + loader: c.Options.SourceMapLoader, + wrapped: wrap, + compatibilityMode: c.Options.CompatibilityMode, + logger: c.logger, + compiler: c, + } + return state.parseImpl(src, filename, wrap) } // sourceMapLoader is to be used with Sobek's WithSourceMapLoader // it not only gets the file from disk in the simple case, but also returns it if the map was generated from babel // additioanlly it fixes off by one error in commonjs dependencies due to having to wrap them in a function. -func (c *compilationState) sourceMapLoader(path string) ([]byte, error) { +func (ps *parsingState) sourceMapLoader(path string) ([]byte, error) { if path == sourceMapURLFromBabel { - if c.wrapped { - return c.increaseMappingsByOne(c.srcMap) + if ps.wrapped { + return ps.increaseMappingsByOne(ps.srcMap) } - return c.srcMap, nil + return ps.srcMap, nil } - c.srcMap, c.srcMapError = c.compiler.Options.SourceMapLoader(path) - if c.srcMapError != nil { - c.couldntLoadSourceMap = true - return nil, c.srcMapError + ps.srcMap, ps.srcMapError = ps.loader(path) + if ps.srcMapError != nil { + ps.couldntLoadSourceMap = true + return nil, ps.srcMapError } - _, c.srcMapError = sourcemap.Parse(path, c.srcMap) - if c.srcMapError != nil { - c.couldntLoadSourceMap = true - c.srcMap = nil - return nil, c.srcMapError + _, ps.srcMapError = sourcemap.Parse(path, ps.srcMap) + if ps.srcMapError != nil { + ps.couldntLoadSourceMap = true + ps.srcMap = nil + return nil, ps.srcMapError } - if c.wrapped { - return c.increaseMappingsByOne(c.srcMap) + if ps.wrapped { + return ps.increaseMappingsByOne(ps.srcMap) } - return c.srcMap, nil + return ps.srcMap, nil } -func (c *Compiler) compileImpl( - src, filename string, wrap bool, compatibilityMode lib.CompatibilityMode, srcMap []byte, -) (*sobek.Program, string, error) { +func (ps *parsingState) parseImpl(src, filename string, wrap bool) (*ast.Program, string, error) { code := src - state := compilationState{srcMap: srcMap, compiler: c, wrapped: wrap} - if wrap { - conditionalNewLine := "" - if index := strings.LastIndex(code, "//# sourceMappingURL="); index != -1 { - // the lines in the sourcemap (if available) will be fixed by increaseMappingsByOne - conditionalNewLine = "\n" - newCode, err := state.updateInlineSourceMap(code, index) - if err != nil { - c.logger.Warnf("while compiling %q, couldn't update its inline sourcemap which might lead "+ - "to some line numbers being off: %s", filename, err) - } else { - code = newCode - } - - // if there is no sourcemap - bork only the first line of code, but leave the remaining ones. - } - code = "(function(module, exports){" + conditionalNewLine + code + "\n})\n" + if wrap { // the lines in the sourcemap (if available) will be fixed by increaseMappingsByOne + code = ps.wrap(code, filename) + ps.wrapped = true } opts := parser.WithDisableSourceMaps - if c.Options.SourceMapLoader != nil { - opts = parser.WithSourceMapLoader(state.sourceMapLoader) + if ps.loader != nil { + opts = parser.WithSourceMapLoader(ps.sourceMapLoader) } - ast, err := parser.ParseFile(nil, filename, code, 0, opts) + prg, err := parser.ParseFile(nil, filename, code, 0, opts) - if state.couldntLoadSourceMap { - state.couldntLoadSourceMap = false // reset + if ps.couldntLoadSourceMap { + ps.couldntLoadSourceMap = false // reset // we probably don't want to abort scripts which have source maps but they can't be found, // this also will be a breaking change, so if we couldn't we retry with it disabled - c.logger.WithError(state.srcMapError).Warnf("Couldn't load source map for %s", filename) - ast, err = parser.ParseFile(nil, filename, code, 0, parser.WithDisableSourceMaps) + ps.logger.WithError(ps.srcMapError).Warnf("Couldn't load source map for %s", filename) + prg, err = parser.ParseFile(nil, filename, code, 0, parser.WithDisableSourceMaps) } if err == nil { - pgm, err := sobek.CompileAST(ast, c.Options.Strict) - return pgm, code, err + return prg, code, nil } - if compatibilityMode == lib.CompatibilityModeExtended { - code, state.srcMap, err = c.Transform(src, filename, state.srcMap) + if ps.compatibilityMode == lib.CompatibilityModeExtended { + code, ps.srcMap, err = ps.compiler.Transform(src, filename, ps.srcMap) if err != nil { return nil, code, err } - // the compatibility mode "decreases" here as we shouldn't transform twice - var prg *sobek.Program - prg, code, err = c.compileImpl(code, filename, wrap, lib.CompatibilityModeBase, state.srcMap) + ps.wrapped = false + ps.compatibilityMode = lib.CompatibilityModeBase + prg, code, err = ps.parseImpl(code, filename, wrap) if err == nil && strings.Contains(src, "module.exports") { - c.logger.Warningf( + ps.logger.Warningf( "During the compilation of %q, it has been detected that the file combines ECMAScript modules (ESM) "+ "import/export syntax with commonJS module.exports. "+ "Mixing these two module systems is non-standard and will not be supported anymore in future releases. "+ @@ -263,52 +261,41 @@ func (c *Compiler) compileImpl( } return prg, code, err } - - if compatibilityMode == lib.CompatibilityModeExperimentalEnhanced { - code, state.srcMap, err = esbuildTransform(src, filename) + if ps.compatibilityMode == lib.CompatibilityModeExperimentalEnhanced { + code, ps.srcMap, err = esbuildTransform(src, filename) if err != nil { - return nil, code, err + return nil, "", err } - if c.Options.SourceMapLoader != nil { + if ps.loader != nil { // This hack is required for the source map to work code += "\n//# sourceMappingURL=" + sourceMapURLFromBabel } - return c.compileImpl(code, filename, wrap, lib.CompatibilityModeBase, state.srcMap) + ps.wrapped = false + ps.compatibilityMode = lib.CompatibilityModeBase + return ps.parseImpl(code, filename, wrap) } - return nil, code, err + return nil, "", err } -type babel struct { - vm *sobek.Runtime - this sobek.Value - transform sobek.Callable - m sync.Mutex -} - -func newBabel() (*babel, error) { - onceBabelCode.Do(func() { - globalBabelCode, errGlobalBabelCode = sobek.Compile("", babelSrc, false) - }) - if errGlobalBabelCode != nil { - return nil, errGlobalBabelCode - } - vm := sobek.New() - _, err := vm.RunProgram(globalBabelCode) - if err != nil { - return nil, err - } +func (ps *parsingState) wrap(code, filename string) string { + conditionalNewLine := "" + if index := strings.LastIndex(code, "//# sourceMappingURL="); index != -1 { + // the lines in the sourcemap (if available) will be fixed by increaseMappingsByOne + conditionalNewLine = "\n" + newCode, err := ps.updateInlineSourceMap(code, index) + if err != nil { + ps.logger.Warnf("while compiling %q, couldn't update its inline sourcemap which might lead "+ + "to some line numbers being off: %s", filename, err) + } else { + code = newCode + } - this := vm.Get("Babel") - bObj := this.ToObject(vm) - result := &babel{vm: vm, this: this} - if err = vm.ExportTo(bObj.Get("transform"), &result.transform); err != nil { - return nil, err + // if there is no sourcemap - bork only the first line of code, but leave the remaining ones. } - - return result, err + return "(function(module, exports){" + conditionalNewLine + code + "\n})\n" } -func (c *compilationState) updateInlineSourceMap(code string, index int) (string, error) { +func (ps *parsingState) updateInlineSourceMap(code string, index int) (string, error) { nextnewline := strings.Index(code[index:], "\n") if nextnewline == -1 { nextnewline = len(code[index:]) @@ -321,19 +308,19 @@ func (c *compilationState) updateInlineSourceMap(code string, index int) (string if err != nil { return code, err } - b, err = c.increaseMappingsByOne(b) + b, err = ps.increaseMappingsByOne(b) if err != nil { return code, err } encoded := base64.StdEncoding.EncodeToString(b) - code = code[:index] + "//# sourcemappingurl=data:application/json;base64," + encoded + code[nextnewline:] + code = code[:index] + "//# sourceMappingURL=data:application/json;base64," + encoded + code[index+nextnewline:] } return code, nil } // increaseMappingsByOne increases the lines in the sourcemap by line so that it fixes the case where we need to wrap a // required file in a function to support/emulate commonjs -func (c *compilationState) increaseMappingsByOne(sourceMap []byte) ([]byte, error) { +func (ps *parsingState) increaseMappingsByOne(sourceMap []byte) ([]byte, error) { var err error m := make(map[string]interface{}) if err = json.Unmarshal(sourceMap, &m); err != nil { @@ -356,7 +343,7 @@ func (c *compilationState) increaseMappingsByOne(sourceMap []byte) ([]byte, erro } else { // we have mappings but it's not a string - this is some kind of error // we still won't abort the test but just not load the sourcemap - c.couldntLoadSourceMap = true + ps.couldntLoadSourceMap = true return nil, errors.New(`missing "mappings" in sourcemap`) } @@ -482,3 +469,33 @@ func verifySourceMapForBabel(srcMap []byte) error { } return nil } + +type babel struct { + vm *sobek.Runtime + this sobek.Value + transform sobek.Callable + m sync.Mutex +} + +func newBabel() (*babel, error) { + onceBabelCode.Do(func() { + globalBabelCode, errGlobalBabelCode = sobek.Compile("", babelSrc, false) + }) + if errGlobalBabelCode != nil { + return nil, errGlobalBabelCode + } + vm := sobek.New() + _, err := vm.RunProgram(globalBabelCode) + if err != nil { + return nil, err + } + + this := vm.Get("Babel") + bObj := this.ToObject(vm) + result := &babel{vm: vm, this: this} + if err = vm.ExportTo(bObj.Get("transform"), &result.transform); err != nil { + return nil, err + } + + return result, err +} diff --git a/js/compiler/compiler_test.go b/js/compiler/compiler_test.go index 89b44cc971d..47c48a4243f 100644 --- a/js/compiler/compiler_test.go +++ b/js/compiler/compiler_test.go @@ -70,9 +70,11 @@ func TestCompile(t *testing.T) { t.Parallel() c := New(testutils.NewLogger(t)) src := `1+(function() { return 2; })()` - pgm, code, err := c.Compile(src, "script.js", true) + prg, code, err := c.Parse(src, "script.js", false) require.NoError(t, err) assert.Equal(t, src, code) + pgm, err := sobek.CompileAST(prg, true) + require.NoError(t, err) v, err := sobek.New().RunProgram(pgm) require.NoError(t, err) assert.Equal(t, int64(3), v.Export()) @@ -82,18 +84,23 @@ func TestCompile(t *testing.T) { t.Parallel() c := New(testutils.NewLogger(t)) src := `exports.d=1+(function() { return 2; })()` - pgm, code, err := c.Compile(src, "script.js", false) + prg, code, err := c.Parse(src, "script.js", true) + require.NoError(t, err) + pgm, err := sobek.CompileAST(prg, true) require.NoError(t, err) assert.Equal(t, "(function(module, exports){exports.d=1+(function() { return 2; })()\n})\n", code) rt := sobek.New() v, err := rt.RunProgram(pgm) - require.NoError(t, err) - fn, ok := sobek.AssertFunction(v) - require.True(t, ok, "not a function") - exp := make(map[string]sobek.Value) - _, err = fn(sobek.Undefined(), sobek.Undefined(), rt.ToValue(exp)) - require.NoError(t, err) - assert.Equal(t, int64(3), exp["d"].Export()) + if assert.NoError(t, err) { + fn, ok := sobek.AssertFunction(v) + if assert.True(t, ok, "not a function") { + exp := make(map[string]sobek.Value) + _, err := fn(sobek.Undefined(), sobek.Undefined(), rt.ToValue(exp)) + if assert.NoError(t, err) { + assert.Equal(t, int64(3), exp["d"].Export()) + } + } + } }) t.Run("ES5 Invalid", func(t *testing.T) { @@ -101,7 +108,7 @@ func TestCompile(t *testing.T) { c := New(testutils.NewLogger(t)) src := `1+(function() { return 2; )()` c.Options.CompatibilityMode = lib.CompatibilityModeExtended - _, _, err := c.Compile(src, "script.js", false) + _, _, err := c.Parse(src, "script.js", false) assert.IsType(t, &sobek.Exception{}, err) assert.Contains(t, err.Error(), `SyntaxError: script.js: Unexpected token (1:26) > 1 | 1+(function() { return 2; )()`) @@ -110,10 +117,11 @@ func TestCompile(t *testing.T) { t.Parallel() c := New(testutils.NewLogger(t)) c.Options.CompatibilityMode = lib.CompatibilityModeExtended - pgm, code, err := c.Compile(`import "something"`, "script.js", true) + prg, code, err := c.Parse(`import "something"`, "script.js", false) + require.NoError(t, err) + assert.Equal(t, `"use strict";require("something");`, code) + pgm, err := sobek.CompileAST(prg, true) require.NoError(t, err) - assert.Equal(t, `"use strict";require("something");`, - code) rt := sobek.New() var requireCalled bool require.NoError(t, rt.Set("require", func(s string) { @@ -126,12 +134,13 @@ func TestCompile(t *testing.T) { }) t.Run("Wrap", func(t *testing.T) { + // This only works with `require` as wrapping means the import/export won't be top level and that is forbidden t.Parallel() c := New(testutils.NewLogger(t)) c.Options.CompatibilityMode = lib.CompatibilityModeExtended - pgm, code, err := c.Compile(`import "something";`, "script.js", false) + prg, code, err := c.Parse(`require("something");`, "script.js", true) require.NoError(t, err) - assert.Equal(t, `(function(module, exports){"use strict";require("something"); + assert.Equal(t, `(function(module, exports){require("something"); }) `, code) var requireCalled bool @@ -140,6 +149,9 @@ func TestCompile(t *testing.T) { assert.Equal(t, "something", s) requireCalled = true })) + + pgm, err := sobek.CompileAST(prg, true) + require.NoError(t, err) v, err := rt.RunProgram(pgm) require.NoError(t, err) fn, ok := sobek.AssertFunction(v) @@ -153,7 +165,7 @@ func TestCompile(t *testing.T) { t.Parallel() c := New(testutils.NewLogger(t)) c.Options.CompatibilityMode = lib.CompatibilityModeExtended - _, _, err := c.Compile(`1+(=>2)()`, "script.js", true) + _, _, err := c.Parse(`1+(=>2)()`, "script.js", false) assert.IsType(t, &sobek.Exception{}, err) assert.Contains(t, err.Error(), `SyntaxError: script.js: Unexpected token (1:3) > 1 | 1+(=>2)()`) @@ -167,8 +179,10 @@ func TestCorruptSourceMap(t *testing.T) { logger := logrus.New() logger.SetLevel(logrus.DebugLevel) logger.Out = io.Discard - hook := testutils.NewLogHook(logrus.InfoLevel, logrus.WarnLevel) - logger.AddHook(hook) + hook := testutils.SimpleLogrusHook{ + HookedLevels: []logrus.Level{logrus.InfoLevel, logrus.WarnLevel}, + } + logger.AddHook(&hook) compiler := New(logger) compiler.Options = Options{ @@ -177,7 +191,7 @@ func TestCorruptSourceMap(t *testing.T) { return corruptSourceMap, nil }, } - _, _, err := compiler.Compile("var s = 5;\n//# sourceMappingURL=somefile", "somefile", false) + _, _, err := compiler.Parse("var s = 5;\n//# sourceMappingURL=somefile", "somefile", false) require.NoError(t, err) entries := hook.Drain() require.Len(t, entries, 1) @@ -190,14 +204,17 @@ func TestCorruptSourceMap(t *testing.T) { func TestCorruptSourceMapOnlyForBabel(t *testing.T) { t.Parallel() + // This test is now kind of pointless // this a valid source map for the go implementation but babel doesn't like it corruptSourceMap := []byte(`{"mappings": ";"}`) logger := logrus.New() logger.SetLevel(logrus.DebugLevel) logger.Out = io.Discard - hook := testutils.NewLogHook(logrus.InfoLevel, logrus.WarnLevel) - logger.AddHook(hook) + hook := testutils.SimpleLogrusHook{ + HookedLevels: []logrus.Level{logrus.InfoLevel, logrus.WarnLevel}, + } + logger.AddHook(&hook) compiler := New(logger) compiler.Options = Options{ @@ -207,7 +224,9 @@ func TestCorruptSourceMapOnlyForBabel(t *testing.T) { return corruptSourceMap, nil }, } - _, _, err := compiler.Compile("import 'something';\n//# sourceMappingURL=somefile", "somefile", false) + prg, _, err := compiler.Parse("import 'something';\n//# sourceMappingURL=somefile", "somefile", false) + require.NoError(t, err) + _, err = sobek.CompileAST(prg, true) require.NoError(t, err) entries := hook.Drain() require.Len(t, entries, 1) @@ -226,8 +245,10 @@ func TestMinimalSourceMap(t *testing.T) { logger := logrus.New() logger.SetLevel(logrus.DebugLevel) logger.Out = io.Discard - hook := testutils.NewLogHook(logrus.InfoLevel, logrus.WarnLevel) - logger.AddHook(hook) + hook := testutils.SimpleLogrusHook{ + HookedLevels: []logrus.Level{logrus.InfoLevel, logrus.WarnLevel}, + } + logger.AddHook(&hook) compiler := New(logger) compiler.Options = Options{ @@ -237,7 +258,7 @@ func TestMinimalSourceMap(t *testing.T) { return corruptSourceMap, nil }, } - _, _, err := compiler.Compile("class s {};\n//# sourceMappingURL=somefile", "somefile", false) + _, _, err := compiler.Parse("class s {};\n//# sourceMappingURL=somefile", "somefile", false) require.NoError(t, err) require.Empty(t, hook.Drain()) } @@ -255,7 +276,7 @@ func TestMixingImportExport(t *testing.T) { CompatibilityMode: lib.CompatibilityModeExtended, Strict: true, } - _, _, err := compiler.Compile("export let s = 5;\nmodule.exports = 'something';", "somefile", false) + _, _, err := compiler.Parse("export let s = 5;\nmodule.exports = 'something';", "somefile", false) require.NoError(t, err) entries := hook.Drain() require.Len(t, entries, 1) diff --git a/js/compiler/enhanced_test.go b/js/compiler/enhanced_test.go index e1c8bd55390..6007c345056 100644 --- a/js/compiler/enhanced_test.go +++ b/js/compiler/enhanced_test.go @@ -61,7 +61,7 @@ func TestCompile_experimental_enhanced(t *testing.T) { c := New(testutils.NewLogger(t)) src := `1+(function() { return 2; )()` c.Options.CompatibilityMode = lib.CompatibilityModeExperimentalEnhanced - _, _, err := c.Compile(src, "script.js", false) + _, _, err := c.Parse(src, "script.js", false) assert.IsType(t, &parser.Error{}, err) assert.Contains(t, err.Error(), `script.js: Line 1:26 Unexpected ")"`) }) @@ -69,10 +69,12 @@ func TestCompile_experimental_enhanced(t *testing.T) { t.Parallel() c := New(testutils.NewLogger(t)) c.Options.CompatibilityMode = lib.CompatibilityModeExperimentalEnhanced - pgm, code, err := c.Compile(`import "something"`, "script.js", true) + prg, code, err := c.Parse(`import "something"`, "script.js", false) require.NoError(t, err) assert.Equal(t, `var import_something = require("something"); `, code) + pgm, err := sobek.CompileAST(prg, true) + require.NoError(t, err) rt := sobek.New() var requireCalled bool require.NoError(t, rt.Set("require", func(s string) { @@ -88,7 +90,7 @@ func TestCompile_experimental_enhanced(t *testing.T) { c := New(testutils.NewLogger(t)) c.Options.CompatibilityMode = lib.CompatibilityModeExperimentalEnhanced c.Options.SourceMapLoader = func(_ string) ([]byte, error) { return nil, nil } - _, code, err := c.Compile(`import "something"`, "script.js", true) + _, code, err := c.Parse(`import "something"`, "script.js", false) require.NoError(t, err) assert.Equal(t, `var import_something = require("something"); diff --git a/js/modules/cjsmodule.go b/js/modules/cjsmodule.go index 495e0a9f63a..4612f1368c0 100644 --- a/js/modules/cjsmodule.go +++ b/js/modules/cjsmodule.go @@ -65,9 +65,14 @@ func (c *cjsModuleInstance) exports() *sobek.Object { // TODO: extract this to not make this package dependant on compilers. // this is potentially a moot point after ESM when the compiler will likely get mostly dropped. func cjsModuleFromString(fileURL *url.URL, data []byte, c *compiler.Compiler) (*cjsModule, error) { - pgm, _, err := c.Compile(string(data), fileURL.String(), false) + astProgram, _, err := c.Parse(string(data), fileURL.String(), true) if err != nil { return nil, err } + pgm, err := sobek.CompileAST(astProgram, true) + if err != nil { + return nil, err + } + return &cjsModule{prg: pgm, url: fileURL}, nil } diff --git a/js/tc39/breaking_test_errors-experimental_enhanced.json b/js/tc39/breaking_test_errors-experimental_enhanced.json index 7cab7aff117..23f5b50bc9a 100644 --- a/js/tc39/breaking_test_errors-experimental_enhanced.json +++ b/js/tc39/breaking_test_errors-experimental_enhanced.json @@ -36,11 +36,17 @@ "test/language/expressions/assignment/fn-name-lhs-cover.js-strict:true": "test/language/expressions/assignment/fn-name-lhs-cover.js: Test262Error: descriptor value should be ", "test/language/expressions/class/class-name-ident-await-escaped-module.js-strict:true": "test/language/expressions/class/class-name-ident-await-escaped-module.js: error is not an object (Test262: This statement should not be evaluated.)", "test/language/expressions/class/class-name-ident-await-module.js-strict:true": "test/language/expressions/class/class-name-ident-await-module.js: error is not an object (Test262: This statement should not be evaluated.)", + "test/language/expressions/class/cpn-class-expr-accessors-computed-property-name-from-integer-separators.js-strict:true": "test/language/expressions/class/cpn-class-expr-accessors-computed-property-name-from-integer-separators.js: SyntaxError: test/language/expressions/class/cpn-class-expr-accessors-computed-property-name-from-integer-separators.js: Identifier directly after number (40:8)\n 38 | \n 39 | let C = class {\n> 40 | get [1_2_3_4_5_6_7_8]() {\n | ^\n 41 | return 1_2_3_4_5_6_7_8;\n 42 | }\n 43 | ", + "test/language/expressions/class/cpn-class-expr-computed-property-name-from-integer-separators.js-strict:true": "test/language/expressions/class/cpn-class-expr-computed-property-name-from-integer-separators.js: SyntaxError: test/language/expressions/class/cpn-class-expr-computed-property-name-from-integer-separators.js: Identifier directly after number (40:4)\n 38 | \n 39 | let C = class {\n> 40 | [1_2_3_4_5_6_7_8]() {\n | ^\n 41 | return 1_2_3_4_5_6_7_8;\n 42 | }\n 43 | static [1_2_3_4_5_6_7_8]() { ", + "test/language/expressions/class/cpn-class-expr-fields-computed-property-name-from-integer-separators.js-strict:true": "test/language/expressions/class/cpn-class-expr-fields-computed-property-name-from-integer-separators.js: SyntaxError: test/language/expressions/class/cpn-class-expr-fields-computed-property-name-from-integer-separators.js: Identifier directly after number (40:4)\n 38 | \n 39 | let C = class {\n> 40 | [1_2_3_4_5_6_7_8] = 1_2_3_4_5_6_7_8;\n | ^\n 41 | \n 42 | static [1_2_3_4_5_6_7_8] = 1_2_3_4_5_6_7_8;\n 43 | }; ", + "test/language/expressions/class/cpn-class-expr-fields-methods-computed-property-name-from-integer-separators.js-strict:true": "test/language/expressions/class/cpn-class-expr-fields-methods-computed-property-name-from-integer-separators.js: SyntaxError: test/language/expressions/class/cpn-class-expr-fields-methods-computed-property-name-from-integer-separators.js: Identifier directly after number (40:4)\n 38 | \n 39 | let C = class {\n> 40 | [1_2_3_4_5_6_7_8] = () => {\n | ^\n 41 | return 1_2_3_4_5_6_7_8;\n 42 | };\n 43 | ", "test/language/expressions/class/elements/class-name-static-initializer-default-export.js-strict:true": "test/language/expressions/class/elements/class-name-static-initializer-default-export.js: Test262Error: Expected SameValue(«class_name_static_initializer_default_export_default», «default») to be true ", "test/language/expressions/class/elements/private-getter-is-not-a-own-property.js-strict:true": "test/language/expressions/class/elements/private-getter-is-not-a-own-property.js: TypeError: Object has no member '__lookupGetter__' ", "test/language/expressions/class/elements/private-setter-is-not-a-own-property.js-strict:true": "test/language/expressions/class/elements/private-setter-is-not-a-own-property.js: TypeError: Object has no member '__lookupSetter__' ", - "test/language/expressions/optional-chaining/iteration-statement-for-await-of.js-strict:true": "test/language/expressions/optional-chaining/iteration-statement-for-await-of.js: test/language/expressions/optional-chaining/iteration-statement-for-await-of.js: Line 19:7 Unexpected token await (and 9 more errors)", + "test/language/expressions/object/cpn-obj-lit-computed-property-name-from-integer-separators.js-strict:true": "test/language/expressions/object/cpn-obj-lit-computed-property-name-from-integer-separators.js: SyntaxError: test/language/expressions/object/cpn-obj-lit-computed-property-name-from-integer-separators.js: Identifier directly after number (29:4)\n 27 | \n 28 | let o = {\n> 29 | [1_2_3_4_5_6_7_8]: 1_2_3_4_5_6_7_8\n | ^\n 30 | };\n 31 | \n 32 | assert.sameValue( ", + "test/language/expressions/optional-chaining/iteration-statement-for-await-of.js-strict:true": "test/language/expressions/optional-chaining/iteration-statement-for-await-of.js: SyntaxError: test/language/expressions/optional-chaining/iteration-statement-for-await-of.js: Unexpected token, expected ( (31:6)\n 29 | async function checkAssertions() {\n 30 | let count = 0;\n> 31 | for await (const num of obj?.iterable) {\n | ^\n 32 | count += num;\n 33 | }\n 34 | assert.sameValue(3, count); ", "test/language/expressions/optional-chaining/member-expression.js-strict:true": "test/language/expressions/optional-chaining/member-expression.js: SyntaxError: Async generators are not supported yet ", + "test/language/literals/numeric/non-octal-decimal-integer.js-strict:false": "test/language/literals/numeric/non-octal-decimal-integer.js: SyntaxError: test/language/literals/numeric/non-octal-decimal-integer.js: Invalid number (28:17)\n 26 | // NonOctalDecimalIntegerLiteral ::\n 27 | // 0 NonOctalDigit\n> 28 | assert.sameValue(08, 8, '08');\n | ^\n 29 | assert.sameValue(09, 9, '09');\n 30 | \n 31 | // NonOctalDecimalIntegerLiteral :: ", "test/language/literals/regexp/S7.8.5_A1.1_T2.js-strict:true": "test/language/literals/regexp/S7.8.5_A1.1_T2.js: Test262Error: Code unit: d800 Expected SameValue(«\\ud800», «�») to be true ", "test/language/literals/regexp/S7.8.5_A1.4_T2.js-strict:true": "test/language/literals/regexp/S7.8.5_A1.4_T2.js: Test262Error: Code unit: d800 Expected SameValue(«\\\\\\ud800», «\\�») to be true ", "test/language/literals/regexp/S7.8.5_A2.1_T2.js-strict:true": "test/language/literals/regexp/S7.8.5_A2.1_T2.js: Test262Error: Code unit: d800 Expected SameValue(«nnnn\\ud800», «nnnn�») to be true ", @@ -103,11 +109,9 @@ "test/language/module-code/instn-named-bndng-dflt-gen-anon.js-strict:true": "test/language/module-code/instn-named-bndng-dflt-gen-anon.js: Test262Error: correct name is assigned Expected SameValue(«instn_named_bndng_dflt_gen_anon_default», «default») to be true ", "test/language/module-code/instn-named-bndng-dflt-named.js-strict:true": "test/language/module-code/instn-named-bndng-dflt-named.js: Test262Error: Expected a ReferenceError to be thrown but no exception was thrown at all ", "test/language/module-code/instn-named-bndng-dflt-star.js-strict:true": "test/language/module-code/instn-named-bndng-dflt-star.js: Test262Error: Expected a ReferenceError to be thrown but no exception was thrown at all ", - "test/language/module-code/instn-named-bndng-fun.js-strict:true": "test/language/module-code/instn-named-bndng-fun.js: Test262Error: binding rejects assignment Expected a TypeError to be thrown but no exception was thrown at all ", - "test/language/module-code/instn-named-bndng-gen.js-strict:true": "test/language/module-code/instn-named-bndng-gen.js: Test262Error: binding rejects assignment Expected a TypeError to be thrown but no exception was thrown at all ", "test/language/module-code/instn-named-bndng-let.js-strict:true": "test/language/module-code/instn-named-bndng-let.js: Test262Error: binding is created but not initialized Expected a ReferenceError to be thrown but no exception was thrown at all ", - "test/language/module-code/instn-named-bndng-trlng-comma.js-strict:true": "test/language/module-code/instn-named-bndng-trlng-comma.js: Test262Error: binding rejects assignment Expected a TypeError to be thrown but no exception was thrown at all ", - "test/language/module-code/instn-named-bndng-var.js-strict:true": "test/language/module-code/instn-named-bndng-var.js: Test262Error: binding rejects assignment Expected a TypeError to be thrown but no exception was thrown at all ", + "test/language/module-code/instn-named-bndng-trlng-comma.js-strict:true": "test/language/module-code/instn-named-bndng-trlng-comma.js: Test262Error: binding is initialized to `undefined` prior to module evaulation Expected SameValue(«23», «undefined») to be true ", + "test/language/module-code/instn-named-bndng-var.js-strict:true": "test/language/module-code/instn-named-bndng-var.js: Test262Error: binding is initialized to `undefined` prior to module evaulation Expected SameValue(«23», «undefined») to be true ", "test/language/module-code/instn-named-err-ambiguous-as.js-strict:true": "test/language/module-code/instn-named-err-ambiguous-as.js: Expected error: ", "test/language/module-code/instn-named-err-ambiguous.js-strict:true": "test/language/module-code/instn-named-err-ambiguous.js: Expected error: ", "test/language/module-code/instn-named-err-dflt-thru-star-as.js-strict:true": "test/language/module-code/instn-named-err-dflt-thru-star-as.js: Expected error: ", @@ -129,9 +133,8 @@ "test/language/module-code/instn-star-iee-cycle.js-strict:true": "test/language/module-code/instn-star-iee-cycle.js: TypeError: Cannot read property 'b' of undefined ", "test/language/module-code/namespace/Symbol.toStringTag.js-strict:true": "test/language/module-code/namespace/Symbol.toStringTag.js: Test262Error: Expected SameValue(«undefined», «Module») to be true ", "test/language/module-code/namespace/internals/define-own-property.js-strict:true": "test/language/module-code/namespace/internals/define-own-property.js: Test262Error: Reflect.defineProperty: local2 Expected SameValue(«true», «false») to be true ", - "test/language/module-code/namespace/internals/delete-exported-init.js-strict:true": "test/language/module-code/namespace/internals/delete-exported-init.js: Test262Error: delete: local1 Expected a TypeError to be thrown but no exception was thrown at all ", - "test/language/module-code/namespace/internals/delete-exported-uninit.js-strict:true": "test/language/module-code/namespace/internals/delete-exported-uninit.js: Test262Error: delete: local1 Expected a TypeError to be thrown but no exception was thrown at all ", - "test/language/module-code/namespace/internals/delete-non-exported.js-strict:true": "test/language/module-code/namespace/internals/delete-non-exported.js: Test262Error: delete: default ", + "test/language/module-code/namespace/internals/delete-exported-uninit.js-strict:true": "test/language/module-code/namespace/internals/delete-exported-uninit.js: Test262Error: binding unmodified: default Expected a ReferenceError to be thrown but no exception was thrown at all ", + "test/language/module-code/namespace/internals/delete-non-exported.js-strict:true": "test/language/module-code/namespace/internals/delete-non-exported.js: TypeError: Cannot delete property 'default' of [object Object] ", "test/language/module-code/namespace/internals/enumerate-binding-uninit.js-strict:true": "test/language/module-code/namespace/internals/enumerate-binding-uninit.js: Test262Error: Expected a ReferenceError but got a Test262Error ", "test/language/module-code/namespace/internals/get-own-property-str-found-init.js-strict:true": "test/language/module-code/namespace/internals/get-own-property-str-found-init.js: Test262Error: Expected SameValue(«undefined», «201») to be true ", "test/language/module-code/namespace/internals/get-own-property-str-found-uninit.js-strict:true": "test/language/module-code/namespace/internals/get-own-property-str-found-uninit.js: Test262Error: hasOwnProperty: local1 Expected a ReferenceError to be thrown but no exception was thrown at all ", @@ -149,12 +152,16 @@ "test/language/module-code/namespace/internals/own-property-keys-binding-types.js-strict:true": "test/language/module-code/namespace/internals/own-property-keys-binding-types.js: Test262Error: Expected SameValue(«8», «10») to be true ", "test/language/module-code/namespace/internals/own-property-keys-sort.js-strict:true": "test/language/module-code/namespace/internals/own-property-keys-sort.js: Test262Error: Expected SameValue(«17», «16») to be true ", "test/language/module-code/namespace/internals/set-prototype-of.js-strict:true": "test/language/module-code/namespace/internals/set-prototype-of.js: Test262Error: Expected a TypeError to be thrown but no exception was thrown at all ", - "test/language/module-code/namespace/internals/set.js-strict:true": "test/language/module-code/namespace/internals/set.js: Test262Error: AssignmentExpression: local1 Expected a TypeError to be thrown but no exception was thrown at all ", + "test/language/module-code/namespace/internals/set.js-strict:true": "test/language/module-code/namespace/internals/set.js: Test262Error: Reflect.set: local2 Expected SameValue(«true», «false») to be true ", "test/language/module-code/parse-err-hoist-lex-fun.js-strict:true": "test/language/module-code/parse-err-hoist-lex-fun.js: error is not an object (Test262: This statement should not be evaluated.)", "test/language/module-code/parse-err-return.js-strict:true": "test/language/module-code/parse-err-return.js: error is not an object (Test262: This statement should not be evaluated.)", "test/language/reserved-words/await-module.js-strict:true": "test/language/reserved-words/await-module.js: error is not an object (Test262: This statement should not be evaluated.)", "test/language/statements/class/class-name-ident-await-escaped-module.js-strict:true": "test/language/statements/class/class-name-ident-await-escaped-module.js: error is not an object (Test262: This statement should not be evaluated.)", "test/language/statements/class/class-name-ident-await-module.js-strict:true": "test/language/statements/class/class-name-ident-await-module.js: error is not an object (Test262: This statement should not be evaluated.)", + "test/language/statements/class/cpn-class-decl-accessors-computed-property-name-from-integer-separators.js-strict:true": "test/language/statements/class/cpn-class-decl-accessors-computed-property-name-from-integer-separators.js: SyntaxError: test/language/statements/class/cpn-class-decl-accessors-computed-property-name-from-integer-separators.js: Identifier directly after number (40:8)\n 38 | \n 39 | class C {\n> 40 | get [1_2_3_4_5_6_7_8]() {\n | ^\n 41 | return 1_2_3_4_5_6_7_8;\n 42 | }\n 43 | ", + "test/language/statements/class/cpn-class-decl-computed-property-name-from-integer-separators.js-strict:true": "test/language/statements/class/cpn-class-decl-computed-property-name-from-integer-separators.js: SyntaxError: test/language/statements/class/cpn-class-decl-computed-property-name-from-integer-separators.js: Identifier directly after number (40:4)\n 38 | \n 39 | class C {\n> 40 | [1_2_3_4_5_6_7_8]() {\n | ^\n 41 | return 1_2_3_4_5_6_7_8;\n 42 | }\n 43 | static [1_2_3_4_5_6_7_8]() { ", + "test/language/statements/class/cpn-class-decl-fields-computed-property-name-from-integer-separators.js-strict:true": "test/language/statements/class/cpn-class-decl-fields-computed-property-name-from-integer-separators.js: SyntaxError: test/language/statements/class/cpn-class-decl-fields-computed-property-name-from-integer-separators.js: Identifier directly after number (40:4)\n 38 | \n 39 | let C = class {\n> 40 | [1_2_3_4_5_6_7_8] = 1_2_3_4_5_6_7_8;\n | ^\n 41 | \n 42 | static [1_2_3_4_5_6_7_8] = 1_2_3_4_5_6_7_8;\n 43 | }; ", + "test/language/statements/class/cpn-class-decl-fields-methods-computed-property-name-from-integer-separators.js-strict:true": "test/language/statements/class/cpn-class-decl-fields-methods-computed-property-name-from-integer-separators.js: SyntaxError: test/language/statements/class/cpn-class-decl-fields-methods-computed-property-name-from-integer-separators.js: Identifier directly after number (40:4)\n 38 | \n 39 | let C = class {\n> 40 | [1_2_3_4_5_6_7_8] = () => {\n | ^\n 41 | return 1_2_3_4_5_6_7_8;\n 42 | };\n 43 | ", "test/language/statements/class/elements/private-getter-is-not-a-own-property.js-strict:true": "test/language/statements/class/elements/private-getter-is-not-a-own-property.js: TypeError: Object has no member '__lookupGetter__' ", "test/language/statements/class/elements/private-setter-is-not-a-own-property.js-strict:true": "test/language/statements/class/elements/private-setter-is-not-a-own-property.js: TypeError: Object has no member '__lookupSetter__' ", "test/language/statements/labeled/value-await-module-escaped.js-strict:true": "test/language/statements/labeled/value-await-module-escaped.js: error is not an object (Test262: This statement should not be evaluated.)", diff --git a/js/tc39/tc39_test.go b/js/tc39/tc39_test.go index 1bb494a2eac..c44f2395874 100644 --- a/js/tc39/tc39_test.go +++ b/js/tc39/tc39_test.go @@ -603,11 +603,7 @@ func (ctx *tc39TestCtx) compile(base, name string) (*sobek.Program, error) { return nil, err } - str := string(b) - comp := ctx.compilerPool.Get() - defer ctx.compilerPool.Put(comp) - comp.Options = compiler.Options{CompatibilityMode: ctx.compatibilityMode} - prg, _, err = comp.Compile(str, name, true) + prg, err = ctx.compileOnly(string(b), name, ctx.compatibilityMode) if err != nil { return nil, err } @@ -626,6 +622,17 @@ func (ctx *tc39TestCtx) runFile(base, name string, vm *sobek.Runtime) error { return err } +func (ctx *tc39TestCtx) compileOnly(src, name string, compatibilityMode lib.CompatibilityMode) (*sobek.Program, error) { + comp := ctx.compilerPool.Get() + defer ctx.compilerPool.Put(comp) + comp.Options = compiler.Options{Strict: false, CompatibilityMode: compatibilityMode} + astProgram, _, err := comp.Parse(src, name, false) + if err != nil { + return nil, err + } + return sobek.CompileAST(astProgram, false) +} + func (ctx *tc39TestCtx) runTC39Script(name, src string, includes []string, vm *sobek.Runtime, expectsError bool) (early bool, err error) { early = true err = ctx.runFile(ctx.base, path.Join("harness", "assert.js"), vm) @@ -645,14 +652,9 @@ func (ctx *tc39TestCtx) runTC39Script(name, src string, includes []string, vm *s } } - var p *sobek.Program - comp := ctx.compilerPool.Get() - defer ctx.compilerPool.Put(comp) - comp.Options = compiler.Options{Strict: false, CompatibilityMode: lib.CompatibilityModeBase} - p, _, err = comp.Compile(src, name, true) + p, err := ctx.compileOnly(src, name, lib.CompatibilityModeBase) if err != nil && !expectsError { - comp.Options.CompatibilityMode = ctx.compatibilityMode - p, _, err = comp.Compile(src, name, true) + p, err = ctx.compileOnly(src, name, lib.CompatibilityModeExtended) } if err != nil { return early, err