Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add the AST json file generation feature for external analysis purpose #840

Closed
wants to merge 1 commit into from
Closed

Add the AST json file generation feature for external analysis purpose #840

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions docs/compiler_options.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ return {
| `-I --include-dir` | `include_dir` | `{string}` | `check` `gen` `run` | Prepend this directory to the module search path.
| `--gen-compat` | `gen_compat` | `string` | `gen` `run` | Generate compatibility code for targeting different Lua VM versions. See [below](#generated-code) for details.
| `--gen-target` | `gen_target` | `string` | `gen` `run` | Minimum targeted Lua version for generated code. Options are `5.1`, `5.3` and `5.4`. See [below](#generated-code) for details.
| | `gen_ast` | `number` | `gen` `run` | generate the AST json file for specific analysis purpose, the number expected is the max extraction depth from AST (use 10 by default, or increase this number for deeper AST info to extract)
| `--keep-hashbang` | | | `gen` | Preserve hashbang line (`#!`) at the top of file if present.
| `-p --pretend` | | | `gen` | Don't compile/write to any files, but type check and log what files would be written to.
| `--wdisable` | `disable_warnings` | `{string}` | `check` `run` | Disable the given warnings.
Expand Down
109 changes: 95 additions & 14 deletions tl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -139,6 +139,58 @@ local function filename_to_module_name(filename)
return (filename:gsub("%.lua$", ""):gsub("%.d%.tl$", ""):gsub("%.tl$", ""):gsub("[/\\]", "."))
end

local function serializeAstTableToJSON(tbl, maxDepth)
local function escapeString(str)
return string.gsub(str, '[%c"\\]', {
['\t'] = '\\t',
['\r'] = '\\r',
['\n'] = '\\n',
['"'] = '\\"',
['\\'] = '\\\\'
})
end
maxDepth = maxDepth or 10 -- Default max depth

local function serialize(val, stack, depth)
if depth > maxDepth then
return '"MAX_DEPTH_REACHED"'
end

local t = type(val)
if t == 'string' then
return '"' .. escapeString(val) .. '"'
elseif t == 'number' or t == 'boolean' or t == 'nil' then
return tostring(val)
elseif t == 'table' then
if stack[val] then
return '"CIRCULAR_REF_' .. tostring(stack[val]) .. '"'
end
stack[val] = depth

local result = {}
local isArray = #val > 0
local opening, closing = "{", "}"
if isArray then
opening, closing = "[", "]"
for i, v in ipairs(val) do
table.insert(result, serialize(v, stack, depth + 1))
end
else
for k, v in pairs(val) do
if type(k) ~= 'string' then k = tostring(k) end
table.insert(result, '"' .. escapeString(k) .. '":' .. serialize(v, stack, depth + 1))
end
end
stack[val] = nil
return opening .. table.concat(result, ",") .. closing
else
return '"UNSUPPORTED_TYPE"'
end
end

return serialize(tbl, {}, 1)
end

--------------------------------------------------------------------------------
-- Common driver backend
--------------------------------------------------------------------------------
Expand Down Expand Up @@ -168,6 +220,7 @@ local function setup_env(tlconfig, filename)
feat_arity = tlconfig["feat_arity"],
gen_compat = tlconfig["gen_compat"],
gen_target = tlconfig["gen_target"],
gen_ast = tlconfig["gen_ast"],
},
predefined_modules = tlconfig._init_env_modules,
}
Expand Down Expand Up @@ -268,28 +321,50 @@ local function type_check_and_load(tlconfig, filename)
return chunk
end

local function write_out(tlconfig, result, output_file, pp_opts)
local function write_out(tlconfig, result, output_files, pp_opts)
if tlconfig["pretend"] then
print("Would Write: " .. output_file)
print("Would write lua file: " .. output_files[1])
if tlconfig["gen_ast"]>0 then
print("Would write ast file: " .. output_files[2])
end
return
end

local ofd, err = io.open(output_file, "wb")
local ofd, err = io.open(output_files[1], "wb")

if not ofd then
die("cannot write " .. output_file .. ": " .. err)
die("cannot write " .. output_files[1] .. ": " .. err)
end

local _
_, err = ofd:write(tl.generate(result.ast, tlconfig.gen_target, pp_opts) .. "\n")
if err then
die("error writing " .. output_file .. ": " .. err)
die("error writing " .. output_files[1] .. ": " .. err)
end

ofd:close()

if not tlconfig["quiet"] then
print("Wrote: " .. output_file)
print("Wrote: " .. output_files[1])
end

if tlconfig["gen_ast"]>0 then
local ofd_ast,err = io.open(output_files[2], "w")
if not ofd_ast then
die("cannot write " .. output_files[2] .. ": " .. err)
else
print("Write ast file: " .. output_files[2])
_, err =ofd_ast:write('{"ast":\n')
_, err =ofd_ast:write(serializeAstTableToJSON(result.ast,tlconfig["gen_ast"]))
_, err =ofd_ast:write("\n}")

if err then
die("error writing " .. output_files[2] .. ": " .. err)
end
ofd_ast:close()
if not tlconfig["quiet"] then
print("Wrote: " .. output_files[2])
end
end
end
end

Expand Down Expand Up @@ -331,6 +406,7 @@ local function validate_config(config)
gen_target = { ["5.1"] = true, ["5.3"] = true, ["5.4"] = true },
disable_warnings = "{string}",
warning_error = "{string}",
gen_ast = "number",
}

for k, v in pairs(config) do
Expand Down Expand Up @@ -408,6 +484,8 @@ local function get_args_parser()

parser:flag("-p --pretend", "Do not write to any files, type check and output what files would be generated.")

parser:flag("-a --gen-ast", "AST file will be generated.")

parser:require_command(false)
parser:command_target("command")

Expand Down Expand Up @@ -443,7 +521,8 @@ local function get_config(cmd)
include_dir = {},
disable_warnings = {},
warning_error = {},
quiet = false
quiet = false,
gen_ast = 0
}

local config_path = find_file_in_parent_dirs("tlconfig.lua") or "tlconfig.lua"
Expand Down Expand Up @@ -530,6 +609,8 @@ local function merge_config_and_args(tlconfig, args)
tlconfig["pretend"] = true
end

tlconfig["gen_ast"] = args["gen_ast"] or tlconfig["gen_ast"]

tlconfig["feat_arity"] = args["feat_arity"] or tlconfig["feat_arity"]

tlconfig["gen_target"] = args["gen_target"] or tlconfig["gen_target"]
Expand All @@ -548,9 +629,9 @@ local function get_output_filename(file_name)
local name, ext = tail:match("(.+)%.([a-zA-Z]+)$")
if not name then name = tail end
if ext ~= "lua" then
return name .. ".lua"
return {name .. ".lua",name .. "_ast.json"}
else
return name .. ".out.lua"
return {name .. ".out.lua",name .. "_out_ast.json"}
end
end

Expand Down Expand Up @@ -750,7 +831,7 @@ commands["check"] = function(tlconfig, args)
if ok and tlconfig["quiet"] == false and #args["file"] == 1 then
local file_name = args["file"][1]

local output_file = get_output_filename(file_name)
local output_files = get_output_filename(file_name)
print("========================================")
print("Type checked " .. file_name)
print("0 errors detected -- you can use:")
Expand All @@ -761,7 +842,7 @@ commands["check"] = function(tlconfig, args)
print()
print(" tl gen " .. file_name)
print()
print(" to generate " .. output_file)
print(" to generate " .. output_files[1] .. " and " .. output_files[2])
end

os.exit(ok and 0 or 1)
Expand Down Expand Up @@ -794,7 +875,7 @@ commands["gen"] = function(tlconfig, args)

local res = {
input_file = input_file,
output_file = get_output_filename(input_file)
output_files = get_output_filename(input_file)
}

res.tl_result, err = process_module(input_file, env)
Expand All @@ -808,7 +889,7 @@ commands["gen"] = function(tlconfig, args)

for _, res in ipairs(results) do
if #res.tl_result.syntax_errors == 0 then
write_out(tlconfig, res.tl_result, args["output"] or res.output_file, pp_opts)
write_out(tlconfig, res.tl_result, args["output"] or res.output_files, pp_opts)
end
end

Expand Down
Loading