diff --git a/.buildbot.sh b/.buildbot.sh index 3d42111..4d9d2dd 100644 --- a/.buildbot.sh +++ b/.buildbot.sh @@ -31,23 +31,4 @@ cd .. export YK_BUILD_TYPE=debug make -j `nproc` -cd tests -# YKFIXME: The JIT can't yet run the test suite, but the following commented -# commands are what we are aiming at having work. -# -# ../src/lua -e"_U=true" all.lua -# YKD_SERIALISE_COMPILATION=1 ../src/lua -e"_U=true" all.lua -# -# (Adding `YKD_SERIALISE_COMPILATION` exercises different threading behaviour -# that could help to shake out more bugs) -# -# Until we can run `all.lua` reliably, we just run the tests that we know to -# run within reasonable time). -LUA=../src/lua -for serialise in 0 1; do - for test in api bwcoercion closure code events \ - gengc pm tpack tracegc vararg goto cstack locals; do - echo "### YKD_SERIALISE_COMPILATION=$serialise $test.lua ###" - YKD_SERIALISE_COMPILATION=$serialise ${LUA} -e"_U=true" ${test}.lua - done -done +sh test.sh diff --git a/README.md b/README.md index 25b00b1..56bc80a 100644 --- a/README.md +++ b/README.md @@ -37,6 +37,15 @@ cd tests # navigate to tests directory ```shell run_docker_ci_job # local path to https://github.com/softdevteam/buildbot_config/blob/master/bin/run_docker_ci_job + +``` +## Debugging + +Use `LYK_VERBOSE` environment variable to print LYK (lua yk) debug logs: +```shell +LYK_VERBOSE=1 gdb --batch -ex 'r' -ex 'bt' --args ../src/lua all.lua +LYK_VERBOSE=1 ../src/lua all.lua +LYK_VERBOSE=1 sh ./test.sh ``` ### State of Tests diff --git a/src/Makefile b/src/Makefile index fd864b4..898a013 100644 --- a/src/Makefile +++ b/src/Makefile @@ -21,7 +21,8 @@ SYSCFLAGS= SYSLDFLAGS= SYSLIBS= -MYCFLAGS= +# YKFIXME: LYK_DEBUG is a debug flag for the Yk integration. Once YkLua is stable we can remove it. +MYCFLAGS=-DLYK_DEBUG MYLDFLAGS= MYLIBS= MYOBJS= diff --git a/src/lcode.c b/src/lcode.c index ceb135a..96f0c45 100644 --- a/src/lcode.c +++ b/src/lcode.c @@ -391,7 +391,7 @@ int luaK_code (FuncState *fs, Instruction i) { MAX_INT, "opcodes"); f->code[fs->pc++] = i; #ifdef USE_YK - yk_set_location(f, i, idx, fs->pc); + yk_ok_instruction_loaded(f, i, idx); #endif savelineinfo(fs, f, fs->ls->lastline); return idx; /* index of new instruction */ diff --git a/src/lfunc.c b/src/lfunc.c index d110c83..88df14a 100644 --- a/src/lfunc.c +++ b/src/lfunc.c @@ -251,7 +251,7 @@ Proto *luaF_newproto (lua_State *L) { f->sizep = 0; f->code = NULL; #ifdef USE_YK - f->yklocs = NULL; + yk_on_newproto(f); #endif f->sizecode = 0; f->lineinfo = NULL; @@ -273,7 +273,7 @@ Proto *luaF_newproto (lua_State *L) { void luaF_freeproto (lua_State *L, Proto *f) { #ifdef USE_YK - yk_free_locactions(f); + yk_on_proto_free(f); #endif luaM_freearray(L, f->code, f->sizecode); luaM_freearray(L, f->p, f->sizep); diff --git a/src/lobject.h b/src/lobject.h index 3bda66d..642a8d9 100644 --- a/src/lobject.h +++ b/src/lobject.h @@ -556,7 +556,8 @@ typedef struct Proto { TValue *k; /* constants used by the function */ Instruction *code; /* opcodes */ #ifdef USE_YK - YkLocation *yklocs; /* JIT locations */ + YkLocation **yklocs; /* JIT locations */ + int yklocs_size; #endif struct Proto **p; /* functions defined inside the function */ Upvaldesc *upvalues; /* upvalue information */ diff --git a/src/lundump.c b/src/lundump.c index faa540b..dd990bd 100644 --- a/src/lundump.c +++ b/src/lundump.c @@ -267,7 +267,7 @@ static void loadFunction (LoadState *S, Proto *f, TString *psource) { f->maxstacksize = loadByte(S); loadCode(S, f); #ifdef USE_YK - yk_set_locations(f); + yk_on_proto_loaded(f); #endif loadConstants(S, f); loadUpvalues(S, f); diff --git a/src/lyk.c b/src/lyk.c index e901748..6ecebc1 100644 --- a/src/lyk.c +++ b/src/lyk.c @@ -6,75 +6,148 @@ #include "ldebug.h" #include #include - -#define _DEFAULT_SOURCE /* for reallocarray() */ - +#include "stdio.h" /* * Function prototypes in Lua are loaded through two methods: * 1. They are loaded from text source via the `luaY_parser`. * 2. They can also be loaded from binary representation using `luaU_undump`, where * prototypes are previously dumped and saved. - * - * Yk tracing locations are allocated using both of these methods + * + * Yk tracing locations are allocated using both of these methods * using `yk_set_location` and `yk_set_locations` respectively. -*/ + */ + +#ifdef LYK_DEBUG +static int LYK_VERBOSE = -1; + +int is_verbose() { + if (LYK_VERBOSE == -1) { + LYK_VERBOSE = getenv("LYK_VERBOSE") != NULL; + } + return LYK_VERBOSE; +} + +void print_proto_info(char *msg, Proto *f) { + if (!is_verbose()){ + return; + } + char *source = NULL; + if (f->source != NULL) { + source = getstr(f->source); + } + char *vars = NULL; + if (f->locvars != NULL && f->locvars->varname != NULL) { + vars = getstr(f->locvars->varname); + } + printf("[LYK] %s. \t f:\t%p \t source: %s: \t vars: %s", msg, f, source, vars); + if (f->yklocs != NULL) { + printf("\t sizecode: %d", f->sizecode); + } + printf("\n"); +} +#endif // LYK_DEBUG + +void yk_on_newproto(Proto *f) { + #ifdef LYK_DEBUG + if (is_verbose()) { + printf("[LYK] yk_new_proto %p\n", f); + } + #endif // LYK_DEBUG + f->yklocs = NULL; +} /* * Is the instruction `i` the start of a loop? * - * YKFIXME: Numeric and Generic loops can be identified by `OP_FORLOOP` and `OP_TFORLOOP` opcodes. - * Other loops like while and repeat-until are harder to identify since they are based on + * YKFIXME: Numeric and Generic loops can be identified by `OP_FORLOOP` and `OP_TFORLOOP` opcodes. + * Other loops like while and repeat-until are harder to identify since they are based on * `OP_JMP` instruction. */ int is_loop_start(Instruction i) { return (GET_OPCODE(i) == OP_FORLOOP || GET_OPCODE(i) == OP_TFORLOOP); } -inline YkLocation *yk_lookup_ykloc(CallInfo *ci, Instruction *pc){ +inline YkLocation *yk_lookup_ykloc(CallInfo *ci, Instruction *pc) { YkLocation *ykloc = NULL; lua_assert(isLua(ci)); Proto *p = ci_func(ci)->p; lua_assert(p->code <= pc && pc <= p->code + p->sizecode); if (is_loop_start(*pc)) { - ykloc = &p->yklocs[pc - p->code]; + ykloc = p->yklocs[pc - p->code]; } return ykloc; } -inline void yk_set_location(Proto *f, Instruction i, int idx, int pc) { - // YKOPT: Reallocating for every instruction is inefficient. - f->yklocs = reallocarray(f->yklocs, pc, sizeof(YkLocation)); - lua_assert(f->yklocs != NULL && "Expected yklocs to be defined!"); - if (is_loop_start(i)) - { - f->yklocs[idx] = yk_location_new(); +void set_location(Proto *f, int i) { + YkLocation *loc = (YkLocation *)malloc(sizeof(YkLocation)); + lua_assert(loc != NULL && "Expected loc to be defined!"); + *loc = yk_location_new(); + f->yklocs[i] = loc; + #ifdef LYK_DEBUG + if (is_verbose()){ + printf("[LYK] yk_location_new. %p->yklocs[%d]=%p\n", f, i, loc); } + #endif // LYK_DEBUG } -inline void yk_set_locations(Proto *f) { - f->yklocs = calloc(f->sizecode, sizeof(YkLocation)); - lua_assert(f->yklocs != NULL && "Expected yklocs to be defined!"); - for (int i = 0; i < f->sizecode; i++){ - if (is_loop_start(i)){ - f->yklocs[i] = yk_location_new(); +inline void yk_ok_instruction_loaded(Proto *f, Instruction i, int idx) { + // YKOPT: Reallocating for every instruction is inefficient. + YkLocation **new_locations = calloc(f->sizecode, sizeof(YkLocation *)); + lua_assert(new_locations != NULL && "Expected yklocs to be defined!"); + + // copy previous locations over + if (f->yklocs != NULL) { + for (int i = 0; i < f->yklocs_size; i++) { + if (f->yklocs[i] != NULL) { + new_locations[i] = f->yklocs[i]; + } + else { + new_locations[i] = NULL; + } } + free(f->yklocs); } + f->yklocs = new_locations; + f->yklocs_size = f->sizecode; + #ifdef LYK_DEBUG + if (is_loop_start(i)) { + set_location(f, idx); + } + #endif // LYK_DEBUG } -void free_loc(Proto *f, Instruction i, int idx) { - if (is_loop_start(i)) { - yk_location_drop(f->yklocs[idx]); +inline void yk_on_proto_loaded(Proto *f) { + #ifdef LYK_DEBUG + print_proto_info("yk_set_locations", f); + #endif // LYK_DEBUG + f->yklocs = calloc(f->sizecode, sizeof(YkLocation *)); + lua_assert(f->yklocs != NULL && "Expected yklocs to be defined!"); + f->yklocs_size = f->sizecode; + for (int i = 0; i < f->sizecode; i++) { + if (is_loop_start(i)) { + set_location(f, i); + } } } -inline void yk_free_locactions(Proto *f) { - // YK locations are initialised as close as possible to the function loading, +inline void yk_on_proto_free(Proto *f) { + // YK locations are initialised as close as possible to the function loading, // However, this load can fail before we initialise `yklocs`. // This NULL check is a workaround for that. - if (f->yklocs != NULL) { + if (f->yklocs != NULL) { for (int i = 0; i < f->sizecode; i++) { - free_loc(f, f->code[i], i); + YkLocation *loc = f->yklocs[i]; + if (loc != NULL) { + #ifdef LYK_DEBUG + if (is_verbose()){ + printf("[LYK] yk_location_drop. %p->yklocs[%d]=%p\n", f, i, loc); + } + #endif // LYK_DEBUG + yk_location_drop(*loc); + free(loc); + } } + free(f->yklocs); f->yklocs = NULL; } diff --git a/src/lyk.h b/src/lyk.h index 5e10992..3f6523b 100644 --- a/src/lyk.h +++ b/src/lyk.h @@ -5,11 +5,13 @@ #include "lopcodes.h" #include "lstate.h" -void yk_set_location(Proto *f, Instruction i, int idx, int pc); +void yk_on_newproto(Proto *f); -void yk_set_locations(Proto *f); +void yk_ok_instruction_loaded(Proto *f, Instruction i, int idx); -void yk_free_locactions(Proto *f); +void yk_on_proto_loaded(Proto *f); + +void yk_on_proto_free(Proto *f); YkLocation* yk_lookup_ykloc(CallInfo *ci, Instruction *pc); diff --git a/test.sh b/test.sh new file mode 100644 index 0000000..3ba710c --- /dev/null +++ b/test.sh @@ -0,0 +1,29 @@ +#!/bin/sh + +set -e + +cd tests + +# YKFIXME: The JIT can't yet run the test suite, but the following commented +# commands are what we are aiming at having work. +# +# ../src/lua -e"_U=true" all.lua +# YKD_SERIALISE_COMPILATION=1 ../src/lua -e"_U=true" all.lua +# +# (Adding `YKD_SERIALISE_COMPILATION` exercises different threading behaviour +# that could help to shake out more bugs) +# +# Until we can run `all.lua` reliably, we just run the tests that we know to +# run within reasonable time). + +LUA=../src/lua + +# Non-serialised compilation tests +# YKFIXME: The following tests are known to work with non-serialised JIT +for test in api bwcoercion closure code events \ + gengc pm tpack tracegc vararg goto cstack locals; do + YKD_SERIALISE_COMPILATION=0 ${LUA} -e"_U=true" ${test}.lua +done + +# Serialised compilation tests +YKD_SERIALISE_COMPILATION=1 ${LUA} -e"_U=true" all.lua