From f8d0100565ffb7782a79220a362f09c3b806df22 Mon Sep 17 00:00:00 2001 From: Agustin Date: Wed, 13 Mar 2019 12:35:32 +0100 Subject: [PATCH] Dd version (#238) * WIP first integration * integrating imem_meta record structure for ddVersion * fixed preloaded path * implement ddVersion virtual table * commenting out debug logs * #236 tested with gen erl file lookaround and .app * #236 fix top level app origin url --- include/imem_meta.hrl | 10 ++ src/imem.erl | 297 ++++++++++++++++++++++++++++++++++++++++++ src/imem_meta.erl | 22 +++- 3 files changed, 322 insertions(+), 7 deletions(-) diff --git a/include/imem_meta.hrl b/include/imem_meta.hrl index 94b7db9..f2a56a2 100755 --- a/include/imem_meta.hrl +++ b/include/imem_meta.hrl @@ -184,6 +184,16 @@ ). -define(ddTermDiff, [number, binstr, binstr, binstr]). +-record(ddVersion, %% code version + { app ::binary() %% application + , appVsn ::binary() %% application version + , file ::binary() %% module / file name + , fileVsn ::binary() %% module / file version + , filePath = <<>> ::binary() %% module / file binary path + , fileOrigin = <<>> ::binary() %% module / file origin path (e.g. git commit url) + } + ). +-define(ddVersion, [binstr, binstr, binstr, binstr, binstr, binstr]). -define(OneWeek, 7.0). %% span of datetime or timestamp (fraction of 1 day) -define(OneDay, 1.0). %% span of datetime or timestamp (fraction of 1 day) diff --git a/src/imem.erl b/src/imem.erl index bdc7d40..67a721f 100755 --- a/src/imem.erl +++ b/src/imem.erl @@ -10,6 +10,7 @@ -include("imem.hrl"). -include("imem_if.hrl"). +-include("imem_meta.hrl"). % shell start/stop helper -export([ start/0 @@ -27,6 +28,8 @@ , get_swap_space/0 , spawn_sync_mfa/3 , priv_dir/0 + , all_apps_version_info/0 + , all_apps_version_info/1 ]). -safe([get_os_memory/0, get_vm_memory/0, get_swap_space/0]). @@ -302,3 +305,297 @@ priv_dir() -> code:which(?MODULE))), "priv"); D -> D end. + +all_apps_version_info() -> all_apps_version_info(_Opts = #{}). + +all_apps_version_info(Opts) when is_list(Opts) -> + all_apps_version_info(maps:from_list(Opts)); +all_apps_version_info(Opts) -> + AllModulesDetails = [ + { + Mod, + if is_atom(ModPath) -> atom_to_list(ModPath); + true -> ModPath + end + } + || {Mod, ModPath} <- code:all_loaded() + ], + AllApps = [ + { + App, list_to_binary(AppVsn), + element(2, application:get_key(App, modules)) + } + || {App, _, AppVsn} <- application:which_applications() + ], + {FAllApps, FAllModulesDetails} = + case Opts of + #{apps := FilterApps} -> + FilterAppsBin = [ + if is_atom(A) -> atom_to_binary(A, utf8); true -> A end + || A <- FilterApps + ], + FilteredAllApps = [ + App || {A, _, _} = App <- AllApps, + lists:member(atom_to_binary(A, utf8), FilterAppsBin) + ], + FilteredAllModules = lists:flatten( + [Modules || {_, _, Modules} <- FilteredAllApps] + ), + FilteredAllModulesDetails = lists:filter( + fun({Mod, _}) -> lists:member(Mod, FilteredAllModules) end, + AllModulesDetails + ), + {FilteredAllApps, FilteredAllModulesDetails}; + _ -> + {AllApps, AllModulesDetails} + end, + all_apps_version_info(merge(FAllApps, FAllModulesDetails), Opts). + +merge(AllApps, AllModulesDetails) -> + merge(AllApps, AllModulesDetails, _Acc = []). + +merge([], [], Acc) -> lists:usort(Acc); +merge([], ModulesDetails, Acc) -> + merge([], [], [{undefined, undefined, ModulesDetails} | Acc]); +merge([{App, AppVsn, Modules} | AllApps], ModulesDetails, Acc) -> + {ProcessModuleDetails, RestModulesDetails} = lists:partition( + fun({Mod, _}) -> lists:member(Mod, Modules) end, + ModulesDetails + ), + merge(AllApps, RestModulesDetails, + [{App, AppVsn, ProcessModuleDetails} | Acc]). + +all_apps_version_info(AllApps, Opts) -> + all_apps_version_info( + AllApps, Opts#{git => git()}, _ProcessPriv = true, _Acc = [] + ). + +all_apps_version_info([], _Opts, _ProcessPriv, Acc) -> lists:usort(Acc); +all_apps_version_info([{_, _, []} | AllApps], Opts, _, Acc) -> + all_apps_version_info(AllApps, Opts, _ProcessPriv = true, Acc); +all_apps_version_info( + [{App, AppVsn, [{Mod, ModPath} | Rest]} | AllApps], Opts, ProcessPriv, Acc +) -> + ModVsn = list_to_binary( + io_lib:format( + "vsn:~p", + [proplists:get_value(vsn, Mod:module_info(attributes))] + ) + ), + DDRec = #ddVersion{ + app = if App /= undefined -> atom_to_binary(App, utf8); + true -> App end, + appVsn = AppVsn, + file = atom_to_binary(Mod, utf8), + fileVsn = ModVsn, + filePath = list_to_binary(ModPath) + }, + if ProcessPriv -> + io:format( + "~p: ~p modules of ~p-~s~n", + [{?MODULE,?FUNCTION_NAME,?LINE}, length(Rest) + 1, + App, AppVsn] + ), + {GitRoot, Repo} = git_info(Opts, mod_source(Mod)), + Opts1 = Opts#{gitRoot => GitRoot, gitRepo => Repo}, + FileOrigin = git_file(Opts1, mod_source(Mod)), + DDRec1 = DDRec#ddVersion{fileOrigin = FileOrigin}, + Acc1 = process_app(DDRec1, Opts1, Mod) ++ Acc, + PrivDir = code:priv_dir(App), + NewAcc = + case filelib:is_dir(PrivDir) of + true -> + PrivFiles = filelib:wildcard("*", PrivDir), + {match, [PathRoot]} = re:run( + PrivDir, + <<"(",(DDRec1#ddVersion.app)/binary, ".*)">>, + [{capture, [1], list}] + ), + io:format( + "~p: ~p files @ ~s of ~p-~s~n", + [{?MODULE,?FUNCTION_NAME,?LINE}, + length(PrivFiles), PathRoot, App, AppVsn] + ), + walk_priv( + DDRec1#ddVersion.app, + DDRec1#ddVersion.appVsn, + PrivDir, PrivFiles, Acc1 + ); + _ -> Acc1 + end, + all_apps_version_info( + [{App, AppVsn, Rest} | AllApps], Opts1, _ProcessPriv = false, + [DDRec1 | NewAcc] + ); + true -> + FileOrigin = git_file(Opts, mod_source(Mod)), + all_apps_version_info( + [{App, AppVsn, Rest} | AllApps], Opts, _ProcessPriv = false, + [DDRec#ddVersion{fileOrigin = FileOrigin} | Acc]) + end. + +process_app(#ddVersion{app = undefined}, _Opts, _Module) -> []; +process_app(#ddVersion{app = App} = DDRec, Opts, Module) -> + case filename:dirname( + proplists:get_value(source, Module:module_info(compile)) + ) of + Dir when is_list(Dir) -> + File = <>, + FilePath = filename:join(Dir, File), + case file_phash2(FilePath) of + <<>> -> + File1 = <>, + FilePath1 = filename:join(Dir, File1), + case file_phash2(FilePath1) of + <<>> -> []; + FileHash1 -> + FileOrigin = git_file(Opts, FilePath1), + [DDRec#ddVersion{ + file = File1, + fileVsn = FileHash1, + filePath = FilePath1, + fileOrigin = FileOrigin + }] + end; + FileHash -> + FileOrigin = git_file(Opts, FilePath), + [DDRec#ddVersion{ + file = File, + fileVsn = FileHash, + filePath = FilePath, + fileOrigin = FileOrigin + }] + end; + _ -> [] + end. + +git() -> + case os:find_executable("git") of + false -> false; + GitExe when is_list(GitExe) -> true + end. + +mod_source(Module) when is_atom(Module) -> + case proplists:get_value(source, Module:module_info(compile)) of + Path when is_list(Path) -> Path; + _ -> <<>> + end. + +git_info(#{git := true} = Opts, Path) when is_list(Path); is_binary(Path) -> + case file:set_cwd(filename:dirname(Path)) of + ok -> + case list_to_binary(os:cmd("git rev-parse HEAD")) of + <<"fatal:", _>> -> {<<>>, <<>>}; + Revision -> + case re:run( + os:cmd("git remote -v"), + "(http[^ ]+)", [{capture, [1], list}] + ) of + {match,[Url|_]} -> + git_info(Opts, {Url, Revision}); + _ -> {<<>>, <<>>} + end + end; + _ -> {<<>>, <<>>} + end; +git_info(_Opts, {Url, Revision}) -> + CleanUrl = re:replace(Url, "\\.git", "", [{return, list}]), + {list_to_binary([CleanUrl, "/raw/", string:trim(Revision)]), + lists:last(filename:split(CleanUrl))}; +git_info(_Opts, _) -> {<<>>, <<>>}. + +git_file(#{git := true, gitRoot := <<>>}, _Path) -> <<>>; +git_file(#{git := true, gitRepo := Repo} = Opts, Path) -> + case re:run(Path, Repo++"(.*)", [{capture, [1], list}]) of + {match, [RelativePath]} -> + git_file(Opts#{absPath => Path}, Path, RelativePath); + _ -> <<>> + end. +git_file(#{git := true} = Opts, Path, RelativePath) -> + BaseName = filename:basename(RelativePath), + git_file( + Opts, Path, + BaseName, RelativePath + ). +git_file( + #{git := true, gitRoot := GitRoot} = Opts, Path, BaseName, RelativePath +) -> + case { + list_to_binary(os:cmd("git ls-files --error-unmatch " ++ BaseName)), + filename:extension(RelativePath) + } of + {<<"error: pathspec", _/binary>>, ".erl"} -> + git_file( + Opts, Path, + re:replace( + RelativePath, "\\.erl", "\\.yrl", + [{return, list}] + ) + ); + {<<"error: pathspec", _/binary>>, ".yrl"} -> + git_file( + Opts, Path, + re:replace( + RelativePath, "\\.yrl", "\\.xrl", + [{return, list}] + ) + ); + {<<"error: pathspec", _/binary>>, _} -> + #{absPath := AbsPath, gitRepo := Repo} = Opts, + case re:run( + AbsPath, "lib/"++Repo++"(.*)", [{capture, [1], list}] + ) of + nomatch -> notfound; + {match, [RelativePath1]} -> + list_to_binary([GitRoot, RelativePath1]) + end; + _ -> list_to_binary([GitRoot, RelativePath]) + end. + +walk_priv(_, _, _, [], Acc) -> Acc; +walk_priv(App, AppVsn, Path, [FileOrFolder | Rest], Acc) -> + FullPath = filename:join(Path, FileOrFolder), + case filelib:is_dir(FullPath) of + true -> + PrivFiles = filelib:wildcard("*", FullPath), + {match, [PathRoot]} = re:run( + FullPath, + <<"(",App/binary, ".*)">>, + [{capture, [1], list}] + ), + io:format( + "~p: ~p files @ ~s of ~s-~s~n", + [{?MODULE,?FUNCTION_NAME,?LINE}, length(PrivFiles), PathRoot, + App, AppVsn] + ), + %log(FullPath, "dderl\-3\.2\.0", "FullPath ~p, Files ~p~n", [FullPath, Files]), + walk_priv( + App, AppVsn, Path, Rest, + walk_priv(App, AppVsn, FullPath, PrivFiles, Acc) + ); + _ -> + walk_priv( + App, AppVsn, Path, Rest, + [#ddVersion{ + app = App, + appVsn = AppVsn, + file = list_to_binary(FileOrFolder), + fileVsn = file_phash2(FullPath), + filePath = list_to_binary(Path) + } | Acc] + ) + end. + +file_phash2(FilePath) -> + case file:read_file(FilePath) of + {ok, FBin} -> + <<"ph2:",(integer_to_binary(erlang:phash2(FBin)))/binary>>; + _ -> <<>> + end. + +%log(T, M, F, A) -> +% case re:run(T, M) of +% {match, _} -> +% io:format("~s: "++F, [?ME|A]); +% _ -> skip +% end. \ No newline at end of file diff --git a/src/imem_meta.erl b/src/imem_meta.erl index ab60802..bdb44f2 100755 --- a/src/imem_meta.erl +++ b/src/imem_meta.erl @@ -424,6 +424,7 @@ init(_Args) -> init_create_check_table(ddSchema, {record_info(fields, ddSchema), ?ddSchema, #ddSchema{}}, [], system), init_create_check_table(ddSize, {record_info(fields, ddSize), ?ddSize, #ddSize{}}, [], system), init_create_check_table(?LOG_TABLE, {record_info(fields, ddLog), ?ddLog, #ddLog{}}, ?LOG_TABLE_OPTS, system), + init_create_check_table(ddVersion, {record_info(fields, ddVersion), ?ddVersion, #ddVersion{}}, [], system), imem_tracer:init(), init_create_table(dual, {record_info(fields, dual), ?dual, #dual{}}, [], system), @@ -2335,6 +2336,7 @@ table_record_name(ddNode) -> ddNode; table_record_name(ddSnap) -> ddSnap; table_record_name(ddSchema) -> ddSchema; table_record_name(ddSize) -> ddSize; +table_record_name(ddVersion) -> ddVersion; table_record_name(Table) -> PTN = physical_table_name(Table), case is_virtual_table(PTN) of @@ -2355,6 +2357,7 @@ table_size(ddNode) -> length(read(ddNode)); table_size(ddSnap) -> imem_snap:snap_file_count(); table_size(ddSchema) -> length(read(ddSchema)); table_size(ddSize) -> 1; +table_size(ddVersion) -> 0; table_size(Table) -> %% ToDo: for an Alias, sum should be returned for all local time partitions imem_if_mnesia:table_size(physical_table_name(Table)). @@ -2436,7 +2439,7 @@ fetch_start(Pid, {?CSV_SCHEMA_PATTERN = S,FileName}, MatchSpec, BlockSize, Opts) fetch_start(Pid, {_Schema,Table}, MatchSpec, BlockSize, Opts) -> fetch_start(Pid, Table, MatchSpec, BlockSize, Opts); %% ToDo: may depend on schema fetch_start(Pid, Tab, MatchSpec, BlockSize, Opts) when - Tab==ddNode;Tab==ddSnap;Tab==ddSchema;Tab==ddSize -> + Tab==ddNode;Tab==ddSnap;Tab==ddSchema;Tab==ddSize;Tab==ddVersion -> fetch_start_calculated(Pid, Tab, MatchSpec, BlockSize, Opts); fetch_start(Pid, Table, MatchSpec, BlockSize, Opts) -> imem_if_mnesia:fetch_start(Pid, physical_table_name(Table), MatchSpec, BlockSize, Opts). @@ -2492,6 +2495,8 @@ read(ddSchema) -> [{ddSchema,{Schema,Node},[]} || {Schema,Node} <- data_nodes()]; read(ddSize) -> [hd(read(ddSize,Name)) || Name <- all_tables()]; +read(ddVersion) -> + imem:all_apps_version_info(); read(Table) -> imem_if_mnesia:read(physical_table_name(Table)). @@ -2521,6 +2526,8 @@ read(ddNode,_) -> []; read(ddSchema,Key) when is_tuple(Key) -> [ S || #ddSchema{schemaNode=K} = S <- read(ddSchema), K==Key]; read(ddSchema,_) -> []; +read(ddVersion, App) -> + imem:all_apps_version_info([{apps,[App]}]); read(ddSize,Table) -> PTN = physical_table_name(Table), case is_time_partitioned_table(PTN) of @@ -2549,6 +2556,7 @@ dirty_read({_Schema,Table}, Key) -> dirty_read(Table, Key); dirty_read(ddNode,Node) -> read(ddNode,Node); dirty_read(ddSchema,Key) -> read(ddSchema,Key); dirty_read(ddSize,Table) -> read(ddSize,Table); +dirty_read(ddVersion,App) -> read(ddVersion,App); dirty_read(Table, Key) -> imem_if_mnesia:dirty_read(physical_table_name(Table), Key). dirty_index_read({_Schema,Table}, SecKey,Index) -> @@ -2643,8 +2651,8 @@ select({ddSysConf,Table}, _MatchSpec, _Limit) -> select({_Schema,Table}, MatchSpec, Limit) -> select(Table, MatchSpec, Limit); %% ToDo: may depend on schema select(Tab, MatchSpec, Limit) when - Tab==ddNode;Tab==ddSnap;Tab==ddSchema;Tab==ddSize;Tab==ddSize;Tab==integer -> - select_virtual(Tab, MatchSpec,Limit); + Tab==ddNode;Tab==ddSnap;Tab==ddSchema;Tab==ddSize;Tab==ddVersion;Tab==integer -> + select_virtual(Tab, MatchSpec, Limit); select(Table, MatchSpec, Limit) -> imem_if_mnesia:select(physical_table_name(Table), MatchSpec, Limit). @@ -2670,13 +2678,13 @@ select_virtual(Table, [{MatchHead, [Guard], ['$_']}]=MatchSpec,_Limit) -> % ?Info("Virtual Select Tag / MatchSpec: ~p / ~p~n", [Tag,MatchSpec]), Candidates = case operand_match(Tag,Guard) of false -> read(Table); - {'==',Tag,{element,N,Tup1}} -> % ?Debug("Virtual Select Key : ~p~n", [element(N,Tup1)]), + {'==',Tag,{element,N,Tup1}} -> % ?Info("Virtual Select Key : ~p~n", [element(N,Tup1)]), read(Table,element(N,Tup1)); - {'==',{element,N,Tup2},Tag} -> % ?Debug("Virtual Select Key : ~p~n", [element(N,Tup2)]), + {'==',{element,N,Tup2},Tag} -> % ?Info("Virtual Select Key : ~p~n", [element(N,Tup2)]), read(Table,element(N,Tup2)); - {'==',Tag,Val1} -> % ?Debug("Virtual Select Key : ~p~n", [Val1]), + {'==',Tag,Val1} -> % ?Info("Virtual Select Key : ~p~n", [Val1]), read(Table,Val1); - {'==',Val2,Tag} -> % ?Debug("Virtual Select Key : ~p~n", [Val2]), + {'==',Val2,Tag} -> % ?Info("Virtual Select Key : ~p~n", [Val2]), read(Table,Val2); _ -> read(Table) end,