diff --git a/src/aotcompile.cpp b/src/aotcompile.cpp index 5e5fa131d260a..8388f238c25de 100644 --- a/src/aotcompile.cpp +++ b/src/aotcompile.cpp @@ -605,7 +605,8 @@ void addMachinePasses(legacy::PassManagerBase *PM, TargetMachine *TM, int optlev // this defines the set of optimization passes defined for Julia at various optimization levels. // it assumes that the TLI and TTI wrapper passes have already been added. void addOptimizationPasses(legacy::PassManagerBase *PM, int opt_level, - bool lower_intrinsics, bool dump_native) + bool lower_intrinsics, bool dump_native, + bool external_use) { // Note: LLVM 12 disabled the hoisting of common instruction // before loop vectorization (https://reviews.llvm.org/D84108). @@ -654,7 +655,7 @@ void addOptimizationPasses(legacy::PassManagerBase *PM, int opt_level, } PM->add(createLowerSimdLoopPass()); // Annotate loop marked with "loopinfo" as LLVM parallel loop if (dump_native) { - PM->add(createMultiVersioningPass()); + PM->add(createMultiVersioningPass(external_use)); PM->add(createCPUFeaturesPass()); // minimal clean-up to get rid of CPU feature checks if (opt_level == 1) { @@ -696,7 +697,7 @@ void addOptimizationPasses(legacy::PassManagerBase *PM, int opt_level, PM->add(createInstructionCombiningPass()); PM->add(createCFGSimplificationPass(simplifyCFGOptions)); if (dump_native) - PM->add(createMultiVersioningPass()); + PM->add(createMultiVersioningPass(external_use)); PM->add(createCPUFeaturesPass()); PM->add(createSROAPass()); PM->add(createInstSimplifyLegacyPass()); @@ -834,7 +835,7 @@ void addOptimizationPasses(legacy::PassManagerBase *PM, int opt_level, // An LLVM module pass that just runs all julia passes in order. Useful for // debugging -template +template class JuliaPipeline : public Pass { public: static char ID; @@ -849,7 +850,7 @@ class JuliaPipeline : public Pass { PMTopLevelManager *TPM = Stack.top()->getTopLevelManager(); TPMAdapter Adapter(TPM); addTargetPasses(&Adapter, &jl_ExecutionEngine->getTargetMachine()); - addOptimizationPasses(&Adapter, OptLevel); + addOptimizationPasses(&Adapter, OptLevel, true, dump_native, true); addMachinePasses(&Adapter, &jl_ExecutionEngine->getTargetMachine(), OptLevel); } JuliaPipeline() : Pass(PT_PassManager, ID) {} @@ -857,12 +858,19 @@ class JuliaPipeline : public Pass { return createPrintModulePass(O, Banner); } }; -template<> char JuliaPipeline<0>::ID = 0; -template<> char JuliaPipeline<2>::ID = 0; -template<> char JuliaPipeline<3>::ID = 0; -static RegisterPass> X("juliaO0", "Runs the entire julia pipeline (at -O0)", false, false); -static RegisterPass> Y("julia", "Runs the entire julia pipeline (at -O2)", false, false); -static RegisterPass> Z("juliaO3", "Runs the entire julia pipeline (at -O3)", false, false); +template<> char JuliaPipeline<0,false>::ID = 0; +template<> char JuliaPipeline<2,false>::ID = 0; +template<> char JuliaPipeline<3,false>::ID = 0; +template<> char JuliaPipeline<0,true>::ID = 0; +template<> char JuliaPipeline<2,true>::ID = 0; +template<> char JuliaPipeline<3,true>::ID = 0; +static RegisterPass> X("juliaO0", "Runs the entire julia pipeline (at -O0)", false, false); +static RegisterPass> Y("julia", "Runs the entire julia pipeline (at -O2)", false, false); +static RegisterPass> Z("juliaO3", "Runs the entire julia pipeline (at -O3)", false, false); + +static RegisterPass> XS("juliaO0-sysimg", "Runs the entire julia pipeline (at -O0/sysimg mode)", false, false); +static RegisterPass> YS("julia-sysimg", "Runs the entire julia pipeline (at -O2/sysimg mode)", false, false); +static RegisterPass> ZS("juliaO3-sysimg", "Runs the entire julia pipeline (at -O3/sysimg mode)", false, false); extern "C" JL_DLLEXPORT void jl_add_optimization_passes_impl(LLVMPassManagerRef PM, int opt_level, int lower_intrinsics) { diff --git a/src/jitlayers.cpp b/src/jitlayers.cpp index ac8ef9591a563..92614586a5456 100644 --- a/src/jitlayers.cpp +++ b/src/jitlayers.cpp @@ -18,6 +18,7 @@ #include #include #include +#include // target machine computation #include diff --git a/src/jitlayers.h b/src/jitlayers.h index 16f6eb6bc372f..d1545dca772f6 100644 --- a/src/jitlayers.h +++ b/src/jitlayers.h @@ -51,7 +51,7 @@ extern "C" jl_cgparams_t jl_default_cgparams; extern bool imaging_mode; void addTargetPasses(legacy::PassManagerBase *PM, TargetMachine *TM); -void addOptimizationPasses(legacy::PassManagerBase *PM, int opt_level, bool lower_intrinsics=true, bool dump_native=false); +void addOptimizationPasses(legacy::PassManagerBase *PM, int opt_level, bool lower_intrinsics=true, bool dump_native=false, bool external_use=false); void addMachinePasses(legacy::PassManagerBase *PM, TargetMachine *TM, int optlevel); void jl_finalize_module(std::unique_ptr m); void jl_merge_module(Module *dest, std::unique_ptr src); @@ -283,7 +283,7 @@ Pass *createPropagateJuliaAddrspaces(); Pass *createRemoveJuliaAddrspacesPass(); Pass *createRemoveNIPass(); Pass *createJuliaLICMPass(); -Pass *createMultiVersioningPass(); +Pass *createMultiVersioningPass(bool external_use); Pass *createAllocOptPass(); Pass *createDemoteFloat16Pass(); Pass *createCPUFeaturesPass(); diff --git a/src/llvm-multiversioning.cpp b/src/llvm-multiversioning.cpp index 3a377547da7eb..7f145e31499e7 100644 --- a/src/llvm-multiversioning.cpp +++ b/src/llvm-multiversioning.cpp @@ -242,7 +242,7 @@ struct CloneCtx { return cast(vmap->lookup(orig_f)); } }; - CloneCtx(Module &M, function_ref GetLI, function_ref GetCG); + CloneCtx(Module &M, function_ref GetLI, function_ref GetCG, bool allow_bad_fvars); void clone_bases(); void collect_func_infos(); void clone_all_partials(); @@ -292,10 +292,11 @@ struct CloneCtx { std::set alias_relocs; bool has_veccall{false}; bool has_cloneall{false}; + bool allow_bad_fvars{false}; }; template -static inline std::vector consume_gv(Module &M, const char *name) +static inline std::vector consume_gv(Module &M, const char *name, bool allow_bad_fvars) { // Get information about sysimg export functions from the two global variables. // Strip them from the Module so that it's easier to handle the uses. @@ -304,8 +305,17 @@ static inline std::vector consume_gv(Module &M, const char *name) auto *ary = cast(gv->getInitializer()); unsigned nele = ary->getNumOperands(); std::vector res(nele); - for (unsigned i = 0; i < nele; i++) - res[i] = cast(ary->getOperand(i)->stripPointerCasts()); + unsigned i = 0; + while (i < nele) { + llvm::Value *val = ary->getOperand(i)->stripPointerCasts(); + if (allow_bad_fvars && (!isa(val) || (isa(val) && cast(val)->isDeclaration()))) { + // Shouldn't happen in regular use, but can happen in bugpoint. + nele--; + continue; + } + res[i++] = cast(val); + } + res.resize(nele); assert(gv->use_empty()); gv->eraseFromParent(); if (ary->use_empty()) @@ -314,14 +324,15 @@ static inline std::vector consume_gv(Module &M, const char *name) } // Collect basic information about targets and functions. -CloneCtx::CloneCtx(Module &M, function_ref GetLI, function_ref GetCG) +CloneCtx::CloneCtx(Module &M, function_ref GetLI, function_ref GetCG, bool allow_bad_fvars) : tbaa_const(tbaa_make_child_with_context(M.getContext(), "jtbaa_const", nullptr, true).first), specs(jl_get_llvm_clone_targets()), - fvars(consume_gv(M, "jl_sysimg_fvars")), - gvars(consume_gv(M, "jl_sysimg_gvars")), + fvars(consume_gv(M, "jl_sysimg_fvars", allow_bad_fvars)), + gvars(consume_gv(M, "jl_sysimg_gvars", false)), M(M), GetLI(GetLI), - GetCG(GetCG) + GetCG(GetCG), + allow_bad_fvars(allow_bad_fvars) { groups.emplace_back(0, specs[0]); uint32_t ntargets = specs.size(); @@ -636,6 +647,12 @@ uint32_t CloneCtx::get_func_id(Function *F) { auto &ref = func_ids[F]; if (!ref) { + if (allow_bad_fvars && F->isDeclaration()) { + // This should never happen in regular use, but can happen if + // bugpoint deletes the function. Just do something here to + // allow bugpoint to proceed. + return (uint32_t)-1; + } fvars.push_back(F); ref = fvars.size(); } @@ -927,10 +944,15 @@ Constant *CloneCtx::emit_offset_table(const std::vector &vars, StringRef nam void CloneCtx::emit_metadata() { + uint32_t nfvars = fvars.size(); + if (allow_bad_fvars && nfvars == 0) { + // Will result in a non-loadable sysimg, but `allow_bad_fvars` is for bugpoint only + return; + } + // Store back the information about exported functions. auto fbase = emit_offset_table(fvars, "jl_sysimg_fvars"); auto gbase = emit_offset_table(gvars, "jl_sysimg_gvars"); - uint32_t nfvars = fvars.size(); uint32_t ntargets = specs.size(); SmallVector targets(ntargets); @@ -1057,7 +1079,7 @@ void CloneCtx::emit_metadata() } } -static bool runMultiVersioning(Module &M, function_ref GetLI, function_ref GetCG) +static bool runMultiVersioning(Module &M, function_ref GetLI, function_ref GetCG, bool allow_bad_fvars) { // Group targets and identify cloning bases. // Also initialize function info maps (we'll update these maps as we go) @@ -1070,7 +1092,13 @@ static bool runMultiVersioning(Module &M, function_ref Get if (M.getName() == "sysimage") return false; - CloneCtx clone(M, GetLI, GetCG); + GlobalVariable *fvars = M.getGlobalVariable("jl_sysimg_fvars"); + GlobalVariable *gvars = M.getGlobalVariable("jl_sysimg_gvars"); + if (allow_bad_fvars && (!fvars || !fvars->hasInitializer() || !isa(fvars->getInitializer()) || + !gvars || !gvars->hasInitializer() || !isa(gvars->getInitializer()))) + return false; + + CloneCtx clone(M, GetLI, GetCG, allow_bad_fvars); // Collect a list of original functions and clone base functions clone.clone_bases(); @@ -1110,8 +1138,8 @@ static bool runMultiVersioning(Module &M, function_ref Get struct MultiVersioningLegacy: public ModulePass { static char ID; - MultiVersioningLegacy() - : ModulePass(ID) + MultiVersioningLegacy(bool allow_bad_fvars=false) + : ModulePass(ID), allow_bad_fvars(allow_bad_fvars) {} private: @@ -1122,6 +1150,7 @@ struct MultiVersioningLegacy: public ModulePass { AU.addRequired(); AU.addPreserved(); } + bool allow_bad_fvars; }; bool MultiVersioningLegacy::runOnModule(Module &M) @@ -1132,7 +1161,7 @@ bool MultiVersioningLegacy::runOnModule(Module &M) auto GetCG = [this]() -> CallGraph & { return getAnalysis().getCallGraph(); }; - return runMultiVersioning(M, GetLI, GetCG); + return runMultiVersioning(M, GetLI, GetCG, allow_bad_fvars); } @@ -1152,7 +1181,7 @@ PreservedAnalyses MultiVersioning::run(Module &M, ModuleAnalysisManager &AM) auto GetCG = [&]() -> CallGraph & { return AM.getResult(M); }; - if (runMultiVersioning(M, GetLI, GetCG)) { + if (runMultiVersioning(M, GetLI, GetCG, false)) { auto preserved = PreservedAnalyses::allInSet(); preserved.preserve(); return preserved; @@ -1160,12 +1189,12 @@ PreservedAnalyses MultiVersioning::run(Module &M, ModuleAnalysisManager &AM) return PreservedAnalyses::all(); } -Pass *createMultiVersioningPass() +Pass *createMultiVersioningPass(bool allow_bad_fvars) { - return new MultiVersioningLegacy(); + return new MultiVersioningLegacy(allow_bad_fvars); } extern "C" JL_DLLEXPORT void LLVMExtraAddMultiVersioningPass_impl(LLVMPassManagerRef PM) { - unwrap(PM)->add(createMultiVersioningPass()); + unwrap(PM)->add(createMultiVersioningPass(false)); }