diff --git a/rir/src/compiler/analysis/verifier.cpp b/rir/src/compiler/analysis/verifier.cpp index 2ae69d1e8..156bdbac4 100644 --- a/rir/src/compiler/analysis/verifier.cpp +++ b/rir/src/compiler/analysis/verifier.cpp @@ -272,7 +272,7 @@ class TheVerifier { } if (auto assume = Assume::Cast(i)) { if (IsType::Cast(assume->arg(0).val())) { - if (assume->feedbackOrigin.empty()) { + if (!assume->reason.pc()) { std::cerr << "Error: instruction '"; i->print(std::cerr); std::cerr << "' typecheck without origin information\n"; diff --git a/rir/src/compiler/backend.cpp b/rir/src/compiler/backend.cpp index 19ea9738a..3ed677851 100644 --- a/rir/src/compiler/backend.cpp +++ b/rir/src/compiler/backend.cpp @@ -204,7 +204,7 @@ static void lower(Code* code) { next = it + 1; } else if (auto expect = Assume::Cast(*it)) { - if (expect->arg(0).val() == True::instance()) { + if (expect->triviallyHolds()) { next = bb->remove(it); } else { auto expectation = expect->assumeTrue; diff --git a/rir/src/compiler/native/lower_function_llvm.cpp b/rir/src/compiler/native/lower_function_llvm.cpp index 00d446c33..55645e1c3 100644 --- a/rir/src/compiler/native/lower_function_llvm.cpp +++ b/rir/src/compiler/native/lower_function_llvm.cpp @@ -80,7 +80,8 @@ class NativeAllocator : public SSAAllocator { // Ensure we preserve slots for variables with typefeedback to make them // accessible to the runtime profiler. // TODO: this needs to be replaced by proper mapping of slots. - if (a != b && (a->typeFeedback().origin || b->typeFeedback().origin)) + if (a != b && (a->typeFeedback().feedbackOrigin.pc() || + b->typeFeedback().feedbackOrigin.pc())) return true; return SSAAllocator::interfere(a, b); } @@ -1927,14 +1928,15 @@ llvm::Value* LowerFunctionLLVM::fastVeceltOkNative(llvm::Value* v) { auto attrs = attr(v); auto isNil = builder.CreateICmpEQ(attrs, constant(R_NilValue, t::SEXP)); auto ok = builder.CreateAnd(builder.CreateNot(isObj(v)), isNil); - return createSelect2(ok, [&]() { return builder.getTrue(); }, - [&]() { - auto isMatr1 = builder.CreateICmpEQ( - tag(attrs), constant(R_DimSymbol, t::SEXP)); - auto isMatr2 = builder.CreateICmpEQ( - cdr(attrs), constant(R_NilValue, t::SEXP)); - return builder.CreateAnd(isMatr1, isMatr2); - }); + return createSelect2( + ok, [&]() { return builder.getTrue(); }, + [&]() { + auto isMatr1 = builder.CreateICmpEQ(tag(attrs), + constant(R_DimSymbol, t::SEXP)); + auto isMatr2 = + builder.CreateICmpEQ(cdr(attrs), constant(R_NilValue, t::SEXP)); + return builder.CreateAnd(isMatr1, isMatr2); + }); }; llvm::Value* LowerFunctionLLVM::isAltrep(llvm::Value* v) { @@ -2291,15 +2293,14 @@ void LowerFunctionLLVM::compile() { llvm::ConstantInt::get( PirJitLLVM::getContext(), llvm::APInt( - 64, reinterpret_cast(rec->reason.srcCode), + 64, + reinterpret_cast(rec->reason.srcCode()), false)), t::voidPtr); auto reason = llvm::ConstantStruct::get( - t::DeoptReason, { - c(rec->reason.reason, 32), - srcAddr, - c(rec->reason.originOffset), - }); + t::DeoptReason, + {c(rec->reason.reason, 32), + c(rec->reason.origin.offset(), 32), srcAddr}); call(NativeBuiltins::get(NativeBuiltins::Id::recordDeopt), {loadSxp(rec->arg<0>().val()), globalConst(reason)}); break; @@ -5896,12 +5897,12 @@ void LowerFunctionLLVM::compile() { auto i = var.first; if (Representation::Of(i) != Representation::Sexp) continue; - if (!i->typeFeedback().origin) + if (!i->typeFeedback().feedbackOrigin.pc()) continue; if (!var.second.initialized) continue; if (var.second.stackSlot < PirTypeFeedback::MAX_SLOT_IDX) { - codes.insert(i->typeFeedback().srcCode); + codes.insert(i->typeFeedback().feedbackOrigin.srcCode()); variableMapping.emplace(var.second.stackSlot, i->typeFeedback()); #ifdef DEBUG_REGISTER_MAP assert(!usedSlots.count(var.second.stackSlot)); diff --git a/rir/src/compiler/native/types_llvm.cpp b/rir/src/compiler/native/types_llvm.cpp index 51fd4212c..a1cb1faac 100644 --- a/rir/src/compiler/native/types_llvm.cpp +++ b/rir/src/compiler/native/types_llvm.cpp @@ -109,7 +109,7 @@ void initializeTypes(LLVMContext& context) { t::RCNTXT->setBody(fields); t::DeoptReason = StructType::create(context, "DeoptReason"); - fields = {t::i32, t::voidPtr, t::i32}; + fields = {t::i32, t::i32, t::voidPtr}; t::DeoptReason->setBody(fields, true); #define DECLARE(name, ret, ...) \ diff --git a/rir/src/compiler/opt/assumptions.cpp b/rir/src/compiler/opt/assumptions.cpp index 916aedfcd..83cecacbb 100644 --- a/rir/src/compiler/opt/assumptions.cpp +++ b/rir/src/compiler/opt/assumptions.cpp @@ -20,36 +20,44 @@ struct AAssumption { } explicit AAssumption(Assume* a) : yesNo(a->assumeTrue) { auto cond = a->condition(); - if (auto t = IsType::Cast(cond)) { - kind = Typecheck; - c.typecheck = {t->arg(0).val(), t->typeTest}; - } else if (auto t = Identical::Cast(cond)) { - kind = Equality; - c.equality = {t->arg(0).val(), t->arg(1).val()}; - } else if (auto t = IsEnvStub::Cast(cond)) { - kind = IsEnvStub; - c.env = t->env(); - } else { - kind = Other; - c.misc = cond; + switch (a->reason.reason) { + case DeoptReason::Typecheck: { + if (auto t = IsType::Cast(cond)) { + c.typecheck = {t->arg(0).val(), t->typeTest}; + kind = Typecheck; + return; + } + break; } - } - explicit AAssumption(Branch* b, bool yesNo) : yesNo(yesNo) { - auto cond = b->arg(0).val(); - if (auto t = IsType::Cast(cond)) { - kind = Typecheck; - c.typecheck = {t->arg(0).val(), t->typeTest}; - } else if (auto t = Identical::Cast(cond)) { - kind = Equality; - c.equality = {t->arg(0).val(), t->arg(1).val()}; - } else if (auto t = IsEnvStub::Cast(cond)) { - kind = IsEnvStub; - c.env = t->env(); - } else { - kind = Other; - c.misc = cond; + case DeoptReason::Calltarget: { + if (auto t = Identical::Cast(cond)) { + c.equality = {t->arg(0).val(), t->arg(1).val()}; + kind = Equality; + return; + } + break; + } + case DeoptReason::EnvStubMaterialized: { + if (auto t = IsEnvStub::Cast(cond)) { + c.env = t->env(); + kind = IsEnvStub; + return; + } + break; } + case DeoptReason::DeadBranchReached: + break; + case DeoptReason::DeadCall: + assert(false); + break; + } + + // Fallthrough generic case. Assumptions are identified by the concrete + // condition alone. + kind = Other; + c.misc = cond; } + AAssumption& operator=(const AAssumption& o) { yesNo = o.yesNo; kind = o.kind; @@ -253,14 +261,6 @@ bool OptimizeAssumptions::apply(Compiler&, ClosureVersion* vers, Code* code, return false; }; - if (auto br = Branch::Cast(instr)) { - if (assumptionsIncludes(AAssumption(br, true))) { - br->arg(0).val() = True::instance(); - } else if (assumptionsIncludes(AAssumption(br, false))) { - br->arg(0).val() = False::instance(); - } - } - if (auto assume = Assume::Cast(instr)) { if (assumptionsIncludes(AAssumption(assume))) { anyChange = true; @@ -327,11 +327,9 @@ bool OptimizeAssumptions::apply(Compiler&, ClosureVersion* vers, Code* code, if (h != hoistAssume.end()) { auto g = h->second; ip++; - auto assume = new Assume(std::get<0>(g), std::get<1>(g)); - assume->feedbackOrigin.insert( - assume->feedbackOrigin.end(), - std::get<2>(g)->feedbackOrigin.begin(), - std::get<2>(g)->feedbackOrigin.end()); + auto assume = new Assume(std::get(g), + std::get(g), + std::get(g)->reason); ip = bb->insert(ip, assume); anyChange = true; } diff --git a/rir/src/compiler/opt/constantfold.cpp b/rir/src/compiler/opt/constantfold.cpp index 8157c874c..4dbfec030 100644 --- a/rir/src/compiler/opt/constantfold.cpp +++ b/rir/src/compiler/opt/constantfold.cpp @@ -302,7 +302,7 @@ bool Constantfold::apply(Compiler& cmp, ClosureVersion* cls, Code* code, auto next = ip + 1; auto killUnreachable = [&]() { - if (ip == bb->end() || Unreachable::Cast(*(ip + 1))) + if (ip == bb->end() || Unreachable::Cast(bb->last())) return; ip = bb->insert(ip + 1, new Unreachable()) + 1; while (ip != bb->end()) diff --git a/rir/src/compiler/opt/eager_calls.cpp b/rir/src/compiler/opt/eager_calls.cpp index 14eadb627..628f1bd2c 100644 --- a/rir/src/compiler/opt/eager_calls.cpp +++ b/rir/src/compiler/opt/eager_calls.cpp @@ -18,16 +18,22 @@ bool EagerCalls::apply(Compiler& cmp, ClosureVersion* cls, Code* code, LogStream& log) const { AvailableCheckpoints checkpoint(cls, code, log); + struct Speculation { + SEXP builtin; + Checkpoint* cp; + FeedbackOrigin origin; + }; + auto replaceLdFunBuiltinWithDeopt = [&](BB* bb, BB::Instrs::iterator ip, - Checkpoint* cp, SEXP builtin, + const Speculation& speculation, LdFun* ldfun) { assert(LdFun::Cast(*ip)); - assert(cp); + assert(speculation.cp); // skip ldfun ++ip; - auto expected = new LdConst(builtin); + auto expected = new LdConst(speculation.builtin); ip = bb->insert(ip, expected); ++ip; Instruction* given = ldfun; @@ -54,7 +60,9 @@ bool EagerCalls::apply(Compiler& cmp, ClosureVersion* cls, Code* code, ip = bb->insert(ip, test); ++ip; - auto assume = new Assume(test, cp); + auto assume = new Assume( + test, speculation.cp, + DeoptReason(speculation.origin, DeoptReason::Calltarget)); ip = bb->insert(ip, assume); ++ip; @@ -105,7 +113,8 @@ bool EagerCalls::apply(Compiler& cmp, ClosureVersion* cls, Code* code, }; // Search for calls that likely point to a builtin. - std::unordered_map> needsGuard; + std::unordered_map needsGuard; + Visitor::run(code->entry, [&](BB* bb) { auto ip = bb->begin(); while (ip != bb->end()) { @@ -144,8 +153,8 @@ bool EagerCalls::apply(Compiler& cmp, ClosureVersion* cls, Code* code, Effect::DependsOnAssume)); } } else if (auto ldfun = LdFun::Cast(call->cls())) { - if (ldfun->hint && !ldfun->hintIsInnerFunction) { - auto kind = TYPEOF(ldfun->hint); + if (ldfun->hint() && !ldfun->hintIsInnerFunction) { + auto kind = TYPEOF(ldfun->hint()); // We also speculate on calls to CLOSXPs, these will // be picked up by MatchArgs opt pass and turned // into a static call. TODO, for inner functions we @@ -157,9 +166,10 @@ bool EagerCalls::apply(Compiler& cmp, ClosureVersion* cls, Code* code, if (auto cp = checkpoint.at(ldfun)) { if (kind == BUILTINSXP) { ip = replaceCallWithCallBuiltin( - bb, ip, call, ldfun->hint, true); + bb, ip, call, ldfun->hint(), true); } - needsGuard[ldfun] = {ldfun->hint, cp}; + needsGuard[ldfun] = {ldfun->hint(), cp, + ldfun->hintOrigin()}; } } } else { @@ -218,8 +228,7 @@ bool EagerCalls::apply(Compiler& cmp, ClosureVersion* cls, Code* code, if (auto ldfun = LdFun::Cast(*ip)) { auto r = needsGuard.find(ldfun); if (r != needsGuard.end()) { - ip = replaceLdFunBuiltinWithDeopt(bb, ip, r->second.second, - r->second.first, ldfun); + ip = replaceLdFunBuiltinWithDeopt(bb, ip, r->second, ldfun); needsGuard.erase(r); continue; } diff --git a/rir/src/compiler/opt/elide_env_spec.cpp b/rir/src/compiler/opt/elide_env_spec.cpp index 87438d142..b81e2af53 100644 --- a/rir/src/compiler/opt/elide_env_spec.cpp +++ b/rir/src/compiler/opt/elide_env_spec.cpp @@ -87,8 +87,9 @@ bool ElideEnvSpec::apply(Compiler&, ClosureVersion* cls, Code* code, arg, seen, suggested, required, [&](TypeTest::Info info) { BBTransform::insertAssume( - info.test, cp, bb, ip, info.expectation, - info.srcCode, info.origin); + info.test, info.expectation, cp, + info.feedbackOrigin, DeoptReason::Typecheck, + bb, ip); if (argi) { auto cast = new CastType( @@ -129,6 +130,7 @@ bool ElideEnvSpec::apply(Compiler&, ClosureVersion* cls, Code* code, !arg->type.maybeMissing()) { force->replaceUsesWith(arg); next = bb->remove(ip); + anyChange = true; } } } @@ -281,8 +283,10 @@ bool ElideEnvSpec::apply(Compiler&, ClosureVersion* cls, Code* code, if (!bannedEnvs.count(env)) { auto condition = new IsEnvStub(env); BBTransform::insertAssume( - condition, cp, true, - env->typeFeedback().srcCode, nullptr); + condition, true, cp, + env->typeFeedback().feedbackOrigin, + DeoptReason::EnvStubMaterialized); + anyChange = true; assert(cp->bb()->trueBranch() != bb); } } @@ -301,6 +305,7 @@ bool ElideEnvSpec::apply(Compiler&, ClosureVersion* cls, Code* code, for (auto n : additionalEntries[env]) { env->varName.push_back(n); env->pushArg(UnboundValue::instance(), PirType::any()); + anyChange = true; } // After eliding an env we must ensure to add a // materialization before every usage in deopt branches @@ -331,17 +336,20 @@ bool ElideEnvSpec::apply(Compiler&, ClosureVersion* cls, Code* code, !i->effects.contains(Effect::DependsOnAssume) && MkEnv::Cast(i->env()) && MkEnv::Cast(i->env())->stub) { i->effects.set(Effect::DependsOnAssume); + anyChange = true; } if (auto is = IsEnvStub::Cast(i)) { if (!materializableStubs.count(i->env())) { is->replaceUsesWith(True::instance()); is->effects.reset(); + anyChange = true; } } if (auto mk = MkArg::Cast(i)) { if (materializableStubs.count(mk->env())) { if (auto e = mk->prom()->env()) { e->stub = true; + anyChange = true; } } } diff --git a/rir/src/compiler/opt/match_call_args.cpp b/rir/src/compiler/opt/match_call_args.cpp index f39800543..28f2c29f2 100644 --- a/rir/src/compiler/opt/match_call_args.cpp +++ b/rir/src/compiler/opt/match_call_args.cpp @@ -203,7 +203,8 @@ bool MatchCallArgs::apply(Compiler& cmp, ClosureVersion* cls, Code* code, false, [&](ClosureVersion* fun) { target = fun; }, []() {}, {}); - } else if (auto cnst = LdConst::Cast(calli->tryGetClsArg())) { + } else if (auto cnst = + LdConst::Cast(calli->tryGetClsArg())) { if (auto dt = DispatchTable::check(BODY(cnst->c()))) { if (dt->baseline()->body()->codeSize < Parameter::RECOMPILE_THRESHOLD) @@ -222,14 +223,14 @@ bool MatchCallArgs::apply(Compiler& cmp, ClosureVersion* cls, Code* code, auto srcRef = mk->srcRef; if (dt->baseline()->body()->codeSize < Parameter::RECOMPILE_THRESHOLD) - cmp.compileFunction(dt, "unknown--fromMkFunCls", - formals, srcRef, asmpt, - [&](ClosureVersion* fun) { - mk->setCls( - fun->owner()); - target = fun; - }, - []() {}, {}); + cmp.compileFunction( + dt, "unknown--fromMkFunCls", formals, + srcRef, asmpt, + [&](ClosureVersion* fun) { + mk->setCls(fun->owner()); + target = fun; + }, + []() {}, {}); } } } diff --git a/rir/src/compiler/opt/type_speculation.cpp b/rir/src/compiler/opt/type_speculation.cpp index c42b8d899..f0c0a066f 100644 --- a/rir/src/compiler/opt/type_speculation.cpp +++ b/rir/src/compiler/opt/type_speculation.cpp @@ -130,8 +130,9 @@ bool TypeSpeculation::apply(Compiler&, ClosureVersion* cls, Code* code, auto cp = sp.second.first; TypeTest::Info& info = sp.second.second; - BBTransform::insertAssume(info.test, cp, bb, ip, info.expectation, - info.srcCode, info.origin); + BBTransform::insertAssume(info.test, info.expectation, cp, + info.feedbackOrigin, + DeoptReason::Typecheck, bb, ip); auto cast = new CastType(i, CastType::Downcast, PirType::any(), info.result); diff --git a/rir/src/compiler/opt/type_test.h b/rir/src/compiler/opt/type_test.h index 950b14620..d45a2048a 100644 --- a/rir/src/compiler/opt/type_test.h +++ b/rir/src/compiler/opt/type_test.h @@ -12,8 +12,7 @@ class TypeTest { PirType result; Instruction* test; bool expectation; - rir::Code* srcCode; - Opcode* origin; + FeedbackOrigin feedbackOrigin; }; static void Create(Value* i, const TypeFeedback& feedback, const PirType& suggested, const PirType& required, @@ -31,14 +30,14 @@ class TypeTest { return failed(); } - assert(feedback.origin); + assert(feedback.feedbackOrigin.pc()); // First try to refine the type if (!expected.maybeObj() && // TODO: Is this right? (expected.noAttribsOrObject().isA(RType::integer) || expected.noAttribsOrObject().isA(RType::real) || expected.noAttribsOrObject().isA(RType::logical))) { return action({expected, new IsType(expected, i), true, - feedback.srcCode, feedback.origin}); + feedback.feedbackOrigin}); } // Second try to test for object-ness, or attribute-ness. @@ -53,14 +52,14 @@ class TypeTest { assert(!expected.maybeObj()); assert(!expected.maybeHasAttrs()); return action({checkFor, new IsType(checkFor, i), true, - feedback.srcCode, feedback.origin}); + feedback.feedbackOrigin}); } checkFor = i->type.notLazy().notObject(); if (expected.isA(checkFor)) { assert(!expected.maybeObj()); return action({checkFor, new IsType(checkFor, i), true, - feedback.srcCode, feedback.origin}); + feedback.feedbackOrigin}); } if (i->type.isA(required)) diff --git a/rir/src/compiler/pir/builder.cpp b/rir/src/compiler/pir/builder.cpp index fd63c9f5e..1099b3891 100644 --- a/rir/src/compiler/pir/builder.cpp +++ b/rir/src/compiler/pir/builder.cpp @@ -132,7 +132,7 @@ Builder::Builder(ClosureVersion* version, Value* closureEnv) auto rirCode = version->owner()->rirFunction()->body(); if (rirCode->flags.contains(rir::Code::NeedsFullEnv)) mkenv->neverStub = true; - mkenv->updateTypeFeedback().srcCode = rirCode; + mkenv->updateTypeFeedback().feedbackOrigin.srcCode(rirCode); add(mkenv); this->env = mkenv; } diff --git a/rir/src/compiler/pir/instruction.cpp b/rir/src/compiler/pir/instruction.cpp index af316a36c..cff770e34 100644 --- a/rir/src/compiler/pir/instruction.cpp +++ b/rir/src/compiler/pir/instruction.cpp @@ -615,8 +615,8 @@ void LdFun::printArgs(std::ostream& out, bool tty) const { guessedBinding()->printRef(out); out << ">, "; } - if (hint && hint != symbol::ambiguousCallTarget) { - out << "<" << hint << ">, "; + if (hint_ && hint_ != symbol::ambiguousCallTarget) { + out << "<" << hint_ << ">, "; } } @@ -1280,6 +1280,45 @@ void Checkpoint::printGraphBranches(std::ostream& out, size_t bbId) const { BB* Checkpoint::deoptBranch() { return bb()->falseBranch(); } BB* Checkpoint::nextBB() { return bb()->trueBranch(); } +Value* Assume::valueUnderTest() const { + switch (reason.reason) { + case DeoptReason::Typecheck: { + if (auto t = IsType::Cast(condition())) + return t->arg<0>().val(); + break; + } + case DeoptReason::Calltarget: { + if (auto t = Identical::Cast(condition())) { + auto value = t->arg<0>().val(); + if (LdConst::Cast(value)) + value = t->arg<1>().val(); + assert(!LdConst::Cast(value)); + return value; + } + break; + } + case DeoptReason::EnvStubMaterialized: { + if (auto t = IsEnvStub::Cast(condition())) + return t->arg(0).val(); + break; + } + case DeoptReason::DeadBranchReached: { + return condition(); + } + // DeadCall is an unconditional deopt and never associated with an + // assume. + case DeoptReason::DeadCall: + assert(false); + } + + if (Instruction::Cast(condition())) { + printRecursive(std::cerr, 2); + assert(false && "Unexpected condition for this kind of assumption"); + } + + return nullptr; +} + PirType Colon::inferType(const GetType& getType) const { auto convertsToInt = [](Value* a) { if (a->type.isA(RType::integer)) diff --git a/rir/src/compiler/pir/instruction.h b/rir/src/compiler/pir/instruction.h index cc6d099af..91eb35abd 100644 --- a/rir/src/compiler/pir/instruction.h +++ b/rir/src/compiler/pir/instruction.h @@ -19,6 +19,7 @@ #include #include #include +#include #include #include @@ -143,8 +144,7 @@ enum class VisibilityFlag : uint8_t { struct TypeFeedback { PirType type = PirType::optimistic(); Value* value = nullptr; - rir::Code* srcCode = nullptr; - Opcode* origin = nullptr; + FeedbackOrigin feedbackOrigin; bool used = false; }; @@ -935,9 +935,18 @@ class VLIE(FrameState, }; class FLIE(LdFun, 2, Effects::Any()) { + private: + SEXP hint_ = nullptr; + FeedbackOrigin hintOrigin_; + public: + SEXP hint() { return hint_; } + const FeedbackOrigin& hintOrigin() { return hintOrigin_; } + void hint(SEXP hint, const FeedbackOrigin& hintOrigin) { + hint_ = hint; + hintOrigin_ = hintOrigin; + } SEXP varName; - SEXP hint = nullptr; bool hintIsInnerFunction = false; LdFun(const char* name, Value* env) @@ -2643,23 +2652,33 @@ class Deopt : public FixedLenInstruction> feedbackOrigin; - bool assumeTrue = true; - Assume(Value* test, Value* checkpoint) + const bool assumeTrue = true; + const DeoptReason reason; + + Assume(Value* test, Value* checkpoint, const DeoptReason& r, + bool expectation = true) : FixedLenInstruction(PirType::voyd(), {{PirType::test(), NativeType::checkpoint}}, - {{test, checkpoint}}) {} + {{test, checkpoint}}), + assumeTrue(expectation), reason(r) { + assert(reason.reason != DeoptReason::DeadCall); + } Checkpoint* checkpoint() const { return Checkpoint::Cast(arg(1).val()); } void checkpoint(Checkpoint* cp) { arg(1).val() = cp; } Value* condition() const { return arg(0).val(); } - Assume* Not() { - assumeTrue = !assumeTrue; - return this; - } std::string name() const override { return assumeTrue ? "Assume" : "AssumeNot"; } + + bool triviallyHolds() const { + + auto pirBool = + assumeTrue ? (Value*)True::instance() : (Value*)False::instance(); + return arg(0).val() == pirBool; + } + + Value* valueUnderTest() const; }; class ScheduledDeopt diff --git a/rir/src/compiler/rir2pir/rir2pir.cpp b/rir/src/compiler/rir2pir/rir2pir.cpp index b3763bbcc..00544ce9d 100644 --- a/rir/src/compiler/rir2pir/rir2pir.cpp +++ b/rir/src/compiler/rir2pir/rir2pir.cpp @@ -190,6 +190,7 @@ struct TargetInfo { SEXP monomorphic; size_t taken; bool stableEnv; + FeedbackOrigin feedbackOrigin; }; static TargetInfo checkCallTarget(Value* callee, rir::Code* srcCode, @@ -203,6 +204,8 @@ checkCallTarget(Value* callee, rir::Code* srcCode, auto& feedback = std::get(feedbackIt->second); result.taken = feedback.taken; if (result.taken > 1) { + result.feedbackOrigin = + FeedbackOrigin(srcCode, std::get(feedbackIt->second)); if (feedback.numTargets == 1) { result.monomorphic = feedback.getTarget(srcCode, 0); result.stableEnv = true; @@ -294,8 +297,9 @@ static Value* insertLdFunGuard(const TargetInfo& trg, Value* callee, auto t = new Identical(calleeForGuard, expected, PirType::any()); pos = bb->insert(pos, t) + 1; - auto assumption = new Assume(t, cp); - assumption->feedbackOrigin.push_back({srcCode, pc}); + auto assumption = new Assume( + t, cp, + DeoptReason(FeedbackOrigin(srcCode, pc), DeoptReason::Calltarget)); pos = bb->insert(pos, assumption) + 1; if (trg.stableEnv) @@ -508,8 +512,7 @@ bool Rir2Pir::compileBC(const BC& bc, Opcode* pos, Opcode* nextPos, if (!i->typeFeedback().value) { auto& t = i->updateTypeFeedback(); t.value = v; - t.srcCode = srcCode; - t.origin = pos; + t.feedbackOrigin = FeedbackOrigin(srcCode, pos); } else if (i->typeFeedback().value != v) { i->updateTypeFeedback().value = nullptr; } @@ -540,8 +543,7 @@ bool Rir2Pir::compileBC(const BC& bc, Opcode* pos, Opcode* nextPos, // TODO: deal with multiple locations auto& t = i->updateTypeFeedback(); t.type.merge(feedback); - t.srcCode = srcCode; - t.origin = pos; + t.feedbackOrigin = FeedbackOrigin(srcCode, pos); if (auto force = Force::Cast(i)) { force->observed = static_cast( feedback.stateBeforeLastForce); @@ -562,9 +564,10 @@ bool Rir2Pir::compileBC(const BC& bc, Opcode* pos, Opcode* nextPos, srcCode->funInvocationCount > 1 && srcCode->deadCallReached < 3) { auto sp = insert.registerFrameState(srcCode, pos, stack, inPromise()); - auto offset = (uintptr_t)pos - (uintptr_t)srcCode; - DeoptReason reason = {DeoptReason::DeadCall, srcCode, - (uint32_t)offset}; + + DeoptReason reason = DeoptReason(FeedbackOrigin(srcCode, pos), + DeoptReason::DeadCall); + insert(new RecordDeoptReason(reason, target)); insert(new Deopt(sp)); stack.clear(); @@ -642,11 +645,11 @@ bool Rir2Pir::compileBC(const BC& bc, Opcode* pos, Opcode* nextPos, auto ldfun = LdFun::Cast(callee); if (ldfun) { if (ti.monomorphic) { - ldfun->hint = ti.monomorphic; + ldfun->hint(ti.monomorphic, ti.feedbackOrigin); if (!ti.stableEnv) ldfun->hintIsInnerFunction = true; } else { - ldfun->hint = symbol::ambiguousCallTarget; + ldfun->hint(symbol::ambiguousCallTarget, {}); } } @@ -1386,6 +1389,7 @@ Value* Rir2Pir::tryTranslate(rir::Code* srcCode, Builder& insert) { } const auto pos = finger; BC bc = BC::advance(&finger, srcCode); + // cppcheck-suppress variableScope const auto nextPos = finger; assert(pos != end); @@ -1396,105 +1400,91 @@ Value* Rir2Pir::tryTranslate(rir::Code* srcCode, Builder& insert) { continue; } - bool swapTrueFalse = false; - Instruction* deoptCondition = nullptr; - Value* branchCondition; - bool assumeBB0 = false; + if (bc.bc == Opcode::beginloop_) { + log.warn("Cannot compile Function. Unsupported beginloop bc"); + return nullptr; + } // Conditional jump - switch (bc.bc) { - case Opcode::brtrue_: - case Opcode::brfalse_: { - auto v = branchCondition = cur.stack.pop(); + assert(bc.isCondJmp()); + auto branchCondition = cur.stack.top(); + auto branchReason = bc.bc == Opcode::brtrue_ + ? (Value*)True::instance() + : (Value*)False::instance(); + auto asBool = insert( + new Identical(branchCondition, branchReason, PirType::val())); + + if (!inPromise()) { if (auto c = Instruction::Cast(branchCondition)) { - if (c->typeFeedback().value == True::instance()) { - assumeBB0 = bc.bc == Opcode::brtrue_; - deoptCondition = c; - } - if (c->typeFeedback().value == False::instance()) { - assumeBB0 = bc.bc == Opcode::brfalse_; - deoptCondition = c; + auto likely = c->typeFeedback().value; + if (likely == True::instance() || + likely == False::instance()) { + if (auto cp = addCheckpoint(srcCode, pos, cur.stack, + insert)) { + bool expectBranch = likely == branchReason; + if (expectBranch) + finger = trg; + + auto assume = new Assume( + asBool, cp, + DeoptReason(c->typeFeedback().feedbackOrigin, + DeoptReason::DeadBranchReached), + expectBranch); + insert(assume); + + // If we deopt on a typecheck, then we should record + // that information by casting the value. + // TODO: also do this for negative type checks + if (expectBranch) + if (auto tt = IsType::Cast(branchCondition)) { + auto checkedValue = tt->arg<0>().val(); + auto checkedType = checkedValue->type; + + for (auto& e : cur.stack) { + if (e == checkedValue) { + if (!e->type.isA(checkedType)) { + bool block = false; + if (auto j = + Instruction::Cast(e)) { + // In case the typefeedback + // is more precise than the + if (!j->typeFeedback() + .type.isVoid() && + !checkedType.isA( + j->typeFeedback() + .type)) + block = true; + } + if (!block) { + auto cast = + insert(new CastType( + e, + CastType::Downcast, + PirType::any(), + checkedType)); + cast->effects.set( + Effect:: + DependsOnAssume); + e = cast; + } + } + } + } + } + cur.stack.pop(); + continue; + } } } - - if (!branchCondition->type.isA(PirType::test())) { - v = insert(new Identical(branchCondition, - bc.bc == Opcode::brtrue_ - ? (Value*)True::instance() - : (Value*)False::instance(), - PirType::val())); - } else { - swapTrueFalse = bc.bc == Opcode::brfalse_; - } - insert(new Branch(v)); - break; - } - case Opcode::beginloop_: - log.warn("Cannot compile Function. Unsupported beginloop bc"); - return nullptr; - default: - assert(false); } + cur.stack.pop(); + insert(new Branch(asBool)); + BB* branch = insert.createBB(); BB* fall = insert.createBB(); - if (swapTrueFalse) { - insert.setBranch(fall, branch); - assumeBB0 = !assumeBB0; - } else { - insert.setBranch(branch, fall); - } - - if (deoptCondition && !inPromise() && !inlining()) { - auto deopt = assumeBB0 ? insert.getCurrentBB()->falseBranch() - : insert.getCurrentBB()->trueBranch(); - insert.enterBB(deopt); - - auto sp = insert.registerFrameState( - srcCode, (deopt == fall) ? nextPos : trg, cur.stack, - inPromise()); - auto offset = (uintptr_t)deoptCondition->typeFeedback().origin - - (uintptr_t)srcCode; - DeoptReason reason = {DeoptReason::DeadBranchReached, srcCode, - (uint32_t)offset}; - insert(new RecordDeoptReason(reason, deoptCondition)); - insert(new Deopt(sp)); - - insert.enterBB(deopt == fall ? branch : fall); - finger = (deopt == fall) ? trg : nextPos; - - // If we deopt on a typecheck, then we should record that - // information by casting the value. - if (assumeBB0) - if (auto tt = IsType::Cast(branchCondition)) { - for (auto& e : cur.stack) { - if (tt->arg<0>().val() == e) { - if (!e->type.isA(tt->typeTest)) { - bool block = false; - if (auto j = Instruction::Cast(e)) { - // In case the typefeedback is more - // precise than the - if (!j->typeFeedback().type.isVoid() && - !tt->typeTest.isA( - j->typeFeedback().type)) - block = true; - } - if (!block) { - auto cast = insert(new CastType( - e, CastType::Downcast, - PirType::any(), tt->typeTest)); - cast->effects.set( - Effect::DependsOnAssume); - e = cast; - } - } - } - } - } - - continue; - } + insert.setBranch(branch, fall); pushWorklist(branch, trg); insert.enterBB(fall); diff --git a/rir/src/compiler/util/bb_transform.cpp b/rir/src/compiler/util/bb_transform.cpp index 271bb18cc..a143082fa 100644 --- a/rir/src/compiler/util/bb_transform.cpp +++ b/rir/src/compiler/util/bb_transform.cpp @@ -171,18 +171,21 @@ Value* BBTransform::forInline(BB* inlinee, BB* splice, Value* context, return found; } -BB* BBTransform::lowerExpect(Code* code, BB* src, BB::Instrs::iterator position, - Assume* assume, bool condition, BB* deoptBlock_, +BB* BBTransform::lowerExpect(Code* code, BB* srcBlock, + BB::Instrs::iterator position, Assume* assume, + bool condition, BB* deoptBlock_, const std::string& debugMessage, bool triggerAnyway) { - auto split = BBTransform::split(code->nextBBId++, src, position + 1, code); - static SEXP print = Rf_findFun(Rf_install("cat"), R_GlobalEnv); + auto split = + BBTransform::split(code->nextBBId++, srcBlock, position + 1, code); BB* deoptBlock = new BB(code, code->nextBBId++); deoptBlock->setNext(deoptBlock_); if (debugMessage.size() != 0) { + static SEXP print = Rf_findFun(Rf_install("cat"), R_GlobalEnv); + SEXP msg = Rf_mkString(debugMessage.c_str()); auto ldprint = new LdConst(print); Instruction* ldmsg = new LdConst(msg); @@ -196,75 +199,26 @@ BB* BBTransform::lowerExpect(Code* code, BB* src, BB::Instrs::iterator position, Tombstone::framestate(), 0)); } - if (!assume->feedbackOrigin.empty()) { - for (auto& origin : assume->feedbackOrigin) { - Value* src = nullptr; - auto cond = assume->condition(); - auto r = DeoptReason::None; - if (auto t = IsType::Cast(cond)) { - r = DeoptReason::Typecheck; - src = t->arg<0>().val(); - } else if (auto t = Identical::Cast(cond)) { - src = t->arg<0>().val(); - if (LdConst::Cast(src)) - src = t->arg<1>().val(); - assert(!LdConst::Cast(src)); - r = DeoptReason::Calltarget; - } else if (auto t = IsEnvStub::Cast(cond)) { - src = t->arg(0).val(); - r = DeoptReason::EnvStubMaterialized; - } else { - if (auto c = Instruction::Cast(cond)) { - c->print(std::cerr); - assert(src && "Don't know how to report deopt reason"); - } - } - switch (r) { - case DeoptReason::Typecheck: - case DeoptReason::DeadCall: - case DeoptReason::Calltarget: { - auto offset = - (uintptr_t)origin.second - (uintptr_t)origin.first; - auto o = *((Opcode*)origin.first + offset); - assert(o == Opcode::record_call_ || o == Opcode::record_type_ || - o == Opcode::record_test_); - assert((uintptr_t)origin.second > (uintptr_t)origin.first); - auto rec = new RecordDeoptReason( - {r, origin.first, (uint32_t)offset}, src); - deoptBlock->append(rec); - break; - } - case DeoptReason::EnvStubMaterialized: { - auto rec = new RecordDeoptReason({r, origin.first, 0}, src); - deoptBlock->append(rec); - break; - } - case DeoptReason::DeadBranchReached: { - assert(false); - break; - } - case DeoptReason::None: - break; - } - } - } + if (assume->valueUnderTest()) + deoptBlock->append( + new RecordDeoptReason(assume->reason, assume->valueUnderTest())); Value* test = assume->condition(); if (triggerAnyway) { test = condition ? (Value*)False::instance() : (Value*)True::instance(); } - src->replace(position, new Branch(test)); + srcBlock->replace(position, new Branch(test)); if (condition) - src->overrideSuccessors({split, deoptBlock}); + srcBlock->overrideSuccessors({split, deoptBlock}); else - src->overrideSuccessors({deoptBlock, split}); + srcBlock->overrideSuccessors({deoptBlock, split}); // If visibility was tainted between the last checkpoint and the bailout, // we try (best-effort) to recover the correct setting, by scanning for the // last known-good setting. bool wrongViz = false; - for (auto i : *src) { + for (auto i : *srcBlock) { if (i->effects.contains(Effect::Visibility)) wrongViz = true; } @@ -293,27 +247,24 @@ BB* BBTransform::lowerExpect(Code* code, BB* src, BB::Instrs::iterator position, return split; } -void BBTransform::insertAssume(Instruction* condition, Checkpoint* cp, BB* bb, - BB::Instrs::iterator& position, - bool assumePositive, rir::Code* srcCode, - Opcode* origin) { +void BBTransform::insertAssume(Instruction* condition, bool assumePositive, + Checkpoint* cp, const FeedbackOrigin& origin, + DeoptReason::Reason reason, BB* bb, + BB::Instrs::iterator& position) { position = bb->insert(position, condition); - auto assume = new Assume(condition, cp); - if (srcCode) - assume->feedbackOrigin.push_back({srcCode, origin}); - if (!assumePositive) - assume->Not(); + auto assume = + new Assume(condition, cp, DeoptReason(origin, reason), assumePositive); position = bb->insert(position + 1, assume); position++; }; -void BBTransform::insertAssume(Instruction* condition, Checkpoint* cp, - bool assumePositive, rir::Code* srcCode, - Opcode* origin) { +void BBTransform::insertAssume(Instruction* condition, bool assumePositive, + Checkpoint* cp, const FeedbackOrigin& origin, + DeoptReason::Reason reason) { auto contBB = cp->bb()->trueBranch(); auto contBegin = contBB->begin(); - insertAssume(condition, cp, contBB, contBegin, assumePositive, srcCode, - origin); + insertAssume(condition, assumePositive, cp, origin, reason, contBB, + contBegin); } void BBTransform::mergeRedundantBBs(Code* closure) { diff --git a/rir/src/compiler/util/bb_transform.h b/rir/src/compiler/util/bb_transform.h index c97135ab9..5d790278a 100644 --- a/rir/src/compiler/util/bb_transform.h +++ b/rir/src/compiler/util/bb_transform.h @@ -4,6 +4,7 @@ #include "../pir/bb.h" #include "../pir/pir.h" #include "compiler/analysis/cfg.h" +#include "runtime/TypeFeedback.h" namespace rir { @@ -29,13 +30,13 @@ class BBTransform { bool condition, BB* deoptBlock, const std::string& debugMesage, bool triggerAnyway = false); - static void insertAssume(Instruction* condition, Checkpoint* cp, BB* bb, - BB::Instrs::iterator& position, - bool assumePositive, rir::Code* srcCode, - Opcode* origin = nullptr); - static void insertAssume(Instruction* condition, Checkpoint* cp, - bool assumePositive, rir::Code* srcCode, - Opcode* origin = nullptr); + static void insertAssume(Instruction* condition, bool assumePositive, + Checkpoint* cp, const FeedbackOrigin& origin, + DeoptReason::Reason reason, BB* bb, + BB::Instrs::iterator& position); + static void insertAssume(Instruction* condition, bool assumePositive, + Checkpoint* cp, const FeedbackOrigin& origin, + DeoptReason::Reason reason); static void mergeRedundantBBs(Code* closure); diff --git a/rir/src/interpreter/interp.cpp b/rir/src/interpreter/interp.cpp index fe1f2014d..8f24c70b6 100644 --- a/rir/src/interpreter/interp.cpp +++ b/rir/src/interpreter/interp.cpp @@ -569,7 +569,9 @@ void checkUserInterrupt() { } void recordDeoptReason(SEXP val, const DeoptReason& reason) { - Opcode* pos = (Opcode*)reason.srcCode + reason.originOffset; + + auto pos = reason.pc(); + switch (reason.reason) { case DeoptReason::DeadBranchReached: { assert(*pos == Opcode::record_test_); @@ -593,23 +595,20 @@ void recordDeoptReason(SEXP val, const DeoptReason& reason) { break; } case DeoptReason::DeadCall: - reason.srcCode->deadCallReached++; + reason.srcCode()->deadCallReached++; // fall through [[clang::fallthrough]]; case DeoptReason::Calltarget: { assert(*pos == Opcode::record_call_); ObservedCallees* feedback = (ObservedCallees*)(pos + 1); - feedback->record(reason.srcCode, val); + feedback->record(reason.srcCode(), val); assert(feedback->taken > 0); break; } case DeoptReason::EnvStubMaterialized: { - reason.srcCode->flags.set(Code::NeedsFullEnv); + reason.srcCode()->flags.set(Code::NeedsFullEnv); break; } - case DeoptReason::None: - assert(false); - break; } } diff --git a/rir/src/ir/BC_noarg_list.h b/rir/src/ir/BC_noarg_list.h index 1da8c47a9..7ef0b8976 100644 --- a/rir/src/ir/BC_noarg_list.h +++ b/rir/src/ir/BC_noarg_list.h @@ -37,7 +37,7 @@ V(NESTED, sub, sub) \ V(NESTED, uplus, uplus) \ V(NESTED, uminus, uminus) \ - V(NESTED, not_, not) \ + V(NESTED, not_, not ) \ V(NESTED, lt, lt) \ V(NESTED, gt, gt) \ V(NESTED, le, le) \ diff --git a/rir/src/runtime/PirTypeFeedback.cpp b/rir/src/runtime/PirTypeFeedback.cpp index c52f32299..66e0e0bdb 100644 --- a/rir/src/runtime/PirTypeFeedback.cpp +++ b/rir/src/runtime/PirTypeFeedback.cpp @@ -39,18 +39,18 @@ PirTypeFeedback::PirTypeFeedback( auto typeFeedback = s.second; assert(slot < MAX_SLOT_IDX); - auto e = reverseMapping.find(typeFeedback.origin); + auto e = reverseMapping.find(typeFeedback.feedbackOrigin.pc()); if (e != reverseMapping.end()) { entry[slot] = e->second; assert(mdEntries()[e->second].previousType == typeFeedback.type); } else { - assert(codes.count(typeFeedback.srcCode)); + assert(codes.count(typeFeedback.feedbackOrigin.srcCode())); new (&mdEntries()[idx]) MDEntry; - mdEntries()[idx].srcCode = srcCodeMap.at(typeFeedback.srcCode); - mdEntries()[idx].offset = - typeFeedback.origin - typeFeedback.srcCode->code(); + mdEntries()[idx].srcCode = + srcCodeMap.at(typeFeedback.feedbackOrigin.srcCode()); + mdEntries()[idx].offset = typeFeedback.feedbackOrigin.offset(); mdEntries()[idx].previousType = typeFeedback.type; - reverseMapping[typeFeedback.origin] = idx; + reverseMapping[typeFeedback.feedbackOrigin.pc()] = idx; entry[slot] = idx++; } } diff --git a/rir/src/runtime/TypeFeedback.cpp b/rir/src/runtime/TypeFeedback.cpp index 6eb4c5165..0b68c6e66 100644 --- a/rir/src/runtime/TypeFeedback.cpp +++ b/rir/src/runtime/TypeFeedback.cpp @@ -26,4 +26,31 @@ SEXP ObservedCallees::getTarget(const Code* code, size_t pos) const { return code->getExtraPoolEntry(targets[pos]); } +FeedbackOrigin::FeedbackOrigin(rir::Code* src, Opcode* p) + : offset_((uintptr_t)p - (uintptr_t)src), srcCode_(src) { + assert(p); + assert(p >= src->code()); + assert(p < src->endCode()); + assert(pc() == p); +} + +DeoptReason::DeoptReason(const FeedbackOrigin& origin, + DeoptReason::Reason reason) + : reason(reason), origin(origin) { + switch (reason) { + case DeoptReason::Typecheck: + case DeoptReason::DeadCall: + case DeoptReason::Calltarget: + case DeoptReason::DeadBranchReached: { + assert(pc()); + auto o = *pc(); + assert(o == Opcode::record_call_ || o == Opcode::record_type_ || + o == Opcode::record_test_); + break; + } + case DeoptReason::EnvStubMaterialized: + break; + } +} + } // namespace rir diff --git a/rir/src/runtime/TypeFeedback.h b/rir/src/runtime/TypeFeedback.h index 757041537..ec4f401fc 100644 --- a/rir/src/runtime/TypeFeedback.h +++ b/rir/src/runtime/TypeFeedback.h @@ -124,7 +124,7 @@ struct ObservedValues { RIR_INLINE void record(SEXP e) { - // Set attribs flag for every object even if the SEXP does not + // Set attribs flag for every object even if the SEXP does not // have attributes. The assumption used to be that e having no // attributes implies that it is not an object, but this is not // the case in some very specific cases: @@ -155,18 +155,42 @@ static_assert(sizeof(ObservedValues) == sizeof(uint32_t), enum class Opcode : uint8_t; +struct FeedbackOrigin { + private: + uint32_t offset_ = 0; + Code* srcCode_ = nullptr; + + public: + FeedbackOrigin() {} + FeedbackOrigin(rir::Code* src, Opcode* pc); + + Opcode* pc() const { + if (offset_ == 0) + return nullptr; + return (Opcode*)((uintptr_t)srcCode() + offset_); + } + uint32_t offset() const { return offset_; } + Code* srcCode() const { return srcCode_; } + void srcCode(Code* src) { srcCode_ = src; } +}; + struct DeoptReason { enum Reason : uint32_t { - None, Typecheck, DeadCall, Calltarget, EnvStubMaterialized, DeadBranchReached, }; - Reason reason; - Code* srcCode; - uint32_t originOffset; + + DeoptReason::Reason reason; + FeedbackOrigin origin; + + DeoptReason() = delete; + DeoptReason(const FeedbackOrigin& origin, DeoptReason::Reason reason); + + Code* srcCode() const { return origin.srcCode(); } + Opcode* pc() const { return origin.pc(); } }; static_assert(sizeof(DeoptReason) == 4 * sizeof(uint32_t), "Size needs to fit inside a record_deopt_ bc immediate args");