diff --git a/llvm/docs/LangRef.rst b/llvm/docs/LangRef.rst index cee8175a189b8..8f0c45f674ead 100644 --- a/llvm/docs/LangRef.rst +++ b/llvm/docs/LangRef.rst @@ -959,10 +959,11 @@ IFuncs ------- IFuncs, like as aliases, don't create any new data or func. They are just a new -symbol that dynamic linker resolves at runtime by calling a resolver function. +symbol that is resolved at runtime by calling a resolver function. -IFuncs have a name and a resolver that is a function called by dynamic linker -that returns address of another function associated with the name. +On ELF platforms, IFuncs are resolved by the dynamic linker at load time. On +Mach-O platforms, they are lowered in terms of ``.symbol_resolver`` functions, +which lazily resolve the callee the first time they are called. IFunc may have an optional :ref:`linkage type ` and an optional :ref:`visibility style `. diff --git a/llvm/include/llvm/CodeGen/AsmPrinter.h b/llvm/include/llvm/CodeGen/AsmPrinter.h index 2731ef452c79c..5ec246ee7015c 100644 --- a/llvm/include/llvm/CodeGen/AsmPrinter.h +++ b/llvm/include/llvm/CodeGen/AsmPrinter.h @@ -599,6 +599,26 @@ class AsmPrinter : public MachineFunctionPass { /// instructions in verbose mode. virtual void emitImplicitDef(const MachineInstr *MI) const; + /// getSubtargetInfo() cannot be used where this is needed because we don't + /// have a MachineFunction when we're lowering a GlobalIFunc, and + /// getSubtargetInfo requires one. Override the implementation in targets + /// that support the Mach-O IFunc lowering. + virtual const MCSubtargetInfo *getIFuncMCSubtargetInfo() const { + return nullptr; + } + + virtual void emitMachOIFuncStubBody(Module &M, const GlobalIFunc &GI, + MCSymbol *LazyPointer) { + llvm_unreachable( + "Mach-O IFunc lowering is not yet supported on this target"); + } + + virtual void emitMachOIFuncStubHelperBody(Module &M, const GlobalIFunc &GI, + MCSymbol *LazyPointer) { + llvm_unreachable( + "Mach-O IFunc lowering is not yet supported on this target"); + } + /// Emit N NOP instructions. void emitNops(unsigned N); @@ -614,7 +634,7 @@ class AsmPrinter : public MachineFunctionPass { StringRef Suffix) const; /// Return the MCSymbol for the specified ExternalSymbol. - MCSymbol *GetExternalSymbolSymbol(StringRef Sym) const; + MCSymbol *GetExternalSymbolSymbol(Twine Sym) const; /// Return the symbol for the specified jump table entry. MCSymbol *GetJTISymbol(unsigned JTID, bool isLinkerPrivate = false) const; @@ -884,6 +904,7 @@ class AsmPrinter : public MachineFunctionPass { void emitGlobalAlias(Module &M, const GlobalAlias &GA); void emitGlobalIFunc(Module &M, const GlobalIFunc &GI); +private: /// This method decides whether the specified basic block requires a label. bool shouldEmitLabelForBasicBlock(const MachineBasicBlock &MBB) const; diff --git a/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp b/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp index 3a679f1576b7b..61309c51336e5 100644 --- a/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp +++ b/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp @@ -59,6 +59,7 @@ #include "llvm/CodeGen/TargetLowering.h" #include "llvm/CodeGen/TargetOpcodes.h" #include "llvm/CodeGen/TargetRegisterInfo.h" +#include "llvm/CodeGen/TargetSubtargetInfo.h" #include "llvm/Config/config.h" #include "llvm/IR/BasicBlock.h" #include "llvm/IR/Comdat.h" @@ -2147,24 +2148,80 @@ void AsmPrinter::emitGlobalIFunc(Module &M, const GlobalIFunc &GI) { assert(!TM.getTargetTriple().isOSBinFormatXCOFF() && "IFunc is not supported on AIX."); - MCSymbol *Name = getSymbol(&GI); + auto EmitLinkage = [&](MCSymbol *Sym) { + if (GI.hasExternalLinkage() || !MAI->getWeakRefDirective()) + OutStreamer->emitSymbolAttribute(Sym, MCSA_Global); + else if (GI.hasWeakLinkage() || GI.hasLinkOnceLinkage()) + OutStreamer->emitSymbolAttribute(Sym, MCSA_WeakReference); + else + assert(GI.hasLocalLinkage() && "Invalid ifunc linkage"); + }; - if (GI.hasExternalLinkage() || !MAI->getWeakRefDirective()) - OutStreamer->emitSymbolAttribute(Name, MCSA_Global); - else if (GI.hasWeakLinkage() || GI.hasLinkOnceLinkage()) - OutStreamer->emitSymbolAttribute(Name, MCSA_WeakReference); - else - assert(GI.hasLocalLinkage() && "Invalid ifunc linkage"); + if (TM.getTargetTriple().isOSBinFormatELF()) { + MCSymbol *Name = getSymbol(&GI); + EmitLinkage(Name); + OutStreamer->emitSymbolAttribute(Name, MCSA_ELF_TypeIndFunction); + emitVisibility(Name, GI.getVisibility()); + + // Emit the directives as assignments aka .set: + const MCExpr *Expr = lowerConstant(GI.getResolver()); + OutStreamer->emitAssignment(Name, Expr); + MCSymbol *LocalAlias = getSymbolPreferLocal(GI); + if (LocalAlias != Name) + OutStreamer->emitAssignment(LocalAlias, Expr); + + return; + } - OutStreamer->emitSymbolAttribute(Name, MCSA_ELF_TypeIndFunction); - emitVisibility(Name, GI.getVisibility()); + if (!TM.getTargetTriple().isOSBinFormatMachO() || !getIFuncMCSubtargetInfo()) + llvm::report_fatal_error("IFuncs are not supported on this platform"); - // Emit the directives as assignments aka .set: - const MCExpr *Expr = lowerConstant(GI.getResolver()); - OutStreamer->emitAssignment(Name, Expr); - MCSymbol *LocalAlias = getSymbolPreferLocal(GI); - if (LocalAlias != Name) - OutStreamer->emitAssignment(LocalAlias, Expr); + // On Darwin platforms, emit a manually-constructed .symbol_resolver that + // implements the symbol resolution duties of the IFunc. + // + // Normally, this would be handled by linker magic, but unfortunately there + // are a few limitations in ld64 and ld-prime's implementation of + // .symbol_resolver that mean we can't always use them: + // + // * resolvers cannot be the target of an alias + // * resolvers cannot have private linkage + // * resolvers cannot have linkonce linkage + // * resolvers cannot appear in executables + // * resolvers cannot appear in bundles + // + // This works around that by emitting a close approximation of what the + // linker would have done. + + MCSymbol *LazyPointer = + GetExternalSymbolSymbol(GI.getName() + ".lazy_pointer"); + MCSymbol *StubHelper = GetExternalSymbolSymbol(GI.getName() + ".stub_helper"); + + OutStreamer->switchSection(OutContext.getObjectFileInfo()->getDataSection()); + + const DataLayout &DL = M.getDataLayout(); + emitAlignment(Align(DL.getPointerSize())); + OutStreamer->emitLabel(LazyPointer); + emitVisibility(LazyPointer, GI.getVisibility()); + OutStreamer->emitValue(MCSymbolRefExpr::create(StubHelper, OutContext), 8); + + OutStreamer->switchSection(OutContext.getObjectFileInfo()->getTextSection()); + + const TargetSubtargetInfo *STI = + TM.getSubtargetImpl(*GI.getResolverFunction()); + const TargetLowering *TLI = STI->getTargetLowering(); + Align TextAlign(TLI->getMinFunctionAlignment()); + + MCSymbol *Stub = getSymbol(&GI); + EmitLinkage(Stub); + OutStreamer->emitCodeAlignment(TextAlign, getIFuncMCSubtargetInfo()); + OutStreamer->emitLabel(Stub); + emitVisibility(Stub, GI.getVisibility()); + emitMachOIFuncStubBody(M, GI, LazyPointer); + + OutStreamer->emitCodeAlignment(TextAlign, getIFuncMCSubtargetInfo()); + OutStreamer->emitLabel(StubHelper); + emitVisibility(StubHelper, GI.getVisibility()); + emitMachOIFuncStubHelperBody(M, GI, LazyPointer); } void AsmPrinter::emitRemarksSection(remarks::RemarkStreamer &RS) { @@ -2311,6 +2368,32 @@ bool AsmPrinter::doFinalization(Module &M) { // through user plugins. emitStackMaps(); + // Print aliases in topological order, that is, for each alias a = b, + // b must be printed before a. + // This is because on some targets (e.g. PowerPC) linker expects aliases in + // such an order to generate correct TOC information. + SmallVector AliasStack; + SmallPtrSet AliasVisited; + for (const auto &Alias : M.aliases()) { + if (Alias.hasAvailableExternallyLinkage()) + continue; + for (const GlobalAlias *Cur = &Alias; Cur; + Cur = dyn_cast(Cur->getAliasee())) { + if (!AliasVisited.insert(Cur).second) + break; + AliasStack.push_back(Cur); + } + for (const GlobalAlias *AncestorAlias : llvm::reverse(AliasStack)) + emitGlobalAlias(M, *AncestorAlias); + AliasStack.clear(); + } + + // IFuncs must come before deubginfo in case the backend decides to emit them + // as actual functions, since on Mach-O targets, we cannot create regular + // sections after DWARF. + for (const auto &IFunc : M.ifuncs()) + emitGlobalIFunc(M, IFunc); + // Finalize debug and EH information. for (const HandlerInfo &HI : Handlers) { NamedRegionTimer T(HI.TimerName, HI.TimerDescription, HI.TimerGroupName, @@ -2350,28 +2433,6 @@ bool AsmPrinter::doFinalization(Module &M) { } } - // Print aliases in topological order, that is, for each alias a = b, - // b must be printed before a. - // This is because on some targets (e.g. PowerPC) linker expects aliases in - // such an order to generate correct TOC information. - SmallVector AliasStack; - SmallPtrSet AliasVisited; - for (const auto &Alias : M.aliases()) { - if (Alias.hasAvailableExternallyLinkage()) - continue; - for (const GlobalAlias *Cur = &Alias; Cur; - Cur = dyn_cast(Cur->getAliasee())) { - if (!AliasVisited.insert(Cur).second) - break; - AliasStack.push_back(Cur); - } - for (const GlobalAlias *AncestorAlias : llvm::reverse(AliasStack)) - emitGlobalAlias(M, *AncestorAlias); - AliasStack.clear(); - } - for (const auto &IFunc : M.ifuncs()) - emitGlobalIFunc(M, IFunc); - GCModuleInfo *MI = getAnalysisIfAvailable(); assert(MI && "AsmPrinter didn't require GCModuleInfo?"); for (GCModuleInfo::iterator I = MI->end(), E = MI->begin(); I != E; ) @@ -3745,7 +3806,7 @@ MCSymbol *AsmPrinter::getSymbolWithGlobalValueBase(const GlobalValue *GV, } /// Return the MCSymbol for the specified ExternalSymbol. -MCSymbol *AsmPrinter::GetExternalSymbolSymbol(StringRef Sym) const { +MCSymbol *AsmPrinter::GetExternalSymbolSymbol(Twine Sym) const { SmallString<60> NameStr; Mangler::getNameWithPrefix(NameStr, Sym, getDataLayout()); return OutContext.getOrCreateSymbol(NameStr); diff --git a/llvm/lib/IR/Verifier.cpp b/llvm/lib/IR/Verifier.cpp index cdc556ba7df82..8aba28026306a 100644 --- a/llvm/lib/IR/Verifier.cpp +++ b/llvm/lib/IR/Verifier.cpp @@ -2229,13 +2229,11 @@ void Verifier::verifyFunctionAttrs(FunctionType *FT, AttributeList Attrs, } // Check EVEX512 feature. - if (MaxParameterWidth >= 512 && Attrs.hasFnAttr("target-features")) { - Triple T(M.getTargetTriple()); - if (T.isX86()) { - StringRef TF = Attrs.getFnAttr("target-features").getValueAsString(); - Check(!TF.contains("+avx512f") || !TF.contains("-evex512"), - "512-bit vector arguments require 'evex512' for AVX512", V); - } + if (MaxParameterWidth >= 512 && Attrs.hasFnAttr("target-features") && + TT.isX86()) { + StringRef TF = Attrs.getFnAttr("target-features").getValueAsString(); + Check(!TF.contains("+avx512f") || !TF.contains("-evex512"), + "512-bit vector arguments require 'evex512' for AVX512", V); } checkUnsignedBaseTenFuncAttr(Attrs, "patchable-function-prefix", V); diff --git a/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp b/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp index f4d3a85f34c88..90e1ce9ddf66b 100644 --- a/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp +++ b/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp @@ -30,6 +30,7 @@ #include "llvm/ADT/Twine.h" #include "llvm/BinaryFormat/COFF.h" #include "llvm/BinaryFormat/ELF.h" +#include "llvm/BinaryFormat/MachO.h" #include "llvm/CodeGen/AsmPrinter.h" #include "llvm/CodeGen/FaultMaps.h" #include "llvm/CodeGen/MachineBasicBlock.h" @@ -47,10 +48,12 @@ #include "llvm/MC/MCInst.h" #include "llvm/MC/MCInstBuilder.h" #include "llvm/MC/MCSectionELF.h" +#include "llvm/MC/MCSectionMachO.h" #include "llvm/MC/MCStreamer.h" #include "llvm/MC/MCSymbol.h" #include "llvm/MC/TargetRegistry.h" #include "llvm/Support/Casting.h" +#include "llvm/Support/CommandLine.h" #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/raw_ostream.h" #include "llvm/Target/TargetMachine.h" @@ -198,6 +201,15 @@ class AArch64AsmPrinter : public AsmPrinter { bool shouldEmitWeakSwiftAsyncExtendedFramePointerFlags() const override { return ShouldEmitWeakSwiftAsyncExtendedFramePointerFlags; } + + const MCSubtargetInfo *getIFuncMCSubtargetInfo() const override { + assert(STI); + return STI; + } + void emitMachOIFuncStubBody(Module &M, const GlobalIFunc &GI, + MCSymbol *LazyPointer) override; + void emitMachOIFuncStubHelperBody(Module &M, const GlobalIFunc &GI, + MCSymbol *LazyPointer) override; }; } // end anonymous namespace @@ -1809,6 +1821,201 @@ void AArch64AsmPrinter::emitInstruction(const MachineInstr *MI) { EmitToStreamer(*OutStreamer, TmpInst); } +void AArch64AsmPrinter::emitMachOIFuncStubBody(Module &M, const GlobalIFunc &GI, + MCSymbol *LazyPointer) { + // _ifunc: + // adrp x16, lazy_pointer@GOTPAGE + // ldr x16, [x16, lazy_pointer@GOTPAGEOFF] + // ldr x16, [x16] + // br x16 + + { + MCInst Adrp; + Adrp.setOpcode(AArch64::ADRP); + Adrp.addOperand(MCOperand::createReg(AArch64::X16)); + MCOperand SymPage; + MCInstLowering.lowerOperand( + MachineOperand::CreateMCSymbol(LazyPointer, + AArch64II::MO_GOT | AArch64II::MO_PAGE), + SymPage); + Adrp.addOperand(SymPage); + OutStreamer->emitInstruction(Adrp, *STI); + } + + { + MCInst Ldr; + Ldr.setOpcode(AArch64::LDRXui); + Ldr.addOperand(MCOperand::createReg(AArch64::X16)); + Ldr.addOperand(MCOperand::createReg(AArch64::X16)); + MCOperand SymPageOff; + MCInstLowering.lowerOperand( + MachineOperand::CreateMCSymbol(LazyPointer, AArch64II::MO_GOT | + AArch64II::MO_PAGEOFF), + SymPageOff); + Ldr.addOperand(SymPageOff); + Ldr.addOperand(MCOperand::createImm(0)); + OutStreamer->emitInstruction(Ldr, *STI); + } + + OutStreamer->emitInstruction(MCInstBuilder(AArch64::LDRXui) + .addReg(AArch64::X16) + .addReg(AArch64::X16) + .addImm(0), + *STI); + + OutStreamer->emitInstruction(MCInstBuilder(TM.getTargetTriple().isArm64e() + ? AArch64::BRAAZ + : AArch64::BR) + .addReg(AArch64::X16), + *STI); +} + +void AArch64AsmPrinter::emitMachOIFuncStubHelperBody(Module &M, + const GlobalIFunc &GI, + MCSymbol *LazyPointer) { + // These stub helpers are only ever called once, so here we're optimizing for + // minimum size by using the pre-indexed store variants, which saves a few + // bytes of instructions to bump & restore sp. + + // _ifunc.stub_helper: + // stp fp, lr, [sp, #-16]! + // mov fp, sp + // stp x1, x0, [sp, #-16]! + // stp x3, x2, [sp, #-16]! + // stp x5, x4, [sp, #-16]! + // stp x7, x6, [sp, #-16]! + // stp d1, d0, [sp, #-16]! + // stp d3, d2, [sp, #-16]! + // stp d5, d4, [sp, #-16]! + // stp d7, d6, [sp, #-16]! + // bl _resolver + // adrp x16, lazy_pointer@GOTPAGE + // ldr x16, [x16, lazy_pointer@GOTPAGEOFF] + // str x0, [x16] + // mov x16, x0 + // ldp d7, d6, [sp], #16 + // ldp d5, d4, [sp], #16 + // ldp d3, d2, [sp], #16 + // ldp d1, d0, [sp], #16 + // ldp x7, x6, [sp], #16 + // ldp x5, x4, [sp], #16 + // ldp x3, x2, [sp], #16 + // ldp x1, x0, [sp], #16 + // ldp fp, lr, [sp], #16 + // br x16 + + OutStreamer->emitInstruction(MCInstBuilder(AArch64::STPXpre) + .addReg(AArch64::SP) + .addReg(AArch64::FP) + .addReg(AArch64::LR) + .addReg(AArch64::SP) + .addImm(-2), + *STI); + + OutStreamer->emitInstruction(MCInstBuilder(AArch64::ADDXri) + .addReg(AArch64::FP) + .addReg(AArch64::SP) + .addImm(0) + .addImm(0), + *STI); + + for (int I = 0; I != 4; ++I) + OutStreamer->emitInstruction(MCInstBuilder(AArch64::STPXpre) + .addReg(AArch64::SP) + .addReg(AArch64::X1 + 2 * I) + .addReg(AArch64::X0 + 2 * I) + .addReg(AArch64::SP) + .addImm(-2), + *STI); + + for (int I = 0; I != 4; ++I) + OutStreamer->emitInstruction(MCInstBuilder(AArch64::STPDpre) + .addReg(AArch64::SP) + .addReg(AArch64::D1 + 2 * I) + .addReg(AArch64::D0 + 2 * I) + .addReg(AArch64::SP) + .addImm(-2), + *STI); + + OutStreamer->emitInstruction( + MCInstBuilder(AArch64::BL) + .addOperand(MCOperand::createExpr(lowerConstant(GI.getResolver()))), + *STI); + + { + MCInst Adrp; + Adrp.setOpcode(AArch64::ADRP); + Adrp.addOperand(MCOperand::createReg(AArch64::X16)); + MCOperand SymPage; + MCInstLowering.lowerOperand( + MachineOperand::CreateES(LazyPointer->getName().data() + 1, + AArch64II::MO_GOT | AArch64II::MO_PAGE), + SymPage); + Adrp.addOperand(SymPage); + OutStreamer->emitInstruction(Adrp, *STI); + } + + { + MCInst Ldr; + Ldr.setOpcode(AArch64::LDRXui); + Ldr.addOperand(MCOperand::createReg(AArch64::X16)); + Ldr.addOperand(MCOperand::createReg(AArch64::X16)); + MCOperand SymPageOff; + MCInstLowering.lowerOperand( + MachineOperand::CreateES(LazyPointer->getName().data() + 1, + AArch64II::MO_GOT | AArch64II::MO_PAGEOFF), + SymPageOff); + Ldr.addOperand(SymPageOff); + Ldr.addOperand(MCOperand::createImm(0)); + OutStreamer->emitInstruction(Ldr, *STI); + } + + OutStreamer->emitInstruction(MCInstBuilder(AArch64::STRXui) + .addReg(AArch64::X0) + .addReg(AArch64::X16) + .addImm(0), + *STI); + + OutStreamer->emitInstruction(MCInstBuilder(AArch64::ADDXri) + .addReg(AArch64::X16) + .addReg(AArch64::X0) + .addImm(0) + .addImm(0), + *STI); + + for (int I = 3; I != -1; --I) + OutStreamer->emitInstruction(MCInstBuilder(AArch64::LDPDpost) + .addReg(AArch64::SP) + .addReg(AArch64::D1 + 2 * I) + .addReg(AArch64::D0 + 2 * I) + .addReg(AArch64::SP) + .addImm(2), + *STI); + + for (int I = 3; I != -1; --I) + OutStreamer->emitInstruction(MCInstBuilder(AArch64::LDPXpost) + .addReg(AArch64::SP) + .addReg(AArch64::X1 + 2 * I) + .addReg(AArch64::X0 + 2 * I) + .addReg(AArch64::SP) + .addImm(2), + *STI); + + OutStreamer->emitInstruction(MCInstBuilder(AArch64::LDPXpost) + .addReg(AArch64::SP) + .addReg(AArch64::FP) + .addReg(AArch64::LR) + .addReg(AArch64::SP) + .addImm(2), + *STI); + + OutStreamer->emitInstruction(MCInstBuilder(TM.getTargetTriple().isArm64e() + ? AArch64::BRAAZ + : AArch64::BR) + .addReg(AArch64::X16), + *STI); +} + // Force static initialization. extern "C" LLVM_EXTERNAL_VISIBILITY void LLVMInitializeAArch64AsmPrinter() { RegisterAsmPrinter X(getTheAArch64leTarget()); diff --git a/llvm/lib/Target/X86/X86AsmPrinter.cpp b/llvm/lib/Target/X86/X86AsmPrinter.cpp index 73c7450620966..15cfd247f125c 100644 --- a/llvm/lib/Target/X86/X86AsmPrinter.cpp +++ b/llvm/lib/Target/X86/X86AsmPrinter.cpp @@ -14,6 +14,7 @@ #include "X86AsmPrinter.h" #include "MCTargetDesc/X86ATTInstPrinter.h" #include "MCTargetDesc/X86BaseInfo.h" +#include "MCTargetDesc/X86MCTargetDesc.h" #include "MCTargetDesc/X86TargetStreamer.h" #include "TargetInfo/X86TargetInfo.h" #include "X86InstrInfo.h" @@ -34,6 +35,7 @@ #include "llvm/MC/MCCodeEmitter.h" #include "llvm/MC/MCContext.h" #include "llvm/MC/MCExpr.h" +#include "llvm/MC/MCInst.h" #include "llvm/MC/MCInstBuilder.h" #include "llvm/MC/MCSectionCOFF.h" #include "llvm/MC/MCSectionELF.h" @@ -530,6 +532,86 @@ void X86AsmPrinter::PrintIntelMemReference(const MachineInstr *MI, O << ']'; } +const MCSubtargetInfo *X86AsmPrinter::getIFuncMCSubtargetInfo() const { + assert(Subtarget); + return Subtarget; +} + +void X86AsmPrinter::emitMachOIFuncStubBody(Module &M, const GlobalIFunc &GI, + MCSymbol *LazyPointer) { + // _ifunc: + // jmpq *lazy_pointer(%rip) + + OutStreamer->emitInstruction( + MCInstBuilder(X86::JMP32m) + .addReg(X86::RIP) + .addImm(1) + .addReg(0) + .addOperand(MCOperand::createExpr( + MCSymbolRefExpr::create(LazyPointer, OutContext))) + .addReg(0), + *Subtarget); +} + +void X86AsmPrinter::emitMachOIFuncStubHelperBody(Module &M, + const GlobalIFunc &GI, + MCSymbol *LazyPointer) { + // _ifunc.stub_helper: + // push %rax + // push %rdi + // push %rsi + // push %rdx + // push %rcx + // push %r8 + // push %r9 + // callq foo + // movq %rax,lazy_pointer(%rip) + // pop %r9 + // pop %r8 + // pop %rcx + // pop %rdx + // pop %rsi + // pop %rdi + // pop %rax + // jmpq *lazy_pointer(%rip) + + for (int Reg : + {X86::RAX, X86::RDI, X86::RSI, X86::RDX, X86::RCX, X86::R8, X86::R9}) + OutStreamer->emitInstruction(MCInstBuilder(X86::PUSH64r).addReg(Reg), + *Subtarget); + + OutStreamer->emitInstruction( + MCInstBuilder(X86::CALL64pcrel32) + .addOperand(MCOperand::createExpr(lowerConstant(GI.getResolver()))), + *Subtarget); + + OutStreamer->emitInstruction( + MCInstBuilder(X86::MOV64mr) + .addReg(X86::RIP) + .addImm(1) + .addReg(0) + .addOperand(MCOperand::createExpr( + MCSymbolRefExpr::create(LazyPointer, OutContext))) + .addReg(0) + .addReg(X86::RAX), + *Subtarget); + + for (int Reg : + {X86::R9, X86::R8, X86::RCX, X86::RDX, X86::RSI, X86::RDI, X86::RAX}) + OutStreamer->emitInstruction(MCInstBuilder(X86::POP64r).addReg(Reg), + *Subtarget); + + OutStreamer->emitInstruction( + MCInstBuilder(X86::JMP32m) + .addReg(X86::RIP) + .addImm(1) + .addReg(0) + .addOperand(MCOperand::createExpr( + MCSymbolRefExpr::create(LazyPointer, OutContext))) + .addReg(0), + *Subtarget); +} + static bool printAsmMRegister(const X86AsmPrinter &P, const MachineOperand &MO, char Mode, raw_ostream &O) { Register Reg = MO.getReg(); diff --git a/llvm/lib/Target/X86/X86AsmPrinter.h b/llvm/lib/Target/X86/X86AsmPrinter.h index c81651cf7f2f0..693021eca3295 100644 --- a/llvm/lib/Target/X86/X86AsmPrinter.h +++ b/llvm/lib/Target/X86/X86AsmPrinter.h @@ -120,6 +120,11 @@ class LLVM_LIBRARY_VISIBILITY X86AsmPrinter : public AsmPrinter { const char *Modifier); void PrintIntelMemReference(const MachineInstr *MI, unsigned OpNo, raw_ostream &O, const char *Modifier); + const MCSubtargetInfo *getIFuncMCSubtargetInfo() const override; + void emitMachOIFuncStubBody(Module &M, const GlobalIFunc &GI, + MCSymbol *LazyPointer) override; + void emitMachOIFuncStubHelperBody(Module &M, const GlobalIFunc &GI, + MCSymbol *LazyPointer) override; public: X86AsmPrinter(TargetMachine &TM, std::unique_ptr Streamer); diff --git a/llvm/test/CodeGen/AArch64/addrsig-macho.ll b/llvm/test/CodeGen/AArch64/addrsig-macho.ll index 360876fccaad3..62bc764e0251b 100644 --- a/llvm/test/CodeGen/AArch64/addrsig-macho.ll +++ b/llvm/test/CodeGen/AArch64/addrsig-macho.ll @@ -3,6 +3,19 @@ ; RUN: llvm-objdump --macho --section-headers %t | FileCheck %s --check-prefix=SECTIONS ; RUN: llvm-objdump --macho --reloc %t | FileCheck %s --check-prefix=RELOCS +; CHECK: .section __DATA,__data +; CHECK: _i1.lazy_pointer: +; CHECK: .section __TEXT,__text,regular,pure_instructions +; CHECK: _i1: +; CHECK: _i1.stub_helper: +; CHECK: .section __DATA,__data +; CHECK: _i2.lazy_pointer: +; CHECK: .section __TEXT,__text,regular,pure_instructions +; CHECK: _i2: +; CHECK: _i2.stub_helper: + +; CHECK: .section __DWARF + ; CHECK: .addrsig{{$}} ; CHECK-NEXT: .addrsig_sym _func03_takeaddr ; CHECK-NEXT: .addrsig_sym _f1 diff --git a/llvm/test/CodeGen/AArch64/ifunc-asm.ll b/llvm/test/CodeGen/AArch64/ifunc-asm.ll new file mode 100644 index 0000000000000..57fc2f0c9d7f5 --- /dev/null +++ b/llvm/test/CodeGen/AArch64/ifunc-asm.ll @@ -0,0 +1,70 @@ +; RUN: llc -mtriple=arm64-unknown-linux-gnu %s -o - | FileCheck %s --check-prefixes=ELF +; RUN: llc -mtriple=arm64-apple-darwin %s -o - | FileCheck %s --check-prefix=MACHO +; RUN: llc -mtriple=arm64-apple-darwin %s -global-isel -o - | FileCheck %s --check-prefix=MACHO + +define internal ptr @the_resolver() { +entry: + ret ptr null +} +; ELF: .type the_resolver,@function +; ELF-NEXT: the_resolver: + +; MACHO: .p2align 2 +; MACHO-NEXT: _the_resolver: + + +@global_ifunc = ifunc i32 (i32), ptr @the_resolver +; ELF: .globl global_ifunc +; ELF-NEXT: .type global_ifunc,@gnu_indirect_function +; ELF-NEXT: .set global_ifunc, the_resolver + +; MACHO: .section __DATA,__data +; MACHO-NEXT: .p2align 3, 0x0 +; MACHO-NEXT: _global_ifunc.lazy_pointer: +; MACHO-NEXT: .quad _global_ifunc.stub_helper + +; MACHO: .section __TEXT,__text,regular,pure_instructions +; MACHO-NEXT: .globl _global_ifunc +; MACHO-NEXT: .p2align 2 +; MACHO-NEXT: _global_ifunc: +; MACHO-NEXT: adrp x16, _global_ifunc.lazy_pointer@GOTPAGE +; MACHO-NEXT: ldr x16, [x16, _global_ifunc.lazy_pointer@GOTPAGEOFF] +; MACHO-NEXT: ldr x16, [x16] +; MACHO-NEXT: br x16 +; MACHO-NEXT: .p2align 2 +; MACHO-NEXT: _global_ifunc.stub_helper: +; MACHO-NEXT: stp x29, x30, [sp, #-16]! +; MACHO-NEXT: mov x29, sp +; MACHO-NEXT: stp x1, x0, [sp, #-16]! +; MACHO-NEXT: stp x3, x2, [sp, #-16]! +; MACHO-NEXT: stp x5, x4, [sp, #-16]! +; MACHO-NEXT: stp x7, x6, [sp, #-16]! +; MACHO-NEXT: stp d1, d0, [sp, #-16]! +; MACHO-NEXT: stp d3, d2, [sp, #-16]! +; MACHO-NEXT: stp d5, d4, [sp, #-16]! +; MACHO-NEXT: stp d7, d6, [sp, #-16]! +; MACHO-NEXT: bl _the_resolver +; MACHO-NEXT: adrp x16, _global_ifunc.lazy_pointer@GOTPAGE +; MACHO-NEXT: ldr x16, [x16, _global_ifunc.lazy_pointer@GOTPAGEOFF] +; MACHO-NEXT: str x0, [x16] +; MACHO-NEXT: add x16, x0, #0 +; MACHO-NEXT: ldp d7, d6, [sp], #16 +; MACHO-NEXT: ldp d5, d4, [sp], #16 +; MACHO-NEXT: ldp d3, d2, [sp], #16 +; MACHO-NEXT: ldp d1, d0, [sp], #16 +; MACHO-NEXT: ldp x7, x6, [sp], #16 +; MACHO-NEXT: ldp x5, x4, [sp], #16 +; MACHO-NEXT: ldp x3, x2, [sp], #16 +; MACHO-NEXT: ldp x1, x0, [sp], #16 +; MACHO-NEXT: ldp x29, x30, [sp], #16 +; MACHO-NEXT: br x16 + + +@weak_ifunc = weak ifunc i32 (i32), ptr @the_resolver +; ELF: .type weak_ifunc,@gnu_indirect_function +; MACHO-NOT: .weak_reference _weak_ifunc.lazy_pointer +; MACHO: _weak_ifunc.lazy_pointer: +; MACHO: .weak_reference _weak_ifunc +; MACHO: _weak_ifunc: +; MACHO-NOT: .weak_reference _weak_ifunc.stub_helper +; MACHO: _weak_ifunc.stub_helper: \ No newline at end of file diff --git a/llvm/test/CodeGen/X86/ifunc-asm.ll b/llvm/test/CodeGen/X86/ifunc-asm.ll index 4b380c8ae3301..6cfd37604ff5e 100644 --- a/llvm/test/CodeGen/X86/ifunc-asm.ll +++ b/llvm/test/CodeGen/X86/ifunc-asm.ll @@ -1,14 +1,56 @@ -; RUN: llvm-as < %s -o - | llc -filetype=asm | FileCheck %s +; RUN: llc -mtriple=x86_64-unknown-linux-gnu %s -o - | FileCheck %s --check-prefixes=ELF +; RUN: llc -mtriple=x86_64-apple-darwin %s -o - | FileCheck %s --check-prefixes=MACHO -target triple = "x86_64-unknown-linux-gnu" - -define internal ptr @foo_ifunc() { +define internal ptr @foo_resolver() { entry: ret ptr null } -; CHECK: .type foo_ifunc,@function -; CHECK-NEXT: foo_ifunc: +; ELF: .type foo_resolver,@function +; ELF-NEXT: foo_resolver: + +; MACHO: .p2align 4, 0x90 +; MACHO-NEXT: _foo_resolver + + +@foo_ifunc = ifunc i32 (i32), ptr @foo_resolver +; ELF: .globl foo_ifunc +; ELF-NEXT: .type foo_ifunc,@gnu_indirect_function +; ELF-NEXT: .set foo_ifunc, foo_resolver + +; MACHO: .section __DATA,__data +; MACHO-NEXT: .p2align 3, 0x0 +; MACHO-NEXT: _foo_ifunc.lazy_pointer: +; MACHO-NEXT: .quad _foo_ifunc.stub_helper +; MACHO-NEXT: .section __TEXT,__text,regular,pure_instructions +; MACHO-NEXT: .globl _foo_ifunc +; MACHO-NEXT: .p2align 0, 0x90 +; MACHO-NEXT: _foo_ifunc: +; MACHO-NEXT: jmpl *_foo_ifunc.lazy_pointer(%rip) +; MACHO-NEXT: .p2align 0, 0x90 +; MACHO-NEXT: _foo_ifunc.stub_helper: +; MACHO-NEXT: pushq %rax +; MACHO-NEXT: pushq %rdi +; MACHO-NEXT: pushq %rsi +; MACHO-NEXT: pushq %rdx +; MACHO-NEXT: pushq %rcx +; MACHO-NEXT: pushq %r8 +; MACHO-NEXT: pushq %r9 +; MACHO-NEXT: callq _foo_resolver +; MACHO-NEXT: movq %rax, _foo_ifunc.lazy_pointer(%rip) +; MACHO-NEXT: popq %r9 +; MACHO-NEXT: popq %r8 +; MACHO-NEXT: popq %rcx +; MACHO-NEXT: popq %rdx +; MACHO-NEXT: popq %rsi +; MACHO-NEXT: popq %rdi +; MACHO-NEXT: popq %rax +; MACHO-NEXT: jmpl *_foo_ifunc.lazy_pointer(%rip) -@foo = ifunc i32 (i32), ptr @foo_ifunc -; CHECK: .type foo,@gnu_indirect_function -; CHECK-NEXT: .set foo, foo_ifunc +@weak_ifunc = weak ifunc i32 (i32), ptr @foo_resolver +; ELF: .type weak_ifunc,@gnu_indirect_function +; MACHO-NOT: .weak_reference _weak_ifunc.lazy_pointer +; MACHO: _weak_ifunc.lazy_pointer: +; MACHO: .weak_reference _weak_ifunc +; MACHO: _weak_ifunc: +; MACHO-NOT: .weak_reference _weak_ifunc.stub_helper +; MACHO: _weak_ifunc.stub_helper: