[builtins][arm64] Build __init_cpu_features_resolver on Apple platforms

clang/include/clang/Basic/Attr.td

@@ -455,6 +455,9 @@ def TargetMicrosoftCXXABI : TargetArch<["x86", "x86_64", "arm", "thumb", "aarch6
 def TargetELF : TargetSpec {
   let ObjectFormats = ["ELF"];
 }
+def TargetELFOrMachO : TargetSpec {
+  let ObjectFormats = ["ELF", "MachO"];
+}
 
 def TargetSupportsInitPriority : TargetSpec {
   let CustomCode = [{ !Target.getTriple().isOSzOS() }];
@@ -1665,7 +1668,7 @@ def IBOutletCollection : InheritableAttr {
   let Documentation = [Undocumented];
 }
 
-def IFunc : Attr, TargetSpecificAttr<TargetELF> {
+def IFunc : Attr, TargetSpecificAttr<TargetELFOrMachO> {
   let Spellings = [GCC<"ifunc">];
   let Args = [StringArgument<"Resolver">];
   let Subjects = SubjectList<[Function]>;

clang/include/clang/Basic/AttrDocs.td

@@ -5408,7 +5408,9 @@ considered inline.
 Not all targets support this attribute. ELF target support depends on both the
 linker and runtime linker, and is available in at least lld 4.0 and later,
 binutils 2.20.1 and later, glibc v2.11.1 and later, and FreeBSD 9.1 and later.
-Non-ELF targets currently do not support this attribute.
+MachO targets support it, but with slightly different semantics: the resolver is
+run at first call, instead of at load time by the runtime linker. Targets other
+than ELF and MachO currently do not support this attribute.
   }];
 }
 

clang/include/clang/Basic/TargetInfo.h

@@ -1424,6 +1424,8 @@ class TargetInfo : public TransferrableTargetInfo,
 
   /// Identify whether this target supports IFuncs.
   bool supportsIFunc() const {
+    if (getTriple().isOSBinFormatMachO())
+      return true;
     return getTriple().isOSBinFormatELF() &&
            ((getTriple().isOSLinux() && !getTriple().isMusl()) ||
             getTriple().isOSFreeBSD());

clang/test/CodeGen/attr-target-mv-va-args.c

@@ -3,6 +3,7 @@
 // RUN: %clang_cc1 -triple x86_64-windows-pc -emit-llvm %s -o - | FileCheck %s --check-prefixes=NO-IFUNC,WINDOWS
 // RUN: %clang_cc1 -triple x86_64-linux-musl -emit-llvm %s -o - | FileCheck %s --check-prefixes=NO-IFUNC,NO-IFUNC-ELF
 // RUN: %clang_cc1 -triple x86_64-fuchsia -emit-llvm %s -o - | FileCheck %s --check-prefixes=NO-IFUNC,NO-IFUNC-ELF
+// RUN: %clang_cc1 -triple x86_64-apple-macho -emit-llvm %s -o - | FileCheck %s --check-prefix=IFUNC-MACHO
 int __attribute__((target("sse4.2"))) foo(int i, ...) { return 0; }
 int __attribute__((target("arch=sandybridge"))) foo(int i, ...);
 int __attribute__((target("arch=ivybridge"))) foo(int i, ...) {return 1;}
@@ -30,6 +31,24 @@ int bar(void) {
 // IFUNC-ELF: ret ptr @foo
 // IFUNC-ELF: declare i32 @foo.arch_sandybridge(i32 noundef, ...)
 
+// IFUNC-MACHO: @foo.ifunc = weak_odr ifunc i32 (i32, ...), ptr @foo.resolver
+// IFUNC-MACHO: define{{.*}} i32 @foo.sse4.2(i32 noundef %i, ...)
+// IFUNC-MACHO: ret i32 0
+// IFUNC-MACHO: define{{.*}} i32 @foo.arch_ivybridge(i32 noundef %i, ...)
+// IFUNC-MACHO: ret i32 1
+// IFUNC-MACHO: define{{.*}} i32 @foo(i32 noundef %i, ...)
+// IFUNC-MACHO: ret i32 2
+// IFUNC-MACHO: define{{.*}} i32 @bar()
+// IFUNC-MACHO: call i32 (i32, ...) @foo.ifunc(i32 noundef 1, i32 noundef 97, double
+// IFUNC-MACHO: call i32 (i32, ...) @foo.ifunc(i32 noundef 2, double noundef 2.2{{[0-9Ee+]+}}, ptr noundef
+
+// IFUNC-MACHO: define weak_odr ptr @foo.resolver()
+// IFUNC-MACHO: ret ptr @foo.arch_sandybridge
+// IFUNC-MACHO: ret ptr @foo.arch_ivybridge
+// IFUNC-MACHO: ret ptr @foo.sse4.2
+// IFUNC-MACHO: ret ptr @foo
+// IFUNC-MACHO: declare i32 @foo.arch_sandybridge(i32 noundef, ...)
+
 // NO-IFUNC: define dso_local i32 @foo.sse4.2(i32 noundef %i, ...)
 // NO-IFUNC: ret i32 0
 // NO-IFUNC: define dso_local i32 @foo.arch_ivybridge(i32 noundef %i, ...)

clang/test/CodeGen/ifunc.c

@@ -3,6 +3,10 @@
 // RUN: %clang_cc1 -triple i386-unknown-linux-gnu -fsanitize=thread -O2 -emit-llvm -o - %s | FileCheck %s --check-prefix=SAN
 // RUN: %clang_cc1 -triple i386-unknown-linux-gnu -fsanitize=address -O2 -emit-llvm -o - %s | FileCheck %s --check-prefix=SAN
 // RUN: %clang_cc1 -triple i386-unknown-linux-gnu -fsanitize=memory -O2 -emit-llvm -o - %s | FileCheck %s --check-prefix=SAN
+// RUN: %clang_cc1 -triple x86_64-apple-macosx -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple x86_64-apple-macosx -O2 -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple x86_64-apple-macosx -fsanitize=thread -O2 -emit-llvm -o - %s | FileCheck %s --check-prefix=MACSAN
+// RUN: %clang_cc1 -triple x86_64-apple-macosx -fsanitize=address -O2 -emit-llvm -o - %s | FileCheck %s --check-prefix=MACSAN
 
 int foo(int) __attribute__ ((ifunc("foo_ifunc")));
 
@@ -44,9 +48,13 @@ void* goo_ifunc(void) {
 // CHECK: call void @goo()
 
 // SAN: define internal nonnull ptr @foo_ifunc() #[[#FOO_IFUNC:]] {
+// MACSAN: define internal nonnull ptr @foo_ifunc() #[[#FOO_IFUNC:]] {
 
 // SAN: define dso_local noalias ptr @goo_ifunc() #[[#GOO_IFUNC:]] {
+// MACSAN: define noalias ptr @goo_ifunc() #[[#GOO_IFUNC:]] {
 
 // SAN-DAG: attributes #[[#FOO_IFUNC]] = {{{.*}} disable_sanitizer_instrumentation {{.*}}
+// MACSAN-DAG: attributes #[[#FOO_IFUNC]] = {{{.*}} disable_sanitizer_instrumentation {{.*}}
 
 // SAN-DAG: attributes #[[#GOO_IFUNC]] = {{{.*}} disable_sanitizer_instrumentation {{.*}}
+// MACSAN-DAG: attributes #[[#GOO_IFUNC]] = {{{.*}} disable_sanitizer_instrumentation {{.*}}

compiler-rt/lib/builtins/cpu_model.c

@@ -948,6 +948,8 @@ _Bool __aarch64_have_lse_atomics
 #if defined(__has_include)
 #if __has_include(<sys/auxv.h>)
 #include <sys/auxv.h>
+#define HAVE_SYS_AUXV
+#endif
 
 #if __has_include(<sys/ifunc.h>)
 #include <sys/ifunc.h>
@@ -961,6 +963,8 @@ typedef struct __ifunc_arg_t {
 
 #if __has_include(<asm/hwcap.h>)
 #include <asm/hwcap.h>
+#include HAVE_SYS_HWCAP
+#endif
 
 #if defined(__ANDROID__)
 #include <string.h>
@@ -997,6 +1001,9 @@ typedef struct __ifunc_arg_t {
 #ifndef HWCAP_SHA2
 #define HWCAP_SHA2 (1 << 6)
 #endif
+#ifndef HWCAP_CRC32
+#define HWCAP_CRC32 (1 << 7)
+#endif
 #ifndef HWCAP_ATOMICS
 #define HWCAP_ATOMICS (1 << 8)
 #endif
@@ -1149,6 +1156,7 @@ typedef struct __ifunc_arg_t {
   if (__system_property_get("ro.arch", arch) > 0 &&                            \
       strncmp(arch, "exynos9810", sizeof("exynos9810") - 1) == 0)
 
+#if !defined(__APPLE__)
 static void CONSTRUCTOR_ATTRIBUTE init_have_lse_atomics(void) {
 #if defined(__FreeBSD__)
   unsigned long hwcap;
@@ -1162,7 +1170,7 @@ static void CONSTRUCTOR_ATTRIBUTE init_have_lse_atomics(void) {
   zx_status_t status = _zx_system_get_features(ZX_FEATURE_KIND_CPU, &features);
   __aarch64_have_lse_atomics =
       status == ZX_OK && (features & ZX_ARM64_FEATURE_ISA_ATOMICS) != 0;
-#else
+#elif defined(HAVE_SYS_AUXV)
   unsigned long hwcap = getauxval(AT_HWCAP);
   _Bool result = (hwcap & HWCAP_ATOMICS) != 0;
 #if defined(__ANDROID__)
@@ -1180,8 +1188,11 @@ static void CONSTRUCTOR_ATTRIBUTE init_have_lse_atomics(void) {
   }
 #endif // defined(__ANDROID__)
   __aarch64_have_lse_atomics = result;
+#else
+#error No support for checking for lse atomics on this platfrom yet.
 #endif // defined(__FreeBSD__)
 }
+#endif // !defined(__APPLE__)
 
 #if !defined(DISABLE_AARCH64_FMV)
 // CPUFeatures must correspond to the same AArch64 features in
@@ -1259,6 +1270,64 @@ struct {
   // As features grows new fields could be added
 } __aarch64_cpu_features __attribute__((visibility("hidden"), nocommon));
 
+#if defined(__APPLE__)
+#include <TargetConditionals.h>
+#if TARGET_OS_OSX || TARGET_OS_IPHONE
+#include <dispatch/dispatch.h>
+#include <sys/sysctl.h>
+
+static bool isKnownAndSupported(const char *name) {
+  int32_t val = 0;
+  size_t size = sizeof(val);
+  if (sysctlbyname(name, &val, &size, NULL, 0))
+    return false;
+  return val;
+}
+
+void __init_cpu_features_resolver(void) {
+  static dispatch_once_t onceToken = 0;
+  dispatch_once(&onceToken, ^{
+    // https://developer.apple.com/documentation/kernel/1387446-sysctlbyname/determining_instruction_set_characteristics
+    static struct {
+      const char *sysctl_name;
+      enum CPUFeatures feature;
+    } Features[] = {
+        {"hw.optional.arm.FEAT_FlagM", FEAT_FLAGM},
+        {"hw.optional.arm.FEAT_FlagM2", FEAT_FLAGM2},
+        {"hw.optional.arm.FEAT_FHM", FEAT_FP16FML},
+        {"hw.optional.arm.FEAT_DotProd", FEAT_DOTPROD},
+        {"hw.optional.arm.FEAT_RDM", FEAT_RDM},
+        {"hw.optional.arm.FEAT_LSE", FEAT_LSE},
+        {"hw.optional.floatingpoint", FEAT_FP},
+        {"hw.optional.AdvSIMD", FEAT_SIMD},
+        {"hw.optional.armv8_crc32", FEAT_CRC},
+        {"hw.optional.arm.FEAT_SHA1", FEAT_SHA1},
+        {"hw.optional.arm.FEAT_SHA256", FEAT_SHA2},
+        {"hw.optional.armv8_2_sha3", FEAT_SHA3},
+        {"hw.optional.arm.FEAT_AES", FEAT_AES},
+        {"hw.optional.arm.FEAT_PMULL", FEAT_PMULL},
+        {"hw.optional.arm.FEAT_FP16", FEAT_FP16},
+        {"hw.optional.arm.FEAT_JSCVT", FEAT_JSCVT},
+        {"hw.optional.arm.FEAT_FCMA", FEAT_FCMA},
+        {"hw.optional.arm.FEAT_LRCPC", FEAT_RCPC},
+        {"hw.optional.arm.FEAT_LRCPC2", FEAT_RCPC2},
+        {"hw.optional.arm.FEAT_FRINTTS", FEAT_FRINTTS},
+        {"hw.optional.arm.FEAT_I8MM", FEAT_I8MM},
+        {"hw.optional.arm.FEAT_BF16", FEAT_BF16},
+        {"hw.optional.arm.FEAT_SB", FEAT_SB},
+        {"hw.optional.arm.FEAT_SSBS", FEAT_SSBS2},
+        {"hw.optional.arm.FEAT_BTI", FEAT_BTI},
+    };
+
+    for (size_t I = 0, E = sizeof(Features) / sizeof(Features[0]); I != E; ++I)
+      if (isKnownAndSupported(Features[I].sysctl_name))
+        __aarch64_cpu_features.features |= (1ULL << Features[I].feature);
+
+    __aarch64_cpu_features.features |= (1ULL << FEAT_INIT);
+  });
+}
+#endif // TARGET_OS_OSX || TARGET_OS_IPHONE
+#else  // defined(__APPLE__)
 static void __init_cpu_features_constructor(unsigned long hwcap,
                                             const __ifunc_arg_t *arg) {
 #define setCPUFeature(F) __aarch64_cpu_features.features |= 1ULL << F
@@ -1467,8 +1536,8 @@ void __init_cpu_features_resolver(unsigned long hwcap,
 }
 
 void CONSTRUCTOR_ATTRIBUTE __init_cpu_features(void) {
-  unsigned long hwcap;
-  unsigned long hwcap2;
+  unsigned long hwcap = 0;
+  unsigned long hwcap2 = 0;
   // CPU features already initialized.
   if (__aarch64_cpu_features.features)
     return;
@@ -1478,14 +1547,16 @@ void CONSTRUCTOR_ATTRIBUTE __init_cpu_features(void) {
   res |= elf_aux_info(AT_HWCAP2, &hwcap2, sizeof hwcap2);
   if (res)
     return;
-#else
+#elif defined(HAVE_SYS_AUXV)
 #if defined(__ANDROID__)
   // Don't set any CPU features,
   // detection could be wrong on Exynos 9810.
   IF_EXYNOS9810 return;
 #endif // defined(__ANDROID__)
   hwcap = getauxval(AT_HWCAP);
   hwcap2 = getauxval(AT_HWCAP2);
+#else
+#error No support for checking hwcap on this platform yet.
 #endif // defined(__FreeBSD__)
   __ifunc_arg_t arg;
   arg._size = sizeof(__ifunc_arg_t);
@@ -1497,8 +1568,7 @@ void CONSTRUCTOR_ATTRIBUTE __init_cpu_features(void) {
 #undef setCPUFeature
 #undef IF_EXYNOS9810
 }
+#endif // defined(__APPLE__)
 #endif // !defined(DISABLE_AARCH64_FMV)
 #endif // defined(__has_include)
-#endif // __has_include(<sys/auxv.h>)
-#endif // __has_include(<asm/hwcap.h>)
 #endif // defined(__aarch64__)

llvm/docs/LangRef.rst

@@ -934,10 +934,11 @@ IFuncs
 -------
 
 IFuncs, like as aliases, don't create any new data or func. They are just a new
-symbol that dynamic linker resolves at runtime by calling a resolver function.
+symbol that is resolved at runtime by calling a resolver function.
 
-IFuncs have a name and a resolver that is a function called by dynamic linker
-that returns address of another function associated with the name.
+On ELF platforms, IFuncs are resolved by the dynamic linker at load time. On
+MachO platforms, they are lowered in terms of ``.symbol_resolver``s, which
+lazily resolve the callee the first time they are called.
 
 IFunc may have an optional :ref:`linkage type <linkage>` and an optional
 :ref:`visibility style <visibility>`.

llvm/include/llvm/CodeGen/AsmPrinter.h

@@ -882,7 +882,11 @@ class AsmPrinter : public MachineFunctionPass {
 
   GCMetadataPrinter *getOrCreateGCPrinter(GCStrategy &S);
   void emitGlobalAlias(Module &M, const GlobalAlias &GA);
-  void emitGlobalIFunc(Module &M, const GlobalIFunc &GI);
+
+protected:
+  virtual void emitGlobalIFunc(Module &M, const GlobalIFunc &GI);
+
+private:
 
   /// This method decides whether the specified basic block requires a label.
   bool shouldEmitLabelForBasicBlock(const MachineBasicBlock &MBB) const;

llvm/lib/CodeGen/GlobalISel/CallLowering.cpp

@@ -144,7 +144,12 @@ bool CallLowering::lowerCall(MachineIRBuilder &MIRBuilder, const CallBase &CB,
   // Try looking through a bitcast from one function type to another.
   // Commonly happens with calls to objc_msgSend().
   const Value *CalleeV = CB.getCalledOperand()->stripPointerCasts();
-  if (const Function *F = dyn_cast<Function>(CalleeV))
+  if (const GlobalIFunc *IF = dyn_cast<GlobalIFunc>(CalleeV);
+      IF && MF.getTarget().getTargetTriple().isOSBinFormatMachO()) {
+    // ld64 requires that .symbol_resolvers to be called via a stub, so these
+    // must always be a diret call.
+    Info.Callee = MachineOperand::CreateGA(IF, 0);
+  } else if (const Function *F = dyn_cast<Function>(CalleeV))
     Info.Callee = MachineOperand::CreateGA(F, 0);
   else
     Info.Callee = MachineOperand::CreateReg(GetCalleeReg(), false);

llvm/lib/IR/Verifier.cpp

@@ -959,6 +959,7 @@ void Verifier::visitGlobalIFunc(const GlobalIFunc &GI) {
       GlobalIFunc::getResolverFunctionType(GI.getValueType());
   Check(ResolverTy == ResolverFuncTy->getPointerTo(GI.getAddressSpace()),
         "IFunc resolver has incorrect type", &GI);
+
 }
 
 void Verifier::visitNamedMDNode(const NamedMDNode &NMD) {
@@ -2239,13 +2240,10 @@ void Verifier::verifyFunctionAttrs(FunctionType *FT, AttributeList Attrs,
   }
 
   // Check EVEX512 feature.
-  if (MaxParameterWidth >= 512 && Attrs.hasFnAttr("target-features")) {
-    Triple T(M.getTargetTriple());
-    if (T.isX86()) {
-      StringRef TF = Attrs.getFnAttr("target-features").getValueAsString();
-      Check(!TF.contains("+avx512f") || !TF.contains("-evex512"),
-            "512-bit vector arguments require 'evex512' for AVX512", V);
-    }
+  if (MaxParameterWidth >= 512 && Attrs.hasFnAttr("target-features") && TT.isX86()) {
+    StringRef TF = Attrs.getFnAttr("target-features").getValueAsString();
+    Check(!TF.contains("+avx512f") || !TF.contains("-evex512"),
+          "512-bit vector arguments require 'evex512' for AVX512", V);
   }
 
   checkUnsignedBaseTenFuncAttr(Attrs, "patchable-function-prefix", V);