diff --git a/compiler-rt/include/profile/InstrProfData.inc b/compiler-rt/include/profile/InstrProfData.inc index e9866d94b762c..6d2df2195c739 100644 --- a/compiler-rt/include/profile/InstrProfData.inc +++ b/compiler-rt/include/profile/InstrProfData.inc @@ -738,6 +738,7 @@ serializeValueProfDataFrom(ValueProfRecordClosure *Closure, #define INSTR_PROF_RAW_VERSION_VAR __llvm_profile_raw_version #define INSTR_PROF_PROFILE_RUNTIME_VAR __llvm_profile_runtime #define INSTR_PROF_PROFILE_COUNTER_BIAS_VAR __llvm_profile_counter_bias +#define INSTR_PROF_PROFILE_BITMAP_BIAS_VAR __llvm_profile_bitmap_bias #define INSTR_PROF_PROFILE_SET_TIMESTAMP __llvm_profile_set_timestamp /* The variable that holds the name of the profile data diff --git a/compiler-rt/lib/profile/InstrProfilingFile.c b/compiler-rt/lib/profile/InstrProfilingFile.c index 1c58584d2d4f7..1116ea0bb2396 100644 --- a/compiler-rt/lib/profile/InstrProfilingFile.c +++ b/compiler-rt/lib/profile/InstrProfilingFile.c @@ -101,6 +101,8 @@ static const int UseBiasVar = 0; static const char *FileOpenMode = "a+b"; static void *BiasAddr = NULL; static void *BiasDefaultAddr = NULL; +static void *BitmapBiasAddr = NULL; +static void *BitmapBiasDefaultAddr = NULL; static int mmapForContinuousMode(uint64_t CurrentFileOffset, FILE *File) { /* Get the sizes of various profile data sections. Taken from * __llvm_profile_get_size_for_buffer(). */ @@ -199,11 +201,15 @@ static int mmapForContinuousMode(uint64_t CurrentFileOffset, FILE *File) { #define INSTR_PROF_PROFILE_COUNTER_BIAS_DEFAULT_VAR \ INSTR_PROF_CONCAT(INSTR_PROF_PROFILE_COUNTER_BIAS_VAR, _default) COMPILER_RT_VISIBILITY intptr_t INSTR_PROF_PROFILE_COUNTER_BIAS_DEFAULT_VAR = 0; +#define INSTR_PROF_PROFILE_BITMAP_BIAS_DEFAULT_VAR \ + INSTR_PROF_CONCAT(INSTR_PROF_PROFILE_BITMAP_BIAS_VAR, _default) +COMPILER_RT_VISIBILITY intptr_t INSTR_PROF_PROFILE_BITMAP_BIAS_DEFAULT_VAR = 0; /* This variable is a weak external reference which could be used to detect * whether or not the compiler defined this symbol. */ #if defined(_MSC_VER) COMPILER_RT_VISIBILITY extern intptr_t INSTR_PROF_PROFILE_COUNTER_BIAS_VAR; +COMPILER_RT_VISIBILITY extern intptr_t INSTR_PROF_PROFILE_BITMAP_BIAS_VAR; #if defined(_M_IX86) || defined(__i386__) #define WIN_SYM_PREFIX "_" #else @@ -213,10 +219,17 @@ COMPILER_RT_VISIBILITY extern intptr_t INSTR_PROF_PROFILE_COUNTER_BIAS_VAR; linker, "/alternatename:" WIN_SYM_PREFIX INSTR_PROF_QUOTE( \ INSTR_PROF_PROFILE_COUNTER_BIAS_VAR) "=" WIN_SYM_PREFIX \ INSTR_PROF_QUOTE(INSTR_PROF_PROFILE_COUNTER_BIAS_DEFAULT_VAR)) +#pragma comment( \ + linker, "/alternatename:" WIN_SYM_PREFIX INSTR_PROF_QUOTE( \ + INSTR_PROF_PROFILE_BITMAP_BIAS_VAR) "=" WIN_SYM_PREFIX \ + INSTR_PROF_QUOTE(INSTR_PROF_PROFILE_BITMAP_BIAS_DEFAULT_VAR)) #else COMPILER_RT_VISIBILITY extern intptr_t INSTR_PROF_PROFILE_COUNTER_BIAS_VAR __attribute__((weak, alias(INSTR_PROF_QUOTE( INSTR_PROF_PROFILE_COUNTER_BIAS_DEFAULT_VAR)))); +COMPILER_RT_VISIBILITY extern intptr_t INSTR_PROF_PROFILE_BITMAP_BIAS_VAR + __attribute__((weak, alias(INSTR_PROF_QUOTE( + INSTR_PROF_PROFILE_BITMAP_BIAS_DEFAULT_VAR)))); #endif static const int ContinuousModeSupported = 1; static const int UseBiasVar = 1; @@ -227,6 +240,9 @@ static const char *FileOpenMode = "w+b"; * used and runtime provides a weak alias so we can check if it's defined. */ static void *BiasAddr = &INSTR_PROF_PROFILE_COUNTER_BIAS_VAR; static void *BiasDefaultAddr = &INSTR_PROF_PROFILE_COUNTER_BIAS_DEFAULT_VAR; +static void *BitmapBiasAddr = &INSTR_PROF_PROFILE_BITMAP_BIAS_VAR; +static void *BitmapBiasDefaultAddr = + &INSTR_PROF_PROFILE_BITMAP_BIAS_DEFAULT_VAR; static int mmapForContinuousMode(uint64_t CurrentFileOffset, FILE *File) { /* Get the sizes of various profile data sections. Taken from * __llvm_profile_get_size_for_buffer(). */ @@ -237,12 +253,18 @@ static int mmapForContinuousMode(uint64_t CurrentFileOffset, FILE *File) { const char *BitmapBegin = __llvm_profile_begin_bitmap(); const char *BitmapEnd = __llvm_profile_end_bitmap(); uint64_t DataSize = __llvm_profile_get_data_size(DataBegin, DataEnd); + uint64_t CountersSize = + __llvm_profile_get_counters_size(CountersBegin, CountersEnd); + uint64_t NumBitmapBytes = + __llvm_profile_get_num_bitmap_bytes(BitmapBegin, BitmapEnd); /* Get the file size. */ uint64_t FileSize = 0; if (getProfileFileSizeForMerging(File, &FileSize)) return 1; int Fileno = fileno(File); + uint64_t PaddingBytesAfterCounters = + __llvm_profile_get_num_padding_bytes(CountersSize); uint64_t FileOffsetToCounters = sizeof(__llvm_profile_header) + __llvm_write_binary_ids(NULL) + DataSize; @@ -260,7 +282,17 @@ static int mmapForContinuousMode(uint64_t CurrentFileOffset, FILE *File) { /* Return the memory allocated for counters to OS. */ lprofReleaseMemoryPagesToOS((uintptr_t)CountersBegin, (uintptr_t)CountersEnd); - /* BIAS MODE not supported yet for Bitmap (MCDC). */ + /* Also mmap MCDC bitmap bytes. If there aren't any bitmap bytes, mmap() + * will fail with EINVAL. */ + if (NumBitmapBytes == 0) + return 0; + + /* Update profbm_bias. */ + uint64_t FileOffsetToBitmap = + FileOffsetToCounters + CountersSize + PaddingBytesAfterCounters; + /* Update the profile fields based on the current mapping. */ + INSTR_PROF_PROFILE_BITMAP_BIAS_VAR = + (uintptr_t)Profile - (uintptr_t)BitmapBegin + FileOffsetToBitmap; /* Return the memory allocated for counters to OS. */ lprofReleaseMemoryPagesToOS((uintptr_t)BitmapBegin, (uintptr_t)BitmapEnd); @@ -619,8 +651,10 @@ static void initializeProfileForContinuousMode(void) { PROF_ERR("%s\n", "continuous mode is unsupported on this platform"); return; } - if (UseBiasVar && BiasAddr == BiasDefaultAddr) { - PROF_ERR("%s\n", "__llvm_profile_counter_bias is undefined"); + if (UseBiasVar && BiasAddr == BiasDefaultAddr && + BitmapBiasAddr == BitmapBiasDefaultAddr) { + PROF_ERR("%s\n", "Neither __llvm_profile_counter_bias nor " + "__llvm_profile_bitmap_bias is defined"); return; } diff --git a/llvm/include/llvm/ProfileData/InstrProf.h b/llvm/include/llvm/ProfileData/InstrProf.h index 7fa6d44990a14..ff812923ed01d 100644 --- a/llvm/include/llvm/ProfileData/InstrProf.h +++ b/llvm/include/llvm/ProfileData/InstrProf.h @@ -174,6 +174,10 @@ inline StringRef getInstrProfCounterBiasVarName() { return INSTR_PROF_QUOTE(INSTR_PROF_PROFILE_COUNTER_BIAS_VAR); } +inline StringRef getInstrProfBitmapBiasVarName() { + return INSTR_PROF_QUOTE(INSTR_PROF_PROFILE_BITMAP_BIAS_VAR); +} + /// Return the marker used to separate PGO names during serialization. inline StringRef getInstrProfNameSeparator() { return "\01"; } diff --git a/llvm/include/llvm/ProfileData/InstrProfData.inc b/llvm/include/llvm/ProfileData/InstrProfData.inc index e9866d94b762c..6d2df2195c739 100644 --- a/llvm/include/llvm/ProfileData/InstrProfData.inc +++ b/llvm/include/llvm/ProfileData/InstrProfData.inc @@ -738,6 +738,7 @@ serializeValueProfDataFrom(ValueProfRecordClosure *Closure, #define INSTR_PROF_RAW_VERSION_VAR __llvm_profile_raw_version #define INSTR_PROF_PROFILE_RUNTIME_VAR __llvm_profile_runtime #define INSTR_PROF_PROFILE_COUNTER_BIAS_VAR __llvm_profile_counter_bias +#define INSTR_PROF_PROFILE_BITMAP_BIAS_VAR __llvm_profile_bitmap_bias #define INSTR_PROF_PROFILE_SET_TIMESTAMP __llvm_profile_set_timestamp /* The variable that holds the name of the profile data diff --git a/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp b/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp index f994f8a62c320..c90525140cd84 100644 --- a/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp +++ b/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp @@ -1014,18 +1014,22 @@ CallInst *InstrLowerer::getRMWOrCall(Value *Addr, Value *Val) { Value *InstrLowerer::getBitmapAddress(InstrProfMCDCTVBitmapUpdate *I) { auto *Bitmaps = getOrCreateRegionBitmaps(I); - IRBuilder<> Builder(I); - - if (isRuntimeCounterRelocationEnabled()) { - LLVMContext &Ctx = M.getContext(); - Ctx.diagnose(DiagnosticInfoPGOProfile( - M.getName().data(), - Twine("Runtime counter relocation is presently not supported for MC/DC " - "bitmaps."), - DS_Warning)); - } + if (!isRuntimeCounterRelocationEnabled()) + return Bitmaps; - return Bitmaps; + // Put BiasLI onto the entry block. + Type *Int64Ty = Type::getInt64Ty(M.getContext()); + Function *Fn = I->getFunction(); + IRBuilder<> EntryBuilder(&Fn->getEntryBlock().front()); + auto *Bias = getOrCreateBiasVar(getInstrProfBitmapBiasVarName()); + auto *BiasLI = EntryBuilder.CreateLoad(Int64Ty, Bias, "profbm_bias"); + // Assume BiasLI invariant (in the function at least) + BiasLI->setMetadata(LLVMContext::MD_invariant_load, + MDNode::get(M.getContext(), std::nullopt)); + + // Add Bias to Bitmaps and put it before the intrinsic. + IRBuilder<> Builder(I); + return Builder.CreatePtrAdd(Bitmaps, BiasLI, "profbm_addr"); } void InstrLowerer::lowerCover(InstrProfCoverInst *CoverInstruction) { diff --git a/llvm/test/Instrumentation/InstrProfiling/mcdc.ll b/llvm/test/Instrumentation/InstrProfiling/mcdc.ll index 20c002e0565db..8bed7be325fa3 100644 --- a/llvm/test/Instrumentation/InstrProfiling/mcdc.ll +++ b/llvm/test/Instrumentation/InstrProfiling/mcdc.ll @@ -1,9 +1,7 @@ ; Check that MC/DC intrinsics are properly lowered ; RUN: opt < %s -passes=instrprof -S | FileCheck %s --check-prefixes=CHECK,BASIC ; RUN: opt < %s -passes=instrprof -S -instrprof-atomic-counter-update-all | FileCheck %s --check-prefixes=CHECK,ATOMIC -; RUN: opt < %s -passes=instrprof -runtime-counter-relocation -S 2>&1 | FileCheck %s --check-prefix RELOC - -; RELOC: Runtime counter relocation is presently not supported for MC/DC bitmaps +; RUN: opt < %s -passes=instrprof -S -runtime-counter-relocation | FileCheck %s --check-prefixes=CHECK,RELOC target triple = "x86_64-unknown-linux-gnu" @@ -14,10 +12,15 @@ target triple = "x86_64-unknown-linux-gnu" define dso_local void @test(i32 noundef %A) { entry: + ; RELOC: %profbm_bias = load i64, ptr @__llvm_profile_bitmap_bias, align 8, !invariant.load !0 + ; RELOC: %profc_bias = load i64, ptr @__llvm_profile_counter_bias, align 8 %A.addr = alloca i32, align 4 %mcdc.addr = alloca i32, align 4 call void @llvm.instrprof.cover(ptr @__profn_test, i64 99278, i32 5, i32 0) ; BASIC: store i8 0, ptr @__profc_test, align 1 + ; RELOC: %[[PROFC_INTADDR:.+]] = add i64 ptrtoint (ptr @__profc_test to i64), %profc_bias + ; RELOC: %[[PROFC_ADDR:.+]] = inttoptr i64 %[[PROFC_INTADDR]] to ptr + ; RELOC: store i8 0, ptr %[[PROFC_ADDR]], align 1 call void @llvm.instrprof.mcdc.parameters(ptr @__profn_test, i64 99278, i32 1) store i32 0, ptr %mcdc.addr, align 4 @@ -25,6 +28,7 @@ entry: %tobool = icmp ne i32 %0, 0 call void @llvm.instrprof.mcdc.tvbitmap.update(ptr @__profn_test, i64 99278, i32 0, ptr %mcdc.addr) + ; RELOC: [[PROFBM_ADDR:%.+]] = getelementptr i8, ptr @__profbm_test, i64 %profbm_bias ; CHECK: %[[TEMP0:mcdc.*]] = load i32, ptr %mcdc.addr, align 4 ; CHECK-NEXT: %[[TEMP:[0-9]+]] = add i32 %[[TEMP0]], 0 ; CHECK-NEXT: %[[LAB4:[0-9]+]] = lshr i32 %[[TEMP]], 3 @@ -39,7 +43,9 @@ entry: ; CHECK: define private void @[[RMW_OR]](ptr %[[ARGPTR:.+]], i8 %[[ARGVAL:.+]]) ; CHECK: %[[BITS:.+]] = load i8, ptr %[[ARGPTR]], align 1 ; BASIC-NEXT: %[[LAB11:[0-9]+]] = or i8 %[[BITS]], %[[ARGVAL]] +; RELOC-NEXT: %[[LAB11:[0-9]+]] = or i8 %[[BITS]], %[[ARGVAL]] ; BASIC-NEXT: store i8 %[[LAB11]], ptr %[[ARGPTR]], align 1 +; RELOC-NEXT: store i8 %[[LAB11]], ptr %[[ARGPTR]], align 1 ; ATOMIC-NEXT: %[[MASKED:.+]] = and i8 %[[BITS]], %[[ARGVAL]] ; ATOMIC-NEXT: %[[SHOULDWRITE:.+]] = icmp ne i8 %[[MASKED]], %[[ARGVAL]] ; ATOMIC-NEXT: br i1 %[[SHOULDWRITE]], label %[[WRITE:.+]], label %[[SKIP:.+]], !prof ![[MDPROF:[0-9]+]]