aboutsummaryrefslogtreecommitdiffstats
path: root/lib/CodeGen
diff options
context:
space:
mode:
authorDimitry Andric <dim@FreeBSD.org>2015-05-27 18:47:56 +0000
committerDimitry Andric <dim@FreeBSD.org>2015-05-27 18:47:56 +0000
commit5e20cdd81c44a443562a09007668ffdf76c455af (patch)
treedbbd4047878da71c1a706e26ce05b4e7791b14cc /lib/CodeGen
parentd5f23b0b7528b5c3caed1ba14f897cc4aaa9e3c3 (diff)
downloadsrc-5e20cdd81c44a443562a09007668ffdf76c455af.tar.gz
src-5e20cdd81c44a443562a09007668ffdf76c455af.zip
Vendor import of clang trunk r238337:vendor/clang/clang-trunk-r238337
Notes
Notes: svn path=/vendor/clang/dist/; revision=283627 svn path=/vendor/clang/clang-trunk-r238337/; revision=283628; tag=vendor/clang/clang-trunk-r238337
Diffstat (limited to 'lib/CodeGen')
-rw-r--r--lib/CodeGen/ABIInfo.h2
-rw-r--r--lib/CodeGen/BackendUtil.cpp165
-rw-r--r--lib/CodeGen/CGAtomic.cpp962
-rw-r--r--lib/CodeGen/CGBlocks.cpp132
-rw-r--r--lib/CodeGen/CGBuilder.h2
-rw-r--r--lib/CodeGen/CGBuiltin.cpp719
-rw-r--r--lib/CodeGen/CGCUDANV.cpp218
-rw-r--r--lib/CodeGen/CGCUDARuntime.h17
-rw-r--r--lib/CodeGen/CGCXX.cpp7
-rw-r--r--lib/CodeGen/CGCXXABI.cpp7
-rw-r--r--lib/CodeGen/CGCXXABI.h11
-rw-r--r--lib/CodeGen/CGCall.cpp193
-rw-r--r--lib/CodeGen/CGClass.cpp414
-rw-r--r--lib/CodeGen/CGCleanup.cpp46
-rw-r--r--lib/CodeGen/CGCleanup.h13
-rw-r--r--lib/CodeGen/CGDebugInfo.cpp1186
-rw-r--r--lib/CodeGen/CGDebugInfo.h421
-rw-r--r--lib/CodeGen/CGDecl.cpp89
-rw-r--r--lib/CodeGen/CGDeclCXX.cpp68
-rw-r--r--lib/CodeGen/CGException.cpp1026
-rw-r--r--lib/CodeGen/CGExpr.cpp123
-rw-r--r--lib/CodeGen/CGExprAgg.cpp78
-rw-r--r--lib/CodeGen/CGExprCXX.cpp50
-rw-r--r--lib/CodeGen/CGExprComplex.cpp29
-rw-r--r--lib/CodeGen/CGExprConstant.cpp20
-rw-r--r--lib/CodeGen/CGExprScalar.cpp321
-rw-r--r--lib/CodeGen/CGLoopInfo.cpp5
-rw-r--r--lib/CodeGen/CGLoopInfo.h4
-rw-r--r--lib/CodeGen/CGObjC.cpp64
-rw-r--r--lib/CodeGen/CGObjCGNU.cpp171
-rw-r--r--lib/CodeGen/CGObjCMac.cpp68
-rw-r--r--lib/CodeGen/CGObjCRuntime.cpp2
-rw-r--r--lib/CodeGen/CGOpenMPRuntime.cpp2202
-rw-r--r--lib/CodeGen/CGOpenMPRuntime.h392
-rw-r--r--lib/CodeGen/CGRecordLayout.h4
-rw-r--r--lib/CodeGen/CGRecordLayoutBuilder.cpp48
-rw-r--r--lib/CodeGen/CGStmt.cpp125
-rw-r--r--lib/CodeGen/CGStmtOpenMP.cpp2112
-rw-r--r--lib/CodeGen/CGVTT.cpp16
-rw-r--r--lib/CodeGen/CGVTables.cpp92
-rw-r--r--lib/CodeGen/CGValue.h2
-rw-r--r--lib/CodeGen/CMakeLists.txt2
-rw-r--r--lib/CodeGen/CodeGenAction.cpp74
-rw-r--r--lib/CodeGen/CodeGenFunction.cpp153
-rw-r--r--lib/CodeGen/CodeGenFunction.h407
-rw-r--r--lib/CodeGen/CodeGenModule.cpp274
-rw-r--r--lib/CodeGen/CodeGenModule.h25
-rw-r--r--lib/CodeGen/CodeGenPGO.cpp910
-rw-r--r--lib/CodeGen/CodeGenPGO.h135
-rw-r--r--lib/CodeGen/CodeGenTypes.cpp15
-rw-r--r--lib/CodeGen/CodeGenTypes.h29
-rw-r--r--lib/CodeGen/CoverageMappingGen.cpp1257
-rw-r--r--lib/CodeGen/EHScopeStack.h148
-rw-r--r--lib/CodeGen/ItaniumCXXABI.cpp461
-rw-r--r--lib/CodeGen/MicrosoftCXXABI.cpp1268
-rw-r--r--lib/CodeGen/ModuleBuilder.cpp10
-rw-r--r--lib/CodeGen/SanitizerMetadata.h4
-rw-r--r--lib/CodeGen/TargetInfo.cpp838
58 files changed, 11789 insertions, 5847 deletions
diff --git a/lib/CodeGen/ABIInfo.h b/lib/CodeGen/ABIInfo.h
index 7e7f7fa20679..cc8652e169d8 100644
--- a/lib/CodeGen/ABIInfo.h
+++ b/lib/CodeGen/ABIInfo.h
@@ -87,6 +87,8 @@ namespace clang {
virtual bool isHomogeneousAggregateSmallEnough(const Type *Base,
uint64_t Members) const;
+ virtual bool shouldSignExtUnsignedType(QualType Ty) const;
+
bool isHomogeneousAggregate(QualType Ty, const Type *&Base,
uint64_t &Members) const;
diff --git a/lib/CodeGen/BackendUtil.cpp b/lib/CodeGen/BackendUtil.cpp
index 25ecec586244..7f0c7bafc046 100644
--- a/lib/CodeGen/BackendUtil.cpp
+++ b/lib/CodeGen/BackendUtil.cpp
@@ -15,22 +15,22 @@
#include "clang/Frontend/FrontendDiagnostic.h"
#include "clang/Frontend/Utils.h"
#include "llvm/ADT/StringSwitch.h"
+#include "llvm/Analysis/TargetLibraryInfo.h"
+#include "llvm/Analysis/TargetTransformInfo.h"
#include "llvm/Bitcode/BitcodeWriterPass.h"
#include "llvm/CodeGen/RegAllocRegistry.h"
#include "llvm/CodeGen/SchedulerRegistry.h"
#include "llvm/IR/DataLayout.h"
#include "llvm/IR/IRPrintingPasses.h"
+#include "llvm/IR/LegacyPassManager.h"
#include "llvm/IR/Module.h"
#include "llvm/IR/Verifier.h"
#include "llvm/MC/SubtargetFeature.h"
-#include "llvm/PassManager.h"
#include "llvm/Support/CommandLine.h"
-#include "llvm/Support/FormattedStream.h"
#include "llvm/Support/PrettyStackTrace.h"
#include "llvm/Support/TargetRegistry.h"
#include "llvm/Support/Timer.h"
#include "llvm/Support/raw_ostream.h"
-#include "llvm/Target/TargetLibraryInfo.h"
#include "llvm/Target/TargetMachine.h"
#include "llvm/Target/TargetOptions.h"
#include "llvm/Target/TargetSubtargetInfo.h"
@@ -55,44 +55,48 @@ class EmitAssemblyHelper {
Timer CodeGenerationTime;
- mutable PassManager *CodeGenPasses;
- mutable PassManager *PerModulePasses;
- mutable FunctionPassManager *PerFunctionPasses;
+ mutable legacy::PassManager *CodeGenPasses;
+ mutable legacy::PassManager *PerModulePasses;
+ mutable legacy::FunctionPassManager *PerFunctionPasses;
private:
- PassManager *getCodeGenPasses() const {
+ TargetIRAnalysis getTargetIRAnalysis() const {
+ if (TM)
+ return TM->getTargetIRAnalysis();
+
+ return TargetIRAnalysis();
+ }
+
+ legacy::PassManager *getCodeGenPasses() const {
if (!CodeGenPasses) {
- CodeGenPasses = new PassManager();
- CodeGenPasses->add(new DataLayoutPass());
- if (TM)
- TM->addAnalysisPasses(*CodeGenPasses);
+ CodeGenPasses = new legacy::PassManager();
+ CodeGenPasses->add(
+ createTargetTransformInfoWrapperPass(getTargetIRAnalysis()));
}
return CodeGenPasses;
}
- PassManager *getPerModulePasses() const {
+ legacy::PassManager *getPerModulePasses() const {
if (!PerModulePasses) {
- PerModulePasses = new PassManager();
- PerModulePasses->add(new DataLayoutPass());
- if (TM)
- TM->addAnalysisPasses(*PerModulePasses);
+ PerModulePasses = new legacy::PassManager();
+ PerModulePasses->add(
+ createTargetTransformInfoWrapperPass(getTargetIRAnalysis()));
}
return PerModulePasses;
}
- FunctionPassManager *getPerFunctionPasses() const {
+ legacy::FunctionPassManager *getPerFunctionPasses() const {
if (!PerFunctionPasses) {
- PerFunctionPasses = new FunctionPassManager(TheModule);
- PerFunctionPasses->add(new DataLayoutPass());
- if (TM)
- TM->addAnalysisPasses(*PerFunctionPasses);
+ PerFunctionPasses = new legacy::FunctionPassManager(TheModule);
+ PerFunctionPasses->add(
+ createTargetTransformInfoWrapperPass(getTargetIRAnalysis()));
}
return PerFunctionPasses;
}
void CreatePasses();
- /// CreateTargetMachine - Generates the TargetMachine.
+ /// Generates the TargetMachine.
/// Returns Null if it is unable to create the target machine.
/// Some of our clang tests specify triples which are not built
/// into clang. This is okay because these tests check the generated
@@ -102,10 +106,10 @@ private:
/// the requested target.
TargetMachine *CreateTargetMachine(bool MustCreateTM);
- /// AddEmitPasses - Add passes necessary to emit assembly or LLVM IR.
+ /// Add passes necessary to emit assembly or LLVM IR.
///
/// \return True on success.
- bool AddEmitPasses(BackendAction Action, formatted_raw_ostream &OS);
+ bool AddEmitPasses(BackendAction Action, raw_pwrite_stream &OS);
public:
EmitAssemblyHelper(DiagnosticsEngine &_Diags,
@@ -128,7 +132,7 @@ public:
std::unique_ptr<TargetMachine> TM;
- void EmitAssembly(BackendAction Action, raw_ostream *OS);
+ void EmitAssembly(BackendAction Action, raw_pwrite_stream *OS);
};
// We need this wrapper to access LangOpts and CGOpts from extension functions
@@ -163,7 +167,7 @@ static void addObjCARCOptPass(const PassManagerBuilder &Builder, PassManagerBase
}
static void addSampleProfileLoaderPass(const PassManagerBuilder &Builder,
- PassManagerBase &PM) {
+ legacy::PassManagerBase &PM) {
const PassManagerBuilderWrapper &BuilderWrapper =
static_cast<const PassManagerBuilderWrapper &>(Builder);
const CodeGenOptions &CGOpts = BuilderWrapper.getCGOpts();
@@ -171,31 +175,38 @@ static void addSampleProfileLoaderPass(const PassManagerBuilder &Builder,
}
static void addAddDiscriminatorsPass(const PassManagerBuilder &Builder,
- PassManagerBase &PM) {
+ legacy::PassManagerBase &PM) {
PM.add(createAddDiscriminatorsPass());
}
static void addBoundsCheckingPass(const PassManagerBuilder &Builder,
- PassManagerBase &PM) {
+ legacy::PassManagerBase &PM) {
PM.add(createBoundsCheckingPass());
}
static void addSanitizerCoveragePass(const PassManagerBuilder &Builder,
- PassManagerBase &PM) {
+ legacy::PassManagerBase &PM) {
const PassManagerBuilderWrapper &BuilderWrapper =
static_cast<const PassManagerBuilderWrapper&>(Builder);
const CodeGenOptions &CGOpts = BuilderWrapper.getCGOpts();
- PM.add(createSanitizerCoverageModulePass(CGOpts.SanitizeCoverage));
+ SanitizerCoverageOptions Opts;
+ Opts.CoverageType =
+ static_cast<SanitizerCoverageOptions::Type>(CGOpts.SanitizeCoverageType);
+ Opts.IndirectCalls = CGOpts.SanitizeCoverageIndirectCalls;
+ Opts.TraceBB = CGOpts.SanitizeCoverageTraceBB;
+ Opts.TraceCmp = CGOpts.SanitizeCoverageTraceCmp;
+ Opts.Use8bitCounters = CGOpts.SanitizeCoverage8bitCounters;
+ PM.add(createSanitizerCoverageModulePass(Opts));
}
static void addAddressSanitizerPasses(const PassManagerBuilder &Builder,
- PassManagerBase &PM) {
+ legacy::PassManagerBase &PM) {
PM.add(createAddressSanitizerFunctionPass());
PM.add(createAddressSanitizerModulePass());
}
static void addMemorySanitizerPass(const PassManagerBuilder &Builder,
- PassManagerBase &PM) {
+ legacy::PassManagerBase &PM) {
const PassManagerBuilderWrapper &BuilderWrapper =
static_cast<const PassManagerBuilderWrapper&>(Builder);
const CodeGenOptions &CGOpts = BuilderWrapper.getCGOpts();
@@ -215,28 +226,36 @@ static void addMemorySanitizerPass(const PassManagerBuilder &Builder,
}
static void addThreadSanitizerPass(const PassManagerBuilder &Builder,
- PassManagerBase &PM) {
+ legacy::PassManagerBase &PM) {
PM.add(createThreadSanitizerPass());
}
static void addDataFlowSanitizerPass(const PassManagerBuilder &Builder,
- PassManagerBase &PM) {
+ legacy::PassManagerBase &PM) {
const PassManagerBuilderWrapper &BuilderWrapper =
static_cast<const PassManagerBuilderWrapper&>(Builder);
const LangOptions &LangOpts = BuilderWrapper.getLangOpts();
- PM.add(createDataFlowSanitizerPass(LangOpts.SanitizerBlacklistFile));
+ PM.add(createDataFlowSanitizerPass(LangOpts.SanitizerBlacklistFiles));
}
-static TargetLibraryInfo *createTLI(llvm::Triple &TargetTriple,
- const CodeGenOptions &CodeGenOpts) {
- TargetLibraryInfo *TLI = new TargetLibraryInfo(TargetTriple);
+static TargetLibraryInfoImpl *createTLII(llvm::Triple &TargetTriple,
+ const CodeGenOptions &CodeGenOpts) {
+ TargetLibraryInfoImpl *TLII = new TargetLibraryInfoImpl(TargetTriple);
if (!CodeGenOpts.SimplifyLibCalls)
- TLI->disableAllFunctions();
- return TLI;
+ TLII->disableAllFunctions();
+
+ switch (CodeGenOpts.getVecLib()) {
+ case CodeGenOptions::Accelerate:
+ TLII->addVectorizableFunctionsFromVecLib(TargetLibraryInfoImpl::Accelerate);
+ break;
+ default:
+ break;
+ }
+ return TLII;
}
static void addSymbolRewriterPass(const CodeGenOptions &Opts,
- PassManager *MPM) {
+ legacy::PassManager *MPM) {
llvm::SymbolRewriter::RewriteDescriptorList DL;
llvm::SymbolRewriter::RewriteMapParser MapParser;
@@ -294,7 +313,9 @@ void EmitAssemblyHelper::CreatePasses() {
addBoundsCheckingPass);
}
- if (CodeGenOpts.SanitizeCoverage) {
+ if (CodeGenOpts.SanitizeCoverageType ||
+ CodeGenOpts.SanitizeCoverageIndirectCalls ||
+ CodeGenOpts.SanitizeCoverageTraceCmp) {
PMBuilder.addExtension(PassManagerBuilder::EP_OptimizerLast,
addSanitizerCoveragePass);
PMBuilder.addExtension(PassManagerBuilder::EP_EnabledOnOptLevel0,
@@ -331,7 +352,7 @@ void EmitAssemblyHelper::CreatePasses() {
// Figure out TargetLibraryInfo.
Triple TargetTriple(TheModule->getTargetTriple());
- PMBuilder.LibraryInfo = createTLI(TargetTriple, CodeGenOpts);
+ PMBuilder.LibraryInfo = createTLII(TargetTriple, CodeGenOpts);
switch (Inlining) {
case CodeGenOptions::NoInlining: break;
@@ -351,17 +372,15 @@ void EmitAssemblyHelper::CreatePasses() {
}
// Set up the per-function pass manager.
- FunctionPassManager *FPM = getPerFunctionPasses();
+ legacy::FunctionPassManager *FPM = getPerFunctionPasses();
if (CodeGenOpts.VerifyModule)
FPM->add(createVerifierPass());
PMBuilder.populateFunctionPassManager(*FPM);
// Set up the per-module pass manager.
- PassManager *MPM = getPerModulePasses();
+ legacy::PassManager *MPM = getPerModulePasses();
if (!CodeGenOpts.RewriteMapFiles.empty())
addSymbolRewriterPass(CodeGenOpts, MPM);
- if (CodeGenOpts.VerifyModule)
- MPM->add(createDebugInfoVerifierPass());
if (!CodeGenOpts.DisableGCov &&
(CodeGenOpts.EmitGcovArcs || CodeGenOpts.EmitGcovNotes)) {
@@ -375,6 +394,7 @@ void EmitAssemblyHelper::CreatePasses() {
Options.NoRedZone = CodeGenOpts.DisableRedZone;
Options.FunctionNamesInData =
!CodeGenOpts.CoverageNoFunctionNamesInData;
+ Options.ExitBlockBeforeBody = CodeGenOpts.CoverageExitBlockBeforeBody;
MPM->add(createGCOVProfilerPass(Options));
if (CodeGenOpts.getDebugInfo() == CodeGenOptions::NoDebugInfo)
MPM->add(createStripSymbolsPass(true));
@@ -383,6 +403,7 @@ void EmitAssemblyHelper::CreatePasses() {
if (CodeGenOpts.ProfileInstrGenerate) {
InstrProfOptions Options;
Options.NoRedZone = CodeGenOpts.DisableRedZone;
+ Options.InstrProfileOutput = CodeGenOpts.InstrProfileOutput;
MPM->add(createInstrProfilingPass(Options));
}
@@ -425,14 +446,12 @@ TargetMachine *EmitAssemblyHelper::CreateTargetMachine(bool MustCreateTM) {
BackendArgs.push_back("-time-passes");
for (unsigned i = 0, e = CodeGenOpts.BackendOptions.size(); i != e; ++i)
BackendArgs.push_back(CodeGenOpts.BackendOptions[i].c_str());
- if (CodeGenOpts.NoGlobalMerge)
- BackendArgs.push_back("-enable-global-merge=false");
BackendArgs.push_back(nullptr);
llvm::cl::ParseCommandLineOptions(BackendArgs.size() - 1,
BackendArgs.data());
std::string FeaturesStr;
- if (TargetOpts.Features.size()) {
+ if (!TargetOpts.Features.empty()) {
SubtargetFeatures Features;
for (std::vector<std::string>::const_iterator
it = TargetOpts.Features.begin(),
@@ -472,15 +491,6 @@ TargetMachine *EmitAssemblyHelper::CreateTargetMachine(bool MustCreateTM) {
if (CodeGenOpts.CompressDebugSections)
Options.CompressDebugSections = true;
- // Set frame pointer elimination mode.
- if (!CodeGenOpts.DisableFPElim) {
- Options.NoFramePointerElim = false;
- } else if (CodeGenOpts.OmitLeafFramePointer) {
- Options.NoFramePointerElim = false;
- } else {
- Options.NoFramePointerElim = true;
- }
-
if (CodeGenOpts.UseInitArray)
Options.UseInitArray = true;
@@ -512,13 +522,13 @@ TargetMachine *EmitAssemblyHelper::CreateTargetMachine(bool MustCreateTM) {
Options.NoNaNsFPMath = CodeGenOpts.NoNaNsFPMath;
Options.NoZerosInBSS = CodeGenOpts.NoZeroInitializedInBSS;
Options.UnsafeFPMath = CodeGenOpts.UnsafeFPMath;
- Options.UseSoftFloat = CodeGenOpts.SoftFloat;
Options.StackAlignmentOverride = CodeGenOpts.StackAlignment;
Options.DisableTailCalls = CodeGenOpts.DisableTailCalls;
Options.TrapFuncName = CodeGenOpts.TrapFuncName;
Options.PositionIndependentExecutable = LangOpts.PIELevel != 0;
Options.FunctionSections = CodeGenOpts.FunctionSections;
Options.DataSections = CodeGenOpts.DataSections;
+ Options.UniqueSectionNames = CodeGenOpts.UniqueSectionNames;
Options.MCOptions.MCRelaxAll = CodeGenOpts.RelaxAll;
Options.MCOptions.MCSaveTempLabels = CodeGenOpts.SaveTempLabels;
@@ -536,17 +546,16 @@ TargetMachine *EmitAssemblyHelper::CreateTargetMachine(bool MustCreateTM) {
}
bool EmitAssemblyHelper::AddEmitPasses(BackendAction Action,
- formatted_raw_ostream &OS) {
+ raw_pwrite_stream &OS) {
// Create the code generator passes.
- PassManager *PM = getCodeGenPasses();
+ legacy::PassManager *PM = getCodeGenPasses();
// Add LibraryInfo.
llvm::Triple TargetTriple(TheModule->getTargetTriple());
- PM->add(createTLI(TargetTriple, CodeGenOpts));
-
- // Add Target specific analysis passes.
- TM->addAnalysisPasses(*PM);
+ std::unique_ptr<TargetLibraryInfoImpl> TLII(
+ createTLII(TargetTriple, CodeGenOpts));
+ PM->add(new TargetLibraryInfoWrapperPass(*TLII));
// Normal mode, emit a .s or .o file by running the code generator. Note,
// this also adds codegenerator level optimization passes.
@@ -561,8 +570,7 @@ bool EmitAssemblyHelper::AddEmitPasses(BackendAction Action,
// Add ObjC ARC final-cleanup optimizations. This is done as part of the
// "codegen" passes so that it isn't run multiple times when there is
// inlining happening.
- if (LangOpts.ObjCAutoRefCount &&
- CodeGenOpts.OptimizationLevel > 0)
+ if (CodeGenOpts.OptimizationLevel > 0)
PM->add(createObjCARCContractPass());
if (TM->addPassesToEmitFile(*PM, OS, CGFT,
@@ -574,9 +582,9 @@ bool EmitAssemblyHelper::AddEmitPasses(BackendAction Action,
return true;
}
-void EmitAssemblyHelper::EmitAssembly(BackendAction Action, raw_ostream *OS) {
+void EmitAssemblyHelper::EmitAssembly(BackendAction Action,
+ raw_pwrite_stream *OS) {
TimeRegion Region(llvm::TimePassesIsEnabled ? &CodeGenerationTime : nullptr);
- llvm::formatted_raw_ostream FormattedOS;
bool UsesCodeGen = (Action != Backend_EmitNothing &&
Action != Backend_EmitBC &&
@@ -592,17 +600,17 @@ void EmitAssemblyHelper::EmitAssembly(BackendAction Action, raw_ostream *OS) {
break;
case Backend_EmitBC:
- getPerModulePasses()->add(createBitcodeWriterPass(*OS));
+ getPerModulePasses()->add(
+ createBitcodeWriterPass(*OS, CodeGenOpts.EmitLLVMUseLists));
break;
case Backend_EmitLL:
- FormattedOS.setStream(*OS, formatted_raw_ostream::PRESERVE_STREAM);
- getPerModulePasses()->add(createPrintModulePass(FormattedOS));
+ getPerModulePasses()->add(
+ createPrintModulePass(*OS, "", CodeGenOpts.EmitLLVMUseLists));
break;
default:
- FormattedOS.setStream(*OS, formatted_raw_ostream::PRESERVE_STREAM);
- if (!AddEmitPasses(Action, FormattedOS))
+ if (!AddEmitPasses(Action, *OS))
return;
}
@@ -639,7 +647,7 @@ void clang::EmitBackendOutput(DiagnosticsEngine &Diags,
const clang::TargetOptions &TOpts,
const LangOptions &LOpts, StringRef TDesc,
Module *M, BackendAction Action,
- raw_ostream *OS) {
+ raw_pwrite_stream *OS) {
EmitAssemblyHelper AsmHelper(Diags, CGOpts, TOpts, LOpts, M);
AsmHelper.EmitAssembly(Action, OS);
@@ -647,9 +655,8 @@ void clang::EmitBackendOutput(DiagnosticsEngine &Diags,
// If an optional clang TargetInfo description string was passed in, use it to
// verify the LLVM TargetMachine's DataLayout.
if (AsmHelper.TM && !TDesc.empty()) {
- std::string DLDesc = AsmHelper.TM->getSubtargetImpl()
- ->getDataLayout()
- ->getStringRepresentation();
+ std::string DLDesc =
+ AsmHelper.TM->getDataLayout()->getStringRepresentation();
if (DLDesc != TDesc) {
unsigned DiagID = Diags.getCustomDiagID(
DiagnosticsEngine::Error, "backend data layout '%0' does not match "
diff --git a/lib/CodeGen/CGAtomic.cpp b/lib/CodeGen/CGAtomic.cpp
index daac174c8e0c..da82249fe114 100644
--- a/lib/CodeGen/CGAtomic.cpp
+++ b/lib/CodeGen/CGAtomic.cpp
@@ -13,6 +13,7 @@
#include "CodeGenFunction.h"
#include "CGCall.h"
+#include "CGRecordLayout.h"
#include "CodeGenModule.h"
#include "clang/AST/ASTContext.h"
#include "clang/CodeGen/CGFunctionInfo.h"
@@ -36,34 +37,94 @@ namespace {
CharUnits LValueAlign;
TypeEvaluationKind EvaluationKind;
bool UseLibcall;
+ LValue LVal;
+ CGBitFieldInfo BFI;
public:
- AtomicInfo(CodeGenFunction &CGF, LValue &lvalue) : CGF(CGF) {
- assert(lvalue.isSimple());
-
- AtomicTy = lvalue.getType();
- ValueTy = AtomicTy->castAs<AtomicType>()->getValueType();
- EvaluationKind = CGF.getEvaluationKind(ValueTy);
-
+ AtomicInfo(CodeGenFunction &CGF, LValue &lvalue)
+ : CGF(CGF), AtomicSizeInBits(0), ValueSizeInBits(0),
+ EvaluationKind(TEK_Scalar), UseLibcall(true) {
+ assert(!lvalue.isGlobalReg());
ASTContext &C = CGF.getContext();
-
- uint64_t ValueAlignInBits;
- uint64_t AtomicAlignInBits;
- TypeInfo ValueTI = C.getTypeInfo(ValueTy);
- ValueSizeInBits = ValueTI.Width;
- ValueAlignInBits = ValueTI.Align;
-
- TypeInfo AtomicTI = C.getTypeInfo(AtomicTy);
- AtomicSizeInBits = AtomicTI.Width;
- AtomicAlignInBits = AtomicTI.Align;
-
- assert(ValueSizeInBits <= AtomicSizeInBits);
- assert(ValueAlignInBits <= AtomicAlignInBits);
-
- AtomicAlign = C.toCharUnitsFromBits(AtomicAlignInBits);
- ValueAlign = C.toCharUnitsFromBits(ValueAlignInBits);
- if (lvalue.getAlignment().isZero())
- lvalue.setAlignment(AtomicAlign);
-
+ if (lvalue.isSimple()) {
+ AtomicTy = lvalue.getType();
+ if (auto *ATy = AtomicTy->getAs<AtomicType>())
+ ValueTy = ATy->getValueType();
+ else
+ ValueTy = AtomicTy;
+ EvaluationKind = CGF.getEvaluationKind(ValueTy);
+
+ uint64_t ValueAlignInBits;
+ uint64_t AtomicAlignInBits;
+ TypeInfo ValueTI = C.getTypeInfo(ValueTy);
+ ValueSizeInBits = ValueTI.Width;
+ ValueAlignInBits = ValueTI.Align;
+
+ TypeInfo AtomicTI = C.getTypeInfo(AtomicTy);
+ AtomicSizeInBits = AtomicTI.Width;
+ AtomicAlignInBits = AtomicTI.Align;
+
+ assert(ValueSizeInBits <= AtomicSizeInBits);
+ assert(ValueAlignInBits <= AtomicAlignInBits);
+
+ AtomicAlign = C.toCharUnitsFromBits(AtomicAlignInBits);
+ ValueAlign = C.toCharUnitsFromBits(ValueAlignInBits);
+ if (lvalue.getAlignment().isZero())
+ lvalue.setAlignment(AtomicAlign);
+
+ LVal = lvalue;
+ } else if (lvalue.isBitField()) {
+ ValueTy = lvalue.getType();
+ ValueSizeInBits = C.getTypeSize(ValueTy);
+ auto &OrigBFI = lvalue.getBitFieldInfo();
+ auto Offset = OrigBFI.Offset % C.toBits(lvalue.getAlignment());
+ AtomicSizeInBits = C.toBits(
+ C.toCharUnitsFromBits(Offset + OrigBFI.Size + C.getCharWidth() - 1)
+ .RoundUpToAlignment(lvalue.getAlignment()));
+ auto VoidPtrAddr = CGF.EmitCastToVoidPtr(lvalue.getBitFieldAddr());
+ auto OffsetInChars =
+ (C.toCharUnitsFromBits(OrigBFI.Offset) / lvalue.getAlignment()) *
+ lvalue.getAlignment();
+ VoidPtrAddr = CGF.Builder.CreateConstGEP1_64(
+ VoidPtrAddr, OffsetInChars.getQuantity());
+ auto Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
+ VoidPtrAddr,
+ CGF.Builder.getIntNTy(AtomicSizeInBits)->getPointerTo(),
+ "atomic_bitfield_base");
+ BFI = OrigBFI;
+ BFI.Offset = Offset;
+ BFI.StorageSize = AtomicSizeInBits;
+ LVal = LValue::MakeBitfield(Addr, BFI, lvalue.getType(),
+ lvalue.getAlignment());
+ LVal.setTBAAInfo(lvalue.getTBAAInfo());
+ AtomicTy = C.getIntTypeForBitwidth(AtomicSizeInBits, OrigBFI.IsSigned);
+ if (AtomicTy.isNull()) {
+ llvm::APInt Size(
+ /*numBits=*/32,
+ C.toCharUnitsFromBits(AtomicSizeInBits).getQuantity());
+ AtomicTy = C.getConstantArrayType(C.CharTy, Size, ArrayType::Normal,
+ /*IndexTypeQuals=*/0);
+ }
+ AtomicAlign = ValueAlign = lvalue.getAlignment();
+ } else if (lvalue.isVectorElt()) {
+ ValueTy = lvalue.getType()->getAs<VectorType>()->getElementType();
+ ValueSizeInBits = C.getTypeSize(ValueTy);
+ AtomicTy = lvalue.getType();
+ AtomicSizeInBits = C.getTypeSize(AtomicTy);
+ AtomicAlign = ValueAlign = lvalue.getAlignment();
+ LVal = lvalue;
+ } else {
+ assert(lvalue.isExtVectorElt());
+ ValueTy = lvalue.getType();
+ ValueSizeInBits = C.getTypeSize(ValueTy);
+ AtomicTy = ValueTy = CGF.getContext().getExtVectorType(
+ lvalue.getType(), lvalue.getExtVectorAddr()
+ ->getType()
+ ->getPointerElementType()
+ ->getVectorNumElements());
+ AtomicSizeInBits = C.getTypeSize(AtomicTy);
+ AtomicAlign = ValueAlign = lvalue.getAlignment();
+ LVal = lvalue;
+ }
UseLibcall = !C.getTargetInfo().hasBuiltinAtomic(
AtomicSizeInBits, C.toBits(lvalue.getAlignment()));
}
@@ -76,6 +137,17 @@ namespace {
uint64_t getValueSizeInBits() const { return ValueSizeInBits; }
TypeEvaluationKind getEvaluationKind() const { return EvaluationKind; }
bool shouldUseLibcall() const { return UseLibcall; }
+ const LValue &getAtomicLValue() const { return LVal; }
+ llvm::Value *getAtomicAddress() const {
+ if (LVal.isSimple())
+ return LVal.getAddress();
+ else if (LVal.isBitField())
+ return LVal.getBitFieldAddr();
+ else if (LVal.isVectorElt())
+ return LVal.getVectorAddr();
+ assert(LVal.isExtVectorElt());
+ return LVal.getExtVectorAddr();
+ }
/// Is the atomic size larger than the underlying value type?
///
@@ -87,7 +159,7 @@ namespace {
return (ValueSizeInBits != AtomicSizeInBits);
}
- bool emitMemSetZeroIfNecessary(LValue dest) const;
+ bool emitMemSetZeroIfNecessary() const;
llvm::Value *getAtomicSizeValue() const {
CharUnits size = CGF.getContext().toCharUnitsFromBits(AtomicSizeInBits);
@@ -99,37 +171,141 @@ namespace {
llvm::Value *emitCastToAtomicIntPointer(llvm::Value *addr) const;
/// Turn an atomic-layout object into an r-value.
- RValue convertTempToRValue(llvm::Value *addr,
- AggValueSlot resultSlot,
- SourceLocation loc) const;
+ RValue convertTempToRValue(llvm::Value *addr, AggValueSlot resultSlot,
+ SourceLocation loc, bool AsValue) const;
/// \brief Converts a rvalue to integer value.
llvm::Value *convertRValueToInt(RValue RVal) const;
- RValue convertIntToValue(llvm::Value *IntVal, AggValueSlot ResultSlot,
- SourceLocation Loc) const;
+ RValue ConvertIntToValueOrAtomic(llvm::Value *IntVal,
+ AggValueSlot ResultSlot,
+ SourceLocation Loc, bool AsValue) const;
/// Copy an atomic r-value into atomic-layout memory.
- void emitCopyIntoMemory(RValue rvalue, LValue lvalue) const;
+ void emitCopyIntoMemory(RValue rvalue) const;
/// Project an l-value down to the value field.
- LValue projectValue(LValue lvalue) const {
- llvm::Value *addr = lvalue.getAddress();
+ LValue projectValue() const {
+ assert(LVal.isSimple());
+ llvm::Value *addr = getAtomicAddress();
if (hasPadding())
- addr = CGF.Builder.CreateStructGEP(addr, 0);
+ addr = CGF.Builder.CreateStructGEP(nullptr, addr, 0);
- return LValue::MakeAddr(addr, getValueType(), lvalue.getAlignment(),
- CGF.getContext(), lvalue.getTBAAInfo());
+ return LValue::MakeAddr(addr, getValueType(), LVal.getAlignment(),
+ CGF.getContext(), LVal.getTBAAInfo());
}
+ /// \brief Emits atomic load.
+ /// \returns Loaded value.
+ RValue EmitAtomicLoad(AggValueSlot ResultSlot, SourceLocation Loc,
+ bool AsValue, llvm::AtomicOrdering AO,
+ bool IsVolatile);
+
+ /// \brief Emits atomic compare-and-exchange sequence.
+ /// \param Expected Expected value.
+ /// \param Desired Desired value.
+ /// \param Success Atomic ordering for success operation.
+ /// \param Failure Atomic ordering for failed operation.
+ /// \param IsWeak true if atomic operation is weak, false otherwise.
+ /// \returns Pair of values: previous value from storage (value type) and
+ /// boolean flag (i1 type) with true if success and false otherwise.
+ std::pair<RValue, llvm::Value *> EmitAtomicCompareExchange(
+ RValue Expected, RValue Desired,
+ llvm::AtomicOrdering Success = llvm::SequentiallyConsistent,
+ llvm::AtomicOrdering Failure = llvm::SequentiallyConsistent,
+ bool IsWeak = false);
+
+ /// \brief Emits atomic update.
+ /// \param AO Atomic ordering.
+ /// \param UpdateOp Update operation for the current lvalue.
+ void EmitAtomicUpdate(llvm::AtomicOrdering AO,
+ const llvm::function_ref<RValue(RValue)> &UpdateOp,
+ bool IsVolatile);
+ /// \brief Emits atomic update.
+ /// \param AO Atomic ordering.
+ void EmitAtomicUpdate(llvm::AtomicOrdering AO, RValue UpdateRVal,
+ bool IsVolatile);
+
/// Materialize an atomic r-value in atomic-layout memory.
llvm::Value *materializeRValue(RValue rvalue) const;
+ /// \brief Translates LLVM atomic ordering to GNU atomic ordering for
+ /// libcalls.
+ static AtomicExpr::AtomicOrderingKind
+ translateAtomicOrdering(const llvm::AtomicOrdering AO);
+
private:
bool requiresMemSetZero(llvm::Type *type) const;
+
+ /// \brief Creates temp alloca for intermediate operations on atomic value.
+ llvm::Value *CreateTempAlloca() const;
+
+ /// \brief Emits atomic load as a libcall.
+ void EmitAtomicLoadLibcall(llvm::Value *AddForLoaded,
+ llvm::AtomicOrdering AO, bool IsVolatile);
+ /// \brief Emits atomic load as LLVM instruction.
+ llvm::Value *EmitAtomicLoadOp(llvm::AtomicOrdering AO, bool IsVolatile);
+ /// \brief Emits atomic compare-and-exchange op as a libcall.
+ llvm::Value *EmitAtomicCompareExchangeLibcall(
+ llvm::Value *ExpectedAddr, llvm::Value *DesiredAddr,
+ llvm::AtomicOrdering Success = llvm::SequentiallyConsistent,
+ llvm::AtomicOrdering Failure = llvm::SequentiallyConsistent);
+ /// \brief Emits atomic compare-and-exchange op as LLVM instruction.
+ std::pair<llvm::Value *, llvm::Value *> EmitAtomicCompareExchangeOp(
+ llvm::Value *ExpectedVal, llvm::Value *DesiredVal,
+ llvm::AtomicOrdering Success = llvm::SequentiallyConsistent,
+ llvm::AtomicOrdering Failure = llvm::SequentiallyConsistent,
+ bool IsWeak = false);
+ /// \brief Emit atomic update as libcalls.
+ void
+ EmitAtomicUpdateLibcall(llvm::AtomicOrdering AO,
+ const llvm::function_ref<RValue(RValue)> &UpdateOp,
+ bool IsVolatile);
+ /// \brief Emit atomic update as LLVM instructions.
+ void EmitAtomicUpdateOp(llvm::AtomicOrdering AO,
+ const llvm::function_ref<RValue(RValue)> &UpdateOp,
+ bool IsVolatile);
+ /// \brief Emit atomic update as libcalls.
+ void EmitAtomicUpdateLibcall(llvm::AtomicOrdering AO, RValue UpdateRVal,
+ bool IsVolatile);
+ /// \brief Emit atomic update as LLVM instructions.
+ void EmitAtomicUpdateOp(llvm::AtomicOrdering AO, RValue UpdateRal,
+ bool IsVolatile);
};
}
+AtomicExpr::AtomicOrderingKind
+AtomicInfo::translateAtomicOrdering(const llvm::AtomicOrdering AO) {
+ switch (AO) {
+ case llvm::Unordered:
+ case llvm::NotAtomic:
+ case llvm::Monotonic:
+ return AtomicExpr::AO_ABI_memory_order_relaxed;
+ case llvm::Acquire:
+ return AtomicExpr::AO_ABI_memory_order_acquire;
+ case llvm::Release:
+ return AtomicExpr::AO_ABI_memory_order_release;
+ case llvm::AcquireRelease:
+ return AtomicExpr::AO_ABI_memory_order_acq_rel;
+ case llvm::SequentiallyConsistent:
+ return AtomicExpr::AO_ABI_memory_order_seq_cst;
+ }
+ llvm_unreachable("Unhandled AtomicOrdering");
+}
+
+llvm::Value *AtomicInfo::CreateTempAlloca() const {
+ auto *TempAlloca = CGF.CreateMemTemp(
+ (LVal.isBitField() && ValueSizeInBits > AtomicSizeInBits) ? ValueTy
+ : AtomicTy,
+ "atomic-temp");
+ TempAlloca->setAlignment(getAtomicAlignment().getQuantity());
+ // Cast to pointer to value type for bitfields.
+ if (LVal.isBitField())
+ return CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
+ TempAlloca, getAtomicAddress()->getType());
+ return TempAlloca;
+}
+
static RValue emitAtomicLibcall(CodeGenFunction &CGF,
StringRef fnName,
QualType resultType,
@@ -172,14 +348,16 @@ bool AtomicInfo::requiresMemSetZero(llvm::Type *type) const {
llvm_unreachable("bad evaluation kind");
}
-bool AtomicInfo::emitMemSetZeroIfNecessary(LValue dest) const {
- llvm::Value *addr = dest.getAddress();
+bool AtomicInfo::emitMemSetZeroIfNecessary() const {
+ assert(LVal.isSimple());
+ llvm::Value *addr = LVal.getAddress();
if (!requiresMemSetZero(addr->getType()->getPointerElementType()))
return false;
- CGF.Builder.CreateMemSet(addr, llvm::ConstantInt::get(CGF.Int8Ty, 0),
- AtomicSizeInBits / 8,
- dest.getAlignment().getQuantity());
+ CGF.Builder.CreateMemSet(
+ addr, llvm::ConstantInt::get(CGF.Int8Ty, 0),
+ CGF.getContext().toCharUnitsFromBits(AtomicSizeInBits).getQuantity(),
+ LVal.getAlignment().getQuantity());
return true;
}
@@ -901,29 +1079,53 @@ llvm::Value *AtomicInfo::emitCastToAtomicIntPointer(llvm::Value *addr) const {
RValue AtomicInfo::convertTempToRValue(llvm::Value *addr,
AggValueSlot resultSlot,
- SourceLocation loc) const {
- if (EvaluationKind == TEK_Aggregate)
- return resultSlot.asRValue();
-
- // Drill into the padding structure if we have one.
- if (hasPadding())
- addr = CGF.Builder.CreateStructGEP(addr, 0);
-
- // Otherwise, just convert the temporary to an r-value using the
- // normal conversion routine.
- return CGF.convertTempToRValue(addr, getValueType(), loc);
+ SourceLocation loc, bool AsValue) const {
+ if (LVal.isSimple()) {
+ if (EvaluationKind == TEK_Aggregate)
+ return resultSlot.asRValue();
+
+ // Drill into the padding structure if we have one.
+ if (hasPadding())
+ addr = CGF.Builder.CreateStructGEP(nullptr, addr, 0);
+
+ // Otherwise, just convert the temporary to an r-value using the
+ // normal conversion routine.
+ return CGF.convertTempToRValue(addr, getValueType(), loc);
+ }
+ if (!AsValue)
+ // Get RValue from temp memory as atomic for non-simple lvalues
+ return RValue::get(
+ CGF.Builder.CreateAlignedLoad(addr, AtomicAlign.getQuantity()));
+ if (LVal.isBitField())
+ return CGF.EmitLoadOfBitfieldLValue(LValue::MakeBitfield(
+ addr, LVal.getBitFieldInfo(), LVal.getType(), LVal.getAlignment()));
+ if (LVal.isVectorElt())
+ return CGF.EmitLoadOfLValue(LValue::MakeVectorElt(addr, LVal.getVectorIdx(),
+ LVal.getType(),
+ LVal.getAlignment()),
+ loc);
+ assert(LVal.isExtVectorElt());
+ return CGF.EmitLoadOfExtVectorElementLValue(LValue::MakeExtVectorElt(
+ addr, LVal.getExtVectorElts(), LVal.getType(), LVal.getAlignment()));
}
-RValue AtomicInfo::convertIntToValue(llvm::Value *IntVal,
- AggValueSlot ResultSlot,
- SourceLocation Loc) const {
+RValue AtomicInfo::ConvertIntToValueOrAtomic(llvm::Value *IntVal,
+ AggValueSlot ResultSlot,
+ SourceLocation Loc,
+ bool AsValue) const {
// Try not to in some easy cases.
assert(IntVal->getType()->isIntegerTy() && "Expected integer value");
- if (getEvaluationKind() == TEK_Scalar && !hasPadding()) {
- auto *ValTy = CGF.ConvertTypeForMem(ValueTy);
+ if (getEvaluationKind() == TEK_Scalar &&
+ (((!LVal.isBitField() ||
+ LVal.getBitFieldInfo().Size == ValueSizeInBits) &&
+ !hasPadding()) ||
+ !AsValue)) {
+ auto *ValTy = AsValue
+ ? CGF.ConvertTypeForMem(ValueTy)
+ : getAtomicAddress()->getType()->getPointerElementType();
if (ValTy->isIntegerTy()) {
assert(IntVal->getType() == ValTy && "Different integer types.");
- return RValue::get(IntVal);
+ return RValue::get(CGF.EmitFromMemory(IntVal, ValueTy));
} else if (ValTy->isPointerTy())
return RValue::get(CGF.Builder.CreateIntToPtr(IntVal, ValTy));
else if (llvm::CastInst::isBitCastable(IntVal->getType(), ValTy))
@@ -935,13 +1137,13 @@ RValue AtomicInfo::convertIntToValue(llvm::Value *IntVal,
llvm::Value *Temp;
bool TempIsVolatile = false;
CharUnits TempAlignment;
- if (getEvaluationKind() == TEK_Aggregate) {
+ if (AsValue && getEvaluationKind() == TEK_Aggregate) {
assert(!ResultSlot.isIgnored());
Temp = ResultSlot.getAddr();
TempAlignment = getValueAlignment();
TempIsVolatile = ResultSlot.isVolatile();
} else {
- Temp = CGF.CreateMemTemp(getAtomicType(), "atomic-temp");
+ Temp = CreateTempAlloca();
TempAlignment = getAtomicAlignment();
}
@@ -950,93 +1152,146 @@ RValue AtomicInfo::convertIntToValue(llvm::Value *IntVal,
CGF.Builder.CreateAlignedStore(IntVal, CastTemp, TempAlignment.getQuantity())
->setVolatile(TempIsVolatile);
- return convertTempToRValue(Temp, ResultSlot, Loc);
+ return convertTempToRValue(Temp, ResultSlot, Loc, AsValue);
}
-/// Emit a load from an l-value of atomic type. Note that the r-value
-/// we produce is an r-value of the atomic *value* type.
-RValue CodeGenFunction::EmitAtomicLoad(LValue src, SourceLocation loc,
- AggValueSlot resultSlot) {
- AtomicInfo atomics(*this, src);
+void AtomicInfo::EmitAtomicLoadLibcall(llvm::Value *AddForLoaded,
+ llvm::AtomicOrdering AO, bool) {
+ // void __atomic_load(size_t size, void *mem, void *return, int order);
+ CallArgList Args;
+ Args.add(RValue::get(getAtomicSizeValue()), CGF.getContext().getSizeType());
+ Args.add(RValue::get(CGF.EmitCastToVoidPtr(getAtomicAddress())),
+ CGF.getContext().VoidPtrTy);
+ Args.add(RValue::get(CGF.EmitCastToVoidPtr(AddForLoaded)),
+ CGF.getContext().VoidPtrTy);
+ Args.add(RValue::get(
+ llvm::ConstantInt::get(CGF.IntTy, translateAtomicOrdering(AO))),
+ CGF.getContext().IntTy);
+ emitAtomicLibcall(CGF, "__atomic_load", CGF.getContext().VoidTy, Args);
+}
- // Check whether we should use a library call.
- if (atomics.shouldUseLibcall()) {
- llvm::Value *tempAddr;
- if (!resultSlot.isIgnored()) {
- assert(atomics.getEvaluationKind() == TEK_Aggregate);
- tempAddr = resultSlot.getAddr();
- } else {
- tempAddr = CreateMemTemp(atomics.getAtomicType(), "atomic-load-temp");
- }
+llvm::Value *AtomicInfo::EmitAtomicLoadOp(llvm::AtomicOrdering AO,
+ bool IsVolatile) {
+ // Okay, we're doing this natively.
+ llvm::Value *Addr = emitCastToAtomicIntPointer(getAtomicAddress());
+ llvm::LoadInst *Load = CGF.Builder.CreateLoad(Addr, "atomic-load");
+ Load->setAtomic(AO);
- // void __atomic_load(size_t size, void *mem, void *return, int order);
- CallArgList args;
- args.add(RValue::get(atomics.getAtomicSizeValue()),
- getContext().getSizeType());
- args.add(RValue::get(EmitCastToVoidPtr(src.getAddress())),
- getContext().VoidPtrTy);
- args.add(RValue::get(EmitCastToVoidPtr(tempAddr)),
- getContext().VoidPtrTy);
- args.add(RValue::get(llvm::ConstantInt::get(
- IntTy, AtomicExpr::AO_ABI_memory_order_seq_cst)),
- getContext().IntTy);
- emitAtomicLibcall(*this, "__atomic_load", getContext().VoidTy, args);
+ // Other decoration.
+ Load->setAlignment(getAtomicAlignment().getQuantity());
+ if (IsVolatile)
+ Load->setVolatile(true);
+ if (LVal.getTBAAInfo())
+ CGF.CGM.DecorateInstruction(Load, LVal.getTBAAInfo());
+ return Load;
+}
+
+/// An LValue is a candidate for having its loads and stores be made atomic if
+/// we are operating under /volatile:ms *and* the LValue itself is volatile and
+/// performing such an operation can be performed without a libcall.
+bool CodeGenFunction::LValueIsSuitableForInlineAtomic(LValue LV) {
+ AtomicInfo AI(*this, LV);
+ bool IsVolatile = LV.isVolatile() || hasVolatileMember(LV.getType());
+ // An atomic is inline if we don't need to use a libcall.
+ bool AtomicIsInline = !AI.shouldUseLibcall();
+ return CGM.getCodeGenOpts().MSVolatile && IsVolatile && AtomicIsInline;
+}
- // Produce the r-value.
- return atomics.convertTempToRValue(tempAddr, resultSlot, loc);
+/// An type is a candidate for having its loads and stores be made atomic if
+/// we are operating under /volatile:ms *and* we know the access is volatile and
+/// performing such an operation can be performed without a libcall.
+bool CodeGenFunction::typeIsSuitableForInlineAtomic(QualType Ty,
+ bool IsVolatile) const {
+ // An atomic is inline if we don't need to use a libcall (e.g. it is builtin).
+ bool AtomicIsInline = getContext().getTargetInfo().hasBuiltinAtomic(
+ getContext().getTypeSize(Ty), getContext().getTypeAlign(Ty));
+ return CGM.getCodeGenOpts().MSVolatile && IsVolatile && AtomicIsInline;
+}
+
+RValue CodeGenFunction::EmitAtomicLoad(LValue LV, SourceLocation SL,
+ AggValueSlot Slot) {
+ llvm::AtomicOrdering AO;
+ bool IsVolatile = LV.isVolatileQualified();
+ if (LV.getType()->isAtomicType()) {
+ AO = llvm::SequentiallyConsistent;
+ } else {
+ AO = llvm::Acquire;
+ IsVolatile = true;
}
+ return EmitAtomicLoad(LV, SL, AO, IsVolatile, Slot);
+}
- // Okay, we're doing this natively.
- llvm::Value *addr = atomics.emitCastToAtomicIntPointer(src.getAddress());
- llvm::LoadInst *load = Builder.CreateLoad(addr, "atomic-load");
- load->setAtomic(llvm::SequentiallyConsistent);
+RValue AtomicInfo::EmitAtomicLoad(AggValueSlot ResultSlot, SourceLocation Loc,
+ bool AsValue, llvm::AtomicOrdering AO,
+ bool IsVolatile) {
+ // Check whether we should use a library call.
+ if (shouldUseLibcall()) {
+ llvm::Value *TempAddr;
+ if (LVal.isSimple() && !ResultSlot.isIgnored()) {
+ assert(getEvaluationKind() == TEK_Aggregate);
+ TempAddr = ResultSlot.getAddr();
+ } else
+ TempAddr = CreateTempAlloca();
+
+ EmitAtomicLoadLibcall(TempAddr, AO, IsVolatile);
+
+ // Okay, turn that back into the original value or whole atomic (for
+ // non-simple lvalues) type.
+ return convertTempToRValue(TempAddr, ResultSlot, Loc, AsValue);
+ }
- // Other decoration.
- load->setAlignment(src.getAlignment().getQuantity());
- if (src.isVolatileQualified())
- load->setVolatile(true);
- if (src.getTBAAInfo())
- CGM.DecorateInstruction(load, src.getTBAAInfo());
+ // Okay, we're doing this natively.
+ auto *Load = EmitAtomicLoadOp(AO, IsVolatile);
// If we're ignoring an aggregate return, don't do anything.
- if (atomics.getEvaluationKind() == TEK_Aggregate && resultSlot.isIgnored())
+ if (getEvaluationKind() == TEK_Aggregate && ResultSlot.isIgnored())
return RValue::getAggregate(nullptr, false);
- // Okay, turn that back into the original value type.
- return atomics.convertIntToValue(load, resultSlot, loc);
+ // Okay, turn that back into the original value or atomic (for non-simple
+ // lvalues) type.
+ return ConvertIntToValueOrAtomic(Load, ResultSlot, Loc, AsValue);
}
-
+/// Emit a load from an l-value of atomic type. Note that the r-value
+/// we produce is an r-value of the atomic *value* type.
+RValue CodeGenFunction::EmitAtomicLoad(LValue src, SourceLocation loc,
+ llvm::AtomicOrdering AO, bool IsVolatile,
+ AggValueSlot resultSlot) {
+ AtomicInfo Atomics(*this, src);
+ return Atomics.EmitAtomicLoad(resultSlot, loc, /*AsValue=*/true, AO,
+ IsVolatile);
+}
/// Copy an r-value into memory as part of storing to an atomic type.
/// This needs to create a bit-pattern suitable for atomic operations.
-void AtomicInfo::emitCopyIntoMemory(RValue rvalue, LValue dest) const {
+void AtomicInfo::emitCopyIntoMemory(RValue rvalue) const {
+ assert(LVal.isSimple());
// If we have an r-value, the rvalue should be of the atomic type,
// which means that the caller is responsible for having zeroed
// any padding. Just do an aggregate copy of that type.
if (rvalue.isAggregate()) {
- CGF.EmitAggregateCopy(dest.getAddress(),
+ CGF.EmitAggregateCopy(getAtomicAddress(),
rvalue.getAggregateAddr(),
getAtomicType(),
(rvalue.isVolatileQualified()
- || dest.isVolatileQualified()),
- dest.getAlignment());
+ || LVal.isVolatileQualified()),
+ LVal.getAlignment());
return;
}
// Okay, otherwise we're copying stuff.
// Zero out the buffer if necessary.
- emitMemSetZeroIfNecessary(dest);
+ emitMemSetZeroIfNecessary();
// Drill past the padding if present.
- dest = projectValue(dest);
+ LValue TempLVal = projectValue();
// Okay, store the rvalue in.
if (rvalue.isScalar()) {
- CGF.EmitStoreOfScalar(rvalue.getScalarVal(), dest, /*init*/ true);
+ CGF.EmitStoreOfScalar(rvalue.getScalarVal(), TempLVal, /*init*/ true);
} else {
- CGF.EmitStoreOfComplex(rvalue.getComplexVal(), dest, /*init*/ true);
+ CGF.EmitStoreOfComplex(rvalue.getComplexVal(), TempLVal, /*init*/ true);
}
}
@@ -1050,22 +1305,24 @@ llvm::Value *AtomicInfo::materializeRValue(RValue rvalue) const {
return rvalue.getAggregateAddr();
// Otherwise, make a temporary and materialize into it.
- llvm::Value *temp = CGF.CreateMemTemp(getAtomicType(), "atomic-store-temp");
- LValue tempLV = CGF.MakeAddrLValue(temp, getAtomicType(), getAtomicAlignment());
- emitCopyIntoMemory(rvalue, tempLV);
- return temp;
+ LValue TempLV = CGF.MakeAddrLValue(CreateTempAlloca(), getAtomicType(),
+ getAtomicAlignment());
+ AtomicInfo Atomics(CGF, TempLV);
+ Atomics.emitCopyIntoMemory(rvalue);
+ return TempLV.getAddress();
}
llvm::Value *AtomicInfo::convertRValueToInt(RValue RVal) const {
// If we've got a scalar value of the right size, try to avoid going
// through memory.
- if (RVal.isScalar() && !hasPadding()) {
+ if (RVal.isScalar() && (!hasPadding() || !LVal.isSimple())) {
llvm::Value *Value = RVal.getScalarVal();
if (isa<llvm::IntegerType>(Value->getType()))
- return Value;
+ return CGF.EmitToMemory(Value, ValueTy);
else {
- llvm::IntegerType *InputIntTy =
- llvm::IntegerType::get(CGF.getLLVMContext(), getValueSizeInBits());
+ llvm::IntegerType *InputIntTy = llvm::IntegerType::get(
+ CGF.getLLVMContext(),
+ LVal.isSimple() ? getValueSizeInBits() : getAtomicSizeInBits());
if (isa<llvm::PointerType>(Value->getType()))
return CGF.Builder.CreatePtrToInt(Value, InputIntTy);
else if (llvm::BitCastInst::isBitCastable(Value->getType(), InputIntTy))
@@ -1082,12 +1339,324 @@ llvm::Value *AtomicInfo::convertRValueToInt(RValue RVal) const {
getAtomicAlignment().getQuantity());
}
+std::pair<llvm::Value *, llvm::Value *> AtomicInfo::EmitAtomicCompareExchangeOp(
+ llvm::Value *ExpectedVal, llvm::Value *DesiredVal,
+ llvm::AtomicOrdering Success, llvm::AtomicOrdering Failure, bool IsWeak) {
+ // Do the atomic store.
+ auto *Addr = emitCastToAtomicIntPointer(getAtomicAddress());
+ auto *Inst = CGF.Builder.CreateAtomicCmpXchg(Addr, ExpectedVal, DesiredVal,
+ Success, Failure);
+ // Other decoration.
+ Inst->setVolatile(LVal.isVolatileQualified());
+ Inst->setWeak(IsWeak);
+
+ // Okay, turn that back into the original value type.
+ auto *PreviousVal = CGF.Builder.CreateExtractValue(Inst, /*Idxs=*/0);
+ auto *SuccessFailureVal = CGF.Builder.CreateExtractValue(Inst, /*Idxs=*/1);
+ return std::make_pair(PreviousVal, SuccessFailureVal);
+}
+
+llvm::Value *
+AtomicInfo::EmitAtomicCompareExchangeLibcall(llvm::Value *ExpectedAddr,
+ llvm::Value *DesiredAddr,
+ llvm::AtomicOrdering Success,
+ llvm::AtomicOrdering Failure) {
+ // bool __atomic_compare_exchange(size_t size, void *obj, void *expected,
+ // void *desired, int success, int failure);
+ CallArgList Args;
+ Args.add(RValue::get(getAtomicSizeValue()), CGF.getContext().getSizeType());
+ Args.add(RValue::get(CGF.EmitCastToVoidPtr(getAtomicAddress())),
+ CGF.getContext().VoidPtrTy);
+ Args.add(RValue::get(CGF.EmitCastToVoidPtr(ExpectedAddr)),
+ CGF.getContext().VoidPtrTy);
+ Args.add(RValue::get(CGF.EmitCastToVoidPtr(DesiredAddr)),
+ CGF.getContext().VoidPtrTy);
+ Args.add(RValue::get(llvm::ConstantInt::get(
+ CGF.IntTy, translateAtomicOrdering(Success))),
+ CGF.getContext().IntTy);
+ Args.add(RValue::get(llvm::ConstantInt::get(
+ CGF.IntTy, translateAtomicOrdering(Failure))),
+ CGF.getContext().IntTy);
+ auto SuccessFailureRVal = emitAtomicLibcall(CGF, "__atomic_compare_exchange",
+ CGF.getContext().BoolTy, Args);
+
+ return SuccessFailureRVal.getScalarVal();
+}
+
+std::pair<RValue, llvm::Value *> AtomicInfo::EmitAtomicCompareExchange(
+ RValue Expected, RValue Desired, llvm::AtomicOrdering Success,
+ llvm::AtomicOrdering Failure, bool IsWeak) {
+ if (Failure >= Success)
+ // Don't assert on undefined behavior.
+ Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(Success);
+
+ // Check whether we should use a library call.
+ if (shouldUseLibcall()) {
+ // Produce a source address.
+ auto *ExpectedAddr = materializeRValue(Expected);
+ auto *DesiredAddr = materializeRValue(Desired);
+ auto *Res = EmitAtomicCompareExchangeLibcall(ExpectedAddr, DesiredAddr,
+ Success, Failure);
+ return std::make_pair(
+ convertTempToRValue(ExpectedAddr, AggValueSlot::ignored(),
+ SourceLocation(), /*AsValue=*/false),
+ Res);
+ }
+
+ // If we've got a scalar value of the right size, try to avoid going
+ // through memory.
+ auto *ExpectedVal = convertRValueToInt(Expected);
+ auto *DesiredVal = convertRValueToInt(Desired);
+ auto Res = EmitAtomicCompareExchangeOp(ExpectedVal, DesiredVal, Success,
+ Failure, IsWeak);
+ return std::make_pair(
+ ConvertIntToValueOrAtomic(Res.first, AggValueSlot::ignored(),
+ SourceLocation(), /*AsValue=*/false),
+ Res.second);
+}
+
+static void
+EmitAtomicUpdateValue(CodeGenFunction &CGF, AtomicInfo &Atomics, RValue OldRVal,
+ const llvm::function_ref<RValue(RValue)> &UpdateOp,
+ llvm::Value *DesiredAddr) {
+ llvm::Value *Ptr = nullptr;
+ LValue UpdateLVal;
+ RValue UpRVal;
+ LValue AtomicLVal = Atomics.getAtomicLValue();
+ LValue DesiredLVal;
+ if (AtomicLVal.isSimple()) {
+ UpRVal = OldRVal;
+ DesiredLVal =
+ LValue::MakeAddr(DesiredAddr, AtomicLVal.getType(),
+ AtomicLVal.getAlignment(), CGF.CGM.getContext());
+ } else {
+ // Build new lvalue for temp address
+ Ptr = Atomics.materializeRValue(OldRVal);
+ if (AtomicLVal.isBitField()) {
+ UpdateLVal =
+ LValue::MakeBitfield(Ptr, AtomicLVal.getBitFieldInfo(),
+ AtomicLVal.getType(), AtomicLVal.getAlignment());
+ DesiredLVal =
+ LValue::MakeBitfield(DesiredAddr, AtomicLVal.getBitFieldInfo(),
+ AtomicLVal.getType(), AtomicLVal.getAlignment());
+ } else if (AtomicLVal.isVectorElt()) {
+ UpdateLVal = LValue::MakeVectorElt(Ptr, AtomicLVal.getVectorIdx(),
+ AtomicLVal.getType(),
+ AtomicLVal.getAlignment());
+ DesiredLVal = LValue::MakeVectorElt(
+ DesiredAddr, AtomicLVal.getVectorIdx(), AtomicLVal.getType(),
+ AtomicLVal.getAlignment());
+ } else {
+ assert(AtomicLVal.isExtVectorElt());
+ UpdateLVal = LValue::MakeExtVectorElt(Ptr, AtomicLVal.getExtVectorElts(),
+ AtomicLVal.getType(),
+ AtomicLVal.getAlignment());
+ DesiredLVal = LValue::MakeExtVectorElt(
+ DesiredAddr, AtomicLVal.getExtVectorElts(), AtomicLVal.getType(),
+ AtomicLVal.getAlignment());
+ }
+ UpdateLVal.setTBAAInfo(AtomicLVal.getTBAAInfo());
+ DesiredLVal.setTBAAInfo(AtomicLVal.getTBAAInfo());
+ UpRVal = CGF.EmitLoadOfLValue(UpdateLVal, SourceLocation());
+ }
+ // Store new value in the corresponding memory area
+ RValue NewRVal = UpdateOp(UpRVal);
+ if (NewRVal.isScalar()) {
+ CGF.EmitStoreThroughLValue(NewRVal, DesiredLVal);
+ } else {
+ assert(NewRVal.isComplex());
+ CGF.EmitStoreOfComplex(NewRVal.getComplexVal(), DesiredLVal,
+ /*isInit=*/false);
+ }
+}
+
+void AtomicInfo::EmitAtomicUpdateLibcall(
+ llvm::AtomicOrdering AO, const llvm::function_ref<RValue(RValue)> &UpdateOp,
+ bool IsVolatile) {
+ auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO);
+
+ llvm::Value *ExpectedAddr = CreateTempAlloca();
+
+ EmitAtomicLoadLibcall(ExpectedAddr, AO, IsVolatile);
+ auto *ContBB = CGF.createBasicBlock("atomic_cont");
+ auto *ExitBB = CGF.createBasicBlock("atomic_exit");
+ CGF.EmitBlock(ContBB);
+ auto *DesiredAddr = CreateTempAlloca();
+ if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) ||
+ requiresMemSetZero(
+ getAtomicAddress()->getType()->getPointerElementType())) {
+ auto *OldVal = CGF.Builder.CreateAlignedLoad(
+ ExpectedAddr, getAtomicAlignment().getQuantity());
+ CGF.Builder.CreateAlignedStore(OldVal, DesiredAddr,
+ getAtomicAlignment().getQuantity());
+ }
+ auto OldRVal = convertTempToRValue(ExpectedAddr, AggValueSlot::ignored(),
+ SourceLocation(), /*AsValue=*/false);
+ EmitAtomicUpdateValue(CGF, *this, OldRVal, UpdateOp, DesiredAddr);
+ auto *Res =
+ EmitAtomicCompareExchangeLibcall(ExpectedAddr, DesiredAddr, AO, Failure);
+ CGF.Builder.CreateCondBr(Res, ExitBB, ContBB);
+ CGF.EmitBlock(ExitBB, /*IsFinished=*/true);
+}
+
+void AtomicInfo::EmitAtomicUpdateOp(
+ llvm::AtomicOrdering AO, const llvm::function_ref<RValue(RValue)> &UpdateOp,
+ bool IsVolatile) {
+ auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO);
+
+ // Do the atomic load.
+ auto *OldVal = EmitAtomicLoadOp(AO, IsVolatile);
+ // For non-simple lvalues perform compare-and-swap procedure.
+ auto *ContBB = CGF.createBasicBlock("atomic_cont");
+ auto *ExitBB = CGF.createBasicBlock("atomic_exit");
+ auto *CurBB = CGF.Builder.GetInsertBlock();
+ CGF.EmitBlock(ContBB);
+ llvm::PHINode *PHI = CGF.Builder.CreatePHI(OldVal->getType(),
+ /*NumReservedValues=*/2);
+ PHI->addIncoming(OldVal, CurBB);
+ auto *NewAtomicAddr = CreateTempAlloca();
+ auto *NewAtomicIntAddr = emitCastToAtomicIntPointer(NewAtomicAddr);
+ if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) ||
+ requiresMemSetZero(
+ getAtomicAddress()->getType()->getPointerElementType())) {
+ CGF.Builder.CreateAlignedStore(PHI, NewAtomicIntAddr,
+ getAtomicAlignment().getQuantity());
+ }
+ auto OldRVal = ConvertIntToValueOrAtomic(PHI, AggValueSlot::ignored(),
+ SourceLocation(), /*AsValue=*/false);
+ EmitAtomicUpdateValue(CGF, *this, OldRVal, UpdateOp, NewAtomicAddr);
+ auto *DesiredVal = CGF.Builder.CreateAlignedLoad(
+ NewAtomicIntAddr, getAtomicAlignment().getQuantity());
+ // Try to write new value using cmpxchg operation
+ auto Res = EmitAtomicCompareExchangeOp(PHI, DesiredVal, AO, Failure);
+ PHI->addIncoming(Res.first, CGF.Builder.GetInsertBlock());
+ CGF.Builder.CreateCondBr(Res.second, ExitBB, ContBB);
+ CGF.EmitBlock(ExitBB, /*IsFinished=*/true);
+}
+
+static void EmitAtomicUpdateValue(CodeGenFunction &CGF, AtomicInfo &Atomics,
+ RValue UpdateRVal, llvm::Value *DesiredAddr) {
+ LValue AtomicLVal = Atomics.getAtomicLValue();
+ LValue DesiredLVal;
+ // Build new lvalue for temp address
+ if (AtomicLVal.isBitField()) {
+ DesiredLVal =
+ LValue::MakeBitfield(DesiredAddr, AtomicLVal.getBitFieldInfo(),
+ AtomicLVal.getType(), AtomicLVal.getAlignment());
+ } else if (AtomicLVal.isVectorElt()) {
+ DesiredLVal =
+ LValue::MakeVectorElt(DesiredAddr, AtomicLVal.getVectorIdx(),
+ AtomicLVal.getType(), AtomicLVal.getAlignment());
+ } else {
+ assert(AtomicLVal.isExtVectorElt());
+ DesiredLVal = LValue::MakeExtVectorElt(
+ DesiredAddr, AtomicLVal.getExtVectorElts(), AtomicLVal.getType(),
+ AtomicLVal.getAlignment());
+ }
+ DesiredLVal.setTBAAInfo(AtomicLVal.getTBAAInfo());
+ // Store new value in the corresponding memory area
+ assert(UpdateRVal.isScalar());
+ CGF.EmitStoreThroughLValue(UpdateRVal, DesiredLVal);
+}
+
+void AtomicInfo::EmitAtomicUpdateLibcall(llvm::AtomicOrdering AO,
+ RValue UpdateRVal, bool IsVolatile) {
+ auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO);
+
+ llvm::Value *ExpectedAddr = CreateTempAlloca();
+
+ EmitAtomicLoadLibcall(ExpectedAddr, AO, IsVolatile);
+ auto *ContBB = CGF.createBasicBlock("atomic_cont");
+ auto *ExitBB = CGF.createBasicBlock("atomic_exit");
+ CGF.EmitBlock(ContBB);
+ auto *DesiredAddr = CreateTempAlloca();
+ if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) ||
+ requiresMemSetZero(
+ getAtomicAddress()->getType()->getPointerElementType())) {
+ auto *OldVal = CGF.Builder.CreateAlignedLoad(
+ ExpectedAddr, getAtomicAlignment().getQuantity());
+ CGF.Builder.CreateAlignedStore(OldVal, DesiredAddr,
+ getAtomicAlignment().getQuantity());
+ }
+ EmitAtomicUpdateValue(CGF, *this, UpdateRVal, DesiredAddr);
+ auto *Res =
+ EmitAtomicCompareExchangeLibcall(ExpectedAddr, DesiredAddr, AO, Failure);
+ CGF.Builder.CreateCondBr(Res, ExitBB, ContBB);
+ CGF.EmitBlock(ExitBB, /*IsFinished=*/true);
+}
+
+void AtomicInfo::EmitAtomicUpdateOp(llvm::AtomicOrdering AO, RValue UpdateRVal,
+ bool IsVolatile) {
+ auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO);
+
+ // Do the atomic load.
+ auto *OldVal = EmitAtomicLoadOp(AO, IsVolatile);
+ // For non-simple lvalues perform compare-and-swap procedure.
+ auto *ContBB = CGF.createBasicBlock("atomic_cont");
+ auto *ExitBB = CGF.createBasicBlock("atomic_exit");
+ auto *CurBB = CGF.Builder.GetInsertBlock();
+ CGF.EmitBlock(ContBB);
+ llvm::PHINode *PHI = CGF.Builder.CreatePHI(OldVal->getType(),
+ /*NumReservedValues=*/2);
+ PHI->addIncoming(OldVal, CurBB);
+ auto *NewAtomicAddr = CreateTempAlloca();
+ auto *NewAtomicIntAddr = emitCastToAtomicIntPointer(NewAtomicAddr);
+ if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) ||
+ requiresMemSetZero(
+ getAtomicAddress()->getType()->getPointerElementType())) {
+ CGF.Builder.CreateAlignedStore(PHI, NewAtomicIntAddr,
+ getAtomicAlignment().getQuantity());
+ }
+ EmitAtomicUpdateValue(CGF, *this, UpdateRVal, NewAtomicAddr);
+ auto *DesiredVal = CGF.Builder.CreateAlignedLoad(
+ NewAtomicIntAddr, getAtomicAlignment().getQuantity());
+ // Try to write new value using cmpxchg operation
+ auto Res = EmitAtomicCompareExchangeOp(PHI, DesiredVal, AO, Failure);
+ PHI->addIncoming(Res.first, CGF.Builder.GetInsertBlock());
+ CGF.Builder.CreateCondBr(Res.second, ExitBB, ContBB);
+ CGF.EmitBlock(ExitBB, /*IsFinished=*/true);
+}
+
+void AtomicInfo::EmitAtomicUpdate(
+ llvm::AtomicOrdering AO, const llvm::function_ref<RValue(RValue)> &UpdateOp,
+ bool IsVolatile) {
+ if (shouldUseLibcall()) {
+ EmitAtomicUpdateLibcall(AO, UpdateOp, IsVolatile);
+ } else {
+ EmitAtomicUpdateOp(AO, UpdateOp, IsVolatile);
+ }
+}
+
+void AtomicInfo::EmitAtomicUpdate(llvm::AtomicOrdering AO, RValue UpdateRVal,
+ bool IsVolatile) {
+ if (shouldUseLibcall()) {
+ EmitAtomicUpdateLibcall(AO, UpdateRVal, IsVolatile);
+ } else {
+ EmitAtomicUpdateOp(AO, UpdateRVal, IsVolatile);
+ }
+}
+
+void CodeGenFunction::EmitAtomicStore(RValue rvalue, LValue lvalue,
+ bool isInit) {
+ bool IsVolatile = lvalue.isVolatileQualified();
+ llvm::AtomicOrdering AO;
+ if (lvalue.getType()->isAtomicType()) {
+ AO = llvm::SequentiallyConsistent;
+ } else {
+ AO = llvm::Release;
+ IsVolatile = true;
+ }
+ return EmitAtomicStore(rvalue, lvalue, AO, IsVolatile, isInit);
+}
+
/// Emit a store to an l-value of atomic type.
///
/// Note that the r-value is expected to be an r-value *of the atomic
/// type*; this means that for aggregate r-values, it should include
/// storage for any padding that was necessary.
-void CodeGenFunction::EmitAtomicStore(RValue rvalue, LValue dest, bool isInit) {
+void CodeGenFunction::EmitAtomicStore(RValue rvalue, LValue dest,
+ llvm::AtomicOrdering AO, bool IsVolatile,
+ bool isInit) {
// If this is an aggregate r-value, it should agree in type except
// maybe for address-space qualification.
assert(!rvalue.isAggregate() ||
@@ -1095,54 +1664,64 @@ void CodeGenFunction::EmitAtomicStore(RValue rvalue, LValue dest, bool isInit) {
== dest.getAddress()->getType()->getPointerElementType());
AtomicInfo atomics(*this, dest);
+ LValue LVal = atomics.getAtomicLValue();
// If this is an initialization, just put the value there normally.
- if (isInit) {
- atomics.emitCopyIntoMemory(rvalue, dest);
- return;
- }
+ if (LVal.isSimple()) {
+ if (isInit) {
+ atomics.emitCopyIntoMemory(rvalue);
+ return;
+ }
- // Check whether we should use a library call.
- if (atomics.shouldUseLibcall()) {
- // Produce a source address.
- llvm::Value *srcAddr = atomics.materializeRValue(rvalue);
+ // Check whether we should use a library call.
+ if (atomics.shouldUseLibcall()) {
+ // Produce a source address.
+ llvm::Value *srcAddr = atomics.materializeRValue(rvalue);
- // void __atomic_store(size_t size, void *mem, void *val, int order)
- CallArgList args;
- args.add(RValue::get(atomics.getAtomicSizeValue()),
- getContext().getSizeType());
- args.add(RValue::get(EmitCastToVoidPtr(dest.getAddress())),
- getContext().VoidPtrTy);
- args.add(RValue::get(EmitCastToVoidPtr(srcAddr)),
- getContext().VoidPtrTy);
- args.add(RValue::get(llvm::ConstantInt::get(
- IntTy, AtomicExpr::AO_ABI_memory_order_seq_cst)),
- getContext().IntTy);
- emitAtomicLibcall(*this, "__atomic_store", getContext().VoidTy, args);
+ // void __atomic_store(size_t size, void *mem, void *val, int order)
+ CallArgList args;
+ args.add(RValue::get(atomics.getAtomicSizeValue()),
+ getContext().getSizeType());
+ args.add(RValue::get(EmitCastToVoidPtr(atomics.getAtomicAddress())),
+ getContext().VoidPtrTy);
+ args.add(RValue::get(EmitCastToVoidPtr(srcAddr)), getContext().VoidPtrTy);
+ args.add(RValue::get(llvm::ConstantInt::get(
+ IntTy, AtomicInfo::translateAtomicOrdering(AO))),
+ getContext().IntTy);
+ emitAtomicLibcall(*this, "__atomic_store", getContext().VoidTy, args);
+ return;
+ }
+
+ // Okay, we're doing this natively.
+ llvm::Value *intValue = atomics.convertRValueToInt(rvalue);
+
+ // Do the atomic store.
+ llvm::Value *addr =
+ atomics.emitCastToAtomicIntPointer(atomics.getAtomicAddress());
+ intValue = Builder.CreateIntCast(
+ intValue, addr->getType()->getPointerElementType(), /*isSigned=*/false);
+ llvm::StoreInst *store = Builder.CreateStore(intValue, addr);
+
+ // Initializations don't need to be atomic.
+ if (!isInit)
+ store->setAtomic(AO);
+
+ // Other decoration.
+ store->setAlignment(dest.getAlignment().getQuantity());
+ if (IsVolatile)
+ store->setVolatile(true);
+ if (dest.getTBAAInfo())
+ CGM.DecorateInstruction(store, dest.getTBAAInfo());
return;
}
- // Okay, we're doing this natively.
- llvm::Value *intValue = atomics.convertRValueToInt(rvalue);
-
- // Do the atomic store.
- llvm::Value *addr = atomics.emitCastToAtomicIntPointer(dest.getAddress());
- llvm::StoreInst *store = Builder.CreateStore(intValue, addr);
-
- // Initializations don't need to be atomic.
- if (!isInit) store->setAtomic(llvm::SequentiallyConsistent);
-
- // Other decoration.
- store->setAlignment(dest.getAlignment().getQuantity());
- if (dest.isVolatileQualified())
- store->setVolatile(true);
- if (dest.getTBAAInfo())
- CGM.DecorateInstruction(store, dest.getTBAAInfo());
+ // Emit simple atomic update operation.
+ atomics.EmitAtomicUpdate(AO, rvalue, IsVolatile);
}
/// Emit a compare-and-exchange op for atomic type.
///
-std::pair<RValue, RValue> CodeGenFunction::EmitAtomicCompareExchange(
+std::pair<RValue, llvm::Value *> CodeGenFunction::EmitAtomicCompareExchange(
LValue Obj, RValue Expected, RValue Desired, SourceLocation Loc,
llvm::AtomicOrdering Success, llvm::AtomicOrdering Failure, bool IsWeak,
AggValueSlot Slot) {
@@ -1156,56 +1735,15 @@ std::pair<RValue, RValue> CodeGenFunction::EmitAtomicCompareExchange(
Obj.getAddress()->getType()->getPointerElementType());
AtomicInfo Atomics(*this, Obj);
- if (Failure >= Success)
- // Don't assert on undefined behavior.
- Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(Success);
-
- auto Alignment = Atomics.getValueAlignment();
- // Check whether we should use a library call.
- if (Atomics.shouldUseLibcall()) {
- auto *ExpectedAddr = Atomics.materializeRValue(Expected);
- // Produce a source address.
- auto *DesiredAddr = Atomics.materializeRValue(Desired);
- // bool __atomic_compare_exchange(size_t size, void *obj, void *expected,
- // void *desired, int success, int failure);
- CallArgList Args;
- Args.add(RValue::get(Atomics.getAtomicSizeValue()),
- getContext().getSizeType());
- Args.add(RValue::get(EmitCastToVoidPtr(Obj.getAddress())),
- getContext().VoidPtrTy);
- Args.add(RValue::get(EmitCastToVoidPtr(ExpectedAddr)),
- getContext().VoidPtrTy);
- Args.add(RValue::get(EmitCastToVoidPtr(DesiredAddr)),
- getContext().VoidPtrTy);
- Args.add(RValue::get(llvm::ConstantInt::get(IntTy, Success)),
- getContext().IntTy);
- Args.add(RValue::get(llvm::ConstantInt::get(IntTy, Failure)),
- getContext().IntTy);
- auto SuccessFailureRVal = emitAtomicLibcall(
- *this, "__atomic_compare_exchange", getContext().BoolTy, Args);
- auto *PreviousVal =
- Builder.CreateAlignedLoad(ExpectedAddr, Alignment.getQuantity());
- return std::make_pair(RValue::get(PreviousVal), SuccessFailureRVal);
- }
-
- // If we've got a scalar value of the right size, try to avoid going
- // through memory.
- auto *ExpectedIntVal = Atomics.convertRValueToInt(Expected);
- auto *DesiredIntVal = Atomics.convertRValueToInt(Desired);
-
- // Do the atomic store.
- auto *Addr = Atomics.emitCastToAtomicIntPointer(Obj.getAddress());
- auto *Inst = Builder.CreateAtomicCmpXchg(Addr, ExpectedIntVal, DesiredIntVal,
- Success, Failure);
- // Other decoration.
- Inst->setVolatile(Obj.isVolatileQualified());
- Inst->setWeak(IsWeak);
+ return Atomics.EmitAtomicCompareExchange(Expected, Desired, Success, Failure,
+ IsWeak);
+}
- // Okay, turn that back into the original value type.
- auto *PreviousVal = Builder.CreateExtractValue(Inst, /*Idxs=*/0);
- auto *SuccessFailureVal = Builder.CreateExtractValue(Inst, /*Idxs=*/1);
- return std::make_pair(Atomics.convertIntToValue(PreviousVal, Slot, Loc),
- RValue::get(SuccessFailureVal));
+void CodeGenFunction::EmitAtomicUpdate(
+ LValue LVal, llvm::AtomicOrdering AO,
+ const llvm::function_ref<RValue(RValue)> &UpdateOp, bool IsVolatile) {
+ AtomicInfo Atomics(*this, LVal);
+ Atomics.EmitAtomicUpdate(AO, UpdateOp, IsVolatile);
}
void CodeGenFunction::EmitAtomicInit(Expr *init, LValue dest) {
@@ -1214,13 +1752,13 @@ void CodeGenFunction::EmitAtomicInit(Expr *init, LValue dest) {
switch (atomics.getEvaluationKind()) {
case TEK_Scalar: {
llvm::Value *value = EmitScalarExpr(init);
- atomics.emitCopyIntoMemory(RValue::get(value), dest);
+ atomics.emitCopyIntoMemory(RValue::get(value));
return;
}
case TEK_Complex: {
ComplexPairTy value = EmitComplexExpr(init);
- atomics.emitCopyIntoMemory(RValue::getComplex(value), dest);
+ atomics.emitCopyIntoMemory(RValue::getComplex(value));
return;
}
@@ -1229,8 +1767,8 @@ void CodeGenFunction::EmitAtomicInit(Expr *init, LValue dest) {
// of atomic type.
bool Zeroed = false;
if (!init->getType()->isAtomicType()) {
- Zeroed = atomics.emitMemSetZeroIfNecessary(dest);
- dest = atomics.projectValue(dest);
+ Zeroed = atomics.emitMemSetZeroIfNecessary();
+ dest = atomics.projectValue();
}
// Evaluate the expression directly into the destination.
diff --git a/lib/CodeGen/CGBlocks.cpp b/lib/CodeGen/CGBlocks.cpp
index b98460a9ddd8..3fd344c389a5 100644
--- a/lib/CodeGen/CGBlocks.cpp
+++ b/lib/CodeGen/CGBlocks.cpp
@@ -621,8 +621,8 @@ static void enterBlockScope(CodeGenFunction &CGF, BlockDecl *block) {
}
// GEP down to the address.
- llvm::Value *addr = CGF.Builder.CreateStructGEP(blockInfo.Address,
- capture.getIndex());
+ llvm::Value *addr = CGF.Builder.CreateStructGEP(
+ blockInfo.StructureType, blockInfo.Address, capture.getIndex());
// We can use that GEP as the dominating IP.
if (!blockInfo.DominatingIP)
@@ -721,6 +721,7 @@ llvm::Value *CodeGenFunction::EmitBlockLiteral(const CGBlockInfo &blockInfo) {
// Build the block descriptor.
llvm::Constant *descriptor = buildBlockDescriptor(CGM, blockInfo);
+ llvm::Type *blockTy = blockInfo.StructureType;
llvm::AllocaInst *blockAddr = blockInfo.Address;
assert(blockAddr && "block has no address!");
@@ -732,14 +733,17 @@ llvm::Value *CodeGenFunction::EmitBlockLiteral(const CGBlockInfo &blockInfo) {
if (blockInfo.UsesStret) flags |= BLOCK_USE_STRET;
// Initialize the block literal.
- Builder.CreateStore(isa, Builder.CreateStructGEP(blockAddr, 0, "block.isa"));
- Builder.CreateStore(llvm::ConstantInt::get(IntTy, flags.getBitMask()),
- Builder.CreateStructGEP(blockAddr, 1, "block.flags"));
- Builder.CreateStore(llvm::ConstantInt::get(IntTy, 0),
- Builder.CreateStructGEP(blockAddr, 2, "block.reserved"));
- Builder.CreateStore(blockFn, Builder.CreateStructGEP(blockAddr, 3,
- "block.invoke"));
- Builder.CreateStore(descriptor, Builder.CreateStructGEP(blockAddr, 4,
+ Builder.CreateStore(
+ isa, Builder.CreateStructGEP(blockTy, blockAddr, 0, "block.isa"));
+ Builder.CreateStore(
+ llvm::ConstantInt::get(IntTy, flags.getBitMask()),
+ Builder.CreateStructGEP(blockTy, blockAddr, 1, "block.flags"));
+ Builder.CreateStore(
+ llvm::ConstantInt::get(IntTy, 0),
+ Builder.CreateStructGEP(blockTy, blockAddr, 2, "block.reserved"));
+ Builder.CreateStore(
+ blockFn, Builder.CreateStructGEP(blockTy, blockAddr, 3, "block.invoke"));
+ Builder.CreateStore(descriptor, Builder.CreateStructGEP(blockTy, blockAddr, 4,
"block.descriptor"));
// Finally, capture all the values into the block.
@@ -747,9 +751,8 @@ llvm::Value *CodeGenFunction::EmitBlockLiteral(const CGBlockInfo &blockInfo) {
// First, 'this'.
if (blockDecl->capturesCXXThis()) {
- llvm::Value *addr = Builder.CreateStructGEP(blockAddr,
- blockInfo.CXXThisIndex,
- "block.captured-this.addr");
+ llvm::Value *addr = Builder.CreateStructGEP(
+ blockTy, blockAddr, blockInfo.CXXThisIndex, "block.captured-this.addr");
Builder.CreateStore(LoadCXXThis(), addr);
}
@@ -766,9 +769,8 @@ llvm::Value *CodeGenFunction::EmitBlockLiteral(const CGBlockInfo &blockInfo) {
// This will be a [[type]]*, except that a byref entry will just be
// an i8**.
- llvm::Value *blockField =
- Builder.CreateStructGEP(blockAddr, capture.getIndex(),
- "block.captured");
+ llvm::Value *blockField = Builder.CreateStructGEP(
+ blockTy, blockAddr, capture.getIndex(), "block.captured");
// Compute the address of the thing we're going to move into the
// block literal.
@@ -779,7 +781,7 @@ llvm::Value *CodeGenFunction::EmitBlockLiteral(const CGBlockInfo &blockInfo) {
BlockInfo->getCapture(variable);
// This is a [[type]]*, except that a byref entry wil just be an i8**.
- src = Builder.CreateStructGEP(LoadBlockStruct(),
+ src = Builder.CreateStructGEP(BlockInfo->StructureType, LoadBlockStruct(),
enclosingCapture.getIndex(),
"block.capture.addr");
} else if (blockDecl->isConversionFromLambda()) {
@@ -964,7 +966,8 @@ RValue CodeGenFunction::EmitBlockCallExpr(const CallExpr *E,
Builder.CreateBitCast(Callee, BlockLiteralTy, "block.literal");
// Get the function pointer from the literal.
- llvm::Value *FuncPtr = Builder.CreateStructGEP(BlockLiteral, 3);
+ llvm::Value *FuncPtr = Builder.CreateStructGEP(
+ CGM.getGenericBlockLiteralType(), BlockLiteral, 3);
BlockLiteral = Builder.CreateBitCast(BlockLiteral, VoidPtrTy);
@@ -1004,26 +1007,27 @@ llvm::Value *CodeGenFunction::GetAddrOfBlockDecl(const VarDecl *variable,
if (capture.isConstant()) return LocalDeclMap[variable];
llvm::Value *addr =
- Builder.CreateStructGEP(LoadBlockStruct(), capture.getIndex(),
- "block.capture.addr");
+ Builder.CreateStructGEP(BlockInfo->StructureType, LoadBlockStruct(),
+ capture.getIndex(), "block.capture.addr");
if (isByRef) {
// addr should be a void** right now. Load, then cast the result
// to byref*.
addr = Builder.CreateLoad(addr);
- llvm::PointerType *byrefPointerType
- = llvm::PointerType::get(BuildByRefType(variable), 0);
+ auto *byrefType = BuildByRefType(variable);
+ llvm::PointerType *byrefPointerType = llvm::PointerType::get(byrefType, 0);
addr = Builder.CreateBitCast(addr, byrefPointerType,
"byref.addr");
// Follow the forwarding pointer.
- addr = Builder.CreateStructGEP(addr, 1, "byref.forwarding");
+ addr = Builder.CreateStructGEP(byrefType, addr, 1, "byref.forwarding");
addr = Builder.CreateLoad(addr, "byref.addr.forwarded");
// Cast back to byref* and GEP over to the actual object.
addr = Builder.CreateBitCast(addr, byrefPointerType);
- addr = Builder.CreateStructGEP(addr, getByRefValueLLVMField(variable),
+ addr = Builder.CreateStructGEP(byrefType, addr,
+ getByRefValueLLVMField(variable).second,
variable->getNameAsString());
}
@@ -1136,8 +1140,7 @@ CodeGenFunction::GenerateBlockFunction(GlobalDecl GD,
args.push_back(&selfDecl);
// Now add the rest of the parameters.
- for (auto i : blockDecl->params())
- args.push_back(i);
+ args.append(blockDecl->param_begin(), blockDecl->param_end());
// Create the function declaration.
const FunctionProtoType *fnType = blockInfo.getBlockExpr()->getFunctionType();
@@ -1178,7 +1181,7 @@ CodeGenFunction::GenerateBlockFunction(GlobalDecl GD,
Alloca->setAlignment(Align);
// Set the DebugLocation to empty, so the store is recognized as a
// frame setup instruction by llvm::DwarfDebug::beginFunction().
- ApplyDebugLocation NL(*this);
+ auto NL = ApplyDebugLocation::CreateEmpty(*this);
Builder.CreateAlignedStore(BlockPointer, Alloca, Align);
BlockPointerDbgLoc = Alloca;
}
@@ -1186,9 +1189,9 @@ CodeGenFunction::GenerateBlockFunction(GlobalDecl GD,
// If we have a C++ 'this' reference, go ahead and force it into
// existence now.
if (blockDecl->capturesCXXThis()) {
- llvm::Value *addr = Builder.CreateStructGEP(BlockPointer,
- blockInfo.CXXThisIndex,
- "block.captured-this");
+ llvm::Value *addr =
+ Builder.CreateStructGEP(blockInfo.StructureType, BlockPointer,
+ blockInfo.CXXThisIndex, "block.captured-this");
CXXThisValue = Builder.CreateLoad(addr, "this");
}
@@ -1218,8 +1221,7 @@ CodeGenFunction::GenerateBlockFunction(GlobalDecl GD,
EmitLambdaBlockInvokeBody();
else {
PGO.assignRegionCounters(blockDecl, fn);
- RegionCounter Cnt = getPGORegionCounter(blockDecl->getBody());
- Cnt.beginRegion(Builder);
+ incrementProfileCounter(blockDecl->getBody());
EmitStmt(blockDecl->getBody());
}
@@ -1328,11 +1330,10 @@ CodeGenFunction::GenerateCopyHelperFunction(const CGBlockInfo &blockInfo) {
nullptr, SC_Static,
false,
false);
- // Create a scope with an artificial location for the body of this function.
- ApplyDebugLocation NL(*this);
+ auto NL = ApplyDebugLocation::CreateEmpty(*this);
StartFunction(FD, C.VoidTy, Fn, FI, args);
- ArtificialLocation AL(*this);
-
+ // Create a scope with an artificial location for the body of this function.
+ auto AL = ApplyDebugLocation::CreateArtificial(*this);
llvm::Type *structPtrTy = blockInfo.StructureType->getPointerTo();
llvm::Value *src = GetAddrOfLocalVar(&srcDecl);
@@ -1404,8 +1405,10 @@ CodeGenFunction::GenerateCopyHelperFunction(const CGBlockInfo &blockInfo) {
}
unsigned index = capture.getIndex();
- llvm::Value *srcField = Builder.CreateStructGEP(src, index);
- llvm::Value *dstField = Builder.CreateStructGEP(dst, index);
+ llvm::Value *srcField =
+ Builder.CreateStructGEP(blockInfo.StructureType, src, index);
+ llvm::Value *dstField =
+ Builder.CreateStructGEP(blockInfo.StructureType, dst, index);
// If there's an explicit copy expression, we do that.
if (copyExpr) {
@@ -1500,9 +1503,9 @@ CodeGenFunction::GenerateDestroyHelperFunction(const CGBlockInfo &blockInfo) {
nullptr, SC_Static,
false, false);
// Create a scope with an artificial location for the body of this function.
- ApplyDebugLocation NL(*this);
+ auto NL = ApplyDebugLocation::CreateEmpty(*this);
StartFunction(FD, C.VoidTy, Fn, FI, args);
- ArtificialLocation AL(*this);
+ auto AL = ApplyDebugLocation::CreateArtificial(*this);
llvm::Type *structPtrTy = blockInfo.StructureType->getPointerTo();
@@ -1562,7 +1565,8 @@ CodeGenFunction::GenerateDestroyHelperFunction(const CGBlockInfo &blockInfo) {
}
unsigned index = capture.getIndex();
- llvm::Value *srcField = Builder.CreateStructGEP(src, index);
+ llvm::Value *srcField =
+ Builder.CreateStructGEP(blockInfo.StructureType, src, index);
// If there's an explicit copy expression, we do that.
if (dtor) {
@@ -1801,13 +1805,15 @@ generateByrefCopyHelper(CodeGenFunction &CGF,
llvm::Value *destField = CGF.GetAddrOfLocalVar(&dst);
destField = CGF.Builder.CreateLoad(destField);
destField = CGF.Builder.CreateBitCast(destField, byrefPtrType);
- destField = CGF.Builder.CreateStructGEP(destField, valueFieldIndex, "x");
+ destField = CGF.Builder.CreateStructGEP(&byrefType, destField,
+ valueFieldIndex, "x");
// src->x
llvm::Value *srcField = CGF.GetAddrOfLocalVar(&src);
srcField = CGF.Builder.CreateLoad(srcField);
srcField = CGF.Builder.CreateBitCast(srcField, byrefPtrType);
- srcField = CGF.Builder.CreateStructGEP(srcField, valueFieldIndex, "x");
+ srcField =
+ CGF.Builder.CreateStructGEP(&byrefType, srcField, valueFieldIndex, "x");
byrefInfo.emitCopy(CGF, destField, srcField);
}
@@ -1868,7 +1874,7 @@ generateByrefDisposeHelper(CodeGenFunction &CGF,
llvm::Value *V = CGF.GetAddrOfLocalVar(&src);
V = CGF.Builder.CreateLoad(V);
V = CGF.Builder.CreateBitCast(V, byrefType.getPointerTo(0));
- V = CGF.Builder.CreateStructGEP(V, byrefValueIndex, "x");
+ V = CGF.Builder.CreateStructGEP(&byrefType, V, byrefValueIndex, "x");
byrefInfo.emitDispose(CGF, V);
}
@@ -1925,7 +1931,7 @@ CodeGenFunction::buildByrefHelpers(llvm::StructType &byrefType,
const VarDecl &var = *emission.Variable;
QualType type = var.getType();
- unsigned byrefValueIndex = getByRefValueLLVMField(&var);
+ unsigned byrefValueIndex = getByRefValueLLVMField(&var).second;
if (const CXXRecordDecl *record = type->getAsCXXRecordDecl()) {
const Expr *copyExpr = CGM.getContext().getBlockVarCopyInits(&var);
@@ -1995,18 +2001,20 @@ CodeGenFunction::buildByrefHelpers(llvm::StructType &byrefType,
return ::buildByrefHelpers(CGM, byrefType, byrefValueIndex, byrefInfo);
}
-unsigned CodeGenFunction::getByRefValueLLVMField(const ValueDecl *VD) const {
+std::pair<llvm::Type *, unsigned>
+CodeGenFunction::getByRefValueLLVMField(const ValueDecl *VD) const {
assert(ByRefValueInfo.count(VD) && "Did not find value!");
-
- return ByRefValueInfo.find(VD)->second.second;
+
+ return ByRefValueInfo.find(VD)->second;
}
llvm::Value *CodeGenFunction::BuildBlockByrefAddress(llvm::Value *BaseAddr,
const VarDecl *V) {
- llvm::Value *Loc = Builder.CreateStructGEP(BaseAddr, 1, "forwarding");
+ auto P = getByRefValueLLVMField(V);
+ llvm::Value *Loc =
+ Builder.CreateStructGEP(P.first, BaseAddr, 1, "forwarding");
Loc = Builder.CreateLoad(Loc);
- Loc = Builder.CreateStructGEP(Loc, getByRefValueLLVMField(V),
- V->getNameAsString());
+ Loc = Builder.CreateStructGEP(P.first, Loc, P.second, V->getNameAsString());
return Loc;
}
@@ -2143,11 +2151,12 @@ void CodeGenFunction::emitByrefStructureInit(const AutoVarEmission &emission) {
if (type.isObjCGCWeak())
isa = 1;
V = Builder.CreateIntToPtr(Builder.getInt32(isa), Int8PtrTy, "isa");
- Builder.CreateStore(V, Builder.CreateStructGEP(addr, 0, "byref.isa"));
+ Builder.CreateStore(V,
+ Builder.CreateStructGEP(nullptr, addr, 0, "byref.isa"));
// Store the address of the variable into its own forwarding pointer.
- Builder.CreateStore(addr,
- Builder.CreateStructGEP(addr, 1, "byref.forwarding"));
+ Builder.CreateStore(
+ addr, Builder.CreateStructGEP(nullptr, addr, 1, "byref.forwarding"));
// Blocks ABI:
// c) the flags field is set to either 0 if no helper functions are
@@ -2193,25 +2202,26 @@ void CodeGenFunction::emitByrefStructureInit(const AutoVarEmission &emission) {
printf("\n");
}
}
-
+
Builder.CreateStore(llvm::ConstantInt::get(IntTy, flags.getBitMask()),
- Builder.CreateStructGEP(addr, 2, "byref.flags"));
+ Builder.CreateStructGEP(nullptr, addr, 2, "byref.flags"));
CharUnits byrefSize = CGM.GetTargetTypeStoreSize(byrefType);
V = llvm::ConstantInt::get(IntTy, byrefSize.getQuantity());
- Builder.CreateStore(V, Builder.CreateStructGEP(addr, 3, "byref.size"));
+ Builder.CreateStore(V,
+ Builder.CreateStructGEP(nullptr, addr, 3, "byref.size"));
if (helpers) {
- llvm::Value *copy_helper = Builder.CreateStructGEP(addr, 4);
+ llvm::Value *copy_helper = Builder.CreateStructGEP(nullptr, addr, 4);
Builder.CreateStore(helpers->CopyHelper, copy_helper);
- llvm::Value *destroy_helper = Builder.CreateStructGEP(addr, 5);
+ llvm::Value *destroy_helper = Builder.CreateStructGEP(nullptr, addr, 5);
Builder.CreateStore(helpers->DisposeHelper, destroy_helper);
}
if (ByRefHasLifetime && HasByrefExtendedLayout) {
llvm::Constant* ByrefLayoutInfo = CGM.getObjCRuntime().BuildByrefLayout(CGM, type);
- llvm::Value *ByrefInfoAddr = Builder.CreateStructGEP(addr, helpers ? 6 : 4,
- "byref.layout");
+ llvm::Value *ByrefInfoAddr =
+ Builder.CreateStructGEP(nullptr, addr, helpers ? 6 : 4, "byref.layout");
// cast destination to pointer to source type.
llvm::Type *DesTy = ByrefLayoutInfo->getType();
DesTy = DesTy->getPointerTo();
diff --git a/lib/CodeGen/CGBuilder.h b/lib/CodeGen/CGBuilder.h
index 72ba4faa3c7c..6610659131f7 100644
--- a/lib/CodeGen/CGBuilder.h
+++ b/lib/CodeGen/CGBuilder.h
@@ -33,7 +33,7 @@ protected:
llvm::BasicBlock *BB,
llvm::BasicBlock::iterator InsertPt) const;
private:
- void operator=(const CGBuilderInserter &) LLVM_DELETED_FUNCTION;
+ void operator=(const CGBuilderInserter &) = delete;
CodeGenFunction *CGF;
};
diff --git a/lib/CodeGen/CGBuiltin.cpp b/lib/CodeGen/CGBuiltin.cpp
index 635e34207de7..272baac80897 100644
--- a/lib/CodeGen/CGBuiltin.cpp
+++ b/lib/CodeGen/CGBuiltin.cpp
@@ -12,6 +12,7 @@
//===----------------------------------------------------------------------===//
#include "CodeGenFunction.h"
+#include "CGCXXABI.h"
#include "CGObjCRuntime.h"
#include "CodeGenModule.h"
#include "TargetInfo.h"
@@ -21,9 +22,11 @@
#include "clang/Basic/TargetInfo.h"
#include "clang/CodeGen/CGFunctionInfo.h"
#include "llvm/ADT/StringExtras.h"
+#include "llvm/IR/CallSite.h"
#include "llvm/IR/DataLayout.h"
#include "llvm/IR/InlineAsm.h"
#include "llvm/IR/Intrinsics.h"
+#include <sstream>
using namespace clang;
using namespace CodeGen;
@@ -156,6 +159,27 @@ static Value *EmitFAbs(CodeGenFunction &CGF, Value *V) {
return Call;
}
+/// Emit the computation of the sign bit for a floating point value. Returns
+/// the i1 sign bit value.
+static Value *EmitSignBit(CodeGenFunction &CGF, Value *V) {
+ LLVMContext &C = CGF.CGM.getLLVMContext();
+
+ llvm::Type *Ty = V->getType();
+ int Width = Ty->getPrimitiveSizeInBits();
+ llvm::Type *IntTy = llvm::IntegerType::get(C, Width);
+ V = CGF.Builder.CreateBitCast(V, IntTy);
+ if (Ty->isPPC_FP128Ty()) {
+ // The higher-order double comes first, and so we need to truncate the
+ // pair to extract the overall sign. The order of the pair is the same
+ // in both little- and big-Endian modes.
+ Width >>= 1;
+ IntTy = llvm::IntegerType::get(C, Width);
+ V = CGF.Builder.CreateTrunc(V, IntTy);
+ }
+ Value *Zero = llvm::Constant::getNullValue(IntTy);
+ return CGF.Builder.CreateICmpSLT(V, Zero);
+}
+
static RValue emitLibraryCall(CodeGenFunction &CGF, const FunctionDecl *Fn,
const CallExpr *E, llvm::Value *calleeValue) {
return CGF.EmitCall(E->getCallee()->getType(), calleeValue, E,
@@ -181,7 +205,7 @@ static llvm::Value *EmitOverflowIntrinsic(CodeGenFunction &CGF,
"arguments have the same integer width?)");
llvm::Value *Callee = CGF.CGM.getIntrinsic(IntrinsicID, X->getType());
- llvm::Value *Tmp = CGF.Builder.CreateCall2(Callee, X, Y);
+ llvm::Value *Tmp = CGF.Builder.CreateCall(Callee, {X, Y});
Carry = CGF.Builder.CreateExtractValue(Tmp, 1);
return CGF.Builder.CreateExtractValue(Tmp, 0);
}
@@ -230,8 +254,8 @@ RValue CodeGenFunction::EmitBuiltinExpr(const FunctionDecl *FD,
DstPtr = Builder.CreateBitCast(DstPtr, Type);
SrcPtr = Builder.CreateBitCast(SrcPtr, Type);
- return RValue::get(Builder.CreateCall2(CGM.getIntrinsic(Intrinsic::vacopy),
- DstPtr, SrcPtr));
+ return RValue::get(Builder.CreateCall(CGM.getIntrinsic(Intrinsic::vacopy),
+ {DstPtr, SrcPtr}));
}
case Builtin::BI__builtin_abs:
case Builtin::BI__builtin_labs:
@@ -309,7 +333,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const FunctionDecl *FD,
llvm::Type *ResultType = ConvertType(E->getType());
Value *ZeroUndef = Builder.getInt1(getTarget().isCLZForZeroUndef());
- Value *Result = Builder.CreateCall2(F, ArgValue, ZeroUndef);
+ Value *Result = Builder.CreateCall(F, {ArgValue, ZeroUndef});
if (Result->getType() != ResultType)
Result = Builder.CreateIntCast(Result, ResultType, /*isSigned*/true,
"cast");
@@ -326,7 +350,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const FunctionDecl *FD,
llvm::Type *ResultType = ConvertType(E->getType());
Value *ZeroUndef = Builder.getInt1(getTarget().isCLZForZeroUndef());
- Value *Result = Builder.CreateCall2(F, ArgValue, ZeroUndef);
+ Value *Result = Builder.CreateCall(F, {ArgValue, ZeroUndef});
if (Result->getType() != ResultType)
Result = Builder.CreateIntCast(Result, ResultType, /*isSigned*/true,
"cast");
@@ -342,9 +366,9 @@ RValue CodeGenFunction::EmitBuiltinExpr(const FunctionDecl *FD,
Value *F = CGM.getIntrinsic(Intrinsic::cttz, ArgType);
llvm::Type *ResultType = ConvertType(E->getType());
- Value *Tmp = Builder.CreateAdd(Builder.CreateCall2(F, ArgValue,
- Builder.getTrue()),
- llvm::ConstantInt::get(ArgType, 1));
+ Value *Tmp =
+ Builder.CreateAdd(Builder.CreateCall(F, {ArgValue, Builder.getTrue()}),
+ llvm::ConstantInt::get(ArgType, 1));
Value *Zero = llvm::Constant::getNullValue(ArgType);
Value *IsZero = Builder.CreateICmpEQ(ArgValue, Zero, "iszero");
Value *Result = Builder.CreateSelect(IsZero, Zero, Tmp, "ffs");
@@ -389,11 +413,16 @@ RValue CodeGenFunction::EmitBuiltinExpr(const FunctionDecl *FD,
Value *ArgValue = EmitScalarExpr(E->getArg(0));
llvm::Type *ArgType = ArgValue->getType();
- Value *FnExpect = CGM.getIntrinsic(Intrinsic::expect, ArgType);
Value *ExpectedValue = EmitScalarExpr(E->getArg(1));
+ // Don't generate llvm.expect on -O0 as the backend won't use it for
+ // anything.
+ // Note, we still IRGen ExpectedValue because it could have side-effects.
+ if (CGM.getCodeGenOpts().OptimizationLevel == 0)
+ return RValue::get(ArgValue);
- Value *Result = Builder.CreateCall2(FnExpect, ArgValue, ExpectedValue,
- "expval");
+ Value *FnExpect = CGM.getIntrinsic(Intrinsic::expect, ArgType);
+ Value *Result =
+ Builder.CreateCall(FnExpect, {ArgValue, ExpectedValue}, "expval");
return RValue::get(Result);
}
case Builtin::BI__builtin_assume_aligned: {
@@ -444,7 +473,8 @@ RValue CodeGenFunction::EmitBuiltinExpr(const FunctionDecl *FD,
// FIXME: Get right address space.
llvm::Type *Tys[] = { ResType, Builder.getInt8PtrTy(0) };
Value *F = CGM.getIntrinsic(Intrinsic::objectsize, Tys);
- return RValue::get(Builder.CreateCall2(F, EmitScalarExpr(E->getArg(0)),CI));
+ return RValue::get(
+ Builder.CreateCall(F, {EmitScalarExpr(E->getArg(0)), CI}));
}
case Builtin::BI__builtin_prefetch: {
Value *Locality, *RW, *Address = EmitScalarExpr(E->getArg(0));
@@ -455,25 +485,25 @@ RValue CodeGenFunction::EmitBuiltinExpr(const FunctionDecl *FD,
llvm::ConstantInt::get(Int32Ty, 3);
Value *Data = llvm::ConstantInt::get(Int32Ty, 1);
Value *F = CGM.getIntrinsic(Intrinsic::prefetch);
- return RValue::get(Builder.CreateCall4(F, Address, RW, Locality, Data));
+ return RValue::get(Builder.CreateCall(F, {Address, RW, Locality, Data}));
}
case Builtin::BI__builtin_readcyclecounter: {
Value *F = CGM.getIntrinsic(Intrinsic::readcyclecounter);
- return RValue::get(Builder.CreateCall(F));
+ return RValue::get(Builder.CreateCall(F, {}));
}
case Builtin::BI__builtin___clear_cache: {
Value *Begin = EmitScalarExpr(E->getArg(0));
Value *End = EmitScalarExpr(E->getArg(1));
Value *F = CGM.getIntrinsic(Intrinsic::clear_cache);
- return RValue::get(Builder.CreateCall2(F, Begin, End));
+ return RValue::get(Builder.CreateCall(F, {Begin, End}));
}
case Builtin::BI__builtin_trap: {
Value *F = CGM.getIntrinsic(Intrinsic::trap);
- return RValue::get(Builder.CreateCall(F));
+ return RValue::get(Builder.CreateCall(F, {}));
}
case Builtin::BI__debugbreak: {
Value *F = CGM.getIntrinsic(Intrinsic::debugtrap);
- return RValue::get(Builder.CreateCall(F));
+ return RValue::get(Builder.CreateCall(F, {}));
}
case Builtin::BI__builtin_unreachable: {
if (SanOpts.has(SanitizerKind::Unreachable)) {
@@ -498,7 +528,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const FunctionDecl *FD,
Value *Exponent = EmitScalarExpr(E->getArg(1));
llvm::Type *ArgType = Base->getType();
Value *F = CGM.getIntrinsic(Intrinsic::powi, ArgType);
- return RValue::get(Builder.CreateCall2(F, Base, Exponent));
+ return RValue::get(Builder.CreateCall(F, {Base, Exponent}));
}
case Builtin::BI__builtin_isgreater:
@@ -551,8 +581,22 @@ RValue CodeGenFunction::EmitBuiltinExpr(const FunctionDecl *FD,
return RValue::get(Builder.CreateZExt(V, ConvertType(E->getType())));
}
- // TODO: BI__builtin_isinf_sign
- // isinf_sign(x) -> isinf(x) ? (signbit(x) ? -1 : 1) : 0
+ case Builtin::BI__builtin_isinf_sign: {
+ // isinf_sign(x) -> fabs(x) == infinity ? (signbit(x) ? -1 : 1) : 0
+ Value *Arg = EmitScalarExpr(E->getArg(0));
+ Value *AbsArg = EmitFAbs(*this, Arg);
+ Value *IsInf = Builder.CreateFCmpOEQ(
+ AbsArg, ConstantFP::getInfinity(Arg->getType()), "isinf");
+ Value *IsNeg = EmitSignBit(*this, Arg);
+
+ llvm::Type *IntTy = ConvertType(E->getType());
+ Value *Zero = Constant::getNullValue(IntTy);
+ Value *One = ConstantInt::get(IntTy, 1);
+ Value *NegativeOne = ConstantInt::get(IntTy, -1);
+ Value *SignResult = Builder.CreateSelect(IsNeg, NegativeOne, One);
+ Value *Result = Builder.CreateSelect(IsInf, SignResult, Zero);
+ return RValue::get(Result);
+ }
case Builtin::BI__builtin_isnormal: {
// isnormal(x) --> x == x && fabsf(x) < infinity && fabsf(x) >= float_min
@@ -815,7 +859,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const FunctionDecl *FD,
Value *F = CGM.getIntrinsic(IntTy->getBitWidth() == 32
? Intrinsic::eh_return_i32
: Intrinsic::eh_return_i64);
- Builder.CreateCall2(F, Int, Ptr);
+ Builder.CreateCall(F, {Int, Ptr});
Builder.CreateUnreachable();
// We do need to preserve an insertion point.
@@ -825,7 +869,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const FunctionDecl *FD,
}
case Builtin::BI__builtin_unwind_init: {
Value *F = CGM.getIntrinsic(Intrinsic::eh_unwind_init);
- return RValue::get(Builder.CreateCall(F));
+ return RValue::get(Builder.CreateCall(F, {}));
}
case Builtin::BI__builtin_extend_pointer: {
// Extends a pointer to the size of an _Unwind_Word, which is
@@ -864,7 +908,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const FunctionDecl *FD,
// Store the stack pointer to the setjmp buffer.
Value *StackAddr =
- Builder.CreateCall(CGM.getIntrinsic(Intrinsic::stacksave));
+ Builder.CreateCall(CGM.getIntrinsic(Intrinsic::stacksave), {});
Value *StackSaveSlot =
Builder.CreateGEP(Buf, ConstantInt::get(Int32Ty, 2));
Builder.CreateStore(StackAddr, StackSaveSlot);
@@ -1357,6 +1401,9 @@ RValue CodeGenFunction::EmitBuiltinExpr(const FunctionDecl *FD,
return RValue::get(Builder.CreateCall(F, Arg0));
}
+ case Builtin::BI__builtin_pow:
+ case Builtin::BI__builtin_powf:
+ case Builtin::BI__builtin_powl:
case Builtin::BIpow:
case Builtin::BIpowf:
case Builtin::BIpowl: {
@@ -1367,7 +1414,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const FunctionDecl *FD,
Value *Exponent = EmitScalarExpr(E->getArg(1));
llvm::Type *ArgType = Base->getType();
Value *F = CGM.getIntrinsic(Intrinsic::pow, ArgType);
- return RValue::get(Builder.CreateCall2(F, Base, Exponent));
+ return RValue::get(Builder.CreateCall(F, {Base, Exponent}));
}
case Builtin::BIfma:
@@ -1380,32 +1427,17 @@ RValue CodeGenFunction::EmitBuiltinExpr(const FunctionDecl *FD,
Value *FirstArg = EmitScalarExpr(E->getArg(0));
llvm::Type *ArgType = FirstArg->getType();
Value *F = CGM.getIntrinsic(Intrinsic::fma, ArgType);
- return RValue::get(Builder.CreateCall3(F, FirstArg,
- EmitScalarExpr(E->getArg(1)),
- EmitScalarExpr(E->getArg(2))));
+ return RValue::get(
+ Builder.CreateCall(F, {FirstArg, EmitScalarExpr(E->getArg(1)),
+ EmitScalarExpr(E->getArg(2))}));
}
case Builtin::BI__builtin_signbit:
case Builtin::BI__builtin_signbitf:
case Builtin::BI__builtin_signbitl: {
- LLVMContext &C = CGM.getLLVMContext();
-
- Value *Arg = EmitScalarExpr(E->getArg(0));
- llvm::Type *ArgTy = Arg->getType();
- int ArgWidth = ArgTy->getPrimitiveSizeInBits();
- llvm::Type *ArgIntTy = llvm::IntegerType::get(C, ArgWidth);
- Value *BCArg = Builder.CreateBitCast(Arg, ArgIntTy);
- if (ArgTy->isPPC_FP128Ty()) {
- // The higher-order double comes first, and so we need to truncate the
- // pair to extract the overall sign. The order of the pair is the same
- // in both little- and big-Endian modes.
- ArgWidth >>= 1;
- ArgIntTy = llvm::IntegerType::get(C, ArgWidth);
- BCArg = Builder.CreateTrunc(BCArg, ArgIntTy);
- }
- Value *ZeroCmp = llvm::Constant::getNullValue(ArgIntTy);
- Value *Result = Builder.CreateICmpSLT(BCArg, ZeroCmp);
- return RValue::get(Builder.CreateZExt(Result, ConvertType(E->getType())));
+ return RValue::get(
+ Builder.CreateZExt(EmitSignBit(*this, EmitScalarExpr(E->getArg(0))),
+ ConvertType(E->getType())));
}
case Builtin::BI__builtin_annotation: {
llvm::Value *AnnVal = EmitScalarExpr(E->getArg(0));
@@ -1650,6 +1682,76 @@ RValue CodeGenFunction::EmitBuiltinExpr(const FunctionDecl *FD,
Builder.CreateAlignedLoad(IntToPtr, /*Align=*/4, /*isVolatile=*/true);
return RValue::get(Load);
}
+
+ case Builtin::BI__exception_code:
+ case Builtin::BI_exception_code:
+ return RValue::get(EmitSEHExceptionCode());
+ case Builtin::BI__exception_info:
+ case Builtin::BI_exception_info:
+ return RValue::get(EmitSEHExceptionInfo());
+ case Builtin::BI__abnormal_termination:
+ case Builtin::BI_abnormal_termination:
+ return RValue::get(EmitSEHAbnormalTermination());
+ case Builtin::BI_setjmpex: {
+ if (getTarget().getTriple().isOSMSVCRT()) {
+ llvm::Type *ArgTypes[] = {Int8PtrTy, Int8PtrTy};
+ llvm::AttributeSet ReturnsTwiceAttr =
+ AttributeSet::get(getLLVMContext(), llvm::AttributeSet::FunctionIndex,
+ llvm::Attribute::ReturnsTwice);
+ llvm::Constant *SetJmpEx = CGM.CreateRuntimeFunction(
+ llvm::FunctionType::get(IntTy, ArgTypes, /*isVarArg=*/false),
+ "_setjmpex", ReturnsTwiceAttr);
+ llvm::Value *Buf = Builder.CreateBitOrPointerCast(
+ EmitScalarExpr(E->getArg(0)), Int8PtrTy);
+ llvm::Value *FrameAddr =
+ Builder.CreateCall(CGM.getIntrinsic(Intrinsic::frameaddress),
+ ConstantInt::get(Int32Ty, 0));
+ llvm::Value *Args[] = {Buf, FrameAddr};
+ llvm::CallSite CS = EmitRuntimeCallOrInvoke(SetJmpEx, Args);
+ CS.setAttributes(ReturnsTwiceAttr);
+ return RValue::get(CS.getInstruction());
+ }
+ break;
+ }
+ case Builtin::BI_setjmp: {
+ if (getTarget().getTriple().isOSMSVCRT()) {
+ llvm::AttributeSet ReturnsTwiceAttr =
+ AttributeSet::get(getLLVMContext(), llvm::AttributeSet::FunctionIndex,
+ llvm::Attribute::ReturnsTwice);
+ llvm::Value *Buf = Builder.CreateBitOrPointerCast(
+ EmitScalarExpr(E->getArg(0)), Int8PtrTy);
+ llvm::CallSite CS;
+ if (getTarget().getTriple().getArch() == llvm::Triple::x86) {
+ llvm::Type *ArgTypes[] = {Int8PtrTy, IntTy};
+ llvm::Constant *SetJmp3 = CGM.CreateRuntimeFunction(
+ llvm::FunctionType::get(IntTy, ArgTypes, /*isVarArg=*/true),
+ "_setjmp3", ReturnsTwiceAttr);
+ llvm::Value *Count = ConstantInt::get(IntTy, 0);
+ llvm::Value *Args[] = {Buf, Count};
+ CS = EmitRuntimeCallOrInvoke(SetJmp3, Args);
+ } else {
+ llvm::Type *ArgTypes[] = {Int8PtrTy, Int8PtrTy};
+ llvm::Constant *SetJmp = CGM.CreateRuntimeFunction(
+ llvm::FunctionType::get(IntTy, ArgTypes, /*isVarArg=*/false),
+ "_setjmp", ReturnsTwiceAttr);
+ llvm::Value *FrameAddr =
+ Builder.CreateCall(CGM.getIntrinsic(Intrinsic::frameaddress),
+ ConstantInt::get(Int32Ty, 0));
+ llvm::Value *Args[] = {Buf, FrameAddr};
+ CS = EmitRuntimeCallOrInvoke(SetJmp, Args);
+ }
+ CS.setAttributes(ReturnsTwiceAttr);
+ return RValue::get(CS.getInstruction());
+ }
+ break;
+ }
+
+ case Builtin::BI__GetExceptionInfo: {
+ if (llvm::GlobalVariable *GV =
+ CGM.getCXXABI().getThrowInfo(FD->getParamDecl(0)->getType()))
+ return RValue::get(llvm::ConstantExpr::getBitCast(GV, CGM.Int8PtrTy));
+ break;
+ }
}
// If this is an alias for a lib function (e.g. __builtin_sin), emit
@@ -1764,6 +1866,8 @@ Value *CodeGenFunction::EmitTargetBuiltinExpr(unsigned BuiltinID,
case llvm::Triple::r600:
case llvm::Triple::amdgcn:
return EmitR600BuiltinExpr(BuiltinID, E);
+ case llvm::Triple::systemz:
+ return EmitSystemZBuiltinExpr(BuiltinID, E);
default:
return nullptr;
}
@@ -2534,7 +2638,7 @@ Function *CodeGenFunction::LookupNeonLLVMIntrinsic(unsigned IntrinsicID,
// Return type.
SmallVector<llvm::Type *, 3> Tys;
if (Modifier & AddRetType) {
- llvm::Type *Ty = ConvertType(E->getCallReturnType());
+ llvm::Type *Ty = ConvertType(E->getCallReturnType(getContext()));
if (Modifier & VectorizeRetType)
Ty = llvm::VectorType::get(
Ty, VectorSize ? VectorSize / Ty->getPrimitiveSizeInBits() : 1);
@@ -2812,7 +2916,7 @@ Value *CodeGenFunction::EmitCommonNeonBuiltinExpr(
Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
// NEON intrinsic puts accumulator first, unlike the LLVM fma.
- return Builder.CreateCall3(F, Ops[1], Ops[2], Ops[0]);
+ return Builder.CreateCall(F, {Ops[1], Ops[2], Ops[0]});
}
case NEON::BI__builtin_neon_vld1_v:
case NEON::BI__builtin_neon_vld1q_v:
@@ -2825,7 +2929,7 @@ Value *CodeGenFunction::EmitCommonNeonBuiltinExpr(
case NEON::BI__builtin_neon_vld4_v:
case NEON::BI__builtin_neon_vld4q_v: {
Function *F = CGM.getIntrinsic(LLVMIntrinsic, Ty);
- Ops[1] = Builder.CreateCall2(F, Ops[1], Align, NameHint);
+ Ops[1] = Builder.CreateCall(F, {Ops[1], Align}, NameHint);
Ty = llvm::PointerType::getUnqual(Ops[1]->getType());
Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
return Builder.CreateStore(Ops[1], Ops[0]);
@@ -3004,7 +3108,7 @@ Value *CodeGenFunction::EmitCommonNeonBuiltinExpr(
Indices.push_back(Builder.getInt32(i+vi));
Indices.push_back(Builder.getInt32(i+e+vi));
}
- Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ops[0], vi);
+ Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi);
SV = llvm::ConstantVector::get(Indices);
SV = Builder.CreateShuffleVector(Ops[1], Ops[2], SV, "vtrn");
SV = Builder.CreateStore(SV, Addr);
@@ -3032,7 +3136,7 @@ Value *CodeGenFunction::EmitCommonNeonBuiltinExpr(
for (unsigned i = 0, e = VTy->getNumElements(); i != e; ++i)
Indices.push_back(ConstantInt::get(Int32Ty, 2*i+vi));
- Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ops[0], vi);
+ Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi);
SV = llvm::ConstantVector::get(Indices);
SV = Builder.CreateShuffleVector(Ops[1], Ops[2], SV, "vuzp");
SV = Builder.CreateStore(SV, Addr);
@@ -3052,7 +3156,7 @@ Value *CodeGenFunction::EmitCommonNeonBuiltinExpr(
Indices.push_back(ConstantInt::get(Int32Ty, (i + vi*e) >> 1));
Indices.push_back(ConstantInt::get(Int32Ty, ((i + vi*e) >> 1)+e));
}
- Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ops[0], vi);
+ Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi);
SV = llvm::ConstantVector::get(Indices);
SV = Builder.CreateShuffleVector(Ops[1], Ops[2], SV, "vzip");
SV = Builder.CreateStore(SV, Addr);
@@ -3185,7 +3289,7 @@ Value *CodeGenFunction::EmitARMBuiltinExpr(unsigned BuiltinID,
: InlineAsm::get(FTy, ".inst 0x" + utohexstr(ZExtValue), "",
/*SideEffects=*/true);
- return Builder.CreateCall(Emit);
+ return Builder.CreateCall(Emit, {});
}
if (BuiltinID == ARM::BI__builtin_arm_dbg) {
@@ -3202,7 +3306,7 @@ Value *CodeGenFunction::EmitARMBuiltinExpr(unsigned BuiltinID,
Value *Locality = llvm::ConstantInt::get(Int32Ty, 3);
Value *F = CGM.getIntrinsic(Intrinsic::prefetch);
- return Builder.CreateCall4(F, Address, RW, Locality, IsData);
+ return Builder.CreateCall(F, {Address, RW, Locality, IsData});
}
if (BuiltinID == ARM::BI__builtin_arm_rbit) {
@@ -3300,7 +3404,7 @@ Value *CodeGenFunction::EmitARMBuiltinExpr(unsigned BuiltinID,
Value *Arg0 = Builder.CreateExtractValue(Val, 0);
Value *Arg1 = Builder.CreateExtractValue(Val, 1);
Value *StPtr = Builder.CreateBitCast(EmitScalarExpr(E->getArg(1)), Int8PtrTy);
- return Builder.CreateCall3(F, Arg0, Arg1, StPtr, "strexd");
+ return Builder.CreateCall(F, {Arg0, Arg1, StPtr}, "strexd");
}
if (BuiltinID == ARM::BI__builtin_arm_strex ||
@@ -3324,12 +3428,12 @@ Value *CodeGenFunction::EmitARMBuiltinExpr(unsigned BuiltinID,
? Intrinsic::arm_stlex
: Intrinsic::arm_strex,
StoreAddr->getType());
- return Builder.CreateCall2(F, StoreVal, StoreAddr, "strex");
+ return Builder.CreateCall(F, {StoreVal, StoreAddr}, "strex");
}
if (BuiltinID == ARM::BI__builtin_arm_clrex) {
Function *F = CGM.getIntrinsic(Intrinsic::arm_clrex);
- return Builder.CreateCall(F);
+ return Builder.CreateCall(F, {});
}
// CRC32
@@ -3365,13 +3469,13 @@ Value *CodeGenFunction::EmitARMBuiltinExpr(unsigned BuiltinID,
Arg1b = Builder.CreateTruncOrBitCast(Arg1b, Int32Ty);
Function *F = CGM.getIntrinsic(CRCIntrinsicID);
- Value *Res = Builder.CreateCall2(F, Arg0, Arg1a);
- return Builder.CreateCall2(F, Res, Arg1b);
+ Value *Res = Builder.CreateCall(F, {Arg0, Arg1a});
+ return Builder.CreateCall(F, {Res, Arg1b});
} else {
Arg1 = Builder.CreateZExtOrBitCast(Arg1, Int32Ty);
Function *F = CGM.getIntrinsic(CRCIntrinsicID);
- return Builder.CreateCall2(F, Arg0, Arg1);
+ return Builder.CreateCall(F, {Arg0, Arg1});
}
}
@@ -3547,7 +3651,7 @@ Value *CodeGenFunction::EmitARMBuiltinExpr(unsigned BuiltinID,
// Load the value as a one-element vector.
Ty = llvm::VectorType::get(VTy->getElementType(), 1);
Function *F = CGM.getIntrinsic(Intrinsic::arm_neon_vld1, Ty);
- Value *Ld = Builder.CreateCall2(F, Ops[0], Align);
+ Value *Ld = Builder.CreateCall(F, {Ops[0], Align});
// Combine them.
SmallVector<Constant*, 2> Indices;
Indices.push_back(ConstantInt::get(Int32Ty, 1-Lane));
@@ -3582,7 +3686,7 @@ Value *CodeGenFunction::EmitARMBuiltinExpr(unsigned BuiltinID,
default: llvm_unreachable("unknown vld_dup intrinsic?");
}
Function *F = CGM.getIntrinsic(Int, Ty);
- Ops[1] = Builder.CreateCall2(F, Ops[1], Align, "vld_dup");
+ Ops[1] = Builder.CreateCall(F, {Ops[1], Align}, "vld_dup");
Ty = llvm::PointerType::getUnqual(Ops[1]->getType());
Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
return Builder.CreateStore(Ops[1], Ops[0]);
@@ -3651,7 +3755,7 @@ Value *CodeGenFunction::EmitARMBuiltinExpr(unsigned BuiltinID,
Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
Ops[2] = EmitNeonShiftVector(Ops[2], Ty, true);
Int = usgn ? Intrinsic::arm_neon_vrshiftu : Intrinsic::arm_neon_vrshifts;
- Ops[1] = Builder.CreateCall2(CGM.getIntrinsic(Int, Ty), Ops[1], Ops[2]);
+ Ops[1] = Builder.CreateCall(CGM.getIntrinsic(Int, Ty), {Ops[1], Ops[2]});
return Builder.CreateAdd(Ops[0], Ops[1], "vrsra_n");
case NEON::BI__builtin_neon_vsri_n_v:
case NEON::BI__builtin_neon_vsriq_n_v:
@@ -3979,7 +4083,7 @@ Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID,
// FIXME: We need AArch64 specific LLVM intrinsic if we want to specify
// PLDL3STRM or PLDL2STRM.
Value *F = CGM.getIntrinsic(Intrinsic::prefetch);
- return Builder.CreateCall4(F, Address, RW, Locality, IsData);
+ return Builder.CreateCall(F, {Address, RW, Locality, IsData});
}
if (BuiltinID == AArch64::BI__builtin_arm_rbit) {
@@ -4074,9 +4178,11 @@ Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID,
Value *Arg1 = Builder.CreateExtractValue(Val, 1);
Value *StPtr = Builder.CreateBitCast(EmitScalarExpr(E->getArg(1)),
Int8PtrTy);
- return Builder.CreateCall3(F, Arg0, Arg1, StPtr, "stxp");
- } else if (BuiltinID == AArch64::BI__builtin_arm_strex ||
- BuiltinID == AArch64::BI__builtin_arm_stlex) {
+ return Builder.CreateCall(F, {Arg0, Arg1, StPtr}, "stxp");
+ }
+
+ if (BuiltinID == AArch64::BI__builtin_arm_strex ||
+ BuiltinID == AArch64::BI__builtin_arm_stlex) {
Value *StoreVal = EmitScalarExpr(E->getArg(0));
Value *StoreAddr = EmitScalarExpr(E->getArg(1));
@@ -4096,12 +4202,12 @@ Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID,
? Intrinsic::aarch64_stlxr
: Intrinsic::aarch64_stxr,
StoreAddr->getType());
- return Builder.CreateCall2(F, StoreVal, StoreAddr, "stxr");
+ return Builder.CreateCall(F, {StoreVal, StoreAddr}, "stxr");
}
if (BuiltinID == AArch64::BI__builtin_arm_clrex) {
Function *F = CGM.getIntrinsic(Intrinsic::aarch64_clrex);
- return Builder.CreateCall(F);
+ return Builder.CreateCall(F, {});
}
// CRC32
@@ -4133,7 +4239,7 @@ Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID,
llvm::Type *DataTy = F->getFunctionType()->getParamType(1);
Arg1 = Builder.CreateZExtOrBitCast(Arg1, DataTy);
- return Builder.CreateCall2(F, Arg0, Arg1);
+ return Builder.CreateCall(F, {Arg0, Arg1});
}
llvm::SmallVector<Value*, 4> Ops;
@@ -4248,36 +4354,36 @@ Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID,
case NEON::BI__builtin_neon_vceqzs_f32:
Ops.push_back(EmitScalarExpr(E->getArg(0)));
return EmitAArch64CompareBuiltinExpr(
- Ops[0], ConvertType(E->getCallReturnType()), ICmpInst::FCMP_OEQ,
- ICmpInst::ICMP_EQ, "vceqz");
+ Ops[0], ConvertType(E->getCallReturnType(getContext())),
+ ICmpInst::FCMP_OEQ, ICmpInst::ICMP_EQ, "vceqz");
case NEON::BI__builtin_neon_vcgezd_s64:
case NEON::BI__builtin_neon_vcgezd_f64:
case NEON::BI__builtin_neon_vcgezs_f32:
Ops.push_back(EmitScalarExpr(E->getArg(0)));
return EmitAArch64CompareBuiltinExpr(
- Ops[0], ConvertType(E->getCallReturnType()), ICmpInst::FCMP_OGE,
- ICmpInst::ICMP_SGE, "vcgez");
+ Ops[0], ConvertType(E->getCallReturnType(getContext())),
+ ICmpInst::FCMP_OGE, ICmpInst::ICMP_SGE, "vcgez");
case NEON::BI__builtin_neon_vclezd_s64:
case NEON::BI__builtin_neon_vclezd_f64:
case NEON::BI__builtin_neon_vclezs_f32:
Ops.push_back(EmitScalarExpr(E->getArg(0)));
return EmitAArch64CompareBuiltinExpr(
- Ops[0], ConvertType(E->getCallReturnType()), ICmpInst::FCMP_OLE,
- ICmpInst::ICMP_SLE, "vclez");
+ Ops[0], ConvertType(E->getCallReturnType(getContext())),
+ ICmpInst::FCMP_OLE, ICmpInst::ICMP_SLE, "vclez");
case NEON::BI__builtin_neon_vcgtzd_s64:
case NEON::BI__builtin_neon_vcgtzd_f64:
case NEON::BI__builtin_neon_vcgtzs_f32:
Ops.push_back(EmitScalarExpr(E->getArg(0)));
return EmitAArch64CompareBuiltinExpr(
- Ops[0], ConvertType(E->getCallReturnType()), ICmpInst::FCMP_OGT,
- ICmpInst::ICMP_SGT, "vcgtz");
+ Ops[0], ConvertType(E->getCallReturnType(getContext())),
+ ICmpInst::FCMP_OGT, ICmpInst::ICMP_SGT, "vcgtz");
case NEON::BI__builtin_neon_vcltzd_s64:
case NEON::BI__builtin_neon_vcltzd_f64:
case NEON::BI__builtin_neon_vcltzs_f32:
Ops.push_back(EmitScalarExpr(E->getArg(0)));
return EmitAArch64CompareBuiltinExpr(
- Ops[0], ConvertType(E->getCallReturnType()), ICmpInst::FCMP_OLT,
- ICmpInst::ICMP_SLT, "vcltz");
+ Ops[0], ConvertType(E->getCallReturnType(getContext())),
+ ICmpInst::FCMP_OLT, ICmpInst::ICMP_SLT, "vcltz");
case NEON::BI__builtin_neon_vceqzd_u64: {
llvm::Type *Ty = llvm::Type::getInt64Ty(getLLVMContext());
@@ -4528,8 +4634,8 @@ Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID,
: Intrinsic::aarch64_neon_srshl;
Ops[1] = Builder.CreateBitCast(Ops[1], Int64Ty);
Ops.push_back(Builder.CreateNeg(EmitScalarExpr(E->getArg(2))));
- Ops[1] = Builder.CreateCall2(CGM.getIntrinsic(Int, Int64Ty), Ops[1],
- Builder.CreateSExt(Ops[2], Int64Ty));
+ Ops[1] = Builder.CreateCall(CGM.getIntrinsic(Int, Int64Ty),
+ {Ops[1], Builder.CreateSExt(Ops[2], Int64Ty)});
return Builder.CreateAdd(Ops[0], Builder.CreateBitCast(Ops[1], Int64Ty));
}
case NEON::BI__builtin_neon_vshld_n_s64:
@@ -4699,7 +4805,7 @@ Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID,
Ops[2] = Builder.CreateBitCast(Ops[2], VTy);
Ops[2] = Builder.CreateExtractElement(Ops[2], Ops[3], "extract");
Value *F = CGM.getIntrinsic(Intrinsic::fma, DoubleTy);
- Value *Result = Builder.CreateCall3(F, Ops[1], Ops[2], Ops[0]);
+ Value *Result = Builder.CreateCall(F, {Ops[1], Ops[2], Ops[0]});
return Builder.CreateBitCast(Result, Ty);
}
Value *F = CGM.getIntrinsic(Intrinsic::fma, Ty);
@@ -4713,7 +4819,7 @@ Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID,
cast<ConstantInt>(Ops[3]));
Ops[2] = Builder.CreateShuffleVector(Ops[2], Ops[2], SV, "lane");
- return Builder.CreateCall3(F, Ops[2], Ops[1], Ops[0]);
+ return Builder.CreateCall(F, {Ops[2], Ops[1], Ops[0]});
}
case NEON::BI__builtin_neon_vfmaq_laneq_v: {
Value *F = CGM.getIntrinsic(Intrinsic::fma, Ty);
@@ -4722,17 +4828,17 @@ Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID,
Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
Ops[2] = EmitNeonSplat(Ops[2], cast<ConstantInt>(Ops[3]));
- return Builder.CreateCall3(F, Ops[2], Ops[1], Ops[0]);
+ return Builder.CreateCall(F, {Ops[2], Ops[1], Ops[0]});
}
case NEON::BI__builtin_neon_vfmas_lane_f32:
case NEON::BI__builtin_neon_vfmas_laneq_f32:
case NEON::BI__builtin_neon_vfmad_lane_f64:
case NEON::BI__builtin_neon_vfmad_laneq_f64: {
Ops.push_back(EmitScalarExpr(E->getArg(3)));
- llvm::Type *Ty = ConvertType(E->getCallReturnType());
+ llvm::Type *Ty = ConvertType(E->getCallReturnType(getContext()));
Value *F = CGM.getIntrinsic(Intrinsic::fma, Ty);
Ops[2] = Builder.CreateExtractElement(Ops[2], Ops[3], "extract");
- return Builder.CreateCall3(F, Ops[1], Ops[2], Ops[0]);
+ return Builder.CreateCall(F, {Ops[1], Ops[2], Ops[0]});
}
case NEON::BI__builtin_neon_vfms_v:
case NEON::BI__builtin_neon_vfmsq_v: { // Only used for FP types
@@ -5667,7 +5773,7 @@ Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID,
Indices.push_back(ConstantInt::get(Int32Ty, i+vi));
Indices.push_back(ConstantInt::get(Int32Ty, i+e+vi));
}
- Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ops[0], vi);
+ Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi);
SV = llvm::ConstantVector::get(Indices);
SV = Builder.CreateShuffleVector(Ops[1], Ops[2], SV, "vtrn");
SV = Builder.CreateStore(SV, Addr);
@@ -5686,7 +5792,7 @@ Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID,
for (unsigned i = 0, e = VTy->getNumElements(); i != e; ++i)
Indices.push_back(ConstantInt::get(Int32Ty, 2*i+vi));
- Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ops[0], vi);
+ Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi);
SV = llvm::ConstantVector::get(Indices);
SV = Builder.CreateShuffleVector(Ops[1], Ops[2], SV, "vuzp");
SV = Builder.CreateStore(SV, Addr);
@@ -5706,7 +5812,7 @@ Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID,
Indices.push_back(ConstantInt::get(Int32Ty, (i + vi*e) >> 1));
Indices.push_back(ConstantInt::get(Int32Ty, ((i + vi*e) >> 1)+e));
}
- Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ops[0], vi);
+ Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi);
SV = llvm::ConstantVector::get(Indices);
SV = Builder.CreateShuffleVector(Ops[1], Ops[2], SV, "vzip");
SV = Builder.CreateStore(SV, Addr);
@@ -5817,7 +5923,7 @@ Value *CodeGenFunction::EmitX86BuiltinExpr(unsigned BuiltinID,
Value *Locality = EmitScalarExpr(E->getArg(1));
Value *Data = ConstantInt::get(Int32Ty, 1);
Value *F = CGM.getIntrinsic(Intrinsic::prefetch);
- return Builder.CreateCall4(F, Address, RW, Locality, Data);
+ return Builder.CreateCall(F, {Address, RW, Locality, Data});
}
case X86::BI__builtin_ia32_vec_init_v8qi:
case X86::BI__builtin_ia32_vec_init_v4hi:
@@ -5856,104 +5962,95 @@ Value *CodeGenFunction::EmitX86BuiltinExpr(unsigned BuiltinID,
Ops[0] = Builder.CreateBitCast(Ops[0], PtrTy);
return Builder.CreateStore(Ops[1], Ops[0]);
}
- case X86::BI__builtin_ia32_palignr: {
- unsigned shiftVal = cast<llvm::ConstantInt>(Ops[2])->getZExtValue();
-
- // If palignr is shifting the pair of input vectors less than 9 bytes,
- // emit a shuffle instruction.
- if (shiftVal <= 8) {
- SmallVector<llvm::Constant*, 8> Indices;
- for (unsigned i = 0; i != 8; ++i)
- Indices.push_back(llvm::ConstantInt::get(Int32Ty, shiftVal + i));
-
- Value* SV = llvm::ConstantVector::get(Indices);
- return Builder.CreateShuffleVector(Ops[1], Ops[0], SV, "palignr");
+ case X86::BI__builtin_ia32_palignr128:
+ case X86::BI__builtin_ia32_palignr256: {
+ unsigned ShiftVal = cast<llvm::ConstantInt>(Ops[2])->getZExtValue();
+
+ unsigned NumElts =
+ cast<llvm::VectorType>(Ops[0]->getType())->getNumElements();
+ assert(NumElts % 16 == 0);
+ unsigned NumLanes = NumElts / 16;
+ unsigned NumLaneElts = NumElts / NumLanes;
+
+ // If palignr is shifting the pair of vectors more than the size of two
+ // lanes, emit zero.
+ if (ShiftVal >= (2 * NumLaneElts))
+ return llvm::Constant::getNullValue(ConvertType(E->getType()));
+
+ // If palignr is shifting the pair of input vectors more than one lane,
+ // but less than two lanes, convert to shifting in zeroes.
+ if (ShiftVal > NumLaneElts) {
+ ShiftVal -= NumLaneElts;
+ Ops[0] = llvm::Constant::getNullValue(Ops[0]->getType());
}
- // If palignr is shifting the pair of input vectors more than 8 but less
- // than 16 bytes, emit a logical right shift of the destination.
- if (shiftVal < 16) {
- // MMX has these as 1 x i64 vectors for some odd optimization reasons.
- llvm::Type *VecTy = llvm::VectorType::get(Int64Ty, 1);
-
- Ops[0] = Builder.CreateBitCast(Ops[0], VecTy, "cast");
- Ops[1] = llvm::ConstantInt::get(VecTy, (shiftVal-8) * 8);
-
- // create i32 constant
- llvm::Function *F = CGM.getIntrinsic(Intrinsic::x86_mmx_psrl_q);
- return Builder.CreateCall(F, makeArrayRef(Ops.data(), 2), "palignr");
+ SmallVector<llvm::Constant*, 32> Indices;
+ // 256-bit palignr operates on 128-bit lanes so we need to handle that
+ for (unsigned l = 0; l != NumElts; l += NumLaneElts) {
+ for (unsigned i = 0; i != NumLaneElts; ++i) {
+ unsigned Idx = ShiftVal + i;
+ if (Idx >= NumLaneElts)
+ Idx += NumElts - NumLaneElts; // End of lane, switch operand.
+ Indices.push_back(llvm::ConstantInt::get(Int32Ty, Idx + l));
+ }
}
- // If palignr is shifting the pair of vectors more than 16 bytes, emit zero.
- return llvm::Constant::getNullValue(ConvertType(E->getType()));
+ Value* SV = llvm::ConstantVector::get(Indices);
+ return Builder.CreateShuffleVector(Ops[1], Ops[0], SV, "palignr");
}
- case X86::BI__builtin_ia32_palignr128: {
- unsigned shiftVal = cast<llvm::ConstantInt>(Ops[2])->getZExtValue();
+ case X86::BI__builtin_ia32_pslldqi256: {
+ // Shift value is in bits so divide by 8.
+ unsigned shiftVal = cast<llvm::ConstantInt>(Ops[1])->getZExtValue() >> 3;
- // If palignr is shifting the pair of input vectors less than 17 bytes,
- // emit a shuffle instruction.
- if (shiftVal <= 16) {
- SmallVector<llvm::Constant*, 16> Indices;
- for (unsigned i = 0; i != 16; ++i)
- Indices.push_back(llvm::ConstantInt::get(Int32Ty, shiftVal + i));
+ // If pslldq is shifting the vector more than 15 bytes, emit zero.
+ if (shiftVal >= 16)
+ return llvm::Constant::getNullValue(ConvertType(E->getType()));
- Value* SV = llvm::ConstantVector::get(Indices);
- return Builder.CreateShuffleVector(Ops[1], Ops[0], SV, "palignr");
+ SmallVector<llvm::Constant*, 32> Indices;
+ // 256-bit pslldq operates on 128-bit lanes so we need to handle that
+ for (unsigned l = 0; l != 32; l += 16) {
+ for (unsigned i = 0; i != 16; ++i) {
+ unsigned Idx = 32 + i - shiftVal;
+ if (Idx < 32) Idx -= 16; // end of lane, switch operand.
+ Indices.push_back(llvm::ConstantInt::get(Int32Ty, Idx + l));
+ }
}
- // If palignr is shifting the pair of input vectors more than 16 but less
- // than 32 bytes, emit a logical right shift of the destination.
- if (shiftVal < 32) {
- llvm::Type *VecTy = llvm::VectorType::get(Int64Ty, 2);
-
- Ops[0] = Builder.CreateBitCast(Ops[0], VecTy, "cast");
- Ops[1] = llvm::ConstantInt::get(Int32Ty, (shiftVal-16) * 8);
-
- // create i32 constant
- llvm::Function *F = CGM.getIntrinsic(Intrinsic::x86_sse2_psrl_dq);
- return Builder.CreateCall(F, makeArrayRef(Ops.data(), 2), "palignr");
- }
+ llvm::Type *VecTy = llvm::VectorType::get(Int8Ty, 32);
+ Ops[0] = Builder.CreateBitCast(Ops[0], VecTy, "cast");
+ Value *Zero = llvm::Constant::getNullValue(VecTy);
- // If palignr is shifting the pair of vectors more than 32 bytes, emit zero.
- return llvm::Constant::getNullValue(ConvertType(E->getType()));
- }
- case X86::BI__builtin_ia32_palignr256: {
- unsigned shiftVal = cast<llvm::ConstantInt>(Ops[2])->getZExtValue();
-
- // If palignr is shifting the pair of input vectors less than 17 bytes,
- // emit a shuffle instruction.
- if (shiftVal <= 16) {
- SmallVector<llvm::Constant*, 32> Indices;
- // 256-bit palignr operates on 128-bit lanes so we need to handle that
- for (unsigned l = 0; l != 2; ++l) {
- unsigned LaneStart = l * 16;
- unsigned LaneEnd = (l+1) * 16;
- for (unsigned i = 0; i != 16; ++i) {
- unsigned Idx = shiftVal + i + LaneStart;
- if (Idx >= LaneEnd) Idx += 16; // end of lane, switch operand
- Indices.push_back(llvm::ConstantInt::get(Int32Ty, Idx));
- }
+ Value *SV = llvm::ConstantVector::get(Indices);
+ SV = Builder.CreateShuffleVector(Zero, Ops[0], SV, "pslldq");
+ llvm::Type *ResultType = ConvertType(E->getType());
+ return Builder.CreateBitCast(SV, ResultType, "cast");
+ }
+ case X86::BI__builtin_ia32_psrldqi256: {
+ // Shift value is in bits so divide by 8.
+ unsigned shiftVal = cast<llvm::ConstantInt>(Ops[1])->getZExtValue() >> 3;
+
+ // If psrldq is shifting the vector more than 15 bytes, emit zero.
+ if (shiftVal >= 16)
+ return llvm::Constant::getNullValue(ConvertType(E->getType()));
+
+ SmallVector<llvm::Constant*, 32> Indices;
+ // 256-bit psrldq operates on 128-bit lanes so we need to handle that
+ for (unsigned l = 0; l != 32; l += 16) {
+ for (unsigned i = 0; i != 16; ++i) {
+ unsigned Idx = i + shiftVal;
+ if (Idx >= 16) Idx += 16; // end of lane, switch operand.
+ Indices.push_back(llvm::ConstantInt::get(Int32Ty, Idx + l));
}
-
- Value* SV = llvm::ConstantVector::get(Indices);
- return Builder.CreateShuffleVector(Ops[1], Ops[0], SV, "palignr");
}
- // If palignr is shifting the pair of input vectors more than 16 but less
- // than 32 bytes, emit a logical right shift of the destination.
- if (shiftVal < 32) {
- llvm::Type *VecTy = llvm::VectorType::get(Int64Ty, 4);
-
- Ops[0] = Builder.CreateBitCast(Ops[0], VecTy, "cast");
- Ops[1] = llvm::ConstantInt::get(Int32Ty, (shiftVal-16) * 8);
-
- // create i32 constant
- llvm::Function *F = CGM.getIntrinsic(Intrinsic::x86_avx2_psrl_dq);
- return Builder.CreateCall(F, makeArrayRef(Ops.data(), 2), "palignr");
- }
+ llvm::Type *VecTy = llvm::VectorType::get(Int8Ty, 32);
+ Ops[0] = Builder.CreateBitCast(Ops[0], VecTy, "cast");
+ Value *Zero = llvm::Constant::getNullValue(VecTy);
- // If palignr is shifting the pair of vectors more than 32 bytes, emit zero.
- return llvm::Constant::getNullValue(ConvertType(E->getType()));
+ Value *SV = llvm::ConstantVector::get(Indices);
+ SV = Builder.CreateShuffleVector(Ops[0], Zero, SV, "psrldq");
+ llvm::Type *ResultType = ConvertType(E->getType());
+ return Builder.CreateBitCast(SV, ResultType, "cast");
}
case X86::BI__builtin_ia32_movntps:
case X86::BI__builtin_ia32_movntps256:
@@ -5987,20 +6084,10 @@ Value *CodeGenFunction::EmitX86BuiltinExpr(unsigned BuiltinID,
// 3DNow!
case X86::BI__builtin_ia32_pswapdsf:
case X86::BI__builtin_ia32_pswapdsi: {
- const char *name;
- Intrinsic::ID ID;
- switch(BuiltinID) {
- default: llvm_unreachable("Unsupported intrinsic!");
- case X86::BI__builtin_ia32_pswapdsf:
- case X86::BI__builtin_ia32_pswapdsi:
- name = "pswapd";
- ID = Intrinsic::x86_3dnowa_pswapd;
- break;
- }
llvm::Type *MMXTy = llvm::Type::getX86_MMXTy(getLLVMContext());
Ops[0] = Builder.CreateBitCast(Ops[0], MMXTy, "cast");
- llvm::Function *F = CGM.getIntrinsic(ID);
- return Builder.CreateCall(F, Ops, name);
+ llvm::Function *F = CGM.getIntrinsic(Intrinsic::x86_3dnowa_pswapd);
+ return Builder.CreateCall(F, Ops, "pswapd");
}
case X86::BI__builtin_ia32_rdrand16_step:
case X86::BI__builtin_ia32_rdrand32_step:
@@ -6031,17 +6118,10 @@ Value *CodeGenFunction::EmitX86BuiltinExpr(unsigned BuiltinID,
break;
}
- Value *Call = Builder.CreateCall(CGM.getIntrinsic(ID));
+ Value *Call = Builder.CreateCall(CGM.getIntrinsic(ID), {});
Builder.CreateStore(Builder.CreateExtractValue(Call, 0), Ops[0]);
return Builder.CreateExtractValue(Call, 1);
}
- // AVX2 broadcast
- case X86::BI__builtin_ia32_vbroadcastsi256: {
- Value *VecTmp = CreateMemTemp(E->getArg(0)->getType());
- Builder.CreateStore(Ops[0], VecTmp);
- Value *F = CGM.getIntrinsic(Intrinsic::x86_avx2_vbroadcasti128);
- return Builder.CreateCall(F, Builder.CreateBitCast(VecTmp, Int8PtrTy));
- }
// SSE comparison intrisics
case X86::BI__builtin_ia32_cmpeqps:
case X86::BI__builtin_ia32_cmpltps:
@@ -6318,7 +6398,7 @@ static Value *emitTernaryFPBuiltin(CodeGenFunction &CGF,
llvm::Value *Src2 = CGF.EmitScalarExpr(E->getArg(2));
Value *F = CGF.CGM.getIntrinsic(IntrinsicID, Src0->getType());
- return CGF.Builder.CreateCall3(F, Src0, Src1, Src2);
+ return CGF.Builder.CreateCall(F, {Src0, Src1, Src2});
}
// Emit an intrinsic that has 1 float or double operand, and 1 integer.
@@ -6329,7 +6409,7 @@ static Value *emitFPIntBuiltin(CodeGenFunction &CGF,
llvm::Value *Src1 = CGF.EmitScalarExpr(E->getArg(1));
Value *F = CGF.CGM.getIntrinsic(IntrinsicID, Src0->getType());
- return CGF.Builder.CreateCall2(F, Src0, Src1);
+ return CGF.Builder.CreateCall(F, {Src0, Src1});
}
Value *CodeGenFunction::EmitR600BuiltinExpr(unsigned BuiltinID,
@@ -6350,7 +6430,7 @@ Value *CodeGenFunction::EmitR600BuiltinExpr(unsigned BuiltinID,
llvm::Value *Callee = CGM.getIntrinsic(Intrinsic::AMDGPU_div_scale,
X->getType());
- llvm::Value *Tmp = Builder.CreateCall3(Callee, X, Y, Z);
+ llvm::Value *Tmp = Builder.CreateCall(Callee, {X, Y, Z});
llvm::Value *Result = Builder.CreateExtractValue(Tmp, 0);
llvm::Value *Flag = Builder.CreateExtractValue(Tmp, 1);
@@ -6373,7 +6453,7 @@ Value *CodeGenFunction::EmitR600BuiltinExpr(unsigned BuiltinID,
llvm::Value *F = CGM.getIntrinsic(Intrinsic::AMDGPU_div_fmas,
Src0->getType());
llvm::Value *Src3ToBool = Builder.CreateIsNotNull(Src3);
- return Builder.CreateCall4(F, Src0, Src1, Src2, Src3ToBool);
+ return Builder.CreateCall(F, {Src0, Src1, Src2, Src3ToBool});
}
case R600::BI__builtin_amdgpu_div_fixup:
case R600::BI__builtin_amdgpu_div_fixupf:
@@ -6400,3 +6480,246 @@ Value *CodeGenFunction::EmitR600BuiltinExpr(unsigned BuiltinID,
return nullptr;
}
}
+
+/// Handle a SystemZ function in which the final argument is a pointer
+/// to an int that receives the post-instruction CC value. At the LLVM level
+/// this is represented as a function that returns a {result, cc} pair.
+static Value *EmitSystemZIntrinsicWithCC(CodeGenFunction &CGF,
+ unsigned IntrinsicID,
+ const CallExpr *E) {
+ unsigned NumArgs = E->getNumArgs() - 1;
+ SmallVector<Value *, 8> Args(NumArgs);
+ for (unsigned I = 0; I < NumArgs; ++I)
+ Args[I] = CGF.EmitScalarExpr(E->getArg(I));
+ Value *CCPtr = CGF.EmitScalarExpr(E->getArg(NumArgs));
+ Value *F = CGF.CGM.getIntrinsic(IntrinsicID);
+ Value *Call = CGF.Builder.CreateCall(F, Args);
+ Value *CC = CGF.Builder.CreateExtractValue(Call, 1);
+ CGF.Builder.CreateStore(CC, CCPtr);
+ return CGF.Builder.CreateExtractValue(Call, 0);
+}
+
+Value *CodeGenFunction::EmitSystemZBuiltinExpr(unsigned BuiltinID,
+ const CallExpr *E) {
+ switch (BuiltinID) {
+ case SystemZ::BI__builtin_tbegin: {
+ Value *TDB = EmitScalarExpr(E->getArg(0));
+ Value *Control = llvm::ConstantInt::get(Int32Ty, 0xff0c);
+ Value *F = CGM.getIntrinsic(Intrinsic::s390_tbegin);
+ return Builder.CreateCall(F, {TDB, Control});
+ }
+ case SystemZ::BI__builtin_tbegin_nofloat: {
+ Value *TDB = EmitScalarExpr(E->getArg(0));
+ Value *Control = llvm::ConstantInt::get(Int32Ty, 0xff0c);
+ Value *F = CGM.getIntrinsic(Intrinsic::s390_tbegin_nofloat);
+ return Builder.CreateCall(F, {TDB, Control});
+ }
+ case SystemZ::BI__builtin_tbeginc: {
+ Value *TDB = llvm::ConstantPointerNull::get(Int8PtrTy);
+ Value *Control = llvm::ConstantInt::get(Int32Ty, 0xff08);
+ Value *F = CGM.getIntrinsic(Intrinsic::s390_tbeginc);
+ return Builder.CreateCall(F, {TDB, Control});
+ }
+ case SystemZ::BI__builtin_tabort: {
+ Value *Data = EmitScalarExpr(E->getArg(0));
+ Value *F = CGM.getIntrinsic(Intrinsic::s390_tabort);
+ return Builder.CreateCall(F, Builder.CreateSExt(Data, Int64Ty, "tabort"));
+ }
+ case SystemZ::BI__builtin_non_tx_store: {
+ Value *Address = EmitScalarExpr(E->getArg(0));
+ Value *Data = EmitScalarExpr(E->getArg(1));
+ Value *F = CGM.getIntrinsic(Intrinsic::s390_ntstg);
+ return Builder.CreateCall(F, {Data, Address});
+ }
+
+ // Vector builtins. Note that most vector builtins are mapped automatically
+ // to target-specific LLVM intrinsics. The ones handled specially here can
+ // be represented via standard LLVM IR, which is preferable to enable common
+ // LLVM optimizations.
+
+ case SystemZ::BI__builtin_s390_vpopctb:
+ case SystemZ::BI__builtin_s390_vpopcth:
+ case SystemZ::BI__builtin_s390_vpopctf:
+ case SystemZ::BI__builtin_s390_vpopctg: {
+ llvm::Type *ResultType = ConvertType(E->getType());
+ Value *X = EmitScalarExpr(E->getArg(0));
+ Function *F = CGM.getIntrinsic(Intrinsic::ctpop, ResultType);
+ return Builder.CreateCall(F, X);
+ }
+
+ case SystemZ::BI__builtin_s390_vclzb:
+ case SystemZ::BI__builtin_s390_vclzh:
+ case SystemZ::BI__builtin_s390_vclzf:
+ case SystemZ::BI__builtin_s390_vclzg: {
+ llvm::Type *ResultType = ConvertType(E->getType());
+ Value *X = EmitScalarExpr(E->getArg(0));
+ Value *Undef = ConstantInt::get(Builder.getInt1Ty(), false);
+ Function *F = CGM.getIntrinsic(Intrinsic::ctlz, ResultType);
+ return Builder.CreateCall(F, {X, Undef});
+ }
+
+ case SystemZ::BI__builtin_s390_vctzb:
+ case SystemZ::BI__builtin_s390_vctzh:
+ case SystemZ::BI__builtin_s390_vctzf:
+ case SystemZ::BI__builtin_s390_vctzg: {
+ llvm::Type *ResultType = ConvertType(E->getType());
+ Value *X = EmitScalarExpr(E->getArg(0));
+ Value *Undef = ConstantInt::get(Builder.getInt1Ty(), false);
+ Function *F = CGM.getIntrinsic(Intrinsic::cttz, ResultType);
+ return Builder.CreateCall(F, {X, Undef});
+ }
+
+ case SystemZ::BI__builtin_s390_vfsqdb: {
+ llvm::Type *ResultType = ConvertType(E->getType());
+ Value *X = EmitScalarExpr(E->getArg(0));
+ Function *F = CGM.getIntrinsic(Intrinsic::sqrt, ResultType);
+ return Builder.CreateCall(F, X);
+ }
+ case SystemZ::BI__builtin_s390_vfmadb: {
+ llvm::Type *ResultType = ConvertType(E->getType());
+ Value *X = EmitScalarExpr(E->getArg(0));
+ Value *Y = EmitScalarExpr(E->getArg(1));
+ Value *Z = EmitScalarExpr(E->getArg(2));
+ Function *F = CGM.getIntrinsic(Intrinsic::fma, ResultType);
+ return Builder.CreateCall(F, {X, Y, Z});
+ }
+ case SystemZ::BI__builtin_s390_vfmsdb: {
+ llvm::Type *ResultType = ConvertType(E->getType());
+ Value *X = EmitScalarExpr(E->getArg(0));
+ Value *Y = EmitScalarExpr(E->getArg(1));
+ Value *Z = EmitScalarExpr(E->getArg(2));
+ Value *Zero = llvm::ConstantFP::getZeroValueForNegation(ResultType);
+ Function *F = CGM.getIntrinsic(Intrinsic::fma, ResultType);
+ return Builder.CreateCall(F, {X, Y, Builder.CreateFSub(Zero, Z, "sub")});
+ }
+ case SystemZ::BI__builtin_s390_vflpdb: {
+ llvm::Type *ResultType = ConvertType(E->getType());
+ Value *X = EmitScalarExpr(E->getArg(0));
+ Function *F = CGM.getIntrinsic(Intrinsic::fabs, ResultType);
+ return Builder.CreateCall(F, X);
+ }
+ case SystemZ::BI__builtin_s390_vflndb: {
+ llvm::Type *ResultType = ConvertType(E->getType());
+ Value *X = EmitScalarExpr(E->getArg(0));
+ Value *Zero = llvm::ConstantFP::getZeroValueForNegation(ResultType);
+ Function *F = CGM.getIntrinsic(Intrinsic::fabs, ResultType);
+ return Builder.CreateFSub(Zero, Builder.CreateCall(F, X), "sub");
+ }
+ case SystemZ::BI__builtin_s390_vfidb: {
+ llvm::Type *ResultType = ConvertType(E->getType());
+ Value *X = EmitScalarExpr(E->getArg(0));
+ // Constant-fold the M4 and M5 mask arguments.
+ llvm::APSInt M4, M5;
+ bool IsConstM4 = E->getArg(1)->isIntegerConstantExpr(M4, getContext());
+ bool IsConstM5 = E->getArg(2)->isIntegerConstantExpr(M5, getContext());
+ assert(IsConstM4 && IsConstM5 && "Constant arg isn't actually constant?");
+ (void)IsConstM4; (void)IsConstM5;
+ // Check whether this instance of vfidb can be represented via a LLVM
+ // standard intrinsic. We only support some combinations of M4 and M5.
+ Intrinsic::ID ID = Intrinsic::not_intrinsic;
+ switch (M4.getZExtValue()) {
+ default: break;
+ case 0: // IEEE-inexact exception allowed
+ switch (M5.getZExtValue()) {
+ default: break;
+ case 0: ID = Intrinsic::rint; break;
+ }
+ break;
+ case 4: // IEEE-inexact exception suppressed
+ switch (M5.getZExtValue()) {
+ default: break;
+ case 0: ID = Intrinsic::nearbyint; break;
+ case 1: ID = Intrinsic::round; break;
+ case 5: ID = Intrinsic::trunc; break;
+ case 6: ID = Intrinsic::ceil; break;
+ case 7: ID = Intrinsic::floor; break;
+ }
+ break;
+ }
+ if (ID != Intrinsic::not_intrinsic) {
+ Function *F = CGM.getIntrinsic(ID, ResultType);
+ return Builder.CreateCall(F, X);
+ }
+ Function *F = CGM.getIntrinsic(Intrinsic::s390_vfidb);
+ Value *M4Value = llvm::ConstantInt::get(getLLVMContext(), M4);
+ Value *M5Value = llvm::ConstantInt::get(getLLVMContext(), M5);
+ return Builder.CreateCall(F, {X, M4Value, M5Value});
+ }
+
+ // Vector intrisincs that output the post-instruction CC value.
+
+#define INTRINSIC_WITH_CC(NAME) \
+ case SystemZ::BI__builtin_##NAME: \
+ return EmitSystemZIntrinsicWithCC(*this, Intrinsic::NAME, E)
+
+ INTRINSIC_WITH_CC(s390_vpkshs);
+ INTRINSIC_WITH_CC(s390_vpksfs);
+ INTRINSIC_WITH_CC(s390_vpksgs);
+
+ INTRINSIC_WITH_CC(s390_vpklshs);
+ INTRINSIC_WITH_CC(s390_vpklsfs);
+ INTRINSIC_WITH_CC(s390_vpklsgs);
+
+ INTRINSIC_WITH_CC(s390_vceqbs);
+ INTRINSIC_WITH_CC(s390_vceqhs);
+ INTRINSIC_WITH_CC(s390_vceqfs);
+ INTRINSIC_WITH_CC(s390_vceqgs);
+
+ INTRINSIC_WITH_CC(s390_vchbs);
+ INTRINSIC_WITH_CC(s390_vchhs);
+ INTRINSIC_WITH_CC(s390_vchfs);
+ INTRINSIC_WITH_CC(s390_vchgs);
+
+ INTRINSIC_WITH_CC(s390_vchlbs);
+ INTRINSIC_WITH_CC(s390_vchlhs);
+ INTRINSIC_WITH_CC(s390_vchlfs);
+ INTRINSIC_WITH_CC(s390_vchlgs);
+
+ INTRINSIC_WITH_CC(s390_vfaebs);
+ INTRINSIC_WITH_CC(s390_vfaehs);
+ INTRINSIC_WITH_CC(s390_vfaefs);
+
+ INTRINSIC_WITH_CC(s390_vfaezbs);
+ INTRINSIC_WITH_CC(s390_vfaezhs);
+ INTRINSIC_WITH_CC(s390_vfaezfs);
+
+ INTRINSIC_WITH_CC(s390_vfeebs);
+ INTRINSIC_WITH_CC(s390_vfeehs);
+ INTRINSIC_WITH_CC(s390_vfeefs);
+
+ INTRINSIC_WITH_CC(s390_vfeezbs);
+ INTRINSIC_WITH_CC(s390_vfeezhs);
+ INTRINSIC_WITH_CC(s390_vfeezfs);
+
+ INTRINSIC_WITH_CC(s390_vfenebs);
+ INTRINSIC_WITH_CC(s390_vfenehs);
+ INTRINSIC_WITH_CC(s390_vfenefs);
+
+ INTRINSIC_WITH_CC(s390_vfenezbs);
+ INTRINSIC_WITH_CC(s390_vfenezhs);
+ INTRINSIC_WITH_CC(s390_vfenezfs);
+
+ INTRINSIC_WITH_CC(s390_vistrbs);
+ INTRINSIC_WITH_CC(s390_vistrhs);
+ INTRINSIC_WITH_CC(s390_vistrfs);
+
+ INTRINSIC_WITH_CC(s390_vstrcbs);
+ INTRINSIC_WITH_CC(s390_vstrchs);
+ INTRINSIC_WITH_CC(s390_vstrcfs);
+
+ INTRINSIC_WITH_CC(s390_vstrczbs);
+ INTRINSIC_WITH_CC(s390_vstrczhs);
+ INTRINSIC_WITH_CC(s390_vstrczfs);
+
+ INTRINSIC_WITH_CC(s390_vfcedbs);
+ INTRINSIC_WITH_CC(s390_vfchdbs);
+ INTRINSIC_WITH_CC(s390_vfchedbs);
+
+ INTRINSIC_WITH_CC(s390_vftcidb);
+
+#undef INTRINSIC_WITH_CC
+
+ default:
+ return nullptr;
+ }
+}
diff --git a/lib/CodeGen/CGCUDANV.cpp b/lib/CodeGen/CGCUDANV.cpp
index fb11751fc91e..67d0ab7a82f7 100644
--- a/lib/CodeGen/CGCUDANV.cpp
+++ b/lib/CodeGen/CGCUDANV.cpp
@@ -20,7 +20,6 @@
#include "llvm/IR/CallSite.h"
#include "llvm/IR/Constants.h"
#include "llvm/IR/DerivedTypes.h"
-#include <vector>
using namespace clang;
using namespace CodeGen;
@@ -30,29 +29,66 @@ namespace {
class CGNVCUDARuntime : public CGCUDARuntime {
private:
- llvm::Type *IntTy, *SizeTy;
- llvm::PointerType *CharPtrTy, *VoidPtrTy;
+ llvm::Type *IntTy, *SizeTy, *VoidTy;
+ llvm::PointerType *CharPtrTy, *VoidPtrTy, *VoidPtrPtrTy;
+
+ /// Convenience reference to LLVM Context
+ llvm::LLVMContext &Context;
+ /// Convenience reference to the current module
+ llvm::Module &TheModule;
+ /// Keeps track of kernel launch stubs emitted in this module
+ llvm::SmallVector<llvm::Function *, 16> EmittedKernels;
+ /// Keeps track of variables containing handles of GPU binaries. Populated by
+ /// ModuleCtorFunction() and used to create corresponding cleanup calls in
+ /// ModuleDtorFunction()
+ llvm::SmallVector<llvm::GlobalVariable *, 16> GpuBinaryHandles;
llvm::Constant *getSetupArgumentFn() const;
llvm::Constant *getLaunchFn() const;
+ /// Creates a function to register all kernel stubs generated in this module.
+ llvm::Function *makeRegisterKernelsFn();
+
+ /// Helper function that generates a constant string and returns a pointer to
+ /// the start of the string. The result of this function can be used anywhere
+ /// where the C code specifies const char*.
+ llvm::Constant *makeConstantString(const std::string &Str,
+ const std::string &Name = "",
+ unsigned Alignment = 0) {
+ llvm::Constant *Zeros[] = {llvm::ConstantInt::get(SizeTy, 0),
+ llvm::ConstantInt::get(SizeTy, 0)};
+ auto *ConstStr = CGM.GetAddrOfConstantCString(Str, Name.c_str());
+ return llvm::ConstantExpr::getGetElementPtr(ConstStr->getValueType(),
+ ConstStr, Zeros);
+ }
+
+ void emitDeviceStubBody(CodeGenFunction &CGF, FunctionArgList &Args);
+
public:
CGNVCUDARuntime(CodeGenModule &CGM);
- void EmitDeviceStubBody(CodeGenFunction &CGF, FunctionArgList &Args) override;
+ void emitDeviceStub(CodeGenFunction &CGF, FunctionArgList &Args) override;
+ /// Creates module constructor function
+ llvm::Function *makeModuleCtorFunction() override;
+ /// Creates module destructor function
+ llvm::Function *makeModuleDtorFunction() override;
};
}
-CGNVCUDARuntime::CGNVCUDARuntime(CodeGenModule &CGM) : CGCUDARuntime(CGM) {
+CGNVCUDARuntime::CGNVCUDARuntime(CodeGenModule &CGM)
+ : CGCUDARuntime(CGM), Context(CGM.getLLVMContext()),
+ TheModule(CGM.getModule()) {
CodeGen::CodeGenTypes &Types = CGM.getTypes();
ASTContext &Ctx = CGM.getContext();
IntTy = Types.ConvertType(Ctx.IntTy);
SizeTy = Types.ConvertType(Ctx.getSizeType());
+ VoidTy = llvm::Type::getVoidTy(Context);
CharPtrTy = llvm::PointerType::getUnqual(Types.ConvertType(Ctx.CharTy));
VoidPtrTy = cast<llvm::PointerType>(Types.ConvertType(Ctx.VoidPtrTy));
+ VoidPtrPtrTy = VoidPtrTy->getPointerTo();
}
llvm::Constant *CGNVCUDARuntime::getSetupArgumentFn() const {
@@ -68,14 +104,17 @@ llvm::Constant *CGNVCUDARuntime::getSetupArgumentFn() const {
llvm::Constant *CGNVCUDARuntime::getLaunchFn() const {
// cudaError_t cudaLaunch(char *)
- std::vector<llvm::Type*> Params;
- Params.push_back(CharPtrTy);
- return CGM.CreateRuntimeFunction(llvm::FunctionType::get(IntTy,
- Params, false),
- "cudaLaunch");
+ return CGM.CreateRuntimeFunction(
+ llvm::FunctionType::get(IntTy, CharPtrTy, false), "cudaLaunch");
+}
+
+void CGNVCUDARuntime::emitDeviceStub(CodeGenFunction &CGF,
+ FunctionArgList &Args) {
+ EmittedKernels.push_back(CGF.CurFn);
+ emitDeviceStubBody(CGF, Args);
}
-void CGNVCUDARuntime::EmitDeviceStubBody(CodeGenFunction &CGF,
+void CGNVCUDARuntime::emitDeviceStubBody(CodeGenFunction &CGF,
FunctionArgList &Args) {
// Build the argument value list and the argument stack struct type.
SmallVector<llvm::Value *, 16> ArgValues;
@@ -87,8 +126,7 @@ void CGNVCUDARuntime::EmitDeviceStubBody(CodeGenFunction &CGF,
assert(isa<llvm::PointerType>(V->getType()) && "Arg type not PointerType");
ArgTypes.push_back(cast<llvm::PointerType>(V->getType())->getElementType());
}
- llvm::StructType *ArgStackTy = llvm::StructType::get(
- CGF.getLLVMContext(), ArgTypes);
+ llvm::StructType *ArgStackTy = llvm::StructType::get(Context, ArgTypes);
llvm::BasicBlock *EndBlock = CGF.createBasicBlock("setup.end");
@@ -120,6 +158,160 @@ void CGNVCUDARuntime::EmitDeviceStubBody(CodeGenFunction &CGF,
CGF.EmitBlock(EndBlock);
}
+/// Creates internal function to register all kernel stubs generated in this
+/// module with the CUDA runtime.
+/// \code
+/// void __cuda_register_kernels(void** GpuBinaryHandle) {
+/// __cudaRegisterFunction(GpuBinaryHandle,Kernel0,...);
+/// ...
+/// __cudaRegisterFunction(GpuBinaryHandle,KernelM,...);
+/// }
+/// \endcode
+llvm::Function *CGNVCUDARuntime::makeRegisterKernelsFn() {
+ llvm::Function *RegisterKernelsFunc = llvm::Function::Create(
+ llvm::FunctionType::get(VoidTy, VoidPtrPtrTy, false),
+ llvm::GlobalValue::InternalLinkage, "__cuda_register_kernels", &TheModule);
+ llvm::BasicBlock *EntryBB =
+ llvm::BasicBlock::Create(Context, "entry", RegisterKernelsFunc);
+ CGBuilderTy Builder(Context);
+ Builder.SetInsertPoint(EntryBB);
+
+ // void __cudaRegisterFunction(void **, const char *, char *, const char *,
+ // int, uint3*, uint3*, dim3*, dim3*, int*)
+ std::vector<llvm::Type *> RegisterFuncParams = {
+ VoidPtrPtrTy, CharPtrTy, CharPtrTy, CharPtrTy, IntTy,
+ VoidPtrTy, VoidPtrTy, VoidPtrTy, VoidPtrTy, IntTy->getPointerTo()};
+ llvm::Constant *RegisterFunc = CGM.CreateRuntimeFunction(
+ llvm::FunctionType::get(IntTy, RegisterFuncParams, false),
+ "__cudaRegisterFunction");
+
+ // Extract GpuBinaryHandle passed as the first argument passed to
+ // __cuda_register_kernels() and generate __cudaRegisterFunction() call for
+ // each emitted kernel.
+ llvm::Argument &GpuBinaryHandlePtr = *RegisterKernelsFunc->arg_begin();
+ for (llvm::Function *Kernel : EmittedKernels) {
+ llvm::Constant *KernelName = makeConstantString(Kernel->getName());
+ llvm::Constant *NullPtr = llvm::ConstantPointerNull::get(VoidPtrTy);
+ llvm::Value *args[] = {
+ &GpuBinaryHandlePtr, Builder.CreateBitCast(Kernel, VoidPtrTy),
+ KernelName, KernelName, llvm::ConstantInt::get(IntTy, -1), NullPtr,
+ NullPtr, NullPtr, NullPtr,
+ llvm::ConstantPointerNull::get(IntTy->getPointerTo())};
+ Builder.CreateCall(RegisterFunc, args);
+ }
+
+ Builder.CreateRetVoid();
+ return RegisterKernelsFunc;
+}
+
+/// Creates a global constructor function for the module:
+/// \code
+/// void __cuda_module_ctor(void*) {
+/// Handle0 = __cudaRegisterFatBinary(GpuBinaryBlob0);
+/// __cuda_register_kernels(Handle0);
+/// ...
+/// HandleN = __cudaRegisterFatBinary(GpuBinaryBlobN);
+/// __cuda_register_kernels(HandleN);
+/// }
+/// \endcode
+llvm::Function *CGNVCUDARuntime::makeModuleCtorFunction() {
+ // void __cuda_register_kernels(void* handle);
+ llvm::Function *RegisterKernelsFunc = makeRegisterKernelsFn();
+ // void ** __cudaRegisterFatBinary(void *);
+ llvm::Constant *RegisterFatbinFunc = CGM.CreateRuntimeFunction(
+ llvm::FunctionType::get(VoidPtrPtrTy, VoidPtrTy, false),
+ "__cudaRegisterFatBinary");
+ // struct { int magic, int version, void * gpu_binary, void * dont_care };
+ llvm::StructType *FatbinWrapperTy =
+ llvm::StructType::get(IntTy, IntTy, VoidPtrTy, VoidPtrTy, nullptr);
+
+ llvm::Function *ModuleCtorFunc = llvm::Function::Create(
+ llvm::FunctionType::get(VoidTy, VoidPtrTy, false),
+ llvm::GlobalValue::InternalLinkage, "__cuda_module_ctor", &TheModule);
+ llvm::BasicBlock *CtorEntryBB =
+ llvm::BasicBlock::Create(Context, "entry", ModuleCtorFunc);
+ CGBuilderTy CtorBuilder(Context);
+
+ CtorBuilder.SetInsertPoint(CtorEntryBB);
+
+ // For each GPU binary, register it with the CUDA runtime and store returned
+ // handle in a global variable and save the handle in GpuBinaryHandles vector
+ // to be cleaned up in destructor on exit. Then associate all known kernels
+ // with the GPU binary handle so CUDA runtime can figure out what to call on
+ // the GPU side.
+ for (const std::string &GpuBinaryFileName :
+ CGM.getCodeGenOpts().CudaGpuBinaryFileNames) {
+ llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> GpuBinaryOrErr =
+ llvm::MemoryBuffer::getFileOrSTDIN(GpuBinaryFileName);
+ if (std::error_code EC = GpuBinaryOrErr.getError()) {
+ CGM.getDiags().Report(diag::err_cannot_open_file) << GpuBinaryFileName
+ << EC.message();
+ continue;
+ }
+
+ // Create initialized wrapper structure that points to the loaded GPU binary
+ llvm::Constant *Values[] = {
+ llvm::ConstantInt::get(IntTy, 0x466243b1), // Fatbin wrapper magic.
+ llvm::ConstantInt::get(IntTy, 1), // Fatbin version.
+ makeConstantString(GpuBinaryOrErr.get()->getBuffer(), "", 16), // Data.
+ llvm::ConstantPointerNull::get(VoidPtrTy)}; // Unused in fatbin v1.
+ llvm::GlobalVariable *FatbinWrapper = new llvm::GlobalVariable(
+ TheModule, FatbinWrapperTy, true, llvm::GlobalValue::InternalLinkage,
+ llvm::ConstantStruct::get(FatbinWrapperTy, Values),
+ "__cuda_fatbin_wrapper");
+
+ // GpuBinaryHandle = __cudaRegisterFatBinary(&FatbinWrapper);
+ llvm::CallInst *RegisterFatbinCall = CtorBuilder.CreateCall(
+ RegisterFatbinFunc,
+ CtorBuilder.CreateBitCast(FatbinWrapper, VoidPtrTy));
+ llvm::GlobalVariable *GpuBinaryHandle = new llvm::GlobalVariable(
+ TheModule, VoidPtrPtrTy, false, llvm::GlobalValue::InternalLinkage,
+ llvm::ConstantPointerNull::get(VoidPtrPtrTy), "__cuda_gpubin_handle");
+ CtorBuilder.CreateStore(RegisterFatbinCall, GpuBinaryHandle, false);
+
+ // Call __cuda_register_kernels(GpuBinaryHandle);
+ CtorBuilder.CreateCall(RegisterKernelsFunc, RegisterFatbinCall);
+
+ // Save GpuBinaryHandle so we can unregister it in destructor.
+ GpuBinaryHandles.push_back(GpuBinaryHandle);
+ }
+
+ CtorBuilder.CreateRetVoid();
+ return ModuleCtorFunc;
+}
+
+/// Creates a global destructor function that unregisters all GPU code blobs
+/// registered by constructor.
+/// \code
+/// void __cuda_module_dtor(void*) {
+/// __cudaUnregisterFatBinary(Handle0);
+/// ...
+/// __cudaUnregisterFatBinary(HandleN);
+/// }
+/// \endcode
+llvm::Function *CGNVCUDARuntime::makeModuleDtorFunction() {
+ // void __cudaUnregisterFatBinary(void ** handle);
+ llvm::Constant *UnregisterFatbinFunc = CGM.CreateRuntimeFunction(
+ llvm::FunctionType::get(VoidTy, VoidPtrPtrTy, false),
+ "__cudaUnregisterFatBinary");
+
+ llvm::Function *ModuleDtorFunc = llvm::Function::Create(
+ llvm::FunctionType::get(VoidTy, VoidPtrTy, false),
+ llvm::GlobalValue::InternalLinkage, "__cuda_module_dtor", &TheModule);
+ llvm::BasicBlock *DtorEntryBB =
+ llvm::BasicBlock::Create(Context, "entry", ModuleDtorFunc);
+ CGBuilderTy DtorBuilder(Context);
+ DtorBuilder.SetInsertPoint(DtorEntryBB);
+
+ for (llvm::GlobalVariable *GpuBinaryHandle : GpuBinaryHandles) {
+ DtorBuilder.CreateCall(UnregisterFatbinFunc,
+ DtorBuilder.CreateLoad(GpuBinaryHandle, false));
+ }
+
+ DtorBuilder.CreateRetVoid();
+ return ModuleDtorFunc;
+}
+
CGCUDARuntime *CodeGen::CreateNVCUDARuntime(CodeGenModule &CGM) {
return new CGNVCUDARuntime(CGM);
}
diff --git a/lib/CodeGen/CGCUDARuntime.h b/lib/CodeGen/CGCUDARuntime.h
index 8c162fb05ab9..dcacf9703277 100644
--- a/lib/CodeGen/CGCUDARuntime.h
+++ b/lib/CodeGen/CGCUDARuntime.h
@@ -16,6 +16,10 @@
#ifndef LLVM_CLANG_LIB_CODEGEN_CGCUDARUNTIME_H
#define LLVM_CLANG_LIB_CODEGEN_CGCUDARUNTIME_H
+namespace llvm {
+class Function;
+}
+
namespace clang {
class CUDAKernelCallExpr;
@@ -39,10 +43,17 @@ public:
virtual RValue EmitCUDAKernelCallExpr(CodeGenFunction &CGF,
const CUDAKernelCallExpr *E,
ReturnValueSlot ReturnValue);
-
- virtual void EmitDeviceStubBody(CodeGenFunction &CGF,
- FunctionArgList &Args) = 0;
+ /// Emits a kernel launch stub.
+ virtual void emitDeviceStub(CodeGenFunction &CGF, FunctionArgList &Args) = 0;
+
+ /// Constructs and returns a module initialization function or nullptr if it's
+ /// not needed. Must be called after all kernels have been emitted.
+ virtual llvm::Function *makeModuleCtorFunction() = 0;
+
+ /// Returns a module cleanup function or nullptr if it's not needed.
+ /// Must be called after ModuleCtorFunction
+ virtual llvm::Function *makeModuleDtorFunction() = 0;
};
/// Creates an instance of a CUDA runtime class.
diff --git a/lib/CodeGen/CGCXX.cpp b/lib/CodeGen/CGCXX.cpp
index 9f0e67e42176..29a199d6973a 100644
--- a/lib/CodeGen/CGCXX.cpp
+++ b/lib/CodeGen/CGCXX.cpp
@@ -182,8 +182,8 @@ bool CodeGenModule::TryEmitDefinitionAsAlias(GlobalDecl AliasDecl,
return true;
// Create the alias with no name.
- auto *Alias = llvm::GlobalAlias::create(AliasType->getElementType(), 0,
- Linkage, "", Aliasee, &getModule());
+ auto *Alias =
+ llvm::GlobalAlias::create(AliasType, Linkage, "", Aliasee, &getModule());
// Switch any previous uses to the alias.
if (Entry) {
@@ -231,8 +231,7 @@ llvm::GlobalValue *CodeGenModule::getAddrOfCXXStructor(
if (auto *CD = dyn_cast<CXXConstructorDecl>(MD)) {
GD = GlobalDecl(CD, toCXXCtorType(Type));
} else {
- auto *DD = dyn_cast<CXXDestructorDecl>(MD);
- GD = GlobalDecl(DD, toCXXDtorType(Type));
+ GD = GlobalDecl(cast<CXXDestructorDecl>(MD), toCXXDtorType(Type));
}
StringRef Name = getMangledName(GD);
diff --git a/lib/CodeGen/CGCXXABI.cpp b/lib/CodeGen/CGCXXABI.cpp
index d31331de6868..cb7e6dfca1c5 100644
--- a/lib/CodeGen/CGCXXABI.cpp
+++ b/lib/CodeGen/CGCXXABI.cpp
@@ -302,3 +302,10 @@ CGCXXABI::EmitCtorCompleteObjectHandler(CodeGenFunction &CGF,
bool CGCXXABI::NeedsVTTParameter(GlobalDecl GD) {
return false;
}
+
+llvm::CallInst *
+CGCXXABI::emitTerminateForUnexpectedException(CodeGenFunction &CGF,
+ llvm::Value *Exn) {
+ // Just call std::terminate and ignore the violating exception.
+ return CGF.EmitNounwindRuntimeCall(CGF.CGM.getTerminateFn());
+}
diff --git a/lib/CodeGen/CGCXXABI.h b/lib/CodeGen/CGCXXABI.h
index cc5c1b2e0ae6..2c7392188f78 100644
--- a/lib/CodeGen/CGCXXABI.h
+++ b/lib/CodeGen/CGCXXABI.h
@@ -22,6 +22,7 @@ namespace llvm {
class Constant;
class Type;
class Value;
+class CallInst;
}
namespace clang {
@@ -214,8 +215,18 @@ public:
llvm::Value *Ptr, QualType ElementType,
const CXXDestructorDecl *Dtor) = 0;
virtual void emitRethrow(CodeGenFunction &CGF, bool isNoReturn) = 0;
+ virtual void emitThrow(CodeGenFunction &CGF, const CXXThrowExpr *E) = 0;
+ virtual llvm::GlobalVariable *getThrowInfo(QualType T) { return nullptr; }
+
+ virtual void emitBeginCatch(CodeGenFunction &CGF, const CXXCatchStmt *C) = 0;
+
+ virtual llvm::CallInst *
+ emitTerminateForUnexpectedException(CodeGenFunction &CGF,
+ llvm::Value *Exn);
virtual llvm::Constant *getAddrOfRTTIDescriptor(QualType Ty) = 0;
+ virtual llvm::Constant *
+ getAddrOfCXXCatchHandlerType(QualType Ty, QualType CatchHandlerType) = 0;
virtual bool shouldTypeidBeNullChecked(bool IsDeref,
QualType SrcRecordTy) = 0;
diff --git a/lib/CodeGen/CGCall.cpp b/lib/CodeGen/CGCall.cpp
index 6403fa99aa7b..f2340530d66d 100644
--- a/lib/CodeGen/CGCall.cpp
+++ b/lib/CodeGen/CGCall.cpp
@@ -30,7 +30,9 @@
#include "llvm/IR/DataLayout.h"
#include "llvm/IR/InlineAsm.h"
#include "llvm/IR/Intrinsics.h"
+#include "llvm/IR/IntrinsicInst.h"
#include "llvm/Transforms/Utils/Local.h"
+#include <sstream>
using namespace clang;
using namespace CodeGen;
@@ -51,6 +53,8 @@ static unsigned ClangCallConvToLLVMCallConv(CallingConv CC) {
case CC_X86Pascal: return llvm::CallingConv::C;
// TODO: Add support for __vectorcall to LLVM.
case CC_X86VectorCall: return llvm::CallingConv::X86_VectorCall;
+ case CC_SpirFunction: return llvm::CallingConv::SPIR_FUNC;
+ case CC_SpirKernel: return llvm::CallingConv::SPIR_KERNEL;
}
}
@@ -96,8 +100,7 @@ arrangeLLVMFunctionInfo(CodeGenTypes &CGT, bool instanceMethod,
CanQual<FunctionProtoType> FTP) {
RequiredArgs required = RequiredArgs::forPrototypePlus(FTP, prefix.size());
// FIXME: Kill copy.
- for (unsigned i = 0, e = FTP->getNumParams(); i != e; ++i)
- prefix.push_back(FTP->getParamType(i));
+ prefix.append(FTP->param_type_begin(), FTP->param_type_end());
CanQualType resultType = FTP->getReturnType().getUnqualifiedType();
return CGT.arrangeLLVMFunctionInfo(resultType, instanceMethod,
/*chainCall=*/false, prefix,
@@ -133,9 +136,6 @@ static CallingConv getCallingConventionForDecl(const Decl *D, bool IsWindows) {
if (PcsAttr *PCS = D->getAttr<PcsAttr>())
return (PCS->getPCS() == PcsAttr::AAPCS ? CC_AAPCS : CC_AAPCS_VFP);
- if (D->hasAttr<PnaclCallAttr>())
- return CC_PnaclCall;
-
if (D->hasAttr<IntelOclBiccAttr>())
return CC_IntelOclBicc;
@@ -208,8 +208,7 @@ CodeGenTypes::arrangeCXXStructorDeclaration(const CXXMethodDecl *MD,
CanQual<FunctionProtoType> FTP = GetFormalType(MD);
// Add the formal parameters.
- for (unsigned i = 0, e = FTP->getNumParams(); i != e; ++i)
- argTypes.push_back(FTP->getParamType(i));
+ argTypes.append(FTP->param_type_begin(), FTP->param_type_end());
TheCXXABI.buildStructorSignature(MD, Type, argTypes);
@@ -349,6 +348,26 @@ CodeGenTypes::arrangeMSMemberPointerThunk(const CXXMethodDecl *MD) {
FTP->getExtInfo(), RequiredArgs(1));
}
+const CGFunctionInfo &
+CodeGenTypes::arrangeMSCtorClosure(const CXXConstructorDecl *CD,
+ CXXCtorType CT) {
+ assert(CT == Ctor_CopyingClosure || CT == Ctor_DefaultClosure);
+
+ CanQual<FunctionProtoType> FTP = GetFormalType(CD);
+ SmallVector<CanQualType, 2> ArgTys;
+ const CXXRecordDecl *RD = CD->getParent();
+ ArgTys.push_back(GetThisType(Context, RD));
+ if (CT == Ctor_CopyingClosure)
+ ArgTys.push_back(*FTP->param_type_begin());
+ if (RD->getNumVBases() > 0)
+ ArgTys.push_back(Context.IntTy);
+ CallingConv CC = Context.getDefaultCallingConvention(
+ /*IsVariadic=*/false, /*IsCXXMethod=*/true);
+ return arrangeLLVMFunctionInfo(Context.VoidTy, /*instanceMethod=*/true,
+ /*chainCall=*/false, ArgTys,
+ FunctionType::ExtInfo(CC), RequiredArgs::All);
+}
+
/// Arrange a call as unto a free function, except possibly with an
/// additional number of formal parameters considered required.
static const CGFunctionInfo &
@@ -716,7 +735,8 @@ void CodeGenFunction::ExpandTypeFromArgs(
auto Exp = getTypeExpansion(Ty, getContext());
if (auto CAExp = dyn_cast<ConstantArrayExpansion>(Exp.get())) {
for (int i = 0, n = CAExp->NumElts; i < n; i++) {
- llvm::Value *EltAddr = Builder.CreateConstGEP2_32(LV.getAddress(), 0, i);
+ llvm::Value *EltAddr =
+ Builder.CreateConstGEP2_32(nullptr, LV.getAddress(), 0, i);
LValue LV = MakeAddrLValue(EltAddr, CAExp->EltTy);
ExpandTypeFromArgs(CAExp->EltTy, LV, AI);
}
@@ -738,10 +758,12 @@ void CodeGenFunction::ExpandTypeFromArgs(
ExpandTypeFromArgs(FD->getType(), SubLV, AI);
}
} else if (auto CExp = dyn_cast<ComplexExpansion>(Exp.get())) {
- llvm::Value *RealAddr = Builder.CreateStructGEP(LV.getAddress(), 0, "real");
+ llvm::Value *RealAddr =
+ Builder.CreateStructGEP(nullptr, LV.getAddress(), 0, "real");
EmitStoreThroughLValue(RValue::get(*AI++),
MakeAddrLValue(RealAddr, CExp->EltTy));
- llvm::Value *ImagAddr = Builder.CreateStructGEP(LV.getAddress(), 1, "imag");
+ llvm::Value *ImagAddr =
+ Builder.CreateStructGEP(nullptr, LV.getAddress(), 1, "imag");
EmitStoreThroughLValue(RValue::get(*AI++),
MakeAddrLValue(ImagAddr, CExp->EltTy));
} else {
@@ -757,7 +779,7 @@ void CodeGenFunction::ExpandTypeToArgs(
if (auto CAExp = dyn_cast<ConstantArrayExpansion>(Exp.get())) {
llvm::Value *Addr = RV.getAggregateAddr();
for (int i = 0, n = CAExp->NumElts; i < n; i++) {
- llvm::Value *EltAddr = Builder.CreateConstGEP2_32(Addr, 0, i);
+ llvm::Value *EltAddr = Builder.CreateConstGEP2_32(nullptr, Addr, 0, i);
RValue EltRV =
convertTempToRValue(EltAddr, CAExp->EltTy, SourceLocation());
ExpandTypeToArgs(CAExp->EltTy, EltRV, IRFuncTy, IRCallArgs, IRCallArgPos);
@@ -825,7 +847,7 @@ EnterStructPointerForCoercedAccess(llvm::Value *SrcPtr,
return SrcPtr;
// GEP into the first element.
- SrcPtr = CGF.Builder.CreateConstGEP2_32(SrcPtr, 0, 0, "coerce.dive");
+ SrcPtr = CGF.Builder.CreateConstGEP2_32(SrcSTy, SrcPtr, 0, 0, "coerce.dive");
// If the first element is a struct, recurse.
llvm::Type *SrcTy =
@@ -963,7 +985,7 @@ static void BuildAggStore(CodeGenFunction &CGF, llvm::Value *Val,
if (llvm::StructType *STy =
dyn_cast<llvm::StructType>(Val->getType())) {
for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i) {
- llvm::Value *EltPtr = CGF.Builder.CreateConstGEP2_32(DestPtr, 0, i);
+ llvm::Value *EltPtr = CGF.Builder.CreateConstGEP2_32(STy, DestPtr, 0, i);
llvm::Value *Elt = CGF.Builder.CreateExtractValue(Val, i);
llvm::StoreInst *SI = CGF.Builder.CreateStore(Elt, EltPtr,
DestIsVolatile);
@@ -1403,7 +1425,7 @@ void CodeGenModule::ConstructAttributeList(const CGFunctionInfo &FI,
FuncAttrs.addAttribute(llvm::Attribute::ReadOnly);
FuncAttrs.addAttribute(llvm::Attribute::NoUnwind);
}
- if (TargetDecl->hasAttr<MallocAttr>())
+ if (TargetDecl->hasAttr<RestrictAttr>())
RetAttrs.addAttribute(llvm::Attribute::NoAlias);
if (TargetDecl->hasAttr<ReturnsNonNullAttr>())
RetAttrs.addAttribute(llvm::Attribute::NonNull);
@@ -1458,6 +1480,26 @@ void CodeGenModule::ConstructAttributeList(const CGFunctionInfo &FI,
if (!CodeGenOpts.StackRealignment)
FuncAttrs.addAttribute("no-realign-stack");
+
+ // Add target-cpu and target-features work if they differ from the defaults.
+ std::string &CPU = getTarget().getTargetOpts().CPU;
+ if (CPU != "")
+ FuncAttrs.addAttribute("target-cpu", CPU);
+
+ // TODO: Features gets us the features on the command line including
+ // feature dependencies. For canonicalization purposes we might want to
+ // avoid putting features in the target-features set if we know it'll be one
+ // of the default features in the backend, e.g. corei7-avx and +avx or figure
+ // out non-explicit dependencies.
+ std::vector<std::string> &Features = getTarget().getTargetOpts().Features;
+ if (!Features.empty()) {
+ std::stringstream S;
+ std::copy(Features.begin(), Features.end(),
+ std::ostream_iterator<std::string>(S, ","));
+ // The drop_back gets rid of the trailing space.
+ FuncAttrs.addAttribute("target-features",
+ StringRef(S.str()).drop_back(1));
+ }
}
ClangToLLVMArgMapping IRFunctionArgs(getContext(), FI);
@@ -1546,8 +1588,12 @@ void CodeGenModule::ConstructAttributeList(const CGFunctionInfo &FI,
case ABIArgInfo::Extend:
if (ParamType->isSignedIntegerOrEnumerationType())
Attrs.addAttribute(llvm::Attribute::SExt);
- else if (ParamType->isUnsignedIntegerOrEnumerationType())
- Attrs.addAttribute(llvm::Attribute::ZExt);
+ else if (ParamType->isUnsignedIntegerOrEnumerationType()) {
+ if (getTypes().getABIInfo().shouldSignExtUnsignedType(ParamType))
+ Attrs.addAttribute(llvm::Attribute::SExt);
+ else
+ Attrs.addAttribute(llvm::Attribute::ZExt);
+ }
// FALL THROUGH
case ABIArgInfo::Direct:
if (ArgNo == 0 && FI.isChainCall())
@@ -1734,8 +1780,9 @@ void CodeGenFunction::EmitFunctionProlog(const CGFunctionInfo &FI,
switch (ArgI.getKind()) {
case ABIArgInfo::InAlloca: {
assert(NumIRArgs == 0);
- llvm::Value *V = Builder.CreateStructGEP(
- ArgStruct, ArgI.getInAllocaFieldIndex(), Arg->getName());
+ llvm::Value *V =
+ Builder.CreateStructGEP(FI.getArgStruct(), ArgStruct,
+ ArgI.getInAllocaFieldIndex(), Arg->getName());
ArgVals.push_back(ValueAndIsPtr(V, HavePointer));
break;
}
@@ -1770,8 +1817,7 @@ void CodeGenFunction::EmitFunctionProlog(const CGFunctionInfo &FI,
ArgVals.push_back(ValueAndIsPtr(V, HavePointer));
} else {
// Load scalar value from indirect argument.
- CharUnits Alignment = getContext().getTypeAlignInChars(Ty);
- V = EmitLoadOfScalar(V, false, Alignment.getQuantity(), Ty,
+ V = EmitLoadOfScalar(V, false, ArgI.getIndirectAlign(), Ty,
Arg->getLocStart());
if (isPromoted)
@@ -1901,7 +1947,7 @@ void CodeGenFunction::EmitFunctionProlog(const CGFunctionInfo &FI,
// If the value is offset in memory, apply the offset now.
if (unsigned Offs = ArgI.getDirectOffset()) {
Ptr = Builder.CreateBitCast(Ptr, Builder.getInt8PtrTy());
- Ptr = Builder.CreateConstGEP1_32(Ptr, Offs);
+ Ptr = Builder.CreateConstGEP1_32(Builder.getInt8Ty(), Ptr, Offs);
Ptr = Builder.CreateBitCast(Ptr,
llvm::PointerType::getUnqual(ArgI.getCoerceToType()));
}
@@ -1923,7 +1969,7 @@ void CodeGenFunction::EmitFunctionProlog(const CGFunctionInfo &FI,
for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i) {
auto AI = FnArgs[FirstIRArg + i];
AI->setName(Arg->getName() + ".coerce" + Twine(i));
- llvm::Value *EltPtr = Builder.CreateConstGEP2_32(Ptr, 0, i);
+ llvm::Value *EltPtr = Builder.CreateConstGEP2_32(STy, Ptr, 0, i);
Builder.CreateStore(AI, EltPtr);
}
} else {
@@ -1936,7 +1982,8 @@ void CodeGenFunction::EmitFunctionProlog(const CGFunctionInfo &FI,
for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i) {
auto AI = FnArgs[FirstIRArg + i];
AI->setName(Arg->getName() + ".coerce" + Twine(i));
- llvm::Value *EltPtr = Builder.CreateConstGEP2_32(TempV, 0, i);
+ llvm::Value *EltPtr =
+ Builder.CreateConstGEP2_32(ArgI.getCoerceToType(), TempV, 0, i);
Builder.CreateStore(AI, EltPtr);
}
@@ -2173,7 +2220,29 @@ static llvm::StoreInst *findDominatingStoreToReturnValue(CodeGenFunction &CGF) {
if (!CGF.ReturnValue->hasOneUse()) {
llvm::BasicBlock *IP = CGF.Builder.GetInsertBlock();
if (IP->empty()) return nullptr;
- llvm::StoreInst *store = dyn_cast<llvm::StoreInst>(&IP->back());
+ llvm::Instruction *I = &IP->back();
+
+ // Skip lifetime markers
+ for (llvm::BasicBlock::reverse_iterator II = IP->rbegin(),
+ IE = IP->rend();
+ II != IE; ++II) {
+ if (llvm::IntrinsicInst *Intrinsic =
+ dyn_cast<llvm::IntrinsicInst>(&*II)) {
+ if (Intrinsic->getIntrinsicID() == llvm::Intrinsic::lifetime_end) {
+ const llvm::Value *CastAddr = Intrinsic->getArgOperand(1);
+ ++II;
+ if (isa<llvm::BitCastInst>(&*II)) {
+ if (CastAddr == &*II) {
+ continue;
+ }
+ }
+ }
+ }
+ I = &*II;
+ break;
+ }
+
+ llvm::StoreInst *store = dyn_cast<llvm::StoreInst>(I);
if (!store) return nullptr;
if (store->getPointerOperand() != CGF.ReturnValue) return nullptr;
assert(!store->isAtomic() && !store->isVolatile()); // see below
@@ -2231,8 +2300,8 @@ void CodeGenFunction::EmitFunctionEpilog(const CGFunctionInfo &FI,
llvm::Function::arg_iterator EI = CurFn->arg_end();
--EI;
llvm::Value *ArgStruct = EI;
- llvm::Value *SRet =
- Builder.CreateStructGEP(ArgStruct, RetAI.getInAllocaFieldIndex());
+ llvm::Value *SRet = Builder.CreateStructGEP(
+ nullptr, ArgStruct, RetAI.getInAllocaFieldIndex());
RV = Builder.CreateLoad(SRet, "sret");
}
break;
@@ -2271,7 +2340,8 @@ void CodeGenFunction::EmitFunctionEpilog(const CGFunctionInfo &FI,
// If there is a dominating store to ReturnValue, we can elide
// the load, zap the store, and usually zap the alloca.
- if (llvm::StoreInst *SI = findDominatingStoreToReturnValue(*this)) {
+ if (llvm::StoreInst *SI =
+ findDominatingStoreToReturnValue(*this)) {
// Reuse the debug location from the store unless there is
// cleanup code to be emitted between the store and return
// instruction.
@@ -2296,7 +2366,7 @@ void CodeGenFunction::EmitFunctionEpilog(const CGFunctionInfo &FI,
// If the value is offset in memory, apply the offset now.
if (unsigned Offs = RetAI.getDirectOffset()) {
V = Builder.CreateBitCast(V, Builder.getInt8PtrTy());
- V = Builder.CreateConstGEP1_32(V, Offs);
+ V = Builder.CreateConstGEP1_32(Builder.getInt8Ty(), V, Offs);
V = Builder.CreateBitCast(V,
llvm::PointerType::getUnqual(RetAI.getCoerceToType()));
}
@@ -2342,8 +2412,8 @@ void CodeGenFunction::EmitFunctionEpilog(const CGFunctionInfo &FI,
Ret = Builder.CreateRetVoid();
}
- if (!RetDbgLoc.isUnknown())
- Ret->setDebugLoc(RetDbgLoc);
+ if (RetDbgLoc)
+ Ret->setDebugLoc(std::move(RetDbgLoc));
}
static bool isInAllocaArgument(CGCXXABI &ABI, QualType type) {
@@ -2626,7 +2696,7 @@ void CallArgList::allocateArgumentMemory(CodeGenFunction &CGF) {
// Save the stack.
llvm::Function *F = CGF.CGM.getIntrinsic(llvm::Intrinsic::stacksave);
- StackBase = CGF.Builder.CreateCall(F, "inalloca.save");
+ StackBase = CGF.Builder.CreateCall(F, {}, "inalloca.save");
// Control gets really tied up in landing pads, so we have to spill the
// stacksave to an alloca to avoid violating SSA form.
@@ -2678,12 +2748,7 @@ void CodeGenFunction::EmitCallArgs(CallArgList &Args,
CallExpr::const_arg_iterator ArgBeg,
CallExpr::const_arg_iterator ArgEnd,
const FunctionDecl *CalleeDecl,
- unsigned ParamsToSkip,
- bool ForceColumnInfo) {
- CGDebugInfo *DI = getDebugInfo();
- SourceLocation CallLoc;
- if (DI) CallLoc = DI->getLocation();
-
+ unsigned ParamsToSkip) {
// We *have* to evaluate arguments from right to left in the MS C++ ABI,
// because arguments are destroyed left to right in the callee.
if (CGM.getTarget().getCXXABI().areArgsDestroyedLeftToRightInCallee()) {
@@ -2704,8 +2769,6 @@ void CodeGenFunction::EmitCallArgs(CallArgList &Args,
EmitCallArg(Args, *Arg, ArgTypes[I]);
emitNonNullArgCheck(*this, Args.back().RV, ArgTypes[I], Arg->getExprLoc(),
CalleeDecl, ParamsToSkip + I);
- // Restore the debug location.
- if (DI) DI->EmitLocation(Builder, CallLoc, ForceColumnInfo);
}
// Un-reverse the arguments we just evaluated so they match up with the LLVM
@@ -2720,8 +2783,6 @@ void CodeGenFunction::EmitCallArgs(CallArgList &Args,
EmitCallArg(Args, *Arg, ArgTypes[I]);
emitNonNullArgCheck(*this, Args.back().RV, ArgTypes[I], Arg->getExprLoc(),
CalleeDecl, ParamsToSkip + I);
- // Restore the debug location.
- if (DI) DI->EmitLocation(Builder, CallLoc, ForceColumnInfo);
}
}
@@ -2744,8 +2805,22 @@ struct DestroyUnpassedArg : EHScopeStack::Cleanup {
}
+struct DisableDebugLocationUpdates {
+ CodeGenFunction &CGF;
+ bool disabledDebugInfo;
+ DisableDebugLocationUpdates(CodeGenFunction &CGF, const Expr *E) : CGF(CGF) {
+ if ((disabledDebugInfo = isa<CXXDefaultArgExpr>(E) && CGF.getDebugInfo()))
+ CGF.disableDebugInfo();
+ }
+ ~DisableDebugLocationUpdates() {
+ if (disabledDebugInfo)
+ CGF.enableDebugInfo();
+ }
+};
+
void CodeGenFunction::EmitCallArg(CallArgList &args, const Expr *E,
QualType type) {
+ DisableDebugLocationUpdates Dis(*this, E);
if (const ObjCIndirectCopyRestoreExpr *CRE
= dyn_cast<ObjCIndirectCopyRestoreExpr>(E)) {
assert(getLangOpts().ObjCAutoRefCount);
@@ -2900,7 +2975,6 @@ void CodeGenFunction::EmitNoreturnRuntimeCallOrInvoke(llvm::Value *callee,
call->setCallingConv(getRuntimeCC());
Builder.CreateUnreachable();
}
- PGO.setCurrentRegionUnreachable();
}
/// Emits a call or invoke instruction to the given nullary runtime
@@ -2949,7 +3023,7 @@ CodeGenFunction::EmitCallOrInvoke(llvm::Value *Callee,
if (CGM.getLangOpts().ObjCAutoRefCount)
AddObjCARCExceptionMetadata(Inst);
- return Inst;
+ return llvm::CallSite(Inst);
}
/// \brief Store a non-aggregate value to an address to initialize it. For
@@ -2986,7 +3060,7 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo,
// If we're using inalloca, insert the allocation after the stack save.
// FIXME: Do this earlier rather than hacking it in here!
- llvm::Value *ArgMemory = nullptr;
+ llvm::AllocaInst *ArgMemory = nullptr;
if (llvm::StructType *ArgStruct = CallInfo.getArgStruct()) {
llvm::Instruction *IP = CallArgs.getStackBase();
llvm::AllocaInst *AI;
@@ -3015,7 +3089,8 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo,
IRCallArgs[IRFunctionArgs.getSRetArgNo()] = SRetPtr;
} else {
llvm::Value *Addr =
- Builder.CreateStructGEP(ArgMemory, RetAI.getInAllocaFieldIndex());
+ Builder.CreateStructGEP(ArgMemory->getAllocatedType(), ArgMemory,
+ RetAI.getInAllocaFieldIndex());
Builder.CreateStore(SRetPtr, Addr);
}
}
@@ -3049,14 +3124,16 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo,
cast<llvm::Instruction>(RV.getAggregateAddr());
CGBuilderTy::InsertPoint IP = Builder.saveIP();
Builder.SetInsertPoint(Placeholder);
- llvm::Value *Addr = Builder.CreateStructGEP(
- ArgMemory, ArgInfo.getInAllocaFieldIndex());
+ llvm::Value *Addr =
+ Builder.CreateStructGEP(ArgMemory->getAllocatedType(), ArgMemory,
+ ArgInfo.getInAllocaFieldIndex());
Builder.restoreIP(IP);
deferPlaceholderReplacement(Placeholder, Addr);
} else {
// Store the RValue into the argument struct.
llvm::Value *Addr =
- Builder.CreateStructGEP(ArgMemory, ArgInfo.getInAllocaFieldIndex());
+ Builder.CreateStructGEP(ArgMemory->getAllocatedType(), ArgMemory,
+ ArgInfo.getInAllocaFieldIndex());
unsigned AS = Addr->getType()->getPointerAddressSpace();
llvm::Type *MemType = ConvertTypeForMem(I->Ty)->getPointerTo(AS);
// There are some cases where a trivial bitcast is not avoidable. The
@@ -3100,8 +3177,8 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo,
: 0);
if ((!ArgInfo.getIndirectByVal() && I->NeedsCopy) ||
(ArgInfo.getIndirectByVal() && TypeAlign.getQuantity() < Align &&
- llvm::getOrEnforceKnownAlignment(Addr, Align, TD) < Align) ||
- (ArgInfo.getIndirectByVal() && (RVAddrSpace != ArgAddrSpace))) {
+ llvm::getOrEnforceKnownAlignment(Addr, Align, *TD) < Align) ||
+ (ArgInfo.getIndirectByVal() && (RVAddrSpace != ArgAddrSpace))) {
// Create an aligned temporary, and copy to it.
llvm::AllocaInst *AI = CreateMemTemp(I->Ty);
if (Align > AI->getAlignment())
@@ -3158,7 +3235,7 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo,
// If the value is offset in memory, apply the offset now.
if (unsigned Offs = ArgInfo.getDirectOffset()) {
SrcPtr = Builder.CreateBitCast(SrcPtr, Builder.getInt8PtrTy());
- SrcPtr = Builder.CreateConstGEP1_32(SrcPtr, Offs);
+ SrcPtr = Builder.CreateConstGEP1_32(Builder.getInt8Ty(), SrcPtr, Offs);
SrcPtr = Builder.CreateBitCast(SrcPtr,
llvm::PointerType::getUnqual(ArgInfo.getCoerceToType()));
@@ -3190,7 +3267,7 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo,
assert(NumIRArgs == STy->getNumElements());
for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i) {
- llvm::Value *EltPtr = Builder.CreateConstGEP2_32(SrcPtr, 0, i);
+ llvm::Value *EltPtr = Builder.CreateConstGEP2_32(STy, SrcPtr, 0, i);
llvm::LoadInst *LI = Builder.CreateLoad(EltPtr);
// We don't know what we're loading from.
LI->setAlignment(1);
@@ -3300,7 +3377,8 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo,
llvm::BasicBlock *InvokeDest = nullptr;
if (!Attrs.hasAttribute(llvm::AttributeSet::FunctionIndex,
- llvm::Attribute::NoUnwind))
+ llvm::Attribute::NoUnwind) ||
+ currentFunctionUsesSEHTry())
InvokeDest = getInvokeDest();
llvm::CallSite CS;
@@ -3320,6 +3398,12 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo,
Attrs.addAttribute(getLLVMContext(), llvm::AttributeSet::FunctionIndex,
llvm::Attribute::AlwaysInline);
+ // Disable inlining inside SEH __try blocks.
+ if (isSEHTryScope())
+ Attrs =
+ Attrs.addAttribute(getLLVMContext(), llvm::AttributeSet::FunctionIndex,
+ llvm::Attribute::NoInline);
+
CS.setAttributes(Attrs);
CS.setCallingConv(static_cast<llvm::CallingConv::ID>(CallingConv));
@@ -3413,7 +3497,8 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo,
llvm::Value *StorePtr = DestPtr;
if (unsigned Offs = RetAI.getDirectOffset()) {
StorePtr = Builder.CreateBitCast(StorePtr, Builder.getInt8PtrTy());
- StorePtr = Builder.CreateConstGEP1_32(StorePtr, Offs);
+ StorePtr =
+ Builder.CreateConstGEP1_32(Builder.getInt8Ty(), StorePtr, Offs);
StorePtr = Builder.CreateBitCast(StorePtr,
llvm::PointerType::getUnqual(RetAI.getCoerceToType()));
}
diff --git a/lib/CodeGen/CGClass.cpp b/lib/CodeGen/CGClass.cpp
index d72eda95d7cf..cd75da276287 100644
--- a/lib/CodeGen/CGClass.cpp
+++ b/lib/CodeGen/CGClass.cpp
@@ -24,35 +24,36 @@
#include "clang/Basic/TargetBuiltins.h"
#include "clang/CodeGen/CGFunctionInfo.h"
#include "clang/Frontend/CodeGenOptions.h"
+#include "llvm/IR/Intrinsics.h"
using namespace clang;
using namespace CodeGen;
-static CharUnits
-ComputeNonVirtualBaseClassOffset(ASTContext &Context,
+static CharUnits
+ComputeNonVirtualBaseClassOffset(ASTContext &Context,
const CXXRecordDecl *DerivedClass,
CastExpr::path_const_iterator Start,
CastExpr::path_const_iterator End) {
CharUnits Offset = CharUnits::Zero();
-
+
const CXXRecordDecl *RD = DerivedClass;
-
+
for (CastExpr::path_const_iterator I = Start; I != End; ++I) {
const CXXBaseSpecifier *Base = *I;
assert(!Base->isVirtual() && "Should not see virtual bases here!");
// Get the layout.
const ASTRecordLayout &Layout = Context.getASTRecordLayout(RD);
-
- const CXXRecordDecl *BaseDecl =
+
+ const CXXRecordDecl *BaseDecl =
cast<CXXRecordDecl>(Base->getType()->getAs<RecordType>()->getDecl());
-
+
// Add the offset.
Offset += Layout.getBaseClassOffset(BaseDecl);
-
+
RD = BaseDecl;
}
-
+
return Offset;
}
@@ -62,15 +63,15 @@ CodeGenModule::GetNonVirtualBaseClassOffset(const CXXRecordDecl *ClassDecl,
CastExpr::path_const_iterator PathEnd) {
assert(PathBegin != PathEnd && "Base path should not be empty!");
- CharUnits Offset =
+ CharUnits Offset =
ComputeNonVirtualBaseClassOffset(getContext(), ClassDecl,
PathBegin, PathEnd);
if (Offset.isZero())
return nullptr;
- llvm::Type *PtrDiffTy =
+ llvm::Type *PtrDiffTy =
Types.ConvertType(getContext().getPointerDiffType());
-
+
return llvm::ConstantInt::get(PtrDiffTy, Offset.getQuantity());
}
@@ -127,7 +128,7 @@ ApplyNonVirtualAndVirtualOffset(CodeGenFunction &CGF, llvm::Value *ptr,
} else {
baseOffset = virtualOffset;
}
-
+
// Apply the base offset.
ptr = CGF.Builder.CreateBitCast(ptr, CGF.Int8PtrTy);
ptr = CGF.Builder.CreateInBoundsGEP(ptr, baseOffset, "add.ptr");
@@ -149,7 +150,7 @@ llvm::Value *CodeGenFunction::GetAddressOfBaseClass(
// *start* with a step down to the correct virtual base subobject,
// and hence will not require any further steps.
if ((*Start)->isVirtual()) {
- VBase =
+ VBase =
cast<CXXRecordDecl>((*Start)->getType()->getAs<RecordType>()->getDecl());
++Start;
}
@@ -157,7 +158,7 @@ llvm::Value *CodeGenFunction::GetAddressOfBaseClass(
// Compute the static offset of the ultimate destination within its
// allocating subobject (the virtual base, if there is one, or else
// the "complete" object that we see).
- CharUnits NonVirtualOffset =
+ CharUnits NonVirtualOffset =
ComputeNonVirtualBaseClassOffset(getContext(), VBase ? VBase : Derived,
Start, PathEnd);
@@ -172,7 +173,7 @@ llvm::Value *CodeGenFunction::GetAddressOfBaseClass(
}
// Get the base pointer type.
- llvm::Type *BasePtrTy =
+ llvm::Type *BasePtrTy =
ConvertType((PathEnd[-1])->getType())->getPointerTo();
QualType DerivedTy = getContext().getRecordType(Derived);
@@ -197,7 +198,7 @@ llvm::Value *CodeGenFunction::GetAddressOfBaseClass(
origBB = Builder.GetInsertBlock();
llvm::BasicBlock *notNullBB = createBasicBlock("cast.notnull");
endBB = createBasicBlock("cast.end");
-
+
llvm::Value *isNull = Builder.CreateIsNull(Value);
Builder.CreateCondBr(isNull, endBB, notNullBB);
EmitBlock(notNullBB);
@@ -216,10 +217,10 @@ llvm::Value *CodeGenFunction::GetAddressOfBaseClass(
}
// Apply both offsets.
- Value = ApplyNonVirtualAndVirtualOffset(*this, Value,
+ Value = ApplyNonVirtualAndVirtualOffset(*this, Value,
NonVirtualOffset,
VirtualOffset);
-
+
// Cast to the destination type.
Value = Builder.CreateBitCast(Value, BasePtrTy);
@@ -228,13 +229,13 @@ llvm::Value *CodeGenFunction::GetAddressOfBaseClass(
llvm::BasicBlock *notNullBB = Builder.GetInsertBlock();
Builder.CreateBr(endBB);
EmitBlock(endBB);
-
+
llvm::PHINode *PHI = Builder.CreatePHI(BasePtrTy, 2, "cast.result");
PHI->addIncoming(Value, notNullBB);
PHI->addIncoming(llvm::Constant::getNullValue(BasePtrTy), origBB);
Value = PHI;
}
-
+
return Value;
}
@@ -252,7 +253,7 @@ CodeGenFunction::GetAddressOfDerivedClass(llvm::Value *Value,
llvm::Value *NonVirtualOffset =
CGM.GetNonVirtualBaseClassOffset(Derived, PathBegin, PathEnd);
-
+
if (!NonVirtualOffset) {
// No offset, we can just cast back.
return Builder.CreateBitCast(Value, DerivedPtrTy);
@@ -266,12 +267,12 @@ CodeGenFunction::GetAddressOfDerivedClass(llvm::Value *Value,
CastNull = createBasicBlock("cast.null");
CastNotNull = createBasicBlock("cast.notnull");
CastEnd = createBasicBlock("cast.end");
-
+
llvm::Value *IsNull = Builder.CreateIsNull(Value);
Builder.CreateCondBr(IsNull, CastNull, CastNotNull);
EmitBlock(CastNotNull);
}
-
+
// Apply the offset.
Value = Builder.CreateBitCast(Value, Int8PtrTy);
Value = Builder.CreateGEP(Value, Builder.CreateNeg(NonVirtualOffset),
@@ -285,14 +286,14 @@ CodeGenFunction::GetAddressOfDerivedClass(llvm::Value *Value,
EmitBlock(CastNull);
Builder.CreateBr(CastEnd);
EmitBlock(CastEnd);
-
+
llvm::PHINode *PHI = Builder.CreatePHI(Value->getType(), 2);
PHI->addIncoming(Value, CastNotNull);
- PHI->addIncoming(llvm::Constant::getNullValue(Value->getType()),
+ PHI->addIncoming(llvm::Constant::getNullValue(Value->getType()),
CastNull);
Value = PHI;
}
-
+
return Value;
}
@@ -303,7 +304,7 @@ llvm::Value *CodeGenFunction::GetVTTParameter(GlobalDecl GD,
// This constructor/destructor does not need a VTT parameter.
return nullptr;
}
-
+
const CXXRecordDecl *RD = cast<CXXMethodDecl>(CurCodeDecl)->getParent();
const CXXRecordDecl *Base = cast<CXXMethodDecl>(GD.getDecl())->getParent();
@@ -323,15 +324,15 @@ llvm::Value *CodeGenFunction::GetVTTParameter(GlobalDecl GD,
SubVTTIndex = 0;
} else {
const ASTRecordLayout &Layout = getContext().getASTRecordLayout(RD);
- CharUnits BaseOffset = ForVirtualBase ?
- Layout.getVBaseClassOffset(Base) :
+ CharUnits BaseOffset = ForVirtualBase ?
+ Layout.getVBaseClassOffset(Base) :
Layout.getBaseClassOffset(Base);
- SubVTTIndex =
+ SubVTTIndex =
CGM.getVTables().getSubVTTIndex(RD, BaseSubobject(Base, BaseOffset));
assert(SubVTTIndex != 0 && "Sub-VTT index must be greater than zero!");
}
-
+
if (CGM.getCXXABI().NeedsVTTParameter(CurGD)) {
// A VTT parameter was passed to the constructor, use it.
VTT = LoadCXXVTT();
@@ -358,7 +359,7 @@ namespace {
cast<CXXMethodDecl>(CGF.CurCodeDecl)->getParent();
const CXXDestructorDecl *D = BaseClass->getDestructor();
- llvm::Value *Addr =
+ llvm::Value *Addr =
CGF.GetAddressOfDirectBaseInCompleteClass(CGF.LoadCXXThis(),
DerivedClass, BaseClass,
BaseIsVirtual);
@@ -391,7 +392,7 @@ static bool BaseInitializerUsesThis(ASTContext &C, const Expr *Init) {
return Checker.UsesThis;
}
-static void EmitBaseInitializer(CodeGenFunction &CGF,
+static void EmitBaseInitializer(CodeGenFunction &CGF,
const CXXRecordDecl *ClassDecl,
CXXCtorInitializer *BaseInit,
CXXCtorType CtorType) {
@@ -399,7 +400,7 @@ static void EmitBaseInitializer(CodeGenFunction &CGF,
"Must have base initializer!");
llvm::Value *ThisPtr = CGF.LoadCXXThis();
-
+
const Type *BaseType = BaseInit->getBaseClass();
CXXRecordDecl *BaseClassDecl =
cast<CXXRecordDecl>(BaseType->getAs<RecordType>()->getDecl());
@@ -418,7 +419,7 @@ static void EmitBaseInitializer(CodeGenFunction &CGF,
// We can pretend to be a complete class because it only matters for
// virtual bases, and we only do virtual bases for complete ctors.
- llvm::Value *V =
+ llvm::Value *V =
CGF.GetAddressOfDirectBaseInCompleteClass(ThisPtr, ClassDecl,
BaseClassDecl,
isBaseVirtual);
@@ -430,8 +431,8 @@ static void EmitBaseInitializer(CodeGenFunction &CGF,
AggValueSlot::IsNotAliased);
CGF.EmitAggExpr(BaseInit->getInit(), AggSlot);
-
- if (CGF.CGM.getLangOpts().Exceptions &&
+
+ if (CGF.CGM.getLangOpts().Exceptions &&
!BaseClassDecl->hasTrivialDestructor())
CGF.EHStack.pushCleanup<CallBaseDtor>(EHCleanup, BaseClassDecl,
isBaseVirtual);
@@ -490,17 +491,17 @@ static void EmitAggMemberInitializer(CodeGenFunction &CGF,
llvm::Value *IndexVar
= CGF.GetAddrOfLocalVar(ArrayIndexes[Index]);
assert(IndexVar && "Array index variable not loaded");
-
+
// Initialize this index variable to zero.
llvm::Value* Zero
= llvm::Constant::getNullValue(
CGF.ConvertType(CGF.getContext().getSizeType()));
CGF.Builder.CreateStore(Zero, IndexVar);
-
+
// Start the loop with a block that tests the condition.
llvm::BasicBlock *CondBlock = CGF.createBasicBlock("for.cond");
llvm::BasicBlock *AfterFor = CGF.createBasicBlock("for.end");
-
+
CGF.EmitBlock(CondBlock);
llvm::BasicBlock *ForBody = CGF.createBasicBlock("for.body");
@@ -512,7 +513,7 @@ static void EmitAggMemberInitializer(CodeGenFunction &CGF,
llvm::ConstantInt::get(Counter->getType(), NumElements);
llvm::Value *IsLess = CGF.Builder.CreateICmpULT(Counter, NumElementsPtr,
"isless");
-
+
// If the condition is true, execute the body.
CGF.Builder.CreateCondBr(IsLess, ForBody, AfterFor);
@@ -539,6 +540,23 @@ static void EmitAggMemberInitializer(CodeGenFunction &CGF,
CGF.EmitBlock(AfterFor, true);
}
+static bool isMemcpyEquivalentSpecialMember(const CXXMethodDecl *D) {
+ auto *CD = dyn_cast<CXXConstructorDecl>(D);
+ if (!(CD && CD->isCopyOrMoveConstructor()) &&
+ !D->isCopyAssignmentOperator() && !D->isMoveAssignmentOperator())
+ return false;
+
+ // We can emit a memcpy for a trivial copy or move constructor/assignment.
+ if (D->isTrivial() && !D->getParent()->mayInsertExtraPadding())
+ return true;
+
+ // We *must* emit a memcpy for a defaulted union copy or move op.
+ if (D->getParent()->isUnion() && D->isDefaulted())
+ return true;
+
+ return false;
+}
+
static void EmitMemberInitializer(CodeGenFunction &CGF,
const CXXRecordDecl *ClassDecl,
CXXCtorInitializer *MemberInit,
@@ -548,7 +566,7 @@ static void EmitMemberInitializer(CodeGenFunction &CGF,
assert(MemberInit->isAnyMemberInitializer() &&
"Must have member initializer!");
assert(MemberInit->getInit() && "Must have initializer!");
-
+
// non-static data member initializers.
FieldDecl *Field = MemberInit->getAnyMember();
QualType FieldType = Field->getType();
@@ -580,14 +598,14 @@ static void EmitMemberInitializer(CodeGenFunction &CGF,
QualType BaseElementTy = CGF.getContext().getBaseElementType(Array);
CXXConstructExpr *CE = dyn_cast<CXXConstructExpr>(MemberInit->getInit());
if (BaseElementTy.isPODType(CGF.getContext()) ||
- (CE && CE->getConstructor()->isTrivial())) {
+ (CE && isMemcpyEquivalentSpecialMember(CE->getConstructor()))) {
unsigned SrcArgIndex =
CGF.CGM.getCXXABI().getSrcArgforCopyCtor(Constructor, Args);
llvm::Value *SrcPtr
= CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(Args[SrcArgIndex]));
LValue ThisRHSLV = CGF.MakeNaturalAlignAddrLValue(SrcPtr, RecordTy);
LValue Src = CGF.EmitLValueForFieldInitialization(ThisRHSLV, Field);
-
+
// Copy the aggregate.
CGF.EmitAggregateCopy(LHS.getAddress(), Src.getAddress(), FieldType,
LHS.isVolatileQualified());
@@ -621,28 +639,28 @@ void CodeGenFunction::EmitInitializerForField(
llvm::Value *ArrayIndexVar = nullptr;
if (ArrayIndexes.size()) {
llvm::Type *SizeTy = ConvertType(getContext().getSizeType());
-
+
// The LHS is a pointer to the first object we'll be constructing, as
// a flat array.
QualType BaseElementTy = getContext().getBaseElementType(FieldType);
llvm::Type *BasePtr = ConvertType(BaseElementTy);
BasePtr = llvm::PointerType::getUnqual(BasePtr);
- llvm::Value *BaseAddrPtr = Builder.CreateBitCast(LHS.getAddress(),
+ llvm::Value *BaseAddrPtr = Builder.CreateBitCast(LHS.getAddress(),
BasePtr);
LHS = MakeAddrLValue(BaseAddrPtr, BaseElementTy);
-
+
// Create an array index that will be used to walk over all of the
// objects we're constructing.
ArrayIndexVar = CreateTempAlloca(SizeTy, "object.index");
llvm::Value *Zero = llvm::Constant::getNullValue(SizeTy);
Builder.CreateStore(Zero, ArrayIndexVar);
-
-
+
+
// Emit the block variables for the array indices, if any.
for (unsigned I = 0, N = ArrayIndexes.size(); I != N; ++I)
EmitAutoVarDecl(*ArrayIndexes[I]);
}
-
+
EmitAggMemberInitializer(*this, LHS, Init, ArrayIndexVar, FieldType,
ArrayIndexes, 0);
}
@@ -762,9 +780,9 @@ void CodeGenFunction::EmitAsanPrologueOrEpilogue(bool Prologue) {
if (PoisonSize < AsanAlignment || !SSV[i].Size ||
(NextField % AsanAlignment) != 0)
continue;
- Builder.CreateCall2(
- F, Builder.CreateAdd(ThisPtr, Builder.getIntN(PtrSize, EndOffset)),
- Builder.getIntN(PtrSize, PoisonSize));
+ Builder.CreateCall(
+ F, {Builder.CreateAdd(ThisPtr, Builder.getIntN(PtrSize, EndOffset)),
+ Builder.getIntN(PtrSize, PoisonSize)});
}
}
@@ -796,8 +814,7 @@ void CodeGenFunction::EmitConstructorBody(FunctionArgList &Args) {
if (IsTryBody)
EnterCXXTryStmt(*cast<CXXTryStmt>(Body), true);
- RegionCounter Cnt = getPGORegionCounter(Body);
- Cnt.beginRegion(Builder);
+ incrementProfileCounter(Body);
RunCleanupsScope RunCleanups(*this);
@@ -850,7 +867,7 @@ namespace {
public:
FieldMemcpyizer(CodeGenFunction &CGF, const CXXRecordDecl *ClassDecl,
const VarDecl *SrcRec)
- : CGF(CGF), ClassDecl(ClassDecl), SrcRec(SrcRec),
+ : CGF(CGF), ClassDecl(ClassDecl), SrcRec(SrcRec),
RecLayout(CGF.getContext().getASTRecordLayout(ClassDecl)),
FirstField(nullptr), LastField(nullptr), FirstFieldOffset(0),
LastFieldOffset(0), LastAddedFieldIndex(0) {}
@@ -876,7 +893,7 @@ namespace {
unsigned LastFieldSize =
LastField->isBitField() ?
LastField->getBitWidthValue(CGF.getContext()) :
- CGF.getContext().getTypeSize(LastField->getType());
+ CGF.getContext().getTypeSize(LastField->getType());
uint64_t MemcpySizeBits =
LastFieldOffset + LastFieldSize - FirstByteOffset +
CGF.getContext().getCharWidth() - 1;
@@ -1021,8 +1038,8 @@ namespace {
QualType FieldType = Field->getType();
CXXConstructExpr *CE = dyn_cast<CXXConstructExpr>(MemberInit->getInit());
- // Bail out on non-POD, not-trivially-constructable members.
- if (!(CE && CE->getConstructor()->isTrivial()) &&
+ // Bail out on non-memcpyable, not-trivially-copyable members.
+ if (!(CE && isMemcpyEquivalentSpecialMember(CE->getConstructor())) &&
!(FieldType.isTriviallyCopyableType(CGF.getContext()) ||
FieldType->isReferenceType()))
return false;
@@ -1127,9 +1144,7 @@ namespace {
return Field;
} else if (CXXMemberCallExpr *MCE = dyn_cast<CXXMemberCallExpr>(S)) {
CXXMethodDecl *MD = dyn_cast<CXXMethodDecl>(MCE->getCalleeDecl());
- if (!(MD && (MD->isCopyAssignmentOperator() ||
- MD->isMoveAssignmentOperator()) &&
- MD->isTrivial()))
+ if (!(MD && isMemcpyEquivalentSpecialMember(MD)))
return nullptr;
MemberExpr *IOA = dyn_cast<MemberExpr>(MCE->getImplicitObjectArgument());
if (!IOA)
@@ -1189,7 +1204,7 @@ namespace {
if (F) {
addMemcpyableField(F);
AggregatedStmts.push_back(S);
- } else {
+ } else {
emitAggregatedStmts();
CGF.EmitStmt(S);
}
@@ -1274,7 +1289,7 @@ static bool
FieldHasTrivialDestructorBody(ASTContext &Context, const FieldDecl *Field);
static bool
-HasTrivialDestructorBody(ASTContext &Context,
+HasTrivialDestructorBody(ASTContext &Context,
const CXXRecordDecl *BaseClassDecl,
const CXXRecordDecl *MostDerivedClassDecl)
{
@@ -1309,7 +1324,7 @@ HasTrivialDestructorBody(ASTContext &Context,
cast<CXXRecordDecl>(I.getType()->castAs<RecordType>()->getDecl());
if (!HasTrivialDestructorBody(Context, VirtualBase,
MostDerivedClassDecl))
- return false;
+ return false;
}
}
@@ -1325,7 +1340,7 @@ FieldHasTrivialDestructorBody(ASTContext &Context,
const RecordType *RT = FieldBaseElementType->getAs<RecordType>();
if (!RT)
return true;
-
+
CXXRecordDecl *FieldClassDecl = cast<CXXRecordDecl>(RT->getDecl());
return HasTrivialDestructorBody(Context, FieldClassDecl, FieldClassDecl);
}
@@ -1351,6 +1366,10 @@ void CodeGenFunction::EmitDestructorBody(FunctionArgList &Args) {
const CXXDestructorDecl *Dtor = cast<CXXDestructorDecl>(CurGD.getDecl());
CXXDtorType DtorType = CurGD.getDtorType();
+ Stmt *Body = Dtor->getBody();
+ if (Body)
+ incrementProfileCounter(Body);
+
// The call to operator delete in a deleting destructor happens
// outside of the function-try-block, which means it's always
// possible to delegate the destructor body to the complete
@@ -1363,8 +1382,6 @@ void CodeGenFunction::EmitDestructorBody(FunctionArgList &Args) {
return;
}
- Stmt *Body = Dtor->getBody();
-
// If the body is a function-try-block, enter the try before
// anything else.
bool isTryBody = (Body && isa<CXXTryStmt>(Body));
@@ -1374,11 +1391,11 @@ void CodeGenFunction::EmitDestructorBody(FunctionArgList &Args) {
// Enter the epilogue cleanups.
RunCleanupsScope DtorEpilogue(*this);
-
+
// If this is the complete variant, just invoke the base variant;
// the epilogue will destruct the virtual bases. But we can't do
// this optimization if the body is a function-try-block, because
- // we'd introduce *two* handler blocks. In the Microsoft ABI, we
+ // we'd introduce *two* handler blocks. In the Microsoft ABI, we
// always delegate because we might not have a definition in this TU.
switch (DtorType) {
case Dtor_Comdat:
@@ -1399,13 +1416,10 @@ void CodeGenFunction::EmitDestructorBody(FunctionArgList &Args) {
break;
}
// Fallthrough: act like we're in the base variant.
-
+
case Dtor_Base:
assert(Body);
- RegionCounter Cnt = getPGORegionCounter(Body);
- Cnt.beginRegion(Builder);
-
// Enter the cleanup scopes for fields and non-virtual bases.
EnterDtorCleanups(Dtor, Dtor_Base);
@@ -1447,7 +1461,7 @@ void CodeGenFunction::emitImplicitAssignmentOperatorBody(FunctionArgList &Args)
AssignmentMemcpyizer AM(*this, AssignOp, Args);
for (auto *I : RootCS->body())
- AM.emitAssignment(I);
+ AM.emitAssignment(I);
AM.finish();
}
@@ -1508,7 +1522,7 @@ namespace {
LValue ThisLV = CGF.MakeAddrLValue(thisValue, RecordTy);
LValue LV = CGF.EmitLValueForField(ThisLV, field);
assert(LV.isSimple());
-
+
CGF.emitDestroy(LV.getAddress(), field->getType(), destroyer,
flags.isForNormalCleanup() && useEHCleanupForArray);
}
@@ -1526,7 +1540,7 @@ void CodeGenFunction::EnterDtorCleanups(const CXXDestructorDecl *DD,
// The deleting-destructor phase just needs to call the appropriate
// operator delete that Sema picked up.
if (DtorType == Dtor_Deleting) {
- assert(DD->getOperatorDelete() &&
+ assert(DD->getOperatorDelete() &&
"operator delete missing - EnterDtorCleanups");
if (CXXStructorImplicitParamValue) {
// If there is an implicit param to the deleting dtor, it's a boolean
@@ -1553,7 +1567,7 @@ void CodeGenFunction::EnterDtorCleanups(const CXXDestructorDecl *DD,
for (const auto &Base : ClassDecl->vbases()) {
CXXRecordDecl *BaseClassDecl
= cast<CXXRecordDecl>(Base.getType()->getAs<RecordType>()->getDecl());
-
+
// Ignore trivial destructors.
if (BaseClassDecl->hasTrivialDestructor())
continue;
@@ -1567,15 +1581,15 @@ void CodeGenFunction::EnterDtorCleanups(const CXXDestructorDecl *DD,
}
assert(DtorType == Dtor_Base);
-
+
// Destroy non-virtual bases.
for (const auto &Base : ClassDecl->bases()) {
// Ignore virtual bases.
if (Base.isVirtual())
continue;
-
+
CXXRecordDecl *BaseClassDecl = Base.getType()->getAsCXXRecordDecl();
-
+
// Ignore trivial destructors.
if (BaseClassDecl->hasTrivialDestructor())
continue;
@@ -1656,7 +1670,7 @@ void CodeGenFunction::EmitCXXAggrConstructorCall(const CXXConstructorDecl *ctor,
zeroCheckBranch = Builder.CreateCondBr(iszero, loopBB, loopBB);
EmitBlock(loopBB);
}
-
+
// Find the end of the array.
llvm::Value *arrayEnd = Builder.CreateInBoundsGEP(arrayBegin, numElements,
"arrayctor.end");
@@ -1676,15 +1690,15 @@ void CodeGenFunction::EmitCXXAggrConstructorCall(const CXXConstructorDecl *ctor,
// Zero initialize the storage, if requested.
if (zeroInitialize)
EmitNullInitialization(cur, type);
-
- // C++ [class.temporary]p4:
+
+ // C++ [class.temporary]p4:
// There are two contexts in which temporaries are destroyed at a different
// point than the end of the full-expression. The first context is when a
- // default constructor is called to initialize an element of an array.
- // If the constructor has one or more default arguments, the destruction of
- // every temporary created in a default argument expression is sequenced
+ // default constructor is called to initialize an element of an array.
+ // If the constructor has one or more default arguments, the destruction of
+ // every temporary created in a default argument expression is sequenced
// before the construction of the next array element, if any.
-
+
{
RunCleanupsScope Scope(*this);
@@ -1733,33 +1747,32 @@ void CodeGenFunction::EmitCXXConstructorCall(const CXXConstructorDecl *D,
bool ForVirtualBase,
bool Delegating, llvm::Value *This,
const CXXConstructExpr *E) {
- // If this is a trivial constructor, just emit what's needed.
- if (D->isTrivial() && !D->getParent()->mayInsertExtraPadding()) {
- if (E->getNumArgs() == 0) {
- // Trivial default constructor, no codegen required.
- assert(D->isDefaultConstructor() &&
- "trivial 0-arg ctor not a default ctor");
- return;
- }
+ // C++11 [class.mfct.non-static]p2:
+ // If a non-static member function of a class X is called for an object that
+ // is not of type X, or of a type derived from X, the behavior is undefined.
+ // FIXME: Provide a source location here.
+ EmitTypeCheck(CodeGenFunction::TCK_ConstructorCall, SourceLocation(), This,
+ getContext().getRecordType(D->getParent()));
+
+ if (D->isTrivial() && D->isDefaultConstructor()) {
+ assert(E->getNumArgs() == 0 && "trivial default ctor with args");
+ return;
+ }
+ // If this is a trivial constructor, just emit what's needed. If this is a
+ // union copy constructor, we must emit a memcpy, because the AST does not
+ // model that copy.
+ if (isMemcpyEquivalentSpecialMember(D)) {
assert(E->getNumArgs() == 1 && "unexpected argcount for trivial ctor");
- assert(D->isCopyOrMoveConstructor() &&
- "trivial 1-arg ctor not a copy/move ctor");
const Expr *Arg = E->getArg(0);
- QualType Ty = Arg->getType();
+ QualType SrcTy = Arg->getType();
llvm::Value *Src = EmitLValue(Arg).getAddress();
- EmitAggregateCopy(This, Src, Ty);
+ QualType DestTy = getContext().getTypeDeclType(D->getParent());
+ EmitAggregateCopyCtor(This, Src, DestTy, SrcTy);
return;
}
- // C++11 [class.mfct.non-static]p2:
- // If a non-static member function of a class X is called for an object that
- // is not of type X, or of a type derived from X, the behavior is undefined.
- // FIXME: Provide a source location here.
- EmitTypeCheck(CodeGenFunction::TCK_ConstructorCall, SourceLocation(), This,
- getContext().getRecordType(D->getParent()));
-
CallArgList Args;
// Push the this ptr.
@@ -1784,25 +1797,26 @@ void
CodeGenFunction::EmitSynthesizedCXXCopyCtorCall(const CXXConstructorDecl *D,
llvm::Value *This, llvm::Value *Src,
const CXXConstructExpr *E) {
- if (D->isTrivial() &&
- !D->getParent()->mayInsertExtraPadding()) {
+ if (isMemcpyEquivalentSpecialMember(D)) {
assert(E->getNumArgs() == 1 && "unexpected argcount for trivial ctor");
assert(D->isCopyOrMoveConstructor() &&
"trivial 1-arg ctor not a copy/move ctor");
- EmitAggregateCopy(This, Src, E->arg_begin()->getType());
+ EmitAggregateCopyCtor(This, Src,
+ getContext().getTypeDeclType(D->getParent()),
+ E->arg_begin()->getType());
return;
}
llvm::Value *Callee = CGM.getAddrOfCXXStructor(D, StructorType::Complete);
assert(D->isInstance() &&
"Trying to emit a member call expr on a static method!");
-
+
const FunctionProtoType *FPT = D->getType()->castAs<FunctionProtoType>();
-
+
CallArgList Args;
-
+
// Push the this ptr.
Args.add(RValue::get(This), D->getThisType(getContext()));
-
+
// Push the src ptr.
QualType QT = *(FPT->param_type_begin());
llvm::Type *t = CGM.getTypes().ConvertType(QT);
@@ -1945,10 +1959,18 @@ void CodeGenFunction::PushDestructorCleanup(QualType T, llvm::Value *Addr) {
}
void
-CodeGenFunction::InitializeVTablePointer(BaseSubobject Base,
+CodeGenFunction::InitializeVTablePointer(BaseSubobject Base,
const CXXRecordDecl *NearestVBase,
CharUnits OffsetFromNearestVBase,
const CXXRecordDecl *VTableClass) {
+ const CXXRecordDecl *RD = Base.getBase();
+
+ // Don't initialize the vtable pointer if the class is marked with the
+ // 'novtable' attribute.
+ if ((RD == VTableClass || RD == NearestVBase) &&
+ VTableClass->hasAttr<MSNoVTableAttr>())
+ return;
+
// Compute the address point.
bool NeedsVirtualOffset;
llvm::Value *VTableAddressPoint =
@@ -1960,7 +1982,7 @@ CodeGenFunction::InitializeVTablePointer(BaseSubobject Base,
// Compute where to store the address point.
llvm::Value *VirtualOffset = nullptr;
CharUnits NonVirtualOffset = CharUnits::Zero();
-
+
if (NeedsVirtualOffset) {
// We need to use the virtual base offset offset because the virtual base
// might have a different offset in the most derived class.
@@ -1973,12 +1995,12 @@ CodeGenFunction::InitializeVTablePointer(BaseSubobject Base,
// We can just use the base offset in the complete class.
NonVirtualOffset = Base.getBaseOffset();
}
-
+
// Apply the offsets.
llvm::Value *VTableField = LoadCXXThis();
-
+
if (!NonVirtualOffset.isZero() || VirtualOffset)
- VTableField = ApplyNonVirtualAndVirtualOffset(*this, VTableField,
+ VTableField = ApplyNonVirtualAndVirtualOffset(*this, VTableField,
NonVirtualOffset,
VirtualOffset);
@@ -1995,7 +2017,7 @@ CodeGenFunction::InitializeVTablePointer(BaseSubobject Base,
}
void
-CodeGenFunction::InitializeVTablePointers(BaseSubobject Base,
+CodeGenFunction::InitializeVTablePointers(BaseSubobject Base,
const CXXRecordDecl *NearestVBase,
CharUnits OffsetFromNearestVBase,
bool BaseIsNonVirtualPrimaryBase,
@@ -2008,7 +2030,7 @@ CodeGenFunction::InitializeVTablePointers(BaseSubobject Base,
InitializeVTablePointer(Base, NearestVBase, OffsetFromNearestVBase,
VTableClass);
}
-
+
const CXXRecordDecl *RD = Base.getBase();
// Traverse bases.
@@ -2029,7 +2051,7 @@ CodeGenFunction::InitializeVTablePointers(BaseSubobject Base,
if (!VBases.insert(BaseDecl).second)
continue;
- const ASTRecordLayout &Layout =
+ const ASTRecordLayout &Layout =
getContext().getASTRecordLayout(VTableClass);
BaseOffset = Layout.getVBaseClassOffset(BaseDecl);
@@ -2039,15 +2061,15 @@ CodeGenFunction::InitializeVTablePointers(BaseSubobject Base,
const ASTRecordLayout &Layout = getContext().getASTRecordLayout(RD);
BaseOffset = Base.getBaseOffset() + Layout.getBaseClassOffset(BaseDecl);
- BaseOffsetFromNearestVBase =
+ BaseOffsetFromNearestVBase =
OffsetFromNearestVBase + Layout.getBaseClassOffset(BaseDecl);
BaseDeclIsNonVirtualPrimaryBase = Layout.getPrimaryBase() == BaseDecl;
}
-
- InitializeVTablePointers(BaseSubobject(BaseDecl, BaseOffset),
+
+ InitializeVTablePointers(BaseSubobject(BaseDecl, BaseOffset),
I.isVirtual() ? BaseDecl : NearestVBase,
BaseOffsetFromNearestVBase,
- BaseDeclIsNonVirtualPrimaryBase,
+ BaseDeclIsNonVirtualPrimaryBase,
VTableClass, VBases);
}
}
@@ -2059,7 +2081,7 @@ void CodeGenFunction::InitializeVTablePointers(const CXXRecordDecl *RD) {
// Initialize the vtable pointers for this class and all of its bases.
VisitedVirtualBasesSetTy VBases;
- InitializeVTablePointers(BaseSubobject(RD, CharUnits::Zero()),
+ InitializeVTablePointers(BaseSubobject(RD, CharUnits::Zero()),
/*NearestVBase=*/nullptr,
/*OffsetFromNearestVBase=*/CharUnits::Zero(),
/*BaseIsNonVirtualPrimaryBase=*/false, RD, VBases);
@@ -2076,6 +2098,128 @@ llvm::Value *CodeGenFunction::GetVTablePtr(llvm::Value *This,
return VTable;
}
+// If a class has a single non-virtual base and does not introduce or override
+// virtual member functions or fields, it will have the same layout as its base.
+// This function returns the least derived such class.
+//
+// Casting an instance of a base class to such a derived class is technically
+// undefined behavior, but it is a relatively common hack for introducing member
+// functions on class instances with specific properties (e.g. llvm::Operator)
+// that works under most compilers and should not have security implications, so
+// we allow it by default. It can be disabled with -fsanitize=cfi-cast-strict.
+static const CXXRecordDecl *
+LeastDerivedClassWithSameLayout(const CXXRecordDecl *RD) {
+ if (!RD->field_empty())
+ return RD;
+
+ if (RD->getNumVBases() != 0)
+ return RD;
+
+ if (RD->getNumBases() != 1)
+ return RD;
+
+ for (const CXXMethodDecl *MD : RD->methods()) {
+ if (MD->isVirtual()) {
+ // Virtual member functions are only ok if they are implicit destructors
+ // because the implicit destructor will have the same semantics as the
+ // base class's destructor if no fields are added.
+ if (isa<CXXDestructorDecl>(MD) && MD->isImplicit())
+ continue;
+ return RD;
+ }
+ }
+
+ return LeastDerivedClassWithSameLayout(
+ RD->bases_begin()->getType()->getAsCXXRecordDecl());
+}
+
+void CodeGenFunction::EmitVTablePtrCheckForCall(const CXXMethodDecl *MD,
+ llvm::Value *VTable) {
+ const CXXRecordDecl *ClassDecl = MD->getParent();
+ if (!SanOpts.has(SanitizerKind::CFICastStrict))
+ ClassDecl = LeastDerivedClassWithSameLayout(ClassDecl);
+
+ EmitVTablePtrCheck(ClassDecl, VTable);
+}
+
+void CodeGenFunction::EmitVTablePtrCheckForCast(QualType T,
+ llvm::Value *Derived,
+ bool MayBeNull) {
+ if (!getLangOpts().CPlusPlus)
+ return;
+
+ auto *ClassTy = T->getAs<RecordType>();
+ if (!ClassTy)
+ return;
+
+ const CXXRecordDecl *ClassDecl = cast<CXXRecordDecl>(ClassTy->getDecl());
+
+ if (!ClassDecl->isCompleteDefinition() || !ClassDecl->isDynamicClass())
+ return;
+
+ SmallString<64> MangledName;
+ llvm::raw_svector_ostream Out(MangledName);
+ CGM.getCXXABI().getMangleContext().mangleCXXRTTI(T.getUnqualifiedType(),
+ Out);
+
+ // Blacklist based on the mangled type.
+ if (CGM.getContext().getSanitizerBlacklist().isBlacklistedType(Out.str()))
+ return;
+
+ if (!SanOpts.has(SanitizerKind::CFICastStrict))
+ ClassDecl = LeastDerivedClassWithSameLayout(ClassDecl);
+
+ llvm::BasicBlock *ContBlock = 0;
+
+ if (MayBeNull) {
+ llvm::Value *DerivedNotNull =
+ Builder.CreateIsNotNull(Derived, "cast.nonnull");
+
+ llvm::BasicBlock *CheckBlock = createBasicBlock("cast.check");
+ ContBlock = createBasicBlock("cast.cont");
+
+ Builder.CreateCondBr(DerivedNotNull, CheckBlock, ContBlock);
+
+ EmitBlock(CheckBlock);
+ }
+
+ llvm::Value *VTable = GetVTablePtr(Derived, Int8PtrTy);
+ EmitVTablePtrCheck(ClassDecl, VTable);
+
+ if (MayBeNull) {
+ Builder.CreateBr(ContBlock);
+ EmitBlock(ContBlock);
+ }
+}
+
+void CodeGenFunction::EmitVTablePtrCheck(const CXXRecordDecl *RD,
+ llvm::Value *VTable) {
+ // FIXME: Add blacklisting scheme.
+ if (RD->isInStdNamespace())
+ return;
+
+ std::string OutName;
+ llvm::raw_string_ostream Out(OutName);
+ CGM.getCXXABI().getMangleContext().mangleCXXVTableBitSet(RD, Out);
+
+ llvm::Value *BitSetName = llvm::MetadataAsValue::get(
+ getLLVMContext(), llvm::MDString::get(getLLVMContext(), Out.str()));
+
+ llvm::Value *BitSetTest = Builder.CreateCall(
+ CGM.getIntrinsic(llvm::Intrinsic::bitset_test),
+ {Builder.CreateBitCast(VTable, CGM.Int8PtrTy), BitSetName});
+
+ llvm::BasicBlock *ContBlock = createBasicBlock("vtable.check.cont");
+ llvm::BasicBlock *TrapBlock = createBasicBlock("vtable.check.trap");
+
+ Builder.CreateCondBr(BitSetTest, ContBlock, TrapBlock);
+
+ EmitBlock(TrapBlock);
+ Builder.CreateCall(CGM.getIntrinsic(llvm::Intrinsic::trap), {});
+ Builder.CreateUnreachable();
+
+ EmitBlock(ContBlock);
+}
// FIXME: Ideally Expr::IgnoreParenNoopCasts should do this, but it doesn't do
// quite what we want.
@@ -2140,7 +2284,7 @@ CodeGenFunction::CanDevirtualizeMemberFunctionCall(const Expr *Base,
// This is a record decl. We know the type and can devirtualize it.
return VD->getType()->isRecordType();
}
-
+
return false;
}
@@ -2154,14 +2298,14 @@ CodeGenFunction::CanDevirtualizeMemberFunctionCall(const Expr *Base,
// We can always devirtualize calls on temporary object expressions.
if (isa<CXXConstructExpr>(Base))
return true;
-
+
// And calls on bound temporaries.
if (isa<CXXBindTemporaryExpr>(Base))
return true;
-
+
// Check if this is a call expr that returns a record type.
if (const CallExpr *CE = dyn_cast<CallExpr>(Base))
- return CE->getCallReturnType()->isRecordType();
+ return CE->getCallReturnType(getContext())->isRecordType();
// We can't devirtualize the call.
return false;
@@ -2190,7 +2334,7 @@ void CodeGenFunction::EmitForwardingCallToLambda(
// We don't need to separately arrange the call arguments because
// the call can't be variadic anyway --- it's impossible to forward
// variadic arguments.
-
+
// Now emit our call.
RValue RV = EmitCall(calleeFnInfo, callee, returnSlot,
callArgs, callOperator);
@@ -2218,7 +2362,7 @@ void CodeGenFunction::EmitLambdaBlockInvokeBody() {
for (auto param : BD->params())
EmitDelegateCallArg(CallArgs, param, param->getLocStart());
- assert(!Lambda->isGenericLambda() &&
+ assert(!Lambda->isGenericLambda() &&
"generic lambda interconversion to block not implemented");
EmitForwardingCallToLambda(Lambda->getLambdaCallOperator(), CallArgs);
}
@@ -2256,7 +2400,7 @@ void CodeGenFunction::EmitLambdaDelegatingInvokeBody(const CXXMethodDecl *MD) {
const TemplateArgumentList *TAL = MD->getTemplateSpecializationArgs();
FunctionTemplateDecl *CallOpTemplate = CallOp->getDescribedFunctionTemplate();
void *InsertPos = nullptr;
- FunctionDecl *CorrespondingCallOpSpecialization =
+ FunctionDecl *CorrespondingCallOpSpecialization =
CallOpTemplate->findSpecialization(TAL->asArray(), InsertPos);
assert(CorrespondingCallOpSpecialization);
CallOp = cast<CXXMethodDecl>(CorrespondingCallOpSpecialization);
diff --git a/lib/CodeGen/CGCleanup.cpp b/lib/CodeGen/CGCleanup.cpp
index 18ed3e543d20..d97e40554ef2 100644
--- a/lib/CodeGen/CGCleanup.cpp
+++ b/lib/CodeGen/CGCleanup.cpp
@@ -52,8 +52,10 @@ DominatingValue<RValue>::saved_type::save(CodeGenFunction &CGF, RValue rv) {
llvm::StructType::get(V.first->getType(), V.second->getType(),
(void*) nullptr);
llvm::Value *addr = CGF.CreateTempAlloca(ComplexTy, "saved-complex");
- CGF.Builder.CreateStore(V.first, CGF.Builder.CreateStructGEP(addr, 0));
- CGF.Builder.CreateStore(V.second, CGF.Builder.CreateStructGEP(addr, 1));
+ CGF.Builder.CreateStore(V.first,
+ CGF.Builder.CreateStructGEP(ComplexTy, addr, 0));
+ CGF.Builder.CreateStore(V.second,
+ CGF.Builder.CreateStructGEP(ComplexTy, addr, 1));
return saved_type(addr, ComplexAddress);
}
@@ -82,9 +84,9 @@ RValue DominatingValue<RValue>::saved_type::restore(CodeGenFunction &CGF) {
return RValue::getAggregate(CGF.Builder.CreateLoad(Value));
case ComplexAddress: {
llvm::Value *real =
- CGF.Builder.CreateLoad(CGF.Builder.CreateStructGEP(Value, 0));
+ CGF.Builder.CreateLoad(CGF.Builder.CreateStructGEP(nullptr, Value, 0));
llvm::Value *imag =
- CGF.Builder.CreateLoad(CGF.Builder.CreateStructGEP(Value, 1));
+ CGF.Builder.CreateLoad(CGF.Builder.CreateStructGEP(nullptr, Value, 1));
return RValue::getComplex(real, imag);
}
}
@@ -123,6 +125,17 @@ char *EHScopeStack::allocate(size_t Size) {
return StartOfData;
}
+bool EHScopeStack::containsOnlyLifetimeMarkers(
+ EHScopeStack::stable_iterator Old) const {
+ for (EHScopeStack::iterator it = begin(); stabilize(it) != Old; it++) {
+ EHCleanupScope *cleanup = dyn_cast<EHCleanupScope>(&*it);
+ if (!cleanup || !cleanup->isLifetimeMarker())
+ return false;
+ }
+
+ return true;
+}
+
EHScopeStack::stable_iterator
EHScopeStack::getInnermostActiveNormalCleanup() const {
for (stable_iterator si = getInnermostNormalCleanup(), se = stable_end();
@@ -469,8 +482,14 @@ static void EmitCleanup(CodeGenFunction &CGF,
EHScopeStack::Cleanup *Fn,
EHScopeStack::Cleanup::Flags flags,
llvm::Value *ActiveFlag) {
- // EH cleanups always occur within a terminate scope.
- if (flags.isForEHCleanup()) CGF.EHStack.pushTerminate();
+ // Itanium EH cleanups occur within a terminate scope. Microsoft SEH doesn't
+ // have this behavior, and the Microsoft C++ runtime will call terminate for
+ // us if the cleanup throws.
+ bool PushedTerminate = false;
+ if (flags.isForEHCleanup() && !CGF.getTarget().getCXXABI().isMicrosoft()) {
+ CGF.EHStack.pushTerminate();
+ PushedTerminate = true;
+ }
// If there's an active flag, load it and skip the cleanup if it's
// false.
@@ -493,7 +512,8 @@ static void EmitCleanup(CodeGenFunction &CGF,
CGF.EmitBlock(ContBB);
// Leave the terminate scope.
- if (flags.isForEHCleanup()) CGF.EHStack.popTerminate();
+ if (PushedTerminate)
+ CGF.EHStack.popTerminate();
}
static void ForwardPrebranchedFallthrough(llvm::BasicBlock *Exit,
@@ -739,7 +759,15 @@ void CodeGenFunction::PopCleanupBlock(bool FallthroughIsBranchThrough) {
Scope.getNumBranchAfters() == 1) {
assert(!BranchThroughDest || !IsActive);
- // TODO: clean up the possibly dead stores to the cleanup dest slot.
+ // Clean up the possibly dead store to the cleanup dest slot.
+ llvm::Instruction *NormalCleanupDestSlot =
+ cast<llvm::Instruction>(getNormalCleanupDestSlot());
+ if (NormalCleanupDestSlot->hasOneUse()) {
+ NormalCleanupDestSlot->user_back()->eraseFromParent();
+ NormalCleanupDestSlot->eraseFromParent();
+ NormalCleanupDest = nullptr;
+ }
+
llvm::BasicBlock *BranchAfter = Scope.getBranchAfterBlock(0);
InstsToAppend.push_back(llvm::BranchInst::Create(BranchAfter));
@@ -861,8 +889,6 @@ void CodeGenFunction::PopCleanupBlock(bool FallthroughIsBranchThrough) {
// Emit the EH cleanup if required.
if (RequiresEHCleanup) {
- ApplyDebugLocation AutoRestoreLocation(*this, CurEHLocation);
-
CGBuilderTy::InsertPoint SavedIP = Builder.saveAndClearIP();
EmitBlock(EHEntry);
diff --git a/lib/CodeGen/CGCleanup.h b/lib/CodeGen/CGCleanup.h
index dd156c696ad3..81c64123dfdb 100644
--- a/lib/CodeGen/CGCleanup.h
+++ b/lib/CodeGen/CGCleanup.h
@@ -62,6 +62,9 @@ protected:
/// Whether this cleanup is currently active.
unsigned IsActive : 1;
+ /// Whether this cleanup is a lifetime marker
+ unsigned IsLifetimeMarker : 1;
+
/// Whether the normal cleanup should test the activation flag.
unsigned TestFlagInNormalCleanup : 1;
@@ -75,7 +78,7 @@ protected:
/// The number of fixups required by enclosing scopes (not including
/// this one). If this is the top cleanup scope, all the fixups
/// from this index onwards belong to this scope.
- unsigned FixupDepth : 32 - 17 - NumCommonBits; // currently 13
+ unsigned FixupDepth : 32 - 18 - NumCommonBits; // currently 13
};
class FilterBitFields {
@@ -272,6 +275,7 @@ public:
CleanupBits.IsNormalCleanup = isNormal;
CleanupBits.IsEHCleanup = isEH;
CleanupBits.IsActive = isActive;
+ CleanupBits.IsLifetimeMarker = false;
CleanupBits.TestFlagInNormalCleanup = false;
CleanupBits.TestFlagInEHCleanup = false;
CleanupBits.CleanupSize = cleanupSize;
@@ -284,19 +288,20 @@ public:
delete ExtInfo;
}
// Objects of EHCleanupScope are not destructed. Use Destroy().
- ~EHCleanupScope() LLVM_DELETED_FUNCTION;
+ ~EHCleanupScope() = delete;
bool isNormalCleanup() const { return CleanupBits.IsNormalCleanup; }
llvm::BasicBlock *getNormalBlock() const { return NormalBlock; }
void setNormalBlock(llvm::BasicBlock *BB) { NormalBlock = BB; }
bool isEHCleanup() const { return CleanupBits.IsEHCleanup; }
- llvm::BasicBlock *getEHBlock() const { return getCachedEHDispatchBlock(); }
- void setEHBlock(llvm::BasicBlock *BB) { setCachedEHDispatchBlock(BB); }
bool isActive() const { return CleanupBits.IsActive; }
void setActive(bool A) { CleanupBits.IsActive = A; }
+ bool isLifetimeMarker() const { return CleanupBits.IsLifetimeMarker; }
+ void setLifetimeMarker() { CleanupBits.IsLifetimeMarker = true; }
+
llvm::AllocaInst *getActiveFlag() const { return ActiveFlag; }
void setActiveFlag(llvm::AllocaInst *Var) { ActiveFlag = Var; }
diff --git a/lib/CodeGen/CGDebugInfo.cpp b/lib/CodeGen/CGDebugInfo.cpp
index 978e1bb5b81f..48458dbd6003 100644
--- a/lib/CodeGen/CGDebugInfo.cpp
+++ b/lib/CodeGen/CGDebugInfo.cpp
@@ -52,35 +52,48 @@ CGDebugInfo::~CGDebugInfo() {
"Region stack mismatch, stack not empty!");
}
-ArtificialLocation::ArtificialLocation(CodeGenFunction &CGF)
- : ApplyDebugLocation(CGF) {
- if (auto *DI = CGF.getDebugInfo()) {
- // Construct a location that has a valid scope, but no line info.
- assert(!DI->LexicalBlockStack.empty());
- llvm::DIDescriptor Scope(DI->LexicalBlockStack.back());
- CGF.Builder.SetCurrentDebugLocation(llvm::DebugLoc::get(0, 0, Scope));
- }
+ApplyDebugLocation::ApplyDebugLocation(CodeGenFunction &CGF,
+ SourceLocation TemporaryLocation)
+ : CGF(CGF) {
+ init(TemporaryLocation);
}
ApplyDebugLocation::ApplyDebugLocation(CodeGenFunction &CGF,
- SourceLocation TemporaryLocation,
- bool ForceColumnInfo)
+ bool DefaultToEmpty,
+ SourceLocation TemporaryLocation)
: CGF(CGF) {
+ init(TemporaryLocation, DefaultToEmpty);
+}
+
+void ApplyDebugLocation::init(SourceLocation TemporaryLocation,
+ bool DefaultToEmpty) {
if (auto *DI = CGF.getDebugInfo()) {
OriginalLocation = CGF.Builder.getCurrentDebugLocation();
- if (TemporaryLocation.isInvalid())
- CGF.Builder.SetCurrentDebugLocation(llvm::DebugLoc());
- else
- DI->EmitLocation(CGF.Builder, TemporaryLocation, ForceColumnInfo);
+ if (TemporaryLocation.isInvalid()) {
+ if (DefaultToEmpty)
+ CGF.Builder.SetCurrentDebugLocation(llvm::DebugLoc());
+ else {
+ // Construct a location that has a valid scope, but no line info.
+ assert(!DI->LexicalBlockStack.empty());
+ CGF.Builder.SetCurrentDebugLocation(
+ llvm::DebugLoc::get(0, 0, DI->LexicalBlockStack.back()));
+ }
+ } else
+ DI->EmitLocation(CGF.Builder, TemporaryLocation);
}
}
+ApplyDebugLocation::ApplyDebugLocation(CodeGenFunction &CGF, const Expr *E)
+ : CGF(CGF) {
+ init(E->getExprLoc());
+}
+
ApplyDebugLocation::ApplyDebugLocation(CodeGenFunction &CGF, llvm::DebugLoc Loc)
: CGF(CGF) {
if (CGF.getDebugInfo()) {
OriginalLocation = CGF.Builder.getCurrentDebugLocation();
- if (!Loc.isUnknown())
- CGF.Builder.SetCurrentDebugLocation(Loc);
+ if (Loc)
+ CGF.Builder.SetCurrentDebugLocation(std::move(Loc));
}
}
@@ -88,7 +101,7 @@ ApplyDebugLocation::~ApplyDebugLocation() {
// Query CGF so the location isn't overwritten when location updates are
// temporarily disabled (for C++ default function arguments)
if (CGF.getDebugInfo())
- CGF.Builder.SetCurrentDebugLocation(OriginalLocation);
+ CGF.Builder.SetCurrentDebugLocation(std::move(OriginalLocation));
}
/// ArtificialLocation - An RAII object that temporarily switches to
@@ -107,37 +120,33 @@ void CGDebugInfo::setLocation(SourceLocation Loc) {
return;
SourceManager &SM = CGM.getContext().getSourceManager();
- llvm::DIScope Scope(LexicalBlockStack.back());
+ auto *Scope = cast<llvm::DIScope>(LexicalBlockStack.back());
PresumedLoc PCLoc = SM.getPresumedLoc(CurLoc);
- if (PCLoc.isInvalid() || Scope.getFilename() == PCLoc.getFilename())
+ if (PCLoc.isInvalid() || Scope->getFilename() == PCLoc.getFilename())
return;
- if (Scope.isLexicalBlockFile()) {
- llvm::DILexicalBlockFile LBF = llvm::DILexicalBlockFile(Scope);
- llvm::DIDescriptor D = DBuilder.createLexicalBlockFile(
- LBF.getScope(), getOrCreateFile(CurLoc));
- llvm::MDNode *N = D;
+ if (auto *LBF = dyn_cast<llvm::DILexicalBlockFile>(Scope)) {
LexicalBlockStack.pop_back();
- LexicalBlockStack.emplace_back(N);
- } else if (Scope.isLexicalBlock() || Scope.isSubprogram()) {
- llvm::DIDescriptor D =
- DBuilder.createLexicalBlockFile(Scope, getOrCreateFile(CurLoc));
- llvm::MDNode *N = D;
+ LexicalBlockStack.emplace_back(DBuilder.createLexicalBlockFile(
+ LBF->getScope(), getOrCreateFile(CurLoc)));
+ } else if (isa<llvm::DILexicalBlock>(Scope) ||
+ isa<llvm::DISubprogram>(Scope)) {
LexicalBlockStack.pop_back();
- LexicalBlockStack.emplace_back(N);
+ LexicalBlockStack.emplace_back(
+ DBuilder.createLexicalBlockFile(Scope, getOrCreateFile(CurLoc)));
}
}
/// getContextDescriptor - Get context info for the decl.
-llvm::DIScope CGDebugInfo::getContextDescriptor(const Decl *Context) {
+llvm::DIScope *CGDebugInfo::getContextDescriptor(const Decl *Context) {
if (!Context)
return TheCU;
auto I = RegionMap.find(Context);
if (I != RegionMap.end()) {
llvm::Metadata *V = I->second;
- return llvm::DIScope(dyn_cast_or_null<llvm::MDNode>(V));
+ return dyn_cast_or_null<llvm::DIScope>(V);
}
// Check namespace.
@@ -234,17 +243,17 @@ StringRef CGDebugInfo::getClassName(const RecordDecl *RD) {
}
/// getOrCreateFile - Get the file debug info descriptor for the input location.
-llvm::DIFile CGDebugInfo::getOrCreateFile(SourceLocation Loc) {
+llvm::DIFile *CGDebugInfo::getOrCreateFile(SourceLocation Loc) {
if (!Loc.isValid())
// If Location is not valid then use main input file.
- return DBuilder.createFile(TheCU.getFilename(), TheCU.getDirectory());
+ return DBuilder.createFile(TheCU->getFilename(), TheCU->getDirectory());
SourceManager &SM = CGM.getContext().getSourceManager();
PresumedLoc PLoc = SM.getPresumedLoc(Loc);
if (PLoc.isInvalid() || StringRef(PLoc.getFilename()).empty())
// If the location is not valid then use main input file.
- return DBuilder.createFile(TheCU.getFilename(), TheCU.getDirectory());
+ return DBuilder.createFile(TheCU->getFilename(), TheCU->getDirectory());
// Cache the results.
const char *fname = PLoc.getFilename();
@@ -253,18 +262,19 @@ llvm::DIFile CGDebugInfo::getOrCreateFile(SourceLocation Loc) {
if (it != DIFileCache.end()) {
// Verify that the information still exists.
if (llvm::Metadata *V = it->second)
- return llvm::DIFile(cast<llvm::MDNode>(V));
+ return cast<llvm::DIFile>(V);
}
- llvm::DIFile F = DBuilder.createFile(PLoc.getFilename(), getCurrentDirname());
+ llvm::DIFile *F =
+ DBuilder.createFile(PLoc.getFilename(), getCurrentDirname());
DIFileCache[fname].reset(F);
return F;
}
/// getOrCreateMainFile - Get the file info for main compile unit.
-llvm::DIFile CGDebugInfo::getOrCreateMainFile() {
- return DBuilder.createFile(TheCU.getFilename(), TheCU.getDirectory());
+llvm::DIFile *CGDebugInfo::getOrCreateMainFile() {
+ return DBuilder.createFile(TheCU->getFilename(), TheCU->getDirectory());
}
/// getLineNumber - Get line number for the location. If location is invalid
@@ -369,12 +379,13 @@ void CGDebugInfo::CreateCompileUnit() {
DebugKind <= CodeGenOptions::DebugLineTablesOnly
? llvm::DIBuilder::LineTablesOnly
: llvm::DIBuilder::FullDebug,
+ 0 /* DWOid */,
DebugKind != CodeGenOptions::LocTrackingOnly);
}
/// CreateType - Get the Basic type from the cache or create a new
/// one if necessary.
-llvm::DIType CGDebugInfo::CreateType(const BuiltinType *BT) {
+llvm::DIType *CGDebugInfo::CreateType(const BuiltinType *BT) {
llvm::dwarf::TypeKind Encoding;
StringRef BTName;
switch (BT->getKind()) {
@@ -386,7 +397,7 @@ llvm::DIType CGDebugInfo::CreateType(const BuiltinType *BT) {
case BuiltinType::NullPtr:
return DBuilder.createNullPtrType();
case BuiltinType::Void:
- return llvm::DIType();
+ return nullptr;
case BuiltinType::ObjCClass:
if (!ClassTy)
ClassTy = DBuilder.createForwardDecl(llvm::dwarf::DW_TAG_structure_type,
@@ -409,11 +420,11 @@ llvm::DIType CGDebugInfo::CreateType(const BuiltinType *BT) {
unsigned Size = CGM.getContext().getTypeSize(CGM.getContext().VoidPtrTy);
- llvm::DIType ISATy = DBuilder.createPointerType(ClassTy, Size);
+ auto *ISATy = DBuilder.createPointerType(ClassTy, Size);
ObjTy =
DBuilder.createStructType(TheCU, "objc_object", getOrCreateMainFile(),
- 0, 0, 0, 0, llvm::DIType(), llvm::DIArray());
+ 0, 0, 0, 0, nullptr, llvm::DINodeArray());
DBuilder.replaceArrays(
ObjTy,
@@ -510,11 +521,10 @@ llvm::DIType CGDebugInfo::CreateType(const BuiltinType *BT) {
// Bit size, align and offset of the type.
uint64_t Size = CGM.getContext().getTypeSize(BT);
uint64_t Align = CGM.getContext().getTypeAlign(BT);
- llvm::DIType DbgTy = DBuilder.createBasicType(BTName, Size, Align, Encoding);
- return DbgTy;
+ return DBuilder.createBasicType(BTName, Size, Align, Encoding);
}
-llvm::DIType CGDebugInfo::CreateType(const ComplexType *Ty) {
+llvm::DIType *CGDebugInfo::CreateType(const ComplexType *Ty) {
// Bit size, align and offset of the type.
llvm::dwarf::TypeKind Encoding = llvm::dwarf::DW_ATE_complex_float;
if (Ty->isComplexIntegerType())
@@ -522,15 +532,13 @@ llvm::DIType CGDebugInfo::CreateType(const ComplexType *Ty) {
uint64_t Size = CGM.getContext().getTypeSize(Ty);
uint64_t Align = CGM.getContext().getTypeAlign(Ty);
- llvm::DIType DbgTy =
- DBuilder.createBasicType("complex", Size, Align, Encoding);
-
- return DbgTy;
+ return DBuilder.createBasicType("complex", Size, Align, Encoding);
}
/// CreateCVRType - Get the qualified type from the cache or create
/// a new one if necessary.
-llvm::DIType CGDebugInfo::CreateQualifiedType(QualType Ty, llvm::DIFile Unit) {
+llvm::DIType *CGDebugInfo::CreateQualifiedType(QualType Ty,
+ llvm::DIFile *Unit) {
QualifierCollector Qc;
const Type *T = Qc.strip(Ty);
@@ -556,17 +564,15 @@ llvm::DIType CGDebugInfo::CreateQualifiedType(QualType Ty, llvm::DIFile Unit) {
return getOrCreateType(QualType(T, 0), Unit);
}
- llvm::DIType FromTy = getOrCreateType(Qc.apply(CGM.getContext(), T), Unit);
+ auto *FromTy = getOrCreateType(Qc.apply(CGM.getContext(), T), Unit);
// No need to fill in the Name, Line, Size, Alignment, Offset in case of
// CVR derived types.
- llvm::DIType DbgTy = DBuilder.createQualifiedType(Tag, FromTy);
-
- return DbgTy;
+ return DBuilder.createQualifiedType(Tag, FromTy);
}
-llvm::DIType CGDebugInfo::CreateType(const ObjCObjectPointerType *Ty,
- llvm::DIFile Unit) {
+llvm::DIType *CGDebugInfo::CreateType(const ObjCObjectPointerType *Ty,
+ llvm::DIFile *Unit) {
// The frontend treats 'id' as a typedef to an ObjCObjectType,
// whereas 'id<protocol>' is treated as an ObjCPointerType. For the
@@ -574,12 +580,12 @@ llvm::DIType CGDebugInfo::CreateType(const ObjCObjectPointerType *Ty,
if (Ty->isObjCQualifiedIdType())
return getOrCreateType(CGM.getContext().getObjCIdType(), Unit);
- llvm::DIType DbgTy = CreatePointerLikeType(llvm::dwarf::DW_TAG_pointer_type,
- Ty, Ty->getPointeeType(), Unit);
- return DbgTy;
+ return CreatePointerLikeType(llvm::dwarf::DW_TAG_pointer_type, Ty,
+ Ty->getPointeeType(), Unit);
}
-llvm::DIType CGDebugInfo::CreateType(const PointerType *Ty, llvm::DIFile Unit) {
+llvm::DIType *CGDebugInfo::CreateType(const PointerType *Ty,
+ llvm::DIFile *Unit) {
return CreatePointerLikeType(llvm::dwarf::DW_TAG_pointer_type, Ty,
Ty->getPointeeType(), Unit);
}
@@ -588,12 +594,12 @@ llvm::DIType CGDebugInfo::CreateType(const PointerType *Ty, llvm::DIFile Unit) {
/// on their mangled names, if they're external.
static SmallString<256> getUniqueTagTypeName(const TagType *Ty,
CodeGenModule &CGM,
- llvm::DICompileUnit TheCU) {
+ llvm::DICompileUnit *TheCU) {
SmallString<256> FullName;
// FIXME: ODR should apply to ObjC++ exactly the same wasy it does to C++.
// For now, only apply ODR with C++.
const TagDecl *TD = Ty->getDecl();
- if (TheCU.getLanguage() != llvm::dwarf::DW_LANG_C_plus_plus ||
+ if (TheCU->getSourceLanguage() != llvm::dwarf::DW_LANG_C_plus_plus ||
!TD->isExternallyVisible())
return FullName;
// Microsoft Mangler does not have support for mangleCXXRTTIName yet.
@@ -608,41 +614,56 @@ static SmallString<256> getUniqueTagTypeName(const TagType *Ty,
return FullName;
}
-// Creates a forward declaration for a RecordDecl in the given context.
-llvm::DICompositeType
-CGDebugInfo::getOrCreateRecordFwdDecl(const RecordType *Ty,
- llvm::DIDescriptor Ctx) {
- const RecordDecl *RD = Ty->getDecl();
- if (llvm::DIType T = getTypeOrNull(CGM.getContext().getRecordType(RD)))
- return llvm::DICompositeType(T);
- llvm::DIFile DefUnit = getOrCreateFile(RD->getLocation());
- unsigned Line = getLineNumber(RD->getLocation());
- StringRef RDName = getClassName(RD);
-
- llvm::dwarf::Tag Tag;
+static llvm::dwarf::Tag getTagForRecord(const RecordDecl *RD) {
+ llvm::dwarf::Tag Tag;
if (RD->isStruct() || RD->isInterface())
Tag = llvm::dwarf::DW_TAG_structure_type;
else if (RD->isUnion())
Tag = llvm::dwarf::DW_TAG_union_type;
else {
+ // FIXME: This could be a struct type giving a default visibility different
+ // than C++ class type, but needs llvm metadata changes first.
assert(RD->isClass());
Tag = llvm::dwarf::DW_TAG_class_type;
}
+ return Tag;
+}
+
+// Creates a forward declaration for a RecordDecl in the given context.
+llvm::DICompositeType *
+CGDebugInfo::getOrCreateRecordFwdDecl(const RecordType *Ty,
+ llvm::DIScope *Ctx) {
+ const RecordDecl *RD = Ty->getDecl();
+ if (llvm::DIType *T = getTypeOrNull(CGM.getContext().getRecordType(RD)))
+ return cast<llvm::DICompositeType>(T);
+ llvm::DIFile *DefUnit = getOrCreateFile(RD->getLocation());
+ unsigned Line = getLineNumber(RD->getLocation());
+ StringRef RDName = getClassName(RD);
+
+ uint64_t Size = 0;
+ uint64_t Align = 0;
+
+ const RecordDecl *D = RD->getDefinition();
+ if (D && D->isCompleteDefinition()) {
+ Size = CGM.getContext().getTypeSize(Ty);
+ Align = CGM.getContext().getTypeAlign(Ty);
+ }
// Create the type.
SmallString<256> FullName = getUniqueTagTypeName(Ty, CGM, TheCU);
- llvm::DICompositeType RetTy = DBuilder.createReplaceableForwardDecl(
- Tag, RDName, Ctx, DefUnit, Line, 0, 0, 0, FullName);
+ llvm::DICompositeType *RetTy = DBuilder.createReplaceableCompositeType(
+ getTagForRecord(RD), RDName, Ctx, DefUnit, Line, 0, Size, Align,
+ llvm::DINode::FlagFwdDecl, FullName);
ReplaceMap.emplace_back(
std::piecewise_construct, std::make_tuple(Ty),
std::make_tuple(static_cast<llvm::Metadata *>(RetTy)));
return RetTy;
}
-llvm::DIType CGDebugInfo::CreatePointerLikeType(llvm::dwarf::Tag Tag,
- const Type *Ty,
- QualType PointeeTy,
- llvm::DIFile Unit) {
+llvm::DIType *CGDebugInfo::CreatePointerLikeType(llvm::dwarf::Tag Tag,
+ const Type *Ty,
+ QualType PointeeTy,
+ llvm::DIFile *Unit) {
if (Tag == llvm::dwarf::DW_TAG_reference_type ||
Tag == llvm::dwarf::DW_TAG_rvalue_reference_type)
return DBuilder.createReferenceType(Tag, getOrCreateType(PointeeTy, Unit));
@@ -658,8 +679,8 @@ llvm::DIType CGDebugInfo::CreatePointerLikeType(llvm::dwarf::Tag Tag,
Align);
}
-llvm::DIType CGDebugInfo::getOrCreateStructPtrType(StringRef Name,
- llvm::DIType &Cache) {
+llvm::DIType *CGDebugInfo::getOrCreateStructPtrType(StringRef Name,
+ llvm::DIType *&Cache) {
if (Cache)
return Cache;
Cache = DBuilder.createForwardDecl(llvm::dwarf::DW_TAG_structure_type, Name,
@@ -669,18 +690,16 @@ llvm::DIType CGDebugInfo::getOrCreateStructPtrType(StringRef Name,
return Cache;
}
-llvm::DIType CGDebugInfo::CreateType(const BlockPointerType *Ty,
- llvm::DIFile Unit) {
+llvm::DIType *CGDebugInfo::CreateType(const BlockPointerType *Ty,
+ llvm::DIFile *Unit) {
if (BlockLiteralGeneric)
return BlockLiteralGeneric;
SmallVector<llvm::Metadata *, 8> EltTys;
- llvm::DIType FieldTy;
QualType FType;
uint64_t FieldSize, FieldOffset;
unsigned FieldAlign;
- llvm::DIArray Elements;
- llvm::DIType EltTy, DescTy;
+ llvm::DINodeArray Elements;
FieldOffset = 0;
FType = CGM.getContext().UnsignedLongTy;
@@ -690,17 +709,17 @@ llvm::DIType CGDebugInfo::CreateType(const BlockPointerType *Ty,
Elements = DBuilder.getOrCreateArray(EltTys);
EltTys.clear();
- unsigned Flags = llvm::DIDescriptor::FlagAppleBlock;
+ unsigned Flags = llvm::DINode::FlagAppleBlock;
unsigned LineNo = getLineNumber(CurLoc);
- EltTy = DBuilder.createStructType(Unit, "__block_descriptor", Unit, LineNo,
- FieldOffset, 0, Flags, llvm::DIType(),
- Elements);
+ auto *EltTy =
+ DBuilder.createStructType(Unit, "__block_descriptor", Unit, LineNo,
+ FieldOffset, 0, Flags, nullptr, Elements);
// Bit size, align and offset of the type.
uint64_t Size = CGM.getContext().getTypeSize(Ty);
- DescTy = DBuilder.createPointerType(EltTy, Size);
+ auto *DescTy = DBuilder.createPointerType(EltTy, Size);
FieldOffset = 0;
FType = CGM.getContext().getPointerType(CGM.getContext().VoidTy);
@@ -712,29 +731,27 @@ llvm::DIType CGDebugInfo::CreateType(const BlockPointerType *Ty,
EltTys.push_back(CreateMemberType(Unit, FType, "__FuncPtr", &FieldOffset));
FType = CGM.getContext().getPointerType(CGM.getContext().VoidTy);
- FieldTy = DescTy;
FieldSize = CGM.getContext().getTypeSize(Ty);
FieldAlign = CGM.getContext().getTypeAlign(Ty);
- FieldTy =
- DBuilder.createMemberType(Unit, "__descriptor", Unit, LineNo, FieldSize,
- FieldAlign, FieldOffset, 0, FieldTy);
- EltTys.push_back(FieldTy);
+ EltTys.push_back(DBuilder.createMemberType(Unit, "__descriptor", Unit, LineNo,
+ FieldSize, FieldAlign, FieldOffset,
+ 0, DescTy));
FieldOffset += FieldSize;
Elements = DBuilder.getOrCreateArray(EltTys);
- EltTy = DBuilder.createStructType(Unit, "__block_literal_generic", Unit,
- LineNo, FieldOffset, 0, Flags,
- llvm::DIType(), Elements);
+ EltTy =
+ DBuilder.createStructType(Unit, "__block_literal_generic", Unit, LineNo,
+ FieldOffset, 0, Flags, nullptr, Elements);
BlockLiteralGeneric = DBuilder.createPointerType(EltTy, Size);
return BlockLiteralGeneric;
}
-llvm::DIType CGDebugInfo::CreateType(const TemplateSpecializationType *Ty,
- llvm::DIFile Unit) {
+llvm::DIType *CGDebugInfo::CreateType(const TemplateSpecializationType *Ty,
+ llvm::DIFile *Unit) {
assert(Ty->isTypeAlias());
- llvm::DIType Src = getOrCreateType(Ty->getAliasedType(), Unit);
+ llvm::DIType *Src = getOrCreateType(Ty->getAliasedType(), Unit);
SmallString<128> NS;
llvm::raw_svector_ostream OS(NS);
@@ -749,35 +766,26 @@ llvm::DIType CGDebugInfo::CreateType(const TemplateSpecializationType *Ty,
Ty->getTemplateName().getAsTemplateDecl())->getTemplatedDecl();
SourceLocation Loc = AliasDecl->getLocation();
- llvm::DIFile File = getOrCreateFile(Loc);
- unsigned Line = getLineNumber(Loc);
-
- llvm::DIDescriptor Ctxt =
- getContextDescriptor(cast<Decl>(AliasDecl->getDeclContext()));
-
- return DBuilder.createTypedef(Src, internString(OS.str()), File, Line, Ctxt);
+ return DBuilder.createTypedef(
+ Src, internString(OS.str()), getOrCreateFile(Loc), getLineNumber(Loc),
+ getContextDescriptor(cast<Decl>(AliasDecl->getDeclContext())));
}
-llvm::DIType CGDebugInfo::CreateType(const TypedefType *Ty, llvm::DIFile Unit) {
- // Typedefs are derived from some other type. If we have a typedef of a
- // typedef, make sure to emit the whole chain.
- llvm::DIType Src = getOrCreateType(Ty->getDecl()->getUnderlyingType(), Unit);
+llvm::DIType *CGDebugInfo::CreateType(const TypedefType *Ty,
+ llvm::DIFile *Unit) {
// We don't set size information, but do specify where the typedef was
// declared.
SourceLocation Loc = Ty->getDecl()->getLocation();
- llvm::DIFile File = getOrCreateFile(Loc);
- unsigned Line = getLineNumber(Loc);
- const TypedefNameDecl *TyDecl = Ty->getDecl();
- llvm::DIDescriptor TypedefContext =
- getContextDescriptor(cast<Decl>(Ty->getDecl()->getDeclContext()));
-
- return DBuilder.createTypedef(Src, TyDecl->getName(), File, Line,
- TypedefContext);
+ // Typedefs are derived from some other type.
+ return DBuilder.createTypedef(
+ getOrCreateType(Ty->getDecl()->getUnderlyingType(), Unit),
+ Ty->getDecl()->getName(), getOrCreateFile(Loc), getLineNumber(Loc),
+ getContextDescriptor(cast<Decl>(Ty->getDecl()->getDeclContext())));
}
-llvm::DIType CGDebugInfo::CreateType(const FunctionType *Ty,
- llvm::DIFile Unit) {
+llvm::DIType *CGDebugInfo::CreateType(const FunctionType *Ty,
+ llvm::DIFile *Unit) {
SmallVector<llvm::Metadata *, 16> EltTys;
// Add the result type at least.
@@ -794,11 +802,11 @@ llvm::DIType CGDebugInfo::CreateType(const FunctionType *Ty,
EltTys.push_back(DBuilder.createUnspecifiedParameter());
}
- llvm::DITypeArray EltTypeArray = DBuilder.getOrCreateTypeArray(EltTys);
+ llvm::DITypeRefArray EltTypeArray = DBuilder.getOrCreateTypeArray(EltTys);
return DBuilder.createSubroutineType(Unit, EltTypeArray);
}
-/// Convert an AccessSpecifier into the corresponding DIDescriptor flag.
+/// Convert an AccessSpecifier into the corresponding DINode flag.
/// As an optimization, return 0 if the access specifier equals the
/// default for the containing type.
static unsigned getAccessFlag(AccessSpecifier Access, const RecordDecl *RD) {
@@ -813,25 +821,25 @@ static unsigned getAccessFlag(AccessSpecifier Access, const RecordDecl *RD) {
switch (Access) {
case clang::AS_private:
- return llvm::DIDescriptor::FlagPrivate;
+ return llvm::DINode::FlagPrivate;
case clang::AS_protected:
- return llvm::DIDescriptor::FlagProtected;
+ return llvm::DINode::FlagProtected;
case clang::AS_public:
- return llvm::DIDescriptor::FlagPublic;
+ return llvm::DINode::FlagPublic;
case clang::AS_none:
return 0;
}
llvm_unreachable("unexpected access enumerator");
}
-llvm::DIType CGDebugInfo::createFieldType(
+llvm::DIType *CGDebugInfo::createFieldType(
StringRef name, QualType type, uint64_t sizeInBitsOverride,
SourceLocation loc, AccessSpecifier AS, uint64_t offsetInBits,
- llvm::DIFile tunit, llvm::DIScope scope, const RecordDecl *RD) {
- llvm::DIType debugType = getOrCreateType(type, tunit);
+ llvm::DIFile *tunit, llvm::DIScope *scope, const RecordDecl *RD) {
+ llvm::DIType *debugType = getOrCreateType(type, tunit);
// Get the location for the field.
- llvm::DIFile file = getOrCreateFile(loc);
+ llvm::DIFile *file = getOrCreateFile(loc);
unsigned line = getLineNumber(loc);
uint64_t SizeInBits = 0;
@@ -853,7 +861,7 @@ llvm::DIType CGDebugInfo::createFieldType(
/// CollectRecordLambdaFields - Helper for CollectRecordFields.
void CGDebugInfo::CollectRecordLambdaFields(
const CXXRecordDecl *CXXDecl, SmallVectorImpl<llvm::Metadata *> &elements,
- llvm::DIType RecordTy) {
+ llvm::DIType *RecordTy) {
// For C++11 Lambdas a Field will be the same as a Capture, but the Capture
// has the name and the location of the variable so we should iterate over
// both concurrently.
@@ -866,14 +874,14 @@ void CGDebugInfo::CollectRecordLambdaFields(
const LambdaCapture &C = *I;
if (C.capturesVariable()) {
VarDecl *V = C.getCapturedVar();
- llvm::DIFile VUnit = getOrCreateFile(C.getLocation());
+ llvm::DIFile *VUnit = getOrCreateFile(C.getLocation());
StringRef VName = V->getName();
uint64_t SizeInBitsOverride = 0;
if (Field->isBitField()) {
SizeInBitsOverride = Field->getBitWidthValue(CGM.getContext());
assert(SizeInBitsOverride && "found named 0-width bitfield");
}
- llvm::DIType fieldType = createFieldType(
+ llvm::DIType *fieldType = createFieldType(
VName, Field->getType(), SizeInBitsOverride, C.getLocation(),
Field->getAccess(), layout.getFieldOffset(fieldno), VUnit, RecordTy,
CXXDecl);
@@ -884,9 +892,9 @@ void CGDebugInfo::CollectRecordLambdaFields(
// by using AT_object_pointer for the function and having that be
// used as 'this' for semantic references.
FieldDecl *f = *Field;
- llvm::DIFile VUnit = getOrCreateFile(f->getLocation());
+ llvm::DIFile *VUnit = getOrCreateFile(f->getLocation());
QualType type = f->getType();
- llvm::DIType fieldType = createFieldType(
+ llvm::DIType *fieldType = createFieldType(
"this", type, 0, f->getLocation(), f->getAccess(),
layout.getFieldOffset(fieldno), VUnit, RecordTy, CXXDecl);
@@ -896,14 +904,14 @@ void CGDebugInfo::CollectRecordLambdaFields(
}
/// Helper for CollectRecordFields.
-llvm::DIDerivedType CGDebugInfo::CreateRecordStaticField(const VarDecl *Var,
- llvm::DIType RecordTy,
- const RecordDecl *RD) {
+llvm::DIDerivedType *
+CGDebugInfo::CreateRecordStaticField(const VarDecl *Var, llvm::DIType *RecordTy,
+ const RecordDecl *RD) {
// Create the descriptor for the static variable, with or without
// constant initializers.
Var = Var->getCanonicalDecl();
- llvm::DIFile VUnit = getOrCreateFile(Var->getLocation());
- llvm::DIType VTy = getOrCreateType(Var->getType(), VUnit);
+ llvm::DIFile *VUnit = getOrCreateFile(Var->getLocation());
+ llvm::DIType *VTy = getOrCreateType(Var->getType(), VUnit);
unsigned LineNumber = getLineNumber(Var->getLocation());
StringRef VName = Var->getName();
@@ -919,7 +927,7 @@ llvm::DIDerivedType CGDebugInfo::CreateRecordStaticField(const VarDecl *Var,
}
unsigned Flags = getAccessFlag(Var->getAccess(), RD);
- llvm::DIDerivedType GV = DBuilder.createStaticMemberType(
+ llvm::DIDerivedType *GV = DBuilder.createStaticMemberType(
RecordTy, VName, VUnit, LineNumber, VTy, Flags, C);
StaticDataMemberCache[Var->getCanonicalDecl()].reset(GV);
return GV;
@@ -927,8 +935,8 @@ llvm::DIDerivedType CGDebugInfo::CreateRecordStaticField(const VarDecl *Var,
/// CollectRecordNormalField - Helper for CollectRecordFields.
void CGDebugInfo::CollectRecordNormalField(
- const FieldDecl *field, uint64_t OffsetInBits, llvm::DIFile tunit,
- SmallVectorImpl<llvm::Metadata *> &elements, llvm::DIType RecordTy,
+ const FieldDecl *field, uint64_t OffsetInBits, llvm::DIFile *tunit,
+ SmallVectorImpl<llvm::Metadata *> &elements, llvm::DIType *RecordTy,
const RecordDecl *RD) {
StringRef name = field->getName();
QualType type = field->getType();
@@ -943,7 +951,7 @@ void CGDebugInfo::CollectRecordNormalField(
assert(SizeInBitsOverride && "found named 0-width bitfield");
}
- llvm::DIType fieldType =
+ llvm::DIType *fieldType =
createFieldType(name, type, SizeInBitsOverride, field->getLocation(),
field->getAccess(), OffsetInBits, tunit, RecordTy, RD);
@@ -953,9 +961,9 @@ void CGDebugInfo::CollectRecordNormalField(
/// CollectRecordFields - A helper function to collect debug info for
/// record fields. This is used while creating debug info entry for a Record.
void CGDebugInfo::CollectRecordFields(
- const RecordDecl *record, llvm::DIFile tunit,
+ const RecordDecl *record, llvm::DIFile *tunit,
SmallVectorImpl<llvm::Metadata *> &elements,
- llvm::DICompositeType RecordTy) {
+ llvm::DICompositeType *RecordTy) {
const CXXRecordDecl *CXXDecl = dyn_cast<CXXRecordDecl>(record);
if (CXXDecl && CXXDecl->isLambda())
@@ -975,8 +983,7 @@ void CGDebugInfo::CollectRecordFields(
if (MI != StaticDataMemberCache.end()) {
assert(MI->second &&
"Static data member declaration should still exist");
- elements.push_back(
- llvm::DIDerivedType(cast<llvm::MDNode>(MI->second)));
+ elements.push_back(cast<llvm::DIDerivedTypeBase>(MI->second));
} else {
auto Field = CreateRecordStaticField(V, RecordTy, record);
elements.push_back(Field);
@@ -994,27 +1001,29 @@ void CGDebugInfo::CollectRecordFields(
/// getOrCreateMethodType - CXXMethodDecl's type is a FunctionType. This
/// function type is not updated to include implicit "this" pointer. Use this
/// routine to get a method type which includes "this" pointer.
-llvm::DICompositeType
+llvm::DISubroutineType *
CGDebugInfo::getOrCreateMethodType(const CXXMethodDecl *Method,
- llvm::DIFile Unit) {
+ llvm::DIFile *Unit) {
const FunctionProtoType *Func = Method->getType()->getAs<FunctionProtoType>();
if (Method->isStatic())
- return llvm::DICompositeType(getOrCreateType(QualType(Func, 0), Unit));
+ return cast_or_null<llvm::DISubroutineType>(
+ getOrCreateType(QualType(Func, 0), Unit));
return getOrCreateInstanceMethodType(Method->getThisType(CGM.getContext()),
Func, Unit);
}
-llvm::DICompositeType CGDebugInfo::getOrCreateInstanceMethodType(
- QualType ThisPtr, const FunctionProtoType *Func, llvm::DIFile Unit) {
+llvm::DISubroutineType *CGDebugInfo::getOrCreateInstanceMethodType(
+ QualType ThisPtr, const FunctionProtoType *Func, llvm::DIFile *Unit) {
// Add "this" pointer.
- llvm::DITypeArray Args = llvm::DISubroutineType(
- getOrCreateType(QualType(Func, 0), Unit)).getTypeArray();
- assert(Args.getNumElements() && "Invalid number of arguments!");
+ llvm::DITypeRefArray Args(
+ cast<llvm::DISubroutineType>(getOrCreateType(QualType(Func, 0), Unit))
+ ->getTypeArray());
+ assert(Args.size() && "Invalid number of arguments!");
SmallVector<llvm::Metadata *, 16> Elts;
// First element is always return type. For 'void' functions it is NULL.
- Elts.push_back(Args.getElement(0));
+ Elts.push_back(Args[0]);
// "this" pointer is always first argument.
const CXXRecordDecl *RD = ThisPtr->getPointeeCXXRecordDecl();
@@ -1025,8 +1034,8 @@ llvm::DICompositeType CGDebugInfo::getOrCreateInstanceMethodType(
unsigned AS = CGM.getContext().getTargetAddressSpace(PointeeTy);
uint64_t Size = CGM.getTarget().getPointerWidth(AS);
uint64_t Align = CGM.getContext().getTypeAlign(ThisPtrTy);
- llvm::DIType PointeeType = getOrCreateType(PointeeTy, Unit);
- llvm::DIType ThisPtrType =
+ llvm::DIType *PointeeType = getOrCreateType(PointeeTy, Unit);
+ llvm::DIType *ThisPtrType =
DBuilder.createPointerType(PointeeType, Size, Align);
TypeCache[ThisPtr.getAsOpaquePtr()].reset(ThisPtrType);
// TODO: This and the artificial type below are misleading, the
@@ -1035,23 +1044,23 @@ llvm::DICompositeType CGDebugInfo::getOrCreateInstanceMethodType(
ThisPtrType = DBuilder.createObjectPointerType(ThisPtrType);
Elts.push_back(ThisPtrType);
} else {
- llvm::DIType ThisPtrType = getOrCreateType(ThisPtr, Unit);
+ llvm::DIType *ThisPtrType = getOrCreateType(ThisPtr, Unit);
TypeCache[ThisPtr.getAsOpaquePtr()].reset(ThisPtrType);
ThisPtrType = DBuilder.createObjectPointerType(ThisPtrType);
Elts.push_back(ThisPtrType);
}
// Copy rest of the arguments.
- for (unsigned i = 1, e = Args.getNumElements(); i != e; ++i)
- Elts.push_back(Args.getElement(i));
+ for (unsigned i = 1, e = Args.size(); i != e; ++i)
+ Elts.push_back(Args[i]);
- llvm::DITypeArray EltTypeArray = DBuilder.getOrCreateTypeArray(Elts);
+ llvm::DITypeRefArray EltTypeArray = DBuilder.getOrCreateTypeArray(Elts);
unsigned Flags = 0;
if (Func->getExtProtoInfo().RefQualifier == RQ_LValue)
- Flags |= llvm::DIDescriptor::FlagLValueReference;
+ Flags |= llvm::DINode::FlagLValueReference;
if (Func->getExtProtoInfo().RefQualifier == RQ_RValue)
- Flags |= llvm::DIDescriptor::FlagRValueReference;
+ Flags |= llvm::DINode::FlagRValueReference;
return DBuilder.createSubroutineType(Unit, EltTypeArray, Flags);
}
@@ -1066,16 +1075,15 @@ static bool isFunctionLocalClass(const CXXRecordDecl *RD) {
return false;
}
-/// CreateCXXMemberFunction - A helper function to create a DISubprogram for
+/// CreateCXXMemberFunction - A helper function to create a subprogram for
/// a single member function GlobalDecl.
-llvm::DISubprogram
-CGDebugInfo::CreateCXXMemberFunction(const CXXMethodDecl *Method,
- llvm::DIFile Unit, llvm::DIType RecordTy) {
+llvm::DISubprogram *CGDebugInfo::CreateCXXMemberFunction(
+ const CXXMethodDecl *Method, llvm::DIFile *Unit, llvm::DIType *RecordTy) {
bool IsCtorOrDtor =
isa<CXXConstructorDecl>(Method) || isa<CXXDestructorDecl>(Method);
StringRef MethodName = getFunctionName(Method);
- llvm::DICompositeType MethodTy = getOrCreateMethodType(Method, Unit);
+ llvm::DISubroutineType *MethodTy = getOrCreateMethodType(Method, Unit);
// Since a single ctor/dtor corresponds to multiple functions, it doesn't
// make sense to give a single ctor/dtor a linkage name.
@@ -1084,7 +1092,7 @@ CGDebugInfo::CreateCXXMemberFunction(const CXXMethodDecl *Method,
MethodLinkageName = CGM.getMangledName(Method);
// Get the location for the method.
- llvm::DIFile MethodDefUnit;
+ llvm::DIFile *MethodDefUnit = nullptr;
unsigned MethodLine = 0;
if (!Method->isImplicit()) {
MethodDefUnit = getOrCreateFile(Method->getLocation());
@@ -1092,7 +1100,7 @@ CGDebugInfo::CreateCXXMemberFunction(const CXXMethodDecl *Method,
}
// Collect virtual method info.
- llvm::DIType ContainingType;
+ llvm::DIType *ContainingType = nullptr;
unsigned Virtuality = 0;
unsigned VIndex = 0;
@@ -1115,29 +1123,29 @@ CGDebugInfo::CreateCXXMemberFunction(const CXXMethodDecl *Method,
unsigned Flags = 0;
if (Method->isImplicit())
- Flags |= llvm::DIDescriptor::FlagArtificial;
+ Flags |= llvm::DINode::FlagArtificial;
Flags |= getAccessFlag(Method->getAccess(), Method->getParent());
if (const CXXConstructorDecl *CXXC = dyn_cast<CXXConstructorDecl>(Method)) {
if (CXXC->isExplicit())
- Flags |= llvm::DIDescriptor::FlagExplicit;
+ Flags |= llvm::DINode::FlagExplicit;
} else if (const CXXConversionDecl *CXXC =
dyn_cast<CXXConversionDecl>(Method)) {
if (CXXC->isExplicit())
- Flags |= llvm::DIDescriptor::FlagExplicit;
+ Flags |= llvm::DINode::FlagExplicit;
}
if (Method->hasPrototype())
- Flags |= llvm::DIDescriptor::FlagPrototyped;
+ Flags |= llvm::DINode::FlagPrototyped;
if (Method->getRefQualifier() == RQ_LValue)
- Flags |= llvm::DIDescriptor::FlagLValueReference;
+ Flags |= llvm::DINode::FlagLValueReference;
if (Method->getRefQualifier() == RQ_RValue)
- Flags |= llvm::DIDescriptor::FlagRValueReference;
+ Flags |= llvm::DINode::FlagRValueReference;
- llvm::DIArray TParamsArray = CollectFunctionTemplateParams(Method, Unit);
- llvm::DISubprogram SP = DBuilder.createMethod(
+ llvm::DINodeArray TParamsArray = CollectFunctionTemplateParams(Method, Unit);
+ llvm::DISubprogram *SP = DBuilder.createMethod(
RecordTy, MethodName, MethodLinkageName, MethodDefUnit, MethodLine,
MethodTy, /*isLocalToUnit=*/false,
/* isDefinition=*/false, Virtuality, VIndex, ContainingType, Flags,
- CGM.getLangOpts().Optimize, nullptr, TParamsArray);
+ CGM.getLangOpts().Optimize, nullptr, TParamsArray.get());
SPCache[Method->getCanonicalDecl()].reset(SP);
@@ -1148,8 +1156,8 @@ CGDebugInfo::CreateCXXMemberFunction(const CXXMethodDecl *Method,
/// C++ member functions. This is used while creating debug info entry for
/// a Record.
void CGDebugInfo::CollectCXXMemberFunctions(
- const CXXRecordDecl *RD, llvm::DIFile Unit,
- SmallVectorImpl<llvm::Metadata *> &EltTys, llvm::DIType RecordTy) {
+ const CXXRecordDecl *RD, llvm::DIFile *Unit,
+ SmallVectorImpl<llvm::Metadata *> &EltTys, llvm::DIType *RecordTy) {
// Since we want more than just the individual member decls if we
// have templated functions iterate over every declaration to gather
@@ -1187,10 +1195,9 @@ void CGDebugInfo::CollectCXXMemberFunctions(
/// CollectCXXBases - A helper function to collect debug info for
/// C++ base classes. This is used while creating debug info entry for
/// a Record.
-void CGDebugInfo::CollectCXXBases(const CXXRecordDecl *RD, llvm::DIFile Unit,
+void CGDebugInfo::CollectCXXBases(const CXXRecordDecl *RD, llvm::DIFile *Unit,
SmallVectorImpl<llvm::Metadata *> &EltTys,
- llvm::DIType RecordTy) {
-
+ llvm::DIType *RecordTy) {
const ASTRecordLayout &RL = CGM.getContext().getASTRecordLayout(RD);
for (const auto &BI : RD->bases()) {
unsigned BFlags = 0;
@@ -1212,24 +1219,24 @@ void CGDebugInfo::CollectCXXBases(const CXXRecordDecl *RD, llvm::DIFile Unit,
BaseOffset =
4 * CGM.getMicrosoftVTableContext().getVBTableIndex(RD, Base);
}
- BFlags = llvm::DIDescriptor::FlagVirtual;
+ BFlags = llvm::DINode::FlagVirtual;
} else
BaseOffset = CGM.getContext().toBits(RL.getBaseClassOffset(Base));
// FIXME: Inconsistent units for BaseOffset. It is in bytes when
// BI->isVirtual() and bits when not.
BFlags |= getAccessFlag(BI.getAccessSpecifier(), RD);
- llvm::DIType DTy = DBuilder.createInheritance(
+ llvm::DIType *DTy = DBuilder.createInheritance(
RecordTy, getOrCreateType(BI.getType(), Unit), BaseOffset, BFlags);
EltTys.push_back(DTy);
}
}
/// CollectTemplateParams - A helper function to collect template parameters.
-llvm::DIArray
+llvm::DINodeArray
CGDebugInfo::CollectTemplateParams(const TemplateParameterList *TPList,
ArrayRef<TemplateArgument> TAList,
- llvm::DIFile Unit) {
+ llvm::DIFile *Unit) {
SmallVector<llvm::Metadata *, 16> TemplateParams;
for (unsigned i = 0, e = TAList.size(); i != e; ++i) {
const TemplateArgument &TA = TAList[i];
@@ -1238,23 +1245,20 @@ CGDebugInfo::CollectTemplateParams(const TemplateParameterList *TPList,
Name = TPList->getParam(i)->getName();
switch (TA.getKind()) {
case TemplateArgument::Type: {
- llvm::DIType TTy = getOrCreateType(TA.getAsType(), Unit);
- llvm::DITemplateTypeParameter TTP =
- DBuilder.createTemplateTypeParameter(TheCU, Name, TTy);
- TemplateParams.push_back(TTP);
+ llvm::DIType *TTy = getOrCreateType(TA.getAsType(), Unit);
+ TemplateParams.push_back(
+ DBuilder.createTemplateTypeParameter(TheCU, Name, TTy));
} break;
case TemplateArgument::Integral: {
- llvm::DIType TTy = getOrCreateType(TA.getIntegralType(), Unit);
- llvm::DITemplateValueParameter TVP =
- DBuilder.createTemplateValueParameter(
- TheCU, Name, TTy,
- llvm::ConstantInt::get(CGM.getLLVMContext(), TA.getAsIntegral()));
- TemplateParams.push_back(TVP);
+ llvm::DIType *TTy = getOrCreateType(TA.getIntegralType(), Unit);
+ TemplateParams.push_back(DBuilder.createTemplateValueParameter(
+ TheCU, Name, TTy,
+ llvm::ConstantInt::get(CGM.getLLVMContext(), TA.getAsIntegral())));
} break;
case TemplateArgument::Declaration: {
const ValueDecl *D = TA.getAsDecl();
QualType T = TA.getParamTypeForDecl().getDesugaredType(CGM.getContext());
- llvm::DIType TTy = getOrCreateType(T, Unit);
+ llvm::DIType *TTy = getOrCreateType(T, Unit);
llvm::Constant *V = nullptr;
const CXXMethodDecl *MD;
// Variable pointer template parameters have a value that is the address
@@ -1278,15 +1282,13 @@ CGDebugInfo::CollectTemplateParams(const TemplateParameterList *TPList,
CGM.getContext().toCharUnitsFromBits((int64_t)fieldOffset);
V = CGM.getCXXABI().EmitMemberDataPointer(MPT, chars);
}
- llvm::DITemplateValueParameter TVP =
- DBuilder.createTemplateValueParameter(
- TheCU, Name, TTy,
- cast_or_null<llvm::Constant>(V->stripPointerCasts()));
- TemplateParams.push_back(TVP);
+ TemplateParams.push_back(DBuilder.createTemplateValueParameter(
+ TheCU, Name, TTy,
+ cast_or_null<llvm::Constant>(V->stripPointerCasts())));
} break;
case TemplateArgument::NullPtr: {
QualType T = TA.getNullPtrType();
- llvm::DIType TTy = getOrCreateType(T, Unit);
+ llvm::DIType *TTy = getOrCreateType(T, Unit);
llvm::Constant *V = nullptr;
// Special case member data pointer null values since they're actually -1
// instead of zero.
@@ -1301,24 +1303,19 @@ CGDebugInfo::CollectTemplateParams(const TemplateParameterList *TPList,
V = CGM.getCXXABI().EmitNullMemberPointer(MPT);
if (!V)
V = llvm::ConstantInt::get(CGM.Int8Ty, 0);
- llvm::DITemplateValueParameter TVP =
- DBuilder.createTemplateValueParameter(TheCU, Name, TTy,
- cast<llvm::Constant>(V));
- TemplateParams.push_back(TVP);
- } break;
- case TemplateArgument::Template: {
- llvm::DITemplateValueParameter
- TVP = DBuilder.createTemplateTemplateParameter(
- TheCU, Name, llvm::DIType(),
- TA.getAsTemplate().getAsTemplateDecl()->getQualifiedNameAsString());
- TemplateParams.push_back(TVP);
- } break;
- case TemplateArgument::Pack: {
- llvm::DITemplateValueParameter TVP = DBuilder.createTemplateParameterPack(
- TheCU, Name, llvm::DIType(),
- CollectTemplateParams(nullptr, TA.getPackAsArray(), Unit));
- TemplateParams.push_back(TVP);
+ TemplateParams.push_back(DBuilder.createTemplateValueParameter(
+ TheCU, Name, TTy, cast<llvm::Constant>(V)));
} break;
+ case TemplateArgument::Template:
+ TemplateParams.push_back(DBuilder.createTemplateTemplateParameter(
+ TheCU, Name, nullptr,
+ TA.getAsTemplate().getAsTemplateDecl()->getQualifiedNameAsString()));
+ break;
+ case TemplateArgument::Pack:
+ TemplateParams.push_back(DBuilder.createTemplateParameterPack(
+ TheCU, Name, nullptr,
+ CollectTemplateParams(nullptr, TA.getPackAsArray(), Unit)));
+ break;
case TemplateArgument::Expression: {
const Expr *E = TA.getAsExpr();
QualType T = E->getType();
@@ -1326,11 +1323,9 @@ CGDebugInfo::CollectTemplateParams(const TemplateParameterList *TPList,
T = CGM.getContext().getLValueReferenceType(T);
llvm::Constant *V = CGM.EmitConstantExpr(E, T);
assert(V && "Expression in template argument isn't constant");
- llvm::DIType TTy = getOrCreateType(T, Unit);
- llvm::DITemplateValueParameter TVP =
- DBuilder.createTemplateValueParameter(
- TheCU, Name, TTy, cast<llvm::Constant>(V->stripPointerCasts()));
- TemplateParams.push_back(TVP);
+ llvm::DIType *TTy = getOrCreateType(T, Unit);
+ TemplateParams.push_back(DBuilder.createTemplateValueParameter(
+ TheCU, Name, TTy, cast<llvm::Constant>(V->stripPointerCasts())));
} break;
// And the following should never occur:
case TemplateArgument::TemplateExpansion:
@@ -1344,8 +1339,9 @@ CGDebugInfo::CollectTemplateParams(const TemplateParameterList *TPList,
/// CollectFunctionTemplateParams - A helper function to collect debug
/// info for function template parameters.
-llvm::DIArray CGDebugInfo::CollectFunctionTemplateParams(const FunctionDecl *FD,
- llvm::DIFile Unit) {
+llvm::DINodeArray
+CGDebugInfo::CollectFunctionTemplateParams(const FunctionDecl *FD,
+ llvm::DIFile *Unit) {
if (FD->getTemplatedKind() ==
FunctionDecl::TK_FunctionTemplateSpecialization) {
const TemplateParameterList *TList = FD->getTemplateSpecializationInfo()
@@ -1354,13 +1350,13 @@ llvm::DIArray CGDebugInfo::CollectFunctionTemplateParams(const FunctionDecl *FD,
return CollectTemplateParams(
TList, FD->getTemplateSpecializationArgs()->asArray(), Unit);
}
- return llvm::DIArray();
+ return llvm::DINodeArray();
}
/// CollectCXXTemplateParams - A helper function to collect debug info for
/// template parameters.
-llvm::DIArray CGDebugInfo::CollectCXXTemplateParams(
- const ClassTemplateSpecializationDecl *TSpecial, llvm::DIFile Unit) {
+llvm::DINodeArray CGDebugInfo::CollectCXXTemplateParams(
+ const ClassTemplateSpecializationDecl *TSpecial, llvm::DIFile *Unit) {
// Always get the full list of parameters, not just the ones from
// the specialization.
TemplateParameterList *TPList =
@@ -1370,18 +1366,18 @@ llvm::DIArray CGDebugInfo::CollectCXXTemplateParams(
}
/// getOrCreateVTablePtrType - Return debug info descriptor for vtable.
-llvm::DIType CGDebugInfo::getOrCreateVTablePtrType(llvm::DIFile Unit) {
- if (VTablePtrType.isValid())
+llvm::DIType *CGDebugInfo::getOrCreateVTablePtrType(llvm::DIFile *Unit) {
+ if (VTablePtrType)
return VTablePtrType;
ASTContext &Context = CGM.getContext();
/* Function type */
llvm::Metadata *STy = getOrCreateType(Context.IntTy, Unit);
- llvm::DITypeArray SElements = DBuilder.getOrCreateTypeArray(STy);
- llvm::DIType SubTy = DBuilder.createSubroutineType(Unit, SElements);
+ llvm::DITypeRefArray SElements = DBuilder.getOrCreateTypeArray(STy);
+ llvm::DIType *SubTy = DBuilder.createSubroutineType(Unit, SElements);
unsigned Size = Context.getTypeSize(Context.VoidPtrTy);
- llvm::DIType vtbl_ptr_type =
+ llvm::DIType *vtbl_ptr_type =
DBuilder.createPointerType(SubTy, Size, 0, "__vtbl_ptr_type");
VTablePtrType = DBuilder.createPointerType(vtbl_ptr_type, Size);
return VTablePtrType;
@@ -1395,7 +1391,7 @@ StringRef CGDebugInfo::getVTableName(const CXXRecordDecl *RD) {
/// CollectVTableInfo - If the C++ class has vtable info then insert appropriate
/// debug info entry in EltTys vector.
-void CGDebugInfo::CollectVTableInfo(const CXXRecordDecl *RD, llvm::DIFile Unit,
+void CGDebugInfo::CollectVTableInfo(const CXXRecordDecl *RD, llvm::DIFile *Unit,
SmallVectorImpl<llvm::Metadata *> &EltTys) {
const ASTRecordLayout &RL = CGM.getContext().getASTRecordLayout(RD);
@@ -1408,26 +1404,26 @@ void CGDebugInfo::CollectVTableInfo(const CXXRecordDecl *RD, llvm::DIFile Unit,
return;
unsigned Size = CGM.getContext().getTypeSize(CGM.getContext().VoidPtrTy);
- llvm::DIType VPTR = DBuilder.createMemberType(
+ llvm::DIType *VPTR = DBuilder.createMemberType(
Unit, getVTableName(RD), Unit, 0, Size, 0, 0,
- llvm::DIDescriptor::FlagArtificial, getOrCreateVTablePtrType(Unit));
+ llvm::DINode::FlagArtificial, getOrCreateVTablePtrType(Unit));
EltTys.push_back(VPTR);
}
/// getOrCreateRecordType - Emit record type's standalone debug info.
-llvm::DIType CGDebugInfo::getOrCreateRecordType(QualType RTy,
- SourceLocation Loc) {
+llvm::DIType *CGDebugInfo::getOrCreateRecordType(QualType RTy,
+ SourceLocation Loc) {
assert(DebugKind >= CodeGenOptions::LimitedDebugInfo);
- llvm::DIType T = getOrCreateType(RTy, getOrCreateFile(Loc));
+ llvm::DIType *T = getOrCreateType(RTy, getOrCreateFile(Loc));
return T;
}
/// getOrCreateInterfaceType - Emit an objective c interface type standalone
/// debug info.
-llvm::DIType CGDebugInfo::getOrCreateInterfaceType(QualType D,
- SourceLocation Loc) {
+llvm::DIType *CGDebugInfo::getOrCreateInterfaceType(QualType D,
+ SourceLocation Loc) {
assert(DebugKind >= CodeGenOptions::LimitedDebugInfo);
- llvm::DIType T = getOrCreateType(D, getOrCreateFile(Loc));
+ llvm::DIType *T = getOrCreateType(D, getOrCreateFile(Loc));
RetainedTypes.push_back(D.getAsOpaquePtr());
return T;
}
@@ -1438,11 +1434,10 @@ void CGDebugInfo::completeType(const EnumDecl *ED) {
QualType Ty = CGM.getContext().getEnumType(ED);
void *TyPtr = Ty.getAsOpaquePtr();
auto I = TypeCache.find(TyPtr);
- if (I == TypeCache.end() ||
- !llvm::DIType(cast<llvm::MDNode>(I->second)).isForwardDecl())
+ if (I == TypeCache.end() || !cast<llvm::DIType>(I->second)->isForwardDecl())
return;
- llvm::DIType Res = CreateTypeDefinition(Ty->castAs<EnumType>());
- assert(!Res.isForwardDecl());
+ llvm::DIType *Res = CreateTypeDefinition(Ty->castAs<EnumType>());
+ assert(!Res->isForwardDecl());
TypeCache[TyPtr].reset(Res);
}
@@ -1461,8 +1456,8 @@ void CGDebugInfo::completeRequiredType(const RecordDecl *RD) {
return;
QualType Ty = CGM.getContext().getRecordType(RD);
- llvm::DIType T = getTypeOrNull(Ty);
- if (T && T.isForwardDecl())
+ llvm::DIType *T = getTypeOrNull(Ty);
+ if (T && T->isForwardDecl())
completeClassData(RD);
}
@@ -1472,11 +1467,10 @@ void CGDebugInfo::completeClassData(const RecordDecl *RD) {
QualType Ty = CGM.getContext().getRecordType(RD);
void *TyPtr = Ty.getAsOpaquePtr();
auto I = TypeCache.find(TyPtr);
- if (I != TypeCache.end() &&
- !llvm::DIType(cast<llvm::MDNode>(I->second)).isForwardDecl())
+ if (I != TypeCache.end() && !cast<llvm::DIType>(I->second)->isForwardDecl())
return;
- llvm::DIType Res = CreateTypeDefinition(Ty->castAs<RecordType>());
- assert(!Res.isForwardDecl());
+ llvm::DIType *Res = CreateTypeDefinition(Ty->castAs<RecordType>());
+ assert(!Res->isForwardDecl());
TypeCache[TyPtr].reset(Res);
}
@@ -1524,9 +1518,9 @@ static bool shouldOmitDefinition(CodeGenOptions::DebugInfoKind DebugKind,
}
/// CreateType - get structure or union type.
-llvm::DIType CGDebugInfo::CreateType(const RecordType *Ty) {
+llvm::DIType *CGDebugInfo::CreateType(const RecordType *Ty) {
RecordDecl *RD = Ty->getDecl();
- llvm::DICompositeType T(getTypeOrNull(QualType(Ty, 0)));
+ llvm::DIType *T = cast_or_null<llvm::DIType>(getTypeOrNull(QualType(Ty, 0)));
if (T || shouldOmitDefinition(DebugKind, RD, CGM.getLangOpts())) {
if (!T)
T = getOrCreateRecordFwdDecl(
@@ -1537,11 +1531,11 @@ llvm::DIType CGDebugInfo::CreateType(const RecordType *Ty) {
return CreateTypeDefinition(Ty);
}
-llvm::DIType CGDebugInfo::CreateTypeDefinition(const RecordType *Ty) {
+llvm::DIType *CGDebugInfo::CreateTypeDefinition(const RecordType *Ty) {
RecordDecl *RD = Ty->getDecl();
// Get overall information about the record type for the debug info.
- llvm::DIFile DefUnit = getOrCreateFile(RD->getLocation());
+ llvm::DIFile *DefUnit = getOrCreateFile(RD->getLocation());
// Records and classes and unions can all be recursive. To handle them, we
// first generate a debug descriptor for the struct as a forward declaration.
@@ -1550,11 +1544,11 @@ llvm::DIType CGDebugInfo::CreateTypeDefinition(const RecordType *Ty) {
// may refer to the forward decl if the struct is recursive) and replace all
// uses of the forward declaration with the final definition.
- llvm::DICompositeType FwdDecl(getOrCreateLimitedType(Ty, DefUnit));
- assert(FwdDecl.isCompositeType() &&
- "The debug type of a RecordType should be a llvm::DICompositeType");
+ auto *FwdDecl =
+ cast<llvm::DICompositeType>(getOrCreateLimitedType(Ty, DefUnit));
- if (FwdDecl.isForwardDecl())
+ const RecordDecl *D = RD->getDefinition();
+ if (!D || !D->isCompleteDefinition())
return FwdDecl;
if (const CXXRecordDecl *CXXDecl = dyn_cast<CXXRecordDecl>(RD))
@@ -1586,16 +1580,20 @@ llvm::DIType CGDebugInfo::CreateTypeDefinition(const RecordType *Ty) {
LexicalBlockStack.pop_back();
RegionMap.erase(Ty->getDecl());
- llvm::DIArray Elements = DBuilder.getOrCreateArray(EltTys);
+ llvm::DINodeArray Elements = DBuilder.getOrCreateArray(EltTys);
DBuilder.replaceArrays(FwdDecl, Elements);
+ if (FwdDecl->isTemporary())
+ FwdDecl =
+ llvm::MDNode::replaceWithPermanent(llvm::TempDICompositeType(FwdDecl));
+
RegionMap[Ty->getDecl()].reset(FwdDecl);
return FwdDecl;
}
/// CreateType - get objective-c object type.
-llvm::DIType CGDebugInfo::CreateType(const ObjCObjectType *Ty,
- llvm::DIFile Unit) {
+llvm::DIType *CGDebugInfo::CreateType(const ObjCObjectType *Ty,
+ llvm::DIFile *Unit) {
// Ignore protocols.
return getOrCreateType(Ty->getBaseType(), Unit);
}
@@ -1625,22 +1623,23 @@ static bool hasDefaultSetterName(const ObjCPropertyDecl *PD,
}
/// CreateType - get objective-c interface type.
-llvm::DIType CGDebugInfo::CreateType(const ObjCInterfaceType *Ty,
- llvm::DIFile Unit) {
+llvm::DIType *CGDebugInfo::CreateType(const ObjCInterfaceType *Ty,
+ llvm::DIFile *Unit) {
ObjCInterfaceDecl *ID = Ty->getDecl();
if (!ID)
- return llvm::DIType();
+ return nullptr;
// Get overall information about the record type for the debug info.
- llvm::DIFile DefUnit = getOrCreateFile(ID->getLocation());
+ llvm::DIFile *DefUnit = getOrCreateFile(ID->getLocation());
unsigned Line = getLineNumber(ID->getLocation());
- llvm::dwarf::SourceLanguage RuntimeLang = TheCU.getLanguage();
+ auto RuntimeLang =
+ static_cast<llvm::dwarf::SourceLanguage>(TheCU->getSourceLanguage());
// If this is just a forward declaration return a special forward-declaration
// debug type since we won't be able to lay out the entire type.
ObjCInterfaceDecl *Def = ID->getDefinition();
if (!Def || !Def->getImplementation()) {
- llvm::DIType FwdDecl = DBuilder.createReplaceableForwardDecl(
+ llvm::DIType *FwdDecl = DBuilder.createReplaceableCompositeType(
llvm::dwarf::DW_TAG_structure_type, ID->getName(), TheCU, DefUnit, Line,
RuntimeLang);
ObjCInterfaceCache.push_back(ObjCInterfaceCacheEntry(Ty, FwdDecl, Unit));
@@ -1650,12 +1649,12 @@ llvm::DIType CGDebugInfo::CreateType(const ObjCInterfaceType *Ty,
return CreateTypeDefinition(Ty, Unit);
}
-llvm::DIType CGDebugInfo::CreateTypeDefinition(const ObjCInterfaceType *Ty,
- llvm::DIFile Unit) {
+llvm::DIType *CGDebugInfo::CreateTypeDefinition(const ObjCInterfaceType *Ty,
+ llvm::DIFile *Unit) {
ObjCInterfaceDecl *ID = Ty->getDecl();
- llvm::DIFile DefUnit = getOrCreateFile(ID->getLocation());
+ llvm::DIFile *DefUnit = getOrCreateFile(ID->getLocation());
unsigned Line = getLineNumber(ID->getLocation());
- unsigned RuntimeLang = TheCU.getLanguage();
+ unsigned RuntimeLang = TheCU->getSourceLanguage();
// Bit size, align and offset of the type.
uint64_t Size = CGM.getContext().getTypeSize(Ty);
@@ -1663,17 +1662,17 @@ llvm::DIType CGDebugInfo::CreateTypeDefinition(const ObjCInterfaceType *Ty,
unsigned Flags = 0;
if (ID->getImplementation())
- Flags |= llvm::DIDescriptor::FlagObjcClassComplete;
+ Flags |= llvm::DINode::FlagObjcClassComplete;
- llvm::DICompositeType RealDecl = DBuilder.createStructType(
- Unit, ID->getName(), DefUnit, Line, Size, Align, Flags, llvm::DIType(),
- llvm::DIArray(), RuntimeLang);
+ llvm::DICompositeType *RealDecl = DBuilder.createStructType(
+ Unit, ID->getName(), DefUnit, Line, Size, Align, Flags, nullptr,
+ llvm::DINodeArray(), RuntimeLang);
QualType QTy(Ty, 0);
TypeCache[QTy.getAsOpaquePtr()].reset(RealDecl);
// Push the struct on region stack.
- LexicalBlockStack.emplace_back(static_cast<llvm::MDNode *>(RealDecl));
+ LexicalBlockStack.emplace_back(RealDecl);
RegionMap[Ty->getDecl()].reset(RealDecl);
// Convert all the elements.
@@ -1681,19 +1680,19 @@ llvm::DIType CGDebugInfo::CreateTypeDefinition(const ObjCInterfaceType *Ty,
ObjCInterfaceDecl *SClass = ID->getSuperClass();
if (SClass) {
- llvm::DIType SClassTy =
+ llvm::DIType *SClassTy =
getOrCreateType(CGM.getContext().getObjCInterfaceType(SClass), Unit);
- if (!SClassTy.isValid())
- return llvm::DIType();
+ if (!SClassTy)
+ return nullptr;
- llvm::DIType InhTag = DBuilder.createInheritance(RealDecl, SClassTy, 0, 0);
+ llvm::DIType *InhTag = DBuilder.createInheritance(RealDecl, SClassTy, 0, 0);
EltTys.push_back(InhTag);
}
// Create entries for all of the properties.
for (const auto *PD : ID->properties()) {
SourceLocation Loc = PD->getLocation();
- llvm::DIFile PUnit = getOrCreateFile(Loc);
+ llvm::DIFile *PUnit = getOrCreateFile(Loc);
unsigned PLine = getLineNumber(Loc);
ObjCMethodDecl *Getter = PD->getGetterMethodDecl();
ObjCMethodDecl *Setter = PD->getSetterMethodDecl();
@@ -1711,9 +1710,9 @@ llvm::DIType CGDebugInfo::CreateTypeDefinition(const ObjCInterfaceType *Ty,
unsigned FieldNo = 0;
for (ObjCIvarDecl *Field = ID->all_declared_ivar_begin(); Field;
Field = Field->getNextIvar(), ++FieldNo) {
- llvm::DIType FieldTy = getOrCreateType(Field->getType(), Unit);
- if (!FieldTy.isValid())
- return llvm::DIType();
+ llvm::DIType *FieldTy = getOrCreateType(Field->getType(), Unit);
+ if (!FieldTy)
+ return nullptr;
StringRef FieldName = Field->getName();
@@ -1722,7 +1721,7 @@ llvm::DIType CGDebugInfo::CreateTypeDefinition(const ObjCInterfaceType *Ty,
continue;
// Get the location for the field.
- llvm::DIFile FieldDefUnit = getOrCreateFile(Field->getLocation());
+ llvm::DIFile *FieldDefUnit = getOrCreateFile(Field->getLocation());
unsigned FieldLine = getLineNumber(Field->getLocation());
QualType FType = Field->getType();
uint64_t FieldSize = 0;
@@ -1755,11 +1754,11 @@ llvm::DIType CGDebugInfo::CreateTypeDefinition(const ObjCInterfaceType *Ty,
unsigned Flags = 0;
if (Field->getAccessControl() == ObjCIvarDecl::Protected)
- Flags = llvm::DIDescriptor::FlagProtected;
+ Flags = llvm::DINode::FlagProtected;
else if (Field->getAccessControl() == ObjCIvarDecl::Private)
- Flags = llvm::DIDescriptor::FlagPrivate;
+ Flags = llvm::DINode::FlagPrivate;
else if (Field->getAccessControl() == ObjCIvarDecl::Public)
- Flags = llvm::DIDescriptor::FlagPublic;
+ Flags = llvm::DINode::FlagPublic;
llvm::MDNode *PropertyNode = nullptr;
if (ObjCImplementationDecl *ImpD = ID->getImplementation()) {
@@ -1767,7 +1766,7 @@ llvm::DIType CGDebugInfo::CreateTypeDefinition(const ObjCInterfaceType *Ty,
ImpD->FindPropertyImplIvarDecl(Field->getIdentifier())) {
if (ObjCPropertyDecl *PD = PImpD->getPropertyDecl()) {
SourceLocation Loc = PD->getLocation();
- llvm::DIFile PUnit = getOrCreateFile(Loc);
+ llvm::DIFile *PUnit = getOrCreateFile(Loc);
unsigned PLine = getLineNumber(Loc);
ObjCMethodDecl *Getter = PD->getGetterMethodDecl();
ObjCMethodDecl *Setter = PD->getSetterMethodDecl();
@@ -1788,15 +1787,16 @@ llvm::DIType CGDebugInfo::CreateTypeDefinition(const ObjCInterfaceType *Ty,
EltTys.push_back(FieldTy);
}
- llvm::DIArray Elements = DBuilder.getOrCreateArray(EltTys);
+ llvm::DINodeArray Elements = DBuilder.getOrCreateArray(EltTys);
DBuilder.replaceArrays(RealDecl, Elements);
LexicalBlockStack.pop_back();
return RealDecl;
}
-llvm::DIType CGDebugInfo::CreateType(const VectorType *Ty, llvm::DIFile Unit) {
- llvm::DIType ElementTy = getOrCreateType(Ty->getElementType(), Unit);
+llvm::DIType *CGDebugInfo::CreateType(const VectorType *Ty,
+ llvm::DIFile *Unit) {
+ llvm::DIType *ElementTy = getOrCreateType(Ty->getElementType(), Unit);
int64_t Count = Ty->getNumElements();
if (Count == 0)
// If number of elements are not known then this is an unbounded array.
@@ -1804,7 +1804,7 @@ llvm::DIType CGDebugInfo::CreateType(const VectorType *Ty, llvm::DIFile Unit) {
Count = -1;
llvm::Metadata *Subscript = DBuilder.getOrCreateSubrange(0, Count);
- llvm::DIArray SubscriptArray = DBuilder.getOrCreateArray(Subscript);
+ llvm::DINodeArray SubscriptArray = DBuilder.getOrCreateArray(Subscript);
uint64_t Size = CGM.getContext().getTypeSize(Ty);
uint64_t Align = CGM.getContext().getTypeAlign(Ty);
@@ -1812,7 +1812,7 @@ llvm::DIType CGDebugInfo::CreateType(const VectorType *Ty, llvm::DIFile Unit) {
return DBuilder.createVectorType(Size, Align, ElementTy, SubscriptArray);
}
-llvm::DIType CGDebugInfo::CreateType(const ArrayType *Ty, llvm::DIFile Unit) {
+llvm::DIType *CGDebugInfo::CreateType(const ArrayType *Ty, llvm::DIFile *Unit) {
uint64_t Size;
uint64_t Align;
@@ -1858,32 +1858,33 @@ llvm::DIType CGDebugInfo::CreateType(const ArrayType *Ty, llvm::DIFile Unit) {
EltTy = Ty->getElementType();
}
- llvm::DIArray SubscriptArray = DBuilder.getOrCreateArray(Subscripts);
+ llvm::DINodeArray SubscriptArray = DBuilder.getOrCreateArray(Subscripts);
- llvm::DIType DbgTy = DBuilder.createArrayType(
- Size, Align, getOrCreateType(EltTy, Unit), SubscriptArray);
- return DbgTy;
+ return DBuilder.createArrayType(Size, Align, getOrCreateType(EltTy, Unit),
+ SubscriptArray);
}
-llvm::DIType CGDebugInfo::CreateType(const LValueReferenceType *Ty,
- llvm::DIFile Unit) {
+llvm::DIType *CGDebugInfo::CreateType(const LValueReferenceType *Ty,
+ llvm::DIFile *Unit) {
return CreatePointerLikeType(llvm::dwarf::DW_TAG_reference_type, Ty,
Ty->getPointeeType(), Unit);
}
-llvm::DIType CGDebugInfo::CreateType(const RValueReferenceType *Ty,
- llvm::DIFile Unit) {
+llvm::DIType *CGDebugInfo::CreateType(const RValueReferenceType *Ty,
+ llvm::DIFile *Unit) {
return CreatePointerLikeType(llvm::dwarf::DW_TAG_rvalue_reference_type, Ty,
Ty->getPointeeType(), Unit);
}
-llvm::DIType CGDebugInfo::CreateType(const MemberPointerType *Ty,
- llvm::DIFile U) {
- llvm::DIType ClassType = getOrCreateType(QualType(Ty->getClass(), 0), U);
- if (!Ty->getPointeeType()->isFunctionType())
+llvm::DIType *CGDebugInfo::CreateType(const MemberPointerType *Ty,
+ llvm::DIFile *U) {
+ uint64_t Size = CGM.getCXXABI().isTypeInfoCalculable(QualType(Ty, 0))
+ ? CGM.getContext().getTypeSize(Ty)
+ : 0;
+ llvm::DIType *ClassType = getOrCreateType(QualType(Ty->getClass(), 0), U);
+ if (Ty->isMemberDataPointerType())
return DBuilder.createMemberPointerType(
- getOrCreateType(Ty->getPointeeType(), U), ClassType,
- CGM.getContext().getTypeSize(Ty));
+ getOrCreateType(Ty->getPointeeType(), U), ClassType, Size);
const FunctionProtoType *FPT =
Ty->getPointeeType()->getAs<FunctionProtoType>();
@@ -1891,17 +1892,17 @@ llvm::DIType CGDebugInfo::CreateType(const MemberPointerType *Ty,
getOrCreateInstanceMethodType(CGM.getContext().getPointerType(QualType(
Ty->getClass(), FPT->getTypeQuals())),
FPT, U),
- ClassType, CGM.getContext().getTypeSize(Ty));
+ ClassType, Size);
}
-llvm::DIType CGDebugInfo::CreateType(const AtomicType *Ty, llvm::DIFile U) {
+llvm::DIType *CGDebugInfo::CreateType(const AtomicType *Ty, llvm::DIFile *U) {
// Ignore the atomic wrapping
// FIXME: What is the correct representation?
return getOrCreateType(Ty->getValueType(), U);
}
/// CreateEnumType - get enumeration type.
-llvm::DIType CGDebugInfo::CreateEnumType(const EnumType *Ty) {
+llvm::DIType *CGDebugInfo::CreateEnumType(const EnumType *Ty) {
const EnumDecl *ED = Ty->getDecl();
uint64_t Size = 0;
uint64_t Align = 0;
@@ -1915,14 +1916,14 @@ llvm::DIType CGDebugInfo::CreateEnumType(const EnumType *Ty) {
// If this is just a forward declaration, construct an appropriately
// marked node and just return it.
if (!ED->getDefinition()) {
- llvm::DIDescriptor EDContext;
- EDContext = getContextDescriptor(cast<Decl>(ED->getDeclContext()));
- llvm::DIFile DefUnit = getOrCreateFile(ED->getLocation());
+ llvm::DIScope *EDContext =
+ getContextDescriptor(cast<Decl>(ED->getDeclContext()));
+ llvm::DIFile *DefUnit = getOrCreateFile(ED->getLocation());
unsigned Line = getLineNumber(ED->getLocation());
StringRef EDName = ED->getName();
- llvm::DIType RetTy = DBuilder.createReplaceableForwardDecl(
+ llvm::DIType *RetTy = DBuilder.createReplaceableCompositeType(
llvm::dwarf::DW_TAG_enumeration_type, EDName, EDContext, DefUnit, Line,
- 0, Size, Align, FullName);
+ 0, Size, Align, llvm::DINode::FlagFwdDecl, FullName);
ReplaceMap.emplace_back(
std::piecewise_construct, std::make_tuple(Ty),
std::make_tuple(static_cast<llvm::Metadata *>(RetTy)));
@@ -1932,7 +1933,7 @@ llvm::DIType CGDebugInfo::CreateEnumType(const EnumType *Ty) {
return CreateTypeDefinition(Ty);
}
-llvm::DIType CGDebugInfo::CreateTypeDefinition(const EnumType *Ty) {
+llvm::DIType *CGDebugInfo::CreateTypeDefinition(const EnumType *Ty) {
const EnumDecl *ED = Ty->getDecl();
uint64_t Size = 0;
uint64_t Align = 0;
@@ -1943,7 +1944,7 @@ llvm::DIType CGDebugInfo::CreateTypeDefinition(const EnumType *Ty) {
SmallString<256> FullName = getUniqueTagTypeName(Ty, CGM, TheCU);
- // Create DIEnumerator elements for each enumerator.
+ // Create elements for each enumerator.
SmallVector<llvm::Metadata *, 16> Enumerators;
ED = ED->getDefinition();
for (const auto *Enum : ED->enumerators()) {
@@ -1952,19 +1953,17 @@ llvm::DIType CGDebugInfo::CreateTypeDefinition(const EnumType *Ty) {
}
// Return a CompositeType for the enum itself.
- llvm::DIArray EltArray = DBuilder.getOrCreateArray(Enumerators);
+ llvm::DINodeArray EltArray = DBuilder.getOrCreateArray(Enumerators);
- llvm::DIFile DefUnit = getOrCreateFile(ED->getLocation());
+ llvm::DIFile *DefUnit = getOrCreateFile(ED->getLocation());
unsigned Line = getLineNumber(ED->getLocation());
- llvm::DIDescriptor EnumContext =
+ llvm::DIScope *EnumContext =
getContextDescriptor(cast<Decl>(ED->getDeclContext()));
- llvm::DIType ClassTy = ED->isFixed()
- ? getOrCreateType(ED->getIntegerType(), DefUnit)
- : llvm::DIType();
- llvm::DIType DbgTy =
- DBuilder.createEnumerationType(EnumContext, ED->getName(), DefUnit, Line,
- Size, Align, EltArray, ClassTy, FullName);
- return DbgTy;
+ llvm::DIType *ClassTy =
+ ED->isFixed() ? getOrCreateType(ED->getIntegerType(), DefUnit) : nullptr;
+ return DBuilder.createEnumerationType(EnumContext, ED->getName(), DefUnit,
+ Line, Size, Align, EltArray, ClassTy,
+ FullName);
}
static QualType UnwrapTypeForDebugInfo(QualType T, const ASTContext &C) {
@@ -2024,7 +2023,7 @@ static QualType UnwrapTypeForDebugInfo(QualType T, const ASTContext &C) {
/// getType - Get the type from the cache or return null type if it doesn't
/// exist.
-llvm::DIType CGDebugInfo::getTypeOrNull(QualType Ty) {
+llvm::DIType *CGDebugInfo::getTypeOrNull(QualType Ty) {
// Unwrap the type as needed for debug information.
Ty = UnwrapTypeForDebugInfo(Ty, CGM.getContext());
@@ -2033,10 +2032,10 @@ llvm::DIType CGDebugInfo::getTypeOrNull(QualType Ty) {
if (it != TypeCache.end()) {
// Verify that the debug info still exists.
if (llvm::Metadata *V = it->second)
- return llvm::DIType(cast<llvm::MDNode>(V));
+ return cast<llvm::DIType>(V);
}
- return llvm::DIType();
+ return nullptr;
}
void CGDebugInfo::completeTemplateDefinition(
@@ -2052,18 +2051,18 @@ void CGDebugInfo::completeTemplateDefinition(
/// getOrCreateType - Get the type from the cache or create a new
/// one if necessary.
-llvm::DIType CGDebugInfo::getOrCreateType(QualType Ty, llvm::DIFile Unit) {
+llvm::DIType *CGDebugInfo::getOrCreateType(QualType Ty, llvm::DIFile *Unit) {
if (Ty.isNull())
- return llvm::DIType();
+ return nullptr;
// Unwrap the type as needed for debug information.
Ty = UnwrapTypeForDebugInfo(Ty, CGM.getContext());
- if (llvm::DIType T = getTypeOrNull(Ty))
+ if (auto *T = getTypeOrNull(Ty))
return T;
// Otherwise create the type.
- llvm::DIType Res = CreateTypeNode(Ty, Unit);
+ llvm::DIType *Res = CreateTypeNode(Ty, Unit);
void *TyPtr = Ty.getAsOpaquePtr();
// And update the type cache.
@@ -2099,7 +2098,7 @@ ObjCInterfaceDecl *CGDebugInfo::getObjCInterfaceDecl(QualType Ty) {
}
/// CreateTypeNode - Create a new debug type node.
-llvm::DIType CGDebugInfo::CreateTypeNode(QualType Ty, llvm::DIFile Unit) {
+llvm::DIType *CGDebugInfo::CreateTypeNode(QualType Ty, llvm::DIFile *Unit) {
// Handle qualifiers, which recursively handles what they refer to.
if (Ty.hasLocalQualifiers())
return CreateQualifiedType(Ty, Unit);
@@ -2181,25 +2180,25 @@ llvm::DIType CGDebugInfo::CreateTypeNode(QualType Ty, llvm::DIFile Unit) {
/// getOrCreateLimitedType - Get the type from the cache or create a new
/// limited type if necessary.
-llvm::DIType CGDebugInfo::getOrCreateLimitedType(const RecordType *Ty,
- llvm::DIFile Unit) {
+llvm::DIType *CGDebugInfo::getOrCreateLimitedType(const RecordType *Ty,
+ llvm::DIFile *Unit) {
QualType QTy(Ty, 0);
- llvm::DICompositeType T(getTypeOrNull(QTy));
+ auto *T = cast_or_null<llvm::DICompositeTypeBase>(getTypeOrNull(QTy));
// We may have cached a forward decl when we could have created
// a non-forward decl. Go ahead and create a non-forward decl
// now.
- if (T && !T.isForwardDecl())
+ if (T && !T->isForwardDecl())
return T;
// Otherwise create the type.
- llvm::DICompositeType Res = CreateLimitedType(Ty);
+ llvm::DICompositeType *Res = CreateLimitedType(Ty);
// Propagate members from the declaration to the definition
// CreateType(const RecordType*) will overwrite this with the members in the
// correct order if the full type is needed.
- DBuilder.replaceArrays(Res, T.getElements());
+ DBuilder.replaceArrays(Res, T ? T->getElements() : llvm::DINodeArray());
// And update the type cache.
TypeCache[QTy.getAsOpaquePtr()].reset(Res);
@@ -2207,21 +2206,22 @@ llvm::DIType CGDebugInfo::getOrCreateLimitedType(const RecordType *Ty,
}
// TODO: Currently used for context chains when limiting debug info.
-llvm::DICompositeType CGDebugInfo::CreateLimitedType(const RecordType *Ty) {
+llvm::DICompositeType *CGDebugInfo::CreateLimitedType(const RecordType *Ty) {
RecordDecl *RD = Ty->getDecl();
// Get overall information about the record type for the debug info.
- llvm::DIFile DefUnit = getOrCreateFile(RD->getLocation());
+ llvm::DIFile *DefUnit = getOrCreateFile(RD->getLocation());
unsigned Line = getLineNumber(RD->getLocation());
StringRef RDName = getClassName(RD);
- llvm::DIDescriptor RDContext =
+ llvm::DIScope *RDContext =
getContextDescriptor(cast<Decl>(RD->getDeclContext()));
// If we ended up creating the type during the context chain construction,
// just return that.
- llvm::DICompositeType T(getTypeOrNull(CGM.getContext().getRecordType(RD)));
- if (T && (!T.isForwardDecl() || !RD->getDefinition()))
+ auto *T = cast_or_null<llvm::DICompositeType>(
+ getTypeOrNull(CGM.getContext().getRecordType(RD)));
+ if (T && (!T->isForwardDecl() || !RD->getDefinition()))
return T;
// If this is just a forward or incomplete declaration, construct an
@@ -2232,38 +2232,27 @@ llvm::DICompositeType CGDebugInfo::CreateLimitedType(const RecordType *Ty) {
uint64_t Size = CGM.getContext().getTypeSize(Ty);
uint64_t Align = CGM.getContext().getTypeAlign(Ty);
- llvm::DICompositeType RealDecl;
SmallString<256> FullName = getUniqueTagTypeName(Ty, CGM, TheCU);
- if (RD->isUnion())
- RealDecl = DBuilder.createUnionType(RDContext, RDName, DefUnit, Line, Size,
- Align, 0, llvm::DIArray(), 0, FullName);
- else if (RD->isClass()) {
- // FIXME: This could be a struct type giving a default visibility different
- // than C++ class type, but needs llvm metadata changes first.
- RealDecl = DBuilder.createClassType(
- RDContext, RDName, DefUnit, Line, Size, Align, 0, 0, llvm::DIType(),
- llvm::DIArray(), llvm::DIType(), llvm::DIArray(), FullName);
- } else
- RealDecl = DBuilder.createStructType(
- RDContext, RDName, DefUnit, Line, Size, Align, 0, llvm::DIType(),
- llvm::DIArray(), 0, llvm::DIType(), FullName);
+ llvm::DICompositeType *RealDecl = DBuilder.createReplaceableCompositeType(
+ getTagForRecord(RD), RDName, RDContext, DefUnit, Line, 0, Size, Align, 0,
+ FullName);
RegionMap[Ty->getDecl()].reset(RealDecl);
TypeCache[QualType(Ty, 0).getAsOpaquePtr()].reset(RealDecl);
if (const ClassTemplateSpecializationDecl *TSpecial =
dyn_cast<ClassTemplateSpecializationDecl>(RD))
- DBuilder.replaceArrays(RealDecl, llvm::DIArray(),
+ DBuilder.replaceArrays(RealDecl, llvm::DINodeArray(),
CollectCXXTemplateParams(TSpecial, DefUnit));
return RealDecl;
}
void CGDebugInfo::CollectContainingType(const CXXRecordDecl *RD,
- llvm::DICompositeType RealDecl) {
+ llvm::DICompositeType *RealDecl) {
// A class's primary base or the class itself contains the vtable.
- llvm::DICompositeType ContainingType;
+ llvm::DICompositeType *ContainingType = nullptr;
const ASTRecordLayout &RL = CGM.getContext().getASTRecordLayout(RD);
if (const CXXRecordDecl *PBase = RL.getPrimaryBase()) {
// Seek non-virtual primary base root.
@@ -2275,7 +2264,7 @@ void CGDebugInfo::CollectContainingType(const CXXRecordDecl *RD,
else
break;
}
- ContainingType = llvm::DICompositeType(
+ ContainingType = cast<llvm::DICompositeType>(
getOrCreateType(QualType(PBase->getTypeForDecl(), 0),
getOrCreateFile(RD->getLocation())));
} else if (RD->isDynamicClass())
@@ -2285,29 +2274,29 @@ void CGDebugInfo::CollectContainingType(const CXXRecordDecl *RD,
}
/// CreateMemberType - Create new member and increase Offset by FType's size.
-llvm::DIType CGDebugInfo::CreateMemberType(llvm::DIFile Unit, QualType FType,
- StringRef Name, uint64_t *Offset) {
- llvm::DIType FieldTy = CGDebugInfo::getOrCreateType(FType, Unit);
+llvm::DIType *CGDebugInfo::CreateMemberType(llvm::DIFile *Unit, QualType FType,
+ StringRef Name, uint64_t *Offset) {
+ llvm::DIType *FieldTy = CGDebugInfo::getOrCreateType(FType, Unit);
uint64_t FieldSize = CGM.getContext().getTypeSize(FType);
unsigned FieldAlign = CGM.getContext().getTypeAlign(FType);
- llvm::DIType Ty = DBuilder.createMemberType(Unit, Name, Unit, 0, FieldSize,
- FieldAlign, *Offset, 0, FieldTy);
+ llvm::DIType *Ty = DBuilder.createMemberType(Unit, Name, Unit, 0, FieldSize,
+ FieldAlign, *Offset, 0, FieldTy);
*Offset += FieldSize;
return Ty;
}
-void CGDebugInfo::collectFunctionDeclProps(GlobalDecl GD,
- llvm::DIFile Unit,
- StringRef &Name, StringRef &LinkageName,
- llvm::DIDescriptor &FDContext,
- llvm::DIArray &TParamsArray,
+void CGDebugInfo::collectFunctionDeclProps(GlobalDecl GD, llvm::DIFile *Unit,
+ StringRef &Name,
+ StringRef &LinkageName,
+ llvm::DIScope *&FDContext,
+ llvm::DINodeArray &TParamsArray,
unsigned &Flags) {
const FunctionDecl *FD = cast<FunctionDecl>(GD.getDecl());
Name = getFunctionName(FD);
// Use mangled name as linkage name for C/C++ functions.
if (FD->hasPrototype()) {
LinkageName = CGM.getMangledName(GD);
- Flags |= llvm::DIDescriptor::FlagPrototyped;
+ Flags |= llvm::DINode::FlagPrototyped;
}
// No need to replicate the linkage name if it isn't different from the
// subprogram name, no need to have it at all unless coverage is enabled or
@@ -2330,10 +2319,10 @@ void CGDebugInfo::collectFunctionDeclProps(GlobalDecl GD,
}
}
-void CGDebugInfo::collectVarDeclProps(const VarDecl *VD, llvm::DIFile &Unit,
+void CGDebugInfo::collectVarDeclProps(const VarDecl *VD, llvm::DIFile *&Unit,
unsigned &LineNo, QualType &T,
StringRef &Name, StringRef &LinkageName,
- llvm::DIDescriptor &VDContext) {
+ llvm::DIScope *&VDContext) {
Unit = getOrCreateFile(VD->getLocation());
LineNo = getLineNumber(VD->getLocation());
@@ -2362,19 +2351,27 @@ void CGDebugInfo::collectVarDeclProps(const VarDecl *VD, llvm::DIFile &Unit,
// FIXME: Generalize this for even non-member global variables where the
// declaration and definition may have different lexical decl contexts, once
// we have support for emitting declarations of (non-member) global variables.
- VDContext = getContextDescriptor(
- dyn_cast<Decl>(VD->isStaticDataMember() ? VD->getLexicalDeclContext()
- : VD->getDeclContext()));
-}
-
-llvm::DISubprogram
+ const DeclContext *DC = VD->isStaticDataMember() ? VD->getLexicalDeclContext()
+ : VD->getDeclContext();
+ // When a record type contains an in-line initialization of a static data
+ // member, and the record type is marked as __declspec(dllexport), an implicit
+ // definition of the member will be created in the record context. DWARF
+ // doesn't seem to have a nice way to describe this in a form that consumers
+ // are likely to understand, so fake the "normal" situation of a definition
+ // outside the class by putting it in the global scope.
+ if (DC->isRecord())
+ DC = CGM.getContext().getTranslationUnitDecl();
+ VDContext = getContextDescriptor(dyn_cast<Decl>(DC));
+}
+
+llvm::DISubprogram *
CGDebugInfo::getFunctionForwardDeclaration(const FunctionDecl *FD) {
- llvm::DIArray TParamsArray;
+ llvm::DINodeArray TParamsArray;
StringRef Name, LinkageName;
unsigned Flags = 0;
SourceLocation Loc = FD->getLocation();
- llvm::DIFile Unit = getOrCreateFile(Loc);
- llvm::DIDescriptor DContext(Unit);
+ llvm::DIFile *Unit = getOrCreateFile(Loc);
+ llvm::DIScope *DContext = Unit;
unsigned Line = getLineNumber(Loc);
collectFunctionDeclProps(FD, Unit, Name, LinkageName, DContext,
@@ -2386,35 +2383,31 @@ CGDebugInfo::getFunctionForwardDeclaration(const FunctionDecl *FD) {
QualType FnType =
CGM.getContext().getFunctionType(FD->getReturnType(), ArgTypes,
FunctionProtoType::ExtProtoInfo());
- llvm::DISubprogram SP =
- DBuilder.createTempFunctionFwdDecl(DContext, Name, LinkageName, Unit, Line,
- getOrCreateFunctionType(FD, FnType, Unit),
- !FD->isExternallyVisible(),
- false /*declaration*/, 0, Flags,
- CGM.getLangOpts().Optimize, nullptr,
- TParamsArray, getFunctionDeclaration(FD));
+ llvm::DISubprogram *SP = DBuilder.createTempFunctionFwdDecl(
+ DContext, Name, LinkageName, Unit, Line,
+ getOrCreateFunctionType(FD, FnType, Unit), !FD->isExternallyVisible(),
+ false /*declaration*/, 0, Flags, CGM.getLangOpts().Optimize, nullptr,
+ TParamsArray.get(), getFunctionDeclaration(FD));
const FunctionDecl *CanonDecl = cast<FunctionDecl>(FD->getCanonicalDecl());
- FwdDeclReplaceMap.emplace_back(
- std::piecewise_construct, std::make_tuple(CanonDecl),
- std::make_tuple(static_cast<llvm::Metadata *>(SP)));
+ FwdDeclReplaceMap.emplace_back(std::piecewise_construct,
+ std::make_tuple(CanonDecl),
+ std::make_tuple(SP));
return SP;
}
-llvm::DIGlobalVariable
+llvm::DIGlobalVariable *
CGDebugInfo::getGlobalVariableForwardDeclaration(const VarDecl *VD) {
QualType T;
StringRef Name, LinkageName;
SourceLocation Loc = VD->getLocation();
- llvm::DIFile Unit = getOrCreateFile(Loc);
- llvm::DIDescriptor DContext(Unit);
+ llvm::DIFile *Unit = getOrCreateFile(Loc);
+ llvm::DIScope *DContext = Unit;
unsigned Line = getLineNumber(Loc);
collectVarDeclProps(VD, Unit, Line, T, Name, LinkageName, DContext);
- llvm::DIGlobalVariable GV =
- DBuilder.createTempGlobalVariableFwdDecl(DContext, Name, LinkageName, Unit,
- Line, getOrCreateType(T, Unit),
- !VD->isExternallyVisible(),
- nullptr, nullptr);
+ auto *GV = DBuilder.createTempGlobalVariableFwdDecl(
+ DContext, Name, LinkageName, Unit, Line, getOrCreateType(T, Unit),
+ !VD->isExternallyVisible(), nullptr, nullptr);
FwdDeclReplaceMap.emplace_back(
std::piecewise_construct,
std::make_tuple(cast<VarDecl>(VD->getCanonicalDecl())),
@@ -2422,7 +2415,7 @@ CGDebugInfo::getGlobalVariableForwardDeclaration(const VarDecl *VD) {
return GV;
}
-llvm::DIDescriptor CGDebugInfo::getDeclarationOrDefinition(const Decl *D) {
+llvm::DINode *CGDebugInfo::getDeclarationOrDefinition(const Decl *D) {
// We only need a declaration (not a definition) of the type - so use whatever
// we would otherwise do to get a type for a pointee. (forward declarations in
// limited debug info, full definitions (if the type definition is available)
@@ -2433,7 +2426,7 @@ llvm::DIDescriptor CGDebugInfo::getDeclarationOrDefinition(const Decl *D) {
auto I = DeclCache.find(D->getCanonicalDecl());
if (I != DeclCache.end())
- return llvm::DIDescriptor(dyn_cast_or_null<llvm::MDNode>(I->second));
+ return dyn_cast_or_null<llvm::DINode>(I->second);
// No definition for now. Emit a forward definition that might be
// merged with a potential upcoming definition.
@@ -2442,59 +2435,55 @@ llvm::DIDescriptor CGDebugInfo::getDeclarationOrDefinition(const Decl *D) {
else if (const auto *VD = dyn_cast<VarDecl>(D))
return getGlobalVariableForwardDeclaration(VD);
- return llvm::DIDescriptor();
+ return nullptr;
}
/// getFunctionDeclaration - Return debug info descriptor to describe method
/// declaration for the given method definition.
-llvm::DISubprogram CGDebugInfo::getFunctionDeclaration(const Decl *D) {
+llvm::DISubprogram *CGDebugInfo::getFunctionDeclaration(const Decl *D) {
if (!D || DebugKind <= CodeGenOptions::DebugLineTablesOnly)
- return llvm::DISubprogram();
+ return nullptr;
const FunctionDecl *FD = dyn_cast<FunctionDecl>(D);
if (!FD)
- return llvm::DISubprogram();
+ return nullptr;
// Setup context.
- llvm::DIScope S = getContextDescriptor(cast<Decl>(D->getDeclContext()));
+ auto *S = getContextDescriptor(cast<Decl>(D->getDeclContext()));
auto MI = SPCache.find(FD->getCanonicalDecl());
if (MI == SPCache.end()) {
if (const CXXMethodDecl *MD =
dyn_cast<CXXMethodDecl>(FD->getCanonicalDecl())) {
- llvm::DICompositeType T(S);
- llvm::DISubprogram SP =
- CreateCXXMemberFunction(MD, getOrCreateFile(MD->getLocation()), T);
- return SP;
+ return CreateCXXMemberFunction(MD, getOrCreateFile(MD->getLocation()),
+ cast<llvm::DICompositeType>(S));
}
}
if (MI != SPCache.end()) {
- llvm::DISubprogram SP(dyn_cast_or_null<llvm::MDNode>(MI->second));
- if (SP.isSubprogram() && !SP.isDefinition())
+ auto *SP = dyn_cast_or_null<llvm::DISubprogram>(MI->second);
+ if (SP && !SP->isDefinition())
return SP;
}
for (auto NextFD : FD->redecls()) {
auto MI = SPCache.find(NextFD->getCanonicalDecl());
if (MI != SPCache.end()) {
- llvm::DISubprogram SP(dyn_cast_or_null<llvm::MDNode>(MI->second));
- if (SP.isSubprogram() && !SP.isDefinition())
+ auto *SP = dyn_cast_or_null<llvm::DISubprogram>(MI->second);
+ if (SP && !SP->isDefinition())
return SP;
}
}
- return llvm::DISubprogram();
+ return nullptr;
}
-// getOrCreateFunctionType - Construct DIType. If it is a c++ method, include
+// getOrCreateFunctionType - Construct type. If it is a c++ method, include
// implicit parameter "this".
-llvm::DICompositeType CGDebugInfo::getOrCreateFunctionType(const Decl *D,
- QualType FnType,
- llvm::DIFile F) {
+llvm::DISubroutineType *CGDebugInfo::getOrCreateFunctionType(const Decl *D,
+ QualType FnType,
+ llvm::DIFile *F) {
if (!D || DebugKind <= CodeGenOptions::DebugLineTablesOnly)
- // Create fake but valid subroutine type. Otherwise
- // llvm::DISubprogram::Verify() would return false, and
- // subprogram DIE will miss DW_AT_decl_file and
- // DW_AT_decl_line fields.
+ // Create fake but valid subroutine type. Otherwise -verify would fail, and
+ // subprogram DIE will miss DW_AT_decl_file and DW_AT_decl_line fields.
return DBuilder.createSubroutineType(F,
DBuilder.getOrCreateTypeArray(None));
@@ -2515,11 +2504,10 @@ llvm::DICompositeType CGDebugInfo::getOrCreateFunctionType(const Decl *D,
Elts.push_back(getOrCreateType(ResultTy, F));
// "self" pointer is always first argument.
QualType SelfDeclTy = OMethod->getSelfDecl()->getType();
- llvm::DIType SelfTy = getOrCreateType(SelfDeclTy, F);
- Elts.push_back(CreateSelfType(SelfDeclTy, SelfTy));
+ Elts.push_back(CreateSelfType(SelfDeclTy, getOrCreateType(SelfDeclTy, F)));
// "_cmd" pointer is always second argument.
- llvm::DIType CmdTy = getOrCreateType(OMethod->getCmdDecl()->getType(), F);
- Elts.push_back(DBuilder.createArtificialType(CmdTy));
+ Elts.push_back(DBuilder.createArtificialType(
+ getOrCreateType(OMethod->getCmdDecl()->getType(), F)));
// Get rest of the arguments.
for (const auto *PI : OMethod->params())
Elts.push_back(getOrCreateType(PI->getType(), F));
@@ -2527,7 +2515,7 @@ llvm::DICompositeType CGDebugInfo::getOrCreateFunctionType(const Decl *D,
if (OMethod->isVariadic())
Elts.push_back(DBuilder.createUnspecifiedParameter());
- llvm::DITypeArray EltTypeArray = DBuilder.getOrCreateTypeArray(Elts);
+ llvm::DITypeRefArray EltTypeArray = DBuilder.getOrCreateTypeArray(Elts);
return DBuilder.createSubroutineType(F, EltTypeArray);
}
@@ -2541,11 +2529,11 @@ llvm::DICompositeType CGDebugInfo::getOrCreateFunctionType(const Decl *D,
for (unsigned i = 0, e = FPT->getNumParams(); i != e; ++i)
EltTys.push_back(getOrCreateType(FPT->getParamType(i), F));
EltTys.push_back(DBuilder.createUnspecifiedParameter());
- llvm::DITypeArray EltTypeArray = DBuilder.getOrCreateTypeArray(EltTys);
+ llvm::DITypeRefArray EltTypeArray = DBuilder.getOrCreateTypeArray(EltTys);
return DBuilder.createSubroutineType(F, EltTypeArray);
}
- return llvm::DICompositeType(getOrCreateType(FnType, F));
+ return cast<llvm::DISubroutineType>(getOrCreateType(FnType, F));
}
/// EmitFunctionStart - Constructs the debug code for entering a function.
@@ -2562,20 +2550,19 @@ void CGDebugInfo::EmitFunctionStart(GlobalDecl GD, SourceLocation Loc,
bool HasDecl = (D != nullptr);
unsigned Flags = 0;
- llvm::DIFile Unit = getOrCreateFile(Loc);
- llvm::DIDescriptor FDContext(Unit);
- llvm::DIArray TParamsArray;
+ llvm::DIFile *Unit = getOrCreateFile(Loc);
+ llvm::DIScope *FDContext = Unit;
+ llvm::DINodeArray TParamsArray;
if (!HasDecl) {
// Use llvm function name.
LinkageName = Fn->getName();
} else if (const FunctionDecl *FD = dyn_cast<FunctionDecl>(D)) {
- // If there is a DISubprogram for this function available then use it.
+ // If there is a subprogram for this function available then use it.
auto FI = SPCache.find(FD->getCanonicalDecl());
if (FI != SPCache.end()) {
- llvm::DIDescriptor SP(dyn_cast_or_null<llvm::MDNode>(FI->second));
- if (SP.isSubprogram() && llvm::DISubprogram(SP).isDefinition()) {
- llvm::MDNode *SPN = SP;
- LexicalBlockStack.emplace_back(SPN);
+ auto *SP = dyn_cast_or_null<llvm::DISubprogram>(FI->second);
+ if (SP && SP->isDefinition()) {
+ LexicalBlockStack.emplace_back(SP);
RegionMap[D].reset(SP);
return;
}
@@ -2584,17 +2571,17 @@ void CGDebugInfo::EmitFunctionStart(GlobalDecl GD, SourceLocation Loc,
TParamsArray, Flags);
} else if (const ObjCMethodDecl *OMD = dyn_cast<ObjCMethodDecl>(D)) {
Name = getObjCMethodName(OMD);
- Flags |= llvm::DIDescriptor::FlagPrototyped;
+ Flags |= llvm::DINode::FlagPrototyped;
} else {
// Use llvm function name.
Name = Fn->getName();
- Flags |= llvm::DIDescriptor::FlagPrototyped;
+ Flags |= llvm::DINode::FlagPrototyped;
}
if (!Name.empty() && Name[0] == '\01')
Name = Name.substr(1);
if (!HasDecl || D->isImplicit()) {
- Flags |= llvm::DIDescriptor::FlagArtificial;
+ Flags |= llvm::DINode::FlagArtificial;
// Artificial functions without a location should not silently reuse CurLoc.
if (Loc.isInvalid())
CurLoc = SourceLocation();
@@ -2607,11 +2594,11 @@ void CGDebugInfo::EmitFunctionStart(GlobalDecl GD, SourceLocation Loc,
// FunctionDecls. When/if we fix this we can have FDContext be TheCU/null for
// all subprograms instead of the actual context since subprogram definitions
// are emitted as CU level entities by the backend.
- llvm::DISubprogram SP = DBuilder.createFunction(
+ llvm::DISubprogram *SP = DBuilder.createFunction(
FDContext, Name, LinkageName, Unit, LineNo,
getOrCreateFunctionType(D, FnType, Unit), Fn->hasInternalLinkage(),
true /*definition*/, ScopeLine, Flags, CGM.getLangOpts().Optimize, Fn,
- TParamsArray, getFunctionDeclaration(D));
+ TParamsArray.get(), getFunctionDeclaration(D));
// We might get here with a VarDecl in the case we're generating
// code for the initialization of globals. Do not record these decls
// as they will overwrite the actual VarDecl Decl in the cache.
@@ -2619,8 +2606,7 @@ void CGDebugInfo::EmitFunctionStart(GlobalDecl GD, SourceLocation Loc,
DeclCache[D->getCanonicalDecl()].reset(static_cast<llvm::Metadata *>(SP));
// Push the function onto the lexical block stack.
- llvm::MDNode *SPN = SP;
- LexicalBlockStack.emplace_back(SPN);
+ LexicalBlockStack.emplace_back(SP);
if (HasDecl)
RegionMap[D].reset(SP);
@@ -2629,8 +2615,7 @@ void CGDebugInfo::EmitFunctionStart(GlobalDecl GD, SourceLocation Loc,
/// EmitLocation - Emit metadata to indicate a change in line/column
/// information in the source file. If the location is invalid, the
/// previous location will be reused.
-void CGDebugInfo::EmitLocation(CGBuilderTy &Builder, SourceLocation Loc,
- bool ForceColumnInfo) {
+void CGDebugInfo::EmitLocation(CGBuilderTy &Builder, SourceLocation Loc) {
// Update our current location
setLocation(Loc);
@@ -2639,7 +2624,7 @@ void CGDebugInfo::EmitLocation(CGBuilderTy &Builder, SourceLocation Loc,
llvm::MDNode *Scope = LexicalBlockStack.back();
Builder.SetCurrentDebugLocation(llvm::DebugLoc::get(
- getLineNumber(CurLoc), getColumnNumber(CurLoc, ForceColumnInfo), Scope));
+ getLineNumber(CurLoc), getColumnNumber(CurLoc), Scope));
}
/// CreateLexicalBlock - Creates a new lexical block node and pushes it on
@@ -2648,11 +2633,9 @@ void CGDebugInfo::CreateLexicalBlock(SourceLocation Loc) {
llvm::MDNode *Back = nullptr;
if (!LexicalBlockStack.empty())
Back = LexicalBlockStack.back().get();
- llvm::DIDescriptor D = DBuilder.createLexicalBlock(
- llvm::DIDescriptor(Back), getOrCreateFile(CurLoc), getLineNumber(CurLoc),
- getColumnNumber(CurLoc));
- llvm::MDNode *DN = D;
- LexicalBlockStack.emplace_back(DN);
+ LexicalBlockStack.emplace_back(DBuilder.createLexicalBlock(
+ cast<llvm::DIScope>(Back), getOrCreateFile(CurLoc), getLineNumber(CurLoc),
+ getColumnNumber(CurLoc)));
}
/// EmitLexicalBlockStart - Constructs the debug code for entering a declarative
@@ -2705,15 +2688,15 @@ void CGDebugInfo::EmitFunctionEnd(CGBuilderTy &Builder) {
// EmitTypeForVarWithBlocksAttr - Build up structure info for the byref.
// See BuildByRefType.
-llvm::DIType CGDebugInfo::EmitTypeForVarWithBlocksAttr(const VarDecl *VD,
- uint64_t *XOffset) {
+llvm::DIType *CGDebugInfo::EmitTypeForVarWithBlocksAttr(const VarDecl *VD,
+ uint64_t *XOffset) {
SmallVector<llvm::Metadata *, 5> EltTys;
QualType FType;
uint64_t FieldSize, FieldOffset;
unsigned FieldAlign;
- llvm::DIFile Unit = getOrCreateFile(VD->getLocation());
+ llvm::DIFile *Unit = getOrCreateFile(VD->getLocation());
QualType Type = VD->getType();
FieldOffset = 0;
@@ -2760,7 +2743,7 @@ llvm::DIType CGDebugInfo::EmitTypeForVarWithBlocksAttr(const VarDecl *VD,
}
FType = Type;
- llvm::DIType FieldTy = getOrCreateType(FType, Unit);
+ llvm::DIType *FieldTy = getOrCreateType(FType, Unit);
FieldSize = CGM.getContext().getTypeSize(FType);
FieldAlign = CGM.getContext().toBits(Align);
@@ -2770,16 +2753,16 @@ llvm::DIType CGDebugInfo::EmitTypeForVarWithBlocksAttr(const VarDecl *VD,
EltTys.push_back(FieldTy);
FieldOffset += FieldSize;
- llvm::DIArray Elements = DBuilder.getOrCreateArray(EltTys);
+ llvm::DINodeArray Elements = DBuilder.getOrCreateArray(EltTys);
- unsigned Flags = llvm::DIDescriptor::FlagBlockByrefStruct;
+ unsigned Flags = llvm::DINode::FlagBlockByrefStruct;
return DBuilder.createStructType(Unit, "", Unit, 0, FieldOffset, 0, Flags,
- llvm::DIType(), Elements);
+ nullptr, Elements);
}
/// EmitDeclare - Emit local variable declaration debug info.
-void CGDebugInfo::EmitDeclare(const VarDecl *VD, llvm::dwarf::LLVMConstants Tag,
+void CGDebugInfo::EmitDeclare(const VarDecl *VD, llvm::dwarf::Tag Tag,
llvm::Value *Storage, unsigned ArgNo,
CGBuilderTy &Builder) {
assert(DebugKind >= CodeGenOptions::LimitedDebugInfo);
@@ -2788,10 +2771,10 @@ void CGDebugInfo::EmitDeclare(const VarDecl *VD, llvm::dwarf::LLVMConstants Tag,
bool Unwritten =
VD->isImplicit() || (isa<Decl>(VD->getDeclContext()) &&
cast<Decl>(VD->getDeclContext())->isImplicit());
- llvm::DIFile Unit;
+ llvm::DIFile *Unit = nullptr;
if (!Unwritten)
Unit = getOrCreateFile(VD->getLocation());
- llvm::DIType Ty;
+ llvm::DIType *Ty;
uint64_t XOffset = 0;
if (VD->hasAttr<BlocksAttr>())
Ty = EmitTypeForVarWithBlocksAttr(VD, &XOffset);
@@ -2810,58 +2793,64 @@ void CGDebugInfo::EmitDeclare(const VarDecl *VD, llvm::dwarf::LLVMConstants Tag,
Line = getLineNumber(VD->getLocation());
Column = getColumnNumber(VD->getLocation());
}
+ SmallVector<int64_t, 9> Expr;
unsigned Flags = 0;
if (VD->isImplicit())
- Flags |= llvm::DIDescriptor::FlagArtificial;
+ Flags |= llvm::DINode::FlagArtificial;
// If this is the first argument and it is implicit then
// give it an object pointer flag.
// FIXME: There has to be a better way to do this, but for static
// functions there won't be an implicit param at arg1 and
// otherwise it is 'self' or 'this'.
if (isa<ImplicitParamDecl>(VD) && ArgNo == 1)
- Flags |= llvm::DIDescriptor::FlagObjectPointer;
+ Flags |= llvm::DINode::FlagObjectPointer;
if (llvm::Argument *Arg = dyn_cast<llvm::Argument>(Storage))
if (Arg->getType()->isPointerTy() && !Arg->hasByValAttr() &&
!VD->getType()->isPointerType())
- Flags |= llvm::DIDescriptor::FlagIndirectVariable;
+ Expr.push_back(llvm::dwarf::DW_OP_deref);
- llvm::MDNode *Scope = LexicalBlockStack.back();
+ auto *Scope = cast<llvm::DIScope>(LexicalBlockStack.back());
StringRef Name = VD->getName();
if (!Name.empty()) {
if (VD->hasAttr<BlocksAttr>()) {
CharUnits offset = CharUnits::fromQuantity(32);
- SmallVector<int64_t, 9> addr;
- addr.push_back(llvm::dwarf::DW_OP_plus);
+ Expr.push_back(llvm::dwarf::DW_OP_plus);
// offset of __forwarding field
offset = CGM.getContext().toCharUnitsFromBits(
CGM.getTarget().getPointerWidth(0));
- addr.push_back(offset.getQuantity());
- addr.push_back(llvm::dwarf::DW_OP_deref);
- addr.push_back(llvm::dwarf::DW_OP_plus);
+ Expr.push_back(offset.getQuantity());
+ Expr.push_back(llvm::dwarf::DW_OP_deref);
+ Expr.push_back(llvm::dwarf::DW_OP_plus);
// offset of x field
offset = CGM.getContext().toCharUnitsFromBits(XOffset);
- addr.push_back(offset.getQuantity());
+ Expr.push_back(offset.getQuantity());
// Create the descriptor for the variable.
- llvm::DIVariable D = DBuilder.createLocalVariable(
- Tag, llvm::DIDescriptor(Scope), VD->getName(), Unit, Line, Ty, ArgNo);
+ auto *D = DBuilder.createLocalVariable(Tag, Scope, VD->getName(), Unit,
+ Line, Ty, ArgNo);
// Insert an llvm.dbg.declare into the current block.
- llvm::Instruction *Call =
- DBuilder.insertDeclare(Storage, D, DBuilder.createExpression(addr),
- Builder.GetInsertBlock());
- Call->setDebugLoc(llvm::DebugLoc::get(Line, Column, Scope));
+ DBuilder.insertDeclare(Storage, D, DBuilder.createExpression(Expr),
+ llvm::DebugLoc::get(Line, Column, Scope),
+ Builder.GetInsertBlock());
return;
} else if (isa<VariableArrayType>(VD->getType()))
- Flags |= llvm::DIDescriptor::FlagIndirectVariable;
+ Expr.push_back(llvm::dwarf::DW_OP_deref);
} else if (const RecordType *RT = dyn_cast<RecordType>(VD->getType())) {
// If VD is an anonymous union then Storage represents value for
// all union fields.
const RecordDecl *RD = cast<RecordDecl>(RT->getDecl());
if (RD->isUnion() && RD->isAnonymousStructOrUnion()) {
+ // GDB has trouble finding local variables in anonymous unions, so we emit
+ // artifical local variables for each of the members.
+ //
+ // FIXME: Remove this code as soon as GDB supports this.
+ // The debug info verifier in LLVM operates based on the assumption that a
+ // variable has the same size as its storage and we had to disable the check
+ // for artificial variables.
for (const auto *Field : RD->fields()) {
- llvm::DIType FieldTy = getOrCreateType(Field->getType(), Unit);
+ llvm::DIType *FieldTy = getOrCreateType(Field->getType(), Unit);
StringRef FieldName = Field->getName();
// Ignore unnamed fields. Do not ignore unnamed records.
@@ -2869,28 +2858,28 @@ void CGDebugInfo::EmitDeclare(const VarDecl *VD, llvm::dwarf::LLVMConstants Tag,
continue;
// Use VarDecl's Tag, Scope and Line number.
- llvm::DIVariable D = DBuilder.createLocalVariable(
- Tag, llvm::DIDescriptor(Scope), FieldName, Unit, Line, FieldTy,
- CGM.getLangOpts().Optimize, Flags, ArgNo);
+ auto *D = DBuilder.createLocalVariable(
+ Tag, Scope, FieldName, Unit, Line, FieldTy,
+ CGM.getLangOpts().Optimize, Flags | llvm::DINode::FlagArtificial,
+ ArgNo);
// Insert an llvm.dbg.declare into the current block.
- llvm::Instruction *Call = DBuilder.insertDeclare(
- Storage, D, DBuilder.createExpression(), Builder.GetInsertBlock());
- Call->setDebugLoc(llvm::DebugLoc::get(Line, Column, Scope));
+ DBuilder.insertDeclare(Storage, D, DBuilder.createExpression(Expr),
+ llvm::DebugLoc::get(Line, Column, Scope),
+ Builder.GetInsertBlock());
}
- return;
}
}
// Create the descriptor for the variable.
- llvm::DIVariable D = DBuilder.createLocalVariable(
- Tag, llvm::DIDescriptor(Scope), Name, Unit, Line, Ty,
- CGM.getLangOpts().Optimize, Flags, ArgNo);
+ auto *D =
+ DBuilder.createLocalVariable(Tag, Scope, Name, Unit, Line, Ty,
+ CGM.getLangOpts().Optimize, Flags, ArgNo);
// Insert an llvm.dbg.declare into the current block.
- llvm::Instruction *Call = DBuilder.insertDeclare(
- Storage, D, DBuilder.createExpression(), Builder.GetInsertBlock());
- Call->setDebugLoc(llvm::DebugLoc::get(Line, Column, Scope));
+ DBuilder.insertDeclare(Storage, D, DBuilder.createExpression(Expr),
+ llvm::DebugLoc::get(Line, Column, Scope),
+ Builder.GetInsertBlock());
}
void CGDebugInfo::EmitDeclareOfAutoVariable(const VarDecl *VD,
@@ -2906,9 +2895,9 @@ void CGDebugInfo::EmitDeclareOfAutoVariable(const VarDecl *VD,
/// never happen though, since creating a type for the implicit self
/// argument implies that we already parsed the interface definition
/// and the ivar declarations in the implementation.
-llvm::DIType CGDebugInfo::CreateSelfType(const QualType &QualTy,
- llvm::DIType Ty) {
- llvm::DIType CachedTy = getTypeOrNull(QualTy);
+llvm::DIType *CGDebugInfo::CreateSelfType(const QualType &QualTy,
+ llvm::DIType *Ty) {
+ llvm::DIType *CachedTy = getTypeOrNull(QualTy);
if (CachedTy)
Ty = CachedTy;
return DBuilder.createObjectPointerType(Ty);
@@ -2926,8 +2915,8 @@ void CGDebugInfo::EmitDeclareOfBlockDeclRefVariable(
bool isByRef = VD->hasAttr<BlocksAttr>();
uint64_t XOffset = 0;
- llvm::DIFile Unit = getOrCreateFile(VD->getLocation());
- llvm::DIType Ty;
+ llvm::DIFile *Unit = getOrCreateFile(VD->getLocation());
+ llvm::DIType *Ty;
if (isByRef)
Ty = EmitTypeForVarWithBlocksAttr(VD, &XOffset);
else
@@ -2968,19 +2957,19 @@ void CGDebugInfo::EmitDeclareOfBlockDeclRefVariable(
}
// Create the descriptor for the variable.
- llvm::DIVariable D =
- DBuilder.createLocalVariable(llvm::dwarf::DW_TAG_auto_variable,
- llvm::DIDescriptor(LexicalBlockStack.back()),
- VD->getName(), Unit, Line, Ty);
+ auto *D = DBuilder.createLocalVariable(
+ llvm::dwarf::DW_TAG_auto_variable,
+ cast<llvm::DILocalScope>(LexicalBlockStack.back()), VD->getName(), Unit,
+ Line, Ty);
// Insert an llvm.dbg.declare into the current block.
- llvm::Instruction *Call = InsertPoint ?
- DBuilder.insertDeclare(Storage, D, DBuilder.createExpression(addr),
- InsertPoint)
- : DBuilder.insertDeclare(Storage, D, DBuilder.createExpression(addr),
- Builder.GetInsertBlock());
- Call->setDebugLoc(
- llvm::DebugLoc::get(Line, Column, LexicalBlockStack.back()));
+ auto DL = llvm::DebugLoc::get(Line, Column, LexicalBlockStack.back());
+ if (InsertPoint)
+ DBuilder.insertDeclare(Storage, D, DBuilder.createExpression(addr), DL,
+ InsertPoint);
+ else
+ DBuilder.insertDeclare(Storage, D, DBuilder.createExpression(addr), DL,
+ Builder.GetInsertBlock());
}
/// EmitDeclareOfArgVariable - Emit call to llvm.dbg.declare for an argument
@@ -3013,7 +3002,7 @@ void CGDebugInfo::EmitDeclareOfBlockLiteralArgVariable(const CGBlockInfo &block,
// Collect some general information about the block's location.
SourceLocation loc = blockDecl->getCaretLocation();
- llvm::DIFile tunit = getOrCreateFile(loc);
+ llvm::DIFile *tunit = getOrCreateFile(loc);
unsigned line = getLineNumber(loc);
unsigned column = getColumnNumber(loc);
@@ -3096,7 +3085,7 @@ void CGDebugInfo::EmitDeclareOfBlockLiteralArgVariable(const CGBlockInfo &block,
const VarDecl *variable = capture->getVariable();
StringRef name = variable->getName();
- llvm::DIType fieldType;
+ llvm::DIType *fieldType;
if (capture->isByRef()) {
TypeInfo PtrInfo = C.getTypeInfo(C.VoidPtrTy);
@@ -3118,70 +3107,67 @@ void CGDebugInfo::EmitDeclareOfBlockLiteralArgVariable(const CGBlockInfo &block,
llvm::raw_svector_ostream(typeName) << "__block_literal_"
<< CGM.getUniqueBlockCount();
- llvm::DIArray fieldsArray = DBuilder.getOrCreateArray(fields);
+ llvm::DINodeArray fieldsArray = DBuilder.getOrCreateArray(fields);
- llvm::DIType type =
- DBuilder.createStructType(tunit, typeName.str(), tunit, line,
- CGM.getContext().toBits(block.BlockSize),
- CGM.getContext().toBits(block.BlockAlign), 0,
- llvm::DIType(), fieldsArray);
+ llvm::DIType *type = DBuilder.createStructType(
+ tunit, typeName.str(), tunit, line,
+ CGM.getContext().toBits(block.BlockSize),
+ CGM.getContext().toBits(block.BlockAlign), 0, nullptr, fieldsArray);
type = DBuilder.createPointerType(type, CGM.PointerWidthInBits);
// Get overall information about the block.
- unsigned flags = llvm::DIDescriptor::FlagArtificial;
- llvm::MDNode *scope = LexicalBlockStack.back();
+ unsigned flags = llvm::DINode::FlagArtificial;
+ auto *scope = cast<llvm::DILocalScope>(LexicalBlockStack.back());
// Create the descriptor for the parameter.
- llvm::DIVariable debugVar = DBuilder.createLocalVariable(
- llvm::dwarf::DW_TAG_arg_variable, llvm::DIDescriptor(scope),
- Arg->getName(), tunit, line, type, CGM.getLangOpts().Optimize, flags,
- ArgNo);
+ auto *debugVar = DBuilder.createLocalVariable(
+ llvm::dwarf::DW_TAG_arg_variable, scope, Arg->getName(), tunit, line,
+ type, CGM.getLangOpts().Optimize, flags, ArgNo);
if (LocalAddr) {
// Insert an llvm.dbg.value into the current block.
- llvm::Instruction *DbgVal = DBuilder.insertDbgValueIntrinsic(
+ DBuilder.insertDbgValueIntrinsic(
LocalAddr, 0, debugVar, DBuilder.createExpression(),
- Builder.GetInsertBlock());
- DbgVal->setDebugLoc(llvm::DebugLoc::get(line, column, scope));
+ llvm::DebugLoc::get(line, column, scope), Builder.GetInsertBlock());
}
// Insert an llvm.dbg.declare into the current block.
- llvm::Instruction *DbgDecl = DBuilder.insertDeclare(
- Arg, debugVar, DBuilder.createExpression(), Builder.GetInsertBlock());
- DbgDecl->setDebugLoc(llvm::DebugLoc::get(line, column, scope));
+ DBuilder.insertDeclare(Arg, debugVar, DBuilder.createExpression(),
+ llvm::DebugLoc::get(line, column, scope),
+ Builder.GetInsertBlock());
}
/// If D is an out-of-class definition of a static data member of a class, find
/// its corresponding in-class declaration.
-llvm::DIDerivedType
+llvm::DIDerivedType *
CGDebugInfo::getOrCreateStaticDataMemberDeclarationOrNull(const VarDecl *D) {
if (!D->isStaticDataMember())
- return llvm::DIDerivedType();
+ return nullptr;
+
auto MI = StaticDataMemberCache.find(D->getCanonicalDecl());
if (MI != StaticDataMemberCache.end()) {
assert(MI->second && "Static data member declaration should still exist");
- return llvm::DIDerivedType(cast<llvm::MDNode>(MI->second));
+ return cast<llvm::DIDerivedType>(MI->second);
}
// If the member wasn't found in the cache, lazily construct and add it to the
// type (used when a limited form of the type is emitted).
auto DC = D->getDeclContext();
- llvm::DICompositeType Ctxt(getContextDescriptor(cast<Decl>(DC)));
+ auto *Ctxt =
+ cast<llvm::DICompositeType>(getContextDescriptor(cast<Decl>(DC)));
return CreateRecordStaticField(D, Ctxt, cast<RecordDecl>(DC));
}
/// Recursively collect all of the member fields of a global anonymous decl and
/// create static variables for them. The first time this is called it needs
/// to be on a union and then from there we can have additional unnamed fields.
-llvm::DIGlobalVariable
-CGDebugInfo::CollectAnonRecordDecls(const RecordDecl *RD, llvm::DIFile Unit,
- unsigned LineNo, StringRef LinkageName,
- llvm::GlobalVariable *Var,
- llvm::DIDescriptor DContext) {
- llvm::DIGlobalVariable GV;
+llvm::DIGlobalVariable *CGDebugInfo::CollectAnonRecordDecls(
+ const RecordDecl *RD, llvm::DIFile *Unit, unsigned LineNo,
+ StringRef LinkageName, llvm::GlobalVariable *Var, llvm::DIScope *DContext) {
+ llvm::DIGlobalVariable *GV = nullptr;
for (const auto *Field : RD->fields()) {
- llvm::DIType FieldTy = getOrCreateType(Field->getType(), Unit);
+ llvm::DIType *FieldTy = getOrCreateType(Field->getType(), Unit);
StringRef FieldName = Field->getName();
// Ignore unnamed fields, but recurse into anonymous records.
@@ -3193,9 +3179,9 @@ CGDebugInfo::CollectAnonRecordDecls(const RecordDecl *RD, llvm::DIFile Unit,
continue;
}
// Use VarDecl's Tag, Scope and Line number.
- GV = DBuilder.createGlobalVariable(
- DContext, FieldName, LinkageName, Unit, LineNo, FieldTy,
- Var->hasInternalLinkage(), Var, llvm::DIDerivedType());
+ GV = DBuilder.createGlobalVariable(DContext, FieldName, LinkageName, Unit,
+ LineNo, FieldTy,
+ Var->hasInternalLinkage(), Var, nullptr);
}
return GV;
}
@@ -3205,8 +3191,8 @@ void CGDebugInfo::EmitGlobalVariable(llvm::GlobalVariable *Var,
const VarDecl *D) {
assert(DebugKind >= CodeGenOptions::LimitedDebugInfo);
// Create global variable debug descriptor.
- llvm::DIFile Unit;
- llvm::DIDescriptor DContext;
+ llvm::DIFile *Unit = nullptr;
+ llvm::DIScope *DContext = nullptr;
unsigned LineNo;
StringRef DeclName, LinkageName;
QualType T;
@@ -3214,7 +3200,7 @@ void CGDebugInfo::EmitGlobalVariable(llvm::GlobalVariable *Var,
// Attempt to store one global variable for the declaration - even if we
// emit a lot of fields.
- llvm::DIGlobalVariable GV;
+ llvm::DIGlobalVariable *GV = nullptr;
// If this is an anonymous union then we'll want to emit a global
// variable for each member of the anonymous union so that it's possible
@@ -3238,16 +3224,18 @@ void CGDebugInfo::EmitGlobalVariable(const ValueDecl *VD,
llvm::Constant *Init) {
assert(DebugKind >= CodeGenOptions::LimitedDebugInfo);
// Create the descriptor for the variable.
- llvm::DIFile Unit = getOrCreateFile(VD->getLocation());
+ llvm::DIFile *Unit = getOrCreateFile(VD->getLocation());
StringRef Name = VD->getName();
- llvm::DIType Ty = getOrCreateType(VD->getType(), Unit);
+ llvm::DIType *Ty = getOrCreateType(VD->getType(), Unit);
if (const EnumConstantDecl *ECD = dyn_cast<EnumConstantDecl>(VD)) {
const EnumDecl *ED = cast<EnumDecl>(ECD->getDeclContext());
assert(isa<EnumType>(ED->getTypeForDecl()) && "Enum without EnumType?");
Ty = getOrCreateType(QualType(ED->getTypeForDecl(), 0), Unit);
}
- // Do not use DIGlobalVariable for enums.
- if (Ty.getTag() == llvm::dwarf::DW_TAG_enumeration_type)
+ // Do not use global variables for enums.
+ //
+ // FIXME: why not?
+ if (Ty->getTag() == llvm::dwarf::DW_TAG_enumeration_type)
return;
// Do not emit separate definitions for function local const/statics.
if (isa<FunctionDecl>(VD->getDeclContext()))
@@ -3263,7 +3251,7 @@ void CGDebugInfo::EmitGlobalVariable(const ValueDecl *VD,
return;
}
- llvm::DIDescriptor DContext =
+ llvm::DIScope *DContext =
getContextDescriptor(dyn_cast<Decl>(VD->getDeclContext()));
auto &GV = DeclCache[VD];
@@ -3274,9 +3262,9 @@ void CGDebugInfo::EmitGlobalVariable(const ValueDecl *VD,
true, Init, getOrCreateStaticDataMemberDeclarationOrNull(VarD)));
}
-llvm::DIScope CGDebugInfo::getCurrentContextDescriptor(const Decl *D) {
+llvm::DIScope *CGDebugInfo::getCurrentContextDescriptor(const Decl *D) {
if (!LexicalBlockStack.empty())
- return llvm::DIScope(LexicalBlockStack.back());
+ return LexicalBlockStack.back();
return getContextDescriptor(D);
}
@@ -3297,21 +3285,21 @@ void CGDebugInfo::EmitUsingDecl(const UsingDecl &UD) {
// Emitting one decl is sufficient - debuggers can detect that this is an
// overloaded name & provide lookup for all the overloads.
const UsingShadowDecl &USD = **UD.shadow_begin();
- if (llvm::DIDescriptor Target =
+ if (llvm::DINode *Target =
getDeclarationOrDefinition(USD.getUnderlyingDecl()))
DBuilder.createImportedDeclaration(
getCurrentContextDescriptor(cast<Decl>(USD.getDeclContext())), Target,
getLineNumber(USD.getLocation()));
}
-llvm::DIImportedEntity
+llvm::DIImportedEntity *
CGDebugInfo::EmitNamespaceAlias(const NamespaceAliasDecl &NA) {
if (CGM.getCodeGenOpts().getDebugInfo() < CodeGenOptions::LimitedDebugInfo)
- return llvm::DIImportedEntity(nullptr);
+ return nullptr;
auto &VH = NamespaceAliasCache[&NA];
if (VH)
- return llvm::DIImportedEntity(cast<llvm::MDNode>(VH));
- llvm::DIImportedEntity R(nullptr);
+ return cast<llvm::DIImportedEntity>(VH);
+ llvm::DIImportedEntity *R;
if (const NamespaceAliasDecl *Underlying =
dyn_cast<NamespaceAliasDecl>(NA.getAliasedNamespace()))
// This could cache & dedup here rather than relying on metadata deduping.
@@ -3330,19 +3318,19 @@ CGDebugInfo::EmitNamespaceAlias(const NamespaceAliasDecl &NA) {
/// getOrCreateNamesSpace - Return namespace descriptor for the given
/// namespace decl.
-llvm::DINameSpace
+llvm::DINamespace *
CGDebugInfo::getOrCreateNameSpace(const NamespaceDecl *NSDecl) {
NSDecl = NSDecl->getCanonicalDecl();
auto I = NameSpaceCache.find(NSDecl);
if (I != NameSpaceCache.end())
- return llvm::DINameSpace(cast<llvm::MDNode>(I->second));
+ return cast<llvm::DINamespace>(I->second);
unsigned LineNo = getLineNumber(NSDecl->getLocation());
- llvm::DIFile FileD = getOrCreateFile(NSDecl->getLocation());
- llvm::DIDescriptor Context =
- getContextDescriptor(dyn_cast<Decl>(NSDecl->getDeclContext()));
- llvm::DINameSpace NS =
- DBuilder.createNameSpace(Context, NSDecl->getName(), FileD, LineNo);
+ llvm::DIFile *FileD = getOrCreateFile(NSDecl->getLocation());
+ llvm::DIScope *Context =
+ getContextDescriptor(dyn_cast<Decl>(NSDecl->getDeclContext()));
+ llvm::DINamespace *NS =
+ DBuilder.createNameSpace(Context, NSDecl->getName(), FileD, LineNo);
NameSpaceCache[NSDecl].reset(NS);
return NS;
}
@@ -3352,28 +3340,28 @@ void CGDebugInfo::finalize() {
// element and the size(), so don't cache/reference them.
for (size_t i = 0; i != ObjCInterfaceCache.size(); ++i) {
ObjCInterfaceCacheEntry E = ObjCInterfaceCache[i];
- E.Decl.replaceAllUsesWith(CGM.getLLVMContext(),
- E.Type->getDecl()->getDefinition()
- ? CreateTypeDefinition(E.Type, E.Unit)
- : E.Decl);
+ llvm::DIType *Ty = E.Type->getDecl()->getDefinition()
+ ? CreateTypeDefinition(E.Type, E.Unit)
+ : E.Decl;
+ DBuilder.replaceTemporary(llvm::TempDIType(E.Decl), Ty);
}
for (auto p : ReplaceMap) {
assert(p.second);
- llvm::DIType Ty(cast<llvm::MDNode>(p.second));
- assert(Ty.isForwardDecl());
+ auto *Ty = cast<llvm::DIType>(p.second);
+ assert(Ty->isForwardDecl());
auto it = TypeCache.find(p.first);
assert(it != TypeCache.end());
assert(it->second);
- llvm::DIType RepTy(cast<llvm::MDNode>(it->second));
- Ty.replaceAllUsesWith(CGM.getLLVMContext(), RepTy);
+ DBuilder.replaceTemporary(llvm::TempDIType(Ty),
+ cast<llvm::DIType>(it->second));
}
for (const auto &p : FwdDeclReplaceMap) {
assert(p.second);
- llvm::DIDescriptor FwdDecl(cast<llvm::MDNode>(p.second));
+ llvm::TempMDNode FwdDecl(cast<llvm::MDNode>(p.second));
llvm::Metadata *Repl;
auto it = DeclCache.find(p.first);
@@ -3385,15 +3373,14 @@ void CGDebugInfo::finalize() {
else
Repl = it->second;
- FwdDecl.replaceAllUsesWith(CGM.getLLVMContext(),
- llvm::DIDescriptor(cast<llvm::MDNode>(Repl)));
+ DBuilder.replaceTemporary(std::move(FwdDecl), cast<llvm::MDNode>(Repl));
}
// We keep our own list of retained types, because we need to look
// up the final type in the type cache.
for (std::vector<void *>::const_iterator RI = RetainedTypes.begin(),
RE = RetainedTypes.end(); RI != RE; ++RI)
- DBuilder.retainType(llvm::DIType(cast<llvm::MDNode>(TypeCache[*RI])));
+ DBuilder.retainType(cast<llvm::DIType>(TypeCache[*RI]));
DBuilder.finalize();
}
@@ -3401,7 +3388,8 @@ void CGDebugInfo::finalize() {
void CGDebugInfo::EmitExplicitCastType(QualType Ty) {
if (CGM.getCodeGenOpts().getDebugInfo() < CodeGenOptions::LimitedDebugInfo)
return;
- llvm::DIType DieTy = getOrCreateType(Ty, getOrCreateMainFile());
- // Don't ignore in case of explicit cast where it is referenced indirectly.
- DBuilder.retainType(DieTy);
+
+ if (auto *DieTy = getOrCreateType(Ty, getOrCreateMainFile()))
+ // Don't ignore in case of explicit cast where it is referenced indirectly.
+ DBuilder.retainType(DieTy);
}
diff --git a/lib/CodeGen/CGDebugInfo.h b/lib/CodeGen/CGDebugInfo.h
index 0be032c1d790..8509e0770db6 100644
--- a/lib/CodeGen/CGDebugInfo.h
+++ b/lib/CodeGen/CGDebugInfo.h
@@ -43,47 +43,50 @@ namespace CodeGen {
class CodeGenFunction;
class CGBlockInfo;
-/// CGDebugInfo - This class gathers all debug information during compilation
+/// \brief This class gathers all debug information during compilation
/// and is responsible for emitting to llvm globals or pass directly to
/// the backend.
class CGDebugInfo {
- friend class ArtificialLocation;
+ friend class ApplyDebugLocation;
friend class SaveAndRestoreLocation;
CodeGenModule &CGM;
const CodeGenOptions::DebugInfoKind DebugKind;
llvm::DIBuilder DBuilder;
- llvm::DICompileUnit TheCU;
+ llvm::DICompileUnit *TheCU = nullptr;
SourceLocation CurLoc;
- llvm::DIType VTablePtrType;
- llvm::DIType ClassTy;
- llvm::DICompositeType ObjTy;
- llvm::DIType SelTy;
- llvm::DIType OCLImage1dDITy, OCLImage1dArrayDITy, OCLImage1dBufferDITy;
- llvm::DIType OCLImage2dDITy, OCLImage2dArrayDITy;
- llvm::DIType OCLImage3dDITy;
- llvm::DIType OCLEventDITy;
- llvm::DIType BlockLiteralGeneric;
-
- /// TypeCache - Cache of previously constructed Types.
+ llvm::DIType *VTablePtrType = nullptr;
+ llvm::DIType *ClassTy = nullptr;
+ llvm::DICompositeType *ObjTy = nullptr;
+ llvm::DIType *SelTy = nullptr;
+ llvm::DIType *OCLImage1dDITy = nullptr;
+ llvm::DIType *OCLImage1dArrayDITy = nullptr;
+ llvm::DIType *OCLImage1dBufferDITy = nullptr;
+ llvm::DIType *OCLImage2dDITy = nullptr;
+ llvm::DIType *OCLImage2dArrayDITy = nullptr;
+ llvm::DIType *OCLImage3dDITy = nullptr;
+ llvm::DIType *OCLEventDITy = nullptr;
+ llvm::DIType *BlockLiteralGeneric = nullptr;
+
+ /// \brief Cache of previously constructed Types.
llvm::DenseMap<const void *, llvm::TrackingMDRef> TypeCache;
struct ObjCInterfaceCacheEntry {
const ObjCInterfaceType *Type;
- llvm::DIType Decl;
- llvm::DIFile Unit;
- ObjCInterfaceCacheEntry(const ObjCInterfaceType *Type, llvm::DIType Decl,
- llvm::DIFile Unit)
+ llvm::DIType *Decl;
+ llvm::DIFile *Unit;
+ ObjCInterfaceCacheEntry(const ObjCInterfaceType *Type, llvm::DIType *Decl,
+ llvm::DIFile *Unit)
: Type(Type), Decl(Decl), Unit(Unit) {}
};
- /// ObjCInterfaceCache - Cache of previously constructed interfaces
+ /// \brief Cache of previously constructed interfaces
/// which may change.
llvm::SmallVector<ObjCInterfaceCacheEntry, 32> ObjCInterfaceCache;
- /// RetainedTypes - list of interfaces we want to keep even if orphaned.
+ /// \brief list of interfaces we want to keep even if orphaned.
std::vector<void *> RetainedTypes;
- /// ReplaceMap - Cache of forward declared types to RAUW at the end of
+ /// \brief Cache of forward declared types to RAUW at the end of
/// compilation.
std::vector<std::pair<const TagType *, llvm::TrackingMDRef>> ReplaceMap;
@@ -93,14 +96,14 @@ class CGDebugInfo {
FwdDeclReplaceMap;
// LexicalBlockStack - Keep track of our current nested lexical block.
- std::vector<llvm::TrackingMDNodeRef> LexicalBlockStack;
+ std::vector<llvm::TypedTrackingMDRef<llvm::DIScope>> LexicalBlockStack;
llvm::DenseMap<const Decl *, llvm::TrackingMDRef> RegionMap;
// FnBeginRegionCount - Keep track of LexicalBlockStack counter at the
// beginning of a function. This is used to pop unbalanced regions at
// the end of a function.
std::vector<unsigned> FnBeginRegionCount;
- /// DebugInfoNames - This is a storage for names that are
+ /// \brief This is a storage for names that are
/// constructed on demand. For example, C++ destructors, C++ operators etc..
llvm::BumpPtrAllocator DebugInfoNames;
StringRef CWDName;
@@ -117,95 +120,94 @@ class CGDebugInfo {
/// Helper functions for getOrCreateType.
unsigned Checksum(const ObjCInterfaceDecl *InterfaceDecl);
- llvm::DIType CreateType(const BuiltinType *Ty);
- llvm::DIType CreateType(const ComplexType *Ty);
- llvm::DIType CreateQualifiedType(QualType Ty, llvm::DIFile Fg);
- llvm::DIType CreateType(const TypedefType *Ty, llvm::DIFile Fg);
- llvm::DIType CreateType(const TemplateSpecializationType *Ty, llvm::DIFile Fg);
- llvm::DIType CreateType(const ObjCObjectPointerType *Ty,
- llvm::DIFile F);
- llvm::DIType CreateType(const PointerType *Ty, llvm::DIFile F);
- llvm::DIType CreateType(const BlockPointerType *Ty, llvm::DIFile F);
- llvm::DIType CreateType(const FunctionType *Ty, llvm::DIFile F);
- llvm::DIType CreateType(const RecordType *Tyg);
- llvm::DIType CreateTypeDefinition(const RecordType *Ty);
- llvm::DICompositeType CreateLimitedType(const RecordType *Ty);
- void CollectContainingType(const CXXRecordDecl *RD, llvm::DICompositeType CT);
- llvm::DIType CreateType(const ObjCInterfaceType *Ty, llvm::DIFile F);
- llvm::DIType CreateTypeDefinition(const ObjCInterfaceType *Ty, llvm::DIFile F);
- llvm::DIType CreateType(const ObjCObjectType *Ty, llvm::DIFile F);
- llvm::DIType CreateType(const VectorType *Ty, llvm::DIFile F);
- llvm::DIType CreateType(const ArrayType *Ty, llvm::DIFile F);
- llvm::DIType CreateType(const LValueReferenceType *Ty, llvm::DIFile F);
- llvm::DIType CreateType(const RValueReferenceType *Ty, llvm::DIFile Unit);
- llvm::DIType CreateType(const MemberPointerType *Ty, llvm::DIFile F);
- llvm::DIType CreateType(const AtomicType *Ty, llvm::DIFile F);
- llvm::DIType CreateEnumType(const EnumType *Ty);
- llvm::DIType CreateTypeDefinition(const EnumType *Ty);
- llvm::DIType CreateSelfType(const QualType &QualTy, llvm::DIType Ty);
- llvm::DIType getTypeOrNull(const QualType);
- llvm::DICompositeType getOrCreateMethodType(const CXXMethodDecl *Method,
- llvm::DIFile F);
- llvm::DICompositeType getOrCreateInstanceMethodType(
- QualType ThisPtr, const FunctionProtoType *Func, llvm::DIFile Unit);
- llvm::DICompositeType getOrCreateFunctionType(const Decl *D, QualType FnType,
- llvm::DIFile F);
- llvm::DIType getOrCreateVTablePtrType(llvm::DIFile F);
- llvm::DINameSpace getOrCreateNameSpace(const NamespaceDecl *N);
- llvm::DIType getOrCreateTypeDeclaration(QualType PointeeTy, llvm::DIFile F);
- llvm::DIType CreatePointerLikeType(llvm::dwarf::Tag Tag,
- const Type *Ty, QualType PointeeTy,
- llvm::DIFile F);
+ llvm::DIType *CreateType(const BuiltinType *Ty);
+ llvm::DIType *CreateType(const ComplexType *Ty);
+ llvm::DIType *CreateQualifiedType(QualType Ty, llvm::DIFile *Fg);
+ llvm::DIType *CreateType(const TypedefType *Ty, llvm::DIFile *Fg);
+ llvm::DIType *CreateType(const TemplateSpecializationType *Ty,
+ llvm::DIFile *Fg);
+ llvm::DIType *CreateType(const ObjCObjectPointerType *Ty, llvm::DIFile *F);
+ llvm::DIType *CreateType(const PointerType *Ty, llvm::DIFile *F);
+ llvm::DIType *CreateType(const BlockPointerType *Ty, llvm::DIFile *F);
+ llvm::DIType *CreateType(const FunctionType *Ty, llvm::DIFile *F);
+ llvm::DIType *CreateType(const RecordType *Tyg);
+ llvm::DIType *CreateTypeDefinition(const RecordType *Ty);
+ llvm::DICompositeType *CreateLimitedType(const RecordType *Ty);
+ void CollectContainingType(const CXXRecordDecl *RD,
+ llvm::DICompositeType *CT);
+ llvm::DIType *CreateType(const ObjCInterfaceType *Ty, llvm::DIFile *F);
+ llvm::DIType *CreateTypeDefinition(const ObjCInterfaceType *Ty,
+ llvm::DIFile *F);
+ llvm::DIType *CreateType(const ObjCObjectType *Ty, llvm::DIFile *F);
+ llvm::DIType *CreateType(const VectorType *Ty, llvm::DIFile *F);
+ llvm::DIType *CreateType(const ArrayType *Ty, llvm::DIFile *F);
+ llvm::DIType *CreateType(const LValueReferenceType *Ty, llvm::DIFile *F);
+ llvm::DIType *CreateType(const RValueReferenceType *Ty, llvm::DIFile *Unit);
+ llvm::DIType *CreateType(const MemberPointerType *Ty, llvm::DIFile *F);
+ llvm::DIType *CreateType(const AtomicType *Ty, llvm::DIFile *F);
+ llvm::DIType *CreateEnumType(const EnumType *Ty);
+ llvm::DIType *CreateTypeDefinition(const EnumType *Ty);
+ llvm::DIType *CreateSelfType(const QualType &QualTy, llvm::DIType *Ty);
+ llvm::DIType *getTypeOrNull(const QualType);
+ llvm::DISubroutineType *getOrCreateMethodType(const CXXMethodDecl *Method,
+ llvm::DIFile *F);
+ llvm::DISubroutineType *
+ getOrCreateInstanceMethodType(QualType ThisPtr, const FunctionProtoType *Func,
+ llvm::DIFile *Unit);
+ llvm::DISubroutineType *
+ getOrCreateFunctionType(const Decl *D, QualType FnType, llvm::DIFile *F);
+ llvm::DIType *getOrCreateVTablePtrType(llvm::DIFile *F);
+ llvm::DINamespace *getOrCreateNameSpace(const NamespaceDecl *N);
+ llvm::DIType *getOrCreateTypeDeclaration(QualType PointeeTy, llvm::DIFile *F);
+ llvm::DIType *CreatePointerLikeType(llvm::dwarf::Tag Tag, const Type *Ty,
+ QualType PointeeTy, llvm::DIFile *F);
llvm::Value *getCachedInterfaceTypeOrNull(const QualType Ty);
- llvm::DIType getOrCreateStructPtrType(StringRef Name, llvm::DIType &Cache);
+ llvm::DIType *getOrCreateStructPtrType(StringRef Name, llvm::DIType *&Cache);
- llvm::DISubprogram CreateCXXMemberFunction(const CXXMethodDecl *Method,
- llvm::DIFile F,
- llvm::DIType RecordTy);
+ llvm::DISubprogram *CreateCXXMemberFunction(const CXXMethodDecl *Method,
+ llvm::DIFile *F,
+ llvm::DIType *RecordTy);
- void CollectCXXMemberFunctions(const CXXRecordDecl *Decl, llvm::DIFile F,
+ void CollectCXXMemberFunctions(const CXXRecordDecl *Decl, llvm::DIFile *F,
SmallVectorImpl<llvm::Metadata *> &E,
- llvm::DIType T);
+ llvm::DIType *T);
- void CollectCXXBases(const CXXRecordDecl *Decl, llvm::DIFile F,
+ void CollectCXXBases(const CXXRecordDecl *Decl, llvm::DIFile *F,
SmallVectorImpl<llvm::Metadata *> &EltTys,
- llvm::DIType RecordTy);
-
- llvm::DIArray
- CollectTemplateParams(const TemplateParameterList *TPList,
- ArrayRef<TemplateArgument> TAList,
- llvm::DIFile Unit);
- llvm::DIArray
- CollectFunctionTemplateParams(const FunctionDecl *FD, llvm::DIFile Unit);
- llvm::DIArray
+ llvm::DIType *RecordTy);
+
+ llvm::DINodeArray CollectTemplateParams(const TemplateParameterList *TPList,
+ ArrayRef<TemplateArgument> TAList,
+ llvm::DIFile *Unit);
+ llvm::DINodeArray CollectFunctionTemplateParams(const FunctionDecl *FD,
+ llvm::DIFile *Unit);
+ llvm::DINodeArray
CollectCXXTemplateParams(const ClassTemplateSpecializationDecl *TS,
- llvm::DIFile F);
+ llvm::DIFile *F);
- llvm::DIType createFieldType(StringRef name, QualType type,
- uint64_t sizeInBitsOverride, SourceLocation loc,
- AccessSpecifier AS,
- uint64_t offsetInBits,
- llvm::DIFile tunit,
- llvm::DIScope scope,
- const RecordDecl* RD = nullptr);
+ llvm::DIType *createFieldType(StringRef name, QualType type,
+ uint64_t sizeInBitsOverride, SourceLocation loc,
+ AccessSpecifier AS, uint64_t offsetInBits,
+ llvm::DIFile *tunit, llvm::DIScope *scope,
+ const RecordDecl *RD = nullptr);
// Helpers for collecting fields of a record.
void CollectRecordLambdaFields(const CXXRecordDecl *CXXDecl,
SmallVectorImpl<llvm::Metadata *> &E,
- llvm::DIType RecordTy);
- llvm::DIDerivedType CreateRecordStaticField(const VarDecl *Var,
- llvm::DIType RecordTy,
- const RecordDecl* RD);
+ llvm::DIType *RecordTy);
+ llvm::DIDerivedType *CreateRecordStaticField(const VarDecl *Var,
+ llvm::DIType *RecordTy,
+ const RecordDecl *RD);
void CollectRecordNormalField(const FieldDecl *Field, uint64_t OffsetInBits,
- llvm::DIFile F,
+ llvm::DIFile *F,
SmallVectorImpl<llvm::Metadata *> &E,
- llvm::DIType RecordTy, const RecordDecl *RD);
- void CollectRecordFields(const RecordDecl *Decl, llvm::DIFile F,
+ llvm::DIType *RecordTy, const RecordDecl *RD);
+ void CollectRecordFields(const RecordDecl *Decl, llvm::DIFile *F,
SmallVectorImpl<llvm::Metadata *> &E,
- llvm::DICompositeType RecordTy);
+ llvm::DICompositeType *RecordTy);
- void CollectVTableInfo(const CXXRecordDecl *Decl, llvm::DIFile F,
+ void CollectVTableInfo(const CXXRecordDecl *Decl, llvm::DIFile *F,
SmallVectorImpl<llvm::Metadata *> &EltTys);
// CreateLexicalBlock - Create a new lexical block node and push it on
@@ -218,20 +220,15 @@ public:
void finalize();
- /// setLocation - Update the current source location. If \arg loc is
+ /// \brief Update the current source location. If \arg loc is
/// invalid it is ignored.
void setLocation(SourceLocation Loc);
- /// getLocation - Return the current source location.
- SourceLocation getLocation() const { return CurLoc; }
-
- /// EmitLocation - Emit metadata to indicate a change in line/column
+ /// \brief Emit metadata to indicate a change in line/column
/// information in the source file.
- /// \param ForceColumnInfo Assume DebugColumnInfo option is true.
- void EmitLocation(CGBuilderTy &Builder, SourceLocation Loc,
- bool ForceColumnInfo = false);
+ void EmitLocation(CGBuilderTy &Builder, SourceLocation Loc);
- /// EmitFunctionStart - Emit a call to llvm.dbg.function.start to indicate
+ /// \brief Emit a call to llvm.dbg.function.start to indicate
/// start of a new function.
/// \param Loc The location of the function header.
/// \param ScopeLoc The location of the function body.
@@ -240,23 +237,23 @@ public:
QualType FnType, llvm::Function *Fn,
CGBuilderTy &Builder);
- /// EmitFunctionEnd - Constructs the debug code for exiting a function.
+ /// \brief Constructs the debug code for exiting a function.
void EmitFunctionEnd(CGBuilderTy &Builder);
- /// EmitLexicalBlockStart - Emit metadata to indicate the beginning of a
+ /// \brief Emit metadata to indicate the beginning of a
/// new lexical block and push the block onto the stack.
void EmitLexicalBlockStart(CGBuilderTy &Builder, SourceLocation Loc);
- /// EmitLexicalBlockEnd - Emit metadata to indicate the end of a new lexical
+ /// \brief Emit metadata to indicate the end of a new lexical
/// block and pop the current block.
void EmitLexicalBlockEnd(CGBuilderTy &Builder, SourceLocation Loc);
- /// EmitDeclareOfAutoVariable - Emit call to llvm.dbg.declare for an automatic
+ /// \brief Emit call to llvm.dbg.declare for an automatic
/// variable declaration.
void EmitDeclareOfAutoVariable(const VarDecl *Decl, llvm::Value *AI,
CGBuilderTy &Builder);
- /// EmitDeclareOfBlockDeclRefVariable - Emit call to llvm.dbg.declare for an
+ /// \brief Emit call to llvm.dbg.declare for an
/// imported variable declaration in a block.
void EmitDeclareOfBlockDeclRefVariable(const VarDecl *variable,
llvm::Value *storage,
@@ -264,12 +261,12 @@ public:
const CGBlockInfo &blockInfo,
llvm::Instruction *InsertPoint = 0);
- /// EmitDeclareOfArgVariable - Emit call to llvm.dbg.declare for an argument
+ /// \brief Emit call to llvm.dbg.declare for an argument
/// variable declaration.
void EmitDeclareOfArgVariable(const VarDecl *Decl, llvm::Value *AI,
unsigned ArgNo, CGBuilderTy &Builder);
- /// EmitDeclareOfBlockLiteralArgVariable - Emit call to
+ /// \brief Emit call to
/// llvm.dbg.declare for the block-literal argument to a block
/// invocation function.
void EmitDeclareOfBlockLiteralArgVariable(const CGBlockInfo &block,
@@ -277,31 +274,30 @@ public:
llvm::Value *LocalAddr,
CGBuilderTy &Builder);
- /// EmitGlobalVariable - Emit information about a global variable.
+ /// \brief Emit information about a global variable.
void EmitGlobalVariable(llvm::GlobalVariable *GV, const VarDecl *Decl);
- /// EmitGlobalVariable - Emit global variable's debug info.
+ /// \brief Emit global variable's debug info.
void EmitGlobalVariable(const ValueDecl *VD, llvm::Constant *Init);
- /// \brief - Emit C++ using directive.
+ /// \brief Emit C++ using directive.
void EmitUsingDirective(const UsingDirectiveDecl &UD);
- /// EmitExplicitCastType - Emit the type explicitly casted to.
+ /// \brief Emit the type explicitly casted to.
void EmitExplicitCastType(QualType Ty);
- /// \brief - Emit C++ using declaration.
+ /// \brief Emit C++ using declaration.
void EmitUsingDecl(const UsingDecl &UD);
- /// \brief - Emit C++ namespace alias.
- llvm::DIImportedEntity EmitNamespaceAlias(const NamespaceAliasDecl &NA);
+ /// \brief Emit C++ namespace alias.
+ llvm::DIImportedEntity *EmitNamespaceAlias(const NamespaceAliasDecl &NA);
- /// getOrCreateRecordType - Emit record type's standalone debug info.
- llvm::DIType getOrCreateRecordType(QualType Ty, SourceLocation L);
+ /// \brief Emit record type's standalone debug info.
+ llvm::DIType *getOrCreateRecordType(QualType Ty, SourceLocation L);
- /// getOrCreateInterfaceType - Emit an objective c interface type standalone
+ /// \brief Emit an objective c interface type standalone
/// debug info.
- llvm::DIType getOrCreateInterfaceType(QualType Ty,
- SourceLocation Loc);
+ llvm::DIType *getOrCreateInterfaceType(QualType Ty, SourceLocation Loc);
void completeType(const EnumDecl *ED);
void completeType(const RecordDecl *RD);
@@ -311,133 +307,129 @@ public:
void completeTemplateDefinition(const ClassTemplateSpecializationDecl &SD);
private:
- /// EmitDeclare - Emit call to llvm.dbg.declare for a variable declaration.
+ /// \brief Emit call to llvm.dbg.declare for a variable declaration.
/// Tag accepts custom types DW_TAG_arg_variable and DW_TAG_auto_variable,
/// otherwise would be of type llvm::dwarf::Tag.
- void EmitDeclare(const VarDecl *decl, llvm::dwarf::LLVMConstants Tag,
- llvm::Value *AI, unsigned ArgNo, CGBuilderTy &Builder);
+ void EmitDeclare(const VarDecl *decl, llvm::dwarf::Tag Tag, llvm::Value *AI,
+ unsigned ArgNo, CGBuilderTy &Builder);
// EmitTypeForVarWithBlocksAttr - Build up structure info for the byref.
// See BuildByRefType.
- llvm::DIType EmitTypeForVarWithBlocksAttr(const VarDecl *VD,
- uint64_t *OffSet);
+ llvm::DIType *EmitTypeForVarWithBlocksAttr(const VarDecl *VD,
+ uint64_t *OffSet);
- /// getContextDescriptor - Get context info for the decl.
- llvm::DIScope getContextDescriptor(const Decl *Decl);
+ /// \brief Get context info for the decl.
+ llvm::DIScope *getContextDescriptor(const Decl *Decl);
- llvm::DIScope getCurrentContextDescriptor(const Decl *Decl);
+ llvm::DIScope *getCurrentContextDescriptor(const Decl *Decl);
/// \brief Create a forward decl for a RecordType in a given context.
- llvm::DICompositeType getOrCreateRecordFwdDecl(const RecordType *,
- llvm::DIDescriptor);
-
- /// createContextChain - Create a set of decls for the context chain.
- llvm::DIDescriptor createContextChain(const Decl *Decl);
+ llvm::DICompositeType *getOrCreateRecordFwdDecl(const RecordType *,
+ llvm::DIScope *);
- /// getCurrentDirname - Return current directory name.
+ /// \brief Return current directory name.
StringRef getCurrentDirname();
- /// CreateCompileUnit - Create new compile unit.
+ /// \brief Create new compile unit.
void CreateCompileUnit();
- /// getOrCreateFile - Get the file debug info descriptor for the input
+ /// \brief Get the file debug info descriptor for the input
/// location.
- llvm::DIFile getOrCreateFile(SourceLocation Loc);
+ llvm::DIFile *getOrCreateFile(SourceLocation Loc);
- /// getOrCreateMainFile - Get the file info for main compile unit.
- llvm::DIFile getOrCreateMainFile();
+ /// \brief Get the file info for main compile unit.
+ llvm::DIFile *getOrCreateMainFile();
- /// getOrCreateType - Get the type from the cache or create a new type if
+ /// \brief Get the type from the cache or create a new type if
/// necessary.
- llvm::DIType getOrCreateType(QualType Ty, llvm::DIFile Fg);
+ llvm::DIType *getOrCreateType(QualType Ty, llvm::DIFile *Fg);
- /// getOrCreateLimitedType - Get the type from the cache or create a new
+ /// \brief Get the type from the cache or create a new
/// partial type if necessary.
- llvm::DIType getOrCreateLimitedType(const RecordType *Ty, llvm::DIFile F);
+ llvm::DIType *getOrCreateLimitedType(const RecordType *Ty, llvm::DIFile *F);
- /// CreateTypeNode - Create type metadata for a source language type.
- llvm::DIType CreateTypeNode(QualType Ty, llvm::DIFile Fg);
+ /// \brief Create type metadata for a source language type.
+ llvm::DIType *CreateTypeNode(QualType Ty, llvm::DIFile *Fg);
- /// getObjCInterfaceDecl - return the underlying ObjCInterfaceDecl
+ /// \brief return the underlying ObjCInterfaceDecl
/// if Ty is an ObjCInterface or a pointer to one.
ObjCInterfaceDecl* getObjCInterfaceDecl(QualType Ty);
- /// CreateMemberType - Create new member and increase Offset by FType's size.
- llvm::DIType CreateMemberType(llvm::DIFile Unit, QualType FType,
- StringRef Name, uint64_t *Offset);
+ /// \brief Create new member and increase Offset by FType's size.
+ llvm::DIType *CreateMemberType(llvm::DIFile *Unit, QualType FType,
+ StringRef Name, uint64_t *Offset);
/// \brief Retrieve the DIDescriptor, if any, for the canonical form of this
/// declaration.
- llvm::DIDescriptor getDeclarationOrDefinition(const Decl *D);
+ llvm::DINode *getDeclarationOrDefinition(const Decl *D);
- /// getFunctionDeclaration - Return debug info descriptor to describe method
+ /// \brief Return debug info descriptor to describe method
/// declaration for the given method definition.
- llvm::DISubprogram getFunctionDeclaration(const Decl *D);
+ llvm::DISubprogram *getFunctionDeclaration(const Decl *D);
/// Return debug info descriptor to describe in-class static data member
/// declaration for the given out-of-class definition.
- llvm::DIDerivedType
+ llvm::DIDerivedType *
getOrCreateStaticDataMemberDeclarationOrNull(const VarDecl *D);
- /// \brief Create a DISubprogram describing the forward
+ /// \brief Create a subprogram describing the forward
/// decalration represented in the given FunctionDecl.
- llvm::DISubprogram getFunctionForwardDeclaration(const FunctionDecl *FD);
+ llvm::DISubprogram *getFunctionForwardDeclaration(const FunctionDecl *FD);
- /// \brief Create a DIGlobalVariable describing the forward
- /// decalration represented in the given VarDecl.
- llvm::DIGlobalVariable getGlobalVariableForwardDeclaration(const VarDecl *VD);
+ /// \brief Create a global variable describing the forward decalration
+ /// represented in the given VarDecl.
+ llvm::DIGlobalVariable *
+ getGlobalVariableForwardDeclaration(const VarDecl *VD);
/// Return a global variable that represents one of the collection of
/// global variables created for an anonmyous union.
- llvm::DIGlobalVariable
- CollectAnonRecordDecls(const RecordDecl *RD, llvm::DIFile Unit, unsigned LineNo,
- StringRef LinkageName, llvm::GlobalVariable *Var,
- llvm::DIDescriptor DContext);
+ llvm::DIGlobalVariable *
+ CollectAnonRecordDecls(const RecordDecl *RD, llvm::DIFile *Unit,
+ unsigned LineNo, StringRef LinkageName,
+ llvm::GlobalVariable *Var, llvm::DIScope *DContext);
- /// getFunctionName - Get function name for the given FunctionDecl. If the
+ /// \brief Get function name for the given FunctionDecl. If the
/// name is constructed on demand (e.g. C++ destructor) then the name
/// is stored on the side.
StringRef getFunctionName(const FunctionDecl *FD);
- /// getObjCMethodName - Returns the unmangled name of an Objective-C method.
+ /// \brief Returns the unmangled name of an Objective-C method.
/// This is the display name for the debugging info.
StringRef getObjCMethodName(const ObjCMethodDecl *FD);
- /// getSelectorName - Return selector name. This is used for debugging
+ /// \brief Return selector name. This is used for debugging
/// info.
StringRef getSelectorName(Selector S);
- /// getClassName - Get class name including template argument list.
+ /// \brief Get class name including template argument list.
StringRef getClassName(const RecordDecl *RD);
- /// getVTableName - Get vtable name for the given Class.
+ /// \brief Get vtable name for the given Class.
StringRef getVTableName(const CXXRecordDecl *Decl);
- /// getLineNumber - Get line number for the location. If location is invalid
+ /// \brief Get line number for the location. If location is invalid
/// then use current location.
unsigned getLineNumber(SourceLocation Loc);
- /// getColumnNumber - Get column number for the location. If location is
+ /// \brief Get column number for the location. If location is
/// invalid then use current location.
/// \param Force Assume DebugColumnInfo option is true.
unsigned getColumnNumber(SourceLocation Loc, bool Force=false);
/// \brief Collect various properties of a FunctionDecl.
/// \param GD A GlobalDecl whose getDecl() must return a FunctionDecl.
- void collectFunctionDeclProps(GlobalDecl GD,
- llvm::DIFile Unit,
+ void collectFunctionDeclProps(GlobalDecl GD, llvm::DIFile *Unit,
StringRef &Name, StringRef &LinkageName,
- llvm::DIDescriptor &FDContext,
- llvm::DIArray &TParamsArray,
+ llvm::DIScope *&FDContext,
+ llvm::DINodeArray &TParamsArray,
unsigned &Flags);
/// \brief Collect various properties of a VarDecl.
- void collectVarDeclProps(const VarDecl *VD, llvm::DIFile &Unit,
- unsigned &LineNo, QualType &T,
- StringRef &Name, StringRef &LinkageName,
- llvm::DIDescriptor &VDContext);
+ void collectVarDeclProps(const VarDecl *VD, llvm::DIFile *&Unit,
+ unsigned &LineNo, QualType &T, StringRef &Name,
+ StringRef &LinkageName, llvm::DIScope *&VDContext);
- /// internString - Allocate a copy of \p A using the DebugInfoNames allocator
+ /// \brief Allocate a copy of \p A using the DebugInfoNames allocator
/// and return a reference to it. If multiple arguments are given the strings
/// are concatenated.
StringRef internString(StringRef A, StringRef B = StringRef()) {
@@ -448,33 +440,60 @@ private:
}
};
+/// \brief A scoped helper to set the current debug location to the specified
+/// location or preferred location of the specified Expr.
class ApplyDebugLocation {
-protected:
+private:
+ void init(SourceLocation TemporaryLocation, bool DefaultToEmpty = false);
+ ApplyDebugLocation(CodeGenFunction &CGF, bool DefaultToEmpty,
+ SourceLocation TemporaryLocation);
+
llvm::DebugLoc OriginalLocation;
CodeGenFunction &CGF;
-
public:
- ApplyDebugLocation(CodeGenFunction &CGF,
- SourceLocation TemporaryLocation = SourceLocation(),
- bool ForceColumnInfo = false);
+
+ /// \brief Set the location to the (valid) TemporaryLocation.
+ ApplyDebugLocation(CodeGenFunction &CGF, SourceLocation TemporaryLocation);
+ ApplyDebugLocation(CodeGenFunction &CGF, const Expr *E);
ApplyDebugLocation(CodeGenFunction &CGF, llvm::DebugLoc Loc);
+
~ApplyDebugLocation();
-};
-/// ArtificialLocation - An RAII object that temporarily switches to
-/// an artificial debug location that has a valid scope, but no line
-/// information. This is useful when emitting compiler-generated
-/// helper functions that have no source location associated with
-/// them. The DWARF specification allows the compiler to use the
-/// special line number 0 to indicate code that can not be attributed
-/// to any source location.
-///
-/// This is necessary because passing an empty SourceLocation to
-/// CGDebugInfo::setLocation() will result in the last valid location
-/// being reused.
-class ArtificialLocation : public ApplyDebugLocation {
-public:
- ArtificialLocation(CodeGenFunction &CGF);
+ /// \brief Apply TemporaryLocation if it is valid. Otherwise switch to an
+ /// artificial debug location that has a valid scope, but no line information.
+ ///
+ /// Artificial locations are useful when emitting compiler-generated helper
+ /// functions that have no source location associated with them. The DWARF
+ /// specification allows the compiler to use the special line number 0 to
+ /// indicate code that can not be attributed to any source location. Note that
+ /// passing an empty SourceLocation to CGDebugInfo::setLocation() will result
+ /// in the last valid location being reused.
+ static ApplyDebugLocation CreateArtificial(CodeGenFunction &CGF) {
+ return ApplyDebugLocation(CGF, false, SourceLocation());
+ }
+ /// \brief Apply TemporaryLocation if it is valid. Otherwise switch to an
+ /// artificial debug location that has a valid scope, but no line information.
+ static ApplyDebugLocation CreateDefaultArtificial(CodeGenFunction &CGF,
+ SourceLocation TemporaryLocation) {
+ return ApplyDebugLocation(CGF, false, TemporaryLocation);
+ }
+
+ /// \brief Set the IRBuilder to not attach debug locations. Note that passing
+ /// an empty SourceLocation to CGDebugInfo::setLocation() will result in the
+ /// last valid location being reused. Note that all instructions that do not
+ /// have a location at the beginning of a function are counted towards to
+ /// funciton prologue.
+ static ApplyDebugLocation CreateEmpty(CodeGenFunction &CGF) {
+ return ApplyDebugLocation(CGF, true, SourceLocation());
+ }
+
+ /// \brief Apply TemporaryLocation if it is valid. Otherwise set the IRBuilder
+ /// to not attach debug locations.
+ static ApplyDebugLocation CreateDefaultEmpty(CodeGenFunction &CGF,
+ SourceLocation TemporaryLocation) {
+ return ApplyDebugLocation(CGF, true, TemporaryLocation);
+ }
+
};
diff --git a/lib/CodeGen/CGDecl.cpp b/lib/CodeGen/CGDecl.cpp
index 15a1a7fb5f12..579a04145567 100644
--- a/lib/CodeGen/CGDecl.cpp
+++ b/lib/CodeGen/CGDecl.cpp
@@ -12,6 +12,7 @@
//===----------------------------------------------------------------------===//
#include "CodeGenFunction.h"
+#include "CGCleanup.h"
#include "CGDebugInfo.h"
#include "CGOpenCLRuntime.h"
#include "CodeGenModule.h"
@@ -34,6 +35,7 @@ using namespace CodeGen;
void CodeGenFunction::EmitDecl(const Decl &D) {
switch (D.getKind()) {
case Decl::TranslationUnit:
+ case Decl::ExternCContext:
case Decl::Namespace:
case Decl::UnresolvedUsingTypename:
case Decl::ClassTemplateSpecialization:
@@ -154,6 +156,8 @@ static std::string getStaticDeclName(CodeGenModule &CGM, const VarDecl &D) {
assert(!D.isExternallyVisible() && "name shouldn't matter");
std::string ContextName;
const DeclContext *DC = D.getDeclContext();
+ if (auto *CD = dyn_cast<CapturedDecl>(DC))
+ DC = cast<DeclContext>(CD->getNonClosureContext());
if (const auto *FD = dyn_cast<FunctionDecl>(DC))
ContextName = CGM.getMangledName(FD);
else if (const auto *BD = dyn_cast<BlockDecl>(DC))
@@ -206,6 +210,9 @@ llvm::Constant *CodeGenModule::getOrCreateStaticVarDecl(
GV->setAlignment(getContext().getDeclAlign(&D).getQuantity());
setGlobalVisibility(GV, &D);
+ if (supportsCOMDAT() && GV->isWeakForLinker())
+ GV->setComdat(TheModule.getOrInsertComdat(GV->getName()));
+
if (D.getTLSKind())
setTLSMode(GV, D);
@@ -512,10 +519,7 @@ namespace {
: Addr(addr), Size(size) {}
void Emit(CodeGenFunction &CGF, Flags flags) override {
- llvm::Value *castAddr = CGF.Builder.CreateBitCast(Addr, CGF.Int8PtrTy);
- CGF.Builder.CreateCall2(CGF.CGM.getLLVMLifetimeEndFn(),
- Size, castAddr)
- ->setDoesNotThrow();
+ CGF.EmitLifetimeEnd(Size, Addr);
}
};
}
@@ -631,8 +635,9 @@ void CodeGenFunction::EmitScalarInit(const Expr *init, const ValueDecl *D,
if (capturedByInit) {
// We can use a simple GEP for this because it can't have been
// moved yet.
- tempLV.setAddress(Builder.CreateStructGEP(tempLV.getAddress(),
- getByRefValueLLVMField(cast<VarDecl>(D))));
+ tempLV.setAddress(Builder.CreateStructGEP(
+ nullptr, tempLV.getAddress(),
+ getByRefValueLLVMField(cast<VarDecl>(D)).second));
}
llvm::PointerType *ty
@@ -793,8 +798,9 @@ static void emitStoresForInitAfterMemset(llvm::Constant *Init, llvm::Value *Loc,
// If necessary, get a pointer to the element and emit it.
if (!Elt->isNullValue() && !isa<llvm::UndefValue>(Elt))
- emitStoresForInitAfterMemset(Elt, Builder.CreateConstGEP2_32(Loc, 0, i),
- isVolatile, Builder);
+ emitStoresForInitAfterMemset(
+ Elt, Builder.CreateConstGEP2_32(Init->getType(), Loc, 0, i),
+ isVolatile, Builder);
}
return;
}
@@ -807,8 +813,9 @@ static void emitStoresForInitAfterMemset(llvm::Constant *Init, llvm::Value *Loc,
// If necessary, get a pointer to the element and emit it.
if (!Elt->isNullValue() && !isa<llvm::UndefValue>(Elt))
- emitStoresForInitAfterMemset(Elt, Builder.CreateConstGEP2_32(Loc, 0, i),
- isVolatile, Builder);
+ emitStoresForInitAfterMemset(
+ Elt, Builder.CreateConstGEP2_32(Init->getType(), Loc, 0, i),
+ isVolatile, Builder);
}
}
@@ -833,21 +840,6 @@ static bool shouldUseMemSetPlusStoresToInitialize(llvm::Constant *Init,
canEmitInitWithFewStoresAfterMemset(Init, StoreBudget);
}
-/// Should we use the LLVM lifetime intrinsics for the given local variable?
-static bool shouldUseLifetimeMarkers(CodeGenFunction &CGF, const VarDecl &D,
- unsigned Size) {
- // For now, only in optimized builds.
- if (CGF.CGM.getCodeGenOpts().OptimizationLevel == 0)
- return false;
-
- // Limit the size of marked objects to 32 bytes. We don't want to increase
- // compile time by marking tiny objects.
- unsigned SizeThreshold = 32;
-
- return Size > SizeThreshold;
-}
-
-
/// EmitAutoVarDecl - Emit code and set up an entry in LocalDeclMap for a
/// variable declaration with auto, register, or no storage class specifier.
/// These turn into simple stack objects, or GlobalValues depending on target.
@@ -857,6 +849,38 @@ void CodeGenFunction::EmitAutoVarDecl(const VarDecl &D) {
EmitAutoVarCleanups(emission);
}
+/// Emit a lifetime.begin marker if some criteria are satisfied.
+/// \return a pointer to the temporary size Value if a marker was emitted, null
+/// otherwise
+llvm::Value *CodeGenFunction::EmitLifetimeStart(uint64_t Size,
+ llvm::Value *Addr) {
+ // For now, only in optimized builds.
+ if (CGM.getCodeGenOpts().OptimizationLevel == 0)
+ return nullptr;
+
+ // Disable lifetime markers in msan builds.
+ // FIXME: Remove this when msan works with lifetime markers.
+ if (getLangOpts().Sanitize.has(SanitizerKind::Memory))
+ return nullptr;
+
+ llvm::Value *SizeV = llvm::ConstantInt::get(Int64Ty, Size);
+ llvm::Value *Args[] = {
+ SizeV,
+ new llvm::BitCastInst(Addr, Int8PtrTy, "", Builder.GetInsertBlock())};
+ llvm::CallInst *C = llvm::CallInst::Create(CGM.getLLVMLifetimeStartFn(), Args,
+ "", Builder.GetInsertBlock());
+ C->setDoesNotThrow();
+ return SizeV;
+}
+
+void CodeGenFunction::EmitLifetimeEnd(llvm::Value *Size, llvm::Value *Addr) {
+ llvm::Value *Args[] = {Size, new llvm::BitCastInst(Addr, Int8PtrTy, "",
+ Builder.GetInsertBlock())};
+ llvm::CallInst *C = llvm::CallInst::Create(CGM.getLLVMLifetimeEndFn(), Args,
+ "", Builder.GetInsertBlock());
+ C->setDoesNotThrow();
+}
+
/// EmitAutoVarAlloca - Emit the alloca and debug information for a
/// local variable. Does not emit initialization or destruction.
CodeGenFunction::AutoVarEmission
@@ -952,13 +976,8 @@ CodeGenFunction::EmitAutoVarAlloca(const VarDecl &D) {
// Emit a lifetime intrinsic if meaningful. There's no point
// in doing this if we don't have a valid insertion point (?).
uint64_t size = CGM.getDataLayout().getTypeAllocSize(LTy);
- if (HaveInsertPoint() && shouldUseLifetimeMarkers(*this, D, size)) {
- llvm::Value *sizeV = llvm::ConstantInt::get(Int64Ty, size);
-
- emission.SizeForLifetimeMarkers = sizeV;
- llvm::Value *castAddr = Builder.CreateBitCast(Alloc, Int8PtrTy);
- Builder.CreateCall2(CGM.getLLVMLifetimeStartFn(), sizeV, castAddr)
- ->setDoesNotThrow();
+ if (HaveInsertPoint()) {
+ emission.SizeForLifetimeMarkers = EmitLifetimeStart(size, Alloc);
} else {
assert(!emission.useLifetimeMarkers());
}
@@ -971,7 +990,7 @@ CodeGenFunction::EmitAutoVarAlloca(const VarDecl &D) {
llvm::Value *Stack = CreateTempAlloca(Int8PtrTy, "saved_stack");
llvm::Value *F = CGM.getIntrinsic(llvm::Intrinsic::stacksave);
- llvm::Value *V = Builder.CreateCall(F);
+ llvm::Value *V = Builder.CreateCall(F, {});
Builder.CreateStore(V, Stack);
@@ -1087,7 +1106,7 @@ void CodeGenFunction::EmitAutoVarInit(const AutoVarEmission &emission) {
if (emission.wasEmittedAsGlobal()) return;
const VarDecl &D = *emission.Variable;
- ApplyDebugLocation DL(*this, D.getLocation());
+ auto DL = ApplyDebugLocation::CreateDefaultArtificial(*this, D.getLocation());
QualType type = D.getType();
// If this local has an initializer, emit it now.
@@ -1304,6 +1323,8 @@ void CodeGenFunction::EmitAutoVarCleanups(const AutoVarEmission &emission) {
EHStack.pushCleanup<CallLifetimeEnd>(NormalCleanup,
emission.getAllocatedAddress(),
emission.getSizeForLifetimeMarkers());
+ EHCleanupScope &cleanup = cast<EHCleanupScope>(*EHStack.begin());
+ cleanup.setLifetimeMarker();
}
// Check the type for a cleanup.
diff --git a/lib/CodeGen/CGDeclCXX.cpp b/lib/CodeGen/CGDeclCXX.cpp
index 19e4bdd00cfe..06d157bd82e7 100644
--- a/lib/CodeGen/CGDeclCXX.cpp
+++ b/lib/CodeGen/CGDeclCXX.cpp
@@ -139,9 +139,32 @@ void CodeGenFunction::EmitCXXGlobalVarDeclInit(const VarDecl &D,
const Expr *Init = D.getInit();
QualType T = D.getType();
+ // The address space of a static local variable (DeclPtr) may be different
+ // from the address space of the "this" argument of the constructor. In that
+ // case, we need an addrspacecast before calling the constructor.
+ //
+ // struct StructWithCtor {
+ // __device__ StructWithCtor() {...}
+ // };
+ // __device__ void foo() {
+ // __shared__ StructWithCtor s;
+ // ...
+ // }
+ //
+ // For example, in the above CUDA code, the static local variable s has a
+ // "shared" address space qualifier, but the constructor of StructWithCtor
+ // expects "this" in the "generic" address space.
+ unsigned ExpectedAddrSpace = getContext().getTargetAddressSpace(T);
+ unsigned ActualAddrSpace = DeclPtr->getType()->getPointerAddressSpace();
+ if (ActualAddrSpace != ExpectedAddrSpace) {
+ llvm::Type *LTy = CGM.getTypes().ConvertTypeForMem(T);
+ llvm::PointerType *PTy = llvm::PointerType::get(LTy, ExpectedAddrSpace);
+ DeclPtr = llvm::ConstantExpr::getAddrSpaceCast(DeclPtr, PTy);
+ }
+
if (!T->isReferenceType()) {
if (getLangOpts().OpenMP && D.hasAttr<OMPThreadPrivateDeclAttr>())
- (void)CGM.getOpenMPRuntime().EmitOMPThreadPrivateVarDefinition(
+ (void)CGM.getOpenMPRuntime().emitThreadPrivateVarDefinition(
&D, DeclPtr, D.getAttr<OMPThreadPrivateDeclAttr>()->getLocation(),
PerformInit, this);
if (PerformInit)
@@ -236,6 +259,8 @@ llvm::Function *CodeGenModule::CreateGlobalInitOrDestructFunction(
Fn->setSection(Section);
}
+ SetLLVMFunctionAttributes(nullptr, getTypes().arrangeNullaryFunction(), Fn);
+
Fn->setCallingConv(getRuntimeCC());
if (!getLangOpts().Exceptions)
@@ -267,15 +292,7 @@ void CodeGenModule::EmitPointerToInitFunc(const VarDecl *D,
addUsedGlobal(PtrArray);
// If the GV is already in a comdat group, then we have to join it.
- llvm::Comdat *C = GV->getComdat();
-
- // LinkOnce and Weak linkage are lowered down to a single-member comdat group.
- // Make an explicit group so we can join it.
- if (!C && (GV->hasWeakLinkage() || GV->hasLinkOnceLinkage())) {
- C = TheModule.getOrInsertComdat(GV->getName());
- GV->setComdat(C);
- }
- if (C)
+ if (llvm::Comdat *C = GV->getComdat())
PtrArray->setComdat(C);
}
@@ -283,6 +300,11 @@ void
CodeGenModule::EmitCXXGlobalVarDeclInitFunc(const VarDecl *D,
llvm::GlobalVariable *Addr,
bool PerformInit) {
+ // Check if we've already initialized this decl.
+ auto I = DelayedCXXInitPosition.find(D);
+ if (I != DelayedCXXInitPosition.end() && I->second == ~0U)
+ return;
+
llvm::FunctionType *FTy = llvm::FunctionType::get(VoidTy, false);
SmallString<256> FnName;
{
@@ -312,11 +334,9 @@ CodeGenModule::EmitCXXGlobalVarDeclInitFunc(const VarDecl *D,
CXXThreadLocalInitVars.push_back(Addr);
} else if (PerformInit && ISA) {
EmitPointerToInitFunc(D, Addr, Fn, ISA);
- DelayedCXXInitPosition.erase(D);
} else if (auto *IPA = D->getAttr<InitPriorityAttr>()) {
OrderGlobalInits Key(IPA->getPriority(), PrioritizedCXXGlobalInits.size());
PrioritizedCXXGlobalInits.push_back(std::make_pair(Key, Fn));
- DelayedCXXInitPosition.erase(D);
} else if (isTemplateInstantiation(D->getTemplateSpecializationKind())) {
// C++ [basic.start.init]p2:
// Definitions of explicitly specialized class template static data
@@ -331,24 +351,24 @@ CodeGenModule::EmitCXXGlobalVarDeclInitFunc(const VarDecl *D,
// minor startup time optimization. In the MS C++ ABI, there are no guard
// variables, so this COMDAT key is required for correctness.
AddGlobalCtor(Fn, 65535, COMDATKey);
- DelayedCXXInitPosition.erase(D);
} else if (D->hasAttr<SelectAnyAttr>()) {
// SelectAny globals will be comdat-folded. Put the initializer into a
// COMDAT group associated with the global, so the initializers get folded
// too.
AddGlobalCtor(Fn, 65535, COMDATKey);
- DelayedCXXInitPosition.erase(D);
} else {
- llvm::DenseMap<const Decl *, unsigned>::iterator I =
- DelayedCXXInitPosition.find(D);
+ I = DelayedCXXInitPosition.find(D); // Re-do lookup in case of re-hash.
if (I == DelayedCXXInitPosition.end()) {
CXXGlobalInits.push_back(Fn);
- } else {
- assert(CXXGlobalInits[I->second] == nullptr);
+ } else if (I->second != ~0U) {
+ assert(I->second < CXXGlobalInits.size() &&
+ CXXGlobalInits[I->second] == nullptr);
CXXGlobalInits[I->second] = Fn;
- DelayedCXXInitPosition.erase(I);
}
}
+
+ // Remember that we already emitted the initializer for this global.
+ DelayedCXXInitPosition[D] = ~0U;
}
void CodeGenModule::EmitCXXThreadLocalInitFunc() {
@@ -411,7 +431,7 @@ CodeGenModule::EmitCXXGlobalInitFunc() {
// priority emitted above.
FileName = llvm::sys::path::filename(MainFile->getName());
} else {
- FileName = SmallString<128>("<null>");
+ FileName = "<null>";
}
for (size_t i = 0; i < FileName.size(); ++i) {
@@ -477,11 +497,11 @@ CodeGenFunction::GenerateCXXGlobalInitFunc(llvm::Function *Fn,
ArrayRef<llvm::Function *> Decls,
llvm::GlobalVariable *Guard) {
{
- ApplyDebugLocation NL(*this);
+ auto NL = ApplyDebugLocation::CreateEmpty(*this);
StartFunction(GlobalDecl(), getContext().VoidTy, Fn,
getTypes().arrangeNullaryFunction(), FunctionArgList());
// Emit an artificial location for this function.
- ArtificialLocation AL(*this);
+ auto AL = ApplyDebugLocation::CreateArtificial(*this);
llvm::BasicBlock *ExitBlock = nullptr;
if (Guard) {
@@ -528,11 +548,11 @@ void CodeGenFunction::GenerateCXXGlobalDtorsFunc(llvm::Function *Fn,
const std::vector<std::pair<llvm::WeakVH, llvm::Constant*> >
&DtorsAndObjects) {
{
- ApplyDebugLocation NL(*this);
+ auto NL = ApplyDebugLocation::CreateEmpty(*this);
StartFunction(GlobalDecl(), getContext().VoidTy, Fn,
getTypes().arrangeNullaryFunction(), FunctionArgList());
// Emit an artificial location for this function.
- ArtificialLocation AL(*this);
+ auto AL = ApplyDebugLocation::CreateArtificial(*this);
// Emit the dtors, in reverse order from construction.
for (unsigned i = 0, e = DtorsAndObjects.size(); i != e; ++i) {
diff --git a/lib/CodeGen/CGException.cpp b/lib/CodeGen/CGException.cpp
index cb8eb8fa490c..d9a3f0b252a5 100644
--- a/lib/CodeGen/CGException.cpp
+++ b/lib/CodeGen/CGException.cpp
@@ -16,23 +16,18 @@
#include "CGCleanup.h"
#include "CGObjCRuntime.h"
#include "TargetInfo.h"
+#include "clang/AST/Mangle.h"
#include "clang/AST/StmtCXX.h"
#include "clang/AST/StmtObjC.h"
+#include "clang/AST/StmtVisitor.h"
#include "llvm/IR/CallSite.h"
#include "llvm/IR/Intrinsics.h"
+#include "llvm/IR/IntrinsicInst.h"
+#include "llvm/Support/SaveAndRestore.h"
using namespace clang;
using namespace CodeGen;
-static llvm::Constant *getAllocateExceptionFn(CodeGenModule &CGM) {
- // void *__cxa_allocate_exception(size_t thrown_size);
-
- llvm::FunctionType *FTy =
- llvm::FunctionType::get(CGM.Int8PtrTy, CGM.SizeTy, /*IsVarArgs=*/false);
-
- return CGM.CreateRuntimeFunction(FTy, "__cxa_allocate_exception");
-}
-
static llvm::Constant *getFreeExceptionFn(CodeGenModule &CGM) {
// void __cxa_free_exception(void *thrown_exception);
@@ -42,44 +37,6 @@ static llvm::Constant *getFreeExceptionFn(CodeGenModule &CGM) {
return CGM.CreateRuntimeFunction(FTy, "__cxa_free_exception");
}
-static llvm::Constant *getThrowFn(CodeGenModule &CGM) {
- // void __cxa_throw(void *thrown_exception, std::type_info *tinfo,
- // void (*dest) (void *));
-
- llvm::Type *Args[3] = { CGM.Int8PtrTy, CGM.Int8PtrTy, CGM.Int8PtrTy };
- llvm::FunctionType *FTy =
- llvm::FunctionType::get(CGM.VoidTy, Args, /*IsVarArgs=*/false);
-
- return CGM.CreateRuntimeFunction(FTy, "__cxa_throw");
-}
-
-static llvm::Constant *getGetExceptionPtrFn(CodeGenModule &CGM) {
- // void *__cxa_get_exception_ptr(void*);
-
- llvm::FunctionType *FTy =
- llvm::FunctionType::get(CGM.Int8PtrTy, CGM.Int8PtrTy, /*IsVarArgs=*/false);
-
- return CGM.CreateRuntimeFunction(FTy, "__cxa_get_exception_ptr");
-}
-
-static llvm::Constant *getBeginCatchFn(CodeGenModule &CGM) {
- // void *__cxa_begin_catch(void*);
-
- llvm::FunctionType *FTy =
- llvm::FunctionType::get(CGM.Int8PtrTy, CGM.Int8PtrTy, /*IsVarArgs=*/false);
-
- return CGM.CreateRuntimeFunction(FTy, "__cxa_begin_catch");
-}
-
-static llvm::Constant *getEndCatchFn(CodeGenModule &CGM) {
- // void __cxa_end_catch();
-
- llvm::FunctionType *FTy =
- llvm::FunctionType::get(CGM.VoidTy, /*IsVarArgs=*/false);
-
- return CGM.CreateRuntimeFunction(FTy, "__cxa_end_catch");
-}
-
static llvm::Constant *getUnexpectedFn(CodeGenModule &CGM) {
// void __cxa_call_unexpected(void *thrown_exception);
@@ -89,23 +46,30 @@ static llvm::Constant *getUnexpectedFn(CodeGenModule &CGM) {
return CGM.CreateRuntimeFunction(FTy, "__cxa_call_unexpected");
}
-static llvm::Constant *getTerminateFn(CodeGenModule &CGM) {
+llvm::Constant *CodeGenModule::getTerminateFn() {
// void __terminate();
llvm::FunctionType *FTy =
- llvm::FunctionType::get(CGM.VoidTy, /*IsVarArgs=*/false);
+ llvm::FunctionType::get(VoidTy, /*IsVarArgs=*/false);
StringRef name;
// In C++, use std::terminate().
- if (CGM.getLangOpts().CPlusPlus)
- name = "_ZSt9terminatev"; // FIXME: mangling!
- else if (CGM.getLangOpts().ObjC1 &&
- CGM.getLangOpts().ObjCRuntime.hasTerminate())
+ if (getLangOpts().CPlusPlus &&
+ getTarget().getCXXABI().isItaniumFamily()) {
+ name = "_ZSt9terminatev";
+ } else if (getLangOpts().CPlusPlus &&
+ getTarget().getCXXABI().isMicrosoft()) {
+ if (getLangOpts().isCompatibleWithMSVC(LangOptions::MSVC2015))
+ name = "__std_terminate";
+ else
+ name = "\01?terminate@@YAXXZ";
+ } else if (getLangOpts().ObjC1 &&
+ getLangOpts().ObjCRuntime.hasTerminate())
name = "objc_terminate";
else
name = "abort";
- return CGM.CreateRuntimeFunction(FTy, name);
+ return CreateRuntimeFunction(FTy, name);
}
static llvm::Constant *getCatchallRethrowFn(CodeGenModule &CGM,
@@ -126,7 +90,12 @@ namespace {
// This function must have prototype void(void*).
const char *CatchallRethrowFn;
- static const EHPersonality &get(CodeGenModule &CGM);
+ static const EHPersonality &get(CodeGenModule &CGM,
+ const FunctionDecl *FD);
+ static const EHPersonality &get(CodeGenFunction &CGF) {
+ return get(CGF.CGM, dyn_cast_or_null<FunctionDecl>(CGF.CurCodeDecl));
+ }
+
static const EHPersonality GNU_C;
static const EHPersonality GNU_C_SJLJ;
static const EHPersonality GNU_C_SEH;
@@ -137,6 +106,9 @@ namespace {
static const EHPersonality GNU_CPlusPlus;
static const EHPersonality GNU_CPlusPlus_SJLJ;
static const EHPersonality GNU_CPlusPlus_SEH;
+ static const EHPersonality MSVC_except_handler;
+ static const EHPersonality MSVC_C_specific_handler;
+ static const EHPersonality MSVC_CxxFrameHandler3;
};
}
@@ -159,6 +131,12 @@ const EHPersonality
EHPersonality::GNU_ObjCXX = { "__gnustep_objcxx_personality_v0", nullptr };
const EHPersonality
EHPersonality::GNUstep_ObjC = { "__gnustep_objc_personality_v0", nullptr };
+const EHPersonality
+EHPersonality::MSVC_except_handler = { "_except_handler3", nullptr };
+const EHPersonality
+EHPersonality::MSVC_C_specific_handler = { "__C_specific_handler", nullptr };
+const EHPersonality
+EHPersonality::MSVC_CxxFrameHandler3 = { "__CxxFrameHandler3", nullptr };
/// On Win64, use libgcc's SEH personality function. We fall back to dwarf on
/// other platforms, unless the user asked for SjLj exceptions.
@@ -231,9 +209,29 @@ static const EHPersonality &getObjCXXPersonality(const llvm::Triple &T,
llvm_unreachable("bad runtime kind");
}
-const EHPersonality &EHPersonality::get(CodeGenModule &CGM) {
+static const EHPersonality &getSEHPersonalityMSVC(const llvm::Triple &T) {
+ if (T.getArch() == llvm::Triple::x86)
+ return EHPersonality::MSVC_except_handler;
+ return EHPersonality::MSVC_C_specific_handler;
+}
+
+const EHPersonality &EHPersonality::get(CodeGenModule &CGM,
+ const FunctionDecl *FD) {
const llvm::Triple &T = CGM.getTarget().getTriple();
const LangOptions &L = CGM.getLangOpts();
+
+ // Try to pick a personality function that is compatible with MSVC if we're
+ // not compiling Obj-C. Obj-C users better have an Obj-C runtime that supports
+ // the GCC-style personality function.
+ if (T.isWindowsMSVCEnvironment() && !L.ObjC1) {
+ if (L.SjLjExceptions)
+ return EHPersonality::GNU_CPlusPlus_SJLJ;
+ else if (FD && FD->usesSEHTry())
+ return getSEHPersonalityMSVC(T);
+ else
+ return EHPersonality::MSVC_CxxFrameHandler3;
+ }
+
if (L.CPlusPlus && L.ObjC1)
return getObjCXXPersonality(T, L);
else if (L.CPlusPlus)
@@ -318,7 +316,7 @@ void CodeGenModule::SimplifyPersonality() {
if (!LangOpts.ObjCRuntime.isNeXTFamily())
return;
- const EHPersonality &ObjCXX = EHPersonality::get(*this);
+ const EHPersonality &ObjCXX = EHPersonality::get(*this, /*FD=*/nullptr);
const EHPersonality &CXX =
getCXXPersonality(getTarget().getTriple(), LangOpts);
if (&ObjCXX == &CXX)
@@ -369,17 +367,16 @@ namespace {
// differs from EmitAnyExprToMem only in that, if a final copy-ctor
// call is required, an exception within that copy ctor causes
// std::terminate to be invoked.
-static void EmitAnyExprToExn(CodeGenFunction &CGF, const Expr *e,
- llvm::Value *addr) {
+void CodeGenFunction::EmitAnyExprToExn(const Expr *e, llvm::Value *addr) {
// Make sure the exception object is cleaned up if there's an
// exception during initialization.
- CGF.pushFullExprCleanup<FreeException>(EHCleanup, addr);
- EHScopeStack::stable_iterator cleanup = CGF.EHStack.stable_begin();
+ pushFullExprCleanup<FreeException>(EHCleanup, addr);
+ EHScopeStack::stable_iterator cleanup = EHStack.stable_begin();
// __cxa_allocate_exception returns a void*; we need to cast this
// to the appropriate type for the object.
- llvm::Type *ty = CGF.ConvertTypeForMem(e->getType())->getPointerTo();
- llvm::Value *typedAddr = CGF.Builder.CreateBitCast(addr, ty);
+ llvm::Type *ty = ConvertTypeForMem(e->getType())->getPointerTo();
+ llvm::Value *typedAddr = Builder.CreateBitCast(addr, ty);
// FIXME: this isn't quite right! If there's a final unelided call
// to a copy constructor, then according to [except.terminate]p1 we
@@ -388,11 +385,11 @@ static void EmitAnyExprToExn(CodeGenFunction &CGF, const Expr *e,
// evaluated but before the exception is caught. But the best way
// to handle that is to teach EmitAggExpr to do the final copy
// differently if it can't be elided.
- CGF.EmitAnyExprToMem(e, typedAddr, e->getType().getQualifiers(),
- /*IsInit*/ true);
+ EmitAnyExprToMem(e, typedAddr, e->getType().getQualifiers(),
+ /*IsInit*/ true);
// Deactivate the cleanup block.
- CGF.DeactivateCleanupBlock(cleanup, cast<llvm::Instruction>(typedAddr));
+ DeactivateCleanupBlock(cleanup, cast<llvm::Instruction>(typedAddr));
}
llvm::Value *CodeGenFunction::getExceptionSlot() {
@@ -417,67 +414,18 @@ llvm::Value *CodeGenFunction::getSelectorFromSlot() {
void CodeGenFunction::EmitCXXThrowExpr(const CXXThrowExpr *E,
bool KeepInsertionPoint) {
- if (!E->getSubExpr()) {
- CGM.getCXXABI().emitRethrow(*this, /*isNoReturn*/true);
-
- // throw is an expression, and the expression emitters expect us
- // to leave ourselves at a valid insertion point.
- if (KeepInsertionPoint)
- EmitBlock(createBasicBlock("throw.cont"));
-
- return;
- }
-
- if (CGM.getTarget().getTriple().isKnownWindowsMSVCEnvironment()) {
- ErrorUnsupported(E, "throw expression");
- return;
- }
-
- QualType ThrowType = E->getSubExpr()->getType();
-
- if (ThrowType->isObjCObjectPointerType()) {
- const Stmt *ThrowStmt = E->getSubExpr();
- const ObjCAtThrowStmt S(E->getExprLoc(),
- const_cast<Stmt *>(ThrowStmt));
- CGM.getObjCRuntime().EmitThrowStmt(*this, S, false);
- // This will clear insertion point which was not cleared in
- // call to EmitThrowStmt.
- if (KeepInsertionPoint)
- EmitBlock(createBasicBlock("throw.cont"));
- return;
- }
-
- // Now allocate the exception object.
- llvm::Type *SizeTy = ConvertType(getContext().getSizeType());
- uint64_t TypeSize = getContext().getTypeSizeInChars(ThrowType).getQuantity();
-
- llvm::Constant *AllocExceptionFn = getAllocateExceptionFn(CGM);
- llvm::CallInst *ExceptionPtr =
- EmitNounwindRuntimeCall(AllocExceptionFn,
- llvm::ConstantInt::get(SizeTy, TypeSize),
- "exception");
-
- EmitAnyExprToExn(*this, E->getSubExpr(), ExceptionPtr);
-
- // Now throw the exception.
- llvm::Constant *TypeInfo = CGM.GetAddrOfRTTIDescriptor(ThrowType,
- /*ForEH=*/true);
-
- // The address of the destructor. If the exception type has a
- // trivial destructor (or isn't a record), we just pass null.
- llvm::Constant *Dtor = nullptr;
- if (const RecordType *RecordTy = ThrowType->getAs<RecordType>()) {
- CXXRecordDecl *Record = cast<CXXRecordDecl>(RecordTy->getDecl());
- if (!Record->hasTrivialDestructor()) {
- CXXDestructorDecl *DtorD = Record->getDestructor();
- Dtor = CGM.getAddrOfCXXStructor(DtorD, StructorType::Complete);
- Dtor = llvm::ConstantExpr::getBitCast(Dtor, Int8PtrTy);
+ if (const Expr *SubExpr = E->getSubExpr()) {
+ QualType ThrowType = SubExpr->getType();
+ if (ThrowType->isObjCObjectPointerType()) {
+ const Stmt *ThrowStmt = E->getSubExpr();
+ const ObjCAtThrowStmt S(E->getExprLoc(), const_cast<Stmt *>(ThrowStmt));
+ CGM.getObjCRuntime().EmitThrowStmt(*this, S, false);
+ } else {
+ CGM.getCXXABI().emitThrow(*this, E);
}
+ } else {
+ CGM.getCXXABI().emitRethrow(*this, /*isNoReturn=*/true);
}
- if (!Dtor) Dtor = llvm::Constant::getNullValue(Int8PtrTy);
-
- llvm::Value *args[] = { ExceptionPtr, TypeInfo, Dtor };
- EmitNoreturnRuntimeCallOrInvoke(getThrowFn(CGM), args);
// throw is an expression, and the expression emitters expect us
// to leave ourselves at a valid insertion point.
@@ -509,6 +457,10 @@ void CodeGenFunction::EmitStartEHSpec(const Decl *D) {
EHStack.pushTerminate();
}
} else if (EST == EST_Dynamic || EST == EST_DynamicNone) {
+ // TODO: Revisit exception specifications for the MS ABI. There is a way to
+ // encode these in an object file but MSVC doesn't do anything with it.
+ if (getTarget().getCXXABI().isMicrosoft())
+ return;
unsigned NumExceptions = Proto->getNumExceptions();
EHFilterScope *Filter = EHStack.pushFilter(NumExceptions);
@@ -543,8 +495,9 @@ static void emitFilterDispatchBlock(CodeGenFunction &CGF,
llvm::Value *zero = CGF.Builder.getInt32(0);
llvm::Value *failsFilter =
- CGF.Builder.CreateICmpSLT(selector, zero, "ehspec.fails");
- CGF.Builder.CreateCondBr(failsFilter, unexpectedBB, CGF.getEHResumeBlock(false));
+ CGF.Builder.CreateICmpSLT(selector, zero, "ehspec.fails");
+ CGF.Builder.CreateCondBr(failsFilter, unexpectedBB,
+ CGF.getEHResumeBlock(false));
CGF.EmitBlock(unexpectedBB);
}
@@ -582,6 +535,10 @@ void CodeGenFunction::EmitEndEHSpec(const Decl *D) {
EHStack.popTerminate();
}
} else if (EST == EST_Dynamic || EST == EST_DynamicNone) {
+ // TODO: Revisit exception specifications for the MS ABI. There is a way to
+ // encode these in an object file but MSVC doesn't do anything with it.
+ if (getTarget().getCXXABI().isMicrosoft())
+ return;
EHFilterScope &filterScope = cast<EHFilterScope>(*EHStack.begin());
emitFilterDispatchBlock(*this, filterScope);
EHStack.popFilter();
@@ -589,11 +546,6 @@ void CodeGenFunction::EmitEndEHSpec(const Decl *D) {
}
void CodeGenFunction::EmitCXXTryStmt(const CXXTryStmt &S) {
- if (CGM.getTarget().getTriple().isKnownWindowsMSVCEnvironment()) {
- ErrorUnsupported(&S, "try statement");
- return;
- }
-
EnterCXXTryStmt(S);
EmitStmt(S.getTryBlock());
ExitCXXTryStmt(S);
@@ -622,7 +574,8 @@ void CodeGenFunction::EnterCXXTryStmt(const CXXTryStmt &S, bool IsFnTryBlock) {
if (CaughtType->isObjCObjectPointerType())
TypeInfo = CGM.getObjCRuntime().GetEHType(CaughtType);
else
- TypeInfo = CGM.GetAddrOfRTTIDescriptor(CaughtType, /*ForEH=*/true);
+ TypeInfo =
+ CGM.getAddrOfCXXCatchHandlerType(CaughtType, C->getCaughtType());
CatchScope->setHandler(I, TypeInfo, Handler);
} else {
// No exception decl indicates '...', a catch-all.
@@ -695,8 +648,15 @@ llvm::BasicBlock *CodeGenFunction::getInvokeDestImpl() {
assert(EHStack.requiresLandingPad());
assert(!EHStack.empty());
- if (!CGM.getLangOpts().Exceptions)
- return nullptr;
+ // If exceptions are disabled, there are usually no landingpads. However, when
+ // SEH is enabled, functions using SEH still get landingpads.
+ const LangOptions &LO = CGM.getLangOpts();
+ if (!LO.Exceptions) {
+ if (!LO.Borland && !LO.MicrosoftExt)
+ return nullptr;
+ if (!currentFunctionUsesSEHTry())
+ return nullptr;
+ }
// Check the innermost scope for a cached landing pad. If this is
// a non-EH cleanup, we'll check enclosing scopes in EmitLandingPad.
@@ -734,9 +694,9 @@ llvm::BasicBlock *CodeGenFunction::EmitLandingPad() {
// Save the current IR generation state.
CGBuilderTy::InsertPoint savedIP = Builder.saveAndClearIP();
- ApplyDebugLocation AutoRestoreLocation(*this, CurEHLocation);
+ auto DL = ApplyDebugLocation::CreateDefaultArtificial(*this, CurEHLocation);
- const EHPersonality &personality = EHPersonality::get(CGM);
+ const EHPersonality &personality = EHPersonality::get(*this);
// Create and configure the landing pad.
llvm::BasicBlock *lpad = createBasicBlock("lpad");
@@ -762,8 +722,8 @@ llvm::BasicBlock *CodeGenFunction::EmitLandingPad() {
bool hasFilter = false;
SmallVector<llvm::Value*, 4> filterTypes;
llvm::SmallPtrSet<llvm::Value*, 4> catchTypes;
- for (EHScopeStack::iterator I = EHStack.begin(), E = EHStack.end();
- I != E; ++I) {
+ for (EHScopeStack::iterator I = EHStack.begin(), E = EHStack.end(); I != E;
+ ++I) {
switch (I->getKind()) {
case EHScope::Cleanup:
@@ -857,263 +817,6 @@ llvm::BasicBlock *CodeGenFunction::EmitLandingPad() {
return lpad;
}
-namespace {
- /// A cleanup to call __cxa_end_catch. In many cases, the caught
- /// exception type lets us state definitively that the thrown exception
- /// type does not have a destructor. In particular:
- /// - Catch-alls tell us nothing, so we have to conservatively
- /// assume that the thrown exception might have a destructor.
- /// - Catches by reference behave according to their base types.
- /// - Catches of non-record types will only trigger for exceptions
- /// of non-record types, which never have destructors.
- /// - Catches of record types can trigger for arbitrary subclasses
- /// of the caught type, so we have to assume the actual thrown
- /// exception type might have a throwing destructor, even if the
- /// caught type's destructor is trivial or nothrow.
- struct CallEndCatch : EHScopeStack::Cleanup {
- CallEndCatch(bool MightThrow) : MightThrow(MightThrow) {}
- bool MightThrow;
-
- void Emit(CodeGenFunction &CGF, Flags flags) override {
- if (!MightThrow) {
- CGF.EmitNounwindRuntimeCall(getEndCatchFn(CGF.CGM));
- return;
- }
-
- CGF.EmitRuntimeCallOrInvoke(getEndCatchFn(CGF.CGM));
- }
- };
-}
-
-/// Emits a call to __cxa_begin_catch and enters a cleanup to call
-/// __cxa_end_catch.
-///
-/// \param EndMightThrow - true if __cxa_end_catch might throw
-static llvm::Value *CallBeginCatch(CodeGenFunction &CGF,
- llvm::Value *Exn,
- bool EndMightThrow) {
- llvm::CallInst *call =
- CGF.EmitNounwindRuntimeCall(getBeginCatchFn(CGF.CGM), Exn);
-
- CGF.EHStack.pushCleanup<CallEndCatch>(NormalAndEHCleanup, EndMightThrow);
-
- return call;
-}
-
-/// A "special initializer" callback for initializing a catch
-/// parameter during catch initialization.
-static void InitCatchParam(CodeGenFunction &CGF,
- const VarDecl &CatchParam,
- llvm::Value *ParamAddr,
- SourceLocation Loc) {
- // Load the exception from where the landing pad saved it.
- llvm::Value *Exn = CGF.getExceptionFromSlot();
-
- CanQualType CatchType =
- CGF.CGM.getContext().getCanonicalType(CatchParam.getType());
- llvm::Type *LLVMCatchTy = CGF.ConvertTypeForMem(CatchType);
-
- // If we're catching by reference, we can just cast the object
- // pointer to the appropriate pointer.
- if (isa<ReferenceType>(CatchType)) {
- QualType CaughtType = cast<ReferenceType>(CatchType)->getPointeeType();
- bool EndCatchMightThrow = CaughtType->isRecordType();
-
- // __cxa_begin_catch returns the adjusted object pointer.
- llvm::Value *AdjustedExn = CallBeginCatch(CGF, Exn, EndCatchMightThrow);
-
- // We have no way to tell the personality function that we're
- // catching by reference, so if we're catching a pointer,
- // __cxa_begin_catch will actually return that pointer by value.
- if (const PointerType *PT = dyn_cast<PointerType>(CaughtType)) {
- QualType PointeeType = PT->getPointeeType();
-
- // When catching by reference, generally we should just ignore
- // this by-value pointer and use the exception object instead.
- if (!PointeeType->isRecordType()) {
-
- // Exn points to the struct _Unwind_Exception header, which
- // we have to skip past in order to reach the exception data.
- unsigned HeaderSize =
- CGF.CGM.getTargetCodeGenInfo().getSizeOfUnwindException();
- AdjustedExn = CGF.Builder.CreateConstGEP1_32(Exn, HeaderSize);
-
- // However, if we're catching a pointer-to-record type that won't
- // work, because the personality function might have adjusted
- // the pointer. There's actually no way for us to fully satisfy
- // the language/ABI contract here: we can't use Exn because it
- // might have the wrong adjustment, but we can't use the by-value
- // pointer because it's off by a level of abstraction.
- //
- // The current solution is to dump the adjusted pointer into an
- // alloca, which breaks language semantics (because changing the
- // pointer doesn't change the exception) but at least works.
- // The better solution would be to filter out non-exact matches
- // and rethrow them, but this is tricky because the rethrow
- // really needs to be catchable by other sites at this landing
- // pad. The best solution is to fix the personality function.
- } else {
- // Pull the pointer for the reference type off.
- llvm::Type *PtrTy =
- cast<llvm::PointerType>(LLVMCatchTy)->getElementType();
-
- // Create the temporary and write the adjusted pointer into it.
- llvm::Value *ExnPtrTmp = CGF.CreateTempAlloca(PtrTy, "exn.byref.tmp");
- llvm::Value *Casted = CGF.Builder.CreateBitCast(AdjustedExn, PtrTy);
- CGF.Builder.CreateStore(Casted, ExnPtrTmp);
-
- // Bind the reference to the temporary.
- AdjustedExn = ExnPtrTmp;
- }
- }
-
- llvm::Value *ExnCast =
- CGF.Builder.CreateBitCast(AdjustedExn, LLVMCatchTy, "exn.byref");
- CGF.Builder.CreateStore(ExnCast, ParamAddr);
- return;
- }
-
- // Scalars and complexes.
- TypeEvaluationKind TEK = CGF.getEvaluationKind(CatchType);
- if (TEK != TEK_Aggregate) {
- llvm::Value *AdjustedExn = CallBeginCatch(CGF, Exn, false);
-
- // If the catch type is a pointer type, __cxa_begin_catch returns
- // the pointer by value.
- if (CatchType->hasPointerRepresentation()) {
- llvm::Value *CastExn =
- CGF.Builder.CreateBitCast(AdjustedExn, LLVMCatchTy, "exn.casted");
-
- switch (CatchType.getQualifiers().getObjCLifetime()) {
- case Qualifiers::OCL_Strong:
- CastExn = CGF.EmitARCRetainNonBlock(CastExn);
- // fallthrough
-
- case Qualifiers::OCL_None:
- case Qualifiers::OCL_ExplicitNone:
- case Qualifiers::OCL_Autoreleasing:
- CGF.Builder.CreateStore(CastExn, ParamAddr);
- return;
-
- case Qualifiers::OCL_Weak:
- CGF.EmitARCInitWeak(ParamAddr, CastExn);
- return;
- }
- llvm_unreachable("bad ownership qualifier!");
- }
-
- // Otherwise, it returns a pointer into the exception object.
-
- llvm::Type *PtrTy = LLVMCatchTy->getPointerTo(0); // addrspace 0 ok
- llvm::Value *Cast = CGF.Builder.CreateBitCast(AdjustedExn, PtrTy);
-
- LValue srcLV = CGF.MakeNaturalAlignAddrLValue(Cast, CatchType);
- LValue destLV = CGF.MakeAddrLValue(ParamAddr, CatchType,
- CGF.getContext().getDeclAlign(&CatchParam));
- switch (TEK) {
- case TEK_Complex:
- CGF.EmitStoreOfComplex(CGF.EmitLoadOfComplex(srcLV, Loc), destLV,
- /*init*/ true);
- return;
- case TEK_Scalar: {
- llvm::Value *ExnLoad = CGF.EmitLoadOfScalar(srcLV, Loc);
- CGF.EmitStoreOfScalar(ExnLoad, destLV, /*init*/ true);
- return;
- }
- case TEK_Aggregate:
- llvm_unreachable("evaluation kind filtered out!");
- }
- llvm_unreachable("bad evaluation kind");
- }
-
- assert(isa<RecordType>(CatchType) && "unexpected catch type!");
-
- llvm::Type *PtrTy = LLVMCatchTy->getPointerTo(0); // addrspace 0 ok
-
- // Check for a copy expression. If we don't have a copy expression,
- // that means a trivial copy is okay.
- const Expr *copyExpr = CatchParam.getInit();
- if (!copyExpr) {
- llvm::Value *rawAdjustedExn = CallBeginCatch(CGF, Exn, true);
- llvm::Value *adjustedExn = CGF.Builder.CreateBitCast(rawAdjustedExn, PtrTy);
- CGF.EmitAggregateCopy(ParamAddr, adjustedExn, CatchType);
- return;
- }
-
- // We have to call __cxa_get_exception_ptr to get the adjusted
- // pointer before copying.
- llvm::CallInst *rawAdjustedExn =
- CGF.EmitNounwindRuntimeCall(getGetExceptionPtrFn(CGF.CGM), Exn);
-
- // Cast that to the appropriate type.
- llvm::Value *adjustedExn = CGF.Builder.CreateBitCast(rawAdjustedExn, PtrTy);
-
- // The copy expression is defined in terms of an OpaqueValueExpr.
- // Find it and map it to the adjusted expression.
- CodeGenFunction::OpaqueValueMapping
- opaque(CGF, OpaqueValueExpr::findInCopyConstruct(copyExpr),
- CGF.MakeAddrLValue(adjustedExn, CatchParam.getType()));
-
- // Call the copy ctor in a terminate scope.
- CGF.EHStack.pushTerminate();
-
- // Perform the copy construction.
- CharUnits Alignment = CGF.getContext().getDeclAlign(&CatchParam);
- CGF.EmitAggExpr(copyExpr,
- AggValueSlot::forAddr(ParamAddr, Alignment, Qualifiers(),
- AggValueSlot::IsNotDestructed,
- AggValueSlot::DoesNotNeedGCBarriers,
- AggValueSlot::IsNotAliased));
-
- // Leave the terminate scope.
- CGF.EHStack.popTerminate();
-
- // Undo the opaque value mapping.
- opaque.pop();
-
- // Finally we can call __cxa_begin_catch.
- CallBeginCatch(CGF, Exn, true);
-}
-
-/// Begins a catch statement by initializing the catch variable and
-/// calling __cxa_begin_catch.
-static void BeginCatch(CodeGenFunction &CGF, const CXXCatchStmt *S) {
- // We have to be very careful with the ordering of cleanups here:
- // C++ [except.throw]p4:
- // The destruction [of the exception temporary] occurs
- // immediately after the destruction of the object declared in
- // the exception-declaration in the handler.
- //
- // So the precise ordering is:
- // 1. Construct catch variable.
- // 2. __cxa_begin_catch
- // 3. Enter __cxa_end_catch cleanup
- // 4. Enter dtor cleanup
- //
- // We do this by using a slightly abnormal initialization process.
- // Delegation sequence:
- // - ExitCXXTryStmt opens a RunCleanupsScope
- // - EmitAutoVarAlloca creates the variable and debug info
- // - InitCatchParam initializes the variable from the exception
- // - CallBeginCatch calls __cxa_begin_catch
- // - CallBeginCatch enters the __cxa_end_catch cleanup
- // - EmitAutoVarCleanups enters the variable destructor cleanup
- // - EmitCXXTryStmt emits the code for the catch body
- // - EmitCXXTryStmt close the RunCleanupsScope
-
- VarDecl *CatchParam = S->getExceptionDecl();
- if (!CatchParam) {
- llvm::Value *Exn = CGF.getExceptionFromSlot();
- CallBeginCatch(CGF, Exn, true);
- return;
- }
-
- // Emit the local.
- CodeGenFunction::AutoVarEmission var = CGF.EmitAutoVarAlloca(*CatchParam);
- InitCatchParam(CGF, *CatchParam, var.getObjectAddress(CGF), S->getLocStart());
- CGF.EmitAutoVarCleanups(var);
-}
-
/// Emit the structure of the dispatch block for the given catch scope.
/// It is an invariant that the dispatch block already exists.
static void emitCatchDispatchBlock(CodeGenFunction &CGF,
@@ -1252,11 +955,10 @@ void CodeGenFunction::ExitCXXTryStmt(const CXXTryStmt &S, bool IsFnTryBlock) {
RunCleanupsScope CatchScope(*this);
// Initialize the catch variable and set up the cleanups.
- BeginCatch(*this, C);
+ CGM.getCXXABI().emitBeginCatch(*this, C);
// Emit the PGO counter increment.
- RegionCounter CatchCnt = getPGORegionCounter(C);
- CatchCnt.beginRegion(Builder);
+ incrementProfileCounter(C);
// Perform the body of the catch.
EmitStmt(C->getHandlerBlock());
@@ -1284,9 +986,8 @@ void CodeGenFunction::ExitCXXTryStmt(const CXXTryStmt &S, bool IsFnTryBlock) {
Builder.CreateBr(ContBB);
}
- RegionCounter ContCnt = getPGORegionCounter(&S);
EmitBlock(ContBB);
- ContCnt.beginRegion(Builder);
+ incrementProfileCounter(&S);
}
namespace {
@@ -1480,68 +1181,6 @@ void CodeGenFunction::FinallyInfo::exit(CodeGenFunction &CGF) {
CGF.PopCleanupBlock();
}
-/// In a terminate landing pad, should we use __clang__call_terminate
-/// or just a naked call to std::terminate?
-///
-/// __clang_call_terminate calls __cxa_begin_catch, which then allows
-/// std::terminate to usefully report something about the
-/// violating exception.
-static bool useClangCallTerminate(CodeGenModule &CGM) {
- // Only do this for Itanium-family ABIs in C++ mode.
- return (CGM.getLangOpts().CPlusPlus &&
- CGM.getTarget().getCXXABI().isItaniumFamily());
-}
-
-/// Get or define the following function:
-/// void @__clang_call_terminate(i8* %exn) nounwind noreturn
-/// This code is used only in C++.
-static llvm::Constant *getClangCallTerminateFn(CodeGenModule &CGM) {
- llvm::FunctionType *fnTy =
- llvm::FunctionType::get(CGM.VoidTy, CGM.Int8PtrTy, /*IsVarArgs=*/false);
- llvm::Constant *fnRef =
- CGM.CreateRuntimeFunction(fnTy, "__clang_call_terminate");
-
- llvm::Function *fn = dyn_cast<llvm::Function>(fnRef);
- if (fn && fn->empty()) {
- fn->setDoesNotThrow();
- fn->setDoesNotReturn();
-
- // What we really want is to massively penalize inlining without
- // forbidding it completely. The difference between that and
- // 'noinline' is negligible.
- fn->addFnAttr(llvm::Attribute::NoInline);
-
- // Allow this function to be shared across translation units, but
- // we don't want it to turn into an exported symbol.
- fn->setLinkage(llvm::Function::LinkOnceODRLinkage);
- fn->setVisibility(llvm::Function::HiddenVisibility);
-
- // Set up the function.
- llvm::BasicBlock *entry =
- llvm::BasicBlock::Create(CGM.getLLVMContext(), "", fn);
- CGBuilderTy builder(entry);
-
- // Pull the exception pointer out of the parameter list.
- llvm::Value *exn = &*fn->arg_begin();
-
- // Call __cxa_begin_catch(exn).
- llvm::CallInst *catchCall = builder.CreateCall(getBeginCatchFn(CGM), exn);
- catchCall->setDoesNotThrow();
- catchCall->setCallingConv(CGM.getRuntimeCC());
-
- // Call std::terminate().
- llvm::CallInst *termCall = builder.CreateCall(getTerminateFn(CGM));
- termCall->setDoesNotThrow();
- termCall->setDoesNotReturn();
- termCall->setCallingConv(CGM.getRuntimeCC());
-
- // std::terminate cannot return.
- builder.CreateUnreachable();
- }
-
- return fnRef;
-}
-
llvm::BasicBlock *CodeGenFunction::getTerminateLandingPad() {
if (TerminateLandingPad)
return TerminateLandingPad;
@@ -1553,20 +1192,17 @@ llvm::BasicBlock *CodeGenFunction::getTerminateLandingPad() {
Builder.SetInsertPoint(TerminateLandingPad);
// Tell the backend that this is a landing pad.
- const EHPersonality &Personality = EHPersonality::get(CGM);
+ const EHPersonality &Personality = EHPersonality::get(*this);
llvm::LandingPadInst *LPadInst =
Builder.CreateLandingPad(llvm::StructType::get(Int8PtrTy, Int32Ty, nullptr),
getOpaquePersonalityFn(CGM, Personality), 0);
LPadInst->addClause(getCatchAllValue(*this));
- llvm::CallInst *terminateCall;
- if (useClangCallTerminate(CGM)) {
- // Extract out the exception pointer.
- llvm::Value *exn = Builder.CreateExtractValue(LPadInst, 0);
- terminateCall = EmitNounwindRuntimeCall(getClangCallTerminateFn(CGM), exn);
- } else {
- terminateCall = EmitNounwindRuntimeCall(getTerminateFn(CGM));
- }
+ llvm::Value *Exn = 0;
+ if (getLangOpts().CPlusPlus)
+ Exn = Builder.CreateExtractValue(LPadInst, 0);
+ llvm::CallInst *terminateCall =
+ CGM.getCXXABI().emitTerminateForUnexpectedException(*this, Exn);
terminateCall->setDoesNotReturn();
Builder.CreateUnreachable();
@@ -1586,14 +1222,11 @@ llvm::BasicBlock *CodeGenFunction::getTerminateHandler() {
// end of the function by FinishFunction.
TerminateHandler = createBasicBlock("terminate.handler");
Builder.SetInsertPoint(TerminateHandler);
- llvm::CallInst *terminateCall;
- if (useClangCallTerminate(CGM)) {
- // Load the exception pointer.
- llvm::Value *exn = getExceptionFromSlot();
- terminateCall = EmitNounwindRuntimeCall(getClangCallTerminateFn(CGM), exn);
- } else {
- terminateCall = EmitNounwindRuntimeCall(getTerminateFn(CGM));
- }
+ llvm::Value *Exn = 0;
+ if (getLangOpts().CPlusPlus)
+ Exn = getExceptionFromSlot();
+ llvm::CallInst *terminateCall =
+ CGM.getCXXABI().emitTerminateForUnexpectedException(*this, Exn);
terminateCall->setDoesNotReturn();
Builder.CreateUnreachable();
@@ -1612,15 +1245,14 @@ llvm::BasicBlock *CodeGenFunction::getEHResumeBlock(bool isCleanup) {
EHResumeBlock = createBasicBlock("eh.resume");
Builder.SetInsertPoint(EHResumeBlock);
- const EHPersonality &Personality = EHPersonality::get(CGM);
+ const EHPersonality &Personality = EHPersonality::get(*this);
// This can always be a call because we necessarily didn't find
// anything on the EH stack which needs our help.
const char *RethrowName = Personality.CatchallRethrowFn;
if (RethrowName != nullptr && !isCleanup) {
EmitRuntimeCall(getCatchallRethrowFn(CGM, RethrowName),
- getExceptionFromSlot())
- ->setDoesNotReturn();
+ getExceptionFromSlot())->setDoesNotReturn();
Builder.CreateUnreachable();
Builder.restoreIP(SavedIP);
return EHResumeBlock;
@@ -1642,9 +1274,433 @@ llvm::BasicBlock *CodeGenFunction::getEHResumeBlock(bool isCleanup) {
}
void CodeGenFunction::EmitSEHTryStmt(const SEHTryStmt &S) {
- CGM.ErrorUnsupported(&S, "SEH __try");
+ // FIXME: Implement SEH on other architectures.
+ const llvm::Triple &T = CGM.getTarget().getTriple();
+ if (T.getArch() != llvm::Triple::x86_64 ||
+ !T.isKnownWindowsMSVCEnvironment()) {
+ ErrorUnsupported(&S, "__try statement");
+ return;
+ }
+
+ EnterSEHTryStmt(S);
+ {
+ JumpDest TryExit = getJumpDestInCurrentScope("__try.__leave");
+
+ SEHTryEpilogueStack.push_back(&TryExit);
+ EmitStmt(S.getTryBlock());
+ SEHTryEpilogueStack.pop_back();
+
+ if (!TryExit.getBlock()->use_empty())
+ EmitBlock(TryExit.getBlock(), /*IsFinished=*/true);
+ else
+ delete TryExit.getBlock();
+ }
+ ExitSEHTryStmt(S);
+}
+
+namespace {
+struct PerformSEHFinally : EHScopeStack::Cleanup {
+ llvm::Function *OutlinedFinally;
+ PerformSEHFinally(llvm::Function *OutlinedFinally)
+ : OutlinedFinally(OutlinedFinally) {}
+
+ void Emit(CodeGenFunction &CGF, Flags F) override {
+ ASTContext &Context = CGF.getContext();
+ QualType ArgTys[2] = {Context.UnsignedCharTy, Context.VoidPtrTy};
+ FunctionProtoType::ExtProtoInfo EPI;
+ const auto *FTP = cast<FunctionType>(
+ Context.getFunctionType(Context.VoidTy, ArgTys, EPI));
+
+ CallArgList Args;
+ llvm::Value *IsForEH =
+ llvm::ConstantInt::get(CGF.ConvertType(ArgTys[0]), F.isForEHCleanup());
+ Args.add(RValue::get(IsForEH), ArgTys[0]);
+
+ CodeGenModule &CGM = CGF.CGM;
+ llvm::Value *Zero = llvm::ConstantInt::get(CGM.Int32Ty, 0);
+ llvm::Value *FrameAddr = CGM.getIntrinsic(llvm::Intrinsic::frameaddress);
+ llvm::Value *FP = CGF.Builder.CreateCall(FrameAddr, Zero);
+ Args.add(RValue::get(FP), ArgTys[1]);
+
+ const CGFunctionInfo &FnInfo =
+ CGM.getTypes().arrangeFreeFunctionCall(Args, FTP, /*chainCall=*/false);
+ CGF.EmitCall(FnInfo, OutlinedFinally, ReturnValueSlot(), Args);
+ }
+};
+}
+
+namespace {
+/// Find all local variable captures in the statement.
+struct CaptureFinder : ConstStmtVisitor<CaptureFinder> {
+ CodeGenFunction &ParentCGF;
+ const VarDecl *ParentThis;
+ SmallVector<const VarDecl *, 4> Captures;
+ CaptureFinder(CodeGenFunction &ParentCGF, const VarDecl *ParentThis)
+ : ParentCGF(ParentCGF), ParentThis(ParentThis) {}
+
+ void Visit(const Stmt *S) {
+ // See if this is a capture, then recurse.
+ ConstStmtVisitor<CaptureFinder>::Visit(S);
+ for (const Stmt *Child : S->children())
+ if (Child)
+ Visit(Child);
+ }
+
+ void VisitDeclRefExpr(const DeclRefExpr *E) {
+ // If this is already a capture, just make sure we capture 'this'.
+ if (E->refersToEnclosingVariableOrCapture()) {
+ Captures.push_back(ParentThis);
+ return;
+ }
+
+ const auto *D = dyn_cast<VarDecl>(E->getDecl());
+ if (D && D->isLocalVarDeclOrParm() && D->hasLocalStorage())
+ Captures.push_back(D);
+ }
+
+ void VisitCXXThisExpr(const CXXThisExpr *E) {
+ Captures.push_back(ParentThis);
+ }
+};
+}
+
+void CodeGenFunction::EmitCapturedLocals(CodeGenFunction &ParentCGF,
+ const Stmt *OutlinedStmt,
+ llvm::Value *ParentFP) {
+ // Find all captures in the Stmt.
+ CaptureFinder Finder(ParentCGF, ParentCGF.CXXABIThisDecl);
+ Finder.Visit(OutlinedStmt);
+
+ // Typically there are no captures and we can exit early.
+ if (Finder.Captures.empty())
+ return;
+
+ // Prepare the first two arguments to llvm.framerecover.
+ llvm::Function *FrameRecoverFn = llvm::Intrinsic::getDeclaration(
+ &CGM.getModule(), llvm::Intrinsic::framerecover);
+ llvm::Constant *ParentI8Fn =
+ llvm::ConstantExpr::getBitCast(ParentCGF.CurFn, Int8PtrTy);
+
+ // Create llvm.framerecover calls for all captures.
+ for (const VarDecl *VD : Finder.Captures) {
+ if (isa<ImplicitParamDecl>(VD)) {
+ CGM.ErrorUnsupported(VD, "'this' captured by SEH");
+ CXXThisValue = llvm::UndefValue::get(ConvertTypeForMem(VD->getType()));
+ continue;
+ }
+ if (VD->getType()->isVariablyModifiedType()) {
+ CGM.ErrorUnsupported(VD, "VLA captured by SEH");
+ continue;
+ }
+ assert((isa<ImplicitParamDecl>(VD) || VD->isLocalVarDeclOrParm()) &&
+ "captured non-local variable");
+
+ // If this decl hasn't been declared yet, it will be declared in the
+ // OutlinedStmt.
+ auto I = ParentCGF.LocalDeclMap.find(VD);
+ if (I == ParentCGF.LocalDeclMap.end())
+ continue;
+ llvm::Value *ParentVar = I->second;
+
+ llvm::CallInst *RecoverCall = nullptr;
+ CGBuilderTy Builder(AllocaInsertPt);
+ if (auto *ParentAlloca = dyn_cast<llvm::AllocaInst>(ParentVar)) {
+ // Mark the variable escaped if nobody else referenced it and compute the
+ // frameescape index.
+ auto InsertPair =
+ ParentCGF.EscapedLocals.insert(std::make_pair(ParentAlloca, -1));
+ if (InsertPair.second)
+ InsertPair.first->second = ParentCGF.EscapedLocals.size() - 1;
+ int FrameEscapeIdx = InsertPair.first->second;
+ // call i8* @llvm.framerecover(i8* bitcast(@parentFn), i8* %fp, i32 N)
+ RecoverCall = Builder.CreateCall(
+ FrameRecoverFn, {ParentI8Fn, ParentFP,
+ llvm::ConstantInt::get(Int32Ty, FrameEscapeIdx)});
+
+ } else {
+ // If the parent didn't have an alloca, we're doing some nested outlining.
+ // Just clone the existing framerecover call, but tweak the FP argument to
+ // use our FP value. All other arguments are constants.
+ auto *ParentRecover =
+ cast<llvm::IntrinsicInst>(ParentVar->stripPointerCasts());
+ assert(ParentRecover->getIntrinsicID() == llvm::Intrinsic::framerecover &&
+ "expected alloca or framerecover in parent LocalDeclMap");
+ RecoverCall = cast<llvm::CallInst>(ParentRecover->clone());
+ RecoverCall->setArgOperand(1, ParentFP);
+ RecoverCall->insertBefore(AllocaInsertPt);
+ }
+
+ // Bitcast the variable, rename it, and insert it in the local decl map.
+ llvm::Value *ChildVar =
+ Builder.CreateBitCast(RecoverCall, ParentVar->getType());
+ ChildVar->setName(ParentVar->getName());
+ LocalDeclMap[VD] = ChildVar;
+ }
+}
+
+/// Arrange a function prototype that can be called by Windows exception
+/// handling personalities. On Win64, the prototype looks like:
+/// RetTy func(void *EHPtrs, void *ParentFP);
+void CodeGenFunction::startOutlinedSEHHelper(CodeGenFunction &ParentCGF,
+ StringRef Name, QualType RetTy,
+ FunctionArgList &Args,
+ const Stmt *OutlinedStmt) {
+ llvm::Function *ParentFn = ParentCGF.CurFn;
+ const CGFunctionInfo &FnInfo = CGM.getTypes().arrangeFreeFunctionDeclaration(
+ RetTy, Args, FunctionType::ExtInfo(), /*isVariadic=*/false);
+
+ llvm::FunctionType *FnTy = CGM.getTypes().GetFunctionType(FnInfo);
+ llvm::Function *Fn = llvm::Function::Create(
+ FnTy, llvm::GlobalValue::InternalLinkage, Name.str(), &CGM.getModule());
+ // The filter is either in the same comdat as the function, or it's internal.
+ if (llvm::Comdat *C = ParentFn->getComdat()) {
+ Fn->setComdat(C);
+ } else if (ParentFn->hasWeakLinkage() || ParentFn->hasLinkOnceLinkage()) {
+ llvm::Comdat *C = CGM.getModule().getOrInsertComdat(ParentFn->getName());
+ ParentFn->setComdat(C);
+ Fn->setComdat(C);
+ } else {
+ Fn->setLinkage(llvm::GlobalValue::InternalLinkage);
+ }
+
+ IsOutlinedSEHHelper = true;
+
+ StartFunction(GlobalDecl(), RetTy, Fn, FnInfo, Args,
+ OutlinedStmt->getLocStart(), OutlinedStmt->getLocStart());
+
+ CGM.SetLLVMFunctionAttributes(nullptr, FnInfo, CurFn);
+
+ auto AI = Fn->arg_begin();
+ ++AI;
+ EmitCapturedLocals(ParentCGF, OutlinedStmt, &*AI);
+}
+
+/// Create a stub filter function that will ultimately hold the code of the
+/// filter expression. The EH preparation passes in LLVM will outline the code
+/// from the main function body into this stub.
+llvm::Function *
+CodeGenFunction::GenerateSEHFilterFunction(CodeGenFunction &ParentCGF,
+ const SEHExceptStmt &Except) {
+ const Expr *FilterExpr = Except.getFilterExpr();
+ SourceLocation StartLoc = FilterExpr->getLocStart();
+
+ SEHPointersDecl = ImplicitParamDecl::Create(
+ getContext(), nullptr, StartLoc,
+ &getContext().Idents.get("exception_pointers"), getContext().VoidPtrTy);
+ FunctionArgList Args;
+ Args.push_back(SEHPointersDecl);
+ Args.push_back(ImplicitParamDecl::Create(
+ getContext(), nullptr, StartLoc,
+ &getContext().Idents.get("frame_pointer"), getContext().VoidPtrTy));
+
+ // Get the mangled function name.
+ SmallString<128> Name;
+ {
+ llvm::raw_svector_ostream OS(Name);
+ const Decl *ParentCodeDecl = ParentCGF.CurCodeDecl;
+ const NamedDecl *Parent = dyn_cast_or_null<NamedDecl>(ParentCodeDecl);
+ assert(Parent && "FIXME: handle unnamed decls (lambdas, blocks) with SEH");
+ CGM.getCXXABI().getMangleContext().mangleSEHFilterExpression(Parent, OS);
+ }
+
+ startOutlinedSEHHelper(ParentCGF, Name, getContext().LongTy, Args,
+ FilterExpr);
+
+ // Mark finally block calls as nounwind and noinline to make LLVM's job a
+ // little easier.
+ // FIXME: Remove these restrictions in the future.
+ CurFn->addFnAttr(llvm::Attribute::NoUnwind);
+ CurFn->addFnAttr(llvm::Attribute::NoInline);
+
+ EmitSEHExceptionCodeSave();
+
+ // Emit the original filter expression, convert to i32, and return.
+ llvm::Value *R = EmitScalarExpr(FilterExpr);
+ R = Builder.CreateIntCast(R, ConvertType(getContext().LongTy),
+ FilterExpr->getType()->isSignedIntegerType());
+ Builder.CreateStore(R, ReturnValue);
+
+ FinishFunction(FilterExpr->getLocEnd());
+
+ return CurFn;
+}
+
+llvm::Function *
+CodeGenFunction::GenerateSEHFinallyFunction(CodeGenFunction &ParentCGF,
+ const SEHFinallyStmt &Finally) {
+ const Stmt *FinallyBlock = Finally.getBlock();
+ SourceLocation StartLoc = FinallyBlock->getLocStart();
+
+ FunctionArgList Args;
+ Args.push_back(ImplicitParamDecl::Create(
+ getContext(), nullptr, StartLoc,
+ &getContext().Idents.get("abnormal_termination"),
+ getContext().UnsignedCharTy));
+ Args.push_back(ImplicitParamDecl::Create(
+ getContext(), nullptr, StartLoc,
+ &getContext().Idents.get("frame_pointer"), getContext().VoidPtrTy));
+
+ // Get the mangled function name.
+ SmallString<128> Name;
+ {
+ llvm::raw_svector_ostream OS(Name);
+ const Decl *ParentCodeDecl = ParentCGF.CurCodeDecl;
+ const NamedDecl *Parent = dyn_cast_or_null<NamedDecl>(ParentCodeDecl);
+ assert(Parent && "FIXME: handle unnamed decls (lambdas, blocks) with SEH");
+ CGM.getCXXABI().getMangleContext().mangleSEHFinallyBlock(Parent, OS);
+ }
+
+ startOutlinedSEHHelper(ParentCGF, Name, getContext().VoidTy, Args,
+ FinallyBlock);
+
+ // Emit the original filter expression, convert to i32, and return.
+ EmitStmt(FinallyBlock);
+
+ FinishFunction(FinallyBlock->getLocEnd());
+
+ return CurFn;
+}
+
+void CodeGenFunction::EmitSEHExceptionCodeSave() {
+ // Save the exception code in the exception slot to unify exception access in
+ // the filter function and the landing pad.
+ // struct EXCEPTION_POINTERS {
+ // EXCEPTION_RECORD *ExceptionRecord;
+ // CONTEXT *ContextRecord;
+ // };
+ // void *exn.slot =
+ // (void *)(uintptr_t)exception_pointers->ExceptionRecord->ExceptionCode;
+ llvm::Value *Ptrs = Builder.CreateLoad(GetAddrOfLocalVar(SEHPointersDecl));
+ llvm::Type *RecordTy = CGM.Int32Ty->getPointerTo();
+ llvm::Type *PtrsTy = llvm::StructType::get(RecordTy, CGM.VoidPtrTy, nullptr);
+ Ptrs = Builder.CreateBitCast(Ptrs, PtrsTy->getPointerTo());
+ llvm::Value *Rec = Builder.CreateStructGEP(PtrsTy, Ptrs, 0);
+ Rec = Builder.CreateLoad(Rec);
+ llvm::Value *Code = Builder.CreateLoad(Rec);
+ Code = Builder.CreateZExt(Code, CGM.IntPtrTy);
+ // FIXME: Change landing pads to produce {i32, i32} and make the exception
+ // slot an i32.
+ Code = Builder.CreateIntToPtr(Code, CGM.VoidPtrTy);
+ Builder.CreateStore(Code, getExceptionSlot());
+}
+
+llvm::Value *CodeGenFunction::EmitSEHExceptionInfo() {
+ // Sema should diagnose calling this builtin outside of a filter context, but
+ // don't crash if we screw up.
+ if (!SEHPointersDecl)
+ return llvm::UndefValue::get(Int8PtrTy);
+ return Builder.CreateLoad(GetAddrOfLocalVar(SEHPointersDecl));
+}
+
+llvm::Value *CodeGenFunction::EmitSEHExceptionCode() {
+ // If we're in a landing pad or filter function, the exception slot contains
+ // the code.
+ assert(ExceptionSlot);
+ llvm::Value *Code =
+ Builder.CreatePtrToInt(getExceptionFromSlot(), CGM.IntPtrTy);
+ return Builder.CreateTrunc(Code, CGM.Int32Ty);
+}
+
+llvm::Value *CodeGenFunction::EmitSEHAbnormalTermination() {
+ // Abnormal termination is just the first parameter to the outlined finally
+ // helper.
+ auto AI = CurFn->arg_begin();
+ return Builder.CreateZExt(&*AI, Int32Ty);
+}
+
+void CodeGenFunction::EnterSEHTryStmt(const SEHTryStmt &S) {
+ CodeGenFunction HelperCGF(CGM, /*suppressNewContext=*/true);
+ if (const SEHFinallyStmt *Finally = S.getFinallyHandler()) {
+ // Push a cleanup for __finally blocks.
+ llvm::Function *FinallyFunc =
+ HelperCGF.GenerateSEHFinallyFunction(*this, *Finally);
+ EHStack.pushCleanup<PerformSEHFinally>(NormalAndEHCleanup, FinallyFunc);
+ return;
+ }
+
+ // Otherwise, we must have an __except block.
+ const SEHExceptStmt *Except = S.getExceptHandler();
+ assert(Except);
+ EHCatchScope *CatchScope = EHStack.pushCatch(1);
+
+ // If the filter is known to evaluate to 1, then we can use the clause "catch
+ // i8* null".
+ llvm::Constant *C =
+ CGM.EmitConstantExpr(Except->getFilterExpr(), getContext().IntTy, this);
+ if (C && C->isOneValue()) {
+ CatchScope->setCatchAllHandler(0, createBasicBlock("__except"));
+ return;
+ }
+
+ // In general, we have to emit an outlined filter function. Use the function
+ // in place of the RTTI typeinfo global that C++ EH uses.
+ llvm::Function *FilterFunc =
+ HelperCGF.GenerateSEHFilterFunction(*this, *Except);
+ llvm::Constant *OpaqueFunc =
+ llvm::ConstantExpr::getBitCast(FilterFunc, Int8PtrTy);
+ CatchScope->setHandler(0, OpaqueFunc, createBasicBlock("__except"));
+}
+
+void CodeGenFunction::ExitSEHTryStmt(const SEHTryStmt &S) {
+ // Just pop the cleanup if it's a __finally block.
+ if (S.getFinallyHandler()) {
+ PopCleanupBlock();
+ return;
+ }
+
+ // Otherwise, we must have an __except block.
+ const SEHExceptStmt *Except = S.getExceptHandler();
+ assert(Except && "__try must have __finally xor __except");
+ EHCatchScope &CatchScope = cast<EHCatchScope>(*EHStack.begin());
+
+ // Don't emit the __except block if the __try block lacked invokes.
+ // TODO: Model unwind edges from instructions, either with iload / istore or
+ // a try body function.
+ if (!CatchScope.hasEHBranches()) {
+ CatchScope.clearHandlerBlocks();
+ EHStack.popCatch();
+ return;
+ }
+
+ // The fall-through block.
+ llvm::BasicBlock *ContBB = createBasicBlock("__try.cont");
+
+ // We just emitted the body of the __try; jump to the continue block.
+ if (HaveInsertPoint())
+ Builder.CreateBr(ContBB);
+
+ // Check if our filter function returned true.
+ emitCatchDispatchBlock(*this, CatchScope);
+
+ // Grab the block before we pop the handler.
+ llvm::BasicBlock *ExceptBB = CatchScope.getHandler(0).Block;
+ EHStack.popCatch();
+
+ EmitBlockAfterUses(ExceptBB);
+
+ // Emit the __except body.
+ EmitStmt(Except->getBlock());
+
+ if (HaveInsertPoint())
+ Builder.CreateBr(ContBB);
+
+ EmitBlock(ContBB);
}
void CodeGenFunction::EmitSEHLeaveStmt(const SEHLeaveStmt &S) {
- CGM.ErrorUnsupported(&S, "SEH __leave");
+ // If this code is reachable then emit a stop point (if generating
+ // debug info). We have to do this ourselves because we are on the
+ // "simple" statement path.
+ if (HaveInsertPoint())
+ EmitStopPoint(&S);
+
+ // This must be a __leave from a __finally block, which we warn on and is UB.
+ // Just emit unreachable.
+ if (!isSEHTryScope()) {
+ Builder.CreateUnreachable();
+ Builder.ClearInsertionPoint();
+ return;
+ }
+
+ EmitBranchThroughCleanup(*SEHTryEpilogueStack.back());
}
diff --git a/lib/CodeGen/CGExpr.cpp b/lib/CodeGen/CGExpr.cpp
index ce7679c836e4..1ed45a33d0fd 100644
--- a/lib/CodeGen/CGExpr.cpp
+++ b/lib/CodeGen/CGExpr.cpp
@@ -31,6 +31,7 @@
#include "llvm/IR/LLVMContext.h"
#include "llvm/IR/MDBuilder.h"
#include "llvm/Support/ConvertUTF.h"
+#include "llvm/Support/MathExtras.h"
using namespace clang;
using namespace CodeGen;
@@ -300,9 +301,26 @@ createReferenceTemporary(CodeGenFunction &CGF,
const MaterializeTemporaryExpr *M, const Expr *Inner) {
switch (M->getStorageDuration()) {
case SD_FullExpression:
- case SD_Automatic:
- return CGF.CreateMemTemp(Inner->getType(), "ref.tmp");
-
+ case SD_Automatic: {
+ // If we have a constant temporary array or record try to promote it into a
+ // constant global under the same rules a normal constant would've been
+ // promoted. This is easier on the optimizer and generally emits fewer
+ // instructions.
+ QualType Ty = Inner->getType();
+ if (CGF.CGM.getCodeGenOpts().MergeAllConstants &&
+ (Ty->isArrayType() || Ty->isRecordType()) &&
+ CGF.CGM.isTypeConstant(Ty, true))
+ if (llvm::Constant *Init = CGF.CGM.EmitConstantExpr(Inner, Ty, &CGF)) {
+ auto *GV = new llvm::GlobalVariable(
+ CGF.CGM.getModule(), Init->getType(), /*isConstant=*/true,
+ llvm::GlobalValue::PrivateLinkage, Init, ".ref.tmp");
+ GV->setAlignment(
+ CGF.getContext().getTypeAlignInChars(Ty).getQuantity());
+ // FIXME: Should we put the new global into a COMDAT?
+ return GV;
+ }
+ return CGF.CreateMemTemp(Ty, "ref.tmp");
+ }
case SD_Thread:
case SD_Static:
return CGF.CGM.GetAddrOfGlobalTemporary(M, Inner);
@@ -324,14 +342,15 @@ EmitMaterializeTemporaryExpr(const MaterializeTemporaryExpr *M) {
M->getType().getObjCLifetime() != Qualifiers::OCL_None &&
M->getType().getObjCLifetime() != Qualifiers::OCL_ExplicitNone) {
llvm::Value *Object = createReferenceTemporary(*this, M, E);
- LValue RefTempDst = MakeAddrLValue(Object, M->getType());
-
if (auto *Var = dyn_cast<llvm::GlobalVariable>(Object)) {
+ Object = llvm::ConstantExpr::getBitCast(
+ Var, ConvertTypeForMem(E->getType())->getPointerTo());
// We should not have emitted the initializer for this temporary as a
// constant.
assert(!Var->hasInitializer());
Var->setInitializer(CGM.EmitNullConstant(E->getType()));
}
+ LValue RefTempDst = MakeAddrLValue(Object, M->getType());
switch (getEvaluationKind(E->getType())) {
default: llvm_unreachable("expected scalar or aggregate expression");
@@ -370,8 +389,11 @@ EmitMaterializeTemporaryExpr(const MaterializeTemporaryExpr *M) {
// Create and initialize the reference temporary.
llvm::Value *Object = createReferenceTemporary(*this, M, E);
if (auto *Var = dyn_cast<llvm::GlobalVariable>(Object)) {
- // If the temporary is a global and has a constant initializer, we may
- // have already initialized it.
+ Object = llvm::ConstantExpr::getBitCast(
+ Var, ConvertTypeForMem(E->getType())->getPointerTo());
+ // If the temporary is a global and has a constant initializer or is a
+ // constant temporary that we promoted to a global, we may have already
+ // initialized it.
if (!Var->hasInitializer()) {
Var->setInitializer(CGM.EmitNullConstant(E->getType()));
EmitAnyExprToMem(E, Object, Qualifiers(), /*IsInit*/true);
@@ -478,7 +500,7 @@ void CodeGenFunction::EmitTypeCheck(TypeCheckKind TCK, SourceLocation Loc,
SanitizerScope SanScope(this);
- SmallVector<std::pair<llvm::Value *, SanitizerKind>, 3> Checks;
+ SmallVector<std::pair<llvm::Value *, SanitizerMask>, 3> Checks;
llvm::BasicBlock *Done = nullptr;
bool AllowNullPointers = TCK == TCK_DowncastPointer || TCK == TCK_Upcast ||
@@ -513,7 +535,7 @@ void CodeGenFunction::EmitTypeCheck(TypeCheckKind TCK, SourceLocation Loc,
llvm::Value *Min = Builder.getFalse();
llvm::Value *CastAddr = Builder.CreateBitCast(Address, Int8PtrTy);
llvm::Value *LargeEnough =
- Builder.CreateICmpUGE(Builder.CreateCall2(F, CastAddr, Min),
+ Builder.CreateICmpUGE(Builder.CreateCall(F, {CastAddr, Min}),
llvm::ConstantInt::get(IntPtrTy, Size));
Checks.push_back(std::make_pair(LargeEnough, SanitizerKind::ObjectSize));
}
@@ -807,6 +829,7 @@ LValue CodeGenFunction::EmitCheckedLValue(const Expr *E, TypeCheckKind TCK) {
/// length type, this is not possible.
///
LValue CodeGenFunction::EmitLValue(const Expr *E) {
+ ApplyDebugLocation DL(*this, E);
switch (E->getStmtClass()) {
default: return EmitUnsupportedLValue(E, "l-value expression");
@@ -819,10 +842,14 @@ LValue CodeGenFunction::EmitLValue(const Expr *E) {
return EmitObjCIsaExpr(cast<ObjCIsaExpr>(E));
case Expr::BinaryOperatorClass:
return EmitBinaryOperatorLValue(cast<BinaryOperator>(E));
- case Expr::CompoundAssignOperatorClass:
- if (!E->getType()->isAnyComplexType())
+ case Expr::CompoundAssignOperatorClass: {
+ QualType Ty = E->getType();
+ if (const AtomicType *AT = Ty->getAs<AtomicType>())
+ Ty = AT->getValueType();
+ if (!Ty->isAnyComplexType())
return EmitCompoundAssignmentLValue(cast<CompoundAssignOperator>(E));
return EmitComplexCompoundAssignmentLValue(cast<CompoundAssignOperator>(E));
+ }
case Expr::CallExprClass:
case Expr::CXXMemberCallExprClass:
case Expr::CXXOperatorCallExprClass:
@@ -1135,7 +1162,7 @@ llvm::Value *CodeGenFunction::EmitLoadOfScalar(llvm::Value *Addr, bool Volatile,
}
// Atomic operations have to be done on integral types.
- if (Ty->isAtomicType()) {
+ if (Ty->isAtomicType() || typeIsSuitableForInlineAtomic(Ty, Volatile)) {
LValue lvalue = LValue::MakeAddr(Addr, Ty,
CharUnits::fromQuantity(Alignment),
getContext(), TBAAInfo);
@@ -1178,7 +1205,7 @@ llvm::Value *CodeGenFunction::EmitLoadOfScalar(llvm::Value *Addr, bool Volatile,
EmitCheckSourceLocation(Loc),
EmitCheckTypeDescriptor(Ty)
};
- SanitizerKind Kind = NeedsEnumCheck ? SanitizerKind::Enum : SanitizerKind::Bool;
+ SanitizerMask Kind = NeedsEnumCheck ? SanitizerKind::Enum : SanitizerKind::Bool;
EmitCheck(std::make_pair(Check, Kind), "load_invalid_value", StaticArgs,
EmitCheckValue(Load));
}
@@ -1254,7 +1281,8 @@ void CodeGenFunction::EmitStoreOfScalar(llvm::Value *Value, llvm::Value *Addr,
Value = EmitToMemory(Value, Ty);
- if (Ty->isAtomicType()) {
+ if (Ty->isAtomicType() ||
+ (!isInit && typeIsSuitableForInlineAtomic(Ty, Volatile))) {
EmitAtomicStore(RValue::get(Value),
LValue::MakeAddr(Addr, Ty,
CharUnits::fromQuantity(Alignment),
@@ -1692,8 +1720,8 @@ void CodeGenFunction::EmitStoreThroughGlobalRegLValue(RValue Src, LValue Dst) {
llvm::Value *Value = Src.getScalarVal();
if (OrigTy->isPointerTy())
Value = Builder.CreatePtrToInt(Value, Ty);
- Builder.CreateCall2(F, llvm::MetadataAsValue::get(Ty->getContext(), RegName),
- Value);
+ Builder.CreateCall(
+ F, {llvm::MetadataAsValue::get(Ty->getContext(), RegName), Value});
}
// setObjCGCLValueClass - sets class of the lvalue for the purpose of
@@ -1807,7 +1835,7 @@ EmitBitCastOfLValueToProperType(CodeGenFunction &CGF,
static LValue EmitThreadPrivateVarDeclLValue(
CodeGenFunction &CGF, const VarDecl *VD, QualType T, llvm::Value *V,
llvm::Type *RealVarTy, CharUnits Alignment, SourceLocation Loc) {
- V = CGF.CGM.getOpenMPRuntime().getOMPAddrOfThreadPrivate(CGF, VD, V, Loc);
+ V = CGF.CGM.getOpenMPRuntime().getAddrOfThreadPrivate(CGF, VD, V, Loc);
V = EmitBitCastOfLValueToProperType(CGF, V, RealVarTy);
return CGF.MakeAddrLValue(V, T, Alignment);
}
@@ -2050,9 +2078,8 @@ LValue CodeGenFunction::EmitUnaryOpLValue(const UnaryOperator *E) {
assert(E->getSubExpr()->getType()->isAnyComplexType());
unsigned Idx = E->getOpcode() == UO_Imag;
- return MakeAddrLValue(Builder.CreateStructGEP(LV.getAddress(),
- Idx, "idx"),
- ExprTy);
+ return MakeAddrLValue(
+ Builder.CreateStructGEP(nullptr, LV.getAddress(), Idx, "idx"), ExprTy);
}
case UO_PreInc:
case UO_PreDec: {
@@ -2217,7 +2244,8 @@ enum class CheckRecoverableKind {
};
}
-static CheckRecoverableKind getRecoverableKind(SanitizerKind Kind) {
+static CheckRecoverableKind getRecoverableKind(SanitizerMask Kind) {
+ assert(llvm::countPopulation(Kind) == 1);
switch (Kind) {
case SanitizerKind::Vptr:
return CheckRecoverableKind::AlwaysRecoverable;
@@ -2264,7 +2292,7 @@ static void emitCheckHandlerCall(CodeGenFunction &CGF,
}
void CodeGenFunction::EmitCheck(
- ArrayRef<std::pair<llvm::Value *, SanitizerKind>> Checked,
+ ArrayRef<std::pair<llvm::Value *, SanitizerMask>> Checked,
StringRef CheckName, ArrayRef<llvm::Constant *> StaticArgs,
ArrayRef<llvm::Value *> DynamicArgs) {
assert(IsSanitizerScope);
@@ -2376,7 +2404,7 @@ void CodeGenFunction::EmitTrapCheck(llvm::Value *Checked) {
Builder.CreateCondBr(Checked, Cont, TrapBB);
EmitBlock(TrapBB);
llvm::Value *F = CGM.getIntrinsic(llvm::Intrinsic::trap);
- llvm::CallInst *TrapCall = Builder.CreateCall(F);
+ llvm::CallInst *TrapCall = Builder.CreateCall(F, {});
TrapCall->setDoesNotReturn();
TrapCall->setDoesNotThrow();
Builder.CreateUnreachable();
@@ -2648,7 +2676,7 @@ LValue CodeGenFunction::EmitLValueForField(LValue base,
unsigned Idx = RL.getLLVMFieldNo(field);
if (Idx != 0)
// For structs, we GEP to the field that the record layout suggests.
- Addr = Builder.CreateStructGEP(Addr, Idx, field->getName());
+ Addr = Builder.CreateStructGEP(nullptr, Addr, Idx, field->getName());
// Get the access type.
llvm::Type *PtrTy = llvm::Type::getIntNPtrTy(
getLLVMContext(), Info.StorageSize,
@@ -2683,7 +2711,7 @@ LValue CodeGenFunction::EmitLValueForField(LValue base,
} else {
// For structs, we GEP to the field that the record layout suggests.
unsigned idx = CGM.getTypes().getCGRecordLayout(rec).getLLVMFieldNo(field);
- addr = Builder.CreateStructGEP(addr, idx, field->getName());
+ addr = Builder.CreateStructGEP(nullptr, addr, idx, field->getName());
// If this is a reference field, load the reference right now.
if (const ReferenceType *refType = type->getAs<ReferenceType>()) {
@@ -2762,7 +2790,7 @@ CodeGenFunction::EmitLValueForFieldInitialization(LValue Base,
const CGRecordLayout &RL =
CGM.getTypes().getCGRecordLayout(Field->getParent());
unsigned idx = RL.getLLVMFieldNo(Field);
- llvm::Value *V = Builder.CreateStructGEP(Base.getAddress(), idx);
+ llvm::Value *V = Builder.CreateStructGEP(nullptr, Base.getAddress(), idx);
assert(!FieldType.getObjCGCAttr() && "fields cannot have GC attrs");
// Make sure that the address is pointing to the right type. This is critical
@@ -2834,7 +2862,6 @@ EmitConditionalOperatorLValue(const AbstractConditionalOperator *expr) {
}
OpaqueValueMapping binding(*this, expr);
- RegionCounter Cnt = getPGORegionCounter(expr);
const Expr *condExpr = expr->getCond();
bool CondExprBool;
@@ -2845,7 +2872,7 @@ EmitConditionalOperatorLValue(const AbstractConditionalOperator *expr) {
if (!ContainsLabel(dead)) {
// If the true case is live, we need to track its region.
if (CondExprBool)
- Cnt.beginRegion(Builder);
+ incrementProfileCounter(expr);
return EmitLValue(live);
}
}
@@ -2855,11 +2882,11 @@ EmitConditionalOperatorLValue(const AbstractConditionalOperator *expr) {
llvm::BasicBlock *contBlock = createBasicBlock("cond.end");
ConditionalEvaluation eval(*this);
- EmitBranchOnBoolExpr(condExpr, lhsBlock, rhsBlock, Cnt.getCount());
+ EmitBranchOnBoolExpr(condExpr, lhsBlock, rhsBlock, getProfileCount(expr));
// Any temporaries created here are conditional.
EmitBlock(lhsBlock);
- Cnt.beginRegion(Builder);
+ incrementProfileCounter(expr);
eval.begin(*this);
Optional<LValue> lhs =
EmitLValueOrThrowExpression(*this, expr->getTrueExpr());
@@ -3007,6 +3034,9 @@ LValue CodeGenFunction::EmitCastLValue(const CastExpr *E) {
EmitTypeCheck(TCK_DowncastReference, E->getExprLoc(),
Derived, E->getType());
+ if (SanOpts.has(SanitizerKind::CFIDerivedCast))
+ EmitVTablePtrCheckForCast(E->getType(), Derived, /*MayBeNull=*/false);
+
return MakeAddrLValue(Derived, E->getType());
}
case CK_LValueBitCast: {
@@ -3016,6 +3046,10 @@ LValue CodeGenFunction::EmitCastLValue(const CastExpr *E) {
LValue LV = EmitLValue(E->getSubExpr());
llvm::Value *V = Builder.CreateBitCast(LV.getAddress(),
ConvertType(CE->getTypeAsWritten()));
+
+ if (SanOpts.has(SanitizerKind::CFIUnrelatedCast))
+ EmitVTablePtrCheckForCast(E->getType(), V, /*MayBeNull=*/false);
+
return MakeAddrLValue(V, E->getType());
}
case CK_ObjCObjectLValueCast: {
@@ -3059,16 +3093,6 @@ RValue CodeGenFunction::EmitRValueForField(LValue LV,
RValue CodeGenFunction::EmitCallExpr(const CallExpr *E,
ReturnValueSlot ReturnValue) {
- // Force column info to be generated so we can differentiate
- // multiple call sites on the same line in the debug info.
- // FIXME: This is insufficient. Two calls coming from the same macro
- // expansion will still get the same line/column and break debug info. It's
- // possible that LLVM can be fixed to not rely on this uniqueness, at which
- // point this workaround can be removed.
- ApplyDebugLocation DL(*this, E->getLocStart(),
- E->getDirectCallee() &&
- E->getDirectCallee()->isInlineSpecified());
-
// Builtins never have block type.
if (E->getCallee()->getType()->isBlockPointerType())
return EmitBlockCallExpr(E, ReturnValue);
@@ -3202,7 +3226,7 @@ LValue CodeGenFunction::EmitCallExprLValue(const CallExpr *E) {
if (!RV.isScalar())
return MakeAddrLValue(RV.getAggregateAddr(), E->getType());
- assert(E->getCallReturnType()->isReferenceType() &&
+ assert(E->getCallReturnType(getContext())->isReferenceType() &&
"Can't have a scalar return unless the return type is a "
"reference type!");
@@ -3328,16 +3352,6 @@ RValue CodeGenFunction::EmitCall(QualType CalleeType, llvm::Value *Callee,
const auto *FnType =
cast<FunctionType>(cast<PointerType>(CalleeType)->getPointeeType());
- // Force column info to differentiate multiple inlined call sites on
- // the same line, analoguous to EmitCallExpr.
- // FIXME: This is insufficient. Two calls coming from the same macro expansion
- // will still get the same line/column and break debug info. It's possible
- // that LLVM can be fixed to not rely on this uniqueness, at which point this
- // workaround can be removed.
- bool ForceColumnInfo = false;
- if (const FunctionDecl* FD = dyn_cast_or_null<const FunctionDecl>(TargetDecl))
- ForceColumnInfo = FD->isInlineSpecified();
-
if (getLangOpts().CPlusPlus && SanOpts.has(SanitizerKind::Function) &&
(!TargetDecl || !isa<FunctionDecl>(TargetDecl))) {
if (llvm::Constant *PrefixSig =
@@ -3355,7 +3369,7 @@ RValue CodeGenFunction::EmitCall(QualType CalleeType, llvm::Value *Callee,
llvm::Value *CalleePrefixStruct = Builder.CreateBitCast(
Callee, llvm::PointerType::getUnqual(PrefixStructTy));
llvm::Value *CalleeSigPtr =
- Builder.CreateConstGEP2_32(CalleePrefixStruct, 0, 0);
+ Builder.CreateConstGEP2_32(PrefixStructTy, CalleePrefixStruct, 0, 0);
llvm::Value *CalleeSig = Builder.CreateLoad(CalleeSigPtr);
llvm::Value *CalleeSigMatch = Builder.CreateICmpEQ(CalleeSig, PrefixSig);
@@ -3365,7 +3379,7 @@ RValue CodeGenFunction::EmitCall(QualType CalleeType, llvm::Value *Callee,
EmitBlock(TypeCheck);
llvm::Value *CalleeRTTIPtr =
- Builder.CreateConstGEP2_32(CalleePrefixStruct, 0, 1);
+ Builder.CreateConstGEP2_32(PrefixStructTy, CalleePrefixStruct, 0, 1);
llvm::Value *CalleeRTTI = Builder.CreateLoad(CalleeRTTIPtr);
llvm::Value *CalleeRTTIMatch =
Builder.CreateICmpEQ(CalleeRTTI, FTRTTIConst);
@@ -3386,8 +3400,7 @@ RValue CodeGenFunction::EmitCall(QualType CalleeType, llvm::Value *Callee,
Args.add(RValue::get(Builder.CreateBitCast(Chain, CGM.VoidPtrTy)),
CGM.getContext().VoidPtrTy);
EmitCallArgs(Args, dyn_cast<FunctionProtoType>(FnType), E->arg_begin(),
- E->arg_end(), E->getDirectCallee(), /*ParamsToSkip*/ 0,
- ForceColumnInfo);
+ E->arg_end(), E->getDirectCallee(), /*ParamsToSkip*/ 0);
const CGFunctionInfo &FnInfo = CGM.getTypes().arrangeFreeFunctionCall(
Args, FnType, /*isChainCall=*/Chain);
diff --git a/lib/CodeGen/CGExprAgg.cpp b/lib/CodeGen/CGExprAgg.cpp
index 4cf94c033bb4..6fedf0efda9d 100644
--- a/lib/CodeGen/CGExprAgg.cpp
+++ b/lib/CodeGen/CGExprAgg.cpp
@@ -98,6 +98,11 @@ public:
// Visitor Methods
//===--------------------------------------------------------------------===//
+ void Visit(Expr *E) {
+ ApplyDebugLocation DL(CGF, E);
+ StmtVisitor<AggExprEmitter>::Visit(E);
+ }
+
void VisitStmt(Stmt *S) {
CGF.ErrorUnsupported(S, "aggregate expression");
}
@@ -207,7 +212,7 @@ void AggExprEmitter::EmitAggLoadOfLValue(const Expr *E) {
LValue LV = CGF.EmitLValue(E);
// If the type of the l-value is atomic, then do an atomic load.
- if (LV.getType()->isAtomicType()) {
+ if (LV.getType()->isAtomicType() || CGF.LValueIsSuitableForInlineAtomic(LV)) {
CGF.EmitAtomicLoad(LV, E->getExprLoc(), Dest);
return;
}
@@ -579,7 +584,12 @@ void AggExprEmitter::VisitCastExpr(CastExpr *E) {
}
case CK_ToUnion: {
- if (Dest.isIgnored()) break;
+ // Evaluate even if the destination is ignored.
+ if (Dest.isIgnored()) {
+ CGF.EmitAnyExpr(E->getSubExpr(), AggValueSlot::ignored(),
+ /*ignoreResult=*/true);
+ break;
+ }
// GCC union extension
QualType Ty = E->getSubExpr()->getType();
@@ -640,7 +650,7 @@ void AggExprEmitter::VisitCastExpr(CastExpr *E) {
// Build a GEP to refer to the subobject.
llvm::Value *valueAddr =
- CGF.Builder.CreateStructGEP(valueDest.getAddr(), 0);
+ CGF.Builder.CreateStructGEP(nullptr, valueDest.getAddr(), 0);
valueDest = AggValueSlot::forAddr(valueAddr,
valueDest.getAlignment(),
valueDest.getQualifiers(),
@@ -661,7 +671,7 @@ void AggExprEmitter::VisitCastExpr(CastExpr *E) {
CGF.EmitAggExpr(E->getSubExpr(), atomicSlot);
llvm::Value *valueAddr =
- Builder.CreateStructGEP(atomicSlot.getAddr(), 0);
+ Builder.CreateStructGEP(nullptr, atomicSlot.getAddr(), 0);
RValue rvalue = RValue::getAggregate(valueAddr, atomicSlot.isVolatile());
return EmitFinalDestCopy(valueType, rvalue);
}
@@ -736,7 +746,7 @@ void AggExprEmitter::VisitCastExpr(CastExpr *E) {
}
void AggExprEmitter::VisitCallExpr(const CallExpr *E) {
- if (E->getCallReturnType()->isReferenceType()) {
+ if (E->getCallReturnType(CGF.getContext())->isReferenceType()) {
EmitAggLoadOfLValue(E);
return;
}
@@ -860,7 +870,8 @@ void AggExprEmitter::VisitBinAssign(const BinaryOperator *E) {
LValue LHS = CGF.EmitCheckedLValue(E->getLHS(), CodeGenFunction::TCK_Store);
// That copy is an atomic copy if the LHS is atomic.
- if (LHS.getType()->isAtomicType()) {
+ if (LHS.getType()->isAtomicType() ||
+ CGF.LValueIsSuitableForInlineAtomic(LHS)) {
CGF.EmitAtomicStore(Dest.asRValue(), LHS, /*isInit*/ false);
return;
}
@@ -877,7 +888,8 @@ void AggExprEmitter::VisitBinAssign(const BinaryOperator *E) {
// If we have an atomic type, evaluate into the destination and then
// do an atomic copy.
- if (LHS.getType()->isAtomicType()) {
+ if (LHS.getType()->isAtomicType() ||
+ CGF.LValueIsSuitableForInlineAtomic(LHS)) {
EnsureDest(E->getRHS()->getType());
Visit(E->getRHS());
CGF.EmitAtomicStore(Dest.asRValue(), LHS, /*isInit*/ false);
@@ -909,16 +921,16 @@ VisitAbstractConditionalOperator(const AbstractConditionalOperator *E) {
// Bind the common expression if necessary.
CodeGenFunction::OpaqueValueMapping binding(CGF, E);
- RegionCounter Cnt = CGF.getPGORegionCounter(E);
CodeGenFunction::ConditionalEvaluation eval(CGF);
- CGF.EmitBranchOnBoolExpr(E->getCond(), LHSBlock, RHSBlock, Cnt.getCount());
+ CGF.EmitBranchOnBoolExpr(E->getCond(), LHSBlock, RHSBlock,
+ CGF.getProfileCount(E));
// Save whether the destination's lifetime is externally managed.
bool isExternallyDestructed = Dest.isExternallyDestructed();
eval.begin(CGF);
CGF.EmitBlock(LHSBlock);
- Cnt.beginRegion(Builder);
+ CGF.incrementProfileCounter(E);
Visit(E->getTrueExpr());
eval.end(CGF);
@@ -1408,7 +1420,8 @@ void CodeGenFunction::EmitAggregateCopy(llvm::Value *DestPtr,
assert((Record->hasTrivialCopyConstructor() ||
Record->hasTrivialCopyAssignment() ||
Record->hasTrivialMoveConstructor() ||
- Record->hasTrivialMoveAssignment()) &&
+ Record->hasTrivialMoveAssignment() ||
+ Record->isUnion()) &&
"Trying to aggregate-copy a type without a trivial copy/move "
"constructor or assignment operator");
// Ignore empty classes in C++.
@@ -1439,7 +1452,34 @@ void CodeGenFunction::EmitAggregateCopy(llvm::Value *DestPtr,
if (alignment.isZero())
alignment = TypeInfo.second;
- // FIXME: Handle variable sized types.
+ llvm::Value *SizeVal = nullptr;
+ if (TypeInfo.first.isZero()) {
+ // But note that getTypeInfo returns 0 for a VLA.
+ if (auto *VAT = dyn_cast_or_null<VariableArrayType>(
+ getContext().getAsArrayType(Ty))) {
+ QualType BaseEltTy;
+ SizeVal = emitArrayLength(VAT, BaseEltTy, DestPtr);
+ TypeInfo = getContext().getTypeInfoDataSizeInChars(BaseEltTy);
+ std::pair<CharUnits, CharUnits> LastElementTypeInfo;
+ if (!isAssignment)
+ LastElementTypeInfo = getContext().getTypeInfoInChars(BaseEltTy);
+ assert(!TypeInfo.first.isZero());
+ SizeVal = Builder.CreateNUWMul(
+ SizeVal,
+ llvm::ConstantInt::get(SizeTy, TypeInfo.first.getQuantity()));
+ if (!isAssignment) {
+ SizeVal = Builder.CreateNUWSub(
+ SizeVal,
+ llvm::ConstantInt::get(SizeTy, TypeInfo.first.getQuantity()));
+ SizeVal = Builder.CreateNUWAdd(
+ SizeVal, llvm::ConstantInt::get(
+ SizeTy, LastElementTypeInfo.first.getQuantity()));
+ }
+ }
+ }
+ if (!SizeVal) {
+ SizeVal = llvm::ConstantInt::get(SizeTy, TypeInfo.first.getQuantity());
+ }
// FIXME: If we have a volatile struct, the optimizer can remove what might
// appear to be `extra' memory ops:
@@ -1470,9 +1510,6 @@ void CodeGenFunction::EmitAggregateCopy(llvm::Value *DestPtr,
} else if (const RecordType *RecordTy = Ty->getAs<RecordType>()) {
RecordDecl *Record = RecordTy->getDecl();
if (Record->hasObjectMember()) {
- CharUnits size = TypeInfo.first;
- llvm::Type *SizeTy = ConvertType(getContext().getSizeType());
- llvm::Value *SizeVal = llvm::ConstantInt::get(SizeTy, size.getQuantity());
CGM.getObjCRuntime().EmitGCMemmoveCollectable(*this, DestPtr, SrcPtr,
SizeVal);
return;
@@ -1481,10 +1518,6 @@ void CodeGenFunction::EmitAggregateCopy(llvm::Value *DestPtr,
QualType BaseType = getContext().getBaseElementType(Ty);
if (const RecordType *RecordTy = BaseType->getAs<RecordType>()) {
if (RecordTy->getDecl()->hasObjectMember()) {
- CharUnits size = TypeInfo.first;
- llvm::Type *SizeTy = ConvertType(getContext().getSizeType());
- llvm::Value *SizeVal =
- llvm::ConstantInt::get(SizeTy, size.getQuantity());
CGM.getObjCRuntime().EmitGCMemmoveCollectable(*this, DestPtr, SrcPtr,
SizeVal);
return;
@@ -1497,9 +1530,6 @@ void CodeGenFunction::EmitAggregateCopy(llvm::Value *DestPtr,
// the optimizer wishes to expand it in to scalar memory operations.
llvm::MDNode *TBAAStructTag = CGM.getTBAAStructInfo(Ty);
- Builder.CreateMemCpy(DestPtr, SrcPtr,
- llvm::ConstantInt::get(IntPtrTy,
- TypeInfo.first.getQuantity()),
- alignment.getQuantity(), isVolatile,
- /*TBAATag=*/nullptr, TBAAStructTag);
+ Builder.CreateMemCpy(DestPtr, SrcPtr, SizeVal, alignment.getQuantity(),
+ isVolatile, /*TBAATag=*/nullptr, TBAAStructTag);
}
diff --git a/lib/CodeGen/CGExprCXX.cpp b/lib/CodeGen/CGExprCXX.cpp
index 6d63b3ae9cf8..13dfbb38816f 100644
--- a/lib/CodeGen/CGExprCXX.cpp
+++ b/lib/CodeGen/CGExprCXX.cpp
@@ -173,7 +173,7 @@ RValue CodeGenFunction::EmitCXXMemberOrOperatorMemberCallExpr(
This = EmitLValue(Base).getAddress();
- if (MD->isTrivial()) {
+ if (MD->isTrivial() || (MD->isDefaulted() && MD->getParent()->isUnion())) {
if (isa<CXXDestructorDecl>(MD)) return RValue::get(nullptr);
if (isa<CXXConstructorDecl>(MD) &&
cast<CXXConstructorDecl>(MD)->isDefaultConstructor())
@@ -256,6 +256,12 @@ RValue CodeGenFunction::EmitCXXMemberOrOperatorMemberCallExpr(
} else if (UseVirtualCall) {
Callee = CGM.getCXXABI().getVirtualFunctionPointer(*this, MD, This, Ty);
} else {
+ if (SanOpts.has(SanitizerKind::CFINVCall) &&
+ MD->getParent()->isDynamicClass()) {
+ llvm::Value *VTable = GetVTablePtr(This, Int8PtrTy);
+ EmitVTablePtrCheckForCall(MD, VTable);
+ }
+
if (getLangOpts().AppleKext && MD->isVirtual() && HasQualifier)
Callee = BuildAppleKextVirtualCall(MD, Qualifier, Ty);
else if (!DevirtualizedMethod)
@@ -684,7 +690,7 @@ static llvm::Value *EmitCXXNewAllocSize(CodeGenFunction &CGF,
llvm::Value *tsmV =
llvm::ConstantInt::get(CGF.SizeTy, typeSizeMultiplier);
llvm::Value *result =
- CGF.Builder.CreateCall2(umul_with_overflow, size, tsmV);
+ CGF.Builder.CreateCall(umul_with_overflow, {size, tsmV});
llvm::Value *overflowed = CGF.Builder.CreateExtractValue(result, 1);
if (hasOverflow)
@@ -723,7 +729,7 @@ static llvm::Value *EmitCXXNewAllocSize(CodeGenFunction &CGF,
llvm::Value *cookieSizeV = llvm::ConstantInt::get(CGF.SizeTy, cookieSize);
llvm::Value *result =
- CGF.Builder.CreateCall2(uadd_with_overflow, size, cookieSizeV);
+ CGF.Builder.CreateCall(uadd_with_overflow, {size, cookieSizeV});
llvm::Value *overflowed = CGF.Builder.CreateExtractValue(result, 1);
if (hasOverflow)
@@ -778,12 +784,10 @@ static void StoreAnyExprIntoOneUnit(CodeGenFunction &CGF, const Expr *Init,
llvm_unreachable("bad evaluation kind");
}
-void
-CodeGenFunction::EmitNewArrayInitializer(const CXXNewExpr *E,
- QualType ElementType,
- llvm::Value *BeginPtr,
- llvm::Value *NumElements,
- llvm::Value *AllocSizeWithoutCookie) {
+void CodeGenFunction::EmitNewArrayInitializer(
+ const CXXNewExpr *E, QualType ElementType, llvm::Type *ElementTy,
+ llvm::Value *BeginPtr, llvm::Value *NumElements,
+ llvm::Value *AllocSizeWithoutCookie) {
// If we have a type with trivial initialization and no initializer,
// there's nothing to do.
if (!E->hasInitializer())
@@ -809,7 +813,8 @@ CodeGenFunction::EmitNewArrayInitializer(const CXXNewExpr *E,
if (const ConstantArrayType *CAT = dyn_cast_or_null<ConstantArrayType>(
AllocType->getAsArrayTypeUnsafe())) {
unsigned AS = CurPtr->getType()->getPointerAddressSpace();
- llvm::Type *AllocPtrTy = ConvertTypeForMem(AllocType)->getPointerTo(AS);
+ ElementTy = ConvertTypeForMem(AllocType);
+ llvm::Type *AllocPtrTy = ElementTy->getPointerTo(AS);
CurPtr = Builder.CreateBitCast(CurPtr, AllocPtrTy);
InitListElements *= getContext().getConstantArrayElementCount(CAT);
}
@@ -839,7 +844,8 @@ CodeGenFunction::EmitNewArrayInitializer(const CXXNewExpr *E,
// initialization loops.
StoreAnyExprIntoOneUnit(*this, ILE->getInit(i),
ILE->getInit(i)->getType(), CurPtr);
- CurPtr = Builder.CreateConstInBoundsGEP1_32(CurPtr, 1, "array.exp.next");
+ CurPtr = Builder.CreateConstInBoundsGEP1_32(ElementTy, CurPtr, 1,
+ "array.exp.next");
}
// The remaining elements are filled with the array filler expression.
@@ -1000,7 +1006,7 @@ CodeGenFunction::EmitNewArrayInitializer(const CXXNewExpr *E,
// Advance to the next element by adjusting the pointer type as necessary.
llvm::Value *NextPtr =
- Builder.CreateConstInBoundsGEP1_32(CurPtr, 1, "array.next");
+ Builder.CreateConstInBoundsGEP1_32(ElementTy, CurPtr, 1, "array.next");
// Check whether we've gotten to the end of the array and, if so,
// exit the loop.
@@ -1012,13 +1018,12 @@ CodeGenFunction::EmitNewArrayInitializer(const CXXNewExpr *E,
}
static void EmitNewInitializer(CodeGenFunction &CGF, const CXXNewExpr *E,
- QualType ElementType,
- llvm::Value *NewPtr,
- llvm::Value *NumElements,
+ QualType ElementType, llvm::Type *ElementTy,
+ llvm::Value *NewPtr, llvm::Value *NumElements,
llvm::Value *AllocSizeWithoutCookie) {
- ApplyDebugLocation DL(CGF, E->getStartLoc());
+ ApplyDebugLocation DL(CGF, E);
if (E->isArray())
- CGF.EmitNewArrayInitializer(E, ElementType, NewPtr, NumElements,
+ CGF.EmitNewArrayInitializer(E, ElementType, ElementTy, NewPtr, NumElements,
AllocSizeWithoutCookie);
else if (const Expr *Init = E->getInitializer())
StoreAnyExprIntoOneUnit(CGF, Init, E->getAllocatedType(), NewPtr);
@@ -1279,10 +1284,9 @@ llvm::Value *CodeGenFunction::EmitCXXNewExpr(const CXXNewExpr *E) {
// Emit a null check on the allocation result if the allocation
// function is allowed to return null (because it has a non-throwing
- // exception spec; for this part, we inline
- // CXXNewExpr::shouldNullCheckAllocation()) and we have an
+ // exception spec or is the reserved placement new) and we have an
// interesting initializer.
- bool nullCheck = allocatorType->isNothrow(getContext()) &&
+ bool nullCheck = E->shouldNullCheckAllocation(getContext()) &&
(!allocType.isPODType(getContext()) || E->hasInitializer());
llvm::BasicBlock *nullCheckBB = nullptr;
@@ -1327,11 +1331,11 @@ llvm::Value *CodeGenFunction::EmitCXXNewExpr(const CXXNewExpr *E) {
E, allocType);
}
- llvm::Type *elementPtrTy
- = ConvertTypeForMem(allocType)->getPointerTo(AS);
+ llvm::Type *elementTy = ConvertTypeForMem(allocType);
+ llvm::Type *elementPtrTy = elementTy->getPointerTo(AS);
llvm::Value *result = Builder.CreateBitCast(allocation, elementPtrTy);
- EmitNewInitializer(*this, E, allocType, result, numElements,
+ EmitNewInitializer(*this, E, allocType, elementTy, result, numElements,
allocSizeWithoutCookie);
if (E->isArray()) {
// NewPtr is a pointer to the base element type. If we're
diff --git a/lib/CodeGen/CGExprComplex.cpp b/lib/CodeGen/CGExprComplex.cpp
index 1580bbe6a294..27d1c689966b 100644
--- a/lib/CodeGen/CGExprComplex.cpp
+++ b/lib/CodeGen/CGExprComplex.cpp
@@ -95,6 +95,7 @@ public:
//===--------------------------------------------------------------------===//
ComplexPairTy Visit(Expr *E) {
+ ApplyDebugLocation DL(CGF, E);
return StmtVisitor<ComplexExprEmitter, ComplexPairTy>::Visit(E);
}
@@ -316,14 +317,14 @@ ComplexPairTy ComplexExprEmitter::EmitLoadOfLValue(LValue lvalue,
llvm::Value *Real=nullptr, *Imag=nullptr;
if (!IgnoreReal || isVolatile) {
- llvm::Value *RealP = Builder.CreateStructGEP(SrcPtr, 0,
+ llvm::Value *RealP = Builder.CreateStructGEP(nullptr, SrcPtr, 0,
SrcPtr->getName() + ".realp");
Real = Builder.CreateAlignedLoad(RealP, AlignR, isVolatile,
SrcPtr->getName() + ".real");
}
if (!IgnoreImag || isVolatile) {
- llvm::Value *ImagP = Builder.CreateStructGEP(SrcPtr, 1,
+ llvm::Value *ImagP = Builder.CreateStructGEP(nullptr, SrcPtr, 1,
SrcPtr->getName() + ".imagp");
Imag = Builder.CreateAlignedLoad(ImagP, AlignI, isVolatile,
SrcPtr->getName() + ".imag");
@@ -335,12 +336,13 @@ ComplexPairTy ComplexExprEmitter::EmitLoadOfLValue(LValue lvalue,
/// specified value pointer.
void ComplexExprEmitter::EmitStoreOfComplex(ComplexPairTy Val, LValue lvalue,
bool isInit) {
- if (lvalue.getType()->isAtomicType())
+ if (lvalue.getType()->isAtomicType() ||
+ (!isInit && CGF.LValueIsSuitableForInlineAtomic(lvalue)))
return CGF.EmitAtomicStore(RValue::getComplex(Val), lvalue, isInit);
llvm::Value *Ptr = lvalue.getAddress();
- llvm::Value *RealPtr = Builder.CreateStructGEP(Ptr, 0, "real");
- llvm::Value *ImagPtr = Builder.CreateStructGEP(Ptr, 1, "imag");
+ llvm::Value *RealPtr = Builder.CreateStructGEP(nullptr, Ptr, 0, "real");
+ llvm::Value *ImagPtr = Builder.CreateStructGEP(nullptr, Ptr, 1, "imag");
unsigned AlignR = lvalue.getAlignment().getQuantity();
ASTContext &C = CGF.getContext();
QualType ComplexTy = lvalue.getType();
@@ -375,7 +377,7 @@ VisitImaginaryLiteral(const ImaginaryLiteral *IL) {
ComplexPairTy ComplexExprEmitter::VisitCallExpr(const CallExpr *E) {
- if (E->getCallReturnType()->isReferenceType())
+ if (E->getCallReturnType(CGF.getContext())->isReferenceType())
return EmitLoadOfLValue(E);
return CGF.EmitCallExpr(E).getComplexVal();
@@ -818,6 +820,8 @@ EmitCompoundAssignLValue(const CompoundAssignOperator *E,
TestAndClearIgnoreReal();
TestAndClearIgnoreImag();
QualType LHSTy = E->getLHS()->getType();
+ if (const AtomicType *AT = LHSTy->getAs<AtomicType>())
+ LHSTy = AT->getValueType();
BinOpInfo OpInfo;
@@ -945,13 +949,14 @@ VisitAbstractConditionalOperator(const AbstractConditionalOperator *E) {
// Bind the common expression if necessary.
CodeGenFunction::OpaqueValueMapping binding(CGF, E);
- RegionCounter Cnt = CGF.getPGORegionCounter(E);
+
CodeGenFunction::ConditionalEvaluation eval(CGF);
- CGF.EmitBranchOnBoolExpr(E->getCond(), LHSBlock, RHSBlock, Cnt.getCount());
+ CGF.EmitBranchOnBoolExpr(E->getCond(), LHSBlock, RHSBlock,
+ CGF.getProfileCount(E));
eval.begin(CGF);
CGF.EmitBlock(LHSBlock);
- Cnt.beginRegion(Builder);
+ CGF.incrementProfileCounter(E);
ComplexPairTy LHS = Visit(E->getTrueExpr());
LHSBlock = Builder.GetInsertBlock();
CGF.EmitBranch(ContBlock);
@@ -1033,7 +1038,7 @@ ComplexPairTy CodeGenFunction::EmitComplexExpr(const Expr *E, bool IgnoreReal,
"Invalid complex expression to emit");
return ComplexExprEmitter(*this, IgnoreReal, IgnoreImag)
- .Visit(const_cast<Expr*>(E));
+ .Visit(const_cast<Expr *>(E));
}
void CodeGenFunction::EmitComplexExprIntoLValue(const Expr *E, LValue dest,
@@ -1085,8 +1090,8 @@ EmitComplexCompoundAssignmentLValue(const CompoundAssignOperator *E) {
}
LValue CodeGenFunction::
-EmitScalarCompooundAssignWithComplex(const CompoundAssignOperator *E,
- llvm::Value *&Result) {
+EmitScalarCompoundAssignWithComplex(const CompoundAssignOperator *E,
+ llvm::Value *&Result) {
CompoundFunc Op = getComplexOp(E->getOpcode());
RValue Val;
LValue Ret = ComplexExprEmitter(*this).EmitCompoundAssignLValue(E, Op, Val);
diff --git a/lib/CodeGen/CGExprConstant.cpp b/lib/CodeGen/CGExprConstant.cpp
index 54f7eee6791e..73ca0cc1c3d5 100644
--- a/lib/CodeGen/CGExprConstant.cpp
+++ b/lib/CodeGen/CGExprConstant.cpp
@@ -383,14 +383,19 @@ bool ConstStructBuilder::Build(InitListExpr *ILE) {
if (!EltInit)
return false;
-
+
if (!Field->isBitField()) {
// Handle non-bitfield members.
AppendField(*Field, Layout.getFieldOffset(FieldNo), EltInit);
} else {
// Otherwise we have a bitfield.
- AppendBitField(*Field, Layout.getFieldOffset(FieldNo),
- cast<llvm::ConstantInt>(EltInit));
+ if (auto *CI = dyn_cast<llvm::ConstantInt>(EltInit)) {
+ AppendBitField(*Field, Layout.getFieldOffset(FieldNo), CI);
+ } else {
+ // We are trying to initialize a bitfield with a non-trivial constant,
+ // this must require run-time code.
+ return false;
+ }
}
}
@@ -1110,7 +1115,7 @@ llvm::Constant *CodeGenModule::EmitConstantValue(const APValue &Value,
unsigned AS = C->getType()->getPointerAddressSpace();
llvm::Type *CharPtrTy = Int8Ty->getPointerTo(AS);
llvm::Constant *Casted = llvm::ConstantExpr::getBitCast(C, CharPtrTy);
- Casted = llvm::ConstantExpr::getGetElementPtr(Casted, Offset);
+ Casted = llvm::ConstantExpr::getGetElementPtr(Int8Ty, Casted, Offset);
C = llvm::ConstantExpr::getPointerCast(Casted, C->getType());
}
@@ -1403,10 +1408,6 @@ llvm::Constant *CodeGenModule::EmitNullConstant(QualType T) {
llvm::Constant *Element = EmitNullConstant(ElementTy);
unsigned NumElements = CAT->getSize().getZExtValue();
-
- if (Element->isNullValue())
- return llvm::ConstantAggregateZero::get(ATy);
-
SmallVector<llvm::Constant *, 8> Array(NumElements, Element);
return llvm::ConstantArray::get(ATy, Array);
}
@@ -1416,8 +1417,7 @@ llvm::Constant *CodeGenModule::EmitNullConstant(QualType T) {
return ::EmitNullConstant(*this, RD, /*complete object*/ true);
}
- assert(T->isMemberPointerType() && "Should only see member pointers here!");
- assert(!T->getAs<MemberPointerType>()->getPointeeType()->isFunctionType() &&
+ assert(T->isMemberDataPointerType() &&
"Should only see pointers to data members here!");
return getCXXABI().EmitNullMemberPointer(T->castAs<MemberPointerType>());
diff --git a/lib/CodeGen/CGExprScalar.cpp b/lib/CodeGen/CGExprScalar.cpp
index 3be14c8c3ee6..08c81c012c76 100644
--- a/lib/CodeGen/CGExprScalar.cpp
+++ b/lib/CodeGen/CGExprScalar.cpp
@@ -85,7 +85,7 @@ public:
return CGF.EmitCheckedLValue(E, TCK);
}
- void EmitBinOpCheck(ArrayRef<std::pair<Value *, SanitizerKind>> Checks,
+ void EmitBinOpCheck(ArrayRef<std::pair<Value *, SanitizerMask>> Checks,
const BinOpInfo &Info);
Value *EmitLoadOfLValue(LValue LV, SourceLocation Loc) {
@@ -196,7 +196,7 @@ public:
//===--------------------------------------------------------------------===//
Value *Visit(Expr *E) {
- ApplyDebugLocation DL(CGF, E->getLocStart());
+ ApplyDebugLocation DL(CGF, E);
return StmtVisitor<ScalarExprEmitter, Value*>::Visit(E);
}
@@ -320,7 +320,7 @@ public:
Value *VisitCastExpr(CastExpr *E);
Value *VisitCallExpr(const CallExpr *E) {
- if (E->getCallReturnType()->isReferenceType())
+ if (E->getCallReturnType(CGF.getContext())->isReferenceType())
return EmitLoadOfLValue(E);
Value *V = CGF.EmitCallExpr(E).getScalarVal();
@@ -349,10 +349,9 @@ public:
return EmitScalarPrePostIncDec(E, LV, true, true);
}
- llvm::Value *EmitAddConsiderOverflowBehavior(const UnaryOperator *E,
- llvm::Value *InVal,
- llvm::Value *NextVal,
- bool IsInc);
+ llvm::Value *EmitIncDecConsiderOverflowBehavior(const UnaryOperator *E,
+ llvm::Value *InVal,
+ bool IsInc);
llvm::Value *EmitScalarPrePostIncDec(const UnaryOperator *E, LValue LV,
bool isInc, bool isPre);
@@ -745,23 +744,37 @@ Value *ScalarExprEmitter::EmitScalarConversion(Value *Src, QualType SrcType,
QualType OrigSrcType = SrcType;
llvm::Type *SrcTy = Src->getType();
- // If casting to/from storage-only half FP, use special intrinsics.
- if (SrcType->isHalfType() && !CGF.getContext().getLangOpts().NativeHalfType &&
- !CGF.getContext().getLangOpts().HalfArgsAndReturns) {
- Src = Builder.CreateCall(
- CGF.CGM.getIntrinsic(llvm::Intrinsic::convert_from_fp16,
- CGF.CGM.FloatTy),
- Src);
- SrcType = CGF.getContext().FloatTy;
- SrcTy = CGF.FloatTy;
- }
-
// Handle conversions to bool first, they are special: comparisons against 0.
if (DstType->isBooleanType())
return EmitConversionToBool(Src, SrcType);
llvm::Type *DstTy = ConvertType(DstType);
+ // Cast from half through float if half isn't a native type.
+ if (SrcType->isHalfType() && !CGF.getContext().getLangOpts().NativeHalfType) {
+ // Cast to FP using the intrinsic if the half type itself isn't supported.
+ if (DstTy->isFloatingPointTy()) {
+ if (!CGF.getContext().getLangOpts().HalfArgsAndReturns)
+ return Builder.CreateCall(
+ CGF.CGM.getIntrinsic(llvm::Intrinsic::convert_from_fp16, DstTy),
+ Src);
+ } else {
+ // Cast to other types through float, using either the intrinsic or FPExt,
+ // depending on whether the half type itself is supported
+ // (as opposed to operations on half, available with NativeHalfType).
+ if (!CGF.getContext().getLangOpts().HalfArgsAndReturns) {
+ Src = Builder.CreateCall(
+ CGF.CGM.getIntrinsic(llvm::Intrinsic::convert_from_fp16,
+ CGF.CGM.FloatTy),
+ Src);
+ } else {
+ Src = Builder.CreateFPExt(Src, CGF.CGM.FloatTy, "conv");
+ }
+ SrcType = CGF.getContext().FloatTy;
+ SrcTy = CGF.FloatTy;
+ }
+ }
+
// Ignore conversions like int -> uint.
if (SrcTy == DstTy)
return Src;
@@ -818,10 +831,20 @@ Value *ScalarExprEmitter::EmitScalarConversion(Value *Src, QualType SrcType,
EmitFloatConversionCheck(OrigSrc, OrigSrcType, Src, SrcType, DstType,
DstTy);
- // Cast to half via float
- if (DstType->isHalfType() && !CGF.getContext().getLangOpts().NativeHalfType &&
- !CGF.getContext().getLangOpts().HalfArgsAndReturns)
+ // Cast to half through float if half isn't a native type.
+ if (DstType->isHalfType() && !CGF.getContext().getLangOpts().NativeHalfType) {
+ // Make sure we cast in a single step if from another FP type.
+ if (SrcTy->isFloatingPointTy()) {
+ // Use the intrinsic if the half type itself isn't supported
+ // (as opposed to operations on half, available with NativeHalfType).
+ if (!CGF.getContext().getLangOpts().HalfArgsAndReturns)
+ return Builder.CreateCall(
+ CGF.CGM.getIntrinsic(llvm::Intrinsic::convert_to_fp16, SrcTy), Src);
+ // If the half type is supported, just use an fptrunc.
+ return Builder.CreateFPTrunc(Src, DstTy);
+ }
DstTy = CGF.FloatTy;
+ }
if (isa<llvm::IntegerType>(SrcTy)) {
bool InputSigned = SrcType->isSignedIntegerOrEnumerationType();
@@ -847,10 +870,14 @@ Value *ScalarExprEmitter::EmitScalarConversion(Value *Src, QualType SrcType,
}
if (DstTy != ResTy) {
- assert(ResTy->isIntegerTy(16) && "Only half FP requires extra conversion");
- Res = Builder.CreateCall(
+ if (!CGF.getContext().getLangOpts().HalfArgsAndReturns) {
+ assert(ResTy->isIntegerTy(16) && "Only half FP requires extra conversion");
+ Res = Builder.CreateCall(
CGF.CGM.getIntrinsic(llvm::Intrinsic::convert_to_fp16, CGF.CGM.FloatTy),
Res);
+ } else {
+ Res = Builder.CreateFPTrunc(Res, ResTy, "conv");
+ }
}
return Res;
@@ -889,7 +916,7 @@ Value *ScalarExprEmitter::EmitNullValue(QualType Ty) {
/// operation). The check passes if all values in \p Checks (which are \c i1),
/// are \c true.
void ScalarExprEmitter::EmitBinOpCheck(
- ArrayRef<std::pair<Value *, SanitizerKind>> Checks, const BinOpInfo &Info) {
+ ArrayRef<std::pair<Value *, SanitizerMask>> Checks, const BinOpInfo &Info) {
assert(CGF.IsSanitizerScope);
StringRef CheckName;
SmallVector<llvm::Constant *, 4> StaticData;
@@ -1355,6 +1382,13 @@ Value *ScalarExprEmitter::VisitCastExpr(CastExpr *CE) {
llvm_unreachable("wrong cast for pointers in different address spaces"
"(must be an address space cast)!");
}
+
+ if (CGF.SanOpts.has(SanitizerKind::CFIUnrelatedCast)) {
+ if (auto PT = DestTy->getAs<PointerType>())
+ CGF.EmitVTablePtrCheckForCast(PT->getPointeeType(), Src,
+ /*MayBeNull=*/true);
+ }
+
return Builder.CreateBitCast(Src, DstTy);
}
case CK_AddressSpaceConversion: {
@@ -1384,6 +1418,10 @@ Value *ScalarExprEmitter::VisitCastExpr(CastExpr *CE) {
CGF.EmitTypeCheck(CodeGenFunction::TCK_DowncastPointer, CE->getExprLoc(),
Derived, DestTy->getPointeeType());
+ if (CGF.SanOpts.has(SanitizerKind::CFIDerivedCast))
+ CGF.EmitVTablePtrCheckForCast(DestTy->getPointeeType(), Derived,
+ /*MayBeNull=*/true);
+
return Derived;
}
case CK_UncheckedDerivedToBase:
@@ -1412,13 +1450,13 @@ Value *ScalarExprEmitter::VisitCastExpr(CastExpr *CE) {
// anything here.
if (!E->getType()->isVariableArrayType()) {
assert(isa<llvm::PointerType>(V->getType()) && "Expected pointer");
+ llvm::Type *NewTy = ConvertType(E->getType());
V = CGF.Builder.CreatePointerCast(
- V, ConvertType(E->getType())->getPointerTo(
- V->getType()->getPointerAddressSpace()));
+ V, NewTy->getPointerTo(V->getType()->getPointerAddressSpace()));
assert(isa<llvm::ArrayType>(V->getType()->getPointerElementType()) &&
"Expected pointer to array");
- V = Builder.CreateStructGEP(V, 0, "arraydecay");
+ V = Builder.CreateStructGEP(NewTy, V, 0, "arraydecay");
}
// Make sure the array decay ends up being the right type. This matters if
@@ -1571,26 +1609,32 @@ Value *ScalarExprEmitter::VisitStmtExpr(const StmtExpr *E) {
// Unary Operators
//===----------------------------------------------------------------------===//
-llvm::Value *ScalarExprEmitter::
-EmitAddConsiderOverflowBehavior(const UnaryOperator *E,
- llvm::Value *InVal,
- llvm::Value *NextVal, bool IsInc) {
+static BinOpInfo createBinOpInfoFromIncDec(const UnaryOperator *E,
+ llvm::Value *InVal, bool IsInc) {
+ BinOpInfo BinOp;
+ BinOp.LHS = InVal;
+ BinOp.RHS = llvm::ConstantInt::get(InVal->getType(), 1, false);
+ BinOp.Ty = E->getType();
+ BinOp.Opcode = IsInc ? BO_Add : BO_Sub;
+ BinOp.FPContractable = false;
+ BinOp.E = E;
+ return BinOp;
+}
+
+llvm::Value *ScalarExprEmitter::EmitIncDecConsiderOverflowBehavior(
+ const UnaryOperator *E, llvm::Value *InVal, bool IsInc) {
+ llvm::Value *Amount =
+ llvm::ConstantInt::get(InVal->getType(), IsInc ? 1 : -1, true);
+ StringRef Name = IsInc ? "inc" : "dec";
switch (CGF.getLangOpts().getSignedOverflowBehavior()) {
case LangOptions::SOB_Defined:
- return Builder.CreateAdd(InVal, NextVal, IsInc ? "inc" : "dec");
+ return Builder.CreateAdd(InVal, Amount, Name);
case LangOptions::SOB_Undefined:
if (!CGF.SanOpts.has(SanitizerKind::SignedIntegerOverflow))
- return Builder.CreateNSWAdd(InVal, NextVal, IsInc ? "inc" : "dec");
+ return Builder.CreateNSWAdd(InVal, Amount, Name);
// Fall through.
case LangOptions::SOB_Trapping:
- BinOpInfo BinOp;
- BinOp.LHS = InVal;
- BinOp.RHS = NextVal;
- BinOp.Ty = E->getType();
- BinOp.Opcode = BO_Add;
- BinOp.FPContractable = false;
- BinOp.E = E;
- return EmitOverflowCheckedBinOp(BinOp);
+ return EmitOverflowCheckedBinOp(createBinOpInfoFromIncDec(E, InVal, IsInc));
}
llvm_unreachable("Unknown SignedOverflowBehaviorTy");
}
@@ -1668,27 +1712,20 @@ ScalarExprEmitter::EmitScalarPrePostIncDec(const UnaryOperator *E, LValue LV,
// Most common case by far: integer increment.
} else if (type->isIntegerType()) {
-
- llvm::Value *amt = llvm::ConstantInt::get(value->getType(), amount, true);
-
// Note that signed integer inc/dec with width less than int can't
// overflow because of promotion rules; we're just eliding a few steps here.
bool CanOverflow = value->getType()->getIntegerBitWidth() >=
CGF.IntTy->getIntegerBitWidth();
if (CanOverflow && type->isSignedIntegerOrEnumerationType()) {
- value = EmitAddConsiderOverflowBehavior(E, value, amt, isInc);
+ value = EmitIncDecConsiderOverflowBehavior(E, value, isInc);
} else if (CanOverflow && type->isUnsignedIntegerType() &&
CGF.SanOpts.has(SanitizerKind::UnsignedIntegerOverflow)) {
- BinOpInfo BinOp;
- BinOp.LHS = value;
- BinOp.RHS = llvm::ConstantInt::get(value->getType(), 1, false);
- BinOp.Ty = E->getType();
- BinOp.Opcode = isInc ? BO_Add : BO_Sub;
- BinOp.FPContractable = false;
- BinOp.E = E;
- value = EmitOverflowCheckedBinOp(BinOp);
- } else
+ value =
+ EmitOverflowCheckedBinOp(createBinOpInfoFromIncDec(E, value, isInc));
+ } else {
+ llvm::Value *amt = llvm::ConstantInt::get(value->getType(), amount, true);
value = Builder.CreateAdd(value, amt, isInc ? "inc" : "dec");
+ }
// Next most common: pointer increment.
} else if (const PointerType *ptr = type->getAs<PointerType>()) {
@@ -1742,13 +1779,16 @@ ScalarExprEmitter::EmitScalarPrePostIncDec(const UnaryOperator *E, LValue LV,
// Add the inc/dec to the real part.
llvm::Value *amt;
- if (type->isHalfType() && !CGF.getContext().getLangOpts().NativeHalfType &&
- !CGF.getContext().getLangOpts().HalfArgsAndReturns) {
+ if (type->isHalfType() && !CGF.getContext().getLangOpts().NativeHalfType) {
// Another special case: half FP increment should be done via float
- value = Builder.CreateCall(
- CGF.CGM.getIntrinsic(llvm::Intrinsic::convert_from_fp16,
- CGF.CGM.FloatTy),
- input);
+ if (!CGF.getContext().getLangOpts().HalfArgsAndReturns) {
+ value = Builder.CreateCall(
+ CGF.CGM.getIntrinsic(llvm::Intrinsic::convert_from_fp16,
+ CGF.CGM.FloatTy),
+ input, "incdec.conv");
+ } else {
+ value = Builder.CreateFPExt(input, CGF.CGM.FloatTy, "incdec.conv");
+ }
}
if (value->getType()->isFloatTy())
@@ -1758,20 +1798,29 @@ ScalarExprEmitter::EmitScalarPrePostIncDec(const UnaryOperator *E, LValue LV,
amt = llvm::ConstantFP::get(VMContext,
llvm::APFloat(static_cast<double>(amount)));
else {
+ // Remaining types are either Half or LongDouble. Convert from float.
llvm::APFloat F(static_cast<float>(amount));
bool ignored;
- F.convert(CGF.getTarget().getLongDoubleFormat(),
+ // Don't use getFloatTypeSemantics because Half isn't
+ // necessarily represented using the "half" LLVM type.
+ F.convert(value->getType()->isHalfTy()
+ ? CGF.getTarget().getHalfFormat()
+ : CGF.getTarget().getLongDoubleFormat(),
llvm::APFloat::rmTowardZero, &ignored);
amt = llvm::ConstantFP::get(VMContext, F);
}
value = Builder.CreateFAdd(value, amt, isInc ? "inc" : "dec");
- if (type->isHalfType() && !CGF.getContext().getLangOpts().NativeHalfType &&
- !CGF.getContext().getLangOpts().HalfArgsAndReturns)
- value = Builder.CreateCall(
- CGF.CGM.getIntrinsic(llvm::Intrinsic::convert_to_fp16,
- CGF.CGM.FloatTy),
- value);
+ if (type->isHalfType() && !CGF.getContext().getLangOpts().NativeHalfType) {
+ if (!CGF.getContext().getLangOpts().HalfArgsAndReturns) {
+ value = Builder.CreateCall(
+ CGF.CGM.getIntrinsic(llvm::Intrinsic::convert_to_fp16,
+ CGF.CGM.FloatTy),
+ value, "incdec.conv");
+ } else {
+ value = Builder.CreateFPTrunc(value, input->getType(), "incdec.conv");
+ }
+ }
// Objective-C pointer types.
} else {
@@ -1794,10 +1843,9 @@ ScalarExprEmitter::EmitScalarPrePostIncDec(const UnaryOperator *E, LValue LV,
llvm::BasicBlock *opBB = Builder.GetInsertBlock();
llvm::BasicBlock *contBB = CGF.createBasicBlock("atomic_cont", CGF.CurFn);
auto Pair = CGF.EmitAtomicCompareExchange(
- LV, RValue::get(atomicPHI), RValue::get(CGF.EmitToMemory(value, type)),
- E->getExprLoc());
- llvm::Value *old = Pair.first.getScalarVal();
- llvm::Value *success = Pair.second.getScalarVal();
+ LV, RValue::get(atomicPHI), RValue::get(value), E->getExprLoc());
+ llvm::Value *old = CGF.EmitToMemory(Pair.first.getScalarVal(), type);
+ llvm::Value *success = Pair.second;
atomicPHI->addIncoming(old, opBB);
Builder.CreateCondBr(success, contBB, opBB);
Builder.SetInsertPoint(contBB);
@@ -2056,7 +2104,7 @@ LValue ScalarExprEmitter::EmitCompoundAssignLValue(
BinOpInfo OpInfo;
if (E->getComputationResultType()->isAnyComplexType())
- return CGF.EmitScalarCompooundAssignWithComplex(E, Result);
+ return CGF.EmitScalarCompoundAssignWithComplex(E, Result);
// Emit the RHS first. __block variables need to have the rhs evaluated
// first, plus this should improve codegen a little.
@@ -2138,10 +2186,9 @@ LValue ScalarExprEmitter::EmitCompoundAssignLValue(
llvm::BasicBlock *opBB = Builder.GetInsertBlock();
llvm::BasicBlock *contBB = CGF.createBasicBlock("atomic_cont", CGF.CurFn);
auto Pair = CGF.EmitAtomicCompareExchange(
- LHSLV, RValue::get(atomicPHI),
- RValue::get(CGF.EmitToMemory(Result, LHSTy)), E->getExprLoc());
- llvm::Value *old = Pair.first.getScalarVal();
- llvm::Value *success = Pair.second.getScalarVal();
+ LHSLV, RValue::get(atomicPHI), RValue::get(Result), E->getExprLoc());
+ llvm::Value *old = CGF.EmitToMemory(Pair.first.getScalarVal(), LHSTy);
+ llvm::Value *success = Pair.second;
atomicPHI->addIncoming(old, opBB);
Builder.CreateCondBr(success, contBB, opBB);
Builder.SetInsertPoint(contBB);
@@ -2184,7 +2231,7 @@ Value *ScalarExprEmitter::EmitCompoundAssign(const CompoundAssignOperator *E,
void ScalarExprEmitter::EmitUndefinedBehaviorIntegerDivAndRemCheck(
const BinOpInfo &Ops, llvm::Value *Zero, bool isDiv) {
- SmallVector<std::pair<llvm::Value *, SanitizerKind>, 2> Checks;
+ SmallVector<std::pair<llvm::Value *, SanitizerMask>, 2> Checks;
if (CGF.SanOpts.has(SanitizerKind::IntegerDivideByZero)) {
Checks.push_back(std::make_pair(Builder.CreateICmpNE(Ops.RHS, Zero),
@@ -2296,7 +2343,7 @@ Value *ScalarExprEmitter::EmitOverflowCheckedBinOp(const BinOpInfo &Ops) {
llvm::Function *intrinsic = CGF.CGM.getIntrinsic(IID, opTy);
- Value *resultAndOverflow = Builder.CreateCall2(intrinsic, Ops.LHS, Ops.RHS);
+ Value *resultAndOverflow = Builder.CreateCall(intrinsic, {Ops.LHS, Ops.RHS});
Value *result = Builder.CreateExtractValue(resultAndOverflow, 0);
Value *overflow = Builder.CreateExtractValue(resultAndOverflow, 1);
@@ -2309,7 +2356,7 @@ Value *ScalarExprEmitter::EmitOverflowCheckedBinOp(const BinOpInfo &Ops) {
if (!isSigned || CGF.SanOpts.has(SanitizerKind::SignedIntegerOverflow)) {
CodeGenFunction::SanitizerScope SanScope(&CGF);
llvm::Value *NotOverflow = Builder.CreateNot(overflow);
- SanitizerKind Kind = isSigned ? SanitizerKind::SignedIntegerOverflow
+ SanitizerMask Kind = isSigned ? SanitizerKind::SignedIntegerOverflow
: SanitizerKind::UnsignedIntegerOverflow;
EmitBinOpCheck(std::make_pair(NotOverflow, Kind), Ops);
} else
@@ -2476,10 +2523,9 @@ static Value* buildFMulAdd(llvm::BinaryOperator *MulOp, Value *Addend,
"neg");
}
- Value *FMulAdd =
- Builder.CreateCall3(
+ Value *FMulAdd = Builder.CreateCall(
CGF.CGM.getIntrinsic(llvm::Intrinsic::fmuladd, Addend->getType()),
- MulOp0, MulOp1, Addend);
+ {MulOp0, MulOp1, Addend});
MulOp->eraseFromParent();
return FMulAdd;
@@ -2664,21 +2710,34 @@ Value *ScalarExprEmitter::EmitShl(const BinOpInfo &Ops) {
if (Ops.LHS->getType() != RHS->getType())
RHS = Builder.CreateIntCast(RHS, Ops.LHS->getType(), false, "sh_prom");
- if (CGF.SanOpts.has(SanitizerKind::Shift) && !CGF.getLangOpts().OpenCL &&
- isa<llvm::IntegerType>(Ops.LHS->getType())) {
+ bool SanitizeBase = CGF.SanOpts.has(SanitizerKind::ShiftBase) &&
+ Ops.Ty->hasSignedIntegerRepresentation();
+ bool SanitizeExponent = CGF.SanOpts.has(SanitizerKind::ShiftExponent);
+ // OpenCL 6.3j: shift values are effectively % word size of LHS.
+ if (CGF.getLangOpts().OpenCL)
+ RHS =
+ Builder.CreateAnd(RHS, GetWidthMinusOneValue(Ops.LHS, RHS), "shl.mask");
+ else if ((SanitizeBase || SanitizeExponent) &&
+ isa<llvm::IntegerType>(Ops.LHS->getType())) {
CodeGenFunction::SanitizerScope SanScope(&CGF);
+ SmallVector<std::pair<Value *, SanitizerMask>, 2> Checks;
llvm::Value *WidthMinusOne = GetWidthMinusOneValue(Ops.LHS, RHS);
- llvm::Value *Valid = Builder.CreateICmpULE(RHS, WidthMinusOne);
+ llvm::Value *ValidExponent = Builder.CreateICmpULE(RHS, WidthMinusOne);
- if (Ops.Ty->hasSignedIntegerRepresentation()) {
- llvm::BasicBlock *Orig = Builder.GetInsertBlock();
- llvm::BasicBlock *Cont = CGF.createBasicBlock("cont");
- llvm::BasicBlock *CheckBitsShifted = CGF.createBasicBlock("check");
- Builder.CreateCondBr(Valid, CheckBitsShifted, Cont);
+ if (SanitizeExponent) {
+ Checks.push_back(
+ std::make_pair(ValidExponent, SanitizerKind::ShiftExponent));
+ }
+ if (SanitizeBase) {
// Check whether we are shifting any non-zero bits off the top of the
- // integer.
- CGF.EmitBlock(CheckBitsShifted);
+ // integer. We only emit this check if exponent is valid - otherwise
+ // instructions below will have undefined behavior themselves.
+ llvm::BasicBlock *Orig = Builder.GetInsertBlock();
+ llvm::BasicBlock *Cont = CGF.createBasicBlock("cont");
+ llvm::BasicBlock *CheckShiftBase = CGF.createBasicBlock("check");
+ Builder.CreateCondBr(ValidExponent, CheckShiftBase, Cont);
+ CGF.EmitBlock(CheckShiftBase);
llvm::Value *BitsShiftedOff =
Builder.CreateLShr(Ops.LHS,
Builder.CreateSub(WidthMinusOne, RHS, "shl.zeros",
@@ -2693,19 +2752,17 @@ Value *ScalarExprEmitter::EmitShl(const BinOpInfo &Ops) {
BitsShiftedOff = Builder.CreateLShr(BitsShiftedOff, One);
}
llvm::Value *Zero = llvm::ConstantInt::get(BitsShiftedOff->getType(), 0);
- llvm::Value *SecondCheck = Builder.CreateICmpEQ(BitsShiftedOff, Zero);
+ llvm::Value *ValidBase = Builder.CreateICmpEQ(BitsShiftedOff, Zero);
CGF.EmitBlock(Cont);
- llvm::PHINode *P = Builder.CreatePHI(Valid->getType(), 2);
- P->addIncoming(Valid, Orig);
- P->addIncoming(SecondCheck, CheckBitsShifted);
- Valid = P;
+ llvm::PHINode *BaseCheck = Builder.CreatePHI(ValidBase->getType(), 2);
+ BaseCheck->addIncoming(Builder.getTrue(), Orig);
+ BaseCheck->addIncoming(ValidBase, CheckShiftBase);
+ Checks.push_back(std::make_pair(BaseCheck, SanitizerKind::ShiftBase));
}
- EmitBinOpCheck(std::make_pair(Valid, SanitizerKind::Shift), Ops);
+ assert(!Checks.empty());
+ EmitBinOpCheck(Checks, Ops);
}
- // OpenCL 6.3j: shift values are effectively % word size of LHS.
- if (CGF.getLangOpts().OpenCL)
- RHS = Builder.CreateAnd(RHS, GetWidthMinusOneValue(Ops.LHS, RHS), "shl.mask");
return Builder.CreateShl(Ops.LHS, RHS, "shl");
}
@@ -2717,18 +2774,18 @@ Value *ScalarExprEmitter::EmitShr(const BinOpInfo &Ops) {
if (Ops.LHS->getType() != RHS->getType())
RHS = Builder.CreateIntCast(RHS, Ops.LHS->getType(), false, "sh_prom");
- if (CGF.SanOpts.has(SanitizerKind::Shift) && !CGF.getLangOpts().OpenCL &&
- isa<llvm::IntegerType>(Ops.LHS->getType())) {
+ // OpenCL 6.3j: shift values are effectively % word size of LHS.
+ if (CGF.getLangOpts().OpenCL)
+ RHS =
+ Builder.CreateAnd(RHS, GetWidthMinusOneValue(Ops.LHS, RHS), "shr.mask");
+ else if (CGF.SanOpts.has(SanitizerKind::ShiftExponent) &&
+ isa<llvm::IntegerType>(Ops.LHS->getType())) {
CodeGenFunction::SanitizerScope SanScope(&CGF);
llvm::Value *Valid =
Builder.CreateICmpULE(RHS, GetWidthMinusOneValue(Ops.LHS, RHS));
- EmitBinOpCheck(std::make_pair(Valid, SanitizerKind::Shift), Ops);
+ EmitBinOpCheck(std::make_pair(Valid, SanitizerKind::ShiftExponent), Ops);
}
- // OpenCL 6.3j: shift values are effectively % word size of LHS.
- if (CGF.getLangOpts().OpenCL)
- RHS = Builder.CreateAnd(RHS, GetWidthMinusOneValue(Ops.LHS, RHS), "shr.mask");
-
if (Ops.Ty->hasUnsignedIntegerRepresentation())
return Builder.CreateLShr(Ops.LHS, RHS, "shr");
return Builder.CreateAShr(Ops.LHS, RHS, "shr");
@@ -2846,7 +2903,7 @@ Value *ScalarExprEmitter::EmitCompare(const BinaryOperator *E,unsigned UICmpOpc,
Value *CR6Param = Builder.getInt32(CR6);
llvm::Function *F = CGF.CGM.getIntrinsic(ID);
- Result = Builder.CreateCall3(F, CR6Param, FirstVecArg, SecondVecArg, "");
+ Result = Builder.CreateCall(F, {CR6Param, FirstVecArg, SecondVecArg});
return EmitScalarConversion(Result, CGF.getContext().BoolTy, E->getType());
}
@@ -2975,11 +3032,9 @@ Value *ScalarExprEmitter::VisitBinAssign(const BinaryOperator *E) {
}
Value *ScalarExprEmitter::VisitBinLAnd(const BinaryOperator *E) {
- RegionCounter Cnt = CGF.getPGORegionCounter(E);
-
// Perform vector logical and on comparisons with zero vectors.
if (E->getType()->isVectorType()) {
- Cnt.beginRegion(Builder);
+ CGF.incrementProfileCounter(E);
Value *LHS = Visit(E->getLHS());
Value *RHS = Visit(E->getRHS());
@@ -3002,7 +3057,7 @@ Value *ScalarExprEmitter::VisitBinLAnd(const BinaryOperator *E) {
bool LHSCondVal;
if (CGF.ConstantFoldsToSimpleInteger(E->getLHS(), LHSCondVal)) {
if (LHSCondVal) { // If we have 1 && X, just emit X.
- Cnt.beginRegion(Builder);
+ CGF.incrementProfileCounter(E);
Value *RHSCond = CGF.EvaluateExprAsBool(E->getRHS());
// ZExt result to int or bool.
@@ -3020,7 +3075,8 @@ Value *ScalarExprEmitter::VisitBinLAnd(const BinaryOperator *E) {
CodeGenFunction::ConditionalEvaluation eval(CGF);
// Branch on the LHS first. If it is false, go to the failure (cont) block.
- CGF.EmitBranchOnBoolExpr(E->getLHS(), RHSBlock, ContBlock, Cnt.getCount());
+ CGF.EmitBranchOnBoolExpr(E->getLHS(), RHSBlock, ContBlock,
+ CGF.getProfileCount(E->getRHS()));
// Any edges into the ContBlock are now from an (indeterminate number of)
// edges from this first condition. All of these values will be false. Start
@@ -3033,7 +3089,7 @@ Value *ScalarExprEmitter::VisitBinLAnd(const BinaryOperator *E) {
eval.begin(CGF);
CGF.EmitBlock(RHSBlock);
- Cnt.beginRegion(Builder);
+ CGF.incrementProfileCounter(E);
Value *RHSCond = CGF.EvaluateExprAsBool(E->getRHS());
eval.end(CGF);
@@ -3043,7 +3099,7 @@ Value *ScalarExprEmitter::VisitBinLAnd(const BinaryOperator *E) {
// Emit an unconditional branch from this block to ContBlock.
{
// There is no need to emit line number for unconditional branch.
- ApplyDebugLocation DL(CGF);
+ auto NL = ApplyDebugLocation::CreateEmpty(CGF);
CGF.EmitBlock(ContBlock);
}
// Insert an entry into the phi node for the edge with the value of RHSCond.
@@ -3054,11 +3110,9 @@ Value *ScalarExprEmitter::VisitBinLAnd(const BinaryOperator *E) {
}
Value *ScalarExprEmitter::VisitBinLOr(const BinaryOperator *E) {
- RegionCounter Cnt = CGF.getPGORegionCounter(E);
-
// Perform vector logical or on comparisons with zero vectors.
if (E->getType()->isVectorType()) {
- Cnt.beginRegion(Builder);
+ CGF.incrementProfileCounter(E);
Value *LHS = Visit(E->getLHS());
Value *RHS = Visit(E->getRHS());
@@ -3081,7 +3135,7 @@ Value *ScalarExprEmitter::VisitBinLOr(const BinaryOperator *E) {
bool LHSCondVal;
if (CGF.ConstantFoldsToSimpleInteger(E->getLHS(), LHSCondVal)) {
if (!LHSCondVal) { // If we have 0 || X, just emit X.
- Cnt.beginRegion(Builder);
+ CGF.incrementProfileCounter(E);
Value *RHSCond = CGF.EvaluateExprAsBool(E->getRHS());
// ZExt result to int or bool.
@@ -3100,7 +3154,8 @@ Value *ScalarExprEmitter::VisitBinLOr(const BinaryOperator *E) {
// Branch on the LHS first. If it is true, go to the success (cont) block.
CGF.EmitBranchOnBoolExpr(E->getLHS(), ContBlock, RHSBlock,
- Cnt.getParentCount() - Cnt.getCount());
+ CGF.getCurrentProfileCount() -
+ CGF.getProfileCount(E->getRHS()));
// Any edges into the ContBlock are now from an (indeterminate number of)
// edges from this first condition. All of these values will be true. Start
@@ -3115,7 +3170,7 @@ Value *ScalarExprEmitter::VisitBinLOr(const BinaryOperator *E) {
// Emit the RHS condition as a bool value.
CGF.EmitBlock(RHSBlock);
- Cnt.beginRegion(Builder);
+ CGF.incrementProfileCounter(E);
Value *RHSCond = CGF.EvaluateExprAsBool(E->getRHS());
eval.end(CGF);
@@ -3166,7 +3221,6 @@ VisitAbstractConditionalOperator(const AbstractConditionalOperator *E) {
// Bind the common expression if necessary.
CodeGenFunction::OpaqueValueMapping binding(CGF, E);
- RegionCounter Cnt = CGF.getPGORegionCounter(E);
Expr *condExpr = E->getCond();
Expr *lhsExpr = E->getTrueExpr();
@@ -3182,7 +3236,7 @@ VisitAbstractConditionalOperator(const AbstractConditionalOperator *E) {
// If the dead side doesn't have labels we need, just emit the Live part.
if (!CGF.ContainsLabel(dead)) {
if (CondExprBool)
- Cnt.beginRegion(Builder);
+ CGF.incrementProfileCounter(E);
Value *Result = Visit(live);
// If the live part is a throw expression, it acts like it has a void
@@ -3199,7 +3253,7 @@ VisitAbstractConditionalOperator(const AbstractConditionalOperator *E) {
// the select function.
if (CGF.getLangOpts().OpenCL
&& condExpr->getType()->isVectorType()) {
- Cnt.beginRegion(Builder);
+ CGF.incrementProfileCounter(E);
llvm::Value *CondV = CGF.EmitScalarExpr(condExpr);
llvm::Value *LHS = Visit(lhsExpr);
@@ -3244,7 +3298,7 @@ VisitAbstractConditionalOperator(const AbstractConditionalOperator *E) {
// safe to evaluate the LHS and RHS unconditionally.
if (isCheapEnoughToEvaluateUnconditionally(lhsExpr, CGF) &&
isCheapEnoughToEvaluateUnconditionally(rhsExpr, CGF)) {
- Cnt.beginRegion(Builder);
+ CGF.incrementProfileCounter(E);
llvm::Value *CondV = CGF.EvaluateExprAsBool(condExpr);
llvm::Value *LHS = Visit(lhsExpr);
@@ -3262,10 +3316,11 @@ VisitAbstractConditionalOperator(const AbstractConditionalOperator *E) {
llvm::BasicBlock *ContBlock = CGF.createBasicBlock("cond.end");
CodeGenFunction::ConditionalEvaluation eval(CGF);
- CGF.EmitBranchOnBoolExpr(condExpr, LHSBlock, RHSBlock, Cnt.getCount());
+ CGF.EmitBranchOnBoolExpr(condExpr, LHSBlock, RHSBlock,
+ CGF.getProfileCount(lhsExpr));
CGF.EmitBlock(LHSBlock);
- Cnt.beginRegion(Builder);
+ CGF.incrementProfileCounter(E);
eval.begin(CGF);
Value *LHS = Visit(lhsExpr);
eval.end(CGF);
@@ -3393,14 +3448,8 @@ Value *CodeGenFunction::EmitScalarExpr(const Expr *E, bool IgnoreResultAssign) {
assert(E && hasScalarEvaluationKind(E->getType()) &&
"Invalid scalar expression to emit");
- bool hasDebugInfo = getDebugInfo();
- if (isa<CXXDefaultArgExpr>(E))
- disableDebugInfo();
- Value *V = ScalarExprEmitter(*this, IgnoreResultAssign)
- .Visit(const_cast<Expr*>(E));
- if (isa<CXXDefaultArgExpr>(E) && hasDebugInfo)
- enableDebugInfo();
- return V;
+ return ScalarExprEmitter(*this, IgnoreResultAssign)
+ .Visit(const_cast<Expr *>(E));
}
/// EmitScalarConversion - Emit a conversion from the specified type to the
diff --git a/lib/CodeGen/CGLoopInfo.cpp b/lib/CodeGen/CGLoopInfo.cpp
index 89f43c281590..011ae7e836ae 100644
--- a/lib/CodeGen/CGLoopInfo.cpp
+++ b/lib/CodeGen/CGLoopInfo.cpp
@@ -26,8 +26,8 @@ static MDNode *createMetadata(LLVMContext &Ctx, const LoopAttributes &Attrs) {
SmallVector<Metadata *, 4> Args;
// Reserve operand 0 for loop id self reference.
- MDNode *TempNode = MDNode::getTemporary(Ctx, None);
- Args.push_back(TempNode);
+ auto TempNode = MDNode::getTemporary(Ctx, None);
+ Args.push_back(TempNode.get());
// Setting vectorizer.width
if (Attrs.VectorizerWidth > 0) {
@@ -58,7 +58,6 @@ static MDNode *createMetadata(LLVMContext &Ctx, const LoopAttributes &Attrs) {
// Set the first operand to itself.
MDNode *LoopID = MDNode::get(Ctx, Args);
LoopID->replaceOperandWith(0, LoopID);
- MDNode::deleteTemporary(TempNode);
return LoopID;
}
diff --git a/lib/CodeGen/CGLoopInfo.h b/lib/CodeGen/CGLoopInfo.h
index b1693996507e..aee162118a2f 100644
--- a/lib/CodeGen/CGLoopInfo.h
+++ b/lib/CodeGen/CGLoopInfo.h
@@ -78,8 +78,8 @@ private:
/// This stack can be used to prepare attributes which are applied when a loop
/// is emitted.
class LoopInfoStack {
- LoopInfoStack(const LoopInfoStack &) LLVM_DELETED_FUNCTION;
- void operator=(const LoopInfoStack &) LLVM_DELETED_FUNCTION;
+ LoopInfoStack(const LoopInfoStack &) = delete;
+ void operator=(const LoopInfoStack &) = delete;
public:
LoopInfoStack() {}
diff --git a/lib/CodeGen/CGObjC.cpp b/lib/CodeGen/CGObjC.cpp
index 34c6d94f8817..ef9a92dfc35b 100644
--- a/lib/CodeGen/CGObjC.cpp
+++ b/lib/CodeGen/CGObjC.cpp
@@ -102,8 +102,8 @@ llvm::Value *CodeGenFunction::EmitObjCCollectionLiteral(const Expr *E,
ArrayType::Normal, /*IndexTypeQuals=*/0);
// Allocate the temporary array(s).
- llvm::Value *Objects = CreateMemTemp(ElementArrayType, "objects");
- llvm::Value *Keys = nullptr;
+ llvm::AllocaInst *Objects = CreateMemTemp(ElementArrayType, "objects");
+ llvm::AllocaInst *Keys = nullptr;
if (DLE)
Keys = CreateMemTemp(ElementArrayType, "keys");
@@ -119,10 +119,9 @@ llvm::Value *CodeGenFunction::EmitObjCCollectionLiteral(const Expr *E,
if (ALE) {
// Emit the element and store it to the appropriate array slot.
const Expr *Rhs = ALE->getElement(i);
- LValue LV = LValue::MakeAddr(Builder.CreateStructGEP(Objects, i),
- ElementType,
- Context.getTypeAlignInChars(Rhs->getType()),
- Context);
+ LValue LV = LValue::MakeAddr(
+ Builder.CreateStructGEP(Objects->getAllocatedType(), Objects, i),
+ ElementType, Context.getTypeAlignInChars(Rhs->getType()), Context);
llvm::Value *value = EmitScalarExpr(Rhs);
EmitStoreThroughLValue(RValue::get(value), LV, true);
@@ -132,19 +131,17 @@ llvm::Value *CodeGenFunction::EmitObjCCollectionLiteral(const Expr *E,
} else {
// Emit the key and store it to the appropriate array slot.
const Expr *Key = DLE->getKeyValueElement(i).Key;
- LValue KeyLV = LValue::MakeAddr(Builder.CreateStructGEP(Keys, i),
- ElementType,
- Context.getTypeAlignInChars(Key->getType()),
- Context);
+ LValue KeyLV = LValue::MakeAddr(
+ Builder.CreateStructGEP(Keys->getAllocatedType(), Keys, i),
+ ElementType, Context.getTypeAlignInChars(Key->getType()), Context);
llvm::Value *keyValue = EmitScalarExpr(Key);
EmitStoreThroughLValue(RValue::get(keyValue), KeyLV, /*isInit=*/true);
// Emit the value and store it to the appropriate array slot.
- const Expr *Value = DLE->getKeyValueElement(i).Value;
- LValue ValueLV = LValue::MakeAddr(Builder.CreateStructGEP(Objects, i),
- ElementType,
- Context.getTypeAlignInChars(Value->getType()),
- Context);
+ const Expr *Value = DLE->getKeyValueElement(i).Value;
+ LValue ValueLV = LValue::MakeAddr(
+ Builder.CreateStructGEP(Objects->getAllocatedType(), Objects, i),
+ ElementType, Context.getTypeAlignInChars(Value->getType()), Context);
llvm::Value *valueValue = EmitScalarExpr(Value);
EmitStoreThroughLValue(RValue::get(valueValue), ValueLV, /*isInit=*/true);
if (TrackNeededObjects) {
@@ -472,8 +469,7 @@ void CodeGenFunction::StartObjCMethod(const ObjCMethodDecl *OMD,
args.push_back(OMD->getSelfDecl());
args.push_back(OMD->getCmdDecl());
- for (const auto *PI : OMD->params())
- args.push_back(PI);
+ args.append(OMD->param_begin(), OMD->param_end());
CurGD = OMD;
CurEHLocation = OMD->getLocEnd();
@@ -501,8 +497,7 @@ void CodeGenFunction::GenerateObjCMethod(const ObjCMethodDecl *OMD) {
StartObjCMethod(OMD, OMD->getClassInterface());
PGO.assignRegionCounters(OMD, CurFn);
assert(isa<CompoundStmt>(OMD->getBody()));
- RegionCounter Cnt = getPGORegionCounter(OMD->getBody());
- Cnt.beginRegion(Builder);
+ incrementProfileCounter(OMD->getBody());
EmitCompoundStmtWithoutScope(*cast<CompoundStmt>(OMD->getBody()));
FinishFunction(OMD->getBodyRBrace());
}
@@ -1435,7 +1430,7 @@ void CodeGenFunction::EmitObjCForCollectionStmt(const ObjCForCollectionStmt &S){
// Fast enumeration state.
QualType StateTy = CGM.getObjCFastEnumerationStateType();
- llvm::Value *StatePtr = CreateMemTemp(StateTy, "state.ptr");
+ llvm::AllocaInst *StatePtr = CreateMemTemp(StateTy, "state.ptr");
EmitNullInitialization(StatePtr, StateTy);
// Number of elements in the items array.
@@ -1507,11 +1502,11 @@ void CodeGenFunction::EmitObjCForCollectionStmt(const ObjCForCollectionStmt &S){
// If the limit pointer was zero to begin with, the collection is
// empty; skip all this. Set the branch weight assuming this has the same
// probability of exiting the loop as any other loop exit.
- uint64_t EntryCount = PGO.getCurrentRegionCount();
- RegionCounter Cnt = getPGORegionCounter(&S);
- Builder.CreateCondBr(Builder.CreateICmpEQ(initialBufferLimit, zero, "iszero"),
- EmptyBB, LoopInitBB,
- PGO.createBranchWeights(EntryCount, Cnt.getCount()));
+ uint64_t EntryCount = getCurrentProfileCount();
+ Builder.CreateCondBr(
+ Builder.CreateICmpEQ(initialBufferLimit, zero, "iszero"), EmptyBB,
+ LoopInitBB,
+ createProfileWeights(EntryCount, getProfileCount(S.getBody())));
// Otherwise, initialize the loop.
EmitBlock(LoopInitBB);
@@ -1519,8 +1514,8 @@ void CodeGenFunction::EmitObjCForCollectionStmt(const ObjCForCollectionStmt &S){
// Save the initial mutations value. This is the value at an
// address that was written into the state object by
// countByEnumeratingWithState:objects:count:.
- llvm::Value *StateMutationsPtrPtr =
- Builder.CreateStructGEP(StatePtr, 2, "mutationsptr.ptr");
+ llvm::Value *StateMutationsPtrPtr = Builder.CreateStructGEP(
+ StatePtr->getAllocatedType(), StatePtr, 2, "mutationsptr.ptr");
llvm::Value *StateMutationsPtr = Builder.CreateLoad(StateMutationsPtrPtr,
"mutationsptr");
@@ -1540,7 +1535,7 @@ void CodeGenFunction::EmitObjCForCollectionStmt(const ObjCForCollectionStmt &S){
llvm::PHINode *count = Builder.CreatePHI(UnsignedLongLTy, 3, "forcoll.count");
count->addIncoming(initialBufferLimit, LoopInitBB);
- Cnt.beginRegion(Builder);
+ incrementProfileCounter(&S);
// Check whether the mutations value has changed from where it was
// at start. StateMutationsPtr should actually be invariant between
@@ -1600,8 +1595,8 @@ void CodeGenFunction::EmitObjCForCollectionStmt(const ObjCForCollectionStmt &S){
// Fetch the buffer out of the enumeration state.
// TODO: this pointer should actually be invariant between
// refreshes, which would help us do certain loop optimizations.
- llvm::Value *StateItemsPtr =
- Builder.CreateStructGEP(StatePtr, 1, "stateitems.ptr");
+ llvm::Value *StateItemsPtr = Builder.CreateStructGEP(
+ StatePtr->getAllocatedType(), StatePtr, 1, "stateitems.ptr");
llvm::Value *EnumStateItems =
Builder.CreateLoad(StateItemsPtr, "stateitems");
@@ -1652,9 +1647,9 @@ void CodeGenFunction::EmitObjCForCollectionStmt(const ObjCForCollectionStmt &S){
// Set the branch weights based on the simplifying assumption that this is
// like a while-loop, i.e., ignoring that the false branch fetches more
// elements and then returns to the loop.
- Builder.CreateCondBr(Builder.CreateICmpULT(indexPlusOne, count),
- LoopBodyBB, FetchMoreBB,
- PGO.createBranchWeights(Cnt.getCount(), EntryCount));
+ Builder.CreateCondBr(
+ Builder.CreateICmpULT(indexPlusOne, count), LoopBodyBB, FetchMoreBB,
+ createProfileWeights(getProfileCount(S.getBody()), EntryCount));
index->addIncoming(indexPlusOne, AfterBody.getBlock());
count->addIncoming(count, AfterBody.getBlock());
@@ -1985,7 +1980,8 @@ CodeGenFunction::EmitARCRetainAutoreleasedReturnValue(llvm::Value *value) {
}
// Call the marker asm if we made one, which we do only at -O0.
- if (marker) Builder.CreateCall(marker);
+ if (marker)
+ Builder.CreateCall(marker, {});
return emitARCValueOperation(*this, value,
CGM.getARCEntrypoints().objc_retainAutoreleasedReturnValue,
diff --git a/lib/CodeGen/CGObjCGNU.cpp b/lib/CodeGen/CGObjCGNU.cpp
index c0dc3b8002d8..1580c777ea47 100644
--- a/lib/CodeGen/CGObjCGNU.cpp
+++ b/lib/CodeGen/CGObjCGNU.cpp
@@ -46,54 +46,49 @@ namespace {
/// avoids constructing the type more than once if it's used more than once.
class LazyRuntimeFunction {
CodeGenModule *CGM;
- std::vector<llvm::Type*> ArgTys;
+ llvm::FunctionType *FTy;
const char *FunctionName;
llvm::Constant *Function;
- public:
- /// Constructor leaves this class uninitialized, because it is intended to
- /// be used as a field in another class and not all of the types that are
- /// used as arguments will necessarily be available at construction time.
- LazyRuntimeFunction()
- : CGM(nullptr), FunctionName(nullptr), Function(nullptr) {}
- /// Initialises the lazy function with the name, return type, and the types
- /// of the arguments.
- LLVM_END_WITH_NULL
- void init(CodeGenModule *Mod, const char *name,
- llvm::Type *RetTy, ...) {
- CGM =Mod;
- FunctionName = name;
- Function = nullptr;
- ArgTys.clear();
- va_list Args;
- va_start(Args, RetTy);
- while (llvm::Type *ArgTy = va_arg(Args, llvm::Type*))
- ArgTys.push_back(ArgTy);
- va_end(Args);
- // Push the return type on at the end so we can pop it off easily
- ArgTys.push_back(RetTy);
- }
- /// Overloaded cast operator, allows the class to be implicitly cast to an
- /// LLVM constant.
- operator llvm::Constant*() {
- if (!Function) {
- if (!FunctionName) return nullptr;
- // We put the return type on the end of the vector, so pop it back off
- llvm::Type *RetTy = ArgTys.back();
- ArgTys.pop_back();
- llvm::FunctionType *FTy = llvm::FunctionType::get(RetTy, ArgTys, false);
- Function =
- cast<llvm::Constant>(CGM->CreateRuntimeFunction(FTy, FunctionName));
- // We won't need to use the types again, so we may as well clean up the
- // vector now
- ArgTys.resize(0);
- }
- return Function;
- }
- operator llvm::Function*() {
- return cast<llvm::Function>((llvm::Constant*)*this);
- }
+public:
+ /// Constructor leaves this class uninitialized, because it is intended to
+ /// be used as a field in another class and not all of the types that are
+ /// used as arguments will necessarily be available at construction time.
+ LazyRuntimeFunction()
+ : CGM(nullptr), FunctionName(nullptr), Function(nullptr) {}
+ /// Initialises the lazy function with the name, return type, and the types
+ /// of the arguments.
+ LLVM_END_WITH_NULL
+ void init(CodeGenModule *Mod, const char *name, llvm::Type *RetTy, ...) {
+ CGM = Mod;
+ FunctionName = name;
+ Function = nullptr;
+ std::vector<llvm::Type *> ArgTys;
+ va_list Args;
+ va_start(Args, RetTy);
+ while (llvm::Type *ArgTy = va_arg(Args, llvm::Type *))
+ ArgTys.push_back(ArgTy);
+ va_end(Args);
+ FTy = llvm::FunctionType::get(RetTy, ArgTys, false);
+ }
+
+ llvm::FunctionType *getType() { return FTy; }
+
+ /// Overloaded cast operator, allows the class to be implicitly cast to an
+ /// LLVM constant.
+ operator llvm::Constant *() {
+ if (!Function) {
+ if (!FunctionName)
+ return nullptr;
+ Function =
+ cast<llvm::Constant>(CGM->CreateRuntimeFunction(FTy, FunctionName));
+ }
+ return Function;
+ }
+ operator llvm::Function *() {
+ return cast<llvm::Function>((llvm::Constant *)*this);
+ }
};
@@ -171,8 +166,9 @@ protected:
/// where the C code specifies const char*.
llvm::Constant *MakeConstantString(const std::string &Str,
const std::string &Name="") {
- llvm::Constant *ConstStr = CGM.GetAddrOfConstantCString(Str, Name.c_str());
- return llvm::ConstantExpr::getGetElementPtr(ConstStr, Zeros);
+ auto *ConstStr = CGM.GetAddrOfConstantCString(Str, Name.c_str());
+ return llvm::ConstantExpr::getGetElementPtr(ConstStr->getValueType(),
+ ConstStr, Zeros);
}
/// Emits a linkonce_odr string, whose name is the prefix followed by the
/// string value. This allows the linker to combine the strings between
@@ -181,13 +177,14 @@ protected:
llvm::Constant *ExportUniqueString(const std::string &Str,
const std::string prefix) {
std::string name = prefix + Str;
- llvm::Constant *ConstStr = TheModule.getGlobalVariable(name);
+ auto *ConstStr = TheModule.getGlobalVariable(name);
if (!ConstStr) {
llvm::Constant *value = llvm::ConstantDataArray::getString(VMContext,Str);
ConstStr = new llvm::GlobalVariable(TheModule, value->getType(), true,
llvm::GlobalValue::LinkOnceODRLinkage, value, prefix + Str);
}
- return llvm::ConstantExpr::getGetElementPtr(ConstStr, Zeros);
+ return llvm::ConstantExpr::getGetElementPtr(ConstStr->getValueType(),
+ ConstStr, Zeros);
}
/// Generates a global structure, initialized by the elements in the vector.
/// The element types must match the types of the structure elements in the
@@ -237,8 +234,9 @@ protected:
NameAndAttributes += TypeStr;
NameAndAttributes += '\0';
NameAndAttributes += PD->getNameAsString();
- return llvm::ConstantExpr::getGetElementPtr(
- CGM.GetAddrOfConstantCString(NameAndAttributes), Zeros);
+ auto *ConstStr = CGM.GetAddrOfConstantCString(NameAndAttributes);
+ return llvm::ConstantExpr::getGetElementPtr(ConstStr->getValueType(),
+ ConstStr, Zeros);
}
return MakeConstantString(PD->getNameAsString());
}
@@ -672,8 +670,8 @@ class CGObjCGNUstep : public CGObjCGNU {
slot->setMetadata(msgSendMDKind, node);
// Load the imp from the slot
- llvm::Value *imp =
- Builder.CreateLoad(Builder.CreateStructGEP(slot.getInstruction(), 4));
+ llvm::Value *imp = Builder.CreateLoad(
+ Builder.CreateStructGEP(nullptr, slot.getInstruction(), 4));
// The lookup function may have changed the receiver, so make sure we use
// the new one.
@@ -690,7 +688,7 @@ class CGObjCGNUstep : public CGObjCGNU {
CGF.EmitNounwindRuntimeCall(SlotLookupSuperFn, lookupArgs);
slot->setOnlyReadsMemory();
- return Builder.CreateLoad(Builder.CreateStructGEP(slot, 4));
+ return Builder.CreateLoad(Builder.CreateStructGEP(nullptr, slot, 4));
}
public:
CGObjCGNUstep(CodeGenModule &Mod) : CGObjCGNU(Mod, 9, 3) {
@@ -1013,7 +1011,7 @@ CGObjCGNU::CGObjCGNU(CodeGenModule &cgm, unsigned runtimeABIVersion,
llvm::Value *CGObjCGNU::GetClassNamed(CodeGenFunction &CGF,
const std::string &Name,
bool isWeak) {
- llvm::Value *ClassName = CGM.GetAddrOfConstantCString(Name);
+ llvm::GlobalVariable *ClassNameGV = CGM.GetAddrOfConstantCString(Name);
// With the incompatible ABI, this will need to be replaced with a direct
// reference to the class symbol. For the compatible nonfragile ABI we are
// still performing this lookup at run time but emitting the symbol for the
@@ -1023,7 +1021,8 @@ llvm::Value *CGObjCGNU::GetClassNamed(CodeGenFunction &CGF,
// with memoized versions or with static references if it's safe to do so.
if (!isWeak)
EmitClassRef(Name);
- ClassName = CGF.Builder.CreateStructGEP(ClassName, 0);
+ llvm::Value *ClassName =
+ CGF.Builder.CreateStructGEP(ClassNameGV->getValueType(), ClassNameGV, 0);
llvm::Constant *ClassLookupFn =
CGM.CreateRuntimeFunction(llvm::FunctionType::get(IdTy, PtrToInt8Ty, true),
@@ -1056,7 +1055,7 @@ llvm::Value *CGObjCGNU::GetSelector(CodeGenFunction &CGF, Selector Sel,
}
if (!SelValue) {
SelValue = llvm::GlobalAlias::create(
- SelectorTy->getElementType(), 0, llvm::GlobalValue::PrivateLinkage,
+ SelectorTy, llvm::GlobalValue::PrivateLinkage,
".objc_selector_" + Sel.getAsString(), &TheModule);
Types.push_back(TypedSelector(TypeEncoding, SelValue));
}
@@ -1143,21 +1142,22 @@ llvm::Constant *CGObjCGNUstep::GetEHType(QualType T) {
// It's quite ugly hard-coding this. Ideally we'd generate it using the host
// platform's name mangling.
const char *vtableName = "_ZTVN7gnustep7libobjc22__objc_class_type_infoE";
- llvm::Constant *Vtable = TheModule.getGlobalVariable(vtableName);
+ auto *Vtable = TheModule.getGlobalVariable(vtableName);
if (!Vtable) {
Vtable = new llvm::GlobalVariable(TheModule, PtrToInt8Ty, true,
llvm::GlobalValue::ExternalLinkage,
nullptr, vtableName);
}
llvm::Constant *Two = llvm::ConstantInt::get(IntTy, 2);
- Vtable = llvm::ConstantExpr::getGetElementPtr(Vtable, Two);
- Vtable = llvm::ConstantExpr::getBitCast(Vtable, PtrToInt8Ty);
+ auto *BVtable = llvm::ConstantExpr::getBitCast(
+ llvm::ConstantExpr::getGetElementPtr(Vtable->getValueType(), Vtable, Two),
+ PtrToInt8Ty);
llvm::Constant *typeName =
ExportUniqueString(className, "__objc_eh_typename_");
std::vector<llvm::Constant*> fields;
- fields.push_back(Vtable);
+ fields.push_back(BVtable);
fields.push_back(typeName);
llvm::Constant *TI =
MakeGlobal(llvm::StructType::get(PtrToInt8Ty, PtrToInt8Ty,
@@ -1261,25 +1261,25 @@ CGObjCGNU::GenerateMessageSendSuper(CodeGenFunction &CGF,
if (IsClassMessage) {
if (!MetaClassPtrAlias) {
MetaClassPtrAlias = llvm::GlobalAlias::create(
- IdTy->getElementType(), 0, llvm::GlobalValue::InternalLinkage,
+ IdTy, llvm::GlobalValue::InternalLinkage,
".objc_metaclass_ref" + Class->getNameAsString(), &TheModule);
}
ReceiverClass = MetaClassPtrAlias;
} else {
if (!ClassPtrAlias) {
ClassPtrAlias = llvm::GlobalAlias::create(
- IdTy->getElementType(), 0, llvm::GlobalValue::InternalLinkage,
+ IdTy, llvm::GlobalValue::InternalLinkage,
".objc_class_ref" + Class->getNameAsString(), &TheModule);
}
ReceiverClass = ClassPtrAlias;
}
}
// Cast the pointer to a simplified version of the class structure
+ llvm::Type *CastTy = llvm::StructType::get(IdTy, IdTy, nullptr);
ReceiverClass = Builder.CreateBitCast(ReceiverClass,
- llvm::PointerType::getUnqual(
- llvm::StructType::get(IdTy, IdTy, nullptr)));
+ llvm::PointerType::getUnqual(CastTy));
// Get the superclass pointer
- ReceiverClass = Builder.CreateStructGEP(ReceiverClass, 1);
+ ReceiverClass = Builder.CreateStructGEP(CastTy, ReceiverClass, 1);
// Load the superclass pointer
ReceiverClass = Builder.CreateLoad(ReceiverClass);
// Construct the structure used to look up the IMP
@@ -1287,8 +1287,10 @@ CGObjCGNU::GenerateMessageSendSuper(CodeGenFunction &CGF,
Receiver->getType(), IdTy, nullptr);
llvm::Value *ObjCSuper = Builder.CreateAlloca(ObjCSuperTy);
- Builder.CreateStore(Receiver, Builder.CreateStructGEP(ObjCSuper, 0));
- Builder.CreateStore(ReceiverClass, Builder.CreateStructGEP(ObjCSuper, 1));
+ Builder.CreateStore(Receiver,
+ Builder.CreateStructGEP(ObjCSuperTy, ObjCSuper, 0));
+ Builder.CreateStore(ReceiverClass,
+ Builder.CreateStructGEP(ObjCSuperTy, ObjCSuper, 1));
ObjCSuper = EnforceType(Builder, ObjCSuper, PtrToObjCSuperTy);
@@ -2294,7 +2296,8 @@ void CGObjCGNU::GenerateClass(const ObjCImplementationDecl *OID) {
offsetPointerIndexes[2] = llvm::ConstantInt::get(IndexTy, ivarIndex);
// Get the correct ivar field
llvm::Constant *offsetValue = llvm::ConstantExpr::getGetElementPtr(
- IvarList, offsetPointerIndexes);
+ cast<llvm::GlobalVariable>(IvarList)->getValueType(), IvarList,
+ offsetPointerIndexes);
// Get the existing variable, if one exists.
llvm::GlobalVariable *offset = TheModule.getNamedGlobal(Name);
if (offset) {
@@ -2366,7 +2369,7 @@ llvm::Function *CGObjCGNU::ModuleInitFunction() {
std::vector<llvm::Constant*> Elements;
llvm::Constant *Statics = NULLPtr;
// Generate statics list:
- if (ConstantStrings.size()) {
+ if (!ConstantStrings.empty()) {
llvm::ArrayType *StaticsArrayTy = llvm::ArrayType::get(PtrToInt8Ty,
ConstantStrings.size() + 1);
ConstantStrings.push_back(NULLPtr);
@@ -2439,8 +2442,8 @@ llvm::Function *CGObjCGNU::ModuleInitFunction() {
// Number of static selectors
Elements.push_back(llvm::ConstantInt::get(LongTy, SelectorCount));
- llvm::Constant *SelectorList = MakeGlobalArray(SelStructTy, Selectors,
- ".objc_selector_list");
+ llvm::GlobalVariable *SelectorList =
+ MakeGlobalArray(SelStructTy, Selectors, ".objc_selector_list");
Elements.push_back(llvm::ConstantExpr::getBitCast(SelectorList,
SelStructPtrTy));
@@ -2450,8 +2453,8 @@ llvm::Function *CGObjCGNU::ModuleInitFunction() {
llvm::Constant *Idxs[] = {Zeros[0],
llvm::ConstantInt::get(Int32Ty, i), Zeros[0]};
// FIXME: We're generating redundant loads and stores here!
- llvm::Constant *SelPtr = llvm::ConstantExpr::getGetElementPtr(SelectorList,
- makeArrayRef(Idxs, 2));
+ llvm::Constant *SelPtr = llvm::ConstantExpr::getGetElementPtr(
+ SelectorList->getValueType(), SelectorList, makeArrayRef(Idxs, 2));
// If selectors are defined as an opaque type, cast the pointer to this
// type.
SelPtr = llvm::ConstantExpr::getBitCast(SelPtr, SelectorTy);
@@ -2562,8 +2565,8 @@ llvm::Function *CGObjCGNU::ModuleInitFunction() {
true);
if (TheClass) {
TheClass = llvm::ConstantExpr::getBitCast(TheClass, PtrTy);
- Builder.CreateCall2(RegisterAlias, TheClass,
- MakeConstantString(iter->second));
+ Builder.CreateCall(RegisterAlias,
+ {TheClass, MakeConstantString(iter->second)});
}
}
// Jump to end:
@@ -2679,7 +2682,7 @@ llvm::Value * CGObjCGNU::EmitObjCWeakRead(CodeGenFunction &CGF,
llvm::Value *AddrWeakObj) {
CGBuilderTy &B = CGF.Builder;
AddrWeakObj = EnforceType(B, AddrWeakObj, PtrToIdTy);
- return B.CreateCall(WeakReadFn, AddrWeakObj);
+ return B.CreateCall(WeakReadFn.getType(), WeakReadFn, AddrWeakObj);
}
void CGObjCGNU::EmitObjCWeakAssign(CodeGenFunction &CGF,
@@ -2687,7 +2690,7 @@ void CGObjCGNU::EmitObjCWeakAssign(CodeGenFunction &CGF,
CGBuilderTy &B = CGF.Builder;
src = EnforceType(B, src, IdTy);
dst = EnforceType(B, dst, PtrToIdTy);
- B.CreateCall2(WeakAssignFn, src, dst);
+ B.CreateCall(WeakAssignFn.getType(), WeakAssignFn, {src, dst});
}
void CGObjCGNU::EmitObjCGlobalAssign(CodeGenFunction &CGF,
@@ -2696,11 +2699,9 @@ void CGObjCGNU::EmitObjCGlobalAssign(CodeGenFunction &CGF,
CGBuilderTy &B = CGF.Builder;
src = EnforceType(B, src, IdTy);
dst = EnforceType(B, dst, PtrToIdTy);
- if (!threadlocal)
- B.CreateCall2(GlobalAssignFn, src, dst);
- else
- // FIXME. Add threadloca assign API
- llvm_unreachable("EmitObjCGlobalAssign - Threal Local API NYI");
+ // FIXME. Add threadloca assign API
+ assert(!threadlocal && "EmitObjCGlobalAssign - Threal Local API NYI");
+ B.CreateCall(GlobalAssignFn.getType(), GlobalAssignFn, {src, dst});
}
void CGObjCGNU::EmitObjCIvarAssign(CodeGenFunction &CGF,
@@ -2709,7 +2710,7 @@ void CGObjCGNU::EmitObjCIvarAssign(CodeGenFunction &CGF,
CGBuilderTy &B = CGF.Builder;
src = EnforceType(B, src, IdTy);
dst = EnforceType(B, dst, IdTy);
- B.CreateCall3(IvarAssignFn, src, dst, ivarOffset);
+ B.CreateCall(IvarAssignFn.getType(), IvarAssignFn, {src, dst, ivarOffset});
}
void CGObjCGNU::EmitObjCStrongCastAssign(CodeGenFunction &CGF,
@@ -2717,7 +2718,7 @@ void CGObjCGNU::EmitObjCStrongCastAssign(CodeGenFunction &CGF,
CGBuilderTy &B = CGF.Builder;
src = EnforceType(B, src, IdTy);
dst = EnforceType(B, dst, PtrToIdTy);
- B.CreateCall2(StrongCastAssignFn, src, dst);
+ B.CreateCall(StrongCastAssignFn.getType(), StrongCastAssignFn, {src, dst});
}
void CGObjCGNU::EmitGCMemmoveCollectable(CodeGenFunction &CGF,
@@ -2728,7 +2729,7 @@ void CGObjCGNU::EmitGCMemmoveCollectable(CodeGenFunction &CGF,
DestPtr = EnforceType(B, DestPtr, PtrTy);
SrcPtr = EnforceType(B, SrcPtr, PtrTy);
- B.CreateCall3(MemMoveFn, DestPtr, SrcPtr, Size);
+ B.CreateCall(MemMoveFn.getType(), MemMoveFn, {DestPtr, SrcPtr, Size});
}
llvm::GlobalVariable *CGObjCGNU::ObjCIvarOffsetVariable(
diff --git a/lib/CodeGen/CGObjCMac.cpp b/lib/CodeGen/CGObjCMac.cpp
index f91e8e15b039..a45446a7065a 100644
--- a/lib/CodeGen/CGObjCMac.cpp
+++ b/lib/CodeGen/CGObjCMac.cpp
@@ -486,7 +486,6 @@ public:
}
ObjCCommonTypesHelper(CodeGen::CodeGenModule &cgm);
- ~ObjCCommonTypesHelper(){}
};
/// ObjCTypesHelper - Helper class that encapsulates lazy
@@ -595,7 +594,6 @@ public:
public:
ObjCTypesHelper(CodeGen::CodeGenModule &cgm);
- ~ObjCTypesHelper() {}
};
/// ObjCNonFragileABITypesHelper - will have all types needed by objective-c's
@@ -733,7 +731,6 @@ public:
llvm::Type *EHTypePtrTy;
ObjCNonFragileABITypesHelper(CodeGen::CodeGenModule &cgm);
- ~ObjCNonFragileABITypesHelper(){}
};
class CGObjCCommonMac : public CodeGen::CGObjCRuntime {
@@ -1678,14 +1675,13 @@ struct NullReturnState {
/// getConstantGEP() - Help routine to construct simple GEPs.
static llvm::Constant *getConstantGEP(llvm::LLVMContext &VMContext,
- llvm::Constant *C,
- unsigned idx0,
+ llvm::GlobalVariable *C, unsigned idx0,
unsigned idx1) {
llvm::Value *Idxs[] = {
llvm::ConstantInt::get(llvm::Type::getInt32Ty(VMContext), idx0),
llvm::ConstantInt::get(llvm::Type::getInt32Ty(VMContext), idx1)
};
- return llvm::ConstantExpr::getGetElementPtr(C, Idxs);
+ return llvm::ConstantExpr::getGetElementPtr(C->getValueType(), C, Idxs);
}
/// hasObjCExceptionAttribute - Return true if this class or any super
@@ -1791,8 +1787,9 @@ CGObjCMac::GenerateMessageSendSuper(CodeGen::CodeGenFunction &CGF,
CGF.CreateTempAlloca(ObjCTypes.SuperTy, "objc_super");
llvm::Value *ReceiverAsObject =
CGF.Builder.CreateBitCast(Receiver, ObjCTypes.ObjectPtrTy);
- CGF.Builder.CreateStore(ReceiverAsObject,
- CGF.Builder.CreateStructGEP(ObjCSuper, 0));
+ CGF.Builder.CreateStore(
+ ReceiverAsObject,
+ CGF.Builder.CreateStructGEP(ObjCTypes.SuperTy, ObjCSuper, 0));
// If this is a class message the metaclass is passed as the target.
llvm::Value *Target;
@@ -1805,20 +1802,20 @@ CGObjCMac::GenerateMessageSendSuper(CodeGen::CodeGenFunction &CGF,
// the class's "isa" pointer. The following assumes that
// isa" is the first ivar in a class (which it must be).
Target = EmitClassRef(CGF, Class->getSuperClass());
- Target = CGF.Builder.CreateStructGEP(Target, 0);
+ Target = CGF.Builder.CreateStructGEP(ObjCTypes.ClassTy, Target, 0);
Target = CGF.Builder.CreateLoad(Target);
} else {
- llvm::Value *MetaClassPtr = EmitMetaClassRef(Class);
- llvm::Value *SuperPtr = CGF.Builder.CreateStructGEP(MetaClassPtr, 1);
+ llvm::Constant *MetaClassPtr = EmitMetaClassRef(Class);
+ llvm::Value *SuperPtr =
+ CGF.Builder.CreateStructGEP(ObjCTypes.ClassTy, MetaClassPtr, 1);
llvm::Value *Super = CGF.Builder.CreateLoad(SuperPtr);
Target = Super;
}
- }
- else if (isCategoryImpl)
+ } else if (isCategoryImpl)
Target = EmitClassRef(CGF, Class->getSuperClass());
else {
llvm::Value *ClassPtr = EmitSuperClassRef(Class);
- ClassPtr = CGF.Builder.CreateStructGEP(ClassPtr, 1);
+ ClassPtr = CGF.Builder.CreateStructGEP(ObjCTypes.ClassTy, ClassPtr, 1);
Target = CGF.Builder.CreateLoad(ClassPtr);
}
// FIXME: We shouldn't need to do this cast, rectify the ASTContext and
@@ -1826,8 +1823,8 @@ CGObjCMac::GenerateMessageSendSuper(CodeGen::CodeGenFunction &CGF,
llvm::Type *ClassTy =
CGM.getTypes().ConvertType(CGF.getContext().getObjCClassType());
Target = CGF.Builder.CreateBitCast(Target, ClassTy);
- CGF.Builder.CreateStore(Target,
- CGF.Builder.CreateStructGEP(ObjCSuper, 1));
+ CGF.Builder.CreateStore(
+ Target, CGF.Builder.CreateStructGEP(ObjCTypes.SuperTy, ObjCSuper, 1));
return EmitMessageSend(CGF, Return, ResultType,
EmitSelector(CGF, Sel),
ObjCSuper, ObjCTypes.SuperPtrCTy,
@@ -3810,15 +3807,16 @@ void CGObjCMac::EmitTryOrSynchronizedStmt(CodeGen::CodeGenFunction &CGF,
// Enter a try block:
// - Call objc_exception_try_enter to push ExceptionData on top of
// the EH stack.
- CGF.EmitNounwindRuntimeCall(ObjCTypes.getExceptionTryEnterFn(), ExceptionData);
+ CGF.EmitNounwindRuntimeCall(ObjCTypes.getExceptionTryEnterFn(),
+ ExceptionData);
// - Call setjmp on the exception data buffer.
llvm::Constant *Zero = llvm::ConstantInt::get(CGF.Builder.getInt32Ty(), 0);
llvm::Value *GEPIndexes[] = { Zero, Zero, Zero };
- llvm::Value *SetJmpBuffer =
- CGF.Builder.CreateGEP(ExceptionData, GEPIndexes, "setjmp_buffer");
- llvm::CallInst *SetJmpResult =
- CGF.EmitNounwindRuntimeCall(ObjCTypes.getSetJmpFn(), SetJmpBuffer, "setjmp_result");
+ llvm::Value *SetJmpBuffer = CGF.Builder.CreateGEP(
+ ObjCTypes.ExceptionDataTy, ExceptionData, GEPIndexes, "setjmp_buffer");
+ llvm::CallInst *SetJmpResult = CGF.EmitNounwindRuntimeCall(
+ ObjCTypes.getSetJmpFn(), SetJmpBuffer, "setjmp_result");
SetJmpResult->setCanReturnTwice();
// If setjmp returned 0, enter the protected block; otherwise,
@@ -5263,6 +5261,7 @@ ObjCNonFragileABITypesHelper::ObjCNonFragileABITypesHelper(CodeGen::CodeGenModul
// const uint32_t size; // sizeof(struct _protocol_t)
// const uint32_t flags; // = 0
// const char ** extendedMethodTypes;
+ // const char *demangledName;
// }
// Holder for struct _protocol_list_t *
@@ -5275,6 +5274,7 @@ ObjCNonFragileABITypesHelper::ObjCNonFragileABITypesHelper(CodeGen::CodeGenModul
MethodListnfABIPtrTy, MethodListnfABIPtrTy,
MethodListnfABIPtrTy, MethodListnfABIPtrTy,
PropertyListPtrTy, IntTy, IntTy, Int8PtrPtrTy,
+ Int8PtrTy,
nullptr);
// struct _protocol_t*
@@ -6207,6 +6207,7 @@ llvm::Constant *CGObjCNonFragileABIMac::GetOrEmitProtocolRef(
/// const uint32_t size; // sizeof(struct _protocol_t)
/// const uint32_t flags; // = 0
/// const char ** extendedMethodTypes;
+/// const char *demangledName;
/// }
/// @endcode
///
@@ -6258,7 +6259,7 @@ llvm::Constant *CGObjCNonFragileABIMac::GetOrEmitProtocol(
MethodTypesExt.insert(MethodTypesExt.end(),
OptMethodTypesExt.begin(), OptMethodTypesExt.end());
- llvm::Constant *Values[11];
+ llvm::Constant *Values[12];
// isa is NULL
Values[0] = llvm::Constant::getNullValue(ObjCTypes.ObjectPtrTy);
Values[1] = GetClassName(PD->getObjCRuntimeNameAsString());
@@ -6291,6 +6292,9 @@ llvm::Constant *CGObjCNonFragileABIMac::GetOrEmitProtocol(
Values[10] = EmitProtocolMethodTypes("\01l_OBJC_$_PROTOCOL_METHOD_TYPES_"
+ PD->getObjCRuntimeNameAsString(),
MethodTypesExt, ObjCTypes);
+ // const char *demangledName;
+ Values[11] = llvm::Constant::getNullValue(ObjCTypes.Int8PtrTy);
+
llvm::Constant *Init = llvm::ConstantStruct::get(ObjCTypes.ProtocolnfABITy,
Values);
@@ -6562,7 +6566,8 @@ CGObjCNonFragileABIMac::EmitVTableMessageSend(CodeGenFunction &CGF,
args[1].RV = RValue::get(mref);
// Load the function to call from the message ref table.
- llvm::Value *callee = CGF.Builder.CreateStructGEP(mref, 0);
+ llvm::Value *callee =
+ CGF.Builder.CreateStructGEP(ObjCTypes.MessageRefTy, mref, 0);
callee = CGF.Builder.CreateLoad(callee, "msgSend_fn");
callee = CGF.Builder.CreateBitCast(callee, MSI.MessengerType);
@@ -6727,8 +6732,9 @@ CGObjCNonFragileABIMac::GenerateMessageSendSuper(CodeGen::CodeGenFunction &CGF,
llvm::Value *ReceiverAsObject =
CGF.Builder.CreateBitCast(Receiver, ObjCTypes.ObjectPtrTy);
- CGF.Builder.CreateStore(ReceiverAsObject,
- CGF.Builder.CreateStructGEP(ObjCSuper, 0));
+ CGF.Builder.CreateStore(
+ ReceiverAsObject,
+ CGF.Builder.CreateStructGEP(ObjCTypes.SuperTy, ObjCSuper, 0));
// If this is a class message the metaclass is passed as the target.
llvm::Value *Target;
@@ -6742,8 +6748,8 @@ CGObjCNonFragileABIMac::GenerateMessageSendSuper(CodeGen::CodeGenFunction &CGF,
llvm::Type *ClassTy =
CGM.getTypes().ConvertType(CGF.getContext().getObjCClassType());
Target = CGF.Builder.CreateBitCast(Target, ClassTy);
- CGF.Builder.CreateStore(Target,
- CGF.Builder.CreateStructGEP(ObjCSuper, 1));
+ CGF.Builder.CreateStore(
+ Target, CGF.Builder.CreateStructGEP(ObjCTypes.SuperTy, ObjCSuper, 1));
return (isVTableDispatchedSelector(Sel))
? EmitVTableMessageSend(CGF, Return, ResultType, Sel,
@@ -6992,10 +6998,10 @@ CGObjCNonFragileABIMac::GetInterfaceEHType(const ObjCInterfaceDecl *ID,
llvm::Value *VTableIdx = llvm::ConstantInt::get(CGM.Int32Ty, 2);
llvm::Constant *Values[] = {
- llvm::ConstantExpr::getGetElementPtr(VTableGV, VTableIdx),
- GetClassName(ID->getObjCRuntimeNameAsString()),
- GetClassGlobal(ClassName.str())
- };
+ llvm::ConstantExpr::getGetElementPtr(VTableGV->getValueType(), VTableGV,
+ VTableIdx),
+ GetClassName(ID->getObjCRuntimeNameAsString()),
+ GetClassGlobal(ClassName.str())};
llvm::Constant *Init =
llvm::ConstantStruct::get(ObjCTypes.EHTypeTy, Values);
diff --git a/lib/CodeGen/CGObjCRuntime.cpp b/lib/CodeGen/CGObjCRuntime.cpp
index 3d013da51e65..5290a87cebf2 100644
--- a/lib/CodeGen/CGObjCRuntime.cpp
+++ b/lib/CodeGen/CGObjCRuntime.cpp
@@ -160,7 +160,7 @@ namespace {
void Emit(CodeGenFunction &CGF, Flags flags) override {
if (!MightThrow) {
- CGF.Builder.CreateCall(Fn)->setDoesNotThrow();
+ CGF.Builder.CreateCall(Fn, {})->setDoesNotThrow();
return;
}
diff --git a/lib/CodeGen/CGOpenMPRuntime.cpp b/lib/CodeGen/CGOpenMPRuntime.cpp
index 22ee00f2c7ae..1238accf42d7 100644
--- a/lib/CodeGen/CGOpenMPRuntime.cpp
+++ b/lib/CodeGen/CGOpenMPRuntime.cpp
@@ -13,6 +13,7 @@
#include "CGOpenMPRuntime.h"
#include "CodeGenFunction.h"
+#include "CGCleanup.h"
#include "clang/AST/Decl.h"
#include "clang/AST/StmtOpenMP.h"
#include "llvm/ADT/ArrayRef.h"
@@ -27,61 +28,236 @@ using namespace clang;
using namespace CodeGen;
namespace {
-/// \brief API for captured statement code generation in OpenMP constructs.
+/// \brief Base class for handling code generation inside OpenMP regions.
class CGOpenMPRegionInfo : public CodeGenFunction::CGCapturedStmtInfo {
public:
- CGOpenMPRegionInfo(const OMPExecutableDirective &D, const CapturedStmt &CS,
- const VarDecl *ThreadIDVar)
- : CGCapturedStmtInfo(CS, CR_OpenMP), ThreadIDVar(ThreadIDVar),
- Directive(D) {
- assert(ThreadIDVar != nullptr && "No ThreadID in OpenMP region.");
- }
+ /// \brief Kinds of OpenMP regions used in codegen.
+ enum CGOpenMPRegionKind {
+ /// \brief Region with outlined function for standalone 'parallel'
+ /// directive.
+ ParallelOutlinedRegion,
+ /// \brief Region with outlined function for standalone 'task' directive.
+ TaskOutlinedRegion,
+ /// \brief Region for constructs that do not require function outlining,
+ /// like 'for', 'sections', 'atomic' etc. directives.
+ InlinedRegion,
+ };
+
+ CGOpenMPRegionInfo(const CapturedStmt &CS,
+ const CGOpenMPRegionKind RegionKind,
+ const RegionCodeGenTy &CodeGen)
+ : CGCapturedStmtInfo(CS, CR_OpenMP), RegionKind(RegionKind),
+ CodeGen(CodeGen) {}
+
+ CGOpenMPRegionInfo(const CGOpenMPRegionKind RegionKind,
+ const RegionCodeGenTy &CodeGen)
+ : CGCapturedStmtInfo(CR_OpenMP), RegionKind(RegionKind),
+ CodeGen(CodeGen) {}
- /// \brief Gets a variable or parameter for storing global thread id
+ /// \brief Get a variable or parameter for storing global thread id
/// inside OpenMP construct.
- const VarDecl *getThreadIDVariable() const { return ThreadIDVar; }
+ virtual const VarDecl *getThreadIDVariable() const = 0;
- /// \brief Gets an LValue for the current ThreadID variable.
- LValue getThreadIDVariableLValue(CodeGenFunction &CGF);
+ /// \brief Emit the captured statement body.
+ virtual void EmitBody(CodeGenFunction &CGF, const Stmt *S) override;
+
+ /// \brief Get an LValue for the current ThreadID variable.
+ /// \return LValue for thread id variable. This LValue always has type int32*.
+ virtual LValue getThreadIDVariableLValue(CodeGenFunction &CGF);
+
+ CGOpenMPRegionKind getRegionKind() const { return RegionKind; }
static bool classof(const CGCapturedStmtInfo *Info) {
return Info->getKind() == CR_OpenMP;
}
- /// \brief Emit the captured statement body.
- void EmitBody(CodeGenFunction &CGF, Stmt *S) override;
+protected:
+ CGOpenMPRegionKind RegionKind;
+ const RegionCodeGenTy &CodeGen;
+};
+
+/// \brief API for captured statement code generation in OpenMP constructs.
+class CGOpenMPOutlinedRegionInfo : public CGOpenMPRegionInfo {
+public:
+ CGOpenMPOutlinedRegionInfo(const CapturedStmt &CS, const VarDecl *ThreadIDVar,
+ const RegionCodeGenTy &CodeGen)
+ : CGOpenMPRegionInfo(CS, ParallelOutlinedRegion, CodeGen),
+ ThreadIDVar(ThreadIDVar) {
+ assert(ThreadIDVar != nullptr && "No ThreadID in OpenMP region.");
+ }
+ /// \brief Get a variable or parameter for storing global thread id
+ /// inside OpenMP construct.
+ const VarDecl *getThreadIDVariable() const override { return ThreadIDVar; }
/// \brief Get the name of the capture helper.
StringRef getHelperName() const override { return ".omp_outlined."; }
+ static bool classof(const CGCapturedStmtInfo *Info) {
+ return CGOpenMPRegionInfo::classof(Info) &&
+ cast<CGOpenMPRegionInfo>(Info)->getRegionKind() ==
+ ParallelOutlinedRegion;
+ }
+
private:
/// \brief A variable or parameter storing global thread id for OpenMP
/// constructs.
const VarDecl *ThreadIDVar;
- /// \brief OpenMP executable directive associated with the region.
- const OMPExecutableDirective &Directive;
};
+
+/// \brief API for captured statement code generation in OpenMP constructs.
+class CGOpenMPTaskOutlinedRegionInfo : public CGOpenMPRegionInfo {
+public:
+ CGOpenMPTaskOutlinedRegionInfo(const CapturedStmt &CS,
+ const VarDecl *ThreadIDVar,
+ const RegionCodeGenTy &CodeGen)
+ : CGOpenMPRegionInfo(CS, TaskOutlinedRegion, CodeGen),
+ ThreadIDVar(ThreadIDVar) {
+ assert(ThreadIDVar != nullptr && "No ThreadID in OpenMP region.");
+ }
+ /// \brief Get a variable or parameter for storing global thread id
+ /// inside OpenMP construct.
+ const VarDecl *getThreadIDVariable() const override { return ThreadIDVar; }
+
+ /// \brief Get an LValue for the current ThreadID variable.
+ LValue getThreadIDVariableLValue(CodeGenFunction &CGF) override;
+
+ /// \brief Get the name of the capture helper.
+ StringRef getHelperName() const override { return ".omp_outlined."; }
+
+ static bool classof(const CGCapturedStmtInfo *Info) {
+ return CGOpenMPRegionInfo::classof(Info) &&
+ cast<CGOpenMPRegionInfo>(Info)->getRegionKind() ==
+ TaskOutlinedRegion;
+ }
+
+private:
+ /// \brief A variable or parameter storing global thread id for OpenMP
+ /// constructs.
+ const VarDecl *ThreadIDVar;
+};
+
+/// \brief API for inlined captured statement code generation in OpenMP
+/// constructs.
+class CGOpenMPInlinedRegionInfo : public CGOpenMPRegionInfo {
+public:
+ CGOpenMPInlinedRegionInfo(CodeGenFunction::CGCapturedStmtInfo *OldCSI,
+ const RegionCodeGenTy &CodeGen)
+ : CGOpenMPRegionInfo(InlinedRegion, CodeGen), OldCSI(OldCSI),
+ OuterRegionInfo(dyn_cast_or_null<CGOpenMPRegionInfo>(OldCSI)) {}
+ // \brief Retrieve the value of the context parameter.
+ llvm::Value *getContextValue() const override {
+ if (OuterRegionInfo)
+ return OuterRegionInfo->getContextValue();
+ llvm_unreachable("No context value for inlined OpenMP region");
+ }
+ virtual void setContextValue(llvm::Value *V) override {
+ if (OuterRegionInfo) {
+ OuterRegionInfo->setContextValue(V);
+ return;
+ }
+ llvm_unreachable("No context value for inlined OpenMP region");
+ }
+ /// \brief Lookup the captured field decl for a variable.
+ const FieldDecl *lookup(const VarDecl *VD) const override {
+ if (OuterRegionInfo)
+ return OuterRegionInfo->lookup(VD);
+ // If there is no outer outlined region,no need to lookup in a list of
+ // captured variables, we can use the original one.
+ return nullptr;
+ }
+ FieldDecl *getThisFieldDecl() const override {
+ if (OuterRegionInfo)
+ return OuterRegionInfo->getThisFieldDecl();
+ return nullptr;
+ }
+ /// \brief Get a variable or parameter for storing global thread id
+ /// inside OpenMP construct.
+ const VarDecl *getThreadIDVariable() const override {
+ if (OuterRegionInfo)
+ return OuterRegionInfo->getThreadIDVariable();
+ return nullptr;
+ }
+
+ /// \brief Get the name of the capture helper.
+ StringRef getHelperName() const override {
+ if (auto *OuterRegionInfo = getOldCSI())
+ return OuterRegionInfo->getHelperName();
+ llvm_unreachable("No helper name for inlined OpenMP construct");
+ }
+
+ CodeGenFunction::CGCapturedStmtInfo *getOldCSI() const { return OldCSI; }
+
+ static bool classof(const CGCapturedStmtInfo *Info) {
+ return CGOpenMPRegionInfo::classof(Info) &&
+ cast<CGOpenMPRegionInfo>(Info)->getRegionKind() == InlinedRegion;
+ }
+
+private:
+ /// \brief CodeGen info about outer OpenMP region.
+ CodeGenFunction::CGCapturedStmtInfo *OldCSI;
+ CGOpenMPRegionInfo *OuterRegionInfo;
+};
+
+/// \brief RAII for emitting code of OpenMP constructs.
+class InlinedOpenMPRegionRAII {
+ CodeGenFunction &CGF;
+
+public:
+ /// \brief Constructs region for combined constructs.
+ /// \param CodeGen Code generation sequence for combined directives. Includes
+ /// a list of functions used for code generation of implicitly inlined
+ /// regions.
+ InlinedOpenMPRegionRAII(CodeGenFunction &CGF, const RegionCodeGenTy &CodeGen)
+ : CGF(CGF) {
+ // Start emission for the construct.
+ CGF.CapturedStmtInfo =
+ new CGOpenMPInlinedRegionInfo(CGF.CapturedStmtInfo, CodeGen);
+ }
+ ~InlinedOpenMPRegionRAII() {
+ // Restore original CapturedStmtInfo only if we're done with code emission.
+ auto *OldCSI =
+ cast<CGOpenMPInlinedRegionInfo>(CGF.CapturedStmtInfo)->getOldCSI();
+ delete CGF.CapturedStmtInfo;
+ CGF.CapturedStmtInfo = OldCSI;
+ }
+};
+
} // namespace
LValue CGOpenMPRegionInfo::getThreadIDVariableLValue(CodeGenFunction &CGF) {
return CGF.MakeNaturalAlignAddrLValue(
- CGF.GetAddrOfLocalVar(ThreadIDVar),
- CGF.getContext().getPointerType(ThreadIDVar->getType()));
+ CGF.Builder.CreateAlignedLoad(
+ CGF.GetAddrOfLocalVar(getThreadIDVariable()),
+ CGF.PointerAlignInBytes),
+ getThreadIDVariable()
+ ->getType()
+ ->castAs<PointerType>()
+ ->getPointeeType());
}
-void CGOpenMPRegionInfo::EmitBody(CodeGenFunction &CGF, Stmt *S) {
- CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
- CGF.EmitOMPPrivateClause(Directive, PrivateScope);
- CGF.EmitOMPFirstprivateClause(Directive, PrivateScope);
- if (PrivateScope.Privatize())
- // Emit implicit barrier to synchronize threads and avoid data races.
- CGF.CGM.getOpenMPRuntime().EmitOMPBarrierCall(CGF, Directive.getLocStart(),
- /*IsExplicit=*/false);
- CGCapturedStmtInfo::EmitBody(CGF, S);
+void CGOpenMPRegionInfo::EmitBody(CodeGenFunction &CGF, const Stmt * /*S*/) {
+ // 1.2.2 OpenMP Language Terminology
+ // Structured block - An executable statement with a single entry at the
+ // top and a single exit at the bottom.
+ // The point of exit cannot be a branch out of the structured block.
+ // longjmp() and throw() must not violate the entry/exit criteria.
+ CGF.EHStack.pushTerminate();
+ {
+ CodeGenFunction::RunCleanupsScope Scope(CGF);
+ CodeGen(CGF);
+ }
+ CGF.EHStack.popTerminate();
+}
+
+LValue CGOpenMPTaskOutlinedRegionInfo::getThreadIDVariableLValue(
+ CodeGenFunction &CGF) {
+ return CGF.MakeNaturalAlignAddrLValue(
+ CGF.GetAddrOfLocalVar(getThreadIDVariable()),
+ getThreadIDVariable()->getType());
}
CGOpenMPRuntime::CGOpenMPRuntime(CodeGenModule &CGM)
- : CGM(CGM), DefaultOpenMPPSource(nullptr) {
+ : CGM(CGM), DefaultOpenMPPSource(nullptr), KmpRoutineEntryPtrTy(nullptr) {
IdentTy = llvm::StructType::create(
"ident_t", CGM.Int32Ty /* reserved_1 */, CGM.Int32Ty /* flags */,
CGM.Int32Ty /* reserved_2 */, CGM.Int32Ty /* reserved_3 */,
@@ -93,18 +269,38 @@ CGOpenMPRuntime::CGOpenMPRuntime(CodeGenModule &CGM)
KmpCriticalNameTy = llvm::ArrayType::get(CGM.Int32Ty, /*NumElements*/ 8);
}
+void CGOpenMPRuntime::clear() {
+ InternalVars.clear();
+}
+
llvm::Value *
-CGOpenMPRuntime::EmitOpenMPOutlinedFunction(const OMPExecutableDirective &D,
- const VarDecl *ThreadIDVar) {
+CGOpenMPRuntime::emitParallelOutlinedFunction(const OMPExecutableDirective &D,
+ const VarDecl *ThreadIDVar,
+ const RegionCodeGenTy &CodeGen) {
+ assert(ThreadIDVar->getType()->isPointerType() &&
+ "thread id variable must be of type kmp_int32 *");
const CapturedStmt *CS = cast<CapturedStmt>(D.getAssociatedStmt());
CodeGenFunction CGF(CGM, true);
- CGOpenMPRegionInfo CGInfo(D, *CS, ThreadIDVar);
+ CGOpenMPOutlinedRegionInfo CGInfo(*CS, ThreadIDVar, CodeGen);
CGF.CapturedStmtInfo = &CGInfo;
return CGF.GenerateCapturedStmtFunction(*CS);
}
llvm::Value *
-CGOpenMPRuntime::GetOrCreateDefaultOpenMPLocation(OpenMPLocationFlags Flags) {
+CGOpenMPRuntime::emitTaskOutlinedFunction(const OMPExecutableDirective &D,
+ const VarDecl *ThreadIDVar,
+ const RegionCodeGenTy &CodeGen) {
+ assert(!ThreadIDVar->getType()->isPointerType() &&
+ "thread id variable must be of type kmp_int32 for tasks");
+ auto *CS = cast<CapturedStmt>(D.getAssociatedStmt());
+ CodeGenFunction CGF(CGM, true);
+ CGOpenMPTaskOutlinedRegionInfo CGInfo(*CS, ThreadIDVar, CodeGen);
+ CGF.CapturedStmtInfo = &CGInfo;
+ return CGF.GenerateCapturedStmtFunction(*CS);
+}
+
+llvm::Value *
+CGOpenMPRuntime::getOrCreateDefaultLocation(OpenMPLocationFlags Flags) {
llvm::Value *Entry = OpenMPDefaultLocMap.lookup(Flags);
if (!Entry) {
if (!DefaultOpenMPPSource) {
@@ -134,12 +330,13 @@ CGOpenMPRuntime::GetOrCreateDefaultOpenMPLocation(OpenMPLocationFlags Flags) {
return Entry;
}
-llvm::Value *CGOpenMPRuntime::EmitOpenMPUpdateLocation(
- CodeGenFunction &CGF, SourceLocation Loc, OpenMPLocationFlags Flags) {
+llvm::Value *CGOpenMPRuntime::emitUpdateLocation(CodeGenFunction &CGF,
+ SourceLocation Loc,
+ OpenMPLocationFlags Flags) {
// If no debug info is generated - return global default location.
if (CGM.getCodeGenOpts().getDebugInfo() == CodeGenOptions::NoDebugInfo ||
Loc.isInvalid())
- return GetOrCreateDefaultOpenMPLocation(Flags);
+ return getOrCreateDefaultLocation(Flags);
assert(CGF.CurFn && "No function in current CodeGenFunction.");
@@ -159,14 +356,14 @@ llvm::Value *CGOpenMPRuntime::EmitOpenMPUpdateLocation(
CGBuilderTy::InsertPointGuard IPG(CGF.Builder);
CGF.Builder.SetInsertPoint(CGF.AllocaInsertPt);
- CGF.Builder.CreateMemCpy(LocValue, GetOrCreateDefaultOpenMPLocation(Flags),
+ CGF.Builder.CreateMemCpy(LocValue, getOrCreateDefaultLocation(Flags),
llvm::ConstantExpr::getSizeOf(IdentTy),
CGM.PointerAlignInBytes);
}
// char **psource = &.kmpc_loc_<flags>.addr.psource;
- auto *PSource =
- CGF.Builder.CreateConstInBoundsGEP2_32(LocValue, 0, IdentField_PSource);
+ auto *PSource = CGF.Builder.CreateConstInBoundsGEP2_32(IdentTy, LocValue, 0,
+ IdentField_PSource);
auto OMPDebugLoc = OpenMPDebugLocMap.lookup(Loc.getRawEncoding());
if (OMPDebugLoc == nullptr) {
@@ -189,8 +386,8 @@ llvm::Value *CGOpenMPRuntime::EmitOpenMPUpdateLocation(
return LocValue;
}
-llvm::Value *CGOpenMPRuntime::GetOpenMPThreadID(CodeGenFunction &CGF,
- SourceLocation Loc) {
+llvm::Value *CGOpenMPRuntime::getThreadID(CodeGenFunction &CGF,
+ SourceLocation Loc) {
assert(CGF.CurFn && "No function in current CodeGenFunction.");
llvm::Value *ThreadID = nullptr;
@@ -204,36 +401,35 @@ llvm::Value *CGOpenMPRuntime::GetOpenMPThreadID(CodeGenFunction &CGF,
}
if (auto OMPRegionInfo =
dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo)) {
- // Check if this an outlined function with thread id passed as argument.
- auto ThreadIDVar = OMPRegionInfo->getThreadIDVariable();
- auto LVal = OMPRegionInfo->getThreadIDVariableLValue(CGF);
- auto RVal = CGF.EmitLoadOfLValue(LVal, Loc);
- LVal = CGF.MakeNaturalAlignAddrLValue(RVal.getScalarVal(),
- ThreadIDVar->getType());
- ThreadID = CGF.EmitLoadOfLValue(LVal, Loc).getScalarVal();
- // If value loaded in entry block, cache it and use it everywhere in
- // function.
- if (CGF.Builder.GetInsertBlock() == CGF.AllocaInsertPt->getParent()) {
- auto &Elem = OpenMPLocThreadIDMap.FindAndConstruct(CGF.CurFn);
- Elem.second.ThreadID = ThreadID;
+ if (OMPRegionInfo->getThreadIDVariable()) {
+ // Check if this an outlined function with thread id passed as argument.
+ auto LVal = OMPRegionInfo->getThreadIDVariableLValue(CGF);
+ ThreadID = CGF.EmitLoadOfLValue(LVal, Loc).getScalarVal();
+ // If value loaded in entry block, cache it and use it everywhere in
+ // function.
+ if (CGF.Builder.GetInsertBlock() == CGF.AllocaInsertPt->getParent()) {
+ auto &Elem = OpenMPLocThreadIDMap.FindAndConstruct(CGF.CurFn);
+ Elem.second.ThreadID = ThreadID;
+ }
+ return ThreadID;
}
- } else {
- // This is not an outlined function region - need to call __kmpc_int32
- // kmpc_global_thread_num(ident_t *loc).
- // Generate thread id value and cache this value for use across the
- // function.
- CGBuilderTy::InsertPointGuard IPG(CGF.Builder);
- CGF.Builder.SetInsertPoint(CGF.AllocaInsertPt);
- llvm::Value *Args[] = {EmitOpenMPUpdateLocation(CGF, Loc)};
- ThreadID = CGF.EmitRuntimeCall(
- CreateRuntimeFunction(OMPRTL__kmpc_global_thread_num), Args);
- auto &Elem = OpenMPLocThreadIDMap.FindAndConstruct(CGF.CurFn);
- Elem.second.ThreadID = ThreadID;
}
+
+ // This is not an outlined function region - need to call __kmpc_int32
+ // kmpc_global_thread_num(ident_t *loc).
+ // Generate thread id value and cache this value for use across the
+ // function.
+ CGBuilderTy::InsertPointGuard IPG(CGF.Builder);
+ CGF.Builder.SetInsertPoint(CGF.AllocaInsertPt);
+ ThreadID =
+ CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_global_thread_num),
+ emitUpdateLocation(CGF, Loc));
+ auto &Elem = OpenMPLocThreadIDMap.FindAndConstruct(CGF.CurFn);
+ Elem.second.ThreadID = ThreadID;
return ThreadID;
}
-void CGOpenMPRuntime::FunctionFinished(CodeGenFunction &CGF) {
+void CGOpenMPRuntime::functionFinished(CodeGenFunction &CGF) {
assert(CGF.CurFn && "No function in current CodeGenFunction.");
if (OpenMPLocThreadIDMap.count(CGF.CurFn))
OpenMPLocThreadIDMap.erase(CGF.CurFn);
@@ -248,7 +444,7 @@ llvm::Type *CGOpenMPRuntime::getKmpc_MicroPointerTy() {
}
llvm::Constant *
-CGOpenMPRuntime::CreateRuntimeFunction(OpenMPRTLFunction Function) {
+CGOpenMPRuntime::createRuntimeFunction(OpenMPRTLFunction Function) {
llvm::Constant *RTLFn = nullptr;
switch (Function) {
case OMPRTL__kmpc_fork_call: {
@@ -334,87 +530,6 @@ CGOpenMPRuntime::CreateRuntimeFunction(OpenMPRTLFunction Function) {
RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name*/ "__kmpc_cancel_barrier");
break;
}
- // Build __kmpc_for_static_init*(
- // ident_t *loc, kmp_int32 tid, kmp_int32 schedtype,
- // kmp_int32 *p_lastiter, kmp_int[32|64] *p_lower,
- // kmp_int[32|64] *p_upper, kmp_int[32|64] *p_stride,
- // kmp_int[32|64] incr, kmp_int[32|64] chunk);
- case OMPRTL__kmpc_for_static_init_4: {
- auto ITy = CGM.Int32Ty;
- auto PtrTy = llvm::PointerType::getUnqual(ITy);
- llvm::Type *TypeParams[] = {
- getIdentTyPointerTy(), // loc
- CGM.Int32Ty, // tid
- CGM.Int32Ty, // schedtype
- llvm::PointerType::getUnqual(CGM.Int32Ty), // p_lastiter
- PtrTy, // p_lower
- PtrTy, // p_upper
- PtrTy, // p_stride
- ITy, // incr
- ITy // chunk
- };
- llvm::FunctionType *FnTy =
- llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
- RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_for_static_init_4");
- break;
- }
- case OMPRTL__kmpc_for_static_init_4u: {
- auto ITy = CGM.Int32Ty;
- auto PtrTy = llvm::PointerType::getUnqual(ITy);
- llvm::Type *TypeParams[] = {
- getIdentTyPointerTy(), // loc
- CGM.Int32Ty, // tid
- CGM.Int32Ty, // schedtype
- llvm::PointerType::getUnqual(CGM.Int32Ty), // p_lastiter
- PtrTy, // p_lower
- PtrTy, // p_upper
- PtrTy, // p_stride
- ITy, // incr
- ITy // chunk
- };
- llvm::FunctionType *FnTy =
- llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
- RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_for_static_init_4u");
- break;
- }
- case OMPRTL__kmpc_for_static_init_8: {
- auto ITy = CGM.Int64Ty;
- auto PtrTy = llvm::PointerType::getUnqual(ITy);
- llvm::Type *TypeParams[] = {
- getIdentTyPointerTy(), // loc
- CGM.Int32Ty, // tid
- CGM.Int32Ty, // schedtype
- llvm::PointerType::getUnqual(CGM.Int32Ty), // p_lastiter
- PtrTy, // p_lower
- PtrTy, // p_upper
- PtrTy, // p_stride
- ITy, // incr
- ITy // chunk
- };
- llvm::FunctionType *FnTy =
- llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
- RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_for_static_init_8");
- break;
- }
- case OMPRTL__kmpc_for_static_init_8u: {
- auto ITy = CGM.Int64Ty;
- auto PtrTy = llvm::PointerType::getUnqual(ITy);
- llvm::Type *TypeParams[] = {
- getIdentTyPointerTy(), // loc
- CGM.Int32Ty, // tid
- CGM.Int32Ty, // schedtype
- llvm::PointerType::getUnqual(CGM.Int32Ty), // p_lastiter
- PtrTy, // p_lower
- PtrTy, // p_upper
- PtrTy, // p_stride
- ITy, // incr
- ITy // chunk
- };
- llvm::FunctionType *FnTy =
- llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
- RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_for_static_init_8u");
- break;
- }
case OMPRTL__kmpc_for_static_fini: {
// Build void __kmpc_for_static_fini(ident_t *loc, kmp_int32 global_tid);
llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
@@ -452,10 +567,10 @@ CGOpenMPRuntime::CreateRuntimeFunction(OpenMPRTLFunction Function) {
break;
}
case OMPRTL__kmpc_flush: {
- // Build void __kmpc_flush(ident_t *loc, ...);
+ // Build void __kmpc_flush(ident_t *loc);
llvm::Type *TypeParams[] = {getIdentTyPointerTy()};
llvm::FunctionType *FnTy =
- llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ true);
+ llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_flush");
break;
}
@@ -475,38 +590,291 @@ CGOpenMPRuntime::CreateRuntimeFunction(OpenMPRTLFunction Function) {
RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_end_master");
break;
}
+ case OMPRTL__kmpc_omp_taskyield: {
+ // Build kmp_int32 __kmpc_omp_taskyield(ident_t *, kmp_int32 global_tid,
+ // int end_part);
+ llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty, CGM.IntTy};
+ llvm::FunctionType *FnTy =
+ llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg=*/false);
+ RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_omp_taskyield");
+ break;
+ }
+ case OMPRTL__kmpc_single: {
+ // Build kmp_int32 __kmpc_single(ident_t *loc, kmp_int32 global_tid);
+ llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
+ llvm::FunctionType *FnTy =
+ llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg=*/false);
+ RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_single");
+ break;
+ }
+ case OMPRTL__kmpc_end_single: {
+ // Build void __kmpc_end_single(ident_t *loc, kmp_int32 global_tid);
+ llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
+ llvm::FunctionType *FnTy =
+ llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
+ RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_end_single");
+ break;
+ }
+ case OMPRTL__kmpc_omp_task_alloc: {
+ // Build kmp_task_t *__kmpc_omp_task_alloc(ident_t *, kmp_int32 gtid,
+ // kmp_int32 flags, size_t sizeof_kmp_task_t, size_t sizeof_shareds,
+ // kmp_routine_entry_t *task_entry);
+ assert(KmpRoutineEntryPtrTy != nullptr &&
+ "Type kmp_routine_entry_t must be created.");
+ llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty, CGM.Int32Ty,
+ CGM.SizeTy, CGM.SizeTy, KmpRoutineEntryPtrTy};
+ // Return void * and then cast to particular kmp_task_t type.
+ llvm::FunctionType *FnTy =
+ llvm::FunctionType::get(CGM.VoidPtrTy, TypeParams, /*isVarArg=*/false);
+ RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_omp_task_alloc");
+ break;
+ }
+ case OMPRTL__kmpc_omp_task: {
+ // Build kmp_int32 __kmpc_omp_task(ident_t *, kmp_int32 gtid, kmp_task_t
+ // *new_task);
+ llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty,
+ CGM.VoidPtrTy};
+ llvm::FunctionType *FnTy =
+ llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg=*/false);
+ RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_omp_task");
+ break;
+ }
+ case OMPRTL__kmpc_copyprivate: {
+ // Build void __kmpc_copyprivate(ident_t *loc, kmp_int32 global_tid,
+ // size_t cpy_size, void *cpy_data, void(*cpy_func)(void *, void *),
+ // kmp_int32 didit);
+ llvm::Type *CpyTypeParams[] = {CGM.VoidPtrTy, CGM.VoidPtrTy};
+ auto *CpyFnTy =
+ llvm::FunctionType::get(CGM.VoidTy, CpyTypeParams, /*isVarArg=*/false);
+ llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty, CGM.SizeTy,
+ CGM.VoidPtrTy, CpyFnTy->getPointerTo(),
+ CGM.Int32Ty};
+ llvm::FunctionType *FnTy =
+ llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
+ RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_copyprivate");
+ break;
+ }
+ case OMPRTL__kmpc_reduce: {
+ // Build kmp_int32 __kmpc_reduce(ident_t *loc, kmp_int32 global_tid,
+ // kmp_int32 num_vars, size_t reduce_size, void *reduce_data, void
+ // (*reduce_func)(void *lhs_data, void *rhs_data), kmp_critical_name *lck);
+ llvm::Type *ReduceTypeParams[] = {CGM.VoidPtrTy, CGM.VoidPtrTy};
+ auto *ReduceFnTy = llvm::FunctionType::get(CGM.VoidTy, ReduceTypeParams,
+ /*isVarArg=*/false);
+ llvm::Type *TypeParams[] = {
+ getIdentTyPointerTy(), CGM.Int32Ty, CGM.Int32Ty, CGM.SizeTy,
+ CGM.VoidPtrTy, ReduceFnTy->getPointerTo(),
+ llvm::PointerType::getUnqual(KmpCriticalNameTy)};
+ llvm::FunctionType *FnTy =
+ llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg=*/false);
+ RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_reduce");
+ break;
+ }
+ case OMPRTL__kmpc_reduce_nowait: {
+ // Build kmp_int32 __kmpc_reduce_nowait(ident_t *loc, kmp_int32
+ // global_tid, kmp_int32 num_vars, size_t reduce_size, void *reduce_data,
+ // void (*reduce_func)(void *lhs_data, void *rhs_data), kmp_critical_name
+ // *lck);
+ llvm::Type *ReduceTypeParams[] = {CGM.VoidPtrTy, CGM.VoidPtrTy};
+ auto *ReduceFnTy = llvm::FunctionType::get(CGM.VoidTy, ReduceTypeParams,
+ /*isVarArg=*/false);
+ llvm::Type *TypeParams[] = {
+ getIdentTyPointerTy(), CGM.Int32Ty, CGM.Int32Ty, CGM.SizeTy,
+ CGM.VoidPtrTy, ReduceFnTy->getPointerTo(),
+ llvm::PointerType::getUnqual(KmpCriticalNameTy)};
+ llvm::FunctionType *FnTy =
+ llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg=*/false);
+ RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_reduce_nowait");
+ break;
+ }
+ case OMPRTL__kmpc_end_reduce: {
+ // Build void __kmpc_end_reduce(ident_t *loc, kmp_int32 global_tid,
+ // kmp_critical_name *lck);
+ llvm::Type *TypeParams[] = {
+ getIdentTyPointerTy(), CGM.Int32Ty,
+ llvm::PointerType::getUnqual(KmpCriticalNameTy)};
+ llvm::FunctionType *FnTy =
+ llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
+ RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_end_reduce");
+ break;
+ }
+ case OMPRTL__kmpc_end_reduce_nowait: {
+ // Build __kmpc_end_reduce_nowait(ident_t *loc, kmp_int32 global_tid,
+ // kmp_critical_name *lck);
+ llvm::Type *TypeParams[] = {
+ getIdentTyPointerTy(), CGM.Int32Ty,
+ llvm::PointerType::getUnqual(KmpCriticalNameTy)};
+ llvm::FunctionType *FnTy =
+ llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
+ RTLFn =
+ CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_end_reduce_nowait");
+ break;
+ }
+ case OMPRTL__kmpc_omp_task_begin_if0: {
+ // Build void __kmpc_omp_task(ident_t *, kmp_int32 gtid, kmp_task_t
+ // *new_task);
+ llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty,
+ CGM.VoidPtrTy};
+ llvm::FunctionType *FnTy =
+ llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
+ RTLFn =
+ CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_omp_task_begin_if0");
+ break;
+ }
+ case OMPRTL__kmpc_omp_task_complete_if0: {
+ // Build void __kmpc_omp_task(ident_t *, kmp_int32 gtid, kmp_task_t
+ // *new_task);
+ llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty,
+ CGM.VoidPtrTy};
+ llvm::FunctionType *FnTy =
+ llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
+ RTLFn = CGM.CreateRuntimeFunction(FnTy,
+ /*Name=*/"__kmpc_omp_task_complete_if0");
+ break;
+ }
+ case OMPRTL__kmpc_ordered: {
+ // Build void __kmpc_ordered(ident_t *loc, kmp_int32 global_tid);
+ llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
+ llvm::FunctionType *FnTy =
+ llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
+ RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_ordered");
+ break;
+ }
+ case OMPRTL__kmpc_end_ordered: {
+ // Build void __kmpc_ordered(ident_t *loc, kmp_int32 global_tid);
+ llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
+ llvm::FunctionType *FnTy =
+ llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
+ RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_end_ordered");
+ break;
+ }
+ case OMPRTL__kmpc_omp_taskwait: {
+ // Build kmp_int32 __kmpc_omp_taskwait(ident_t *loc, kmp_int32 global_tid);
+ llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
+ llvm::FunctionType *FnTy =
+ llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg=*/false);
+ RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_omp_taskwait");
+ break;
+ }
}
return RTLFn;
}
+llvm::Constant *CGOpenMPRuntime::createForStaticInitFunction(unsigned IVSize,
+ bool IVSigned) {
+ assert((IVSize == 32 || IVSize == 64) &&
+ "IV size is not compatible with the omp runtime");
+ auto Name = IVSize == 32 ? (IVSigned ? "__kmpc_for_static_init_4"
+ : "__kmpc_for_static_init_4u")
+ : (IVSigned ? "__kmpc_for_static_init_8"
+ : "__kmpc_for_static_init_8u");
+ auto ITy = IVSize == 32 ? CGM.Int32Ty : CGM.Int64Ty;
+ auto PtrTy = llvm::PointerType::getUnqual(ITy);
+ llvm::Type *TypeParams[] = {
+ getIdentTyPointerTy(), // loc
+ CGM.Int32Ty, // tid
+ CGM.Int32Ty, // schedtype
+ llvm::PointerType::getUnqual(CGM.Int32Ty), // p_lastiter
+ PtrTy, // p_lower
+ PtrTy, // p_upper
+ PtrTy, // p_stride
+ ITy, // incr
+ ITy // chunk
+ };
+ llvm::FunctionType *FnTy =
+ llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
+ return CGM.CreateRuntimeFunction(FnTy, Name);
+}
+
+llvm::Constant *CGOpenMPRuntime::createDispatchInitFunction(unsigned IVSize,
+ bool IVSigned) {
+ assert((IVSize == 32 || IVSize == 64) &&
+ "IV size is not compatible with the omp runtime");
+ auto Name =
+ IVSize == 32
+ ? (IVSigned ? "__kmpc_dispatch_init_4" : "__kmpc_dispatch_init_4u")
+ : (IVSigned ? "__kmpc_dispatch_init_8" : "__kmpc_dispatch_init_8u");
+ auto ITy = IVSize == 32 ? CGM.Int32Ty : CGM.Int64Ty;
+ llvm::Type *TypeParams[] = { getIdentTyPointerTy(), // loc
+ CGM.Int32Ty, // tid
+ CGM.Int32Ty, // schedtype
+ ITy, // lower
+ ITy, // upper
+ ITy, // stride
+ ITy // chunk
+ };
+ llvm::FunctionType *FnTy =
+ llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
+ return CGM.CreateRuntimeFunction(FnTy, Name);
+}
+
+llvm::Constant *CGOpenMPRuntime::createDispatchFiniFunction(unsigned IVSize,
+ bool IVSigned) {
+ assert((IVSize == 32 || IVSize == 64) &&
+ "IV size is not compatible with the omp runtime");
+ auto Name =
+ IVSize == 32
+ ? (IVSigned ? "__kmpc_dispatch_fini_4" : "__kmpc_dispatch_fini_4u")
+ : (IVSigned ? "__kmpc_dispatch_fini_8" : "__kmpc_dispatch_fini_8u");
+ llvm::Type *TypeParams[] = {
+ getIdentTyPointerTy(), // loc
+ CGM.Int32Ty, // tid
+ };
+ llvm::FunctionType *FnTy =
+ llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
+ return CGM.CreateRuntimeFunction(FnTy, Name);
+}
+
+llvm::Constant *CGOpenMPRuntime::createDispatchNextFunction(unsigned IVSize,
+ bool IVSigned) {
+ assert((IVSize == 32 || IVSize == 64) &&
+ "IV size is not compatible with the omp runtime");
+ auto Name =
+ IVSize == 32
+ ? (IVSigned ? "__kmpc_dispatch_next_4" : "__kmpc_dispatch_next_4u")
+ : (IVSigned ? "__kmpc_dispatch_next_8" : "__kmpc_dispatch_next_8u");
+ auto ITy = IVSize == 32 ? CGM.Int32Ty : CGM.Int64Ty;
+ auto PtrTy = llvm::PointerType::getUnqual(ITy);
+ llvm::Type *TypeParams[] = {
+ getIdentTyPointerTy(), // loc
+ CGM.Int32Ty, // tid
+ llvm::PointerType::getUnqual(CGM.Int32Ty), // p_lastiter
+ PtrTy, // p_lower
+ PtrTy, // p_upper
+ PtrTy // p_stride
+ };
+ llvm::FunctionType *FnTy =
+ llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg*/ false);
+ return CGM.CreateRuntimeFunction(FnTy, Name);
+}
+
llvm::Constant *
CGOpenMPRuntime::getOrCreateThreadPrivateCache(const VarDecl *VD) {
// Lookup the entry, lazily creating it if necessary.
- return GetOrCreateInternalVariable(CGM.Int8PtrPtrTy,
+ return getOrCreateInternalVariable(CGM.Int8PtrPtrTy,
Twine(CGM.getMangledName(VD)) + ".cache.");
}
-llvm::Value *CGOpenMPRuntime::getOMPAddrOfThreadPrivate(CodeGenFunction &CGF,
- const VarDecl *VD,
- llvm::Value *VDAddr,
- SourceLocation Loc) {
+llvm::Value *CGOpenMPRuntime::getAddrOfThreadPrivate(CodeGenFunction &CGF,
+ const VarDecl *VD,
+ llvm::Value *VDAddr,
+ SourceLocation Loc) {
auto VarTy = VDAddr->getType()->getPointerElementType();
- llvm::Value *Args[] = {EmitOpenMPUpdateLocation(CGF, Loc),
- GetOpenMPThreadID(CGF, Loc),
+ llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),
CGF.Builder.CreatePointerCast(VDAddr, CGM.Int8PtrTy),
CGM.getSize(CGM.GetTargetTypeStoreSize(VarTy)),
getOrCreateThreadPrivateCache(VD)};
return CGF.EmitRuntimeCall(
- CreateRuntimeFunction(OMPRTL__kmpc_threadprivate_cached), Args);
+ createRuntimeFunction(OMPRTL__kmpc_threadprivate_cached), Args);
}
-void CGOpenMPRuntime::EmitOMPThreadPrivateVarInit(
+void CGOpenMPRuntime::emitThreadPrivateVarInit(
CodeGenFunction &CGF, llvm::Value *VDAddr, llvm::Value *Ctor,
llvm::Value *CopyCtor, llvm::Value *Dtor, SourceLocation Loc) {
// Call kmp_int32 __kmpc_global_thread_num(&loc) to init OpenMP runtime
// library.
- auto OMPLoc = EmitOpenMPUpdateLocation(CGF, Loc);
- CGF.EmitRuntimeCall(CreateRuntimeFunction(OMPRTL__kmpc_global_thread_num),
+ auto OMPLoc = emitUpdateLocation(CGF, Loc);
+ CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_global_thread_num),
OMPLoc);
// Call __kmpc_threadprivate_register(&loc, &var, ctor, cctor/*NULL*/, dtor)
// to register constructor/destructor for variable.
@@ -514,10 +882,10 @@ void CGOpenMPRuntime::EmitOMPThreadPrivateVarInit(
CGF.Builder.CreatePointerCast(VDAddr, CGM.VoidPtrTy),
Ctor, CopyCtor, Dtor};
CGF.EmitRuntimeCall(
- CreateRuntimeFunction(OMPRTL__kmpc_threadprivate_register), Args);
+ createRuntimeFunction(OMPRTL__kmpc_threadprivate_register), Args);
}
-llvm::Function *CGOpenMPRuntime::EmitOMPThreadPrivateVarDefinition(
+llvm::Function *CGOpenMPRuntime::emitThreadPrivateVarDefinition(
const VarDecl *VD, llvm::Value *VDAddr, SourceLocation Loc,
bool PerformInit, CodeGenFunction *CGF) {
VD = VD->getDefinition(CGM.getContext());
@@ -620,54 +988,121 @@ llvm::Function *CGOpenMPRuntime::EmitOMPThreadPrivateVarDefinition(
InitCGF.StartFunction(GlobalDecl(), CGM.getContext().VoidTy, InitFunction,
CGM.getTypes().arrangeNullaryFunction(), ArgList,
Loc);
- EmitOMPThreadPrivateVarInit(InitCGF, VDAddr, Ctor, CopyCtor, Dtor, Loc);
+ emitThreadPrivateVarInit(InitCGF, VDAddr, Ctor, CopyCtor, Dtor, Loc);
InitCGF.FinishFunction();
return InitFunction;
}
- EmitOMPThreadPrivateVarInit(*CGF, VDAddr, Ctor, CopyCtor, Dtor, Loc);
+ emitThreadPrivateVarInit(*CGF, VDAddr, Ctor, CopyCtor, Dtor, Loc);
}
return nullptr;
}
-void CGOpenMPRuntime::EmitOMPParallelCall(CodeGenFunction &CGF,
- SourceLocation Loc,
- llvm::Value *OutlinedFn,
- llvm::Value *CapturedStruct) {
- // Build call __kmpc_fork_call(loc, 1, microtask, captured_struct/*context*/)
- llvm::Value *Args[] = {
- EmitOpenMPUpdateLocation(CGF, Loc),
- CGF.Builder.getInt32(1), // Number of arguments after 'microtask' argument
- // (there is only one additional argument - 'context')
- CGF.Builder.CreateBitCast(OutlinedFn, getKmpc_MicroPointerTy()),
- CGF.EmitCastToVoidPtr(CapturedStruct)};
- auto RTLFn = CreateRuntimeFunction(OMPRTL__kmpc_fork_call);
- CGF.EmitRuntimeCall(RTLFn, Args);
-}
-
-void CGOpenMPRuntime::EmitOMPSerialCall(CodeGenFunction &CGF,
- SourceLocation Loc,
- llvm::Value *OutlinedFn,
- llvm::Value *CapturedStruct) {
- auto ThreadID = GetOpenMPThreadID(CGF, Loc);
- // Build calls:
- // __kmpc_serialized_parallel(&Loc, GTid);
- llvm::Value *SerArgs[] = {EmitOpenMPUpdateLocation(CGF, Loc), ThreadID};
- auto RTLFn = CreateRuntimeFunction(OMPRTL__kmpc_serialized_parallel);
- CGF.EmitRuntimeCall(RTLFn, SerArgs);
-
- // OutlinedFn(&GTid, &zero, CapturedStruct);
- auto ThreadIDAddr = EmitThreadIDAddress(CGF, Loc);
- auto Int32Ty =
- CGF.getContext().getIntTypeForBitwidth(/*DestWidth*/ 32, /*Signed*/ true);
- auto ZeroAddr = CGF.CreateMemTemp(Int32Ty, /*Name*/ ".zero.addr");
- CGF.InitTempAlloca(ZeroAddr, CGF.Builder.getInt32(/*C*/ 0));
- llvm::Value *OutlinedFnArgs[] = {ThreadIDAddr, ZeroAddr, CapturedStruct};
- CGF.EmitCallOrInvoke(OutlinedFn, OutlinedFnArgs);
+/// \brief Emits code for OpenMP 'if' clause using specified \a CodeGen
+/// function. Here is the logic:
+/// if (Cond) {
+/// ThenGen();
+/// } else {
+/// ElseGen();
+/// }
+static void emitOMPIfClause(CodeGenFunction &CGF, const Expr *Cond,
+ const RegionCodeGenTy &ThenGen,
+ const RegionCodeGenTy &ElseGen) {
+ CodeGenFunction::LexicalScope ConditionScope(CGF, Cond->getSourceRange());
+
+ // If the condition constant folds and can be elided, try to avoid emitting
+ // the condition and the dead arm of the if/else.
+ bool CondConstant;
+ if (CGF.ConstantFoldsToSimpleInteger(Cond, CondConstant)) {
+ CodeGenFunction::RunCleanupsScope Scope(CGF);
+ if (CondConstant) {
+ ThenGen(CGF);
+ } else {
+ ElseGen(CGF);
+ }
+ return;
+ }
+
+ // Otherwise, the condition did not fold, or we couldn't elide it. Just
+ // emit the conditional branch.
+ auto ThenBlock = CGF.createBasicBlock("omp_if.then");
+ auto ElseBlock = CGF.createBasicBlock("omp_if.else");
+ auto ContBlock = CGF.createBasicBlock("omp_if.end");
+ CGF.EmitBranchOnBoolExpr(Cond, ThenBlock, ElseBlock, /*TrueCount=*/0);
- // __kmpc_end_serialized_parallel(&Loc, GTid);
- llvm::Value *EndSerArgs[] = {EmitOpenMPUpdateLocation(CGF, Loc), ThreadID};
- RTLFn = CreateRuntimeFunction(OMPRTL__kmpc_end_serialized_parallel);
- CGF.EmitRuntimeCall(RTLFn, EndSerArgs);
+ // Emit the 'then' code.
+ CGF.EmitBlock(ThenBlock);
+ {
+ CodeGenFunction::RunCleanupsScope ThenScope(CGF);
+ ThenGen(CGF);
+ }
+ CGF.EmitBranch(ContBlock);
+ // Emit the 'else' code if present.
+ {
+ // There is no need to emit line number for unconditional branch.
+ auto NL = ApplyDebugLocation::CreateEmpty(CGF);
+ CGF.EmitBlock(ElseBlock);
+ }
+ {
+ CodeGenFunction::RunCleanupsScope ThenScope(CGF);
+ ElseGen(CGF);
+ }
+ {
+ // There is no need to emit line number for unconditional branch.
+ auto NL = ApplyDebugLocation::CreateEmpty(CGF);
+ CGF.EmitBranch(ContBlock);
+ }
+ // Emit the continuation block for code after the if.
+ CGF.EmitBlock(ContBlock, /*IsFinished=*/true);
+}
+
+void CGOpenMPRuntime::emitParallelCall(CodeGenFunction &CGF, SourceLocation Loc,
+ llvm::Value *OutlinedFn,
+ llvm::Value *CapturedStruct,
+ const Expr *IfCond) {
+ auto *RTLoc = emitUpdateLocation(CGF, Loc);
+ auto &&ThenGen =
+ [this, OutlinedFn, CapturedStruct, RTLoc](CodeGenFunction &CGF) {
+ // Build call __kmpc_fork_call(loc, 1, microtask,
+ // captured_struct/*context*/)
+ llvm::Value *Args[] = {
+ RTLoc,
+ CGF.Builder.getInt32(
+ 1), // Number of arguments after 'microtask' argument
+ // (there is only one additional argument - 'context')
+ CGF.Builder.CreateBitCast(OutlinedFn, getKmpc_MicroPointerTy()),
+ CGF.EmitCastToVoidPtr(CapturedStruct)};
+ auto RTLFn = createRuntimeFunction(OMPRTL__kmpc_fork_call);
+ CGF.EmitRuntimeCall(RTLFn, Args);
+ };
+ auto &&ElseGen = [this, OutlinedFn, CapturedStruct, RTLoc, Loc](
+ CodeGenFunction &CGF) {
+ auto ThreadID = getThreadID(CGF, Loc);
+ // Build calls:
+ // __kmpc_serialized_parallel(&Loc, GTid);
+ llvm::Value *Args[] = {RTLoc, ThreadID};
+ CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_serialized_parallel),
+ Args);
+
+ // OutlinedFn(&GTid, &zero, CapturedStruct);
+ auto ThreadIDAddr = emitThreadIDAddress(CGF, Loc);
+ auto Int32Ty = CGF.getContext().getIntTypeForBitwidth(/*DestWidth*/ 32,
+ /*Signed*/ true);
+ auto ZeroAddr = CGF.CreateMemTemp(Int32Ty, /*Name*/ ".zero.addr");
+ CGF.InitTempAlloca(ZeroAddr, CGF.Builder.getInt32(/*C*/ 0));
+ llvm::Value *OutlinedFnArgs[] = {ThreadIDAddr, ZeroAddr, CapturedStruct};
+ CGF.EmitCallOrInvoke(OutlinedFn, OutlinedFnArgs);
+
+ // __kmpc_end_serialized_parallel(&Loc, GTid);
+ llvm::Value *EndArgs[] = {emitUpdateLocation(CGF, Loc), ThreadID};
+ CGF.EmitRuntimeCall(
+ createRuntimeFunction(OMPRTL__kmpc_end_serialized_parallel), EndArgs);
+ };
+ if (IfCond) {
+ emitOMPIfClause(CGF, IfCond, ThenGen, ElseGen);
+ } else {
+ CodeGenFunction::RunCleanupsScope Scope(CGF);
+ ThenGen(CGF);
+ }
}
// If we're inside an (outlined) parallel region, use the region info's
@@ -676,13 +1111,14 @@ void CGOpenMPRuntime::EmitOMPSerialCall(CodeGenFunction &CGF,
// regular serial code region, get thread ID by calling kmp_int32
// kmpc_global_thread_num(ident_t *loc), stash this thread ID in a temporary and
// return the address of that temp.
-llvm::Value *CGOpenMPRuntime::EmitThreadIDAddress(CodeGenFunction &CGF,
+llvm::Value *CGOpenMPRuntime::emitThreadIDAddress(CodeGenFunction &CGF,
SourceLocation Loc) {
if (auto OMPRegionInfo =
dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo))
- return CGF.EmitLoadOfLValue(OMPRegionInfo->getThreadIDVariableLValue(CGF),
- SourceLocation()).getScalarVal();
- auto ThreadID = GetOpenMPThreadID(CGF, Loc);
+ if (OMPRegionInfo->getThreadIDVariable())
+ return OMPRegionInfo->getThreadIDVariableLValue(CGF).getAddress();
+
+ auto ThreadID = getThreadID(CGF, Loc);
auto Int32Ty =
CGF.getContext().getIntTypeForBitwidth(/*DestWidth*/ 32, /*Signed*/ true);
auto ThreadIDTemp = CGF.CreateMemTemp(Int32Ty, /*Name*/ ".threadid_temp.");
@@ -693,7 +1129,7 @@ llvm::Value *CGOpenMPRuntime::EmitThreadIDAddress(CodeGenFunction &CGF,
}
llvm::Constant *
-CGOpenMPRuntime::GetOrCreateInternalVariable(llvm::Type *Ty,
+CGOpenMPRuntime::getOrCreateInternalVariable(llvm::Type *Ty,
const llvm::Twine &Name) {
SmallString<256> Buffer;
llvm::raw_svector_ostream Out(Buffer);
@@ -712,31 +1148,51 @@ CGOpenMPRuntime::GetOrCreateInternalVariable(llvm::Type *Ty,
Elem.first());
}
-llvm::Value *CGOpenMPRuntime::GetCriticalRegionLock(StringRef CriticalName) {
+llvm::Value *CGOpenMPRuntime::getCriticalRegionLock(StringRef CriticalName) {
llvm::Twine Name(".gomp_critical_user_", CriticalName);
- return GetOrCreateInternalVariable(KmpCriticalNameTy, Name.concat(".var"));
+ return getOrCreateInternalVariable(KmpCriticalNameTy, Name.concat(".var"));
}
-void CGOpenMPRuntime::EmitOMPCriticalRegion(
- CodeGenFunction &CGF, StringRef CriticalName,
- const std::function<void()> &CriticalOpGen, SourceLocation Loc) {
- auto RegionLock = GetCriticalRegionLock(CriticalName);
+namespace {
+template <size_t N> class CallEndCleanup : public EHScopeStack::Cleanup {
+ llvm::Value *Callee;
+ llvm::Value *Args[N];
+
+public:
+ CallEndCleanup(llvm::Value *Callee, ArrayRef<llvm::Value *> CleanupArgs)
+ : Callee(Callee) {
+ assert(CleanupArgs.size() == N);
+ std::copy(CleanupArgs.begin(), CleanupArgs.end(), std::begin(Args));
+ }
+ void Emit(CodeGenFunction &CGF, Flags /*flags*/) override {
+ CGF.EmitRuntimeCall(Callee, Args);
+ }
+};
+} // namespace
+
+void CGOpenMPRuntime::emitCriticalRegion(CodeGenFunction &CGF,
+ StringRef CriticalName,
+ const RegionCodeGenTy &CriticalOpGen,
+ SourceLocation Loc) {
// __kmpc_critical(ident_t *, gtid, Lock);
// CriticalOpGen();
// __kmpc_end_critical(ident_t *, gtid, Lock);
// Prepare arguments and build a call to __kmpc_critical
- llvm::Value *Args[] = {EmitOpenMPUpdateLocation(CGF, Loc),
- GetOpenMPThreadID(CGF, Loc), RegionLock};
- auto RTLFn = CreateRuntimeFunction(OMPRTL__kmpc_critical);
- CGF.EmitRuntimeCall(RTLFn, Args);
- CriticalOpGen();
- // Build a call to __kmpc_end_critical
- RTLFn = CreateRuntimeFunction(OMPRTL__kmpc_end_critical);
- CGF.EmitRuntimeCall(RTLFn, Args);
-}
-
-static void EmitOMPIfStmt(CodeGenFunction &CGF, llvm::Value *IfCond,
- const std::function<void()> &BodyOpGen) {
+ {
+ CodeGenFunction::RunCleanupsScope Scope(CGF);
+ llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),
+ getCriticalRegionLock(CriticalName)};
+ CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_critical), Args);
+ // Build a call to __kmpc_end_critical
+ CGF.EHStack.pushCleanup<CallEndCleanup<std::extent<decltype(Args)>::value>>(
+ NormalAndEHCleanup, createRuntimeFunction(OMPRTL__kmpc_end_critical),
+ llvm::makeArrayRef(Args));
+ emitInlinedDirective(CGF, CriticalOpGen);
+ }
+}
+
+static void emitIfStmt(CodeGenFunction &CGF, llvm::Value *IfCond,
+ const RegionCodeGenTy &BodyOpGen) {
llvm::Value *CallBool = CGF.EmitScalarConversion(
IfCond,
CGF.getContext().getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/true),
@@ -747,61 +1203,234 @@ static void EmitOMPIfStmt(CodeGenFunction &CGF, llvm::Value *IfCond,
// Generate the branch (If-stmt)
CGF.Builder.CreateCondBr(CallBool, ThenBlock, ContBlock);
CGF.EmitBlock(ThenBlock);
- BodyOpGen();
+ CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, BodyOpGen);
// Emit the rest of bblocks/branches
CGF.EmitBranch(ContBlock);
CGF.EmitBlock(ContBlock, true);
}
-void CGOpenMPRuntime::EmitOMPMasterRegion(
- CodeGenFunction &CGF, const std::function<void()> &MasterOpGen,
- SourceLocation Loc) {
+void CGOpenMPRuntime::emitMasterRegion(CodeGenFunction &CGF,
+ const RegionCodeGenTy &MasterOpGen,
+ SourceLocation Loc) {
// if(__kmpc_master(ident_t *, gtid)) {
// MasterOpGen();
// __kmpc_end_master(ident_t *, gtid);
// }
// Prepare arguments and build a call to __kmpc_master
- llvm::Value *Args[] = {EmitOpenMPUpdateLocation(CGF, Loc),
- GetOpenMPThreadID(CGF, Loc)};
- auto RTLFn = CreateRuntimeFunction(OMPRTL__kmpc_master);
- auto *IsMaster = CGF.EmitRuntimeCall(RTLFn, Args);
- EmitOMPIfStmt(CGF, IsMaster, [&]() -> void {
- MasterOpGen();
- // Build a call to __kmpc_end_master.
- // OpenMP [1.2.2 OpenMP Language Terminology]
- // For C/C++, an executable statement, possibly compound, with a single
- // entry at the top and a single exit at the bottom, or an OpenMP construct.
- // * Access to the structured block must not be the result of a branch.
- // * The point of exit cannot be a branch out of the structured block.
- // * The point of entry must not be a call to setjmp().
- // * longjmp() and throw() must not violate the entry/exit criteria.
- // * An expression statement, iteration statement, selection statement, or
- // try block is considered to be a structured block if the corresponding
- // compound statement obtained by enclosing it in { and } would be a
- // structured block.
- // It is analyzed in Sema, so we can just call __kmpc_end_master() on
- // fallthrough rather than pushing a normal cleanup for it.
- RTLFn = CreateRuntimeFunction(OMPRTL__kmpc_end_master);
- CGF.EmitRuntimeCall(RTLFn, Args);
+ llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)};
+ auto *IsMaster =
+ CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_master), Args);
+ typedef CallEndCleanup<std::extent<decltype(Args)>::value>
+ MasterCallEndCleanup;
+ emitIfStmt(CGF, IsMaster, [&](CodeGenFunction &CGF) -> void {
+ CodeGenFunction::RunCleanupsScope Scope(CGF);
+ CGF.EHStack.pushCleanup<MasterCallEndCleanup>(
+ NormalAndEHCleanup, createRuntimeFunction(OMPRTL__kmpc_end_master),
+ llvm::makeArrayRef(Args));
+ MasterOpGen(CGF);
});
}
-void CGOpenMPRuntime::EmitOMPBarrierCall(CodeGenFunction &CGF,
- SourceLocation Loc, bool IsExplicit) {
+void CGOpenMPRuntime::emitTaskyieldCall(CodeGenFunction &CGF,
+ SourceLocation Loc) {
+ // Build call __kmpc_omp_taskyield(loc, thread_id, 0);
+ llvm::Value *Args[] = {
+ emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),
+ llvm::ConstantInt::get(CGM.IntTy, /*V=*/0, /*isSigned=*/true)};
+ CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_omp_taskyield), Args);
+}
+
+static llvm::Value *emitCopyprivateCopyFunction(
+ CodeGenModule &CGM, llvm::Type *ArgsType,
+ ArrayRef<const Expr *> CopyprivateVars, ArrayRef<const Expr *> DestExprs,
+ ArrayRef<const Expr *> SrcExprs, ArrayRef<const Expr *> AssignmentOps) {
+ auto &C = CGM.getContext();
+ // void copy_func(void *LHSArg, void *RHSArg);
+ FunctionArgList Args;
+ ImplicitParamDecl LHSArg(C, /*DC=*/nullptr, SourceLocation(), /*Id=*/nullptr,
+ C.VoidPtrTy);
+ ImplicitParamDecl RHSArg(C, /*DC=*/nullptr, SourceLocation(), /*Id=*/nullptr,
+ C.VoidPtrTy);
+ Args.push_back(&LHSArg);
+ Args.push_back(&RHSArg);
+ FunctionType::ExtInfo EI;
+ auto &CGFI = CGM.getTypes().arrangeFreeFunctionDeclaration(
+ C.VoidTy, Args, EI, /*isVariadic=*/false);
+ auto *Fn = llvm::Function::Create(
+ CGM.getTypes().GetFunctionType(CGFI), llvm::GlobalValue::InternalLinkage,
+ ".omp.copyprivate.copy_func", &CGM.getModule());
+ CGM.SetLLVMFunctionAttributes(/*D=*/nullptr, CGFI, Fn);
+ CodeGenFunction CGF(CGM);
+ CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, CGFI, Args);
+ // Dest = (void*[n])(LHSArg);
+ // Src = (void*[n])(RHSArg);
+ auto *LHS = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
+ CGF.Builder.CreateAlignedLoad(CGF.GetAddrOfLocalVar(&LHSArg),
+ CGF.PointerAlignInBytes),
+ ArgsType);
+ auto *RHS = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
+ CGF.Builder.CreateAlignedLoad(CGF.GetAddrOfLocalVar(&RHSArg),
+ CGF.PointerAlignInBytes),
+ ArgsType);
+ // *(Type0*)Dst[0] = *(Type0*)Src[0];
+ // *(Type1*)Dst[1] = *(Type1*)Src[1];
+ // ...
+ // *(Typen*)Dst[n] = *(Typen*)Src[n];
+ for (unsigned I = 0, E = AssignmentOps.size(); I < E; ++I) {
+ auto *DestAddr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
+ CGF.Builder.CreateAlignedLoad(
+ CGF.Builder.CreateStructGEP(nullptr, LHS, I),
+ CGM.PointerAlignInBytes),
+ CGF.ConvertTypeForMem(C.getPointerType(SrcExprs[I]->getType())));
+ auto *SrcAddr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
+ CGF.Builder.CreateAlignedLoad(
+ CGF.Builder.CreateStructGEP(nullptr, RHS, I),
+ CGM.PointerAlignInBytes),
+ CGF.ConvertTypeForMem(C.getPointerType(SrcExprs[I]->getType())));
+ auto *VD = cast<DeclRefExpr>(CopyprivateVars[I])->getDecl();
+ QualType Type = VD->getType();
+ CGF.EmitOMPCopy(CGF, Type, DestAddr, SrcAddr,
+ cast<VarDecl>(cast<DeclRefExpr>(DestExprs[I])->getDecl()),
+ cast<VarDecl>(cast<DeclRefExpr>(SrcExprs[I])->getDecl()),
+ AssignmentOps[I]);
+ }
+ CGF.FinishFunction();
+ return Fn;
+}
+
+void CGOpenMPRuntime::emitSingleRegion(CodeGenFunction &CGF,
+ const RegionCodeGenTy &SingleOpGen,
+ SourceLocation Loc,
+ ArrayRef<const Expr *> CopyprivateVars,
+ ArrayRef<const Expr *> SrcExprs,
+ ArrayRef<const Expr *> DstExprs,
+ ArrayRef<const Expr *> AssignmentOps) {
+ assert(CopyprivateVars.size() == SrcExprs.size() &&
+ CopyprivateVars.size() == DstExprs.size() &&
+ CopyprivateVars.size() == AssignmentOps.size());
+ auto &C = CGM.getContext();
+ // int32 did_it = 0;
+ // if(__kmpc_single(ident_t *, gtid)) {
+ // SingleOpGen();
+ // __kmpc_end_single(ident_t *, gtid);
+ // did_it = 1;
+ // }
+ // call __kmpc_copyprivate(ident_t *, gtid, <buf_size>, <copyprivate list>,
+ // <copy_func>, did_it);
+
+ llvm::AllocaInst *DidIt = nullptr;
+ if (!CopyprivateVars.empty()) {
+ // int32 did_it = 0;
+ auto KmpInt32Ty = C.getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/1);
+ DidIt = CGF.CreateMemTemp(KmpInt32Ty, ".omp.copyprivate.did_it");
+ CGF.Builder.CreateAlignedStore(CGF.Builder.getInt32(0), DidIt,
+ DidIt->getAlignment());
+ }
+ // Prepare arguments and build a call to __kmpc_single
+ llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)};
+ auto *IsSingle =
+ CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_single), Args);
+ typedef CallEndCleanup<std::extent<decltype(Args)>::value>
+ SingleCallEndCleanup;
+ emitIfStmt(CGF, IsSingle, [&](CodeGenFunction &CGF) -> void {
+ CodeGenFunction::RunCleanupsScope Scope(CGF);
+ CGF.EHStack.pushCleanup<SingleCallEndCleanup>(
+ NormalAndEHCleanup, createRuntimeFunction(OMPRTL__kmpc_end_single),
+ llvm::makeArrayRef(Args));
+ SingleOpGen(CGF);
+ if (DidIt) {
+ // did_it = 1;
+ CGF.Builder.CreateAlignedStore(CGF.Builder.getInt32(1), DidIt,
+ DidIt->getAlignment());
+ }
+ });
+ // call __kmpc_copyprivate(ident_t *, gtid, <buf_size>, <copyprivate list>,
+ // <copy_func>, did_it);
+ if (DidIt) {
+ llvm::APInt ArraySize(/*unsigned int numBits=*/32, CopyprivateVars.size());
+ auto CopyprivateArrayTy =
+ C.getConstantArrayType(C.VoidPtrTy, ArraySize, ArrayType::Normal,
+ /*IndexTypeQuals=*/0);
+ // Create a list of all private variables for copyprivate.
+ auto *CopyprivateList =
+ CGF.CreateMemTemp(CopyprivateArrayTy, ".omp.copyprivate.cpr_list");
+ for (unsigned I = 0, E = CopyprivateVars.size(); I < E; ++I) {
+ auto *Elem = CGF.Builder.CreateStructGEP(
+ CopyprivateList->getAllocatedType(), CopyprivateList, I);
+ CGF.Builder.CreateAlignedStore(
+ CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
+ CGF.EmitLValue(CopyprivateVars[I]).getAddress(), CGF.VoidPtrTy),
+ Elem, CGM.PointerAlignInBytes);
+ }
+ // Build function that copies private values from single region to all other
+ // threads in the corresponding parallel region.
+ auto *CpyFn = emitCopyprivateCopyFunction(
+ CGM, CGF.ConvertTypeForMem(CopyprivateArrayTy)->getPointerTo(),
+ CopyprivateVars, SrcExprs, DstExprs, AssignmentOps);
+ auto *BufSize = llvm::ConstantInt::get(
+ CGM.SizeTy, C.getTypeSizeInChars(CopyprivateArrayTy).getQuantity());
+ auto *CL = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(CopyprivateList,
+ CGF.VoidPtrTy);
+ auto *DidItVal =
+ CGF.Builder.CreateAlignedLoad(DidIt, CGF.PointerAlignInBytes);
+ llvm::Value *Args[] = {
+ emitUpdateLocation(CGF, Loc), // ident_t *<loc>
+ getThreadID(CGF, Loc), // i32 <gtid>
+ BufSize, // size_t <buf_size>
+ CL, // void *<copyprivate list>
+ CpyFn, // void (*) (void *, void *) <copy_func>
+ DidItVal // i32 did_it
+ };
+ CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_copyprivate), Args);
+ }
+}
+
+void CGOpenMPRuntime::emitOrderedRegion(CodeGenFunction &CGF,
+ const RegionCodeGenTy &OrderedOpGen,
+ SourceLocation Loc) {
+ // __kmpc_ordered(ident_t *, gtid);
+ // OrderedOpGen();
+ // __kmpc_end_ordered(ident_t *, gtid);
+ // Prepare arguments and build a call to __kmpc_ordered
+ {
+ CodeGenFunction::RunCleanupsScope Scope(CGF);
+ llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)};
+ CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_ordered), Args);
+ // Build a call to __kmpc_end_ordered
+ CGF.EHStack.pushCleanup<CallEndCleanup<std::extent<decltype(Args)>::value>>(
+ NormalAndEHCleanup, createRuntimeFunction(OMPRTL__kmpc_end_ordered),
+ llvm::makeArrayRef(Args));
+ emitInlinedDirective(CGF, OrderedOpGen);
+ }
+}
+
+void CGOpenMPRuntime::emitBarrierCall(CodeGenFunction &CGF, SourceLocation Loc,
+ OpenMPDirectiveKind Kind) {
// Build call __kmpc_cancel_barrier(loc, thread_id);
- auto Flags = static_cast<OpenMPLocationFlags>(
- OMP_IDENT_KMPC |
- (IsExplicit ? OMP_IDENT_BARRIER_EXPL : OMP_IDENT_BARRIER_IMPL));
+ OpenMPLocationFlags Flags = OMP_IDENT_KMPC;
+ if (Kind == OMPD_for) {
+ Flags =
+ static_cast<OpenMPLocationFlags>(Flags | OMP_IDENT_BARRIER_IMPL_FOR);
+ } else if (Kind == OMPD_sections) {
+ Flags = static_cast<OpenMPLocationFlags>(Flags |
+ OMP_IDENT_BARRIER_IMPL_SECTIONS);
+ } else if (Kind == OMPD_single) {
+ Flags =
+ static_cast<OpenMPLocationFlags>(Flags | OMP_IDENT_BARRIER_IMPL_SINGLE);
+ } else if (Kind == OMPD_barrier) {
+ Flags = static_cast<OpenMPLocationFlags>(Flags | OMP_IDENT_BARRIER_EXPL);
+ } else {
+ Flags = static_cast<OpenMPLocationFlags>(Flags | OMP_IDENT_BARRIER_IMPL);
+ }
// Build call __kmpc_cancel_barrier(loc, thread_id);
// Replace __kmpc_barrier() function by __kmpc_cancel_barrier() because this
// one provides the same functionality and adds initial support for
// cancellation constructs introduced in OpenMP 4.0. __kmpc_cancel_barrier()
// is provided default by the runtime library so it safe to make such
// replacement.
- llvm::Value *Args[] = {EmitOpenMPUpdateLocation(CGF, Loc, Flags),
- GetOpenMPThreadID(CGF, Loc)};
- auto RTLFn = CreateRuntimeFunction(OMPRTL__kmpc_cancel_barrier);
- CGF.EmitRuntimeCall(RTLFn, Args);
+ llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc, Flags),
+ getThreadID(CGF, Loc)};
+ CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_cancel_barrier), Args);
}
/// \brief Schedule types for 'omp for' loops (these enumerators are taken from
@@ -817,106 +1446,1017 @@ enum OpenMPSchedType {
OMP_sch_auto = 38,
/// \brief Lower bound for 'ordered' versions.
OMP_ord_lower = 64,
- /// \brief Lower bound for 'nomerge' versions.
- OMP_nm_lower = 160,
+ OMP_ord_static_chunked = 65,
+ OMP_ord_static = 66,
+ OMP_ord_dynamic_chunked = 67,
+ OMP_ord_guided_chunked = 68,
+ OMP_ord_runtime = 69,
+ OMP_ord_auto = 70,
+ OMP_sch_default = OMP_sch_static,
};
/// \brief Map the OpenMP loop schedule to the runtime enumeration.
static OpenMPSchedType getRuntimeSchedule(OpenMPScheduleClauseKind ScheduleKind,
- bool Chunked) {
+ bool Chunked, bool Ordered) {
switch (ScheduleKind) {
case OMPC_SCHEDULE_static:
- return Chunked ? OMP_sch_static_chunked : OMP_sch_static;
+ return Chunked ? (Ordered ? OMP_ord_static_chunked : OMP_sch_static_chunked)
+ : (Ordered ? OMP_ord_static : OMP_sch_static);
case OMPC_SCHEDULE_dynamic:
- return OMP_sch_dynamic_chunked;
+ return Ordered ? OMP_ord_dynamic_chunked : OMP_sch_dynamic_chunked;
case OMPC_SCHEDULE_guided:
- return OMP_sch_guided_chunked;
- case OMPC_SCHEDULE_auto:
- return OMP_sch_auto;
+ return Ordered ? OMP_ord_guided_chunked : OMP_sch_guided_chunked;
case OMPC_SCHEDULE_runtime:
- return OMP_sch_runtime;
+ return Ordered ? OMP_ord_runtime : OMP_sch_runtime;
+ case OMPC_SCHEDULE_auto:
+ return Ordered ? OMP_ord_auto : OMP_sch_auto;
case OMPC_SCHEDULE_unknown:
assert(!Chunked && "chunk was specified but schedule kind not known");
- return OMP_sch_static;
+ return Ordered ? OMP_ord_static : OMP_sch_static;
}
llvm_unreachable("Unexpected runtime schedule");
}
bool CGOpenMPRuntime::isStaticNonchunked(OpenMPScheduleClauseKind ScheduleKind,
bool Chunked) const {
- auto Schedule = getRuntimeSchedule(ScheduleKind, Chunked);
+ auto Schedule = getRuntimeSchedule(ScheduleKind, Chunked, /*Ordered=*/false);
return Schedule == OMP_sch_static;
}
-void CGOpenMPRuntime::EmitOMPForInit(CodeGenFunction &CGF, SourceLocation Loc,
- OpenMPScheduleClauseKind ScheduleKind,
- unsigned IVSize, bool IVSigned,
- llvm::Value *IL, llvm::Value *LB,
- llvm::Value *UB, llvm::Value *ST,
- llvm::Value *Chunk) {
- OpenMPSchedType Schedule = getRuntimeSchedule(ScheduleKind, Chunk != nullptr);
- // Call __kmpc_for_static_init(
- // ident_t *loc, kmp_int32 tid, kmp_int32 schedtype,
- // kmp_int32 *p_lastiter, kmp_int[32|64] *p_lower,
- // kmp_int[32|64] *p_upper, kmp_int[32|64] *p_stride,
- // kmp_int[32|64] incr, kmp_int[32|64] chunk);
- // TODO: Implement dynamic schedule.
-
- // If the Chunk was not specified in the clause - use default value 1.
- if (Chunk == nullptr)
- Chunk = CGF.Builder.getIntN(IVSize, /*C*/ 1);
+bool CGOpenMPRuntime::isDynamic(OpenMPScheduleClauseKind ScheduleKind) const {
+ auto Schedule =
+ getRuntimeSchedule(ScheduleKind, /*Chunked=*/false, /*Ordered=*/false);
+ assert(Schedule != OMP_sch_static_chunked && "cannot be chunked here");
+ return Schedule != OMP_sch_static;
+}
+
+void CGOpenMPRuntime::emitForInit(CodeGenFunction &CGF, SourceLocation Loc,
+ OpenMPScheduleClauseKind ScheduleKind,
+ unsigned IVSize, bool IVSigned, bool Ordered,
+ llvm::Value *IL, llvm::Value *LB,
+ llvm::Value *UB, llvm::Value *ST,
+ llvm::Value *Chunk) {
+ OpenMPSchedType Schedule =
+ getRuntimeSchedule(ScheduleKind, Chunk != nullptr, Ordered);
+ if (Ordered ||
+ (Schedule != OMP_sch_static && Schedule != OMP_sch_static_chunked &&
+ Schedule != OMP_ord_static && Schedule != OMP_ord_static_chunked)) {
+ // Call __kmpc_dispatch_init(
+ // ident_t *loc, kmp_int32 tid, kmp_int32 schedule,
+ // kmp_int[32|64] lower, kmp_int[32|64] upper,
+ // kmp_int[32|64] stride, kmp_int[32|64] chunk);
+
+ // If the Chunk was not specified in the clause - use default value 1.
+ if (Chunk == nullptr)
+ Chunk = CGF.Builder.getIntN(IVSize, 1);
+ llvm::Value *Args[] = { emitUpdateLocation(CGF, Loc, OMP_IDENT_KMPC),
+ getThreadID(CGF, Loc),
+ CGF.Builder.getInt32(Schedule), // Schedule type
+ CGF.Builder.getIntN(IVSize, 0), // Lower
+ UB, // Upper
+ CGF.Builder.getIntN(IVSize, 1), // Stride
+ Chunk // Chunk
+ };
+ CGF.EmitRuntimeCall(createDispatchInitFunction(IVSize, IVSigned), Args);
+ } else {
+ // Call __kmpc_for_static_init(
+ // ident_t *loc, kmp_int32 tid, kmp_int32 schedtype,
+ // kmp_int32 *p_lastiter, kmp_int[32|64] *p_lower,
+ // kmp_int[32|64] *p_upper, kmp_int[32|64] *p_stride,
+ // kmp_int[32|64] incr, kmp_int[32|64] chunk);
+ if (Chunk == nullptr) {
+ assert((Schedule == OMP_sch_static || Schedule == OMP_ord_static) &&
+ "expected static non-chunked schedule");
+ // If the Chunk was not specified in the clause - use default value 1.
+ Chunk = CGF.Builder.getIntN(IVSize, 1);
+ } else
+ assert((Schedule == OMP_sch_static_chunked ||
+ Schedule == OMP_ord_static_chunked) &&
+ "expected static chunked schedule");
+ llvm::Value *Args[] = { emitUpdateLocation(CGF, Loc, OMP_IDENT_KMPC),
+ getThreadID(CGF, Loc),
+ CGF.Builder.getInt32(Schedule), // Schedule type
+ IL, // &isLastIter
+ LB, // &LB
+ UB, // &UB
+ ST, // &Stride
+ CGF.Builder.getIntN(IVSize, 1), // Incr
+ Chunk // Chunk
+ };
+ CGF.EmitRuntimeCall(createForStaticInitFunction(IVSize, IVSigned), Args);
+ }
+}
+void CGOpenMPRuntime::emitForStaticFinish(CodeGenFunction &CGF,
+ SourceLocation Loc) {
+ // Call __kmpc_for_static_fini(ident_t *loc, kmp_int32 tid);
+ llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc, OMP_IDENT_KMPC),
+ getThreadID(CGF, Loc)};
+ CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_for_static_fini),
+ Args);
+}
+
+void CGOpenMPRuntime::emitForOrderedIterationEnd(CodeGenFunction &CGF,
+ SourceLocation Loc,
+ unsigned IVSize,
+ bool IVSigned) {
+ // Call __kmpc_for_dynamic_fini_(4|8)[u](ident_t *loc, kmp_int32 tid);
+ llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc, OMP_IDENT_KMPC),
+ getThreadID(CGF, Loc)};
+ CGF.EmitRuntimeCall(createDispatchFiniFunction(IVSize, IVSigned), Args);
+}
+
+llvm::Value *CGOpenMPRuntime::emitForNext(CodeGenFunction &CGF,
+ SourceLocation Loc, unsigned IVSize,
+ bool IVSigned, llvm::Value *IL,
+ llvm::Value *LB, llvm::Value *UB,
+ llvm::Value *ST) {
+ // Call __kmpc_dispatch_next(
+ // ident_t *loc, kmp_int32 tid, kmp_int32 *p_lastiter,
+ // kmp_int[32|64] *p_lower, kmp_int[32|64] *p_upper,
+ // kmp_int[32|64] *p_stride);
llvm::Value *Args[] = {
- EmitOpenMPUpdateLocation(CGF, Loc, OMP_IDENT_KMPC),
- GetOpenMPThreadID(CGF, Loc),
- CGF.Builder.getInt32(Schedule), // Schedule type
- IL, // &isLastIter
- LB, // &LB
- UB, // &UB
- ST, // &Stride
- CGF.Builder.getIntN(IVSize, 1), // Incr
- Chunk // Chunk
+ emitUpdateLocation(CGF, Loc, OMP_IDENT_KMPC), getThreadID(CGF, Loc),
+ IL, // &isLastIter
+ LB, // &Lower
+ UB, // &Upper
+ ST // &Stride
};
- assert((IVSize == 32 || IVSize == 64) &&
- "Index size is not compatible with the omp runtime");
- auto F = IVSize == 32 ? (IVSigned ? OMPRTL__kmpc_for_static_init_4
- : OMPRTL__kmpc_for_static_init_4u)
- : (IVSigned ? OMPRTL__kmpc_for_static_init_8
- : OMPRTL__kmpc_for_static_init_8u);
- auto RTLFn = CreateRuntimeFunction(F);
- CGF.EmitRuntimeCall(RTLFn, Args);
-}
-
-void CGOpenMPRuntime::EmitOMPForFinish(CodeGenFunction &CGF, SourceLocation Loc,
- OpenMPScheduleClauseKind ScheduleKind) {
- assert((ScheduleKind == OMPC_SCHEDULE_static ||
- ScheduleKind == OMPC_SCHEDULE_unknown) &&
- "Non-static schedule kinds are not yet implemented");
- // Call __kmpc_for_static_fini(ident_t *loc, kmp_int32 tid);
- llvm::Value *Args[] = {EmitOpenMPUpdateLocation(CGF, Loc, OMP_IDENT_KMPC),
- GetOpenMPThreadID(CGF, Loc)};
- auto RTLFn = CreateRuntimeFunction(OMPRTL__kmpc_for_static_fini);
- CGF.EmitRuntimeCall(RTLFn, Args);
+ llvm::Value *Call =
+ CGF.EmitRuntimeCall(createDispatchNextFunction(IVSize, IVSigned), Args);
+ return CGF.EmitScalarConversion(
+ Call, CGF.getContext().getIntTypeForBitwidth(32, /* Signed */ true),
+ CGF.getContext().BoolTy);
}
-void CGOpenMPRuntime::EmitOMPNumThreadsClause(CodeGenFunction &CGF,
- llvm::Value *NumThreads,
- SourceLocation Loc) {
+void CGOpenMPRuntime::emitNumThreadsClause(CodeGenFunction &CGF,
+ llvm::Value *NumThreads,
+ SourceLocation Loc) {
// Build call __kmpc_push_num_threads(&loc, global_tid, num_threads)
llvm::Value *Args[] = {
- EmitOpenMPUpdateLocation(CGF, Loc), GetOpenMPThreadID(CGF, Loc),
+ emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),
CGF.Builder.CreateIntCast(NumThreads, CGF.Int32Ty, /*isSigned*/ true)};
- llvm::Constant *RTLFn = CreateRuntimeFunction(OMPRTL__kmpc_push_num_threads);
- CGF.EmitRuntimeCall(RTLFn, Args);
+ CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_push_num_threads),
+ Args);
+}
+
+void CGOpenMPRuntime::emitFlush(CodeGenFunction &CGF, ArrayRef<const Expr *>,
+ SourceLocation Loc) {
+ // Build call void __kmpc_flush(ident_t *loc)
+ CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_flush),
+ emitUpdateLocation(CGF, Loc));
+}
+
+namespace {
+/// \brief Indexes of fields for type kmp_task_t.
+enum KmpTaskTFields {
+ /// \brief List of shared variables.
+ KmpTaskTShareds,
+ /// \brief Task routine.
+ KmpTaskTRoutine,
+ /// \brief Partition id for the untied tasks.
+ KmpTaskTPartId,
+ /// \brief Function with call of destructors for private variables.
+ KmpTaskTDestructors,
+};
+} // namespace
+
+void CGOpenMPRuntime::emitKmpRoutineEntryT(QualType KmpInt32Ty) {
+ if (!KmpRoutineEntryPtrTy) {
+ // Build typedef kmp_int32 (* kmp_routine_entry_t)(kmp_int32, void *); type.
+ auto &C = CGM.getContext();
+ QualType KmpRoutineEntryTyArgs[] = {KmpInt32Ty, C.VoidPtrTy};
+ FunctionProtoType::ExtProtoInfo EPI;
+ KmpRoutineEntryPtrQTy = C.getPointerType(
+ C.getFunctionType(KmpInt32Ty, KmpRoutineEntryTyArgs, EPI));
+ KmpRoutineEntryPtrTy = CGM.getTypes().ConvertType(KmpRoutineEntryPtrQTy);
+ }
+}
+
+static void addFieldToRecordDecl(ASTContext &C, DeclContext *DC,
+ QualType FieldTy) {
+ auto *Field = FieldDecl::Create(
+ C, DC, SourceLocation(), SourceLocation(), /*Id=*/nullptr, FieldTy,
+ C.getTrivialTypeSourceInfo(FieldTy, SourceLocation()),
+ /*BW=*/nullptr, /*Mutable=*/false, /*InitStyle=*/ICIS_NoInit);
+ Field->setAccess(AS_public);
+ DC->addDecl(Field);
+}
+
+namespace {
+struct PrivateHelpersTy {
+ PrivateHelpersTy(const VarDecl *Original, const VarDecl *PrivateCopy,
+ const VarDecl *PrivateElemInit)
+ : Original(Original), PrivateCopy(PrivateCopy),
+ PrivateElemInit(PrivateElemInit) {}
+ const VarDecl *Original;
+ const VarDecl *PrivateCopy;
+ const VarDecl *PrivateElemInit;
+};
+typedef std::pair<CharUnits /*Align*/, PrivateHelpersTy> PrivateDataTy;
+} // namespace
+
+static RecordDecl *
+createPrivatesRecordDecl(CodeGenModule &CGM,
+ const ArrayRef<PrivateDataTy> Privates) {
+ if (!Privates.empty()) {
+ auto &C = CGM.getContext();
+ // Build struct .kmp_privates_t. {
+ // /* private vars */
+ // };
+ auto *RD = C.buildImplicitRecord(".kmp_privates.t");
+ RD->startDefinition();
+ for (auto &&Pair : Privates) {
+ auto Type = Pair.second.Original->getType();
+ Type = Type.getNonReferenceType();
+ addFieldToRecordDecl(C, RD, Type);
+ }
+ RD->completeDefinition();
+ return RD;
+ }
+ return nullptr;
}
-void CGOpenMPRuntime::EmitOMPFlush(CodeGenFunction &CGF, ArrayRef<const Expr *>,
- SourceLocation Loc) {
- // Build call void __kmpc_flush(ident_t *loc, ...)
- // FIXME: List of variables is ignored by libiomp5 runtime, no need to
- // generate it, just request full memory fence.
- llvm::Value *Args[] = {EmitOpenMPUpdateLocation(CGF, Loc),
- llvm::ConstantInt::get(CGM.Int32Ty, 0)};
- auto *RTLFn = CreateRuntimeFunction(OMPRTL__kmpc_flush);
- CGF.EmitRuntimeCall(RTLFn, Args);
+static RecordDecl *
+createKmpTaskTRecordDecl(CodeGenModule &CGM, QualType KmpInt32Ty,
+ QualType KmpRoutineEntryPointerQTy) {
+ auto &C = CGM.getContext();
+ // Build struct kmp_task_t {
+ // void * shareds;
+ // kmp_routine_entry_t routine;
+ // kmp_int32 part_id;
+ // kmp_routine_entry_t destructors;
+ // };
+ auto *RD = C.buildImplicitRecord("kmp_task_t");
+ RD->startDefinition();
+ addFieldToRecordDecl(C, RD, C.VoidPtrTy);
+ addFieldToRecordDecl(C, RD, KmpRoutineEntryPointerQTy);
+ addFieldToRecordDecl(C, RD, KmpInt32Ty);
+ addFieldToRecordDecl(C, RD, KmpRoutineEntryPointerQTy);
+ RD->completeDefinition();
+ return RD;
+}
+
+static RecordDecl *
+createKmpTaskTWithPrivatesRecordDecl(CodeGenModule &CGM, QualType KmpTaskTQTy,
+ const ArrayRef<PrivateDataTy> Privates) {
+ auto &C = CGM.getContext();
+ // Build struct kmp_task_t_with_privates {
+ // kmp_task_t task_data;
+ // .kmp_privates_t. privates;
+ // };
+ auto *RD = C.buildImplicitRecord("kmp_task_t_with_privates");
+ RD->startDefinition();
+ addFieldToRecordDecl(C, RD, KmpTaskTQTy);
+ if (auto *PrivateRD = createPrivatesRecordDecl(CGM, Privates)) {
+ addFieldToRecordDecl(C, RD, C.getRecordType(PrivateRD));
+ }
+ RD->completeDefinition();
+ return RD;
}
+
+/// \brief Emit a proxy function which accepts kmp_task_t as the second
+/// argument.
+/// \code
+/// kmp_int32 .omp_task_entry.(kmp_int32 gtid, kmp_task_t *tt) {
+/// TaskFunction(gtid, tt->part_id, &tt->privates, task_privates_map,
+/// tt->shareds);
+/// return 0;
+/// }
+/// \endcode
+static llvm::Value *
+emitProxyTaskFunction(CodeGenModule &CGM, SourceLocation Loc,
+ QualType KmpInt32Ty, QualType KmpTaskTWithPrivatesPtrQTy,
+ QualType KmpTaskTWithPrivatesQTy, QualType KmpTaskTQTy,
+ QualType SharedsPtrTy, llvm::Value *TaskFunction,
+ llvm::Value *TaskPrivatesMap) {
+ auto &C = CGM.getContext();
+ FunctionArgList Args;
+ ImplicitParamDecl GtidArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, KmpInt32Ty);
+ ImplicitParamDecl TaskTypeArg(C, /*DC=*/nullptr, Loc,
+ /*Id=*/nullptr, KmpTaskTWithPrivatesPtrQTy);
+ Args.push_back(&GtidArg);
+ Args.push_back(&TaskTypeArg);
+ FunctionType::ExtInfo Info;
+ auto &TaskEntryFnInfo =
+ CGM.getTypes().arrangeFreeFunctionDeclaration(KmpInt32Ty, Args, Info,
+ /*isVariadic=*/false);
+ auto *TaskEntryTy = CGM.getTypes().GetFunctionType(TaskEntryFnInfo);
+ auto *TaskEntry =
+ llvm::Function::Create(TaskEntryTy, llvm::GlobalValue::InternalLinkage,
+ ".omp_task_entry.", &CGM.getModule());
+ CGM.SetLLVMFunctionAttributes(/*D=*/nullptr, TaskEntryFnInfo, TaskEntry);
+ CodeGenFunction CGF(CGM);
+ CGF.disableDebugInfo();
+ CGF.StartFunction(GlobalDecl(), KmpInt32Ty, TaskEntry, TaskEntryFnInfo, Args);
+
+ // TaskFunction(gtid, tt->task_data.part_id, &tt->privates, task_privates_map,
+ // tt->task_data.shareds);
+ auto *GtidParam = CGF.EmitLoadOfScalar(
+ CGF.GetAddrOfLocalVar(&GtidArg), /*Volatile=*/false,
+ C.getTypeAlignInChars(KmpInt32Ty).getQuantity(), KmpInt32Ty, Loc);
+ auto *TaskTypeArgAddr = CGF.Builder.CreateAlignedLoad(
+ CGF.GetAddrOfLocalVar(&TaskTypeArg), CGM.PointerAlignInBytes);
+ LValue TDBase =
+ CGF.MakeNaturalAlignAddrLValue(TaskTypeArgAddr, KmpTaskTWithPrivatesQTy);
+ auto *KmpTaskTWithPrivatesQTyRD =
+ cast<RecordDecl>(KmpTaskTWithPrivatesQTy->getAsTagDecl());
+ LValue Base =
+ CGF.EmitLValueForField(TDBase, *KmpTaskTWithPrivatesQTyRD->field_begin());
+ auto *KmpTaskTQTyRD = cast<RecordDecl>(KmpTaskTQTy->getAsTagDecl());
+ auto PartIdFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTPartId);
+ auto PartIdLVal = CGF.EmitLValueForField(Base, *PartIdFI);
+ auto *PartidParam = CGF.EmitLoadOfLValue(PartIdLVal, Loc).getScalarVal();
+
+ auto SharedsFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTShareds);
+ auto SharedsLVal = CGF.EmitLValueForField(Base, *SharedsFI);
+ auto *SharedsParam = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
+ CGF.EmitLoadOfLValue(SharedsLVal, Loc).getScalarVal(),
+ CGF.ConvertTypeForMem(SharedsPtrTy));
+
+ auto PrivatesFI = std::next(KmpTaskTWithPrivatesQTyRD->field_begin(), 1);
+ llvm::Value *PrivatesParam;
+ if (PrivatesFI != KmpTaskTWithPrivatesQTyRD->field_end()) {
+ auto PrivatesLVal = CGF.EmitLValueForField(TDBase, *PrivatesFI);
+ PrivatesParam = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
+ PrivatesLVal.getAddress(), CGF.VoidPtrTy);
+ } else {
+ PrivatesParam = llvm::ConstantPointerNull::get(CGF.VoidPtrTy);
+ }
+
+ llvm::Value *CallArgs[] = {GtidParam, PartidParam, PrivatesParam,
+ TaskPrivatesMap, SharedsParam};
+ CGF.EmitCallOrInvoke(TaskFunction, CallArgs);
+ CGF.EmitStoreThroughLValue(
+ RValue::get(CGF.Builder.getInt32(/*C=*/0)),
+ CGF.MakeNaturalAlignAddrLValue(CGF.ReturnValue, KmpInt32Ty));
+ CGF.FinishFunction();
+ return TaskEntry;
+}
+
+static llvm::Value *emitDestructorsFunction(CodeGenModule &CGM,
+ SourceLocation Loc,
+ QualType KmpInt32Ty,
+ QualType KmpTaskTWithPrivatesPtrQTy,
+ QualType KmpTaskTWithPrivatesQTy) {
+ auto &C = CGM.getContext();
+ FunctionArgList Args;
+ ImplicitParamDecl GtidArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, KmpInt32Ty);
+ ImplicitParamDecl TaskTypeArg(C, /*DC=*/nullptr, Loc,
+ /*Id=*/nullptr, KmpTaskTWithPrivatesPtrQTy);
+ Args.push_back(&GtidArg);
+ Args.push_back(&TaskTypeArg);
+ FunctionType::ExtInfo Info;
+ auto &DestructorFnInfo =
+ CGM.getTypes().arrangeFreeFunctionDeclaration(KmpInt32Ty, Args, Info,
+ /*isVariadic=*/false);
+ auto *DestructorFnTy = CGM.getTypes().GetFunctionType(DestructorFnInfo);
+ auto *DestructorFn =
+ llvm::Function::Create(DestructorFnTy, llvm::GlobalValue::InternalLinkage,
+ ".omp_task_destructor.", &CGM.getModule());
+ CGM.SetLLVMFunctionAttributes(/*D=*/nullptr, DestructorFnInfo, DestructorFn);
+ CodeGenFunction CGF(CGM);
+ CGF.disableDebugInfo();
+ CGF.StartFunction(GlobalDecl(), KmpInt32Ty, DestructorFn, DestructorFnInfo,
+ Args);
+
+ auto *TaskTypeArgAddr = CGF.Builder.CreateAlignedLoad(
+ CGF.GetAddrOfLocalVar(&TaskTypeArg), CGM.PointerAlignInBytes);
+ LValue Base =
+ CGF.MakeNaturalAlignAddrLValue(TaskTypeArgAddr, KmpTaskTWithPrivatesQTy);
+ auto *KmpTaskTWithPrivatesQTyRD =
+ cast<RecordDecl>(KmpTaskTWithPrivatesQTy->getAsTagDecl());
+ auto FI = std::next(KmpTaskTWithPrivatesQTyRD->field_begin());
+ Base = CGF.EmitLValueForField(Base, *FI);
+ for (auto *Field :
+ cast<RecordDecl>(FI->getType()->getAsTagDecl())->fields()) {
+ if (auto DtorKind = Field->getType().isDestructedType()) {
+ auto FieldLValue = CGF.EmitLValueForField(Base, Field);
+ CGF.pushDestroy(DtorKind, FieldLValue.getAddress(), Field->getType());
+ }
+ }
+ CGF.FinishFunction();
+ return DestructorFn;
+}
+
+/// \brief Emit a privates mapping function for correct handling of private and
+/// firstprivate variables.
+/// \code
+/// void .omp_task_privates_map.(const .privates. *noalias privs, <ty1>
+/// **noalias priv1,..., <tyn> **noalias privn) {
+/// *priv1 = &.privates.priv1;
+/// ...;
+/// *privn = &.privates.privn;
+/// }
+/// \endcode
+static llvm::Value *
+emitTaskPrivateMappingFunction(CodeGenModule &CGM, SourceLocation Loc,
+ const ArrayRef<const Expr *> PrivateVars,
+ const ArrayRef<const Expr *> FirstprivateVars,
+ QualType PrivatesQTy,
+ const ArrayRef<PrivateDataTy> Privates) {
+ auto &C = CGM.getContext();
+ FunctionArgList Args;
+ ImplicitParamDecl TaskPrivatesArg(
+ C, /*DC=*/nullptr, Loc, /*Id=*/nullptr,
+ C.getPointerType(PrivatesQTy).withConst().withRestrict());
+ Args.push_back(&TaskPrivatesArg);
+ llvm::DenseMap<const VarDecl *, unsigned> PrivateVarsPos;
+ unsigned Counter = 1;
+ for (auto *E: PrivateVars) {
+ Args.push_back(ImplicitParamDecl::Create(
+ C, /*DC=*/nullptr, Loc,
+ /*Id=*/nullptr, C.getPointerType(C.getPointerType(E->getType()))
+ .withConst()
+ .withRestrict()));
+ auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
+ PrivateVarsPos[VD] = Counter;
+ ++Counter;
+ }
+ for (auto *E : FirstprivateVars) {
+ Args.push_back(ImplicitParamDecl::Create(
+ C, /*DC=*/nullptr, Loc,
+ /*Id=*/nullptr, C.getPointerType(C.getPointerType(E->getType()))
+ .withConst()
+ .withRestrict()));
+ auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
+ PrivateVarsPos[VD] = Counter;
+ ++Counter;
+ }
+ FunctionType::ExtInfo Info;
+ auto &TaskPrivatesMapFnInfo =
+ CGM.getTypes().arrangeFreeFunctionDeclaration(C.VoidTy, Args, Info,
+ /*isVariadic=*/false);
+ auto *TaskPrivatesMapTy =
+ CGM.getTypes().GetFunctionType(TaskPrivatesMapFnInfo);
+ auto *TaskPrivatesMap = llvm::Function::Create(
+ TaskPrivatesMapTy, llvm::GlobalValue::InternalLinkage,
+ ".omp_task_privates_map.", &CGM.getModule());
+ CGM.SetLLVMFunctionAttributes(/*D=*/nullptr, TaskPrivatesMapFnInfo,
+ TaskPrivatesMap);
+ TaskPrivatesMap->addFnAttr(llvm::Attribute::AlwaysInline);
+ CodeGenFunction CGF(CGM);
+ CGF.disableDebugInfo();
+ CGF.StartFunction(GlobalDecl(), C.VoidTy, TaskPrivatesMap,
+ TaskPrivatesMapFnInfo, Args);
+
+ // *privi = &.privates.privi;
+ auto *TaskPrivatesArgAddr = CGF.Builder.CreateAlignedLoad(
+ CGF.GetAddrOfLocalVar(&TaskPrivatesArg), CGM.PointerAlignInBytes);
+ LValue Base =
+ CGF.MakeNaturalAlignAddrLValue(TaskPrivatesArgAddr, PrivatesQTy);
+ auto *PrivatesQTyRD = cast<RecordDecl>(PrivatesQTy->getAsTagDecl());
+ Counter = 0;
+ for (auto *Field : PrivatesQTyRD->fields()) {
+ auto FieldLVal = CGF.EmitLValueForField(Base, Field);
+ auto *VD = Args[PrivateVarsPos[Privates[Counter].second.Original]];
+ auto RefLVal = CGF.MakeNaturalAlignAddrLValue(CGF.GetAddrOfLocalVar(VD),
+ VD->getType());
+ auto RefLoadRVal = CGF.EmitLoadOfLValue(RefLVal, Loc);
+ CGF.EmitStoreOfScalar(
+ FieldLVal.getAddress(),
+ CGF.MakeNaturalAlignAddrLValue(RefLoadRVal.getScalarVal(),
+ RefLVal.getType()->getPointeeType()));
+ ++Counter;
+ }
+ CGF.FinishFunction();
+ return TaskPrivatesMap;
+}
+
+static int array_pod_sort_comparator(const PrivateDataTy *P1,
+ const PrivateDataTy *P2) {
+ return P1->first < P2->first ? 1 : (P2->first < P1->first ? -1 : 0);
+}
+
+void CGOpenMPRuntime::emitTaskCall(
+ CodeGenFunction &CGF, SourceLocation Loc, const OMPExecutableDirective &D,
+ bool Tied, llvm::PointerIntPair<llvm::Value *, 1, bool> Final,
+ llvm::Value *TaskFunction, QualType SharedsTy, llvm::Value *Shareds,
+ const Expr *IfCond, const ArrayRef<const Expr *> PrivateVars,
+ const ArrayRef<const Expr *> PrivateCopies,
+ const ArrayRef<const Expr *> FirstprivateVars,
+ const ArrayRef<const Expr *> FirstprivateCopies,
+ const ArrayRef<const Expr *> FirstprivateInits) {
+ auto &C = CGM.getContext();
+ llvm::SmallVector<PrivateDataTy, 8> Privates;
+ // Aggregate privates and sort them by the alignment.
+ auto I = PrivateCopies.begin();
+ for (auto *E : PrivateVars) {
+ auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
+ Privates.push_back(std::make_pair(
+ C.getTypeAlignInChars(VD->getType()),
+ PrivateHelpersTy(VD, cast<VarDecl>(cast<DeclRefExpr>(*I)->getDecl()),
+ /*PrivateElemInit=*/nullptr)));
+ ++I;
+ }
+ I = FirstprivateCopies.begin();
+ auto IElemInitRef = FirstprivateInits.begin();
+ for (auto *E : FirstprivateVars) {
+ auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
+ Privates.push_back(std::make_pair(
+ C.getTypeAlignInChars(VD->getType()),
+ PrivateHelpersTy(
+ VD, cast<VarDecl>(cast<DeclRefExpr>(*I)->getDecl()),
+ cast<VarDecl>(cast<DeclRefExpr>(*IElemInitRef)->getDecl()))));
+ ++I, ++IElemInitRef;
+ }
+ llvm::array_pod_sort(Privates.begin(), Privates.end(),
+ array_pod_sort_comparator);
+ auto KmpInt32Ty = C.getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/1);
+ // Build type kmp_routine_entry_t (if not built yet).
+ emitKmpRoutineEntryT(KmpInt32Ty);
+ // Build type kmp_task_t (if not built yet).
+ if (KmpTaskTQTy.isNull()) {
+ KmpTaskTQTy = C.getRecordType(
+ createKmpTaskTRecordDecl(CGM, KmpInt32Ty, KmpRoutineEntryPtrQTy));
+ }
+ auto *KmpTaskTQTyRD = cast<RecordDecl>(KmpTaskTQTy->getAsTagDecl());
+ // Build particular struct kmp_task_t for the given task.
+ auto *KmpTaskTWithPrivatesQTyRD =
+ createKmpTaskTWithPrivatesRecordDecl(CGM, KmpTaskTQTy, Privates);
+ auto KmpTaskTWithPrivatesQTy = C.getRecordType(KmpTaskTWithPrivatesQTyRD);
+ QualType KmpTaskTWithPrivatesPtrQTy =
+ C.getPointerType(KmpTaskTWithPrivatesQTy);
+ auto *KmpTaskTWithPrivatesTy = CGF.ConvertType(KmpTaskTWithPrivatesQTy);
+ auto *KmpTaskTWithPrivatesPtrTy = KmpTaskTWithPrivatesTy->getPointerTo();
+ auto KmpTaskTWithPrivatesTySize =
+ CGM.getSize(C.getTypeSizeInChars(KmpTaskTWithPrivatesQTy));
+ QualType SharedsPtrTy = C.getPointerType(SharedsTy);
+
+ // Emit initial values for private copies (if any).
+ llvm::Value *TaskPrivatesMap = nullptr;
+ auto *TaskPrivatesMapTy =
+ std::next(cast<llvm::Function>(TaskFunction)->getArgumentList().begin(),
+ 3)
+ ->getType();
+ if (!Privates.empty()) {
+ auto FI = std::next(KmpTaskTWithPrivatesQTyRD->field_begin());
+ TaskPrivatesMap = emitTaskPrivateMappingFunction(
+ CGM, Loc, PrivateVars, FirstprivateVars, FI->getType(), Privates);
+ TaskPrivatesMap = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
+ TaskPrivatesMap, TaskPrivatesMapTy);
+ } else {
+ TaskPrivatesMap = llvm::ConstantPointerNull::get(
+ cast<llvm::PointerType>(TaskPrivatesMapTy));
+ }
+ // Build a proxy function kmp_int32 .omp_task_entry.(kmp_int32 gtid,
+ // kmp_task_t *tt);
+ auto *TaskEntry = emitProxyTaskFunction(
+ CGM, Loc, KmpInt32Ty, KmpTaskTWithPrivatesPtrQTy, KmpTaskTWithPrivatesQTy,
+ KmpTaskTQTy, SharedsPtrTy, TaskFunction, TaskPrivatesMap);
+
+ // Build call kmp_task_t * __kmpc_omp_task_alloc(ident_t *, kmp_int32 gtid,
+ // kmp_int32 flags, size_t sizeof_kmp_task_t, size_t sizeof_shareds,
+ // kmp_routine_entry_t *task_entry);
+ // Task flags. Format is taken from
+ // http://llvm.org/svn/llvm-project/openmp/trunk/runtime/src/kmp.h,
+ // description of kmp_tasking_flags struct.
+ const unsigned TiedFlag = 0x1;
+ const unsigned FinalFlag = 0x2;
+ unsigned Flags = Tied ? TiedFlag : 0;
+ auto *TaskFlags =
+ Final.getPointer()
+ ? CGF.Builder.CreateSelect(Final.getPointer(),
+ CGF.Builder.getInt32(FinalFlag),
+ CGF.Builder.getInt32(/*C=*/0))
+ : CGF.Builder.getInt32(Final.getInt() ? FinalFlag : 0);
+ TaskFlags = CGF.Builder.CreateOr(TaskFlags, CGF.Builder.getInt32(Flags));
+ auto SharedsSize = C.getTypeSizeInChars(SharedsTy);
+ llvm::Value *AllocArgs[] = {
+ emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc), TaskFlags,
+ KmpTaskTWithPrivatesTySize, CGM.getSize(SharedsSize),
+ CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(TaskEntry,
+ KmpRoutineEntryPtrTy)};
+ auto *NewTask = CGF.EmitRuntimeCall(
+ createRuntimeFunction(OMPRTL__kmpc_omp_task_alloc), AllocArgs);
+ auto *NewTaskNewTaskTTy = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
+ NewTask, KmpTaskTWithPrivatesPtrTy);
+ LValue Base = CGF.MakeNaturalAlignAddrLValue(NewTaskNewTaskTTy,
+ KmpTaskTWithPrivatesQTy);
+ LValue TDBase =
+ CGF.EmitLValueForField(Base, *KmpTaskTWithPrivatesQTyRD->field_begin());
+ // Fill the data in the resulting kmp_task_t record.
+ // Copy shareds if there are any.
+ llvm::Value *KmpTaskSharedsPtr = nullptr;
+ if (!SharedsTy->getAsStructureType()->getDecl()->field_empty()) {
+ KmpTaskSharedsPtr = CGF.EmitLoadOfScalar(
+ CGF.EmitLValueForField(
+ TDBase, *std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTShareds)),
+ Loc);
+ CGF.EmitAggregateCopy(KmpTaskSharedsPtr, Shareds, SharedsTy);
+ }
+ // Emit initial values for private copies (if any).
+ bool NeedsCleanup = false;
+ if (!Privates.empty()) {
+ auto FI = std::next(KmpTaskTWithPrivatesQTyRD->field_begin());
+ auto PrivatesBase = CGF.EmitLValueForField(Base, *FI);
+ FI = cast<RecordDecl>(FI->getType()->getAsTagDecl())->field_begin();
+ LValue SharedsBase;
+ if (!FirstprivateVars.empty()) {
+ SharedsBase = CGF.MakeNaturalAlignAddrLValue(
+ CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
+ KmpTaskSharedsPtr, CGF.ConvertTypeForMem(SharedsPtrTy)),
+ SharedsTy);
+ }
+ CodeGenFunction::CGCapturedStmtInfo CapturesInfo(
+ cast<CapturedStmt>(*D.getAssociatedStmt()));
+ for (auto &&Pair : Privates) {
+ auto *VD = Pair.second.PrivateCopy;
+ auto *Init = VD->getAnyInitializer();
+ LValue PrivateLValue = CGF.EmitLValueForField(PrivatesBase, *FI);
+ if (Init) {
+ if (auto *Elem = Pair.second.PrivateElemInit) {
+ auto *OriginalVD = Pair.second.Original;
+ auto *SharedField = CapturesInfo.lookup(OriginalVD);
+ auto SharedRefLValue =
+ CGF.EmitLValueForField(SharedsBase, SharedField);
+ QualType Type = OriginalVD->getType();
+ if (Type->isArrayType()) {
+ // Initialize firstprivate array.
+ if (!isa<CXXConstructExpr>(Init) ||
+ CGF.isTrivialInitializer(Init)) {
+ // Perform simple memcpy.
+ CGF.EmitAggregateAssign(PrivateLValue.getAddress(),
+ SharedRefLValue.getAddress(), Type);
+ } else {
+ // Initialize firstprivate array using element-by-element
+ // intialization.
+ CGF.EmitOMPAggregateAssign(
+ PrivateLValue.getAddress(), SharedRefLValue.getAddress(),
+ Type, [&CGF, Elem, Init, &CapturesInfo](
+ llvm::Value *DestElement, llvm::Value *SrcElement) {
+ // Clean up any temporaries needed by the initialization.
+ CodeGenFunction::OMPPrivateScope InitScope(CGF);
+ InitScope.addPrivate(Elem, [SrcElement]() -> llvm::Value *{
+ return SrcElement;
+ });
+ (void)InitScope.Privatize();
+ // Emit initialization for single element.
+ auto *OldCapturedStmtInfo = CGF.CapturedStmtInfo;
+ CGF.CapturedStmtInfo = &CapturesInfo;
+ CGF.EmitAnyExprToMem(Init, DestElement,
+ Init->getType().getQualifiers(),
+ /*IsInitializer=*/false);
+ CGF.CapturedStmtInfo = OldCapturedStmtInfo;
+ });
+ }
+ } else {
+ CodeGenFunction::OMPPrivateScope InitScope(CGF);
+ InitScope.addPrivate(Elem, [SharedRefLValue]() -> llvm::Value *{
+ return SharedRefLValue.getAddress();
+ });
+ (void)InitScope.Privatize();
+ auto *OldCapturedStmtInfo = CGF.CapturedStmtInfo;
+ CGF.CapturedStmtInfo = &CapturesInfo;
+ CGF.EmitExprAsInit(Init, VD, PrivateLValue,
+ /*capturedByInit=*/false);
+ CGF.CapturedStmtInfo = OldCapturedStmtInfo;
+ }
+ } else {
+ CGF.EmitExprAsInit(Init, VD, PrivateLValue, /*capturedByInit=*/false);
+ }
+ }
+ NeedsCleanup = NeedsCleanup || FI->getType().isDestructedType();
+ ++FI;
+ }
+ }
+ // Provide pointer to function with destructors for privates.
+ llvm::Value *DestructorFn =
+ NeedsCleanup ? emitDestructorsFunction(CGM, Loc, KmpInt32Ty,
+ KmpTaskTWithPrivatesPtrQTy,
+ KmpTaskTWithPrivatesQTy)
+ : llvm::ConstantPointerNull::get(
+ cast<llvm::PointerType>(KmpRoutineEntryPtrTy));
+ LValue Destructor = CGF.EmitLValueForField(
+ TDBase, *std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTDestructors));
+ CGF.EmitStoreOfScalar(CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
+ DestructorFn, KmpRoutineEntryPtrTy),
+ Destructor);
+ // NOTE: routine and part_id fields are intialized by __kmpc_omp_task_alloc()
+ // libcall.
+ // Build kmp_int32 __kmpc_omp_task(ident_t *, kmp_int32 gtid, kmp_task_t
+ // *new_task);
+ auto *ThreadID = getThreadID(CGF, Loc);
+ llvm::Value *TaskArgs[] = {emitUpdateLocation(CGF, Loc), ThreadID, NewTask};
+ auto &&ThenCodeGen = [this, &TaskArgs](CodeGenFunction &CGF) {
+ // TODO: add check for untied tasks.
+ CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_omp_task), TaskArgs);
+ };
+ typedef CallEndCleanup<std::extent<decltype(TaskArgs)>::value>
+ IfCallEndCleanup;
+ auto &&ElseCodeGen =
+ [this, &TaskArgs, ThreadID, NewTaskNewTaskTTy, TaskEntry](
+ CodeGenFunction &CGF) {
+ CodeGenFunction::RunCleanupsScope LocalScope(CGF);
+ CGF.EmitRuntimeCall(
+ createRuntimeFunction(OMPRTL__kmpc_omp_task_begin_if0), TaskArgs);
+ // Build void __kmpc_omp_task_complete_if0(ident_t *, kmp_int32 gtid,
+ // kmp_task_t *new_task);
+ CGF.EHStack.pushCleanup<IfCallEndCleanup>(
+ NormalAndEHCleanup,
+ createRuntimeFunction(OMPRTL__kmpc_omp_task_complete_if0),
+ llvm::makeArrayRef(TaskArgs));
+
+ // Call proxy_task_entry(gtid, new_task);
+ llvm::Value *OutlinedFnArgs[] = {ThreadID, NewTaskNewTaskTTy};
+ CGF.EmitCallOrInvoke(TaskEntry, OutlinedFnArgs);
+ };
+ if (IfCond) {
+ emitOMPIfClause(CGF, IfCond, ThenCodeGen, ElseCodeGen);
+ } else {
+ CodeGenFunction::RunCleanupsScope Scope(CGF);
+ ThenCodeGen(CGF);
+ }
+}
+
+static llvm::Value *emitReductionFunction(CodeGenModule &CGM,
+ llvm::Type *ArgsType,
+ ArrayRef<const Expr *> LHSExprs,
+ ArrayRef<const Expr *> RHSExprs,
+ ArrayRef<const Expr *> ReductionOps) {
+ auto &C = CGM.getContext();
+
+ // void reduction_func(void *LHSArg, void *RHSArg);
+ FunctionArgList Args;
+ ImplicitParamDecl LHSArg(C, /*DC=*/nullptr, SourceLocation(), /*Id=*/nullptr,
+ C.VoidPtrTy);
+ ImplicitParamDecl RHSArg(C, /*DC=*/nullptr, SourceLocation(), /*Id=*/nullptr,
+ C.VoidPtrTy);
+ Args.push_back(&LHSArg);
+ Args.push_back(&RHSArg);
+ FunctionType::ExtInfo EI;
+ auto &CGFI = CGM.getTypes().arrangeFreeFunctionDeclaration(
+ C.VoidTy, Args, EI, /*isVariadic=*/false);
+ auto *Fn = llvm::Function::Create(
+ CGM.getTypes().GetFunctionType(CGFI), llvm::GlobalValue::InternalLinkage,
+ ".omp.reduction.reduction_func", &CGM.getModule());
+ CGM.SetLLVMFunctionAttributes(/*D=*/nullptr, CGFI, Fn);
+ CodeGenFunction CGF(CGM);
+ CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, CGFI, Args);
+
+ // Dst = (void*[n])(LHSArg);
+ // Src = (void*[n])(RHSArg);
+ auto *LHS = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
+ CGF.Builder.CreateAlignedLoad(CGF.GetAddrOfLocalVar(&LHSArg),
+ CGF.PointerAlignInBytes),
+ ArgsType);
+ auto *RHS = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
+ CGF.Builder.CreateAlignedLoad(CGF.GetAddrOfLocalVar(&RHSArg),
+ CGF.PointerAlignInBytes),
+ ArgsType);
+
+ // ...
+ // *(Type<i>*)lhs[i] = RedOp<i>(*(Type<i>*)lhs[i], *(Type<i>*)rhs[i]);
+ // ...
+ CodeGenFunction::OMPPrivateScope Scope(CGF);
+ for (unsigned I = 0, E = ReductionOps.size(); I < E; ++I) {
+ Scope.addPrivate(
+ cast<VarDecl>(cast<DeclRefExpr>(RHSExprs[I])->getDecl()),
+ [&]() -> llvm::Value *{
+ return CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
+ CGF.Builder.CreateAlignedLoad(
+ CGF.Builder.CreateStructGEP(/*Ty=*/nullptr, RHS, I),
+ CGM.PointerAlignInBytes),
+ CGF.ConvertTypeForMem(C.getPointerType(RHSExprs[I]->getType())));
+ });
+ Scope.addPrivate(
+ cast<VarDecl>(cast<DeclRefExpr>(LHSExprs[I])->getDecl()),
+ [&]() -> llvm::Value *{
+ return CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
+ CGF.Builder.CreateAlignedLoad(
+ CGF.Builder.CreateStructGEP(/*Ty=*/nullptr, LHS, I),
+ CGM.PointerAlignInBytes),
+ CGF.ConvertTypeForMem(C.getPointerType(LHSExprs[I]->getType())));
+ });
+ }
+ Scope.Privatize();
+ for (auto *E : ReductionOps) {
+ CGF.EmitIgnoredExpr(E);
+ }
+ Scope.ForceCleanup();
+ CGF.FinishFunction();
+ return Fn;
+}
+
+void CGOpenMPRuntime::emitReduction(CodeGenFunction &CGF, SourceLocation Loc,
+ ArrayRef<const Expr *> LHSExprs,
+ ArrayRef<const Expr *> RHSExprs,
+ ArrayRef<const Expr *> ReductionOps,
+ bool WithNowait) {
+ // Next code should be emitted for reduction:
+ //
+ // static kmp_critical_name lock = { 0 };
+ //
+ // void reduce_func(void *lhs[<n>], void *rhs[<n>]) {
+ // *(Type0*)lhs[0] = ReductionOperation0(*(Type0*)lhs[0], *(Type0*)rhs[0]);
+ // ...
+ // *(Type<n>-1*)lhs[<n>-1] = ReductionOperation<n>-1(*(Type<n>-1*)lhs[<n>-1],
+ // *(Type<n>-1*)rhs[<n>-1]);
+ // }
+ //
+ // ...
+ // void *RedList[<n>] = {&<RHSExprs>[0], ..., &<RHSExprs>[<n>-1]};
+ // switch (__kmpc_reduce{_nowait}(<loc>, <gtid>, <n>, sizeof(RedList),
+ // RedList, reduce_func, &<lock>)) {
+ // case 1:
+ // ...
+ // <LHSExprs>[i] = RedOp<i>(*<LHSExprs>[i], *<RHSExprs>[i]);
+ // ...
+ // __kmpc_end_reduce{_nowait}(<loc>, <gtid>, &<lock>);
+ // break;
+ // case 2:
+ // ...
+ // Atomic(<LHSExprs>[i] = RedOp<i>(*<LHSExprs>[i], *<RHSExprs>[i]));
+ // ...
+ // [__kmpc_end_reduce(<loc>, <gtid>, &<lock>);]
+ // break;
+ // default:;
+ // }
+
+ auto &C = CGM.getContext();
+
+ // 1. Build a list of reduction variables.
+ // void *RedList[<n>] = {<ReductionVars>[0], ..., <ReductionVars>[<n>-1]};
+ llvm::APInt ArraySize(/*unsigned int numBits=*/32, RHSExprs.size());
+ QualType ReductionArrayTy =
+ C.getConstantArrayType(C.VoidPtrTy, ArraySize, ArrayType::Normal,
+ /*IndexTypeQuals=*/0);
+ auto *ReductionList =
+ CGF.CreateMemTemp(ReductionArrayTy, ".omp.reduction.red_list");
+ for (unsigned I = 0, E = RHSExprs.size(); I < E; ++I) {
+ auto *Elem = CGF.Builder.CreateStructGEP(/*Ty=*/nullptr, ReductionList, I);
+ CGF.Builder.CreateAlignedStore(
+ CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
+ CGF.EmitLValue(RHSExprs[I]).getAddress(), CGF.VoidPtrTy),
+ Elem, CGM.PointerAlignInBytes);
+ }
+
+ // 2. Emit reduce_func().
+ auto *ReductionFn = emitReductionFunction(
+ CGM, CGF.ConvertTypeForMem(ReductionArrayTy)->getPointerTo(), LHSExprs,
+ RHSExprs, ReductionOps);
+
+ // 3. Create static kmp_critical_name lock = { 0 };
+ auto *Lock = getCriticalRegionLock(".reduction");
+
+ // 4. Build res = __kmpc_reduce{_nowait}(<loc>, <gtid>, <n>, sizeof(RedList),
+ // RedList, reduce_func, &<lock>);
+ auto *IdentTLoc = emitUpdateLocation(
+ CGF, Loc,
+ static_cast<OpenMPLocationFlags>(OMP_IDENT_KMPC | OMP_ATOMIC_REDUCE));
+ auto *ThreadId = getThreadID(CGF, Loc);
+ auto *ReductionArrayTySize = llvm::ConstantInt::get(
+ CGM.SizeTy, C.getTypeSizeInChars(ReductionArrayTy).getQuantity());
+ auto *RL = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(ReductionList,
+ CGF.VoidPtrTy);
+ llvm::Value *Args[] = {
+ IdentTLoc, // ident_t *<loc>
+ ThreadId, // i32 <gtid>
+ CGF.Builder.getInt32(RHSExprs.size()), // i32 <n>
+ ReductionArrayTySize, // size_type sizeof(RedList)
+ RL, // void *RedList
+ ReductionFn, // void (*) (void *, void *) <reduce_func>
+ Lock // kmp_critical_name *&<lock>
+ };
+ auto Res = CGF.EmitRuntimeCall(
+ createRuntimeFunction(WithNowait ? OMPRTL__kmpc_reduce_nowait
+ : OMPRTL__kmpc_reduce),
+ Args);
+
+ // 5. Build switch(res)
+ auto *DefaultBB = CGF.createBasicBlock(".omp.reduction.default");
+ auto *SwInst = CGF.Builder.CreateSwitch(Res, DefaultBB, /*NumCases=*/2);
+
+ // 6. Build case 1:
+ // ...
+ // <LHSExprs>[i] = RedOp<i>(*<LHSExprs>[i], *<RHSExprs>[i]);
+ // ...
+ // __kmpc_end_reduce{_nowait}(<loc>, <gtid>, &<lock>);
+ // break;
+ auto *Case1BB = CGF.createBasicBlock(".omp.reduction.case1");
+ SwInst->addCase(CGF.Builder.getInt32(1), Case1BB);
+ CGF.EmitBlock(Case1BB);
+
+ {
+ CodeGenFunction::RunCleanupsScope Scope(CGF);
+ // Add emission of __kmpc_end_reduce{_nowait}(<loc>, <gtid>, &<lock>);
+ llvm::Value *EndArgs[] = {
+ IdentTLoc, // ident_t *<loc>
+ ThreadId, // i32 <gtid>
+ Lock // kmp_critical_name *&<lock>
+ };
+ CGF.EHStack
+ .pushCleanup<CallEndCleanup<std::extent<decltype(EndArgs)>::value>>(
+ NormalAndEHCleanup,
+ createRuntimeFunction(WithNowait ? OMPRTL__kmpc_end_reduce_nowait
+ : OMPRTL__kmpc_end_reduce),
+ llvm::makeArrayRef(EndArgs));
+ for (auto *E : ReductionOps) {
+ CGF.EmitIgnoredExpr(E);
+ }
+ }
+
+ CGF.EmitBranch(DefaultBB);
+
+ // 7. Build case 2:
+ // ...
+ // Atomic(<LHSExprs>[i] = RedOp<i>(*<LHSExprs>[i], *<