aboutsummaryrefslogtreecommitdiffstats
path: root/lib/CodeGen/ItaniumCXXABI.cpp
diff options
context:
space:
mode:
Diffstat (limited to 'lib/CodeGen/ItaniumCXXABI.cpp')
-rw-r--r--lib/CodeGen/ItaniumCXXABI.cpp246
1 files changed, 174 insertions, 72 deletions
diff --git a/lib/CodeGen/ItaniumCXXABI.cpp b/lib/CodeGen/ItaniumCXXABI.cpp
index 3b2413d960d6..8f9b16470b64 100644
--- a/lib/CodeGen/ItaniumCXXABI.cpp
+++ b/lib/CodeGen/ItaniumCXXABI.cpp
@@ -43,6 +43,10 @@ class ItaniumCXXABI : public CodeGen::CGCXXABI {
/// VTables - All the vtables which have been defined.
llvm::DenseMap<const CXXRecordDecl *, llvm::GlobalVariable *> VTables;
+ /// All the thread wrapper functions that have been used.
+ llvm::SmallVector<std::pair<const VarDecl *, llvm::Function *>, 8>
+ ThreadWrappers;
+
protected:
bool UseARMMethodPtrABI;
bool UseARMGuardVarABI;
@@ -322,7 +326,43 @@ public:
ArrayRef<llvm::Function *> CXXThreadLocalInits,
ArrayRef<const VarDecl *> CXXThreadLocalInitVars) override;
- bool usesThreadWrapperFunction() const override { return true; }
+ /// Determine whether we will definitely emit this variable with a constant
+ /// initializer, either because the language semantics demand it or because
+ /// we know that the initializer is a constant.
+ bool isEmittedWithConstantInitializer(const VarDecl *VD) const {
+ VD = VD->getMostRecentDecl();
+ if (VD->hasAttr<ConstInitAttr>())
+ return true;
+
+ // All later checks examine the initializer specified on the variable. If
+ // the variable is weak, such examination would not be correct.
+ if (VD->isWeak() || VD->hasAttr<SelectAnyAttr>())
+ return false;
+
+ const VarDecl *InitDecl = VD->getInitializingDeclaration();
+ if (!InitDecl)
+ return false;
+
+ // If there's no initializer to run, this is constant initialization.
+ if (!InitDecl->hasInit())
+ return true;
+
+ // If we have the only definition, we don't need a thread wrapper if we
+ // will emit the value as a constant.
+ if (isUniqueGVALinkage(getContext().GetGVALinkageForVariable(VD)))
+ return !VD->needsDestruction(getContext()) && InitDecl->evaluateValue();
+
+ // Otherwise, we need a thread wrapper unless we know that every
+ // translation unit will emit the value as a constant. We rely on
+ // ICE-ness not varying between translation units, which isn't actually
+ // guaranteed by the standard but is necessary for sanity.
+ return InitDecl->isInitKnownICE() && InitDecl->isInitICE();
+ }
+
+ bool usesThreadWrapperFunction(const VarDecl *VD) const override {
+ return !isEmittedWithConstantInitializer(VD) ||
+ VD->needsDestruction(getContext());
+ }
LValue EmitThreadLocalVarDeclLValue(CodeGenFunction &CGF, const VarDecl *VD,
QualType LValType) override;
@@ -415,8 +455,8 @@ public:
class ARMCXXABI : public ItaniumCXXABI {
public:
ARMCXXABI(CodeGen::CodeGenModule &CGM) :
- ItaniumCXXABI(CGM, /* UseARMMethodPtrABI = */ true,
- /* UseARMGuardVarABI = */ true) {}
+ ItaniumCXXABI(CGM, /*UseARMMethodPtrABI=*/true,
+ /*UseARMGuardVarABI=*/true) {}
bool HasThisReturn(GlobalDecl GD) const override {
return (isa<CXXConstructorDecl>(GD.getDecl()) || (
@@ -480,11 +520,11 @@ CodeGen::CGCXXABI *CodeGen::CreateItaniumCXXABI(CodeGenModule &CGM) {
// include the other 32-bit ARM oddities: constructor/destructor return values
// and array cookies.
case TargetCXXABI::GenericAArch64:
- return new ItaniumCXXABI(CGM, /* UseARMMethodPtrABI = */ true,
- /* UseARMGuardVarABI = */ true);
+ return new ItaniumCXXABI(CGM, /*UseARMMethodPtrABI=*/true,
+ /*UseARMGuardVarABI=*/true);
case TargetCXXABI::GenericMIPS:
- return new ItaniumCXXABI(CGM, /* UseARMMethodPtrABI = */ true);
+ return new ItaniumCXXABI(CGM, /*UseARMMethodPtrABI=*/true);
case TargetCXXABI::WebAssembly:
return new WebAssemblyCXXABI(CGM);
@@ -495,8 +535,7 @@ CodeGen::CGCXXABI *CodeGen::CreateItaniumCXXABI(CodeGenModule &CGM) {
// For PNaCl, use ARM-style method pointers so that PNaCl code
// does not assume anything about the alignment of function
// pointers.
- return new ItaniumCXXABI(CGM, /* UseARMMethodPtrABI = */ true,
- /* UseARMGuardVarABI = */ false);
+ return new ItaniumCXXABI(CGM, /*UseARMMethodPtrABI=*/true);
}
return new ItaniumCXXABI(CGM);
@@ -541,8 +580,8 @@ CGCallee ItaniumCXXABI::EmitLoadOfMemberFunctionPointer(
const FunctionProtoType *FPT =
MPT->getPointeeType()->getAs<FunctionProtoType>();
- const CXXRecordDecl *RD =
- cast<CXXRecordDecl>(MPT->getClass()->getAs<RecordType>()->getDecl());
+ auto *RD =
+ cast<CXXRecordDecl>(MPT->getClass()->castAs<RecordType>()->getDecl());
llvm::FunctionType *FTy = CGM.getTypes().GetFunctionType(
CGM.getTypes().arrangeCXXMethodType(RD, FPT, /*FD=*/nullptr));
@@ -605,8 +644,6 @@ CGCallee ItaniumCXXABI::EmitLoadOfMemberFunctionPointer(
VTableOffset = Builder.CreateTrunc(VTableOffset, CGF.Int32Ty);
VTableOffset = Builder.CreateZExt(VTableOffset, CGM.PtrDiffTy);
}
- // Compute the address of the virtual function pointer.
- llvm::Value *VFPAddr = Builder.CreateGEP(VTable, VTableOffset);
// Check the address of the function pointer if CFI on member function
// pointers is enabled.
@@ -614,44 +651,81 @@ CGCallee ItaniumCXXABI::EmitLoadOfMemberFunctionPointer(
llvm::Constant *CheckTypeDesc;
bool ShouldEmitCFICheck = CGF.SanOpts.has(SanitizerKind::CFIMFCall) &&
CGM.HasHiddenLTOVisibility(RD);
- if (ShouldEmitCFICheck) {
- CodeGenFunction::SanitizerScope SanScope(&CGF);
-
- CheckSourceLocation = CGF.EmitCheckSourceLocation(E->getBeginLoc());
- CheckTypeDesc = CGF.EmitCheckTypeDescriptor(QualType(MPT, 0));
- llvm::Constant *StaticData[] = {
- llvm::ConstantInt::get(CGF.Int8Ty, CodeGenFunction::CFITCK_VMFCall),
- CheckSourceLocation,
- CheckTypeDesc,
- };
-
- llvm::Metadata *MD =
- CGM.CreateMetadataIdentifierForVirtualMemPtrType(QualType(MPT, 0));
- llvm::Value *TypeId = llvm::MetadataAsValue::get(CGF.getLLVMContext(), MD);
+ bool ShouldEmitVFEInfo = CGM.getCodeGenOpts().VirtualFunctionElimination &&
+ CGM.HasHiddenLTOVisibility(RD);
+ llvm::Value *VirtualFn = nullptr;
- llvm::Value *TypeTest = Builder.CreateCall(
- CGM.getIntrinsic(llvm::Intrinsic::type_test), {VFPAddr, TypeId});
+ {
+ CodeGenFunction::SanitizerScope SanScope(&CGF);
+ llvm::Value *TypeId = nullptr;
+ llvm::Value *CheckResult = nullptr;
+
+ if (ShouldEmitCFICheck || ShouldEmitVFEInfo) {
+ // If doing CFI or VFE, we will need the metadata node to check against.
+ llvm::Metadata *MD =
+ CGM.CreateMetadataIdentifierForVirtualMemPtrType(QualType(MPT, 0));
+ TypeId = llvm::MetadataAsValue::get(CGF.getLLVMContext(), MD);
+ }
- if (CGM.getCodeGenOpts().SanitizeTrap.has(SanitizerKind::CFIMFCall)) {
- CGF.EmitTrapCheck(TypeTest);
+ llvm::Value *VFPAddr = Builder.CreateGEP(VTable, VTableOffset);
+
+ if (ShouldEmitVFEInfo) {
+ // If doing VFE, load from the vtable with a type.checked.load intrinsic
+ // call. Note that we use the GEP to calculate the address to load from
+ // and pass 0 as the offset to the intrinsic. This is because every
+ // vtable slot of the correct type is marked with matching metadata, and
+ // we know that the load must be from one of these slots.
+ llvm::Value *CheckedLoad = Builder.CreateCall(
+ CGM.getIntrinsic(llvm::Intrinsic::type_checked_load),
+ {VFPAddr, llvm::ConstantInt::get(CGM.Int32Ty, 0), TypeId});
+ CheckResult = Builder.CreateExtractValue(CheckedLoad, 1);
+ VirtualFn = Builder.CreateExtractValue(CheckedLoad, 0);
+ VirtualFn = Builder.CreateBitCast(VirtualFn, FTy->getPointerTo(),
+ "memptr.virtualfn");
} else {
- llvm::Value *AllVtables = llvm::MetadataAsValue::get(
- CGM.getLLVMContext(),
- llvm::MDString::get(CGM.getLLVMContext(), "all-vtables"));
- llvm::Value *ValidVtable = Builder.CreateCall(
- CGM.getIntrinsic(llvm::Intrinsic::type_test), {VTable, AllVtables});
- CGF.EmitCheck(std::make_pair(TypeTest, SanitizerKind::CFIMFCall),
- SanitizerHandler::CFICheckFail, StaticData,
- {VTable, ValidVtable});
+ // When not doing VFE, emit a normal load, as it allows more
+ // optimisations than type.checked.load.
+ if (ShouldEmitCFICheck) {
+ CheckResult = Builder.CreateCall(
+ CGM.getIntrinsic(llvm::Intrinsic::type_test),
+ {Builder.CreateBitCast(VFPAddr, CGF.Int8PtrTy), TypeId});
+ }
+ VFPAddr =
+ Builder.CreateBitCast(VFPAddr, FTy->getPointerTo()->getPointerTo());
+ VirtualFn = Builder.CreateAlignedLoad(VFPAddr, CGF.getPointerAlign(),
+ "memptr.virtualfn");
}
+ assert(VirtualFn && "Virtual fuction pointer not created!");
+ assert((!ShouldEmitCFICheck || !ShouldEmitVFEInfo || CheckResult) &&
+ "Check result required but not created!");
+
+ if (ShouldEmitCFICheck) {
+ // If doing CFI, emit the check.
+ CheckSourceLocation = CGF.EmitCheckSourceLocation(E->getBeginLoc());
+ CheckTypeDesc = CGF.EmitCheckTypeDescriptor(QualType(MPT, 0));
+ llvm::Constant *StaticData[] = {
+ llvm::ConstantInt::get(CGF.Int8Ty, CodeGenFunction::CFITCK_VMFCall),
+ CheckSourceLocation,
+ CheckTypeDesc,
+ };
- FnVirtual = Builder.GetInsertBlock();
- }
+ if (CGM.getCodeGenOpts().SanitizeTrap.has(SanitizerKind::CFIMFCall)) {
+ CGF.EmitTrapCheck(CheckResult);
+ } else {
+ llvm::Value *AllVtables = llvm::MetadataAsValue::get(
+ CGM.getLLVMContext(),
+ llvm::MDString::get(CGM.getLLVMContext(), "all-vtables"));
+ llvm::Value *ValidVtable = Builder.CreateCall(
+ CGM.getIntrinsic(llvm::Intrinsic::type_test), {VTable, AllVtables});
+ CGF.EmitCheck(std::make_pair(CheckResult, SanitizerKind::CFIMFCall),
+ SanitizerHandler::CFICheckFail, StaticData,
+ {VTable, ValidVtable});
+ }
+
+ FnVirtual = Builder.GetInsertBlock();
+ }
+ } // End of sanitizer scope
- // Load the virtual function to call.
- VFPAddr = Builder.CreateBitCast(VFPAddr, FTy->getPointerTo()->getPointerTo());
- llvm::Value *VirtualFn = Builder.CreateAlignedLoad(
- VFPAddr, CGF.getPointerAlign(), "memptr.virtualfn");
CGF.EmitBranch(FnEnd);
// In the non-virtual path, the function pointer is actually a
@@ -1104,7 +1178,7 @@ void ItaniumCXXABI::emitVirtualObjectDelete(CodeGenFunction &CGF,
// Grab the vtable pointer as an intptr_t*.
auto *ClassDecl =
- cast<CXXRecordDecl>(ElementType->getAs<RecordType>()->getDecl());
+ cast<CXXRecordDecl>(ElementType->castAs<RecordType>()->getDecl());
llvm::Value *VTable =
CGF.GetVTablePtr(Ptr, CGF.IntPtrTy->getPointerTo(), ClassDecl);
@@ -1307,7 +1381,7 @@ llvm::Value *ItaniumCXXABI::EmitTypeid(CodeGenFunction &CGF,
Address ThisPtr,
llvm::Type *StdTypeInfoPtrTy) {
auto *ClassDecl =
- cast<CXXRecordDecl>(SrcRecordTy->getAs<RecordType>()->getDecl());
+ cast<CXXRecordDecl>(SrcRecordTy->castAs<RecordType>()->getDecl());
llvm::Value *Value =
CGF.GetVTablePtr(ThisPtr, StdTypeInfoPtrTy->getPointerTo(), ClassDecl);
@@ -1373,7 +1447,7 @@ llvm::Value *ItaniumCXXABI::EmitDynamicCastToVoid(CodeGenFunction &CGF,
llvm::Type *DestLTy = CGF.ConvertType(DestTy);
auto *ClassDecl =
- cast<CXXRecordDecl>(SrcRecordTy->getAs<RecordType>()->getDecl());
+ cast<CXXRecordDecl>(SrcRecordTy->castAs<RecordType>()->getDecl());
// Get the vtable pointer.
llvm::Value *VTable = CGF.GetVTablePtr(ThisAddr, PtrDiffLTy->getPointerTo(),
ClassDecl);
@@ -1595,7 +1669,7 @@ void ItaniumCXXABI::emitVTableDefinitions(CodeGenVTables &CGVT,
EmitFundamentalRTTIDescriptors(RD);
if (!VTable->isDeclarationForLinker())
- CGM.EmitVTableTypeMetadata(VTable, VTLayout);
+ CGM.EmitVTableTypeMetadata(RD, VTable, VTLayout);
}
bool ItaniumCXXABI::isVirtualOffsetNeededForVTableField(
@@ -1755,10 +1829,11 @@ llvm::Value *ItaniumCXXABI::EmitVirtualDestructorCall(
CGCallee Callee = CGCallee::forVirtual(CE, GD, This, Ty);
QualType ThisTy;
- if (CE)
- ThisTy = CE->getImplicitObjectArgument()->getType()->getPointeeType();
- else
+ if (CE) {
+ ThisTy = CE->getObjectType();
+ } else {
ThisTy = D->getDestroyedType();
+ }
CGF.EmitCXXDestructorCall(GD, Callee, This.getPointer(), ThisTy, nullptr,
QualType(), nullptr);
@@ -2154,7 +2229,7 @@ void ItaniumCXXABI::EmitGuardedInit(CodeGenFunction &CGF,
guard->setVisibility(var->getVisibility());
// If the variable is thread-local, so is its guard variable.
guard->setThreadLocalMode(var->getThreadLocalMode());
- guard->setAlignment(guardAlignment.getQuantity());
+ guard->setAlignment(guardAlignment.getAsAlign());
// The ABI says: "It is suggested that it be emitted in the same COMDAT
// group as the associated data object." In practice, this doesn't work for
@@ -2456,9 +2531,6 @@ ItaniumCXXABI::getOrCreateThreadLocalWrapper(const VarDecl *VD,
CGM.SetLLVMFunctionAttributes(GlobalDecl(), FI, Wrapper);
- if (VD->hasDefinition())
- CGM.SetLLVMFunctionAttributesForDefinition(nullptr, Wrapper);
-
// Always resolve references to the wrapper at link time.
if (!Wrapper->hasLocalLinkage())
if (!isThreadWrapperReplaceable(VD, CGM) ||
@@ -2471,6 +2543,8 @@ ItaniumCXXABI::getOrCreateThreadLocalWrapper(const VarDecl *VD,
Wrapper->setCallingConv(llvm::CallingConv::CXX_FAST_TLS);
Wrapper->addFnAttr(llvm::Attribute::NoUnwind);
}
+
+ ThreadWrappers.push_back({VD, Wrapper});
return Wrapper;
}
@@ -2508,7 +2582,7 @@ void ItaniumCXXABI::EmitThreadLocalInitFuncs(
Guard->setThreadLocal(true);
CharUnits GuardAlign = CharUnits::One();
- Guard->setAlignment(GuardAlign.getQuantity());
+ Guard->setAlignment(GuardAlign.getAsAlign());
CodeGenFunction(CGM).GenerateCXXGlobalInitFunc(
InitFunc, OrderedInits, ConstantAddress(Guard, GuardAlign));
@@ -2519,20 +2593,40 @@ void ItaniumCXXABI::EmitThreadLocalInitFuncs(
}
}
- // Emit thread wrappers.
+ // Create declarations for thread wrappers for all thread-local variables
+ // with non-discardable definitions in this translation unit.
for (const VarDecl *VD : CXXThreadLocals) {
+ if (VD->hasDefinition() &&
+ !isDiscardableGVALinkage(getContext().GetGVALinkageForVariable(VD))) {
+ llvm::GlobalValue *GV = CGM.GetGlobalValue(CGM.getMangledName(VD));
+ getOrCreateThreadLocalWrapper(VD, GV);
+ }
+ }
+
+ // Emit all referenced thread wrappers.
+ for (auto VDAndWrapper : ThreadWrappers) {
+ const VarDecl *VD = VDAndWrapper.first;
llvm::GlobalVariable *Var =
cast<llvm::GlobalVariable>(CGM.GetGlobalValue(CGM.getMangledName(VD)));
- llvm::Function *Wrapper = getOrCreateThreadLocalWrapper(VD, Var);
+ llvm::Function *Wrapper = VDAndWrapper.second;
// Some targets require that all access to thread local variables go through
// the thread wrapper. This means that we cannot attempt to create a thread
// wrapper or a thread helper.
- if (isThreadWrapperReplaceable(VD, CGM) && !VD->hasDefinition()) {
- Wrapper->setLinkage(llvm::Function::ExternalLinkage);
- continue;
+ if (!VD->hasDefinition()) {
+ if (isThreadWrapperReplaceable(VD, CGM)) {
+ Wrapper->setLinkage(llvm::Function::ExternalLinkage);
+ continue;
+ }
+
+ // If this isn't a TU in which this variable is defined, the thread
+ // wrapper is discardable.
+ if (Wrapper->getLinkage() == llvm::Function::WeakODRLinkage)
+ Wrapper->setLinkage(llvm::Function::LinkOnceODRLinkage);
}
+ CGM.SetLLVMFunctionAttributesForDefinition(nullptr, Wrapper);
+
// Mangle the name for the thread_local initialization function.
SmallString<256> InitFnName;
{
@@ -2547,7 +2641,10 @@ void ItaniumCXXABI::EmitThreadLocalInitFuncs(
// produce a declaration of the initialization function.
llvm::GlobalValue *Init = nullptr;
bool InitIsInitFunc = false;
- if (VD->hasDefinition()) {
+ bool HasConstantInitialization = false;
+ if (!usesThreadWrapperFunction(VD)) {
+ HasConstantInitialization = true;
+ } else if (VD->hasDefinition()) {
InitIsInitFunc = true;
llvm::Function *InitFuncToUse = InitFunc;
if (isTemplateInstantiation(VD->getTemplateSpecializationKind()))
@@ -2576,7 +2673,9 @@ void ItaniumCXXABI::EmitThreadLocalInitFuncs(
llvm::LLVMContext &Context = CGM.getModule().getContext();
llvm::BasicBlock *Entry = llvm::BasicBlock::Create(Context, "", Wrapper);
CGBuilderTy Builder(CGM, Entry);
- if (InitIsInitFunc) {
+ if (HasConstantInitialization) {
+ // No dynamic initialization to invoke.
+ } else if (InitIsInitFunc) {
if (Init) {
llvm::CallInst *CallVal = Builder.CreateCall(InitFnTy, Init);
if (isThreadWrapperReplaceable(VD, CGM)) {
@@ -2860,6 +2959,9 @@ static bool TypeInfoIsInStandardLibrary(const BuiltinType *Ty) {
case BuiltinType::OCLClkEvent:
case BuiltinType::OCLQueue:
case BuiltinType::OCLReserveID:
+#define SVE_TYPE(Name, Id, SingletonId) \
+ case BuiltinType::Id:
+#include "clang/Basic/AArch64SVEACLETypes.def"
case BuiltinType::ShortAccum:
case BuiltinType::Accum:
case BuiltinType::LongAccum:
@@ -3033,8 +3135,8 @@ static bool CanUseSingleInheritance(const CXXRecordDecl *RD) {
return false;
// Check that the class is dynamic iff the base is.
- const CXXRecordDecl *BaseDecl =
- cast<CXXRecordDecl>(Base->getType()->getAs<RecordType>()->getDecl());
+ auto *BaseDecl =
+ cast<CXXRecordDecl>(Base->getType()->castAs<RecordType>()->getDecl());
if (!BaseDecl->isEmpty() &&
BaseDecl->isDynamicClass() != RD->isDynamicClass())
return false;
@@ -3061,7 +3163,7 @@ void ItaniumRTTIBuilder::BuildVTablePointer(const Type *Ty) {
#define NON_CANONICAL_UNLESS_DEPENDENT_TYPE(Class, Base) case Type::Class:
#define NON_CANONICAL_TYPE(Class, Base) case Type::Class:
#define DEPENDENT_TYPE(Class, Base) case Type::Class:
-#include "clang/AST/TypeNodes.def"
+#include "clang/AST/TypeNodes.inc"
llvm_unreachable("Non-canonical and dependent types shouldn't get here");
case Type::LValueReference:
@@ -3307,7 +3409,7 @@ llvm::Constant *ItaniumRTTIBuilder::BuildTypeInfo(
#define NON_CANONICAL_UNLESS_DEPENDENT_TYPE(Class, Base) case Type::Class:
#define NON_CANONICAL_TYPE(Class, Base) case Type::Class:
#define DEPENDENT_TYPE(Class, Base) case Type::Class:
-#include "clang/AST/TypeNodes.def"
+#include "clang/AST/TypeNodes.inc"
llvm_unreachable("Non-canonical and dependent types shouldn't get here");
// GCC treats vector types as fundamental types.
@@ -3412,7 +3514,7 @@ llvm::Constant *ItaniumRTTIBuilder::BuildTypeInfo(
CharUnits Align =
CGM.getContext().toCharUnitsFromBits(CGM.getTarget().getPointerAlign(0));
- GV->setAlignment(Align.getQuantity());
+ GV->setAlignment(Align.getAsAlign());
// The Itanium ABI specifies that type_info objects must be globally
// unique, with one exception: if the type is an incomplete class
@@ -3497,8 +3599,8 @@ static unsigned ComputeVMIClassTypeInfoFlags(const CXXBaseSpecifier *Base,
unsigned Flags = 0;
- const CXXRecordDecl *BaseDecl =
- cast<CXXRecordDecl>(Base->getType()->getAs<RecordType>()->getDecl());
+ auto *BaseDecl =
+ cast<CXXRecordDecl>(Base->getType()->castAs<RecordType>()->getDecl());
if (Base->isVirtual()) {
// Mark the virtual base as seen.
@@ -3596,8 +3698,8 @@ void ItaniumRTTIBuilder::BuildVMIClassTypeInfo(const CXXRecordDecl *RD) {
// The __base_type member points to the RTTI for the base type.
Fields.push_back(ItaniumRTTIBuilder(CXXABI).BuildTypeInfo(Base.getType()));
- const CXXRecordDecl *BaseDecl =
- cast<CXXRecordDecl>(Base.getType()->getAs<RecordType>()->getDecl());
+ auto *BaseDecl =
+ cast<CXXRecordDecl>(Base.getType()->castAs<RecordType>()->getDecl());
int64_t OffsetFlags = 0;