Update clang to r89205.

32 files changed, 2289 insertions, 1030 deletions

diff --git a/lib/CodeGen/CGBlocks.cpp b/lib/CodeGen/CGBlocks.cpp
index 682cf5da1e72..bc9eb67674ff 100644
--- a/lib/CodeGen/CGBlocks.cpp
+++ b/lib/CodeGen/CGBlocks.cpp
@@ -146,7 +146,23 @@ llvm::Value *CodeGenFunction::BuildBlockLiteralTmp(const BlockExpr *BE) {
   if (0 && CanBlockBeGlobal(Info))
     return CGM.GetAddrOfGlobalBlock(BE, Name.c_str());
 
-  std::vector<llvm::Constant*> Elts(5);
+  size_t BlockFields = 5;
+
+  bool hasIntrospection  = CGM.getContext().getLangOptions().BlockIntrospection;
+
+  if (hasIntrospection) {
+    BlockFields++;
+  }
+  std::vector<llvm::Constant*> Elts(BlockFields);
+
+  if (hasIntrospection) {
+    std::string BlockTypeEncoding;
+    CGM.getContext().getObjCEncodingForBlock(BE, BlockTypeEncoding);
+
+    Elts[5] = llvm::ConstantExpr::getBitCast(
+            CGM.GetAddrOfConstantCString(BlockTypeEncoding), PtrToInt8Ty);
+  }
+
   llvm::Constant *C;
   llvm::Value *V;
 
@@ -154,6 +170,9 @@ llvm::Value *CodeGenFunction::BuildBlockLiteralTmp(const BlockExpr *BE) {
     // C = BuildBlockStructInitlist();
     unsigned int flags = BLOCK_HAS_DESCRIPTOR;
 
+    if (hasIntrospection)
+      flags |= BLOCK_HAS_OBJC_TYPE;
+
     // We run this first so that we set BlockHasCopyDispose from the entire
     // block literal.
     // __invoke
@@ -211,19 +230,21 @@ llvm::Value *CodeGenFunction::BuildBlockLiteralTmp(const BlockExpr *BE) {
       return C;
     }
 
-    std::vector<const llvm::Type *> Types(5+subBlockDeclRefDecls.size());
+    std::vector<const llvm::Type *> Types(BlockFields+subBlockDeclRefDecls.size());
     for (int i=0; i<4; ++i)
       Types[i] = Elts[i]->getType();
     Types[4] = PtrToInt8Ty;
+    if (hasIntrospection)
+      Types[5] = PtrToInt8Ty;
 
     for (unsigned i=0; i < subBlockDeclRefDecls.size(); ++i) {
       const Expr *E = subBlockDeclRefDecls[i];
       const BlockDeclRefExpr *BDRE = dyn_cast<BlockDeclRefExpr>(E);
       QualType Ty = E->getType();
       if (BDRE && BDRE->isByRef()) {
-        Types[i+5] = llvm::PointerType::get(BuildByRefType(BDRE->getDecl()), 0);
+        Types[i+BlockFields] = llvm::PointerType::get(BuildByRefType(BDRE->getDecl()), 0);
       } else
-        Types[i+5] = ConvertType(Ty);
+        Types[i+BlockFields] = ConvertType(Ty);
     }
 
     llvm::StructType *Ty = llvm::StructType::get(VMContext, Types, true);
@@ -237,6 +258,8 @@ llvm::Value *CodeGenFunction::BuildBlockLiteralTmp(const BlockExpr *BE) {
 
     for (unsigned i=0; i<4; ++i)
       Builder.CreateStore(Elts[i], Builder.CreateStructGEP(V, i, "block.tmp"));
+    if (hasIntrospection)
+      Builder.CreateStore(Elts[5], Builder.CreateStructGEP(V, 5, "block.tmp"));
 
     for (unsigned i=0; i < subBlockDeclRefDecls.size(); ++i)
       {
@@ -252,7 +275,7 @@ llvm::Value *CodeGenFunction::BuildBlockLiteralTmp(const BlockExpr *BE) {
         BlockDeclRefExpr *BDRE = dyn_cast<BlockDeclRefExpr>(E);
         VD = BDRE->getDecl();
 
-        llvm::Value* Addr = Builder.CreateStructGEP(V, i+5, "tmp");
+        llvm::Value* Addr = Builder.CreateStructGEP(V, i+BlockFields, "tmp");
         NoteForHelper[helpersize].index = i+5;
         NoteForHelper[helpersize].RequiresCopying
           = BlockRequiresCopying(VD->getType());
@@ -291,7 +314,7 @@ llvm::Value *CodeGenFunction::BuildBlockLiteralTmp(const BlockExpr *BE) {
         RValue r = EmitAnyExpr(E, Addr, false);
         if (r.isScalar()) {
           llvm::Value *Loc = r.getScalarVal();
-          const llvm::Type *Ty = Types[i+5];
+          const llvm::Type *Ty = Types[i+BlockFields];
           if  (BDRE->isByRef()) {
             // E is now the address of the value field, instead, we want the
             // address of the actual ByRef struct.  We optimize this slightly
@@ -375,8 +398,20 @@ const llvm::Type *BlockModule::getGenericBlockLiteralType() {
   //   int __reserved;
   //   void (*__invoke)(void *);
   //   struct __block_descriptor *__descriptor;
+  //   // GNU runtime only:
+  //   const char *types;
   // };
-  GenericBlockLiteralType = llvm::StructType::get(IntTy->getContext(),
+  if (CGM.getContext().getLangOptions().BlockIntrospection)
+    GenericBlockLiteralType = llvm::StructType::get(IntTy->getContext(),
+                                                  PtrToInt8Ty,
+                                                  IntTy,
+                                                  IntTy,
+                                                  PtrToInt8Ty,
+                                                  BlockDescPtrTy,
+                                                  PtrToInt8Ty,
+                                                  NULL);
+  else
+    GenericBlockLiteralType = llvm::StructType::get(IntTy->getContext(),
                                                   PtrToInt8Ty,
                                                   IntTy,
                                                   IntTy,
@@ -571,8 +606,12 @@ BlockModule::GetAddrOfGlobalBlock(const BlockExpr *BE, const char * n) {
                              llvm::GlobalVariable::InternalLinkage,
                              DescriptorStruct, "__block_descriptor_global");
 
+  int FieldCount = 5;
   // Generate the constants for the block literal.
-  llvm::Constant *LiteralFields[5];
+  if (CGM.getContext().getLangOptions().BlockIntrospection)
+    FieldCount = 6;
+
+  std::vector<llvm::Constant*> LiteralFields(FieldCount);
 
   CodeGenFunction::BlockInfo Info(0, n);
   uint64_t subBlockSize, subBlockAlign;
@@ -592,7 +631,9 @@ BlockModule::GetAddrOfGlobalBlock(const BlockExpr *BE, const char * n) {
   LiteralFields[0] = getNSConcreteGlobalBlock();
 
   // Flags
-  LiteralFields[1] =
+  LiteralFields[1] = CGM.getContext().getLangOptions().BlockIntrospection ?
+    llvm::ConstantInt::get(IntTy, BLOCK_IS_GLOBAL | BLOCK_HAS_DESCRIPTOR |
+            BLOCK_HAS_OBJC_TYPE) :
     llvm::ConstantInt::get(IntTy, BLOCK_IS_GLOBAL | BLOCK_HAS_DESCRIPTOR);
 
   // Reserved
@@ -603,9 +644,17 @@ BlockModule::GetAddrOfGlobalBlock(const BlockExpr *BE, const char * n) {
 
   // Descriptor
   LiteralFields[4] = Descriptor;
+  
+  // Type encoding
+  if (CGM.getContext().getLangOptions().BlockIntrospection) {
+    std::string BlockTypeEncoding;
+    CGM.getContext().getObjCEncodingForBlock(BE, BlockTypeEncoding);
+
+    LiteralFields[5] = CGM.GetAddrOfConstantCString(BlockTypeEncoding);
+  }
 
   llvm::Constant *BlockLiteralStruct =
-    llvm::ConstantStruct::get(VMContext, &LiteralFields[0], 5, false);
+    llvm::ConstantStruct::get(VMContext, LiteralFields, false);
 
   llvm::GlobalVariable *BlockLiteral =
     new llvm::GlobalVariable(getModule(), BlockLiteralStruct->getType(), true,
diff --git a/lib/CodeGen/CGBlocks.h b/lib/CodeGen/CGBlocks.h
index 3ab4efb71bee..38e02a70a4e2 100644
--- a/lib/CodeGen/CGBlocks.h
+++ b/lib/CodeGen/CGBlocks.h
@@ -55,7 +55,8 @@ public:
         BLOCK_HAS_CXX_OBJ =       (1 << 26),
         BLOCK_IS_GC =             (1 << 27),
         BLOCK_IS_GLOBAL =         (1 << 28),
-        BLOCK_HAS_DESCRIPTOR =    (1 << 29)
+        BLOCK_HAS_DESCRIPTOR =    (1 << 29),
+        BLOCK_HAS_OBJC_TYPE  =    (1 << 30)
     };
 };
 
diff --git a/lib/CodeGen/CGBuiltin.cpp b/lib/CodeGen/CGBuiltin.cpp
index c26921969a88..399b8733e720 100644
--- a/lib/CodeGen/CGBuiltin.cpp
+++ b/lib/CodeGen/CGBuiltin.cpp
@@ -121,7 +121,8 @@ RValue CodeGenFunction::EmitBuiltinExpr(const FunctionDecl *FD,
     const llvm::Type *ResultType = ConvertType(E->getType());
     Value *Result = Builder.CreateCall(F, ArgValue, "tmp");
     if (Result->getType() != ResultType)
-      Result = Builder.CreateIntCast(Result, ResultType, "cast");
+      Result = Builder.CreateIntCast(Result, ResultType, /*isSigned*/true,
+                                     "cast");
     return RValue::get(Result);
   }
   case Builtin::BI__builtin_clz:
@@ -135,7 +136,8 @@ RValue CodeGenFunction::EmitBuiltinExpr(const FunctionDecl *FD,
     const llvm::Type *ResultType = ConvertType(E->getType());
     Value *Result = Builder.CreateCall(F, ArgValue, "tmp");
     if (Result->getType() != ResultType)
-      Result = Builder.CreateIntCast(Result, ResultType, "cast");
+      Result = Builder.CreateIntCast(Result, ResultType, /*isSigned*/true,
+                                     "cast");
     return RValue::get(Result);
   }
   case Builtin::BI__builtin_ffs:
@@ -154,7 +156,8 @@ RValue CodeGenFunction::EmitBuiltinExpr(const FunctionDecl *FD,
     Value *IsZero = Builder.CreateICmpEQ(ArgValue, Zero, "iszero");
     Value *Result = Builder.CreateSelect(IsZero, Zero, Tmp, "ffs");
     if (Result->getType() != ResultType)
-      Result = Builder.CreateIntCast(Result, ResultType, "cast");
+      Result = Builder.CreateIntCast(Result, ResultType, /*isSigned*/true,
+                                     "cast");
     return RValue::get(Result);
   }
   case Builtin::BI__builtin_parity:
@@ -171,7 +174,8 @@ RValue CodeGenFunction::EmitBuiltinExpr(const FunctionDecl *FD,
     Value *Result = Builder.CreateAnd(Tmp, llvm::ConstantInt::get(ArgType, 1),
                                       "tmp");
     if (Result->getType() != ResultType)
-      Result = Builder.CreateIntCast(Result, ResultType, "cast");
+      Result = Builder.CreateIntCast(Result, ResultType, /*isSigned*/true,
+                                     "cast");
     return RValue::get(Result);
   }
   case Builtin::BI__builtin_popcount:
@@ -185,7 +189,8 @@ RValue CodeGenFunction::EmitBuiltinExpr(const FunctionDecl *FD,
     const llvm::Type *ResultType = ConvertType(E->getType());
     Value *Result = Builder.CreateCall(F, ArgValue, "tmp");
     if (Result->getType() != ResultType)
-      Result = Builder.CreateIntCast(Result, ResultType, "cast");
+      Result = Builder.CreateIntCast(Result, ResultType, /*isSigned*/true,
+                                     "cast");
     return RValue::get(Result);
   }
   case Builtin::BI__builtin_expect:
@@ -199,8 +204,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const FunctionDecl *FD,
     return RValue::get(Builder.CreateCall(F, ArgValue, "tmp"));
   }
   case Builtin::BI__builtin_object_size: {
-    // FIXME: We're awaiting the llvm intrincis.
-#if 0
+#if 1
     // We pass this builtin onto the optimizer so that it can
     // figure out the object size in more complex cases.
     const llvm::Type *ResType[] = {
@@ -211,8 +215,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const FunctionDecl *FD,
                                            EmitScalarExpr(E->getArg(0)),
                                            EmitScalarExpr(E->getArg(1))));
 #else
-    // FIXME: Implement. For now we just always fail and pretend we
-    // don't know the object size.
+    // FIXME: Remove after testing.
     llvm::APSInt TypeArg = E->getArg(1)->EvaluateAsInt(CGM.getContext());
     const llvm::Type *ResType = ConvertType(E->getType());
     //    bool UseSubObject = TypeArg.getZExtValue() & 1;
diff --git a/lib/CodeGen/CGCXX.cpp b/lib/CodeGen/CGCXX.cpp
index cf172b1a1cb6..8f5cff4efafe 100644
--- a/lib/CodeGen/CGCXX.cpp
+++ b/lib/CodeGen/CGCXX.cpp
@@ -27,7 +27,7 @@ using namespace clang;
 using namespace CodeGen;
 
 void
-CodeGenFunction::EmitCXXGlobalDtorRegistration(const CXXDestructorDecl *Dtor,
+CodeGenFunction::EmitCXXGlobalDtorRegistration(llvm::Constant *DtorFn,
                                                llvm::Constant *DeclPtr) {
   const llvm::Type *Int8PtrTy = 
     llvm::Type::getInt8Ty(VMContext)->getPointerTo();
@@ -55,9 +55,6 @@ CodeGenFunction::EmitCXXGlobalDtorRegistration(const CXXDestructorDecl *Dtor,
 
   llvm::Constant *Handle = CGM.CreateRuntimeVariable(Int8PtrTy,
                                                      "__dso_handle");
-
-  llvm::Constant *DtorFn = CGM.GetAddrOfCXXDestructor(Dtor, Dtor_Complete);
-
   llvm::Value *Args[3] = { llvm::ConstantExpr::getBitCast(DtorFn, DtorFnTy),
                            llvm::ConstantExpr::getBitCast(DeclPtr, Int8PtrTy),
                            llvm::ConstantExpr::getBitCast(Handle, Int8PtrTy) };
@@ -82,11 +79,29 @@ void CodeGenFunction::EmitCXXGlobalVarDeclInit(const VarDecl &D,
     EmitComplexExprIntoAddr(Init, DeclPtr, isVolatile);
   } else {
     EmitAggExpr(Init, DeclPtr, isVolatile);
-
+    // Avoid generating destructor(s) for initialized objects. 
+    if (!isa<CXXConstructExpr>(Init))
+      return;
+    const ConstantArrayType *Array = getContext().getAsConstantArrayType(T);
+    if (Array)
+      T = getContext().getBaseElementType(Array);
+    
     if (const RecordType *RT = T->getAs<RecordType>()) {
       CXXRecordDecl *RD = cast<CXXRecordDecl>(RT->getDecl());
-      if (!RD->hasTrivialDestructor())
-        EmitCXXGlobalDtorRegistration(RD->getDestructor(getContext()), DeclPtr);
+      if (!RD->hasTrivialDestructor()) {
+        llvm::Constant *DtorFn;
+        if (Array) {
+          DtorFn = CodeGenFunction(CGM).GenerateCXXAggrDestructorHelper(
+                                                RD->getDestructor(getContext()), 
+                                                Array, DeclPtr);
+          DeclPtr = 
+            llvm::Constant::getNullValue(llvm::Type::getInt8PtrTy(VMContext));
+        }
+        else
+          DtorFn = CGM.GetAddrOfCXXDestructor(RD->getDestructor(getContext()), 
+                                              Dtor_Complete);                                
+        EmitCXXGlobalDtorRegistration(DtorFn, DeclPtr);
+      }
     }
   }
 }
@@ -267,14 +282,20 @@ RValue CodeGenFunction::EmitCXXMemberCallExpr(const CXXMemberCallExpr *CE) {
   // We also don't emit a virtual call if the base expression has a record type
   // because then we know what the type is.
   llvm::Value *Callee;
-  if (MD->isVirtual() && !ME->hasQualifier() && 
-      !canDevirtualizeMemberFunctionCalls(ME->getBase()))
-    Callee = BuildVirtualCall(MD, This, Ty);
-  else if (const CXXDestructorDecl *Destructor
-             = dyn_cast<CXXDestructorDecl>(MD))
-    Callee = CGM.GetAddrOfFunction(GlobalDecl(Destructor, Dtor_Complete), Ty);
-  else
+  if (const CXXDestructorDecl *Destructor
+             = dyn_cast<CXXDestructorDecl>(MD)) {
+    if (MD->isVirtual() && !ME->hasQualifier() && 
+        !canDevirtualizeMemberFunctionCalls(ME->getBase())) {
+      Callee = BuildVirtualCall(Destructor, Dtor_Complete, This, Ty); 
+    } else {
+      Callee = CGM.GetAddrOfFunction(GlobalDecl(Destructor, Dtor_Complete), Ty);
+    }
+  } else if (MD->isVirtual() && !ME->hasQualifier() && 
+             !canDevirtualizeMemberFunctionCalls(ME->getBase())) {
+    Callee = BuildVirtualCall(MD, This, Ty); 
+  } else {
     Callee = CGM.GetAddrOfFunction(MD, Ty);
+  }
 
   return EmitCXXMemberCall(MD, Callee, This,
                            CE->arg_begin(), CE->arg_end());
@@ -410,10 +431,15 @@ CodeGenFunction::EmitCXXOperatorMemberCallExpr(const CXXOperatorCallExpr *E,
   const llvm::Type *Ty =
     CGM.getTypes().GetFunctionType(CGM.getTypes().getFunctionInfo(MD),
                                    FPT->isVariadic());
-  llvm::Constant *Callee = CGM.GetAddrOfFunction(MD, Ty);
 
   llvm::Value *This = EmitLValue(E->getArg(0)).getAddress();
 
+  llvm::Value *Callee;
+  if (MD->isVirtual() && !canDevirtualizeMemberFunctionCalls(E->getArg(0)))
+    Callee = BuildVirtualCall(MD, This, Ty);
+  else
+    Callee = CGM.GetAddrOfFunction(MD, Ty);
+
   return EmitCXXMemberCall(MD, Callee, This,
                            E->arg_begin() + 1, E->arg_end());
 }
@@ -506,16 +532,25 @@ CodeGenFunction::EmitCXXAggrDestructorCall(const CXXDestructorDecl *D,
                                            llvm::Value *This) {
   const ConstantArrayType *CA = dyn_cast<ConstantArrayType>(Array);
   assert(CA && "Do we support VLA for destruction ?");
+  uint64_t ElementCount = getContext().getConstantArrayElementCount(CA);
+  llvm::Value* ElementCountPtr =
+    llvm::ConstantInt::get(llvm::Type::getInt64Ty(VMContext), ElementCount);
+  EmitCXXAggrDestructorCall(D, ElementCountPtr, This);
+}
+
+/// EmitCXXAggrDestructorCall - calls the default destructor on array
+/// elements in reverse order of construction.
+void
+CodeGenFunction::EmitCXXAggrDestructorCall(const CXXDestructorDecl *D,
+                                           llvm::Value *UpperCount,
+                                           llvm::Value *This) {
   llvm::Value *One = llvm::ConstantInt::get(llvm::Type::getInt64Ty(VMContext),
                                             1);
-  uint64_t ElementCount = getContext().getConstantArrayElementCount(CA);
   // Create a temporary for the loop index and initialize it with count of
   // array elements.
   llvm::Value *IndexPtr = CreateTempAlloca(llvm::Type::getInt64Ty(VMContext),
                                            "loop.index");
   // Index = ElementCount;
-  llvm::Value* UpperCount =
-    llvm::ConstantInt::get(llvm::Type::getInt64Ty(VMContext), ElementCount);
   Builder.CreateStore(UpperCount, IndexPtr, false);
 
   // Start the loop with a block that tests the condition.
@@ -543,7 +578,16 @@ CodeGenFunction::EmitCXXAggrDestructorCall(const CXXDestructorDecl *D,
   Counter = Builder.CreateLoad(IndexPtr);
   Counter = Builder.CreateSub(Counter, One);
   llvm::Value *Address = Builder.CreateInBoundsGEP(This, Counter, "arrayidx");
-  EmitCXXDestructorCall(D, Dtor_Complete, Address);
+  if (D->isVirtual()) {
+    const llvm::Type *Ty =
+      CGM.getTypes().GetFunctionType(CGM.getTypes().getFunctionInfo(D),
+                                     /*isVariadic=*/false);
+    
+    llvm::Value *Callee = BuildVirtualCall(D, Dtor_Deleting, Address, Ty);
+    EmitCXXMemberCall(D, Callee, Address, 0, 0);
+  }
+  else
+    EmitCXXDestructorCall(D, Dtor_Complete, Address);
 
   EmitBlock(ContinueBlock);
 
@@ -559,6 +603,50 @@ CodeGenFunction::EmitCXXAggrDestructorCall(const CXXDestructorDecl *D,
   EmitBlock(AfterFor, true);
 }
 
+/// GenerateCXXAggrDestructorHelper - Generates a helper function which when invoked,
+/// calls the default destructor on array elements in reverse order of 
+/// construction.
+llvm::Constant * 
+CodeGenFunction::GenerateCXXAggrDestructorHelper(const CXXDestructorDecl *D,
+                                                 const ArrayType *Array,
+                                                 llvm::Value *This) {
+  static int UniqueCount;
+  FunctionArgList Args;
+  ImplicitParamDecl *Dst =
+    ImplicitParamDecl::Create(getContext(), 0,
+                              SourceLocation(), 0,
+                              getContext().getPointerType(getContext().VoidTy));
+  Args.push_back(std::make_pair(Dst, Dst->getType()));
+  
+  llvm::SmallString<16> Name;
+  llvm::raw_svector_ostream(Name) << "__tcf_" << (++UniqueCount);
+  QualType R = getContext().VoidTy;
+  const CGFunctionInfo &FI = CGM.getTypes().getFunctionInfo(R, Args);
+  const llvm::FunctionType *FTy = CGM.getTypes().GetFunctionType(FI, false);
+  llvm::Function *Fn =
+    llvm::Function::Create(FTy, llvm::GlobalValue::InternalLinkage,
+                           Name.c_str(),
+                           &CGM.getModule());
+  IdentifierInfo *II
+    = &CGM.getContext().Idents.get(Name.c_str());
+  FunctionDecl *FD = FunctionDecl::Create(getContext(),
+                                          getContext().getTranslationUnitDecl(),
+                                          SourceLocation(), II, R, 0,
+                                          FunctionDecl::Static,
+                                          false, true);
+  StartFunction(FD, R, Fn, Args, SourceLocation());
+  QualType BaseElementTy = getContext().getBaseElementType(Array);
+  const llvm::Type *BasePtr = ConvertType(BaseElementTy);
+  BasePtr = llvm::PointerType::getUnqual(BasePtr);
+  llvm::Value *BaseAddrPtr = Builder.CreateBitCast(This, BasePtr);
+  EmitCXXAggrDestructorCall(D, Array, BaseAddrPtr);
+  FinishFunction();
+  llvm::Type *Ptr8Ty = llvm::PointerType::get(llvm::Type::getInt8Ty(VMContext),
+                                              0);
+  llvm::Constant *m = llvm::ConstantExpr::getBitCast(Fn, Ptr8Ty);
+  return m;
+}
+
 void
 CodeGenFunction::EmitCXXConstructorCall(const CXXConstructorDecl *D,
                                         CXXCtorType Type,
@@ -612,8 +700,13 @@ CodeGenFunction::EmitCXXConstructExpr(llvm::Value *Dest,
   // Code gen optimization to eliminate copy constructor and return
   // its first argument instead.
   if (getContext().getLangOptions().ElideConstructors && E->isElidable()) {
-    CXXConstructExpr::const_arg_iterator i = E->arg_begin();
-    EmitAggExpr((*i), Dest, false);
+    const Expr *Arg = E->getArg(0);
+    
+    if (const CXXBindTemporaryExpr *BindExpr = 
+          dyn_cast<CXXBindTemporaryExpr>(Arg))
+      Arg = BindExpr->getSubExpr();
+
+    EmitAggExpr(Arg, Dest, false);
     return;
   }
   if (Array) {
@@ -649,8 +742,10 @@ void CodeGenModule::EmitCXXConstructor(const CXXConstructorDecl *D,
 llvm::Function *
 CodeGenModule::GetAddrOfCXXConstructor(const CXXConstructorDecl *D,
                                        CXXCtorType Type) {
+  const FunctionProtoType *FPT = D->getType()->getAs<FunctionProtoType>();
   const llvm::FunctionType *FTy =
-    getTypes().GetFunctionType(getTypes().getFunctionInfo(D), false);
+    getTypes().GetFunctionType(getTypes().getFunctionInfo(D), 
+                               FPT->isVariadic());
 
   const char *Name = getMangledCXXCtorName(D, Type);
   return cast<llvm::Function>(
@@ -668,6 +763,8 @@ const char *CodeGenModule::getMangledCXXCtorName(const CXXConstructorDecl *D,
 }
 
 void CodeGenModule::EmitCXXDestructors(const CXXDestructorDecl *D) {
+  if (D->isVirtual())
+    EmitCXXDestructor(D, Dtor_Deleting);
   EmitCXXDestructor(D, Dtor_Complete);
   EmitCXXDestructor(D, Dtor_Base);
 }
@@ -919,20 +1016,35 @@ CodeGenFunction::GetVirtualCXXBaseClassOffset(llvm::Value *This,
   return VBaseOffset;
 }
 
+static llvm::Value *BuildVirtualCall(CodeGenFunction &CGF, int64_t VtableIndex, 
+                                     llvm::Value *This, const llvm::Type *Ty) {
+  Ty = Ty->getPointerTo()->getPointerTo()->getPointerTo();
+  
+  llvm::Value *Vtable = CGF.Builder.CreateBitCast(This, Ty);
+  Vtable = CGF.Builder.CreateLoad(Vtable);
+  
+  llvm::Value *VFuncPtr = 
+    CGF.Builder.CreateConstInBoundsGEP1_64(Vtable, VtableIndex, "vfn");
+  return CGF.Builder.CreateLoad(VFuncPtr);
+}
+
 llvm::Value *
-CodeGenFunction::BuildVirtualCall(const CXXMethodDecl *MD, llvm::Value *&This,
+CodeGenFunction::BuildVirtualCall(const CXXMethodDecl *MD, llvm::Value *This,
                                   const llvm::Type *Ty) {
-  int64_t Index = CGM.getVtableInfo().getMethodVtableIndex(MD);
+  MD = MD->getCanonicalDecl();
+  int64_t VtableIndex = CGM.getVtableInfo().getMethodVtableIndex(MD);
   
-  Ty = llvm::PointerType::get(Ty, 0);
-  Ty = llvm::PointerType::get(Ty, 0);
-  Ty = llvm::PointerType::get(Ty, 0);
-  llvm::Value *vtbl = Builder.CreateBitCast(This, Ty);
-  vtbl = Builder.CreateLoad(vtbl);
-  llvm::Value *vfn = Builder.CreateConstInBoundsGEP1_64(vtbl,
-                                                        Index, "vfn");
-  vfn = Builder.CreateLoad(vfn);
-  return vfn;
+  return ::BuildVirtualCall(*this, VtableIndex, This, Ty);
+}
+
+llvm::Value *
+CodeGenFunction::BuildVirtualCall(const CXXDestructorDecl *DD, CXXDtorType Type, 
+                                  llvm::Value *&This, const llvm::Type *Ty) {
+  DD = cast<CXXDestructorDecl>(DD->getCanonicalDecl());
+  int64_t VtableIndex = 
+    CGM.getVtableInfo().getMethodVtableIndex(GlobalDecl(DD, Type));
+
+  return ::BuildVirtualCall(*this, VtableIndex, This, Ty);
 }
 
 /// EmitClassAggrMemberwiseCopy - This routine generates code to copy a class
@@ -1269,8 +1381,16 @@ CodeGenFunction::SynthesizeCXXCopyConstructor(const CXXConstructorDecl *Ctor,
     // Do a built-in assignment of scalar data members.
     LValue LHS = EmitLValueForField(LoadOfThis, *Field, false, 0);
     LValue RHS = EmitLValueForField(LoadOfSrc, *Field, false, 0);
-    RValue RVRHS = EmitLoadOfLValue(RHS, FieldType);
-    EmitStoreThroughLValue(RVRHS, LHS, FieldType);
+    if (!hasAggregateLLVMType(Field->getType())) {
+      RValue RVRHS = EmitLoadOfLValue(RHS, Field->getType());
+      EmitStoreThroughLValue(RVRHS, LHS, Field->getType());
+    } else if (Field->getType()->isAnyComplexType()) {
+      ComplexPairTy Pair = LoadComplexFromAddr(RHS.getAddress(),
+                                               RHS.isVolatileQualified());
+      StoreComplexToAddr(Pair, LHS.getAddress(), LHS.isVolatileQualified());
+    } else {
+      EmitAggregateCopy(LHS.getAddress(), RHS.getAddress(), Field->getType());
+    }
   }
   FinishFunction();
 }
@@ -1366,12 +1486,122 @@ void CodeGenFunction::SynthesizeCXXCopyAssignment(const CXXMethodDecl *CD,
   FinishFunction();
 }
 
+static void EmitBaseInitializer(CodeGenFunction &CGF, 
+                                const CXXRecordDecl *ClassDecl,
+                                CXXBaseOrMemberInitializer *BaseInit,
+                                CXXCtorType CtorType) {
+  assert(BaseInit->isBaseInitializer() &&
+         "Must have base initializer!");
+
+  llvm::Value *ThisPtr = CGF.LoadCXXThis();
+  
+  const Type *BaseType = BaseInit->getBaseClass();
+  CXXRecordDecl *BaseClassDecl =
+    cast<CXXRecordDecl>(BaseType->getAs<RecordType>()->getDecl());
+  llvm::Value *V = CGF.GetAddressCXXOfBaseClass(ThisPtr, ClassDecl,
+                                                BaseClassDecl,
+                                                /*NullCheckValue=*/false);
+  CGF.EmitCXXConstructorCall(BaseInit->getConstructor(),
+                             CtorType, V,
+                             BaseInit->const_arg_begin(),
+                             BaseInit->const_arg_end());
+}
+
+static void EmitMemberInitializer(CodeGenFunction &CGF,
+                                  const CXXRecordDecl *ClassDecl,
+                                  CXXBaseOrMemberInitializer *MemberInit) {
+  assert(MemberInit->isMemberInitializer() &&
+         "Must have member initializer!");
+  
+  // non-static data member initializers.
+  FieldDecl *Field = MemberInit->getMember();
+  QualType FieldType = CGF.getContext().getCanonicalType(Field->getType());
+
+  llvm::Value *ThisPtr = CGF.LoadCXXThis();
+  LValue LHS;
+  if (FieldType->isReferenceType()) {
+    // FIXME: This is really ugly; should be refactored somehow
+    unsigned idx = CGF.CGM.getTypes().getLLVMFieldNo(Field);
+    llvm::Value *V = CGF.Builder.CreateStructGEP(ThisPtr, idx, "tmp");
+    assert(!FieldType.getObjCGCAttr() && "fields cannot have GC attrs");
+    LHS = LValue::MakeAddr(V, CGF.MakeQualifiers(FieldType));
+  } else {
+    LHS = CGF.EmitLValueForField(ThisPtr, Field, ClassDecl->isUnion(), 0);
+  }
+
+  // If we are initializing an anonymous union field, drill down to the field.
+  if (MemberInit->getAnonUnionMember()) {
+    Field = MemberInit->getAnonUnionMember();
+    LHS = CGF.EmitLValueForField(LHS.getAddress(), Field,
+                                 /*IsUnion=*/true, 0);
+    FieldType = Field->getType();
+  }
+
+  // If the field is an array, branch based on the element type.
+  const ConstantArrayType *Array =
+    CGF.getContext().getAsConstantArrayType(FieldType);
+  if (Array)
+    FieldType = CGF.getContext().getBaseElementType(FieldType);
+
+  // We lose the constructor for anonymous union members, so handle them
+  // explicitly.
+  // FIXME: This is somwhat ugly.
+  if (MemberInit->getAnonUnionMember() && FieldType->getAs<RecordType>()) {
+    if (MemberInit->getNumArgs())
+      CGF.EmitAggExpr(*MemberInit->arg_begin(), LHS.getAddress(),
+                      LHS.isVolatileQualified());
+    else
+      CGF.EmitAggregateClear(LHS.getAddress(), Field->getType());
+    return;
+  }
+
+  if (FieldType->getAs<RecordType>()) {
+    assert(MemberInit->getConstructor() &&
+           "EmitCtorPrologue - no constructor to initialize member");
+    if (Array) {
+      const llvm::Type *BasePtr = CGF.ConvertType(FieldType);
+      BasePtr = llvm::PointerType::getUnqual(BasePtr);
+      llvm::Value *BaseAddrPtr =
+        CGF.Builder.CreateBitCast(LHS.getAddress(), BasePtr);
+      CGF.EmitCXXAggrConstructorCall(MemberInit->getConstructor(),
+                                     Array, BaseAddrPtr);
+    }
+    else
+      CGF.EmitCXXConstructorCall(MemberInit->getConstructor(),
+                                 Ctor_Complete, LHS.getAddress(),
+                                 MemberInit->const_arg_begin(),
+                                 MemberInit->const_arg_end());
+    return;
+  }
+
+  assert(MemberInit->getNumArgs() == 1 && "Initializer count must be 1 only");
+  Expr *RhsExpr = *MemberInit->arg_begin();
+  RValue RHS;
+  if (FieldType->isReferenceType()) {
+    RHS = CGF.EmitReferenceBindingToExpr(RhsExpr, FieldType,
+                                    /*IsInitializer=*/true);
+    CGF.EmitStoreThroughLValue(RHS, LHS, FieldType);
+  } else if (Array) {
+    CGF.EmitMemSetToZero(LHS.getAddress(), Field->getType());
+  } else if (!CGF.hasAggregateLLVMType(RhsExpr->getType())) {
+    RHS = RValue::get(CGF.EmitScalarExpr(RhsExpr, true));
+    CGF.EmitStoreThroughLValue(RHS, LHS, FieldType);
+  } else if (RhsExpr->getType()->isAnyComplexType()) {
+    CGF.EmitComplexExprIntoAddr(RhsExpr, LHS.getAddress(),
+                                LHS.isVolatileQualified());
+  } else {
+    // Handle member function pointers; other aggregates shouldn't get this far.
+    CGF.EmitAggExpr(RhsExpr, LHS.getAddress(), LHS.isVolatileQualified());
+  }
+}
+
 /// EmitCtorPrologue - This routine generates necessary code to initialize
 /// base classes and non-static data members belonging to this constructor.
 /// FIXME: This needs to take a CXXCtorType.
 void CodeGenFunction::EmitCtorPrologue(const CXXConstructorDecl *CD,
                                        CXXCtorType CtorType) {
-  const CXXRecordDecl *ClassDecl = cast<CXXRecordDecl>(CD->getDeclContext());
+  const CXXRecordDecl *ClassDecl = CD->getParent();
+  
   // FIXME: Add vbase initialization
   llvm::Value *LoadOfThis = 0;
 
@@ -1379,136 +1609,18 @@ void CodeGenFunction::EmitCtorPrologue(const CXXConstructorDecl *CD,
        E = CD->init_end();
        B != E; ++B) {
     CXXBaseOrMemberInitializer *Member = (*B);
-    if (Member->isBaseInitializer()) {
-      LoadOfThis = LoadCXXThis();
-      Type *BaseType = Member->getBaseClass();
-      CXXRecordDecl *BaseClassDecl =
-        cast<CXXRecordDecl>(BaseType->getAs<RecordType>()->getDecl());
-      llvm::Value *V = GetAddressCXXOfBaseClass(LoadOfThis, ClassDecl,
-                                                BaseClassDecl,
-                                                /*NullCheckValue=*/false);
-      EmitCXXConstructorCall(Member->getConstructor(),
-                             CtorType, V,
-                             Member->const_arg_begin(),
-                             Member->const_arg_end());
-    } else {
-      // non-static data member initilaizers.
-      FieldDecl *Field = Member->getMember();
-      QualType FieldType = getContext().getCanonicalType((Field)->getType());
-      const ConstantArrayType *Array =
-        getContext().getAsConstantArrayType(FieldType);
-      if (Array)
-        FieldType = getContext().getBaseElementType(FieldType);
+    
+    assert(LiveTemporaries.empty() &&
+           "Should not have any live temporaries at initializer start!");
 
-      LoadOfThis = LoadCXXThis();
-      LValue LHS;
-      if (FieldType->isReferenceType()) {
-        // FIXME: This is really ugly; should be refactored somehow
-        unsigned idx = CGM.getTypes().getLLVMFieldNo(Field);
-        llvm::Value *V = Builder.CreateStructGEP(LoadOfThis, idx, "tmp");
-        assert(!FieldType.getObjCGCAttr() && "fields cannot have GC attrs");
-        LHS = LValue::MakeAddr(V, MakeQualifiers(FieldType));
-      } else {
-        LHS = EmitLValueForField(LoadOfThis, Field, false, 0);
-      }
-      if (FieldType->getAs<RecordType>()) {
-        if (!Field->isAnonymousStructOrUnion()) {
-          assert(Member->getConstructor() &&
-                 "EmitCtorPrologue - no constructor to initialize member");
-          if (Array) {
-            const llvm::Type *BasePtr = ConvertType(FieldType);
-            BasePtr = llvm::PointerType::getUnqual(BasePtr);
-            llvm::Value *BaseAddrPtr =
-            Builder.CreateBitCast(LHS.getAddress(), BasePtr);
-            EmitCXXAggrConstructorCall(Member->getConstructor(),
-                                       Array, BaseAddrPtr);
-          }
-          else
-            EmitCXXConstructorCall(Member->getConstructor(),
-                                   Ctor_Complete, LHS.getAddress(),
-                                   Member->const_arg_begin(),
-                                   Member->const_arg_end());
-          continue;
-        }
-        else {
-          // Initializing an anonymous union data member.
-          FieldDecl *anonMember = Member->getAnonUnionMember();
-          LHS = EmitLValueForField(LHS.getAddress(), anonMember,
-                                   /*IsUnion=*/true, 0);
-          FieldType = anonMember->getType();
-        }
-      }
+    if (Member->isBaseInitializer())
+      EmitBaseInitializer(*this, ClassDecl, Member, CtorType);
+    else
+      EmitMemberInitializer(*this, ClassDecl, Member);
 
-      assert(Member->getNumArgs() == 1 && "Initializer count must be 1 only");
-      Expr *RhsExpr = *Member->arg_begin();
-      RValue RHS;
-      if (FieldType->isReferenceType())
-        RHS = EmitReferenceBindingToExpr(RhsExpr, FieldType,
-                                        /*IsInitializer=*/true);
-      else if (FieldType->isMemberFunctionPointerType())
-        RHS = RValue::get(CGM.EmitConstantExpr(RhsExpr, FieldType, this));
-      else
-        RHS = RValue::get(EmitScalarExpr(RhsExpr, true));
-      EmitStoreThroughLValue(RHS, LHS, FieldType);
-    }
-  }
-
-  if (!CD->getNumBaseOrMemberInitializers() && !CD->isTrivial()) {
-    // Nontrivial default constructor with no initializer list. It may still
-    // have bases classes and/or contain non-static data members which require
-    // construction.
-    for (CXXRecordDecl::base_class_const_iterator Base =
-          ClassDecl->bases_begin();
-          Base != ClassDecl->bases_end(); ++Base) {
-      // FIXME. copy assignment of virtual base NYI
-      if (Base->isVirtual())
-        continue;
-
-      CXXRecordDecl *BaseClassDecl
-        = cast<CXXRecordDecl>(Base->getType()->getAs<RecordType>()->getDecl());
-      if (BaseClassDecl->hasTrivialConstructor())
-        continue;
-      if (CXXConstructorDecl *BaseCX =
-            BaseClassDecl->getDefaultConstructor(getContext())) {
-        LoadOfThis = LoadCXXThis();
-        llvm::Value *V = GetAddressCXXOfBaseClass(LoadOfThis, ClassDecl,
-                                                  BaseClassDecl,
-                                                  /*NullCheckValue=*/false);
-        EmitCXXConstructorCall(BaseCX, Ctor_Complete, V, 0, 0);
-      }
-    }
-
-    for (CXXRecordDecl::field_iterator Field = ClassDecl->field_begin(),
-         FieldEnd = ClassDecl->field_end();
-         Field != FieldEnd; ++Field) {
-      QualType FieldType = getContext().getCanonicalType((*Field)->getType());
-      const ConstantArrayType *Array =
-        getContext().getAsConstantArrayType(FieldType);
-      if (Array)
-        FieldType = getContext().getBaseElementType(FieldType);
-      if (!FieldType->getAs<RecordType>() || Field->isAnonymousStructOrUnion())
-        continue;
-      const RecordType *ClassRec = FieldType->getAs<RecordType>();
-      CXXRecordDecl *MemberClassDecl =
-        dyn_cast<CXXRecordDecl>(ClassRec->getDecl());
-      if (!MemberClassDecl || MemberClassDecl->hasTrivialConstructor())
-        continue;
-      if (CXXConstructorDecl *MamberCX =
-            MemberClassDecl->getDefaultConstructor(getContext())) {
-        LoadOfThis = LoadCXXThis();
-        LValue LHS = EmitLValueForField(LoadOfThis, *Field, false, 0);
-        if (Array) {
-          const llvm::Type *BasePtr = ConvertType(FieldType);
-          BasePtr = llvm::PointerType::getUnqual(BasePtr);
-          llvm::Value *BaseAddrPtr =
-            Builder.CreateBitCast(LHS.getAddress(), BasePtr);
-          EmitCXXAggrConstructorCall(MamberCX, Array, BaseAddrPtr);
-        }
-        else
-          EmitCXXConstructorCall(MamberCX, Ctor_Complete, LHS.getAddress(),
-                                 0, 0);
-      }
-    }
+    // Pop any live temporaries that the initializers might have pushed.
+    while (!LiveTemporaries.empty())
+      PopCXXTemporary();
   }
 
   // Initialize the vtable pointer
@@ -1520,7 +1632,7 @@ void CodeGenFunction::EmitCtorPrologue(const CXXConstructorDecl *CD,
     Ptr8Ty = llvm::PointerType::get(llvm::Type::getInt8Ty(VMContext), 0);
     PtrPtr8Ty = llvm::PointerType::get(Ptr8Ty, 0);
     VtableField = Builder.CreateBitCast(LoadOfThis, PtrPtr8Ty);
-    llvm::Value *vtable = GenerateVtable(ClassDecl);
+    llvm::Value *vtable = CGM.getVtableInfo().getVtable(ClassDecl);
     Builder.CreateStore(vtable, VtableField);
   }
 }
@@ -1531,138 +1643,117 @@ void CodeGenFunction::EmitCtorPrologue(const CXXConstructorDecl *CD,
 /// FIXME: This needs to take a CXXDtorType.
 void CodeGenFunction::EmitDtorEpilogue(const CXXDestructorDecl *DD,
                                        CXXDtorType DtorType) {
-  const CXXRecordDecl *ClassDecl = cast<CXXRecordDecl>(DD->getDeclContext());
-  assert(!ClassDecl->getNumVBases() &&
-         "FIXME: Destruction of virtual bases not supported");
-  (void)ClassDecl;  // prevent warning.
-
-  for (CXXDestructorDecl::destr_const_iterator *B = DD->destr_begin(),
-       *E = DD->destr_end(); B != E; ++B) {
-    uintptr_t BaseOrMember = (*B);
-    if (DD->isMemberToDestroy(BaseOrMember)) {
-      FieldDecl *FD = DD->getMemberToDestroy(BaseOrMember);
-      QualType FieldType = getContext().getCanonicalType((FD)->getType());
-      const ConstantArrayType *Array =
-        getContext().getAsConstantArrayType(FieldType);
-      if (Array)
-        FieldType = getContext().getBaseElementType(FieldType);
-      const RecordType *RT = FieldType->getAs<RecordType>();
-      CXXRecordDecl *FieldClassDecl = cast<CXXRecordDecl>(RT->getDecl());
-      if (FieldClassDecl->hasTrivialDestructor())
-        continue;
-      llvm::Value *LoadOfThis = LoadCXXThis();
-      LValue LHS = EmitLValueForField(LoadOfThis, FD, false, 0);
-      if (Array) {
-        const llvm::Type *BasePtr = ConvertType(FieldType);
-        BasePtr = llvm::PointerType::getUnqual(BasePtr);
-        llvm::Value *BaseAddrPtr =
-          Builder.CreateBitCast(LHS.getAddress(), BasePtr);
-        EmitCXXAggrDestructorCall(FieldClassDecl->getDestructor(getContext()),
-                                  Array, BaseAddrPtr);
-      }
-      else
-        EmitCXXDestructorCall(FieldClassDecl->getDestructor(getContext()),
-                              Dtor_Complete, LHS.getAddress());
-    } else {
-      const RecordType *RT =
-        DD->getAnyBaseClassToDestroy(BaseOrMember)->getAs<RecordType>();
-      CXXRecordDecl *BaseClassDecl = cast<CXXRecordDecl>(RT->getDecl());
-      if (BaseClassDecl->hasTrivialDestructor())
-        continue;
-      llvm::Value *V = GetAddressCXXOfBaseClass(LoadCXXThis(),
-                                                ClassDecl, BaseClassDecl,
-                                                /*NullCheckValue=*/false);
-      EmitCXXDestructorCall(BaseClassDecl->getDestructor(getContext()),
-                            DtorType, V);
-    }
-  }
-  if (DD->getNumBaseOrMemberDestructions() || DD->isTrivial())
-    return;
-  // Case of destructor synthesis with fields and base classes
-  // which have non-trivial destructors. They must be destructed in
-  // reverse order of their construction.
-  llvm::SmallVector<FieldDecl *, 16> DestructedFields;
+  assert(!DD->isTrivial() &&
+         "Should not emit dtor epilogue for trivial dtor!");
 
-  for (CXXRecordDecl::field_iterator Field = ClassDecl->field_begin(),
-       FieldEnd = ClassDecl->field_end();
-       Field != FieldEnd; ++Field) {
-    QualType FieldType = getContext().getCanonicalType((*Field)->getType());
-    if (getContext().getAsConstantArrayType(FieldType))
-      FieldType = getContext().getBaseElementType(FieldType);
-    if (const RecordType *RT = FieldType->getAs<RecordType>()) {
-      CXXRecordDecl *FieldClassDecl = cast<CXXRecordDecl>(RT->getDecl());
-      if (FieldClassDecl->hasTrivialDestructor())
+  const CXXRecordDecl *ClassDecl = DD->getParent();
+
+  // Collect the fields.
+  llvm::SmallVector<const FieldDecl *, 16> FieldDecls;
+  for (CXXRecordDecl::field_iterator I = ClassDecl->field_begin(),
+       E = ClassDecl->field_end(); I != E; ++I) {
+    const FieldDecl *Field = *I;
+    
+    QualType FieldType = getContext().getCanonicalType(Field->getType());
+    FieldType = getContext().getBaseElementType(FieldType);
+    
+    const RecordType *RT = FieldType->getAs<RecordType>();
+    if (!RT)
+      continue;
+    
+    CXXRecordDecl *FieldClassDecl = cast<CXXRecordDecl>(RT->getDecl());
+    if (FieldClassDecl->hasTrivialDestructor())
         continue;
-      DestructedFields.push_back(*Field);
-    }
+    
+    FieldDecls.push_back(Field);
   }
-  if (!DestructedFields.empty())
-    for (int i = DestructedFields.size() -1; i >= 0; --i) {
-      FieldDecl *Field = DestructedFields[i];
-      QualType FieldType = Field->getType();
-      const ConstantArrayType *Array =
-        getContext().getAsConstantArrayType(FieldType);
-        if (Array)
-          FieldType = getContext().getBaseElementType(FieldType);
-      const RecordType *RT = FieldType->getAs<RecordType>();
-      CXXRecordDecl *FieldClassDecl = cast<CXXRecordDecl>(RT->getDecl());
-      llvm::Value *LoadOfThis = LoadCXXThis();
-      LValue LHS = EmitLValueForField(LoadOfThis, Field, false, 0);
-      if (Array) {
-        const llvm::Type *BasePtr = ConvertType(FieldType);
-        BasePtr = llvm::PointerType::getUnqual(BasePtr);
-        llvm::Value *BaseAddrPtr =
+  
+  // Now destroy the fields.
+  for (size_t i = FieldDecls.size(); i > 0; --i) {
+    const FieldDecl *Field = FieldDecls[i - 1];
+    
+    QualType FieldType = Field->getType();
+    const ConstantArrayType *Array = 
+      getContext().getAsConstantArrayType(FieldType);
+    if (Array)
+      FieldType = getContext().getBaseElementType(FieldType);
+    
+    const RecordType *RT = FieldType->getAs<RecordType>();
+    CXXRecordDecl *FieldClassDecl = cast<CXXRecordDecl>(RT->getDecl());
+
+    llvm::Value *ThisPtr = LoadCXXThis();
+
+    LValue LHS = EmitLValueForField(ThisPtr, Field, 
+                                    /*isUnion=*/false,
+                                    // FIXME: Qualifiers?
+                                    /*CVRQualifiers=*/0);
+    if (Array) {
+      const llvm::Type *BasePtr = ConvertType(FieldType);
+      BasePtr = llvm::PointerType::getUnqual(BasePtr);
+      llvm::Value *BaseAddrPtr =
         Builder.CreateBitCast(LHS.getAddress(), BasePtr);
-        EmitCXXAggrDestructorCall(FieldClassDecl->getDestructor(getContext()),
-                                  Array, BaseAddrPtr);
-      }
-      else
-        EmitCXXDestructorCall(FieldClassDecl->getDestructor(getContext()),
-                              Dtor_Complete, LHS.getAddress());
-    }
+      EmitCXXAggrDestructorCall(FieldClassDecl->getDestructor(getContext()),
+                                Array, BaseAddrPtr);
+    } else
+      EmitCXXDestructorCall(FieldClassDecl->getDestructor(getContext()),
+                            Dtor_Complete, LHS.getAddress());
+  }
 
-  llvm::SmallVector<CXXRecordDecl*, 4> DestructedBases;
-  for (CXXRecordDecl::base_class_const_iterator Base = ClassDecl->bases_begin();
-       Base != ClassDecl->bases_end(); ++Base) {
-    // FIXME. copy assignment of virtual base NYI
-    if (Base->isVirtual())
+  // Destroy non-virtual bases.
+  for (CXXRecordDecl::reverse_base_class_const_iterator I = 
+        ClassDecl->bases_rbegin(), E = ClassDecl->bases_rend(); I != E; ++I) {
+    const CXXBaseSpecifier &Base = *I;
+    
+    // Ignore virtual bases.
+    if (Base.isVirtual())
       continue;
-
+    
     CXXRecordDecl *BaseClassDecl
-      = cast<CXXRecordDecl>(Base->getType()->getAs<RecordType>()->getDecl());
+      = cast<CXXRecordDecl>(Base.getType()->getAs<RecordType>()->getDecl());
+    
+    // Ignore trivial destructors.
     if (BaseClassDecl->hasTrivialDestructor())
       continue;
-    DestructedBases.push_back(BaseClassDecl);
-  }
-  if (DestructedBases.empty())
-    return;
-  for (int i = DestructedBases.size() -1; i >= 0; --i) {
-    CXXRecordDecl *BaseClassDecl = DestructedBases[i];
+
     llvm::Value *V = GetAddressCXXOfBaseClass(LoadCXXThis(),
-                                              ClassDecl,BaseClassDecl, 
+                                              ClassDecl, BaseClassDecl, 
                                               /*NullCheckValue=*/false);
     EmitCXXDestructorCall(BaseClassDecl->getDestructor(getContext()),
-                          Dtor_Complete, V);
+                          Dtor_Base, V);
   }
+
+  // If we're emitting a base destructor, we don't want to emit calls to the
+  // virtual bases.
+  if (DtorType == Dtor_Base)
+    return;
+  
+  // FIXME: Handle virtual bases.
+  for (CXXRecordDecl::reverse_base_class_const_iterator I = 
+       ClassDecl->vbases_rbegin(), E = ClassDecl->vbases_rend(); I != E; ++I) {
+    assert(false && "FIXME: Handle virtual bases.");
+  }
+    
+  // If we have a deleting destructor, emit a call to the delete operator.
+  if (DtorType == Dtor_Deleting)
+    EmitDeleteCall(DD->getOperatorDelete(), LoadCXXThis(),
+                   getContext().getTagDeclType(ClassDecl));
 }
 
 void CodeGenFunction::SynthesizeDefaultDestructor(const CXXDestructorDecl *Dtor,
                                                   CXXDtorType DtorType,
                                                   llvm::Function *Fn,
                                                   const FunctionArgList &Args) {
-
-  const CXXRecordDecl *ClassDecl = Dtor->getParent();
-  assert(!ClassDecl->hasUserDeclaredDestructor() &&
+  assert(!Dtor->getParent()->hasUserDeclaredDestructor() &&
          "SynthesizeDefaultDestructor - destructor has user declaration");
-  (void) ClassDecl;
 
   StartFunction(GlobalDecl(Dtor, DtorType), Dtor->getResultType(), Fn, Args, 
                 SourceLocation());
+
   EmitDtorEpilogue(Dtor, DtorType);
   FinishFunction();
 }
 
-// FIXME: Move this to CGCXXStmt.cpp
+// FIXME: Move this to CGStmtCXX.cpp
 void CodeGenFunction::EmitCXXTryStmt(const CXXTryStmt &S) {
   // FIXME: We need to do more here.
   EmitStmt(S.getTryBlock());
diff --git a/lib/CodeGen/CGCXXClass.cpp b/lib/CodeGen/CGCXXClass.cpp
index 56a28fc9a007..533aabc8616e 100644
--- a/lib/CodeGen/CGCXXClass.cpp
+++ b/lib/CodeGen/CGCXXClass.cpp
@@ -31,6 +31,9 @@ ComputeNonVirtualBaseClassOffset(ASTContext &Context, CXXBasePaths &Paths,
     const ASTRecordLayout &Layout = Context.getASTRecordLayout(Element.Class);
     
     const CXXBaseSpecifier *BS = Element.Base;
+    // FIXME: enable test3 from virt.cc to not abort.
+    if (BS->isVirtual())
+      return 0;
     assert(!BS->isVirtual() && "Should not see virtual bases here!");
     
     const CXXRecordDecl *Base = 
diff --git a/lib/CodeGen/CGCXXExpr.cpp b/lib/CodeGen/CGCXXExpr.cpp
index 2d62df6c58a4..cd7d21b3b951 100644
--- a/lib/CodeGen/CGCXXExpr.cpp
+++ b/lib/CodeGen/CGCXXExpr.cpp
@@ -218,6 +218,7 @@ llvm::Value *CodeGenFunction::EmitCXXNewExpr(const CXXNewExpr *E) {
 
   if (NullCheckResult) {
     Builder.CreateBr(NewEnd);
+    NewNotNull = Builder.GetInsertBlock();
     EmitBlock(NewNull);
     Builder.CreateBr(NewEnd);
     EmitBlock(NewEnd);
@@ -233,12 +234,35 @@ llvm::Value *CodeGenFunction::EmitCXXNewExpr(const CXXNewExpr *E) {
   return NewPtr;
 }
 
-void CodeGenFunction::EmitCXXDeleteExpr(const CXXDeleteExpr *E) {
-  if (E->isArrayForm()) {
-    ErrorUnsupported(E, "delete[] expression");
-    return;
-  };
+void CodeGenFunction::EmitDeleteCall(const FunctionDecl *DeleteFD,
+                                     llvm::Value *Ptr,
+                                     QualType DeleteTy) {
+  const FunctionProtoType *DeleteFTy =
+    DeleteFD->getType()->getAs<FunctionProtoType>();
+
+  CallArgList DeleteArgs;
+
+  QualType ArgTy = DeleteFTy->getArgType(0);
+  llvm::Value *DeletePtr = Builder.CreateBitCast(Ptr, ConvertType(ArgTy));
+  DeleteArgs.push_back(std::make_pair(RValue::get(DeletePtr), ArgTy));
+
+  if (DeleteFTy->getNumArgs() == 2) {
+    QualType SizeTy = DeleteFTy->getArgType(1);
+    uint64_t SizeVal = getContext().getTypeSize(DeleteTy) / 8;
+    llvm::Constant *Size = llvm::ConstantInt::get(ConvertType(SizeTy),
+                                                  SizeVal);
+    DeleteArgs.push_back(std::make_pair(RValue::get(Size), SizeTy));
+  }
+
+  // Emit the call to delete.
+  EmitCall(CGM.getTypes().getFunctionInfo(DeleteFTy->getResultType(),
+                                          DeleteArgs),
+           CGM.GetAddrOfFunction(DeleteFD),
+           DeleteArgs, DeleteFD);
+}
 
+void CodeGenFunction::EmitCXXDeleteExpr(const CXXDeleteExpr *E) {
+  
   // Get at the argument before we performed the implicit conversion
   // to void*.
   const Expr *Arg = E->getArgument();
@@ -264,41 +288,237 @@ void CodeGenFunction::EmitCXXDeleteExpr(const CXXDeleteExpr *E) {
 
   Builder.CreateCondBr(IsNull, DeleteEnd, DeleteNotNull);
   EmitBlock(DeleteNotNull);
-
+  
+  bool ShouldCallDelete = true;
+  
   // Call the destructor if necessary.
   if (const RecordType *RT = DeleteTy->getAs<RecordType>()) {
     if (CXXRecordDecl *RD = dyn_cast<CXXRecordDecl>(RT->getDecl())) {
       if (!RD->hasTrivialDestructor()) {
         const CXXDestructorDecl *Dtor = RD->getDestructor(getContext());
-        if (Dtor->isVirtual()) {
+        if (E->isArrayForm()) {
+          QualType SizeTy = getContext().getSizeType();
+          uint64_t CookiePadding = std::max(getContext().getTypeSize(SizeTy),
+                static_cast<uint64_t>(getContext().getTypeAlign(DeleteTy))) / 8;
+          if (CookiePadding) {
+            llvm::Type *Ptr8Ty = 
+              llvm::PointerType::get(llvm::Type::getInt8Ty(VMContext), 0);
+            uint64_t CookieOffset =
+              CookiePadding - getContext().getTypeSize(SizeTy) / 8;
+            llvm::Value *AllocatedObjectPtr = 
+              Builder.CreateConstInBoundsGEP1_64(
+                            Builder.CreateBitCast(Ptr, Ptr8Ty), -CookiePadding);
+            llvm::Value *NumElementsPtr =
+              Builder.CreateConstInBoundsGEP1_64(AllocatedObjectPtr, 
+                                                 CookieOffset);
+            NumElementsPtr = Builder.CreateBitCast(NumElementsPtr,
+                                          ConvertType(SizeTy)->getPointerTo());
+            
+            llvm::Value *NumElements = 
+              Builder.CreateLoad(NumElementsPtr);
+            NumElements = 
+              Builder.CreateIntCast(NumElements, 
+                                    llvm::Type::getInt64Ty(VMContext), false, 
+                                    "count.tmp");
+            EmitCXXAggrDestructorCall(Dtor, NumElements, Ptr);
+            Ptr = AllocatedObjectPtr;
+          }
+        }
+        else if (Dtor->isVirtual()) {
           const llvm::Type *Ty =
             CGM.getTypes().GetFunctionType(CGM.getTypes().getFunctionInfo(Dtor),
                                            /*isVariadic=*/false);
           
-          llvm::Value *Callee = BuildVirtualCall(Dtor, Ptr, Ty);
+          llvm::Value *Callee = BuildVirtualCall(Dtor, Dtor_Deleting, Ptr, Ty);
           EmitCXXMemberCall(Dtor, Callee, Ptr, 0, 0);
+
+          // The dtor took care of deleting the object.
+          ShouldCallDelete = false;
         } else 
           EmitCXXDestructorCall(Dtor, Dtor_Complete, Ptr);
       }
     }
   }
 
-  // Call delete.
-  FunctionDecl *DeleteFD = E->getOperatorDelete();
-  const FunctionProtoType *DeleteFTy =
-    DeleteFD->getType()->getAs<FunctionProtoType>();
+  if (ShouldCallDelete)
+    EmitDeleteCall(E->getOperatorDelete(), Ptr, DeleteTy);
 
-  CallArgList DeleteArgs;
+  EmitBlock(DeleteEnd);
+}
 
-  QualType ArgTy = DeleteFTy->getArgType(0);
-  llvm::Value *DeletePtr = Builder.CreateBitCast(Ptr, ConvertType(ArgTy));
-  DeleteArgs.push_back(std::make_pair(RValue::get(DeletePtr), ArgTy));
+llvm::Value * CodeGenFunction::EmitCXXTypeidExpr(const CXXTypeidExpr *E) {
+  QualType Ty = E->getType();
+  const llvm::Type *LTy = ConvertType(Ty)->getPointerTo();
+  if (E->isTypeOperand()) {
+    Ty = E->getTypeOperand();
+    CanQualType CanTy = CGM.getContext().getCanonicalType(Ty);
+    Ty = CanTy.getUnqualifiedType().getNonReferenceType();
+    if (const RecordType *RT = Ty->getAs<RecordType>()) {
+      const CXXRecordDecl *RD = cast<CXXRecordDecl>(RT->getDecl());
+      if (RD->isPolymorphic())
+        return Builder.CreateBitCast(CGM.GenerateRttiRef(RD), LTy);
+      return Builder.CreateBitCast(CGM.GenerateRtti(RD), LTy);
+    }
+    return Builder.CreateBitCast(CGM.GenerateRttiNonClass(Ty), LTy);
+  }
+  Expr *subE = E->getExprOperand();
+  Ty = subE->getType();
+  CanQualType CanTy = CGM.getContext().getCanonicalType(Ty);
+  Ty = CanTy.getUnqualifiedType().getNonReferenceType();
+  if (const RecordType *RT = Ty->getAs<RecordType>()) {
+    const CXXRecordDecl *RD = cast<CXXRecordDecl>(RT->getDecl());
+    if (RD->isPolymorphic()) {
+      // FIXME: if subE is an lvalue do
+      LValue Obj = EmitLValue(subE);
+      llvm::Value *This = Obj.getAddress();
+      LTy = LTy->getPointerTo()->getPointerTo();
+      llvm::Value *V = Builder.CreateBitCast(This, LTy);
+      // We need to do a zero check for *p, unless it has NonNullAttr.
+      // FIXME: PointerType->hasAttr<NonNullAttr>()
+      bool CanBeZero = false;
+      if (UnaryOperator *UO = dyn_cast<UnaryOperator>(subE->IgnoreParens()))
+        if (UO->getOpcode() == UnaryOperator::Deref)
+          CanBeZero = true;
+      if (CanBeZero) {
+        llvm::BasicBlock *NonZeroBlock = createBasicBlock();
+        llvm::BasicBlock *ZeroBlock = createBasicBlock();
+        
+        llvm::Value *Zero = llvm::Constant::getNullValue(LTy);
+        Builder.CreateCondBr(Builder.CreateICmpNE(V, Zero),
+                             NonZeroBlock, ZeroBlock);
+        EmitBlock(ZeroBlock);
+        /// Call __cxa_bad_typeid
+        const llvm::Type *ResultType = llvm::Type::getVoidTy(VMContext);
+        const llvm::FunctionType *FTy;
+        FTy = llvm::FunctionType::get(ResultType, false);
+        llvm::Value *F = CGM.CreateRuntimeFunction(FTy, "__cxa_bad_typeid");
+        Builder.CreateCall(F)->setDoesNotReturn();
+        Builder.CreateUnreachable();
+        EmitBlock(NonZeroBlock);
+      }
+      V = Builder.CreateLoad(V, "vtable");
+      V = Builder.CreateConstInBoundsGEP1_64(V, -1ULL);
+      V = Builder.CreateLoad(V);
+      return V;
+    }      
+    return Builder.CreateBitCast(CGM.GenerateRtti(RD), LTy);
+  }
+  return Builder.CreateBitCast(CGM.GenerateRttiNonClass(Ty), LTy);
+}
 
-  // Emit the call to delete.
-  EmitCall(CGM.getTypes().getFunctionInfo(DeleteFTy->getResultType(),
-                                          DeleteArgs),
-           CGM.GetAddrOfFunction(DeleteFD),
-           DeleteArgs, DeleteFD);
+llvm::Value *CodeGenFunction::EmitDynamicCast(llvm::Value *V,
+                                              const CXXDynamicCastExpr *DCE) {
+  QualType CastTy = DCE->getTypeAsWritten();
+  QualType InnerType = CastTy->getPointeeType();
+  QualType ArgTy = DCE->getSubExpr()->getType();
+  const llvm::Type *LArgTy = ConvertType(ArgTy);
+  const llvm::Type *LTy = ConvertType(DCE->getType());
+
+  bool CanBeZero = false;
+  bool ToVoid = false;
+  bool ThrowOnBad = false;
+  if (CastTy->isPointerType()) {
+    // FIXME: if PointerType->hasAttr<NonNullAttr>(), we don't set this
+    CanBeZero = true;
+    if (InnerType->isVoidType())
+      ToVoid = true;
+  } else {
+    LTy = LTy->getPointerTo();
+    ThrowOnBad = true;
+  }
 
-  EmitBlock(DeleteEnd);
+  CXXRecordDecl *SrcTy;
+  QualType Ty = ArgTy;
+  if (ArgTy.getTypePtr()->isPointerType()
+      || ArgTy.getTypePtr()->isReferenceType())
+    Ty = Ty.getTypePtr()->getPointeeType();
+  CanQualType CanTy = CGM.getContext().getCanonicalType(Ty);
+  Ty = CanTy.getUnqualifiedType();
+  SrcTy = cast<CXXRecordDecl>(Ty->getAs<RecordType>()->getDecl());
+
+  llvm::BasicBlock *ContBlock = createBasicBlock();
+  llvm::BasicBlock *NullBlock = 0;
+  llvm::BasicBlock *NonZeroBlock = 0;
+  if (CanBeZero) {
+    NonZeroBlock = createBasicBlock();
+    NullBlock = createBasicBlock();
+    llvm::Value *Zero = llvm::Constant::getNullValue(LArgTy);
+    Builder.CreateCondBr(Builder.CreateICmpNE(V, Zero),
+                         NonZeroBlock, NullBlock);
+    EmitBlock(NonZeroBlock);
+  }
+
+  llvm::BasicBlock *BadCastBlock = 0;
+
+  const llvm::Type *PtrDiffTy = ConvertType(getContext().getSizeType());
+
+  // See if this is a dynamic_cast(void*)
+  if (ToVoid) {
+    llvm::Value *This = V;
+    V = Builder.CreateBitCast(This, PtrDiffTy->getPointerTo()->getPointerTo());
+    V = Builder.CreateLoad(V, "vtable");
+    V = Builder.CreateConstInBoundsGEP1_64(V, -2ULL);
+    V = Builder.CreateLoad(V, "offset to top");
+    This = Builder.CreateBitCast(This, llvm::Type::getInt8PtrTy(VMContext));
+    V = Builder.CreateInBoundsGEP(This, V);
+    V = Builder.CreateBitCast(V, LTy);
+  } else {
+    /// Call __dynamic_cast
+    const llvm::Type *ResultType = llvm::Type::getInt8PtrTy(VMContext);
+    const llvm::FunctionType *FTy;
+    std::vector<const llvm::Type*> ArgTys;
+    const llvm::Type *PtrToInt8Ty
+      = llvm::Type::getInt8Ty(VMContext)->getPointerTo();
+    ArgTys.push_back(PtrToInt8Ty);
+    ArgTys.push_back(PtrToInt8Ty);
+    ArgTys.push_back(PtrToInt8Ty);
+    ArgTys.push_back(PtrDiffTy);
+    FTy = llvm::FunctionType::get(ResultType, ArgTys, false);
+    CXXRecordDecl *DstTy;
+    Ty = CastTy.getTypePtr()->getPointeeType();
+    CanTy = CGM.getContext().getCanonicalType(Ty);
+    Ty = CanTy.getUnqualifiedType();
+    DstTy = cast<CXXRecordDecl>(Ty->getAs<RecordType>()->getDecl());
+
+    // FIXME: Calculate better hint.
+    llvm::Value *hint = llvm::ConstantInt::get(PtrDiffTy, -1ULL);
+    llvm::Value *SrcArg = CGM.GenerateRttiRef(SrcTy);
+    llvm::Value *DstArg = CGM.GenerateRttiRef(DstTy);
+    V = Builder.CreateBitCast(V, PtrToInt8Ty);
+    V = Builder.CreateCall4(CGM.CreateRuntimeFunction(FTy, "__dynamic_cast"),
+                            V, SrcArg, DstArg, hint);
+    V = Builder.CreateBitCast(V, LTy);
+
+    if (ThrowOnBad) {
+      BadCastBlock = createBasicBlock();
+
+      llvm::Value *Zero = llvm::Constant::getNullValue(LTy);
+      Builder.CreateCondBr(Builder.CreateICmpNE(V, Zero),
+                           ContBlock, BadCastBlock);
+      EmitBlock(BadCastBlock);
+      /// Call __cxa_bad_cast
+      ResultType = llvm::Type::getVoidTy(VMContext);
+      const llvm::FunctionType *FBadTy;
+      FBadTy = llvm::FunctionType::get(ResultType, false);
+      llvm::Value *F = CGM.CreateRuntimeFunction(FBadTy, "__cxa_bad_cast");
+      Builder.CreateCall(F)->setDoesNotReturn();
+      Builder.CreateUnreachable();
+    }
+  }
+  
+  if (CanBeZero) {
+    Builder.CreateBr(ContBlock);
+    EmitBlock(NullBlock);
+    Builder.CreateBr(ContBlock);
+  }
+  EmitBlock(ContBlock);
+  if (CanBeZero) {
+    llvm::PHINode *PHI = Builder.CreatePHI(LTy);
+    PHI->reserveOperandSpace(2);
+    PHI->addIncoming(V, NonZeroBlock);
+    PHI->addIncoming(llvm::Constant::getNullValue(LTy), NullBlock);
+    V = PHI;
+  }
+
+  return V;
 }
diff --git a/lib/CodeGen/CGCall.cpp b/lib/CodeGen/CGCall.cpp
index 06cd05cc75a8..d0c7d03f20ef 100644
--- a/lib/CodeGen/CGCall.cpp
+++ b/lib/CodeGen/CGCall.cpp
@@ -19,7 +19,7 @@
 #include "clang/AST/Decl.h"
 #include "clang/AST/DeclCXX.h"
 #include "clang/AST/DeclObjC.h"
-#include "clang/Frontend/CompileOptions.h"
+#include "clang/CodeGen/CodeGenOptions.h"
 #include "llvm/Attributes.h"
 #include "llvm/Support/CallSite.h"
 #include "llvm/Target/TargetData.h"
@@ -441,18 +441,13 @@ void CodeGenModule::ConstructAttributeList(const CGFunctionInfo &FI,
       RetAttrs |= llvm::Attribute::NoAlias;
   }
 
-  if (CompileOpts.OptimizeSize)
+  if (CodeGenOpts.OptimizeSize)
     FuncAttrs |= llvm::Attribute::OptimizeForSize;
-  if (CompileOpts.DisableRedZone)
+  if (CodeGenOpts.DisableRedZone)
     FuncAttrs |= llvm::Attribute::NoRedZone;
-  if (CompileOpts.NoImplicitFloat)
+  if (CodeGenOpts.NoImplicitFloat)
     FuncAttrs |= llvm::Attribute::NoImplicitFloat;
 
-  if (Features.getStackProtectorMode() == LangOptions::SSPOn)
-    FuncAttrs |= llvm::Attribute::StackProtect;
-  else if (Features.getStackProtectorMode() == LangOptions::SSPReq)
-    FuncAttrs |= llvm::Attribute::StackProtectReq;
-
   QualType RetTy = FI.getReturnType();
   unsigned Index = 1;
   const ABIArgInfo &RetAI = FI.getReturnInfo();
diff --git a/lib/CodeGen/CGDebugInfo.cpp b/lib/CodeGen/CGDebugInfo.cpp
index 1b01e1537b42..055166721085 100644
--- a/lib/CodeGen/CGDebugInfo.cpp
+++ b/lib/CodeGen/CGDebugInfo.cpp
@@ -21,7 +21,7 @@
 #include "clang/Basic/SourceManager.h"
 #include "clang/Basic/FileManager.h"
 #include "clang/Basic/Version.h"
-#include "clang/Frontend/CompileOptions.h"
+#include "clang/CodeGen/CodeGenOptions.h"
 #include "llvm/Constants.h"
 #include "llvm/DerivedTypes.h"
 #include "llvm/Instructions.h"
@@ -57,7 +57,7 @@ llvm::DIDescriptor CGDebugInfo::getContext(const VarDecl *Decl,
   if (Decl->getDeclContext()->isFunctionOrMethod()) {
     // Find the last subprogram in region stack.
     for (unsigned RI = RegionStack.size(), RE = 0; RI != RE; --RI) {
-      llvm::DIDescriptor R = RegionStack[RI - 1];
+      llvm::DIDescriptor R(RegionStack[RI - 1]);
       if (R.isSubprogram())
         return R;
     }
@@ -136,10 +136,11 @@ llvm::DICompileUnit CGDebugInfo::getOrCreateCompileUnit(SourceLocation Loc) {
     RuntimeVers = LO.ObjCNonFragileABI ? 2 : 1;
 
   // Create new compile unit.
-  return Unit = DebugFactory.CreateCompileUnit(LangTag, AbsFileName.getLast(),
-                                               AbsFileName.getDirname(),
-                                               Producer, isMain, isOptimized,
-                                               Flags, RuntimeVers);
+  return Unit = DebugFactory.CreateCompileUnit(LangTag, 
+                                               AbsFileName.getLast().c_str(),
+                                               AbsFileName.getDirname().c_str(),
+                                               Producer.c_str(), isMain, 
+                                               isOptimized, Flags, RuntimeVers);
 }
 
 /// CreateType - Get the Basic type from the cache or create a new
@@ -178,8 +179,6 @@ llvm::DIType CGDebugInfo::CreateType(const BuiltinType *BT,
                                  BT->getName(M->getContext().getLangOptions()),
                                  Unit, 0, Size, Align,
                                  Offset, /*flags*/ 0, Encoding);
-
-  TypeCache[QualType(BT, 0).getAsOpaquePtr()] = DbgTy.getNode();
   return DbgTy;
 }
 
@@ -198,7 +197,6 @@ llvm::DIType CGDebugInfo::CreateType(const ComplexType *Ty,
     DebugFactory.CreateBasicType(Unit, "complex",
                                  Unit, 0, Size, Align,
                                  Offset, /*flags*/ 0, Encoding);
-  TypeCache[QualType(Ty, 0).getAsOpaquePtr()] = DbgTy.getNode();
   return DbgTy;
 }
 
@@ -236,38 +234,41 @@ llvm::DIType CGDebugInfo::CreateQualifiedType(QualType Ty, llvm::DICompileUnit U
   llvm::DIType DbgTy =
     DebugFactory.CreateDerivedType(Tag, Unit, "", llvm::DICompileUnit(),
                                    0, 0, 0, 0, 0, FromTy);
-  TypeCache[Ty.getAsOpaquePtr()] = DbgTy.getNode();
   return DbgTy;
 }
 
 llvm::DIType CGDebugInfo::CreateType(const ObjCObjectPointerType *Ty,
                                      llvm::DICompileUnit Unit) {
-  llvm::DIType EltTy = getOrCreateType(Ty->getPointeeType(), Unit);
-
-  // Bit size, align and offset of the type.
-  uint64_t Size = M->getContext().getTypeSize(Ty);
-  uint64_t Align = M->getContext().getTypeAlign(Ty);
-
   llvm::DIType DbgTy =
-    DebugFactory.CreateDerivedType(llvm::dwarf::DW_TAG_pointer_type, Unit,
-                                   "", llvm::DICompileUnit(),
-                                   0, Size, Align, 0, 0, EltTy);
-  TypeCache[QualType(Ty, 0).getAsOpaquePtr()] = DbgTy.getNode();
+    CreatePointerLikeType(llvm::dwarf::DW_TAG_pointer_type, Ty, 
+                          Ty->getPointeeType(), Unit);
   return DbgTy;
 }
 
 llvm::DIType CGDebugInfo::CreateType(const PointerType *Ty,
                                      llvm::DICompileUnit Unit) {
-  llvm::DIType EltTy = getOrCreateType(Ty->getPointeeType(), Unit);
+  return CreatePointerLikeType(llvm::dwarf::DW_TAG_pointer_type, Ty, 
+                               Ty->getPointeeType(), Unit);
+}
+
+llvm::DIType CGDebugInfo::CreatePointerLikeType(unsigned Tag,
+                                                const Type *Ty, 
+                                                QualType PointeeTy,
+                                                llvm::DICompileUnit Unit) {
+  llvm::DIType EltTy = getOrCreateType(PointeeTy, Unit);
 
   // Bit size, align and offset of the type.
-  uint64_t Size = M->getContext().getTypeSize(Ty);
+  
+  // Size is always the size of a pointer. We can't use getTypeSize here
+  // because that does not return the correct value for references.
+  uint64_t Size = 
+    M->getContext().Target.getPointerWidth(PointeeTy.getAddressSpace());
   uint64_t Align = M->getContext().getTypeAlign(Ty);
 
   return
-    DebugFactory.CreateDerivedType(llvm::dwarf::DW_TAG_pointer_type, Unit,
-                                   "", llvm::DICompileUnit(),
+    DebugFactory.CreateDerivedType(Tag, Unit, "", llvm::DICompileUnit(),
                                    0, Size, Align, 0, 0, EltTy);
+  
 }
 
 llvm::DIType CGDebugInfo::CreateType(const BlockPointerType *Ty,
@@ -407,7 +408,6 @@ llvm::DIType CGDebugInfo::CreateType(const TypedefType *Ty,
 
   // We don't set size information, but do specify where the typedef was
   // declared.
-  std::string TyName = Ty->getDecl()->getNameAsString();
   SourceLocation DefLoc = Ty->getDecl()->getLocation();
   llvm::DICompileUnit DefUnit = getOrCreateCompileUnit(DefLoc);
 
@@ -417,8 +417,8 @@ llvm::DIType CGDebugInfo::CreateType(const TypedefType *Ty,
 
   llvm::DIType DbgTy = 
     DebugFactory.CreateDerivedType(llvm::dwarf::DW_TAG_typedef, Unit,
-                                   TyName, DefUnit, Line, 0, 0, 0, 0, Src);
-  TypeCache[QualType(Ty, 0).getAsOpaquePtr()] = DbgTy.getNode();
+                                   Ty->getDecl()->getNameAsCString(),
+                                   DefUnit, Line, 0, 0, 0, 0, Src);
   return DbgTy;
 }
 
@@ -446,7 +446,6 @@ llvm::DIType CGDebugInfo::CreateType(const FunctionType *Ty,
                                      Unit, "", llvm::DICompileUnit(),
                                      0, 0, 0, 0, 0,
                                      llvm::DIType(), EltTypeArray);
-  TypeCache[QualType(Ty, 0).getAsOpaquePtr()] = DbgTy.getNode();
   return DbgTy;
 }
 
@@ -468,8 +467,6 @@ llvm::DIType CGDebugInfo::CreateType(const RecordType *Ty,
   SourceManager &SM = M->getContext().getSourceManager();
 
   // Get overall information about the record type for the debug info.
-  std::string Name = Decl->getNameAsString();
-
   PresumedLoc PLoc = SM.getPresumedLoc(Decl->getLocation());
   llvm::DICompileUnit DefUnit;
   unsigned Line = 0;
@@ -485,13 +482,15 @@ llvm::DIType CGDebugInfo::CreateType(const RecordType *Ty,
   // may refer to the forward decl if the struct is recursive) and replace all
   // uses of the forward declaration with the final definition.
   llvm::DICompositeType FwdDecl =
-    DebugFactory.CreateCompositeType(Tag, Unit, Name, DefUnit, Line, 0, 0, 0, 0,
+    DebugFactory.CreateCompositeType(Tag, Unit, Decl->getNameAsString().data(), 
+                                     DefUnit, Line, 0, 0, 0, 0,
                                      llvm::DIType(), llvm::DIArray());
 
   // If this is just a forward declaration, return it.
   if (!Decl->getDefinition(M->getContext()))
     return FwdDecl;
 
+  llvm::TrackingVH<llvm::MDNode> FwdDeclNode = FwdDecl.getNode();
   // Otherwise, insert it into the TypeCache so that recursive uses will find
   // it.
   TypeCache[QualType(Ty, 0).getAsOpaquePtr()] = FwdDecl.getNode();
@@ -508,10 +507,10 @@ llvm::DIType CGDebugInfo::CreateType(const RecordType *Ty,
     FieldDecl *Field = *I;
     llvm::DIType FieldTy = getOrCreateType(Field->getType(), Unit);
 
-    std::string FieldName = Field->getNameAsString();
+    const char *FieldName = Field->getNameAsCString();
 
     // Ignore unnamed fields.
-    if (FieldName.empty())
+    if (!FieldName)
       continue;
 
     // Get the location for the field.
@@ -559,15 +558,13 @@ llvm::DIType CGDebugInfo::CreateType(const RecordType *Ty,
   uint64_t Align = M->getContext().getTypeAlign(Ty);
 
   llvm::DICompositeType RealDecl =
-    DebugFactory.CreateCompositeType(Tag, Unit, Name, DefUnit, Line, Size,
-                                     Align, 0, 0, llvm::DIType(), Elements);
-
-  // Update TypeCache.
-  TypeCache[QualType(Ty, 0).getAsOpaquePtr()] = RealDecl.getNode();
+    DebugFactory.CreateCompositeType(Tag, Unit, Decl->getNameAsString().data(),
+                                     DefUnit, Line, Size, Align, 0, 0, 
+                                     llvm::DIType(), Elements);
 
   // Now that we have a real decl for the struct, replace anything using the
   // old decl with the new one.  This will recursively update the debug info.
-  FwdDecl.replaceAllUsesWith(RealDecl);
+  llvm::DIDerivedType(FwdDeclNode).replaceAllUsesWith(RealDecl);
 
   return RealDecl;
 }
@@ -581,8 +578,6 @@ llvm::DIType CGDebugInfo::CreateType(const ObjCInterfaceType *Ty,
   SourceManager &SM = M->getContext().getSourceManager();
 
   // Get overall information about the record type for the debug info.
-  std::string Name = Decl->getNameAsString();
-
   llvm::DICompileUnit DefUnit = getOrCreateCompileUnit(Decl->getLocation());
   PresumedLoc PLoc = SM.getPresumedLoc(Decl->getLocation());
   unsigned Line = PLoc.isInvalid() ? 0 : PLoc.getLine();
@@ -597,7 +592,8 @@ llvm::DIType CGDebugInfo::CreateType(const ObjCInterfaceType *Ty,
   // may refer to the forward decl if the struct is recursive) and replace all
   // uses of the forward declaration with the final definition.
   llvm::DICompositeType FwdDecl =
-    DebugFactory.CreateCompositeType(Tag, Unit, Name, DefUnit, Line, 0, 0, 0, 0,
+    DebugFactory.CreateCompositeType(Tag, Unit, Decl->getNameAsCString(), 
+                                     DefUnit, Line, 0, 0, 0, 0,
                                      llvm::DIType(), llvm::DIArray(),
                                      RuntimeLang);
 
@@ -605,6 +601,7 @@ llvm::DIType CGDebugInfo::CreateType(const ObjCInterfaceType *Ty,
   if (Decl->isForwardDecl())
     return FwdDecl;
 
+  llvm::TrackingVH<llvm::MDNode> FwdDeclNode = FwdDecl.getNode();
   // Otherwise, insert it into the TypeCache so that recursive uses will find
   // it.
   TypeCache[QualType(Ty, 0).getAsOpaquePtr()] = FwdDecl.getNode();
@@ -631,10 +628,10 @@ llvm::DIType CGDebugInfo::CreateType(const ObjCInterfaceType *Ty,
     ObjCIvarDecl *Field = *I;
     llvm::DIType FieldTy = getOrCreateType(Field->getType(), Unit);
 
-    std::string FieldName = Field->getNameAsString();
+    const char *FieldName = Field->getNameAsCString();
 
     // Ignore unnamed fields.
-    if (FieldName.empty())
+    if (!FieldName)
       continue;
 
     // Get the location for the field.
@@ -685,16 +682,13 @@ llvm::DIType CGDebugInfo::CreateType(const ObjCInterfaceType *Ty,
   uint64_t Align = M->getContext().getTypeAlign(Ty);
 
   llvm::DICompositeType RealDecl =
-    DebugFactory.CreateCompositeType(Tag, Unit, Name, DefUnit, Line, Size,
-                                     Align, 0, 0, llvm::DIType(), Elements,
-                                     RuntimeLang);
-
-  // Update TypeCache.
-  TypeCache[QualType(Ty, 0).getAsOpaquePtr()] = RealDecl.getNode();
+    DebugFactory.CreateCompositeType(Tag, Unit, Decl->getNameAsCString(), DefUnit,
+                                     Line, Size, Align, 0, 0, llvm::DIType(), 
+                                     Elements, RuntimeLang);
 
   // Now that we have a real decl for the struct, replace anything using the
   // old decl with the new one.  This will recursively update the debug info.
-  FwdDecl.replaceAllUsesWith(RealDecl);
+  llvm::DIDerivedType(FwdDeclNode).replaceAllUsesWith(RealDecl);
 
   return RealDecl;
 }
@@ -709,7 +703,7 @@ llvm::DIType CGDebugInfo::CreateType(const EnumType *Ty,
   for (EnumDecl::enumerator_iterator
          Enum = Decl->enumerator_begin(), EnumEnd = Decl->enumerator_end();
        Enum != EnumEnd; ++Enum) {
-    Enumerators.push_back(DebugFactory.CreateEnumerator(Enum->getNameAsString(),
+    Enumerators.push_back(DebugFactory.CreateEnumerator(Enum->getNameAsCString(),
                                             Enum->getInitVal().getZExtValue()));
   }
 
@@ -717,7 +711,6 @@ llvm::DIType CGDebugInfo::CreateType(const EnumType *Ty,
   llvm::DIArray EltArray =
     DebugFactory.GetOrCreateArray(Enumerators.data(), Enumerators.size());
 
-  std::string EnumName = Decl->getNameAsString();
   SourceLocation DefLoc = Decl->getLocation();
   llvm::DICompileUnit DefUnit = getOrCreateCompileUnit(DefLoc);
   SourceManager &SM = M->getContext().getSourceManager();
@@ -735,11 +728,9 @@ llvm::DIType CGDebugInfo::CreateType(const EnumType *Ty,
 
   llvm::DIType DbgTy = 
     DebugFactory.CreateCompositeType(llvm::dwarf::DW_TAG_enumeration_type,
-                                     Unit, EnumName, DefUnit, Line,
+                                     Unit, Decl->getNameAsCString(), DefUnit, Line,
                                      Size, Align, 0, 0,
                                      llvm::DIType(), EltArray);
-  
-  TypeCache[QualType(Ty, 0).getAsOpaquePtr()] = DbgTy.getNode();
   return DbgTy;
 }
 
@@ -797,11 +788,37 @@ llvm::DIType CGDebugInfo::CreateType(const ArrayType *Ty,
                                      0, Size, Align, 0, 0,
                                      getOrCreateType(EltTy, Unit),
                                      SubscriptArray);
-  
-  TypeCache[QualType(Ty, 0).getAsOpaquePtr()] = DbgTy.getNode();
   return DbgTy;
 }
 
+llvm::DIType CGDebugInfo::CreateType(const LValueReferenceType *Ty, 
+                                     llvm::DICompileUnit Unit) {
+  return CreatePointerLikeType(llvm::dwarf::DW_TAG_reference_type, 
+                               Ty, Ty->getPointeeType(), Unit);
+}
+
+static QualType CanonicalizeTypeForDebugInfo(QualType T) {
+  switch (T->getTypeClass()) {
+  default:
+    return T;
+  case Type::TemplateSpecialization:
+    return cast<TemplateSpecializationType>(T)->desugar();
+  case Type::TypeOfExpr: {
+    TypeOfExprType *Ty = cast<TypeOfExprType>(T);
+    return CanonicalizeTypeForDebugInfo(Ty->getUnderlyingExpr()->getType());
+  }
+  case Type::TypeOf:
+    return cast<TypeOfType>(T)->getUnderlyingType();
+  case Type::Decltype:
+    return cast<DecltypeType>(T)->getUnderlyingType();
+  case Type::QualifiedName:
+    return cast<QualifiedNameType>(T)->getNamedType();
+  case Type::SubstTemplateTypeParm:
+    return cast<SubstTemplateTypeParmType>(T)->getReplacementType();
+  case Type::Elaborated:
+    return cast<ElaboratedType>(T)->getUnderlyingType();
+  }
+}
 
 /// getOrCreateType - Get the type from the cache or create a new
 /// one if necessary.
@@ -810,6 +827,9 @@ llvm::DIType CGDebugInfo::getOrCreateType(QualType Ty,
   if (Ty.isNull())
     return llvm::DIType();
 
+  // Canonicalize the type.
+  Ty = CanonicalizeTypeForDebugInfo(Ty);
+  
   // Check for existing entry.
   std::map<void *, llvm::WeakVH>::iterator it =
     TypeCache.find(Ty.getAsOpaquePtr());
@@ -821,15 +841,17 @@ llvm::DIType CGDebugInfo::getOrCreateType(QualType Ty,
 
   // Otherwise create the type.
   llvm::DIType Res = CreateTypeNode(Ty, Unit);
+
+  // And update the type cache.
+  TypeCache[Ty.getAsOpaquePtr()] = Res.getNode();  
   return Res;
 }
 
-/// getOrCreateTypeNode - Get the type metadata node from the cache or create a
-/// new one if necessary.
+/// CreateTypeNode - Create a new debug type node.
 llvm::DIType CGDebugInfo::CreateTypeNode(QualType Ty,
                                          llvm::DICompileUnit Unit) {
   // Handle qualifiers, which recursively handles what they refer to.
-  if (Ty.hasQualifiers())
+  if (Ty.hasLocalQualifiers())
     return CreateQualifiedType(Ty, Unit);
 
   // Work out details of type.
@@ -841,16 +863,13 @@ llvm::DIType CGDebugInfo::CreateTypeNode(QualType Ty,
 #include "clang/AST/TypeNodes.def"
     assert(false && "Dependent types cannot show up in debug information");
 
-  default:
-  case Type::LValueReference:
-  case Type::RValueReference:
-  case Type::Vector:
+  // FIXME: Handle these.
   case Type::ExtVector:
+  case Type::Vector:
   case Type::FixedWidthInt:
-  case Type::MemberPointer:
-  case Type::TemplateSpecialization:
-  case Type::QualifiedName:
-    // Unsupported types
+    return llvm::DIType();
+  default:
+    assert(false && "Unhandled type class!");
     return llvm::DIType();
   case Type::ObjCObjectPointer:
     return CreateType(cast<ObjCObjectPointerType>(Ty), Unit);
@@ -868,21 +887,14 @@ llvm::DIType CGDebugInfo::CreateTypeNode(QualType Ty,
   case Type::FunctionProto:
   case Type::FunctionNoProto:
     return CreateType(cast<FunctionType>(Ty), Unit);
-  case Type::Elaborated:
-    return getOrCreateType(cast<ElaboratedType>(Ty)->getUnderlyingType(),
-                           Unit);
-
   case Type::ConstantArray:
   case Type::VariableArray:
   case Type::IncompleteArray:
     return CreateType(cast<ArrayType>(Ty), Unit);
-  case Type::TypeOfExpr:
-    return getOrCreateType(cast<TypeOfExprType>(Ty)->getUnderlyingExpr()
-                           ->getType(), Unit);
-  case Type::TypeOf:
-    return getOrCreateType(cast<TypeOfType>(Ty)->getUnderlyingType(), Unit);
-  case Type::Decltype:
-    return getOrCreateType(cast<DecltypeType>(Ty)->getUnderlyingType(), Unit);
+
+  case Type::LValueReference:
+    return CreateType(cast<LValueReferenceType>(Ty), Unit);
+
   }
 }
 
@@ -909,12 +921,8 @@ void CGDebugInfo::EmitFunctionStart(const char *Name, QualType FnType,
                                   getOrCreateType(FnType, Unit),
                                   Fn->hasInternalLinkage(), true/*definition*/);
 
-#ifndef ATTACH_DEBUG_INFO_TO_AN_INSN
-  DebugFactory.InsertSubprogramStart(SP, Builder.GetInsertBlock());
-#endif
-
   // Push function on region stack.
-  RegionStack.push_back(SP);
+  RegionStack.push_back(SP.getNode());
 }
 
 
@@ -936,31 +944,23 @@ void CGDebugInfo::EmitStopPoint(llvm::Function *Fn, CGBuilderTy &Builder) {
   llvm::DICompileUnit Unit = getOrCreateCompileUnit(CurLoc);
   PresumedLoc PLoc = SM.getPresumedLoc(CurLoc);
 
-#ifdef ATTACH_DEBUG_INFO_TO_AN_INSN
-  llvm::DIDescriptor DR = RegionStack.back();
+  llvm::DIDescriptor DR(RegionStack.back());
   llvm::DIScope DS = llvm::DIScope(DR.getNode());
   llvm::DILocation DO(NULL);
   llvm::DILocation DL = 
     DebugFactory.CreateLocation(PLoc.getLine(), PLoc.getColumn(),
                                 DS, DO);
   Builder.SetCurrentDebugLocation(DL.getNode());
-#else
-  DebugFactory.InsertStopPoint(Unit, PLoc.getLine(), PLoc.getColumn(),
-                               Builder.GetInsertBlock());
-#endif
 }
 
 /// EmitRegionStart- Constructs the debug code for entering a declarative
 /// region - "llvm.dbg.region.start.".
 void CGDebugInfo::EmitRegionStart(llvm::Function *Fn, CGBuilderTy &Builder) {
-  llvm::DIDescriptor D;
-  if (!RegionStack.empty())
-    D = RegionStack.back();
-  D = DebugFactory.CreateLexicalBlock(D);
-  RegionStack.push_back(D);
-#ifndef ATTACH_DEBUG_INFO_TO_AN_INSN
-  DebugFactory.InsertRegionStart(D, Builder.GetInsertBlock());
-#endif
+  llvm::DIDescriptor D =
+    DebugFactory.CreateLexicalBlock(RegionStack.empty() ? 
+                                    llvm::DIDescriptor() : 
+                                    llvm::DIDescriptor(RegionStack.back()));
+  RegionStack.push_back(D.getNode());
 }
 
 /// EmitRegionEnd - Constructs the debug code for exiting a declarative
@@ -971,9 +971,6 @@ void CGDebugInfo::EmitRegionEnd(llvm::Function *Fn, CGBuilderTy &Builder) {
   // Provide an region stop point.
   EmitStopPoint(Fn, Builder);
 
-#ifndef ATTACH_DEBUG_INFO_TO_AN_INSN
-  DebugFactory.InsertRegionEnd(RegionStack.back(), Builder.GetInsertBlock());
-#endif
   RegionStack.pop_back();
 }
 
@@ -985,8 +982,8 @@ void CGDebugInfo::EmitDeclare(const VarDecl *Decl, unsigned Tag,
   // Do not emit variable debug information while generating optimized code.
   // The llvm optimizer and code generator are not yet ready to support
   // optimized code debugging.
-  const CompileOptions &CO = M->getCompileOpts();
-  if (CO.OptimizationLevel)
+  const CodeGenOptions &CGO = M->getCodeGenOpts();
+  if (CGO.OptimizationLevel)
     return;
 
   llvm::DICompileUnit Unit = getOrCreateCompileUnit(Decl->getLocation());
@@ -1105,10 +1102,9 @@ void CGDebugInfo::EmitDeclare(const VarDecl *Decl, unsigned Tag,
     FieldTy = CGDebugInfo::getOrCreateType(FType, Unit);
     FieldSize = M->getContext().getTypeSize(FType);
     FieldAlign = Align*8;
-    std::string Name = Decl->getNameAsString();
     
     FieldTy = DebugFactory.CreateDerivedType(llvm::dwarf::DW_TAG_member, Unit,
-                                             Name, DefUnit,
+                                             Decl->getNameAsCString(), DefUnit,
                                              0, FieldSize, FieldAlign,
                                              FieldOffset, 0, FieldTy);
     EltTys.push_back(FieldTy);
@@ -1128,18 +1124,28 @@ void CGDebugInfo::EmitDeclare(const VarDecl *Decl, unsigned Tag,
   SourceManager &SM = M->getContext().getSourceManager();
   PresumedLoc PLoc = SM.getPresumedLoc(Decl->getLocation());
   unsigned Line = 0;
-  if (!PLoc.isInvalid())
+  unsigned Column = 0;
+  if (!PLoc.isInvalid()) {
     Line = PLoc.getLine();
-  else
+    Column = PLoc.getColumn();
+  } else {
     Unit = llvm::DICompileUnit();
-
+  }
 
   // Create the descriptor for the variable.
   llvm::DIVariable D =
-    DebugFactory.CreateVariable(Tag, RegionStack.back(),Decl->getNameAsString(),
+    DebugFactory.CreateVariable(Tag, llvm::DIDescriptor(RegionStack.back()),
+                                Decl->getNameAsCString(),
                                 Unit, Line, Ty);
   // Insert an llvm.dbg.declare into the current block.
-  DebugFactory.InsertDeclare(Storage, D, Builder.GetInsertBlock());
+  llvm::Instruction *Call =
+    DebugFactory.InsertDeclare(Storage, D, Builder.GetInsertBlock());
+
+  llvm::DIScope DS(RegionStack.back());
+  llvm::DILocation DO(NULL);
+  llvm::DILocation DL = 
+    DebugFactory.CreateLocation(Line, Column, DS, DO);
+  Builder.SetDebugLocation(Call, DL.getNode());
 }
 
 /// EmitDeclare - Emit local variable declaration debug info.
@@ -1152,8 +1158,8 @@ void CGDebugInfo::EmitDeclare(const BlockDeclRefExpr *BDRE, unsigned Tag,
   // Do not emit variable debug information while generating optimized code.
   // The llvm optimizer and code generator are not yet ready to support
   // optimized code debugging.
-  const CompileOptions &CO = M->getCompileOpts();
-  if (CO.OptimizationLevel || Builder.GetInsertBlock() == 0)
+  const CodeGenOptions &CGO = M->getCodeGenOpts();
+  if (CGO.OptimizationLevel || Builder.GetInsertBlock() == 0)
     return;
 
   uint64_t XOffset = 0;
@@ -1273,11 +1279,10 @@ void CGDebugInfo::EmitDeclare(const BlockDeclRefExpr *BDRE, unsigned Tag,
     FieldTy = CGDebugInfo::getOrCreateType(FType, Unit);
     FieldSize = M->getContext().getTypeSize(FType);
     FieldAlign = Align*8;
-    std::string Name = Decl->getNameAsString();
     
     XOffset = FieldOffset;
     FieldTy = DebugFactory.CreateDerivedType(llvm::dwarf::DW_TAG_member, Unit,
-                                             Name, DefUnit,
+                                             Decl->getNameAsCString(), DefUnit,
                                              0, FieldSize, FieldAlign,
                                              FieldOffset, 0, FieldTy);
     EltTys.push_back(FieldTy);
@@ -1330,11 +1335,18 @@ void CGDebugInfo::EmitDeclare(const BlockDeclRefExpr *BDRE, unsigned Tag,
 
   // Create the descriptor for the variable.
   llvm::DIVariable D =
-    DebugFactory.CreateComplexVariable(Tag, RegionStack.back(),
-                                       Decl->getNameAsString(), Unit, Line, Ty,
+    DebugFactory.CreateComplexVariable(Tag, llvm::DIDescriptor(RegionStack.back()),
+                                       Decl->getNameAsCString(), Unit, Line, Ty,
                                        addr);
   // Insert an llvm.dbg.declare into the current block.
-  DebugFactory.InsertDeclare(Storage, D, Builder.GetInsertPoint());
+  llvm::Instruction *Call = 
+    DebugFactory.InsertDeclare(Storage, D, Builder.GetInsertPoint());
+
+  llvm::DIScope DS(RegionStack.back());
+  llvm::DILocation DO(NULL);
+  llvm::DILocation DL = 
+    DebugFactory.CreateLocation(Line, PLoc.getColumn(), DS, DO);
+  Builder.SetDebugLocation(Call, DL.getNode());
 }
 
 void CGDebugInfo::EmitDeclareOfAutoVariable(const VarDecl *Decl,
@@ -1362,21 +1374,12 @@ void CGDebugInfo::EmitDeclareOfArgVariable(const VarDecl *Decl, llvm::Value *AI,
 void CGDebugInfo::EmitGlobalVariable(llvm::GlobalVariable *Var,
                                      const VarDecl *Decl) {
 
-  // Do not emit variable debug information while generating optimized code.
-  // The llvm optimizer and code generator are not yet ready to support
-  // optimized code debugging.
-  const CompileOptions &CO = M->getCompileOpts();
-  if (CO.OptimizationLevel)
-    return;
-
   // Create global variable debug descriptor.
   llvm::DICompileUnit Unit = getOrCreateCompileUnit(Decl->getLocation());
   SourceManager &SM = M->getContext().getSourceManager();
   PresumedLoc PLoc = SM.getPresumedLoc(Decl->getLocation());
   unsigned LineNo = PLoc.isInvalid() ? 0 : PLoc.getLine();
 
-  std::string Name = Var->getName();
-
   QualType T = Decl->getType();
   if (T->isIncompleteArrayType()) {
 
@@ -1389,9 +1392,9 @@ void CGDebugInfo::EmitGlobalVariable(llvm::GlobalVariable *Var,
     T = M->getContext().getConstantArrayType(ET, ConstVal,
                                            ArrayType::Normal, 0);
   }
-
-  DebugFactory.CreateGlobalVariable(getContext(Decl, Unit), 
-                                    Name, Name, Name, Unit, LineNo,
+  const char *DeclName = Decl->getNameAsCString();
+  DebugFactory.CreateGlobalVariable(getContext(Decl, Unit), DeclName, DeclName,
+                                    NULL, Unit, LineNo,
                                     getOrCreateType(T, Unit),
                                     Var->hasInternalLinkage(),
                                     true/*definition*/, Var);
@@ -1406,7 +1409,7 @@ void CGDebugInfo::EmitGlobalVariable(llvm::GlobalVariable *Var,
   PresumedLoc PLoc = SM.getPresumedLoc(Decl->getLocation());
   unsigned LineNo = PLoc.isInvalid() ? 0 : PLoc.getLine();
 
-  std::string Name = Decl->getNameAsString();
+  const char *Name = Decl->getNameAsCString();
 
   QualType T = M->getContext().getObjCInterfaceType(Decl);
   if (T->isIncompleteArrayType()) {
diff --git a/lib/CodeGen/CGDebugInfo.h b/lib/CodeGen/CGDebugInfo.h
index 2e44e09d2590..af86e2b263fd 100644
--- a/lib/CodeGen/CGDebugInfo.h
+++ b/lib/CodeGen/CGDebugInfo.h
@@ -56,7 +56,7 @@ class CGDebugInfo {
   bool BlockLiteralGenericSet;
   llvm::DIType BlockLiteralGeneric;
 
-  std::vector<llvm::DIDescriptor> RegionStack;
+  std::vector<llvm::TrackingVH<llvm::MDNode> > RegionStack;
 
   /// Helper functions for getOrCreateType.
   llvm::DIType CreateType(const BuiltinType *Ty, llvm::DICompileUnit U);
@@ -73,7 +73,11 @@ class CGDebugInfo {
   llvm::DIType CreateType(const ObjCInterfaceType *Ty, llvm::DICompileUnit U);
   llvm::DIType CreateType(const EnumType *Ty, llvm::DICompileUnit U);
   llvm::DIType CreateType(const ArrayType *Ty, llvm::DICompileUnit U);
+  llvm::DIType CreateType(const LValueReferenceType *Ty, llvm::DICompileUnit U);
 
+  llvm::DIType CreatePointerLikeType(unsigned Tag,
+                                     const Type *Ty, QualType PointeeTy,
+                                     llvm::DICompileUnit U);
 public:
   CGDebugInfo(CodeGenModule *m);
   ~CGDebugInfo();
diff --git a/lib/CodeGen/CGDecl.cpp b/lib/CodeGen/CGDecl.cpp
index 2021ced31683..1d2040bae203 100644
--- a/lib/CodeGen/CGDecl.cpp
+++ b/lib/CodeGen/CGDecl.cpp
@@ -19,7 +19,7 @@
 #include "clang/AST/DeclObjC.h"
 #include "clang/Basic/SourceManager.h"
 #include "clang/Basic/TargetInfo.h"
-#include "clang/Frontend/CompileOptions.h"
+#include "clang/CodeGen/CodeGenOptions.h"
 #include "llvm/GlobalVariable.h"
 #include "llvm/Intrinsics.h"
 #include "llvm/Target/TargetData.h"
@@ -320,7 +320,7 @@ void CodeGenFunction::EmitLocalBlockVarDecl(const VarDecl &D) {
       
       // All constant structs and arrays should be global if
       // their initializer is constant and if the element type is POD.
-      if (CGM.getCompileOpts().MergeAllConstants) {
+      if (CGM.getCodeGenOpts().MergeAllConstants) {
         if (Ty.isConstant(getContext())
             && (Ty->isArrayType() || Ty->isRecordType())
             && (D.getInit() 
@@ -507,7 +507,7 @@ void CodeGenFunction::EmitLocalBlockVarDecl(const VarDecl &D) {
 
   // Handle CXX destruction of variables.
   QualType DtorTy(Ty);
-  if (const ArrayType *Array = DtorTy->getAs<ArrayType>())
+  while (const ArrayType *Array = getContext().getAsArrayType(DtorTy))
     DtorTy = getContext().getBaseElementType(Array);
   if (const RecordType *RT = DtorTy->getAs<RecordType>())
     if (CXXRecordDecl *ClassDecl = dyn_cast<CXXRecordDecl>(RT->getDecl())) {
diff --git a/lib/CodeGen/CGExpr.cpp b/lib/CodeGen/CGExpr.cpp
index d9dd70ac9e49..2a544c560931 100644
--- a/lib/CodeGen/CGExpr.cpp
+++ b/lib/CodeGen/CGExpr.cpp
@@ -267,6 +267,12 @@ LValue CodeGenFunction::EmitLValue(const Expr *E) {
     return EmitCXXBindTemporaryLValue(cast<CXXBindTemporaryExpr>(E));
   case Expr::CXXExprWithTemporariesClass:
     return EmitCXXExprWithTemporariesLValue(cast<CXXExprWithTemporaries>(E));
+  case Expr::CXXZeroInitValueExprClass:
+    return EmitNullInitializationLValue(cast<CXXZeroInitValueExpr>(E));
+  case Expr::CXXDefaultArgExprClass:
+    return EmitLValue(cast<CXXDefaultArgExpr>(E)->getExpr());
+  case Expr::CXXTypeidExprClass:
+    return EmitCXXTypeidLValue(cast<CXXTypeidExpr>(E));
 
   case Expr::ObjCMessageExprClass:
     return EmitObjCMessageExprLValue(cast<ObjCMessageExpr>(E));
@@ -303,8 +309,6 @@ LValue CodeGenFunction::EmitLValue(const Expr *E) {
   case Expr::CXXReinterpretCastExprClass:
   case Expr::CXXConstCastExprClass:
     return EmitCastLValue(cast<CastExpr>(E));
-  case Expr::CXXZeroInitValueExprClass:
-    return EmitNullInitializationLValue(cast<CXXZeroInitValueExpr>(E));
   }
 }
 
@@ -813,53 +817,53 @@ static void setObjCGCLValueClass(const ASTContext &Ctx, const Expr *E,
   }
 }
 
+static LValue EmitGlobalVarDeclLValue(CodeGenFunction &CGF,
+                                      const Expr *E, const VarDecl *VD) {
+  assert((VD->hasExternalStorage() || VD->isFileVarDecl()) &&
+         "Var decl must have external storage or be a file var decl!");
+
+  llvm::Value *V = CGF.CGM.GetAddrOfGlobalVar(VD);
+  if (VD->getType()->isReferenceType())
+    V = CGF.Builder.CreateLoad(V, "tmp");
+  LValue LV = LValue::MakeAddr(V, CGF.MakeQualifiers(E->getType()));
+  setObjCGCLValueClass(CGF.getContext(), E, LV);
+  return LV;
+}
+
 LValue CodeGenFunction::EmitDeclRefLValue(const DeclRefExpr *E) {
-  const VarDecl *VD = dyn_cast<VarDecl>(E->getDecl());
-
-  if (VD && (VD->isBlockVarDecl() || isa<ParmVarDecl>(VD) ||
-        isa<ImplicitParamDecl>(VD))) {
-    LValue LV;
-    bool NonGCable = VD->hasLocalStorage() &&
-      !VD->hasAttr<BlocksAttr>();
-    if (VD->hasExternalStorage()) {
-      llvm::Value *V = CGM.GetAddrOfGlobalVar(VD);
-      if (VD->getType()->isReferenceType())
-        V = Builder.CreateLoad(V, "tmp");
-      LV = LValue::MakeAddr(V, MakeQualifiers(E->getType()));
-    } else {
-      llvm::Value *V = LocalDeclMap[VD];
-      assert(V && "DeclRefExpr not entered in LocalDeclMap?");
-
-      Qualifiers Quals = MakeQualifiers(E->getType());
-      // local variables do not get their gc attribute set.
-      // local static?
-      if (NonGCable) Quals.removeObjCGCAttr();
-
-      if (VD->hasAttr<BlocksAttr>()) {
-        V = Builder.CreateStructGEP(V, 1, "forwarding");
-        V = Builder.CreateLoad(V, false);
-        V = Builder.CreateStructGEP(V, getByRefValueLLVMField(VD),
-                                    VD->getNameAsString());
-      }
-      if (VD->getType()->isReferenceType())
-        V = Builder.CreateLoad(V, "tmp");
-      LV = LValue::MakeAddr(V, Quals);
+  const NamedDecl *ND = E->getDecl();
+
+  if (const VarDecl *VD = dyn_cast<VarDecl>(ND)) {
+    
+    // Check if this is a global variable.
+    if (VD->hasExternalStorage() || VD->isFileVarDecl()) 
+      return EmitGlobalVarDeclLValue(*this, E, VD);
+
+    bool NonGCable = VD->hasLocalStorage() && !VD->hasAttr<BlocksAttr>();
+
+    llvm::Value *V = LocalDeclMap[VD];
+    assert(V && "DeclRefExpr not entered in LocalDeclMap?");
+
+    Qualifiers Quals = MakeQualifiers(E->getType());
+    // local variables do not get their gc attribute set.
+    // local static?
+    if (NonGCable) Quals.removeObjCGCAttr();
+
+    if (VD->hasAttr<BlocksAttr>()) {
+      V = Builder.CreateStructGEP(V, 1, "forwarding");
+      V = Builder.CreateLoad(V, false);
+      V = Builder.CreateStructGEP(V, getByRefValueLLVMField(VD),
+                                  VD->getNameAsString());
     }
-    LValue::SetObjCNonGC(LV, NonGCable);
-    setObjCGCLValueClass(getContext(), E, LV);
-    return LV;
-  }
-  
-  if (VD && VD->isFileVarDecl()) {
-    llvm::Value *V = CGM.GetAddrOfGlobalVar(VD);
     if (VD->getType()->isReferenceType())
       V = Builder.CreateLoad(V, "tmp");
-    LValue LV = LValue::MakeAddr(V, MakeQualifiers(E->getType()));
+    LValue LV = LValue::MakeAddr(V, Quals);
+    LValue::SetObjCNonGC(LV, NonGCable);
     setObjCGCLValueClass(getContext(), E, LV);
     return LV;
   }
   
-  if (const FunctionDecl *FD = dyn_cast<FunctionDecl>(E->getDecl())) {
+  if (const FunctionDecl *FD = dyn_cast<FunctionDecl>(ND)) {
     llvm::Value* V = CGM.GetAddrOfFunction(FD);
     if (!FD->hasPrototype()) {
       if (const FunctionProtoType *Proto =
@@ -876,20 +880,15 @@ LValue CodeGenFunction::EmitDeclRefLValue(const DeclRefExpr *E) {
     return LValue::MakeAddr(V, MakeQualifiers(E->getType()));
   }
   
-  if (const ImplicitParamDecl *IPD = dyn_cast<ImplicitParamDecl>(E->getDecl())){
-    llvm::Value *V = LocalDeclMap[IPD];
-    assert(V && "BlockVarDecl not entered in LocalDeclMap?");
-    return LValue::MakeAddr(V, MakeQualifiers(E->getType()));
-  }
-  
   if (E->getQualifier()) {
     // FIXME: the qualifier check does not seem sufficient here
-    return EmitPointerToDataMemberLValue(E);
+    return EmitPointerToDataMemberLValue(cast<FieldDecl>(ND));
   }
   
-  assert(0 && "Unimp declref");
-  //an invalid LValue, but the assert will
-  //ensure that this point is never reached.
+  assert(false && "Unhandled DeclRefExpr");
+  
+  // an invalid LValue, but the assert will
+  // ensure that this point is never reached.
   return LValue();
 }
 
@@ -924,13 +923,17 @@ LValue CodeGenFunction::EmitUnaryOpLValue(const UnaryOperator *E) {
     return LV;
   }
   case UnaryOperator::Real:
-  case UnaryOperator::Imag:
+  case UnaryOperator::Imag: {
     LValue LV = EmitLValue(E->getSubExpr());
     unsigned Idx = E->getOpcode() == UnaryOperator::Imag;
     return LValue::MakeAddr(Builder.CreateStructGEP(LV.getAddress(),
                                                     Idx, "idx"),
                             MakeQualifiers(ExprTy));
   }
+  case UnaryOperator::PreInc:
+  case UnaryOperator::PreDec:
+    return EmitUnsupportedLValue(E, "pre-inc/dec expression");
+  }
 }
 
 LValue CodeGenFunction::EmitStringLiteralLValue(const StringLiteral *E) {
@@ -1151,18 +1154,24 @@ LValue CodeGenFunction::EmitMemberExpr(const MemberExpr *E) {
     BaseQuals = BaseTy.getQualifiers();
   }
 
-  FieldDecl *Field = dyn_cast<FieldDecl>(E->getMemberDecl());
-  // FIXME: Handle non-field member expressions
-  assert(Field && "No code generation for non-field member references");
-  LValue MemExpLV = EmitLValueForField(BaseValue, Field, isUnion,
-                                       BaseQuals.getCVRQualifiers());
-  LValue::SetObjCNonGC(MemExpLV, isNonGC);
-  setObjCGCLValueClass(getContext(), E, MemExpLV);
-  return MemExpLV;
+  NamedDecl *ND = E->getMemberDecl();
+  if (FieldDecl *Field = dyn_cast<FieldDecl>(ND)) {
+    LValue LV = EmitLValueForField(BaseValue, Field, isUnion,
+                                   BaseQuals.getCVRQualifiers());
+    LValue::SetObjCNonGC(LV, isNonGC);
+    setObjCGCLValueClass(getContext(), E, LV);
+    return LV;
+  }
+  
+  if (VarDecl *VD = dyn_cast<VarDecl>(ND))
+    return EmitGlobalVarDeclLValue(*this, E, VD);
+  
+  assert(false && "Unhandled member declaration!");
+  return LValue();
 }
 
 LValue CodeGenFunction::EmitLValueForBitfield(llvm::Value* BaseValue,
-                                              FieldDecl* Field,
+                                              const FieldDecl* Field,
                                               unsigned CVRQualifiers) {
   CodeGenTypes::BitFieldInfo Info = CGM.getTypes().getBitFieldInfo(Field);
 
@@ -1187,7 +1196,7 @@ LValue CodeGenFunction::EmitLValueForBitfield(llvm::Value* BaseValue,
 }
 
 LValue CodeGenFunction::EmitLValueForField(llvm::Value* BaseValue,
-                                           FieldDecl* Field,
+                                           const FieldDecl* Field,
                                            bool isUnion,
                                            unsigned CVRQualifiers) {
   if (Field->isBitField())
@@ -1281,22 +1290,26 @@ CodeGenFunction::EmitConditionalOperatorLValue(const ConditionalOperator* E) {
   return LValue::MakeAddr(Temp, MakeQualifiers(E->getType()));
 }
 
-/// EmitCastLValue - Casts are never lvalues.  If a cast is needed by the code
-/// generator in an lvalue context, then it must mean that we need the address
-/// of an aggregate in order to access one of its fields.  This can happen for
-/// all the reasons that casts are permitted with aggregate result, including
-/// noop aggregate casts, and cast from scalar to union.
+/// EmitCastLValue - Casts are never lvalues unless that cast is a dynamic_cast.
+/// If the cast is a dynamic_cast, we can have the usual lvalue result,
+/// otherwise if a cast is needed by the code generator in an lvalue context,
+/// then it must mean that we need the address of an aggregate in order to
+/// access one of its fields.  This can happen for all the reasons that casts
+/// are permitted with aggregate result, including noop aggregate casts, and
+/// cast from scalar to union.
 LValue CodeGenFunction::EmitCastLValue(const CastExpr *E) {
   switch (E->getCastKind()) {
   default:
-    // If this is an lvalue cast, treat it as a no-op.
-    // FIXME: We shouldn't need to check for this explicitly!
-    if (const ImplicitCastExpr *ICE = dyn_cast<ImplicitCastExpr>(E))
-      if (ICE->isLvalueCast())
-        return EmitLValue(E->getSubExpr());
-    
-    assert(0 && "Unhandled cast!");
-      
+    return EmitUnsupportedLValue(E, "unexpected cast lvalue");
+
+  case CastExpr::CK_Dynamic: {
+    LValue LV = EmitLValue(E->getSubExpr());
+    llvm::Value *V = LV.getAddress();
+    const CXXDynamicCastExpr *DCE = cast<CXXDynamicCastExpr>(E);
+    return LValue::MakeAddr(EmitDynamicCast(V, DCE),
+                            MakeQualifiers(E->getType()));
+  }
+
   case CastExpr::CK_NoOp:
   case CastExpr::CK_ConstructorConversion:
   case CastExpr::CK_UserDefinedConversion:
@@ -1320,13 +1333,24 @@ LValue CodeGenFunction::EmitCastLValue(const CastExpr *E) {
     
     return LValue::MakeAddr(Base, MakeQualifiers(E->getType()));
   }
-
   case CastExpr::CK_ToUnion: {
     llvm::Value *Temp = CreateTempAlloca(ConvertType(E->getType()));
     EmitAnyExpr(E->getSubExpr(), Temp, false);
 
     return LValue::MakeAddr(Temp, MakeQualifiers(E->getType()));
-    }
+  }
+  case CastExpr::CK_BaseToDerived: {
+    return EmitUnsupportedLValue(E, "base-to-derived cast lvalue");
+  }
+  case CastExpr::CK_BitCast: {
+    // This must be a reinterpret_cast (or c-style equivalent).
+    const ExplicitCastExpr *CE = cast<ExplicitCastExpr>(E);
+    
+    LValue LV = EmitLValue(E->getSubExpr());
+    llvm::Value *V = Builder.CreateBitCast(LV.getAddress(),
+                                           ConvertType(CE->getTypeAsWritten()));
+    return LValue::MakeAddr(V, MakeQualifiers(E->getType()));
+  }
   }
 }
 
@@ -1449,6 +1473,12 @@ LValue CodeGenFunction::EmitCXXConstructLValue(const CXXConstructExpr *E) {
 }
 
 LValue
+CodeGenFunction::EmitCXXTypeidLValue(const CXXTypeidExpr *E) {
+  llvm::Value *Temp = EmitCXXTypeidExpr(E);
+  return LValue::MakeAddr(Temp, MakeQualifiers(E->getType()));
+}
+
+LValue
 CodeGenFunction::EmitCXXBindTemporaryLValue(const CXXBindTemporaryExpr *E) {
   LValue LV = EmitLValue(E->getSubExpr());
   PushCXXTemporary(E->getTemporary(), LV.getAddress());
@@ -1526,19 +1556,18 @@ LValue CodeGenFunction::EmitStmtExprLValue(const StmtExpr *E) {
 }
 
 
-LValue CodeGenFunction::EmitPointerToDataMemberLValue(const DeclRefExpr *E) {
-  const FieldDecl *Field = cast<FieldDecl>(E->getDecl());
+LValue CodeGenFunction::EmitPointerToDataMemberLValue(const FieldDecl *Field) {
   const CXXRecordDecl *ClassDecl = cast<CXXRecordDecl>(Field->getDeclContext());
   QualType NNSpecTy = 
     getContext().getCanonicalType(
       getContext().getTypeDeclType(const_cast<CXXRecordDecl*>(ClassDecl)));
   NNSpecTy = getContext().getPointerType(NNSpecTy);
   llvm::Value *V = llvm::Constant::getNullValue(ConvertType(NNSpecTy));
-  LValue MemExpLV = EmitLValueForField(V, const_cast<FieldDecl*>(Field), 
-                                       /*isUnion*/false, /*Qualifiers*/0);
+  LValue MemExpLV = EmitLValueForField(V, Field, /*isUnion=*/false, 
+                                       /*Qualifiers=*/0);
   const llvm::Type *ResultType = ConvertType(getContext().getPointerDiffType());
   V = Builder.CreatePtrToInt(MemExpLV.getAddress(), ResultType, "datamember");
-  return LValue::MakeAddr(V, MakeQualifiers(E->getType()));
+  return LValue::MakeAddr(V, MakeQualifiers(Field->getType()));
 }
 
 RValue CodeGenFunction::EmitCall(llvm::Value *Callee, QualType CalleeType,
@@ -1571,16 +1600,14 @@ RValue CodeGenFunction::EmitCall(llvm::Value *Callee, QualType CalleeType,
 
 LValue CodeGenFunction::
 EmitPointerToDataMemberBinaryExpr(const BinaryOperator *E) {
-  llvm::Value *BaseV = EmitLValue(E->getLHS()).getAddress();
+  llvm::Value *BaseV;
   if (E->getOpcode() == BinaryOperator::PtrMemI)
-    BaseV = Builder.CreateLoad(BaseV, "indir.ptr");
+    BaseV = EmitScalarExpr(E->getLHS());
+  else
+    BaseV = EmitLValue(E->getLHS()).getAddress();
   const llvm::Type *i8Ty = llvm::Type::getInt8PtrTy(getLLVMContext());
   BaseV = Builder.CreateBitCast(BaseV, i8Ty);
-  LValue RHSLV = EmitLValue(E->getRHS());
-  llvm::Value *OffsetV = 
-    EmitLoadOfLValue(RHSLV, E->getRHS()->getType()).getScalarVal();
-  const llvm::Type* ResultType = ConvertType(getContext().getPointerDiffType());
-  OffsetV = Builder.CreateBitCast(OffsetV, ResultType);
+  llvm::Value *OffsetV = EmitScalarExpr(E->getRHS());
   llvm::Value *AddV = Builder.CreateInBoundsGEP(BaseV, OffsetV, "add.ptr");
 
   QualType Ty = E->getRHS()->getType();
diff --git a/lib/CodeGen/CGExprAgg.cpp b/lib/CodeGen/CGExprAgg.cpp
index 901f867a5913..0e10368ab04e 100644
--- a/lib/CodeGen/CGExprAgg.cpp
+++ b/lib/CodeGen/CGExprAgg.cpp
@@ -114,6 +114,7 @@ public:
   void VisitCXXConstructExpr(const CXXConstructExpr *E);
   void VisitCXXExprWithTemporaries(CXXExprWithTemporaries *E);
   void VisitCXXZeroInitValueExpr(CXXZeroInitValueExpr *E);
+  void VisitCXXTypeidExpr(CXXTypeidExpr *E) { EmitAggLoadOfLValue(E); }
 
   void VisitVAArgExpr(VAArgExpr *E);
 
diff --git a/lib/CodeGen/CGExprConstant.cpp b/lib/CodeGen/CGExprConstant.cpp
index 9145d92128a1..40b845dba050 100644
--- a/lib/CodeGen/CGExprConstant.cpp
+++ b/lib/CodeGen/CGExprConstant.cpp
@@ -262,7 +262,7 @@ class VISIBILITY_HIDDEN ConstStructBuilder {
         uint64_t NumBytes =
           AlignedElementOffsetInBytes - ElementOffsetInBytes;
 
-        const llvm::Type *Ty = llvm::Type::getInt8Ty(CGF->getLLVMContext());
+        const llvm::Type *Ty = llvm::Type::getInt8Ty(CGM.getLLVMContext());
         if (NumBytes > 1)
           Ty = llvm::ArrayType::get(Ty, NumBytes);
 
@@ -766,27 +766,21 @@ public:
       return llvm::ConstantExpr::getBitCast(C, ConvertType(E->getType()));
     }
     case Expr::PredefinedExprClass: {
-      // __func__/__FUNCTION__ -> "".  __PRETTY_FUNCTION__ -> "top level".
-      std::string Str;
-      if (cast<PredefinedExpr>(E)->getIdentType() ==
-          PredefinedExpr::PrettyFunction)
-        Str = "top level";
+      unsigned Type = cast<PredefinedExpr>(E)->getIdentType();
+      if (CGF) {
+        LValue Res = CGF->EmitPredefinedFunctionName(Type);
+        return cast<llvm::Constant>(Res.getAddress());
+      } else if (Type == PredefinedExpr::PrettyFunction) {
+        return CGM.GetAddrOfConstantCString("top level", ".tmp");
+      }
 
-      return CGM.GetAddrOfConstantCString(Str, ".tmp");
+      return CGM.GetAddrOfConstantCString("", ".tmp");
     }
     case Expr::AddrLabelExprClass: {
       assert(CGF && "Invalid address of label expression outside function.");
-#ifndef USEINDIRECTBRANCH
-      unsigned id =
-          CGF->GetIDForAddrOfLabel(cast<AddrLabelExpr>(E)->getLabel());
-      llvm::Constant *C =
-            llvm::ConstantInt::get(llvm::Type::getInt32Ty(VMContext), id);
-      return llvm::ConstantExpr::getIntToPtr(C, ConvertType(E->getType()));
-#else
       llvm::Constant *Ptr =
         CGF->GetAddrOfLabel(cast<AddrLabelExpr>(E)->getLabel());
       return llvm::ConstantExpr::getBitCast(Ptr, ConvertType(E->getType()));
-#endif
     }
     case Expr::CallExprClass: {
       CallExpr* CE = cast<CallExpr>(E);
@@ -827,9 +821,7 @@ llvm::Constant *CodeGenModule::EmitConstantExpr(const Expr *E,
   else
     Success = E->Evaluate(Result, Context);
 
-  if (Success) {
-    assert(!Result.HasSideEffects &&
-           "Constant expr should not have any side effects!");
+  if (Success && !Result.HasSideEffects) {
     switch (Result.Val.getKind()) {
     case APValue::Uninitialized:
       assert(0 && "Constant expressions should be initialized.");
diff --git a/lib/CodeGen/CGExprScalar.cpp b/lib/CodeGen/CGExprScalar.cpp
index 96b58d8995ae..e9bbf35fcd8d 100644
--- a/lib/CodeGen/CGExprScalar.cpp
+++ b/lib/CodeGen/CGExprScalar.cpp
@@ -135,16 +135,8 @@ public:
   }
   Value *VisitSizeOfAlignOfExpr(const SizeOfAlignOfExpr *E);
   Value *VisitAddrLabelExpr(const AddrLabelExpr *E) {
-#ifndef USEINDIRECTBRANCH
-    llvm::Value *V =
-      llvm::ConstantInt::get(llvm::Type::getInt32Ty(CGF.getLLVMContext()),
-                             CGF.GetIDForAddrOfLabel(E->getLabel()));
-
-    return Builder.CreateIntToPtr(V, ConvertType(E->getType()));
-#else
     llvm::Value *V = CGF.GetAddrOfLabel(E->getLabel());
     return Builder.CreateBitCast(V, ConvertType(E->getType()));
-#endif
   }
 
   // l-values.
@@ -356,6 +348,9 @@ public:
   Value *VisitBinLOr        (const BinaryOperator *E);
   Value *VisitBinComma      (const BinaryOperator *E);
 
+  Value *VisitBinPtrMemD(const Expr *E) { return EmitLoadOfLValue(E); }
+  Value *VisitBinPtrMemI(const Expr *E) { return EmitLoadOfLValue(E); }
+
   // Other Operators.
   Value *VisitBlockExpr(const BlockExpr *BE);
   Value *VisitConditionalOperator(const ConditionalOperator *CO);
@@ -547,13 +542,6 @@ EmitComplexToScalarConversion(CodeGenFunction::ComplexPairTy Src,
 //===----------------------------------------------------------------------===//
 
 Value *ScalarExprEmitter::VisitExpr(Expr *E) {
-  if (const BinaryOperator *BExpr = dyn_cast<BinaryOperator>(E))
-    if (BExpr->getOpcode() == BinaryOperator::PtrMemD) {
-      LValue LV = CGF.EmitPointerToDataMemberBinaryExpr(BExpr);
-      Value *InVal = CGF.EmitLoadOfLValue(LV, E->getType()).getScalarVal();
-      return InVal;
-    }
-  
   CGF.ErrorUnsupported(E, "scalar expression");
   if (E->getType()->isVoidType())
     return 0;
@@ -773,49 +761,20 @@ Value *ScalarExprEmitter::EmitCastExpr(const CastExpr *CE) {
 
   switch (Kind) {
   default:
-    // FIXME: Assert here.
-    // assert(0 && "Unhandled cast kind!");
+    //return CGF.ErrorUnsupported(E, "type of cast");
     break;
+
   case CastExpr::CK_Unknown:
-    // FIXME: We should really assert here - Unknown casts should never get
-    // as far as to codegen.
+    //assert(0 && "Unknown cast kind!");
     break;
+
   case CastExpr::CK_BitCast: {
     Value *Src = Visit(const_cast<Expr*>(E));
     return Builder.CreateBitCast(Src, ConvertType(DestTy));
   }
-  case CastExpr::CK_ArrayToPointerDecay: {
-    assert(E->getType()->isArrayType() &&
-           "Array to pointer decay must have array source type!");
-
-    Value *V = EmitLValue(E).getAddress();  // Bitfields can't be arrays.
+  case CastExpr::CK_NoOp:
+    return Visit(const_cast<Expr*>(E));
 
-    // Note that VLA pointers are always decayed, so we don't need to do
-    // anything here.
-    if (!E->getType()->isVariableArrayType()) {
-      assert(isa<llvm::PointerType>(V->getType()) && "Expected pointer");
-      assert(isa<llvm::ArrayType>(cast<llvm::PointerType>(V->getType())
-                                 ->getElementType()) &&
-             "Expected pointer to array");
-      V = Builder.CreateStructGEP(V, 0, "arraydecay");
-    }
-
-    // The resultant pointer type can be implicitly casted to other pointer
-    // types as well (e.g. void*) and can be implicitly converted to integer.
-    const llvm::Type *DestLTy = ConvertType(DestTy);
-    if (V->getType() != DestLTy) {
-      if (isa<llvm::PointerType>(DestLTy))
-        V = Builder.CreateBitCast(V, DestLTy, "ptrconv");
-      else {
-        assert(isa<llvm::IntegerType>(DestLTy) && "Unknown array decay");
-        V = Builder.CreatePtrToInt(V, DestLTy, "ptrconv");
-      }
-    }
-    return V;
-  }
-  case CastExpr::CK_NullToMemberPointer:
-    return CGF.CGM.EmitNullConstant(DestTy);
-      
   case CastExpr::CK_DerivedToBase: {
     const RecordType *DerivedClassTy = 
       E->getType()->getAs<PointerType>()->getPointeeType()->getAs<RecordType>();
@@ -841,6 +800,33 @@ Value *ScalarExprEmitter::EmitCastExpr(const CastExpr *CE) {
     return CGF.GetAddressCXXOfBaseClass(Src, DerivedClassDecl, BaseClassDecl,
                                         NullCheckValue);
   }
+  case CastExpr::CK_ToUnion: {
+    assert(0 && "Should be unreachable!");
+    break;
+  }
+  case CastExpr::CK_ArrayToPointerDecay: {
+    assert(E->getType()->isArrayType() &&
+           "Array to pointer decay must have array source type!");
+
+    Value *V = EmitLValue(E).getAddress();  // Bitfields can't be arrays.
+
+    // Note that VLA pointers are always decayed, so we don't need to do
+    // anything here.
+    if (!E->getType()->isVariableArrayType()) {
+      assert(isa<llvm::PointerType>(V->getType()) && "Expected pointer");
+      assert(isa<llvm::ArrayType>(cast<llvm::PointerType>(V->getType())
+                                 ->getElementType()) &&
+             "Expected pointer to array");
+      V = Builder.CreateStructGEP(V, 0, "arraydecay");
+    }
+
+    return V;
+  }
+  case CastExpr::CK_FunctionToPointerDecay:
+    return EmitLValue(E).getAddress();
+
+  case CastExpr::CK_NullToMemberPointer:
+    return CGF.CGM.EmitNullConstant(DestTy);
 
   case CastExpr::CK_IntegralToPointer: {
     Value *Src = Visit(const_cast<Expr*>(E));
@@ -860,7 +846,40 @@ Value *ScalarExprEmitter::EmitCastExpr(const CastExpr *CE) {
     Value *Src = Visit(const_cast<Expr*>(E));
     return Builder.CreatePtrToInt(Src, ConvertType(DestTy));
   }
-  
+
+  case CastExpr::CK_ToVoid: {
+    CGF.EmitAnyExpr(E, 0, false, true);
+    return 0;
+  }
+
+  case CastExpr::CK_Dynamic: {
+    Value *V = Visit(const_cast<Expr*>(E));
+    const CXXDynamicCastExpr *DCE = cast<CXXDynamicCastExpr>(CE);
+    return CGF.EmitDynamicCast(V, DCE);
+  }
+
+  case CastExpr::CK_VectorSplat: {
+    const llvm::Type *DstTy = ConvertType(DestTy);
+    Value *Elt = Visit(const_cast<Expr*>(E));
+
+    // Insert the element in element zero of an undef vector
+    llvm::Value *UnV = llvm::UndefValue::get(DstTy);
+    llvm::Value *Idx =
+        llvm::ConstantInt::get(llvm::Type::getInt32Ty(VMContext), 0);
+    UnV = Builder.CreateInsertElement(UnV, Elt, Idx, "tmp");
+
+    // Splat the element across to all elements
+    llvm::SmallVector<llvm::Constant*, 16> Args;
+    unsigned NumElements = cast<llvm::VectorType>(DstTy)->getNumElements();
+    for (unsigned i = 0; i < NumElements; i++)
+      Args.push_back(llvm::ConstantInt::get(
+                                        llvm::Type::getInt32Ty(VMContext), 0));
+
+    llvm::Constant *Mask = llvm::ConstantVector::get(&Args[0], NumElements);
+    llvm::Value *Yay = Builder.CreateShuffleVector(UnV, UnV, Mask, "splat");
+    return Yay;
+  }
+
   }
 
   // Handle cases where the source is an non-complex type.
diff --git a/lib/CodeGen/CGObjCGNU.cpp b/lib/CodeGen/CGObjCGNU.cpp
index f348bfffcb86..b431daa958f0 100644
--- a/lib/CodeGen/CGObjCGNU.cpp
+++ b/lib/CodeGen/CGObjCGNU.cpp
@@ -608,6 +608,8 @@ llvm::Constant *CGObjCGNU::GenerateIvarList(
     const llvm::SmallVectorImpl<llvm::Constant *>  &IvarNames,
     const llvm::SmallVectorImpl<llvm::Constant *>  &IvarTypes,
     const llvm::SmallVectorImpl<llvm::Constant *>  &IvarOffsets) {
+  if (IvarNames.size() == 0)
+    return NULLPtr;
   // Get the method structure type.
   llvm::StructType *ObjCIvarTy = llvm::StructType::get(VMContext,
     PtrToInt8Ty,
@@ -1189,8 +1191,8 @@ void CGObjCGNU::GenerateClass(const ObjCImplementationDecl *OID) {
       Context.getObjCEncodingForType((*iter)->getType(), TypeStr);
       IvarTypes.push_back(MakeConstantString(TypeStr));
       // Get the offset
-      uint64_t Offset = 0;
       uint64_t BaseOffset = ComputeIvarBaseOffset(CGM, ClassDecl, *iter);
+      uint64_t Offset = BaseOffset;
       if (CGM.getContext().getLangOptions().ObjCNonFragileABI) {
         Offset = BaseOffset - superInstanceSize;
       }
@@ -1301,7 +1303,7 @@ void CGObjCGNU::GenerateClass(const ObjCImplementationDecl *OID) {
   }
   //Generate metaclass for class methods
   llvm::Constant *MetaClassStruct = GenerateClassStructure(NULLPtr,
-      NULLPtr, 0x12L, /*name*/"", 0, Zeros[0], GenerateIvarList(
+      NULLPtr, 0x12L, ClassName.c_str(), 0, Zeros[0], GenerateIvarList(
         empty, empty, empty), ClassMethodList, NULLPtr, NULLPtr, NULLPtr);
 
   // Generate the class structure
diff --git a/lib/CodeGen/CGObjCMac.cpp b/lib/CodeGen/CGObjCMac.cpp
index 9b2f4a1faee7..4355e66feecd 100644
--- a/lib/CodeGen/CGObjCMac.cpp
+++ b/lib/CodeGen/CGObjCMac.cpp
@@ -991,6 +991,9 @@ private:
   /// for the given class.
   llvm::Value *EmitClassRef(CGBuilderTy &Builder,
                             const ObjCInterfaceDecl *ID);
+  
+  /// EmitSuperClassRef - Emits reference to class's main metadata class.
+  llvm::Value *EmitSuperClassRef(const ObjCInterfaceDecl *ID);
 
   CodeGen::RValue EmitMessageSend(CodeGen::CodeGenFunction &CGF,
                                   QualType ResultType,
@@ -1167,6 +1170,9 @@ private:
   /// legacy messaging dispatch.
   llvm::DenseSet<Selector> NonLegacyDispatchMethods;
 
+  /// DefinedMetaClasses - List of defined meta-classes.
+  std::vector<llvm::GlobalValue*> DefinedMetaClasses;
+  
   /// LegacyDispatchedSelector - Returns true if SEL is not in the list of
   /// NonLegacyDispatchMethods; false otherwise.
   bool LegacyDispatchedSelector(Selector Sel);
@@ -1485,8 +1491,13 @@ CGObjCMac::GenerateMessageSendSuper(CodeGen::CodeGenFunction &CGF,
       llvm::Value *Super = CGF.Builder.CreateLoad(SuperPtr);
       Target = Super;
     }
-  } else {
+  } 
+  else if (isCategoryImpl)
     Target = EmitClassRef(CGF.Builder, Class->getSuperClass());
+  else {
+    llvm::Value *ClassPtr = EmitSuperClassRef(Class);
+    ClassPtr = CGF.Builder.CreateStructGEP(ClassPtr, 1);
+    Target = CGF.Builder.CreateLoad(ClassPtr);
   }
   // FIXME: We shouldn't need to do this cast, rectify the ASTContext and
   // ObjCTypes types.
@@ -2051,11 +2062,22 @@ void CGObjCMac::GenerateClass(const ObjCImplementationDecl *ID) {
   Values[11] = EmitClassExtension(ID);
   llvm::Constant *Init = llvm::ConstantStruct::get(ObjCTypes.ClassTy,
                                                    Values);
-
-  llvm::GlobalVariable *GV =
-    CreateMetadataVar("\01L_OBJC_CLASS_" + ClassName, Init,
-                      "__OBJC,__class,regular,no_dead_strip",
-                      4, true);
+  std::string Name("\01L_OBJC_CLASS_");
+  Name += ClassName;
+  const char *Section = "__OBJC,__class,regular,no_dead_strip";
+  // Check for a forward reference.
+  llvm::GlobalVariable *GV = CGM.getModule().getGlobalVariable(Name);
+  if (GV) {
+    assert(GV->getType()->getElementType() == ObjCTypes.ClassTy &&
+           "Forward metaclass reference has incorrect type.");
+    GV->setLinkage(llvm::GlobalValue::InternalLinkage);
+    GV->setInitializer(Init);
+    GV->setSection(Section);
+    GV->setAlignment(4);
+    CGM.AddUsedGlobal(GV);
+  } 
+  else
+    GV = CreateMetadataVar(Name, Init, Section, 4, true);
   DefinedClasses.push_back(GV);
 }
 
@@ -2154,6 +2176,22 @@ llvm::Constant *CGObjCMac::EmitMetaClassRef(const ObjCInterfaceDecl *ID) {
   }
 }
 
+llvm::Value *CGObjCMac::EmitSuperClassRef(const ObjCInterfaceDecl *ID) {
+  std::string Name = "\01L_OBJC_CLASS_" + ID->getNameAsString();
+  
+  if (llvm::GlobalVariable *GV = CGM.getModule().getGlobalVariable(Name,
+                                                                   true)) {
+    assert(GV->getType()->getElementType() == ObjCTypes.ClassTy &&
+           "Forward class metadata reference has incorrect type.");
+    return GV;
+  } else {
+    return new llvm::GlobalVariable(CGM.getModule(), ObjCTypes.ClassTy, false,
+                                    llvm::GlobalValue::ExternalLinkage,
+                                    0,
+                                    Name);
+  }
+}
+
 /*
   struct objc_class_ext {
   uint32_t size;
@@ -4095,6 +4133,25 @@ void CGObjCNonFragileABIMac::FinishNonFragileABIModule() {
   AddModuleClassList(DefinedClasses,
                      "\01L_OBJC_LABEL_CLASS_$",
                      "__DATA, __objc_classlist, regular, no_dead_strip");
+  
+  bool hasWeakImport = false;
+  for (unsigned i = 0; i < DefinedClasses.size(); i++) {
+    llvm::GlobalValue *IMPLGV = DefinedClasses[i];
+    if (IMPLGV->getLinkage() != llvm::GlobalValue::ExternalWeakLinkage)
+      continue;
+    IMPLGV->setLinkage(llvm::GlobalValue::ExternalLinkage);
+    hasWeakImport = true;
+  }
+  
+  if (hasWeakImport) {
+    for (unsigned i = 0; i < DefinedMetaClasses.size(); i++) {
+      llvm::GlobalValue *IMPLGV = DefinedMetaClasses[i];
+      if (IMPLGV->getLinkage() != llvm::GlobalValue::ExternalWeakLinkage)
+        continue;
+      IMPLGV->setLinkage(llvm::GlobalValue::ExternalLinkage);
+    }    
+  }
+  
   AddModuleClassList(DefinedNonLazyClasses,
                      "\01L_OBJC_LABEL_NONLAZY_CLASS_$",
                      "__DATA, __objc_nlclslist, regular, no_dead_strip");
@@ -4384,6 +4441,8 @@ void CGObjCNonFragileABIMac::GenerateClass(const ObjCImplementationDecl *ID) {
     std::string SuperClassName =
       ObjCMetaClassName + ID->getClassInterface()->getSuperClass()->getNameAsString();
     SuperClassGV = GetClassGlobal(SuperClassName);
+    if (ID->getClassInterface()->getSuperClass()->hasAttr<WeakImportAttr>())
+      SuperClassGV->setLinkage(llvm::GlobalValue::ExternalWeakLinkage);
   }
   llvm::GlobalVariable *CLASS_RO_GV = BuildClassRoTInitializer(flags,
                                                                InstanceStart,
@@ -4392,6 +4451,7 @@ void CGObjCNonFragileABIMac::GenerateClass(const ObjCImplementationDecl *ID) {
   llvm::GlobalVariable *MetaTClass =
     BuildClassMetaData(TClassName, IsAGV, SuperClassGV, CLASS_RO_GV,
                        classIsHidden);
+  DefinedMetaClasses.push_back(MetaTClass);
 
   // Metadata for the class
   flags = CLS;
@@ -4409,6 +4469,8 @@ void CGObjCNonFragileABIMac::GenerateClass(const ObjCImplementationDecl *ID) {
     std::string RootClassName =
       ID->getClassInterface()->getSuperClass()->getNameAsString();
     SuperClassGV = GetClassGlobal(ObjCClassName + RootClassName);
+    if (ID->getClassInterface()->getSuperClass()->hasAttr<WeakImportAttr>())
+      SuperClassGV->setLinkage(llvm::GlobalValue::ExternalWeakLinkage);
   }
   GetClassSizeInfo(ID, InstanceStart, InstanceSize);
   CLASS_RO_GV = BuildClassRoTInitializer(flags,
@@ -4489,6 +4551,9 @@ void CGObjCNonFragileABIMac::GenerateCategory(const ObjCCategoryImplDecl *OCD) {
   Values[0] = GetClassName(OCD->getIdentifier());
   // meta-class entry symbol
   llvm::GlobalVariable *ClassGV = GetClassGlobal(ExtClassName);
+  if (Interface->hasAttr<WeakImportAttr>())
+    ClassGV->setLinkage(llvm::GlobalValue::ExternalWeakLinkage);
+  
   Values[1] = ClassGV;
   std::vector<llvm::Constant*> Methods;
   std::string MethodListName(Prefix);
@@ -5178,6 +5243,12 @@ llvm::Value *CGObjCNonFragileABIMac::EmitMetaClassRef(CGBuilderTy &Builder,
 /// decl.
 llvm::Value *CGObjCNonFragileABIMac::GetClass(CGBuilderTy &Builder,
                                               const ObjCInterfaceDecl *ID) {
+  if (ID->hasAttr<WeakImportAttr>()) {
+    std::string ClassName(getClassSymbolPrefix() + ID->getNameAsString());
+    llvm::GlobalVariable *ClassGV = GetClassGlobal(ClassName);
+    ClassGV->setLinkage(llvm::GlobalValue::ExternalWeakLinkage);
+  }
+  
   return EmitClassRef(Builder, ID);
 }
 
diff --git a/lib/CodeGen/CGRecordLayoutBuilder.cpp b/lib/CodeGen/CGRecordLayoutBuilder.cpp
index 7baf69d87689..8e0864b8dd83 100644
--- a/lib/CodeGen/CGRecordLayoutBuilder.cpp
+++ b/lib/CodeGen/CGRecordLayoutBuilder.cpp
@@ -208,7 +208,11 @@ void CGRecordLayoutBuilder::LayoutUnion(const RecordDecl *D) {
       Align = 1;
     }
   }
-
+  if (!Align) {
+    assert((D->field_begin() == D->field_end()) && "LayoutUnion - Align 0");
+    Align = 1;
+  }
+  
   // Append tail padding.
   if (Layout.getSize() / 8 > Size)
     AppendPadding(Layout.getSize() / 8, Align);
diff --git a/lib/CodeGen/CGRtti.cpp b/lib/CodeGen/CGRtti.cpp
index 7af15f0a8ca4..e18843d5cf3f 100644
--- a/lib/CodeGen/CGRtti.cpp
+++ b/lib/CodeGen/CGRtti.cpp
@@ -11,36 +11,378 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "clang/AST/Type.h"
+#include "clang/AST/RecordLayout.h"
 #include "CodeGenModule.h"
 using namespace clang;
 using namespace CodeGen;
 
+class RttiBuilder {
+  CodeGenModule &CGM;  // Per-module state.
+  llvm::LLVMContext &VMContext;
+  const llvm::Type *Int8PtrTy;
+  llvm::SmallSet<const CXXRecordDecl *, 16> SeenVBase;
+  llvm::SmallSet<const CXXRecordDecl *, 32> SeenBase;
+public:
+  RttiBuilder(CodeGenModule &cgm)
+    : CGM(cgm), VMContext(cgm.getModule().getContext()),
+      Int8PtrTy(llvm::Type::getInt8PtrTy(VMContext)) { }
+
+  /// BuildVtableRef - Build a reference to a vtable.
+  llvm::Constant *BuildVtableRef(const char *Name) {
+    // Build a descriptor for Name
+    llvm::Constant *GV = CGM.getModule().getGlobalVariable(Name);
+    if (GV)
+      GV = llvm::ConstantExpr::getBitCast(GV,
+                                          llvm::PointerType::get(Int8PtrTy, 0));
+    else {
+      llvm::GlobalVariable::LinkageTypes linktype;
+      linktype = llvm::GlobalValue::ExternalLinkage;
+      GV = new llvm::GlobalVariable(CGM.getModule(), Int8PtrTy,
+                                    true, linktype, 0, Name);
+    }
+    llvm::Constant *C;
+    C = llvm::ConstantInt::get(llvm::Type::getInt64Ty(VMContext), 2);
+    C = llvm::ConstantExpr::getInBoundsGetElementPtr(GV, &C, 1);
+    return llvm::ConstantExpr::getBitCast(C, Int8PtrTy);
+  }
+
+  llvm::Constant *BuildName(QualType Ty, bool Hidden) {
+    llvm::SmallString<256> OutName;
+    llvm::raw_svector_ostream Out(OutName);
+    mangleCXXRttiName(CGM.getMangleContext(), Ty, Out);
+
+    llvm::GlobalVariable::LinkageTypes linktype;
+    linktype = llvm::GlobalValue::LinkOnceODRLinkage;
+
+    llvm::Constant *C;
+    C = llvm::ConstantArray::get(VMContext, Out.str().substr(4));
+
+    llvm::GlobalVariable * GV = new llvm::GlobalVariable(CGM.getModule(),
+                                                         C->getType(),
+                                                         true, linktype, C,
+                                                         Out.str());
+    if (Hidden)
+      GV->setVisibility(llvm::GlobalVariable::HiddenVisibility);
+    return llvm::ConstantExpr::getBitCast(GV, Int8PtrTy);
+  };
+
+  /// - BuildFlags - Build a psABI __flags value for __vmi_class_type_info.
+  llvm::Constant *BuildFlags(int f) {
+    return llvm::ConstantInt::get(llvm::Type::getInt32Ty(VMContext), f);
+  }
+
+  /// BuildBaseCount - Build a psABI __base_count value for
+  /// __vmi_class_type_info.
+  llvm::Constant *BuildBaseCount(unsigned c) {
+    return llvm::ConstantInt::get(llvm::Type::getInt32Ty(VMContext), c);
+  }
+
+  llvm::Constant *BuildTypeRef(QualType Ty) {
+    llvm::Constant *C;
+
+    if (!CGM.getContext().getLangOptions().Rtti)
+      return llvm::Constant::getNullValue(Int8PtrTy);
+
+    llvm::SmallString<256> OutName;
+    llvm::raw_svector_ostream Out(OutName);
+    mangleCXXRtti(CGM.getMangleContext(), Ty, Out);
+
+    C = CGM.getModule().getGlobalVariable(Out.str());
+    if (C)
+      return llvm::ConstantExpr::getBitCast(C, Int8PtrTy);
+
+    llvm::GlobalVariable::LinkageTypes linktype;
+    linktype = llvm::GlobalValue::ExternalLinkage;;
+
+    C = new llvm::GlobalVariable(CGM.getModule(), Int8PtrTy, true, linktype,
+                                 0, Out.str());
+    return llvm::ConstantExpr::getBitCast(C, Int8PtrTy);
+  }
+
+  llvm::Constant *Buildclass_type_infoRef(const CXXRecordDecl *RD) {
+    return BuildTypeRef(CGM.getContext().getTagDeclType(RD));
+  }
+
+  /// CalculateFlags - Calculate the flags for the __vmi_class_type_info
+  /// datastructure.  1 for non-diamond repeated inheritance, 2 for a dimond
+  /// shaped class.
+  int CalculateFlags(const CXXRecordDecl*RD) {
+    int flags = 0;
+    if (SeenBase.count(RD))
+      flags |= 1;
+    else
+      SeenBase.insert(RD);
+    for (CXXRecordDecl::base_class_const_iterator i = RD->bases_begin(),
+           e = RD->bases_end(); i != e; ++i) {
+      const CXXRecordDecl *Base =
+        cast<CXXRecordDecl>(i->getType()->getAs<RecordType>()->getDecl());
+      if (i->isVirtual()) {
+        if (SeenVBase.count(Base))
+          flags |= 2;
+        else
+          SeenVBase.insert(Base);
+      }
+      flags |= CalculateFlags(Base);
+    }
+    return flags;
+  }
+
+  bool SimpleInheritance(const CXXRecordDecl *RD) {
+    if (RD->getNumBases() != 1)
+      return false;
+    CXXRecordDecl::base_class_const_iterator i = RD->bases_begin();
+    if (i->isVirtual())
+      return false;
+    if (i->getAccessSpecifier() != AS_public)
+      return false;
+
+    const ASTRecordLayout &Layout = CGM.getContext().getASTRecordLayout(RD);
+    const CXXRecordDecl *Base =
+      cast<CXXRecordDecl>(i->getType()->getAs<RecordType>()->getDecl());
+    if (Layout.getBaseClassOffset(Base) != 0)
+      return false;
+    return true;
+  }
+
+  llvm::Constant *finish(std::vector<llvm::Constant *> &info,
+                         llvm::GlobalVariable *GV,
+                         llvm::StringRef Name, bool Hidden) {
+    llvm::GlobalVariable::LinkageTypes linktype;
+    linktype = llvm::GlobalValue::LinkOnceODRLinkage;
+
+    llvm::Constant *C;
+    C = llvm::ConstantStruct::get(VMContext, &info[0], info.size(), false);
+
+    if (GV == 0)
+      GV = new llvm::GlobalVariable(CGM.getModule(), C->getType(), true,
+                                    linktype, C, Name);
+    else {
+      llvm::GlobalVariable *OGV = GV;
+      GV = new llvm::GlobalVariable(CGM.getModule(), C->getType(), true,
+                                    linktype, C, Name);
+      GV->takeName(OGV);
+      llvm::Constant *NewPtr = llvm::ConstantExpr::getBitCast(GV,
+                                                              OGV->getType());
+      OGV->replaceAllUsesWith(NewPtr);
+      OGV->eraseFromParent();
+    }
+    if (Hidden)
+      GV->setVisibility(llvm::GlobalVariable::HiddenVisibility);
+    return llvm::ConstantExpr::getBitCast(GV, Int8PtrTy);
+  }
+
+
+  llvm::Constant *Buildclass_type_info(const CXXRecordDecl *RD) {
+    if (!CGM.getContext().getLangOptions().Rtti)
+      return llvm::Constant::getNullValue(Int8PtrTy);
+
+    llvm::Constant *C;
+
+    llvm::SmallString<256> OutName;
+    llvm::raw_svector_ostream Out(OutName);
+    mangleCXXRtti(CGM.getMangleContext(), CGM.getContext().getTagDeclType(RD),
+                  Out);
+
+    llvm::GlobalVariable *GV;
+    GV = CGM.getModule().getGlobalVariable(Out.str());
+    if (GV && !GV->isDeclaration())
+      return llvm::ConstantExpr::getBitCast(GV, Int8PtrTy);
+
+    std::vector<llvm::Constant *> info;
+
+    bool Hidden = CGM.getDeclVisibilityMode(RD) == LangOptions::Hidden;
+
+    bool simple = false;
+    if (RD->getNumBases() == 0)
+      C = BuildVtableRef("_ZTVN10__cxxabiv117__class_type_infoE");
+    else if (SimpleInheritance(RD)) {
+      simple = true;
+      C = BuildVtableRef("_ZTVN10__cxxabiv120__si_class_type_infoE");
+    } else
+      C = BuildVtableRef("_ZTVN10__cxxabiv121__vmi_class_type_infoE");
+    info.push_back(C);
+    info.push_back(BuildName(CGM.getContext().getTagDeclType(RD), Hidden));
+
+    // If we have no bases, there are no more fields.
+    if (RD->getNumBases()) {
+      if (!simple) {
+        info.push_back(BuildFlags(CalculateFlags(RD)));
+        info.push_back(BuildBaseCount(RD->getNumBases()));
+      }
+
+      const ASTRecordLayout &Layout = CGM.getContext().getASTRecordLayout(RD);
+      for (CXXRecordDecl::base_class_const_iterator i = RD->bases_begin(),
+             e = RD->bases_end(); i != e; ++i) {
+        const CXXRecordDecl *Base =
+          cast<CXXRecordDecl>(i->getType()->getAs<RecordType>()->getDecl());
+        info.push_back(CGM.GenerateRttiRef(Base));
+        if (simple)
+          break;
+        int64_t offset;
+        if (!i->isVirtual())
+          offset = Layout.getBaseClassOffset(Base)/8;
+        else
+          offset = CGM.getVtableInfo().getVirtualBaseOffsetIndex(RD, Base);
+        offset <<= 8;
+        // Now set the flags.
+        offset += i->isVirtual() ? 1 : 0;;
+        offset += i->getAccessSpecifier() == AS_public ? 2 : 0;
+        const llvm::Type *LongTy =
+          CGM.getTypes().ConvertType(CGM.getContext().LongTy);
+        C = llvm::ConstantInt::get(LongTy, offset);
+        info.push_back(C);
+      }
+    }
+
+    return finish(info, GV, Out.str(), Hidden);
+  }
+
+  /// - BuildFlags - Build a __flags value for __pbase_type_info.
+  llvm::Constant *BuildInt(int f) {
+    return llvm::ConstantInt::get(llvm::Type::getInt32Ty(VMContext), f);
+  }
+
+  llvm::Constant *BuildType2(QualType Ty) {
+    if (const RecordType *RT = Ty.getTypePtr()->getAs<RecordType>())
+      if (const CXXRecordDecl *RD = cast<CXXRecordDecl>(RT->getDecl()))
+      return Buildclass_type_info(RD);
+    return BuildType(Ty);
+  }
+
+  llvm::Constant *BuildPointerType(QualType Ty) {
+    llvm::Constant *C;
+
+    llvm::SmallString<256> OutName;
+    llvm::raw_svector_ostream Out(OutName);
+    mangleCXXRtti(CGM.getMangleContext(), Ty, Out);
+
+    llvm::GlobalVariable *GV;
+    GV = CGM.getModule().getGlobalVariable(Out.str());
+    if (GV && !GV->isDeclaration())
+      return llvm::ConstantExpr::getBitCast(GV, Int8PtrTy);
+
+    std::vector<llvm::Constant *> info;
+
+    // FIXME: pointer to hidden should be hidden, we should be able to
+    // grab a bit off the type for this.
+    bool Hidden = false;
+
+    QualType PTy = Ty->getPointeeType();
+    QualType BTy;
+    bool PtrMem = false;
+    if (const MemberPointerType *MPT = dyn_cast<MemberPointerType>(Ty)) {
+      PtrMem = true;
+      BTy = QualType(MPT->getClass(), 0);
+      PTy = MPT->getPointeeType();
+    }
+
+    if (PtrMem)
+      C = BuildVtableRef("_ZTVN10__cxxabiv129__pointer_to_member_type_infoE");
+    else
+      C = BuildVtableRef("_ZTVN10__cxxabiv119__pointer_type_infoE");
+    info.push_back(C);
+    info.push_back(BuildName(Ty, Hidden));
+    Qualifiers Q = PTy.getQualifiers();
+    PTy = CGM.getContext().getCanonicalType(PTy).getUnqualifiedType();
+    int flags = 0;
+    flags += Q.hasConst() ? 0x1 : 0;
+    flags += Q.hasVolatile() ? 0x2 : 0;
+    flags += Q.hasRestrict() ? 0x4 : 0;
+    flags += Ty.getTypePtr()->isIncompleteType() ? 0x8 : 0;
+    if (PtrMem && BTy.getTypePtr()->isIncompleteType())
+      flags += 0x10;
+
+    info.push_back(BuildInt(flags));
+    info.push_back(BuildInt(0));
+    info.push_back(BuildType2(PTy));
+
+    if (PtrMem)
+      info.push_back(BuildType2(BTy));
+
+    return finish(info, GV, Out.str(), true);
+  }
+
+  llvm::Constant *BuildSimpleType(QualType Ty, const char *vtbl) {
+    llvm::Constant *C;
+
+    llvm::SmallString<256> OutName;
+    llvm::raw_svector_ostream Out(OutName);
+    mangleCXXRtti(CGM.getMangleContext(), Ty, Out);
+
+    llvm::GlobalVariable *GV;
+    GV = CGM.getModule().getGlobalVariable(Out.str());
+    if (GV && !GV->isDeclaration())
+      return llvm::ConstantExpr::getBitCast(GV, Int8PtrTy);
+
+    std::vector<llvm::Constant *> info;
+
+    // FIXME: pointer to hidden should be hidden, we should be able to
+    // grab a bit off the type for this.
+    bool Hidden = false;
+
+    C = BuildVtableRef(vtbl);
+    info.push_back(C);
+    info.push_back(BuildName(Ty, Hidden));
+
+    return finish(info, GV, Out.str(), true);
+  }
+
+  llvm::Constant *BuildType(QualType Ty) {
+    const clang::Type &Type
+      = *CGM.getContext().getCanonicalType(Ty).getTypePtr();
+    switch (Type.getTypeClass()) {
+    default: {
+      assert(0 && "typeid expression");
+      return llvm::Constant::getNullValue(Int8PtrTy);
+    }
+
+    case Type::Builtin: {
+      // We expect all type_info objects for builtin types to be in the library.
+      return BuildTypeRef(Ty);
+    }
+
+    case Type::Pointer: {
+      QualType PTy = Ty->getPointeeType();
+      Qualifiers Q = PTy.getQualifiers();
+      Q.removeConst();
+      // T* and const T* for all builtin types T are expected in the library.
+      if (isa<BuiltinType>(PTy) && Q.empty())
+        return BuildTypeRef(Ty);
+
+      return BuildPointerType(Ty);
+    }
+    case Type::MemberPointer:
+      return BuildPointerType(Ty);
+    case Type::FunctionProto:
+    case Type::FunctionNoProto:
+      return BuildSimpleType(Ty, "_ZTVN10__cxxabiv120__function_type_infoE");
+    case Type::ConstantArray:
+    case Type::IncompleteArray:
+    case Type::VariableArray:
+    case Type::Vector:
+    case Type::ExtVector:
+      return BuildSimpleType(Ty, "_ZTVN10__cxxabiv117__array_type_infoE");
+    case Type::Enum:
+      return BuildSimpleType(Ty, "_ZTVN10__cxxabiv116__enum_type_infoE");
+    }
+  }
+};
+
+llvm::Constant *CodeGenModule::GenerateRttiRef(const CXXRecordDecl *RD) {
+  RttiBuilder b(*this);
+
+  return b.Buildclass_type_infoRef(RD);
+}
+
 llvm::Constant *CodeGenModule::GenerateRtti(const CXXRecordDecl *RD) {
-  const llvm::Type *Int8PtrTy = llvm::Type::getInt8PtrTy(VMContext);
-
-  if (!getContext().getLangOptions().Rtti)
-    return llvm::Constant::getNullValue(Int8PtrTy);
-
-  llvm::SmallString<256> OutName;
-  llvm::raw_svector_ostream Out(OutName);
-  mangleCXXRtti(getMangleContext(), Context.getTagDeclType(RD), Out);
-  
-  llvm::GlobalVariable::LinkageTypes linktype;
-  linktype = llvm::GlobalValue::WeakAnyLinkage;
-  std::vector<llvm::Constant *> info;
-  // assert(0 && "FIXME: implement rtti descriptor");
-  // FIXME: descriptor
-  info.push_back(llvm::Constant::getNullValue(Int8PtrTy));
-  // assert(0 && "FIXME: implement rtti ts");
-  // FIXME: TS
-  info.push_back(llvm::Constant::getNullValue(Int8PtrTy));
-
-  llvm::Constant *C;
-  llvm::ArrayType *type = llvm::ArrayType::get(Int8PtrTy, info.size());
-  C = llvm::ConstantArray::get(type, info);
-  llvm::Constant *Rtti = 
-    new llvm::GlobalVariable(getModule(), type, true, linktype, C,
-                             Out.str());
-  Rtti = llvm::ConstantExpr::getBitCast(Rtti, Int8PtrTy);
-  return Rtti;
+  RttiBuilder b(*this);
+
+  return b.Buildclass_type_info(RD);
+}
+
+llvm::Constant *CodeGenModule::GenerateRttiNonClass(QualType Ty) {
+  RttiBuilder b(*this);
+
+  return b.BuildType(Ty);
 }
diff --git a/lib/CodeGen/CGStmt.cpp b/lib/CodeGen/CGStmt.cpp
index 9126c2c99cdf..b6d7b3990452 100644
--- a/lib/CodeGen/CGStmt.cpp
+++ b/lib/CodeGen/CGStmt.cpp
@@ -148,13 +148,8 @@ RValue CodeGenFunction::EmitCompoundStmt(const CompoundStmt &S, bool GetLast,
 
   CGDebugInfo *DI = getDebugInfo();
   if (DI) {
-#ifdef ATTACH_DEBUG_INFO_TO_AN_INSN
     DI->setLocation(S.getLBracLoc());
     DI->EmitRegionStart(CurFn, Builder);
-#else
-    EnsureInsertPoint();
-    DI->setLocation(S.getLBracLoc());
-#endif    
   }
 
   // Keep track of the current cleanup stack depth.
@@ -167,13 +162,8 @@ RValue CodeGenFunction::EmitCompoundStmt(const CompoundStmt &S, bool GetLast,
     EmitStmt(*I);
 
   if (DI) {
-#ifdef ATTACH_DEBUG_INFO_TO_AN_INSN
     DI->setLocation(S.getLBracLoc());
     DI->EmitRegionEnd(CurFn, Builder);
-#else
-    EnsureInsertPoint();
-    DI->setLocation(S.getLBracLoc());
-#endif    
   }
 
   RValue RV;
@@ -284,16 +274,9 @@ void CodeGenFunction::EmitGotoStmt(const GotoStmt &S) {
 
 
 void CodeGenFunction::EmitIndirectGotoStmt(const IndirectGotoStmt &S) {
-  // Emit initial switch which will be patched up later by
-  // EmitIndirectSwitches(). We need a default dest, so we use the
-  // current BB, but this is overwritten.
-#ifndef USEINDIRECTBRANCH
-  llvm::Value *V = Builder.CreatePtrToInt(EmitScalarExpr(S.getTarget()),
-                                          llvm::Type::getInt32Ty(VMContext),
-#else
+  // Ensure that we have an i8* for our PHI node.
   llvm::Value *V = Builder.CreateBitCast(EmitScalarExpr(S.getTarget()),
                                          llvm::Type::getInt8PtrTy(VMContext),
-#endif
                                           "addr");
   llvm::BasicBlock *CurBB = Builder.GetInsertBlock();
   
@@ -491,13 +474,11 @@ void CodeGenFunction::EmitForStmt(const ForStmt &S) {
   BreakContinueStack.push_back(BreakContinue(AfterFor, ContinueBlock));
 
   // If the condition is true, execute the body of the for stmt.
-#ifdef ATTACH_DEBUG_INFO_TO_AN_INSN
   CGDebugInfo *DI = getDebugInfo();
   if (DI) {
     DI->setLocation(S.getSourceRange().getBegin());
     DI->EmitRegionStart(CurFn, Builder);
   }
-#endif
   EmitStmt(S.getBody());
 
   BreakContinueStack.pop_back();
@@ -510,12 +491,10 @@ void CodeGenFunction::EmitForStmt(const ForStmt &S) {
 
   // Finally, branch back up to the condition for the next iteration.
   EmitBranch(CondBlock);
-#ifdef ATTACH_DEBUG_INFO_TO_AN_INSN
   if (DI) {
     DI->setLocation(S.getSourceRange().getEnd());
     DI->EmitRegionEnd(CurFn, Builder);
   }
-#endif
 
   // Emit the fall-through block.
   EmitBlock(AfterFor, true);
@@ -758,7 +737,7 @@ void CodeGenFunction::EmitSwitchStmt(const SwitchStmt &S) {
 }
 
 static std::string
-SimplifyConstraint(const char *Constraint, TargetInfo &Target,
+SimplifyConstraint(const char *Constraint, const TargetInfo &Target,
                  llvm::SmallVectorImpl<TargetInfo::ConstraintInfo> *OutCons=0) {
   std::string Result;
 
diff --git a/lib/CodeGen/CGVtable.cpp b/lib/CodeGen/CGVtable.cpp
index e2e11478de26..4c97a9bdeebf 100644
--- a/lib/CodeGen/CGVtable.cpp
+++ b/lib/CodeGen/CGVtable.cpp
@@ -15,6 +15,7 @@
 #include "CodeGenFunction.h"
 
 #include "clang/AST/RecordLayout.h"
+#include <cstdio>
 
 using namespace clang;
 using namespace CodeGen;
@@ -29,6 +30,11 @@ private:
   llvm::Type *Ptr8Ty;
   /// Class - The most derived class that this vtable is being built for.
   const CXXRecordDecl *Class;
+  /// LayoutClass - The most derived class used for virtual base layout
+  /// information.
+  const CXXRecordDecl *LayoutClass;
+  /// LayoutOffset - The offset for Class in LayoutClass.
+  uint64_t LayoutOffset;
   /// BLayout - Layout for the most derived class that this vtable is being
   /// built for.
   const ASTRecordLayout &BLayout;
@@ -39,38 +45,45 @@ private:
   CodeGenModule &CGM;  // Per-module state.
   /// Index - Maps a method decl into a vtable index.  Useful for virtual
   /// dispatch codegen.
-  llvm::DenseMap<const CXXMethodDecl *, Index_t> Index;
-  llvm::DenseMap<const CXXMethodDecl *, Index_t> VCall;
-  llvm::DenseMap<const CXXMethodDecl *, Index_t> VCallOffset;
+  llvm::DenseMap<GlobalDecl, Index_t> Index;
+  llvm::DenseMap<GlobalDecl, Index_t> VCall;
+  llvm::DenseMap<GlobalDecl, Index_t> VCallOffset;
+  // This is the offset to the nearest virtual base
+  llvm::DenseMap<GlobalDecl, Index_t> NonVirtualOffset;
   llvm::DenseMap<const CXXRecordDecl *, Index_t> VBIndex;
 
-  typedef llvm::DenseMap<const CXXMethodDecl *, int> Pures_t;
+  typedef llvm::DenseMap<GlobalDecl, int> Pures_t;
   Pures_t Pures;
   typedef std::pair<Index_t, Index_t>  CallOffset;
-  typedef llvm::DenseMap<const CXXMethodDecl *, CallOffset> Thunks_t;
+  typedef llvm::DenseMap<GlobalDecl, CallOffset> Thunks_t;
   Thunks_t Thunks;
-  typedef llvm::DenseMap<const CXXMethodDecl *,
+  typedef llvm::DenseMap<GlobalDecl,
                          std::pair<std::pair<CallOffset, CallOffset>,
                                    CanQualType> > CovariantThunks_t;
   CovariantThunks_t CovariantThunks;
   std::vector<Index_t> VCalls;
+
+  typedef std::pair<const CXXRecordDecl *, uint64_t> CtorVtable_t;
+  // CtorVtable - Used to hold the AddressPoints (offsets) into the built vtable
+  // for use in computing the initializers for the VTT.
+  llvm::DenseMap<CtorVtable_t, int64_t> &AddressPoints;
+
   typedef CXXRecordDecl::method_iterator method_iter;
   // FIXME: Linkage should follow vtable
   const bool Extern;
   const uint32_t LLVMPointerWidth;
   Index_t extra;
-  int CurrentVBaseOffset;
   typedef std::vector<std::pair<const CXXRecordDecl *, int64_t> > Path_t;
   llvm::Constant *cxa_pure;
 public:
-  VtableBuilder(std::vector<llvm::Constant *> &meth,
-                const CXXRecordDecl *c,
-                CodeGenModule &cgm)
-    : methods(meth), Class(c), BLayout(cgm.getContext().getASTRecordLayout(c)),
-      rtti(cgm.GenerateRtti(c)), VMContext(cgm.getModule().getContext()),
-      CGM(cgm), Extern(true),
-      LLVMPointerWidth(cgm.getContext().Target.getPointerWidth(0)),
-      CurrentVBaseOffset(0) {
+  VtableBuilder(std::vector<llvm::Constant *> &meth, const CXXRecordDecl *c,
+                const CXXRecordDecl *l, uint64_t lo, CodeGenModule &cgm)
+    : methods(meth), Class(c), LayoutClass(l), LayoutOffset(lo),
+      BLayout(cgm.getContext().getASTRecordLayout(l)),
+      rtti(cgm.GenerateRttiRef(c)), VMContext(cgm.getModule().getContext()),
+      CGM(cgm), AddressPoints(*new llvm::DenseMap<CtorVtable_t, int64_t>),
+      Extern(true),
+      LLVMPointerWidth(cgm.getContext().Target.getPointerWidth(0)) {
     Ptr8Ty = llvm::PointerType::get(llvm::Type::getInt8Ty(VMContext), 0);
 
     // Calculate pointer for ___cxa_pure_virtual.
@@ -81,10 +94,13 @@ public:
     cxa_pure = wrap(CGM.CreateRuntimeFunction(FTy, "__cxa_pure_virtual"));
   }
 
-  llvm::DenseMap<const CXXMethodDecl *, Index_t> &getIndex() { return Index; }
+  llvm::DenseMap<GlobalDecl, Index_t> &getIndex() { return Index; }
   llvm::DenseMap<const CXXRecordDecl *, Index_t> &getVBIndex()
     { return VBIndex; }
 
+  llvm::DenseMap<CtorVtable_t, int64_t> *getAddressPoints()
+    { return &AddressPoints; }
+
   llvm::Constant *wrap(Index_t i) {
     llvm::Constant *m;
     m = llvm::ConstantInt::get(llvm::Type::getInt64Ty(VMContext), i);
@@ -95,8 +111,8 @@ public:
     return llvm::ConstantExpr::getBitCast(m, Ptr8Ty);
   }
 
-#define D1(x)
-//#define D1(X) do { if (getenv("DEBUG")) { X; } } while (0)
+//#define D1(x)
+#define D1(X) do { if (getenv("DEBUG")) { X; } } while (0)
 
   void GenerateVBaseOffsets(const CXXRecordDecl *RD, uint64_t Offset,
                             bool updateVBIndex, Index_t current_vbindex) {
@@ -189,9 +205,11 @@ public:
     return 0;
   }
 
-  bool OverrideMethod(const CXXMethodDecl *MD, llvm::Constant *m,
+  bool OverrideMethod(GlobalDecl GD, llvm::Constant *m,
                       bool MorallyVirtual, Index_t OverrideOffset,
-                      Index_t Offset) {
+                      Index_t Offset, int64_t CurrentVBaseOffset) {
+    const CXXMethodDecl *MD = cast<CXXMethodDecl>(GD.getDecl());
+
     const bool isPure = MD->isPure();
     typedef CXXMethodDecl::method_iterator meth_iter;
     // FIXME: Should OverrideOffset's be Offset?
@@ -204,9 +222,16 @@ public:
     for (meth_iter mi = MD->begin_overridden_methods(),
            e = MD->end_overridden_methods();
          mi != e; ++mi) {
+      GlobalDecl OGD;
+      
       const CXXMethodDecl *OMD = *mi;
+      if (const CXXDestructorDecl *DD = dyn_cast<CXXDestructorDecl>(OMD))
+        OGD = GlobalDecl(DD, GD.getDtorType());
+      else
+        OGD = OMD;
+      
       llvm::Constant *om;
-      om = CGM.GetAddrOfFunction(OMD, Ptr8Ty);
+      om = WrapAddrOf(OGD);
       om = llvm::ConstantExpr::getBitCast(om, Ptr8Ty);
 
       for (Index_t i = 0, e = submethods.size();
@@ -229,57 +254,60 @@ public:
           Index_t nv = getNVOffset(oret, ret)/8;
           ReturnOffset = std::make_pair(nv, getVbaseOffset(oret, ret));
         }
-        Index[MD] = i;
+        Index[GD] = i;
         submethods[i] = m;
         if (isPure)
-          Pures[MD] = 1;
-        Pures.erase(OMD);
-        Thunks.erase(OMD);
-        if (MorallyVirtual) {
-          Index_t &idx = VCall[OMD];
+          Pures[GD] = 1;
+        Pures.erase(OGD);
+        Thunks.erase(OGD);
+        if (MorallyVirtual || VCall.count(OGD)) {
+          Index_t &idx = VCall[OGD];
           if (idx == 0) {
-            VCallOffset[MD] = OverrideOffset/8;
+            NonVirtualOffset[GD] = -OverrideOffset/8 + CurrentVBaseOffset/8;
+            VCallOffset[GD] = OverrideOffset/8;
             idx = VCalls.size()+1;
             VCalls.push_back(0);
             D1(printf("  vcall for %s at %d with delta %d most derived %s\n",
-                      MD->getNameAsCString(),
-                      (int)-VCalls.size()-3, (int)VCallOffset[MD],
-                      Class->getNameAsCString()));
+                      MD->getNameAsString().c_str(), (int)-idx-3,
+                      (int)VCalls[idx-1], Class->getNameAsCString()));
           } else {
-            VCallOffset[MD] = VCallOffset[OMD];
-            VCalls[idx-1] = -VCallOffset[OMD] + OverrideOffset/8;
+            NonVirtualOffset[GD] = NonVirtualOffset[OGD];
+            VCallOffset[GD] = VCallOffset[OGD];
+            VCalls[idx-1] = -VCallOffset[OGD] + OverrideOffset/8;
             D1(printf("  vcall patch for %s at %d with delta %d most derived %s\n",
-                      MD->getNameAsCString(),
-                      (int)-VCalls.size()-3, (int)VCallOffset[MD],
-                      Class->getNameAsCString()));
+                      MD->getNameAsString().c_str(), (int)-idx-3,
+                      (int)VCalls[idx-1], Class->getNameAsCString()));
           }
-          VCall[MD] = idx;
-          CallOffset ThisOffset;
-          ThisOffset = std::make_pair(CurrentVBaseOffset/8 - Offset/8,
-                                      -((idx+extra+2)*LLVMPointerWidth/8));
+          VCall[GD] = idx;
+          int64_t O = NonVirtualOffset[GD];
+          int v = -((idx+extra+2)*LLVMPointerWidth/8);
+          // Optimize out virtual adjustments of 0.
+          if (VCalls[idx-1] == 0)
+            v = 0;
+          CallOffset ThisOffset = std::make_pair(O, v);
           // FIXME: Do we always have to build a covariant thunk to save oret,
           // which is the containing virtual base class?
           if (ReturnOffset.first || ReturnOffset.second)
-            CovariantThunks[MD] = std::make_pair(std::make_pair(ThisOffset,
+            CovariantThunks[GD] = std::make_pair(std::make_pair(ThisOffset,
                                                                 ReturnOffset),
                                                  oret);
-          else if (!isPure)
-            Thunks[MD] = ThisOffset;
+          else if (!isPure && (ThisOffset.first || ThisOffset.second))
+            Thunks[GD] = ThisOffset;
           return true;
         }
 
         // FIXME: finish off
-        int64_t O = VCallOffset[OMD] - OverrideOffset/8;
-        // int64_t O = CurrentVBaseOffset/8 - OverrideOffset/8;
+        int64_t O = VCallOffset[OGD] - OverrideOffset/8;
+
         if (O || ReturnOffset.first || ReturnOffset.second) {
           CallOffset ThisOffset = std::make_pair(O, 0);
           
           if (ReturnOffset.first || ReturnOffset.second)
-            CovariantThunks[MD] = std::make_pair(std::make_pair(ThisOffset,
+            CovariantThunks[GD] = std::make_pair(std::make_pair(ThisOffset,
                                                                 ReturnOffset),
                                                  oret);
           else if (!isPure)
-            Thunks[MD] = ThisOffset;
+            Thunks[GD] = ThisOffset;
         }
         return true;
       }
@@ -291,9 +319,10 @@ public:
   void InstallThunks() {
     for (Thunks_t::iterator i = Thunks.begin(), e = Thunks.end();
          i != e; ++i) {
-      const CXXMethodDecl *MD = i->first;
+      GlobalDecl GD = i->first;
+      const CXXMethodDecl *MD = cast<CXXMethodDecl>(GD.getDecl());
       assert(!MD->isPure() && "Trying to thunk a pure");
-      Index_t idx = Index[MD];
+      Index_t idx = Index[GD];
       Index_t nv_O = i->second.first;
       Index_t v_O = i->second.second;
       submethods[idx] = CGM.BuildThunk(MD, Extern, nv_O, v_O);
@@ -302,10 +331,11 @@ public:
     for (CovariantThunks_t::iterator i = CovariantThunks.begin(),
            e = CovariantThunks.end();
          i != e; ++i) {
-      const CXXMethodDecl *MD = i->first;
+      GlobalDecl GD = i->first;
+      const CXXMethodDecl *MD = cast<CXXMethodDecl>(GD.getDecl());
       if (MD->isPure())
         continue;
-      Index_t idx = Index[MD];
+      Index_t idx = Index[GD];
       Index_t nv_t = i->second.first.first.first;
       Index_t v_t = i->second.first.first.second;
       Index_t nv_r = i->second.first.second.first;
@@ -316,16 +346,18 @@ public:
     CovariantThunks.clear();
     for (Pures_t::iterator i = Pures.begin(), e = Pures.end();
          i != e; ++i) {
-      const CXXMethodDecl *MD = i->first;
-      Index_t idx = Index[MD];
+      GlobalDecl GD = i->first;
+      Index_t idx = Index[GD];
       submethods[idx] = cxa_pure;
     }
     Pures.clear();
   }
 
-  llvm::Constant *WrapAddrOf(const CXXMethodDecl *MD) {
+  llvm::Constant *WrapAddrOf(GlobalDecl GD) {
+    const CXXMethodDecl *MD = cast<CXXMethodDecl>(GD.getDecl());
+
     if (const CXXDestructorDecl *Dtor = dyn_cast<CXXDestructorDecl>(MD))
-      return wrap(CGM.GetAddrOfCXXDestructor(Dtor, Dtor_Complete));
+      return wrap(CGM.GetAddrOfCXXDestructor(Dtor, GD.getDtorType()));
 
     const FunctionProtoType *FPT = MD->getType()->getAs<FunctionProtoType>();
     const llvm::Type *Ty =
@@ -335,63 +367,95 @@ public:
     return wrap(CGM.GetAddrOfFunction(MD, Ty));
   }
 
-  void OverrideMethods(Path_t *Path, bool MorallyVirtual, int64_t Offset) {
+  void OverrideMethods(Path_t *Path, bool MorallyVirtual, int64_t Offset,
+                       int64_t CurrentVBaseOffset) {
     for (Path_t::reverse_iterator i = Path->rbegin(),
            e = Path->rend(); i != e; ++i) {
       const CXXRecordDecl *RD = i->first;
       int64_t OverrideOffset = i->second;
       for (method_iter mi = RD->method_begin(), me = RD->method_end(); mi != me;
            ++mi) {
-        if (!mi->isVirtual())
+        const CXXMethodDecl *MD = *mi;
+
+        if (!MD->isVirtual())
           continue;
 
-        const CXXMethodDecl *MD = *mi;
-        llvm::Constant *m = WrapAddrOf(MD);
-        OverrideMethod(MD, m, MorallyVirtual, OverrideOffset, Offset);
+        if (const CXXDestructorDecl *DD = dyn_cast<CXXDestructorDecl>(MD)) {
+          // Override both the complete and the deleting destructor.
+          GlobalDecl CompDtor(DD, Dtor_Complete);
+          OverrideMethod(CompDtor, WrapAddrOf(CompDtor), MorallyVirtual, 
+                         OverrideOffset, Offset, CurrentVBaseOffset);
+          
+          GlobalDecl DeletingDtor(DD, Dtor_Deleting);
+          OverrideMethod(DeletingDtor, WrapAddrOf(DeletingDtor), MorallyVirtual, 
+                         OverrideOffset, Offset, CurrentVBaseOffset);
+        } else {
+          OverrideMethod(MD, WrapAddrOf(MD), MorallyVirtual, OverrideOffset, 
+                         Offset, CurrentVBaseOffset);
+        }
       }
     }
   }
 
-  void AddMethod(const CXXMethodDecl *MD, bool MorallyVirtual, Index_t Offset,
-    bool ForVirtualBase) {
-    llvm::Constant *m = WrapAddrOf(MD);
+  void AddMethod(const GlobalDecl GD, bool MorallyVirtual, Index_t Offset,
+                 bool ForVirtualBase, int64_t CurrentVBaseOffset) {
+    llvm::Constant *m = WrapAddrOf(GD);
 
     // If we can find a previously allocated slot for this, reuse it.
-    if (OverrideMethod(MD, m, MorallyVirtual, Offset, Offset))
+    if (OverrideMethod(GD, m, MorallyVirtual, Offset, Offset,
+                       CurrentVBaseOffset))
       return;
 
+    const CXXMethodDecl *MD = cast<CXXMethodDecl>(GD.getDecl());
+    
     // else allocate a new slot.
-    Index[MD] = submethods.size();
+    Index[GD] = submethods.size();
     submethods.push_back(m);
-    D1(printf("  vfn for %s at %d\n", MD->getNameAsCString(), (int)Index[MD]));
+    D1(printf("  vfn for %s at %d\n", MD->getNameAsString().c_str(),
+              (int)Index[GD]));
     if (MD->isPure())
-      Pures[MD] = 1;
+      Pures[GD] = 1;
     if (MorallyVirtual) {
-      VCallOffset[MD] = Offset/8;
-      Index_t &idx = VCall[MD];
+      VCallOffset[GD] = Offset/8;
+      Index_t &idx = VCall[GD];
       // Allocate the first one, after that, we reuse the previous one.
       if (idx == 0) {
+        NonVirtualOffset[GD] = CurrentVBaseOffset/8 - Offset/8;
         idx = VCalls.size()+1;
         VCalls.push_back(0);
         D1(printf("  vcall for %s at %d with delta %d\n",
-                  MD->getNameAsCString(), (int)-VCalls.size()-3,
-                  (int)VCallOffset[MD]));
+                  MD->getNameAsString().c_str(), (int)-VCalls.size()-3, 0));
       }
     }
   }
 
   void AddMethods(const CXXRecordDecl *RD, bool MorallyVirtual,
-                  Index_t Offset, bool RDisVirtualBase) {
+                  Index_t Offset, bool RDisVirtualBase,
+                  int64_t CurrentVBaseOffset) {
     for (method_iter mi = RD->method_begin(), me = RD->method_end(); mi != me;
-         ++mi)
-      if (mi->isVirtual())
-        AddMethod(*mi, MorallyVirtual, Offset, RDisVirtualBase);
+         ++mi) {
+      const CXXMethodDecl *MD = *mi;
+      if (!MD->isVirtual())
+        continue;
+      
+      if (const CXXDestructorDecl *DD = dyn_cast<CXXDestructorDecl>(MD)) {
+        // For destructors, add both the complete and the deleting destructor
+        // to the vtable.
+        AddMethod(GlobalDecl(DD, Dtor_Complete), MorallyVirtual, Offset, 
+                  RDisVirtualBase, CurrentVBaseOffset);
+        AddMethod(GlobalDecl(DD, Dtor_Deleting), MorallyVirtual, Offset, 
+                  RDisVirtualBase, CurrentVBaseOffset);
+      } else
+        AddMethod(MD, MorallyVirtual, Offset, RDisVirtualBase,
+                  CurrentVBaseOffset);
+    }
   }
 
   void NonVirtualBases(const CXXRecordDecl *RD, const ASTRecordLayout &Layout,
                        const CXXRecordDecl *PrimaryBase,
                        bool PrimaryBaseWasVirtual, bool MorallyVirtual,
-                       int64_t Offset, Path_t *Path) {
+                       int64_t Offset, int64_t CurrentVBaseOffset,
+                       Path_t *Path) {
     Path->push_back(std::make_pair(RD, Offset));
     for (CXXRecordDecl::base_class_const_iterator i = RD->bases_begin(),
            e = RD->bases_end(); i != e; ++i) {
@@ -402,8 +466,8 @@ public:
       if (Base != PrimaryBase || PrimaryBaseWasVirtual) {
         uint64_t o = Offset + Layout.getBaseClassOffset(Base);
         StartNewTable();
-        CurrentVBaseOffset = Offset;
-        GenerateVtableForBase(Base, MorallyVirtual, o, false, Path);
+        GenerateVtableForBase(Base, o, MorallyVirtual, false,
+                              CurrentVBaseOffset, Path);
       }
     }
     Path->pop_back();
@@ -424,11 +488,39 @@ public:
          i != e; ++i)
       methods[InsertionPoint++] = wrap((0?600:0) + *i);
     VCalls.clear();
+    VCall.clear();
   }
 
+  void AddAddressPoints(const CXXRecordDecl *RD, uint64_t Offset,
+                       Index_t AddressPoint) {
+    D1(printf("XXX address point for %s in %s layout %s at offset %d is %d\n",
+              RD->getNameAsCString(), Class->getNameAsCString(),
+              LayoutClass->getNameAsCString(), (int)Offset, (int)AddressPoint));
+    AddressPoints[std::make_pair(RD, Offset)] = AddressPoint;
+
+    // Now also add the address point for all our primary bases.
+    while (1) {
+      const ASTRecordLayout &Layout = CGM.getContext().getASTRecordLayout(RD);
+      RD = Layout.getPrimaryBase();
+      const bool PrimaryBaseWasVirtual = Layout.getPrimaryBaseWasVirtual();
+      // FIXME: Double check this.
+      if (RD == 0)
+        break;
+      if (PrimaryBaseWasVirtual &&
+          BLayout.getVBaseClassOffset(RD) != Offset)
+        break;
+      D1(printf("XXX address point for %s in %s layout %s at offset %d is %d\n",
+                RD->getNameAsCString(), Class->getNameAsCString(),
+                LayoutClass->getNameAsCString(), (int)Offset, (int)AddressPoint));
+      AddressPoints[std::make_pair(RD, Offset)] = AddressPoint;
+    }
+  }
+
+
   Index_t end(const CXXRecordDecl *RD, const ASTRecordLayout &Layout,
               const CXXRecordDecl *PrimaryBase, bool PrimaryBaseWasVirtual,
               bool MorallyVirtual, int64_t Offset, bool ForVirtualBase,
+              int64_t CurrentVBaseOffset,
               Path_t *Path) {
     bool alloc = false;
     if (Path == 0) {
@@ -446,7 +538,7 @@ public:
       // FIXME: just for extra, or for all uses of VCalls.size post this?
       extra = -VCalls.size();
 
-    methods.push_back(wrap(-(Offset/8)));
+    methods.push_back(wrap(-((Offset-LayoutOffset)/8)));
     methods.push_back(rtti);
     Index_t AddressPoint = methods.size();
 
@@ -457,13 +549,22 @@ public:
 
     // and then the non-virtual bases.
     NonVirtualBases(RD, Layout, PrimaryBase, PrimaryBaseWasVirtual,
-                    MorallyVirtual, Offset, Path);
+                    MorallyVirtual, Offset, CurrentVBaseOffset, Path);
 
     if (ForVirtualBase) {
-      insertVCalls(VCallInsertionPoint);
+      // FIXME: We're adding to VCalls in callers, we need to do the overrides
+      // in the inner part, so that we know the complete set of vcalls during
+      // the build and don't have to insert into methods.  Saving out the
+      // AddressPoint here, would need to be fixed, if we didn't do that.  Also
+      // retroactively adding vcalls for overrides later wind up in the wrong
+      // place, the vcall slot has to be alloted during the walk of the base
+      // when the function is first introduces.
       AddressPoint += VCalls.size();
+      insertVCalls(VCallInsertionPoint);
     }
     
+    AddAddressPoints(RD, Offset, AddressPoint);
+
     if (alloc) {
       delete Path;
     }
@@ -472,7 +573,7 @@ public:
 
   void Primaries(const CXXRecordDecl *RD, bool MorallyVirtual, int64_t Offset,
                  bool updateVBIndex, Index_t current_vbindex,
-                 bool RDisVirtualBase) {
+                 bool RDisVirtualBase, int64_t CurrentVBaseOffset) {
     if (!RD->isDynamicClass())
       return;
 
@@ -485,21 +586,27 @@ public:
       D1(printf(" doing primaries for %s most derived %s\n",
                 RD->getNameAsCString(), Class->getNameAsCString()));
       
+      int BaseCurrentVBaseOffset = CurrentVBaseOffset;
+      if (PrimaryBaseWasVirtual)
+        BaseCurrentVBaseOffset = BLayout.getVBaseClassOffset(PrimaryBase);
+        
       if (!PrimaryBaseWasVirtual)
         Primaries(PrimaryBase, PrimaryBaseWasVirtual|MorallyVirtual, Offset,
-                  updateVBIndex, current_vbindex, PrimaryBaseWasVirtual);
+                  updateVBIndex, current_vbindex, PrimaryBaseWasVirtual,
+                  BaseCurrentVBaseOffset);
     }
 
     D1(printf(" doing vcall entries for %s most derived %s\n",
               RD->getNameAsCString(), Class->getNameAsCString()));
 
     // And add the virtuals for the class to the primary vtable.
-    AddMethods(RD, MorallyVirtual, Offset, RDisVirtualBase);
+    AddMethods(RD, MorallyVirtual, Offset, RDisVirtualBase, CurrentVBaseOffset);
   }
 
   void VBPrimaries(const CXXRecordDecl *RD, bool MorallyVirtual, int64_t Offset,
                    bool updateVBIndex, Index_t current_vbindex,
-                   bool RDisVirtualBase, bool bottom=false) {
+                   bool RDisVirtualBase, int64_t CurrentVBaseOffset,
+                   bool bottom=false) {
     if (!RD->isDynamicClass())
       return;
 
@@ -509,14 +616,18 @@ public:
 
     // vtables are composed from the chain of primaries.
     if (PrimaryBase) {
-      if (PrimaryBaseWasVirtual)
+      int BaseCurrentVBaseOffset = CurrentVBaseOffset;
+      if (PrimaryBaseWasVirtual) {
         IndirectPrimary.insert(PrimaryBase);
+        BaseCurrentVBaseOffset = BLayout.getVBaseClassOffset(PrimaryBase);
+      }
 
       D1(printf(" doing primaries for %s most derived %s\n",
                 RD->getNameAsCString(), Class->getNameAsCString()));
       
       VBPrimaries(PrimaryBase, PrimaryBaseWasVirtual|MorallyVirtual, Offset,
-                  updateVBIndex, current_vbindex, PrimaryBaseWasVirtual);
+                  updateVBIndex, current_vbindex, PrimaryBaseWasVirtual,
+                  BaseCurrentVBaseOffset);
     }
 
     D1(printf(" doing vbase entries for %s most derived %s\n",
@@ -525,17 +636,23 @@ public:
 
     if (RDisVirtualBase || bottom) {
       Primaries(RD, MorallyVirtual, Offset, updateVBIndex, current_vbindex,
-                RDisVirtualBase);
+                RDisVirtualBase, CurrentVBaseOffset);
     }
   }
 
-  int64_t GenerateVtableForBase(const CXXRecordDecl *RD,
-                                bool MorallyVirtual = false, int64_t Offset = 0,
+  int64_t GenerateVtableForBase(const CXXRecordDecl *RD, int64_t Offset = 0,
+                                bool MorallyVirtual = false, 
                                 bool ForVirtualBase = false,
+                                int CurrentVBaseOffset = 0,
                                 Path_t *Path = 0) {
     if (!RD->isDynamicClass())
       return 0;
 
+    // Construction vtable don't need parts that have no virtual bases and
+    // aren't morally virtual.
+    if ((LayoutClass != Class) && RD->getNumVBases() == 0 && !MorallyVirtual)
+      return 0;
+
     const ASTRecordLayout &Layout = CGM.getContext().getASTRecordLayout(RD);
     const CXXRecordDecl *PrimaryBase = Layout.getPrimaryBase();
     const bool PrimaryBaseWasVirtual = Layout.getPrimaryBaseWasVirtual();
@@ -548,13 +665,13 @@ public:
       extra = VCalls.size();
 
     VBPrimaries(RD, MorallyVirtual, Offset, !ForVirtualBase, 0, ForVirtualBase,
-                true);
+                CurrentVBaseOffset, true);
 
     if (Path)
-      OverrideMethods(Path, MorallyVirtual, Offset);
+      OverrideMethods(Path, MorallyVirtual, Offset, CurrentVBaseOffset);
 
     return end(RD, Layout, PrimaryBase, PrimaryBaseWasVirtual, MorallyVirtual,
-               Offset, ForVirtualBase, Path);
+               Offset, ForVirtualBase, CurrentVBaseOffset, Path);
   }
 
   void GenerateVtableForVBases(const CXXRecordDecl *RD,
@@ -578,16 +695,21 @@ public:
         StartNewTable();
         VCall.clear();
         int64_t BaseOffset = BLayout.getVBaseClassOffset(Base);
-        CurrentVBaseOffset = BaseOffset;
+        int64_t CurrentVBaseOffset = BaseOffset;
         D1(printf("vtable %s virtual base %s\n",
                   Class->getNameAsCString(), Base->getNameAsCString()));
-        GenerateVtableForBase(Base, true, BaseOffset, true, Path);
+        GenerateVtableForBase(Base, BaseOffset, true, true, CurrentVBaseOffset,
+                              Path);
       }
-      int64_t BaseOffset = Offset;
+      int64_t BaseOffset;
       if (i->isVirtual())
         BaseOffset = BLayout.getVBaseClassOffset(Base);
+      else {
+        const ASTRecordLayout &Layout = CGM.getContext().getASTRecordLayout(RD);
+        BaseOffset = Offset + Layout.getBaseClassOffset(Base);
+      }
+        
       if (Base->getNumVBases()) {
-        CurrentVBaseOffset = BaseOffset;
         GenerateVtableForVBases(Base, BaseOffset, Path);
       }
     }
@@ -603,19 +725,17 @@ VtableBuilder::Index_t VtableBuilder::VBlookup(CXXRecordDecl *D,
   return CGM.getVtableInfo().getVirtualBaseOffsetIndex(D, B);
 }
 
-int64_t CGVtableInfo::getMethodVtableIndex(const CXXMethodDecl *MD) {
-  MD = MD->getCanonicalDecl();
-
-  MethodVtableIndicesTy::iterator I = MethodVtableIndices.find(MD);
+int64_t CGVtableInfo::getMethodVtableIndex(GlobalDecl GD) {
+  MethodVtableIndicesTy::iterator I = MethodVtableIndices.find(GD);
   if (I != MethodVtableIndices.end())
     return I->second;
   
-  const CXXRecordDecl *RD = MD->getParent();
+  const CXXRecordDecl *RD = cast<CXXMethodDecl>(GD.getDecl())->getParent();
   
   std::vector<llvm::Constant *> methods;
   // FIXME: This seems expensive.  Can we do a partial job to get
   // just this data.
-  VtableBuilder b(methods, RD, CGM);
+  VtableBuilder b(methods, RD, RD, 0, CGM);
   D1(printf("vtable %s\n", RD->getNameAsCString()));
   b.GenerateVtableForBase(RD);
   b.GenerateVtableForVBases(RD);
@@ -623,7 +743,7 @@ int64_t CGVtableInfo::getMethodVtableIndex(const CXXMethodDecl *MD) {
   MethodVtableIndices.insert(b.getIndex().begin(),
                              b.getIndex().end());
   
-  I = MethodVtableIndices.find(MD);
+  I = MethodVtableIndices.find(GD);
   assert(I != MethodVtableIndices.end() && "Did not find index!");
   return I->second;
 }
@@ -640,7 +760,7 @@ int64_t CGVtableInfo::getVirtualBaseOffsetIndex(const CXXRecordDecl *RD,
   std::vector<llvm::Constant *> methods;
   // FIXME: This seems expensive.  Can we do a partial job to get
   // just this data.
-  VtableBuilder b(methods, RD, CGM);
+  VtableBuilder b(methods, RD, RD, 0, CGM);
   D1(printf("vtable %s\n", RD->getNameAsCString()));
   b.GenerateVtableForBase(RD);
   b.GenerateVtableForVBases(RD);
@@ -659,10 +779,15 @@ int64_t CGVtableInfo::getVirtualBaseOffsetIndex(const CXXRecordDecl *RD,
   return I->second;
 }
 
-llvm::Value *CodeGenFunction::GenerateVtable(const CXXRecordDecl *RD) {
+llvm::Constant *CodeGenModule::GenerateVtable(const CXXRecordDecl *LayoutClass,
+                                              const CXXRecordDecl *RD,
+                                              uint64_t Offset) {
   llvm::SmallString<256> OutName;
   llvm::raw_svector_ostream Out(OutName);
-  mangleCXXVtable(CGM.getMangleContext(), RD, Out);
+  if (LayoutClass != RD)
+    mangleCXXCtorVtable(getMangleContext(), LayoutClass, Offset/8, RD, Out);
+  else
+    mangleCXXVtable(getMangleContext(), RD, Out);
 
   llvm::GlobalVariable::LinkageTypes linktype;
   linktype = llvm::GlobalValue::LinkOnceODRLinkage;
@@ -670,23 +795,245 @@ llvm::Value *CodeGenFunction::GenerateVtable(const CXXRecordDecl *RD) {
   llvm::Type *Ptr8Ty=llvm::PointerType::get(llvm::Type::getInt8Ty(VMContext),0);
   int64_t AddressPoint;
 
-  VtableBuilder b(methods, RD, CGM);
+  VtableBuilder b(methods, RD, LayoutClass, Offset, *this);
 
   D1(printf("vtable %s\n", RD->getNameAsCString()));
   // First comes the vtables for all the non-virtual bases...
-  AddressPoint = b.GenerateVtableForBase(RD);
+  AddressPoint = b.GenerateVtableForBase(RD, Offset);
 
   // then the vtables for all the virtual bases.
-  b.GenerateVtableForVBases(RD);
+  b.GenerateVtableForVBases(RD, Offset);
+
+  CodeGenModule::AddrMap_t *&ref = AddressPoints[LayoutClass];
+  if (ref == 0)
+    ref = new CodeGenModule::AddrMap_t;
+    
+  (*ref)[RD] = b.getAddressPoints();
 
   llvm::Constant *C;
   llvm::ArrayType *type = llvm::ArrayType::get(Ptr8Ty, methods.size());
   C = llvm::ConstantArray::get(type, methods);
-  llvm::Value *vtable = new llvm::GlobalVariable(CGM.getModule(), type, true,
-                                                 linktype, C, Out.str());
-  vtable = Builder.CreateBitCast(vtable, Ptr8Ty);
-  vtable = Builder.CreateGEP(vtable,
-                       llvm::ConstantInt::get(llvm::Type::getInt64Ty(VMContext),
-                                              AddressPoint*LLVMPointerWidth/8));
+  llvm::GlobalVariable *GV = new llvm::GlobalVariable(getModule(), type,
+                                                      true, linktype, C,
+                                                      Out.str());
+  bool Hidden = getDeclVisibilityMode(RD) == LangOptions::Hidden;
+  if (Hidden)
+    GV->setVisibility(llvm::GlobalVariable::HiddenVisibility);
+  llvm::Constant *vtable = llvm::ConstantExpr::getBitCast(GV, Ptr8Ty);
+  llvm::Constant *AddressPointC;
+  uint32_t LLVMPointerWidth = getContext().Target.getPointerWidth(0);
+  AddressPointC = llvm::ConstantInt::get(llvm::Type::getInt64Ty(VMContext),
+                                         AddressPoint*LLVMPointerWidth/8);
+  vtable = llvm::ConstantExpr::getInBoundsGetElementPtr(vtable, &AddressPointC,
+                                                        1);
+
   return vtable;
 }
+
+class VTTBuilder {
+  /// Inits - The list of values built for the VTT.
+  std::vector<llvm::Constant *> &Inits;
+  /// Class - The most derived class that this vtable is being built for.
+  const CXXRecordDecl *Class;
+  CodeGenModule &CGM;  // Per-module state.
+  llvm::SmallSet<const CXXRecordDecl *, 32> SeenVBase;
+  /// BLayout - Layout for the most derived class that this vtable is being
+  /// built for.
+  const ASTRecordLayout &BLayout;
+  CodeGenModule::AddrMap_t &AddressPoints;
+  // vtbl - A pointer to the vtable for Class.
+  llvm::Constant *ClassVtbl;
+  llvm::LLVMContext &VMContext;
+
+  /// BuildVtablePtr - Build up a referene to the given secondary vtable
+  llvm::Constant *BuildVtablePtr(llvm::Constant *vtbl,
+                                 const CXXRecordDecl *VtblClass,
+                                 const CXXRecordDecl *RD,
+                                 uint64_t Offset) {
+    int64_t AddressPoint;
+    AddressPoint = (*AddressPoints[VtblClass])[std::make_pair(RD, Offset)];    
+    // FIXME: We can never have 0 address point.  Do this for now so gepping
+    // retains the same structure.
+    if (AddressPoint == 0)
+      AddressPoint = 1;
+    D1(printf("XXX address point for %s in %s layout %s at offset %d was %d\n",
+              RD->getNameAsCString(), VtblClass->getNameAsCString(),
+              Class->getNameAsCString(), (int)Offset, (int)AddressPoint));
+    uint32_t LLVMPointerWidth = CGM.getContext().Target.getPointerWidth(0);
+    llvm::Constant *init;
+    init = llvm::ConstantInt::get(llvm::Type::getInt64Ty(VMContext),
+                                  AddressPoint*LLVMPointerWidth/8);
+    init = llvm::ConstantExpr::getInBoundsGetElementPtr(vtbl, &init, 1);
+    return init;
+  }
+
+  /// Secondary - Add the secondary vtable pointers to Inits.  Offset is the
+  /// current offset in bits to the object we're working on.
+  void Secondary(const CXXRecordDecl *RD, llvm::Constant *vtbl,
+                 const CXXRecordDecl *VtblClass, uint64_t Offset=0,
+                 bool MorallyVirtual=false) {
+    if (RD->getNumVBases() == 0 && ! MorallyVirtual)
+      return;
+
+    for (CXXRecordDecl::base_class_const_iterator i = RD->bases_begin(),
+           e = RD->bases_end(); i != e; ++i) {
+      const CXXRecordDecl *Base =
+        cast<CXXRecordDecl>(i->getType()->getAs<RecordType>()->getDecl());
+      const ASTRecordLayout &Layout = CGM.getContext().getASTRecordLayout(RD);
+      const CXXRecordDecl *PrimaryBase = Layout.getPrimaryBase();
+      const bool PrimaryBaseWasVirtual = Layout.getPrimaryBaseWasVirtual();
+      bool NonVirtualPrimaryBase;
+      NonVirtualPrimaryBase = !PrimaryBaseWasVirtual && Base == PrimaryBase;
+      bool BaseMorallyVirtual = MorallyVirtual | i->isVirtual();
+      uint64_t BaseOffset;
+      if (!i->isVirtual()) {
+        const ASTRecordLayout &Layout = CGM.getContext().getASTRecordLayout(RD);
+        BaseOffset = Offset + Layout.getBaseClassOffset(Base);
+      } else
+        BaseOffset = BLayout.getVBaseClassOffset(Base);
+      llvm::Constant *subvtbl = vtbl;
+      const CXXRecordDecl *subVtblClass = VtblClass;
+      if ((Base->getNumVBases() || BaseMorallyVirtual)
+          && !NonVirtualPrimaryBase) {
+        // FIXME: Slightly too many of these for __ZTT8test8_B2
+        llvm::Constant *init;
+        if (BaseMorallyVirtual)
+          init = BuildVtablePtr(vtbl, VtblClass, RD, Offset);
+        else {
+          init = CGM.getVtableInfo().getCtorVtable(Class, Base, BaseOffset);
+          subvtbl = dyn_cast<llvm::Constant>(init->getOperand(0));
+          subVtblClass = Base;
+        }
+        Inits.push_back(init);
+      }
+      Secondary(Base, subvtbl, subVtblClass, BaseOffset, BaseMorallyVirtual);
+    }
+  }
+
+  /// BuiltVTT - Add the VTT to Inits.  Offset is the offset in bits to the
+  /// currnet object we're working on.
+  void BuildVTT(const CXXRecordDecl *RD, uint64_t Offset, bool MorallyVirtual) {
+    if (RD->getNumVBases() == 0 && !MorallyVirtual)
+      return;
+
+    llvm::Constant *init;
+    const CXXRecordDecl *VtblClass;
+
+    // First comes the primary virtual table pointer...
+    if (MorallyVirtual) {
+      init = BuildVtablePtr(ClassVtbl, Class, RD, Offset);
+      VtblClass = Class;
+    } else {
+      init = CGM.getVtableInfo().getCtorVtable(Class, RD, Offset);
+      VtblClass = RD;
+    }
+    llvm::Constant *vtbl = dyn_cast<llvm::Constant>(init->getOperand(0));
+    Inits.push_back(init);
+
+    // then the secondary VTTs....
+    SecondaryVTTs(RD, Offset, MorallyVirtual);
+
+    // and last the secondary vtable pointers.
+    Secondary(RD, vtbl, VtblClass, Offset, MorallyVirtual);
+  }
+
+  /// SecondaryVTTs - Add the secondary VTTs to Inits.  The secondary VTTs are
+  /// built from each direct non-virtual proper base that requires a VTT in
+  /// declaration order.
+  void SecondaryVTTs(const CXXRecordDecl *RD, uint64_t Offset=0,
+                     bool MorallyVirtual=false) {
+    for (CXXRecordDecl::base_class_const_iterator i = RD->bases_begin(),
+           e = RD->bases_end(); i != e; ++i) {
+      const CXXRecordDecl *Base =
+        cast<CXXRecordDecl>(i->getType()->getAs<RecordType>()->getDecl());
+      if (i->isVirtual())
+        continue;
+      const ASTRecordLayout &Layout = CGM.getContext().getASTRecordLayout(RD);
+      uint64_t BaseOffset = Offset + Layout.getBaseClassOffset(Base);
+      BuildVTT(Base, BaseOffset, MorallyVirtual);
+    }
+  }
+
+  /// VirtualVTTs - Add the VTT for each proper virtual base in inheritance
+  /// graph preorder.
+  void VirtualVTTs(const CXXRecordDecl *RD) {
+    for (CXXRecordDecl::base_class_const_iterator i = RD->bases_begin(),
+           e = RD->bases_end(); i != e; ++i) {
+      const CXXRecordDecl *Base =
+        cast<CXXRecordDecl>(i->getType()->getAs<RecordType>()->getDecl());
+      if (i->isVirtual() && !SeenVBase.count(Base)) {
+        SeenVBase.insert(Base);
+        uint64_t BaseOffset = BLayout.getVBaseClassOffset(Base);
+        BuildVTT(Base, BaseOffset, true);
+      }
+      VirtualVTTs(Base);
+    }
+  }
+public:
+  VTTBuilder(std::vector<llvm::Constant *> &inits, const CXXRecordDecl *c,
+             CodeGenModule &cgm)
+    : Inits(inits), Class(c), CGM(cgm),
+      BLayout(cgm.getContext().getASTRecordLayout(c)),
+      AddressPoints(*cgm.AddressPoints[c]),
+      VMContext(cgm.getModule().getContext()) {
+    
+    // First comes the primary virtual table pointer for the complete class...
+    ClassVtbl = CGM.getVtableInfo().getVtable(Class);
+    Inits.push_back(ClassVtbl);
+    ClassVtbl = dyn_cast<llvm::Constant>(ClassVtbl->getOperand(0));
+    
+    // then the secondary VTTs...
+    SecondaryVTTs(Class);
+
+    // then the secondary vtable pointers...
+    Secondary(Class, ClassVtbl, Class);
+
+    // and last, the virtual VTTs.
+    VirtualVTTs(Class);
+  }
+};
+
+llvm::Constant *CodeGenModule::GenerateVTT(const CXXRecordDecl *RD) {
+  // Only classes that have virtual bases need a VTT.
+  if (RD->getNumVBases() == 0)
+    return 0;
+
+  llvm::SmallString<256> OutName;
+  llvm::raw_svector_ostream Out(OutName);
+  mangleCXXVTT(getMangleContext(), RD, Out);
+
+  llvm::GlobalVariable::LinkageTypes linktype;
+  linktype = llvm::GlobalValue::LinkOnceODRLinkage;
+  std::vector<llvm::Constant *> inits;
+  llvm::Type *Ptr8Ty=llvm::PointerType::get(llvm::Type::getInt8Ty(VMContext),0);
+
+  D1(printf("vtt %s\n", RD->getNameAsCString()));
+
+  VTTBuilder b(inits, RD, *this);
+
+  llvm::Constant *C;
+  llvm::ArrayType *type = llvm::ArrayType::get(Ptr8Ty, inits.size());
+  C = llvm::ConstantArray::get(type, inits);
+  llvm::GlobalVariable *vtt = new llvm::GlobalVariable(getModule(), type, true,
+                                                 linktype, C, Out.str());
+  bool Hidden = getDeclVisibilityMode(RD) == LangOptions::Hidden;
+  if (Hidden)
+    vtt->setVisibility(llvm::GlobalVariable::HiddenVisibility);
+  return llvm::ConstantExpr::getBitCast(vtt, Ptr8Ty);
+}
+
+llvm::Constant *CGVtableInfo::getVtable(const CXXRecordDecl *RD) {
+  llvm::Constant *&vtbl = Vtables[RD];
+  if (vtbl)
+    return vtbl;
+  vtbl = CGM.GenerateVtable(RD, RD);
+  CGM.GenerateRtti(RD);
+  CGM.GenerateVTT(RD);
+  return vtbl;
+}
+
+llvm::Constant *CGVtableInfo::getCtorVtable(const CXXRecordDecl *LayoutClass,
+                                            const CXXRecordDecl *RD,
+                                            uint64_t Offset) {
+  return CGM.GenerateVtable(LayoutClass, RD, Offset);
+}
diff --git a/lib/CodeGen/CGVtable.h b/lib/CodeGen/CGVtable.h
index 69fb1f100599..f9ddf44284b1 100644
--- a/lib/CodeGen/CGVtable.h
+++ b/lib/CodeGen/CGVtable.h
@@ -15,6 +15,7 @@
 #define CLANG_CODEGEN_CGVTABLE_H
 
 #include "llvm/ADT/DenseMap.h"
+#include "GlobalDecl.h"
 
 namespace clang {
   class CXXMethodDecl;
@@ -28,7 +29,7 @@ class CGVtableInfo {
   
   /// MethodVtableIndices - Contains the index (relative to the vtable address
   /// point) where the function pointer for a virtual function is stored.
-  typedef llvm::DenseMap<const CXXMethodDecl *, int64_t> MethodVtableIndicesTy;
+  typedef llvm::DenseMap<GlobalDecl, int64_t> MethodVtableIndicesTy;
   MethodVtableIndicesTy MethodVtableIndices;
   
   typedef std::pair<const CXXRecordDecl *,
@@ -38,6 +39,8 @@ class CGVtableInfo {
   /// offsets for virtual bases of a class are stored.
   typedef llvm::DenseMap<ClassPairTy, int64_t> VirtualBaseClassIndiciesTy;
   VirtualBaseClassIndiciesTy VirtualBaseClassIndicies;
+
+  llvm::DenseMap<const CXXRecordDecl *, llvm::Constant *> Vtables;
 public:
   CGVtableInfo(CodeGenModule &CGM) 
     : CGM(CGM) { }
@@ -45,7 +48,7 @@ public:
   /// getMethodVtableIndex - Return the index (relative to the vtable address
   /// point) where the function pointer for the given virtual function is 
   /// stored.
-  int64_t getMethodVtableIndex(const CXXMethodDecl *MD);
+  int64_t getMethodVtableIndex(GlobalDecl GD);
   
   /// getVirtualBaseOffsetIndex - Return the index (relative to the vtable
   /// address point) where the offset of the virtual base that contains the
@@ -54,6 +57,10 @@ public:
   /// base.
   int64_t getVirtualBaseOffsetIndex(const CXXRecordDecl *RD, 
                                     const CXXRecordDecl *VBase);
+
+  llvm::Constant *getVtable(const CXXRecordDecl *RD);
+  llvm::Constant *getCtorVtable(const CXXRecordDecl *RD,
+                                const CXXRecordDecl *Class, uint64_t Offset);
 };
   
 }
diff --git a/lib/CodeGen/CodeGenFunction.cpp b/lib/CodeGen/CodeGenFunction.cpp
index 4be341311ce4..475c7bfefd1d 100644
--- a/lib/CodeGen/CodeGenFunction.cpp
+++ b/lib/CodeGen/CodeGenFunction.cpp
@@ -27,11 +27,7 @@ CodeGenFunction::CodeGenFunction(CodeGenModule &cgm)
   : BlockFunction(cgm, *this, Builder), CGM(cgm),
     Target(CGM.getContext().Target),
     Builder(cgm.getModule().getContext()),
-#ifndef USEINDIRECTBRANCH
-    DebugInfo(0), IndirectGotoSwitch(0),
-#else
     DebugInfo(0), IndirectBranch(0),
-#endif
     SwitchInsn(0), CaseRangeBlock(0), InvokeDest(0),
     CXXThisDecl(0) {
   LLVMIntTy = ConvertType(getContext().IntTy);
@@ -134,7 +130,6 @@ void CodeGenFunction::FinishFunction(SourceLocation EndLoc) {
 
   EmitFunctionEpilog(*CurFnInfo, ReturnValue);
 
-#ifdef USEINDIRECTBRANCH
   // If someone did an indirect goto, emit the indirect goto block at the end of
   // the function.
   if (IndirectBranch) {
@@ -142,13 +137,10 @@ void CodeGenFunction::FinishFunction(SourceLocation EndLoc) {
     Builder.ClearInsertionPoint();
   }
   
-  
-#endif
   // Remove the AllocaInsertPt instruction, which is just a convenience for us.
   llvm::Instruction *Ptr = AllocaInsertPt;
   AllocaInsertPt = 0;
   Ptr->eraseFromParent();
-#ifdef USEINDIRECTBRANCH
   
   // If someone took the address of a label but never did an indirect goto, we
   // made a zero entry PHI node, which is illegal, zap it now.
@@ -159,8 +151,6 @@ void CodeGenFunction::FinishFunction(SourceLocation EndLoc) {
       PN->eraseFromParent();
     }
   }
-  
-#endif
 }
 
 void CodeGenFunction::StartFunction(GlobalDecl GD, QualType RetTy,
@@ -260,19 +250,21 @@ void CodeGenFunction::GenerateCode(GlobalDecl GD,
   // FIXME: Support CXXTryStmt here, too.
   if (const CompoundStmt *S = FD->getCompoundBody()) {
     StartFunction(GD, FD->getResultType(), Fn, Args, S->getLBracLoc());
-    const CXXDestructorDecl *DD = dyn_cast<CXXDestructorDecl>(FD);
-    llvm::BasicBlock *DtorEpilogue = 0;
-    if (DD) {
-      DtorEpilogue = createBasicBlock("dtor.epilogue");
-    
-      PushCleanupBlock(DtorEpilogue);
-    }
-    
-    if (const CXXConstructorDecl *CD = dyn_cast<CXXConstructorDecl>(FD))
+
+    if (const CXXConstructorDecl *CD = dyn_cast<CXXConstructorDecl>(FD)) {
       EmitCtorPrologue(CD, GD.getCtorType());
-    EmitStmt(S);
+      EmitStmt(S);
+      
+      // If any of the member initializers are temporaries bound to references
+      // make sure to emit their destructors.
+      EmitCleanupBlocks(0);
+      
+    } else if (const CXXDestructorDecl *DD = dyn_cast<CXXDestructorDecl>(FD)) {
+      llvm::BasicBlock *DtorEpilogue  = createBasicBlock("dtor.epilogue");
+      PushCleanupBlock(DtorEpilogue);
+
+      EmitStmt(S);
       
-    if (DD) {
       CleanupBlockInfo Info = PopCleanupBlock();
 
       assert(Info.CleanupBlock == DtorEpilogue && "Block mismatch!");
@@ -283,7 +275,11 @@ void CodeGenFunction::GenerateCode(GlobalDecl GD,
         EmitBlock(Info.SwitchBlock);
       if (Info.EndBlock)
         EmitBlock(Info.EndBlock);
+    } else {
+      // Just a regular function, emit its body.
+      EmitStmt(S);
     }
+    
     FinishFunction(S->getRBracLoc());
   } else if (FD->isImplicit()) {
     const CXXRecordDecl *ClassDecl =
@@ -493,121 +489,32 @@ void CodeGenFunction::EmitMemSetToZero(llvm::Value *DestPtr, QualType Ty) {
                                              TypeInfo.second/8));
 }
 
-#ifndef USEINDIRECTBRANCH
-unsigned CodeGenFunction::GetIDForAddrOfLabel(const LabelStmt *L) {
-  // Use LabelIDs.size()+1 as the new ID if one hasn't been assigned.
-  unsigned &Entry = LabelIDs[L];
-  if (Entry) return Entry;
-#else
-
 llvm::BlockAddress *CodeGenFunction::GetAddrOfLabel(const LabelStmt *L) {
   // Make sure that there is a block for the indirect goto.
   if (IndirectBranch == 0)
     GetIndirectGotoBlock();
-#endif
   
-#ifndef USEINDIRECTBRANCH
-  Entry = LabelIDs.size();
-#else
   llvm::BasicBlock *BB = getBasicBlockForLabel(L);
-#endif
-  
-#ifndef USEINDIRECTBRANCH
-  // If this is the first "address taken" of a label and the indirect goto has
-  // already been seen, add this to it.
-  if (IndirectGotoSwitch) {
-    // If this is the first address-taken label, set it as the default dest.
-    if (Entry == 1)
-      IndirectGotoSwitch->setSuccessor(0, getBasicBlockForLabel(L));
-    else {
-      // Otherwise add it to the switch as a new dest.
-      const llvm::IntegerType *Int32Ty = llvm::Type::getInt32Ty(VMContext);
-      IndirectGotoSwitch->addCase(llvm::ConstantInt::get(Int32Ty, Entry),
-                                  getBasicBlockForLabel(L));
-    }
-  }
   
-  return Entry;
-#else
   // Make sure the indirect branch includes all of the address-taken blocks.
   IndirectBranch->addDestination(BB);
   return llvm::BlockAddress::get(CurFn, BB);
-#endif
 }
 
 llvm::BasicBlock *CodeGenFunction::GetIndirectGotoBlock() {
-#ifndef USEINDIRECTBRANCH
-  // If we already made the switch stmt for indirect goto, return its block.
-  if (IndirectGotoSwitch) return IndirectGotoSwitch->getParent();
-#else
   // If we already made the indirect branch for indirect goto, return its block.
   if (IndirectBranch) return IndirectBranch->getParent();
-#endif
   
-#ifndef USEINDIRECTBRANCH
-  EmitBlock(createBasicBlock("indirectgoto"));
-#else
   CGBuilderTy TmpBuilder(createBasicBlock("indirectgoto"));
-#endif
   
-#ifndef USEINDIRECTBRANCH
-  const llvm::IntegerType *Int32Ty = llvm::Type::getInt32Ty(VMContext);
-#else
   const llvm::Type *Int8PtrTy = llvm::Type::getInt8PtrTy(VMContext);
-#endif
 
   // Create the PHI node that indirect gotos will add entries to.
-#ifndef USEINDIRECTBRANCH
-  llvm::Value *DestVal = Builder.CreatePHI(Int32Ty, "indirect.goto.dest");
-#else
   llvm::Value *DestVal = TmpBuilder.CreatePHI(Int8PtrTy, "indirect.goto.dest");
-#endif
-  
-#ifndef USEINDIRECTBRANCH
-  // Create the switch instruction.  For now, set the insert block to this block
-  // which will be fixed as labels are added.
-  IndirectGotoSwitch = Builder.CreateSwitch(DestVal, Builder.GetInsertBlock());
-  
-  // Clear the insertion point to indicate we are in unreachable code.
-  Builder.ClearInsertionPoint();
   
-  // If we already have labels created, add them.
-  if (!LabelIDs.empty()) {
-    // Invert LabelID's so that the order is determinstic.
-    std::vector<const LabelStmt*> AddrTakenLabelsByID;
-    AddrTakenLabelsByID.resize(LabelIDs.size());
-    
-    for (std::map<const LabelStmt*,unsigned>::iterator 
-         LI = LabelIDs.begin(), LE = LabelIDs.end(); LI != LE; ++LI) {
-      assert(LI->second-1 < AddrTakenLabelsByID.size() &&
-             "Numbering inconsistent");
-      AddrTakenLabelsByID[LI->second-1] = LI->first;
-    }
-    
-    // Set the default entry as the first block.
-    IndirectGotoSwitch->setSuccessor(0,
-                                getBasicBlockForLabel(AddrTakenLabelsByID[0]));
-    
-    // FIXME: The iteration order of this is nondeterminstic!
-    for (unsigned i = 1, e = AddrTakenLabelsByID.size(); i != e; ++i)
-      IndirectGotoSwitch->addCase(llvm::ConstantInt::get(Int32Ty, i+1),
-                                 getBasicBlockForLabel(AddrTakenLabelsByID[i]));
-  } else {
-    // Otherwise, create a dead block and set it as the default dest.  This will
-    // be removed by the optimizers after the indirect goto is set up.
-    llvm::BasicBlock *Dummy = createBasicBlock("indgoto.dummy");
-    EmitBlock(Dummy);
-    IndirectGotoSwitch->setSuccessor(0, Dummy);
-    Builder.CreateUnreachable();
-    Builder.ClearInsertionPoint();
-  }
-
-  return IndirectGotoSwitch->getParent();
-#else
   // Create the indirect branch instruction.
   IndirectBranch = TmpBuilder.CreateIndirectBr(DestVal);
   return IndirectBranch->getParent();
-#endif
 }
 
 llvm::Value *CodeGenFunction::GetVLASize(const VariableArrayType *VAT) {
diff --git a/lib/CodeGen/CodeGenFunction.h b/lib/CodeGen/CodeGenFunction.h
index fe8113e95332..d96c3551010e 100644
--- a/lib/CodeGen/CodeGenFunction.h
+++ b/lib/CodeGen/CodeGenFunction.h
@@ -74,7 +74,7 @@ class CodeGenFunction : public BlockFunction {
   void operator=(const CodeGenFunction&);  // DO NOT IMPLEMENT
 public:
   CodeGenModule &CGM;  // Per-module state.
-  TargetInfo &Target;
+  const TargetInfo &Target;
 
   typedef std::pair<llvm::Value *, llvm::Value *> ComplexPairTy;
   CGBuilderTy Builder;
@@ -193,28 +193,12 @@ public:
 private:
   CGDebugInfo *DebugInfo;
 
-#ifndef USEINDIRECTBRANCH
-  /// LabelIDs - Track arbitrary ids assigned to labels for use in implementing
-  /// the GCC address-of-label extension and indirect goto. IDs are assigned to
-  /// labels inside getIDForAddrOfLabel().
-  std::map<const LabelStmt*, unsigned> LabelIDs;
-#else
   /// IndirectBranch - The first time an indirect goto is seen we create a
   /// block with an indirect branch.  Every time we see the address of a label
   /// taken, we add the label to the indirect goto.  Every subsequent indirect
   /// goto is codegen'd as a jump to the IndirectBranch's basic block.
   llvm::IndirectBrInst *IndirectBranch;
-#endif
-
-#ifndef USEINDIRECTBRANCH
-  /// IndirectGotoSwitch - The first time an indirect goto is seen we create a
-  /// block with the switch for the indirect gotos.  Every time we see the
-  /// address of a label taken, we add the label to the indirect goto.  Every
-  /// subsequent indirect goto is codegen'd as a jump to the
-  /// IndirectGotoSwitch's basic block.
-  llvm::SwitchInst *IndirectGotoSwitch;
 
-#endif
   /// LocalDeclMap - This keeps track of the LLVM allocas or globals for local C
   /// decls.
   llvm::DenseMap<const Decl*, llvm::Value*> LocalDeclMap;
@@ -397,9 +381,6 @@ public:
   /// legal to call this function even if there is no current insertion point.
   void FinishFunction(SourceLocation EndLoc=SourceLocation());
 
-  /// GenerateVtable - Generate the vtable for the given type.
-  llvm::Value *GenerateVtable(const CXXRecordDecl *RD);
-
   /// DynamicTypeAdjust - Do the non-virtual and virtual adjustments on an
   /// object pointer to alter the dynamic type of the pointer.  Used by
   /// GenerateCovariantThunk for building thunks.
@@ -586,11 +567,7 @@ public:
   /// the input field number being accessed.
   static unsigned getAccessedFieldNo(unsigned Idx, const llvm::Constant *Elts);
 
-#ifndef USEINDIRECTBRANCH
-  unsigned GetIDForAddrOfLabel(const LabelStmt *L);
-#else
   llvm::BlockAddress *GetAddrOfLabel(const LabelStmt *L);
-#endif
   llvm::BasicBlock *GetIndirectGotoBlock();
 
   /// EmitMemSetToZero - Generate code to memset a value of the given type to 0.
@@ -669,6 +646,14 @@ public:
                                  const ArrayType *Array,
                                  llvm::Value *This);
 
+  void EmitCXXAggrDestructorCall(const CXXDestructorDecl *D,
+                                 llvm::Value *NumElements,
+                                 llvm::Value *This);
+
+  llvm::Constant * GenerateCXXAggrDestructorHelper(const CXXDestructorDecl *D,
+                                                const ArrayType *Array,
+                                                llvm::Value *This);
+
   void EmitCXXDestructorCall(const CXXDestructorDecl *D, CXXDtorType Type,
                              llvm::Value *This);
 
@@ -678,6 +663,12 @@ public:
   llvm::Value *EmitCXXNewExpr(const CXXNewExpr *E);
   void EmitCXXDeleteExpr(const CXXDeleteExpr *E);
 
+  void EmitDeleteCall(const FunctionDecl *DeleteFD, llvm::Value *Ptr,
+                      QualType DeleteTy);
+
+  llvm::Value* EmitCXXTypeidExpr(const CXXTypeidExpr *E);
+  llvm::Value *EmitDynamicCast(llvm::Value *V, const CXXDynamicCastExpr *DCE);
+
   //===--------------------------------------------------------------------===//
   //                            Declaration Emission
   //===--------------------------------------------------------------------===//
@@ -851,17 +842,18 @@ public:
   LValue EmitConditionalOperatorLValue(const ConditionalOperator *E);
   LValue EmitCastLValue(const CastExpr *E);
   LValue EmitNullInitializationLValue(const CXXZeroInitValueExpr *E);
-  LValue EmitPointerToDataMemberLValue(const DeclRefExpr *E);
+  
+  LValue EmitPointerToDataMemberLValue(const FieldDecl *Field);
   
   llvm::Value *EmitIvarOffset(const ObjCInterfaceDecl *Interface,
                               const ObjCIvarDecl *Ivar);
-  LValue EmitLValueForField(llvm::Value* Base, FieldDecl* Field,
+  LValue EmitLValueForField(llvm::Value* Base, const FieldDecl* Field,
                             bool isUnion, unsigned CVRQualifiers);
   LValue EmitLValueForIvar(QualType ObjectTy,
                            llvm::Value* Base, const ObjCIvarDecl *Ivar,
                            unsigned CVRQualifiers);
 
-  LValue EmitLValueForBitfield(llvm::Value* Base, FieldDecl* Field,
+  LValue EmitLValueForBitfield(llvm::Value* Base, const FieldDecl* Field,
                                 unsigned CVRQualifiers);
 
   LValue EmitBlockDeclRefLValue(const BlockDeclRefExpr *E);
@@ -870,6 +862,7 @@ public:
   LValue EmitCXXConstructLValue(const CXXConstructExpr *E);
   LValue EmitCXXBindTemporaryLValue(const CXXBindTemporaryExpr *E);
   LValue EmitCXXExprWithTemporariesLValue(const CXXExprWithTemporaries *E);
+  LValue EmitCXXTypeidLValue(const CXXTypeidExpr *E);
   
   LValue EmitObjCMessageExprLValue(const ObjCMessageExpr *E);
   LValue EmitObjCIvarRefLValue(const ObjCIvarRefExpr *E);
@@ -901,8 +894,11 @@ public:
                   const Decl *TargetDecl = 0);
   RValue EmitCallExpr(const CallExpr *E);
 
-  llvm::Value *BuildVirtualCall(const CXXMethodDecl *MD, llvm::Value *&This,
+  llvm::Value *BuildVirtualCall(const CXXMethodDecl *MD, llvm::Value *This,
                                 const llvm::Type *Ty);
+  llvm::Value *BuildVirtualCall(const CXXDestructorDecl *DD, CXXDtorType Type, 
+                                llvm::Value *&This, const llvm::Type *Ty);
+
   RValue EmitCXXMemberCall(const CXXMethodDecl *MD,
                            llvm::Value *Callee,
                            llvm::Value *This,
@@ -1016,7 +1012,7 @@ public:
 
   /// EmitCXXGlobalDtorRegistration - Emits a call to register the global ptr
   /// with the C++ runtime so that its destructor will be called at exit.
-  void EmitCXXGlobalDtorRegistration(const CXXDestructorDecl *Dtor,
+  void EmitCXXGlobalDtorRegistration(llvm::Constant *DtorFn,
                                      llvm::Constant *DeclPtr);
 
   /// GenerateCXXGlobalInitFunc - Generates code for initializing global
@@ -1033,7 +1029,7 @@ public:
                                     bool IsInitializer = false);
 
   void EmitCXXThrowExpr(const CXXThrowExpr *E);
-  
+
   //===--------------------------------------------------------------------===//
   //                             Internal Helpers
   //===--------------------------------------------------------------------===//
@@ -1101,6 +1097,7 @@ private:
     if (CallArgTypeInfo) {
       for (typename T::arg_type_iterator I = CallArgTypeInfo->arg_type_begin(),
            E = CallArgTypeInfo->arg_type_end(); I != E; ++I, ++Arg) {
+        assert(Arg != ArgEnd && "Running over edge of argument list!");
         QualType ArgType = *I;
 
         assert(getContext().getCanonicalType(ArgType.getNonReferenceType()).
diff --git a/lib/CodeGen/CodeGenModule.cpp b/lib/CodeGen/CodeGenModule.cpp
index db609f62453c..0e6f4a63a42c 100644
--- a/lib/CodeGen/CodeGenModule.cpp
+++ b/lib/CodeGen/CodeGenModule.cpp
@@ -17,7 +17,7 @@
 #include "CGCall.h"
 #include "CGObjCRuntime.h"
 #include "Mangle.h"
-#include "clang/Frontend/CompileOptions.h"
+#include "clang/CodeGen/CodeGenOptions.h"
 #include "clang/AST/ASTContext.h"
 #include "clang/AST/DeclObjC.h"
 #include "clang/AST/DeclCXX.h"
@@ -30,15 +30,16 @@
 #include "llvm/Module.h"
 #include "llvm/Intrinsics.h"
 #include "llvm/Target/TargetData.h"
+#include "llvm/Support/ErrorHandling.h"
 using namespace clang;
 using namespace CodeGen;
 
 
-CodeGenModule::CodeGenModule(ASTContext &C, const CompileOptions &compileOpts,
+CodeGenModule::CodeGenModule(ASTContext &C, const CodeGenOptions &CGO,
                              llvm::Module &M, const llvm::TargetData &TD,
                              Diagnostic &diags)
   : BlockModule(C, M, TD, Types, *this), Context(C),
-    Features(C.getLangOptions()), CompileOpts(compileOpts), TheModule(M),
+    Features(C.getLangOptions()), CodeGenOpts(CGO), TheModule(M),
     TheTargetData(TD), Diags(diags), Types(C, M, TD), MangleCtx(C), 
     VtableInfo(*this), Runtime(0),
     MemCpyFn(0), MemMoveFn(0), MemSetFn(0), CFConstantStringClassRef(0),
@@ -54,7 +55,7 @@ CodeGenModule::CodeGenModule(ASTContext &C, const CompileOptions &compileOpts,
     Runtime = CreateMacObjCRuntime(*this);
 
   // If debug info generation is enabled, create the CGDebugInfo object.
-  DebugInfo = CompileOpts.DebugInfo ? new CGDebugInfo(this) : 0;
+  DebugInfo = CodeGenOpts.DebugInfo ? new CGDebugInfo(this) : 0;
 }
 
 CodeGenModule::~CodeGenModule() {
@@ -347,6 +348,11 @@ void CodeGenModule::SetLLVMFunctionAttributesForDefinition(const Decl *D,
   if (D->hasAttr<NoInlineAttr>())
     F->addFnAttr(llvm::Attribute::NoInline);
 
+  if (Features.getStackProtectorMode() == LangOptions::SSPOn)
+    F->addFnAttr(llvm::Attribute::StackProtect);
+  else if (Features.getStackProtectorMode() == LangOptions::SSPReq)
+    F->addFnAttr(llvm::Attribute::StackProtectReq);
+  
   if (const AlignedAttr *AA = D->getAttr<AlignedAttr>())
     F->setAlignment(AA->getAlignment()/8);
   // C++ ABI requires 2-byte alignment for member functions.
@@ -543,10 +549,15 @@ bool CodeGenModule::MayDeferGeneration(const ValueDecl *Global) {
       
   // Static data may be deferred, but out-of-line static data members
   // cannot be.
-  // FIXME: What if the initializer has side effects?
-  return VD->isInAnonymousNamespace() ||
-         (VD->getStorageClass() == VarDecl::Static &&
-          !(VD->isStaticDataMember() && VD->isOutOfLine()));
+  if (VD->isInAnonymousNamespace())
+    return true;
+  if (VD->getStorageClass() == VarDecl::Static) {
+    // Initializer has side effects?
+    if (VD->getInit() && VD->getInit()->HasSideEffects(Context))
+      return false;
+    return !(VD->isStaticDataMember() && VD->isOutOfLine());
+  }
+  return false;
 }
 
 void CodeGenModule::EmitGlobal(GlobalDecl GD) {
@@ -639,6 +650,24 @@ llvm::Constant *CodeGenModule::GetOrCreateLLVMFunction(const char *MangledName,
     return llvm::ConstantExpr::getBitCast(Entry, PTy);
   }
 
+  // This function doesn't have a complete type (for example, the return
+  // type is an incomplete struct). Use a fake type instead, and make
+  // sure not to try to set attributes.
+  bool IsIncompleteFunction = false;
+  if (!isa<llvm::FunctionType>(Ty)) {
+    Ty = llvm::FunctionType::get(llvm::Type::getVoidTy(VMContext),
+                                 std::vector<const llvm::Type*>(), false);
+    IsIncompleteFunction = true;
+  }
+  llvm::Function *F = llvm::Function::Create(cast<llvm::FunctionType>(Ty),
+                                             llvm::Function::ExternalLinkage,
+                                             "", &getModule());
+  F->setName(MangledName);
+  if (D.getDecl())
+    SetFunctionAttributes(cast<FunctionDecl>(D.getDecl()), F,
+                          IsIncompleteFunction);
+  Entry = F;
+
   // This is the first use or definition of a mangled name.  If there is a
   // deferred decl with this name, remember that we need to emit it at the end
   // of the file.
@@ -672,23 +701,6 @@ llvm::Constant *CodeGenModule::GetOrCreateLLVMFunction(const char *MangledName,
              DeferredCopyAssignmentToEmit(D);
   }
 
-  // This function doesn't have a complete type (for example, the return
-  // type is an incomplete struct). Use a fake type instead, and make
-  // sure not to try to set attributes.
-  bool IsIncompleteFunction = false;
-  if (!isa<llvm::FunctionType>(Ty)) {
-    Ty = llvm::FunctionType::get(llvm::Type::getVoidTy(VMContext),
-                                 std::vector<const llvm::Type*>(), false);
-    IsIncompleteFunction = true;
-  }
-  llvm::Function *F = llvm::Function::Create(cast<llvm::FunctionType>(Ty),
-                                             llvm::Function::ExternalLinkage,
-                                             "", &getModule());
-  F->setName(MangledName);
-  if (D.getDecl())
-    SetFunctionAttributes(cast<FunctionDecl>(D.getDecl()), F,
-                          IsIncompleteFunction);
-  Entry = F;
   return F;
 }
 
@@ -1075,7 +1087,7 @@ void CodeGenModule::EmitGlobalVarDefinition(const VarDecl *D) {
       GV->setLinkage(llvm::GlobalVariable::WeakAnyLinkage);
   } else if (Linkage == GVA_TemplateInstantiation)
     GV->setLinkage(llvm::GlobalVariable::WeakAnyLinkage);   
-  else if (!CompileOpts.NoCommon &&
+  else if (!CodeGenOpts.NoCommon &&
            !D->hasExternalStorage() && !D->getInit() &&
            !D->getAttr<SectionAttr>()) {
     GV->setLinkage(llvm::GlobalVariable::CommonLinkage);
@@ -1548,7 +1560,14 @@ std::string CodeGenModule::GetStringForStringLiteral(const StringLiteral *E) {
 llvm::Constant *
 CodeGenModule::GetAddrOfConstantStringFromLiteral(const StringLiteral *S) {
   // FIXME: This can be more efficient.
-  return GetAddrOfConstantString(GetStringForStringLiteral(S));
+  // FIXME: We shouldn't need to bitcast the constant in the wide string case.
+  llvm::Constant *C = GetAddrOfConstantString(GetStringForStringLiteral(S));
+  if (S->isWide()) {
+    llvm::Type *DestTy =
+        llvm::PointerType::getUnqual(getTypes().ConvertType(S->getType()));
+    C = llvm::ConstantExpr::getBitCast(C, DestTy);
+  }
+  return C;
 }
 
 /// GetAddrOfConstantStringFromObjCEncode - Return a pointer to a constant
@@ -1697,6 +1716,7 @@ void CodeGenModule::EmitTopLevelDecl(Decl *D) {
     EmitNamespace(cast<NamespaceDecl>(D));
     break;
     // No code generation needed.
+  case Decl::UsingShadow:
   case Decl::Using:
   case Decl::UsingDirective:
   case Decl::ClassTemplate:
diff --git a/lib/CodeGen/CodeGenModule.h b/lib/CodeGen/CodeGenModule.h
index 2e58337ee52d..c8562d6745eb 100644
--- a/lib/CodeGen/CodeGenModule.h
+++ b/lib/CodeGen/CodeGenModule.h
@@ -23,6 +23,7 @@
 #include "CGCXX.h"
 #include "CGVtable.h"
 #include "CodeGenTypes.h"
+#include "GlobalDecl.h"
 #include "Mangle.h"
 #include "llvm/Module.h"
 #include "llvm/ADT/DenseMap.h"
@@ -31,8 +32,6 @@
 #include "llvm/Support/ValueHandle.h"
 #include <list>
 
-#define ATTACH_DEBUG_INFO_TO_AN_INSN 1 
-
 namespace llvm {
   class Module;
   class Constant;
@@ -61,7 +60,7 @@ namespace clang {
   class ValueDecl;
   class VarDecl;
   class LangOptions;
-  class CompileOptions;
+  class CodeGenOptions;
   class Diagnostic;
   class AnnotateAttr;
   class CXXDestructorDecl;
@@ -72,46 +71,7 @@ namespace CodeGen {
   class CGDebugInfo;
   class CGObjCRuntime;
 
-/// GlobalDecl - represents a global declaration. This can either be a
-/// CXXConstructorDecl and the constructor type (Base, Complete).
-/// a CXXDestructorDecl and the destructor type (Base, Complete) or
-/// a VarDecl, a FunctionDecl or a BlockDecl.
-class GlobalDecl {
-  llvm::PointerIntPair<const Decl*, 2> Value;
-
-  void Init(const Decl *D) {
-    assert(!isa<CXXConstructorDecl>(D) && "Use other ctor with ctor decls!");
-    assert(!isa<CXXDestructorDecl>(D) && "Use other ctor with dtor decls!");
-
-    Value.setPointer(D);
-  }
   
-public:
-  GlobalDecl() {}
-
-  GlobalDecl(const VarDecl *D) { Init(D);}
-  GlobalDecl(const FunctionDecl *D) { Init(D); }
-  GlobalDecl(const BlockDecl *D) { Init(D); }
-  GlobalDecl(const ObjCMethodDecl *D) { Init(D); }
-
-  GlobalDecl(const CXXConstructorDecl *D, CXXCtorType Type)
-  : Value(D, Type) {}
-  GlobalDecl(const CXXDestructorDecl *D, CXXDtorType Type)
-  : Value(D, Type) {}
-
-  const Decl *getDecl() const { return Value.getPointer(); }
-
-  CXXCtorType getCtorType() const {
-    assert(isa<CXXConstructorDecl>(getDecl()) && "Decl is not a ctor!");
-    return static_cast<CXXCtorType>(Value.getInt());
-  }
-
-  CXXDtorType getDtorType() const {
-    assert(isa<CXXDestructorDecl>(getDecl()) && "Decl is not a dtor!");
-    return static_cast<CXXDtorType>(Value.getInt());
-  }
-};
-
 /// CodeGenModule - This class organizes the cross-function state that is used
 /// while generating LLVM code.
 class CodeGenModule : public BlockModule {
@@ -122,7 +82,7 @@ class CodeGenModule : public BlockModule {
 
   ASTContext &Context;
   const LangOptions &Features;
-  const CompileOptions &CompileOpts;
+  const CodeGenOptions &CodeGenOpts;
   llvm::Module &TheModule;
   const llvm::TargetData &TheTargetData;
   Diagnostic &Diags;
@@ -201,7 +161,7 @@ class CodeGenModule : public BlockModule {
 
   llvm::LLVMContext &VMContext;
 public:
-  CodeGenModule(ASTContext &C, const CompileOptions &CompileOpts,
+  CodeGenModule(ASTContext &C, const CodeGenOptions &CodeGenOpts,
                 llvm::Module &M, const llvm::TargetData &TD, Diagnostic &Diags);
 
   ~CodeGenModule();
@@ -222,7 +182,7 @@ public:
 
   CGDebugInfo *getDebugInfo() { return DebugInfo; }
   ASTContext &getContext() const { return Context; }
-  const CompileOptions &getCompileOpts() const { return CompileOpts; }
+  const CodeGenOptions &getCodeGenOpts() const { return CodeGenOpts; }
   const LangOptions &getLangOptions() const { return Features; }
   llvm::Module &getModule() const { return TheModule; }
   CodeGenTypes &getTypes() { return Types; }
@@ -252,8 +212,26 @@ public:
   llvm::Constant *GetAddrOfFunction(GlobalDecl GD,
                                     const llvm::Type *Ty = 0);
 
+  /// GenerateVtable - Generate the vtable for the given type.  LayoutClass is
+  /// the class to use for the virtual base layout information.  For
+  /// non-construction vtables, this is always the same as RD.  Offset is the
+  /// offset in bits for the RD object in the LayoutClass, if we're generating a
+  /// construction vtable, otherwise 0.
+  llvm::Constant *GenerateVtable(const CXXRecordDecl *LayoutClass,
+                                 const CXXRecordDecl *RD,
+                                 uint64_t Offset=0);
+
+  /// GenerateVTT - Generate the VTT for the given type.
+  llvm::Constant *GenerateVTT(const CXXRecordDecl *RD);
+
   /// GenerateRtti - Generate the rtti information for the given type.
   llvm::Constant *GenerateRtti(const CXXRecordDecl *RD);
+  /// GenerateRttiRef - Generate a reference to the rtti information for the
+  /// given type.
+  llvm::Constant *GenerateRttiRef(const CXXRecordDecl *RD);
+  /// GenerateRttiNonClass - Generate the rtti information for the given
+  /// non-class type.
+  llvm::Constant *GenerateRttiNonClass(QualType Ty);
 
   /// BuildThunk - Build a thunk for the given method
   llvm::Constant *BuildThunk(const CXXMethodDecl *MD, bool Extern, int64_t nv,
@@ -263,6 +241,11 @@ public:
                                       int64_t nv_t, int64_t v_t,
                                       int64_t nv_r, int64_t v_r);
 
+  typedef std::pair<const CXXRecordDecl *, uint64_t> CtorVtable_t;
+  typedef llvm::DenseMap<const CXXRecordDecl *,
+                         llvm::DenseMap<CtorVtable_t, int64_t>*> AddrMap_t;
+  llvm::DenseMap<const CXXRecordDecl *, AddrMap_t*> AddressPoints;
+
   /// GetCXXBaseClassOffset - Returns the offset from a derived class to its
   /// base class. Returns null if the offset is 0.
   llvm::Constant *GetCXXBaseClassOffset(const CXXRecordDecl *ClassDecl,
@@ -499,7 +482,7 @@ private:
 
   /// EmitCXXGlobalInitFunc - Emit a function that initializes C++ globals.
   void EmitCXXGlobalInitFunc();
-
+  
   // FIXME: Hardcoding priority here is gross.
   void AddGlobalCtor(llvm::Function *Ctor, int Priority=65535);
   void AddGlobalDtor(llvm::Function *Dtor, int Priority=65535);
diff --git a/lib/CodeGen/CodeGenTypes.cpp b/lib/CodeGen/CodeGenTypes.cpp
index d43d13e26bbb..1f83f37e0479 100644
--- a/lib/CodeGen/CodeGenTypes.cpp
+++ b/lib/CodeGen/CodeGenTypes.cpp
@@ -492,7 +492,7 @@ const CGRecordLayout &
 CodeGenTypes::getCGRecordLayout(const TagDecl *TD) const {
   const Type *Key =
     Context.getTagDeclType(TD).getTypePtr();
-  llvm::DenseMap<const Type*, CGRecordLayout *>::iterator I
+  llvm::DenseMap<const Type*, CGRecordLayout *>::const_iterator I
     = CGRecordLayouts.find(Key);
   assert (I != CGRecordLayouts.end()
           && "Unable to find record layout information for type");
diff --git a/lib/CodeGen/CodeGenTypes.h b/lib/CodeGen/CodeGenTypes.h
index a92a019b988e..f447549f669c 100644
--- a/lib/CodeGen/CodeGenTypes.h
+++ b/lib/CodeGen/CodeGenTypes.h
@@ -91,7 +91,7 @@ namespace CodeGen {
 /// while lowering AST types to LLVM types.
 class CodeGenTypes {
   ASTContext &Context;
-  TargetInfo &Target;
+  const TargetInfo &Target;
   llvm::Module& TheModule;
   const llvm::TargetData& TheTargetData;
   mutable const ABIInfo* TheABIInfo;
@@ -153,7 +153,7 @@ public:
   ~CodeGenTypes();
 
   const llvm::TargetData &getTargetData() const { return TheTargetData; }
-  TargetInfo &getTarget() const { return Target; }
+  const TargetInfo &getTarget() const { return Target; }
   ASTContext &getContext() const { return Context; }
   const ABIInfo &getABIInfo() const;
   llvm::LLVMContext &getLLVMContext() { return TheModule.getContext(); }
diff --git a/lib/CodeGen/GlobalDecl.h b/lib/CodeGen/GlobalDecl.h
new file mode 100644
index 000000000000..b812020f2aa4
--- /dev/null
+++ b/lib/CodeGen/GlobalDecl.h
@@ -0,0 +1,110 @@
+//===--- GlobalDecl.h - Global declaration holder ---------------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// A GlobalDecl can hold either a regular variable/function or a C++ ctor/dtor
+// together with its type.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef CLANG_CODEGEN_GLOBALDECL_H
+#define CLANG_CODEGEN_GLOBALDECL_H
+
+namespace clang {
+
+namespace CodeGen {
+
+/// GlobalDecl - represents a global declaration. This can either be a
+/// CXXConstructorDecl and the constructor type (Base, Complete).
+/// a CXXDestructorDecl and the destructor type (Base, Complete) or
+/// a VarDecl, a FunctionDecl or a BlockDecl.
+class GlobalDecl {
+  llvm::PointerIntPair<const Decl*, 2> Value;
+
+  void Init(const Decl *D) {
+    assert(!isa<CXXConstructorDecl>(D) && "Use other ctor with ctor decls!");
+    assert(!isa<CXXDestructorDecl>(D) && "Use other ctor with dtor decls!");
+
+    Value.setPointer(D);
+  }
+  
+public:
+  GlobalDecl() {}
+
+  GlobalDecl(const VarDecl *D) { Init(D);}
+  GlobalDecl(const FunctionDecl *D) { Init(D); }
+  GlobalDecl(const BlockDecl *D) { Init(D); }
+  GlobalDecl(const ObjCMethodDecl *D) { Init(D); }
+
+  GlobalDecl(const CXXConstructorDecl *D, CXXCtorType Type)
+  : Value(D, Type) {}
+  GlobalDecl(const CXXDestructorDecl *D, CXXDtorType Type)
+  : Value(D, Type) {}
+
+  const Decl *getDecl() const { return Value.getPointer(); }
+
+  CXXCtorType getCtorType() const {
+    assert(isa<CXXConstructorDecl>(getDecl()) && "Decl is not a ctor!");
+    return static_cast<CXXCtorType>(Value.getInt());
+  }
+
+  CXXDtorType getDtorType() const {
+    assert(isa<CXXDestructorDecl>(getDecl()) && "Decl is not a dtor!");
+    return static_cast<CXXDtorType>(Value.getInt());
+  }
+  
+  friend bool operator==(const GlobalDecl &LHS, const GlobalDecl &RHS) {
+    return LHS.Value == RHS.Value;
+  }
+  
+  void *getAsOpaquePtr() const { return Value.getOpaqueValue(); }
+
+  static GlobalDecl getFromOpaquePtr(void *P) {
+    GlobalDecl GD;
+    GD.Value.setFromOpaqueValue(P);
+    return GD;
+  }
+};
+
+} // end namespace CodeGen
+} // end namespace clang
+
+namespace llvm {
+  template<class> struct DenseMapInfo;
+
+  template<> struct DenseMapInfo<clang::CodeGen::GlobalDecl> {
+    static inline clang::CodeGen::GlobalDecl getEmptyKey() {
+      return clang::CodeGen::GlobalDecl();
+    }
+  
+    static inline clang::CodeGen::GlobalDecl getTombstoneKey() {
+      return clang::CodeGen::GlobalDecl::
+        getFromOpaquePtr(reinterpret_cast<void*>(-1));
+    }
+
+    static unsigned getHashValue(clang::CodeGen::GlobalDecl GD) {
+      return DenseMapInfo<void*>::getHashValue(GD.getAsOpaquePtr());
+    }
+    
+    static bool isEqual(clang::CodeGen::GlobalDecl LHS, 
+                        clang::CodeGen::GlobalDecl RHS) {
+      return LHS == RHS;
+    }
+      
+    static bool isPod() {
+      // GlobalDecl isn't *technically* a POD type. However, we can get
+      // away with calling it a POD type since its copy constructor,
+      // copy assignment operator, and destructor are all trivial.
+      return true;
+    }
+      
+  };
+  
+}
+
+#endif
diff --git a/lib/CodeGen/Mangle.cpp b/lib/CodeGen/Mangle.cpp
index a5b34527969b..0a7124de3621 100644
--- a/lib/CodeGen/Mangle.cpp
+++ b/lib/CodeGen/Mangle.cpp
@@ -53,7 +53,10 @@ namespace {
 
     void mangleCXXVtable(const CXXRecordDecl *RD);
     void mangleCXXVTT(const CXXRecordDecl *RD);
+    void mangleCXXCtorVtable(const CXXRecordDecl *RD, int64_t Offset,
+                             const CXXRecordDecl *Type);
     void mangleCXXRtti(QualType Ty);
+    void mangleCXXRttiName(QualType Ty);
     void mangleCXXCtor(const CXXConstructorDecl *D, CXXCtorType Type);
     void mangleCXXDtor(const CXXDestructorDecl *D, CXXDtorType Type);
 
@@ -65,6 +68,8 @@ namespace {
     bool mangleStandardSubstitution(const NamedDecl *ND);
     
     void addSubstitution(const NamedDecl *ND) {
+      ND = cast<NamedDecl>(ND->getCanonicalDecl());
+
       addSubstitution(reinterpret_cast<uintptr_t>(ND));
     }
     void addSubstitution(QualType T);
@@ -127,8 +132,10 @@ static bool isInCLinkageSpecification(const Decl *D) {
 
 bool CXXNameMangler::mangleFunctionDecl(const FunctionDecl *FD) {
   // Clang's "overloadable" attribute extension to C/C++ implies name mangling
-  // (always).
-  if (!FD->hasAttr<OverloadableAttr>()) {
+  // (always) as does passing a C++ member function and a function
+  // whose name is not a simple identifier.
+  if (!FD->hasAttr<OverloadableAttr>() && !isa<CXXMethodDecl>(FD) &&
+      FD->getDeclName().isIdentifier()) {
     // C functions are not mangled, and "main" is never mangled.
     if (!Context.getASTContext().getLangOptions().CPlusPlus || FD->isMain())
       return false;
@@ -140,7 +147,7 @@ bool CXXNameMangler::mangleFunctionDecl(const FunctionDecl *FD) {
       return false;
 
     // No name mangling in a C linkage specification.
-    if (!isa<CXXMethodDecl>(FD) && isInCLinkageSpecification(FD))
+    if (isInCLinkageSpecification(FD))
       return false;
   }
 
@@ -212,6 +219,17 @@ void CXXNameMangler::mangleCXXVTT(const CXXRecordDecl *RD) {
   mangleName(RD);
 }
 
+void CXXNameMangler::mangleCXXCtorVtable(const CXXRecordDecl *RD,
+                                         int64_t Offset,
+                                         const CXXRecordDecl *Type) {
+  // <special-name> ::= TC <type> <offset number> _ <base type>
+  Out << "_ZTC";
+  mangleName(RD);
+  Out << Offset;
+  Out << "_";
+  mangleName(Type);
+}
+
 void CXXNameMangler::mangleCXXRtti(QualType Ty) {
   // <special-name> ::= TI <type>  # typeinfo structure
   Out << "_ZTI";
@@ -219,6 +237,13 @@ void CXXNameMangler::mangleCXXRtti(QualType Ty) {
   mangleType(Ty);
 }
 
+void CXXNameMangler::mangleCXXRttiName(QualType Ty) {
+  // <special-name> ::= TS <type>  # typeinfo name (null terminated byte string)
+  Out << "_ZTS";
+
+  mangleType(Ty);
+}
+
 void CXXNameMangler::mangleGuardVariable(const VarDecl *D) {
   //  <special-name> ::= GV <object name>       # Guard variable for one-time
   //                                            # initialization
@@ -719,15 +744,15 @@ void CXXNameMangler::mangleType(QualType T) {
   // Only operate on the canonical type!
   T = Context.getASTContext().getCanonicalType(T);
 
-  bool IsSubstitutable = T.hasQualifiers() || !isa<BuiltinType>(T);
+  bool IsSubstitutable = T.hasLocalQualifiers() || !isa<BuiltinType>(T);
   if (IsSubstitutable && mangleSubstitution(T))
     return;
 
-  if (Qualifiers Quals = T.getQualifiers()) {
+  if (Qualifiers Quals = T.getLocalQualifiers()) {
     mangleQualifiers(Quals);
     // Recurse:  even if the qualified type isn't yet substitutable,
     // the unqualified type might be.
-    mangleType(T.getUnqualifiedType());
+    mangleType(T.getLocalUnqualifiedType());
   } else {
     switch (T->getTypeClass()) {
 #define ABSTRACT_TYPE(CLASS, PARENT)
@@ -1015,6 +1040,11 @@ void CXXNameMangler::mangleExpression(const Expr *E) {
 	//              ::= <expr-primary>
   switch (E->getStmtClass()) {
   default: assert(false && "Unhandled expression kind!");
+
+  case Expr::ParenExprClass:
+    mangleExpression(cast<ParenExpr>(E)->getSubExpr());
+    break;
+
   case Expr::DeclRefExprClass: {
     const Decl *D = cast<DeclRefExpr>(E)->getDecl();
     
@@ -1169,6 +1199,7 @@ bool CXXNameMangler::mangleSubstitution(const NamedDecl *ND) {
   if (mangleStandardSubstitution(ND))
     return true;
   
+  ND = cast<NamedDecl>(ND->getCanonicalDecl());
   return mangleSubstitution(reinterpret_cast<uintptr_t>(ND));
 }
 
@@ -1255,6 +1286,8 @@ static bool isCharSpecialization(QualType T, const char *Name) {
 
 bool CXXNameMangler::mangleStandardSubstitution(const NamedDecl *ND) {
   // <substitution> ::= St # ::std::
+  // FIXME: type_info == comes out as __ZNK3std9type_infoeqERKS0_ instead of
+  // __ZNKSt9type_infoeqERKS_
   if (const NamespaceDecl *NS = dyn_cast<NamespaceDecl>(ND)) {
     if (isStdNamespace(NS)) {
       Out << "St";
@@ -1433,6 +1466,23 @@ namespace clang {
     os.flush();
   }
 
+  void mangleCXXVTT(MangleContext &Context, const CXXRecordDecl *RD,
+                       llvm::raw_ostream &os) {
+    CXXNameMangler Mangler(Context, os);
+    Mangler.mangleCXXVTT(RD);
+
+    os.flush();
+  }
+
+  void mangleCXXCtorVtable(MangleContext &Context, const CXXRecordDecl *RD,
+                           int64_t Offset, const CXXRecordDecl *Type,
+                           llvm::raw_ostream &os) {
+    CXXNameMangler Mangler(Context, os);
+    Mangler.mangleCXXCtorVtable(RD, Offset, Type);
+
+    os.flush();
+  }
+
   void mangleCXXRtti(MangleContext &Context, QualType Ty,
                      llvm::raw_ostream &os) {
     CXXNameMangler Mangler(Context, os);
@@ -1440,4 +1490,12 @@ namespace clang {
 
     os.flush();
   }
+
+  void mangleCXXRttiName(MangleContext &Context, QualType Ty,
+                         llvm::raw_ostream &os) {
+    CXXNameMangler Mangler(Context, os);
+    Mangler.mangleCXXRttiName(Ty);
+
+    os.flush();
+  }
 }
diff --git a/lib/CodeGen/Mangle.h b/lib/CodeGen/Mangle.h
index 7f46a106f32a..458708fca6ad 100644
--- a/lib/CodeGen/Mangle.h
+++ b/lib/CodeGen/Mangle.h
@@ -67,8 +67,13 @@ namespace clang {
                        llvm::raw_ostream &os);
   void mangleCXXVTT(MangleContext &Context, const CXXRecordDecl *RD,
                        llvm::raw_ostream &os);
+  void mangleCXXCtorVtable(MangleContext &Context, const CXXRecordDecl *RD,
+                           int64_t Offset, const CXXRecordDecl *Type,
+                       llvm::raw_ostream &os);
   void mangleCXXRtti(MangleContext &Context, QualType T, 
                      llvm::raw_ostream &os);
+  void mangleCXXRttiName(MangleContext &Context, QualType T, 
+                         llvm::raw_ostream &os);
   void mangleCXXCtor(MangleContext &Context, const CXXConstructorDecl *D, 
                      CXXCtorType Type, llvm::raw_ostream &os);
   void mangleCXXDtor(MangleContext &Context, const CXXDestructorDecl *D, 
diff --git a/lib/CodeGen/ModuleBuilder.cpp b/lib/CodeGen/ModuleBuilder.cpp
index c8f686a06f50..1d8f31dd9b24 100644
--- a/lib/CodeGen/ModuleBuilder.cpp
+++ b/lib/CodeGen/ModuleBuilder.cpp
@@ -13,7 +13,7 @@
 
 #include "clang/CodeGen/ModuleBuilder.h"
 #include "CodeGenModule.h"
-#include "clang/Frontend/CompileOptions.h"
+#include "clang/CodeGen/CodeGenOptions.h"
 #include "clang/AST/ASTContext.h"
 #include "clang/AST/DeclObjC.h"
 #include "clang/AST/Expr.h"
@@ -32,14 +32,14 @@ namespace {
     Diagnostic &Diags;
     llvm::OwningPtr<const llvm::TargetData> TD;
     ASTContext *Ctx;
-    const CompileOptions CompileOpts;  // Intentionally copied in.
+    const CodeGenOptions CodeGenOpts;  // Intentionally copied in.
   protected:
     llvm::OwningPtr<llvm::Module> M;
     llvm::OwningPtr<CodeGen::CodeGenModule> Builder;
   public:
     CodeGeneratorImpl(Diagnostic &diags, const std::string& ModuleName,
-                      const CompileOptions &CO, llvm::LLVMContext& C)
-      : Diags(diags), CompileOpts(CO), M(new llvm::Module(ModuleName, C)) {}
+                      const CodeGenOptions &CGO, llvm::LLVMContext& C)
+      : Diags(diags), CodeGenOpts(CGO), M(new llvm::Module(ModuleName, C)) {}
 
     virtual ~CodeGeneratorImpl() {}
 
@@ -57,7 +57,7 @@ namespace {
       M->setTargetTriple(Ctx->Target.getTriple().getTriple());
       M->setDataLayout(Ctx->Target.getTargetDescription());
       TD.reset(new llvm::TargetData(Ctx->Target.getTargetDescription()));
-      Builder.reset(new CodeGen::CodeGenModule(Context, CompileOpts,
+      Builder.reset(new CodeGen::CodeGenModule(Context, CodeGenOpts,
                                                *M, *TD, Diags));
     }
 
@@ -96,7 +96,7 @@ namespace {
 
 CodeGenerator *clang::CreateLLVMCodeGen(Diagnostic &Diags,
                                         const std::string& ModuleName,
-                                        const CompileOptions &CO,
+                                        const CodeGenOptions &CGO,
                                         llvm::LLVMContext& C) {
-  return new CodeGeneratorImpl(Diags, ModuleName, CO, C);
+  return new CodeGeneratorImpl(Diags, ModuleName, CGO, C);
 }
diff --git a/lib/CodeGen/TargetABIInfo.cpp b/lib/CodeGen/TargetABIInfo.cpp
index 852bba4ef033..ba0bc6668e27 100644
--- a/lib/CodeGen/TargetABIInfo.cpp
+++ b/lib/CodeGen/TargetABIInfo.cpp
@@ -168,8 +168,28 @@ static bool is32Or64BitBasicType(QualType Ty, ASTContext &Context) {
   return Size == 32 || Size == 64;
 }
 
-static bool areAllFields32Or64BitBasicType(const RecordDecl *RD,
-                                           ASTContext &Context) {
+/// canExpandIndirectArgument - Test whether an argument type which is to be
+/// passed indirectly (on the stack) would have the equivalent layout if it was
+/// expanded into separate arguments. If so, we prefer to do the latter to avoid
+/// inhibiting optimizations.
+///
+// FIXME: This predicate is missing many cases, currently it just follows
+// llvm-gcc (checks that all fields are 32-bit or 64-bit primitive types). We
+// should probably make this smarter, or better yet make the LLVM backend
+// capable of handling it.
+static bool canExpandIndirectArgument(QualType Ty, ASTContext &Context) {
+  // We can only expand structure types.
+  const RecordType *RT = Ty->getAs<RecordType>();
+  if (!RT)
+    return false;
+
+  // We can only expand (C) structures.
+  //
+  // FIXME: This needs to be generalized to handle classes as well.
+  const RecordDecl *RD = RT->getDecl();
+  if (!RD->isStruct() || isa<CXXRecordDecl>(RD))
+    return false;
+
   for (RecordDecl::field_iterator i = RD->field_begin(), e = RD->field_end();
          i != e; ++i) {
     const FieldDecl *FD = *i;
@@ -442,14 +462,13 @@ ABIArgInfo X86_32ABIInfo::classifyArgumentType(QualType Ty,
     if (Ty->isStructureType() && Context.getTypeSize(Ty) == 0)
       return ABIArgInfo::getIgnore();
 
-    // Expand structs with size <= 128-bits which consist only of
-    // basic types (int, long long, float, double, xxx*). This is
-    // non-recursive and does not ignore empty fields.
-    if (const RecordType *RT = Ty->getAsStructureType()) {
-      if (Context.getTypeSize(Ty) <= 4*32 &&
-          areAllFields32Or64BitBasicType(RT->getDecl(), Context))
-        return ABIArgInfo::getExpand();
-    }
+    // Expand small (<= 128-bit) record types when we know that the stack layout
+    // of those arguments will match the struct. This is important because the
+    // LLVM backend isn't smart enough to remove byval, which inhibits many
+    // optimizations.
+    if (Context.getTypeSize(Ty) <= 4*32 &&
+        canExpandIndirectArgument(Ty, Context))
+      return ABIArgInfo::getExpand();
 
     return ABIArgInfo::getIndirect(getIndirectArgumentAlignment(Ty, Context));
   } else {