aboutsummaryrefslogtreecommitdiffstats
path: root/lib/Lex/TokenLexer.cpp
diff options
context:
space:
mode:
authorDimitry Andric <dim@FreeBSD.org>2013-12-22 00:07:40 +0000
committerDimitry Andric <dim@FreeBSD.org>2013-12-22 00:07:40 +0000
commitbfef399519ca9b8a4b4c6b563253bad7e0eeffe0 (patch)
treedf8df0b0067b381eab470a3b8f28d14a552a6340 /lib/Lex/TokenLexer.cpp
parent6a0372513edbc473b538d2f724efac50405d6fef (diff)
downloadsrc-bfef399519ca9b8a4b4c6b563253bad7e0eeffe0.tar.gz
src-bfef399519ca9b8a4b4c6b563253bad7e0eeffe0.zip
Vendor import of clang release_34 branch r197841 (effectively, 3.4 RC3):vendor/clang/clang-release_34-r197841
Notes
Notes: svn path=/vendor/clang/dist/; revision=259701 svn path=/vendor/clang/clang-release_34-r197841/; revision=259703; tag=vendor/clang/clang-release_34-r197841
Diffstat (limited to 'lib/Lex/TokenLexer.cpp')
-rw-r--r--lib/Lex/TokenLexer.cpp61
1 files changed, 39 insertions, 22 deletions
diff --git a/lib/Lex/TokenLexer.cpp b/lib/Lex/TokenLexer.cpp
index 07753c7d7c36..0213afcee921 100644
--- a/lib/Lex/TokenLexer.cpp
+++ b/lib/Lex/TokenLexer.cpp
@@ -121,7 +121,7 @@ void TokenLexer::destroy() {
/// Remove comma ahead of __VA_ARGS__, if present, according to compiler dialect
/// settings. Returns true if the comma is removed.
-static bool MaybeRemoveCommaBeforeVaArgs(SmallVector<Token, 128> &ResultToks,
+static bool MaybeRemoveCommaBeforeVaArgs(SmallVectorImpl<Token> &ResultToks,
bool &NextTokGetsSpace,
bool HasPasteOperator,
MacroInfo *Macro, unsigned MacroArgNo,
@@ -244,9 +244,11 @@ void TokenLexer::ExpandFunctionArguments() {
// Otherwise, this is a use of the argument. Find out if there is a paste
// (##) operator before or after the argument.
- bool PasteBefore =
+ bool NonEmptyPasteBefore =
!ResultToks.empty() && ResultToks.back().is(tok::hashhash);
+ bool PasteBefore = i != 0 && Tokens[i-1].is(tok::hashhash);
bool PasteAfter = i+1 != e && Tokens[i+1].is(tok::hashhash);
+ assert(!NonEmptyPasteBefore || PasteBefore);
// In Microsoft mode, remove the comma before __VA_ARGS__ to ensure there
// are no trailing commas if __VA_ARGS__ is empty.
@@ -276,6 +278,14 @@ void TokenLexer::ExpandFunctionArguments() {
unsigned NumToks = MacroArgs::getArgLength(ResultArgToks);
ResultToks.append(ResultArgToks, ResultArgToks+NumToks);
+ // In Microsoft-compatibility mode, we follow MSVC's preprocessing
+ // behavior by not considering single commas from nested macro
+ // expansions as argument separators. Set a flag on the token so we can
+ // test for this later when the macro expansion is processed.
+ if (PP.getLangOpts().MicrosoftMode && NumToks == 1 &&
+ ResultToks.back().is(tok::comma))
+ ResultToks.back().setFlag(Token::IgnoredComma);
+
// If the '##' came from expanding an argument, turn it into 'unknown'
// to avoid pasting.
for (unsigned i = FirstResult, e = ResultToks.size(); i != e; ++i) {
@@ -314,13 +324,12 @@ void TokenLexer::ExpandFunctionArguments() {
// that __VA_ARGS__ expands to multiple tokens, avoid a pasting error when
// the expander trys to paste ',' with the first token of the __VA_ARGS__
// expansion.
- if (PasteBefore && ResultToks.size() >= 2 &&
+ if (NonEmptyPasteBefore && ResultToks.size() >= 2 &&
ResultToks[ResultToks.size()-2].is(tok::comma) &&
(unsigned)ArgNo == Macro->getNumArgs()-1 &&
Macro->isVariadic()) {
// Remove the paste operator, report use of the extension.
- PP.Diag(ResultToks.back().getLocation(), diag::ext_paste_comma);
- ResultToks.pop_back();
+ PP.Diag(ResultToks.pop_back_val().getLocation(), diag::ext_paste_comma);
}
ResultToks.append(ArgToks, ArgToks+NumToks);
@@ -350,7 +359,7 @@ void TokenLexer::ExpandFunctionArguments() {
// case, we do not want the extra whitespace to be added. For example,
// we want ". ## foo" -> ".foo" not ". foo".
if ((CurTok.hasLeadingSpace() || NextTokGetsSpace) &&
- !PasteBefore)
+ !NonEmptyPasteBefore)
ResultToks[ResultToks.size()-NumToks].setFlag(Token::LeadingSpace);
NextTokGetsSpace = false;
@@ -371,10 +380,13 @@ void TokenLexer::ExpandFunctionArguments() {
}
// If this is on the RHS of a paste operator, we've already copied the
- // paste operator to the ResultToks list. Remove it.
- assert(PasteBefore && ResultToks.back().is(tok::hashhash));
- NextTokGetsSpace |= ResultToks.back().hasLeadingSpace();
- ResultToks.pop_back();
+ // paste operator to the ResultToks list, unless the LHS was empty too.
+ // Remove it.
+ assert(PasteBefore);
+ if (NonEmptyPasteBefore) {
+ assert(ResultToks.back().is(tok::hashhash));
+ NextTokGetsSpace |= ResultToks.pop_back_val().hasLeadingSpace();
+ }
// If this is the __VA_ARGS__ token, and if the argument wasn't provided,
// and if the macro had at least one real argument, and if the token before
@@ -404,22 +416,19 @@ void TokenLexer::ExpandFunctionArguments() {
/// Lex - Lex and return a token from this macro stream.
///
-void TokenLexer::Lex(Token &Tok) {
+bool TokenLexer::Lex(Token &Tok) {
// Lexing off the end of the macro, pop this macro off the expansion stack.
if (isAtEnd()) {
// If this is a macro (not a token stream), mark the macro enabled now
// that it is no longer being expanded.
if (Macro) Macro->EnableMacro();
- // Pop this context off the preprocessors lexer stack and get the next
- // token. This will delete "this" so remember the PP instance var.
- Preprocessor &PPCache = PP;
- if (PP.HandleEndOfTokenLexer(Tok))
- return;
-
- // HandleEndOfTokenLexer may not return a token. If it doesn't, lex
- // whatever is next.
- return PPCache.Lex(Tok);
+ Tok.startToken();
+ Tok.setFlagValue(Token::StartOfLine , AtStartOfLine);
+ Tok.setFlagValue(Token::LeadingSpace, HasLeadingSpace);
+ if (CurToken == 0)
+ Tok.setFlag(Token::LeadingEmptyMacro);
+ return PP.HandleEndOfTokenLexer(Tok);
}
SourceManager &SM = PP.getSourceManager();
@@ -439,7 +448,7 @@ void TokenLexer::Lex(Token &Tok) {
// When handling the microsoft /##/ extension, the final token is
// returned by PasteTokens, not the pasted token.
if (PasteTokens(Tok))
- return;
+ return true;
TokenIsFromPaste = true;
}
@@ -470,6 +479,8 @@ void TokenLexer::Lex(Token &Tok) {
if (isFirstToken) {
Tok.setFlagValue(Token::StartOfLine , AtStartOfLine);
Tok.setFlagValue(Token::LeadingSpace, HasLeadingSpace);
+ AtStartOfLine = false;
+ HasLeadingSpace = false;
}
// Handle recursive expansion!
@@ -487,10 +498,11 @@ void TokenLexer::Lex(Token &Tok) {
}
if (!DisableMacroExpansion && II->isHandleIdentifierCase())
- PP.HandleIdentifier(Tok);
+ return PP.HandleIdentifier(Tok);
}
// Otherwise, return a normal token.
+ return true;
}
/// PasteTokens - Tok is the LHS of a ## operator, and CurToken is the ##
@@ -812,3 +824,8 @@ void TokenLexer::updateLocForMacroArgTokens(SourceLocation ArgIdSpellLoc,
updateConsecutiveMacroArgTokens(SM, InstLoc, begin_tokens, end_tokens);
}
}
+
+void TokenLexer::PropagateLineStartLeadingSpaceInfo(Token &Result) {
+ AtStartOfLine = Result.isAtStartOfLine();
+ HasLeadingSpace = Result.hasLeadingSpace();
+}