Skip to content

Commit e3892a9

Browse files
danmardanmar
committed
Fixed danmar#6290 (Tokenizer: Because 'and' is not simplified to '&&' there are false positives)
1 parent d0ff8d8 commit e3892a9

File tree

2 files changed

+63
-43
lines changed

2 files changed

+63
-43
lines changed

‎lib/tokenize.cpp

Lines changed: 34 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -6175,26 +6175,42 @@ static const std::map<std::string, std::string> cAlternativeTokens(cAlternativeT
61756175
// xor_eq => ^=
61766176
bool Tokenizer::simplifyCAlternativeTokens()
61776177
{
6178+
/* For C code: executable scope level */
6179+
unsigned int executableScopeLevel = 0;
6180+
61786181
bool ret = false;
61796182
for (Token *tok = list.front(); tok; tok = tok->next()) {
6180-
Token *start = startOfExecutableScope(tok);
6181-
if (start) { // Check for executable scope
6182-
tok = start;
6183-
Token * const end = tok->link();
6184-
for (Token *tok2 = tok->next(); tok2 && tok2 != end; tok2 = tok2->next()) {
6185-
if (Token::Match(tok2, "%var%|%num%|)|]|> %any% %var%|%num%|(|%op%")) {
6186-
const std::map<std::string, std::string>::const_iterator cOpIt = cAlternativeTokens.find(tok2->next()->str());
6187-
if (cOpIt != cAlternativeTokens.end()) {
6188-
tok2->next()->str(cOpIt->second);
6189-
ret = true;
6190-
}
6191-
}
6192-
if (Token::Match(tok2, "not|compl %var%|(|%op%") &&
6193-
!Token::Match(tok2->previous(), "[;{}]")) { // Don't simplify 'not p;' (in case 'not' is a type)
6194-
tok2->str((tok2->str() == "not") ? "!" : "~");
6195-
ret = true;
6196-
}
6197-
}
6183+
if (tok->str() == "{") {
6184+
if (executableScopeLevel > 0 || Token::Match(tok->previous(), ") {"))
6185+
++executableScopeLevel;
6186+
continue;
6187+
}
6188+
6189+
if (tok->str() == "}") {
6190+
if (executableScopeLevel > 0)
6191+
--executableScopeLevel;
6192+
continue;
6193+
}
6194+
6195+
if (!tok->isName())
6196+
continue;
6197+
6198+
const std::map<std::string, std::string>::const_iterator cOpIt = cAlternativeTokens.find(tok->str());
6199+
if (cOpIt != cAlternativeTokens.end()) {
6200+
if (isC() && !Token::Match(tok->previous(), "%var%|%num%|%char%|)|]|> %var% %var%|%num%|%char%|%op%|("))
6201+
continue;
6202+
tok->str(cOpIt->second);
6203+
ret = true;
6204+
}
6205+
else if (Token::Match(tok, "not|compl")) {
6206+
// Don't simplify 'not p;' (in case 'not' is a type)
6207+
if (isC() && (!Token::Match(tok->next(), "%var%|%op%|(") ||
6208+
Token::Match(tok->previous(), "[;{}]") ||
6209+
(executableScopeLevel == 0U && tok->strAt(-1) == "(")))
6210+
continue;
6211+
6212+
tok->str((tok->str() == "not") ? "!" : "~");
6213+
ret = true;
61986214
}
61996215
}
62006216
return ret;

‎test/testsimplifytokens.cpp

Lines changed: 29 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -316,15 +316,16 @@ class TestSimplifyTokens : public TestFixture {
316316
return tokenizer.tokens()->stringifyList(0, !simplify);
317317
}
318318

319-
std::string tok(const char code[], const char filename[]) {
319+
std::string tok(const char code[], const char filename[], bool simplify = true) {
320320
errout.str("");
321321

322322
Settings settings;
323323
Tokenizer tokenizer(&settings, this);
324324

325325
std::istringstream istr(code);
326326
tokenizer.tokenize(istr, filename);
327-
tokenizer.simplifyTokenList2();
327+
if (simplify)
328+
tokenizer.simplifyTokenList2();
328329

329330
return tokenizer.tokens()->stringifyList(0, false);
330331
}
@@ -1684,68 +1685,71 @@ class TestSimplifyTokens : public TestFixture {
16841685
}
16851686

16861687
void not1() {
1687-
ASSERT_EQUALS("void f ( ) { if ( ! p ) { ; } }", tok("void f() { if (not p); }", false));
1688-
ASSERT_EQUALS("void f ( ) { if ( p && ! q ) { ; } }", tok("void f() { if (p && not q); }", false));
1689-
ASSERT_EQUALS("void f ( ) { a = ! ( p && q ) ; }", tok("void f() { a = not(p && q); }", false));
1688+
ASSERT_EQUALS("void f ( ) { if ( ! p ) { ; } }", tok("void f() { if (not p); }", "test.c", false));
1689+
ASSERT_EQUALS("void f ( ) { if ( p && ! q ) { ; } }", tok("void f() { if (p && not q); }", "test.c", false));
1690+
ASSERT_EQUALS("void f ( ) { a = ! ( p && q ) ; }", tok("void f() { a = not(p && q); }", "test.c", false));
16901691
// Don't simplify 'not' or 'compl' if they are defined as a type;
16911692
// in variable declaration and in function declaration/definition
1692-
ASSERT_EQUALS("struct not { int x ; } ;", tok("struct not { int x; };", false));
1693-
ASSERT_EQUALS("void f ( ) { not p ; compl c ; }", tok(" void f() { not p; compl c; }", false));
1694-
ASSERT_EQUALS("void foo ( not i ) ;", tok("void foo(not i);", false));
1695-
ASSERT_EQUALS("int foo ( not i ) { return g ( i ) ; }", tok("int foo(not i) { return g(i); }", false));
1693+
ASSERT_EQUALS("struct not { int x ; } ;", tok("struct not { int x; };", "test.c", false));
1694+
ASSERT_EQUALS("void f ( ) { not p ; compl c ; }", tok(" void f() { not p; compl c; }", "test.c", false));
1695+
ASSERT_EQUALS("void foo ( not i ) ;", tok("void foo(not i);", "test.c", false));
1696+
ASSERT_EQUALS("int foo ( not i ) { return g ( i ) ; }", tok("int foo(not i) { return g(i); }", "test.c", false));
16961697
}
16971698

16981699
void and1() {
16991700
ASSERT_EQUALS("void f ( ) { if ( p && q ) { ; } }",
1700-
tok("void f() { if (p and q) ; }", false));
1701+
tok("void f() { if (p and q) ; }", "test.c", false));
17011702

17021703
ASSERT_EQUALS("void f ( ) { if ( foo ( ) && q ) { ; } }",
1703-
tok("void f() { if (foo() and q) ; }", false));
1704+
tok("void f() { if (foo() and q) ; }", "test.c", false));
17041705

17051706
ASSERT_EQUALS("void f ( ) { if ( foo ( ) && bar ( ) ) { ; } }",
1706-
tok("void f() { if (foo() and bar()) ; }", false));
1707+
tok("void f() { if (foo() and bar()) ; }", "test.c", false));
17071708

17081709
ASSERT_EQUALS("void f ( ) { if ( p && bar ( ) ) { ; } }",
1709-
tok("void f() { if (p and bar()) ; }", false));
1710+
tok("void f() { if (p and bar()) ; }", "test.c", false));
17101711

17111712
ASSERT_EQUALS("void f ( ) { if ( p && ! q ) { ; } }",
1712-
tok("void f() { if (p and not q) ; }", false));
1713+
tok("void f() { if (p and not q) ; }", "test.c", false));
17131714

17141715
ASSERT_EQUALS("void f ( ) { r = a && b ; }",
1715-
tok("void f() { r = a and b; }", false));
1716+
tok("void f() { r = a and b; }", "test.c", false));
17161717

17171718
ASSERT_EQUALS("void f ( ) { r = ( a || b ) && ( c || d ) ; }",
1718-
tok("void f() { r = (a || b) and (c || d); }", false));
1719+
tok("void f() { r = (a || b) and (c || d); }", "test.c", false));
1720+
1721+
ASSERT_EQUALS("void f ( ) { if ( test1 [ i ] == 'A' && test2 [ i ] == 'C' ) { } }",
1722+
tok("void f() { if (test1[i] == 'A' and test2[i] == 'C') {} }", "test.c", false));
17191723
}
17201724

17211725
void or1() {
17221726
ASSERT_EQUALS("void f ( ) { if ( p || q ) { ; } }",
1723-
tok("void f() { if (p or q) ; }", false));
1727+
tok("void f() { if (p or q) ; }", "test.c", false));
17241728

17251729
ASSERT_EQUALS("void f ( ) { if ( foo ( ) || q ) { ; } }",
1726-
tok("void f() { if (foo() or q) ; }", false));
1730+
tok("void f() { if (foo() or q) ; }", "test.c", false));
17271731

17281732
ASSERT_EQUALS("void f ( ) { if ( foo ( ) || bar ( ) ) { ; } }",
1729-
tok("void f() { if (foo() or bar()) ; }", false));
1733+
tok("void f() { if (foo() or bar()) ; }", "test.c", false));
17301734

17311735
ASSERT_EQUALS("void f ( ) { if ( p || bar ( ) ) { ; } }",
1732-
tok("void f() { if (p or bar()) ; }", false));
1736+
tok("void f() { if (p or bar()) ; }", "test.c", false));
17331737

17341738
ASSERT_EQUALS("void f ( ) { if ( p || ! q ) { ; } }",
1735-
tok("void f() { if (p or not q) ; }", false));
1739+
tok("void f() { if (p or not q) ; }", "test.c", false));
17361740

17371741
ASSERT_EQUALS("void f ( ) { r = a || b ; }",
1738-
tok("void f() { r = a or b; }", false));
1742+
tok("void f() { r = a or b; }", "test.c", false));
17391743

17401744
ASSERT_EQUALS("void f ( ) { r = ( a && b ) || ( c && d ) ; }",
1741-
tok("void f() { r = (a && b) or (c && d); }", false));
1745+
tok("void f() { r = (a && b) or (c && d); }", "test.c", false));
17421746
}
17431747

17441748
void cAlternativeTokens() {
17451749
ASSERT_EQUALS("void f ( ) { err = err | ( ( r & s ) && ! t ) ; }",
1746-
tok("void f() { err or_eq ((r bitand s) and not t); }", false));
1750+
tok("void f() { err or_eq ((r bitand s) and not t); }", "test.c", false));
17471751
ASSERT_EQUALS("void f ( ) const { r = f ( a [ 4 ] | 15 , ~ c , ! d ) ; }",
1748-
tok("void f() const { r = f(a[4] bitor 0x0F, compl c, not d) ; }", false));
1752+
tok("void f() const { r = f(a[4] bitor 0x0F, compl c, not d) ; }", "test.c", false));
17491753

17501754
}
17511755

0 commit comments

Comments
 (0)