diff --git a/addons/cppcheckdata.py b/addons/cppcheckdata.py index 0d3e5d4cf29..dca5964b20b 100755 --- a/addons/cppcheckdata.py +++ b/addons/cppcheckdata.py @@ -1563,37 +1563,6 @@ def simpleMatch(token, pattern): '<*>': lambda tok: tok.link if tok.str == '<' and tok.link else None, } -def match_atom(token, p): - if not token: - return None - if not p: - return None - if token.str == p: - return token - if p in ['!', '|', '||', '%', '!=', '*']: - return None - if p in patterns: - return patterns[p](token) - if '|' in p: - for x in p.split('|'): - t = match_atom(token, x) - if t: - return t - elif p.startswith('!!'): - t = match_atom(token, p[2:]) - if not t: - return token - elif p.startswith('**'): - a = p[2:] - t = token - while t: - if match_atom(t, a): - return t - if t.link and t.str in ['(', '[', '<', '{']: - t = t.link - t = t.next - return None - class MatchResult: def __init__(self, matches, bindings=None, keys=None): self.__dict__.update(bindings or {}) @@ -1617,22 +1586,122 @@ def bind_split(s): return (p[0], p[2]) return (s, None) +class CompiledPattern: + """ + A match() pattern compiled once so it can be matched against many tokens. + + Each word of the pattern is compiled into a matcher: a function that takes + a token (never None) and returns the last token the word matched, or None. + + Attributes: + pattern The pattern string + atoms List of (matcher, binding name) for each word in the pattern + failed The MatchResult returned when the pattern does not match + """ + + _literal_atoms = frozenset(['!', '|', '||', '|=', '%', '!=', '*']) + _link_open = frozenset(['(', '[', '<', '{']) + + def __init__(self, pattern): + self.pattern = pattern + words = [bind_split(word) for word in pattern.split()] + self.atoms = [(self.compile_atom(p), b) for p, b in words] + keys = [b for p, b in words] + ['end'] if words else [] + self.failed = MatchResult(False, keys=keys) + + def __repr__(self): + return "CompiledPattern(pattern={})".format(repr(self.pattern)) + + def match(self, token): + if not self.atoms: + # empty pattern (or only whitespace) never matches + return self.failed + end = None + bindings = {} + for m, b in self.atoms: + if token is None: + return self.failed + t = m(token) + if b: + bindings[b] = token + if t is None: + return self.failed + end = t + token = t.next + bindings['end'] = end + return MatchResult(True, bindings=bindings) + + @classmethod + def compile_atom(cls, p): + """Compile one pattern word into a matcher.""" + if not p: + return cls._match_none + if p in cls._literal_atoms: + return cls._match_literal(p) + if p in patterns: + return patterns[p] + prefix, word = p[:2], p[2:] + # a '!!' or '**' prefix applies to the whole word only when the word + # is a literal: '!!|=' is "not |=", but '!!a|b' is "not a, or b" + if '|' in p and not (prefix in ('!!', '**') and word in cls._literal_atoms): + return cls._match_alternatives([cls.compile_atom(x) for x in p.split('|')]) + if prefix == '!!': + return cls._match_negation(cls.compile_atom(word)) + if prefix == '**': + return cls._match_forward(cls.compile_atom(word)) + return cls._match_literal(p) + + @staticmethod + def _match_none(tok): + return None + + @staticmethod + def _match_literal(p): + def m(tok): + return tok if tok.str == p else None + return m + + @staticmethod + def _match_alternatives(alternatives): + def m(tok): + for a in alternatives: + t = a(tok) + if t: + return t + return None + return m + + @staticmethod + def _match_negation(inner): + def m(tok): + return None if inner(tok) else tok + return m + + @classmethod + def _match_forward(cls, inner): + link_open = cls._link_open + def m(tok): + t = tok + while t: + if inner(t): + return t + if t.link and t.str in link_open: + t = t.link + t = t.next + return None + return m + +# pattern string -> CompiledPattern +_pattern_cache = {} + def match(token, pattern): if not pattern: return MatchResult(False) - end = None - bindings = {} - words = [bind_split(word) for word in pattern.split()] - for p, b in words: - t = match_atom(token, p) - if b: - bindings[b] = token - if not t: - return MatchResult(False, keys=[xx for pp, xx in words]+['end']) - end = t - token = t.next - bindings['end'] = end - return MatchResult(True, bindings=bindings) + compiled = _pattern_cache.get(pattern) + if compiled is None: + compiled = CompiledPattern(pattern) + _pattern_cache[pattern] = compiled + return compiled.match(token) def get_function_call_name_args(token): """Get function name and arguments for function call diff --git a/test/addon/conftest.py b/test/addon/conftest.py index 87012983951..797b44e5cca 100644 --- a/test/addon/conftest.py +++ b/test/addon/conftest.py @@ -139,6 +139,7 @@ class Shape { int arr[3]; arr[0] = a; a += 1; + a |= b; if (a > b) { return a; } diff --git a/test/addon/test_cppcheckdata.py b/test/addon/test_cppcheckdata.py index 0ff3f31a89d..dbb4a2672b5 100644 --- a/test/addon/test_cppcheckdata.py +++ b/test/addon/test_cppcheckdata.py @@ -504,6 +504,7 @@ def test_literal_sequence(self, match_cfg): def test_empty_pattern_and_no_token(self, match_cfg): calc_def = find_token(match_cfg, 'calc') assert not cppcheckdata.match(calc_def, '') + assert not cppcheckdata.match(calc_def, None) assert not cppcheckdata.match(None, 'calc') def test_end(self, match_cfg): @@ -553,6 +554,23 @@ def test_literal_percent(self, match_cfg): assert cppcheckdata.match(mod, '%') assert cppcheckdata.match(mod.previous, '%var% % %var% ;') + def test_literal_or_assign(self, match_cfg): + or_assign = find_token(match_cfg, '|=') + assert cppcheckdata.match(or_assign, '|=') + assert cppcheckdata.match(or_assign.previous, '%var% |= %var% ;') + assert cppcheckdata.match(or_assign, '%assign%') + assert not cppcheckdata.match(find_token(match_cfg, '|'), '|=') + # '|=' is a literal, not an alternation of '' and '=' + # (unlike the C++ Token::Match, which also matches a '=' token here) + assert not cppcheckdata.match(find_token(match_cfg, '='), '|=') + + def test_whitespace_only_pattern(self, match_cfg): + calc_def = find_token(match_cfg, 'calc') + res = cppcheckdata.match(calc_def, ' ') + assert not res + with pytest.raises(AttributeError): + res.end # pylint: disable=W0104 + def test_literal_parentheses(self, match_cfg): calc_call = find_token(match_cfg, 'calc', skip=1) assert cppcheckdata.match(calc_call, 'calc ( %var% , %var% ) ;') @@ -677,6 +695,37 @@ def test_negation_keyword(self, match_cfg): assert cppcheckdata.match(calc_call, '%name% !!%op%') assert not cppcheckdata.match(calc_call, '%name% !!(') + def test_negation_literal_or_assign(self, match_cfg): + # '!!|=' is "not |=", not an alternation of '!!' and '=' + or_assign = find_token(match_cfg, '|=') + assert not cppcheckdata.match(or_assign.previous, '%var% !!|=') + bit_or = find_token(match_cfg, '|') + assert cppcheckdata.match(bit_or.previous, '%var% !!|=') + assert cppcheckdata.match(find_token(match_cfg, '=').previous, '%var% !!|=') + + def test_negation_alternatives(self, match_cfg): + # '!!a|b' is "not a, or b", as in the C++ Token::Match + struct_tok = find_token(match_cfg, 'struct') + assert not cppcheckdata.match(struct_tok, 'struct !!Point|Foo') + assert cppcheckdata.match(struct_tok, 'struct !!Foo|Point') + assert cppcheckdata.match(struct_tok, 'struct !!Foo|Bar') + + def test_find_forward_alternatives(self, match_cfg): + # '**a|b' is "a somewhere ahead, or b here" + calc_def = find_token(match_cfg, 'calc') + res = cppcheckdata.match(calc_def, 'calc **{|nosuchtoken') + assert res + assert res.end.str == '{' + res = cppcheckdata.match(calc_def, 'calc **nosuchtoken|(') + assert res + assert res.end.str == '(' + assert not cppcheckdata.match(calc_def, 'calc **nosuchtoken|{') + # but a literal word is searched for as a whole + arr_decl = find_token(match_cfg, 'arr') + res = cppcheckdata.match(arr_decl, 'arr **|=') + assert res + assert res.end.str == '|=' + @pytest.mark.xfail(strict=False, reason='in the C++ Token::Match a negation also matches when there is no token, ' 'in match() it does not')