| #!/usr/bin/env python3 |
| # SPDX-License-Identifier: GPL-2.0 |
| # Copyright(c) 2025: Mauro Carvalho Chehab <mchehab@kernel.org>. |
| |
| """ |
| Regular expression ancillary classes. |
| |
| Those help caching regular expressions and do matching for kernel-doc. |
| |
| Please notice that the code here may rise exceptions to indicate bad |
| usage inside kdoc to indicate problems at the replace pattern. |
| |
| Other errors are logged via log instance. |
| """ |
| |
| import logging |
| import re |
| |
| from copy import copy |
| |
| from .kdoc_re import KernRe |
| |
| log = logging.getLogger(__name__) |
| |
| def tokenizer_set_log(logger, prefix = ""): |
| """ |
| Replace the module‑level logger with a LoggerAdapter that |
| prepends *prefix* to every message. |
| """ |
| global log |
| |
| class PrefixAdapter(logging.LoggerAdapter): |
| """ |
| Ancillary class to set prefix on all message logs. |
| """ |
| def process(self, msg, kwargs): |
| return f"{prefix}{msg}", kwargs |
| |
| # Wrap the provided logger in our adapter |
| log = PrefixAdapter(logger, {"prefix": prefix}) |
| |
| class CToken(): |
| """ |
| Data class to define a C token. |
| """ |
| |
| # Tokens that can be used by the parser. Works like an C enum. |
| |
| COMMENT = 0 #: A standard C or C99 comment, including delimiter. |
| STRING = 1 #: A string, including quotation marks. |
| CHAR = 2 #: A character, including apostophes. |
| NUMBER = 3 #: A number. |
| PUNC = 4 #: A puntuation mark: / ``,`` / ``.``. |
| BEGIN = 5 #: A begin character: ``{`` / ``[`` / ``(``. |
| END = 6 #: A end character: ``}`` / ``]`` / ``)``. |
| CPP = 7 #: A preprocessor macro. |
| HASH = 8 #: The hash character - useful to handle other macros. |
| OP = 9 #: A C operator (add, subtract, ...). |
| STRUCT = 10 #: A ``struct`` keyword. |
| UNION = 11 #: An ``union`` keyword. |
| ENUM = 12 #: A ``struct`` keyword. |
| TYPEDEF = 13 #: A ``typedef`` keyword. |
| NAME = 14 #: A name. Can be an ID or a type. |
| SPACE = 15 #: Any space characters, including new lines |
| ENDSTMT = 16 #: End of an statement (``;``). |
| |
| BACKREF = 17 #: Not a valid C sequence, but used at sub regex patterns. |
| |
| MISMATCH = 255 #: an error indicator: should never happen in practice. |
| |
| # Dict to convert from an enum interger into a string. |
| _name_by_val = {v: k for k, v in dict(vars()).items() if isinstance(v, int)} |
| |
| # Dict to convert from string to an enum-like integer value. |
| _name_to_val = {k: v for v, k in _name_by_val.items()} |
| |
| @staticmethod |
| def to_name(val): |
| """Convert from an integer value from CToken enum into a string""" |
| |
| return CToken._name_by_val.get(val, f"UNKNOWN({val})") |
| |
| @staticmethod |
| def from_name(name): |
| """Convert a string into a CToken enum value""" |
| if name in CToken._name_to_val: |
| return CToken._name_to_val[name] |
| |
| return CToken.MISMATCH |
| |
| |
| def __init__(self, kind, value=None, pos=0, |
| brace_level=0, paren_level=0, bracket_level=0): |
| self.kind = kind |
| self.value = value |
| self.pos = pos |
| self.level = (bracket_level, paren_level, brace_level) |
| |
| def __repr__(self): |
| name = self.to_name(self.kind) |
| if isinstance(self.value, str): |
| value = '"' + self.value + '"' |
| else: |
| value = self.value |
| |
| return f"CToken(CToken.{name}, {value}, {self.pos}, {self.level})" |
| |
| #: Regexes to parse C code, transforming it into tokens. |
| RE_SCANNER_LIST = [ |
| # |
| # Note that \s\S is different than .*, as it also catches \n |
| # |
| (CToken.COMMENT, r"//[^\n]*|/\*[\s\S]*?\*/"), |
| |
| (CToken.STRING, r'"(?:\\.|[^"\\])*"'), |
| (CToken.CHAR, r"'(?:\\.|[^'\\])'"), |
| |
| (CToken.NUMBER, r"0[xX][\da-fA-F]+[uUlL]*|0[0-7]+[uUlL]*|" |
| r"\d+(?:\.\d*)?(?:[eE][+-]?\d+)?[fFlL]*"), |
| |
| (CToken.ENDSTMT, r"(?:\s+;|;)"), |
| |
| (CToken.PUNC, r"[,\.]"), |
| |
| (CToken.BEGIN, r"[\[\(\{]"), |
| |
| (CToken.END, r"[\]\)\}]"), |
| |
| (CToken.CPP, r"#\s*(?:define|include|ifdef|ifndef|if|else|elif|endif|undef|pragma)\b"), |
| |
| (CToken.HASH, r"#"), |
| |
| (CToken.OP, r"\+\+|\-\-|\->|==|\!=|<=|>=|&&|\|\||<<|>>|\+=|\-=|\*=|/=|%=" |
| r"|&=|\|=|\^=|[=\+\-\*/%<>&\|\^~!\?\:]"), |
| |
| (CToken.STRUCT, r"\bstruct\b"), |
| (CToken.UNION, r"\bunion\b"), |
| (CToken.ENUM, r"\benum\b"), |
| (CToken.TYPEDEF, r"\btypedef\b"), |
| |
| (CToken.NAME, r"[A-Za-z_]\w*"), |
| |
| (CToken.SPACE, r"\s+"), |
| |
| (CToken.BACKREF, r"\\\d+"), |
| |
| (CToken.MISMATCH,r"."), |
| ] |
| |
| def fill_re_scanner(token_list): |
| """Ancillary routine to convert RE_SCANNER_LIST into a finditer regex""" |
| re_tokens = [] |
| |
| for kind, pattern in token_list: |
| name = CToken.to_name(kind) |
| re_tokens.append(f"(?P<{name}>{pattern})") |
| |
| return KernRe("|".join(re_tokens), re.MULTILINE | re.DOTALL) |
| |
| #: Handle C continuation lines. |
| RE_CONT = KernRe(r"\\\n") |
| |
| RE_COMMENT_START = KernRe(r'/\*\s*') |
| |
| #: tokenizer regex. Will be filled at the first CTokenizer usage. |
| RE_SCANNER = fill_re_scanner(RE_SCANNER_LIST) |
| |
| |
| class CTokenizer(): |
| """ |
| Scan C statements and definitions and produce tokens. |
| |
| When converted to string, it drops comments and handle public/private |
| values, respecting depth. |
| """ |
| |
| # This class is inspired and follows the basic concepts of: |
| # https://docs.python.org/3/library/re.html#writing-a-tokenizer |
| |
| def __init__(self, source=None): |
| """ |
| Create a regular expression to handle RE_SCANNER_LIST. |
| |
| While I generally don't like using regex group naming via: |
| (?P<name>...) |
| |
| in this particular case, it makes sense, as we can pick the name |
| when matching a code via RE_SCANNER. |
| """ |
| |
| # |
| # Store logger to allow parser classes to re-use it |
| # |
| global log |
| self.log = log |
| |
| self.tokens = [] |
| |
| if not source: |
| return |
| |
| if isinstance(source, list): |
| self.tokens = source |
| return |
| |
| # |
| # While we could just use _tokenize directly via interator, |
| # As we'll need to use the tokenizer several times inside kernel-doc |
| # to handle macro transforms, cache the results on a list, as |
| # re-using it is cheaper than having to parse everytime. |
| # |
| for tok in self._tokenize(source): |
| self.tokens.append(tok) |
| |
| def _tokenize(self, source): |
| """ |
| Iterator that parses ``source``, splitting it into tokens, as defined |
| at ``self.RE_SCANNER_LIST``. |
| |
| The interactor returns a CToken class object. |
| """ |
| |
| # Handle continuation lines. Note that kdoc_parser already has a |
| # logic to do that. Still, let's keep it for completeness, as we might |
| # end re-using this tokenizer outsize kernel-doc some day - or we may |
| # eventually remove from there as a future cleanup. |
| source = RE_CONT.sub("", source) |
| |
| brace_level = 0 |
| paren_level = 0 |
| bracket_level = 0 |
| |
| for match in RE_SCANNER.finditer(source): |
| kind = CToken.from_name(match.lastgroup) |
| pos = match.start() |
| value = match.group() |
| |
| if kind == CToken.MISMATCH: |
| log.error(f"Unexpected token '{value}' on pos {pos}:\n\t'{source}'") |
| elif kind == CToken.BEGIN: |
| if value == '(': |
| paren_level += 1 |
| elif value == '[': |
| bracket_level += 1 |
| else: # value == '{' |
| brace_level += 1 |
| |
| elif kind == CToken.END: |
| if value == ')' and paren_level > 0: |
| paren_level -= 1 |
| elif value == ']' and bracket_level > 0: |
| bracket_level -= 1 |
| elif brace_level > 0: # value == '}' |
| brace_level -= 1 |
| |
| yield CToken(kind, value, pos, |
| brace_level, paren_level, bracket_level) |
| |
| def __str__(self): |
| out="" |
| show_stack = [True] |
| |
| for i, tok in enumerate(self.tokens): |
| if tok.kind == CToken.BEGIN: |
| show_stack.append(show_stack[-1]) |
| |
| elif tok.kind == CToken.END: |
| prev = show_stack[-1] |
| if len(show_stack) > 1: |
| show_stack.pop() |
| |
| if not prev and show_stack[-1]: |
| # |
| # Try to preserve indent |
| # |
| out += "\t" * (len(show_stack) - 1) |
| |
| out += str(tok.value) |
| continue |
| |
| elif tok.kind == CToken.COMMENT: |
| comment = RE_COMMENT_START.sub("", tok.value) |
| |
| if comment.startswith("private:"): |
| show_stack[-1] = False |
| show = False |
| elif comment.startswith("public:"): |
| show_stack[-1] = True |
| |
| continue |
| |
| if not show_stack[-1]: |
| continue |
| |
| if i < len(self.tokens) - 1: |
| next_tok = self.tokens[i + 1] |
| |
| # Do some cleanups before ";" |
| |
| if tok.kind == CToken.SPACE and next_tok.kind == CToken.ENDSTMT: |
| continue |
| |
| if tok.kind == CToken.ENDSTMT and next_tok.kind == tok.kind: |
| continue |
| |
| out += str(tok.value) |
| |
| return out |
| |
| |
| class CTokenArgs: |
| """ |
| Ancillary class to help using backrefs from sub matches. |
| |
| If the highest backref contain a "+" at the last element, |
| the logic will be greedy, picking all other delims. |
| |
| This is needed to parse struct_group macros with end with ``MEMBERS...``. |
| """ |
| def __init__(self, sub_str): |
| self.sub_groups = set() |
| self.max_group = -1 |
| self.greedy = None |
| |
| for m in KernRe(r'\\(\d+)([+]?)').finditer(sub_str): |
| group = int(m.group(1)) |
| if m.group(2) == "+": |
| if self.greedy and self.greedy != group: |
| raise ValueError("There are multiple greedy patterns!") |
| self.greedy = group |
| |
| self.sub_groups.add(group) |
| self.max_group = max(self.max_group, group) |
| |
| if self.greedy: |
| if self.greedy != self.max_group: |
| raise ValueError("Greedy pattern is not the last one!") |
| |
| sub_str = KernRe(r'(\\\d+)[+]').sub(r"\1", sub_str) |
| |
| self.sub_str = sub_str |
| self.sub_tokeninzer = CTokenizer(sub_str) |
| |
| def groups(self, new_tokenizer): |
| r""" |
| Create replacement arguments for backrefs like: |
| |
| ``\0``, ``\1``, ``\2``, ... ``\{number}`` |
| |
| It also accepts a ``+`` character to the highest backref, like |
| ``\4+``. When used, the backref will be greedy, picking all other |
| arguments afterwards. |
| |
| The logic is smart enough to only go up to the maximum required |
| argument, even if there are more. |
| |
| If there is a backref for an argument above the limit, it will |
| raise an exception. Please notice that, on C, square brackets |
| don't have any separator on it. Trying to use ``\1``..``\n`` for |
| brackets also raise an exception. |
| """ |
| |
| level = (0, 0, 0) |
| |
| if self.max_group < 0: |
| return level, [] |
| |
| tokens = new_tokenizer.tokens |
| |
| # |
| # Fill \0 with the full token contents |
| # |
| groups_list = [ [] ] |
| |
| if 0 in self.sub_groups: |
| inner_level = 0 |
| |
| for i in range(0, len(tokens)): |
| tok = tokens[i] |
| |
| if tok.kind == CToken.BEGIN: |
| inner_level += 1 |
| |
| # |
| # Discard first begin |
| # |
| if not groups_list[0]: |
| continue |
| elif tok.kind == CToken.END: |
| inner_level -= 1 |
| if inner_level < 0: |
| break |
| |
| if inner_level: |
| groups_list[0].append(tok) |
| |
| if not self.max_group: |
| return level, groups_list |
| |
| delim = None |
| |
| # |
| # Ignore everything before BEGIN. The value of begin gives the |
| # delimiter to be used for the matches |
| # |
| for i in range(0, len(tokens)): |
| tok = tokens[i] |
| if tok.kind == CToken.BEGIN: |
| if tok.value == "{": |
| delim = ";" |
| elif tok.value == "(": |
| delim = "," |
| else: |
| self.log.error(fr"Can't handle \1..\n on {sub_str}") |
| |
| level = tok.level |
| break |
| |
| pos = 1 |
| groups_list.append([]) |
| |
| inner_level = 0 |
| for i in range(i + 1, len(tokens)): |
| tok = tokens[i] |
| |
| if tok.kind == CToken.BEGIN: |
| inner_level += 1 |
| if tok.kind == CToken.END: |
| inner_level -= 1 |
| if inner_level < 0: |
| break |
| |
| if tok.kind in [CToken.PUNC, CToken.ENDSTMT] and delim == tok.value: |
| pos += 1 |
| if self.greedy and pos > self.max_group: |
| pos -= 1 |
| else: |
| groups_list.append([]) |
| |
| if pos > self.max_group: |
| break |
| |
| continue |
| |
| groups_list[pos].append(tok) |
| |
| if pos < self.max_group: |
| log.error(fr"{self.sub_str} groups are up to {pos} instead of {self.max_group}") |
| |
| return level, groups_list |
| |
| def tokens(self, new_tokenizer): |
| level, groups = self.groups(new_tokenizer) |
| |
| new = CTokenizer() |
| |
| for tok in self.sub_tokeninzer.tokens: |
| if tok.kind == CToken.BACKREF: |
| group = int(tok.value[1:]) |
| |
| for group_tok in groups[group]: |
| new_tok = copy(group_tok) |
| |
| new_level = [0, 0, 0] |
| |
| for i in range(0, len(level)): |
| new_level[i] = new_tok.level[i] + level[i] |
| |
| new_tok.level = tuple(new_level) |
| |
| new.tokens += [ new_tok ] |
| else: |
| new.tokens += [ tok ] |
| |
| return new.tokens |
| |
| |
| class CMatch: |
| """ |
| Finding nested delimiters is hard with regular expressions. It is |
| even harder on Python with its normal re module, as there are several |
| advanced regular expressions that are missing. |
| |
| This is the case of this pattern:: |
| |
| '\\bSTRUCT_GROUP(\\(((?:(?>[^)(]+)|(?1))*)\\))[^;]*;' |
| |
| which is used to properly match open/close parentheses of the |
| string search STRUCT_GROUP(), |
| |
| Add a class that counts pairs of delimiters, using it to match and |
| replace nested expressions. |
| |
| The original approach was suggested by: |
| |
| https://stackoverflow.com/questions/5454322/python-how-to-match-nested-parentheses-with-regex |
| |
| Although I re-implemented it to make it more generic and match 3 types |
| of delimiters. The logic checks if delimiters are paired. If not, it |
| will ignore the search string. |
| """ |
| |
| |
| def __init__(self, regex, delim="("): |
| self.regex = KernRe("^" + regex + r"\b") |
| self.start_delim = delim |
| |
| def _search(self, tokenizer): |
| """ |
| Finds paired blocks for a regex that ends with a delimiter. |
| |
| The suggestion of using finditer to match pairs came from: |
| https://stackoverflow.com/questions/5454322/python-how-to-match-nested-parentheses-with-regex |
| but I ended using a different implementation to align all three types |
| of delimiters and seek for an initial regular expression. |
| |
| The algorithm seeks for open/close paired delimiters and places them |
| into a stack, yielding a start/stop position of each match when the |
| stack is zeroed. |
| |
| The algorithm should work fine for properly paired lines, but will |
| silently ignore end delimiters that precede a start delimiter. |
| This should be OK for kernel-doc parser, as unaligned delimiters |
| would cause compilation errors. So, we don't need to raise exceptions |
| to cover such issues. |
| """ |
| |
| start = None |
| started = False |
| |
| import sys |
| |
| stack = [] |
| |
| for i, tok in enumerate(tokenizer.tokens): |
| if start is None: |
| if tok.kind == CToken.NAME and self.regex.match(tok.value): |
| start = i |
| stack.append((start, tok.level)) |
| started = False |
| |
| continue |
| |
| if not started: |
| if tok.kind == CToken.SPACE: |
| continue |
| |
| if tok.kind == CToken.BEGIN and tok.value == self.start_delim: |
| started = True |
| continue |
| |
| # Name only token without BEGIN/END |
| if i > start: |
| i -= 1 |
| yield start, i |
| start = None |
| |
| if tok.kind == CToken.END and tok.level == stack[-1][1]: |
| start, level = stack.pop() |
| |
| yield start, i |
| start = None |
| |
| # |
| # If an END zeroing levels is not there, return remaining stuff |
| # This is meant to solve cases where the caller logic might be |
| # picking an incomplete block. |
| # |
| if start and stack: |
| if started: |
| s = str(tokenizer) |
| log.warning(f"can't find a final end at {s}") |
| |
| yield start, len(tokenizer.tokens) |
| |
| def search(self, source): |
| """ |
| This is similar to re.search: |
| |
| It matches a regex that it is followed by a delimiter, |
| returning occurrences only if all delimiters are paired. |
| """ |
| |
| if isinstance(source, CTokenizer): |
| tokenizer = source |
| is_token = True |
| else: |
| tokenizer = CTokenizer(source) |
| is_token = False |
| |
| for start, end in self._search(tokenizer): |
| new_tokenizer = CTokenizer(tokenizer.tokens[start:end + 1]) |
| |
| if is_token: |
| yield new_tokenizer |
| else: |
| yield str(new_tokenizer) |
| |
| def sub(self, sub_str, source, count=0): |
| """ |
| This is similar to re.sub: |
| |
| It matches a regex that it is followed by a delimiter, |
| replacing occurrences only if all delimiters are paired. |
| |
| if the sub argument contains:: |
| |
| r'\0' |
| |
| it will work just like re: it places there the matched paired data |
| with the delimiter stripped. |
| |
| If count is different than zero, it will replace at most count |
| items. |
| """ |
| if isinstance(source, CTokenizer): |
| is_token = True |
| tokenizer = source |
| else: |
| is_token = False |
| tokenizer = CTokenizer(source) |
| |
| # Detect if sub_str contains sub arguments |
| |
| args_match = CTokenArgs(sub_str) |
| |
| new_tokenizer = CTokenizer() |
| pos = 0 |
| n = 0 |
| |
| # |
| # NOTE: the code below doesn't consider overlays at sub. |
| # We may need to add some extra unit tests to check if those |
| # would cause problems. When replacing by "", this should not |
| # be a problem, but other transformations could be problematic |
| # |
| for start, end in self._search(tokenizer): |
| new_tokenizer.tokens += tokenizer.tokens[pos:start] |
| |
| new = CTokenizer(tokenizer.tokens[start:end + 1]) |
| |
| new_tokenizer.tokens += args_match.tokens(new) |
| |
| pos = end + 1 |
| |
| n += 1 |
| if count and n >= count: |
| break |
| |
| new_tokenizer.tokens += tokenizer.tokens[pos:] |
| |
| if not is_token: |
| return str(new_tokenizer) |
| |
| return new_tokenizer |
| |
| def __repr__(self): |
| """ |
| Returns a displayable version of the class init. |
| """ |
| |
| return f'CMatch("{self.regex.regex.pattern}")' |
