Sindbad~EG File Manager

# mako/lexer.py
# Copyright 2006-2019 the Mako authors and contributors <see AUTHORS file>
#
# This module is part of Mako and is released under
# the MIT License: http://www.opensource.org/licenses/mit-license.php

"""provides the Lexer class for parsing template strings into parse trees."""

import codecs
import re

from mako import compat
from mako import exceptions
from mako import parsetree
from mako.pygen import adjust_whitespace

_regexp_cache = {}


class Lexer(object):
    def __init__(
        self,
        text,
        filename=None,
        disable_unicode=False,
        input_encoding=None,
        preprocessor=None,
    ):
        self.text = text
        self.filename = filename
        self.template = parsetree.TemplateNode(self.filename)
        self.matched_lineno = 1
        self.matched_charpos = 0
        self.lineno = 1
        self.match_position = 0
        self.tag = []
        self.control_line = []
        self.ternary_stack = []
        self.disable_unicode = disable_unicode
        self.encoding = input_encoding

        if compat.py3k and disable_unicode:
            raise exceptions.UnsupportedError(
                "Mako for Python 3 does not " "support disabling Unicode"
            )

        if preprocessor is None:
            self.preprocessor = []
        elif not hasattr(preprocessor, "__iter__"):
            self.preprocessor = [preprocessor]
        else:
            self.preprocessor = preprocessor

    @property
    def exception_kwargs(self):
        return {
            "source": self.text,
            "lineno": self.matched_lineno,
            "pos": self.matched_charpos,
            "filename": self.filename,
        }

    def match(self, regexp, flags=None):
        """compile the given regexp, cache the reg, and call match_reg()."""

        try:
            reg = _regexp_cache[(regexp, flags)]
        except KeyError:
            if flags:
                reg = re.compile(regexp, flags)
            else:
                reg = re.compile(regexp)
            _regexp_cache[(regexp, flags)] = reg

        return self.match_reg(reg)

    def match_reg(self, reg):
        """match the given regular expression object to the current text
        position.

        if a match occurs, update the current text and line position.

        """

        mp = self.match_position

        match = reg.match(self.text, self.match_position)
        if match:
            (start, end) = match.span()
            if end == start:
                self.match_position = end + 1
            else:
                self.match_position = end
            self.matched_lineno = self.lineno
            lines = re.findall(r"\n", self.text[mp : self.match_position])
            cp = mp - 1
            while cp >= 0 and cp < self.textlength and self.text[cp] != "\n":
                cp -= 1
            self.matched_charpos = mp - cp
            self.lineno += len(lines)
            # print "MATCHED:", match.group(0), "LINE START:",
            # self.matched_lineno, "LINE END:", self.lineno
        # print "MATCH:", regexp, "\n", self.text[mp : mp + 15], \
        #          (match and "TRUE" or "FALSE")
        return match

    def parse_until_text(self, watch_nesting, *text):
        startpos = self.match_position
        text_re = r"|".join(text)
        brace_level = 0
        paren_level = 0
        bracket_level = 0
        while True:
            match = self.match(r"#.*\n")
            if match:
                continue
            match = self.match(
                r"(\"\"\"|\'\'\'|\"|\')[^\\]*?(\\.[^\\]*?)*\1", re.S
            )
            if match:
                continue
            match = self.match(r"(%s)" % text_re)
            if match and not (
                watch_nesting
                and (brace_level > 0 or paren_level > 0 or bracket_level > 0)
            ):
                return (
                    self.text[
                        startpos : self.match_position - len(match.group(1))
                    ],
                    match.group(1),
                )
            elif not match:
                match = self.match(r"(.*?)(?=\"|\'|#|%s)" % text_re, re.S)
            if match:
                brace_level += match.group(1).count("{")
                brace_level -= match.group(1).count("}")
                paren_level += match.group(1).count("(")
                paren_level -= match.group(1).count(")")
                bracket_level += match.group(1).count("[")
                bracket_level -= match.group(1).count("]")
                continue
            raise exceptions.SyntaxException(
                "Expected: %s" % ",".join(text), **self.exception_kwargs
            )

    def append_node(self, nodecls, *args, **kwargs):
        kwargs.setdefault("source", self.text)
        kwargs.setdefault("lineno", self.matched_lineno)
        kwargs.setdefault("pos", self.matched_charpos)
        kwargs["filename"] = self.filename
        node = nodecls(*args, **kwargs)
        if len(self.tag):
            self.tag[-1].nodes.append(node)
        else:
            self.template.nodes.append(node)
        # build a set of child nodes for the control line
        # (used for loop variable detection)
        # also build a set of child nodes on ternary control lines
        # (used for determining if a pass needs to be auto-inserted
        if self.control_line:
            control_frame = self.control_line[-1]
            control_frame.nodes.append(node)
            if not (
                isinstance(node, parsetree.ControlLine)
                and control_frame.is_ternary(node.keyword)
            ):
                if self.ternary_stack and self.ternary_stack[-1]:
                    self.ternary_stack[-1][-1].nodes.append(node)
        if isinstance(node, parsetree.Tag):
            if len(self.tag):
                node.parent = self.tag[-1]
            self.tag.append(node)
        elif isinstance(node, parsetree.ControlLine):
            if node.isend:
                self.control_line.pop()
                self.ternary_stack.pop()
            elif node.is_primary:
                self.control_line.append(node)
                self.ternary_stack.append([])
            elif self.control_line and self.control_line[-1].is_ternary(
                node.keyword
            ):
                self.ternary_stack[-1].append(node)
            elif self.control_line and not self.control_line[-1].is_ternary(
                node.keyword
            ):
                raise exceptions.SyntaxException(
                    "Keyword '%s' not a legal ternary for keyword '%s'"
                    % (node.keyword, self.control_line[-1].keyword),
                    **self.exception_kwargs
                )

    _coding_re = re.compile(r"#.*coding[:=]\s*([-\w.]+).*\r?\n")

    def decode_raw_stream(self, text, decode_raw, known_encoding, filename):
        """given string/unicode or bytes/string, determine encoding
           from magic encoding comment, return body as unicode
           or raw if decode_raw=False

        """
        if isinstance(text, compat.text_type):
            m = self._coding_re.match(text)
            encoding = m and m.group(1) or known_encoding or "ascii"
            return encoding, text

        if text.startswith(codecs.BOM_UTF8):
            text = text[len(codecs.BOM_UTF8) :]
            parsed_encoding = "utf-8"
            m = self._coding_re.match(text.decode("utf-8", "ignore"))
            if m is not None and m.group(1) != "utf-8":
                raise exceptions.CompileException(
                    "Found utf-8 BOM in file, with conflicting "
                    "magic encoding comment of '%s'" % m.group(1),
                    text.decode("utf-8", "ignore"),
                    0,
                    0,
                    filename,
                )
        else:
            m = self._coding_re.match(text.decode("utf-8", "ignore"))
            if m:
                parsed_encoding = m.group(1)
            else:
                parsed_encoding = known_encoding or "ascii"

        if decode_raw:
            try:
                text = text.decode(parsed_encoding)
            except UnicodeDecodeError:
                raise exceptions.CompileException(
                    "Unicode decode operation of encoding '%s' failed"
                    % parsed_encoding,
                    text.decode("utf-8", "ignore"),
                    0,
                    0,
                    filename,
                )

        return parsed_encoding, text

    def parse(self):
        self.encoding, self.text = self.decode_raw_stream(
            self.text, not self.disable_unicode, self.encoding, self.filename
        )

        for preproc in self.preprocessor:
            self.text = preproc(self.text)

        # push the match marker past the
        # encoding comment.
        self.match_reg(self._coding_re)

        self.textlength = len(self.text)

        while True:
            if self.match_position > self.textlength:
                break

            if self.match_end():
                break
            if self.match_expression():
                continue
            if self.match_control_line():
                continue
            if self.match_comment():
                continue
            if self.match_tag_start():
                continue
            if self.match_tag_end():
                continue
            if self.match_python_block():
                continue
            if self.match_text():
                continue

            if self.match_position > self.textlength:
                break
            raise exceptions.CompileException("assertion failed")

        if len(self.tag):
            raise exceptions.SyntaxException(
                "Unclosed tag: <%%%s>" % self.tag[-1].keyword,
                **self.exception_kwargs
            )
        if len(self.control_line):
            raise exceptions.SyntaxException(
                "Unterminated control keyword: '%s'"
                % self.control_line[-1].keyword,
                self.text,
                self.control_line[-1].lineno,
                self.control_line[-1].pos,
                self.filename,
            )
        return self.template

    def match_tag_start(self):
        match = self.match(
            r"""
            \<%     # opening tag

            ([\w\.\:]+)   # keyword

            ((?:\s+\w+|\s*=\s*|".*?"|'.*?')*)  # attrname, = \
                                               #        sign, string expression

            \s*     # more whitespace

            (/)?>   # closing

            """,
            re.I | re.S | re.X,
        )

        if match:
            keyword, attr, isend = match.groups()
            self.keyword = keyword
            attributes = {}
            if attr:
                for att in re.findall(
                    r"\s*(\w+)\s*=\s*(?:'([^']*)'|\"([^\"]*)\")", attr
                ):
                    key, val1, val2 = att
                    text = val1 or val2
                    text = text.replace("\r\n", "\n")
                    attributes[key] = text
            self.append_node(parsetree.Tag, keyword, attributes)
            if isend:
                self.tag.pop()
            else:
                if keyword == "text":
                    match = self.match(r"(.*?)(?=\</%text>)", re.S)
                    if not match:
                        raise exceptions.SyntaxException(
                            "Unclosed tag: <%%%s>" % self.tag[-1].keyword,
                            **self.exception_kwargs
                        )
                    self.append_node(parsetree.Text, match.group(1))
                    return self.match_tag_end()
            return True
        else:
            return False

    def match_tag_end(self):
        match = self.match(r"\</%[\t ]*(.+?)[\t ]*>")
        if match:
            if not len(self.tag):
                raise exceptions.SyntaxException(
                    "Closing tag without opening tag: </%%%s>"
                    % match.group(1),
                    **self.exception_kwargs
                )
            elif self.tag[-1].keyword != match.group(1):
                raise exceptions.SyntaxException(
                    "Closing tag </%%%s> does not match tag: <%%%s>"
                    % (match.group(1), self.tag[-1].keyword),
                    **self.exception_kwargs
                )
            self.tag.pop()
            return True
        else:
            return False

    def match_end(self):
        match = self.match(r"\Z", re.S)
        if match:
            string = match.group()
            if string:
                return string
            else:
                return True
        else:
            return False

    def match_text(self):
        match = self.match(
            r"""
                (.*?)         # anything, followed by:
                (
                 (?<=\n)(?=[ \t]*(?=%|\#\#)) # an eval or line-based
                                             # comment preceded by a
                                             # consumed newline and whitespace
                 |
                 (?=\${)      # an expression
                 |
                 (?=</?[%&])  # a substitution or block or call start or end
                              # - don't consume
                 |
                 (\\\r?\n)    # an escaped newline  - throw away
                 |
                 \Z           # end of string
                )""",
            re.X | re.S,
        )

        if match:
            text = match.group(1)
            if text:
                self.append_node(parsetree.Text, text)
            return True
        else:
            return False

    def match_python_block(self):
        match = self.match(r"<%(!)?")
        if match:
            line, pos = self.matched_lineno, self.matched_charpos
            text, end = self.parse_until_text(False, r"%>")
            # the trailing newline helps
            # compiler.parse() not complain about indentation
            text = adjust_whitespace(text) + "\n"
            self.append_node(
                parsetree.Code,
                text,
                match.group(1) == "!",
                lineno=line,
                pos=pos,
            )
            return True
        else:
            return False

    def match_expression(self):
        match = self.match(r"\${")
        if match:
            line, pos = self.matched_lineno, self.matched_charpos
            text, end = self.parse_until_text(True, r"\|", r"}")
            if end == "|":
                escapes, end = self.parse_until_text(True, r"}")
            else:
                escapes = ""
            text = text.replace("\r\n", "\n")
            self.append_node(
                parsetree.Expression,
                text,
                escapes.strip(),
                lineno=line,
                pos=pos,
            )
            return True
        else:
            return False

    def match_control_line(self):
        match = self.match(
            r"(?<=^)[\t ]*(%(?!%)|##)[\t ]*((?:(?:\\r?\n)|[^\r\n])*)"
            r"(?:\r?\n|\Z)",
            re.M,
        )
        if match:
            operator = match.group(1)
            text = match.group(2)
            if operator == "%":
                m2 = re.match(r"(end)?(\w+)\s*(.*)", text)
                if not m2:
                    raise exceptions.SyntaxException(
                        "Invalid control line: '%s'" % text,
                        **self.exception_kwargs
                    )
                isend, keyword = m2.group(1, 2)
                isend = isend is not None

                if isend:
                    if not len(self.control_line):
                        raise exceptions.SyntaxException(
                            "No starting keyword '%s' for '%s'"
                            % (keyword, text),
                            **self.exception_kwargs
                        )
                    elif self.control_line[-1].keyword != keyword:
                        raise exceptions.SyntaxException(
                            "Keyword '%s' doesn't match keyword '%s'"
                            % (text, self.control_line[-1].keyword),
                            **self.exception_kwargs
                        )
                self.append_node(parsetree.ControlLine, keyword, isend, text)
            else:
                self.append_node(parsetree.Comment, text)
            return True
        else:
            return False

    def match_comment(self):
        """matches the multiline version of a comment"""
        match = self.match(r"<%doc>(.*?)</%doc>", re.S)
        if match:
            self.append_node(parsetree.Comment, match.group(1))
            return True
        else:
            return False