jinja2.parser

   1"""Parse tokens from the lexer into nodes for the compiler."""
   2
   3import typing
   4import typing as t
   5
   6from . import nodes
   7from .exceptions import TemplateAssertionError
   8from .exceptions import TemplateSyntaxError
   9from .lexer import describe_token
  10from .lexer import describe_token_expr
  11
  12if t.TYPE_CHECKING:
  13    import typing_extensions as te
  14
  15    from .environment import Environment
  16
  17_ImportInclude = t.TypeVar("_ImportInclude", nodes.Import, nodes.Include)
  18_MacroCall = t.TypeVar("_MacroCall", nodes.Macro, nodes.CallBlock)
  19
  20_statement_keywords = frozenset(
  21    [
  22        "for",
  23        "if",
  24        "block",
  25        "extends",
  26        "print",
  27        "macro",
  28        "include",
  29        "from",
  30        "import",
  31        "set",
  32        "with",
  33        "autoescape",
  34    ]
  35)
  36_compare_operators = frozenset(["eq", "ne", "lt", "lteq", "gt", "gteq"])
  37
  38_math_nodes: t.Dict[str, t.Type[nodes.Expr]] = {
  39    "add": nodes.Add,
  40    "sub": nodes.Sub,
  41    "mul": nodes.Mul,
  42    "div": nodes.Div,
  43    "floordiv": nodes.FloorDiv,
  44    "mod": nodes.Mod,
  45}
  46
  47
  48class Parser:
  49    """This is the central parsing class Jinja uses.  It's passed to
  50    extensions and can be used to parse expressions or statements.
  51    """
  52
  53    def __init__(
  54        self,
  55        environment: "Environment",
  56        source: str,
  57        name: t.Optional[str] = None,
  58        filename: t.Optional[str] = None,
  59        state: t.Optional[str] = None,
  60    ) -> None:
  61        self.environment = environment
  62        self.stream = environment._tokenize(source, name, filename, state)
  63        self.name = name
  64        self.filename = filename
  65        self.closed = False
  66        self.extensions: t.Dict[
  67            str, t.Callable[[Parser], t.Union[nodes.Node, t.List[nodes.Node]]]
  68        ] = {}
  69        for extension in environment.iter_extensions():
  70            for tag in extension.tags:
  71                self.extensions[tag] = extension.parse
  72        self._last_identifier = 0
  73        self._tag_stack: t.List[str] = []
  74        self._end_token_stack: t.List[t.Tuple[str, ...]] = []
  75
  76    def fail(
  77        self,
  78        msg: str,
  79        lineno: t.Optional[int] = None,
  80        exc: t.Type[TemplateSyntaxError] = TemplateSyntaxError,
  81    ) -> "te.NoReturn":
  82        """Convenience method that raises `exc` with the message, passed
  83        line number or last line number as well as the current name and
  84        filename.
  85        """
  86        if lineno is None:
  87            lineno = self.stream.current.lineno
  88        raise exc(msg, lineno, self.name, self.filename)
  89
  90    def _fail_ut_eof(
  91        self,
  92        name: t.Optional[str],
  93        end_token_stack: t.List[t.Tuple[str, ...]],
  94        lineno: t.Optional[int],
  95    ) -> "te.NoReturn":
  96        expected: t.Set[str] = set()
  97        for exprs in end_token_stack:
  98            expected.update(map(describe_token_expr, exprs))
  99        if end_token_stack:
 100            currently_looking: t.Optional[str] = " or ".join(
 101                map(repr, map(describe_token_expr, end_token_stack[-1]))
 102            )
 103        else:
 104            currently_looking = None
 105
 106        if name is None:
 107            message = ["Unexpected end of template."]
 108        else:
 109            message = [f"Encountered unknown tag {name!r}."]
 110
 111        if currently_looking:
 112            if name is not None and name in expected:
 113                message.append(
 114                    "You probably made a nesting mistake. Jinja is expecting this tag,"
 115                    f" but currently looking for {currently_looking}."
 116                )
 117            else:
 118                message.append(
 119                    f"Jinja was looking for the following tags: {currently_looking}."
 120                )
 121
 122        if self._tag_stack:
 123            message.append(
 124                "The innermost block that needs to be closed is"
 125                f" {self._tag_stack[-1]!r}."
 126            )
 127
 128        self.fail(" ".join(message), lineno)
 129
 130    def fail_unknown_tag(
 131        self, name: str, lineno: t.Optional[int] = None
 132    ) -> "te.NoReturn":
 133        """Called if the parser encounters an unknown tag.  Tries to fail
 134        with a human readable error message that could help to identify
 135        the problem.
 136        """
 137        self._fail_ut_eof(name, self._end_token_stack, lineno)
 138
 139    def fail_eof(
 140        self,
 141        end_tokens: t.Optional[t.Tuple[str, ...]] = None,
 142        lineno: t.Optional[int] = None,
 143    ) -> "te.NoReturn":
 144        """Like fail_unknown_tag but for end of template situations."""
 145        stack = list(self._end_token_stack)
 146        if end_tokens is not None:
 147            stack.append(end_tokens)
 148        self._fail_ut_eof(None, stack, lineno)
 149
 150    def is_tuple_end(
 151        self, extra_end_rules: t.Optional[t.Tuple[str, ...]] = None
 152    ) -> bool:
 153        """Are we at the end of a tuple?"""
 154        if self.stream.current.type in ("variable_end", "block_end", "rparen"):
 155            return True
 156        elif extra_end_rules is not None:
 157            return self.stream.current.test_any(extra_end_rules)  # type: ignore
 158        return False
 159
 160    def free_identifier(self, lineno: t.Optional[int] = None) -> nodes.InternalName:
 161        """Return a new free identifier as :class:`~jinja2.nodes.InternalName`."""
 162        self._last_identifier += 1
 163        rv = object.__new__(nodes.InternalName)
 164        nodes.Node.__init__(rv, f"fi{self._last_identifier}", lineno=lineno)
 165        return rv
 166
 167    def parse_statement(self) -> t.Union[nodes.Node, t.List[nodes.Node]]:
 168        """Parse a single statement."""
 169        token = self.stream.current
 170        if token.type != "name":
 171            self.fail("tag name expected", token.lineno)
 172        self._tag_stack.append(token.value)
 173        pop_tag = True
 174        try:
 175            if token.value in _statement_keywords:
 176                f = getattr(self, f"parse_{self.stream.current.value}")
 177                return f()  # type: ignore
 178            if token.value == "call":
 179                return self.parse_call_block()
 180            if token.value == "filter":
 181                return self.parse_filter_block()
 182            ext = self.extensions.get(token.value)
 183            if ext is not None:
 184                return ext(self)
 185
 186            # did not work out, remove the token we pushed by accident
 187            # from the stack so that the unknown tag fail function can
 188            # produce a proper error message.
 189            self._tag_stack.pop()
 190            pop_tag = False
 191            self.fail_unknown_tag(token.value, token.lineno)
 192        finally:
 193            if pop_tag:
 194                self._tag_stack.pop()
 195
 196    def parse_statements(
 197        self, end_tokens: t.Tuple[str, ...], drop_needle: bool = False
 198    ) -> t.List[nodes.Node]:
 199        """Parse multiple statements into a list until one of the end tokens
 200        is reached.  This is used to parse the body of statements as it also
 201        parses template data if appropriate.  The parser checks first if the
 202        current token is a colon and skips it if there is one.  Then it checks
 203        for the block end and parses until if one of the `end_tokens` is
 204        reached.  Per default the active token in the stream at the end of
 205        the call is the matched end token.  If this is not wanted `drop_needle`
 206        can be set to `True` and the end token is removed.
 207        """
 208        # the first token may be a colon for python compatibility
 209        self.stream.skip_if("colon")
 210
 211        # in the future it would be possible to add whole code sections
 212        # by adding some sort of end of statement token and parsing those here.
 213        self.stream.expect("block_end")
 214        result = self.subparse(end_tokens)
 215
 216        # we reached the end of the template too early, the subparser
 217        # does not check for this, so we do that now
 218        if self.stream.current.type == "eof":
 219            self.fail_eof(end_tokens)
 220
 221        if drop_needle:
 222            next(self.stream)
 223        return result
 224
 225    def parse_set(self) -> t.Union[nodes.Assign, nodes.AssignBlock]:
 226        """Parse an assign statement."""
 227        lineno = next(self.stream).lineno
 228        target = self.parse_assign_target(with_namespace=True)
 229        if self.stream.skip_if("assign"):
 230            expr = self.parse_tuple()
 231            return nodes.Assign(target, expr, lineno=lineno)
 232        filter_node = self.parse_filter(None)
 233        body = self.parse_statements(("name:endset",), drop_needle=True)
 234        return nodes.AssignBlock(target, filter_node, body, lineno=lineno)
 235
 236    def parse_for(self) -> nodes.For:
 237        """Parse a for loop."""
 238        lineno = self.stream.expect("name:for").lineno
 239        target = self.parse_assign_target(extra_end_rules=("name:in",))
 240        self.stream.expect("name:in")
 241        iter = self.parse_tuple(
 242            with_condexpr=False, extra_end_rules=("name:recursive",)
 243        )
 244        test = None
 245        if self.stream.skip_if("name:if"):
 246            test = self.parse_expression()
 247        recursive = self.stream.skip_if("name:recursive")
 248        body = self.parse_statements(("name:endfor", "name:else"))
 249        if next(self.stream).value == "endfor":
 250            else_ = []
 251        else:
 252            else_ = self.parse_statements(("name:endfor",), drop_needle=True)
 253        return nodes.For(target, iter, body, else_, test, recursive, lineno=lineno)
 254
 255    def parse_if(self) -> nodes.If:
 256        """Parse an if construct."""
 257        node = result = nodes.If(lineno=self.stream.expect("name:if").lineno)
 258        while True:
 259            node.test = self.parse_tuple(with_condexpr=False)
 260            node.body = self.parse_statements(("name:elif", "name:else", "name:endif"))
 261            node.elif_ = []
 262            node.else_ = []
 263            token = next(self.stream)
 264            if token.test("name:elif"):
 265                node = nodes.If(lineno=self.stream.current.lineno)
 266                result.elif_.append(node)
 267                continue
 268            elif token.test("name:else"):
 269                result.else_ = self.parse_statements(("name:endif",), drop_needle=True)
 270            break
 271        return result
 272
 273    def parse_with(self) -> nodes.With:
 274        node = nodes.With(lineno=next(self.stream).lineno)
 275        targets: t.List[nodes.Expr] = []
 276        values: t.List[nodes.Expr] = []
 277        while self.stream.current.type != "block_end":
 278            if targets:
 279                self.stream.expect("comma")
 280            target = self.parse_assign_target()
 281            target.set_ctx("param")
 282            targets.append(target)
 283            self.stream.expect("assign")
 284            values.append(self.parse_expression())
 285        node.targets = targets
 286        node.values = values
 287        node.body = self.parse_statements(("name:endwith",), drop_needle=True)
 288        return node
 289
 290    def parse_autoescape(self) -> nodes.Scope:
 291        node = nodes.ScopedEvalContextModifier(lineno=next(self.stream).lineno)
 292        node.options = [nodes.Keyword("autoescape", self.parse_expression())]
 293        node.body = self.parse_statements(("name:endautoescape",), drop_needle=True)
 294        return nodes.Scope([node])
 295
 296    def parse_block(self) -> nodes.Block:
 297        node = nodes.Block(lineno=next(self.stream).lineno)
 298        node.name = self.stream.expect("name").value
 299        node.scoped = self.stream.skip_if("name:scoped")
 300        node.required = self.stream.skip_if("name:required")
 301
 302        # common problem people encounter when switching from django
 303        # to jinja.  we do not support hyphens in block names, so let's
 304        # raise a nicer error message in that case.
 305        if self.stream.current.type == "sub":
 306            self.fail(
 307                "Block names in Jinja have to be valid Python identifiers and may not"
 308                " contain hyphens, use an underscore instead."
 309            )
 310
 311        node.body = self.parse_statements(("name:endblock",), drop_needle=True)
 312
 313        # enforce that required blocks only contain whitespace or comments
 314        # by asserting that the body, if not empty, is just TemplateData nodes
 315        # with whitespace data
 316        if node.required:
 317            for body_node in node.body:
 318                if not isinstance(body_node, nodes.Output) or any(
 319                    not isinstance(output_node, nodes.TemplateData)
 320                    or not output_node.data.isspace()
 321                    for output_node in body_node.nodes
 322                ):
 323                    self.fail("Required blocks can only contain comments or whitespace")
 324
 325        self.stream.skip_if("name:" + node.name)
 326        return node
 327
 328    def parse_extends(self) -> nodes.Extends:
 329        node = nodes.Extends(lineno=next(self.stream).lineno)
 330        node.template = self.parse_expression()
 331        return node
 332
 333    def parse_import_context(
 334        self, node: _ImportInclude, default: bool
 335    ) -> _ImportInclude:
 336        if self.stream.current.test_any(
 337            "name:with", "name:without"
 338        ) and self.stream.look().test("name:context"):
 339            node.with_context = next(self.stream).value == "with"
 340            self.stream.skip()
 341        else:
 342            node.with_context = default
 343        return node
 344
 345    def parse_include(self) -> nodes.Include:
 346        node = nodes.Include(lineno=next(self.stream).lineno)
 347        node.template = self.parse_expression()
 348        if self.stream.current.test("name:ignore") and self.stream.look().test(
 349            "name:missing"
 350        ):
 351            node.ignore_missing = True
 352            self.stream.skip(2)
 353        else:
 354            node.ignore_missing = False
 355        return self.parse_import_context(node, True)
 356
 357    def parse_import(self) -> nodes.Import:
 358        node = nodes.Import(lineno=next(self.stream).lineno)
 359        node.template = self.parse_expression()
 360        self.stream.expect("name:as")
 361        node.target = self.parse_assign_target(name_only=True).name
 362        return self.parse_import_context(node, False)
 363
 364    def parse_from(self) -> nodes.FromImport:
 365        node = nodes.FromImport(lineno=next(self.stream).lineno)
 366        node.template = self.parse_expression()
 367        self.stream.expect("name:import")
 368        node.names = []
 369
 370        def parse_context() -> bool:
 371            if self.stream.current.value in {
 372                "with",
 373                "without",
 374            } and self.stream.look().test("name:context"):
 375                node.with_context = next(self.stream).value == "with"
 376                self.stream.skip()
 377                return True
 378            return False
 379
 380        while True:
 381            if node.names:
 382                self.stream.expect("comma")
 383            if self.stream.current.type == "name":
 384                if parse_context():
 385                    break
 386                target = self.parse_assign_target(name_only=True)
 387                if target.name.startswith("_"):
 388                    self.fail(
 389                        "names starting with an underline can not be imported",
 390                        target.lineno,
 391                        exc=TemplateAssertionError,
 392                    )
 393                if self.stream.skip_if("name:as"):
 394                    alias = self.parse_assign_target(name_only=True)
 395                    node.names.append((target.name, alias.name))
 396                else:
 397                    node.names.append(target.name)
 398                if parse_context() or self.stream.current.type != "comma":
 399                    break
 400            else:
 401                self.stream.expect("name")
 402        if not hasattr(node, "with_context"):
 403            node.with_context = False
 404        return node
 405
 406    def parse_signature(self, node: _MacroCall) -> None:
 407        args = node.args = []
 408        defaults = node.defaults = []
 409        self.stream.expect("lparen")
 410        while self.stream.current.type != "rparen":
 411            if args:
 412                self.stream.expect("comma")
 413            arg = self.parse_assign_target(name_only=True)
 414            arg.set_ctx("param")
 415            if self.stream.skip_if("assign"):
 416                defaults.append(self.parse_expression())
 417            elif defaults:
 418                self.fail("non-default argument follows default argument")
 419            args.append(arg)
 420        self.stream.expect("rparen")
 421
 422    def parse_call_block(self) -> nodes.CallBlock:
 423        node = nodes.CallBlock(lineno=next(self.stream).lineno)
 424        if self.stream.current.type == "lparen":
 425            self.parse_signature(node)
 426        else:
 427            node.args = []
 428            node.defaults = []
 429
 430        call_node = self.parse_expression()
 431        if not isinstance(call_node, nodes.Call):
 432            self.fail("expected call", node.lineno)
 433        node.call = call_node
 434        node.body = self.parse_statements(("name:endcall",), drop_needle=True)
 435        return node
 436
 437    def parse_filter_block(self) -> nodes.FilterBlock:
 438        node = nodes.FilterBlock(lineno=next(self.stream).lineno)
 439        node.filter = self.parse_filter(None, start_inline=True)  # type: ignore
 440        node.body = self.parse_statements(("name:endfilter",), drop_needle=True)
 441        return node
 442
 443    def parse_macro(self) -> nodes.Macro:
 444        node = nodes.Macro(lineno=next(self.stream).lineno)
 445        node.name = self.parse_assign_target(name_only=True).name
 446        self.parse_signature(node)
 447        node.body = self.parse_statements(("name:endmacro",), drop_needle=True)
 448        return node
 449
 450    def parse_print(self) -> nodes.Output:
 451        node = nodes.Output(lineno=next(self.stream).lineno)
 452        node.nodes = []
 453        while self.stream.current.type != "block_end":
 454            if node.nodes:
 455                self.stream.expect("comma")
 456            node.nodes.append(self.parse_expression())
 457        return node
 458
 459    @typing.overload
 460    def parse_assign_target(
 461        self, with_tuple: bool = ..., name_only: "te.Literal[True]" = ...
 462    ) -> nodes.Name: ...
 463
 464    @typing.overload
 465    def parse_assign_target(
 466        self,
 467        with_tuple: bool = True,
 468        name_only: bool = False,
 469        extra_end_rules: t.Optional[t.Tuple[str, ...]] = None,
 470        with_namespace: bool = False,
 471    ) -> t.Union[nodes.NSRef, nodes.Name, nodes.Tuple]: ...
 472
 473    def parse_assign_target(
 474        self,
 475        with_tuple: bool = True,
 476        name_only: bool = False,
 477        extra_end_rules: t.Optional[t.Tuple[str, ...]] = None,
 478        with_namespace: bool = False,
 479    ) -> t.Union[nodes.NSRef, nodes.Name, nodes.Tuple]:
 480        """Parse an assignment target.  As Jinja allows assignments to
 481        tuples, this function can parse all allowed assignment targets.  Per
 482        default assignments to tuples are parsed, that can be disable however
 483        by setting `with_tuple` to `False`.  If only assignments to names are
 484        wanted `name_only` can be set to `True`.  The `extra_end_rules`
 485        parameter is forwarded to the tuple parsing function.  If
 486        `with_namespace` is enabled, a namespace assignment may be parsed.
 487        """
 488        target: nodes.Expr
 489
 490        if name_only:
 491            token = self.stream.expect("name")
 492            target = nodes.Name(token.value, "store", lineno=token.lineno)
 493        else:
 494            if with_tuple:
 495                target = self.parse_tuple(
 496                    simplified=True,
 497                    extra_end_rules=extra_end_rules,
 498                    with_namespace=with_namespace,
 499                )
 500            else:
 501                target = self.parse_primary(with_namespace=with_namespace)
 502
 503            target.set_ctx("store")
 504
 505        if not target.can_assign():
 506            self.fail(
 507                f"can't assign to {type(target).__name__.lower()!r}", target.lineno
 508            )
 509
 510        return target  # type: ignore
 511
 512    def parse_expression(self, with_condexpr: bool = True) -> nodes.Expr:
 513        """Parse an expression.  Per default all expressions are parsed, if
 514        the optional `with_condexpr` parameter is set to `False` conditional
 515        expressions are not parsed.
 516        """
 517        if with_condexpr:
 518            return self.parse_condexpr()
 519        return self.parse_or()
 520
 521    def parse_condexpr(self) -> nodes.Expr:
 522        lineno = self.stream.current.lineno
 523        expr1 = self.parse_or()
 524        expr3: t.Optional[nodes.Expr]
 525
 526        while self.stream.skip_if("name:if"):
 527            expr2 = self.parse_or()
 528            if self.stream.skip_if("name:else"):
 529                expr3 = self.parse_condexpr()
 530            else:
 531                expr3 = None
 532            expr1 = nodes.CondExpr(expr2, expr1, expr3, lineno=lineno)
 533            lineno = self.stream.current.lineno
 534        return expr1
 535
 536    def parse_or(self) -> nodes.Expr:
 537        lineno = self.stream.current.lineno
 538        left = self.parse_and()
 539        while self.stream.skip_if("name:or"):
 540            right = self.parse_and()
 541            left = nodes.Or(left, right, lineno=lineno)
 542            lineno = self.stream.current.lineno
 543        return left
 544
 545    def parse_and(self) -> nodes.Expr:
 546        lineno = self.stream.current.lineno
 547        left = self.parse_not()
 548        while self.stream.skip_if("name:and"):
 549            right = self.parse_not()
 550            left = nodes.And(left, right, lineno=lineno)
 551            lineno = self.stream.current.lineno
 552        return left
 553
 554    def parse_not(self) -> nodes.Expr:
 555        if self.stream.current.test("name:not"):
 556            lineno = next(self.stream).lineno
 557            return nodes.Not(self.parse_not(), lineno=lineno)
 558        return self.parse_compare()
 559
 560    def parse_compare(self) -> nodes.Expr:
 561        lineno = self.stream.current.lineno
 562        expr = self.parse_math1()
 563        ops = []
 564        while True:
 565            token_type = self.stream.current.type
 566            if token_type in _compare_operators:
 567                next(self.stream)
 568                ops.append(nodes.Operand(token_type, self.parse_math1()))
 569            elif self.stream.skip_if("name:in"):
 570                ops.append(nodes.Operand("in", self.parse_math1()))
 571            elif self.stream.current.test("name:not") and self.stream.look().test(
 572                "name:in"
 573            ):
 574                self.stream.skip(2)
 575                ops.append(nodes.Operand("notin", self.parse_math1()))
 576            else:
 577                break
 578            lineno = self.stream.current.lineno
 579        if not ops:
 580            return expr
 581        return nodes.Compare(expr, ops, lineno=lineno)
 582
 583    def parse_math1(self) -> nodes.Expr:
 584        lineno = self.stream.current.lineno
 585        left = self.parse_concat()
 586        while self.stream.current.type in ("add", "sub"):
 587            cls = _math_nodes[self.stream.current.type]
 588            next(self.stream)
 589            right = self.parse_concat()
 590            left = cls(left, right, lineno=lineno)
 591            lineno = self.stream.current.lineno
 592        return left
 593
 594    def parse_concat(self) -> nodes.Expr:
 595        lineno = self.stream.current.lineno
 596        args = [self.parse_math2()]
 597        while self.stream.current.type == "tilde":
 598            next(self.stream)
 599            args.append(self.parse_math2())
 600        if len(args) == 1:
 601            return args[0]
 602        return nodes.Concat(args, lineno=lineno)
 603
 604    def parse_math2(self) -> nodes.Expr:
 605        lineno = self.stream.current.lineno
 606        left = self.parse_pow()
 607        while self.stream.current.type in ("mul", "div", "floordiv", "mod"):
 608            cls = _math_nodes[self.stream.current.type]
 609            next(self.stream)
 610            right = self.parse_pow()
 611            left = cls(left, right, lineno=lineno)
 612            lineno = self.stream.current.lineno
 613        return left
 614
 615    def parse_pow(self) -> nodes.Expr:
 616        lineno = self.stream.current.lineno
 617        left = self.parse_unary()
 618        while self.stream.current.type == "pow":
 619            next(self.stream)
 620            right = self.parse_unary()
 621            left = nodes.Pow(left, right, lineno=lineno)
 622            lineno = self.stream.current.lineno
 623        return left
 624
 625    def parse_unary(self, with_filter: bool = True) -> nodes.Expr:
 626        token_type = self.stream.current.type
 627        lineno = self.stream.current.lineno
 628        node: nodes.Expr
 629
 630        if token_type == "sub":
 631            next(self.stream)
 632            node = nodes.Neg(self.parse_unary(False), lineno=lineno)
 633        elif token_type == "add":
 634            next(self.stream)
 635            node = nodes.Pos(self.parse_unary(False), lineno=lineno)
 636        else:
 637            node = self.parse_primary()
 638        node = self.parse_postfix(node)
 639        if with_filter:
 640            node = self.parse_filter_expr(node)
 641        return node
 642
 643    def parse_primary(self, with_namespace: bool = False) -> nodes.Expr:
 644        """Parse a name or literal value. If ``with_namespace`` is enabled, also
 645        parse namespace attr refs, for use in assignments."""
 646        token = self.stream.current
 647        node: nodes.Expr
 648        if token.type == "name":
 649            next(self.stream)
 650            if token.value in ("true", "false", "True", "False"):
 651                node = nodes.Const(token.value in ("true", "True"), lineno=token.lineno)
 652            elif token.value in ("none", "None"):
 653                node = nodes.Const(None, lineno=token.lineno)
 654            elif with_namespace and self.stream.current.type == "dot":
 655                # If namespace attributes are allowed at this point, and the next
 656                # token is a dot, produce a namespace reference.
 657                next(self.stream)
 658                attr = self.stream.expect("name")
 659                node = nodes.NSRef(token.value, attr.value, lineno=token.lineno)
 660            else:
 661                node = nodes.Name(token.value, "load", lineno=token.lineno)
 662        elif token.type == "string":
 663            next(self.stream)
 664            buf = [token.value]
 665            lineno = token.lineno
 666            while self.stream.current.type == "string":
 667                buf.append(self.stream.current.value)
 668                next(self.stream)
 669            node = nodes.Const("".join(buf), lineno=lineno)
 670        elif token.type in ("integer", "float"):
 671            next(self.stream)
 672            node = nodes.Const(token.value, lineno=token.lineno)
 673        elif token.type == "lparen":
 674            next(self.stream)
 675            node = self.parse_tuple(explicit_parentheses=True)
 676            self.stream.expect("rparen")
 677        elif token.type == "lbracket":
 678            node = self.parse_list()
 679        elif token.type == "lbrace":
 680            node = self.parse_dict()
 681        else:
 682            self.fail(f"unexpected {describe_token(token)!r}", token.lineno)
 683        return node
 684
 685    def parse_tuple(
 686        self,
 687        simplified: bool = False,
 688        with_condexpr: bool = True,
 689        extra_end_rules: t.Optional[t.Tuple[str, ...]] = None,
 690        explicit_parentheses: bool = False,
 691        with_namespace: bool = False,
 692    ) -> t.Union[nodes.Tuple, nodes.Expr]:
 693        """Works like `parse_expression` but if multiple expressions are
 694        delimited by a comma a :class:`~jinja2.nodes.Tuple` node is created.
 695        This method could also return a regular expression instead of a tuple
 696        if no commas where found.
 697
 698        The default parsing mode is a full tuple.  If `simplified` is `True`
 699        only names and literals are parsed; ``with_namespace`` allows namespace
 700        attr refs as well. The `no_condexpr` parameter is forwarded to
 701        :meth:`parse_expression`.
 702
 703        Because tuples do not require delimiters and may end in a bogus comma
 704        an extra hint is needed that marks the end of a tuple.  For example
 705        for loops support tuples between `for` and `in`.  In that case the
 706        `extra_end_rules` is set to ``['name:in']``.
 707
 708        `explicit_parentheses` is true if the parsing was triggered by an
 709        expression in parentheses.  This is used to figure out if an empty
 710        tuple is a valid expression or not.
 711        """
 712        lineno = self.stream.current.lineno
 713        if simplified:
 714
 715            def parse() -> nodes.Expr:
 716                return self.parse_primary(with_namespace=with_namespace)
 717
 718        else:
 719
 720            def parse() -> nodes.Expr:
 721                return self.parse_expression(with_condexpr=with_condexpr)
 722
 723        args: t.List[nodes.Expr] = []
 724        is_tuple = False
 725
 726        while True:
 727            if args:
 728                self.stream.expect("comma")
 729            if self.is_tuple_end(extra_end_rules):
 730                break
 731            args.append(parse())
 732            if self.stream.current.type == "comma":
 733                is_tuple = True
 734            else:
 735                break
 736            lineno = self.stream.current.lineno
 737
 738        if not is_tuple:
 739            if args:
 740                return args[0]
 741
 742            # if we don't have explicit parentheses, an empty tuple is
 743            # not a valid expression.  This would mean nothing (literally
 744            # nothing) in the spot of an expression would be an empty
 745            # tuple.
 746            if not explicit_parentheses:
 747                self.fail(
 748                    "Expected an expression,"
 749                    f" got {describe_token(self.stream.current)!r}"
 750                )
 751
 752        return nodes.Tuple(args, "load", lineno=lineno)
 753
 754    def parse_list(self) -> nodes.List:
 755        token = self.stream.expect("lbracket")
 756        items: t.List[nodes.Expr] = []
 757        while self.stream.current.type != "rbracket":
 758            if items:
 759                self.stream.expect("comma")
 760            if self.stream.current.type == "rbracket":
 761                break
 762            items.append(self.parse_expression())
 763        self.stream.expect("rbracket")
 764        return nodes.List(items, lineno=token.lineno)
 765
 766    def parse_dict(self) -> nodes.Dict:
 767        token = self.stream.expect("lbrace")
 768        items: t.List[nodes.Pair] = []
 769        while self.stream.current.type != "rbrace":
 770            if items:
 771                self.stream.expect("comma")
 772            if self.stream.current.type == "rbrace":
 773                break
 774            key = self.parse_expression()
 775            self.stream.expect("colon")
 776            value = self.parse_expression()
 777            items.append(nodes.Pair(key, value, lineno=key.lineno))
 778        self.stream.expect("rbrace")
 779        return nodes.Dict(items, lineno=token.lineno)
 780
 781    def parse_postfix(self, node: nodes.Expr) -> nodes.Expr:
 782        while True:
 783            token_type = self.stream.current.type
 784            if token_type == "dot" or token_type == "lbracket":
 785                node = self.parse_subscript(node)
 786            # calls are valid both after postfix expressions (getattr
 787            # and getitem) as well as filters and tests
 788            elif token_type == "lparen":
 789                node = self.parse_call(node)
 790            else:
 791                break
 792        return node
 793
 794    def parse_filter_expr(self, node: nodes.Expr) -> nodes.Expr:
 795        while True:
 796            token_type = self.stream.current.type
 797            if token_type == "pipe":
 798                node = self.parse_filter(node)  # type: ignore
 799            elif token_type == "name" and self.stream.current.value == "is":
 800                node = self.parse_test(node)
 801            # calls are valid both after postfix expressions (getattr
 802            # and getitem) as well as filters and tests
 803            elif token_type == "lparen":
 804                node = self.parse_call(node)
 805            else:
 806                break
 807        return node
 808
 809    def parse_subscript(
 810        self, node: nodes.Expr
 811    ) -> t.Union[nodes.Getattr, nodes.Getitem]:
 812        token = next(self.stream)
 813        arg: nodes.Expr
 814
 815        if token.type == "dot":
 816            attr_token = self.stream.current
 817            next(self.stream)
 818            if attr_token.type == "name":
 819                return nodes.Getattr(
 820                    node, attr_token.value, "load", lineno=token.lineno
 821                )
 822            elif attr_token.type != "integer":
 823                self.fail("expected name or number", attr_token.lineno)
 824            arg = nodes.Const(attr_token.value, lineno=attr_token.lineno)
 825            return nodes.Getitem(node, arg, "load", lineno=token.lineno)
 826        if token.type == "lbracket":
 827            args: t.List[nodes.Expr] = []
 828            while self.stream.current.type != "rbracket":
 829                if args:
 830                    self.stream.expect("comma")
 831                args.append(self.parse_subscribed())
 832            self.stream.expect("rbracket")
 833            if len(args) == 1:
 834                arg = args[0]
 835            else:
 836                arg = nodes.Tuple(args, "load", lineno=token.lineno)
 837            return nodes.Getitem(node, arg, "load", lineno=token.lineno)
 838        self.fail("expected subscript expression", token.lineno)
 839
 840    def parse_subscribed(self) -> nodes.Expr:
 841        lineno = self.stream.current.lineno
 842        args: t.List[t.Optional[nodes.Expr]]
 843
 844        if self.stream.current.type == "colon":
 845            next(self.stream)
 846            args = [None]
 847        else:
 848            node = self.parse_expression()
 849            if self.stream.current.type != "colon":
 850                return node
 851            next(self.stream)
 852            args = [node]
 853
 854        if self.stream.current.type == "colon":
 855            args.append(None)
 856        elif self.stream.current.type not in ("rbracket", "comma"):
 857            args.append(self.parse_expression())
 858        else:
 859            args.append(None)
 860
 861        if self.stream.current.type == "colon":
 862            next(self.stream)
 863            if self.stream.current.type not in ("rbracket", "comma"):
 864                args.append(self.parse_expression())
 865            else:
 866                args.append(None)
 867        else:
 868            args.append(None)
 869
 870        return nodes.Slice(lineno=lineno, *args)  # noqa: B026
 871
 872    def parse_call_args(
 873        self,
 874    ) -> t.Tuple[
 875        t.List[nodes.Expr],
 876        t.List[nodes.Keyword],
 877        t.Optional[nodes.Expr],
 878        t.Optional[nodes.Expr],
 879    ]:
 880        token = self.stream.expect("lparen")
 881        args = []
 882        kwargs = []
 883        dyn_args = None
 884        dyn_kwargs = None
 885        require_comma = False
 886
 887        def ensure(expr: bool) -> None:
 888            if not expr:
 889                self.fail("invalid syntax for function call expression", token.lineno)
 890
 891        while self.stream.current.type != "rparen":
 892            if require_comma:
 893                self.stream.expect("comma")
 894
 895                # support for trailing comma
 896                if self.stream.current.type == "rparen":
 897                    break
 898
 899            if self.stream.current.type == "mul":
 900                ensure(dyn_args is None and dyn_kwargs is None)
 901                next(self.stream)
 902                dyn_args = self.parse_expression()
 903            elif self.stream.current.type == "pow":
 904                ensure(dyn_kwargs is None)
 905                next(self.stream)
 906                dyn_kwargs = self.parse_expression()
 907            else:
 908                if (
 909                    self.stream.current.type == "name"
 910                    and self.stream.look().type == "assign"
 911                ):
 912                    # Parsing a kwarg
 913                    ensure(dyn_kwargs is None)
 914                    key = self.stream.current.value
 915                    self.stream.skip(2)
 916                    value = self.parse_expression()
 917                    kwargs.append(nodes.Keyword(key, value, lineno=value.lineno))
 918                else:
 919                    # Parsing an arg
 920                    ensure(dyn_args is None and dyn_kwargs is None and not kwargs)
 921                    args.append(self.parse_expression())
 922
 923            require_comma = True
 924
 925        self.stream.expect("rparen")
 926        return args, kwargs, dyn_args, dyn_kwargs
 927
 928    def parse_call(self, node: nodes.Expr) -> nodes.Call:
 929        # The lparen will be expected in parse_call_args, but the lineno
 930        # needs to be recorded before the stream is advanced.
 931        token = self.stream.current
 932        args, kwargs, dyn_args, dyn_kwargs = self.parse_call_args()
 933        return nodes.Call(node, args, kwargs, dyn_args, dyn_kwargs, lineno=token.lineno)
 934
 935    def parse_filter(
 936        self, node: t.Optional[nodes.Expr], start_inline: bool = False
 937    ) -> t.Optional[nodes.Expr]:
 938        while self.stream.current.type == "pipe" or start_inline:
 939            if not start_inline:
 940                next(self.stream)
 941            token = self.stream.expect("name")
 942            name = token.value
 943            while self.stream.current.type == "dot":
 944                next(self.stream)
 945                name += "." + self.stream.expect("name").value
 946            if self.stream.current.type == "lparen":
 947                args, kwargs, dyn_args, dyn_kwargs = self.parse_call_args()
 948            else:
 949                args = []
 950                kwargs = []
 951                dyn_args = dyn_kwargs = None
 952            node = nodes.Filter(
 953                node, name, args, kwargs, dyn_args, dyn_kwargs, lineno=token.lineno
 954            )
 955            start_inline = False
 956        return node
 957
 958    def parse_test(self, node: nodes.Expr) -> nodes.Expr:
 959        token = next(self.stream)
 960        if self.stream.current.test("name:not"):
 961            next(self.stream)
 962            negated = True
 963        else:
 964            negated = False
 965        name = self.stream.expect("name").value
 966        while self.stream.current.type == "dot":
 967            next(self.stream)
 968            name += "." + self.stream.expect("name").value
 969        dyn_args = dyn_kwargs = None
 970        kwargs: t.List[nodes.Keyword] = []
 971        if self.stream.current.type == "lparen":
 972            args, kwargs, dyn_args, dyn_kwargs = self.parse_call_args()
 973        elif self.stream.current.type in {
 974            "name",
 975            "string",
 976            "integer",
 977            "float",
 978            "lparen",
 979            "lbracket",
 980            "lbrace",
 981        } and not self.stream.current.test_any("name:else", "name:or", "name:and"):
 982            if self.stream.current.test("name:is"):
 983                self.fail("You cannot chain multiple tests with is")
 984            arg_node = self.parse_primary()
 985            arg_node = self.parse_postfix(arg_node)
 986            args = [arg_node]
 987        else:
 988            args = []
 989        node = nodes.Test(
 990            node, name, args, kwargs, dyn_args, dyn_kwargs, lineno=token.lineno
 991        )
 992        if negated:
 993            node = nodes.Not(node, lineno=token.lineno)
 994        return node
 995
 996    def subparse(
 997        self, end_tokens: t.Optional[t.Tuple[str, ...]] = None
 998    ) -> t.List[nodes.Node]:
 999        body: t.List[nodes.Node] = []
1000        data_buffer: t.List[nodes.Node] = []
1001        add_data = data_buffer.append
1002
1003        if end_tokens is not None:
1004            self._end_token_stack.append(end_tokens)
1005
1006        def flush_data() -> None:
1007            if data_buffer:
1008                lineno = data_buffer[0].lineno
1009                body.append(nodes.Output(data_buffer[:], lineno=lineno))
1010                del data_buffer[:]
1011
1012        try:
1013            while self.stream:
1014                token = self.stream.current
1015                if token.type == "data":
1016                    if token.value:
1017                        add_data(nodes.TemplateData(token.value, lineno=token.lineno))
1018                    next(self.stream)
1019                elif token.type == "variable_begin":
1020                    next(self.stream)
1021                    add_data(self.parse_tuple(with_condexpr=True))
1022                    self.stream.expect("variable_end")
1023                elif token.type == "block_begin":
1024                    flush_data()
1025                    next(self.stream)
1026                    if end_tokens is not None and self.stream.current.test_any(
1027                        *end_tokens
1028                    ):
1029                        return body
1030                    rv = self.parse_statement()
1031                    if isinstance(rv, list):
1032                        body.extend(rv)
1033                    else:
1034                        body.append(rv)
1035                    self.stream.expect("block_end")
1036                else:
1037                    raise AssertionError("internal parsing error")
1038
1039            flush_data()
1040        finally:
1041            if end_tokens is not None:
1042                self._end_token_stack.pop()
1043        return body
1044
1045    def parse(self) -> nodes.Template:
1046        """Parse the whole template into a `Template` node."""
1047        result = nodes.Template(self.subparse(), lineno=1)
1048        result.set_environment(self.environment)
1049        return result