jinja2.loaders

  1"""API and implementations for loading templates from different data
  2sources.
  3"""
  4
  5import importlib.util
  6import os
  7import posixpath
  8import sys
  9import typing as t
 10import weakref
 11import zipimport
 12from collections import abc
 13from hashlib import sha1
 14from importlib import import_module
 15from types import ModuleType
 16
 17from .exceptions import TemplateNotFound
 18from .utils import internalcode
 19
 20if t.TYPE_CHECKING:
 21    from .environment import Environment
 22    from .environment import Template
 23
 24
 25def split_template_path(template: str) -> t.List[str]:
 26    """Split a path into segments and perform a sanity check.  If it detects
 27    '..' in the path it will raise a `TemplateNotFound` error.
 28    """
 29    pieces = []
 30    for piece in template.split("/"):
 31        if (
 32            os.path.sep in piece
 33            or (os.path.altsep and os.path.altsep in piece)
 34            or piece == os.path.pardir
 35        ):
 36            raise TemplateNotFound(template)
 37        elif piece and piece != ".":
 38            pieces.append(piece)
 39    return pieces
 40
 41
 42class BaseLoader:
 43    """Baseclass for all loaders.  Subclass this and override `get_source` to
 44    implement a custom loading mechanism.  The environment provides a
 45    `get_template` method that calls the loader's `load` method to get the
 46    :class:`Template` object.
 47
 48    A very basic example for a loader that looks up templates on the file
 49    system could look like this::
 50
 51        from jinja2 import BaseLoader, TemplateNotFound
 52        from os.path import join, exists, getmtime
 53
 54        class MyLoader(BaseLoader):
 55
 56            def __init__(self, path):
 57                self.path = path
 58
 59            def get_source(self, environment, template):
 60                path = join(self.path, template)
 61                if not exists(path):
 62                    raise TemplateNotFound(template)
 63                mtime = getmtime(path)
 64                with open(path) as f:
 65                    source = f.read()
 66                return source, path, lambda: mtime == getmtime(path)
 67    """
 68
 69    #: if set to `False` it indicates that the loader cannot provide access
 70    #: to the source of templates.
 71    #:
 72    #: .. versionadded:: 2.4
 73    has_source_access = True
 74
 75    def get_source(
 76        self, environment: "Environment", template: str
 77    ) -> t.Tuple[str, t.Optional[str], t.Optional[t.Callable[[], bool]]]:
 78        """Get the template source, filename and reload helper for a template.
 79        It's passed the environment and template name and has to return a
 80        tuple in the form ``(source, filename, uptodate)`` or raise a
 81        `TemplateNotFound` error if it can't locate the template.
 82
 83        The source part of the returned tuple must be the source of the
 84        template as a string. The filename should be the name of the
 85        file on the filesystem if it was loaded from there, otherwise
 86        ``None``. The filename is used by Python for the tracebacks
 87        if no loader extension is used.
 88
 89        The last item in the tuple is the `uptodate` function.  If auto
 90        reloading is enabled it's always called to check if the template
 91        changed.  No arguments are passed so the function must store the
 92        old state somewhere (for example in a closure).  If it returns `False`
 93        the template will be reloaded.
 94        """
 95        if not self.has_source_access:
 96            raise RuntimeError(
 97                f"{type(self).__name__} cannot provide access to the source"
 98            )
 99        raise TemplateNotFound(template)
100
101    def list_templates(self) -> t.List[str]:
102        """Iterates over all templates.  If the loader does not support that
103        it should raise a :exc:`TypeError` which is the default behavior.
104        """
105        raise TypeError("this loader cannot iterate over all templates")
106
107    @internalcode
108    def load(
109        self,
110        environment: "Environment",
111        name: str,
112        globals: t.Optional[t.MutableMapping[str, t.Any]] = None,
113    ) -> "Template":
114        """Loads a template.  This method looks up the template in the cache
115        or loads one by calling :meth:`get_source`.  Subclasses should not
116        override this method as loaders working on collections of other
117        loaders (such as :class:`PrefixLoader` or :class:`ChoiceLoader`)
118        will not call this method but `get_source` directly.
119        """
120        code = None
121        if globals is None:
122            globals = {}
123
124        # first we try to get the source for this template together
125        # with the filename and the uptodate function.
126        source, filename, uptodate = self.get_source(environment, name)
127
128        # try to load the code from the bytecode cache if there is a
129        # bytecode cache configured.
130        bcc = environment.bytecode_cache
131        if bcc is not None:
132            bucket = bcc.get_bucket(environment, name, filename, source)
133            code = bucket.code
134
135        # if we don't have code so far (not cached, no longer up to
136        # date) etc. we compile the template
137        if code is None:
138            code = environment.compile(source, name, filename)
139
140        # if the bytecode cache is available and the bucket doesn't
141        # have a code so far, we give the bucket the new code and put
142        # it back to the bytecode cache.
143        if bcc is not None and bucket.code is None:
144            bucket.code = code
145            bcc.set_bucket(bucket)
146
147        return environment.template_class.from_code(
148            environment, code, globals, uptodate
149        )
150
151
152class FileSystemLoader(BaseLoader):
153    """Load templates from a directory in the file system.
154
155    The path can be relative or absolute. Relative paths are relative to
156    the current working directory.
157
158    .. code-block:: python
159
160        loader = FileSystemLoader("templates")
161
162    A list of paths can be given. The directories will be searched in
163    order, stopping at the first matching template.
164
165    .. code-block:: python
166
167        loader = FileSystemLoader(["/override/templates", "/default/templates"])
168
169    :param searchpath: A path, or list of paths, to the directory that
170        contains the templates.
171    :param encoding: Use this encoding to read the text from template
172        files.
173    :param followlinks: Follow symbolic links in the path.
174
175    .. versionchanged:: 2.8
176        Added the ``followlinks`` parameter.
177    """
178
179    def __init__(
180        self,
181        searchpath: t.Union[
182            str, "os.PathLike[str]", t.Sequence[t.Union[str, "os.PathLike[str]"]]
183        ],
184        encoding: str = "utf-8",
185        followlinks: bool = False,
186    ) -> None:
187        if not isinstance(searchpath, abc.Iterable) or isinstance(searchpath, str):
188            searchpath = [searchpath]
189
190        self.searchpath = [os.fspath(p) for p in searchpath]
191        self.encoding = encoding
192        self.followlinks = followlinks
193
194    def get_source(
195        self, environment: "Environment", template: str
196    ) -> t.Tuple[str, str, t.Callable[[], bool]]:
197        pieces = split_template_path(template)
198
199        for searchpath in self.searchpath:
200            # Use posixpath even on Windows to avoid "drive:" or UNC
201            # segments breaking out of the search directory.
202            filename = posixpath.join(searchpath, *pieces)
203
204            if os.path.isfile(filename):
205                break
206        else:
207            plural = "path" if len(self.searchpath) == 1 else "paths"
208            paths_str = ", ".join(repr(p) for p in self.searchpath)
209            raise TemplateNotFound(
210                template,
211                f"{template!r} not found in search {plural}: {paths_str}",
212            )
213
214        with open(filename, encoding=self.encoding) as f:
215            contents = f.read()
216
217        mtime = os.path.getmtime(filename)
218
219        def uptodate() -> bool:
220            try:
221                return os.path.getmtime(filename) == mtime
222            except OSError:
223                return False
224
225        # Use normpath to convert Windows altsep to sep.
226        return contents, os.path.normpath(filename), uptodate
227
228    def list_templates(self) -> t.List[str]:
229        found = set()
230        for searchpath in self.searchpath:
231            walk_dir = os.walk(searchpath, followlinks=self.followlinks)
232            for dirpath, _, filenames in walk_dir:
233                for filename in filenames:
234                    template = (
235                        os.path.join(dirpath, filename)[len(searchpath) :]
236                        .strip(os.path.sep)
237                        .replace(os.path.sep, "/")
238                    )
239                    if template[:2] == "./":
240                        template = template[2:]
241                    if template not in found:
242                        found.add(template)
243        return sorted(found)
244
245
246if sys.version_info >= (3, 13):
247
248    def _get_zipimporter_files(z: t.Any) -> t.Dict[str, object]:
249        try:
250            get_files = z._get_files
251        except AttributeError as e:
252            raise TypeError(
253                "This zip import does not have the required"
254                " metadata to list templates."
255            ) from e
256        return get_files()
257else:
258
259    def _get_zipimporter_files(z: t.Any) -> t.Dict[str, object]:
260        try:
261            files = z._files
262        except AttributeError as e:
263            raise TypeError(
264                "This zip import does not have the required"
265                " metadata to list templates."
266            ) from e
267        return files  # type: ignore[no-any-return]
268
269
270class PackageLoader(BaseLoader):
271    """Load templates from a directory in a Python package.
272
273    :param package_name: Import name of the package that contains the
274        template directory.
275    :param package_path: Directory within the imported package that
276        contains the templates.
277    :param encoding: Encoding of template files.
278
279    The following example looks up templates in the ``pages`` directory
280    within the ``project.ui`` package.
281
282    .. code-block:: python
283
284        loader = PackageLoader("project.ui", "pages")
285
286    Only packages installed as directories (standard pip behavior) or
287    zip/egg files (less common) are supported. The Python API for
288    introspecting data in packages is too limited to support other
289    installation methods the way this loader requires.
290
291    There is limited support for :pep:`420` namespace packages. The
292    template directory is assumed to only be in one namespace
293    contributor. Zip files contributing to a namespace are not
294    supported.
295
296    .. versionchanged:: 3.0
297        No longer uses ``setuptools`` as a dependency.
298
299    .. versionchanged:: 3.0
300        Limited PEP 420 namespace package support.
301    """
302
303    def __init__(
304        self,
305        package_name: str,
306        package_path: "str" = "templates",
307        encoding: str = "utf-8",
308    ) -> None:
309        package_path = os.path.normpath(package_path).rstrip(os.path.sep)
310
311        # normpath preserves ".", which isn't valid in zip paths.
312        if package_path == os.path.curdir:
313            package_path = ""
314        elif package_path[:2] == os.path.curdir + os.path.sep:
315            package_path = package_path[2:]
316
317        self.package_path = package_path
318        self.package_name = package_name
319        self.encoding = encoding
320
321        # Make sure the package exists. This also makes namespace
322        # packages work, otherwise get_loader returns None.
323        import_module(package_name)
324        spec = importlib.util.find_spec(package_name)
325        assert spec is not None, "An import spec was not found for the package."
326        loader = spec.loader
327        assert loader is not None, "A loader was not found for the package."
328        self._loader = loader
329        self._archive = None
330
331        if isinstance(loader, zipimport.zipimporter):
332            self._archive = loader.archive
333            pkgdir = next(iter(spec.submodule_search_locations))  # type: ignore
334            template_root = os.path.join(pkgdir, package_path).rstrip(os.path.sep)
335        else:
336            roots: t.List[str] = []
337
338            # One element for regular packages, multiple for namespace
339            # packages, or None for single module file.
340            if spec.submodule_search_locations:
341                roots.extend(spec.submodule_search_locations)
342            # A single module file, use the parent directory instead.
343            elif spec.origin is not None:
344                roots.append(os.path.dirname(spec.origin))
345
346            if not roots:
347                raise ValueError(
348                    f"The {package_name!r} package was not installed in a"
349                    " way that PackageLoader understands."
350                )
351
352            for root in roots:
353                root = os.path.join(root, package_path)
354
355                if os.path.isdir(root):
356                    template_root = root
357                    break
358            else:
359                raise ValueError(
360                    f"PackageLoader could not find a {package_path!r} directory"
361                    f" in the {package_name!r} package."
362                )
363
364        self._template_root = template_root
365
366    def get_source(
367        self, environment: "Environment", template: str
368    ) -> t.Tuple[str, str, t.Optional[t.Callable[[], bool]]]:
369        # Use posixpath even on Windows to avoid "drive:" or UNC
370        # segments breaking out of the search directory. Use normpath to
371        # convert Windows altsep to sep.
372        p = os.path.normpath(
373            posixpath.join(self._template_root, *split_template_path(template))
374        )
375        up_to_date: t.Optional[t.Callable[[], bool]]
376
377        if self._archive is None:
378            # Package is a directory.
379            if not os.path.isfile(p):
380                raise TemplateNotFound(template)
381
382            with open(p, "rb") as f:
383                source = f.read()
384
385            mtime = os.path.getmtime(p)
386
387            def up_to_date() -> bool:
388                return os.path.isfile(p) and os.path.getmtime(p) == mtime
389
390        else:
391            # Package is a zip file.
392            try:
393                source = self._loader.get_data(p)  # type: ignore
394            except OSError as e:
395                raise TemplateNotFound(template) from e
396
397            # Could use the zip's mtime for all template mtimes, but
398            # would need to safely reload the module if it's out of
399            # date, so just report it as always current.
400            up_to_date = None
401
402        return source.decode(self.encoding), p, up_to_date
403
404    def list_templates(self) -> t.List[str]:
405        results: t.List[str] = []
406
407        if self._archive is None:
408            # Package is a directory.
409            offset = len(self._template_root)
410
411            for dirpath, _, filenames in os.walk(self._template_root):
412                dirpath = dirpath[offset:].lstrip(os.path.sep)
413                results.extend(
414                    os.path.join(dirpath, name).replace(os.path.sep, "/")
415                    for name in filenames
416                )
417        else:
418            files = _get_zipimporter_files(self._loader)
419
420            # Package is a zip file.
421            prefix = (
422                self._template_root[len(self._archive) :].lstrip(os.path.sep)
423                + os.path.sep
424            )
425            offset = len(prefix)
426
427            for name in files:
428                # Find names under the templates directory that aren't directories.
429                if name.startswith(prefix) and name[-1] != os.path.sep:
430                    results.append(name[offset:].replace(os.path.sep, "/"))
431
432        results.sort()
433        return results
434
435
436class DictLoader(BaseLoader):
437    """Loads a template from a Python dict mapping template names to
438    template source.  This loader is useful for unittesting:
439
440    >>> loader = DictLoader({'index.html': 'source here'})
441
442    Because auto reloading is rarely useful this is disabled by default.
443    """
444
445    def __init__(self, mapping: t.Mapping[str, str]) -> None:
446        self.mapping = mapping
447
448    def get_source(
449        self, environment: "Environment", template: str
450    ) -> t.Tuple[str, None, t.Callable[[], bool]]:
451        if template in self.mapping:
452            source = self.mapping[template]
453            return source, None, lambda: source == self.mapping.get(template)
454        raise TemplateNotFound(template)
455
456    def list_templates(self) -> t.List[str]:
457        return sorted(self.mapping)
458
459
460class FunctionLoader(BaseLoader):
461    """A loader that is passed a function which does the loading.  The
462    function receives the name of the template and has to return either
463    a string with the template source, a tuple in the form ``(source,
464    filename, uptodatefunc)`` or `None` if the template does not exist.
465
466    >>> def load_template(name):
467    ...     if name == 'index.html':
468    ...         return '...'
469    ...
470    >>> loader = FunctionLoader(load_template)
471
472    The `uptodatefunc` is a function that is called if autoreload is enabled
473    and has to return `True` if the template is still up to date.  For more
474    details have a look at :meth:`BaseLoader.get_source` which has the same
475    return value.
476    """
477
478    def __init__(
479        self,
480        load_func: t.Callable[
481            [str],
482            t.Optional[
483                t.Union[
484                    str, t.Tuple[str, t.Optional[str], t.Optional[t.Callable[[], bool]]]
485                ]
486            ],
487        ],
488    ) -> None:
489        self.load_func = load_func
490
491    def get_source(
492        self, environment: "Environment", template: str
493    ) -> t.Tuple[str, t.Optional[str], t.Optional[t.Callable[[], bool]]]:
494        rv = self.load_func(template)
495
496        if rv is None:
497            raise TemplateNotFound(template)
498
499        if isinstance(rv, str):
500            return rv, None, None
501
502        return rv
503
504
505class PrefixLoader(BaseLoader):
506    """A loader that is passed a dict of loaders where each loader is bound
507    to a prefix.  The prefix is delimited from the template by a slash per
508    default, which can be changed by setting the `delimiter` argument to
509    something else::
510
511        loader = PrefixLoader({
512            'app1':     PackageLoader('mypackage.app1'),
513            'app2':     PackageLoader('mypackage.app2')
514        })
515
516    By loading ``'app1/index.html'`` the file from the app1 package is loaded,
517    by loading ``'app2/index.html'`` the file from the second.
518    """
519
520    def __init__(
521        self, mapping: t.Mapping[str, BaseLoader], delimiter: str = "/"
522    ) -> None:
523        self.mapping = mapping
524        self.delimiter = delimiter
525
526    def get_loader(self, template: str) -> t.Tuple[BaseLoader, str]:
527        try:
528            prefix, name = template.split(self.delimiter, 1)
529            loader = self.mapping[prefix]
530        except (ValueError, KeyError) as e:
531            raise TemplateNotFound(template) from e
532        return loader, name
533
534    def get_source(
535        self, environment: "Environment", template: str
536    ) -> t.Tuple[str, t.Optional[str], t.Optional[t.Callable[[], bool]]]:
537        loader, name = self.get_loader(template)
538        try:
539            return loader.get_source(environment, name)
540        except TemplateNotFound as e:
541            # re-raise the exception with the correct filename here.
542            # (the one that includes the prefix)
543            raise TemplateNotFound(template) from e
544
545    @internalcode
546    def load(
547        self,
548        environment: "Environment",
549        name: str,
550        globals: t.Optional[t.MutableMapping[str, t.Any]] = None,
551    ) -> "Template":
552        loader, local_name = self.get_loader(name)
553        try:
554            return loader.load(environment, local_name, globals)
555        except TemplateNotFound as e:
556            # re-raise the exception with the correct filename here.
557            # (the one that includes the prefix)
558            raise TemplateNotFound(name) from e
559
560    def list_templates(self) -> t.List[str]:
561        result = []
562        for prefix, loader in self.mapping.items():
563            for template in loader.list_templates():
564                result.append(prefix + self.delimiter + template)
565        return result
566
567
568class ChoiceLoader(BaseLoader):
569    """This loader works like the `PrefixLoader` just that no prefix is
570    specified.  If a template could not be found by one loader the next one
571    is tried.
572
573    >>> loader = ChoiceLoader([
574    ...     FileSystemLoader('/path/to/user/templates'),
575    ...     FileSystemLoader('/path/to/system/templates')
576    ... ])
577
578    This is useful if you want to allow users to override builtin templates
579    from a different location.
580    """
581
582    def __init__(self, loaders: t.Sequence[BaseLoader]) -> None:
583        self.loaders = loaders
584
585    def get_source(
586        self, environment: "Environment", template: str
587    ) -> t.Tuple[str, t.Optional[str], t.Optional[t.Callable[[], bool]]]:
588        for loader in self.loaders:
589            try:
590                return loader.get_source(environment, template)
591            except TemplateNotFound:
592                pass
593        raise TemplateNotFound(template)
594
595    @internalcode
596    def load(
597        self,
598        environment: "Environment",
599        name: str,
600        globals: t.Optional[t.MutableMapping[str, t.Any]] = None,
601    ) -> "Template":
602        for loader in self.loaders:
603            try:
604                return loader.load(environment, name, globals)
605            except TemplateNotFound:
606                pass
607        raise TemplateNotFound(name)
608
609    def list_templates(self) -> t.List[str]:
610        found = set()
611        for loader in self.loaders:
612            found.update(loader.list_templates())
613        return sorted(found)
614
615
616class _TemplateModule(ModuleType):
617    """Like a normal module but with support for weak references"""
618
619
620class ModuleLoader(BaseLoader):
621    """This loader loads templates from precompiled templates.
622
623    Example usage:
624
625    >>> loader = ModuleLoader('/path/to/compiled/templates')
626
627    Templates can be precompiled with :meth:`Environment.compile_templates`.
628    """
629
630    has_source_access = False
631
632    def __init__(
633        self,
634        path: t.Union[
635            str, "os.PathLike[str]", t.Sequence[t.Union[str, "os.PathLike[str]"]]
636        ],
637    ) -> None:
638        package_name = f"_jinja2_module_templates_{id(self):x}"
639
640        # create a fake module that looks for the templates in the
641        # path given.
642        mod = _TemplateModule(package_name)
643
644        if not isinstance(path, abc.Iterable) or isinstance(path, str):
645            path = [path]
646
647        mod.__path__ = [os.fspath(p) for p in path]
648
649        sys.modules[package_name] = weakref.proxy(
650            mod, lambda x: sys.modules.pop(package_name, None)
651        )
652
653        # the only strong reference, the sys.modules entry is weak
654        # so that the garbage collector can remove it once the
655        # loader that created it goes out of business.
656        self.module = mod
657        self.package_name = package_name
658
659    @staticmethod
660    def get_template_key(name: str) -> str:
661        return "tmpl_" + sha1(name.encode("utf-8")).hexdigest()
662
663    @staticmethod
664    def get_module_filename(name: str) -> str:
665        return ModuleLoader.get_template_key(name) + ".py"
666
667    @internalcode
668    def load(
669        self,
670        environment: "Environment",
671        name: str,
672        globals: t.Optional[t.MutableMapping[str, t.Any]] = None,
673    ) -> "Template":
674        key = self.get_template_key(name)
675        module = f"{self.package_name}.{key}"
676        mod = getattr(self.module, module, None)
677
678        if mod is None:
679            try:
680                mod = __import__(module, None, None, ["root"])
681            except ImportError as e:
682                raise TemplateNotFound(name) from e
683
684            # remove the entry from sys.modules, we only want the attribute
685            # on the module object we have stored on the loader.
686            sys.modules.pop(module, None)
687
688        if globals is None:
689            globals = {}
690
691        return environment.template_class.from_module_dict(
692            environment, mod.__dict__, globals
693        )