From 2fcabb529ff69434b9ce5d15d26acdc302f4f3fc Mon Sep 17 00:00:00 2001 From: David Lord Date: Sat, 11 May 2024 13:41:03 -0700 Subject: [PATCH 01/52] start version 3.1.5 --- CHANGES.rst | 6 ++++++ src/jinja2/__init__.py | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index 860c439fb..92a546974 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,5 +1,11 @@ .. currentmodule:: jinja2 +Version 3.1.5 +------------- + +Unreleased + + Version 3.1.4 ------------- diff --git a/src/jinja2/__init__.py b/src/jinja2/__init__.py index 2f0b5b286..578940091 100644 --- a/src/jinja2/__init__.py +++ b/src/jinja2/__init__.py @@ -35,4 +35,4 @@ from .utils import pass_eval_context as pass_eval_context from .utils import select_autoescape as select_autoescape -__version__ = "3.1.4" +__version__ = "3.1.5.dev" From 5bc613ec45d51535849e7f8a67364a1fe1c94716 Mon Sep 17 00:00:00 2001 From: Thomas Grainger Date: Sun, 7 Apr 2024 09:05:54 +0100 Subject: [PATCH 02/52] use asyncio.run --- CHANGES.rst | 8 +++++--- docs/api.rst | 3 --- src/jinja2/environment.py | 14 +------------- 3 files changed, 6 insertions(+), 19 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index 92a546974..bd085d030 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -5,6 +5,9 @@ Version 3.1.5 Unreleased +- Calling sync ``render`` for an async template uses ``asyncio.run``. + :pr:`1952` + Version 3.1.4 ------------- @@ -138,9 +141,8 @@ Released 2021-05-18 extensions shows more relevant context. :issue:`1429` - Fixed calling deprecated ``jinja2.Markup`` without an argument. Use ``markupsafe.Markup`` instead. :issue:`1438` -- Calling sync ``render`` for an async template uses ``asyncio.run`` - on Python >= 3.7. This fixes a deprecation that Python 3.10 - introduces. :issue:`1443` +- Calling sync ``render`` for an async template uses ``asyncio.new_event_loop`` + This fixes a deprecation that Python 3.10 introduces. :issue:`1443` Version 3.0.0 diff --git a/docs/api.rst b/docs/api.rst index e2c9bd526..cb62f6c32 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -515,9 +515,6 @@ environment to compile different code behind the scenes in order to handle async and sync code in an asyncio event loop. This has the following implications: -- Template rendering requires an event loop to be available to the - current thread. :func:`asyncio.get_running_loop` must return an - event loop. - The compiled code uses ``await`` for functions and attributes, and uses ``async for`` loops. In order to support using both async and sync functions in this context, a small wrapper is placed around diff --git a/src/jinja2/environment.py b/src/jinja2/environment.py index 1d3be0bed..ed4198600 100644 --- a/src/jinja2/environment.py +++ b/src/jinja2/environment.py @@ -1282,19 +1282,7 @@ def render(self, *args: t.Any, **kwargs: t.Any) -> str: if self.environment.is_async: import asyncio - close = False - - try: - loop = asyncio.get_running_loop() - except RuntimeError: - loop = asyncio.new_event_loop() - close = True - - try: - return loop.run_until_complete(self.render_async(*args, **kwargs)) - finally: - if close: - loop.close() + return asyncio.run(self.render_async(*args, **kwargs)) ctx = self.new_context(dict(*args, **kwargs)) From 1655128cfc0e8b598d5b3f361a3983f82098276b Mon Sep 17 00:00:00 2001 From: Thomas Grainger Date: Sat, 11 May 2024 23:01:12 +0100 Subject: [PATCH 03/52] test on trio, fix all missing aclose related warnings (#1960) --- CHANGES.rst | 7 +++ requirements/docs.txt | 2 +- requirements/tests.in | 1 + requirements/tests.txt | 20 +++++- src/jinja2/async_utils.py | 25 ++++++-- src/jinja2/compiler.py | 44 ++++++++----- src/jinja2/environment.py | 12 +++- tests/test_async.py | 122 +++++++++++++++++++++++++++++------- tests/test_async_filters.py | 67 ++++++++++++++++---- 9 files changed, 238 insertions(+), 62 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index bd085d030..7fb729763 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -7,6 +7,13 @@ Unreleased - Calling sync ``render`` for an async template uses ``asyncio.run``. :pr:`1952` +- Avoid unclosed ``auto_aiter`` warnings. :pr:`1960` +- Return an ``aclose``-able ``AsyncGenerator`` from + ``Template.generate_async``. :pr:`1960` +- Avoid leaving ``root_render_func()`` unclosed in + ``Template.generate_async``. :pr:`1960` +- Avoid leaving async generators unclosed in blocks, includes and extends. + :pr:`1960` Version 3.1.4 diff --git a/requirements/docs.txt b/requirements/docs.txt index e125c59a4..27488ade0 100644 --- a/requirements/docs.txt +++ b/requirements/docs.txt @@ -15,7 +15,7 @@ charset-normalizer==3.1.0 # via requests docutils==0.20.1 # via sphinx -idna==3.4 +idna==3.6 # via requests imagesize==1.4.1 # via sphinx diff --git a/requirements/tests.in b/requirements/tests.in index e079f8a60..423e485cc 100644 --- a/requirements/tests.in +++ b/requirements/tests.in @@ -1 +1,2 @@ pytest +trio<=0.22.2 # for Python3.7 support diff --git a/requirements/tests.txt b/requirements/tests.txt index 6168271c8..bb8f55df1 100644 --- a/requirements/tests.txt +++ b/requirements/tests.txt @@ -1,19 +1,35 @@ -# SHA1:0eaa389e1fdb3a1917c0f987514bd561be5718ee +# SHA1:b8d151f902b43c4435188a9d3494fb8d4af07168 # # This file is autogenerated by pip-compile-multi # To update, run: # # pip-compile-multi # +attrs==23.2.0 + # via + # outcome + # trio exceptiongroup==1.1.1 - # via pytest + # via + # pytest + # trio +idna==3.6 + # via trio iniconfig==2.0.0 # via pytest +outcome==1.3.0.post0 + # via trio packaging==23.1 # via pytest pluggy==1.2.0 # via pytest pytest==7.4.0 # via -r requirements/tests.in +sniffio==1.3.1 + # via trio +sortedcontainers==2.4.0 + # via trio tomli==2.0.1 # via pytest +trio==0.22.2 + # via -r requirements/tests.in diff --git a/src/jinja2/async_utils.py b/src/jinja2/async_utils.py index e65219e49..b0d277de7 100644 --- a/src/jinja2/async_utils.py +++ b/src/jinja2/async_utils.py @@ -6,6 +6,9 @@ from .utils import _PassArg from .utils import pass_eval_context +if t.TYPE_CHECKING: + import typing_extensions as te + V = t.TypeVar("V") @@ -67,15 +70,27 @@ async def auto_await(value: t.Union[t.Awaitable["V"], "V"]) -> "V": return t.cast("V", value) -async def auto_aiter( +class _IteratorToAsyncIterator(t.Generic[V]): + def __init__(self, iterator: "t.Iterator[V]"): + self._iterator = iterator + + def __aiter__(self) -> "te.Self": + return self + + async def __anext__(self) -> V: + try: + return next(self._iterator) + except StopIteration as e: + raise StopAsyncIteration(e.value) from e + + +def auto_aiter( iterable: "t.Union[t.AsyncIterable[V], t.Iterable[V]]", ) -> "t.AsyncIterator[V]": if hasattr(iterable, "__aiter__"): - async for item in t.cast("t.AsyncIterable[V]", iterable): - yield item + return iterable.__aiter__() else: - for item in iterable: - yield item + return _IteratorToAsyncIterator(iter(iterable)) async def auto_to_list( diff --git a/src/jinja2/compiler.py b/src/jinja2/compiler.py index 274071750..e18a14004 100644 --- a/src/jinja2/compiler.py +++ b/src/jinja2/compiler.py @@ -902,12 +902,15 @@ def visit_Template( if not self.environment.is_async: self.writeline("yield from parent_template.root_render_func(context)") else: - self.writeline( - "async for event in parent_template.root_render_func(context):" - ) + self.writeline("agen = parent_template.root_render_func(context)") + self.writeline("try:") + self.indent() + self.writeline("async for event in agen:") self.indent() self.writeline("yield event") self.outdent() + self.outdent() + self.writeline("finally: await agen.aclose()") self.outdent(1 + (not self.has_known_extends)) # at this point we now have the blocks collected and can visit them too. @@ -977,14 +980,20 @@ def visit_Block(self, node: nodes.Block, frame: Frame) -> None: f"yield from context.blocks[{node.name!r}][0]({context})", node ) else: + self.writeline(f"gen = context.blocks[{node.name!r}][0]({context})") + self.writeline("try:") + self.indent() self.writeline( - f"{self.choose_async()}for event in" - f" context.blocks[{node.name!r}][0]({context}):", + f"{self.choose_async()}for event in gen:", node, ) self.indent() self.simple_write("event", frame) self.outdent() + self.outdent() + self.writeline( + f"finally: {self.choose_async('await gen.aclose()', 'gen.close()')}" + ) self.outdent(level) @@ -1057,26 +1066,33 @@ def visit_Include(self, node: nodes.Include, frame: Frame) -> None: self.writeline("else:") self.indent() - skip_event_yield = False + def loop_body() -> None: + self.indent() + self.simple_write("event", frame) + self.outdent() + if node.with_context: self.writeline( - f"{self.choose_async()}for event in template.root_render_func(" + f"gen = template.root_render_func(" "template.new_context(context.get_all(), True," - f" {self.dump_local_context(frame)})):" + f" {self.dump_local_context(frame)}))" + ) + self.writeline("try:") + self.indent() + self.writeline(f"{self.choose_async()}for event in gen:") + loop_body() + self.outdent() + self.writeline( + f"finally: {self.choose_async('await gen.aclose()', 'gen.close()')}" ) elif self.environment.is_async: self.writeline( "for event in (await template._get_default_module_async())" "._body_stream:" ) + loop_body() else: self.writeline("yield from template._get_default_module()._body_stream") - skip_event_yield = True - - if not skip_event_yield: - self.indent() - self.simple_write("event", frame) - self.outdent() if node.ignore_missing: self.outdent() diff --git a/src/jinja2/environment.py b/src/jinja2/environment.py index ed4198600..57a7f8966 100644 --- a/src/jinja2/environment.py +++ b/src/jinja2/environment.py @@ -1346,7 +1346,7 @@ async def to_list() -> t.List[str]: async def generate_async( self, *args: t.Any, **kwargs: t.Any - ) -> t.AsyncIterator[str]: + ) -> t.AsyncGenerator[str, object]: """An async version of :meth:`generate`. Works very similarly but returns an async iterator instead. """ @@ -1358,8 +1358,14 @@ async def generate_async( ctx = self.new_context(dict(*args, **kwargs)) try: - async for event in self.root_render_func(ctx): # type: ignore - yield event + agen = self.root_render_func(ctx) + try: + async for event in agen: # type: ignore + yield event + finally: + # we can't use async with aclosing(...) because that's only + # in 3.10+ + await agen.aclose() # type: ignore except Exception: yield self.environment.handle_exception() diff --git a/tests/test_async.py b/tests/test_async.py index c9ba70c3e..4edced9dd 100644 --- a/tests/test_async.py +++ b/tests/test_async.py @@ -1,6 +1,7 @@ import asyncio import pytest +import trio from jinja2 import ChainableUndefined from jinja2 import DictLoader @@ -13,7 +14,16 @@ from jinja2.nativetypes import NativeEnvironment -def test_basic_async(): +def _asyncio_run(async_fn, *args): + return asyncio.run(async_fn(*args)) + + +@pytest.fixture(params=[_asyncio_run, trio.run], ids=["asyncio", "trio"]) +def run_async_fn(request): + return request.param + + +def test_basic_async(run_async_fn): t = Template( "{% for item in [1, 2, 3] %}[{{ item }}]{% endfor %}", enable_async=True ) @@ -21,11 +31,11 @@ def test_basic_async(): async def func(): return await t.render_async() - rv = asyncio.run(func()) + rv = run_async_fn(func) assert rv == "[1][2][3]" -def test_await_on_calls(): +def test_await_on_calls(run_async_fn): t = Template("{{ async_func() + normal_func() }}", enable_async=True) async def async_func(): @@ -37,7 +47,7 @@ def normal_func(): async def func(): return await t.render_async(async_func=async_func, normal_func=normal_func) - rv = asyncio.run(func()) + rv = run_async_fn(func) assert rv == "65" @@ -54,7 +64,7 @@ def normal_func(): assert rv == "65" -def test_await_and_macros(): +def test_await_and_macros(run_async_fn): t = Template( "{% macro foo(x) %}[{{ x }}][{{ async_func() }}]{% endmacro %}{{ foo(42) }}", enable_async=True, @@ -66,11 +76,11 @@ async def async_func(): async def func(): return await t.render_async(async_func=async_func) - rv = asyncio.run(func()) + rv = run_async_fn(func) assert rv == "[42][42]" -def test_async_blocks(): +def test_async_blocks(run_async_fn): t = Template( "{% block foo %}{% endblock %}{{ self.foo() }}", enable_async=True, @@ -80,7 +90,7 @@ def test_async_blocks(): async def func(): return await t.render_async() - rv = asyncio.run(func()) + rv = run_async_fn(func) assert rv == "" @@ -156,8 +166,8 @@ def test_trailing_comma(self, test_env_async): test_env_async.from_string('{% from "foo" import bar, with, context %}') test_env_async.from_string('{% from "foo" import bar, with with context %}') - def test_exports(self, test_env_async): - coro = test_env_async.from_string( + def test_exports(self, test_env_async, run_async_fn): + coro_fn = test_env_async.from_string( """ {% macro toplevel() %}...{% endmacro %} {% macro __private() %}...{% endmacro %} @@ -166,9 +176,9 @@ def test_exports(self, test_env_async): {% macro notthere() %}{% endmacro %} {% endfor %} """ - )._get_default_module_async() - m = asyncio.run(coro) - assert asyncio.run(m.toplevel()) == "..." + )._get_default_module_async + m = run_async_fn(coro_fn) + assert run_async_fn(m.toplevel) == "..." assert not hasattr(m, "__missing") assert m.variable == 42 assert not hasattr(m, "notthere") @@ -457,17 +467,19 @@ def test_reversed_bug(self, test_env_async): ) assert tmpl.render(items=reversed([3, 2, 1])) == "1,2,3" - def test_loop_errors(self, test_env_async): + def test_loop_errors(self, test_env_async, run_async_fn): tmpl = test_env_async.from_string( """{% for item in [1] if loop.index == 0 %}...{% endfor %}""" ) - pytest.raises(UndefinedError, tmpl.render) + with pytest.raises(UndefinedError): + run_async_fn(tmpl.render_async) + tmpl = test_env_async.from_string( """{% for item in [] %}...{% else %}{{ loop }}{% endfor %}""" ) - assert tmpl.render() == "" + assert run_async_fn(tmpl.render_async) == "" def test_loop_filter(self, test_env_async): tmpl = test_env_async.from_string( @@ -597,7 +609,7 @@ def test_awaitable_property_slicing(self, test_env_async): assert t.render(a=dict(b=[1, 2, 3])) == "1" -def test_namespace_awaitable(test_env_async): +def test_namespace_awaitable(test_env_async, run_async_fn): async def _test(): t = test_env_async.from_string( '{% set ns = namespace(foo="Bar") %}{{ ns.foo }}' @@ -605,10 +617,10 @@ async def _test(): actual = await t.render_async() assert actual == "Bar" - asyncio.run(_test()) + run_async_fn(_test) -def test_chainable_undefined_aiter(): +def test_chainable_undefined_aiter(run_async_fn): async def _test(): t = Template( "{% for x in a['b']['c'] %}{{ x }}{% endfor %}", @@ -618,7 +630,7 @@ async def _test(): rv = await t.render_async(a={}) assert rv == "" - asyncio.run(_test()) + run_async_fn(_test) @pytest.fixture @@ -626,22 +638,22 @@ def async_native_env(): return NativeEnvironment(enable_async=True) -def test_native_async(async_native_env): +def test_native_async(async_native_env, run_async_fn): async def _test(): t = async_native_env.from_string("{{ x }}") rv = await t.render_async(x=23) assert rv == 23 - asyncio.run(_test()) + run_async_fn(_test) -def test_native_list_async(async_native_env): +def test_native_list_async(async_native_env, run_async_fn): async def _test(): t = async_native_env.from_string("{{ x }}") rv = await t.render_async(x=list(range(3))) assert rv == [0, 1, 2] - asyncio.run(_test()) + run_async_fn(_test) def test_getitem_after_filter(): @@ -658,3 +670,65 @@ def test_getitem_after_call(): t = env.from_string("{{ add_each(a, 2)[1:] }}") out = t.render(a=range(3)) assert out == "[3, 4]" + + +def test_basic_generate_async(run_async_fn): + t = Template( + "{% for item in [1, 2, 3] %}[{{ item }}]{% endfor %}", enable_async=True + ) + + async def func(): + agen = t.generate_async() + try: + return await agen.__anext__() + finally: + await agen.aclose() + + rv = run_async_fn(func) + assert rv == "[" + + +def test_include_generate_async(run_async_fn, test_env_async): + t = test_env_async.from_string('{% include "header" %}') + + async def func(): + agen = t.generate_async() + try: + return await agen.__anext__() + finally: + await agen.aclose() + + rv = run_async_fn(func) + assert rv == "[" + + +def test_blocks_generate_async(run_async_fn): + t = Template( + "{% block foo %}{% endblock %}{{ self.foo() }}", + enable_async=True, + autoescape=True, + ) + + async def func(): + agen = t.generate_async() + try: + return await agen.__anext__() + finally: + await agen.aclose() + + rv = run_async_fn(func) + assert rv == "" + + +def test_async_extend(run_async_fn, test_env_async): + t = test_env_async.from_string('{% extends "header" %}') + + async def func(): + agen = t.generate_async() + try: + return await agen.__anext__() + finally: + await agen.aclose() + + rv = run_async_fn(func) + assert rv == "[" diff --git a/tests/test_async_filters.py b/tests/test_async_filters.py index f5b2627ad..e8cc350d5 100644 --- a/tests/test_async_filters.py +++ b/tests/test_async_filters.py @@ -1,6 +1,9 @@ +import asyncio +import contextlib from collections import namedtuple import pytest +import trio from markupsafe import Markup from jinja2 import Environment @@ -26,10 +29,39 @@ def env_async(): return Environment(enable_async=True) +def _asyncio_run(async_fn, *args): + return asyncio.run(async_fn(*args)) + + +@pytest.fixture(params=[_asyncio_run, trio.run], ids=["asyncio", "trio"]) +def run_async_fn(request): + return request.param + + +@contextlib.asynccontextmanager +async def closing_factory(): + async with contextlib.AsyncExitStack() as stack: + + def closing(maybe_agen): + try: + aclose = maybe_agen.aclose + except AttributeError: + pass + else: + stack.push_async_callback(aclose) + return maybe_agen + + yield closing + + @mark_dualiter("foo", lambda: range(10)) -def test_first(env_async, foo): - tmpl = env_async.from_string("{{ foo()|first }}") - out = tmpl.render(foo=foo) +def test_first(env_async, foo, run_async_fn): + async def test(): + async with closing_factory() as closing: + tmpl = env_async.from_string("{{ closing(foo())|first }}") + return await tmpl.render_async(foo=foo, closing=closing) + + out = run_async_fn(test) assert out == "0" @@ -245,18 +277,23 @@ def test_slice(env_async, items): ) -def test_custom_async_filter(env_async): +def test_custom_async_filter(env_async, run_async_fn): async def customfilter(val): return str(val) - env_async.filters["customfilter"] = customfilter - tmpl = env_async.from_string("{{ 'static'|customfilter }} {{ arg|customfilter }}") - out = tmpl.render(arg="dynamic") + async def test(): + env_async.filters["customfilter"] = customfilter + tmpl = env_async.from_string( + "{{ 'static'|customfilter }} {{ arg|customfilter }}" + ) + return await tmpl.render_async(arg="dynamic") + + out = run_async_fn(test) assert out == "static dynamic" @mark_dualiter("items", lambda: range(10)) -def test_custom_async_iteratable_filter(env_async, items): +def test_custom_async_iteratable_filter(env_async, items, run_async_fn): async def customfilter(iterable): items = [] async for item in auto_aiter(iterable): @@ -265,9 +302,13 @@ async def customfilter(iterable): break return ",".join(items) - env_async.filters["customfilter"] = customfilter - tmpl = env_async.from_string( - "{{ items()|customfilter }} .. {{ [3, 4, 5, 6]|customfilter }}" - ) - out = tmpl.render(items=items) + async def test(): + async with closing_factory() as closing: + env_async.filters["customfilter"] = customfilter + tmpl = env_async.from_string( + "{{ closing(items())|customfilter }} .. {{ [3, 4, 5, 6]|customfilter }}" + ) + return await tmpl.render_async(items=items, closing=closing) + + out = run_async_fn(test) assert out == "0,1,2 .. 3,4,5" From 004476c22b8093d47e0d92c144757879fc779a69 Mon Sep 17 00:00:00 2001 From: David Lord Date: Sun, 12 May 2024 17:51:31 -0700 Subject: [PATCH 04/52] test on python 3.13 update dev dependencies refactor update tox envs 3.7 requires an old version of trio xfail zip loader template test --- .github/workflows/tests.yaml | 1 + .pre-commit-config.yaml | 2 +- requirements/build.txt | 17 ++--- requirements/dev.txt | 138 ++++++++++++++++++++++++++++------- requirements/docs.txt | 53 +++++++------- requirements/tests.in | 2 +- requirements/tests.txt | 27 +++---- requirements/tests37.in | 2 + requirements/tests37.txt | 43 +++++++++++ requirements/typing.txt | 15 ++-- src/jinja2/compiler.py | 2 +- tests/test_loader.py | 5 +- tox.ini | 35 ++++++--- 13 files changed, 236 insertions(+), 106 deletions(-) create mode 100644 requirements/tests37.in create mode 100644 requirements/tests37.txt diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index c5e5f709a..0cd6f4a21 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -21,6 +21,7 @@ jobs: fail-fast: false matrix: include: + - {python: '3.13'} - {python: '3.12'} - {name: Windows, python: '3.12', os: windows-latest} - {name: Mac, python: '3.12', os: macos-latest} diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index dd86089b8..5b7ebb878 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -2,7 +2,7 @@ ci: autoupdate_schedule: monthly repos: - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.4.1 + rev: v0.4.4 hooks: - id: ruff - id: ruff-format diff --git a/requirements/build.txt b/requirements/build.txt index 98175b856..52fd1f69a 100644 --- a/requirements/build.txt +++ b/requirements/build.txt @@ -1,15 +1,12 @@ -# SHA1:80754af91bfb6d1073585b046fe0a474ce868509 # -# This file is autogenerated by pip-compile-multi -# To update, run: +# This file is autogenerated by pip-compile with Python 3.12 +# by the following command: # -# pip-compile-multi +# pip-compile build.in # -build==0.10.0 - # via -r requirements/build.in -packaging==23.1 +build==1.2.1 + # via -r build.in +packaging==24.0 # via build -pyproject-hooks==1.0.0 - # via build -tomli==2.0.1 +pyproject-hooks==1.1.0 # via build diff --git a/requirements/dev.txt b/requirements/dev.txt index 79fb7c0ba..076912b0b 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -1,62 +1,146 @@ -# SHA1:54b5b77ec8c7a0064ffa93b2fd16cb0130ba177c # -# This file is autogenerated by pip-compile-multi -# To update, run: +# This file is autogenerated by pip-compile with Python 3.12 +# by the following command: # -# pip-compile-multi +# pip-compile dev.in # --r docs.txt --r tests.txt --r typing.txt -build==0.10.0 +alabaster==0.7.16 + # via sphinx +attrs==23.2.0 + # via + # outcome + # trio +babel==2.15.0 + # via sphinx +build==1.2.1 # via pip-tools -cachetools==5.3.1 +cachetools==5.3.3 # via tox -cfgv==3.3.1 +certifi==2024.2.2 + # via requests +cfgv==3.4.0 # via pre-commit -chardet==5.1.0 +chardet==5.2.0 # via tox -click==8.1.3 +charset-normalizer==3.3.2 + # via requests +click==8.1.7 # via # pip-compile-multi # pip-tools colorama==0.4.6 # via tox -distlib==0.3.6 +distlib==0.3.8 # via virtualenv -filelock==3.12.2 +docutils==0.21.2 + # via sphinx +filelock==3.14.0 # via # tox # virtualenv -identify==2.5.24 +identify==2.5.36 # via pre-commit +idna==3.7 + # via + # requests + # trio +imagesize==1.4.1 + # via sphinx +iniconfig==2.0.0 + # via pytest +jinja2==3.1.4 + # via sphinx +markupsafe==2.1.5 + # via jinja2 +mypy==1.10.0 + # via -r typing.in +mypy-extensions==1.0.0 + # via mypy nodeenv==1.8.0 # via pre-commit +outcome==1.3.0.post0 + # via trio +packaging==24.0 + # via + # build + # pallets-sphinx-themes + # pyproject-api + # pytest + # sphinx + # tox +pallets-sphinx-themes==2.1.3 + # via -r docs.in pip-compile-multi==2.6.3 - # via -r requirements/dev.in -pip-tools==6.13.0 + # via -r dev.in +pip-tools==7.4.1 # via pip-compile-multi -platformdirs==3.8.0 +platformdirs==4.2.1 # via # tox # virtualenv -pre-commit==3.3.3 - # via -r requirements/dev.in -pyproject-api==1.5.2 +pluggy==1.5.0 + # via + # pytest + # tox +pre-commit==3.7.1 + # via -r dev.in +pygments==2.18.0 + # via sphinx +pyproject-api==1.6.1 # via tox -pyproject-hooks==1.0.0 - # via build +pyproject-hooks==1.1.0 + # via + # build + # pip-tools +pytest==8.2.0 + # via -r tests.in pyyaml==6.0.1 # via pre-commit +requests==2.31.0 + # via sphinx +sniffio==1.3.1 + # via trio +snowballstemmer==2.2.0 + # via sphinx +sortedcontainers==2.4.0 + # via trio +sphinx==7.3.7 + # via + # -r docs.in + # pallets-sphinx-themes + # sphinx-issues + # sphinxcontrib-log-cabinet +sphinx-issues==4.1.0 + # via -r docs.in +sphinxcontrib-applehelp==1.0.8 + # via sphinx +sphinxcontrib-devhelp==1.0.6 + # via sphinx +sphinxcontrib-htmlhelp==2.0.5 + # via sphinx +sphinxcontrib-jsmath==1.0.1 + # via sphinx +sphinxcontrib-log-cabinet==1.0.1 + # via -r docs.in +sphinxcontrib-qthelp==1.0.7 + # via sphinx +sphinxcontrib-serializinghtml==1.1.10 + # via sphinx toposort==1.10 # via pip-compile-multi -tox==4.6.3 - # via -r requirements/dev.in -virtualenv==20.23.1 +tox==4.15.0 + # via -r dev.in +trio==0.25.0 + # via -r tests.in +typing-extensions==4.11.0 + # via mypy +urllib3==2.2.1 + # via requests +virtualenv==20.26.1 # via # pre-commit # tox -wheel==0.40.0 +wheel==0.43.0 # via pip-tools # The following packages are considered to be unsafe in a requirements file: diff --git a/requirements/docs.txt b/requirements/docs.txt index 27488ade0..2cbd73fa8 100644 --- a/requirements/docs.txt +++ b/requirements/docs.txt @@ -1,61 +1,60 @@ -# SHA1:45c590f97fe95b8bdc755eef796e91adf5fbe4ea # -# This file is autogenerated by pip-compile-multi -# To update, run: +# This file is autogenerated by pip-compile with Python 3.12 +# by the following command: # -# pip-compile-multi +# pip-compile docs.in # -alabaster==0.7.13 +alabaster==0.7.16 # via sphinx -babel==2.12.1 +babel==2.15.0 # via sphinx -certifi==2023.5.7 +certifi==2024.2.2 # via requests -charset-normalizer==3.1.0 +charset-normalizer==3.3.2 # via requests -docutils==0.20.1 +docutils==0.21.2 # via sphinx -idna==3.6 +idna==3.7 # via requests imagesize==1.4.1 # via sphinx -jinja2==3.1.2 +jinja2==3.1.4 # via sphinx -markupsafe==2.1.3 +markupsafe==2.1.5 # via jinja2 -packaging==23.1 +packaging==24.0 # via # pallets-sphinx-themes # sphinx -pallets-sphinx-themes==2.1.1 - # via -r requirements/docs.in -pygments==2.15.1 +pallets-sphinx-themes==2.1.3 + # via -r docs.in +pygments==2.18.0 # via sphinx requests==2.31.0 # via sphinx snowballstemmer==2.2.0 # via sphinx -sphinx==7.0.1 +sphinx==7.3.7 # via - # -r requirements/docs.in + # -r docs.in # pallets-sphinx-themes # sphinx-issues # sphinxcontrib-log-cabinet -sphinx-issues==3.0.1 - # via -r requirements/docs.in -sphinxcontrib-applehelp==1.0.4 +sphinx-issues==4.1.0 + # via -r docs.in +sphinxcontrib-applehelp==1.0.8 # via sphinx -sphinxcontrib-devhelp==1.0.2 +sphinxcontrib-devhelp==1.0.6 # via sphinx -sphinxcontrib-htmlhelp==2.0.1 +sphinxcontrib-htmlhelp==2.0.5 # via sphinx sphinxcontrib-jsmath==1.0.1 # via sphinx sphinxcontrib-log-cabinet==1.0.1 - # via -r requirements/docs.in -sphinxcontrib-qthelp==1.0.3 + # via -r docs.in +sphinxcontrib-qthelp==1.0.7 # via sphinx -sphinxcontrib-serializinghtml==1.1.5 +sphinxcontrib-serializinghtml==1.1.10 # via sphinx -urllib3==2.0.3 +urllib3==2.2.1 # via requests diff --git a/requirements/tests.in b/requirements/tests.in index 423e485cc..5669c6ecd 100644 --- a/requirements/tests.in +++ b/requirements/tests.in @@ -1,2 +1,2 @@ pytest -trio<=0.22.2 # for Python3.7 support +trio diff --git a/requirements/tests.txt b/requirements/tests.txt index bb8f55df1..de18d477d 100644 --- a/requirements/tests.txt +++ b/requirements/tests.txt @@ -1,35 +1,28 @@ -# SHA1:b8d151f902b43c4435188a9d3494fb8d4af07168 # -# This file is autogenerated by pip-compile-multi -# To update, run: +# This file is autogenerated by pip-compile with Python 3.12 +# by the following command: # -# pip-compile-multi +# pip-compile tests.in # attrs==23.2.0 # via # outcome # trio -exceptiongroup==1.1.1 - # via - # pytest - # trio -idna==3.6 +idna==3.7 # via trio iniconfig==2.0.0 # via pytest outcome==1.3.0.post0 # via trio -packaging==23.1 +packaging==24.0 # via pytest -pluggy==1.2.0 +pluggy==1.5.0 # via pytest -pytest==7.4.0 - # via -r requirements/tests.in +pytest==8.2.0 + # via -r tests.in sniffio==1.3.1 # via trio sortedcontainers==2.4.0 # via trio -tomli==2.0.1 - # via pytest -trio==0.22.2 - # via -r requirements/tests.in +trio==0.25.0 + # via -r tests.in diff --git a/requirements/tests37.in b/requirements/tests37.in new file mode 100644 index 000000000..9c2bed180 --- /dev/null +++ b/requirements/tests37.in @@ -0,0 +1,2 @@ +pytest +trio==0.22.2 diff --git a/requirements/tests37.txt b/requirements/tests37.txt new file mode 100644 index 000000000..578789e7a --- /dev/null +++ b/requirements/tests37.txt @@ -0,0 +1,43 @@ +# +# This file is autogenerated by pip-compile with Python 3.7 +# by the following command: +# +# pip-compile tests37.in +# +attrs==23.2.0 + # via + # outcome + # trio +exceptiongroup==1.2.1 + # via + # pytest + # trio +idna==3.7 + # via trio +importlib-metadata==6.7.0 + # via + # attrs + # pluggy + # pytest +iniconfig==2.0.0 + # via pytest +outcome==1.3.0.post0 + # via trio +packaging==24.0 + # via pytest +pluggy==1.2.0 + # via pytest +pytest==7.4.4 + # via -r tests37.in +sniffio==1.3.1 + # via trio +sortedcontainers==2.4.0 + # via trio +tomli==2.0.1 + # via pytest +trio==0.22.2 + # via -r tests37.in +typing-extensions==4.7.1 + # via importlib-metadata +zipp==3.15.0 + # via importlib-metadata diff --git a/requirements/typing.txt b/requirements/typing.txt index e06fb524e..c08a53767 100644 --- a/requirements/typing.txt +++ b/requirements/typing.txt @@ -1,15 +1,12 @@ -# SHA1:7983aaa01d64547827c20395d77e248c41b2572f # -# This file is autogenerated by pip-compile-multi -# To update, run: +# This file is autogenerated by pip-compile with Python 3.12 +# by the following command: # -# pip-compile-multi +# pip-compile typing.in # -mypy==1.4.1 - # via -r requirements/typing.in +mypy==1.10.0 + # via -r typing.in mypy-extensions==1.0.0 # via mypy -tomli==2.0.1 - # via mypy -typing-extensions==4.6.3 +typing-extensions==4.11.0 # via mypy diff --git a/src/jinja2/compiler.py b/src/jinja2/compiler.py index e18a14004..91720c5f9 100644 --- a/src/jinja2/compiler.py +++ b/src/jinja2/compiler.py @@ -55,7 +55,7 @@ def new_func( return f(self, node, frame, **kwargs) - return update_wrapper(t.cast(F, new_func), f) + return update_wrapper(new_func, f) # type: ignore[return-value] def _make_binop(op: str) -> t.Callable[["CodeGenerator", nodes.BinExpr, "Frame"], None]: diff --git a/tests/test_loader.py b/tests/test_loader.py index 77d686ef5..3e64f6237 100644 --- a/tests/test_loader.py +++ b/tests/test_loader.py @@ -2,7 +2,6 @@ import importlib.machinery import importlib.util import os -import platform import shutil import sys import tempfile @@ -364,8 +363,8 @@ def test_package_zip_source(package_zip_loader, template, expect): @pytest.mark.xfail( - platform.python_implementation() == "PyPy", - reason="PyPy's zipimporter doesn't have a '_files' attribute.", + sys.implementation.name == "pypy" or sys.version_info > (3, 13), + reason="zipimporter doesn't have a '_files' attribute", raises=TypeError, ) def test_package_zip_list(package_zip_loader): diff --git a/tox.ini b/tox.ini index 6582e2812..b9e972b93 100644 --- a/tox.ini +++ b/tox.ini @@ -1,6 +1,6 @@ [tox] envlist = - py3{12,11,10,9,8,7} + py3{13,12,11,10,9,8,7} pypy310 style typing @@ -15,6 +15,9 @@ use_frozen_constraints = true deps = -r requirements/tests.txt commands = pytest -v --tb=short --basetemp={envtmpdir} {posargs} +[testenv:py37,py3.7] +deps = -r requirements/tests37.txt + [testenv:style] deps = pre-commit skip_install = true @@ -28,16 +31,28 @@ commands = mypy deps = -r requirements/docs.txt commands = sphinx-build -E -W -b dirhtml docs docs/_build/dirhtml +[testenv:update-pre_commit] +labels = update +deps = pre-commit +skip_install = true +commands = pre-commit autoupdate -j4 + [testenv:update-requirements] -deps = - pip-tools - pre-commit +labels = update +deps = pip-tools skip_install = true change_dir = requirements commands = - pre-commit autoupdate -j4 - pip-compile -U build.in - pip-compile -U docs.in - pip-compile -U tests.in - pip-compile -U typing.in - pip-compile -U dev.in + pip-compile build.in -q {posargs:-U} + pip-compile docs.in -q {posargs:-U} + pip-compile tests.in -q {posargs:-U} + pip-compile typing.in -q {posargs:-U} + pip-compile dev.in -q {posargs:-U} + +[testenv:update-requirements37] +base_python = 3.7 +labels = update +deps = pip-tools +skip_install = true +change_dir = requirements +commands = pip-compile tests37.in -q {posargs:-U} From 679af7f816ced8941ed5cf9b151a0cac543d0336 Mon Sep 17 00:00:00 2001 From: Thomas Grainger Date: Mon, 13 May 2024 18:02:35 +0100 Subject: [PATCH 05/52] fix test_package_zip_list on 3.13 --- src/jinja2/loaders.py | 32 ++++++++++++++++++++++++++------ tests/test_loader.py | 2 +- 2 files changed, 27 insertions(+), 7 deletions(-) diff --git a/src/jinja2/loaders.py b/src/jinja2/loaders.py index 9eaf647ba..8c2c86cd0 100644 --- a/src/jinja2/loaders.py +++ b/src/jinja2/loaders.py @@ -238,6 +238,30 @@ def list_templates(self) -> t.List[str]: return sorted(found) +if sys.version_info >= (3, 13): + + def _get_zipimporter_files(z: t.Any) -> t.Dict[str, object]: + try: + get_files = z._get_files + except AttributeError as e: + raise TypeError( + "This zip import does not have the required" + " metadata to list templates." + ) from e + return get_files() +else: + + def _get_zipimporter_files(z: t.Any) -> t.Dict[str, object]: + try: + files = z._files + except AttributeError as e: + raise TypeError( + "This zip import does not have the required" + " metadata to list templates." + ) from e + return files # type: ignore[no-any-return] + + class PackageLoader(BaseLoader): """Load templates from a directory in a Python package. @@ -382,11 +406,7 @@ def list_templates(self) -> t.List[str]: for name in filenames ) else: - if not hasattr(self._loader, "_files"): - raise TypeError( - "This zip import does not have the required" - " metadata to list templates." - ) + files = _get_zipimporter_files(self._loader) # Package is a zip file. prefix = ( @@ -395,7 +415,7 @@ def list_templates(self) -> t.List[str]: ) offset = len(prefix) - for name in self._loader._files.keys(): + for name in files: # Find names under the templates directory that aren't directories. if name.startswith(prefix) and name[-1] != os.path.sep: results.append(name[offset:].replace(os.path.sep, "/")) diff --git a/tests/test_loader.py b/tests/test_loader.py index 3e64f6237..e0cff6720 100644 --- a/tests/test_loader.py +++ b/tests/test_loader.py @@ -363,7 +363,7 @@ def test_package_zip_source(package_zip_loader, template, expect): @pytest.mark.xfail( - sys.implementation.name == "pypy" or sys.version_info > (3, 13), + sys.implementation.name == "pypy", reason="zipimporter doesn't have a '_files' attribute", raises=TypeError, ) From 1470c17f9f466e7b6342b7537bc36605b6881794 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Mazzucotelli?= Date: Sun, 19 May 2024 15:32:56 +0200 Subject: [PATCH 06/52] Convert rST code block to Markdown in README --- README.md | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index 330970b59..f4aa7cbea 100644 --- a/README.md +++ b/README.md @@ -27,18 +27,17 @@ restricting functionality too much. ## In A Nutshell -.. code-block:: jinja - - {% extends "base.html" %} - {% block title %}Members{% endblock %} - {% block content %} - - {% endblock %} - +```jinja +{% extends "base.html" %} +{% block title %}Members{% endblock %} +{% block content %} + +{% endblock %} +``` ## Donate From a59744f50ed2958987733f7c4aa2cc63cac9b2b4 Mon Sep 17 00:00:00 2001 From: David Lord Date: Fri, 23 Aug 2024 16:43:52 -0700 Subject: [PATCH 07/52] add gha-update --- pyproject.toml | 5 +++++ tox.ini | 6 ++++++ 2 files changed, 11 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index c59feefca..5de076b17 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -96,3 +96,8 @@ ignore-init-module-imports = true [tool.ruff.lint.isort] force-single-line = true order-by-type = false + +[tool.gha-update] +tag-only = [ + "slsa-framework/slsa-github-generator", +] diff --git a/tox.ini b/tox.ini index b9e972b93..a40f36a44 100644 --- a/tox.ini +++ b/tox.ini @@ -31,6 +31,11 @@ commands = mypy deps = -r requirements/docs.txt commands = sphinx-build -E -W -b dirhtml docs docs/_build/dirhtml +[testenv:update-actions] +labels = update +deps = gha-update +commands = gha-update + [testenv:update-pre_commit] labels = update deps = pre-commit @@ -38,6 +43,7 @@ skip_install = true commands = pre-commit autoupdate -j4 [testenv:update-requirements] +base_python = 3.8 labels = update deps = pip-tools skip_install = true From 65b27afb617a1884d7436eba7dd6933e389cfa52 Mon Sep 17 00:00:00 2001 From: David Lord Date: Fri, 23 Aug 2024 16:49:33 -0700 Subject: [PATCH 08/52] update dev dependencies --- .github/workflows/publish.yaml | 16 +++---- .github/workflows/tests.yaml | 8 ++-- .pre-commit-config.yaml | 2 +- requirements/build.txt | 10 +++- requirements/dev.txt | 84 +++++++++++++++++++++------------- requirements/docs.txt | 36 +++++++++------ requirements/tests.txt | 18 +++++--- requirements/tests37.txt | 6 +-- requirements/typing.txt | 8 ++-- 9 files changed, 114 insertions(+), 74 deletions(-) diff --git a/.github/workflows/publish.yaml b/.github/workflows/publish.yaml index 6b134d1f0..7acd92e5b 100644 --- a/.github/workflows/publish.yaml +++ b/.github/workflows/publish.yaml @@ -9,8 +9,8 @@ jobs: outputs: hash: ${{ steps.hash.outputs.hash }} steps: - - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 - - uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5.1.0 + - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + - uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v5.1.1 with: python-version: '3.x' cache: pip @@ -23,7 +23,7 @@ jobs: - name: generate hash id: hash run: cd dist && echo "hash=$(sha256sum * | base64 -w0)" >> $GITHUB_OUTPUT - - uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 # v3.1.3 + - uses: actions/upload-artifact@834a144ee995460fba8ed112a2fc961b36a5ec5a # v4.3.6 with: path: ./dist provenance: @@ -33,7 +33,7 @@ jobs: id-token: write contents: write # Can't pin with hash due to how this workflow works. - uses: slsa-framework/slsa-github-generator/.github/workflows/generator_generic_slsa3.yml@v1.10.0 + uses: slsa-framework/slsa-github-generator/.github/workflows/generator_generic_slsa3.yml@v2.0.0 with: base64-subjects: ${{ needs.build.outputs.hash }} create-release: @@ -44,7 +44,7 @@ jobs: permissions: contents: write steps: - - uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2 + - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 - name: create release run: > gh release create --draft --repo ${{ github.repository }} @@ -63,11 +63,11 @@ jobs: permissions: id-token: write steps: - - uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2 - - uses: pypa/gh-action-pypi-publish@81e9d935c883d0b210363ab89cf05f3894778450 # v1.8.14 + - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 + - uses: pypa/gh-action-pypi-publish@ec4db0b4ddc65acdf4bff5fa45ac92d78b56bdf0 # v1.9.0 with: repository-url: https://test.pypi.org/legacy/ packages-dir: artifact/ - - uses: pypa/gh-action-pypi-publish@81e9d935c883d0b210363ab89cf05f3894778450 # v1.8.14 + - uses: pypa/gh-action-pypi-publish@ec4db0b4ddc65acdf4bff5fa45ac92d78b56bdf0 # v1.9.0 with: packages-dir: artifact/ diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 0cd6f4a21..afba0d753 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -32,8 +32,8 @@ jobs: - {python: '3.7'} - {name: PyPy, python: 'pypy-3.10', tox: pypy310} steps: - - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 - - uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5.1.0 + - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + - uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v5.1.1 with: python-version: ${{ matrix.python }} allow-prereleases: true @@ -44,8 +44,8 @@ jobs: typing: runs-on: ubuntu-latest steps: - - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 - - uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5.1.0 + - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + - uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v5.1.1 with: python-version: '3.x' cache: pip diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 5b7ebb878..5bee1ca4b 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -2,7 +2,7 @@ ci: autoupdate_schedule: monthly repos: - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.4.4 + rev: v0.6.2 hooks: - id: ruff - id: ruff-format diff --git a/requirements/build.txt b/requirements/build.txt index 52fd1f69a..96ee8f08e 100644 --- a/requirements/build.txt +++ b/requirements/build.txt @@ -1,12 +1,18 @@ # -# This file is autogenerated by pip-compile with Python 3.12 +# This file is autogenerated by pip-compile with Python 3.8 # by the following command: # # pip-compile build.in # build==1.2.1 # via -r build.in -packaging==24.0 +importlib-metadata==8.4.0 + # via build +packaging==24.1 # via build pyproject-hooks==1.1.0 # via build +tomli==2.0.1 + # via build +zipp==3.20.0 + # via importlib-metadata diff --git a/requirements/dev.txt b/requirements/dev.txt index 076912b0b..d78f0cf61 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -1,22 +1,22 @@ # -# This file is autogenerated by pip-compile with Python 3.12 +# This file is autogenerated by pip-compile with Python 3.8 # by the following command: # # pip-compile dev.in # -alabaster==0.7.16 +alabaster==0.7.13 # via sphinx -attrs==23.2.0 +attrs==24.2.0 # via # outcome # trio -babel==2.15.0 +babel==2.16.0 # via sphinx build==1.2.1 # via pip-tools -cachetools==5.3.3 +cachetools==5.5.0 # via tox -certifi==2024.2.2 +certifi==2024.7.4 # via requests cfgv==3.4.0 # via pre-commit @@ -32,35 +32,43 @@ colorama==0.4.6 # via tox distlib==0.3.8 # via virtualenv -docutils==0.21.2 +docutils==0.20.1 # via sphinx -filelock==3.14.0 +exceptiongroup==1.2.2 + # via + # pytest + # trio +filelock==3.15.4 # via # tox # virtualenv -identify==2.5.36 +identify==2.6.0 # via pre-commit -idna==3.7 +idna==3.8 # via # requests # trio imagesize==1.4.1 # via sphinx +importlib-metadata==8.4.0 + # via + # build + # sphinx iniconfig==2.0.0 # via pytest jinja2==3.1.4 # via sphinx markupsafe==2.1.5 # via jinja2 -mypy==1.10.0 +mypy==1.11.1 # via -r typing.in mypy-extensions==1.0.0 # via mypy -nodeenv==1.8.0 +nodeenv==1.9.1 # via pre-commit outcome==1.3.0.post0 # via trio -packaging==24.0 +packaging==24.1 # via # build # pallets-sphinx-themes @@ -70,11 +78,11 @@ packaging==24.0 # tox pallets-sphinx-themes==2.1.3 # via -r docs.in -pip-compile-multi==2.6.3 +pip-compile-multi==2.6.4 # via -r dev.in pip-tools==7.4.1 # via pip-compile-multi -platformdirs==4.2.1 +platformdirs==4.2.2 # via # tox # virtualenv @@ -82,21 +90,23 @@ pluggy==1.5.0 # via # pytest # tox -pre-commit==3.7.1 +pre-commit==3.5.0 # via -r dev.in pygments==2.18.0 # via sphinx -pyproject-api==1.6.1 +pyproject-api==1.7.1 # via tox pyproject-hooks==1.1.0 # via # build # pip-tools -pytest==8.2.0 +pytest==8.3.2 # via -r tests.in -pyyaml==6.0.1 +pytz==2024.1 + # via babel +pyyaml==6.0.2 # via pre-commit -requests==2.31.0 +requests==2.32.3 # via sphinx sniffio==1.3.1 # via trio @@ -104,7 +114,7 @@ snowballstemmer==2.2.0 # via sphinx sortedcontainers==2.4.0 # via trio -sphinx==7.3.7 +sphinx==7.1.2 # via # -r docs.in # pallets-sphinx-themes @@ -112,36 +122,46 @@ sphinx==7.3.7 # sphinxcontrib-log-cabinet sphinx-issues==4.1.0 # via -r docs.in -sphinxcontrib-applehelp==1.0.8 +sphinxcontrib-applehelp==1.0.4 # via sphinx -sphinxcontrib-devhelp==1.0.6 +sphinxcontrib-devhelp==1.0.2 # via sphinx -sphinxcontrib-htmlhelp==2.0.5 +sphinxcontrib-htmlhelp==2.0.1 # via sphinx sphinxcontrib-jsmath==1.0.1 # via sphinx sphinxcontrib-log-cabinet==1.0.1 # via -r docs.in -sphinxcontrib-qthelp==1.0.7 +sphinxcontrib-qthelp==1.0.3 # via sphinx -sphinxcontrib-serializinghtml==1.1.10 +sphinxcontrib-serializinghtml==1.1.5 # via sphinx +tomli==2.0.1 + # via + # build + # mypy + # pip-tools + # pyproject-api + # pytest + # tox toposort==1.10 # via pip-compile-multi -tox==4.15.0 +tox==4.18.0 # via -r dev.in -trio==0.25.0 +trio==0.26.2 # via -r tests.in -typing-extensions==4.11.0 +typing-extensions==4.12.2 # via mypy -urllib3==2.2.1 +urllib3==2.2.2 # via requests -virtualenv==20.26.1 +virtualenv==20.26.3 # via # pre-commit # tox -wheel==0.43.0 +wheel==0.44.0 # via pip-tools +zipp==3.20.0 + # via importlib-metadata # The following packages are considered to be unsafe in a requirements file: # pip diff --git a/requirements/docs.txt b/requirements/docs.txt index 2cbd73fa8..083b9cdc0 100644 --- a/requirements/docs.txt +++ b/requirements/docs.txt @@ -1,28 +1,30 @@ # -# This file is autogenerated by pip-compile with Python 3.12 +# This file is autogenerated by pip-compile with Python 3.8 # by the following command: # # pip-compile docs.in # -alabaster==0.7.16 +alabaster==0.7.13 # via sphinx -babel==2.15.0 +babel==2.16.0 # via sphinx -certifi==2024.2.2 +certifi==2024.7.4 # via requests charset-normalizer==3.3.2 # via requests -docutils==0.21.2 +docutils==0.20.1 # via sphinx -idna==3.7 +idna==3.8 # via requests imagesize==1.4.1 # via sphinx +importlib-metadata==8.4.0 + # via sphinx jinja2==3.1.4 # via sphinx markupsafe==2.1.5 # via jinja2 -packaging==24.0 +packaging==24.1 # via # pallets-sphinx-themes # sphinx @@ -30,11 +32,13 @@ pallets-sphinx-themes==2.1.3 # via -r docs.in pygments==2.18.0 # via sphinx -requests==2.31.0 +pytz==2024.1 + # via babel +requests==2.32.3 # via sphinx snowballstemmer==2.2.0 # via sphinx -sphinx==7.3.7 +sphinx==7.1.2 # via # -r docs.in # pallets-sphinx-themes @@ -42,19 +46,21 @@ sphinx==7.3.7 # sphinxcontrib-log-cabinet sphinx-issues==4.1.0 # via -r docs.in -sphinxcontrib-applehelp==1.0.8 +sphinxcontrib-applehelp==1.0.4 # via sphinx -sphinxcontrib-devhelp==1.0.6 +sphinxcontrib-devhelp==1.0.2 # via sphinx -sphinxcontrib-htmlhelp==2.0.5 +sphinxcontrib-htmlhelp==2.0.1 # via sphinx sphinxcontrib-jsmath==1.0.1 # via sphinx sphinxcontrib-log-cabinet==1.0.1 # via -r docs.in -sphinxcontrib-qthelp==1.0.7 +sphinxcontrib-qthelp==1.0.3 # via sphinx -sphinxcontrib-serializinghtml==1.1.10 +sphinxcontrib-serializinghtml==1.1.5 # via sphinx -urllib3==2.2.1 +urllib3==2.2.2 # via requests +zipp==3.20.0 + # via importlib-metadata diff --git a/requirements/tests.txt b/requirements/tests.txt index de18d477d..2b51fdbad 100644 --- a/requirements/tests.txt +++ b/requirements/tests.txt @@ -1,28 +1,34 @@ # -# This file is autogenerated by pip-compile with Python 3.12 +# This file is autogenerated by pip-compile with Python 3.8 # by the following command: # # pip-compile tests.in # -attrs==23.2.0 +attrs==24.2.0 # via # outcome # trio -idna==3.7 +exceptiongroup==1.2.2 + # via + # pytest + # trio +idna==3.8 # via trio iniconfig==2.0.0 # via pytest outcome==1.3.0.post0 # via trio -packaging==24.0 +packaging==24.1 # via pytest pluggy==1.5.0 # via pytest -pytest==8.2.0 +pytest==8.3.2 # via -r tests.in sniffio==1.3.1 # via trio sortedcontainers==2.4.0 # via trio -trio==0.25.0 +tomli==2.0.1 + # via pytest +trio==0.26.2 # via -r tests.in diff --git a/requirements/tests37.txt b/requirements/tests37.txt index 578789e7a..81c6213c0 100644 --- a/requirements/tests37.txt +++ b/requirements/tests37.txt @@ -4,15 +4,15 @@ # # pip-compile tests37.in # -attrs==23.2.0 +attrs==24.2.0 # via # outcome # trio -exceptiongroup==1.2.1 +exceptiongroup==1.2.2 # via # pytest # trio -idna==3.7 +idna==3.8 # via trio importlib-metadata==6.7.0 # via diff --git a/requirements/typing.txt b/requirements/typing.txt index c08a53767..10216b9c4 100644 --- a/requirements/typing.txt +++ b/requirements/typing.txt @@ -1,12 +1,14 @@ # -# This file is autogenerated by pip-compile with Python 3.12 +# This file is autogenerated by pip-compile with Python 3.8 # by the following command: # # pip-compile typing.in # -mypy==1.10.0 +mypy==1.11.1 # via -r typing.in mypy-extensions==1.0.0 # via mypy -typing-extensions==4.11.0 +tomli==2.0.1 + # via mypy +typing-extensions==4.12.2 # via mypy From 3adf44dde21117f784aef4a99d8083166b9f80ef Mon Sep 17 00:00:00 2001 From: David Lord Date: Fri, 23 Aug 2024 16:49:44 -0700 Subject: [PATCH 09/52] apply ruff fixes --- pyproject.toml | 1 - src/jinja2/async_utils.py | 2 +- src/jinja2/debug.py | 2 +- src/jinja2/environment.py | 4 ++-- src/jinja2/filters.py | 2 +- src/jinja2/idtracking.py | 4 ++-- src/jinja2/lexer.py | 2 +- src/jinja2/parser.py | 2 +- src/jinja2/runtime.py | 2 +- src/jinja2/sandbox.py | 2 +- src/jinja2/utils.py | 2 +- 11 files changed, 12 insertions(+), 13 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 5de076b17..a7cfa7f93 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -91,7 +91,6 @@ select = [ "UP", # pyupgrade "W", # pycodestyle warning ] -ignore-init-module-imports = true [tool.ruff.lint.isort] force-single-line = true diff --git a/src/jinja2/async_utils.py b/src/jinja2/async_utils.py index b0d277de7..f0c140205 100644 --- a/src/jinja2/async_utils.py +++ b/src/jinja2/async_utils.py @@ -67,7 +67,7 @@ async def auto_await(value: t.Union[t.Awaitable["V"], "V"]) -> "V": if inspect.isawaitable(value): return await t.cast("t.Awaitable[V]", value) - return t.cast("V", value) + return value class _IteratorToAsyncIterator(t.Generic[V]): diff --git a/src/jinja2/debug.py b/src/jinja2/debug.py index 7ed7e9297..eeeeee78b 100644 --- a/src/jinja2/debug.py +++ b/src/jinja2/debug.py @@ -152,7 +152,7 @@ def get_template_locals(real_locals: t.Mapping[str, t.Any]) -> t.Dict[str, t.Any available at that point in the template. """ # Start with the current template context. - ctx: "t.Optional[Context]" = real_locals.get("context") + ctx: t.Optional[Context] = real_locals.get("context") if ctx is not None: data: t.Dict[str, t.Any] = ctx.get_all().copy() diff --git a/src/jinja2/environment.py b/src/jinja2/environment.py index 57a7f8966..f062e4074 100644 --- a/src/jinja2/environment.py +++ b/src/jinja2/environment.py @@ -706,7 +706,7 @@ def _compile(self, source: str, filename: str) -> CodeType: return compile(source, filename, "exec") @typing.overload - def compile( # type: ignore + def compile( self, source: t.Union[str, nodes.Template], name: t.Optional[str] = None, @@ -1248,7 +1248,7 @@ def _from_namespace( namespace: t.MutableMapping[str, t.Any], globals: t.MutableMapping[str, t.Any], ) -> "Template": - t: "Template" = object.__new__(cls) + t: Template = object.__new__(cls) t.environment = environment t.globals = globals t.name = namespace["name"] diff --git a/src/jinja2/filters.py b/src/jinja2/filters.py index acd11976e..14208770d 100644 --- a/src/jinja2/filters.py +++ b/src/jinja2/filters.py @@ -1116,7 +1116,7 @@ def do_batch( {%- endfor %} """ - tmp: "t.List[V]" = [] + tmp: t.List[V] = [] for item in value: if len(tmp) == linecount: diff --git a/src/jinja2/idtracking.py b/src/jinja2/idtracking.py index 995ebaa0c..d6cb635b2 100644 --- a/src/jinja2/idtracking.py +++ b/src/jinja2/idtracking.py @@ -146,7 +146,7 @@ def branch_update(self, branch_symbols: t.Sequence["Symbols"]) -> None: def dump_stores(self) -> t.Dict[str, str]: rv: t.Dict[str, str] = {} - node: t.Optional["Symbols"] = self + node: t.Optional[Symbols] = self while node is not None: for name in sorted(node.stores): @@ -159,7 +159,7 @@ def dump_stores(self) -> t.Dict[str, str]: def dump_param_targets(self) -> t.Set[str]: rv = set() - node: t.Optional["Symbols"] = self + node: t.Optional[Symbols] = self while node is not None: for target, (instr, _) in self.loads.items(): diff --git a/src/jinja2/lexer.py b/src/jinja2/lexer.py index 62b0471a3..6dc94b67d 100644 --- a/src/jinja2/lexer.py +++ b/src/jinja2/lexer.py @@ -329,7 +329,7 @@ def __init__( filename: t.Optional[str], ): self._iter = iter(generator) - self._pushed: "te.Deque[Token]" = deque() + self._pushed: te.Deque[Token] = deque() self.name = name self.filename = filename self.closed = False diff --git a/src/jinja2/parser.py b/src/jinja2/parser.py index 0ec997fb4..22f3f81f7 100644 --- a/src/jinja2/parser.py +++ b/src/jinja2/parser.py @@ -64,7 +64,7 @@ def __init__( self.filename = filename self.closed = False self.extensions: t.Dict[ - str, t.Callable[["Parser"], t.Union[nodes.Node, t.List[nodes.Node]]] + str, t.Callable[[Parser], t.Union[nodes.Node, t.List[nodes.Node]]] ] = {} for extension in environment.iter_extensions(): for tag in extension.tags: diff --git a/src/jinja2/runtime.py b/src/jinja2/runtime.py index 4325c8deb..53582ae8b 100644 --- a/src/jinja2/runtime.py +++ b/src/jinja2/runtime.py @@ -172,7 +172,7 @@ def __init__( ): self.parent = parent self.vars: t.Dict[str, t.Any] = {} - self.environment: "Environment" = environment + self.environment: Environment = environment self.eval_ctx = EvalContext(self.environment, name) self.exported_vars: t.Set[str] = set() self.name = name diff --git a/src/jinja2/sandbox.py b/src/jinja2/sandbox.py index 0b4fc12d3..ce276156c 100644 --- a/src/jinja2/sandbox.py +++ b/src/jinja2/sandbox.py @@ -5,11 +5,11 @@ import operator import types import typing as t +from _string import formatter_field_name_split # type: ignore from collections import abc from collections import deque from string import Formatter -from _string import formatter_field_name_split # type: ignore from markupsafe import EscapeFormatter from markupsafe import Markup diff --git a/src/jinja2/utils.py b/src/jinja2/utils.py index 7fb76935a..5c1ff5d7b 100644 --- a/src/jinja2/utils.py +++ b/src/jinja2/utils.py @@ -428,7 +428,7 @@ class LRUCache: def __init__(self, capacity: int) -> None: self.capacity = capacity self._mapping: t.Dict[t.Any, t.Any] = {} - self._queue: "te.Deque[t.Any]" = deque() + self._queue: te.Deque[t.Any] = deque() self._postinit() def _postinit(self) -> None: From 896a1d59b7c831a594db8719c784e0823e16d3c7 Mon Sep 17 00:00:00 2001 From: David Lord Date: Fri, 23 Aug 2024 16:50:05 -0700 Subject: [PATCH 10/52] remove dependabot --- .github/dependabot.yml | 24 ------------------------ 1 file changed, 24 deletions(-) delete mode 100644 .github/dependabot.yml diff --git a/.github/dependabot.yml b/.github/dependabot.yml deleted file mode 100644 index fa94b770a..000000000 --- a/.github/dependabot.yml +++ /dev/null @@ -1,24 +0,0 @@ -version: 2 -updates: - - package-ecosystem: github-actions - directory: / - schedule: - interval: monthly - ignore: - # slsa depends on upload/download v3 - - dependency-name: actions/upload-artifact - versions: '>= 4' - - dependency-name: actions/download-artifact - versions: '>= 4' - groups: - github-actions: - patterns: - - '*' - - package-ecosystem: pip - directory: /requirements/ - schedule: - interval: monthly - groups: - python-requirements: - patterns: - - '*' From 3e5b5b2794b3ced0efb36906d80ced7626781751 Mon Sep 17 00:00:00 2001 From: David Lord Date: Fri, 23 Aug 2024 17:15:36 -0700 Subject: [PATCH 11/52] refactor 3.7 test pins --- requirements/build.txt | 8 +------- requirements/dev.txt | 40 ++++++++++------------------------------ requirements/docs.txt | 24 +++++++++--------------- requirements/tests.txt | 8 +------- requirements/tests37.in | 2 -- requirements/tests37.txt | 6 +++--- requirements/typing.txt | 4 +--- tox.ini | 3 +-- 8 files changed, 26 insertions(+), 69 deletions(-) delete mode 100644 requirements/tests37.in diff --git a/requirements/build.txt b/requirements/build.txt index 96ee8f08e..4b289ca7f 100644 --- a/requirements/build.txt +++ b/requirements/build.txt @@ -1,18 +1,12 @@ # -# This file is autogenerated by pip-compile with Python 3.8 +# This file is autogenerated by pip-compile with Python 3.12 # by the following command: # # pip-compile build.in # build==1.2.1 # via -r build.in -importlib-metadata==8.4.0 - # via build packaging==24.1 # via build pyproject-hooks==1.1.0 # via build -tomli==2.0.1 - # via build -zipp==3.20.0 - # via importlib-metadata diff --git a/requirements/dev.txt b/requirements/dev.txt index d78f0cf61..51aecad83 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -1,10 +1,10 @@ # -# This file is autogenerated by pip-compile with Python 3.8 +# This file is autogenerated by pip-compile with Python 3.12 # by the following command: # # pip-compile dev.in # -alabaster==0.7.13 +alabaster==1.0.0 # via sphinx attrs==24.2.0 # via @@ -32,12 +32,8 @@ colorama==0.4.6 # via tox distlib==0.3.8 # via virtualenv -docutils==0.20.1 +docutils==0.21.2 # via sphinx -exceptiongroup==1.2.2 - # via - # pytest - # trio filelock==3.15.4 # via # tox @@ -50,10 +46,6 @@ idna==3.8 # trio imagesize==1.4.1 # via sphinx -importlib-metadata==8.4.0 - # via - # build - # sphinx iniconfig==2.0.0 # via pytest jinja2==3.1.4 @@ -90,7 +82,7 @@ pluggy==1.5.0 # via # pytest # tox -pre-commit==3.5.0 +pre-commit==3.8.0 # via -r dev.in pygments==2.18.0 # via sphinx @@ -102,8 +94,6 @@ pyproject-hooks==1.1.0 # pip-tools pytest==8.3.2 # via -r tests.in -pytz==2024.1 - # via babel pyyaml==6.0.2 # via pre-commit requests==2.32.3 @@ -114,7 +104,7 @@ snowballstemmer==2.2.0 # via sphinx sortedcontainers==2.4.0 # via trio -sphinx==7.1.2 +sphinx==8.0.2 # via # -r docs.in # pallets-sphinx-themes @@ -122,28 +112,20 @@ sphinx==7.1.2 # sphinxcontrib-log-cabinet sphinx-issues==4.1.0 # via -r docs.in -sphinxcontrib-applehelp==1.0.4 +sphinxcontrib-applehelp==2.0.0 # via sphinx -sphinxcontrib-devhelp==1.0.2 +sphinxcontrib-devhelp==2.0.0 # via sphinx -sphinxcontrib-htmlhelp==2.0.1 +sphinxcontrib-htmlhelp==2.1.0 # via sphinx sphinxcontrib-jsmath==1.0.1 # via sphinx sphinxcontrib-log-cabinet==1.0.1 # via -r docs.in -sphinxcontrib-qthelp==1.0.3 +sphinxcontrib-qthelp==2.0.0 # via sphinx -sphinxcontrib-serializinghtml==1.1.5 +sphinxcontrib-serializinghtml==2.0.0 # via sphinx -tomli==2.0.1 - # via - # build - # mypy - # pip-tools - # pyproject-api - # pytest - # tox toposort==1.10 # via pip-compile-multi tox==4.18.0 @@ -160,8 +142,6 @@ virtualenv==20.26.3 # tox wheel==0.44.0 # via pip-tools -zipp==3.20.0 - # via importlib-metadata # The following packages are considered to be unsafe in a requirements file: # pip diff --git a/requirements/docs.txt b/requirements/docs.txt index 083b9cdc0..04e39ac5f 100644 --- a/requirements/docs.txt +++ b/requirements/docs.txt @@ -1,10 +1,10 @@ # -# This file is autogenerated by pip-compile with Python 3.8 +# This file is autogenerated by pip-compile with Python 3.12 # by the following command: # # pip-compile docs.in # -alabaster==0.7.13 +alabaster==1.0.0 # via sphinx babel==2.16.0 # via sphinx @@ -12,14 +12,12 @@ certifi==2024.7.4 # via requests charset-normalizer==3.3.2 # via requests -docutils==0.20.1 +docutils==0.21.2 # via sphinx idna==3.8 # via requests imagesize==1.4.1 # via sphinx -importlib-metadata==8.4.0 - # via sphinx jinja2==3.1.4 # via sphinx markupsafe==2.1.5 @@ -32,13 +30,11 @@ pallets-sphinx-themes==2.1.3 # via -r docs.in pygments==2.18.0 # via sphinx -pytz==2024.1 - # via babel requests==2.32.3 # via sphinx snowballstemmer==2.2.0 # via sphinx -sphinx==7.1.2 +sphinx==8.0.2 # via # -r docs.in # pallets-sphinx-themes @@ -46,21 +42,19 @@ sphinx==7.1.2 # sphinxcontrib-log-cabinet sphinx-issues==4.1.0 # via -r docs.in -sphinxcontrib-applehelp==1.0.4 +sphinxcontrib-applehelp==2.0.0 # via sphinx -sphinxcontrib-devhelp==1.0.2 +sphinxcontrib-devhelp==2.0.0 # via sphinx -sphinxcontrib-htmlhelp==2.0.1 +sphinxcontrib-htmlhelp==2.1.0 # via sphinx sphinxcontrib-jsmath==1.0.1 # via sphinx sphinxcontrib-log-cabinet==1.0.1 # via -r docs.in -sphinxcontrib-qthelp==1.0.3 +sphinxcontrib-qthelp==2.0.0 # via sphinx -sphinxcontrib-serializinghtml==1.1.5 +sphinxcontrib-serializinghtml==2.0.0 # via sphinx urllib3==2.2.2 # via requests -zipp==3.20.0 - # via importlib-metadata diff --git a/requirements/tests.txt b/requirements/tests.txt index 2b51fdbad..03ecbc4f8 100644 --- a/requirements/tests.txt +++ b/requirements/tests.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.8 +# This file is autogenerated by pip-compile with Python 3.12 # by the following command: # # pip-compile tests.in @@ -8,10 +8,6 @@ attrs==24.2.0 # via # outcome # trio -exceptiongroup==1.2.2 - # via - # pytest - # trio idna==3.8 # via trio iniconfig==2.0.0 @@ -28,7 +24,5 @@ sniffio==1.3.1 # via trio sortedcontainers==2.4.0 # via trio -tomli==2.0.1 - # via pytest trio==0.26.2 # via -r tests.in diff --git a/requirements/tests37.in b/requirements/tests37.in deleted file mode 100644 index 9c2bed180..000000000 --- a/requirements/tests37.in +++ /dev/null @@ -1,2 +0,0 @@ -pytest -trio==0.22.2 diff --git a/requirements/tests37.txt b/requirements/tests37.txt index 81c6213c0..8efd75ed3 100644 --- a/requirements/tests37.txt +++ b/requirements/tests37.txt @@ -2,7 +2,7 @@ # This file is autogenerated by pip-compile with Python 3.7 # by the following command: # -# pip-compile tests37.in +# pip-compile --output-file=tests37.txt tests.in # attrs==24.2.0 # via @@ -28,7 +28,7 @@ packaging==24.0 pluggy==1.2.0 # via pytest pytest==7.4.4 - # via -r tests37.in + # via -r tests.in sniffio==1.3.1 # via trio sortedcontainers==2.4.0 @@ -36,7 +36,7 @@ sortedcontainers==2.4.0 tomli==2.0.1 # via pytest trio==0.22.2 - # via -r tests37.in + # via -r tests.in typing-extensions==4.7.1 # via importlib-metadata zipp==3.15.0 diff --git a/requirements/typing.txt b/requirements/typing.txt index 10216b9c4..93354adf5 100644 --- a/requirements/typing.txt +++ b/requirements/typing.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.8 +# This file is autogenerated by pip-compile with Python 3.12 # by the following command: # # pip-compile typing.in @@ -8,7 +8,5 @@ mypy==1.11.1 # via -r typing.in mypy-extensions==1.0.0 # via mypy -tomli==2.0.1 - # via mypy typing-extensions==4.12.2 # via mypy diff --git a/tox.ini b/tox.ini index a40f36a44..5e15000d0 100644 --- a/tox.ini +++ b/tox.ini @@ -43,7 +43,6 @@ skip_install = true commands = pre-commit autoupdate -j4 [testenv:update-requirements] -base_python = 3.8 labels = update deps = pip-tools skip_install = true @@ -61,4 +60,4 @@ labels = update deps = pip-tools skip_install = true change_dir = requirements -commands = pip-compile tests37.in -q {posargs:-U} +commands = pip-compile tests.in -q -o tests37.txt {posargs:-U} From a9a0197e3c63cc8abc965369d252cca9e866be6d Mon Sep 17 00:00:00 2001 From: Aarni Koskela Date: Tue, 10 Sep 2024 18:23:08 +0300 Subject: [PATCH 12/52] Improve documentation for initializing the i18n extension Refs discussion at https://stackoverflow.com/a/78970088 --- docs/extensions.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/extensions.rst b/docs/extensions.rst index 45ead3b71..9b15e813e 100644 --- a/docs/extensions.rst +++ b/docs/extensions.rst @@ -39,6 +39,10 @@ After enabling, an application has to provide functions for ``gettext``, globally or when rendering. A ``_()`` function is added as an alias to the ``gettext`` function. +A convenient way to provide these functions is to call one of the below +methods depending on the translation system in use. If you do not require +actual translation, use ``Environment.install_null_translations`` to +install no-op functions. Environment Methods ~~~~~~~~~~~~~~~~~~~ From 1e383959f7bae1b0dbf84ef47128a4ef55bcb1a7 Mon Sep 17 00:00:00 2001 From: David Lord Date: Thu, 24 Oct 2024 14:13:23 -0700 Subject: [PATCH 13/52] update dev dependencies --- .github/workflows/publish.yaml | 10 ++++---- .github/workflows/tests.yaml | 10 ++++---- .pre-commit-config.yaml | 4 +-- requirements/build.txt | 6 ++--- requirements/dev.txt | 47 ++++++++++++++++++---------------- requirements/docs.txt | 21 ++++++++------- requirements/tests.txt | 8 +++--- requirements/tests37.txt | 2 +- requirements/typing.txt | 4 +-- 9 files changed, 59 insertions(+), 53 deletions(-) diff --git a/.github/workflows/publish.yaml b/.github/workflows/publish.yaml index 7acd92e5b..727518c69 100644 --- a/.github/workflows/publish.yaml +++ b/.github/workflows/publish.yaml @@ -9,8 +9,8 @@ jobs: outputs: hash: ${{ steps.hash.outputs.hash }} steps: - - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 - - uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v5.1.1 + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + - uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0 with: python-version: '3.x' cache: pip @@ -23,7 +23,7 @@ jobs: - name: generate hash id: hash run: cd dist && echo "hash=$(sha256sum * | base64 -w0)" >> $GITHUB_OUTPUT - - uses: actions/upload-artifact@834a144ee995460fba8ed112a2fc961b36a5ec5a # v4.3.6 + - uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3 with: path: ./dist provenance: @@ -64,10 +64,10 @@ jobs: id-token: write steps: - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 - - uses: pypa/gh-action-pypi-publish@ec4db0b4ddc65acdf4bff5fa45ac92d78b56bdf0 # v1.9.0 + - uses: pypa/gh-action-pypi-publish@f7600683efdcb7656dec5b29656edb7bc586e597 # v1.10.3 with: repository-url: https://test.pypi.org/legacy/ packages-dir: artifact/ - - uses: pypa/gh-action-pypi-publish@ec4db0b4ddc65acdf4bff5fa45ac92d78b56bdf0 # v1.9.0 + - uses: pypa/gh-action-pypi-publish@f7600683efdcb7656dec5b29656edb7bc586e597 # v1.10.3 with: packages-dir: artifact/ diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index afba0d753..68736fb24 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -32,8 +32,8 @@ jobs: - {python: '3.7'} - {name: PyPy, python: 'pypy-3.10', tox: pypy310} steps: - - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 - - uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v5.1.1 + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + - uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0 with: python-version: ${{ matrix.python }} allow-prereleases: true @@ -44,14 +44,14 @@ jobs: typing: runs-on: ubuntu-latest steps: - - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 - - uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v5.1.1 + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + - uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0 with: python-version: '3.x' cache: pip cache-dependency-path: requirements*/*.txt - name: cache mypy - uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4.0.2 + uses: actions/cache@6849a6489940f00c2f30c0fb92c6274307ccb58a # v4.1.2 with: path: ./.mypy_cache key: mypy|${{ hashFiles('pyproject.toml') }} diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 5bee1ca4b..74b54e8f1 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -2,12 +2,12 @@ ci: autoupdate_schedule: monthly repos: - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.6.2 + rev: v0.7.1 hooks: - id: ruff - id: ruff-format - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.6.0 + rev: v5.0.0 hooks: - id: check-merge-conflict - id: debug-statements diff --git a/requirements/build.txt b/requirements/build.txt index 4b289ca7f..1b13b0552 100644 --- a/requirements/build.txt +++ b/requirements/build.txt @@ -1,12 +1,12 @@ # -# This file is autogenerated by pip-compile with Python 3.12 +# This file is autogenerated by pip-compile with Python 3.13 # by the following command: # # pip-compile build.in # -build==1.2.1 +build==1.2.2.post1 # via -r build.in packaging==24.1 # via build -pyproject-hooks==1.1.0 +pyproject-hooks==1.2.0 # via build diff --git a/requirements/dev.txt b/requirements/dev.txt index 51aecad83..ba73d911c 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.12 +# This file is autogenerated by pip-compile with Python 3.13 # by the following command: # # pip-compile dev.in @@ -12,17 +12,17 @@ attrs==24.2.0 # trio babel==2.16.0 # via sphinx -build==1.2.1 +build==1.2.2.post1 # via pip-tools cachetools==5.5.0 # via tox -certifi==2024.7.4 +certifi==2024.8.30 # via requests cfgv==3.4.0 # via pre-commit chardet==5.2.0 # via tox -charset-normalizer==3.3.2 +charset-normalizer==3.4.0 # via requests click==8.1.7 # via @@ -30,17 +30,17 @@ click==8.1.7 # pip-tools colorama==0.4.6 # via tox -distlib==0.3.8 +distlib==0.3.9 # via virtualenv docutils==0.21.2 # via sphinx -filelock==3.15.4 +filelock==3.16.1 # via # tox # virtualenv -identify==2.6.0 +identify==2.6.1 # via pre-commit -idna==3.8 +idna==3.10 # via # requests # trio @@ -50,9 +50,9 @@ iniconfig==2.0.0 # via pytest jinja2==3.1.4 # via sphinx -markupsafe==2.1.5 +markupsafe==3.0.2 # via jinja2 -mypy==1.11.1 +mypy==1.13.0 # via -r typing.in mypy-extensions==1.0.0 # via mypy @@ -68,13 +68,13 @@ packaging==24.1 # pytest # sphinx # tox -pallets-sphinx-themes==2.1.3 +pallets-sphinx-themes==2.3.0 # via -r docs.in pip-compile-multi==2.6.4 # via -r dev.in pip-tools==7.4.1 # via pip-compile-multi -platformdirs==4.2.2 +platformdirs==4.3.6 # via # tox # virtualenv @@ -82,17 +82,17 @@ pluggy==1.5.0 # via # pytest # tox -pre-commit==3.8.0 +pre-commit==4.0.1 # via -r dev.in pygments==2.18.0 # via sphinx -pyproject-api==1.7.1 +pyproject-api==1.8.0 # via tox -pyproject-hooks==1.1.0 +pyproject-hooks==1.2.0 # via # build # pip-tools -pytest==8.3.2 +pytest==8.3.3 # via -r tests.in pyyaml==6.0.2 # via pre-commit @@ -104,14 +104,17 @@ snowballstemmer==2.2.0 # via sphinx sortedcontainers==2.4.0 # via trio -sphinx==8.0.2 +sphinx==8.1.3 # via # -r docs.in # pallets-sphinx-themes # sphinx-issues + # sphinx-notfound-page # sphinxcontrib-log-cabinet -sphinx-issues==4.1.0 +sphinx-issues==5.0.0 # via -r docs.in +sphinx-notfound-page==1.0.4 + # via pallets-sphinx-themes sphinxcontrib-applehelp==2.0.0 # via sphinx sphinxcontrib-devhelp==2.0.0 @@ -128,15 +131,15 @@ sphinxcontrib-serializinghtml==2.0.0 # via sphinx toposort==1.10 # via pip-compile-multi -tox==4.18.0 +tox==4.23.2 # via -r dev.in -trio==0.26.2 +trio==0.27.0 # via -r tests.in typing-extensions==4.12.2 # via mypy -urllib3==2.2.2 +urllib3==2.2.3 # via requests -virtualenv==20.26.3 +virtualenv==20.27.0 # via # pre-commit # tox diff --git a/requirements/docs.txt b/requirements/docs.txt index 04e39ac5f..453a7cb5d 100644 --- a/requirements/docs.txt +++ b/requirements/docs.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.12 +# This file is autogenerated by pip-compile with Python 3.13 # by the following command: # # pip-compile docs.in @@ -8,25 +8,25 @@ alabaster==1.0.0 # via sphinx babel==2.16.0 # via sphinx -certifi==2024.7.4 +certifi==2024.8.30 # via requests -charset-normalizer==3.3.2 +charset-normalizer==3.4.0 # via requests docutils==0.21.2 # via sphinx -idna==3.8 +idna==3.10 # via requests imagesize==1.4.1 # via sphinx jinja2==3.1.4 # via sphinx -markupsafe==2.1.5 +markupsafe==3.0.2 # via jinja2 packaging==24.1 # via # pallets-sphinx-themes # sphinx -pallets-sphinx-themes==2.1.3 +pallets-sphinx-themes==2.3.0 # via -r docs.in pygments==2.18.0 # via sphinx @@ -34,14 +34,17 @@ requests==2.32.3 # via sphinx snowballstemmer==2.2.0 # via sphinx -sphinx==8.0.2 +sphinx==8.1.3 # via # -r docs.in # pallets-sphinx-themes # sphinx-issues + # sphinx-notfound-page # sphinxcontrib-log-cabinet -sphinx-issues==4.1.0 +sphinx-issues==5.0.0 # via -r docs.in +sphinx-notfound-page==1.0.4 + # via pallets-sphinx-themes sphinxcontrib-applehelp==2.0.0 # via sphinx sphinxcontrib-devhelp==2.0.0 @@ -56,5 +59,5 @@ sphinxcontrib-qthelp==2.0.0 # via sphinx sphinxcontrib-serializinghtml==2.0.0 # via sphinx -urllib3==2.2.2 +urllib3==2.2.3 # via requests diff --git a/requirements/tests.txt b/requirements/tests.txt index 03ecbc4f8..e019ba988 100644 --- a/requirements/tests.txt +++ b/requirements/tests.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.12 +# This file is autogenerated by pip-compile with Python 3.13 # by the following command: # # pip-compile tests.in @@ -8,7 +8,7 @@ attrs==24.2.0 # via # outcome # trio -idna==3.8 +idna==3.10 # via trio iniconfig==2.0.0 # via pytest @@ -18,11 +18,11 @@ packaging==24.1 # via pytest pluggy==1.5.0 # via pytest -pytest==8.3.2 +pytest==8.3.3 # via -r tests.in sniffio==1.3.1 # via trio sortedcontainers==2.4.0 # via trio -trio==0.26.2 +trio==0.27.0 # via -r tests.in diff --git a/requirements/tests37.txt b/requirements/tests37.txt index 8efd75ed3..e8e3492fe 100644 --- a/requirements/tests37.txt +++ b/requirements/tests37.txt @@ -12,7 +12,7 @@ exceptiongroup==1.2.2 # via # pytest # trio -idna==3.8 +idna==3.10 # via trio importlib-metadata==6.7.0 # via diff --git a/requirements/typing.txt b/requirements/typing.txt index 93354adf5..1cf3727a5 100644 --- a/requirements/typing.txt +++ b/requirements/typing.txt @@ -1,10 +1,10 @@ # -# This file is autogenerated by pip-compile with Python 3.12 +# This file is autogenerated by pip-compile with Python 3.13 # by the following command: # # pip-compile typing.in # -mypy==1.11.1 +mypy==1.13.0 # via -r typing.in mypy-extensions==1.0.0 # via mypy From d3d0910d8a05b3f97a3bcfe881a2d99f15f69ced Mon Sep 17 00:00:00 2001 From: David Lord Date: Thu, 24 Oct 2024 14:13:40 -0700 Subject: [PATCH 14/52] update test workflow trigger --- .github/workflows/tests.yaml | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 68736fb24..515a7a5e4 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -1,18 +1,10 @@ name: Tests on: push: - branches: - - main - - '*.x' - paths-ignore: - - 'docs/**' - - '*.md' - - '*.rst' + branches: [main, stable] + paths-ignore: ['docs/**', '*.md', '*.rst'] pull_request: - paths-ignore: - - 'docs/**' - - '*.md' - - '*.rst' + paths-ignore: [ 'docs/**', '*.md', '*.rst' ] jobs: tests: name: ${{ matrix.name || matrix.python }} From f502aac8dcc743f221f3c5ae131d8722135c349f Mon Sep 17 00:00:00 2001 From: Matheus Felipe Date: Thu, 1 Dec 2022 01:31:40 -0300 Subject: [PATCH 15/52] Add link to MarkupSafe in FAQ --- docs/faq.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/faq.rst b/docs/faq.rst index 493dc38c6..a53ae12ff 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -70,6 +70,8 @@ these document types. While automatic escaping means that you are less likely have an XSS problem, it also requires significant extra processing during compiling -and rendering, which can reduce performance. Jinja uses MarkupSafe for +and rendering, which can reduce performance. Jinja uses `MarkupSafe`_ for escaping, which provides optimized C code for speed, but it still introduces overhead to track escaping across methods and formatting. + +.. _MarkupSafe: https://markupsafe.palletsprojects.com/ From 4e7850ce1b26f3108c79e9f59f641f6dcf4d4d6a Mon Sep 17 00:00:00 2001 From: Clay Sweetser Date: Thu, 2 Mar 2023 16:53:14 -0500 Subject: [PATCH 16/52] Clarify what operations the default Undefined supports --- src/jinja2/runtime.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/jinja2/runtime.py b/src/jinja2/runtime.py index 53582ae8b..d10fe9d06 100644 --- a/src/jinja2/runtime.py +++ b/src/jinja2/runtime.py @@ -792,8 +792,8 @@ def __repr__(self) -> str: class Undefined: - """The default undefined type. This undefined type can be printed and - iterated over, but every other access will raise an :exc:`UndefinedError`: + """The default undefined type. This can be printed, iterated, and treated as + a boolean. Any other operation will raise an :exc:`UndefinedError`. >>> foo = Undefined(name='foo') >>> str(foo) From 9c3622c1af8e8e2342c6e7d44de513e2a6c1c441 Mon Sep 17 00:00:00 2001 From: Hugo Vassard Date: Fri, 3 Mar 2023 11:42:07 +0100 Subject: [PATCH 17/52] fix boolean error about whitespace control --- docs/templates.rst | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/docs/templates.rst b/docs/templates.rst index 2471cea39..2cb1d7a48 100644 --- a/docs/templates.rst +++ b/docs/templates.rst @@ -202,10 +202,11 @@ option can also be set to strip tabs and spaces from the beginning of a line to the start of a block. (Nothing will be stripped if there are other characters before the start of the block.) -With both `trim_blocks` and `lstrip_blocks` enabled, you can put block tags -on their own lines, and the entire block line will be removed when -rendered, preserving the whitespace of the contents. For example, -without the `trim_blocks` and `lstrip_blocks` options, this template:: +With both ``trim_blocks`` and ``lstrip_blocks`` disabled (the default), block +tags on their own lines will be removed, but a blank line will remain and the +spaces in the content will be preserved. For example, this template: + +.. code-block:: jinja
{% if True %} @@ -213,7 +214,10 @@ without the `trim_blocks` and `lstrip_blocks` options, this template:: {% endif %}
-gets rendered with blank lines inside the div:: +With both ``trim_blocks`` and ``lstrip_blocks`` disabled, the template is +rendered with blank lines inside the div: + +.. code-block:: text
@@ -221,8 +225,10 @@ gets rendered with blank lines inside the div::
-But with both `trim_blocks` and `lstrip_blocks` enabled, the template block -lines are removed and other whitespace is preserved:: +With both ``trim_blocks`` and ``lstrip_blocks`` enabled, the template block +lines are completely removed: + +.. code-block:: text
yay From 8a90b760a8cb3cff9cd9fe8a7b899405b044244b Mon Sep 17 00:00:00 2001 From: Meng Xiangzhuo Date: Tue, 29 Aug 2023 13:19:59 +0800 Subject: [PATCH 18/52] fix a typo in docs/templates.rst --- docs/templates.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/templates.rst b/docs/templates.rst index 2cb1d7a48..aff7e172c 100644 --- a/docs/templates.rst +++ b/docs/templates.rst @@ -1784,7 +1784,7 @@ It's possible to translate strings in expressions with these functions: - ``_(message)``: Alias for ``gettext``. - ``gettext(message)``: Translate a message. -- ``ngettext(singluar, plural, n)``: Translate a singular or plural +- ``ngettext(singular, plural, n)``: Translate a singular or plural message based on a count variable. - ``pgettext(context, message)``: Like ``gettext()``, but picks the translation based on the context string. From 7d023e5a8600d0db1e6d11f5434737dbcccfcab6 Mon Sep 17 00:00:00 2001 From: Vitor Buxbaum Date: Fri, 17 Nov 2023 09:04:42 -0300 Subject: [PATCH 19/52] Fix typo on filter name --- docs/api.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/api.rst b/docs/api.rst index cb62f6c32..c0fa163a0 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -666,8 +666,8 @@ Now it can be used in templates: .. sourcecode:: jinja - {{ article.pub_date|datetimeformat }} - {{ article.pub_date|datetimeformat("%B %Y") }} + {{ article.pub_date|datetime_format }} + {{ article.pub_date|datetime_format("%B %Y") }} Some decorators are available to tell Jinja to pass extra information to the filter. The object is passed as the first argument, making the value From 64a6bd1b66fdaa11aa21ac238f40c02c1e0074ca Mon Sep 17 00:00:00 2001 From: Stephen Rosen Date: Fri, 16 Feb 2024 01:39:44 -0600 Subject: [PATCH 20/52] improve clarity of logical bool ops co-authored-by: David Lord --- docs/templates.rst | 30 +++++++++++++++++------------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/docs/templates.rst b/docs/templates.rst index aff7e172c..2bb28f610 100644 --- a/docs/templates.rst +++ b/docs/templates.rst @@ -1412,28 +1412,32 @@ Comparisons Logic ~~~~~ -For ``if`` statements, ``for`` filtering, and ``if`` expressions, it can be useful to -combine multiple expressions: +For ``if`` statements, ``for`` filtering, and ``if`` expressions, it can be +useful to combine multiple expressions. ``and`` - Return true if the left and the right operand are true. + For ``x and y``, if ``x`` is false, then the value is ``x``, else ``y``. In + a boolean context, this will be treated as ``True`` if both operands are + truthy. ``or`` - Return true if the left or the right operand are true. + For ``x or y``, if ``x`` is true, then the value is ``x``, else ``y``. In a + boolean context, this will be treated as ``True`` if at least one operand is + truthy. ``not`` - negate a statement (see below). + For ``not x``, if ``x`` is false, then the value is ``True``, else + ``False``. -``(expr)`` - Parentheses group an expression. - -.. admonition:: Note - - The ``is`` and ``in`` operators support negation using an infix notation, - too: ``foo is not bar`` and ``foo not in bar`` instead of ``not foo is bar`` - and ``not foo in bar``. All other expressions require a prefix notation: + Prefer negating ``is`` and ``in`` using their infix notation: + ``foo is not bar`` instead of ``not foo is bar``; ``foo not in bar`` instead + of ``not foo in bar``. All other expressions require prefix notation: ``not (foo and bar).`` +``(expr)`` + Parentheses group an expression. This is used to change evaluation order, or + to make a long expression easier to read or less ambiguous. + Other Operators ~~~~~~~~~~~~~~~ From 75f0fbf6cb47b531a9c277139a505b9315864330 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20Lindh=C3=A9?= <7773090+lindhe@users.noreply.github.com> Date: Mon, 12 Aug 2024 10:54:47 +0200 Subject: [PATCH 21/52] fix list comprehension example --- src/jinja2/filters.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/jinja2/filters.py b/src/jinja2/filters.py index 14208770d..4c949cbde 100644 --- a/src/jinja2/filters.py +++ b/src/jinja2/filters.py @@ -1629,8 +1629,8 @@ def sync_do_selectattr( .. code-block:: python - (u for user in users if user.is_active) - (u for user in users if test_none(user.email)) + (user for user in users if user.is_active) + (user for user in users if test_none(user.email)) .. versionadded:: 2.7 """ @@ -1667,8 +1667,8 @@ def sync_do_rejectattr( .. code-block:: python - (u for user in users if not user.is_active) - (u for user in users if not test_none(user.email)) + (user for user in users if not user.is_active) + (user for user in users if not test_none(user.email)) .. versionadded:: 2.7 """ From c667d56de3d0129971175d75f0389d5fef48d0e3 Mon Sep 17 00:00:00 2001 From: David Lord Date: Wed, 18 Dec 2024 09:33:31 -0800 Subject: [PATCH 22/52] change "per default" to "by default" --- CHANGES.rst | 2 +- docs/templates.rst | 2 +- docs/tricks.rst | 2 +- src/jinja2/loaders.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index 7fb729763..feb1e6c3d 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1001,7 +1001,7 @@ Released 2008-07-17, codename Jinjavitus evaluates to ``false``. - Improved error reporting for undefined values by providing a position. -- ``filesizeformat`` filter uses decimal prefixes now per default and +- ``filesizeformat`` filter uses decimal prefixes now by default and can be set to binary mode with the second parameter. - Fixed bug in finalizer diff --git a/docs/templates.rst b/docs/templates.rst index 2bb28f610..0eab8e664 100644 --- a/docs/templates.rst +++ b/docs/templates.rst @@ -528,7 +528,7 @@ However, the name after the `endblock` word must match the block name. Block Nesting and Scope ~~~~~~~~~~~~~~~~~~~~~~~ -Blocks can be nested for more complex layouts. However, per default blocks +Blocks can be nested for more complex layouts. However, by default blocks may not access variables from outer scopes:: {% for item in seq %} diff --git a/docs/tricks.rst b/docs/tricks.rst index b58c5bb09..3a7084a6d 100644 --- a/docs/tricks.rst +++ b/docs/tricks.rst @@ -21,7 +21,7 @@ for a neat trick. Usually child templates extend from one template that adds a basic HTML skeleton. However it's possible to put the `extends` tag into an `if` tag to only extend from the layout template if the `standalone` variable evaluates -to false which it does per default if it's not defined. Additionally a very +to false, which it does by default if it's not defined. Additionally a very basic skeleton is added to the file so that if it's indeed rendered with `standalone` set to `True` a very basic HTML skeleton is added:: diff --git a/src/jinja2/loaders.py b/src/jinja2/loaders.py index 8c2c86cd0..b1ad401c0 100644 --- a/src/jinja2/loaders.py +++ b/src/jinja2/loaders.py @@ -430,7 +430,7 @@ class DictLoader(BaseLoader): >>> loader = DictLoader({'index.html': 'source here'}) - Because auto reloading is rarely useful this is disabled per default. + Because auto reloading is rarely useful this is disabled by default. """ def __init__(self, mapping: t.Mapping[str, str]) -> None: From 786d12b529a2ecf1e2fb586a619a79e2650a6d4d Mon Sep 17 00:00:00 2001 From: David Lord Date: Wed, 18 Dec 2024 09:36:11 -0800 Subject: [PATCH 23/52] clarify block outer scope docs --- docs/templates.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/templates.rst b/docs/templates.rst index 0eab8e664..d5f2719e0 100644 --- a/docs/templates.rst +++ b/docs/templates.rst @@ -528,8 +528,8 @@ However, the name after the `endblock` word must match the block name. Block Nesting and Scope ~~~~~~~~~~~~~~~~~~~~~~~ -Blocks can be nested for more complex layouts. However, by default blocks -may not access variables from outer scopes:: +Blocks can be nested for more complex layouts. By default, a block may not +access variables from outside the block (outer scopes):: {% for item in seq %}
  • {% block loop_item %}{{ item }}{% endblock %}
    • From 0c0a3d02d1b103120d41f04e3e8a0974504a244c Mon Sep 17 00:00:00 2001 From: JamesParrott <80779630+JamesParrott@users.noreply.github.com> Date: Mon, 9 Dec 2024 15:03:27 +0000 Subject: [PATCH 24/52] fix Jinja syntax in example --- examples/basic/test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/basic/test.py b/examples/basic/test.py index 7a58e1ad4..30f5dd6b3 100644 --- a/examples/basic/test.py +++ b/examples/basic/test.py @@ -6,9 +6,9 @@ { "child.html": """\ {% extends default_layout or 'default.html' %} -{% include helpers = 'helpers.html' %} +{% import 'helpers.html' as helpers %} {% macro get_the_answer() %}42{% endmacro %} -{% title = 'Hello World' %} +{% set title = 'Hello World' %} {% block body %} {{ get_the_answer() }} {{ helpers.conspirate() }} From 955d7daf3d602cba6b3e9afb6fd425fd1577d15f Mon Sep 17 00:00:00 2001 From: Charles-Axel Dein <120501+charlax@users.noreply.github.com> Date: Tue, 26 Jul 2022 11:03:16 +0200 Subject: [PATCH 25/52] Simplify example for ModuleLoader The `ModuleLoader` example seems copy pasted from `ChoiceLoader`. As a result it's not immediately clear how their API differ. --- src/jinja2/loaders.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/jinja2/loaders.py b/src/jinja2/loaders.py index b1ad401c0..d2373e5e7 100644 --- a/src/jinja2/loaders.py +++ b/src/jinja2/loaders.py @@ -613,10 +613,7 @@ class ModuleLoader(BaseLoader): Example usage: - >>> loader = ChoiceLoader([ - ... ModuleLoader('/path/to/compiled/templates'), - ... FileSystemLoader('/path/to/templates') - ... ]) + >>> loader = ModuleLoader('/path/to/compiled/templates') Templates can be precompiled with :meth:`Environment.compile_templates`. """ From d3a0b1a4abac05a071f542aa83edea4551c88fa3 Mon Sep 17 00:00:00 2001 From: Martin Krizek Date: Tue, 9 Aug 2022 10:12:27 +0200 Subject: [PATCH 26/52] use env.concat when calling block reference --- CHANGES.rst | 2 ++ src/jinja2/runtime.py | 6 ++++-- tests/test_nativetypes.py | 10 ++++++++++ 3 files changed, 16 insertions(+), 2 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index feb1e6c3d..540d5cccb 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -14,6 +14,8 @@ Unreleased ``Template.generate_async``. :pr:`1960` - Avoid leaving async generators unclosed in blocks, includes and extends. :pr:`1960` +- The runtime uses the correct ``concat`` function for the current environment + when calling block references. :issue:`1701` Version 3.1.4 diff --git a/src/jinja2/runtime.py b/src/jinja2/runtime.py index d10fe9d06..c2c7c1937 100644 --- a/src/jinja2/runtime.py +++ b/src/jinja2/runtime.py @@ -367,7 +367,7 @@ def super(self) -> t.Union["BlockReference", "Undefined"]: @internalcode async def _async_call(self) -> str: - rv = concat( + rv = self._context.environment.concat( # type: ignore [x async for x in self._stack[self._depth](self._context)] # type: ignore ) @@ -381,7 +381,9 @@ def __call__(self) -> str: if self._context.environment.is_async: return self._async_call() # type: ignore - rv = concat(self._stack[self._depth](self._context)) + rv = self._context.environment.concat( # type: ignore + self._stack[self._depth](self._context) + ) if self._context.eval_ctx.autoescape: return Markup(rv) diff --git a/tests/test_nativetypes.py b/tests/test_nativetypes.py index 8c8525251..136908180 100644 --- a/tests/test_nativetypes.py +++ b/tests/test_nativetypes.py @@ -160,3 +160,13 @@ def test_macro(env): result = t.render() assert result == 2 assert isinstance(result, int) + + +def test_block(env): + t = env.from_string( + "{% block b %}{% for i in range(1) %}{{ loop.index }}{% endfor %}" + "{% endblock %}{{ self.b() }}" + ) + result = t.render() + assert result == 11 + assert isinstance(result, int) From 76af7110ead6083e75072f9e76242c9ce79b76ed Mon Sep 17 00:00:00 2001 From: Mehdi ABAAKOUK Date: Fri, 23 Dec 2022 09:46:29 +0100 Subject: [PATCH 27/52] make unique filter async-aware --- CHANGES.rst | 2 ++ src/jinja2/filters.py | 14 +++++++++++++- tests/test_async_filters.py | 7 +++++++ 3 files changed, 22 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index 540d5cccb..e4bffbfb9 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -16,6 +16,8 @@ Unreleased :pr:`1960` - The runtime uses the correct ``concat`` function for the current environment when calling block references. :issue:`1701` +- Make ``|unique`` async-aware, allowing it to be used after another + async-aware filter. :issue:`1781` Version 3.1.4 diff --git a/src/jinja2/filters.py b/src/jinja2/filters.py index 4c949cbde..af9f6bc0e 100644 --- a/src/jinja2/filters.py +++ b/src/jinja2/filters.py @@ -438,7 +438,7 @@ def do_sort( @pass_environment -def do_unique( +def sync_do_unique( environment: "Environment", value: "t.Iterable[V]", case_sensitive: bool = False, @@ -470,6 +470,18 @@ def do_unique( yield item +@async_variant(sync_do_unique) # type: ignore +async def do_unique( + environment: "Environment", + value: "t.Union[t.AsyncIterable[V], t.Iterable[V]]", + case_sensitive: bool = False, + attribute: t.Optional[t.Union[str, int]] = None, +) -> "t.Iterator[V]": + return sync_do_unique( + environment, await auto_to_list(value), case_sensitive, attribute + ) + + def _min_or_max( environment: "Environment", value: "t.Iterable[V]", diff --git a/tests/test_async_filters.py b/tests/test_async_filters.py index e8cc350d5..e9892f1ed 100644 --- a/tests/test_async_filters.py +++ b/tests/test_async_filters.py @@ -277,6 +277,13 @@ def test_slice(env_async, items): ) +def test_unique_with_async_gen(env_async): + items = ["a", "b", "c", "c", "a", "d", "z"] + tmpl = env_async.from_string("{{ items|reject('==', 'z')|unique|list }}") + out = tmpl.render(items=items) + assert out == "['a', 'b', 'c', 'd']" + + def test_custom_async_filter(env_async, run_async_fn): async def customfilter(val): return str(val) From 2eb4542cbab101910d220229a8f09c94f76c19d1 Mon Sep 17 00:00:00 2001 From: Felipe Moreno Date: Mon, 20 May 2024 11:02:20 -0400 Subject: [PATCH 28/52] int filter handles OverflowError to handle scientific notation --- CHANGES.rst | 2 ++ src/jinja2/filters.py | 2 +- tests/test_filters.py | 1 + 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index e4bffbfb9..6c998e626 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -18,6 +18,8 @@ Unreleased when calling block references. :issue:`1701` - Make ``|unique`` async-aware, allowing it to be used after another async-aware filter. :issue:`1781` +- ``|int`` filter handles ``OverflowError`` from scientific notation. + :issue:`1921` Version 3.1.4 diff --git a/src/jinja2/filters.py b/src/jinja2/filters.py index af9f6bc0e..a92832a34 100644 --- a/src/jinja2/filters.py +++ b/src/jinja2/filters.py @@ -999,7 +999,7 @@ def do_int(value: t.Any, default: int = 0, base: int = 10) -> int: # this quirk is necessary so that "42.23"|int gives 42. try: return int(float(value)) - except (TypeError, ValueError): + except (TypeError, ValueError, OverflowError): return default diff --git a/tests/test_filters.py b/tests/test_filters.py index d8e9114d0..2cb53ac9d 100644 --- a/tests/test_filters.py +++ b/tests/test_filters.py @@ -196,6 +196,7 @@ def test_indent_width_string(self, env): ("abc", "0"), ("32.32", "32"), ("12345678901234567890", "12345678901234567890"), + ("1e10000", "0"), ), ) def test_int(self, env, value, expect): From 4936e4d48207c449f7713392e1ae83a1407aec1d Mon Sep 17 00:00:00 2001 From: Anentropic Date: Tue, 3 Sep 2024 22:16:33 +0100 Subject: [PATCH 29/52] make tuple unpacking deterministic in compiler --- CHANGES.rst | 2 ++ src/jinja2/compiler.py | 4 +-- tests/test_compile.py | 61 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 65 insertions(+), 2 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index 6c998e626..aebb38b58 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -20,6 +20,8 @@ Unreleased async-aware filter. :issue:`1781` - ``|int`` filter handles ``OverflowError`` from scientific notation. :issue:`1921` +- Make compiling deterministic for tuple unpacking in a ``{% set ... %}`` + call. :issue:`2021` Version 3.1.4 diff --git a/src/jinja2/compiler.py b/src/jinja2/compiler.py index 91720c5f9..074e9b187 100644 --- a/src/jinja2/compiler.py +++ b/src/jinja2/compiler.py @@ -811,7 +811,7 @@ def pop_assign_tracking(self, frame: Frame) -> None: self.writeline("_block_vars.update({") else: self.writeline("context.vars.update({") - for idx, name in enumerate(vars): + for idx, name in enumerate(sorted(vars)): if idx: self.write(", ") ref = frame.symbols.ref(name) @@ -821,7 +821,7 @@ def pop_assign_tracking(self, frame: Frame) -> None: if len(public_names) == 1: self.writeline(f"context.exported_vars.add({public_names[0]!r})") else: - names_str = ", ".join(map(repr, public_names)) + names_str = ", ".join(map(repr, sorted(public_names))) self.writeline(f"context.exported_vars.update(({names_str}))") # -- Statement Visitors diff --git a/tests/test_compile.py b/tests/test_compile.py index 42a773f21..42efa59c0 100644 --- a/tests/test_compile.py +++ b/tests/test_compile.py @@ -26,3 +26,64 @@ def test_import_as_with_context_deterministic(tmp_path): expect = [f"'bar{i}': " for i in range(10)] found = re.findall(r"'bar\d': ", content)[:10] assert found == expect + + +def test_top_level_set_vars_unpacking_deterministic(tmp_path): + src = "\n".join(f"{{% set a{i}, b{i}, c{i} = tuple_var{i} %}}" for i in range(10)) + env = Environment(loader=DictLoader({"foo": src})) + env.compile_templates(tmp_path, zip=None) + name = os.listdir(tmp_path)[0] + content = (tmp_path / name).read_text("utf8") + expect = [ + f"context.vars.update({{'a{i}': l_0_a{i}, 'b{i}': l_0_b{i}, 'c{i}': l_0_c{i}}})" + for i in range(10) + ] + found = re.findall( + r"context\.vars\.update\(\{'a\d': l_0_a\d, 'b\d': l_0_b\d, 'c\d': l_0_c\d\}\)", + content, + )[:10] + assert found == expect + expect = [ + f"context.exported_vars.update(('a{i}', 'b{i}', 'c{i}'))" for i in range(10) + ] + found = re.findall( + r"context\.exported_vars\.update\(\('a\d', 'b\d', 'c\d'\)\)", + content, + )[:10] + assert found == expect + + +def test_loop_set_vars_unpacking_deterministic(tmp_path): + src = "\n".join(f" {{% set a{i}, b{i}, c{i} = tuple_var{i} %}}" for i in range(10)) + src = f"{{% for i in seq %}}\n{src}\n{{% endfor %}}" + env = Environment(loader=DictLoader({"foo": src})) + env.compile_templates(tmp_path, zip=None) + name = os.listdir(tmp_path)[0] + content = (tmp_path / name).read_text("utf8") + expect = [ + f"_loop_vars.update({{'a{i}': l_1_a{i}, 'b{i}': l_1_b{i}, 'c{i}': l_1_c{i}}})" + for i in range(10) + ] + found = re.findall( + r"_loop_vars\.update\(\{'a\d': l_1_a\d, 'b\d': l_1_b\d, 'c\d': l_1_c\d\}\)", + content, + )[:10] + assert found == expect + + +def test_block_set_vars_unpacking_deterministic(tmp_path): + src = "\n".join(f" {{% set a{i}, b{i}, c{i} = tuple_var{i} %}}" for i in range(10)) + src = f"{{% block test %}}\n{src}\n{{% endblock test %}}" + env = Environment(loader=DictLoader({"foo": src})) + env.compile_templates(tmp_path, zip=None) + name = os.listdir(tmp_path)[0] + content = (tmp_path / name).read_text("utf8") + expect = [ + f"_block_vars.update({{'a{i}': l_0_a{i}, 'b{i}': l_0_b{i}, 'c{i}': l_0_c{i}}})" + for i in range(10) + ] + found = re.findall( + r"_block_vars\.update\(\{'a\d': l_0_a\d, 'b\d': l_0_b\d, 'c\d': l_0_c\d\}\)", + content, + )[:10] + assert found == expect From d4fb0e8c401a8596be0e67622174b5ad35daeec2 Mon Sep 17 00:00:00 2001 From: Matt Davis Date: Tue, 1 Oct 2024 12:20:19 -0700 Subject: [PATCH 30/52] preserve `__slots__` on Undefined classes --- CHANGES.rst | 2 ++ src/jinja2/runtime.py | 30 +++++++++++++++----------- tests/test_api.py | 8 ------- tests/test_runtime.py | 50 +++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 69 insertions(+), 21 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index aebb38b58..58dc03214 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -22,6 +22,8 @@ Unreleased :issue:`1921` - Make compiling deterministic for tuple unpacking in a ``{% set ... %}`` call. :issue:`2021` +- Fix dunder protocol (`copy`/`pickle`/etc) interaction with ``Undefined`` + objects. :issue:`2025` Version 3.1.4 diff --git a/src/jinja2/runtime.py b/src/jinja2/runtime.py index c2c7c1937..09119e2ae 100644 --- a/src/jinja2/runtime.py +++ b/src/jinja2/runtime.py @@ -860,7 +860,11 @@ def _fail_with_undefined_error( @internalcode def __getattr__(self, name: str) -> t.Any: - if name[:2] == "__": + # Raise AttributeError on requests for names that appear to be unimplemented + # dunder methods to keep Python's internal protocol probing behaviors working + # properly in cases where another exception type could cause unexpected or + # difficult-to-diagnose failures. + if name[:2] == "__" and name[-2:] == "__": raise AttributeError(name) return self._fail_with_undefined_error() @@ -984,10 +988,20 @@ class ChainableUndefined(Undefined): def __html__(self) -> str: return str(self) - def __getattr__(self, _: str) -> "ChainableUndefined": + def __getattr__(self, name: str) -> "ChainableUndefined": + # Raise AttributeError on requests for names that appear to be unimplemented + # dunder methods to avoid confusing Python with truthy non-method objects that + # do not implement the protocol being probed for. e.g., copy.copy(Undefined()) + # fails spectacularly if getattr(Undefined(), '__setstate__') returns an + # Undefined object instead of raising AttributeError to signal that it does not + # support that style of object initialization. + if name[:2] == "__" and name[-2:] == "__": + raise AttributeError(name) + return self - __getitem__ = __getattr__ # type: ignore + def __getitem__(self, _name: str) -> "ChainableUndefined": # type: ignore[override] + return self class DebugUndefined(Undefined): @@ -1046,13 +1060,3 @@ class StrictUndefined(Undefined): __iter__ = __str__ = __len__ = Undefined._fail_with_undefined_error __eq__ = __ne__ = __bool__ = __hash__ = Undefined._fail_with_undefined_error __contains__ = Undefined._fail_with_undefined_error - - -# Remove slots attributes, after the metaclass is applied they are -# unneeded and contain wrong data for subclasses. -del ( - Undefined.__slots__, - ChainableUndefined.__slots__, - DebugUndefined.__slots__, - StrictUndefined.__slots__, -) diff --git a/tests/test_api.py b/tests/test_api.py index ff3fcb138..ee11a8d69 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -323,8 +323,6 @@ def test_default_undefined(self): assert und1 == und2 assert und1 != 42 assert hash(und1) == hash(und2) == hash(Undefined()) - with pytest.raises(AttributeError): - getattr(Undefined, "__slots__") # noqa: B009 def test_chainable_undefined(self): env = Environment(undefined=ChainableUndefined) @@ -335,8 +333,6 @@ def test_chainable_undefined(self): assert env.from_string("{{ foo.missing }}").render(foo=42) == "" assert env.from_string("{{ not missing }}").render() == "True" pytest.raises(UndefinedError, env.from_string("{{ missing - 1}}").render) - with pytest.raises(AttributeError): - getattr(ChainableUndefined, "__slots__") # noqa: B009 # The following tests ensure subclass functionality works as expected assert env.from_string('{{ missing.bar["baz"] }}').render() == "" @@ -368,8 +364,6 @@ def test_debug_undefined(self): str(DebugUndefined(hint=undefined_hint)) == f"{{{{ undefined value printed: {undefined_hint} }}}}" ) - with pytest.raises(AttributeError): - getattr(DebugUndefined, "__slots__") # noqa: B009 def test_strict_undefined(self): env = Environment(undefined=StrictUndefined) @@ -386,8 +380,6 @@ def test_strict_undefined(self): env.from_string('{{ missing|default("default", true) }}').render() == "default" ) - with pytest.raises(AttributeError): - getattr(StrictUndefined, "__slots__") # noqa: B009 assert env.from_string('{{ "foo" if false }}').render() == "" def test_indexing_gives_undefined(self): diff --git a/tests/test_runtime.py b/tests/test_runtime.py index 1978c6410..3cd3be15f 100644 --- a/tests/test_runtime.py +++ b/tests/test_runtime.py @@ -1,6 +1,15 @@ +import copy import itertools +import pickle +import pytest + +from jinja2 import ChainableUndefined +from jinja2 import DebugUndefined +from jinja2 import StrictUndefined from jinja2 import Template +from jinja2 import TemplateRuntimeError +from jinja2 import Undefined from jinja2.runtime import LoopContext TEST_IDX_TEMPLATE_STR_1 = ( @@ -73,3 +82,44 @@ def __call__(self, *args, **kwargs): out = t.render(calc=Calc()) # Would be "1" if context argument was passed. assert out == "0" + + +_undefined_types = (Undefined, ChainableUndefined, DebugUndefined, StrictUndefined) + + +@pytest.mark.parametrize("undefined_type", _undefined_types) +def test_undefined_copy(undefined_type): + undef = undefined_type("a hint", ["foo"], "a name", TemplateRuntimeError) + copied = copy.copy(undef) + + assert copied is not undef + assert copied._undefined_hint is undef._undefined_hint + assert copied._undefined_obj is undef._undefined_obj + assert copied._undefined_name is undef._undefined_name + assert copied._undefined_exception is undef._undefined_exception + + +@pytest.mark.parametrize("undefined_type", _undefined_types) +def test_undefined_deepcopy(undefined_type): + undef = undefined_type("a hint", ["foo"], "a name", TemplateRuntimeError) + copied = copy.deepcopy(undef) + + assert copied._undefined_hint is undef._undefined_hint + assert copied._undefined_obj is not undef._undefined_obj + assert copied._undefined_obj == undef._undefined_obj + assert copied._undefined_name is undef._undefined_name + assert copied._undefined_exception is undef._undefined_exception + + +@pytest.mark.parametrize("undefined_type", _undefined_types) +def test_undefined_pickle(undefined_type): + undef = undefined_type("a hint", ["foo"], "a name", TemplateRuntimeError) + copied = pickle.loads(pickle.dumps(undef)) + + assert copied._undefined_hint is not undef._undefined_hint + assert copied._undefined_hint == undef._undefined_hint + assert copied._undefined_obj is not undef._undefined_obj + assert copied._undefined_obj == undef._undefined_obj + assert copied._undefined_name is not undef._undefined_name + assert copied._undefined_name == undef._undefined_name + assert copied._undefined_exception is undef._undefined_exception From 7232b8246200155226adb672db8b3ef305cf29da Mon Sep 17 00:00:00 2001 From: Matt Clay Date: Tue, 1 Oct 2024 15:18:27 -0700 Subject: [PATCH 31/52] Fix pickle/copy support for the `missing` singleton --- CHANGES.rst | 2 ++ src/jinja2/utils.py | 13 +++++++++++-- tests/test_utils.py | 12 ++++++++++++ 3 files changed, 25 insertions(+), 2 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index 58dc03214..1a1a526b5 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -24,6 +24,8 @@ Unreleased call. :issue:`2021` - Fix dunder protocol (`copy`/`pickle`/etc) interaction with ``Undefined`` objects. :issue:`2025` +- Fix `copy`/`pickle` support for the internal ``missing`` object. + :issue:`2027` Version 3.1.4 diff --git a/src/jinja2/utils.py b/src/jinja2/utils.py index 5c1ff5d7b..7b52fc03e 100644 --- a/src/jinja2/utils.py +++ b/src/jinja2/utils.py @@ -18,8 +18,17 @@ F = t.TypeVar("F", bound=t.Callable[..., t.Any]) -# special singleton representing missing values for the runtime -missing: t.Any = type("MissingType", (), {"__repr__": lambda x: "missing"})() + +class _MissingType: + def __repr__(self) -> str: + return "missing" + + def __reduce__(self) -> str: + return "missing" + + +missing: t.Any = _MissingType() +"""Special singleton representing missing values for the runtime.""" internal_code: t.MutableSet[CodeType] = set() diff --git a/tests/test_utils.py b/tests/test_utils.py index 7b58af144..86e0f0420 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -1,3 +1,4 @@ +import copy import pickle import random from collections import deque @@ -183,3 +184,14 @@ def test_consume(): consume(x) with pytest.raises(StopIteration): next(x) + + +@pytest.mark.parametrize("protocol", range(pickle.HIGHEST_PROTOCOL + 1)) +def test_pickle_missing(protocol: int) -> None: + """Test that missing can be pickled while remaining a singleton.""" + assert pickle.loads(pickle.dumps(missing, protocol)) is missing + + +def test_copy_missing() -> None: + """Test that missing can be copied while remaining a singleton.""" + assert copy.copy(missing) is missing From b5120582706b9e997cfd3277b02a32fd35b1c9d4 Mon Sep 17 00:00:00 2001 From: Dylan Scott Date: Fri, 4 Oct 2024 13:17:07 -0700 Subject: [PATCH 32/52] sandbox disallows `clear` and `pop` on mutable sequence --- CHANGES.rst | 2 ++ src/jinja2/sandbox.py | 4 +++- tests/test_security.py | 2 ++ 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index 1a1a526b5..f48eb0399 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -26,6 +26,8 @@ Unreleased objects. :issue:`2025` - Fix `copy`/`pickle` support for the internal ``missing`` object. :issue:`2027` +- Sandbox does not allow ``clear`` and ``pop`` on known mutable sequence + types. :issue:`2032` Version 3.1.4 diff --git a/src/jinja2/sandbox.py b/src/jinja2/sandbox.py index ce276156c..8200195db 100644 --- a/src/jinja2/sandbox.py +++ b/src/jinja2/sandbox.py @@ -60,7 +60,9 @@ ), ( abc.MutableSequence, - frozenset(["append", "reverse", "insert", "sort", "extend", "remove"]), + frozenset( + ["append", "clear", "pop", "reverse", "insert", "sort", "extend", "remove"] + ), ), ( deque, diff --git a/tests/test_security.py b/tests/test_security.py index 0e8dc5c03..9c7c4427a 100644 --- a/tests/test_security.py +++ b/tests/test_security.py @@ -58,6 +58,8 @@ def test_unsafe(self, env): def test_immutable_environment(self, env): env = ImmutableSandboxedEnvironment() pytest.raises(SecurityError, env.from_string("{{ [].append(23) }}").render) + pytest.raises(SecurityError, env.from_string("{{ [].clear() }}").render) + pytest.raises(SecurityError, env.from_string("{{ [1].pop() }}").render) pytest.raises(SecurityError, env.from_string("{{ {1:2}.clear() }}").render) def test_restricted(self, env): From 0871c71d0166152801798c1e59b3b32d3fb7469e Mon Sep 17 00:00:00 2001 From: David Lord Date: Thu, 19 Dec 2024 12:07:14 -0800 Subject: [PATCH 33/52] rearrange change entry --- CHANGES.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index f48eb0399..4d201a5dc 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -5,6 +5,8 @@ Version 3.1.5 Unreleased +- Sandbox does not allow ``clear`` and ``pop`` on known mutable sequence + types. :issue:`2032` - Calling sync ``render`` for an async template uses ``asyncio.run``. :pr:`1952` - Avoid unclosed ``auto_aiter`` warnings. :pr:`1960` @@ -26,8 +28,6 @@ Unreleased objects. :issue:`2025` - Fix `copy`/`pickle` support for the internal ``missing`` object. :issue:`2027` -- Sandbox does not allow ``clear`` and ``pop`` on known mutable sequence - types. :issue:`2032` Version 3.1.4 From 91a972f5808973cd441f4dc06873b2f8378f30c7 Mon Sep 17 00:00:00 2001 From: Lydxn Date: Mon, 23 Sep 2024 15:09:10 -0700 Subject: [PATCH 34/52] sandbox indirect calls to str.format --- CHANGES.rst | 3 ++ src/jinja2/sandbox.py | 81 ++++++++++++++++++++++-------------------- tests/test_security.py | 17 +++++++++ 3 files changed, 63 insertions(+), 38 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index 4d201a5dc..0a5694757 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -5,6 +5,9 @@ Version 3.1.5 Unreleased +- The sandboxed environment handles indirect calls to ``str.format``, such as + by passing a stored reference to a filter that calls its argument. + :ghsa:`q2x7-8rv6-6q7h` - Sandbox does not allow ``clear`` and ``pop`` on known mutable sequence types. :issue:`2032` - Calling sync ``render`` for an async template uses ``asyncio.run``. diff --git a/src/jinja2/sandbox.py b/src/jinja2/sandbox.py index 8200195db..9c9dae22f 100644 --- a/src/jinja2/sandbox.py +++ b/src/jinja2/sandbox.py @@ -8,6 +8,7 @@ from _string import formatter_field_name_split # type: ignore from collections import abc from collections import deque +from functools import update_wrapper from string import Formatter from markupsafe import EscapeFormatter @@ -83,20 +84,6 @@ ) -def inspect_format_method(callable: t.Callable[..., t.Any]) -> t.Optional[str]: - if not isinstance( - callable, (types.MethodType, types.BuiltinMethodType) - ) or callable.__name__ not in ("format", "format_map"): - return None - - obj = callable.__self__ - - if isinstance(obj, str): - return obj - - return None - - def safe_range(*args: int) -> range: """A range that can't generate ranges with a length of more than MAX_RANGE items. @@ -316,6 +303,9 @@ def getitem( except AttributeError: pass else: + fmt = self.wrap_str_format(value) + if fmt is not None: + return fmt if self.is_safe_attribute(obj, argument, value): return value return self.unsafe_undefined(obj, argument) @@ -333,6 +323,9 @@ def getattr(self, obj: t.Any, attribute: str) -> t.Union[t.Any, Undefined]: except (TypeError, LookupError): pass else: + fmt = self.wrap_str_format(value) + if fmt is not None: + return fmt if self.is_safe_attribute(obj, attribute, value): return value return self.unsafe_undefined(obj, attribute) @@ -348,34 +341,49 @@ def unsafe_undefined(self, obj: t.Any, attribute: str) -> Undefined: exc=SecurityError, ) - def format_string( - self, - s: str, - args: t.Tuple[t.Any, ...], - kwargs: t.Dict[str, t.Any], - format_func: t.Optional[t.Callable[..., t.Any]] = None, - ) -> str: - """If a format call is detected, then this is routed through this - method so that our safety sandbox can be used for it. + def wrap_str_format(self, value: t.Any) -> t.Optional[t.Callable[..., str]]: + """If the given value is a ``str.format`` or ``str.format_map`` method, + return a new function than handles sandboxing. This is done at access + rather than in :meth:`call`, so that calls made without ``call`` are + also sandboxed. """ + if not isinstance( + value, (types.MethodType, types.BuiltinMethodType) + ) or value.__name__ not in ("format", "format_map"): + return None + + f_self: t.Any = value.__self__ + + if not isinstance(f_self, str): + return None + + str_type: t.Type[str] = type(f_self) + is_format_map = value.__name__ == "format_map" formatter: SandboxedFormatter - if isinstance(s, Markup): - formatter = SandboxedEscapeFormatter(self, escape=s.escape) + + if isinstance(f_self, Markup): + formatter = SandboxedEscapeFormatter(self, escape=f_self.escape) else: formatter = SandboxedFormatter(self) - if format_func is not None and format_func.__name__ == "format_map": - if len(args) != 1 or kwargs: - raise TypeError( - "format_map() takes exactly one argument" - f" {len(args) + (kwargs is not None)} given" - ) + vformat = formatter.vformat + + def wrapper(*args: t.Any, **kwargs: t.Any) -> str: + if is_format_map: + if kwargs: + raise TypeError("format_map() takes no keyword arguments") + + if len(args) != 1: + raise TypeError( + f"format_map() takes exactly one argument ({len(args)} given)" + ) + + kwargs = args[0] + args = () - kwargs = args[0] - args = () + return str_type(vformat(f_self, args, kwargs)) - rv = formatter.vformat(s, args, kwargs) - return type(s)(rv) + return update_wrapper(wrapper, value) def call( __self, # noqa: B902 @@ -385,9 +393,6 @@ def call( **kwargs: t.Any, ) -> t.Any: """Call an object from sandboxed code.""" - fmt = inspect_format_method(__obj) - if fmt is not None: - return __self.format_string(fmt, args, kwargs, __obj) # the double prefixes are to avoid double keyword argument # errors when proxying the call. diff --git a/tests/test_security.py b/tests/test_security.py index 9c7c4427a..864d5f7f9 100644 --- a/tests/test_security.py +++ b/tests/test_security.py @@ -173,3 +173,20 @@ def test_safe_format_all_okay(self): '{{ ("a{x.foo}b{y}"|safe).format_map({"x":{"foo": 42}, "y":""}) }}' ) assert t.render() == "a42b<foo>" + + def test_indirect_call(self): + def run(value, arg): + return value.run(arg) + + env = SandboxedEnvironment() + env.filters["run"] = run + t = env.from_string( + """{% set + ns = namespace(run="{0.__call__.__builtins__[__import__]}".format) + %} + {{ ns | run(not_here) }} + """ + ) + + with pytest.raises(SecurityError): + t.render() From 56a724644b1ad9cb03745c10cca732715cdc79e9 Mon Sep 17 00:00:00 2001 From: Sigurd Spieckermann Date: Fri, 26 May 2023 14:32:36 +0200 Subject: [PATCH 35/52] fix f-string syntax error in code generation --- CHANGES.rst | 3 +++ src/jinja2/compiler.py | 7 ++++++- tests/test_compile.py | 19 +++++++++++++++++++ 3 files changed, 28 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index 0a5694757..5d64b267c 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -8,6 +8,9 @@ Unreleased - The sandboxed environment handles indirect calls to ``str.format``, such as by passing a stored reference to a filter that calls its argument. :ghsa:`q2x7-8rv6-6q7h` +- Escape template name before formatting it into error messages, to avoid + issues with names that contain f-string syntax. + :issue:`1792`, :ghsa:`gmj6-6f8f-6699` - Sandbox does not allow ``clear`` and ``pop`` on known mutable sequence types. :issue:`2032` - Calling sync ``render`` for an async template uses ``asyncio.run``. diff --git a/src/jinja2/compiler.py b/src/jinja2/compiler.py index 074e9b187..23295ec1f 100644 --- a/src/jinja2/compiler.py +++ b/src/jinja2/compiler.py @@ -1141,9 +1141,14 @@ def visit_FromImport(self, node: nodes.FromImport, frame: Frame) -> None: ) self.writeline(f"if {frame.symbols.ref(alias)} is missing:") self.indent() + # The position will contain the template name, and will be formatted + # into a string that will be compiled into an f-string. Curly braces + # in the name must be replaced with escapes so that they will not be + # executed as part of the f-string. + position = self.position(node).replace("{", "{{").replace("}", "}}") message = ( "the template {included_template.__name__!r}" - f" (imported on {self.position(node)})" + f" (imported on {position})" f" does not export the requested name {name!r}" ) self.writeline( diff --git a/tests/test_compile.py b/tests/test_compile.py index 42efa59c0..e1a5391ea 100644 --- a/tests/test_compile.py +++ b/tests/test_compile.py @@ -1,6 +1,9 @@ import os import re +import pytest + +from jinja2 import UndefinedError from jinja2.environment import Environment from jinja2.loaders import DictLoader @@ -87,3 +90,19 @@ def test_block_set_vars_unpacking_deterministic(tmp_path): content, )[:10] assert found == expect + + +def test_undefined_import_curly_name(): + env = Environment( + loader=DictLoader( + { + "{bad}": "{% from 'macro' import m %}{{ m() }}", + "macro": "", + } + ) + ) + + # Must not raise `NameError: 'bad' is not defined`, as that would indicate + # that `{bad}` is being interpreted as an f-string. It must be escaped. + with pytest.raises(UndefinedError): + env.get_template("{bad}").render() From e45bc745a76b7aa573e68888bf49ecef70001815 Mon Sep 17 00:00:00 2001 From: SamyCookie Date: Thu, 19 Dec 2024 10:59:57 +0100 Subject: [PATCH 36/52] Bugfix: wrong default argument for `Environment.overlay(enable_async)` parameter --- CHANGES.rst | 1 + src/jinja2/environment.py | 7 +++++-- tests/test_api.py | 8 ++++++++ 3 files changed, 14 insertions(+), 2 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index 0a5694757..745b9690a 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -31,6 +31,7 @@ Unreleased objects. :issue:`2025` - Fix `copy`/`pickle` support for the internal ``missing`` object. :issue:`2027` +- ``Environment.overlay(enable_async)`` is applied correctly. :pr:`2061` Version 3.1.4 diff --git a/src/jinja2/environment.py b/src/jinja2/environment.py index f062e4074..821971779 100644 --- a/src/jinja2/environment.py +++ b/src/jinja2/environment.py @@ -406,7 +406,7 @@ def overlay( cache_size: int = missing, auto_reload: bool = missing, bytecode_cache: t.Optional["BytecodeCache"] = missing, - enable_async: bool = False, + enable_async: bool = missing, ) -> "Environment": """Create a new overlay environment that shares all the data with the current environment except for cache and the overridden attributes. @@ -419,8 +419,11 @@ def overlay( copied over so modifications on the original environment may not shine through. + .. versionchanged:: 3.1.5 + ``enable_async`` is applied correctly. + .. versionchanged:: 3.1.2 - Added the ``newline_sequence``,, ``keep_trailing_newline``, + Added the ``newline_sequence``, ``keep_trailing_newline``, and ``enable_async`` parameters to match ``__init__``. """ args = dict(locals()) diff --git a/tests/test_api.py b/tests/test_api.py index ee11a8d69..4472b85ac 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -425,3 +425,11 @@ class CustomEnvironment(Environment): env = CustomEnvironment() tmpl = env.from_string("{{ foo }}") assert tmpl.render() == "resolve-foo" + + +def test_overlay_enable_async(env): + assert not env.is_async + assert not env.overlay().is_async + env_async = env.overlay(enable_async=True) + assert env_async.is_async + assert not env_async.overlay(enable_async=False).is_async From ed5f76206aa0a1e86ef087a18bc69643b484f029 Mon Sep 17 00:00:00 2001 From: Yourun-Proger Date: Mon, 2 May 2022 15:42:19 +0300 Subject: [PATCH 37/52] FileSystemLoader includes search paths in error --- CHANGES.rst | 2 ++ src/jinja2/loaders.py | 5 ++++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index 384c6122d..569ae69f7 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -35,6 +35,8 @@ Unreleased - Fix `copy`/`pickle` support for the internal ``missing`` object. :issue:`2027` - ``Environment.overlay(enable_async)`` is applied correctly. :pr:`2061` +- Paths where the loader searched for the template were added + to the error message. :issue:`1661` Version 3.1.4 diff --git a/src/jinja2/loaders.py b/src/jinja2/loaders.py index d2373e5e7..65dfbe1a0 100644 --- a/src/jinja2/loaders.py +++ b/src/jinja2/loaders.py @@ -204,7 +204,10 @@ def get_source( if os.path.isfile(filename): break else: - raise TemplateNotFound(template) + raise TemplateNotFound( + f"{template} not found in the following search path(s):" + f" {self.searchpath}" + ) with open(filename, encoding=self.encoding) as f: contents = f.read() From 227edfd372f174fdae1ff74972de6a532af6c76e Mon Sep 17 00:00:00 2001 From: David Lord Date: Thu, 19 Dec 2024 19:34:34 -0800 Subject: [PATCH 38/52] clean up message, add test --- CHANGES.rst | 4 ++-- src/jinja2/loaders.py | 6 ++++-- tests/test_loader.py | 18 ++++++++++++++++++ 3 files changed, 24 insertions(+), 4 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index 569ae69f7..cd2a4ef08 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -35,8 +35,8 @@ Unreleased - Fix `copy`/`pickle` support for the internal ``missing`` object. :issue:`2027` - ``Environment.overlay(enable_async)`` is applied correctly. :pr:`2061` -- Paths where the loader searched for the template were added - to the error message. :issue:`1661` +- The error message from ``FileSystemLoader`` includes the paths that were + searched. :issue:`1661` Version 3.1.4 diff --git a/src/jinja2/loaders.py b/src/jinja2/loaders.py index 65dfbe1a0..35799584c 100644 --- a/src/jinja2/loaders.py +++ b/src/jinja2/loaders.py @@ -204,9 +204,11 @@ def get_source( if os.path.isfile(filename): break else: + plural = "path" if len(self.searchpath) == 1 else "paths" + paths_str = ", ".join(repr(p) for p in self.searchpath) raise TemplateNotFound( - f"{template} not found in the following search path(s):" - f" {self.searchpath}" + template, + f"{template!r} not found in search {plural}: {paths_str}", ) with open(filename, encoding=self.encoding) as f: diff --git a/tests/test_loader.py b/tests/test_loader.py index e0cff6720..5a4e1a9da 100644 --- a/tests/test_loader.py +++ b/tests/test_loader.py @@ -179,6 +179,24 @@ def test_filename_normpath(self): t = e.get_template("foo/test.html") assert t.filename == str(self.searchpath / "foo" / "test.html") + def test_error_includes_paths(self, env, filesystem_loader): + env.loader = filesystem_loader + + with pytest.raises(TemplateNotFound) as info: + env.get_template("missing") + + e_str = str(info.value) + assert e_str.startswith("'missing' not found in search path: ") + + filesystem_loader.searchpath.append("other") + + with pytest.raises(TemplateNotFound) as info: + env.get_template("missing") + + e_str = str(info.value) + assert e_str.startswith("'missing' not found in search paths: ") + assert ", 'other'" in e_str + class TestModuleLoader: archive = None From f54fa113d38230cd7a8e8de1a8ced0f24e344a4e Mon Sep 17 00:00:00 2001 From: Lily Foote Date: Thu, 11 Aug 2022 15:22:08 +0100 Subject: [PATCH 39/52] Improve the PackageLoader error message This exception is raised when the `package_path` directory (default "templates") is not found, so explain this. --- src/jinja2/loaders.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/jinja2/loaders.py b/src/jinja2/loaders.py index 35799584c..0cdeca14a 100644 --- a/src/jinja2/loaders.py +++ b/src/jinja2/loaders.py @@ -353,8 +353,8 @@ def __init__( if template_root is None: raise ValueError( - f"The {package_name!r} package was not installed in a" - " way that PackageLoader understands." + f"PackageLoader could not find a '{package_path}' directory for the " + f"{package_name!r} package." ) self._template_root = template_root From aaa083d265307f44f00b010acec61b6eb8e3c3a7 Mon Sep 17 00:00:00 2001 From: David Lord Date: Thu, 19 Dec 2024 20:15:10 -0800 Subject: [PATCH 40/52] separate messages, add test --- CHANGES.rst | 2 ++ src/jinja2/loaders.py | 18 +++++++++++------- tests/test_loader.py | 5 +++++ 3 files changed, 18 insertions(+), 7 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index cd2a4ef08..2e83ab3f6 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -37,6 +37,8 @@ Unreleased - ``Environment.overlay(enable_async)`` is applied correctly. :pr:`2061` - The error message from ``FileSystemLoader`` includes the paths that were searched. :issue:`1661` +- ``PackageLoader`` shows a clearer error message when the package does not + contain the templates directory. :issue:`1705` Version 3.1.4 diff --git a/src/jinja2/loaders.py b/src/jinja2/loaders.py index 0cdeca14a..3913ee51e 100644 --- a/src/jinja2/loaders.py +++ b/src/jinja2/loaders.py @@ -327,7 +327,6 @@ def __init__( assert loader is not None, "A loader was not found for the package." self._loader = loader self._archive = None - template_root = None if isinstance(loader, zipimport.zipimporter): self._archive = loader.archive @@ -344,18 +343,23 @@ def __init__( elif spec.origin is not None: roots.append(os.path.dirname(spec.origin)) + if not roots: + raise ValueError( + f"The {package_name!r} package was not installed in a" + " way that PackageLoader understands." + ) + for root in roots: root = os.path.join(root, package_path) if os.path.isdir(root): template_root = root break - - if template_root is None: - raise ValueError( - f"PackageLoader could not find a '{package_path}' directory for the " - f"{package_name!r} package." - ) + else: + raise ValueError( + f"PackageLoader could not find a {package_path!r} directory" + f" in the {package_name!r} package." + ) self._template_root = template_root diff --git a/tests/test_loader.py b/tests/test_loader.py index 5a4e1a9da..377290b71 100644 --- a/tests/test_loader.py +++ b/tests/test_loader.py @@ -429,3 +429,8 @@ def exec_module(self, module): assert "test.html" in package_loader.list_templates() finally: sys.meta_path[:] = before + + +def test_package_loader_no_dir() -> None: + with pytest.raises(ValueError, match="could not find a 'templates' directory"): + PackageLoader("jinja2") From ded9915fc5db23d1a45ad3b210def7b4a96dc287 Mon Sep 17 00:00:00 2001 From: Victor Westerhuis Date: Wed, 23 Aug 2023 09:01:55 +0200 Subject: [PATCH 41/52] improve annotations for methods returning copies --- CHANGES.rst | 1 + src/jinja2/compiler.py | 4 ++-- src/jinja2/environment.py | 4 ++-- src/jinja2/ext.py | 2 +- src/jinja2/idtracking.py | 5 ++++- src/jinja2/utils.py | 2 +- 6 files changed, 11 insertions(+), 7 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index 2e83ab3f6..3955a32ba 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -39,6 +39,7 @@ Unreleased searched. :issue:`1661` - ``PackageLoader`` shows a clearer error message when the package does not contain the templates directory. :issue:`1705` +- Improve annotations for methods returning copies. :pr:`1880` Version 3.1.4 diff --git a/src/jinja2/compiler.py b/src/jinja2/compiler.py index 23295ec1f..ca079070a 100644 --- a/src/jinja2/compiler.py +++ b/src/jinja2/compiler.py @@ -216,7 +216,7 @@ def __init__( # or compile time. self.soft_frame = False - def copy(self) -> "Frame": + def copy(self) -> "te.Self": """Create a copy of the current one.""" rv = object.__new__(self.__class__) rv.__dict__.update(self.__dict__) @@ -229,7 +229,7 @@ def inner(self, isolated: bool = False) -> "Frame": return Frame(self.eval_ctx, level=self.symbols.level + 1) return Frame(self.eval_ctx, self) - def soft(self) -> "Frame": + def soft(self) -> "te.Self": """Return a soft frame. A soft frame may not be modified as standalone thing as it shares the resources with the frame it was created of, but it's not a rootlevel frame any longer. diff --git a/src/jinja2/environment.py b/src/jinja2/environment.py index 821971779..0fc6e5be8 100644 --- a/src/jinja2/environment.py +++ b/src/jinja2/environment.py @@ -123,7 +123,7 @@ def load_extensions( return result -def _environment_config_check(environment: "Environment") -> "Environment": +def _environment_config_check(environment: _env_bound) -> _env_bound: """Perform a sanity check on the environment.""" assert issubclass( environment.undefined, Undefined @@ -407,7 +407,7 @@ def overlay( auto_reload: bool = missing, bytecode_cache: t.Optional["BytecodeCache"] = missing, enable_async: bool = missing, - ) -> "Environment": + ) -> "te.Self": """Create a new overlay environment that shares all the data with the current environment except for cache and the overridden attributes. Extensions cannot be removed for an overlayed environment. An overlayed diff --git a/src/jinja2/ext.py b/src/jinja2/ext.py index 8d0810cd4..c7af8d45f 100644 --- a/src/jinja2/ext.py +++ b/src/jinja2/ext.py @@ -89,7 +89,7 @@ def __init_subclass__(cls) -> None: def __init__(self, environment: Environment) -> None: self.environment = environment - def bind(self, environment: Environment) -> "Extension": + def bind(self, environment: Environment) -> "te.Self": """Create a copy of this extension bound to another environment.""" rv = object.__new__(self.__class__) rv.__dict__.update(self.__dict__) diff --git a/src/jinja2/idtracking.py b/src/jinja2/idtracking.py index d6cb635b2..cb4bccb0e 100644 --- a/src/jinja2/idtracking.py +++ b/src/jinja2/idtracking.py @@ -3,6 +3,9 @@ from . import nodes from .visitor import NodeVisitor +if t.TYPE_CHECKING: + import typing_extensions as te + VAR_LOAD_PARAMETER = "param" VAR_LOAD_RESOLVE = "resolve" VAR_LOAD_ALIAS = "alias" @@ -83,7 +86,7 @@ def ref(self, name: str) -> str: ) return rv - def copy(self) -> "Symbols": + def copy(self) -> "te.Self": rv = object.__new__(self.__class__) rv.__dict__.update(self.__dict__) rv.refs = self.refs.copy() diff --git a/src/jinja2/utils.py b/src/jinja2/utils.py index 7b52fc03e..d7149bc31 100644 --- a/src/jinja2/utils.py +++ b/src/jinja2/utils.py @@ -462,7 +462,7 @@ def __setstate__(self, d: t.Mapping[str, t.Any]) -> None: def __getnewargs__(self) -> t.Tuple[t.Any, ...]: return (self.capacity,) - def copy(self) -> "LRUCache": + def copy(self) -> "te.Self": """Return a shallow copy of the instance.""" rv = self.__class__(self.capacity) rv._mapping.update(self._mapping) From 0cd6948192591b2c31e3dba294ed9300813d1d44 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=BD=D0=B0=D0=B1?= Date: Thu, 20 Jul 2023 04:20:13 +0200 Subject: [PATCH 42/52] don't apply `urlize` to `@a@b` --- CHANGES.rst | 1 + src/jinja2/utils.py | 2 ++ tests/test_utils.py | 8 ++++++++ 3 files changed, 11 insertions(+) diff --git a/CHANGES.rst b/CHANGES.rst index 3955a32ba..e31de857d 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -40,6 +40,7 @@ Unreleased - ``PackageLoader`` shows a clearer error message when the package does not contain the templates directory. :issue:`1705` - Improve annotations for methods returning copies. :pr:`1880` +- ``urlize`` does not add ``mailto:`` to values like `@a@b`. :pr:`1870` Version 3.1.4 diff --git a/src/jinja2/utils.py b/src/jinja2/utils.py index d7149bc31..7c922629a 100644 --- a/src/jinja2/utils.py +++ b/src/jinja2/utils.py @@ -333,6 +333,8 @@ def trim_url(x: str) -> str: elif ( "@" in middle and not middle.startswith("www.") + # ignore values like `@a@b` + and not middle.startswith("@") and ":" not in middle and _email_re.match(middle) ): diff --git a/tests/test_utils.py b/tests/test_utils.py index 86e0f0420..b50a6b4c6 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -142,6 +142,14 @@ def test_escape_urlize_target(self): "http://example.org" ) + def test_urlize_mail_mastodon(self): + fr = "nabijaczleweli@nabijaczleweli.xyz\n@eater@cijber.social\n" + to = ( + '' + "nabijaczleweli@nabijaczleweli.xyz\n@eater@cijber.social\n" + ) + assert urlize(fr) == to + class TestLoremIpsum: def test_lorem_ipsum_markup(self): From d05bd3858c3f4990e91201dae058147caf317462 Mon Sep 17 00:00:00 2001 From: Rens Groothuijsen Date: Wed, 16 Nov 2022 01:25:49 +0100 Subject: [PATCH 43/52] Pass context when using select --- CHANGES.rst | 2 ++ src/jinja2/filters.py | 2 +- tests/test_regression.py | 12 ++++++++++++ 3 files changed, 15 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index e31de857d..521f5a08a 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -41,6 +41,8 @@ Unreleased contain the templates directory. :issue:`1705` - Improve annotations for methods returning copies. :pr:`1880` - ``urlize`` does not add ``mailto:`` to values like `@a@b`. :pr:`1870` +- Tests decorated with `@pass_context`` can be used with the ``|select`` + filter. :issue:`1624` Version 3.1.4 diff --git a/src/jinja2/filters.py b/src/jinja2/filters.py index a92832a34..e5b5a00c5 100644 --- a/src/jinja2/filters.py +++ b/src/jinja2/filters.py @@ -1780,7 +1780,7 @@ def transfunc(x: V) -> V: args = args[1 + off :] def func(item: t.Any) -> t.Any: - return context.environment.call_test(name, item, args, kwargs) + return context.environment.call_test(name, item, args, kwargs, context) except LookupError: func = bool # type: ignore diff --git a/tests/test_regression.py b/tests/test_regression.py index 7bd4d1564..10df2d1bd 100644 --- a/tests/test_regression.py +++ b/tests/test_regression.py @@ -737,6 +737,18 @@ def test_nested_loop_scoping(self, env): ) assert tmpl.render() == "hellohellohello" + def test_pass_context_with_select(self, env): + @pass_context + def is_foo(ctx, s): + assert ctx is not None + return s == "foo" + + env.tests["foo"] = is_foo + tmpl = env.from_string( + "{% for x in ['one', 'foo'] | select('foo') %}{{ x }}{% endfor %}" + ) + assert tmpl.render() == "foo" + @pytest.mark.parametrize("unicode_char", ["\N{FORM FEED}", "\x85"]) def test_unicode_whitespace(env, unicode_char): From ae68c961dc52d48580dc9005c9ebe9117590f690 Mon Sep 17 00:00:00 2001 From: David Lord Date: Fri, 20 Dec 2024 07:57:11 -0800 Subject: [PATCH 44/52] document SandboxedNativeEnvironment pattern --- docs/nativetypes.rst | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/docs/nativetypes.rst b/docs/nativetypes.rst index 1a08700b0..fb2a76718 100644 --- a/docs/nativetypes.rst +++ b/docs/nativetypes.rst @@ -55,6 +55,17 @@ Foo >>> print(result.value) 15 +Sandboxed Native Environment +---------------------------- + +You can combine :class:`.SandboxedEnvironment` and :class:`NativeEnvironment` to +get both behaviors. + +.. code-block:: python + + class SandboxedNativeEnvironment(SandboxedEnvironment, NativeEnvironment): + pass + API --- From d6998ab74e628c9851042248c9e10dc25936630b Mon Sep 17 00:00:00 2001 From: ratchek Date: Tue, 12 Dec 2023 20:43:36 -0500 Subject: [PATCH 45/52] Make ease of use update to template documentation Add the phrases 'multiline comment' and 'triple quotes' to docs in the templates/#block-assignments section. This allows for new users to find this alternative easily. --- docs/templates.rst | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/templates.rst b/docs/templates.rst index d5f2719e0..758ba90ce 100644 --- a/docs/templates.rst +++ b/docs/templates.rst @@ -1090,9 +1090,10 @@ Block Assignments Starting with Jinja 2.8, it's possible to also use block assignments to capture the contents of a block into a variable name. This can be useful -in some situations as an alternative for macros. In that case, instead of -using an equals sign and a value, you just write the variable name and then -everything until ``{% endset %}`` is captured. +in some situations as an alternative for macros. It can also be used to create +multiline strings instead of triple quotes (''' and """), which Jinja does not +support. In that case, instead of using an equals sign and a value, you just +write the variable name and then everything until ``{% endset %}`` is captured. Example:: From 8a8eafc6b992ba177f1d3dd483f8465f18a11116 Mon Sep 17 00:00:00 2001 From: David Lord Date: Fri, 20 Dec 2024 08:29:04 -0800 Subject: [PATCH 46/52] edit block assignment section --- docs/templates.rst | 27 +++++++++++++-------------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/docs/templates.rst b/docs/templates.rst index 758ba90ce..8db8ccaf9 100644 --- a/docs/templates.rst +++ b/docs/templates.rst @@ -1086,35 +1086,34 @@ Assignments use the `set` tag and can have multiple targets:: Block Assignments ~~~~~~~~~~~~~~~~~ -.. versionadded:: 2.8 +It's possible to use `set` as a block to assign the content of the block to a +variable. This can be used to create multi-line strings, since Jinja doesn't +support Python's triple quotes (``"""``, ``'''``). -Starting with Jinja 2.8, it's possible to also use block assignments to -capture the contents of a block into a variable name. This can be useful -in some situations as an alternative for macros. It can also be used to create -multiline strings instead of triple quotes (''' and """), which Jinja does not -support. In that case, instead of using an equals sign and a value, you just -write the variable name and then everything until ``{% endset %}`` is captured. +Instead of using an equals sign and a value, you only write the variable name, +and everything until ``{% endset %}`` is captured. -Example:: +.. code-block:: jinja {% set navigation %}
  • Index
  • Downloads {% endset %} -The `navigation` variable then contains the navigation HTML source. - -.. versionchanged:: 2.10 - -Starting with Jinja 2.10, the block assignment supports filters. +Filters applied to the variable name will be applied to the block's content. -Example:: +.. code-block:: jinja {% set reply | wordwrap %} You wrote: {{ message }} {% endset %} +.. versionadded:: 2.8 + +.. versionchanged:: 2.10 + + Block assignment supports filters. .. _extends: From ee832194cd9f55f75e5a51359b709d535efe957f Mon Sep 17 00:00:00 2001 From: Kevin Brown-Silva Date: Mon, 2 May 2022 12:01:08 -0600 Subject: [PATCH 47/52] Add support for namespaces in tuple assignment This fixes a bug that existed because namespaces within `{% set %}` were treated as a special case. This special case had the side-effect of bypassing the code which allows for tuples to be assigned to. The solution was to make tuple handling (and by extension, primary token handling) aware of namespaces so that namespace tokens can be handled appropriately. This is handled in a backwards-compatible way which ensures that we do not try to parse namespace tokens when we otherwise would be expecting to parse out name tokens with attributes. Namespace instance checks are moved earlier, and deduplicated, so that all checks are done before the assignment. Otherwise, the check could be emitted in the middle of the tuple. --- CHANGES.rst | 2 ++ docs/templates.rst | 3 +++ src/jinja2/compiler.py | 23 ++++++++++++++++------- src/jinja2/parser.py | 30 +++++++++++++++++------------- tests/test_core_tags.py | 8 ++++++++ 5 files changed, 46 insertions(+), 20 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index 521f5a08a..2b8179855 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -43,6 +43,8 @@ Unreleased - ``urlize`` does not add ``mailto:`` to values like `@a@b`. :pr:`1870` - Tests decorated with `@pass_context`` can be used with the ``|select`` filter. :issue:`1624` +- Using ``set`` for multiple assignment (``a, b = 1, 2``) does not fail when the + target is a namespace attribute. :issue:`1413` Version 3.1.4 diff --git a/docs/templates.rst b/docs/templates.rst index 8db8ccaf9..9f376a13c 100644 --- a/docs/templates.rst +++ b/docs/templates.rst @@ -1678,6 +1678,9 @@ The following functions are available in the global scope by default: .. versionadded:: 2.10 + .. versionchanged:: 3.2 + Namespace attributes can be assigned to in multiple assignment. + Extensions ---------- diff --git a/src/jinja2/compiler.py b/src/jinja2/compiler.py index ca079070a..0666cddf7 100644 --- a/src/jinja2/compiler.py +++ b/src/jinja2/compiler.py @@ -1581,6 +1581,22 @@ def visit_Output(self, node: nodes.Output, frame: Frame) -> None: def visit_Assign(self, node: nodes.Assign, frame: Frame) -> None: self.push_assign_tracking() + + # NSRef can only ever be used during assignment so we need to check + # to make sure that it is only being used to assign using a Namespace. + # This check is done here because it is used an expression during the + # assignment and therefore cannot have this check done when the NSRef + # node is visited + for nsref in node.find_all(nodes.NSRef): + ref = frame.symbols.ref(nsref.name) + self.writeline(f"if not isinstance({ref}, Namespace):") + self.indent() + self.writeline( + "raise TemplateRuntimeError" + '("cannot assign attribute on non-namespace object")' + ) + self.outdent() + self.newline(node) self.visit(node.target, frame) self.write(" = ") @@ -1641,13 +1657,6 @@ def visit_NSRef(self, node: nodes.NSRef, frame: Frame) -> None: # `foo.bar` notation they will be parsed as a normal attribute access # when used anywhere but in a `set` context ref = frame.symbols.ref(node.name) - self.writeline(f"if not isinstance({ref}, Namespace):") - self.indent() - self.writeline( - "raise TemplateRuntimeError" - '("cannot assign attribute on non-namespace object")' - ) - self.outdent() self.writeline(f"{ref}[{node.attr!r}]") def visit_Const(self, node: nodes.Const, frame: Frame) -> None: diff --git a/src/jinja2/parser.py b/src/jinja2/parser.py index 22f3f81f7..107232631 100644 --- a/src/jinja2/parser.py +++ b/src/jinja2/parser.py @@ -487,21 +487,18 @@ def parse_assign_target( """ target: nodes.Expr - if with_namespace and self.stream.look().type == "dot": - token = self.stream.expect("name") - next(self.stream) # dot - attr = self.stream.expect("name") - target = nodes.NSRef(token.value, attr.value, lineno=token.lineno) - elif name_only: + if name_only: token = self.stream.expect("name") target = nodes.Name(token.value, "store", lineno=token.lineno) else: if with_tuple: target = self.parse_tuple( - simplified=True, extra_end_rules=extra_end_rules + simplified=True, + extra_end_rules=extra_end_rules, + with_namespace=with_namespace, ) else: - target = self.parse_primary() + target = self.parse_primary(with_namespace=with_namespace) target.set_ctx("store") @@ -643,7 +640,7 @@ def parse_unary(self, with_filter: bool = True) -> nodes.Expr: node = self.parse_filter_expr(node) return node - def parse_primary(self) -> nodes.Expr: + def parse_primary(self, with_namespace: bool = False) -> nodes.Expr: token = self.stream.current node: nodes.Expr if token.type == "name": @@ -651,6 +648,11 @@ def parse_primary(self) -> nodes.Expr: node = nodes.Const(token.value in ("true", "True"), lineno=token.lineno) elif token.value in ("none", "None"): node = nodes.Const(None, lineno=token.lineno) + elif with_namespace and self.stream.look().type == "dot": + next(self.stream) # token + next(self.stream) # dot + attr = self.stream.current + node = nodes.NSRef(token.value, attr.value, lineno=token.lineno) else: node = nodes.Name(token.value, "load", lineno=token.lineno) next(self.stream) @@ -683,6 +685,7 @@ def parse_tuple( with_condexpr: bool = True, extra_end_rules: t.Optional[t.Tuple[str, ...]] = None, explicit_parentheses: bool = False, + with_namespace: bool = False, ) -> t.Union[nodes.Tuple, nodes.Expr]: """Works like `parse_expression` but if multiple expressions are delimited by a comma a :class:`~jinja2.nodes.Tuple` node is created. @@ -704,13 +707,14 @@ def parse_tuple( """ lineno = self.stream.current.lineno if simplified: - parse = self.parse_primary - elif with_condexpr: - parse = self.parse_expression + + def parse() -> nodes.Expr: + return self.parse_primary(with_namespace=with_namespace) + else: def parse() -> nodes.Expr: - return self.parse_expression(with_condexpr=False) + return self.parse_expression(with_condexpr=with_condexpr) args: t.List[nodes.Expr] = [] is_tuple = False diff --git a/tests/test_core_tags.py b/tests/test_core_tags.py index 4bb95e024..2d847a2c9 100644 --- a/tests/test_core_tags.py +++ b/tests/test_core_tags.py @@ -538,6 +538,14 @@ def test_namespace_macro(self, env_trim): ) assert tmpl.render() == "13|37" + def test_namespace_set_tuple(self, env_trim): + tmpl = env_trim.from_string( + "{% set ns = namespace(a=12, b=36) %}" + "{% set ns.a, ns.b = ns.a + 1, ns.b + 1 %}" + "{{ ns.a }}|{{ ns.b }}" + ) + assert tmpl.render() == "13|37" + def test_block_escaping_filtered(self): env = Environment(autoescape=True) tmpl = env.from_string( From b8f4831d41e6a7cb5c40d42f074ffd92d2daccfc Mon Sep 17 00:00:00 2001 From: David Lord Date: Fri, 20 Dec 2024 14:02:31 -0800 Subject: [PATCH 48/52] more comments about nsref assignment only emit nsref instance check once per ref name refactor primary name parsing a bit --- src/jinja2/compiler.py | 24 ++++++++++++++++-------- src/jinja2/parser.py | 18 +++++++++++------- 2 files changed, 27 insertions(+), 15 deletions(-) diff --git a/src/jinja2/compiler.py b/src/jinja2/compiler.py index 0666cddf7..a4ff6a1b1 100644 --- a/src/jinja2/compiler.py +++ b/src/jinja2/compiler.py @@ -1582,12 +1582,19 @@ def visit_Output(self, node: nodes.Output, frame: Frame) -> None: def visit_Assign(self, node: nodes.Assign, frame: Frame) -> None: self.push_assign_tracking() - # NSRef can only ever be used during assignment so we need to check - # to make sure that it is only being used to assign using a Namespace. - # This check is done here because it is used an expression during the - # assignment and therefore cannot have this check done when the NSRef - # node is visited + # ``a.b`` is allowed for assignment, and is parsed as an NSRef. However, + # it is only valid if it references a Namespace object. Emit a check for + # that for each ref here, before assignment code is emitted. This can't + # be done in visit_NSRef as the ref could be in the middle of a tuple. + seen_refs: t.Set[str] = set() + for nsref in node.find_all(nodes.NSRef): + if nsref.name in seen_refs: + # Only emit the check for each reference once, in case the same + # ref is used multiple times in a tuple, `ns.a, ns.b = c, d`. + continue + + seen_refs.add(nsref.name) ref = frame.symbols.ref(nsref.name) self.writeline(f"if not isinstance({ref}, Namespace):") self.indent() @@ -1653,9 +1660,10 @@ def visit_Name(self, node: nodes.Name, frame: Frame) -> None: self.write(ref) def visit_NSRef(self, node: nodes.NSRef, frame: Frame) -> None: - # NSRefs can only be used to store values; since they use the normal - # `foo.bar` notation they will be parsed as a normal attribute access - # when used anywhere but in a `set` context + # NSRef is a dotted assignment target a.b=c, but uses a[b]=c internally. + # visit_Assign emits code to validate that each ref is to a Namespace + # object only. That can't be emitted here as the ref could be in the + # middle of a tuple assignment. ref = frame.symbols.ref(node.name) self.writeline(f"{ref}[{node.attr!r}]") diff --git a/src/jinja2/parser.py b/src/jinja2/parser.py index 107232631..f4117754a 100644 --- a/src/jinja2/parser.py +++ b/src/jinja2/parser.py @@ -641,21 +641,24 @@ def parse_unary(self, with_filter: bool = True) -> nodes.Expr: return node def parse_primary(self, with_namespace: bool = False) -> nodes.Expr: + """Parse a name or literal value. If ``with_namespace`` is enabled, also + parse namespace attr refs, for use in assignments.""" token = self.stream.current node: nodes.Expr if token.type == "name": + next(self.stream) if token.value in ("true", "false", "True", "False"): node = nodes.Const(token.value in ("true", "True"), lineno=token.lineno) elif token.value in ("none", "None"): node = nodes.Const(None, lineno=token.lineno) - elif with_namespace and self.stream.look().type == "dot": - next(self.stream) # token - next(self.stream) # dot - attr = self.stream.current + elif with_namespace and self.stream.current.type == "dot": + # If namespace attributes are allowed at this point, and the next + # token is a dot, produce a namespace reference. + next(self.stream) + attr = self.stream.expect("name") node = nodes.NSRef(token.value, attr.value, lineno=token.lineno) else: node = nodes.Name(token.value, "load", lineno=token.lineno) - next(self.stream) elif token.type == "string": next(self.stream) buf = [token.value] @@ -693,8 +696,9 @@ def parse_tuple( if no commas where found. The default parsing mode is a full tuple. If `simplified` is `True` - only names and literals are parsed. The `no_condexpr` parameter is - forwarded to :meth:`parse_expression`. + only names and literals are parsed; ``with_namespace`` allows namespace + attr refs as well. The `no_condexpr` parameter is forwarded to + :meth:`parse_expression`. Because tuples do not require delimiters and may end in a bogus comma an extra hint is needed that marks the end of a tuple. For example From 66587ce989e5a478e0bb165371fa2b9d42b7040f Mon Sep 17 00:00:00 2001 From: Kevin Brown-Silva Date: Mon, 2 May 2022 15:33:58 -0600 Subject: [PATCH 49/52] Fix bug where set would sometimes fail within if There was a bug that came as the result of an early optimization done within ID tracking that caused loading parameters to fail in a very specific and rare edge case. That edge case only occurred when the parameter was being set within all 3 standard branches of an if block, since the optimization would assume that the parameter was never being referenced and was only ever being set. This would cause the variable to be set to undefined. The fix for this was to remove the optimization and still continue to load in the parameter even if it is set in all 3 branches. --- CHANGES.rst | 3 +++ src/jinja2/idtracking.py | 17 +++++++---------- tests/test_regression.py | 10 ++++++++++ 3 files changed, 20 insertions(+), 10 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index 2b8179855..b6a5a1af5 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -45,6 +45,9 @@ Unreleased filter. :issue:`1624` - Using ``set`` for multiple assignment (``a, b = 1, 2``) does not fail when the target is a namespace attribute. :issue:`1413` +- Using ``set`` in all branches of ``{% if %}{% elif %}{% else %}`` blocks + does not cause the variable to be considered initially undefined. + :issue:`1253` Version 3.1.4 diff --git a/src/jinja2/idtracking.py b/src/jinja2/idtracking.py index cb4bccb0e..e6dd8cd11 100644 --- a/src/jinja2/idtracking.py +++ b/src/jinja2/idtracking.py @@ -121,23 +121,20 @@ def load(self, name: str) -> None: self._define_ref(name, load=(VAR_LOAD_RESOLVE, name)) def branch_update(self, branch_symbols: t.Sequence["Symbols"]) -> None: - stores: t.Dict[str, int] = {} + stores: t.Set[str] = set() + for branch in branch_symbols: - for target in branch.stores: - if target in self.stores: - continue - stores[target] = stores.get(target, 0) + 1 + stores.update(branch.stores) + + stores.difference_update(self.stores) for sym in branch_symbols: self.refs.update(sym.refs) self.loads.update(sym.loads) self.stores.update(sym.stores) - for name, branch_count in stores.items(): - if branch_count == len(branch_symbols): - continue - - target = self.find_ref(name) # type: ignore + for name in stores: + target = self.find_ref(name) assert target is not None, "should not happen" if self.parent is not None: diff --git a/tests/test_regression.py b/tests/test_regression.py index 10df2d1bd..93d72c5e6 100644 --- a/tests/test_regression.py +++ b/tests/test_regression.py @@ -750,6 +750,16 @@ def is_foo(ctx, s): assert tmpl.render() == "foo" +def test_load_parameter_when_set_in_all_if_branches(env): + tmpl = env.from_string( + "{% if True %}{{ a.b }}{% set a = 1 %}" + "{% elif False %}{% set a = 2 %}" + "{% else %}{% set a = 3 %}{% endif %}" + "{{ a }}" + ) + assert tmpl.render(a={"b": 0}) == "01" + + @pytest.mark.parametrize("unicode_char", ["\N{FORM FEED}", "\x85"]) def test_unicode_whitespace(env, unicode_char): content = "Lorem ipsum\n" + unicode_char + "\nMore text" From eda8fe86fd716dfce24910294e9f1fc81fbc740c Mon Sep 17 00:00:00 2001 From: David Lord Date: Sat, 21 Dec 2024 10:14:25 -0800 Subject: [PATCH 50/52] update dev dependencies --- .github/workflows/publish.yaml | 6 +++--- .github/workflows/tests.yaml | 2 +- .pre-commit-config.yaml | 2 +- requirements/build.txt | 2 +- requirements/dev.txt | 32 ++++++++++++++++---------------- requirements/docs.txt | 4 ++-- requirements/tests.txt | 6 +++--- requirements/typing.txt | 2 +- src/jinja2/lexer.py | 6 +++--- 9 files changed, 31 insertions(+), 31 deletions(-) diff --git a/.github/workflows/publish.yaml b/.github/workflows/publish.yaml index 727518c69..af983de40 100644 --- a/.github/workflows/publish.yaml +++ b/.github/workflows/publish.yaml @@ -23,7 +23,7 @@ jobs: - name: generate hash id: hash run: cd dist && echo "hash=$(sha256sum * | base64 -w0)" >> $GITHUB_OUTPUT - - uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3 + - uses: actions/upload-artifact@6f51ac03b9356f520e9adb1b1b7802705f340c2b # v4.5.0 with: path: ./dist provenance: @@ -64,10 +64,10 @@ jobs: id-token: write steps: - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 - - uses: pypa/gh-action-pypi-publish@f7600683efdcb7656dec5b29656edb7bc586e597 # v1.10.3 + - uses: pypa/gh-action-pypi-publish@67339c736fd9354cd4f8cb0b744f2b82a74b5c70 # v1.12.3 with: repository-url: https://test.pypi.org/legacy/ packages-dir: artifact/ - - uses: pypa/gh-action-pypi-publish@f7600683efdcb7656dec5b29656edb7bc586e597 # v1.10.3 + - uses: pypa/gh-action-pypi-publish@67339c736fd9354cd4f8cb0b744f2b82a74b5c70 # v1.12.3 with: packages-dir: artifact/ diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 515a7a5e4..1062ebe44 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -43,7 +43,7 @@ jobs: cache: pip cache-dependency-path: requirements*/*.txt - name: cache mypy - uses: actions/cache@6849a6489940f00c2f30c0fb92c6274307ccb58a # v4.1.2 + uses: actions/cache@1bd1e32a3bdc45362d1e726936510720a7c30a57 # v4.2.0 with: path: ./.mypy_cache key: mypy|${{ hashFiles('pyproject.toml') }} diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 74b54e8f1..a9f102b5e 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -2,7 +2,7 @@ ci: autoupdate_schedule: monthly repos: - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.7.1 + rev: v0.8.4 hooks: - id: ruff - id: ruff-format diff --git a/requirements/build.txt b/requirements/build.txt index 1b13b0552..9d6dd1040 100644 --- a/requirements/build.txt +++ b/requirements/build.txt @@ -6,7 +6,7 @@ # build==1.2.2.post1 # via -r build.in -packaging==24.1 +packaging==24.2 # via build pyproject-hooks==1.2.0 # via build diff --git a/requirements/dev.txt b/requirements/dev.txt index ba73d911c..c90a78168 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -6,7 +6,7 @@ # alabaster==1.0.0 # via sphinx -attrs==24.2.0 +attrs==24.3.0 # via # outcome # trio @@ -16,7 +16,7 @@ build==1.2.2.post1 # via pip-tools cachetools==5.5.0 # via tox -certifi==2024.8.30 +certifi==2024.12.14 # via requests cfgv==3.4.0 # via pre-commit @@ -38,7 +38,7 @@ filelock==3.16.1 # via # tox # virtualenv -identify==2.6.1 +identify==2.6.3 # via pre-commit idna==3.10 # via @@ -52,15 +52,15 @@ jinja2==3.1.4 # via sphinx markupsafe==3.0.2 # via jinja2 -mypy==1.13.0 - # via -r typing.in +mypy==1.14.0 + # via -r /Users/david/Projects/jinja/requirements/typing.in mypy-extensions==1.0.0 # via mypy nodeenv==1.9.1 # via pre-commit outcome==1.3.0.post0 # via trio -packaging==24.1 +packaging==24.2 # via # build # pallets-sphinx-themes @@ -69,8 +69,8 @@ packaging==24.1 # sphinx # tox pallets-sphinx-themes==2.3.0 - # via -r docs.in -pip-compile-multi==2.6.4 + # via -r /Users/david/Projects/jinja/requirements/docs.in +pip-compile-multi==2.7.1 # via -r dev.in pip-tools==7.4.1 # via pip-compile-multi @@ -92,8 +92,8 @@ pyproject-hooks==1.2.0 # via # build # pip-tools -pytest==8.3.3 - # via -r tests.in +pytest==8.3.4 + # via -r /Users/david/Projects/jinja/requirements/tests.in pyyaml==6.0.2 # via pre-commit requests==2.32.3 @@ -106,13 +106,13 @@ sortedcontainers==2.4.0 # via trio sphinx==8.1.3 # via - # -r docs.in + # -r /Users/david/Projects/jinja/requirements/docs.in # pallets-sphinx-themes # sphinx-issues # sphinx-notfound-page # sphinxcontrib-log-cabinet sphinx-issues==5.0.0 - # via -r docs.in + # via -r /Users/david/Projects/jinja/requirements/docs.in sphinx-notfound-page==1.0.4 # via pallets-sphinx-themes sphinxcontrib-applehelp==2.0.0 @@ -124,7 +124,7 @@ sphinxcontrib-htmlhelp==2.1.0 sphinxcontrib-jsmath==1.0.1 # via sphinx sphinxcontrib-log-cabinet==1.0.1 - # via -r docs.in + # via -r /Users/david/Projects/jinja/requirements/docs.in sphinxcontrib-qthelp==2.0.0 # via sphinx sphinxcontrib-serializinghtml==2.0.0 @@ -134,16 +134,16 @@ toposort==1.10 tox==4.23.2 # via -r dev.in trio==0.27.0 - # via -r tests.in + # via -r /Users/david/Projects/jinja/requirements/tests.in typing-extensions==4.12.2 # via mypy urllib3==2.2.3 # via requests -virtualenv==20.27.0 +virtualenv==20.28.0 # via # pre-commit # tox -wheel==0.44.0 +wheel==0.45.1 # via pip-tools # The following packages are considered to be unsafe in a requirements file: diff --git a/requirements/docs.txt b/requirements/docs.txt index 453a7cb5d..2283fa9b5 100644 --- a/requirements/docs.txt +++ b/requirements/docs.txt @@ -8,7 +8,7 @@ alabaster==1.0.0 # via sphinx babel==2.16.0 # via sphinx -certifi==2024.8.30 +certifi==2024.12.14 # via requests charset-normalizer==3.4.0 # via requests @@ -22,7 +22,7 @@ jinja2==3.1.4 # via sphinx markupsafe==3.0.2 # via jinja2 -packaging==24.1 +packaging==24.2 # via # pallets-sphinx-themes # sphinx diff --git a/requirements/tests.txt b/requirements/tests.txt index e019ba988..71dad37da 100644 --- a/requirements/tests.txt +++ b/requirements/tests.txt @@ -4,7 +4,7 @@ # # pip-compile tests.in # -attrs==24.2.0 +attrs==24.3.0 # via # outcome # trio @@ -14,11 +14,11 @@ iniconfig==2.0.0 # via pytest outcome==1.3.0.post0 # via trio -packaging==24.1 +packaging==24.2 # via pytest pluggy==1.5.0 # via pytest -pytest==8.3.3 +pytest==8.3.4 # via -r tests.in sniffio==1.3.1 # via trio diff --git a/requirements/typing.txt b/requirements/typing.txt index 1cf3727a5..f50d6d667 100644 --- a/requirements/typing.txt +++ b/requirements/typing.txt @@ -4,7 +4,7 @@ # # pip-compile typing.in # -mypy==1.13.0 +mypy==1.14.0 # via -r typing.in mypy-extensions==1.0.0 # via mypy diff --git a/src/jinja2/lexer.py b/src/jinja2/lexer.py index 6dc94b67d..9b1c96979 100644 --- a/src/jinja2/lexer.py +++ b/src/jinja2/lexer.py @@ -262,7 +262,7 @@ def __init__( self.message = message self.error_class = cls - def __call__(self, lineno: int, filename: str) -> "te.NoReturn": + def __call__(self, lineno: int, filename: t.Optional[str]) -> "te.NoReturn": raise self.error_class(self.message, lineno, filename) @@ -757,7 +757,7 @@ def tokeniter( for idx, token in enumerate(tokens): # failure group - if token.__class__ is Failure: + if isinstance(token, Failure): raise token(lineno, filename) # bygroup is a bit more complex, in that case we # yield for the current token the first named @@ -778,7 +778,7 @@ def tokeniter( data = groups[idx] if data or token not in ignore_if_empty: - yield lineno, token, data + yield lineno, token, data # type: ignore[misc] lineno += data.count("\n") + newlines_stripped newlines_stripped = 0 From 8d588592653b052f957b720e1fc93196e06f207f Mon Sep 17 00:00:00 2001 From: David Lord Date: Sat, 21 Dec 2024 10:14:49 -0800 Subject: [PATCH 51/52] remove test pypi --- .github/workflows/publish.yaml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.github/workflows/publish.yaml b/.github/workflows/publish.yaml index af983de40..d609abdb6 100644 --- a/.github/workflows/publish.yaml +++ b/.github/workflows/publish.yaml @@ -64,10 +64,6 @@ jobs: id-token: write steps: - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 - - uses: pypa/gh-action-pypi-publish@67339c736fd9354cd4f8cb0b744f2b82a74b5c70 # v1.12.3 - with: - repository-url: https://test.pypi.org/legacy/ - packages-dir: artifact/ - uses: pypa/gh-action-pypi-publish@67339c736fd9354cd4f8cb0b744f2b82a74b5c70 # v1.12.3 with: packages-dir: artifact/ From 877f6e51be8e1765b06d911cfaa9033775f051d1 Mon Sep 17 00:00:00 2001 From: David Lord Date: Sat, 21 Dec 2024 10:16:13 -0800 Subject: [PATCH 52/52] release version 3.1.5 --- CHANGES.rst | 2 +- src/jinja2/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index b6a5a1af5..e1b339198 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -3,7 +3,7 @@ Version 3.1.5 ------------- -Unreleased +Released 2024-12-21 - The sandboxed environment handles indirect calls to ``str.format``, such as by passing a stored reference to a filter that calls its argument. diff --git a/src/jinja2/__init__.py b/src/jinja2/__init__.py index 578940091..d669f295b 100644 --- a/src/jinja2/__init__.py +++ b/src/jinja2/__init__.py @@ -35,4 +35,4 @@ from .utils import pass_eval_context as pass_eval_context from .utils import select_autoescape as select_autoescape -__version__ = "3.1.5.dev" +__version__ = "3.1.5"