diff --git a/Doc/library/tarfile.rst b/Doc/library/tarfile.rst index f19038d837bb526..0d8ce6e497915fd 100644 --- a/Doc/library/tarfile.rst +++ b/Doc/library/tarfile.rst @@ -1112,6 +1112,10 @@ reused in custom filters: paths (in case the name is absolute even after stripping slashes, e.g. ``C:/foo`` on Windows). This raises :class:`~tarfile.AbsolutePathError`. + - Normalize filenames (:attr:`TarInfo.name`) that contain ``..`` components + using :func:`os.path.normpath`. + Note that this removes internal ``..`` components, which may change the + meaning of the name if it traverses symbolic links. - :ref:`Refuse ` to extract files whose absolute path (after following symlinks) would end up outside the destination. This raises :class:`~tarfile.OutsideDestinationError`. @@ -1120,6 +1124,10 @@ reused in custom filters: Return the modified ``TarInfo`` member. + .. versionchanged:: next + + Filenames containing ``..`` components are now normalized. + .. function:: data_filter(member, path) Implements the ``'data'`` filter. diff --git a/Lib/tarfile.py b/Lib/tarfile.py index cee21bfc6fe5aa6..451302715329fe5 100644 --- a/Lib/tarfile.py +++ b/Lib/tarfile.py @@ -833,6 +833,13 @@ def _get_filtered_attrs(member, dest_path, for_data=True): # For example, 'C:/foo' on Windows. raise AbsolutePathError(member) # Ensure we stay in the destination + if '..' in name.replace(os.sep, '/').split('/'): + # Directories are created from the name as given, so a name that + # leaves the destination part-way through would create them + # outside it even if the resolved path stays inside. + normalized = os.path.normpath(name) + if normalized != name: + name = new_attrs['name'] = normalized target_path = os.path.realpath(os.path.join(dest_path, name), strict=os.path.ALLOW_MISSING) if os.path.commonpath([target_path, dest_path]) != dest_path: diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index 31e5508dd9b7907..be71575a6ea06a9 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -1271,15 +1271,20 @@ def set_memlimit(limit: str) -> None: def _memory_watchdog(pid): - """Return a function printing the memory usage of process *pid*.""" + """Return a function printing the memory usage of process *pid*. + + The largest value it saw is kept in its ``peak`` attribute. + """ # Imported here: test.support does not depend on test.libregrtest. from test.libregrtest.utils import get_process_memory_usage def watch(): mem = get_process_memory_usage(pid) if mem is not None: + watch.peak = max(watch.peak, mem) print(f" ... process data size: {mem / (1024 ** 3):.1f} GiB", flush=True) + watch.peak = 0 return watch @@ -1333,7 +1338,23 @@ def wrapper(self): qualname = f'{cls.__qualname__}.{f.__name__}' proc = isolation._start_test(cls.__module__, qualname) watchdog = _memory_watchdog(proc.pid) if verbose else None - isolation._replay_test(self, *proc.wait(tick=watchdog)) + payload, output, returncode = proc.wait(tick=watchdog) + if watchdog: + # The subprocess measures its own peak exactly. What the + # parent sampled is only a lower bound. + maxrss = payload and payload.get('maxrss') + peak = maxrss or watchdog.peak + if peak: + print(f" ... peak memory use: " + f"{peak / (1024 ** 3):.1f} GiB" + f"{'' if maxrss else ' or more'}", flush=True) + majflt = payload and payload.get('majflt') + if majflt: + # The test did not fit in memory, so its timing means + # little. + print(f" ... {majflt} major page faults: the test " + f"waited for the disk", flush=True) + isolation._replay_test(self, payload, output, returncode) return return f(self, maxsize) diff --git a/Lib/test/support/subprocess_runner.py b/Lib/test/support/subprocess_runner.py index 90d74cc757d878d..bb4ab059e2b8b6c 100644 --- a/Lib/test/support/subprocess_runner.py +++ b/Lib/test/support/subprocess_runner.py @@ -72,7 +72,45 @@ def _outcome(kind, test, detail): for t, tb in result.expectedFailures] outcomes += [_outcome('skipped', t, reason) for t, reason in result.skipped] -payload = {'outcomes': outcomes, 'durations': result.id_durations} +def _usage(): + """What this process used: peak resident set size in bytes, and the + number of major page faults it took, either of which can be None. + + A major page fault is served from disk, so a non-zero count means swapping. + + The modules are imported here, after the test has run, so that the test + does not see them. + """ + try: + import resource + except ImportError: + pass + else: + usage = resource.getrusage(resource.RUSAGE_SELF) + # Solaris and illumos leave these fields at 0, which no live process + # has, so treat it as "not supported". + if not usage.ru_maxrss: + return {'maxrss': None, 'majflt': None} + # ru_maxrss is in bytes on macOS, in kilobytes on Linux and the BSDs. + maxrss = usage.ru_maxrss + return {'maxrss': maxrss if sys.platform == 'darwin' else maxrss * 1024, + 'majflt': usage.ru_majflt} + try: + import os + import _winapi + handle = _winapi.OpenProcess( + _winapi.PROCESS_QUERY_LIMITED_INFORMATION, False, os.getpid()) + except (ImportError, OSError): + return {'maxrss': None, 'majflt': None} + try: + info = _winapi.GetProcessMemoryInfo(handle) + finally: + _winapi.CloseHandle(handle) + # PageFaultCount counts all faults, not only the ones served from disk. + return {'maxrss': info['PeakWorkingSetSize'], 'majflt': None} + + +payload = {'outcomes': outcomes, 'durations': result.id_durations, **_usage()} with open(outfile, 'wb') as f: marshal.dump(payload, f) diff --git a/Lib/test/test_clinic.py b/Lib/test/test_clinic.py index 94a69b6d7309df8..d3e4454348d0235 100644 --- a/Lib/test/test_clinic.py +++ b/Lib/test/test_clinic.py @@ -347,7 +347,7 @@ def test_ambiguous_group_and_optional_parameters(self): / [clinic start generated code]*/ """ - self.expect_failure(block, err) + self.expect_failure(block, err, lineno=2) def test_star_after_vararg(self): err = "'my_test_func' uses '*' more than once." @@ -3063,9 +3063,22 @@ def test_state_func_docstring_no_summary(self): m.func docstring1 docstring2 + docstring3 """ + # The line which should have been left blank. self.expect_failure(block, err, lineno=3) + def test_state_func_docstring_long_summary(self): + err = "Summary line for 'm.func' is too long!" + block = f""" + module m + m.func + {'x' * 100} + + Body. + """ + self.expect_failure(block, err, lineno=2) + def test_state_func_docstring_only_one_param_template(self): err = "You may not specify {parameters} more than once in a docstring!" block = """ @@ -3077,6 +3090,7 @@ def test_state_func_docstring_only_one_param_template(self): {parameters} these are the params again: {parameters} + and this is the end of the docstring """ self.expect_failure(block, err, lineno=7) @@ -3667,6 +3681,163 @@ def test_cli_converters_no_converters(self): f.write("/*[clinic input]\n[clinic start generated code]*/\n") self.assertEqual(self.expect_success("--converters", fn), "") + LIST_CODE = dedent(""" + /*[clinic input] + func + a: int + / + + Docstring. + [clinic start generated code]*/ + + /*[clinic input] + cloned = func + [clinic start generated code]*/ + + /*[clinic input] + module m + class m.C "void *" "" + class m.C.D "void *" "" + [clinic start generated code]*/ + + /*[clinic input] + m.C.meth + self: self(type="void *") + a: object + [ + b: object + ] + / + + Docstring. + [clinic start generated code]*/ + + /*[clinic input] + @classmethod + m.C.__new__ + a: object + + Docstring. + [clinic start generated code]*/ + + /*[clinic input] + @getter + m.C.prop + [clinic start generated code]*/ + + /*[clinic input] + @setter + m.C.prop + [clinic start generated code]*/ + + /*[clinic input] + m.C.D.meth + self: self(type="void *") + + Docstring. + [clinic start generated code]*/ + """) + + def make_list_file(self, tmp_dir): + fn = os.path.join(tmp_dir, "test.c") + with open(fn, "w", encoding="utf-8") as f: + f.write(self.LIST_CODE) + return fn + + LIST_OUTPUT = [ + " func($module, a, /)", + " cloned($module, a, /)", + " module m", + " class m.C", + # A signature with an option group is only for the docstring. + " m.C.meth(a, [b])", + " m.C(a)", + " getter m.C.prop", + " setter m.C.prop", + " class m.C.D", + " m.C.D.meth($self, /)", + ] + + def test_cli_list(self): + with os_helper.temp_dir() as tmp_dir: + fn = self.make_list_file(tmp_dir) + pre_mtime = os.stat(fn).st_mtime_ns + out = self.expect_success("--list", fn) + self.assertEqual(out.splitlines(), [fn] + self.LIST_OUTPUT) + # Nothing is written. + with open(fn, encoding="utf-8") as f: + self.assertEqual(f.read(), self.LIST_CODE) + self.assertEqual(os.stat(fn).st_mtime_ns, pre_mtime) + self.assertEqual(os.listdir(tmp_dir), ["test.c"]) + + def test_cli_list_no_clinic_block(self): + with os_helper.temp_dir() as tmp_dir: + fn = os.path.join(tmp_dir, "test.c") + with open(fn, "w", encoding="utf-8") as f: + f.write("int x;\n") + self.assertEqual(self.expect_success("--list", fn), "") + + def test_cli_list_no_definitions(self): + with os_helper.temp_dir() as tmp_dir: + fn = os.path.join(tmp_dir, "test.c") + with open(fn, "w", encoding="utf-8") as f: + f.write("/*[clinic input]\n[clinic start generated code]*/\n") + self.assertEqual(self.expect_success("--list", fn), "") + + def test_cli_list_make(self): + with os_helper.temp_dir() as tmp_dir: + fn = self.make_list_file(tmp_dir) + out = self.expect_success("--list", "--make", "--srcdir", tmp_dir) + self.assertEqual(out.splitlines(), [fn] + self.LIST_OUTPUT) + self.assertEqual(os.listdir(tmp_dir), ["test.c"]) + + def test_cli_list_verbose(self): + with os_helper.temp_dir() as tmp_dir: + fn = self.make_list_file(tmp_dir) + # The progress does not mix with the report. + out, err, code = self.run_clinic("-v", "--list", fn) + self.assertEqual(code, 0) + self.assertEqual(err.splitlines(), [fn]) + self.assertEqual(out.splitlines(), [fn] + self.LIST_OUTPUT) + + def test_cli_list_checksum_mismatch(self): + with os_helper.temp_dir() as tmp_dir: + fn = self.make_list_file(tmp_dir) + with open(fn, "a", encoding="utf-8") as f: + f.write("/*[clinic end generated code: " + "output=0123456789abcdef input=fedcba9876543210]*/\n") + _, err = self.expect_failure("--list", fn) + self.assertIn("Checksum mismatch!", err) + # The check is skipped with --force. + out = self.expect_success("-f", "--list", fn) + self.assertEqual(out.splitlines(), [fn] + self.LIST_OUTPUT) + self.assertEqual(os.listdir(tmp_dir), ["test.c"]) + + def test_cli_list_external(self): + # A file which uses getters, setters and nested classes. + source = support.findfile('clinic.test.c') + out = self.expect_success("--list", source) + lines = out.splitlines() + self.assertEqual(lines[0], source) + for line in (" class Test", + " getter Test.property", + " setter Test.property", + " Test.class_method($type, /)", + " module m", + " class m.T"): + with self.subTest(line=line): + self.assertIn(line, lines) + + def test_cli_fail_list_and_dry_run(self): + for opt in "--dry-run", "--diff": + with self.subTest(opt=opt): + _, err = self.expect_failure("--list", opt, "test.c") + self.assertIn("can't use --dry-run or --diff with --list", err) + + def test_cli_fail_list_and_converters(self): + _, err = self.expect_failure("--list", "--converters", "test.c") + self.assertIn("can't use --converters with --list", err) + def test_cli_fail_directory(self): with os_helper.temp_dir() as tmp_dir: subdir = os.path.join(tmp_dir, "test.c") diff --git a/Lib/test/test_tarfile.py b/Lib/test/test_tarfile.py index c8c9b49fb6fc617..3899eac6be3b3aa 100644 --- a/Lib/test/test_tarfile.py +++ b/Lib/test/test_tarfile.py @@ -4092,6 +4092,20 @@ def test_absolute(self): tarfile.AbsolutePathError, """['"].*escaped.evil['"] has an absolute path""") + def test_parent_dir_out_and_back(self): + # Test a member that leaves the destination and comes back. + # The containment check looks at the resolved path, which stays + # inside, but the intermediate directories are created from the + # name as given, which does not. + with ArchiveMaker() as arc: + arc.add(f'../escaped.evil/../{self.destdir.name}/sub/file', + content='content') + + for filter in 'tar', 'data': + with self.subTest(filter): + with self.check_context(arc.open(), filter): + self.expect_file('sub/file', content='content') + @symlink_test def test_parent_symlink(self): # Test interplaying symlinks diff --git a/Misc/NEWS.d/next/Build/2026-08-02-12-00-01.gh-issue-152023.pT7dKb.rst b/Misc/NEWS.d/next/Build/2026-08-02-12-00-01.gh-issue-152023.pT7dKb.rst new file mode 100644 index 000000000000000..4d81c66d163a7c2 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2026-08-02-12-00-01.gh-issue-152023.pT7dKb.rst @@ -0,0 +1,2 @@ +Update Android builds to SQLite 3.53.4. Additionally, enable the ``median()`` +and ``percentile()`` functions. diff --git a/Misc/NEWS.d/next/Security/2026-08-13-13-08-11.gh-issue-155999.Xt4rWq.rst b/Misc/NEWS.d/next/Security/2026-08-13-13-08-11.gh-issue-155999.Xt4rWq.rst new file mode 100644 index 000000000000000..59b725e55bbffda --- /dev/null +++ b/Misc/NEWS.d/next/Security/2026-08-13-13-08-11.gh-issue-155999.Xt4rWq.rst @@ -0,0 +1,5 @@ +Fix the :mod:`tarfile` ``tar`` and ``data`` extraction filters creating +directories outside the destination for members whose name leaves the +destination and returns to it, such as ``../evil/../dest/sub/file``. The +containment check used the resolved path, but intermediate directories were +created from the name as given. diff --git a/Misc/NEWS.d/next/Tests/2026-08-18-19-40-22.gh-issue-75876.IOiCcK.rst b/Misc/NEWS.d/next/Tests/2026-08-18-19-40-22.gh-issue-75876.IOiCcK.rst new file mode 100644 index 000000000000000..863b49d2f91d8cf --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2026-08-18-19-40-22.gh-issue-75876.IOiCcK.rst @@ -0,0 +1,3 @@ +In verbose mode, a test decorated with :func:`~test.support.bigmemtest` now +reports how much memory it really used, and the number of major page faults +it took, if any. diff --git a/Misc/NEWS.d/next/Tools-Demos/2026-08-05-18-56-17.gh-issue-107570.wXxtw2.rst b/Misc/NEWS.d/next/Tools-Demos/2026-08-05-18-56-17.gh-issue-107570.wXxtw2.rst new file mode 100644 index 000000000000000..c57a5cee1c0d49d --- /dev/null +++ b/Misc/NEWS.d/next/Tools-Demos/2026-08-05-18-56-17.gh-issue-107570.wXxtw2.rst @@ -0,0 +1,3 @@ +Argument Clinic: report errors on the offending line. +Errors in a docstring were reported on the line which ends the block, and +errors detected when generating the code were reported without any position. diff --git a/Misc/NEWS.d/next/Tools-Demos/2026-08-06-08-32-53.gh-issue-155263.RCcjtk.rst b/Misc/NEWS.d/next/Tools-Demos/2026-08-06-08-32-53.gh-issue-155263.RCcjtk.rst new file mode 100644 index 000000000000000..969efa1af808f59 --- /dev/null +++ b/Misc/NEWS.d/next/Tools-Demos/2026-08-06-08-32-53.gh-issue-155263.RCcjtk.rst @@ -0,0 +1,3 @@ +Add the ``--list`` option to Argument Clinic. +It prints the modules, classes and functions which Argument Clinic defines in +the specified files, each function with its signature. diff --git a/Misc/NEWS.d/next/Windows/2026-08-02-12-00-00.gh-issue-152023.wXn4Qs.rst b/Misc/NEWS.d/next/Windows/2026-08-02-12-00-00.gh-issue-152023.wXn4Qs.rst new file mode 100644 index 000000000000000..9e12f2d22173f56 --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2026-08-02-12-00-00.gh-issue-152023.wXn4Qs.rst @@ -0,0 +1,2 @@ +Update Windows builds to SQLite 3.53.4. Additionally, enable the ``median()`` +and ``percentile()`` functions. diff --git a/Misc/externals.spdx.json b/Misc/externals.spdx.json index cdacaf6f5869531..0e1d7aeca16bf9a 100644 --- a/Misc/externals.spdx.json +++ b/Misc/externals.spdx.json @@ -91,21 +91,21 @@ "checksums": [ { "algorithm": "SHA256", - "checksumValue": "53f8711811090cc4d9ffc624c360f81e7b409763b145ab2e948998f1a0d6a612" + "checksumValue": "acc27a72ca7745781bb902269370303e778736a090585cc13acd2ea9ab785666" } ], - "downloadLocation": "https://github.com/python/cpython-source-deps/archive/refs/tags/sqlite-3.53.2.0.tar.gz", + "downloadLocation": "https://github.com/python/cpython-source-deps/archive/refs/tags/sqlite-3.53.4.0.tar.gz", "externalRefs": [ { "referenceCategory": "SECURITY", - "referenceLocator": "cpe:2.3:a:sqlite:sqlite:3.53.2.0:*:*:*:*:*:*:*", + "referenceLocator": "cpe:2.3:a:sqlite:sqlite:3.53.4.0:*:*:*:*:*:*:*", "referenceType": "cpe23Type" } ], "licenseConcluded": "NOASSERTION", "name": "sqlite", "primaryPackagePurpose": "SOURCE", - "versionInfo": "3.53.2.0" + "versionInfo": "3.53.4.0" }, { "SPDXID": "SPDXRef-PACKAGE-tcl", diff --git a/PCbuild/get_externals.bat b/PCbuild/get_externals.bat index 6bedd21299a73e9..b2984f3de7fe52d 100644 --- a/PCbuild/get_externals.bat +++ b/PCbuild/get_externals.bat @@ -56,7 +56,7 @@ set libraries=%libraries% bzip2-1.0.8 if NOT "%IncludeLibffiSrc%"=="false" set libraries=%libraries% libffi-3.4.4 if NOT "%IncludeSSLSrc%"=="false" set libraries=%libraries% openssl-3.5.7 set libraries=%libraries% mpdecimal-4.0.0 -set libraries=%libraries% sqlite-3.53.2.0 +set libraries=%libraries% sqlite-3.53.4.0 if NOT "%IncludeTkinterSrc%"=="false" set libraries=%libraries% tcl-9.0.4.0 if NOT "%IncludeTkinterSrc%"=="false" set libraries=%libraries% tk-9.0.4.1 set libraries=%libraries% xz-5.8.1.1 diff --git a/PCbuild/python.props b/PCbuild/python.props index 8d931bba28a389a..ee9aff599c753f4 100644 --- a/PCbuild/python.props +++ b/PCbuild/python.props @@ -98,7 +98,7 @@ - $(ExternalsDir)sqlite-3.53.2.0\ + $(ExternalsDir)sqlite-3.53.4.0\ $(ExternalsDir)bzip2-1.0.8\ $(ExternalsDir)xz-5.8.1.1\ $(ExternalsDir)libffi-3.4.4\ diff --git a/PCbuild/readme.txt b/PCbuild/readme.txt index 7751bdeb1a3077b..d54c0ed3e3c6333 100644 --- a/PCbuild/readme.txt +++ b/PCbuild/readme.txt @@ -234,7 +234,7 @@ _ssl again when building. _sqlite3 - Wraps SQLite 3.53.2, which is itself built by sqlite3.vcxproj + Wraps SQLite 3.53.4, which is itself built by sqlite3.vcxproj Homepage: https://www.sqlite.org/ diff --git a/PCbuild/sqlite3.vcxproj b/PCbuild/sqlite3.vcxproj index f12ec348b37e297..0673e03a59fc1e1 100644 --- a/PCbuild/sqlite3.vcxproj +++ b/PCbuild/sqlite3.vcxproj @@ -98,7 +98,7 @@ $(sqlite3Dir);%(AdditionalIncludeDirectories) - SQLITE_ENABLE_MATH_FUNCTIONS;SQLITE_ENABLE_FTS4;SQLITE_ENABLE_FTS5;SQLITE_ENABLE_RTREE;SQLITE_OMIT_AUTOINIT;SQLITE_API=__declspec(dllexport);%(PreprocessorDefinitions) + SQLITE_ENABLE_MATH_FUNCTIONS;SQLITE_ENABLE_FTS4;SQLITE_ENABLE_FTS5;SQLITE_ENABLE_PERCENTILE;SQLITE_ENABLE_RTREE;SQLITE_OMIT_AUTOINIT;SQLITE_API=__declspec(dllexport);%(PreprocessorDefinitions) Level1 %(AdditionalOptions) -Wno-unused diff --git a/Platforms/Android/__main__.py b/Platforms/Android/__main__.py index 7c4bf9f390741cb..22123ef5e7fbaae 100755 --- a/Platforms/Android/__main__.py +++ b/Platforms/Android/__main__.py @@ -225,7 +225,7 @@ def unpack_deps(host, prefix_dir, cache_dir): "bzip2-1.0.8-3", "libffi-3.4.4-3", "openssl-3.5.7-0", - "sqlite-3.53.2-0", + "sqlite-3.53.4-0", "xz-5.4.6-1", "zstd-1.5.7-2" ]: diff --git a/Tools/clinic/libclinic/clanguage.py b/Tools/clinic/libclinic/clanguage.py index ab77e7ad6603cdc..e586011a92c3469 100644 --- a/Tools/clinic/libclinic/clanguage.py +++ b/Tools/clinic/libclinic/clanguage.py @@ -92,7 +92,10 @@ def render( for o in signatures: if isinstance(o, Function): if function: - fail("You may specify at most one function per block.\nFound a block containing at least two:\n\t" + repr(function) + " and " + repr(o)) + fail("You may specify at most one function per block.\n" + "Found a block containing at least two:\n\t" + + repr(function) + " and " + repr(o), + line_number=o.line_number) function = o return self.render_function(clinic, function) @@ -337,7 +340,8 @@ def render_option_group_parsing( if count in subsets: fail(f"Function {f.full_name!r} has an ambiguous group " f"configuration: a call with {count} argument(s) " - f"can be parsed in more than one way.") + f"can be parsed in more than one way.", + line_number=f.line_number) subsets[count] = subset if limited_capi: @@ -462,7 +466,8 @@ def render_function( if has_option_groups and (not positional): fail("You cannot use optional groups ('[' and ']') " - "unless all parameters are positional-only ('/').") + "unless all parameters are positional-only ('/').", + line_number=f.line_number) # HACK # when we're METH_O, but have a custom return converter, diff --git a/Tools/clinic/libclinic/cli.py b/Tools/clinic/libclinic/cli.py index 290fc3a6e59408a..9629c1731779454 100644 --- a/Tools/clinic/libclinic/cli.py +++ b/Tools/clinic/libclinic/cli.py @@ -22,6 +22,9 @@ return_converters, ReturnConverterType) from libclinic.clanguage import CLanguage from libclinic.app import Clinic +from libclinic.dsl_parser import render_text_signature +from libclinic.function import ( + Class, Definition, Module, GETTER, SETTER, walk_definitions) # TODO: @@ -54,7 +57,7 @@ def parse_file( output: str | None = None, verify: bool = True, writer: libclinic.FileWriter | None = None, -) -> None: +) -> Clinic | None: if not output: output = filename if writer is None: @@ -78,7 +81,7 @@ def parse_file( # exit quickly if there are no clinic markers in the file find_start_re = BlockParser("", language).find_start_re if not find_start_re.search(raw): - return + return None if LIMITED_CAPI_REGEX.search(raw): limited_capi = True @@ -97,6 +100,31 @@ def parse_file( writer.update_times(output, [fn for fn, _ in files if fn != output], any(changed for _, changed in files)) + return clinic + + +def format_definition(depth: int, name: str, definition: Definition) -> str: + indent = " " * (depth + 1) + if isinstance(definition, Module): + return f"{indent}module {name}" + if isinstance(definition, Class): + return f"{indent}class {name}" + if definition.kind is GETTER: + return f"{indent}getter {name}" + if definition.kind is SETTER: + return f"{indent}setter {name}" + signature = render_text_signature(definition, definition.render_parameters, + name=name, line_width=None) + return indent + signature + + +def print_definitions(clinic: Clinic) -> None: + """Print the modules, classes and functions defined in the parsed file.""" + lines = [format_definition(depth, name, definition) + for depth, name, definition in walk_definitions(clinic)] + if lines: + print(clinic.filename) + print("\n".join(lines)) def create_cli() -> argparse.ArgumentParser: @@ -126,6 +154,10 @@ def create_cli() -> argparse.ArgumentParser: "and return converters; if files are " "specified, print only the converters " "which they define")) + cmdline.add_argument("--list", action='store_true', + help=("don't write any file, only list the modules, " + "classes and functions which the specified " + "files define, with their signatures")) cmdline.add_argument("--make", action='store_true', help="walk --srcdir to run over all relevant files") cmdline.add_argument("--srcdir", type=str, default=os.curdir, @@ -252,7 +284,7 @@ def run_clinic(parser: argparse.ArgumentParser, ns: argparse.Namespace) -> None: dry_run = ns.dry_run or ns.diff # The report is written to the standard output, so the progress # is written to the standard error stream to not mix them. - verbose_file = sys.stderr if dry_run else sys.stdout + verbose_file = sys.stderr if dry_run or ns.list else sys.stdout filenames: Iterable[str] if ns.make: @@ -268,6 +300,12 @@ def run_clinic(parser: argparse.ArgumentParser, ns: argparse.Namespace) -> None: parser.error("can't use -o with multiple filenames") filenames = ns.filename + if ns.list: + if dry_run: + parser.error("can't use --dry-run or --diff with --list") + if ns.converters: + parser.error("can't use --converters with --list") + if ns.converters: if dry_run: parser.error("can't use --dry-run or --diff with --converters") @@ -280,20 +318,22 @@ def run_clinic(parser: argparse.ArgumentParser, ns: argparse.Namespace) -> None: builtin_legacy_converters = dict(legacy_converters) builtin_return_converters = dict(return_converters) - writer = libclinic.FileWriter(dry_run=dry_run or ns.converters) + writer = libclinic.FileWriter(dry_run=dry_run or ns.converters or ns.list) for filename in filenames: if ns.verbose: print(filename, file=verbose_file) - parse_file(filename, output=ns.output, - verify=not ns.force, limited_capi=ns.limited_capi, - writer=writer) + clinic = parse_file(filename, output=ns.output, + verify=not ns.force, limited_capi=ns.limited_capi, + writer=writer) + if ns.list and clinic is not None: + print_definitions(clinic) if ns.converters: print_converters( defined_in_files(converters, builtin_converters), defined_in_files(legacy_converters, builtin_legacy_converters), defined_in_files(return_converters, builtin_return_converters)) - else: + elif not ns.list: report_changes(writer, diff=ns.diff) diff --git a/Tools/clinic/libclinic/dsl_parser.py b/Tools/clinic/libclinic/dsl_parser.py index a6b1d2bed5e5dee..b241f58711e68a4 100644 --- a/Tools/clinic/libclinic/dsl_parser.py +++ b/Tools/clinic/libclinic/dsl_parser.py @@ -263,6 +263,8 @@ class DSLParser: critical_section: bool target_critical_section: list[str] disable_fastcall: bool + # Line of the file which is being parsed. + line_number: int | None from_version_re = re.compile(r'([*/]) +\[from +(.+)\]') permit_long_summary = False permit_long_docstring_body = False @@ -286,6 +288,7 @@ def __init__(self, clinic: Clinic) -> None: def reset(self) -> None: self.function = None + self.line_number = None self.state = self.state_dsl_start self.expecting_parameters = True self.keyword_only = False @@ -509,6 +512,7 @@ def parse(self, block: Block) -> None: if '\t' in line: fail(f'Tab characters are illegal in the Clinic DSL: {line!r}', line_number=block_start) + self.line_number = line_number try: self.state(line) except ClinicError as exc: @@ -517,7 +521,14 @@ def parse(self, block: Block) -> None: raise self.do_post_block_processing_cleanup(line_number) - block.output.extend(self.clinic.language.render(self.clinic, block.signatures)) + try: + block.output.extend( + self.clinic.language.render(self.clinic, block.signatures)) + except ClinicError as exc: + if exc.lineno is None: + exc.lineno = line_number + exc.filename = self.clinic.filename + raise if self.preserve_output: if block.output: @@ -666,6 +677,8 @@ def parse_cloned_function(self, names: FunctionNames, existing: str) -> None: "cls": cls, "c_basename": c_basename, "docstring": "", + "docstring_line_number": None, + "line_number": self.line_number, } if not (existing_function.kind is self.kind and existing_function.coexist == self.coexist): @@ -735,7 +748,8 @@ def state_modulename_name(self, line: str) -> None: critical_section=self.critical_section, disable_fastcall=self.disable_fastcall, target_critical_section=self.target_critical_section, - forced_text_signature=self.forced_text_signature + forced_text_signature=self.forced_text_signature, + line_number=self.line_number, ) self.add_function(func) @@ -1141,7 +1155,8 @@ def bad_node(self, node: ast.AST) -> None: converter=converter, default=value, group=self.group_stack[-1] if self.group_stack else 0, group_depth=len(self.group_stack), - deprecated_positional=self.deprecated_positional) + deprecated_positional=self.deprecated_positional, + line_number=self.line_number) names = [k.name for k in self.function.parameters.values()] if parameter_name in names[1:]: @@ -1338,6 +1353,8 @@ def docstring_append(self, obj: Function | Parameter, line: str) -> None: docstring = obj.docstring if docstring: docstring += "\n" + elif isinstance(obj, Function) and line.rstrip(): + obj.docstring_line_number = self.line_number if stripped := line.rstrip(): docstring += self.indent.dedent(stripped) obj.docstring = docstring @@ -1379,184 +1396,10 @@ def state_function_docstring(self, line: str) -> None: def format_docstring_signature( f: Function, parameters: list[Parameter] ) -> str: - lines = [] - lines.append(f.displayname) - if f.forced_text_signature: - lines.append(f.forced_text_signature) - elif f.kind in ACCESSORS: - # @getter and @setter do not need signatures like a method or a function. - return '' - else: - lines.append('(') - - # populate "right_bracket_count" field for every parameter - assert parameters, "We should always have a self parameter. " + repr(f) - assert isinstance(parameters[0].converter, self_converter) - # self is always positional-only. - assert parameters[0].is_positional_only() - assert parameters[0].right_bracket_count == 0 - positional_only = True - for p in parameters[1:]: - if not p.is_positional_only(): - positional_only = False - else: - assert positional_only - if positional_only: - p.right_bracket_count = p.group_depth - else: - # don't put any right brackets around non-positional-only parameters, ever. - p.right_bracket_count = 0 - - right_bracket_count = 0 - last_group = 0 - - def fix_right_bracket_count(desired: int, group: int = 0) -> str: - nonlocal right_bracket_count, last_group - s = '' - if (group != last_group and right_bracket_count and - ((desired >= right_bracket_count) if group < 0 else - (desired <= right_bracket_count))): - # The group is not nested in the previous group, - # close the brackets of the latter first. - s += ']' * right_bracket_count - right_bracket_count = 0 - last_group = group - while right_bracket_count < desired: - s += '[' - right_bracket_count += 1 - while right_bracket_count > desired: - s += ']' - right_bracket_count -= 1 - return s - - need_slash = False - added_slash = False - need_a_trailing_slash = False - - # we only need a trailing slash: - # * if this is not a "docstring_only" signature - # * and if the last *shown* parameter is - # positional only - if not f.docstring_only: - for p in reversed(parameters): - if not p.converter.show_in_signature: - continue - if p.is_positional_only(): - need_a_trailing_slash = True - break - - - added_star = False - - first_parameter = True - last_p = parameters[-1] - line_length = len(''.join(lines)) - indent = " " * line_length - def add_parameter(text: str) -> None: - nonlocal line_length - nonlocal first_parameter - if first_parameter: - s = text - first_parameter = False - else: - s = ' ' + text - if line_length + len(s) >= 72: - lines.extend(["\n", indent]) - line_length = len(indent) - s = text - line_length += len(s) - lines.append(s) - - for p in parameters: - if not p.converter.show_in_signature: - continue - assert p.name - - is_self = isinstance(p.converter, self_converter) - if is_self and f.docstring_only: - # this isn't a real machine-parsable signature, - # so let's not print the "self" parameter - continue - - if p.is_positional_only(): - need_slash = not f.docstring_only - elif need_slash and not (added_slash or p.is_positional_only()): - added_slash = True - add_parameter('/,') - - if p.is_keyword_only() and not added_star: - added_star = True - add_parameter('*,') - - p_lines = [fix_right_bracket_count(p.right_bracket_count, - p.group)] - - if isinstance(p.converter, self_converter): - # annotate first parameter as being a "self". - # - # if inspect.Signature gets this function, - # and it's already bound, the self parameter - # will be stripped off. - # - # if it's not bound, it should be marked - # as positional-only. - # - # note: we don't print "self" for __init__, - # because this isn't actually the signature - # for __init__. (it can't be, __init__ doesn't - # have a docstring.) if this is an __init__ - # (or __new__), then this signature is for - # calling the class to construct a new instance. - p_lines.append('$') - - if p.is_vararg(): - p_lines.append("*") - added_star = True - if p.is_var_keyword(): - p_lines.append("**") - - name = p.converter.signature_name or p.name - p_lines.append(name) - - if not p.is_variable_length() and p.converter.is_optional(): - p_lines.append('=') - value = p.converter.py_default - if not value: - value = repr(p.converter.default) - p_lines.append(value) - - if (p != last_p) or need_a_trailing_slash: - p_lines.append(',') - - p_output = "".join(p_lines) - add_parameter(p_output) - - lines.append(fix_right_bracket_count(0)) - if need_a_trailing_slash: - add_parameter('/') - lines.append(')') - - # PEP 8 says: - # - # The Python standard library will not use function annotations - # as that would result in a premature commitment to a particular - # annotation style. Instead, the annotations are left for users - # to discover and experiment with useful annotation styles. - # - # therefore this is commented out: - # - # if f.return_converter.py_default: - # lines.append(' -> ') - # lines.append(f.return_converter.py_default) - - if not f.docstring_only: - lines.append("\n" + libclinic.SIG_END_MARKER + "\n") - - signature_line = "".join(lines) - - # now fix up the places where the brackets look wrong - return signature_line.replace(', ]', ',] ') - + signature = render_text_signature(f, parameters) + if signature and not f.docstring_only: + signature += "\n" + libclinic.SIG_END_MARKER + "\n" + return signature @staticmethod def format_docstring_parameters(params: list[Parameter]) -> str: """Create substitution text for {parameters}""" @@ -1581,12 +1424,19 @@ def format_docstring(self) -> str: # Guido said Clinic should enforce this: # http://mail.python.org/pipermail/python-dev/2013-June/127110.html + def docstring_line(index: int) -> int | None: + """Return the line of the file which holds the index-th line.""" + if f.docstring_line_number is None: + return None + return f.docstring_line_number + index + lines = f.docstring.split('\n') if len(lines) >= 2: if lines[1]: fail(f"Docstring for {f.full_name!r} does not have a summary line!\n" "Every non-blank function docstring must start with " - "a single line summary followed by an empty line.") + "a single line summary followed by an empty line.", + line_number=docstring_line(1)) elif len(lines) == 1: # the docstring is only one line right now--the summary line. # add an empty line after the summary line so we have space @@ -1598,28 +1448,36 @@ def format_docstring(self) -> str: # Existing violations are recorded in OVERLONG_{SUMMARY,BODY}. max_width = f.docstring_line_width summary_len = len(lines[0]) - max_body = max(map(len, lines[1:])) + long_body = [i for i, line in enumerate(lines) + if i and len(line) > max_width] if summary_len > max_width: if not self.permit_long_summary: fail(f"Summary line for {f.full_name!r} is too long!\n" - f"The summary line must be no longer than {max_width} characters.") + f"The summary line must be no longer than {max_width} characters.", + line_number=docstring_line(0)) else: if self.permit_long_summary: warn("Remove the @permit_long_summary decorator from " - f"{f.full_name!r}!\n") + f"{f.full_name!r}!\n", filename=self.clinic.filename, + line_number=f.line_number) - if max_body > max_width: + if long_body: if not self.permit_long_docstring_body: warn(f"Docstring lines for {f.full_name!r} are too long!\n" - f"Lines should be no longer than {max_width} characters.") + f"Lines should be no longer than {max_width} characters.", + filename=self.clinic.filename, + line_number=docstring_line(long_body[0])) else: if self.permit_long_docstring_body: warn("Remove the @permit_long_docstring_body decorator from " - f"{f.full_name!r}!\n") + f"{f.full_name!r}!\n", filename=self.clinic.filename, + line_number=f.line_number) + markers = [i for i, line in enumerate(lines) if '{parameters}' in line] parameters_marker_count = len(f.docstring.split('{parameters}')) - 1 if parameters_marker_count > 1: - fail('You may not specify {parameters} more than once in a docstring!') + fail('You may not specify {parameters} more than once in a docstring!', + line_number=docstring_line(markers[-1])) # insert signature at front and params after the summary line if not parameters_marker_count: @@ -1679,6 +1537,195 @@ def do_post_block_processing_cleanup(self, lineno: int) -> None: try: self.function.docstring = self.format_docstring() except ClinicError as exc: - exc.lineno = lineno + if exc.lineno is None: + exc.lineno = lineno exc.filename = self.clinic.filename raise + + +def render_text_signature( + f: Function, + parameters: list[Parameter], + *, + name: str | None = None, + line_width: int | None = 72, +) -> str: + """Render the text signature of the function. + + *name* replaces the name of the function. *line_width* is the width + at which the signature is wrapped, None disables wrapping. + """ + lines = [] + lines.append(f.displayname if name is None else name) + if f.forced_text_signature: + lines.append(f.forced_text_signature) + elif f.kind in ACCESSORS: + # @getter and @setter do not need signatures like a method or a function. + return '' + else: + lines.append('(') + + # populate "right_bracket_count" field for every parameter + assert parameters, "We should always have a self parameter. " + repr(f) + assert isinstance(parameters[0].converter, self_converter) + # self is always positional-only. + assert parameters[0].is_positional_only() + assert parameters[0].right_bracket_count == 0 + positional_only = True + for p in parameters[1:]: + if not p.is_positional_only(): + positional_only = False + else: + assert positional_only + if positional_only: + p.right_bracket_count = p.group_depth + else: + # don't put any right brackets around non-positional-only parameters, ever. + p.right_bracket_count = 0 + + right_bracket_count = 0 + last_group = 0 + + def fix_right_bracket_count(desired: int, group: int = 0) -> str: + nonlocal right_bracket_count, last_group + s = '' + if (group != last_group and right_bracket_count and + ((desired >= right_bracket_count) if group < 0 else + (desired <= right_bracket_count))): + # The group is not nested in the previous group, + # close the brackets of the latter first. + s += ']' * right_bracket_count + right_bracket_count = 0 + last_group = group + while right_bracket_count < desired: + s += '[' + right_bracket_count += 1 + while right_bracket_count > desired: + s += ']' + right_bracket_count -= 1 + return s + + need_slash = False + added_slash = False + need_a_trailing_slash = False + + # we only need a trailing slash: + # * if this is not a "docstring_only" signature + # * and if the last *shown* parameter is + # positional only + if not f.docstring_only: + for p in reversed(parameters): + if not p.converter.show_in_signature: + continue + if p.is_positional_only(): + need_a_trailing_slash = True + break + + + added_star = False + + first_parameter = True + last_p = parameters[-1] + line_length = len(''.join(lines)) + indent = " " * line_length + def add_parameter(text: str) -> None: + nonlocal line_length + nonlocal first_parameter + if first_parameter: + s = text + first_parameter = False + else: + s = ' ' + text + if line_width is not None and line_length + len(s) >= line_width: + lines.extend(["\n", indent]) + line_length = len(indent) + s = text + line_length += len(s) + lines.append(s) + + for p in parameters: + if not p.converter.show_in_signature: + continue + assert p.name + + is_self = isinstance(p.converter, self_converter) + if is_self and f.docstring_only: + # this isn't a real machine-parsable signature, + # so let's not print the "self" parameter + continue + + if p.is_positional_only(): + need_slash = not f.docstring_only + elif need_slash and not (added_slash or p.is_positional_only()): + added_slash = True + add_parameter('/,') + + if p.is_keyword_only() and not added_star: + added_star = True + add_parameter('*,') + + p_lines = [fix_right_bracket_count(p.right_bracket_count, + p.group)] + + if isinstance(p.converter, self_converter): + # annotate first parameter as being a "self". + # + # if inspect.Signature gets this function, + # and it's already bound, the self parameter + # will be stripped off. + # + # if it's not bound, it should be marked + # as positional-only. + # + # note: we don't print "self" for __init__, + # because this isn't actually the signature + # for __init__. (it can't be, __init__ doesn't + # have a docstring.) if this is an __init__ + # (or __new__), then this signature is for + # calling the class to construct a new instance. + p_lines.append('$') + + if p.is_vararg(): + p_lines.append("*") + added_star = True + if p.is_var_keyword(): + p_lines.append("**") + + name = p.converter.signature_name or p.name + p_lines.append(name) + + if not p.is_variable_length() and p.converter.is_optional(): + p_lines.append('=') + value = p.converter.py_default + if not value: + value = repr(p.converter.default) + p_lines.append(value) + + if (p != last_p) or need_a_trailing_slash: + p_lines.append(',') + + p_output = "".join(p_lines) + add_parameter(p_output) + + lines.append(fix_right_bracket_count(0)) + if need_a_trailing_slash: + add_parameter('/') + lines.append(')') + + # PEP 8 says: + # + # The Python standard library will not use function annotations + # as that would result in a premature commitment to a particular + # annotation style. Instead, the annotations are left for users + # to discover and experiment with useful annotation styles. + # + # therefore this is commented out: + # + # if f.return_converter.py_default: + # lines.append(' -> ') + # lines.append(f.return_converter.py_default) + + signature_line = "".join(lines) + + # now fix up the places where the brackets look wrong + return signature_line.replace(', ]', ',] ') diff --git a/Tools/clinic/libclinic/function.py b/Tools/clinic/libclinic/function.py index cad673045d1c26d..af858f2e40e75ec 100644 --- a/Tools/clinic/libclinic/function.py +++ b/Tools/clinic/libclinic/function.py @@ -118,6 +118,10 @@ class Function: critical_section: bool = False disable_fastcall: bool = False target_critical_section: list[str] = dc.field(default_factory=list) + # Line of the file on which the function is declared. + line_number: int | None = None + # Line on which the docstring starts (`None` if there is no docstring). + docstring_line_number: int | None = None def __post_init__(self) -> None: self.parent = self.cls or self.module @@ -220,6 +224,8 @@ class Parameter: # (`None` signifies that there is no deprecation) deprecated_positional: VersionTuple | None = None deprecated_keyword: VersionTuple | None = None + # Line of the file on which the parameter is declared. + line_number: int | None = None right_bracket_count: int = dc.field(init=False, default=0) def __repr__(self) -> str: @@ -276,6 +282,36 @@ def render_docstring(self) -> str: ParamTuple = tuple["Parameter", ...] +Definition = Module | Class | Function + + +def walk_definitions( + parent: Clinic | Module | Class, + prefix: str = '', + depth: int = 0, +) -> Iterator[tuple[int, str, Definition]]: + """Yield (depth, dotted name, definition) for every nested definition. + + The name of a module is already fully qualified, but the name of + a class is not, hence the prefix. + """ + for function in parent.functions: + if function.kind.new_or_init: + # __new__() and __init__() are called as the class itself. + name = prefix + else: + name = f'{prefix}.{function.name}' if prefix else function.name + yield depth, name, function + for cls in parent.classes.values(): + name = f'{prefix}.{cls.name}' if prefix else cls.name + yield depth, name, cls + yield from walk_definitions(cls, name, depth + 1) + if not isinstance(parent, Class): + # Only a module can contain modules. + for module in parent.modules.values(): + yield depth, module.name, module + yield from walk_definitions(module, module.name, depth + 1) + def permute_left_option_groups( l: Sequence[Iterable[Parameter]] diff --git a/Tools/clinic/libclinic/parse_args.py b/Tools/clinic/libclinic/parse_args.py index b08b949028205d2..37d7cb7ffabe51a 100644 --- a/Tools/clinic/libclinic/parse_args.py +++ b/Tools/clinic/libclinic/parse_args.py @@ -346,7 +346,8 @@ def select_prototypes(self) -> None: self.docstring_definition = GETSET_DOCSTRING_PROTOTYPE_STRVAR elif self.func.kind in SETTERS: if self.func.docstring: - fail("docstrings are only supported for @getter, not @setter") + fail("docstrings are only supported for @getter, not @setter", + line_number=self.func.line_number) self.return_value_declaration = "int {parser_retval};" self.methoddef_define = SETTERDEF_PROTOTYPE_DEFINE else: