Coverage for src / mafw / devtools / dependencies / compile.py: 92%
115 statements
« prev ^ index » next coverage.py v7.14.0, created at 2026-06-28 13:34 +0000
1# Copyright 2026 European Union
2# Author: Bulgheroni Antonio (antonio.bulgheroni@ec.europa.eu)
3# SPDX-License-Identifier: EUPL-1.2
4"""
5Dependency compilation utilities for MAFw.
7This module provides functions for compiling dependency lockfiles using ``uv``,
8reading resolved dependency versions, and managing Python version metadata
9from the project configuration.
10"""
12from __future__ import annotations
14import re
15import tempfile
16import tomllib
17from pathlib import Path
18from typing import Any, Final
20import tomlkit
22from mafw.devtools import ensure_devtools_available
24ensure_devtools_available()
26from packaging.version import Version # noqa: E402
27from tomlkit.exceptions import TOMLKitError # noqa: E402
29from mafw.devtools import DevtoolsError # noqa: E402
30from mafw.tools.shell_tools import run as cmd # noqa: E402
32PYPROJECT_FILE: Final[Path] = Path('pyproject.toml')
33"""Path to the TOML file containing the project dependencies."""
35DEFAULT_FREEZE_EXTRAS: Final[tuple[str, ...]] = ('seaborn', 'all-db', 'steering-gui')
36"""Extras used when compiling dependency lockfiles for release freezing and compatibility checks."""
39def load_pylock_packages(pylock_path: Path) -> dict[str, dict[str, Any]]:
40 """Parse a pylock TOML file into a dictionary keyed by lowercase package name.
42 Each value in the returned dictionary contains at minimum the ``name``,
43 ``version``, and optionally ``marker`` fields from the original TOML entry.
45 :param pylock_path: Path to the pylock TOML file.
46 :type pylock_path: Path
47 :return: Dictionary mapping lowercase package names to their package metadata.
48 :rtype: dict[str, dict[str, Any]]
49 :raises FileNotFoundError: If *pylock_path* does not exist.
50 :raises tomllib.TOMLDecodeError: If the file is not valid TOML.
51 """
52 with open(pylock_path, 'rb') as f:
53 data = tomllib.load(f)
55 packages: dict[str, dict[str, Any]] = {}
56 for pkg in data.get('packages', []):
57 # Use lowercase name as the key for case-insensitive matching.
58 name = pkg.get('name', '')
59 packages[name.lower()] = dict(pkg)
60 return packages
63def parse_python_version(version: str) -> tuple[int, int]:
64 """
65 Parse a Python version string in ``major.minor`` form.
67 :param version: Python version string.
68 :type version: str
69 :return: Major/minor version tuple.
70 :rtype: tuple[int, int]
71 :raises DevtoolsError: If the version is invalid.
72 """
73 match = re.fullmatch(r'(\d+)\.(\d+)', version.strip())
74 if match is None: 74 ↛ 75line 74 didn't jump to line 75 because the condition on line 74 was never true
75 raise DevtoolsError(f'Invalid Python version "{version}". Expected format like 3.11.')
76 return int(match.group(1)), int(match.group(2))
79def python_versions_between(
80 min_python_ver: str,
81 max_python_ver: str,
82 supported_versions: list[str],
83) -> list[str]:
84 """
85 Build the inclusive list of Python versions between two bounds.
87 The returned versions must also be present in ``supported_versions``.
89 :param min_python_ver: Minimum Python version.
90 :type min_python_ver: str
91 :param max_python_ver: Maximum Python version.
92 :type max_python_ver: str
93 :param supported_versions: List of supported Python versions from project metadata.
94 :type supported_versions: list[str]
95 :return: Ordered list of version strings.
96 :rtype: list[str]
97 """
98 min_major, min_minor = parse_python_version(min_python_ver)
99 max_major, max_minor = parse_python_version(max_python_ver)
100 if (min_major, min_minor) > (max_major, max_minor): 100 ↛ 101line 100 didn't jump to line 101 because the condition on line 100 was never true
101 raise DevtoolsError('--min-python-ver must be less than or equal to --max-python-ver.')
102 if min_major != 3 or max_major != 3: 102 ↛ 103line 102 didn't jump to line 103 because the condition on line 102 was never true
103 raise DevtoolsError('Python dependency verification currently supports only Python 3.x versions.')
105 versions = [f'{min_major}.{minor}' for minor in range(min_minor, max_minor + 1)]
106 supported_set = set(supported_versions)
107 if min_python_ver not in supported_set: 107 ↛ 108line 107 didn't jump to line 108 because the condition on line 107 was never true
108 raise DevtoolsError(f'--min-python-ver {min_python_ver} is not listed in tool.mafw.supported-python.')
109 if max_python_ver not in supported_set:
110 raise DevtoolsError(f'--max-python-ver {max_python_ver} is not listed in tool.mafw.supported-python.')
111 missing = [version for version in versions if version not in supported_set]
112 if missing:
113 raise DevtoolsError(
114 'Requested Python version range is not fully listed in tool.mafw.supported-python: ' + ', '.join(missing)
115 )
116 return versions
119def ensure_mafw_project_root() -> list[str]:
120 """
121 Ensure the current working directory is the MAFw project root.
123 :return: Validated supported Python versions from ``tool.mafw.supported-python``.
124 :rtype: list[str]
125 :raises DevtoolsError: If ``pyproject.toml`` is missing or does not identify MAFw.
126 """
127 if not PYPROJECT_FILE.exists():
128 raise DevtoolsError(f'Unable to find {PYPROJECT_FILE}. Run the command from the MAFw project root.')
129 pyproject_content = PYPROJECT_FILE.read_text(encoding='utf-8')
130 try:
131 doc = tomlkit.loads(pyproject_content)
132 except TOMLKitError as exc:
133 raise DevtoolsError(f'Unable to parse {PYPROJECT_FILE} as TOML.') from exc
135 project = doc.get('project')
136 if project is None or project.get('name') != 'mafw':
137 raise DevtoolsError(f'{PYPROJECT_FILE} does not describe the MAFw project.')
139 return project_python_versions()
142def project_python_versions_from_doc(doc: tomlkit.TOMLDocument) -> list[str]:
143 """
144 Extract supported CPython versions from a parsed ``pyproject.toml`` document.
146 :param doc: Parsed TOML document.
147 :type doc: tomlkit.TOMLDocument
148 :return: Sorted list of supported CPython versions.
149 :rtype: list[str]
150 :raises DevtoolsError: If the ``tool.mafw.supported-python`` field is invalid.
151 """
152 tool = doc.get('tool')
153 mafw = tool.get('mafw') if tool is not None else None
154 if mafw is None:
155 raise DevtoolsError(f'Missing [tool.mafw] table in {PYPROJECT_FILE}.')
157 supported_python = mafw.get('supported-python')
158 if not isinstance(supported_python, list) or not supported_python:
159 raise DevtoolsError(f'Missing tool.mafw.supported-python in {PYPROJECT_FILE}.')
161 validated: list[tuple[int, int, str]] = []
162 for item in supported_python:
163 if not isinstance(item, str):
164 raise DevtoolsError('tool.mafw.supported-python must contain only strings.')
165 match = re.fullmatch(r'(\d+)\.(\d+)', item.strip())
166 if match is None:
167 raise DevtoolsError(
168 f'Invalid Python version in tool.mafw.supported-python: {item}. Expected major.minor, e.g. 3.14.'
169 )
170 major = int(match.group(1))
171 minor = int(match.group(2))
172 if major != 3:
173 raise DevtoolsError(
174 f'Unsupported Python version in tool.mafw.supported-python: {item}. Only Python 3.x is supported.'
175 )
176 validated.append((major, minor, f'{major}.{minor}'))
178 validated.sort()
179 return [item[2] for item in dict.fromkeys(validated)]
182def project_python_versions() -> list[str]:
183 """
184 Read the supported CPython versions from ``tool.mafw.supported-python`` list in pyproject.toml.
186 The field is expected to be a list of strings representing CPython major.minor
187 versions. The helper validates each entry and returns a sorted, de-duplicated
188 list so downstream callers have deterministic ordering.
190 :return: Sorted list of supported CPython versions.
191 :rtype: list[str]
192 :raises DevtoolsError: If ``pyproject.toml`` cannot be parsed or the
193 field contains unsupported values.
194 """
195 if not PYPROJECT_FILE.exists():
196 raise DevtoolsError(f'Unable to find {PYPROJECT_FILE}.')
198 doc = load_pyproject_doc(PYPROJECT_FILE.read_text(encoding='utf-8'))
199 return project_python_versions_from_doc(doc)
202def compile_python_selector(python_version: str) -> str:
203 """
204 Build the Python selector used by ``uv`` for dependency compilation.
206 For Python 3.14 and newer, request the GIL-enabled variant explicitly so
207 free-threaded interpreters do not leak into dependency resolution.
209 This distinction is still needed because of the psycopg
211 :param python_version: Base Python version in ``major.minor`` form.
212 :type python_version: str
213 :return: Python selector string passed to ``uv``.
214 :rtype: str
215 """
216 major, minor = parse_python_version(python_version)
217 if (major, minor) >= (3, 14):
218 return f'{python_version}+gil'
219 return python_version
222def compile_dependency_lockfile( # pragma: no cover
223 python_version: str,
224 pylock_file: Path,
225 extras: list[str],
226 resolution: str | None = None,
227 output_format: str | None = None,
228 with_hashes: bool = False,
229) -> None:
230 """
231 Compile a dependency lockfile for a specific CPython version.
233 :param python_version: CPython version used for the ``uv pip compile`` run.
234 :type python_version: str
235 :param pylock_file: Output path for the generated lockfile.
236 :type pylock_file: Path
237 :param extras: Project extras requested during compilation.
238 :type extras: list[str]
239 :param resolution: Optional UV resolution strategy (e.g. 'lowest-direct', 'highest').
240 :type resolution: str | None
241 :param output_format: Optional output format (e.g. 'requirements.txt').
242 :type output_format: str | None
243 :param with_hashes: Whether to generate hashes for the compiled requirements.
244 :type with_hashes: bool
245 """
246 cmd_parts = [
247 'uv',
248 'pip',
249 'compile',
250 'pyproject.toml',
251 '--python',
252 compile_python_selector(python_version),
253 '--no-annotate',
254 '-o',
255 str(pylock_file),
256 '-q',
257 ]
258 if resolution is not None:
259 cmd_parts.extend(['--resolution', resolution])
260 if output_format is not None:
261 cmd_parts.extend(['--format', output_format])
262 if with_hashes:
263 cmd_parts.append('--generate-hashes')
264 for extra in extras:
265 cmd_parts.extend(['--extra', extra])
266 cmd(cmd_parts)
269def read_compiled_dependency_versions(pylock_text: str) -> dict[str, Version]:
270 """
271 Read resolved dependency versions from a compiled lockfile payload.
273 The returned mapping stores the highest resolved version seen for each
274 dependency name, normalized to lowercase for stable lookups.
276 :param pylock_text: Raw ``pylock.pyX.Y.toml`` content.
277 :type pylock_text: str
278 :return: Mapping of package name to highest resolved version.
279 :rtype: dict[str, Version]
280 :raises DevtoolsError: If the TOML payload cannot be parsed.
281 """
282 try:
283 doc = tomlkit.loads(pylock_text)
284 except TOMLKitError as exc:
285 raise DevtoolsError('Unable to parse compiled dependency lockfile as TOML.') from exc
287 resolved: dict[str, Version] = {}
288 for item in doc.get('packages', []):
289 if not isinstance(item, dict): 289 ↛ 290line 289 didn't jump to line 290 because the condition on line 289 was never true
290 continue
291 name = item.get('name')
292 version_text = item.get('version')
293 if not isinstance(name, str) or not isinstance(version_text, str): 293 ↛ 294line 293 didn't jump to line 294 because the condition on line 293 was never true
294 continue
295 try:
296 version = Version(version_text)
297 except Exception as exc: # pragma: no cover
298 raise DevtoolsError(
299 f'Unable to parse resolved dependency version "{version_text}" for package "{name}".'
300 ) from exc
301 key = name.lower()
302 if key not in resolved or version > resolved[key]: 302 ↛ 288line 302 didn't jump to line 288 because the condition on line 302 was always true
303 resolved[key] = version
304 return resolved
307def load_pyproject_doc(toml_text: str) -> tomlkit.TOMLDocument:
308 """
309 Parse a ``pyproject.toml`` payload once and return the TOML document.
311 :param toml_text: Raw TOML payload.
312 :type toml_text: str
313 :return: Parsed TOML document.
314 :rtype: tomlkit.TOMLDocument
315 :raises DevtoolsError: If the TOML payload cannot be parsed.
316 """
317 try:
318 return tomlkit.loads(toml_text)
319 except TOMLKitError as exc:
320 raise DevtoolsError(f'Unable to parse {PYPROJECT_FILE} as TOML.') from exc
323def collect_compiled_dependency_versions(python_versions: list[str]) -> dict[str, Version]: # pragma: no cover
324 """
325 Compile dependency lockfiles and collect the resolved versions they report.
327 The command mirrors the dependency verification workflow by invoking
328 ``uv pip compile`` for each supported CPython version. A temporary lockfile
329 is generated for each version and removed afterwards.
331 :param python_versions: Supported Python versions to compile.
332 :type python_versions: list[str]
333 :return: Mapping of package name to highest resolved version across the compiled lockfiles.
334 :rtype: dict[str, Version]
335 """
336 resolved: dict[str, Version] = {}
337 if not python_versions:
338 raise DevtoolsError('Unable to determine supported Python versions for dependency freezing.')
340 with tempfile.TemporaryDirectory() as tmpdir:
341 tmpdir_path = Path(tmpdir)
342 for python_version in python_versions:
343 pylock_file = tmpdir_path / f'pylock.py{python_version}.toml'
344 compile_dependency_lockfile(python_version, pylock_file, list(DEFAULT_FREEZE_EXTRAS))
345 compiled_versions = read_compiled_dependency_versions(pylock_file.read_text(encoding='utf-8'))
346 for name, version in compiled_versions.items():
347 if name not in resolved or version > resolved[name]:
348 resolved[name] = version
349 return resolved