Coverage for src/mafw/devtools/dependencies/compile.py: 92%
116 statements
« prev ^ index » next coverage.py v7.15.0, created at 2026-09-01 19:49 +0000
1# Copyright 2026 European Union
2# Author: Bulgheroni Antonio (antonio.bulgheroni@ec.europa.eu)
3# SPDX-License-Identifier: EUPL-1.2
4"""
5Dependency compilation utilities for MAFw.
7This module provides functions for compiling dependency lockfiles using ``uv``,
8reading resolved dependency versions, and managing Python version metadata
9from the project configuration.
10"""
12from __future__ import annotations
14import re
15import tempfile
16import tomllib
17from pathlib import Path
18from typing import Any, Final
20import tomlkit
22from mafw.devtools import ensure_devtools_available
23from mafw.devtools.gitlab import pylock_version_tag
25ensure_devtools_available()
27from packaging.version import Version # noqa: E402
28from tomlkit.exceptions import TOMLKitError # noqa: E402
30from mafw.devtools import DevtoolsError # noqa: E402
31from mafw.tools.shell_tools import run as cmd # noqa: E402
33PYPROJECT_FILE: Final[Path] = Path('pyproject.toml')
34"""Path to the TOML file containing the project dependencies."""
36DEFAULT_FREEZE_EXTRAS: Final[tuple[str, ...]] = ('seaborn', 'all-db', 'steering-gui')
37"""Extras used when compiling dependency lockfiles for release freezing and compatibility checks."""
40def load_pylock_packages(pylock_path: Path) -> dict[str, dict[str, Any]]:
41 """Parse a pylock TOML file into a dictionary keyed by lowercase package name.
43 Each value in the returned dictionary contains at minimum the ``name``,
44 ``version``, and optionally ``marker`` fields from the original TOML entry.
46 :param pylock_path: Path to the pylock TOML file.
47 :type pylock_path: Path
48 :return: Dictionary mapping lowercase package names to their package metadata.
49 :rtype: dict[str, dict[str, Any]]
50 :raises FileNotFoundError: If *pylock_path* does not exist.
51 :raises tomllib.TOMLDecodeError: If the file is not valid TOML.
52 """
53 with open(pylock_path, 'rb') as f:
54 data = tomllib.load(f)
56 packages: dict[str, dict[str, Any]] = {}
57 for pkg in data.get('packages', []):
58 # Use lowercase name as the key for case-insensitive matching.
59 name = pkg.get('name', '')
60 packages[name.lower()] = dict(pkg)
61 return packages
64def parse_python_version(version: str) -> tuple[int, int]:
65 """
66 Parse a Python version string in ``major.minor`` form.
68 :param version: Python version string.
69 :type version: str
70 :return: Major/minor version tuple.
71 :rtype: tuple[int, int]
72 :raises DevtoolsError: If the version is invalid.
73 """
74 match = re.fullmatch(r'(\d+)\.(\d+)', version.strip())
75 if match is None: 75 ↛ 76line 75 didn't jump to line 76 because the condition on line 75 was never true
76 raise DevtoolsError(f'Invalid Python version "{version}". Expected format like 3.11.')
77 return int(match.group(1)), int(match.group(2))
80def python_versions_between(
81 min_python_ver: str,
82 max_python_ver: str,
83 supported_versions: list[str],
84) -> list[str]:
85 """
86 Build the inclusive list of Python versions between two bounds.
88 The returned versions must also be present in ``supported_versions``.
90 :param min_python_ver: Minimum Python version.
91 :type min_python_ver: str
92 :param max_python_ver: Maximum Python version.
93 :type max_python_ver: str
94 :param supported_versions: List of supported Python versions from project metadata.
95 :type supported_versions: list[str]
96 :return: Ordered list of version strings.
97 :rtype: list[str]
98 """
99 min_major, min_minor = parse_python_version(min_python_ver)
100 max_major, max_minor = parse_python_version(max_python_ver)
101 if (min_major, min_minor) > (max_major, max_minor): 101 ↛ 102line 101 didn't jump to line 102 because the condition on line 101 was never true
102 raise DevtoolsError('--min-python-ver must be less than or equal to --max-python-ver.')
103 if min_major != 3 or max_major != 3: 103 ↛ 104line 103 didn't jump to line 104 because the condition on line 103 was never true
104 raise DevtoolsError('Python dependency verification currently supports only Python 3.x versions.')
106 versions = [f'{min_major}.{minor}' for minor in range(min_minor, max_minor + 1)]
107 supported_set = set(supported_versions)
108 if min_python_ver not in supported_set: 108 ↛ 109line 108 didn't jump to line 109 because the condition on line 108 was never true
109 raise DevtoolsError(f'--min-python-ver {min_python_ver} is not listed in tool.mafw.supported-python.')
110 if max_python_ver not in supported_set:
111 raise DevtoolsError(f'--max-python-ver {max_python_ver} is not listed in tool.mafw.supported-python.')
112 missing = [version for version in versions if version not in supported_set]
113 if missing:
114 raise DevtoolsError(
115 'Requested Python version range is not fully listed in tool.mafw.supported-python: ' + ', '.join(missing)
116 )
117 return versions
120def ensure_mafw_project_root() -> list[str]:
121 """
122 Ensure the current working directory is the MAFw project root.
124 :return: Validated supported Python versions from ``tool.mafw.supported-python``.
125 :rtype: list[str]
126 :raises DevtoolsError: If ``pyproject.toml`` is missing or does not identify MAFw.
127 """
128 if not PYPROJECT_FILE.exists():
129 raise DevtoolsError(f'Unable to find {PYPROJECT_FILE}. Run the command from the MAFw project root.')
130 pyproject_content = PYPROJECT_FILE.read_text(encoding='utf-8')
131 try:
132 doc = tomlkit.loads(pyproject_content)
133 except TOMLKitError as exc:
134 raise DevtoolsError(f'Unable to parse {PYPROJECT_FILE} as TOML.') from exc
136 project = doc.get('project')
137 if project is None or project.get('name') != 'mafw':
138 raise DevtoolsError(f'{PYPROJECT_FILE} does not describe the MAFw project.')
140 return project_python_versions()
143def project_python_versions_from_doc(doc: tomlkit.TOMLDocument) -> list[str]:
144 """
145 Extract supported CPython versions from a parsed ``pyproject.toml`` document.
147 :param doc: Parsed TOML document.
148 :type doc: tomlkit.TOMLDocument
149 :return: Sorted list of supported CPython versions.
150 :rtype: list[str]
151 :raises DevtoolsError: If the ``tool.mafw.supported-python`` field is invalid.
152 """
153 tool = doc.get('tool')
154 mafw = tool.get('mafw') if tool is not None else None
155 if mafw is None:
156 raise DevtoolsError(f'Missing [tool.mafw] table in {PYPROJECT_FILE}.')
158 supported_python = mafw.get('supported-python')
159 if not isinstance(supported_python, list) or not supported_python:
160 raise DevtoolsError(f'Missing tool.mafw.supported-python in {PYPROJECT_FILE}.')
162 validated: list[tuple[int, int, str]] = []
163 for item in supported_python:
164 if not isinstance(item, str):
165 raise DevtoolsError('tool.mafw.supported-python must contain only strings.')
166 match = re.fullmatch(r'(\d+)\.(\d+)', item.strip())
167 if match is None:
168 raise DevtoolsError(
169 f'Invalid Python version in tool.mafw.supported-python: {item}. Expected major.minor, e.g. 3.14.'
170 )
171 major = int(match.group(1))
172 minor = int(match.group(2))
173 if major != 3:
174 raise DevtoolsError(
175 f'Unsupported Python version in tool.mafw.supported-python: {item}. Only Python 3.x is supported.'
176 )
177 validated.append((major, minor, f'{major}.{minor}'))
179 validated.sort()
180 return [item[2] for item in dict.fromkeys(validated)]
183def project_python_versions() -> list[str]:
184 """
185 Read the supported CPython versions from ``tool.mafw.supported-python`` list in pyproject.toml.
187 The field is expected to be a list of strings representing CPython major.minor
188 versions. The helper validates each entry and returns a sorted, de-duplicated
189 list so downstream callers have deterministic ordering.
191 :return: Sorted list of supported CPython versions.
192 :rtype: list[str]
193 :raises DevtoolsError: If ``pyproject.toml`` cannot be parsed or the
194 field contains unsupported values.
195 """
196 if not PYPROJECT_FILE.exists():
197 raise DevtoolsError(f'Unable to find {PYPROJECT_FILE}.')
199 doc = load_pyproject_doc(PYPROJECT_FILE.read_text(encoding='utf-8'))
200 return project_python_versions_from_doc(doc)
203def compile_python_selector(python_version: str) -> str:
204 """
205 Build the Python selector used by ``uv`` for dependency compilation.
207 For Python 3.14 and newer, request the GIL-enabled variant explicitly so
208 free-threaded interpreters do not leak into dependency resolution.
210 This distinction is still needed because of the psycopg
212 :param python_version: Base Python version in ``major.minor`` form.
213 :type python_version: str
214 :return: Python selector string passed to ``uv``.
215 :rtype: str
216 """
217 major, minor = parse_python_version(python_version)
218 if (major, minor) >= (3, 14):
219 return f'{python_version}+gil'
220 return python_version
223def compile_dependency_lockfile( # pragma: no cover
224 python_version: str,
225 pylock_file: Path,
226 extras: list[str],
227 resolution: str | None = None,
228 output_format: str | None = None,
229 with_hashes: bool = False,
230) -> None:
231 """
232 Compile a dependency lockfile for a specific CPython version.
234 :param python_version: CPython version used for the ``uv pip compile`` run.
235 :type python_version: str
236 :param pylock_file: Output path for the generated lockfile.
237 :type pylock_file: Path
238 :param extras: Project extras requested during compilation.
239 :type extras: list[str]
240 :param resolution: Optional UV resolution strategy (e.g. 'lowest-direct', 'highest').
241 :type resolution: str | None
242 :param output_format: Optional output format (e.g. 'requirements.txt').
243 :type output_format: str | None
244 :param with_hashes: Whether to generate hashes for the compiled requirements.
245 :type with_hashes: bool
246 """
247 cmd_parts = [
248 'uv',
249 'pip',
250 'compile',
251 'pyproject.toml',
252 '--python',
253 compile_python_selector(python_version),
254 '--no-annotate',
255 '-o',
256 str(pylock_file),
257 '-q',
258 ]
259 if resolution is not None:
260 cmd_parts.extend(['--resolution', resolution])
261 if output_format is not None:
262 cmd_parts.extend(['--format', output_format])
263 if with_hashes:
264 cmd_parts.append('--generate-hashes')
265 for extra in extras:
266 cmd_parts.extend(['--extra', extra])
267 cmd(cmd_parts)
270def read_compiled_dependency_versions(pylock_text: str) -> dict[str, Version]:
271 """
272 Read resolved dependency versions from a compiled lockfile payload.
274 The returned mapping stores the highest resolved version seen for each
275 dependency name, normalized to lowercase for stable lookups.
277 :param pylock_text: Raw ``pylock.py3_14.toml`` content.
278 :type pylock_text: str
279 :return: Mapping of package name to highest resolved version.
280 :rtype: dict[str, Version]
281 :raises DevtoolsError: If the TOML payload cannot be parsed.
282 """
283 try:
284 doc = tomlkit.loads(pylock_text)
285 except TOMLKitError as exc:
286 raise DevtoolsError('Unable to parse compiled dependency lockfile as TOML.') from exc
288 resolved: dict[str, Version] = {}
289 for item in doc.get('packages', []):
290 if not isinstance(item, dict): 290 ↛ 291line 290 didn't jump to line 291 because the condition on line 290 was never true
291 continue
292 name = item.get('name')
293 version_text = item.get('version')
294 if not isinstance(name, str) or not isinstance(version_text, str): 294 ↛ 295line 294 didn't jump to line 295 because the condition on line 294 was never true
295 continue
296 try:
297 version = Version(version_text)
298 except Exception as exc: # pragma: no cover
299 raise DevtoolsError(
300 f'Unable to parse resolved dependency version "{version_text}" for package "{name}".'
301 ) from exc
302 key = name.lower()
303 if key not in resolved or version > resolved[key]: 303 ↛ 289line 303 didn't jump to line 289 because the condition on line 303 was always true
304 resolved[key] = version
305 return resolved
308def load_pyproject_doc(toml_text: str) -> tomlkit.TOMLDocument:
309 """
310 Parse a ``pyproject.toml`` payload once and return the TOML document.
312 :param toml_text: Raw TOML payload.
313 :type toml_text: str
314 :return: Parsed TOML document.
315 :rtype: tomlkit.TOMLDocument
316 :raises DevtoolsError: If the TOML payload cannot be parsed.
317 """
318 try:
319 return tomlkit.loads(toml_text)
320 except TOMLKitError as exc:
321 raise DevtoolsError(f'Unable to parse {PYPROJECT_FILE} as TOML.') from exc
324def collect_compiled_dependency_versions(python_versions: list[str]) -> dict[str, Version]: # pragma: no cover
325 """
326 Compile dependency lockfiles and collect the resolved versions they report.
328 The command mirrors the dependency verification workflow by invoking
329 ``uv pip compile`` for each supported CPython version. A temporary lockfile
330 is generated for each version and removed afterwards.
332 :param python_versions: Supported Python versions to compile.
333 :type python_versions: list[str]
334 :return: Mapping of package name to highest resolved version across the compiled lockfiles.
335 :rtype: dict[str, Version]
336 """
337 resolved: dict[str, Version] = {}
338 if not python_versions:
339 raise DevtoolsError('Unable to determine supported Python versions for dependency freezing.')
341 with tempfile.TemporaryDirectory() as tmpdir:
342 tmpdir_path = Path(tmpdir)
343 for python_version in python_versions:
344 # Use underscore instead of dot in the filename to comply with uv's
345 # lockfile naming constraint: ^pylock\.([^.]+\.)?toml$
346 version_tag = pylock_version_tag(python_version)
347 pylock_file = tmpdir_path / f'pylock.py{version_tag}.toml'
348 compile_dependency_lockfile(python_version, pylock_file, list(DEFAULT_FREEZE_EXTRAS))
349 compiled_versions = read_compiled_dependency_versions(pylock_file.read_text(encoding='utf-8'))
350 for name, version in compiled_versions.items():
351 if name not in resolved or version > resolved[name]:
352 resolved[name] = version
353 return resolved