Coverage for src/mafw/devtools/cli/dependencies/latest.py: 96%
169 statements
« prev ^ index » next coverage.py v7.15.0, created at 2026-09-01 19:49 +0000
1# Copyright 2026 European Union
2# Author: Bulgheroni Antonio (antonio.bulgheroni@ec.europa.eu)
3# SPDX-License-Identifier: EUPL-1.2
4"""Rolling compatibility commands for the latest versions of MAFw dependencies."""
6from __future__ import annotations
8import subprocess
9import tomllib
10from pathlib import Path
11from typing import Final
13import click
14import tomlkit
16from mafw.__about__ import __version__ as MAFW_VERSION
17from mafw.devtools.dependencies.compare import (
18 VersionComparisonResult,
19 compare_packages,
20 render_comparison_json,
21 render_comparison_markdown,
22 render_comparison_rich,
23 resolve_output_format,
24)
25from mafw.devtools.dependencies.compile import (
26 DEFAULT_FREEZE_EXTRAS,
27 compile_dependency_lockfile,
28 compile_python_selector,
29 ensure_mafw_project_root,
30 load_pylock_packages,
31 python_versions_between,
32)
33from mafw.devtools.gitlab import (
34 GitlabAPIConfiguration,
35 build_gitlab_api_configuration,
36 build_pylock_compiled_filename,
37 build_pylock_reference_filename,
38 download_generic_file,
39 normalize_mafw_version,
40 upload_generic_file,
41)
42from mafw.tools.click_extensions import AbbreviateGroup
43from mafw.tools.shell_tools import CONSOLE
44from mafw.tools.shell_tools import run as cmd
46DEPS_REGISTRY_PACKAGE_NAME: Final[str] = 'mafw-deps'
47"""Generic package name used for dependency reference uploads."""
49DEFAULT_TEST_FILES: Final[list[str]] = [
50 'tests/test_full_integration.py',
51]
52"""List of test files executed by default during dependency verification."""
55def _normalize_mafw_version_option(value: str | None) -> str:
56 """Return the registry label version, defaulting to the current MAFw version."""
57 if value is None:
58 return normalize_mafw_version(MAFW_VERSION)
59 try:
60 return normalize_mafw_version(value)
61 except ValueError as exc:
62 raise click.ClickException(str(exc)) from exc
65def _upload_pylock_reference(
66 api_config: GitlabAPIConfiguration,
67 python_version: str,
68 file_path: Path,
69 package_version: str,
70) -> None:
71 """Upload a single dependency reference file to the GitLab registry."""
72 upload_generic_file(
73 api_config,
74 DEPS_REGISTRY_PACKAGE_NAME,
75 package_version,
76 file_path,
77 replace_existing=True,
78 )
79 CONSOLE.print(
80 f'Uploaded {file_path.name} for Python {python_version} as {DEPS_REGISTRY_PACKAGE_NAME}/{package_version}'
81 )
84def _download_pylock_reference(
85 api_config: GitlabAPIConfiguration,
86 python_version: str,
87 file_name: str,
88 package_version: str,
89 output_dir: Path,
90 *,
91 quiet_if_missing: bool = False,
92) -> Path | None:
93 """Download a single dependency reference file from the GitLab registry."""
94 dest = download_generic_file(
95 api_config,
96 DEPS_REGISTRY_PACKAGE_NAME,
97 package_version,
98 file_name,
99 output_dir,
100 )
101 if dest is not None:
102 CONSOLE.print(f'Downloaded {python_version}: {dest}')
103 elif not quiet_if_missing:
104 CONSOLE.print(f'Not found in registry: {DEPS_REGISTRY_PACKAGE_NAME}/{package_version}/{file_name}')
105 return dest
108@click.group(
109 context_settings={'help_option_names': ['-h', '--help']},
110 help='Rolling compatibility for the latest versions of MAFw dependencies.',
111 cls=AbbreviateGroup,
112)
113def latest() -> None:
114 """Group for latest-dependency verification commands."""
117@latest.command(
118 context_settings={'help_option_names': ['-h', '--help']},
119 help='Verify the latest dependency stack across Python versions.\n\n'
120 'This command will perform a compatibility check with the newest version\n'
121 'of all MAFw dependencies.',
122)
123@click.option(
124 '--min-python-ver',
125 default='3.11',
126 show_default=True,
127 type=click.STRING,
128 help='The oldest version of python to be used for the dependencies verification, constrain it to >=3.11',
129)
130@click.option(
131 '--max-python-ver',
132 default='3.14',
133 show_default=True,
134 type=click.STRING,
135 help='The newest version of python to be used for the dependencies verification',
136)
137@click.option(
138 '--full-unittest',
139 is_flag=True,
140 default=False,
141 help='Perform the test over the whole test suite.',
142)
143@click.option(
144 '--with-dep-file/--without-dep-file',
145 default=True,
146 show_default=True,
147 help='Generate dependency lock files using uv pip compile and uninstall managed python before check.',
148)
149@click.option(
150 '--compare-with-ref/--no-compare-with-ref',
151 default=True,
152 show_default=True,
153 help='Compare the generated dependency file with a reference one and skip tests if identical.',
154)
155@click.option(
156 '--gitlab-ref/--no-gitlab-ref',
157 default=False,
158 show_default=True,
159 help='Download/upload dependency reference files from/to the GitLab generic registry.',
160)
161@click.option('--gitlab-api-url', default=None, envvar='CI_API_V4_URL', help='GitLab API v4 base URL.')
162@click.option('--gitlab-project-id', default=None, type=click.INT, envvar='CI_PROJECT_ID', help='GitLab project ID.')
163@click.option('--gitlab-token', default=None, envvar='CI_JOB_TOKEN', help='GitLab API token.')
164@click.option(
165 '--preserve-envs',
166 is_flag=True,
167 default=False,
168 show_default=True,
169 help='Preserve created hatch environments after the check is completed instead of removing them.',
170)
171def check(
172 min_python_ver: str,
173 max_python_ver: str,
174 full_unittest: bool,
175 with_dep_file: bool,
176 compare_with_ref: bool,
177 gitlab_ref: bool,
178 gitlab_api_url: str | None,
179 gitlab_project_id: int | None,
180 gitlab_token: str | None,
181 preserve_envs: bool,
182) -> None:
183 """Verify the latest dependency stack for a Python version range."""
184 supported_python_versions = ensure_mafw_project_root()
185 api_config: GitlabAPIConfiguration | None = None
186 package_version = _normalize_mafw_version_option(None)
187 if gitlab_ref:
188 try:
189 api_config = build_gitlab_api_configuration(gitlab_api_url, gitlab_project_id, gitlab_token)
190 except ValueError as exc:
191 raise click.ClickException(str(exc)) from exc
193 for python_version in python_versions_between(min_python_ver, max_python_ver, supported_python_versions):
194 pylock_file = Path(build_pylock_compiled_filename(python_version))
195 pylock_ref_file = Path(build_pylock_reference_filename(python_version))
197 if with_dep_file or compare_with_ref or gitlab_ref:
198 pylock_file.unlink(missing_ok=True)
199 compile_dependency_lockfile(python_version, pylock_file, list(DEFAULT_FREEZE_EXTRAS))
200 cmd(['uv', 'python', 'uninstall', '--managed-python', python_version])
202 if gitlab_ref and api_config:
203 dest = _download_pylock_reference(
204 api_config,
205 compile_python_selector(python_version),
206 pylock_ref_file.name,
207 package_version,
208 Path.cwd(),
209 quiet_if_missing=True,
210 )
211 if dest is None:
212 CONSOLE.print(f'WARNING: reference file for Python {python_version} not found in GitLab registry.')
213 if pylock_ref_file.exists(): 213 ↛ 216line 213 didn't jump to line 216 because the condition on line 213 was always true
214 CONSOLE.print(f'A local reference file {pylock_ref_file} exists and will be used instead.')
216 if (compare_with_ref or gitlab_ref) and pylock_ref_file.exists():
217 current_data = tomlkit.loads(pylock_file.read_text(encoding='utf-8'))
218 reference_data = tomlkit.loads(pylock_ref_file.read_text(encoding='utf-8'))
220 current_pkgs = {pkg['name']: pkg for pkg in current_data.get('packages', [])}
221 reference_pkgs = {pkg['name']: pkg for pkg in reference_data.get('packages', [])}
223 differences: list[str] = []
224 for name, pkg in current_pkgs.items():
225 if name not in reference_pkgs:
226 differences.append(f'ADDED {name} {pkg.get("version", "unknown")}')
227 else:
228 ref_pkg = reference_pkgs[name]
229 if pkg.get('version') != ref_pkg.get('version') or pkg.get('marker') != ref_pkg.get('marker'):
230 differences.append(f'UPDATED {name}: {ref_pkg.get("version")} -> {pkg.get("version")}')
232 for name in reference_pkgs:
233 if name not in current_pkgs:
234 differences.append(f'REMOVED {name}')
236 if not differences:
237 CONSOLE.print(f'Dependencies for Python {python_version} are identical to reference. Skipping tests.')
238 continue
240 CONSOLE.print(f'Dependencies for Python {python_version} have changed:')
241 for diff in differences:
242 CONSOLE.print(f' - {diff}')
244 ### UNIT TESTS
246 # Step 1. Remove existing hatch-test env
247 cmd(['hatch', 'env', 'remove', f'hatch-test.py{python_version}'])
249 # Step 2. Create it again from scratch
250 cmd(['hatch', 'env', 'create', f'hatch-test.py{python_version}'])
252 # Step 3. Run the test with either a selection of files or the whole test suite.
253 hatch_test_cmd = ['hatch', 'test', '-py', python_version]
254 files_to_be_tested = []
255 if not full_unittest:
256 files_to_be_tested = DEFAULT_TEST_FILES
257 cmd(hatch_test_cmd + files_to_be_tested)
259 # Step 4. Clean up to save space
260 if not preserve_envs:
261 cmd(['hatch', 'env', 'remove', f'hatch-test.py{python_version}'])
263 ### STATIC TYPING
265 # Step 1. Remove env
266 cmd(['hatch', 'env', 'remove', f'types.py{python_version}'])
268 # Step 2. Create env
269 cmd(['hatch', 'env', 'create', f'types.py{python_version}'])
271 # Step 3. Run the test
272 cmd(['hatch', 'run', f'types.py{python_version}:check', '--python-version', python_version])
274 # Step 4. Clean up
275 if not preserve_envs:
276 cmd(['hatch', 'env', 'remove', f'types.py{python_version}'])
278 if compare_with_ref or gitlab_ref:
279 pylock_file.replace(pylock_ref_file)
280 CONSOLE.print(f'Updated reference file: {pylock_ref_file}')
281 if gitlab_ref and api_config:
282 _upload_pylock_reference(api_config, python_version, pylock_ref_file, package_version)
285@latest.command(
286 context_settings={'help_option_names': ['-h', '--help']},
287 help='Compare latest dependency versions against reference pylock files.\n\n'
288 'This command compiles fresh dependency lockfiles and compares them\n'
289 'against reference pylock files for each Python version in the\n'
290 'specified range, classifying differences as ADDED, REMOVED, or UPDATED.',
291)
292@click.option(
293 '--min-python-ver',
294 default='3.11',
295 show_default=True,
296 type=click.STRING,
297 help='The minimum Python version (major.minor) for the comparison range.',
298)
299@click.option(
300 '--max-python-ver',
301 default='3.14',
302 show_default=True,
303 type=click.STRING,
304 help='The maximum Python version (major.minor) for the comparison range.',
305)
306@click.option(
307 '--gitlab-ref/--no-gitlab-ref',
308 default=False,
309 show_default=True,
310 help='Download reference files from the GitLab generic registry.',
311)
312@click.option(
313 '--gitlab-api-url',
314 default=None,
315 envvar='CI_API_V4_URL',
316 help='GitLab API v4 base URL.',
317)
318@click.option(
319 '--gitlab-project-id',
320 default=None,
321 type=click.INT,
322 envvar='CI_PROJECT_ID',
323 help='GitLab project ID.',
324)
325@click.option(
326 '--gitlab-token',
327 default=None,
328 envvar='CI_JOB_TOKEN',
329 help='GitLab API token.',
330)
331@click.option(
332 '-o',
333 '--output-file',
334 default=None,
335 type=click.Path(path_type=Path),
336 help='Output file path for the comparison results.',
337)
338@click.option(
339 '-f',
340 '--format',
341 'output_format',
342 default=None,
343 type=click.Choice(['json', 'markdown'], case_sensitive=False),
344 help='Output file format. Inferred from --output-file extension if omitted.',
345)
346@click.option(
347 '-q',
348 '--quiet',
349 is_flag=True,
350 default=False,
351 help='Suppress console output (requires --output-file).',
352)
353def compare(
354 min_python_ver: str,
355 max_python_ver: str,
356 gitlab_ref: bool,
357 gitlab_api_url: str | None,
358 gitlab_project_id: int | None,
359 gitlab_token: str | None,
360 output_file: Path | None,
361 output_format: str | None,
362 quiet: bool,
363) -> None:
364 """Compare latest dependency versions against reference pylock files."""
365 supported_python_versions = ensure_mafw_project_root()
367 if quiet and output_file is None:
368 click.echo(
369 'Error: --quiet requires --output-file to be specified.',
370 err=True,
371 )
372 raise SystemExit(1)
374 api_config: GitlabAPIConfiguration | None = None
375 package_version = _normalize_mafw_version_option(None)
376 if gitlab_ref:
377 try:
378 api_config = build_gitlab_api_configuration(gitlab_api_url, gitlab_project_id, gitlab_token)
379 except ValueError as exc:
380 raise click.ClickException(str(exc)) from exc
382 python_versions = python_versions_between(min_python_ver, max_python_ver, supported_python_versions)
384 results: list[VersionComparisonResult] = []
385 for python_version in python_versions:
386 pylock_file = Path(build_pylock_compiled_filename(python_version))
387 pylock_ref_file = Path(build_pylock_reference_filename(python_version))
389 try:
390 pylock_file.unlink(missing_ok=True)
391 compile_dependency_lockfile(python_version, pylock_file, list(DEFAULT_FREEZE_EXTRAS))
392 except (subprocess.CalledProcessError, click.ClickException) as exc:
393 click.echo(
394 f'Error: compilation failed for Python {python_version}: {exc}',
395 err=True,
396 )
397 continue
399 if gitlab_ref and api_config: 399 ↛ 400line 399 didn't jump to line 400 because the condition on line 399 was never true
400 dest = _download_pylock_reference(
401 api_config,
402 compile_python_selector(python_version),
403 pylock_ref_file.name,
404 package_version,
405 Path.cwd(),
406 quiet_if_missing=True,
407 )
408 if dest is None:
409 click.echo(
410 f'Warning: reference file for Python {python_version} not found in GitLab registry, skipping.',
411 err=True,
412 )
413 continue
414 else:
415 if not pylock_ref_file.exists():
416 click.echo(
417 f'Warning: no local reference file {pylock_ref_file} for Python {python_version}, skipping.',
418 err=True,
419 )
420 continue
422 try:
423 latest_packages = load_pylock_packages(pylock_file)
424 except (
425 FileNotFoundError,
426 tomllib.TOMLDecodeError,
427 ) as exc: # pragma: no cover — defensive: filesystem/parse error in CI-only flow
428 click.echo(
429 f'Error: unable to parse latest lockfile for Python {python_version}: {exc}',
430 err=True,
431 )
432 continue
434 try:
435 reference_packages = load_pylock_packages(pylock_ref_file)
436 except (
437 FileNotFoundError,
438 tomllib.TOMLDecodeError,
439 ) as exc: # pragma: no cover — defensive: filesystem/parse error in CI-only flow
440 click.echo(
441 f'Error: unable to parse reference lockfile for Python {python_version}: {exc}',
442 err=True,
443 )
444 continue
446 changes = compare_packages(latest_packages, reference_packages)
447 results.append(VersionComparisonResult(python_version=python_version, changes=changes))
449 if output_file is not None:
450 output_file.parent.mkdir(parents=True, exist_ok=True)
452 effective_format = resolve_output_format(output_format, output_file)
454 if effective_format == 'json':
455 content = render_comparison_json(results)
456 else:
457 content = render_comparison_markdown(results)
459 try:
460 output_file.write_text(content, encoding='utf-8')
461 except OSError as exc: # pragma: no cover — defensive: OS-level write failure
462 raise click.ClickException(f'Unable to write output file {output_file}: {exc}') from exc
464 if not quiet: 464 ↛ 467line 464 didn't jump to line 467 because the condition on line 464 was always true
465 CONSOLE.print(f'Comparison results written to {output_file}')
467 if not quiet: 467 ↛ exitline 467 didn't return from function 'compare' because the condition on line 467 was always true
468 render_comparison_rich(results, CONSOLE)