Coverage for src / mafw / devtools / cli / dependencies / latest.py: 96%
169 statements
« prev ^ index » next coverage.py v7.14.0, created at 2026-06-28 13:34 +0000
1# Copyright 2026 European Union
2# Author: Bulgheroni Antonio (antonio.bulgheroni@ec.europa.eu)
3# SPDX-License-Identifier: EUPL-1.2
4"""Rolling compatibility commands for the latest versions of MAFw dependencies."""
6from __future__ import annotations
8import subprocess
9import tomllib
10from pathlib import Path
11from typing import Final
13import click
14import tomlkit
16from mafw.__about__ import __version__ as MAFW_VERSION
17from mafw.devtools.dependencies.compare import (
18 VersionComparisonResult,
19 compare_packages,
20 render_comparison_json,
21 render_comparison_markdown,
22 render_comparison_rich,
23 resolve_output_format,
24)
25from mafw.devtools.dependencies.compile import (
26 DEFAULT_FREEZE_EXTRAS,
27 compile_dependency_lockfile,
28 compile_python_selector,
29 ensure_mafw_project_root,
30 load_pylock_packages,
31 python_versions_between,
32)
33from mafw.devtools.gitlab import (
34 GitlabAPIConfiguration,
35 build_gitlab_api_configuration,
36 download_generic_file,
37 normalize_mafw_version,
38 upload_generic_file,
39)
40from mafw.tools.click_extensions import AbbreviateGroup
41from mafw.tools.shell_tools import CONSOLE
42from mafw.tools.shell_tools import run as cmd
44DEPS_REGISTRY_PACKAGE_NAME: Final[str] = 'mafw-deps'
45"""Generic package name used for dependency reference uploads."""
47DEFAULT_TEST_FILES: Final[list[str]] = [
48 'tests/test_full_integration.py',
49]
50"""List of test files executed by default during dependency verification."""
53def _normalize_mafw_version_option(value: str | None) -> str:
54 """Return the registry label version, defaulting to the current MAFw version."""
55 if value is None:
56 return normalize_mafw_version(MAFW_VERSION)
57 try:
58 return normalize_mafw_version(value)
59 except ValueError as exc:
60 raise click.ClickException(str(exc)) from exc
63def _upload_pylock_reference(
64 api_config: GitlabAPIConfiguration,
65 python_version: str,
66 file_path: Path,
67 package_version: str,
68) -> None:
69 """Upload a single dependency reference file to the GitLab registry."""
70 upload_generic_file(
71 api_config,
72 DEPS_REGISTRY_PACKAGE_NAME,
73 package_version,
74 file_path,
75 replace_existing=True,
76 )
77 CONSOLE.print(
78 f'Uploaded {file_path.name} for Python {python_version} as {DEPS_REGISTRY_PACKAGE_NAME}/{package_version}'
79 )
82def _download_pylock_reference(
83 api_config: GitlabAPIConfiguration,
84 python_version: str,
85 file_name: str,
86 package_version: str,
87 output_dir: Path,
88 *,
89 quiet_if_missing: bool = False,
90) -> Path | None:
91 """Download a single dependency reference file from the GitLab registry."""
92 dest = download_generic_file(
93 api_config,
94 DEPS_REGISTRY_PACKAGE_NAME,
95 package_version,
96 file_name,
97 output_dir,
98 )
99 if dest is not None:
100 CONSOLE.print(f'Downloaded {python_version}: {dest}')
101 elif not quiet_if_missing:
102 CONSOLE.print(f'Not found in registry: {DEPS_REGISTRY_PACKAGE_NAME}/{package_version}/{file_name}')
103 return dest
106@click.group(
107 context_settings={'help_option_names': ['-h', '--help']},
108 help='Rolling compatibility for the latest versions of MAFw dependencies.',
109 cls=AbbreviateGroup,
110)
111def latest() -> None:
112 """Group for latest-dependency verification commands."""
115@latest.command(
116 context_settings={'help_option_names': ['-h', '--help']},
117 help='Verify the latest dependency stack across Python versions.\n\n'
118 'This command will perform a compatibility check with the newest version\n'
119 'of all MAFw dependencies.',
120)
121@click.option(
122 '--min-python-ver',
123 default='3.11',
124 show_default=True,
125 type=click.STRING,
126 help='The oldest version of python to be used for the dependencies verification, constrain it to >=3.11',
127)
128@click.option(
129 '--max-python-ver',
130 default='3.14',
131 show_default=True,
132 type=click.STRING,
133 help='The newest version of python to be used for the dependencies verification',
134)
135@click.option(
136 '--full-unittest',
137 is_flag=True,
138 default=False,
139 help='Perform the test over the whole test suite.',
140)
141@click.option(
142 '--with-dep-file/--without-dep-file',
143 default=True,
144 show_default=True,
145 help='Generate dependency lock files using uv pip compile and uninstall managed python before check.',
146)
147@click.option(
148 '--compare-with-ref/--no-compare-with-ref',
149 default=True,
150 show_default=True,
151 help='Compare the generated dependency file with a reference one and skip tests if identical.',
152)
153@click.option(
154 '--gitlab-ref/--no-gitlab-ref',
155 default=False,
156 show_default=True,
157 help='Download/upload dependency reference files from/to the GitLab generic registry.',
158)
159@click.option('--gitlab-api-url', default=None, envvar='CI_API_V4_URL', help='GitLab API v4 base URL.')
160@click.option('--gitlab-project-id', default=None, type=click.INT, envvar='CI_PROJECT_ID', help='GitLab project ID.')
161@click.option('--gitlab-token', default=None, envvar='CI_JOB_TOKEN', help='GitLab API token.')
162@click.option(
163 '--preserve-envs',
164 is_flag=True,
165 default=False,
166 show_default=True,
167 help='Preserve created hatch environments after the check is completed instead of removing them.',
168)
169def check(
170 min_python_ver: str,
171 max_python_ver: str,
172 full_unittest: bool,
173 with_dep_file: bool,
174 compare_with_ref: bool,
175 gitlab_ref: bool,
176 gitlab_api_url: str | None,
177 gitlab_project_id: int | None,
178 gitlab_token: str | None,
179 preserve_envs: bool,
180) -> None:
181 """Verify the latest dependency stack for a Python version range."""
182 supported_python_versions = ensure_mafw_project_root()
183 api_config: GitlabAPIConfiguration | None = None
184 package_version = _normalize_mafw_version_option(None)
185 if gitlab_ref:
186 try:
187 api_config = build_gitlab_api_configuration(gitlab_api_url, gitlab_project_id, gitlab_token)
188 except ValueError as exc:
189 raise click.ClickException(str(exc)) from exc
191 for python_version in python_versions_between(min_python_ver, max_python_ver, supported_python_versions):
192 pylock_file = Path(f'pylock.py{python_version}.toml')
193 pylock_ref_file = Path(f'pylock.py{python_version}_ref.toml')
195 if with_dep_file or compare_with_ref or gitlab_ref:
196 pylock_file.unlink(missing_ok=True)
197 compile_dependency_lockfile(python_version, pylock_file, list(DEFAULT_FREEZE_EXTRAS))
198 cmd(['uv', 'python', 'uninstall', '--managed-python', python_version])
200 if gitlab_ref and api_config:
201 dest = _download_pylock_reference(
202 api_config,
203 compile_python_selector(python_version),
204 pylock_ref_file.name,
205 package_version,
206 Path.cwd(),
207 quiet_if_missing=True,
208 )
209 if dest is None:
210 CONSOLE.print(f'WARNING: reference file for Python {python_version} not found in GitLab registry.')
211 if pylock_ref_file.exists(): 211 ↛ 214line 211 didn't jump to line 214 because the condition on line 211 was always true
212 CONSOLE.print(f'A local reference file {pylock_ref_file} exists and will be used instead.')
214 if (compare_with_ref or gitlab_ref) and pylock_ref_file.exists():
215 current_data = tomlkit.loads(pylock_file.read_text(encoding='utf-8'))
216 reference_data = tomlkit.loads(pylock_ref_file.read_text(encoding='utf-8'))
218 current_pkgs = {pkg['name']: pkg for pkg in current_data.get('packages', [])}
219 reference_pkgs = {pkg['name']: pkg for pkg in reference_data.get('packages', [])}
221 differences: list[str] = []
222 for name, pkg in current_pkgs.items():
223 if name not in reference_pkgs:
224 differences.append(f'ADDED {name} {pkg.get("version", "unknown")}')
225 else:
226 ref_pkg = reference_pkgs[name]
227 if pkg.get('version') != ref_pkg.get('version') or pkg.get('marker') != ref_pkg.get('marker'):
228 differences.append(f'UPDATED {name}: {ref_pkg.get("version")} -> {pkg.get("version")}')
230 for name in reference_pkgs:
231 if name not in current_pkgs:
232 differences.append(f'REMOVED {name}')
234 if not differences:
235 CONSOLE.print(f'Dependencies for Python {python_version} are identical to reference. Skipping tests.')
236 continue
238 CONSOLE.print(f'Dependencies for Python {python_version} have changed:')
239 for diff in differences:
240 CONSOLE.print(f' - {diff}')
242 ### UNIT TESTS
244 # Step 1. Remove existing hatch-test env
245 cmd(['hatch', 'env', 'remove', f'hatch-test.py{python_version}'])
247 # Step 2. Create it again from scratch
248 cmd(['hatch', 'env', 'create', f'hatch-test.py{python_version}'])
250 # Step 3. Run the test with either a selection of files or the whole test suite.
251 hatch_test_cmd = ['hatch', 'test', '-py', python_version]
252 files_to_be_tested = []
253 if not full_unittest:
254 files_to_be_tested = DEFAULT_TEST_FILES
255 cmd(hatch_test_cmd + files_to_be_tested)
257 # Step 4. Clean up to save space
258 if not preserve_envs:
259 cmd(['hatch', 'env', 'remove', f'hatch-test.py{python_version}'])
261 ### STATIC TYPING
263 # Step 1. Remove env
264 cmd(['hatch', 'env', 'remove', f'types.py{python_version}'])
266 # Step 2. Create env
267 cmd(['hatch', 'env', 'create', f'types.py{python_version}'])
269 # Step 3. Run the test
270 cmd(['hatch', 'run', f'types.py{python_version}:check', '--python-version', python_version])
272 # Step 4. Clean up
273 if not preserve_envs:
274 cmd(['hatch', 'env', 'remove', f'types.py{python_version}'])
276 if compare_with_ref or gitlab_ref:
277 pylock_file.replace(pylock_ref_file)
278 CONSOLE.print(f'Updated reference file: {pylock_ref_file}')
279 if gitlab_ref and api_config:
280 _upload_pylock_reference(api_config, python_version, pylock_ref_file, package_version)
283@latest.command(
284 context_settings={'help_option_names': ['-h', '--help']},
285 help='Compare latest dependency versions against reference pylock files.\n\n'
286 'This command compiles fresh dependency lockfiles and compares them\n'
287 'against reference pylock files for each Python version in the\n'
288 'specified range, classifying differences as ADDED, REMOVED, or UPDATED.',
289)
290@click.option(
291 '--min-python-ver',
292 default='3.11',
293 show_default=True,
294 type=click.STRING,
295 help='The minimum Python version (major.minor) for the comparison range.',
296)
297@click.option(
298 '--max-python-ver',
299 default='3.14',
300 show_default=True,
301 type=click.STRING,
302 help='The maximum Python version (major.minor) for the comparison range.',
303)
304@click.option(
305 '--gitlab-ref/--no-gitlab-ref',
306 default=False,
307 show_default=True,
308 help='Download reference files from the GitLab generic registry.',
309)
310@click.option(
311 '--gitlab-api-url',
312 default=None,
313 envvar='CI_API_V4_URL',
314 help='GitLab API v4 base URL.',
315)
316@click.option(
317 '--gitlab-project-id',
318 default=None,
319 type=click.INT,
320 envvar='CI_PROJECT_ID',
321 help='GitLab project ID.',
322)
323@click.option(
324 '--gitlab-token',
325 default=None,
326 envvar='CI_JOB_TOKEN',
327 help='GitLab API token.',
328)
329@click.option(
330 '-o',
331 '--output-file',
332 default=None,
333 type=click.Path(path_type=Path),
334 help='Output file path for the comparison results.',
335)
336@click.option(
337 '-f',
338 '--format',
339 'output_format',
340 default=None,
341 type=click.Choice(['json', 'markdown'], case_sensitive=False),
342 help='Output file format. Inferred from --output-file extension if omitted.',
343)
344@click.option(
345 '-q',
346 '--quiet',
347 is_flag=True,
348 default=False,
349 help='Suppress console output (requires --output-file).',
350)
351def compare(
352 min_python_ver: str,
353 max_python_ver: str,
354 gitlab_ref: bool,
355 gitlab_api_url: str | None,
356 gitlab_project_id: int | None,
357 gitlab_token: str | None,
358 output_file: Path | None,
359 output_format: str | None,
360 quiet: bool,
361) -> None:
362 """Compare latest dependency versions against reference pylock files."""
363 supported_python_versions = ensure_mafw_project_root()
365 if quiet and output_file is None:
366 click.echo(
367 'Error: --quiet requires --output-file to be specified.',
368 err=True,
369 )
370 raise SystemExit(1)
372 api_config: GitlabAPIConfiguration | None = None
373 package_version = _normalize_mafw_version_option(None)
374 if gitlab_ref:
375 try:
376 api_config = build_gitlab_api_configuration(gitlab_api_url, gitlab_project_id, gitlab_token)
377 except ValueError as exc:
378 raise click.ClickException(str(exc)) from exc
380 python_versions = python_versions_between(min_python_ver, max_python_ver, supported_python_versions)
382 results: list[VersionComparisonResult] = []
383 for python_version in python_versions:
384 pylock_file = Path(f'pylock.py{python_version}.toml')
385 pylock_ref_file = Path(f'pylock.py{python_version}_ref.toml')
387 try:
388 pylock_file.unlink(missing_ok=True)
389 compile_dependency_lockfile(python_version, pylock_file, list(DEFAULT_FREEZE_EXTRAS))
390 except (subprocess.CalledProcessError, click.ClickException) as exc:
391 click.echo(
392 f'Error: compilation failed for Python {python_version}: {exc}',
393 err=True,
394 )
395 continue
397 if gitlab_ref and api_config: 397 ↛ 398line 397 didn't jump to line 398 because the condition on line 397 was never true
398 dest = _download_pylock_reference(
399 api_config,
400 compile_python_selector(python_version),
401 pylock_ref_file.name,
402 package_version,
403 Path.cwd(),
404 quiet_if_missing=True,
405 )
406 if dest is None:
407 click.echo(
408 f'Warning: reference file for Python {python_version} not found in GitLab registry, skipping.',
409 err=True,
410 )
411 continue
412 else:
413 if not pylock_ref_file.exists():
414 click.echo(
415 f'Warning: no local reference file {pylock_ref_file} for Python {python_version}, skipping.',
416 err=True,
417 )
418 continue
420 try:
421 latest_packages = load_pylock_packages(pylock_file)
422 except (
423 FileNotFoundError,
424 tomllib.TOMLDecodeError,
425 ) as exc: # pragma: no cover — defensive: filesystem/parse error in CI-only flow
426 click.echo(
427 f'Error: unable to parse latest lockfile for Python {python_version}: {exc}',
428 err=True,
429 )
430 continue
432 try:
433 reference_packages = load_pylock_packages(pylock_ref_file)
434 except (
435 FileNotFoundError,
436 tomllib.TOMLDecodeError,
437 ) as exc: # pragma: no cover — defensive: filesystem/parse error in CI-only flow
438 click.echo(
439 f'Error: unable to parse reference lockfile for Python {python_version}: {exc}',
440 err=True,
441 )
442 continue
444 changes = compare_packages(latest_packages, reference_packages)
445 results.append(VersionComparisonResult(python_version=python_version, changes=changes))
447 if output_file is not None:
448 output_file.parent.mkdir(parents=True, exist_ok=True)
450 effective_format = resolve_output_format(output_format, output_file)
452 if effective_format == 'json':
453 content = render_comparison_json(results)
454 else:
455 content = render_comparison_markdown(results)
457 try:
458 output_file.write_text(content, encoding='utf-8')
459 except OSError as exc: # pragma: no cover — defensive: OS-level write failure
460 raise click.ClickException(f'Unable to write output file {output_file}: {exc}') from exc
462 if not quiet: 462 ↛ 465line 462 didn't jump to line 465 because the condition on line 462 was always true
463 CONSOLE.print(f'Comparison results written to {output_file}')
465 if not quiet: 465 ↛ exitline 465 didn't return from function 'compare' because the condition on line 465 was always true
466 render_comparison_rich(results, CONSOLE)