# Copyright 2025–2026 European Union
# Author: Bulgheroni Antonio (antonio.bulgheroni@ec.europa.eu)
# SPDX-License-Identifier: EUPL-1.2
"""
Helper tool for the generation of versioned documentation files.
Build Sphinx docs for every stable tag (excluding rc/alpha/beta),
label highest tag as "stable", optionally label current branch as "dev" if it's ahead.
Generates a versions.json and creates redirect index pages for stable/dev.
Now with optional PDF generation!
Requirements:
- Git with worktree support
- sphinx-build available on PATH (install Sphinx in the env)
- For PDF: latexmk and pdflatex (TeX Live or similar)
.. click:: mafw.scripts.doc_versioning:cli
:prog: multiversion-doc
:nested: full
"""
import json
import os
import re
import shutil
import subprocess
import sys
import tempfile
import zipfile
from dataclasses import dataclass
from datetime import datetime, timezone
from enum import StrEnum
from pathlib import Path
from typing import TYPE_CHECKING, Any, List, Literal, Tuple, cast
from urllib.error import HTTPError, URLError
from urllib.request import urlopen
import click
try:
from packaging.version import InvalidVersion, Version
except ImportError as e:
raise click.ClickException(
'Unable to import the "packaging" package. '
'This usually means you are running outside the MAFw development environment. '
'Install MAFw with the optional [dev] feature, or invoke the helper via Hatch '
'(e.g. "hatch run dev.py3.14:multidoc --help").'
) from e
# ---------------------------
# Configurable defaults
# ---------------------------
DEFAULT_MIN_TAG_REGEX = r'^v([1-9][0-9]*)\.[0-9]+\.[0-9]+(\.[0-9]+)?$'
# The files/directories under each worktree where docs live
DOCS_SUBPATH = Path('docs') / 'source'
SPHINX_BUILD_CMD = 'sphinx-build' # ensure on PATH
OLD_VERSION_TO_BE_PATCHED = ['v1.0.0', 'v1.1.0', 'v1.2.0', 'v1.3.0', 'v1.4.0']
CONTEXT_SETTINGS = {'help_option_names': ['-h', '--help']}
"""Click context settings for command line help aliases."""
# ---------------------------
[docs]
@dataclass(frozen=True, slots=True)
class GitlabAPIConfiguration:
"""Configuration needed to communicate with the GitLab API.
The configuration is typically populated from CI-provided environment variables,
but it can also be supplied explicitly when running locally.
:param on_ci: True when the script is running in GitLab CI (``CI`` env var set)
:type on_ci: bool
:param api_url: Base GitLab API v4 URL (e.g. ``https://gitlab.example/api/v4``)
:type api_url: str
:param project_id: GitLab project numeric ID
:type project_id: int
:param token_type: Token kind used for authentication
:type token_type: Literal['job_token', 'api_token']
:param token: Authentication token value
:type token: str
"""
on_ci: bool
api_url: str
project_id: int
token_type: Literal['job_token', 'api_token']
token: str
[docs]
def build_gitlab_api_configuration(
api_url: str | None,
project_id: int | None,
token: str | None,
) -> GitlabAPIConfiguration:
"""Build a GitLab API configuration from provided values and environment context.
The script detects CI execution by checking the ``CI`` environment variable.
When running on CI, the token is expected to be a job token.
Expected CI environment variables:
- ``CI_API_V4_URL`` (API base URL)
- ``CI_PROJECT_ID`` (project numeric id)
- ``CI_JOB_TOKEN`` (job token)
:param api_url: API base URL, typically from ``CI_API_V4_URL``
:type api_url: str | None
:param project_id: GitLab project id, typically from ``CI_PROJECT_ID``
:type project_id: int | None
:param token: Authentication token, typically from ``CI_JOB_TOKEN``
:type token: str | None
:return: Fully populated configuration
:rtype: GitlabAPIConfiguration
:raises ValueError: If required values are missing
"""
on_ci = bool(os.environ.get('CI'))
token_type: Literal['job_token', 'api_token'] = 'job_token' if on_ci else 'api_token'
missing = []
if not api_url:
missing.append('api_url (env: CI_API_V4_URL)')
if project_id is None:
missing.append('project_id (env: CI_PROJECT_ID)')
if not token:
missing.append('token (env: CI_JOB_TOKEN)')
if missing:
raise ValueError(f'Missing GitLab configuration values: {", ".join(missing)}')
if TYPE_CHECKING:
assert api_url is not None
assert project_id is not None
assert token is not None
return GitlabAPIConfiguration(
on_ci=on_ci,
api_url=api_url,
project_id=project_id,
token_type=token_type,
token=token,
)
[docs]
def parse_mafw_docs_zip_filename(file_name: str) -> Tuple[str, str] | None:
"""Parse and validate a mafw-docs zip filename.
The accepted filename pattern is: ``mafw-docs-vX.Y.Z.zip``.
:param file_name: File name to parse
:type file_name: str
:return: Tuple of (version, normalized_file_name) if valid, otherwise None
:rtype: Tuple[str, str] | None
"""
base = Path(file_name).name
m = re.fullmatch(r'(mafw-docs)-(v[0-9]+\.[0-9]+\.[0-9]+)\.zip', base)
if not m:
return None
version = m.group(2)
return version, f'{m.group(1)}-{version}.zip'
[docs]
def normalize_registry_item(item: str) -> Tuple[str, str]:
"""Normalize a registry item into (version, file_name).
The item can be either:
- a version string: ``vX.Y.Z``
- a file name: ``mafw-docs-vX.Y.Z.zip``
:param item: Input item
:type item: str
:return: Tuple of (version, file_name)
:rtype: Tuple[str, str]
:raises ValueError: If the item cannot be normalized
"""
item = item.strip()
parsed = parse_mafw_docs_zip_filename(item)
if parsed is not None:
return parsed
try:
v = Version(item)
except InvalidVersion as e:
raise ValueError(f'Invalid version or zip filename: {item}') from e
if v.is_prerelease or v.is_devrelease:
raise ValueError(f'Pre-release/dev versions are not supported here: {item}')
version = item
return version, f'mafw-docs-{version}.zip'
[docs]
def iter_local_mafw_docs_zips(zip_dir: Path) -> List[Tuple[str, Path]]:
"""List local mafw-docs zip files in a directory.
Only files matching ``mafw-docs-vX.Y.Z.zip`` are returned.
:param zip_dir: Directory to scan
:type zip_dir: Path
:return: List of (version, file_path) tuples
:rtype: List[Tuple[str, Path]]
"""
zip_dir = Path(zip_dir).resolve()
if not zip_dir.exists():
return []
items: List[Tuple[str, Path]] = []
for fp in zip_dir.iterdir():
if not fp.is_file():
continue
parsed = parse_mafw_docs_zip_filename(fp.name)
if parsed is None:
continue
version, _ = parsed
items.append((version, fp))
items.sort(key=lambda x: parse_version_tuple(x[0]))
return items
[docs]
def filter_versions_in_range(versions: List[str], from_v: str | None, to_v: str | None) -> List[str]:
"""Filter versions within an inclusive semantic-version range.
:param versions: Input versions list
:type versions: List[str]
:param from_v: Range start (inclusive)
:type from_v: str | None
:param to_v: Range end (inclusive)
:type to_v: str | None
:return: Filtered versions list
:rtype: List[str]
:raises ValueError: If range bounds are invalid
"""
if from_v is None and to_v is None:
return versions
if from_v is not None:
Version(from_v) # validate
if to_v is not None:
Version(to_v) # validate
if from_v is not None and to_v is not None:
if Version(from_v) > Version(to_v):
raise ValueError(f'Invalid range: --from {from_v} is greater than --to {to_v}')
out: List[str] = []
for v in versions:
vv = Version(v)
if from_v is not None and vv < Version(from_v):
continue
if to_v is not None and vv > Version(to_v):
continue
out.append(v)
return out
[docs]
def list_mafw_docs_generic_packages(
api_config: GitlabAPIConfiguration,
package_name: str = 'mafw-docs',
) -> List[dict[str, Any]]:
"""List generic packages for mafw-docs in the GitLab Package Registry.
Uses the Packages API:
``GET /projects/:id/packages``
and filters for ``package_type=generic`` and exact ``name == package_name``.
:param api_config: GitLab API configuration
:type api_config: GitlabAPIConfiguration
:param package_name: Package name, defaults to ``mafw-docs``
:type package_name: str
:return: List of package dictionaries from the API
:rtype: List[dict[str, Any]]
"""
try:
import requests
except ModuleNotFoundError as e:
raise ModuleNotFoundError(
'The "requests" package is required for registry operations. Install development extras (e.g. "pip install .[dev]").'
) from e
base_url = api_config.api_url.rstrip('/')
url = f'{base_url}/projects/{api_config.project_id}/packages'
headers = build_gitlab_auth_headers(api_config)
per_page = 100
page = 1
out: List[dict[str, Any]] = []
while True:
params: dict[str, str | int] = {
'package_type': 'generic',
'package_name': package_name,
'per_page': per_page,
'page': page,
'order_by': 'version',
'sort': 'asc',
}
resp = requests.get(url, headers=headers, params=params, timeout=60.0)
if not (200 <= resp.status_code < 300):
body_preview = (resp.text or '')[:500].replace('\n', ' ')
raise RuntimeError(f'GitLab list packages failed ({resp.status_code}): {body_preview}')
data = resp.json()
if not data:
break
for pkg in data:
if pkg.get('package_type') != 'generic':
continue
if pkg.get('name') != package_name:
continue
out.append(pkg)
if len(data) < per_page:
break
page += 1
return out
[docs]
def resolve_mafw_docs_package_ids_by_version(
api_config: GitlabAPIConfiguration, package_name: str = 'mafw-docs'
) -> dict[str, int]:
"""Resolve package IDs for mafw-docs generic packages by version.
:param api_config: GitLab API configuration
:type api_config: GitlabAPIConfiguration
:param package_name: Package name, defaults to ``mafw-docs``
:type package_name: str
:return: Mapping from version string to package id
:rtype: dict[str, int]
"""
pkgs = list_mafw_docs_generic_packages(api_config, package_name=package_name)
mapping: dict[str, int] = {}
for pkg in pkgs:
version = pkg.get('version')
pkg_id = pkg.get('id')
if isinstance(version, str) and isinstance(pkg_id, int):
mapping[version] = pkg_id
return mapping
[docs]
def find_repo_root(start: Path | None = None) -> Path:
"""Find the repository root directory.
The root is detected by walking upwards until a ``pyproject.toml`` file is found.
If no such file is found, the starting directory is returned.
:param start: Directory from which to start searching, defaults to current working directory
:type start: Path | None
:return: Resolved repository root directory
:rtype: Path
"""
current = (start or Path.cwd()).resolve()
while True:
if (current / 'pyproject.toml').exists():
return current
if current.parent == current:
return (start or Path.cwd()).resolve()
current = current.parent
[docs]
def create_docs_zip_for_tag(outdir: Path, tag: str, zip_filepath: Path) -> Path:
"""Create a zip archive for a built documentation version directory.
The built docs are expected under ``outdir / tag`` (e.g. ``docs/build/doc/vX.Y.Z``).
The produced zip is laid out so that extracting it from the repository root recreates
the original directory structure (e.g. ``docs/build/doc/vX.Y.Z/...``).
The zip file is created at ``zip_filepath / f"mafw-docs-{tag}.zip"``.
Notes
-----
- Symlinks are skipped to avoid ambiguous extraction behavior across platforms.
:param outdir: Output directory that contains the built docs
:type outdir: Path
:param tag: Version tag (e.g. ``v2.1.0``)
:type tag: str
:param zip_filepath: Directory where the zip file is written
:type zip_filepath: Path
:return: Path to the created zip archive
:rtype: Path
:raises FileNotFoundError: If the built docs directory does not exist
:raises NotADirectoryError: If the built docs path is not a directory
"""
outdir = Path(outdir).resolve()
zip_filepath = Path(zip_filepath).resolve()
zip_filepath.mkdir(parents=True, exist_ok=True)
built_dir = outdir / tag
if not built_dir.exists():
raise FileNotFoundError(f'Built docs directory not found: {built_dir}')
if not built_dir.is_dir():
raise NotADirectoryError(f'Built docs path is not a directory: {built_dir}')
repo_root = find_repo_root()
try:
prefix = built_dir.relative_to(repo_root)
except ValueError:
prefix = Path('docs') / 'build' / 'doc' / tag
print(f'⚠️ Warning: {built_dir} is not under repo root {repo_root}. Using archive prefix {prefix}.')
zip_path = zip_filepath / f'mafw-docs-{tag}.zip'
with zipfile.ZipFile(zip_path, mode='w', compression=zipfile.ZIP_DEFLATED) as zf:
for fp in built_dir.rglob('*'):
if not fp.is_file():
continue
if fp.is_symlink():
continue
arcname = prefix / fp.relative_to(built_dir)
zf.write(fp, arcname=arcname)
return zip_path
[docs]
def upload_docs_zip_to_gitlab_generic_registry(
api_config: GitlabAPIConfiguration,
package_version: str,
zip_path: Path,
package_name: str = 'mafw-docs',
timeout_s: float = 60.0,
) -> bool:
"""Upload a documentation zip archive to the GitLab Generic Package Registry.
The equivalent GitLab API endpoint is:
``PUT /projects/:id/packages/generic/:package_name/:package_version/:file_name``
Authentication headers:
- ``JOB-TOKEN`` when running on CI (``api_config.on_ci`` is True)
- ``PRIVATE-TOKEN`` for local execution
:param api_config: GitLab API configuration
:type api_config: GitlabAPIConfiguration
:param package_version: Package version (typically the git tag, e.g. ``v2.1.0``)
:type package_version: str
:param zip_path: Path to the zip file to upload
:type zip_path: Path
:param package_name: Generic package name, defaults to ``mafw-docs``
:type package_name: str
:param timeout_s: Request timeout in seconds, defaults to 60.0
:type timeout_s: float
:return: True if the file was uploaded, False if the upload was skipped because the target already exists
:rtype: bool
:raises FileNotFoundError: If ``zip_path`` does not exist
:raises RuntimeError: If the upload fails (non-2xx response)
"""
zip_path = Path(zip_path).resolve()
if not zip_path.exists():
raise FileNotFoundError(f'Zip file not found: {zip_path}')
try:
import requests
except ModuleNotFoundError as e:
raise ModuleNotFoundError(
'The "requests" package is required to upload documentation zips. '
'Install the development extras (e.g. "pip install .[dev]").'
) from e
base_url = api_config.api_url.rstrip('/')
file_name = zip_path.name
url = f'{base_url}/projects/{api_config.project_id}/packages/generic/{package_name}/{package_version}/{file_name}'
headers = build_gitlab_auth_headers(api_config)
head_resp = requests.head(url, headers=headers, timeout=timeout_s)
if head_resp.status_code == 200:
print(f'ℹ️ Zip already present on GitLab, skipping upload: {url}')
return False
if head_resp.status_code != 404:
body_preview = (head_resp.text or '')[:500].replace('\n', ' ')
raise RuntimeError(f'GitLab existence check failed ({head_resp.status_code}): {body_preview}')
with open(zip_path, 'rb') as f:
put_headers = dict(headers)
put_headers['Content-Type'] = 'application/zip'
resp = requests.put(url, headers=put_headers, data=f, timeout=timeout_s)
if not (200 <= resp.status_code < 300):
body_preview = (resp.text or '')[:500].replace('\n', ' ')
raise RuntimeError(f'GitLab upload failed ({resp.status_code}): {body_preview}')
return True
[docs]
def download_docs_zip_from_gitlab_generic_registry(
api_config: GitlabAPIConfiguration,
package_version: str,
download_dir: Path,
package_name: str = 'mafw-docs',
file_name: str | None = None,
timeout_s: float = 60.0,
) -> Path | None:
"""Download a documentation zip archive from the GitLab Generic Package Registry.
The equivalent GitLab API endpoint is:
``GET /projects/:id/packages/generic/:package_name/:package_version/:file_name``
The function first issues a HEAD request to determine if the file exists:
- 404: the package file does not exist (cache miss) and ``None`` is returned.
- 200: the file exists and is downloaded.
- otherwise: a warning is printed and ``None`` is returned.
Authentication headers:
- ``JOB-TOKEN`` when running on CI (``api_config.on_ci`` is True)
- ``PRIVATE-TOKEN`` for local execution
:param api_config: GitLab API configuration
:type api_config: GitlabAPIConfiguration
:param package_version: Package version (typically the git tag, e.g. ``v2.1.0``)
:type package_version: str
:param download_dir: Directory where the downloaded zip is stored
:type download_dir: Path
:param package_name: Generic package name, defaults to ``mafw-docs``
:type package_name: str
:param file_name: File name to retrieve, defaults to ``mafw-docs-<version>.zip``
:type file_name: str | None
:param timeout_s: Request timeout in seconds, defaults to 60.0
:type timeout_s: float
:return: Path to the downloaded zip archive, or None if it does not exist or cannot be retrieved
:rtype: Path | None
"""
try:
import requests
except ModuleNotFoundError as e:
raise ModuleNotFoundError(
'The "requests" package is required to download documentation zips. '
'Install the development extras (e.g. "pip install .[dev]").'
) from e
download_dir = Path(download_dir).resolve()
download_dir.mkdir(parents=True, exist_ok=True)
base_url = api_config.api_url.rstrip('/')
target_file_name = file_name or f'mafw-docs-{package_version}.zip'
url = (
f'{base_url}/projects/{api_config.project_id}/packages/generic/'
f'{package_name}/{package_version}/{target_file_name}'
)
headers = build_gitlab_auth_headers(api_config)
try:
head_resp = requests.head(url, headers=headers, timeout=timeout_s)
except Exception as e: # pragma: no cover
print(f'⚠️ Warning: cache HEAD request failed for {package_version}: {e}')
return None
if head_resp.status_code == 404:
return None
if head_resp.status_code != 200:
body_preview = (head_resp.text or '')[:200].replace('\n', ' ')
print(
f'⚠️ Warning: cache existence check returned {head_resp.status_code} for {package_version}: {body_preview}'
)
return None
dest = download_dir / target_file_name
try:
resp = requests.get(url, headers=headers, stream=True, timeout=timeout_s)
except Exception as e: # pragma: no cover
print(f'⚠️ Warning: cache download request failed for {package_version}: {e}')
return None
if not (200 <= resp.status_code < 300):
body_preview = (resp.text or '')[:200].replace('\n', ' ')
print(f'⚠️ Warning: cache download failed for {package_version} ({resp.status_code}): {body_preview}')
return None
try:
with open(dest, 'wb') as f:
for chunk in resp.iter_content(chunk_size=1024 * 1024):
if chunk:
f.write(chunk)
except OSError as e:
print(f'⚠️ Warning: could not write downloaded zip for {package_version} to {dest}: {e}')
return None
return dest
[docs]
def run(cmd: List[str], cwd: Path | None = None) -> subprocess.CompletedProcess[str]:
"""Helper to run commands with consistent behavior.
:param cmd: Command to execute as a list of strings
:type cmd: List[str]
:param cwd: Working directory for command execution, defaults to None
:type cwd: Path | None
:return: Completed process result
:rtype: subprocess.CompletedProcess[str]
"""
print(f'🧩 Running: {" ".join(cmd)}')
return subprocess.run(cmd, cwd=cwd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, check=False)
[docs]
def filter_latest_micro(versions: List[Tuple[Version, Any]]) -> List[Tuple[Version, Any]]:
"""Keep only the latest micro version per minor (major.minor).
:param versions: List of (Version, tag) tuples
:type versions: List[Tuple[Version, Any]]
:return: Filtered list of (Version, tag) tuples
:rtype: List[Tuple[Version, Any]]
"""
latest_per_minor: dict[Tuple[int, int], Tuple[Version, Any]] = {}
for v, tag in versions:
key = (v.major, v.minor)
if key not in latest_per_minor or v > latest_per_minor[key][0]:
latest_per_minor[key] = (v, tag)
return sorted(latest_per_minor.values())
[docs]
def parse_version_tuple(tag: str) -> Tuple[int, ...]:
"""Parse vX.Y.Z(.W) into tuple of ints for sorting.
:param tag: Version tag string
:type tag: str
:return: Tuple of integers representing the version
:rtype: Tuple[int, ...]
"""
if tag.startswith('v'):
tag = tag[1:]
parts = tag.split('.')
# only take numeric parts
nums = []
for p in parts:
if p.isdigit():
nums.append(int(p))
else:
# stop on strange parts; but ideally regex filters those out
break
return tuple(nums)
[docs]
def get_current_branch() -> str:
"""Get the name of the currently checked out branch.
:return: Name of the current branch
:rtype: str
"""
out = run(['git', 'rev-parse', '--abbrev-ref', 'HEAD'])
return out.stdout.strip()
[docs]
def git_rev_of(ref: str) -> str:
"""Get the git revision hash for a given reference.
:param ref: Git reference (tag, branch, commit hash)
:type ref: str
:return: Git revision hash
:rtype: str
:raises RuntimeError: If git rev-list fails
"""
proc = run(['git', 'rev-list', '-n', '1', ref])
if proc.returncode != 0:
raise RuntimeError(f'git rev-list failed for {ref}:\n{proc.stdout}')
return proc.stdout.strip()
[docs]
def is_ancestor(a: str, b: str) -> bool:
"""Return True if commit a is ancestor of commit b (git merge-base --is-ancestor).
:param a: First commit reference
:type a: str
:param b: Second commit reference
:type b: str
:return: True if a is ancestor of b
:rtype: bool
"""
proc = run(['git', 'merge-base', '--is-ancestor', a, b])
return proc.returncode == 0
[docs]
def copy_patch_files(docs_src: Path) -> None:
"""Copy patch files needed for older versions.
:param docs_src: Path to documentation source directory
:type docs_src: Path
"""
# Define the patch files to copy
patch_files = [
('docs/source/conf.py', docs_src / 'conf.py'),
('docs/source/_static/js/version-switcher.js', docs_src / '_static/js/version-switcher.js'),
('docs/source/_templates/versions.html', docs_src / '_templates/versions.html'),
('docs/source/_ext/procparams.py', docs_src / '_ext/procparams.py'),
]
# Create directories and copy files
for src_path, dst_path in patch_files:
dst_path.parent.mkdir(parents=True, exist_ok=True)
shutil.copy(Path.cwd() / src_path, dst_path)
[docs]
def parse_sphinx_log(log_content: str) -> Tuple[int, int, List[str]]:
"""
Parse Sphinx build log to extract warning and error counts, and warning messages.
Only three warnings are reported
:param log_content: Sphinx build log
:type log_content: str
:return: Tuple of warning, error count, warning messages
:rtype: Tuple[int, int, List[str]]
"""
warnings = 0
warning_messages = []
# Look for patterns like "build succeeded, X warning(s)."
success_pattern = re.compile(r'build succeeded(?:,\s+(\d+)\s+warning)?', re.IGNORECASE)
match = success_pattern.search(log_content)
if match:
if match.group(1):
warnings = int(match.group(1))
# Look for explicit warning lines and extract messages
warning_pattern = re.compile(r'^.*WARNING:.*$', re.MULTILINE | re.IGNORECASE)
warning_lines = warning_pattern.findall(log_content)
warnings = max(warnings, len(warning_lines))
# Extract just the relevant part of warning messages (limit to first 3)
# for line in warning_lines[:3]:
# clean_line = ' '.join(line.split())
# warning_messages.append(clean_line)
warning_messages = warning_lines[:3]
if len(warning_lines) > 3:
warning_messages.append(f'... and {len(warning_lines) - 3} more warning(s)')
# Look for error patterns
error_pattern = re.compile(r'ERROR:|CRITICAL:', re.IGNORECASE)
errors = len(error_pattern.findall(log_content))
return warnings, errors, warning_messages
[docs]
def report_build_status(tag: str, success: bool, log: str, build_type: str = 'HTML') -> None:
"""
Report build status with warning/error summary.
:param tag: Version tag being built
:type tag: str
:param success: Whether build succeeded
:type success: bool
:param log: Build log content
:type log: str
:param build_type: Type of build (HTML or PDF)
:type build_type: str
"""
warnings, errors, warning_messages = parse_sphinx_log(log)
status_icon = '✅' if success else '❌'
status_text = 'OK' if success else 'FAILED'
print(f'{status_icon} {tag} {build_type} build {status_text}', end='')
if warnings > 0 or errors > 0:
details = []
if warnings > 0:
details.append(f'⚠️ {warnings} warning(s)')
if errors > 0:
details.append(f'❌ {errors} error(s)')
print(f' ({", ".join(details)})')
# Display warning messages if present
if warning_messages:
for msg in warning_messages:
print(f' ⚠️ {msg}')
else:
print(' (no warnings)')
[docs]
def build_for_tag(
tag: str, outdir: Path, tmproot: Path, use_latest_conf: bool = False, keep_tmp: bool = False
) -> Tuple[bool, str]:
"""
Create worktree for tag, run sphinx-build, save log.
Returns (success, log_contents).
:param tag: Git tag to build documentation for
:type tag: str
:param outdir: Output directory for built documentation
:type outdir: Path
:param tmproot: Root temporary directory
:type tmproot: Path
:param use_latest_conf: Whether to use latest conf.py, defaults to False
:type use_latest_conf: bool
:param keep_tmp: Whether to keep temporary files, defaults to False
:type keep_tmp: bool
:return: Tuple of (success, log_contents)
:rtype: Tuple[bool, str]
"""
worktree_path = tmproot / tag
try:
proc = run(['git', 'worktree', 'add', '-q', str(worktree_path), tag])
if proc.returncode != 0:
return False, f'git worktree add failed:\n{proc.stdout}'
docs_src = worktree_path / DOCS_SUBPATH
if not docs_src.exists():
return False, f'docs source {docs_src} does not exist for tag {tag}'
if use_latest_conf or tag in OLD_VERSION_TO_BE_PATCHED:
copy_patch_files(docs_src)
out_for_tag = outdir / tag
out_for_tag.mkdir(parents=True, exist_ok=True)
# run sphinx-build and capture output
sp = run([SPHINX_BUILD_CMD, '-b', 'html', str(docs_src), str(out_for_tag)], cwd=worktree_path)
log = sp.stdout
# write log
with open(out_for_tag / 'sphinx-build.log', 'w', encoding='utf-8') as f:
f.write(log)
success = sp.returncode == 0
return success, log
finally:
# cleanup worktree
if not keep_tmp:
# use -f in case the worktree wasn't properly created
run(['git', 'worktree', 'remove', '-f', str(worktree_path)])
[docs]
def build_pdf_for_tag(
tag: str, html_tag_dir: Path, tmproot: Path, use_latest_conf: bool = False, keep_tmp: bool = False
) -> Tuple[bool, str, Path | None]:
"""
Create worktree for tag, run sphinx-build with latex builder, then make PDF.
Places PDF in the same directory as the HTML output for that tag.
Returns (success, log_contents, pdf_path).
:param tag: Git tag to build PDF for
:type tag: str
:param html_tag_dir: Directory containing HTML output for the tag
:type html_tag_dir: Path
:param tmproot: Root temporary directory
:type tmproot: Path
:param use_latest_conf: Whether to use latest conf.py, defaults to False
:type use_latest_conf: bool
:param keep_tmp: Whether to keep temporary files, defaults to False
:type keep_tmp: bool
:return: Tuple of (success, log_contents, pdf_path)
:rtype: Tuple[bool, str, Path | None]
"""
worktree_path = tmproot / f'{tag}_pdf'
pdf_path = None
try:
proc = run(['git', 'worktree', 'add', '-q', str(worktree_path), tag])
if proc.returncode != 0:
return False, f'git worktree add failed:\n{proc.stdout}', None
docs_src = worktree_path / DOCS_SUBPATH
if not docs_src.exists():
return False, f'docs source {docs_src} does not exist for tag {tag}', None
if use_latest_conf or tag in OLD_VERSION_TO_BE_PATCHED:
copy_patch_files(docs_src)
# Build latex
latex_out = tmproot / f'{tag}_latex'
latex_out.mkdir(parents=True, exist_ok=True)
sp = run([SPHINX_BUILD_CMD, '-b', 'latex', str(docs_src), str(latex_out)], cwd=worktree_path)
log = sp.stdout
if sp.returncode != 0:
return False, f'Sphinx latex build failed:\n{log}', None
# Run pdflatex (via make if Makefile exists, otherwise directly)
makefile = latex_out / 'Makefile'
if makefile.exists():
sp_pdf = run(['make'], cwd=latex_out)
else:
# Find .tex file and run pdflatex
tex_files = list(latex_out.glob('*.tex'))
if not tex_files:
return False, 'No .tex file found in latex output', None
sp_pdf = run(['pdflatex', '-interaction=nonstopmode', tex_files[0].name], cwd=latex_out)
log += '\n' + sp_pdf.stdout
# Find generated PDF
pdf_files = list(latex_out.glob('*.pdf'))
if not pdf_files:
return False, f'PDF generation failed:\n{log}', None
# Copy PDF to the HTML tag directory
html_tag_dir.mkdir(parents=True, exist_ok=True)
pdf_path = html_tag_dir / f'{tag}.pdf'
pdf_file = latex_out / 'mafw.pdf'
shutil.copy(pdf_file, pdf_path)
success = sp_pdf.returncode == 0
return success, log, pdf_path
finally:
if not keep_tmp:
run(['git', 'worktree', 'remove', '-f', str(worktree_path)])
[docs]
def generate_pdf_index_page(
html_outdir: Path, pdf_info: List[dict[str, str]], project_name: str = 'Documentation'
) -> None:
"""
Generate an HTML page listing all available PDFs.
This page will be placed in the root html_versions directory.
Order: stable first, then latest, then other releases sorted by version (newest first).
:param html_outdir: Output directory for HTML files
:type html_outdir: Path
:param pdf_info: List of dictionaries containing PDF information
:type pdf_info: List[dict[str, str]]
:param project_name: Name of the project for the page title, defaults to 'Documentation'
:type project_name: str
"""
html_content = f"""<!DOCTYPE html>
<html>
<head>
<meta charset="utf-8">
<title>PDF Downloads - {project_name}</title>
<link rel="shortcut icon" href="stable/_static/mafw-logo.svg"/>
<style>
body {{
font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif;
max-width: 900px;
margin: 40px auto;
padding: 20px;
line-height: 1.6;
}}
h1 {{
color: #2c3e50;
border-bottom: 3px solid #3498db;
padding-bottom: 10px;
}}
.pdf-list {{
list-style: none;
padding: 0;
}}
.pdf-item {{
background: #f8f9fa;
border: 1px solid #dee2e6;
border-radius: 5px;
padding: 15px 20px;
margin: 10px 0;
display: flex;
justify-content: space-between;
align-items: center;
transition: all 0.3s;
}}
.pdf-item:hover {{
background: #e9ecef;
transform: translateX(5px);
}}
.pdf-version {{
font-weight: bold;
font-size: 1.1em;
color: #2c3e50;
}}
.pdf-label {{
display: inline-block;
padding: 3px 8px;
border-radius: 3px;
font-size: 0.85em;
margin-left: 10px;
}}
.label-stable {{
background: #28a745;
color: white;
}}
.label-latest {{
background: #ffc107;
color: #000;
}}
.label-release {{
background: #6c757d;
color: white;
}}
.download-btn {{
background: #3498db;
color: white;
padding: 8px 20px;
text-decoration: none;
border-radius: 5px;
transition: background 0.3s;
}}
.download-btn:hover {{
background: #2980b9;
}}
.failed {{
opacity: 0.5;
}}
.failed .download-btn {{
background: #95a5a6;
pointer-events: none;
}}
</style>
</head>
<body>
<h1>📄 PDF Documentation Downloads</h1>
<p>Download the complete documentation in PDF format for any version:</p>
<ul class="pdf-list">
"""
# Sort: stable first, then latest, then releases by version (newest first)
sorted_info = []
stable_item = None
latest_item = None
release_items = []
for info in pdf_info:
if info['label'] == 'alias':
continue
if info['label'] == 'stable':
stable_item = info
elif info['label'] == 'latest':
latest_item = info
else: # release
release_items.append(info)
# Sort releases by version (newest first)
release_items.sort(key=lambda x: parse_version_tuple(x['version']), reverse=True)
# Build final order
if stable_item:
sorted_info.append(stable_item)
if latest_item:
sorted_info.append(latest_item)
sorted_info.extend(release_items)
for info in sorted_info:
label_class = f'label-{info["label"]}'
label_text = info['label'].upper()
item_class = '' if info['built'] else 'failed'
if info['built']:
# PDF is in the same directory as HTML for each version
pdf_link = f'{info["version"]}/{info["version"]}.pdf'
html_content += f"""
<li class="pdf-item {item_class}">
<div>
<span class="pdf-version">{info['version']}</span>
<span class="pdf-label {label_class}">{label_text}</span>
</div>
<a href="{pdf_link}" class="download-btn" download>Download PDF</a>
</li>
"""
else:
html_content += f"""
<li class="pdf-item {item_class}">
<div>
<span class="pdf-version">{info['version']}</span>
<span class="pdf-label {label_class}">{label_text}</span>
<span style="color: #e74c3c; margin-left: 10px;">(Build failed)</span>
</div>
<span class="download-btn">Unavailable</span>
</li>
"""
html_content += """
</ul>
<p style="margin-top: 40px; color: #6c757d; font-size: 0.9em;">
💡 Tip: The PDF version contains the complete documentation for offline reading.
</p>
</body>
</html>
"""
# Write to root of html_versions
pdf_page = html_outdir / 'pdf_downloads.html'
with open(pdf_page, 'w', encoding='utf-8') as f:
f.write(html_content)
print(f'📝 Generated PDF index page: {pdf_page}')
[docs]
def write_versions_json(outdir: Path, versions: List[dict[str, str]]) -> None:
"""
Write versions information to a JSON file.
:param outdir: Output directory for the JSON file
:type outdir: Path
:param versions: List of version information dictionaries
:type versions: List[dict[str, str]]
"""
p = outdir / 'versions.json'
with open(p, 'w', encoding='utf-8') as f:
json.dump(versions, f, indent=2)
print(f'🧾 Wrote versions.json to {p}')
for v in versions:
if v['label'] == 'alias':
sub = v['version']
else:
sub = v['path']
shutil.copy(p, outdir / sub)
shutil.copy(p, outdir / sub / 'generated')
[docs]
def mirror_version(outdir: Path, src_tag: str, target_tag: str, use_symlink: bool = True) -> None:
"""
Mirror a version directory from one tag to another.
Can use symlinks for efficiency or copy for compatibility.
:param outdir: Output directory containing version directories
:type outdir: Path
:param src_tag: Source tag directory name
:type src_tag: str
:param target_tag: Target tag directory name
:type target_tag: str
:param use_symlink: Whether to use symlink instead of copying, defaults to True
:type use_symlink: bool
"""
src = outdir / src_tag
dst = outdir / target_tag
# Remove existing destination if it exists
if dst.exists() or dst.is_symlink():
if dst.is_symlink():
dst.unlink()
else:
shutil.rmtree(dst)
if use_symlink:
print(f'🔗 Symlinking {target_tag} -> {src_tag}')
# Create relative symlink
dst.symlink_to(src_tag, target_is_directory=True)
else:
print(f'🪞 Mirroring {src_tag} to {target_tag}')
dst.mkdir(parents=True, exist_ok=True)
shutil.copytree(src, dst, dirs_exist_ok=True)
[docs]
def write_redirect_page(outdir: Path, name: str, target_tag: str) -> None:
"""
Create a redirect page for a version alias.
:param outdir: Output directory for the redirect page
:type outdir: Path
:param name: Name of the redirect alias (e.g., 'stable', 'dev')
:type name: str
:param target_tag: Tag that the redirect should point to
:type target_tag: str
"""
d = outdir / name
d.mkdir(parents=True, exist_ok=True)
target = f'../{target_tag}/index.html' # relative path from stable/index.html to tag/index
html = f"""<!doctype html>
<html>
<head>
<meta charset="utf-8">
<meta http-equiv="refresh" content="0; url={target}">
<link rel="canonical" href="{target}">
<title>Redirecting to {target_tag}</title>
</head>
<body>
<p>Redirecting to <a href="{target}">{target}</a></p>
</body>
</html>
"""
with open(d / 'index.html', 'w', encoding='utf-8') as f:
f.write(html)
print(f'🧾 Wrote redirect page {d / "index.html"} -> {target}')
[docs]
def write_legacy_redirect_page(outdir: Path) -> None:
"""
Create a legacy redirect page at the root of the output directory.
:param outdir: Output directory for the redirect page
:type outdir: Path
"""
html = """<!doctype html>
<html>
<head>
<meta charset="utf-8">
<script>
// Detect if we're in /doc/ subdirectory and redirect accordingly
const path = window.location.pathname;
const targetUrl = path.startsWith('/doc/')
? '/doc/stable/index.html'
: 'stable/index.html';
window.location.replace(targetUrl);
</script>
<meta http-equiv="refresh" content="0; url=stable/index.html">
<link rel="canonical" href="stable/index.html">
<title>Redirecting to stable documentation</title>
</head>
<body>
<p>Redirecting to <a href="stable/index.html">Documentation of the last stable release</a></p>
</body>
</html>
"""
d = outdir / Path('index.html')
with open(d, 'w', encoding='utf-8') as f:
f.write(html)
print(f'🧾 Wrote legacy redirect page {d}')
[docs]
def write_redirects_file(outdir: Path) -> None:
"""
Create a _redirects file for GitLab Pages.
:param outdir: Output directory for the redirects file
:type outdir: Path
"""
redirects_content = """# Redirects for GitLab Pages
# See: https://docs.gitlab.com/ee/user/project/pages/redirects.html
# Redirect old PDF URL to new PDF downloads page
/doc/mafw.pdf /doc/pdf_downloads.html 301
# Redirect /doc root to stable documentation
# Note: These are specific patterns to avoid redirecting /doc/pdf_downloads.html
/doc/ /doc/stable/ 301
/doc/index.html /doc/stable/index.html 301
/doc/doc_tutorial.html /doc/stable/doc_tutorial.html 301
"""
redirects_file = outdir / '_redirects'
with open(redirects_file, 'w', encoding='utf-8') as f:
f.write(redirects_content)
print(f'🔀 Wrote _redirects file: {redirects_file}')
print(' Note: Copy this file to the public/ directory root for GitLab Pages')
[docs]
def write_root_landing_page(build_root: Path, project_name: str = 'MAFw') -> None:
"""
Create a landing page for the project root with links to documentation and coverage.
:param build_root: Root build directory (should contain 'doc' subdirectory)
:type build_root: Path
:param project_name: Project name for the page title
:type project_name: str
"""
html_content = f"""<!DOCTYPE html>
<html>
<head>
<meta charset="utf-8">
<title>{project_name} - Documentation Hub</title>
<link rel="shortcut icon" href="doc/stable/_static/mafw-logo.svg"/>
<style>
body {{
font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif;
max-width: 1000px;
margin: 0 auto;
padding: 40px 20px;
line-height: 1.6;
background: linear-gradient(135deg, #667eea 0%, #764ba2 100%);
min-height: 100vh;
}}
.container {{
background: white;
border-radius: 10px;
padding: 40px;
box-shadow: 0 10px 40px rgba(0,0,0,0.1);
}}
h1 {{
color: #2c3e50;
border-bottom: 3px solid #3498db;
padding-bottom: 15px;
margin-top: 0;
}}
.section {{
margin: 30px 0;
padding: 25px;
background: #f8f9fa;
border-radius: 8px;
border-left: 4px solid #3498db;
}}
.section h2 {{
color: #2c3e50;
margin-top: 0;
display: flex;
align-items: center;
gap: 10px;
}}
.links {{
display: flex;
flex-wrap: wrap;
gap: 15px;
margin-top: 15px;
}}
.link-btn {{
display: inline-block;
background: #3498db;
color: white;
padding: 12px 24px;
text-decoration: none;
border-radius: 5px;
transition: all 0.3s;
font-weight: 500;
}}
.link-btn:hover {{
background: #2980b9;
transform: translateY(-2px);
box-shadow: 0 4px 12px rgba(52, 152, 219, 0.4);
}}
.link-btn.secondary {{
background: #95a5a6;
}}
.link-btn.secondary:hover {{
background: #7f8c8d;
}}
.description {{
color: #555;
margin: 10px 0;
}}
.icon {{
font-size: 1.5em;
}}
</style>
</head>
<body>
<div class="container">
<h1>📚 {project_name} Documentation Hub</h1>
<p class="description">
Welcome to the {project_name} project documentation portal.
Access the latest documentation, download PDFs, or view test coverage reports.
</p>
<div class="section">
<h2><span class="icon">📖</span> Documentation</h2>
<p class="description">
Browse the complete documentation with tutorials, API reference, and guides.
</p>
<div class="links">
<a href="doc/stable/index.html" class="link-btn">
📘 Latest Stable Documentation
</a>
<a href="doc/latest/index.html" class="link-btn secondary">
🔬 Development Version
</a>
<a href="doc/pdf_downloads.html" class="link-btn secondary">
📄 Download PDFs
</a>
</div>
</div>
<div class="section">
<h2><span class="icon">🧪</span> Test Coverage</h2>
<p class="description">
View detailed test coverage reports showing which parts of the codebase are tested.
</p>
<div class="links">
<a href="coverage/index.html" class="link-btn">
📊 View Coverage Report
</a>
</div>
</div>
<div style="margin-top: 40px; padding-top: 20px; border-top: 1px solid #dee2e6; color: #6c757d; font-size: 0.9em;">
<p>
💡 <strong>Tip:</strong> Bookmark the stable documentation link for quick access to the latest version.
</p>
</div>
</div>
</body>
</html>
"""
landing_page = build_root / 'index.html'
with open(landing_page, 'w', encoding='utf-8') as f:
f.write(html_content)
print(f'🏠 Generated root landing page: {landing_page}')
print(' Note: This should be copied to public/index.html in GitLab CI')
[docs]
def ensure_sphinx_build_available() -> None:
"""Ensure that the Sphinx Python package is available.
``doc_versioning`` is a development helper shipped with MAFw. The script is
typically executed from the optional ``[dev]`` environment (either by
activating the development environment and using the console entry point,
or via ``hatch run dev.py<version>:multidoc`` on CI/CD).
Checking for the ``sphinx-build`` executable alone is not sufficient because
the effective availability depends on which Python environment is executing
the command. Checking the import spec for ``sphinx`` validates that the
correct optional dependencies are installed for the running interpreter.
:raises click.ClickException: If Sphinx is not available.
"""
import importlib.util
if importlib.util.find_spec('sphinx') is None:
raise click.ClickException(
'Unable to import the "sphinx" package. '
'This usually means you are running outside the MAFw development environment. '
'Install MAFw with the optional [dev] feature, or invoke the helper via Hatch '
'(e.g. "hatch run dev.py3.14:multidoc --help").'
)
@click.group(context_settings=CONTEXT_SETTINGS)
@click.pass_context
def cli(ctx: click.Context) -> None:
"""Build and manage versioned documentation."""
if ctx.resilient_parsing:
return
# Do not require Sphinx for every subcommand: helpers like `server`, `registry`,
# `prune`, and `redirects` are useful outside the development environment.
# Doc-building commands validate Sphinx availability at runtime.
[docs]
def get_directory_size(path: Path) -> int:
"""
Calculate total size of a directory in bytes.
:param path: Directory path
:type path: Path
:return: Total size in bytes
:rtype: int
"""
total = 0
for item in path.rglob('*'):
if item.is_file():
total += item.stat().st_size
return total
[docs]
def prune_old_versions(outdir: Path, max_size_mb: int = 100, dry_run: bool = False) -> Tuple[List[str], int]:
"""
Remove oldest version directories until total size is below threshold.
Always keeps 'stable', 'latest', and 'dev' (if present).
:param outdir: Output directory containing version directories
:type outdir: Path
:param max_size_mb: Maximum size in megabytes
:type max_size_mb: int
:param dry_run: If True, only report what would be deleted
:type dry_run: bool
:return: Tuple of (list of removed versions, final size in bytes)
:rtype: Tuple[List[str], int]
"""
outdir = Path(outdir).resolve()
max_size_bytes = max_size_mb * 1024 * 1024
# Get current total size
current_size = get_directory_size(outdir)
print(f'📊 Current total size: {format_size(current_size)}')
print(f'🎯 Target maximum: {format_size(max_size_bytes)}')
if current_size <= max_size_bytes:
print('✅ Size is within limit. No pruning needed.')
return [], current_size
# Find all version directories
protected_versions = {'stable', 'latest', 'dev'}
version_dirs = []
for item in outdir.iterdir():
if item.is_dir() and item.name not in protected_versions:
# Skip if it's a symlink (it's an alias)
if item.is_symlink():
continue
size = get_directory_size(item)
version_dirs.append((item.name, size, item))
# Sort by version (oldest first) using semantic versioning
version_dirs.sort(key=lambda x: parse_version_tuple(x[0]))
print(f'\n📦 Found {len(version_dirs)} version directories (excluding protected):')
for name, size, _ in version_dirs:
print(f' • {name}: {format_size(size)}')
print(f'\n🛡️ Protected versions (will never be removed): {", ".join(protected_versions)}')
# Remove oldest versions until we're under the limit
removed = []
for name, size, path in version_dirs:
if current_size <= max_size_bytes:
break
print(f'\n🗑️ {"[DRY RUN] Would remove" if dry_run else "Removing"} {name} ({format_size(size)})...')
if not dry_run:
shutil.rmtree(path)
removed.append(name)
current_size -= size
print(f' New total size: {format_size(current_size)}')
if not removed:
print(f'\n⚠️ Warning: Cannot reduce size below {format_size(max_size_bytes)}')
print(' All remaining versions are protected or size target is too aggressive.')
return removed, current_size
[docs]
def regenerate_versions_json_after_pruning(outdir: Path, removed_versions: List[str]) -> None:
"""
Regenerate versions.json after pruning, excluding removed versions.
:param outdir: Output directory containing version directories
:type outdir: Path
:param removed_versions: List of version names that were removed
:type removed_versions: List[str]
"""
versions_file = outdir / 'versions.json'
if not versions_file.exists():
print('⚠️ versions.json not found, skipping regeneration')
return
# Read existing versions.json
with open(versions_file, 'r', encoding='utf-8') as f:
versions = json.load(f)
# Filter out removed versions
original_count = len(versions)
versions = [v for v in versions if v['version'] not in removed_versions and v.get('path') not in removed_versions]
removed_count = original_count - len(versions)
if removed_count == 0:
print('ℹ️ No versions removed from versions.json')
return
print('\n🔄 Regenerating versions.json...')
print(f' Removed {removed_count} entries')
# Write updated versions.json
write_versions_json(outdir, versions)
[docs]
def check_multiversion_structure(outdir: Path) -> bool:
"""
Check if multiversion structure exists (other version directories).
:param outdir: Output directory to check
:type outdir: Path
:return: True if other versions exist
:rtype: bool
"""
if not outdir.exists():
return False
# Count non-latest version directories
version_dirs = []
for item in outdir.iterdir():
if item.is_dir() and item.name != 'latest':
# Check if it's not a symlink or if it is, count it
version_dirs.append(item.name)
return len(version_dirs) > 0
@cli.command()
@click.option('--outdir', '-o', default='docs/build/doc', help='Output directory to check')
@click.option('--max-size', '-s', default=100, help='Maximum size in MB (default: 100)')
@click.option('--dry-run/--no-dry-run', default=False, help='Show what would be removed without actually removing')
@click.option('--auto-prune/--no-auto-prune', default=False, help='Automatically prune without confirmation')
def prune(outdir: Path, max_size: int, dry_run: bool, auto_prune: bool) -> None:
"""Prune old documentation versions to stay within size limit.
This command removes the oldest version directories (keeping stable, latest, dev)
until the total size is below the specified threshold.
:param outdir: Output directory to prune
:type outdir: Path
:param max_size: Maximum size in megabytes
:type max_size: int
:param dry_run: Whether to do a dry run
:type dry_run: bool
:param auto_prune: Whether to prune automatically without confirmation
:type auto_prune: bool
"""
outdir = Path(outdir).resolve()
if not outdir.exists():
print(f'❌ Directory does not exist: {outdir}')
return
print(f'🔍 Analyzing {outdir}...\n')
# First do a dry run to see what would be removed
removed_versions, final_size = prune_old_versions(outdir, max_size, dry_run=True)
if not removed_versions:
return
# If it's already a dry run, we're done
if dry_run:
print('\n📋 Summary:')
print(f' Versions to remove: {", ".join(removed_versions)}')
print(f' Final size: {format_size(final_size)}')
print('\n💡 Run without --dry-run to actually remove these versions')
return
# Ask for confirmation unless auto-prune is enabled
if not auto_prune:
print(f'\n⚠️ This will permanently delete {len(removed_versions)} version(s): {", ".join(removed_versions)}')
response = input('Continue? [y/N]: ')
if response.lower() not in ('y', 'yes'):
print('❌ Aborted')
return
# Actually prune
print('\n🔨 Pruning versions...')
removed_versions, final_size = prune_old_versions(outdir, max_size, dry_run=False)
# Regenerate versions.json
regenerate_versions_json_after_pruning(outdir, removed_versions)
print('\n✅ Pruning complete!')
print(f' Removed: {", ".join(removed_versions)}')
print(f' Final size: {format_size(final_size)} (target: {max_size} MB)')
[docs]
def _coerce_with_zip_file(ctx: click.Context, param: click.Parameter, value: bool) -> bool:
"""Coerce ``--with-zip-file`` and ``--with-upload-zip`` constraints.
If zip creation is disabled, uploading must also be disabled.
:param ctx: Click context
:type ctx: click.Context
:param param: Click parameter being processed
:type param: click.Parameter
:param value: Parsed option value
:type value: bool
:return: Option value (possibly coerced)
:rtype: bool
"""
_ = param
if value is False:
ctx.params['with_upload_zip'] = False
return value
[docs]
def _coerce_with_upload_zip(ctx: click.Context, param: click.Parameter, value: bool) -> bool:
"""Coerce ``--with-upload-zip`` and ``--with-zip-file`` constraints.
If upload is enabled, zip creation is automatically enabled.
:param ctx: Click context
:type ctx: click.Context
:param param: Click parameter being processed
:type param: click.Parameter
:param value: Parsed option value
:type value: bool
:return: Option value (possibly coerced)
:rtype: bool
"""
_ = param
if value is True:
ctx.params['with_zip_file'] = True
return value
@cli.group()
@click.option(
'--gitlab-api-url',
default=None,
envvar='CI_API_V4_URL',
help='GitLab API v4 base URL (env: CI_API_V4_URL).',
)
@click.option(
'--gitlab-project-id',
default=None,
type=int,
envvar='CI_PROJECT_ID',
help='GitLab project numeric id (env: CI_PROJECT_ID).',
)
@click.option(
'--gitlab-token',
default=None,
envvar='CI_JOB_TOKEN',
help='GitLab token value (env: CI_JOB_TOKEN).',
)
@click.pass_context
def registry(
ctx: click.Context, gitlab_api_url: str | None, gitlab_project_id: int | None, gitlab_token: str | None
) -> None:
"""Interact with the GitLab Generic Package Registry for mafw-docs packages.
The GitLab API configuration is collected from CLI options (or CI env vars)
and built lazily by subcommands.
\f
:param ctx: Click context
:type ctx: click.Context
:param gitlab_api_url: GitLab API v4 base URL
:type gitlab_api_url: str | None
:param gitlab_project_id: GitLab project numeric id
:type gitlab_project_id: int | None
:param gitlab_token: GitLab token value
:type gitlab_token: str | None
"""
ctx.obj = ctx.obj or {}
ctx.obj['gitlab_api_url'] = gitlab_api_url
ctx.obj['gitlab_project_id'] = gitlab_project_id
ctx.obj['gitlab_token'] = gitlab_token
[docs]
def _registry_build_api_config(ctx: click.Context) -> GitlabAPIConfiguration:
"""Build the GitLab API configuration from values stored in the Click context.
:param ctx: Click context
:type ctx: click.Context
:return: GitLab API configuration
:rtype: GitlabAPIConfiguration
:raises click.ClickException: If the configuration is missing
"""
obj = ctx.obj or {}
try:
return build_gitlab_api_configuration(
obj.get('gitlab_api_url'),
obj.get('gitlab_project_id'),
obj.get('gitlab_token'),
)
except ValueError as e:
raise click.ClickException(str(e)) from e
[docs]
def _registry_validate_selection(
files: Tuple[str, ...], all_entries: bool, from_v: str | None, to_v: str | None
) -> None:
"""Validate that exactly one selection mode is used for registry commands.
:param files: Repeated -f/--file items
:type files: Tuple[str, ...]
:param all_entries: Whether --all was provided
:type all_entries: bool
:param from_v: Range start
:type from_v: str | None
:param to_v: Range end
:type to_v: str | None
:raises click.ClickException: If selection is invalid
"""
modes = 0
if files:
modes += 1
if all_entries:
modes += 1
if from_v is not None or to_v is not None:
modes += 1
if modes != 1:
raise click.ClickException('Choose exactly one selection mode: -f/--file, --all, or --from/--to.')
@registry.command()
@click.option(
'--zip-filepath',
default='.',
type=click.Path(file_okay=False, dir_okay=True, path_type=Path),
help='Directory containing zip files to upload. (.)',
)
@click.option('-f', '--file', 'files', multiple=True, help='Zip file(s) to upload (repeatable).')
@click.option(
'--all', 'all_entries', is_flag=True, default=False, help='Upload all matching zip files from --zip-filepath.'
)
@click.option('--from', 'from_v', default=None, help='Upload versions from this tag (inclusive).')
@click.option('--to', 'to_v', default=None, help='Upload versions up to this tag (inclusive).')
@click.pass_context
def upload(
ctx: click.Context,
zip_filepath: Path,
files: Tuple[str, ...],
all_entries: bool,
from_v: str | None,
to_v: str | None,
) -> None:
"""Upload local documentation zip files to the registry."""
_registry_validate_selection(files, all_entries, from_v, to_v)
api_config = _registry_build_api_config(ctx)
zip_filepath = Path(zip_filepath).resolve()
candidates: List[Tuple[str, Path]] = []
if files:
for raw in files:
fp = Path(raw)
if not fp.is_absolute():
fp2 = zip_filepath / fp
fp = fp2 if fp2.exists() else fp
parsed = parse_mafw_docs_zip_filename(fp.name)
if parsed is None:
raise click.ClickException(f'File does not match mafw-docs-vX.Y.Z.zip: {fp.name}')
version, _ = parsed
if not fp.exists():
raise click.ClickException(f'File not found: {fp}')
candidates.append((version, fp))
else:
candidates = iter_local_mafw_docs_zips(zip_filepath)
if from_v is not None or to_v is not None:
versions = [v for v, _ in candidates]
try:
allowed = set(filter_versions_in_range(versions, from_v, to_v))
except ValueError as e:
raise click.ClickException(str(e)) from e
candidates = [(v, p) for v, p in candidates if v in allowed]
if not candidates:
print('ℹ️ No matching zip files found.')
return
for version, fp in candidates:
uploaded = upload_docs_zip_to_gitlab_generic_registry(api_config, version, fp, package_name='mafw-docs')
if uploaded:
print(f'☁️ Uploaded {fp.name} -> mafw-docs/{version}/{fp.name}')
@registry.command()
@click.option(
'--zip-filepath',
default='.',
type=click.Path(file_okay=False, dir_okay=True, path_type=Path),
help='Directory where downloaded zip files are stored. (.)',
)
@click.option('-f', '--file', 'items', multiple=True, help='Version(s) or zip file name(s) to download (repeatable).')
@click.option(
'--all', 'all_entries', is_flag=True, default=False, help='Download all mafw-docs versions from the registry.'
)
@click.option('--from', 'from_v', default=None, help='Download versions from this tag (inclusive).')
@click.option('--to', 'to_v', default=None, help='Download versions up to this tag (inclusive).')
@click.pass_context
def download(
ctx: click.Context,
zip_filepath: Path,
items: Tuple[str, ...],
all_entries: bool,
from_v: str | None,
to_v: str | None,
) -> None:
"""Download documentation zip files from the registry."""
_registry_validate_selection(items, all_entries, from_v, to_v)
api_config = _registry_build_api_config(ctx)
zip_filepath = Path(zip_filepath).resolve()
targets: List[Tuple[str, str]] = []
if items:
for it in items:
try:
targets.append(normalize_registry_item(it))
except ValueError as e:
raise click.ClickException(str(e)) from e
else:
pkgs = list_mafw_docs_generic_packages(api_config, package_name='mafw-docs')
versions: List[str] = []
for p in pkgs:
version = p.get('version')
if isinstance(version, str):
versions.append(version)
try:
versions = filter_versions_in_range(sorted(set(versions), key=parse_version_tuple), from_v, to_v)
except ValueError as e:
raise click.ClickException(str(e)) from e
targets = [(v, f'mafw-docs-{v}.zip') for v in versions]
if not targets:
print('ℹ️ No matching registry entries found.')
return
for version, file_name in targets:
dest = download_docs_zip_from_gitlab_generic_registry(
api_config, version, zip_filepath, package_name='mafw-docs', file_name=file_name
)
if dest is None:
print(f'ℹ️ Not found: mafw-docs/{version}/{file_name}')
else:
print(f'⬇️ Downloaded: {dest}')
@registry.command()
@click.option('-f', '--file', 'items', multiple=True, help='Version(s) or zip file name(s) to delete (repeatable).')
@click.option(
'--all', 'all_entries', is_flag=True, default=False, help='Delete all mafw-docs versions from the registry.'
)
@click.option('--from', 'from_v', default=None, help='Delete versions from this tag (inclusive).')
@click.option('--to', 'to_v', default=None, help='Delete versions up to this tag (inclusive).')
@click.pass_context
def delete(ctx: click.Context, items: Tuple[str, ...], all_entries: bool, from_v: str | None, to_v: str | None) -> None:
"""Delete mafw-docs package versions from the registry."""
_registry_validate_selection(items, all_entries, from_v, to_v)
api_config = _registry_build_api_config(ctx)
mapping = resolve_mafw_docs_package_ids_by_version(api_config, package_name='mafw-docs')
targets: List[str] = []
if items:
for it in items:
try:
version, _ = normalize_registry_item(it)
except ValueError as e:
raise click.ClickException(str(e)) from e
targets.append(version)
else:
versions = sorted(mapping.keys(), key=parse_version_tuple)
try:
targets = filter_versions_in_range(versions, from_v, to_v)
except ValueError as e:
raise click.ClickException(str(e)) from e
if not targets:
print('ℹ️ No matching registry entries found.')
return
try:
import requests
except ModuleNotFoundError as e:
raise ModuleNotFoundError(
'The "requests" package is required for registry operations. Install development extras (e.g. "pip install .[dev]").'
) from e
base_url = api_config.api_url.rstrip('/')
headers = build_gitlab_auth_headers(api_config)
for version in targets:
pkg_id = mapping.get(version)
if pkg_id is None:
print(f'ℹ️ Not found: mafw-docs/{version}')
continue
url = f'{base_url}/projects/{api_config.project_id}/packages/{pkg_id}'
resp = requests.delete(url, headers=headers, timeout=60.0)
if resp.status_code == 204:
print(f'🗑️ Deleted package: mafw-docs/{version} (id={pkg_id})')
continue
if resp.status_code in (403, 404):
body_preview = (resp.text or '')[:200].replace('\n', ' ')
print(f'⚠️ Could not delete mafw-docs/{version} (id={pkg_id}): {resp.status_code} {body_preview}')
continue
body_preview = (resp.text or '')[:500].replace('\n', ' ')
raise click.ClickException(
f'GitLab delete failed for mafw-docs/{version} (id={pkg_id}): {resp.status_code} {body_preview}'
)
SERVER_DEFAULT_DIRECTORY = Path('docs') / 'build'
SERVER_METADATA_FILENAME = '.multiversion-doc-server.json'
[docs]
class ServerStatusEnum(StrEnum):
"""Possible status values for the multiversion-doc local documentation server.
:cvar running: The server process exists, matches the expected signature and responds over HTTP.
:cvar stale: The server process exists and matches the expected signature, but does not respond over HTTP.
:cvar stopped: The recorded PID does not exist or does not match the expected signature.
:cvar unknown: The metadata file is missing, so no status can be determined.
"""
running = 'running'
stale = 'stale'
stopped = 'stopped'
unknown = 'unknown'
[docs]
def _server_default_directory() -> Path:
"""Return the default directory served by the local documentation server.
The default is resolved from the current working directory (typically the repository root).
:return: Default documentation build directory
:rtype: Path
"""
return SERVER_DEFAULT_DIRECTORY.resolve()
[docs]
def _import_psutil() -> Any:
"""Import psutil and provide a user-facing error if it is missing.
The local documentation server management commands rely on psutil for
portable process introspection.
:return: Imported psutil module
:rtype: Any
:raises click.ClickException: If psutil is not installed
"""
try:
import psutil
except ModuleNotFoundError as e:
raise click.ClickException(
'The "psutil" package is required for documentation server management. '
'Install development extras (e.g. "pip install .[dev]").'
) from e
return psutil
[docs]
def _process_matches_http_server(proc: Any, address: str, port: int, directory: Path) -> bool:
"""Check whether a process resembles the expected http.server invocation.
This function is intentionally tolerant across platforms: it matches based on
essential command line tokens rather than requiring an exact list.
:param proc: psutil Process instance
:type proc: Any
:param address: Expected bind address
:type address: str
:param port: Expected port
:type port: int
:param directory: Expected served directory
:type directory: Path
:return: True when the process signature matches
:rtype: bool
"""
try:
cmdline: List[str] = list(proc.cmdline())
except Exception:
return False
if '-m' not in cmdline:
return False
m_index = cmdline.index('-m')
if m_index + 1 >= len(cmdline) or cmdline[m_index + 1] != 'http.server':
return False
directory_str = str(Path(directory).resolve())
checks = [
('-b', address),
('-d', directory_str),
]
for flag, expected in checks:
if flag not in cmdline:
return False
i = cmdline.index(flag)
if i + 1 >= len(cmdline) or cmdline[i + 1] != expected:
return False
if str(port) not in cmdline:
return False
return True
[docs]
def _http_probe(address: str, port: int, timeout_s: float = 1.0) -> bool:
"""Probe whether the server responds at the given address:port.
:param address: Server address
:type address: str
:param port: Server port
:type port: int
:param timeout_s: Timeout in seconds
:type timeout_s: float
:return: True if a GET to the server root returns HTTP 200
:rtype: bool
"""
url = f'http://{address}:{port}/'
try:
with urlopen(url, timeout=timeout_s) as resp:
return getattr(resp, 'status', None) == 200
except (HTTPError, URLError, OSError):
return False
[docs]
def _get_server_status(directory: Path) -> Tuple[ServerStatusEnum, dict[str, Any] | None]:
"""Determine the current server status from metadata and process/HTTP checks.
:param directory: Served directory used to locate the metadata file
:type directory: Path
:return: Tuple of (status, metadata). Metadata is None when missing.
:rtype: Tuple[ServerStatusEnum, dict[str, Any] | None]
"""
metadata = _read_server_metadata(directory)
if metadata is None:
return ServerStatusEnum.unknown, None
pid = metadata.get('pid')
address = metadata.get('address')
port = metadata.get('port')
dir_value = metadata.get('directory')
if (
not isinstance(pid, int)
or not isinstance(address, str)
or not isinstance(port, int)
or not isinstance(dir_value, str)
):
raise click.ClickException(f'Invalid server metadata contents: {_server_metadata_path(directory)}')
psutil = _import_psutil()
if not psutil.pid_exists(pid):
return ServerStatusEnum.stopped, metadata
try:
proc = psutil.Process(pid)
except Exception:
return ServerStatusEnum.stopped, metadata
if not _process_matches_http_server(proc, address=address, port=port, directory=Path(dir_value)):
return ServerStatusEnum.stopped, metadata
if _http_probe(address=address, port=port):
return ServerStatusEnum.running, metadata
return ServerStatusEnum.stale, metadata
@cli.group()
def server() -> None:
"""Manage a local web server for browsing documentation over HTTP."""
@server.command(name='start')
@click.option('-b', '--bind', 'address', default='127.0.0.1', help='IP address to bind (default: 127.0.0.1).')
@click.option('-p', '--port', default=8000, type=int, help='Port number where to listen (default: 8000).')
@click.option(
'-d',
'--directory',
default=SERVER_DEFAULT_DIRECTORY,
type=click.Path(file_okay=False, dir_okay=True, path_type=Path),
help='Folder to serve (default: docs/build).',
)
@click.option(
'-l',
'--log-file',
default=Path('doc-server.log'),
type=click.Path(dir_okay=False, path_type=Path),
help='Log file for server output (default: doc-server.log).',
)
@click.option('-f', '--force-restart', is_flag=True, default=False, help='Force restart if already running.')
def server_start(address: str, port: int, directory: Path, log_file: Path, force_restart: bool) -> None:
"""Start a local documentation server (python -m http.server)."""
_server_start_impl(address=address, port=port, directory=directory, log_file=log_file, force_restart=force_restart)
[docs]
def _server_start_impl(address: str, port: int, directory: Path, log_file: Path, force_restart: bool) -> None:
"""Implementation for starting a local documentation server.
:param address: IP address to bind the server to
:type address: str
:param port: TCP port number
:type port: int
:param directory: Directory to serve over HTTP
:type directory: Path
:param log_file: Log file path for server stdout/stderr
:type log_file: Path
:param force_restart: Whether to stop an already running/stale server first
:type force_restart: bool
"""
directory = Path(directory).resolve()
log_file = Path(log_file).resolve()
if not directory.exists():
raise click.ClickException(f'Served directory does not exist: {directory}')
status, metadata = _get_server_status(directory)
if status in (ServerStatusEnum.running, ServerStatusEnum.stale):
assert metadata is not None
url = f'http://{metadata["address"]}:{metadata["port"]}'
print(f'ℹ️ Documentation server already present ({status}) at {url}')
if not force_restart:
return
_server_stop_by_metadata(directory, require_metadata=False)
log_file.parent.mkdir(parents=True, exist_ok=True)
with open(log_file, 'a', encoding='utf-8') as log_handle:
process = subprocess.Popen(
[
sys.executable,
'-m',
'http.server',
'-b',
address,
'-d',
str(directory),
str(port),
],
stdout=log_handle,
stderr=subprocess.STDOUT,
start_new_session=True,
)
metadata_to_write = {
'pid': int(process.pid),
'address': address,
'port': int(port),
'directory': str(directory),
'log_file': str(log_file),
'started_at': _format_started_at_utc(datetime.now(timezone.utc)),
}
_write_server_metadata(directory, metadata_to_write)
print(f'✅ Documentation server launched and responding at http://{address}:{port}')
@server.command(name='status')
def server_status() -> None:
"""Show the current status of the local documentation server."""
directory = _server_default_directory()
meta_path = _server_metadata_path(directory)
try:
status, metadata = _get_server_status(directory)
except click.ClickException as e:
raise click.ClickException(f'{e} (while reading: {meta_path})') from e
if status == ServerStatusEnum.unknown:
print('status: unknown (metadata not found)')
return
assert metadata is not None
url = f'http://{metadata["address"]}:{metadata["port"]}'
print(f'status: {status} ({url})')
@server.command(name='stop')
def server_stop() -> None:
"""Stop the local documentation server."""
directory = _server_default_directory()
_server_stop_by_metadata(directory, require_metadata=True)
@server.command(name='restart')
def server_restart() -> None:
"""Restart the local documentation server."""
directory = _server_default_directory()
metadata = _read_server_metadata(directory)
if metadata is None:
_server_stop_by_metadata(directory, require_metadata=False)
_server_start_impl(
address='127.0.0.1', port=8000, directory=directory, log_file=Path('doc-server.log'), force_restart=False
)
return
address = metadata.get('address', '127.0.0.1')
port = metadata.get('port', 8000)
directory_value = metadata.get('directory', str(directory))
log_file_value = metadata.get('log_file', str(Path('doc-server.log').resolve()))
_server_stop_by_metadata(directory, require_metadata=False)
_server_start_impl(
address=str(address),
port=int(port),
directory=Path(directory_value),
log_file=Path(log_file_value),
force_restart=False,
)
@cli.command()
@click.option('--outdir', '-o', default='docs/build/doc', help='Output directory (docs/build/doc)')
@click.option(
'--include-dev/--no-include-dev',
is_flag=True,
help='If true and current branch is ahead of stable, create dev redirect. (True)',
)
@click.option('--min-vers', default='v1.0.0', help='Minimum version to consider (default: v1.0.0).')
@click.option('--keep-temp/--no-keep-temp', default=False, help='Do not remove temp dir (for debugging).')
@click.option(
'--use-latest-conf/--no-use-latest-conf',
is_flag=True,
default=True,
help='Use the latest conf.py for all builds. (True)',
)
@click.option('--build-pdf/--no-build-pdf', is_flag=True, default=False, help='Also build PDF versions. (False)')
@click.option('--project-name', default='MAFw documentation', help='Project name for PDF index page.')
@click.option(
'--use-symlinks/--no-use-symlinks',
is_flag=True,
default=True,
help='Use symlinks for stable/dev aliases instead of copying. (True)',
)
@click.option(
'--max-size', '-s', default=0, help='Maximum artifact size in MB. If exceeded, prune old versions (0 = no limit)'
)
@click.option(
'--zip-filepath',
default='.',
type=click.Path(file_okay=False, dir_okay=True, path_type=Path),
help='Directory where per-tag zip archives are written when --with-zip-file is enabled. (.)',
)
@click.option(
'--with-zip-file/--without-zip-file',
is_flag=True,
default=False,
callback=_coerce_with_zip_file,
help='Create a per-tag zip archive (mafw-docs-<tag>.zip) for each stable tag under --outdir.',
)
@click.option(
'--with-upload-zip/--without-upload-zip',
is_flag=True,
default=False,
callback=_coerce_with_upload_zip,
help='Upload the per-tag documentation zip to the GitLab Generic Package Registry. Implies --with-zip-file.',
)
@click.option(
'--with-cached-packages/--without-cached-packages',
is_flag=True,
default=False,
help='Use cached documentation zip packages from the GitLab Generic Package Registry instead of rebuilding tags.',
)
@click.option(
'--gitlab-api-url',
default=None,
envvar='CI_API_V4_URL',
help='GitLab API v4 base URL (env: CI_API_V4_URL).',
)
@click.option(
'--gitlab-project-id',
default=None,
type=int,
envvar='CI_PROJECT_ID',
help='GitLab project numeric id (env: CI_PROJECT_ID).',
)
@click.option(
'--gitlab-token',
default=None,
envvar='CI_JOB_TOKEN',
help='GitLab token value (env: CI_JOB_TOKEN).',
)
def build(
outdir: Path,
include_dev: bool,
min_vers: str,
keep_temp: bool,
use_latest_conf: bool,
build_pdf: bool,
project_name: str,
use_symlinks: bool,
max_size: int,
zip_filepath: Path,
with_zip_file: bool,
with_upload_zip: bool,
with_cached_packages: bool,
gitlab_api_url: str | None,
gitlab_project_id: int | None,
gitlab_token: str | None,
) -> None:
"""Build multiversion documentation.
\f
:param outdir: Output directory for built documentation
:type outdir: Path
:param include_dev: Whether to include dev alias if current branch is ahead
:type include_dev: bool
:param min_vers: Minimum version to consider
:type min_vers: str
:param keep_temp: Whether to keep temporary files
:type keep_temp: bool
:param use_latest_conf: Whether to use latest conf.py for all builds
:type use_latest_conf: bool
:param build_pdf: Whether to also build PDF versions
:type build_pdf: bool
:param project_name: Project name for PDF index page
:type project_name: str
:param use_symlinks: Whether to use symlinks instead of copying
:type use_symlinks: bool
:param max_size: Maximum artifact size in MB (0 = no limit)
:type max_size: int
:param zip_filepath: Directory where per-tag zip archives are written
:type zip_filepath: Path
:param with_zip_file: Whether to create per-tag zip archives for stable tags
:type with_zip_file: bool
:param with_upload_zip: Whether to upload per-tag zip archives to the GitLab Generic Package Registry
:type with_upload_zip: bool
:param with_cached_packages: Whether to use cached documentation zip packages from the GitLab Generic Package Registry
:type with_cached_packages: bool
:param gitlab_api_url: GitLab API v4 base URL (typically from ``CI_API_V4_URL``)
:type gitlab_api_url: str | None
:param gitlab_project_id: GitLab project numeric id (typically from ``CI_PROJECT_ID``)
:type gitlab_project_id: int | None
:param gitlab_token: GitLab token value (typically from ``CI_JOB_TOKEN``)
:type gitlab_token: str | None
"""
ensure_sphinx_build_available()
outdir = Path(outdir).resolve()
outdir.mkdir(parents=True, exist_ok=True)
zip_filepath = Path(zip_filepath).resolve()
# Final safety net: enforce constraints even if options are provided in any order.
if with_upload_zip:
with_zip_file = True
if not with_zip_file:
with_upload_zip = False
print('🔍 Fetching remote tags...')
p = run(['git', 'fetch', '--tags', '--quiet'])
if p.returncode != 0:
print('⚠️ Warning: git fetch --tags failed. Continuing with local tags.')
print(f' Error output: {p.stdout[:200]}...' if p.stdout else ' (no output)')
print(' This is normal in CI if tags are already present or fetch is restricted.')
# Collect and filter tags
print('🔍 Collecting git tags...')
versions = get_git_tags(min_vers)
versions = filter_latest_micro(versions)
if not versions:
print('No valid tags found. Aborting.')
sys.exit(1)
stable_tags = [r[1] for r in versions]
print('🌿 Candidate stable tags (sorted):', stable_tags)
# highest version = last element after semver sort
highest = stable_tags[-1]
print('🏷️ Highest stable tag:', highest)
tmproot = Path(tempfile.mkdtemp(prefix='mafw-docs-'))
print('🔧 Temporary root:', tmproot)
versions_list = []
pdf_info_list = []
api_config: GitlabAPIConfiguration | None = None
if with_cached_packages or with_upload_zip:
try:
api_config = build_gitlab_api_configuration(gitlab_api_url, gitlab_project_id, gitlab_token)
except ValueError as e:
raise click.ClickException(str(e)) from e
# build each tag
for tag in stable_tags:
used_cache_for_tag = False
success = False
log = ''
if with_cached_packages and api_config is not None:
print(f'📦 Trying cached docs package for tag {tag} ...')
downloaded = download_docs_zip_from_gitlab_generic_registry(api_config, tag, zip_filepath)
if downloaded is not None:
try:
extract_docs_zip_to_repo_root(downloaded)
html_tag_dir = outdir / tag
if not html_tag_dir.exists():
raise FileNotFoundError(f'Expected extracted directory missing: {html_tag_dir}')
used_cache_for_tag = True
success = True
log = 'Used cached package from GitLab Generic Package Registry.'
print(f'✅ Using cached docs for tag {tag} (downloaded + extracted)')
except Exception as e:
print(f'⚠️ Warning: cached package failed for {tag} ({e}); falling back to build.')
if not used_cache_for_tag:
print(f'📘 Building HTML for tag {tag} ...')
success, log = build_for_tag(tag, outdir, tmproot, use_latest_conf=use_latest_conf, keep_tmp=keep_temp)
versions_list.append(
{
'version': tag,
'label': 'stable' if tag == highest else 'release',
'built': success,
}
)
report_build_status(tag, success, log, 'HTML')
# Build PDF if requested
pdf_built = False
if build_pdf:
html_tag_dir = outdir / tag
expected_pdf = html_tag_dir / f'{tag}.pdf'
if used_cache_for_tag and expected_pdf.exists():
pdf_built = True
pdf_log = 'PDF found in cached package; skipped PDF generation.'
report_build_status(tag, True, pdf_log, 'PDF')
else:
if used_cache_for_tag and not expected_pdf.exists():
print(
f'⚠️ Cached package for {tag} does not include the PDF; generating it locally. '
'The used documentation will differ from the cached package.'
)
print(f'📕 Building PDF for tag {tag} ...')
pdf_success, pdf_log, pdf_path = build_pdf_for_tag(
tag, html_tag_dir, tmproot, use_latest_conf=use_latest_conf, keep_tmp=keep_temp
)
pdf_built = pdf_success
report_build_status(tag, pdf_success, pdf_log, 'PDF')
# Write PDF log in the same directory
if html_tag_dir.exists():
with open(html_tag_dir / f'{tag}_pdf_build.log', 'w', encoding='utf-8') as f:
f.write(pdf_log)
pdf_info_list.append(
{
'version': tag,
'label': 'stable' if tag == highest else 'release',
'built': pdf_built,
}
)
if used_cache_for_tag:
if with_upload_zip:
print(f'ℹ️ Upload skipped for {tag} because cached documentation was used.')
continue
if with_zip_file:
try:
zip_path = create_docs_zip_for_tag(outdir, tag, zip_filepath)
print(f'📦 Created docs zip for {tag}: {zip_path}')
if with_upload_zip:
if api_config is None:
raise click.ClickException('GitLab configuration is required to upload documentation zips.')
uploaded = upload_docs_zip_to_gitlab_generic_registry(api_config, tag, zip_path)
if uploaded:
print(f'☁️ Uploaded docs zip for {tag} (mafw-docs/{tag}/{zip_path.name})')
except (FileNotFoundError, NotADirectoryError) as e:
if with_upload_zip:
raise click.ClickException(f'Cannot create/upload zip for {tag}: {e}') from e
print(f'⚠️ Warning: skipping zip creation for {tag}: {e}')
# create stable redirect (directory stable -> tag)
mirror_version(outdir, highest, 'stable', use_symlink=use_symlinks)
# add a 'latest' build from current branch as useful
print("📘 Building latest (current branch) into 'latest' ...")
curr_docs = Path('docs') / 'source'
if curr_docs.exists():
latest_out = outdir / 'latest'
latest_out.mkdir(parents=True, exist_ok=True)
sp = run([SPHINX_BUILD_CMD, '-b', 'html', str(curr_docs), str(latest_out)])
with open(latest_out / 'sphinx-build.log', 'w', encoding='utf-8') as f:
f.write(sp.stdout)
latest_ok = sp.returncode == 0
versions_list.append({'version': 'latest', 'label': 'latest', 'built': latest_ok})
report_build_status('latest', latest_ok, sp.stdout, 'HTML')
# Build PDF for latest if requested
latest_pdf_built = False
if build_pdf and curr_docs.exists():
print('📕 Building PDF for latest ...')
latex_out = tmproot / 'latest_latex'
latex_out.mkdir(parents=True, exist_ok=True)
sp = run([SPHINX_BUILD_CMD, '-b', 'latex', str(curr_docs), str(latex_out)])
pdf_log = sp.stdout
if sp.returncode == 0:
makefile = latex_out / 'Makefile'
if makefile.exists():
sp_pdf = run(['make'], cwd=latex_out)
else:
tex_files = list(latex_out.glob('*.tex'))
if tex_files:
sp_pdf = run(['pdflatex', '-interaction=nonstopmode', tex_files[0].name], cwd=latex_out)
pdf_log += '\n' + sp_pdf.stdout
pdf_files = list(latex_out.glob('*.pdf'))
if pdf_files:
pdf_file = latex_out / 'mafw.pdf'
shutil.copy(pdf_file, latest_out / 'latest.pdf')
latest_pdf_built = True
report_build_status('latest', True, pdf_log, 'PDF')
# Write PDF log
with open(latest_out / 'latest_pdf_build.log', 'w', encoding='utf-8') as f:
f.write(pdf_log)
pdf_info_list.append({'version': 'latest', 'label': 'latest', 'built': latest_pdf_built})
else:
print('❌ No local docs/source for latest. Skipping latest build.')
# detect dev (is current HEAD a descendant of highest tag?)
head_rev = git_rev_of('HEAD')
highest_rev = git_rev_of(highest)
dev_label = None
if is_ancestor(highest_rev, head_rev) and head_rev != highest_rev:
# HEAD is descendant (ahead) of highest -> label dev
dev_label = 'dev'
print('🔍 Current branch is ahead of stable -> creating dev alias')
if include_dev:
mirror_version(outdir, 'latest', dev_label, use_symlink=use_symlinks)
else:
print('🔍 Current branch is not ahead of stable (or identical) -> no dev alias created')
# add stable and dev labels in JSON with nice mapping
versions_json = []
for v in versions_list:
versions_json.append({'version': v['version'], 'label': v['label'], 'built': v['built'], 'path': v['version']})
# add convenience entries: stable -> highest, dev -> 'dev' if created
versions_json.append({'version': 'stable', 'label': 'alias', 'path': highest})
if dev_label and include_dev:
versions_json.append({'version': 'dev', 'label': 'alias', 'path': 'latest'})
write_versions_json(outdir, versions_json)
write_legacy_redirect_page(outdir)
# Generate root landing page (goes to parent of outdir)
build_root = outdir.parent
write_root_landing_page(build_root, project_name.replace(' documentation', ''))
# Generate redirect (goes to parent of outdir)
write_redirects_file(build_root)
# Generate PDF index page if PDFs were built
if build_pdf:
generate_pdf_index_page(outdir, pdf_info_list, project_name)
# Prune if size limit is specified
if max_size > 0:
print(f'\n📏 Checking artifact size (limit: {max_size} MB)...')
removed_versions, final_size = prune_old_versions(outdir, max_size, dry_run=False)
if removed_versions:
regenerate_versions_json_after_pruning(outdir, removed_versions)
# Regenerate PDF index if PDFs were built
if build_pdf:
# Update pdf_info_list to exclude removed versions
pdf_info_list = [p for p in pdf_info_list if p['version'] not in removed_versions]
generate_pdf_index_page(outdir, pdf_info_list, project_name)
if not keep_temp:
try:
shutil.rmtree(tmproot)
except Exception:
pass
print('🎉 All done. Built versions placed under:', outdir)
@cli.command()
@click.argument('target', type=click.Choice(['all', 'latest'], case_sensitive=False), default='all')
@click.option('--outdir', '-o', default='docs/build/doc', help='Output directory to clean')
def clean(target: str, outdir: Path) -> None:
"""Clean the output directory.
TARGET can be 'all' (remove everything) or 'latest' (remove only latest folder).
\f
:param target: What to clean - 'all' or 'latest'
:type target: str
:param outdir: Output directory to clean
:type outdir: Path
"""
outdir = Path(outdir).resolve()
if not outdir.exists():
print(f'ℹ️ Output directory does not exist: {outdir}')
return
if target == 'latest':
latest_dir = outdir / 'latest'
if latest_dir.exists():
try:
if latest_dir.is_symlink():
latest_dir.unlink()
else:
shutil.rmtree(latest_dir)
print(f'🧹 Cleaned latest directory: {latest_dir}')
except Exception as e:
print(f'❌ Failed to clean directory {latest_dir}: {e}')
else:
print(f'ℹ️ Latest directory does not exist: {latest_dir}')
else: # all
try:
shutil.rmtree(outdir)
print(f'🧹 Cleaned all output: {outdir}')
except Exception as e:
print(f'❌ Failed to clean directory {outdir}: {e}')
@cli.command()
@click.option('--outdir', '-o', default='docs/build/doc', help='Output directory for _redirects file')
@click.option('--old-pdf-path', default='/doc/mafw.pdf', help='Old PDF URL path to redirect from')
@click.option('--new-pdf-path', default='/doc/pdf_downloads.html', help='New PDF downloads page to redirect to')
@click.option('--redirect-root/--no-redirect-root', default=True, help='Redirect /doc/ root to stable')
def redirects(outdir: Path, old_pdf_path: Path, new_pdf_path: Path, redirect_root: bool) -> None:
"""Generate _redirects file for GitLab Pages.
\f
:param outdir: Output directory for _redirects file
:type outdir: Path
:param old_pdf_path: Old PDF URL path to redirect from
:type old_pdf_path: Path
:param new_pdf_path: New PDF downloads page to redirect to
:type new_pdf_path: Path
:param redirect_root: Whether to redirect /doc/ root to stable
:type redirect_root: bool
"""
outdir = Path(outdir).resolve()
redirects_content = f"""# Redirects for GitLab Pages
# See: https://docs.gitlab.com/ee/user/project/pages/redirects.html
# Redirect old PDF URL to new PDF downloads page
{old_pdf_path} {new_pdf_path} 301
"""
if redirect_root:
redirects_content += """
# Redirect /doc root to stable documentation
# Note: These are specific patterns to avoid redirecting /doc/pdf_downloads.html
/doc/ /doc/stable/ 301
/doc/index.html /doc/stable/index.html 301
"""
redirects_file = outdir / '_redirects'
outdir.mkdir(parents=True, exist_ok=True)
with open(redirects_file, 'w', encoding='utf-8') as f:
f.write(redirects_content)
print(f'🔀 Generated _redirects file: {redirects_file}')
print(f' Redirects {old_pdf_path} → {new_pdf_path} (301)')
if redirect_root:
print(' Redirects /doc/ → /doc/stable/ (301)')
print(' Redirects /doc/index.html → /doc/stable/index.html (301)')
print('\n📋 GitLab CI/CD setup:')
print(' Make sure your .gitlab-ci.yml copies this file to public/ root:')
print(' ')
print(' pages:')
print(' script:')
print(' - mkdir -p public')
print(f' - cp -r {outdir}/* public/doc/')
print(f' - cp {redirects_file} public/_redirects')
print(' artifacts:')
print(' paths:')
print(' - public')
@cli.command()
@click.option('--build-root', '-b', default='docs/build', help='Build root directory containing doc/ subdirectory')
@click.option('--project-name', default='MAFw', help='Project name for the landing page')
def landing(build_root: Path, project_name: str) -> None:
"""Generate root landing page for project.
\f
:param build_root: Build root directory
:type build_root: Path
:param project_name: Project name
:type project_name: str
"""
build_root = Path(build_root).resolve()
write_root_landing_page(build_root, project_name)
print('\n📋 GitLab CI/CD: Copy this to public/index.html:')
print(f' cp {build_root}/index.html public/index.html')
[docs]
def ensure_versions_json_exists(outdir: Path) -> bool:
"""
Ensure versions.json exists in outdir. If not, try to copy from another version.
:param outdir: Output directory that should contain versions.json
:type outdir: Path
:return: True if versions.json exists or was successfully copied
:rtype: bool
"""
versions_file = outdir / 'versions.json'
if versions_file.exists():
return True
print('⚠️ versions.json not found in output directory')
# Look for versions.json in other version directories
for item in outdir.iterdir():
if item.is_dir() and not item.is_symlink():
candidate = item / 'versions.json'
if candidate.exists():
print(f'📋 Copying versions.json from {item.name}/')
shutil.copy(candidate, versions_file)
shutil.copy(candidate, outdir / 'generated/versions.json')
return True
print('❌ Could not find versions.json in any version directory')
return False
@cli.command(name='current')
@click.option('--outdir', '-o', default='docs/build/doc', help='Output directory (docs/build/doc)')
@click.option('--build-pdf/--no-build-pdf', is_flag=True, default=False, help='Also build PDF versions. (False)')
@click.option('--yes', '-y', is_flag=True, default=False, help='Automatically answer yes to all questions.')
@click.option(
'--from-scratch', is_flag=True, default=False, help='Remove output and generated folders before building.'
)
def build_current_only(
outdir: Path,
build_pdf: bool = False,
project_name: str = 'Documentation',
yes: bool = False,
from_scratch: bool = False,
) -> None:
"""
Build documentation only for the current working tree (no git worktrees).
Places output in the 'latest' folder.
:param outdir: Output directory for built documentation
:type outdir: Path
:param build_pdf: Whether to also build PDF version
:type build_pdf: bool
:param project_name: Project name for PDF
:type project_name: str
:param yes: Whether to automatically answer yes to prompts
:type yes: bool
:param from_scratch: Whether to remove output and generated folders before building
:type from_scratch: bool
"""
ensure_sphinx_build_available()
outdir = Path(outdir).resolve()
if from_scratch:
print('🧹 Cleaning up before building from scratch...')
latest_out = outdir / 'latest'
if latest_out.exists():
print(f' - Removing {latest_out}')
if latest_out.is_symlink():
latest_out.unlink()
else:
shutil.rmtree(latest_out)
generated_docs = Path('docs') / 'source' / 'generated'
if generated_docs.exists():
print(f' - Removing {generated_docs}')
shutil.rmtree(generated_docs)
print('📘 Building documentation for current working tree...')
# Check if multiversion structure exists
has_other_versions = check_multiversion_structure(outdir)
if not has_other_versions:
print('\n⚠️ Warning: No other version directories found!')
print(' The version switcher and navigation may not work correctly.')
print(' Consider running the full build at least once:')
print(' $ multiversion-doc build')
if yes:
print('\nContinue anyway? [y/N]: y (auto)')
response = 'y'
else:
response = input('\nContinue anyway? [y/N]: ')
if response.lower() not in ('y', 'yes'):
print('❌ Aborted')
sys.exit(0)
curr_docs = Path('docs') / 'source'
if not curr_docs.exists():
print(f'❌ Documentation source not found: {curr_docs}')
sys.exit(1)
# Build HTML
latest_out = outdir / 'latest'
latest_out.mkdir(parents=True, exist_ok=True)
print('\n🔨 Building HTML...')
sp = run([SPHINX_BUILD_CMD, '-b', 'html', str(curr_docs), str(latest_out)])
# Write log
log_file = latest_out / 'sphinx-build.log'
with open(log_file, 'w', encoding='utf-8') as f:
f.write(sp.stdout)
html_success = sp.returncode == 0
report_build_status('latest', html_success, sp.stdout, 'HTML')
if not html_success:
print(f'❌ HTML build failed. Check log: {log_file}')
sys.exit(1)
# Ensure versions.json exists
print('\n🔍 Checking for versions.json...')
if not ensure_versions_json_exists(outdir):
print('⚠️ Version switcher may not work without versions.json')
print(' Run the full build to generate it:')
print(' $ python doc_versioning.py build')
else:
# Copy versions.json to latest folder
shutil.copy(outdir / 'versions.json', latest_out / 'versions.json')
shutil.copy(outdir / 'versions.json', latest_out / 'generated/versions.json')
print('✅ versions.json is available')
# Build PDF if requested
if build_pdf:
print('\n🔨 Building PDF...')
tmproot = Path(tempfile.mkdtemp(prefix='mafw-docs-current-'))
pdf_success = False
try:
latex_out = tmproot / 'latex'
latex_out.mkdir(parents=True, exist_ok=True)
sp = run([SPHINX_BUILD_CMD, '-b', 'latex', str(curr_docs), str(latex_out)])
pdf_log = sp.stdout
if sp.returncode == 0:
makefile = latex_out / 'Makefile'
if makefile.exists():
sp_pdf = run(['make'], cwd=latex_out)
else:
tex_files = list(latex_out.glob('*.tex'))
if tex_files:
sp_pdf = run(['pdflatex', '-interaction=nonstopmode', tex_files[0].name], cwd=latex_out)
else:
print('❌ No .tex file found')
sp_pdf = None
if sp_pdf:
pdf_log += '\n' + sp_pdf.stdout
pdf_files = list(latex_out.glob('*.pdf'))
if pdf_files:
pdf_path = latest_out / 'latest.pdf'
shutil.copy(pdf_files[0], pdf_path)
pdf_success = sp_pdf.returncode == 0
report_build_status('latest', pdf_success, pdf_log, 'PDF')
if pdf_success:
print(f'📄 PDF saved to: {pdf_path}')
else:
print('❌ PDF generation failed: no PDF file produced')
else:
print('❌ LaTeX build failed')
# Write PDF log
with open(latest_out / 'latest_pdf_build.log', 'w', encoding='utf-8') as f:
f.write(pdf_log)
finally:
shutil.rmtree(tmproot)
if not pdf_success:
print(f'❌ PDF build failed. Check log: {latest_out / "latest_pdf_build.log"}')
sys.exit(1)
print('\n✅ Documentation built successfully!')
print(f'📂 Output: {latest_out}')
if __name__ == '__main__':
cli.main()