diff options
| author | saturneric <[email protected]> | 2026-10-24 09:08:53 +0200 |
|---|---|---|
| committer | saturneric <[email protected]> | 2026-10-24 09:08:53 +0200 |
| commit | 9c866f373297bb024f59c983fd222df16f423408 (patch) | |
| tree | d826ed39c612282314afb9c1df2cc54a670b7ccc | |
| parent | docs(appendix): update appendix and dependencies for v2.2.3 (diff) | |
| download | manual-9c866f373297bb024f59c983fd222df16f423408.tar.gz manual-9c866f373297bb024f59c983fd222df16f423408.zip | |
feat(screenshots): add screenshot publishing pipeline
- add manifest-driven rendering for isolated reference environments
- post-process captures with frames, shadows, highlights and WebP output
- compare renders against validated local baselines
- support safe CDN uploads and durable recovery transactions
- add document reference updates and lockfile management
- report image inventory, environment changes and source status
- add comprehensive unit tests, fixtures and setup documentation
27 files changed, 5100 insertions, 0 deletions
diff --git a/scripts/screenshots/.gitignore b/scripts/screenshots/.gitignore new file mode 100644 index 0000000..1bc23ff --- /dev/null +++ b/scripts/screenshots/.gitignore @@ -0,0 +1,3 @@ +.venv/ +.screenshots-cache/ +__pycache__/ diff --git a/scripts/screenshots/README.md b/scripts/screenshots/README.md new file mode 100644 index 0000000..a74e327 --- /dev/null +++ b/scripts/screenshots/README.md @@ -0,0 +1,279 @@ +# Manual screenshots + +These scripts make the manual's screenshots from the GpgFrontend screenshot +mode. They render each image, post-process it, compare it with the last one, +upload it to the image CDN and rewrite the `[img-N]: url` definition in the +page. Everything runs locally. Nothing is uploaded unless you pass `--upload`. + +The scripts never run git in this repository. They only read the GpgFrontend +repository, with `rev-parse`, `cat-file`, `merge-base`, `diff` and `status`. + +## Setup + +Windows (PowerShell): + +```powershell +cd scripts\screenshots +py -3 -m venv .venv +.venv\Scripts\pip install -r requirements.txt +``` + +Linux: + +```bash +cd scripts/screenshots +python3 -m venv .venv +.venv/bin/pip install -r requirements.txt +``` + +You also need `git` on PATH. On Linux you need `xvfb-run` and `fc-match`. +`oxipng` is optional. When it is on PATH it is used and its version is part of +`inputs_hash`; otherwise `none` is recorded. + +You need a GpgFrontend binary built with `GPGFRONTEND_BUILD_SCREENSHOT_MODE=ON`. +Pass it with `--binary`, or set `GF_SCREENSHOT_BINARY`. + +## Files + +| File | Purpose | +|---|---| +| `screenshots.yaml` | the manifest: defaults, global sources, entries | +| `screenshots.lock.json` | committed record per entry: url, hashes, commit, env | +| `.screenshots-cache/` | ignored: previews, baselines, transactions, `last-run.json` | +| `check.py` | status report and inventory | +| `run.py` | select, render, post-process, diff, upload, apply, recover | +| `render.py` | runs the binary in a throwaway environment (also a debug CLI) | +| `postprocess.py` | frame, shadow, padding, highlights, PNG and lossless WebP | +| `diff.py` | compares with the local baseline | +| `upload.py` | EasyImages upload | +| `txn.py` | the durable upload/apply transaction | +| `apply.py` | doc definition and lock writes | +| `manifest.py`, `docrefs.py`, `envhash.py`, `fonts.py`, `gitutil.py`, `common.py` | shared pieces | + +## Manifest + +```yaml +defaults: + platform: windows11 # the only reference platform + locale: en_US + theme: light # light | dark + size: 1100x680 # logical size, or "auto" for a dialog's natural size + fonts: ["Noto Sans", "Noto Sans CJK SC"] + clock: "2026-01-15T10:00:00Z" + timeout_ms: 30000 + frame: {radius: 8, shadow: 12, padding: 24} # logical px, scaled by DPR 2 + diff: {tolerance: 16, threshold: 0.001} + preview_platforms: [linux-xvfb] +global_sources: [src/screenshot/, resource/screenshot/, scripts/screenshot/] +entries: + - id: main-window + doc: src/content/docs/guides/understand-interface.mdx + ref: img-1 + scenario: main-window + sources: [src/ui/main_window/] + highlight: InfoBoardWidget # or {name: X, style: box|arrow}, or a list + - id: drag-to-applications + doc: src/content/docs/overview/getting-started.mdx + ref: img-1 + status: manual-only # or animated; never rendered +``` + +`sources` and `global_sources` are paths in the GpgFrontend repository. Any +entry key can override a default. + +## check.py + +``` +python check.py --gpgfrontend ../../../GpgFrontend [--binary <exe>] [--inventory] [--json] +``` + +| Status | Meaning | +|---|---| +| `missing` | the ref has no definition, an empty or `TODO` URL, or the entry has no lock record | +| `unmanaged` | an image ref (reference, inline or `<img>`) that is not in the manifest | +| `orphan` | a manifest entry whose ref is no longer on the page | +| `unknown-baseline` | the lock commit is not in the repository, or not an ancestor of HEAD | +| `outdated` | the entry's sources or the global sources changed since the lock commit | +| `inputs-changed` | settings, pipeline scripts, requirements or the optimizer changed | +| `env-changed` | the probed environment hash differs from the lock | +| `env-unchecked` | no binary, so the environment was not probed (or the probe failed; see details) | +| `dirty-sources` | uncommitted changes in the relevant sources | +| `platform-mismatch` | the probed platform is not the entry's platform | +| `ok` | none of the above | + +Without a binary, every entry is `env-unchecked` and never `ok`. Each distinct +(locale, theme, fonts, clock, size) is probed once per run. + +`--inventory` adds the observed counts: references, definitions, unique URLs +and formats by URL extension, per page and in total, plus every ref. The format +comes from the URL only; nothing is downloaded. With `--json --inventory` the +output is `{"entries": [...], "inventory": {...}}`; with `--json` alone it is the +entries array. + +## run.py + +``` +python run.py --gpgfrontend <repo> --binary <exe> [--only id,...] [--force] + [--accept-baseline id,...] [--force-replace id,...] + [--allow-dirty] [--apply | --upload] [--repeat N] + [--work-root DIR] [--keep-work] +python run.py --recover [--resolve TXN_ID (--url URL | --retry-upload)] +``` + +Selection: + +1. Candidates are every renderable entry, or exactly `--only`. An unknown id, + or a `manual-only`/`animated` id, is a usage error. +2. Candidates that are `missing`, `unknown-baseline`, `outdated`, + `inputs-changed` or `env-changed` are selected. `--force` selects them all. +3. `--accept-baseline` only applies to a selected id whose baseline is + unavailable. `--force-replace` replaces a selected id whatever the diff says. + Naming an unselected id, or one id in both lists, is a usage error. + +Writes: the default is a dry run (previews, cache, `last-run.json`). `--apply` +writes the lock, docs and cache without network. `--upload` implies `--apply` +and allows the upload. A `replace` is only carried out with `--upload`; +otherwise it is reported as `would-replace`. + +Dirty sources: without `--allow-dirty` a dirty entry is `refused-dirty` and not +rendered. With it, the entry is a preview only. An entry rendered on a +non-reference platform is always `preview-only-platform`. Preview-only entries +are never uploaded, applied or advanced in the lock. + +Binary freshness is a warning, never a block: the binary is older than a +relevant source file, or `BUILD-INFO.json` next to the binary names a commit +other than HEAD. + +Summary statuses: `updated`, `unchanged`, `lock-advanced`, +`unchanged-lock-not-advanced`, `doc-restored`, `would-restore-doc`, +`would-replace`, `baseline-unavailable`, +`refused-dirty`, `preview-only-dirty`, `preview-only-platform`, +`failed:<reason>`, `conflict`, `upload-unknown`, `artifact-corrupt`. + +An entry whose link was emptied or set to `TODO` while its image is unchanged +gets the URL from the lock written back under `--apply` (`doc-restored`); no +upload is needed. A dry run reports `would-restore-doc`. + +Exit codes: 0 when every entry is `updated`, `unchanged`, `lock-advanced`, +`unchanged-lock-not-advanced`, `doc-restored`, `would-restore-doc` or +`would-replace` (and preview-only entries in a dry run). 1 when any entry is refused, failed or `baseline-unavailable`, or is +preview-only under `--apply`/`--upload`. 2 for usage errors. 3 when a +`conflict`, `upload-unknown` or `artifact-corrupt` is pending. + +Previews are in `.screenshots-cache/preview/<id>.{png,webp,json}`. + +### CDN credentials + +`--upload` reads `GF_MANUAL_CDN_API` (an https URL) and `GF_MANUAL_CDN_TOKEN`. +The token is never written anywhere. The response's `del` link is kept only in +the ignored transaction cache, never in the lock, the docs or `last-run.json`. +Nothing on the CDN is deleted automatically. When a run replaces an image that +an earlier run uploaded, or hits a conflict, the summary prints the `del` link +so you can remove the old upload in the admin panel. + +## Transactions and the failure window + +Each replacement is one transaction in `.screenshots-cache/txn/<txn_id>/`. Its +state moves `prepared`, `uploading`, `uploaded`, `doc-applied`, +`lock-applied`, `committed`, and is saved to disk before each step. + +`run.py --recover` finishes interrupted transactions without rendering: + +- `prepared` becomes `abandoned`. Nothing left the machine. +- `uploaded`, `doc-applied` and `lock-applied` continue from where they stopped. +- `uploading` becomes `upload-unknown` and waits for you. +- Changed artifacts become `artifact-corrupt`. Nothing is applied from them. +- A doc or lock changed by someone else becomes `conflict`. Newer data is never + overwritten. + +**The failure window.** If the process dies, or the connection drops, after +the server stored the upload but before `uploaded` was saved, the local side +cannot know whether the upload happened. EasyImages has no idempotency key and +no lookup by hash, so this cannot be made duplicate-free. Such a transaction is +`upload-unknown` and is never retried on its own. Decide with: + +- `run.py --recover --resolve <txn_id> --url <url>` if you found the image in the + EasyImages admin. The `del` link stays unknown. +- `run.py --recover --resolve <txn_id> --retry-upload` to upload the same file + again. This may leave a duplicate on the CDN, which is harmless and can be + deleted in the admin panel. + +While an entry has an unfinished transaction, a normal `--upload` run does not +upload it again. + +## Render environment + +Each render runs `<exe> --temp-profile --screenshot <scenario> ...` in a +throwaway directory, `<work root>/run`, emptied before every render: + +- Windows: `LOCALAPPDATA`, `APPDATA`, `TEMP`/`TMP` and `USERPROFILE` point into + it. It runs in your interactive desktop session. The default work root is + `%SystemDrive%\gf-shot`, not your `%TEMP%`: the temporary profile lives under + `TEMP`, the Key Databases settings page shows that path, and a per-user temp + folder would put your account name into the manual. +- Linux: `HOME`, `XDG_*` and `TMPDIR` point into it, under + `xvfb-run -s "-screen 0 2560x1600x24"`. The default work root is + `/tmp/gf-shot-<uid>`. Keep `--work-root` short: gpg-agent's socket lives + deep inside it and Unix sockets are limited to 108 bytes. + +The directory name and the temporary profile's own name are fixed, so the +paths the application shows are the same on every run. On Linux the +temporary profile itself is made in `/dev/shm/gf-shot-<uid>` +(`GF_PROFILE_PACKAGE_STORAGE_DIR`), which is emptied before every render too; +the application refuses to start on a session a crash left behind +(`stale-session`) rather than render from its old settings. That also means one +pipeline at a time per work root: two runs sharing one would collide. + +`GNUPGHOME`, `QT_QPA_PLATFORM` and the Qt scaling variables are removed from +the inherited environment. After each run, the per-user app data +(`BKTUS/GpgFrontend*`) is checked for new files; any new file fails the entry. +A real GpgFrontend running at the same time could cause a false alarm. + +The final `env_hash` combines the binary's `env_hash_core`, which Python +recomputes and must match, with the platform tag and the sha256 of each font +file. + +### Font detection limit + +Fonts are found through the `...\Windows NT\CurrentVersion\Fonts` registry keys +(HKLM and HKCU) on Windows, and `fc-match` on Linux. The hashed files are those +of every listed family plus the system UI font. Qt does not say which file it +used for each glyph, so a fallback to a font outside that list cannot be seen. +The binary's gate requires every listed family to resolve, which keeps that +risk small. + +## Windows 11 checklist (reference renders) + +1. Dispatch `build.yml` with `screenshot: true`, then download the + `gpgfrontend-screenshot-windows-2022-portable` artifact and extract it. +2. Install Noto Sans and Noto Sans CJK SC if they are not installed. +3. Run a dry run, in light and dark (set `theme` in the manifest, or add + entries): + + ```powershell + .venv\Scripts\python run.py --gpgfrontend <GpgFrontend repo> --binary <path>\GpgFrontend.exe ` + --only main-window,key-generate-dialog,text-decrypt-verify-result,main-window-menu-popup --repeat 2 + ``` + + Any gate failure is reported by name. +4. Review the previews in `.screenshots-cache\preview\`. +5. If anything fails, send `.screenshots-cache\last-run.json` and the env + records in `.screenshots-cache\preview\*.json`. They contain no `del` links. + +## Tests + +``` +.venv/bin/python -m unittest discover -s tests -v +``` + +The tests use temporary git repositories, temporary doc trees, a fake binary +(`tests/fakegf.py`) and a mock EasyImages server on 127.0.0.1. They never +contact the network. The environment hash cross-check reads +`resource/screenshot/env-hash-fixture.json` from the GpgFrontend checkout next +to this repository; set `GF_SHOTS_GPGFRONTEND` to point elsewhere. + +Variables used only by the tests: `GF_SHOTS_NO_XVFB=1` (run without +xvfb-run; the platform tag is then not a reference one), +`GF_SHOTS_APPDATA_ROOTS` (where the footprint check looks) and +`GF_SHOTS_TEST_CRASH_AFTER=<state>` (simulate a crash right after a +transaction state is saved). diff --git a/scripts/screenshots/apply.py b/scripts/screenshots/apply.py new file mode 100644 index 0000000..32303c6 --- /dev/null +++ b/scripts/screenshots/apply.py @@ -0,0 +1,93 @@ +"""Local writes: the doc definition URL and the lock record. + +Both are idempotent and refuse to overwrite anything they did not expect, +raising Conflict instead. +""" + +from __future__ import annotations + +from pathlib import Path +from typing import Any, Optional + +import docrefs +from common import PipelineError, atomic_write_json, atomic_write_text, load_json, read_text_exact, sha256_bytes + +LOCK_FIELDS = ("url", "webp_sha256", "pixels_sha256", "gpgfrontend_commit", + "inputs_hash", "env_hash", "env", "generated_at") +WRITTEN = "written" +ALREADY = "already-applied" + + +class Conflict(PipelineError): + pass + + +def _contains_del(obj: Any) -> bool: + if isinstance(obj, dict): + return "del" in obj or any(_contains_del(v) for v in obj.values()) + if isinstance(obj, list): + return any(_contains_del(v) for v in obj) + return False + + +def read_lock(path: Path) -> dict[str, Any]: + data = load_json(path, {}) + if not isinstance(data, dict): + raise PipelineError(f"{path}: the lock must be a JSON object keyed by entry id") + return data + + +def write_lock(path: Path, lock: dict[str, Any]) -> None: + if _contains_del(lock): + raise PipelineError("refusing to write a `del` capability URL into the committed lock") + atomic_write_json(path, lock) + + +def doc_sha256(path: Path) -> str: + return sha256_bytes(Path(path).read_bytes()) + + +def apply_doc(path: Path, label: str, old_url: str, new_url: str, sha_before: str) -> str: + """Points the single definition of `label` at new_url. + + Unchanged doc: rewrite it. Concurrently edited doc: rewrite only if the + definition still carries old_url. Already new_url: nothing to do. + """ + path = Path(path) + if not path.is_file(): + raise Conflict(f"{path} no longer exists") + text = read_text_exact(path) + if sha256_bytes(text.encode("utf-8")) == sha_before: + atomic_write_text(path, docrefs.replace_definition_url(text, label, new_url, old_url)) + return WRITTEN + defs = docrefs.find_definitions(text, label) + if len(defs) != 1: + raise Conflict(f"{path}: [{label}] now has {len(defs)} definitions") + if defs[0].url == new_url: + return ALREADY + if defs[0].url != old_url: + raise Conflict(f"{path}: [{label}] was changed to {defs[0].url!r}") + atomic_write_text(path, docrefs.replace_url(text, defs[0], new_url)) + return WRITTEN + + +def lock_state(path: Path, entry_id: str, before: Optional[dict[str, Any]], + target: dict[str, Any]) -> str: + current = read_lock(path).get(entry_id) + if current == target: + return "target" + if current == before: + return "before" + return "other" + + +def apply_lock(path: Path, entry_id: str, before: Optional[dict[str, Any]], target: dict[str, Any]) -> str: + lock = read_lock(path) + current = lock.get(entry_id) + if current == target: + return ALREADY + if current != before: + raise Conflict(f"lock record for {entry_id} was changed by something else") + lock[entry_id] = target + write_lock(path, lock) + return WRITTEN diff --git a/scripts/screenshots/check.py b/scripts/screenshots/check.py new file mode 100644 index 0000000..52b8936 --- /dev/null +++ b/scripts/screenshots/check.py @@ -0,0 +1,287 @@ +"""Reports the status of every managed entry and every image ref in the manual. + +usage: check.py --gpgfrontend <repo> [--binary <exe>] [--inventory] [--json] +""" + +from __future__ import annotations + +import argparse +import json +import os +import sys +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any, Optional + +import docrefs +import gitutil +from apply import read_lock +from common import (DEFAULT_LOCK, DEFAULT_MANIFEST, MANUAL_ROOT, SCRIPT_DIR, PipelineError, UsageError) +from manifest import Entry, Manifest, inputs_hash, load_manifest +from postprocess import processing_descriptor +from render import RenderConfig, Renderer + +MISSING = "missing" +UNMANAGED = "unmanaged" +ORPHAN = "orphan" +UNKNOWN_BASELINE = "unknown-baseline" +OUTDATED = "outdated" +INPUTS_CHANGED = "inputs-changed" +ENV_CHANGED = "env-changed" +ENV_UNCHECKED = "env-unchecked" +DIRTY_SOURCES = "dirty-sources" +PLATFORM_MISMATCH = "platform-mismatch" +OK = "ok" + +STALE = (MISSING, UNKNOWN_BASELINE, OUTDATED, INPUTS_CHANGED, ENV_CHANGED) +STATUS_ORDER = (MISSING, UNMANAGED, ORPHAN, UNKNOWN_BASELINE, OUTDATED, INPUTS_CHANGED, ENV_CHANGED, + ENV_UNCHECKED, DIRTY_SOURCES, PLATFORM_MISMATCH, "manual-only", "animated", OK) + + +@dataclass +class Row: + id: Optional[str] + doc: str + ref: Optional[str] + statuses: list[str] + details: dict[str, Any] = field(default_factory=dict) + + def as_dict(self) -> dict[str, Any]: + return {"id": self.id, "doc": self.doc, "ref": self.ref, + "statuses": list(self.statuses), "details": self.details} + + +@dataclass +class CheckContext: + manifest: Manifest + lock: dict[str, Any] + manual_root: Path + gpgfrontend: Path + renderer: Optional[Renderer] = None + pipeline_dir: Path = SCRIPT_DIR + processing: Optional[dict[str, str]] = None + + def processing_info(self) -> dict[str, str]: + if self.processing is None: + self.processing = processing_descriptor() + return self.processing + + +def _page_map(pages: list[docrefs.PageScan]) -> dict[str, docrefs.PageScan]: + return {page.doc: page for page in pages} + + +def _baseline_statuses(ctx: CheckContext, entry: Entry, record: dict[str, Any], row: Row) -> None: + commit = str(record.get("gpgfrontend_commit") or "") + row.details["lock_commit"] = commit or None + if not gitutil.commit_exists(ctx.gpgfrontend, commit): + row.statuses.append(UNKNOWN_BASELINE) + row.details["baseline"] = "lock commit not found in the GpgFrontend repository" + return + if not gitutil.is_ancestor(ctx.gpgfrontend, commit): + row.statuses.append(UNKNOWN_BASELINE) + row.details["baseline"] = "lock commit is not an ancestor of HEAD" + return + changed = gitutil.changed_since(ctx.gpgfrontend, commit, ctx.manifest.relevant_sources(entry)) + if changed: + row.statuses.append(OUTDATED) + row.details["changed_files"] = changed + + +def _env_statuses(ctx: CheckContext, entry: Entry, record: Optional[dict[str, Any]], row: Row) -> None: + if ctx.renderer is None: + row.statuses.append(ENV_UNCHECKED) + return + probe = ctx.renderer.probe(entry) + if not probe.ok or probe.env is None: + row.statuses.append(ENV_UNCHECKED) + row.details["probe_failed"] = probe.failure + return + row.details["platform"] = probe.env.platform + row.details["env_hash"] = probe.env.env_hash + if record and record.get("env_hash") != probe.env.env_hash: + row.statuses.append(ENV_CHANGED) + row.details["lock_env_hash"] = record.get("env_hash") + if probe.env.platform != entry.platform: + row.statuses.append(PLATFORM_MISMATCH) + + +def entry_row(ctx: CheckContext, entry: Entry, pages: dict[str, docrefs.PageScan]) -> Row: + row = Row(entry.id, entry.doc, entry.ref, []) + page = pages.get(entry.doc) + if page is None or not page.has_reference(entry.ref): + row.statuses.append(ORPHAN) + else: + definition = page.definition_for(entry.ref) + url = definition.url if definition else None + row.details["url"] = url + if docrefs.is_missing_url(url): + row.statuses.append(MISSING) + row.details["missing"] = "no definition" if definition is None else "empty or TODO url" + if len(page.definitions.get(docrefs.normalize_label(entry.ref), [])) > 1: + row.details["duplicate_definitions"] = True + if not entry.renderable: + row.statuses.append(entry.status or "") + return row + record = ctx.lock.get(entry.id) + if not record: + if MISSING not in row.statuses: + row.statuses.append(MISSING) + row.details.setdefault("missing", "no lock record") + else: + _baseline_statuses(ctx, entry, record, row) + current = inputs_hash(entry, ctx.processing_info(), ctx.pipeline_dir) + if record.get("inputs_hash") != current: + row.statuses.append(INPUTS_CHANGED) + _env_statuses(ctx, entry, record, row) + dirty = gitutil.dirty(ctx.gpgfrontend, ctx.manifest.relevant_sources(entry)) + if dirty: + row.statuses.append(DIRTY_SOURCES) + row.details["dirty_files"] = dirty + if not row.statuses: + row.statuses.append(OK) + return row + + +def unmanaged_rows(ctx: CheckContext, pages: list[docrefs.PageScan]) -> list[Row]: + managed = {(e.doc, docrefs.normalize_label(e.ref)) for e in ctx.manifest.entries} + rows: list[Row] = [] + for page in pages: + seen: set[tuple[str, str]] = set() + for ref in page.refs: + if ref.kind == "reference": + key = (page.doc, ref.label or "") + if key in managed or key in seen: + continue + seen.add(key) + url = page.ref_url(ref) + statuses = [UNMANAGED] + ([MISSING] if docrefs.is_missing_url(url) else []) + rows.append(Row(None, page.doc, ref.raw_label, statuses, + {"url": url, "line": ref.line, "kind": ref.kind})) + else: + rows.append(Row(None, page.doc, None, [UNMANAGED], + {"url": ref.url, "line": ref.line, "kind": ref.kind})) + return rows + + +def compute(ctx: CheckContext) -> tuple[list[Row], list[docrefs.PageScan]]: + gitutil.head(ctx.gpgfrontend) + pages = docrefs.scan_docs(ctx.manual_root) + page_map = _page_map(pages) + rows = [entry_row(ctx, entry, page_map) for entry in ctx.manifest.entries] + rows.extend(unmanaged_rows(ctx, pages)) + return rows, pages + + +def managed_map(manifest: Manifest) -> dict[tuple[str, str], str]: + return {(e.doc, docrefs.normalize_label(e.ref)): e.id for e in manifest.entries} + + +def _short(row: Row) -> str: + d = row.details + bits = [] + for key in ("missing", "baseline", "probe_failed", "kind"): + if d.get(key): + bits.append(f"{key}={d[key]}") + if d.get("changed_files"): + bits.append(f"changed={len(d['changed_files'])} file(s)") + if d.get("dirty_files"): + bits.append(f"dirty={len(d['dirty_files'])} file(s)") + if d.get("platform"): + bits.append(f"platform={d['platform']}") + if d.get("line"): + bits.append(f"line {d['line']}") + return " ".join(bits) + + +def format_table(rows: list[Row]) -> str: + lines: list[str] = [] + for status in STATUS_ORDER: + group = [r for r in rows if status in r.statuses] + if not group: + continue + lines.append(f"{status} ({len(group)})") + for r in group: + ident = r.id or "-" + ref = f"[{r.ref}]" if r.ref else "" + lines.append(f" {ident:<28} {r.doc} {ref} {_short(r)}".rstrip()) + lines.append("") + return "\n".join(lines) + + +def _fmt_counts(counts: dict[str, int]) -> str: + return ", ".join(f"{k}={v}" for k, v in counts.items()) or "-" + + +def format_inventory(inv: dict[str, Any]) -> str: + t = inv["total"] + lines = [ + "Inventory (observed; formats by URL extension, content not fetched)", + f" pages with images: {t['pages_with_images']}", + f" references: {t['references']}", + f" definitions: {t['definitions']}", + f" unique URLs: {t['unique_urls']}", + f" formats by reference: {_fmt_counts(t['formats_by_reference'])}", + f" formats by unique URL: {_fmt_counts(t['formats_by_unique_url'])}", + "", + "Per page (refs / defs / unique URLs / formats by reference)", + ] + for p in inv["pages"]: + lines.append(f" {p['doc']:<56} {p['references']:>3} {p['definitions']:>3} {p['unique_urls']:>3} " + f"{_fmt_counts(p['formats_by_reference'])}") + lines.append("") + managed = sum(1 for r in inv["refs"] if r["managed_id"]) + lines.append(f"Refs ({len(inv['refs'])}; managed {managed}, unmanaged {len(inv['refs']) - managed})") + for r in inv["refs"]: + label = f"[{r['ref']}]" if r["ref"] else f"({r['kind']})" + who = r["managed_id"] or "unmanaged" + lines.append(f" {r['doc']}:{r['line']:<4} {label:<10} {r['format']:<5} {who:<28} {r['url'] or '-'}") + return "\n".join(lines) + + +def build_context(args: argparse.Namespace) -> CheckContext: + manifest = load_manifest(args.manifest) + renderer = None + binary = args.binary or os.environ.get("GF_SCREENSHOT_BINARY") + if binary: + renderer = Renderer(RenderConfig(Path(binary), args.gpgfrontend, args.work_root)) + return CheckContext(manifest, read_lock(args.lock), args.manual_root, args.gpgfrontend, renderer) + + +def main(argv: Optional[list[str]] = None) -> int: + parser = argparse.ArgumentParser(description="Status of manual screenshots.") + parser.add_argument("--gpgfrontend", required=True, type=Path, help="GpgFrontend repository") + parser.add_argument("--binary", type=Path, help="screenshot-mode binary (else $GF_SCREENSHOT_BINARY)") + parser.add_argument("--inventory", action="store_true", help="also report image counts") + parser.add_argument("--json", action="store_true", help="machine-readable output") + parser.add_argument("--manifest", type=Path, default=DEFAULT_MANIFEST) + parser.add_argument("--lock", type=Path, default=DEFAULT_LOCK) + parser.add_argument("--manual-root", type=Path, default=MANUAL_ROOT) + parser.add_argument("--work-root", type=Path, help="throwaway directory root for probes") + args = parser.parse_args(argv) + try: + ctx = build_context(args) + rows, pages = compute(ctx) + except UsageError as exc: + print(f"error: {exc}", file=sys.stderr) + return 2 + except PipelineError as exc: + print(f"error: {exc}", file=sys.stderr) + return 1 + inv = docrefs.inventory(pages, managed_map(ctx.manifest)) if args.inventory else None + if args.json: + payload: Any = [r.as_dict() for r in rows] + if inv is not None: + payload = {"entries": payload, "inventory": inv} + print(json.dumps(payload, indent=2, ensure_ascii=False)) + return 0 + if ctx.renderer is None: + print("note: no --binary or GF_SCREENSHOT_BINARY; the environment was not probed (env-unchecked)\n") + print(format_table(rows)) + if inv is not None: + print(format_inventory(inv)) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/screenshots/common.py b/scripts/screenshots/common.py new file mode 100644 index 0000000..7914f2a --- /dev/null +++ b/scripts/screenshots/common.py @@ -0,0 +1,172 @@ +"""Shared helpers: paths, hashing, canonical JSON and durable file writes.""" + +from __future__ import annotations + +import errno +import hashlib +import json +import os +import tempfile +import time +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Iterable + +SCRIPT_DIR = Path(__file__).resolve().parent +MANUAL_ROOT = SCRIPT_DIR.parent.parent +DEFAULT_MANIFEST = SCRIPT_DIR / "screenshots.yaml" +DEFAULT_LOCK = SCRIPT_DIR / "screenshots.lock.json" +DEFAULT_CACHE = SCRIPT_DIR / ".screenshots-cache" +DOCS_SUBDIR = Path("src") / "content" / "docs" + +# Windows reports a file held open by another process (an editor, a virus +# scanner) as a sharing violation; os.replace surfaces it as PermissionError. +_REPLACE_ATTEMPTS = 50 +_REPLACE_DELAY_S = 0.1 + + +class PipelineError(Exception): + """An expected failure with a message fit for the user.""" + + +class UsageError(PipelineError): + """Bad command line or manifest; maps to exit code 2.""" + + +def sha256_bytes(data: bytes) -> str: + return hashlib.sha256(data).hexdigest() + + +def sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with Path(path).open("rb") as handle: + for chunk in iter(lambda: handle.read(1 << 20), b""): + digest.update(chunk) + return digest.hexdigest() + + +def canonical_json(obj: Any) -> bytes: + """The one serialization every hash in the pipeline is computed over.""" + return json.dumps( + obj, sort_keys=True, separators=(",", ":"), ensure_ascii=False + ).encode("utf-8") + + +def utc_now_iso() -> str: + return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + + +def utc_stamp_compact() -> str: + return datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ") + + +def fsync_dir(path: Path) -> None: + """Persist a directory entry change. Not possible (or needed) on Windows.""" + if os.name == "nt": + return + fd = os.open(str(path), os.O_RDONLY) + try: + os.fsync(fd) + finally: + os.close(fd) + + +def _is_sharing_violation(exc: OSError) -> bool: + if os.name != "nt": + return False + winerror = getattr(exc, "winerror", None) + return isinstance(exc, PermissionError) or winerror in (5, 32, 33) + + +def replace_with_retry(src: Path, dst: Path) -> None: + for attempt in range(_REPLACE_ATTEMPTS): + try: + os.replace(str(src), str(dst)) + return + except OSError as exc: + if not _is_sharing_violation(exc) or attempt == _REPLACE_ATTEMPTS - 1: + raise + time.sleep(_REPLACE_DELAY_S) + + +def atomic_write_bytes(path: Path, data: bytes) -> None: + """Temp file in the same directory, flush, fsync, os.replace, dir fsync.""" + path = Path(path) + path.parent.mkdir(parents=True, exist_ok=True) + fd, tmp_name = tempfile.mkstemp( + prefix=f".{path.name}.", suffix=".tmp", dir=str(path.parent) + ) + tmp = Path(tmp_name) + try: + with os.fdopen(fd, "wb") as handle: + handle.write(data) + handle.flush() + os.fsync(handle.fileno()) + replace_with_retry(tmp, path) + fsync_dir(path.parent) + except BaseException: + try: + tmp.unlink() + except OSError: + pass + raise + + +def atomic_write_text(path: Path, text: str) -> None: + """Writes text exactly as given: no newline translation, UTF-8.""" + atomic_write_bytes(path, text.encode("utf-8")) + + +def dump_json(obj: Any) -> str: + return json.dumps(obj, indent=2, sort_keys=True, ensure_ascii=False) + "\n" + + +def atomic_write_json(path: Path, obj: Any) -> None: + atomic_write_text(path, dump_json(obj)) + + +def atomic_copy(src: Path, dst: Path) -> None: + atomic_write_bytes(dst, Path(src).read_bytes()) + + +def append_line_durable(path: Path, line: str) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + with Path(path).open("a", encoding="utf-8", newline="\n") as handle: + handle.write(line.rstrip("\n") + "\n") + handle.flush() + os.fsync(handle.fileno()) + + +def read_text_exact(path: Path) -> str: + """Reads UTF-8 without newline translation, so CRLF survives a rewrite.""" + return Path(path).read_bytes().decode("utf-8") + + +def load_json(path: Path, default: Any = None) -> Any: + path = Path(path) + if not path.exists(): + return default + try: + return json.loads(read_text_exact(path)) + except json.JSONDecodeError as exc: + raise PipelineError(f"{path}: invalid JSON: {exc}") from exc + + +def split_ids(value: str | None) -> list[str]: + if not value: + return [] + return [part.strip() for part in value.split(",") if part.strip()] + + +def dedupe(items: Iterable[str]) -> list[str]: + seen: set[str] = set() + out: list[str] = [] + for item in items: + if item not in seen: + seen.add(item) + out.append(item) + return out + + +def is_missing_error(exc: OSError) -> bool: + return exc.errno == errno.ENOENT diff --git a/scripts/screenshots/diff.py b/scripts/screenshots/diff.py new file mode 100644 index 0000000..99c66ae --- /dev/null +++ b/scripts/screenshots/diff.py @@ -0,0 +1,72 @@ +"""Compares a new post-processed PNG with the local baseline. + +The baseline `.screenshots-cache/baseline/<id>.png` only counts when its +pixels hash equals the lock's `pixels_sha256`. It is never fetched from the CDN. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Optional + +from PIL import Image, ImageChops + +from postprocess import pixels_sha256 + +NEW = "new" +REPLACE = "replace" +UNCHANGED = "unchanged" +BASELINE_UNAVAILABLE = "baseline-unavailable" + + +@dataclass +class DiffResult: + result: str + changed_ratio: Optional[float] = None + detail: str = "" + + +def baseline_path(cache: Path, entry_id: str) -> Path: + return Path(cache) / "baseline" / f"{entry_id}.png" + + +def baseline_valid(cache: Path, entry_id: str, lock_record: Optional[dict[str, Any]]) -> bool: + if not lock_record: + return False + path = baseline_path(cache, entry_id) + if not path.is_file(): + return False + try: + with Image.open(path) as img: + return pixels_sha256(img) == lock_record.get("pixels_sha256") + except OSError: + return False + + +def changed_ratio(new: Image.Image, old: Image.Image, tolerance: int) -> float: + """Share of pixels whose largest per-channel delta exceeds `tolerance`.""" + delta = ImageChops.difference(new.convert("RGBA"), old.convert("RGBA")) + bands = delta.split() + worst = bands[0] + for band in bands[1:]: + worst = ImageChops.lighter(worst, band) + changed = worst.point(lambda v: 255 if v > tolerance else 0).histogram()[255] + return changed / float(new.size[0] * new.size[1]) + + +def compare(new_png: Path, cache: Path, entry_id: str, lock_record: Optional[dict[str, Any]], + settings: dict[str, Any]) -> DiffResult: + if not lock_record: + return DiffResult(NEW, detail="no lock record") + if not baseline_valid(cache, entry_id, lock_record): + path = baseline_path(cache, entry_id) + why = "baseline missing" if not path.exists() else "baseline pixels_sha256 differs from the lock" + return DiffResult(BASELINE_UNAVAILABLE, detail=why) + with Image.open(new_png) as new, Image.open(baseline_path(cache, entry_id)) as old: + if new.size != old.size: + return DiffResult(REPLACE, 1.0, f"size {old.size} -> {new.size}") + ratio = changed_ratio(new, old, int(settings["tolerance"])) + if ratio > float(settings["threshold"]): + return DiffResult(REPLACE, ratio, f"{ratio:.6f} of pixels changed") + return DiffResult(UNCHANGED, ratio, f"{ratio:.6f} of pixels changed") diff --git a/scripts/screenshots/docrefs.py b/scripts/screenshots/docrefs.py new file mode 100644 index 0000000..968b31f --- /dev/null +++ b/scripts/screenshots/docrefs.py @@ -0,0 +1,336 @@ +"""Image reference parsing, inventory counts and definition URL rewriting. + +Recognised image forms: + reference style ![alt][label], ![alt][] and ![label] (when defined) + inline  + html <img src="url"> +Definitions are `[label]: url`, including the wrapped form where the URL sits +on the following line. Fenced code blocks and inline code spans are skipped. +""" + +from __future__ import annotations + +import re +from collections import Counter +from dataclasses import dataclass, field +from pathlib import Path, PurePosixPath +from typing import Iterator +from urllib.parse import urlparse + +from common import DOCS_SUBDIR, PipelineError, read_text_exact + +DOC_SUFFIXES = (".md", ".mdx") + +_FENCE_RE = re.compile(r"^[ \t]*(`{3,}|~{3,})") +_CODE_SPAN_RE = re.compile(r"(`+)(?:(?!\1).)+?\1") +_IMG_START_RE = re.compile(r"!\[") +_DEF_RE = re.compile(r"^[ \t]*\[([^\]\^][^\]]*)\]:[ \t]*(.*)$") +_HTML_IMG_RE = re.compile(r"<img\b[^>]*?\bsrc\s*=\s*(?:\"([^\"]*)\"|'([^']*)'|(\{[^}]*\}))", re.IGNORECASE) +_URL_TOKEN_RE = re.compile(r"<([^>\n]*)>|(\S+)") + + +def normalize_label(label: str) -> str: + return " ".join(label.split()).casefold() + + +@dataclass(frozen=True) +class ImageRef: + doc: str + line: int + kind: str # reference | inline | html + label: str | None # normalized, reference style only + raw_label: str | None + url: str | None # inline and html only + + +@dataclass(frozen=True) +class Definition: + label: str + raw_label: str + url: str + line_start: int # 1-based, inclusive + line_end: int + url_start: int # character offsets into the page text + url_end: int + text: str # the original definition lines, line endings included + + +@dataclass +class PageScan: + doc: str + refs: list[ImageRef] = field(default_factory=list) + definitions: dict[str, list[Definition]] = field(default_factory=dict) + + def definition_for(self, label: str) -> Definition | None: + found = self.definitions.get(normalize_label(label)) + return found[0] if found else None + + def ref_url(self, ref: ImageRef) -> str | None: + if ref.kind != "reference": + return ref.url + definition = self.definition_for(ref.label or "") + return definition.url if definition else None + + def image_labels(self) -> list[str]: + out: list[str] = [] + for ref in self.refs: + if ref.kind == "reference" and ref.label not in out: + out.append(ref.label or "") + return out + + def has_reference(self, label: str) -> bool: + wanted = normalize_label(label) + return any(r.kind == "reference" and r.label == wanted for r in self.refs) + + +@dataclass(frozen=True) +class _Line: + number: int + start: int # offset of first character + body: str # without line ending + in_code: bool + + +def _lines(text: str) -> list[_Line]: + out: list[_Line] = [] + offset = 0 + fence: str | None = None + for number, raw in enumerate(text.splitlines(keepends=True), start=1): + body = raw.rstrip("\r\n") + match = _FENCE_RE.match(body) + in_code = fence is not None + if match: + marker = match.group(1) + if fence is None: + fence = marker + in_code = True + elif marker[0] == fence[0] and len(marker) >= len(fence) and not body.strip()[len(marker):].strip(): + fence = None + in_code = True + out.append(_Line(number, offset, body, in_code)) + offset += len(raw) + return out + + +def _mask_code_spans(body: str) -> str: + return _CODE_SPAN_RE.sub(lambda m: " " * len(m.group(0)), body) + + +def _bracket_end(text: str, open_index: int) -> int: + """Index of the `]` closing the `[` at open_index, allowing nesting.""" + depth = 0 + i = open_index + while i < len(text): + ch = text[i] + if ch == "\\": + i += 2 + continue + if ch == "[": + depth += 1 + elif ch == "]": + depth -= 1 + if depth == 0: + return i + i += 1 + return -1 + + +def _parse_definitions(doc: str, text: str, lines: list[_Line]) -> dict[str, list[Definition]]: + defs: dict[str, list[Definition]] = {} + for idx, line in enumerate(lines): + if line.in_code: + continue + match = _DEF_RE.match(line.body) + if not match: + continue + raw_label = match.group(1) + rest_start = line.start + match.start(2) + rest = match.group(2) + line_end = line.number + token = _URL_TOKEN_RE.match(rest) if rest.strip() else None + if token: + group = 1 if token.group(1) is not None else 2 + url = token.group(group) + url_start = rest_start + token.start(group) + url_end = rest_start + token.end(group) + else: + url, url_start, url_end = "", line.start + len(line.body), line.start + len(line.body) + nxt = lines[idx + 1] if idx + 1 < len(lines) else None + if nxt and not nxt.in_code and nxt.body.strip() and not _DEF_RE.match(nxt.body): + lead = len(nxt.body) - len(nxt.body.lstrip()) + wrapped = _URL_TOKEN_RE.match(nxt.body[lead:]) + if wrapped: + group = 1 if wrapped.group(1) is not None else 2 + url = wrapped.group(group) + url_start = nxt.start + lead + wrapped.start(group) + url_end = nxt.start + lead + wrapped.end(group) + line_end = nxt.number + block_end = lines[line_end].start if line_end < len(lines) else len(text) + definition = Definition( + label=normalize_label(raw_label), + raw_label=raw_label, + url=url, + line_start=line.number, + line_end=line_end, + url_start=url_start, + url_end=url_end, + text=text[line.start:block_end], + ) + defs.setdefault(definition.label, []).append(definition) + return defs + + +def _parse_images(doc: str, lines: list[_Line], defs: dict[str, list[Definition]]) -> list[ImageRef]: + refs: list[ImageRef] = [] + for line in lines: + if line.in_code: + continue + body = _mask_code_spans(line.body) + if _DEF_RE.match(body): + continue + for match in _IMG_START_RE.finditer(body): + open_index = match.start() + 1 + close = _bracket_end(body, open_index) + if close < 0: + continue + alt = body[open_index + 1:close] + after = body[close + 1:] + if after.startswith("["): + end = after.find("]") + if end < 0: + continue + label = after[1:end] or alt + refs.append(ImageRef(doc, line.number, "reference", normalize_label(label), label, None)) + elif after.startswith("("): + token = _URL_TOKEN_RE.match(after[1:].lstrip()) + url = None + if token: + url = token.group(1) if token.group(1) is not None else token.group(2).rstrip(")") + refs.append(ImageRef(doc, line.number, "inline", None, None, url)) + elif normalize_label(alt) in defs: + refs.append(ImageRef(doc, line.number, "reference", normalize_label(alt), alt, None)) + for match in _HTML_IMG_RE.finditer(body): + url = next((g for g in match.groups() if g is not None), None) + if url is not None and url.startswith("{"): + url = None # a JSX expression; not a literal URL + refs.append(ImageRef(doc, line.number, "html", None, None, url)) + return refs + + +def scan_text(doc: str, text: str) -> PageScan: + lines = _lines(text) + defs = _parse_definitions(doc, text, lines) + return PageScan(doc=doc, refs=_parse_images(doc, lines, defs), definitions=defs) + + +def doc_files(manual_root: Path) -> list[Path]: + docs_dir = Path(manual_root) / DOCS_SUBDIR + if not docs_dir.is_dir(): + raise PipelineError(f"docs directory not found: {docs_dir}") + return sorted(p for p in docs_dir.rglob("*") if p.is_file() and p.suffix in DOC_SUFFIXES) + + +def rel_doc(manual_root: Path, path: Path) -> str: + return Path(path).resolve().relative_to(Path(manual_root).resolve()).as_posix() + + +def scan_docs(manual_root: Path) -> list[PageScan]: + return [scan_text(rel_doc(manual_root, p), read_text_exact(p)) for p in doc_files(manual_root)] + + +def url_format(url: str | None) -> str: + if not url: + return "(none)" + suffix = PurePosixPath(urlparse(url).path).suffix.lower().lstrip(".") + return suffix or "(none)" + + +def is_missing_url(url: str | None) -> bool: + return url is None or not url.strip() or url.strip().upper() == "TODO" + + +def _iter_ref_urls(page: PageScan) -> Iterator[tuple[ImageRef, str | None]]: + for ref in page.refs: + yield ref, page.ref_url(ref) + + +def inventory(pages: list[PageScan], managed: dict[tuple[str, str], str]) -> dict: + """Observed counts. `managed` maps (doc, normalized label) to an entry id. + + Formats are by URL extension only; the content is never fetched. + """ + page_rows = [] + all_refs = [] + total_refs = 0 + total_defs = 0 + total_urls: set[str] = set() + total_by_ref: Counter[str] = Counter() + for page in pages: + if not page.refs: + continue + urls: list[str] = [] + by_ref: Counter[str] = Counter() + labels = page.image_labels() + defs = sum(len(page.definitions.get(label, [])) for label in labels) + for ref, url in _iter_ref_urls(page): + fmt = url_format(url) + by_ref[fmt] += 1 + if url and not is_missing_url(url): + urls.append(url) + all_refs.append({ + "doc": page.doc, + "line": ref.line, + "kind": ref.kind, + "ref": ref.raw_label if ref.kind == "reference" else None, + "url": url, + "format": fmt, + "managed_id": managed.get((page.doc, ref.label or "")) if ref.kind == "reference" else None, + }) + unique = sorted(set(urls)) + page_rows.append({ + "doc": page.doc, + "references": len(page.refs), + "definitions": defs, + "unique_urls": len(unique), + "formats_by_reference": dict(sorted(by_ref.items())), + "formats_by_unique_url": dict(sorted(Counter(url_format(u) for u in unique).items())), + }) + total_refs += len(page.refs) + total_defs += defs + total_urls.update(unique) + total_by_ref.update(by_ref) + return { + "pages": page_rows, + "total": { + "pages_with_images": len(page_rows), + "references": total_refs, + "definitions": total_defs, + "unique_urls": len(total_urls), + "formats_by_reference": dict(sorted(total_by_ref.items())), + "formats_by_unique_url": dict(sorted(Counter(url_format(u) for u in total_urls).items())), + }, + "refs": all_refs, + } + + +def find_definitions(text: str, label: str) -> list[Definition]: + return scan_text("", text).definitions.get(normalize_label(label), []) + + +def replace_url(text: str, definition: Definition, new_url: str) -> str: + """Swaps only the URL token; every other byte stays as it was.""" + if not new_url or any(c.isspace() for c in new_url): + raise PipelineError(f"refusing to write an invalid URL {new_url!r}") + if definition.url_start == definition.url_end and not definition.url: + return text[:definition.url_start] + " " + new_url + text[definition.url_end:] + return text[:definition.url_start] + new_url + text[definition.url_end:] + + +def replace_definition_url(text: str, label: str, new_url: str, expect_url: str) -> str: + """Rewrites the single definition of `label`, which must carry expect_url.""" + defs = find_definitions(text, label) + if len(defs) != 1: + raise PipelineError(f"ref [{label}] has {len(defs)} definitions, expected exactly one") + if defs[0].url != expect_url: + raise PipelineError(f"ref [{label}] carries {defs[0].url!r}, expected {expect_url!r}") + return replace_url(text, defs[0], new_url) diff --git a/scripts/screenshots/envhash.py b/scripts/screenshots/envhash.py new file mode 100644 index 0000000..b12d25c --- /dev/null +++ b/scripts/screenshots/envhash.py @@ -0,0 +1,113 @@ +"""Environment record hashing, per the C++/Python contract. + +env_hash_core is computed by the binary and recomputed here; both must agree. +The final env_hash adds the platform tag and the font file hashes, which only +the Python side knows. +""" + +from __future__ import annotations + +import math +from typing import Any + +from common import PipelineError, canonical_json, sha256_bytes + +ENV_SCHEMA = 1 + +# Recorded but never hashed. The first two also vary between two runs on the +# same machine, so they are dropped from the lock and from repeat comparisons. +RUN_VOLATILE_KEYS = ("display", "profile_path") + + +def _int(value: Any, where: str) -> int: + """Pixel, DPR and DPI values are hashed as integers. + + Non-integral floats are rounded half away from zero, which matches + qRound() on the C++ side for the non-negative values involved. + """ + if isinstance(value, bool): + raise PipelineError(f"env record: {where} must be a number") + if isinstance(value, int): + return value + if isinstance(value, float): + return int(math.floor(value + 0.5)) if value >= 0 else -int(math.floor(-value + 0.5)) + raise PipelineError(f"env record: {where} must be a number, got {value!r}") + + +def core_subset(env: dict[str, Any]) -> dict[str, Any]: + try: + os_info = env["os"] + screen = env["screen"] + size = screen["size"] + return { + "os": {"product": os_info["product"], "build": os_info["build"]}, + "qt_version": env["qt_version"], + "platform_name": env["platform_name"], + "screen": { + "size": [_int(size[0], "screen.size"), _int(size[1], "screen.size")], + "dpr": _int(screen["dpr"], "screen.dpr"), + "logical_dpi": _int(screen["logical_dpi"], "screen.logical_dpi"), + "physical_dpi": _int(screen["physical_dpi"], "screen.physical_dpi"), + }, + "style": env["style"], + "color_scheme": env["color_scheme"], + "locale": env["locale"], + "clock": env["clock"], + "font_requested": list(env["font_requested"]), + "font_resolved": list(env["font_resolved"]), + "system_font": env["system_font"], + "gnupg_version": env["gnupg_version"], + "gpgme_version": env["gpgme_version"], + "gpg_clock": env["gpg_clock"], + "tz_offset_check": _int(env["tz_offset_check"], "tz_offset_check"), + } + except (KeyError, IndexError, TypeError) as exc: + raise PipelineError(f"env record incomplete: {exc!r}") from exc + + +def canonical_core(env: dict[str, Any]) -> bytes: + return canonical_json(core_subset(env)) + + +def env_hash_core(env: dict[str, Any]) -> str: + return sha256_bytes(canonical_core(env)) + + +def font_file_list(env: dict[str, Any]) -> list[str]: + """Families whose files are hashed: font_resolved order, then system_font.""" + families = list(env.get("font_resolved") or []) + families.append(env.get("system_font") or "") + return families + + +def final_env_hash(core_hash: str, platform: str, font_files: list[dict[str, Any]]) -> str: + payload = { + "core": core_hash, + "platform": platform, + "font_files": [{"family": f["family"], "sha256": f.get("sha256")} for f in font_files], + } + return sha256_bytes(canonical_json(payload)) + + +def _windows_build(env: dict[str, Any]) -> int: + digits = "".join(ch if ch.isdigit() else " " for ch in str(env.get("os", {}).get("build", ""))) + numbers = [int(part) for part in digits.split()] + return max(numbers) if numbers else 0 + + +def platform_tag(env: dict[str, Any], under_xvfb: bool) -> str: + """windows11, linux-xvfb, or a descriptive non-reference tag.""" + name = str(env.get("platform_name", "")) + if name == "windows": + product = str(env.get("os", {}).get("product", "")) + if "Windows 11" in product or _windows_build(env) >= 22000: + return "windows11" + return "windows-other" + if name == "xcb": + return "linux-xvfb" if under_xvfb else "linux-xcb" + return f"other-{name or 'unknown'}" + + +def strip_volatile(env: dict[str, Any]) -> dict[str, Any]: + """The env as stored in the committed lock: no temp paths or display ids.""" + return {k: v for k, v in env.items() if k not in RUN_VOLATILE_KEYS} diff --git a/scripts/screenshots/fonts.py b/scripts/screenshots/fonts.py new file mode 100644 index 0000000..1905bbe --- /dev/null +++ b/scripts/screenshots/fonts.py @@ -0,0 +1,129 @@ +"""Finds the font file a family name resolves to, and hashes it. + +Windows: the HKLM and HKCU `...\\Windows NT\\CurrentVersion\\Fonts` registry +keys. Linux and others: `fc-match`. + +Limit: Qt does not expose which file it used for each glyph, so glyph-level +fallback outside the listed families cannot be observed here. +""" + +from __future__ import annotations + +import os +import re +import shutil +import subprocess +import sys +from pathlib import Path +from typing import Callable, Optional + +from common import sha256_file + +FontResolver = Callable[[str], Optional[Path]] + +_REG_SUFFIX_RE = re.compile(r"\s*\((TrueType|OpenType|All res)\)\s*$", re.IGNORECASE) +_STYLE_WORDS = { + "regular", "book", "normal", "roman", "medium", "bold", "italic", "oblique", + "light", "thin", "black", "heavy", "semibold", "extrabold", "extralight", + "demibold", "ultralight", "condensed", "semicondensed", +} +_FONTS_KEY = r"SOFTWARE\Microsoft\Windows NT\CurrentVersion\Fonts" + + +def _fc_escape(family: str) -> str: + return re.sub(r"([\\\-:,])", r"\\\1", family) + + +def resolve_fc_match(family: str) -> Optional[Path]: + exe = shutil.which("fc-match") + if not exe or not family: + return None + try: + out = subprocess.run( + [exe, "-f", "%{file}", _fc_escape(family)], + capture_output=True, timeout=30, check=False, + ) + except (OSError, subprocess.TimeoutExpired): + return None + path = out.stdout.decode("utf-8", "replace").strip() + return Path(path) if path and Path(path).is_file() else None + + +def _registry_fonts() -> list[tuple[str, str]]: + import winreg # type: ignore[import-not-found] + + found: list[tuple[str, str]] = [] + for hive in (winreg.HKEY_LOCAL_MACHINE, winreg.HKEY_CURRENT_USER): + try: + key = winreg.OpenKey(hive, _FONTS_KEY) + except OSError: + continue + with key: + index = 0 + while True: + try: + name, data, _ = winreg.EnumValue(key, index) + except OSError: + break + index += 1 + if isinstance(data, str): + found.append((name, data)) + return found + + +def match_registry(family: str, values: list[tuple[str, str]]) -> Optional[str]: + """Picks the registry file for `family`: the regular face if listed, + else the first style face in name order. Pure, so it is testable.""" + wanted = family.casefold() + exact: list[tuple[str, str]] = [] + styled: list[tuple[str, str]] = [] + for name, data in values: + for face in _REG_SUFFIX_RE.sub("", name).split(" & "): + face_cf = face.strip().casefold() + if face_cf in (wanted, wanted + " regular"): + exact.append((face_cf, data)) + elif face_cf.startswith(wanted + " "): + rest = face_cf[len(wanted) + 1:].split() + if rest and all(word in _STYLE_WORDS for word in rest): + styled.append((face_cf, data)) + for group in (exact, styled): + if group: + return sorted(group)[0][1] + return None + + +def resolve_windows(family: str) -> Optional[Path]: + data = match_registry(family, _registry_fonts()) + if not data: + return None + path = Path(data) + if not path.is_absolute(): + path = Path(os.environ.get("WINDIR", r"C:\Windows")) / "Fonts" / data + return path if path.is_file() else None + + +def resolve_font_file(family: str) -> Optional[Path]: + if sys.platform == "win32": + return resolve_windows(family) + return resolve_fc_match(family) + + +def font_files(families: list[str], resolver: FontResolver = resolve_font_file, + cache: Optional[dict[str, dict]] = None) -> list[dict]: + """[{family, file, sha256}] in the given order; sha256 is None when the + family resolves to no file.""" + out = [] + for family in families: + if cache is not None and family in cache: + out.append(dict(cache[family])) + continue + path = resolver(family) if family else None + record = { + "family": family, + "file": path.name if path else None, + "sha256": sha256_file(path) if path else None, + } + if cache is not None: + cache[family] = record + out.append(dict(record)) + return out diff --git a/scripts/screenshots/gitutil.py b/scripts/screenshots/gitutil.py new file mode 100644 index 0000000..5761f8c --- /dev/null +++ b/scripts/screenshots/gitutil.py @@ -0,0 +1,63 @@ +"""Read-only git queries against the GpgFrontend repository. + +The pipeline never runs git in the Manual repository. +""" + +from __future__ import annotations + +import subprocess +from pathlib import Path + +from common import PipelineError + + +def _git(repo: Path, *args: str) -> subprocess.CompletedProcess: + try: + return subprocess.run( + ["git", "-c", "core.quotepath=off", "-C", str(repo), *args], + capture_output=True, check=False, timeout=120, + ) + except OSError as exc: + raise PipelineError(f"cannot run git: {exc}") from exc + + +def _out(proc: subprocess.CompletedProcess) -> str: + return proc.stdout.decode("utf-8", "replace") + + +def head(repo: Path) -> str: + proc = _git(repo, "rev-parse", "HEAD") + if proc.returncode != 0: + raise PipelineError(f"{repo}: not a git repository with a HEAD commit") + return _out(proc).strip() + + +def commit_exists(repo: Path, commit: str) -> bool: + if not commit: + return False + return _git(repo, "cat-file", "-e", f"{commit}^{{commit}}").returncode == 0 + + +def is_ancestor(repo: Path, commit: str, ref: str = "HEAD") -> bool: + proc = _git(repo, "merge-base", "--is-ancestor", commit, ref) + if proc.returncode not in (0, 1): + raise PipelineError(f"git merge-base failed: {proc.stderr.decode('utf-8', 'replace').strip()}") + return proc.returncode == 0 + + +def changed_since(repo: Path, commit: str, paths: list[str]) -> list[str]: + if not paths: + return [] + proc = _git(repo, "diff", "--name-only", commit, "HEAD", "--", *paths) + if proc.returncode != 0: + raise PipelineError(f"git diff failed: {proc.stderr.decode('utf-8', 'replace').strip()}") + return [line for line in _out(proc).splitlines() if line.strip()] + + +def dirty(repo: Path, paths: list[str]) -> list[str]: + if not paths: + return [] + proc = _git(repo, "status", "--porcelain", "--", *paths) + if proc.returncode != 0: + raise PipelineError(f"git status failed: {proc.stderr.decode('utf-8', 'replace').strip()}") + return [line for line in _out(proc).splitlines() if line.strip()] diff --git a/scripts/screenshots/manifest.py b/scripts/screenshots/manifest.py new file mode 100644 index 0000000..3850f1b --- /dev/null +++ b/scripts/screenshots/manifest.py @@ -0,0 +1,353 @@ +"""Loads screenshots.yaml and resolves every entry against the defaults. + +check.py, render.py and run.py all go through resolve(), so the +configuration that is probed, rendered and hashed is always the same one. +""" + +from __future__ import annotations + +import re +from dataclasses import dataclass, field +from datetime import datetime +from pathlib import Path +from typing import Any + +import yaml + +from common import SCRIPT_DIR, UsageError, canonical_json, read_text_exact, sha256_bytes, sha256_file + +KNOWN_PLATFORMS = ("windows11", "linux-xvfb") +ENTRY_STATUSES = ("manual-only", "animated") +HIGHLIGHT_STYLES = ("box", "arrow") +THEMES = ("light", "dark") + +_SIZE_RE = re.compile(r"^([1-9][0-9]{1,4})x([1-9][0-9]{1,4})$") +_CLOCK_RE = re.compile(r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z$") +_ID_RE = re.compile(r"^[a-z0-9][a-z0-9-]*$") + +BUILTIN_DEFAULTS: dict[str, Any] = { + "platform": "windows11", + "locale": "en_US", + "theme": "light", + "size": "1100x680", + "fonts": ["Noto Sans", "Noto Sans CJK SC"], + "clock": "2026-01-15T10:00:00Z", + "timeout_ms": 30000, + "frame": {"radius": 8, "shadow": 12, "padding": 24}, + "diff": {"tolerance": 16, "threshold": 0.001}, + "preview_platforms": ["linux-xvfb"], +} + +_ENTRY_KEYS = { + "id", "doc", "ref", "scenario", "locale", "platform", "size", "theme", + "fonts", "clock", "timeout_ms", "sources", "highlight", "status", + "frame", "diff", +} +_DEFAULT_KEYS = set(BUILTIN_DEFAULTS) +_FRAME_KEYS = {"radius", "shadow", "padding"} +_DIFF_KEYS = {"tolerance", "threshold"} + +# Settings that never change pixels; they stay out of inputs_hash so that +# moving an image to another page does not force a re-render. +_NON_RENDER_KEYS = ("id", "doc", "ref", "sources", "status", "timeout_ms") + + +@dataclass(frozen=True) +class Highlight: + name: str + style: str = "box" + + def as_dict(self) -> dict[str, str]: + return {"name": self.name, "style": self.style} + + +@dataclass(frozen=True) +class Entry: + id: str + doc: str + ref: str + scenario: str | None + locale: str + platform: str + size: tuple[int, int] | None # None means "auto": the scenario's natural size + theme: str + fonts: tuple[str, ...] + clock: str + timeout_ms: int + sources: tuple[str, ...] + highlight: tuple[Highlight, ...] + status: str | None + frame: dict[str, Any] = field(hash=False, compare=False) + diff: dict[str, Any] = field(hash=False, compare=False) + + @property + def renderable(self) -> bool: + return self.status is None + + @property + def size_str(self) -> str: + return "auto" if self.size is None else f"{self.size[0]}x{self.size[1]}" + + def probe_key(self) -> tuple[str, str, tuple[str, ...], str, tuple[int, int] | None]: + return (self.locale, self.theme, self.fonts, self.clock, self.size) + + def settings(self) -> dict[str, Any]: + """The fully resolved entry as plain JSON data.""" + return { + "id": self.id, + "doc": self.doc, + "ref": self.ref, + "scenario": self.scenario, + "locale": self.locale, + "platform": self.platform, + "size": self.size_str, + "theme": self.theme, + "fonts": list(self.fonts), + "clock": self.clock, + "timeout_ms": self.timeout_ms, + "sources": list(self.sources), + "highlight": [h.as_dict() for h in self.highlight], + "status": self.status, + "frame": dict(self.frame), + "diff": dict(self.diff), + } + + +@dataclass +class Manifest: + path: Path + defaults: dict[str, Any] + global_sources: tuple[str, ...] + preview_platforms: tuple[str, ...] + entries: list[Entry] + + def by_id(self) -> dict[str, Entry]: + return {entry.id: entry for entry in self.entries} + + def relevant_sources(self, entry: Entry) -> list[str]: + return list(entry.sources) + [s for s in self.global_sources if s not in entry.sources] + + +def _fail(where: str, message: str) -> UsageError: + return UsageError(f"screenshots.yaml: {where}: {message}") + + +def _check_keys(where: str, data: dict[str, Any], allowed: set[str]) -> None: + unknown = sorted(set(data) - allowed) + if unknown: + raise _fail(where, f"unknown key(s): {', '.join(unknown)}") + + +def parse_size(where: str, value: Any) -> tuple[int, int] | None: + if str(value) == "auto": + return None + match = _SIZE_RE.match(str(value)) + if not match: + raise _fail(where, f"size must look like 1100x680 or be auto, got {value!r}") + return int(match.group(1)), int(match.group(2)) + + +def validate_clock(where: str, value: Any) -> str: + text = str(value) + if not _CLOCK_RE.match(text): + raise _fail(where, f"clock must be ISO-8601 UTC like 2026-01-15T10:00:00Z, got {value!r}") + try: + datetime.strptime(text, "%Y-%m-%dT%H:%M:%SZ") + except ValueError as exc: + raise _fail(where, f"clock {text!r} is not a real date: {exc}") from exc + return text + + +def _fonts(where: str, value: Any) -> tuple[str, ...]: + if isinstance(value, str): + value = [v.strip() for v in value.split(",")] + if not isinstance(value, list) or not value: + raise _fail(where, "fonts must be a non-empty list of family names") + out = [] + for family in value: + if not isinstance(family, str) or not family.strip() or "," in family: + raise _fail(where, f"invalid font family {family!r}") + out.append(family.strip()) + return tuple(out) + + +def _merge_section(where: str, base: dict[str, Any], override: Any, keys: set[str]) -> dict[str, Any]: + if override is None: + return dict(base) + if not isinstance(override, dict): + raise _fail(where, "must be a mapping") + _check_keys(where, override, keys) + merged = dict(base) + merged.update(override) + return merged + + +def _highlights(where: str, value: Any) -> tuple[Highlight, ...]: + if value is None: + return () + items = value if isinstance(value, list) else [value] + out: list[Highlight] = [] + for item in items: + if isinstance(item, str): + out.append(Highlight(item)) + continue + if not isinstance(item, dict) or "name" not in item: + raise _fail(where, f"highlight must be a name or {{name, style}}, got {item!r}") + _check_keys(where + ".highlight", item, {"name", "style"}) + style = item.get("style", "box") + if style not in HIGHLIGHT_STYLES: + raise _fail(where, f"highlight style must be box or arrow, got {style!r}") + out.append(Highlight(str(item["name"]), style)) + return tuple(out) + + +def _str_list(where: str, value: Any) -> tuple[str, ...]: + if value is None: + return () + if not isinstance(value, list) or not all(isinstance(v, str) and v for v in value): + raise _fail(where, "must be a list of non-empty strings") + return tuple(value) + + +def _resolve_entry(index: int, raw: Any, defaults: dict[str, Any]) -> Entry: + where = f"entries[{index}]" + if not isinstance(raw, dict): + raise _fail(where, "must be a mapping") + _check_keys(where, raw, _ENTRY_KEYS) + entry_id = raw.get("id") + if not isinstance(entry_id, str) or not _ID_RE.match(entry_id): + raise _fail(where, f"id must be lower-case letters, digits and dashes, got {entry_id!r}") + where = f"entry {entry_id}" + for key in ("doc", "ref"): + if not isinstance(raw.get(key), str) or not raw[key].strip(): + raise _fail(where, f"{key} is required") + status = raw.get("status") + if status is not None and status not in ENTRY_STATUSES: + raise _fail(where, f"status must be one of {', '.join(ENTRY_STATUSES)}") + scenario = raw.get("scenario") + if status is None and (not isinstance(scenario, str) or not scenario): + raise _fail(where, "scenario is required unless status is manual-only or animated") + + def pick(key: str) -> Any: + return raw[key] if key in raw else defaults[key] + + platform = str(pick("platform")) + if platform not in KNOWN_PLATFORMS: + raise _fail(where, f"platform must be one of {', '.join(KNOWN_PLATFORMS)}") + theme = str(pick("theme")) + if theme not in THEMES: + raise _fail(where, "theme must be light or dark") + timeout_ms = pick("timeout_ms") + if not isinstance(timeout_ms, int) or timeout_ms <= 0: + raise _fail(where, "timeout_ms must be a positive integer") + doc = raw["doc"].replace("\\", "/").strip() + return Entry( + id=entry_id, + doc=doc, + ref=raw["ref"].strip(), + scenario=scenario if isinstance(scenario, str) else None, + locale=str(pick("locale")), + platform=platform, + size=parse_size(where, pick("size")), + theme=theme, + fonts=_fonts(where, pick("fonts")), + clock=validate_clock(where, pick("clock")), + timeout_ms=timeout_ms, + sources=_str_list(where + ".sources", raw.get("sources")), + highlight=_highlights(where, raw.get("highlight")), + status=status, + frame=_merge_section(where + ".frame", defaults["frame"], raw.get("frame"), _FRAME_KEYS), + diff=_merge_section(where + ".diff", defaults["diff"], raw.get("diff"), _DIFF_KEYS), + ) + + +def _validate_numbers(where: str, frame: dict[str, Any], diff: dict[str, Any]) -> None: + for key in _FRAME_KEYS: + if not isinstance(frame.get(key), int) or frame[key] < 0: + raise _fail(where, f"frame.{key} must be a non-negative integer") + tol = diff.get("tolerance") + if not isinstance(tol, int) or not 0 <= tol <= 255: + raise _fail(where, "diff.tolerance must be an integer 0..255") + thr = diff.get("threshold") + if not isinstance(thr, (int, float)) or not 0 <= thr <= 1: + raise _fail(where, "diff.threshold must be a number 0..1") + + +def parse_manifest(data: Any, path: Path) -> Manifest: + if not isinstance(data, dict): + raise _fail("top level", "must be a mapping") + _check_keys("top level", data, {"defaults", "global_sources", "entries"}) + raw_defaults = data.get("defaults") or {} + if not isinstance(raw_defaults, dict): + raise _fail("defaults", "must be a mapping") + _check_keys("defaults", raw_defaults, _DEFAULT_KEYS) + defaults = dict(BUILTIN_DEFAULTS) + defaults.update({k: v for k, v in raw_defaults.items() if k not in ("frame", "diff")}) + defaults["frame"] = _merge_section("defaults.frame", BUILTIN_DEFAULTS["frame"], raw_defaults.get("frame"), _FRAME_KEYS) + defaults["diff"] = _merge_section("defaults.diff", BUILTIN_DEFAULTS["diff"], raw_defaults.get("diff"), _DIFF_KEYS) + preview = _str_list("defaults.preview_platforms", defaults["preview_platforms"]) + raw_entries = data.get("entries") or [] + if not isinstance(raw_entries, list): + raise _fail("entries", "must be a list") + entries = [_resolve_entry(i, raw, defaults) for i, raw in enumerate(raw_entries)] + seen: dict[str, str] = {} + targets: dict[tuple[str, str], str] = {} + for entry in entries: + _validate_numbers(f"entry {entry.id}", entry.frame, entry.diff) + if entry.id in seen: + raise _fail(f"entry {entry.id}", "duplicate id") + seen[entry.id] = entry.id + target = (entry.doc, entry.ref.casefold()) + if target in targets: + raise _fail(f"entry {entry.id}", f"doc/ref already used by {targets[target]}") + targets[target] = entry.id + return Manifest( + path=path, + defaults=defaults, + global_sources=_str_list("global_sources", data.get("global_sources")), + preview_platforms=preview, + entries=entries, + ) + + +def load_manifest(path: Path) -> Manifest: + path = Path(path) + if not path.exists(): + raise UsageError(f"manifest not found: {path}") + try: + data = yaml.safe_load(read_text_exact(path)) + except yaml.YAMLError as exc: + raise UsageError(f"{path}: invalid YAML: {exc}") from exc + return parse_manifest(data, path) + + +def pipeline_files(pipeline_dir: Path = SCRIPT_DIR) -> dict[str, str]: + """sha256 of every pipeline file: top-level *.py plus requirements.txt. + + Tests live under tests/ and are not part of the pipeline. + """ + pipeline_dir = Path(pipeline_dir) + files = sorted(pipeline_dir.glob("*.py")) + req = pipeline_dir / "requirements.txt" + if req.exists(): + files.append(req) + return {f.name: sha256_file(f) for f in files} + + +def inputs_hash(entry: Entry, processing: dict[str, Any], pipeline_dir: Path = SCRIPT_DIR) -> str: + """sha256 over resolved render settings, pipeline files and post-processing. + + `processing` carries the optimizer in use (`oxipng <version>` or `none`) + and the encoder versions; see postprocess.processing_descriptor(). + """ + settings = entry.settings() + for key in _NON_RENDER_KEYS: + settings.pop(key, None) + payload = { + "entry": settings, + "frame": dict(entry.frame), + "diff": dict(entry.diff), + "pipeline": pipeline_files(pipeline_dir), + "processing": processing, + } + return sha256_bytes(canonical_json(payload)) diff --git a/scripts/screenshots/postprocess.py b/scripts/screenshots/postprocess.py new file mode 100644 index 0000000..d50d341 --- /dev/null +++ b/scripts/screenshots/postprocess.py @@ -0,0 +1,219 @@ +"""Frame, shadow, padding and highlights for a raw capture; PNG and lossless WebP out. + +Every size is given in logical pixels and scaled by the capture's DPR. All +drawing uses integer coordinates so the output depends only on the inputs and +the Pillow/libwebp versions (both recorded in inputs_hash). +""" + +from __future__ import annotations + +import re +import shutil +import subprocess +from dataclasses import dataclass +from functools import lru_cache +from pathlib import Path +from typing import Any + +import PIL +from PIL import Image, ImageChops, ImageDraw, ImageFilter, features + +from common import PipelineError, atomic_write_bytes, sha256_bytes, sha256_file +from manifest import Entry, Highlight + +HIGHLIGHT_RGBA = (229, 72, 77, 255) +HIGHLIGHT_WIDTH = 2 +HIGHLIGHT_OUTSET = 3 +HIGHLIGHT_RADIUS = 4 +ARROW_LENGTH = 40 +ARROW_HEAD = 10 +ARROW_WIDTH = 3 +ARROW_GAP = 4 +SHADOW_OPACITY = 0.30 +SHADOW_OFFSET_Y = 4 + + +class PostprocessError(PipelineError): + def __init__(self, reason: str, detail: str = "") -> None: + super().__init__(f"{reason}: {detail}" if detail else reason) + self.reason = reason + + +@dataclass +class Processed: + png: Path + webp: Path + png_sha256: str + webp_sha256: str + pixels_sha256: str + + +@lru_cache(maxsize=1) +def optimizer() -> str: + """`oxipng <version>` when available, else `none`.""" + exe = shutil.which("oxipng") + if not exe: + return "none" + try: + out = subprocess.run([exe, "--version"], capture_output=True, timeout=30, check=False) + except (OSError, subprocess.TimeoutExpired): + return "none" + match = re.search(r"(\d+\.\d+(?:\.\d+)?)", out.stdout.decode("utf-8", "replace")) + return f"oxipng {match.group(1)}" if match and out.returncode == 0 else "none" + + +def processing_descriptor() -> dict[str, str]: + return { + "optimizer": optimizer(), + "pillow": PIL.__version__, + "libwebp": str(features.version("webp")), + } + + +def pixels_sha256(image: Image.Image) -> str: + return sha256_bytes(image.convert("RGBA").tobytes()) + + +def png_pixels_sha256(path: Path) -> str: + with Image.open(path) as img: + return pixels_sha256(img) + + +def _rect_for(geometry: dict[str, Any], name: str) -> tuple[int, int, int, int]: + if name in (geometry.get("missing") or []): + raise PostprocessError("highlight-missing", name) + widget = (geometry.get("widgets") or {}).get(name) + if not isinstance(widget, dict) or not isinstance(widget.get("rect"), list) or len(widget["rect"]) != 4: + raise PostprocessError("highlight-unknown", name) + x, y, w, h = (int(v) for v in widget["rect"]) + if w <= 0 or h <= 0: + raise PostprocessError("highlight-missing", f"{name} has an empty rect") + return x, y, w, h + + +def _draw_box(draw: ImageDraw.ImageDraw, rect: tuple[int, int, int, int], size: tuple[int, int], dpr: int) -> None: + x, y, w, h = rect + out = HIGHLIGHT_OUTSET * dpr + width = HIGHLIGHT_WIDTH * dpr + left = max(0, x - out) + top = max(0, y - out) + right = min(size[0] - 1, x + w - 1 + out) + bottom = min(size[1] - 1, y + h - 1 + out) + if right - left < 2 * width or bottom - top < 2 * width: + raise PostprocessError("highlight-too-small", str(rect)) + draw.rounded_rectangle((left, top, right, bottom), radius=HIGHLIGHT_RADIUS * dpr, + outline=HIGHLIGHT_RGBA, width=width) + + +def _draw_arrow(draw: ImageDraw.ImageDraw, rect: tuple[int, int, int, int], size: tuple[int, int], dpr: int) -> None: + """An arrow pointing at the rect, from the left if there is room, else + from the right, else from below.""" + x, y, w, h = rect + length = ARROW_LENGTH * dpr + head = ARROW_HEAD * dpr + gap = ARROW_GAP * dpr + cy = y + h // 2 + cx = x + w // 2 + if x - gap - length >= 0: + tip, tail, axis = (x - gap, cy), (x - gap - length, cy), (-1, 0) + elif x + w + gap + length <= size[0]: + tip, tail, axis = (x + w + gap, cy), (x + w + gap + length, cy), (1, 0) + else: + tip = (cx, min(size[1] - 1, y + h + gap)) + tail, axis = (cx, min(size[1] - 1, tip[1] + length)), (0, 1) + draw.line((tail, tip), fill=HIGHLIGHT_RGBA, width=ARROW_WIDTH * dpr) + # axis points from the tip towards the tail. + ax, ay = axis + base = (tip[0] + ax * head, tip[1] + ay * head) + side = (-ay * (head // 2 + dpr), ax * (head // 2 + dpr)) + draw.polygon([tip, (base[0] + side[0], base[1] + side[1]), (base[0] - side[0], base[1] - side[1])], + fill=HIGHLIGHT_RGBA) + + +def apply_highlights(image: Image.Image, geometry: dict[str, Any], highlights: tuple[Highlight, ...], dpr: int) -> Image.Image: + if not highlights: + return image + overlay = Image.new("RGBA", image.size, (0, 0, 0, 0)) + draw = ImageDraw.Draw(overlay) + for highlight in highlights: + rect = _rect_for(geometry, highlight.name) + if highlight.style == "arrow": + _draw_arrow(draw, rect, image.size, dpr) + else: + _draw_box(draw, rect, image.size, dpr) + return Image.alpha_composite(image, overlay) + + +def apply_frame(image: Image.Image, frame: dict[str, Any], dpr: int) -> Image.Image: + radius = int(frame["radius"]) * dpr + blur = int(frame["shadow"]) * dpr + pad = int(frame["padding"]) * dpr + w, h = image.size + if radius: + mask = Image.new("L", image.size, 0) + ImageDraw.Draw(mask).rounded_rectangle((0, 0, w - 1, h - 1), radius=radius, fill=255) + image = image.copy() + image.putalpha(ImageChops.multiply(image.getchannel("A"), mask)) + canvas = Image.new("RGBA", (w + 2 * pad, h + 2 * pad), (0, 0, 0, 0)) + if blur: + alpha = image.getchannel("A").point(lambda v: int(v * SHADOW_OPACITY)) + shadow_alpha = Image.new("L", canvas.size, 0) + shadow_alpha.paste(alpha, (pad, pad + SHADOW_OFFSET_Y * dpr)) + shadow_alpha = shadow_alpha.filter(ImageFilter.GaussianBlur(blur / 2)) + shadow = Image.new("RGBA", canvas.size, (0, 0, 0, 0)) + shadow.putalpha(shadow_alpha) + canvas = Image.alpha_composite(canvas, shadow) + layer = Image.new("RGBA", canvas.size, (0, 0, 0, 0)) + layer.paste(image, (pad, pad)) + return Image.alpha_composite(canvas, layer) + + +def _encode_png(image: Image.Image, path: Path) -> None: + from io import BytesIO + + buf = BytesIO() + image.save(buf, format="PNG", compress_level=9) + atomic_write_bytes(path, buf.getvalue()) + if optimizer() != "none": + proc = subprocess.run( + [shutil.which("oxipng") or "oxipng", "--opt", "2", "--strip", "safe", "--quiet", str(path)], + capture_output=True, timeout=300, check=False, + ) + if proc.returncode != 0: + raise PostprocessError("oxipng-failed", proc.stderr.decode("utf-8", "replace").strip()) + + +def _encode_webp(image: Image.Image, path: Path) -> None: + from io import BytesIO + + buf = BytesIO() + image.save(buf, format="WEBP", lossless=True, quality=100, method=6, exact=True) + atomic_write_bytes(path, buf.getvalue()) + + +def process(raw_png: Path, geometry: dict[str, Any], entry: Entry, out_dir: Path, stem: str) -> Processed: + dpr = int(geometry.get("dpr") or 0) + if dpr != 2: + raise PostprocessError("dpr-mismatch", f"expected 2, got {geometry.get('dpr')!r}") + with Image.open(raw_png) as opened: + image = opened.convert("RGBA") + if list(image.size) != list(geometry.get("image_size") or []): + raise PostprocessError("size-mismatch", f"PNG {image.size} vs geometry {geometry.get('image_size')}") + image = apply_highlights(image, geometry, entry.highlight, dpr) + image = apply_frame(image, entry.frame, dpr) + png = Path(out_dir) / f"{stem}.png" + webp = Path(out_dir) / f"{stem}.webp" + _encode_png(image, png) + with Image.open(png) as final: + final_rgba = final.convert("RGBA") + _encode_webp(final_rgba, webp) + with Image.open(webp) as check: + if check.convert("RGBA").tobytes() != final_rgba.tobytes(): + raise PostprocessError("webp-not-lossless", str(webp)) + return Processed( + png=png, + webp=webp, + png_sha256=sha256_file(png), + webp_sha256=sha256_file(webp), + pixels_sha256=pixels_sha256(final_rgba), + ) diff --git a/scripts/screenshots/render.py b/scripts/screenshots/render.py new file mode 100644 index 0000000..52c7abf --- /dev/null +++ b/scripts/screenshots/render.py @@ -0,0 +1,483 @@ +"""Runs the GpgFrontend screenshot mode in an isolated throwaway environment. + +Both the render (`--screenshot <scenario>`) and the environment probe +(`--screenshot-env`) are built from the same resolved manifest entry and run +with the same subprocess environment, so their env records are comparable. +""" + +from __future__ import annotations + +import argparse +import json +import os +import shutil +import subprocess +import sys +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any, Optional + +from PIL import Image + +import envhash +import fonts +import gitutil +from common import (DEFAULT_CACHE, DEFAULT_MANIFEST, PipelineError, UsageError, canonical_json, + load_json, split_ids) +from manifest import Entry, Manifest, load_manifest + +XVFB_SCREEN = "-screen 0 2560x1600x24" +EXIT_REASONS = { + 2: "usage-or-precondition", + 3: "timeout", + 4: "output-unwritable", + 5: "unexpected-modal", + 6: "environment-gate", +} +CAPTURE_MODES = ("window", "dialog", "composite") +TIMEOUT_MARGIN_S = 60 +# Unix sockets are limited to 108 bytes. The demo homedir ends up at +# $XDG_DATA_HOME/BKTUS/<AppProfileName>/profiles/.tmp-<16hex>/db and gpg-agent +# puts S.gpg-agent.browser inside it. +_SOCKET_LIMIT = 107 +_SOCKET_TAIL = "/BKTUS/GpgFrontendXXXXXXXXXXXX/profiles/.tmp-0123456789abcdef/db/S.gpg-agent.browser" +# Paths that must not leak from the parent environment into a render. +_SCRUB_ENV = ( + "GNUPGHOME", "QT_SCALE_FACTOR", "QT_SCREEN_SCALE_FACTORS", "QT_AUTO_SCREEN_SCALE_FACTOR", + "QT_SCALE_FACTOR_ROUNDING_POLICY", "QT_ENABLE_HIGHDPI_SCALING", "QT_FONT_DPI", + "QT_QPA_PLATFORM", "QT_STYLE_OVERRIDE", "QT_QPA_PLATFORMTHEME", +) +APPDATA_ROOTS_ENV = "GF_SHOTS_APPDATA_ROOTS" +# Where GpgFrontend creates a temporary profile's tree (core/profile). +SESSION_STORAGE_ENV = "GF_PROFILE_PACKAGE_STORAGE_DIR" +NO_XVFB_ENV = "GF_SHOTS_NO_XVFB" + + +def default_work_root() -> Path: + # Fixed and free of the user's name: on Windows the temporary profile + # lives under TEMP, which points in here, and the settings page shows + # that path. A per-user %TEMP% would put the account name into the manual. + if os.name == "nt": + drive = os.environ.get("SystemDrive") or "C:" + return Path(drive + "\\") / "gf-shot" + return Path(f"/tmp/gf-shot-{os.getuid()}") + + +def fresh_session_base() -> Optional[Path]: + """Linux: the memory-backed folder the temporary profile is created in, + emptied first. + + The profile has the same name on every run, so the folder it lands in must + be one the pipeline owns and empties: a session a crash left in the + application's own default folder could not be found again (its pointer + lived in the throwaway directory) and would stand in the next run's way. + On Windows the session is made under TEMP, which already points into the + throwaway directory. + """ + shm = Path("/dev/shm") + if os.name == "nt" or not shm.is_dir(): + return None + base = shm / f"gf-shot-{os.getuid()}" + if base.is_symlink() or (base.exists() and base.stat().st_uid != os.getuid()): + raise PipelineError(f"{base} is a symlink or not owned by this user") + if base.exists(): + shutil.rmtree(base) + base.mkdir(mode=0o700) + return base + + +def default_xvfb_cmd() -> list[str]: + """xvfb-run on Linux. GF_SHOTS_NO_XVFB=1 runs without it (fake binaries in + tests); such renders get a non-reference platform tag.""" + if os.environ.get(NO_XVFB_ENV) == "1": + return [] + if sys.platform.startswith("linux"): + return ["xvfb-run", "-a", "-s", XVFB_SCREEN] + return [] + + +def real_appdata_roots() -> list[Path]: + """Where a real (non-ephemeral) GpgFrontend profile would write.""" + override = os.environ.get(APPDATA_ROOTS_ENV) + if override is not None: + return [Path(p) for p in override.split(os.pathsep) if p] + if os.name == "nt": + return [Path(os.environ[v]) / "BKTUS" for v in ("LOCALAPPDATA", "APPDATA") if os.environ.get(v)] + home = Path.home() + return [ + Path(os.environ.get("XDG_DATA_HOME") or home / ".local" / "share") / "BKTUS", + Path(os.environ.get("XDG_CONFIG_HOME") or home / ".config") / "BKTUS", + Path(os.environ.get("XDG_STATE_HOME") or home / ".local" / "state") / "BKTUS", + Path(os.environ.get("XDG_CACHE_HOME") or home / ".cache") / "BKTUS", + home / "Library" / "Application Support" / "BKTUS", + ] + + +def footprint_snapshot(roots: list[Path]) -> set[str]: + """Every path under <root>/GpgFrontend* for each root.""" + seen: set[str] = set() + for root in roots: + if not root.is_dir(): + continue + for child in root.iterdir(): + if not child.name.startswith("GpgFrontend"): + continue + seen.add(str(child)) + if child.is_dir(): + for dirpath, dirnames, filenames in os.walk(child): + for name in dirnames + filenames: + seen.add(os.path.join(dirpath, name)) + return seen + + +@dataclass +class EnvInfo: + record: dict[str, Any] + core_hash: str + env_hash: str + platform: str + font_files: list[dict[str, Any]] + + def lock_env(self) -> dict[str, Any]: + env = envhash.strip_volatile(self.record) + env["platform"] = self.platform + env["font_files"] = self.font_files + return env + + +@dataclass +class RunOutcome: + ok: bool + exit_code: Optional[int] + reason: str = "" + detail: str = "" + env: Optional[EnvInfo] = None + png: Optional[Path] = None + json_path: Optional[Path] = None + geometry: Optional[dict[str, Any]] = None + footprint: list[str] = field(default_factory=list) + + @property + def failure(self) -> str: + return f"{self.reason}: {self.detail}" if self.detail else self.reason + + +@dataclass +class RenderConfig: + binary: Path + gpgfrontend: Path + work_root: Optional[Path] = None + keep_work: bool = False + xvfb_cmd: Optional[list[str]] = None + appdata_roots: Optional[list[Path]] = None + font_resolver: fonts.FontResolver = fonts.resolve_font_file + + def resolved_xvfb(self) -> list[str]: + return default_xvfb_cmd() if self.xvfb_cmd is None else list(self.xvfb_cmd) + + +class Renderer: + def __init__(self, cfg: RenderConfig) -> None: + self.cfg = cfg + self.binary = Path(cfg.binary) + if not self.binary.is_file(): + raise UsageError(f"binary not found: {self.binary}") + self.work_root = Path(cfg.work_root) if cfg.work_root else default_work_root() + self.xvfb = cfg.resolved_xvfb() + if self.xvfb and not shutil.which(self.xvfb[0]): + raise UsageError(f"{self.xvfb[0]} not found; it is needed for Linux previews") + self._font_cache: dict[str, dict] = {} + self._probe_cache: dict[tuple, RunOutcome] = {} + + # Paths ----------------------------------------------------------------- + + def demo_gnupghome(self) -> Path: + return Path(self.cfg.gpgfrontend) / "resource" / "screenshot" / "demo-gnupghome" + + def fixtures(self) -> Path: + return Path(self.cfg.gpgfrontend) / "resource" / "screenshot" / "fixtures" + + def _prepare_root(self) -> None: + root = self.work_root + root.mkdir(parents=True, exist_ok=True, mode=0o700) + if os.name != "nt": + st = os.lstat(root) + if os.path.islink(root) or st.st_uid != os.getuid(): + raise PipelineError(f"work root {root} is a symlink or not owned by this user") + if len(str(root)) + 4 + len(_SOCKET_TAIL) > _SOCKET_LIMIT: + raise UsageError( + f"work root {root} is too long: the gpg-agent socket path would exceed " + f"{_SOCKET_LIMIT} bytes. Use a shorter --work-root.") + + def _new_workdir(self) -> Path: + """One fixed directory, emptied first: the application shows paths + under it, so its name must be the same on every run. Renders run one + at a time; two pipelines sharing a work root would collide.""" + self._prepare_root() + candidate = self.work_root / "run" + if candidate.is_symlink(): + raise PipelineError(f"{candidate} is a symlink; refusing to use it") + if candidate.exists(): + shutil.rmtree(candidate) + candidate.mkdir(mode=0o700) + return candidate + + def subprocess_env(self, work: Path) -> dict[str, str]: + env = {k: v for k, v in os.environ.items() if k not in _SCRUB_ENV} + env["TZ"] = "UTC" + if os.name == "nt": + layout = {"LOCALAPPDATA": "la", "APPDATA": "ra", "TEMP": "t", "TMP": "t", "USERPROFILE": "alice"} + else: + layout = {"HOME": "alice", "XDG_DATA_HOME": "d", "XDG_CONFIG_HOME": "c", + "XDG_STATE_HOME": "s", "XDG_CACHE_HOME": "k", "TMPDIR": "t"} + for var, sub in layout.items(): + path = work / sub + path.mkdir(exist_ok=True) + env[var] = str(path) + session = fresh_session_base() + if session is not None: + env[SESSION_STORAGE_ENV] = str(session) + return env + + # Command lines ----------------------------------------------------------- + + def command(self, entry: Entry, out: Path, probe: bool) -> list[str]: + mode = ["--screenshot-env"] if probe else ["--screenshot", entry.scenario or ""] + cmd = [ + str(self.binary), "--temp-profile", *mode, + "--locale", entry.locale, + "--theme", entry.theme, + "--font-family", ",".join(entry.fonts), + *(["--size", entry.size_str] if entry.size is not None else []), + "--clock", entry.clock, + "--demo-gnupghome", str(self.demo_gnupghome()), + "--fixtures", str(self.fixtures()), + "--out", str(out), + "--timeout", str(entry.timeout_ms), + ] + return self.xvfb + cmd + + def list_scenarios(self) -> dict[str, dict[str, str]]: + work = self._new_workdir() + try: + proc = subprocess.run( + self.xvfb + [str(self.binary), "--screenshot-list"], + env=self.subprocess_env(work), capture_output=True, timeout=120, check=False, + ) + finally: + self._cleanup(work) + if proc.returncode != 0: + raise PipelineError(f"--screenshot-list exited {proc.returncode}: " + f"{proc.stderr.decode('utf-8', 'replace').strip()[-400:]}") + scenarios = {} + for line in proc.stdout.decode("utf-8", "replace").splitlines(): + parts = line.strip().split("\t") + if len(parts) == 3: + scenarios[parts[0]] = {"size": parts[1], "capture_mode": parts[2]} + return scenarios + + # Running ----------------------------------------------------------------- + + def _cleanup(self, work: Path) -> None: + if not self.cfg.keep_work: + shutil.rmtree(work, ignore_errors=True) + + def _run(self, entry: Entry, probe: bool, dest_dir: Optional[Path], stem: str) -> RunOutcome: + work = self._new_workdir() + out_dir = work / "o" + out_dir.mkdir() + out = out_dir / (f"{stem}.json" if probe else f"{stem}.png") + json_path = out.with_suffix(".json") + roots = self.cfg.appdata_roots if self.cfg.appdata_roots is not None else real_appdata_roots() + before = footprint_snapshot(roots) + try: + try: + proc = subprocess.run( + self.command(entry, out, probe), env=self.subprocess_env(work), + capture_output=True, timeout=entry.timeout_ms / 1000 + TIMEOUT_MARGIN_S, check=False, + ) + code: Optional[int] = proc.returncode + stderr = proc.stderr.decode("utf-8", "replace").strip() + except subprocess.TimeoutExpired: + code, stderr = None, "the binary did not exit; killed by the runner" + except OSError as exc: + return RunOutcome(False, None, "launch-failed", str(exc)) + new = sorted(footprint_snapshot(roots) - before) + outcome = self._evaluate(entry, probe, code, stderr, out, json_path) + outcome.footprint = new + if new and outcome.ok: + outcome.ok = False + outcome.reason = "footprint" + outcome.detail = "new files outside the throwaway directory: " + ", ".join(new[:10]) + if dest_dir is not None: + dest_dir.mkdir(parents=True, exist_ok=True) + if not probe and out.exists(): + outcome.png = dest_dir / out.name + shutil.copyfile(out, outcome.png) + if json_path.exists(): + outcome.json_path = dest_dir / json_path.name + shutil.copyfile(json_path, outcome.json_path) + return outcome + finally: + self._cleanup(work) + + def _evaluate(self, entry: Entry, probe: bool, code: Optional[int], stderr: str, + out: Path, json_path: Path) -> RunOutcome: + record: Optional[dict[str, Any]] = None + if json_path.exists(): + try: + record = json.loads(json_path.read_bytes().decode("utf-8")) + except (ValueError, UnicodeDecodeError) as exc: + return RunOutcome(False, code, "invalid-json", str(exc)) + if code is None: + return RunOutcome(False, None, "timeout", stderr[-400:]) + if code != 0: + reason = EXIT_REASONS.get(code, f"exit-{code}") + detail = "" + if isinstance(record, dict): + err = record.get("error") or {} + if isinstance(err, dict) and err.get("reason"): + detail = str(err.get("reason")) + if err.get("detail"): + detail += f" ({err['detail']})" + env_part = record if probe else record.get("env") + if isinstance(env_part, dict) and env_part.get("gate_failures"): + detail += " gates: " + ",".join(map(str, env_part["gate_failures"])) + return RunOutcome(False, code, reason, (detail or stderr[-400:]).strip()) + if record is None: + return RunOutcome(False, code, "no-json", f"{json_path.name} was not written") + env_record = record if probe else record.get("env") + if not isinstance(env_record, dict): + return RunOutcome(False, code, "no-env", "the record carries no env") + try: + env = self.env_info(env_record) + except PipelineError as exc: + return RunOutcome(False, code, "env-invalid", str(exc)) + if env_record.get("gate_failures"): + return RunOutcome(False, code, "environment-gate", ",".join(map(str, env_record["gate_failures"])), env) + if probe: + return RunOutcome(True, code, env=env, geometry=None) + problem = check_geometry(record, out) + if problem: + return RunOutcome(False, code, "geometry", problem, env) + return RunOutcome(True, code, env=env, geometry=record) + + def env_info(self, record: dict[str, Any]) -> EnvInfo: + if record.get("schema") != envhash.ENV_SCHEMA: + raise PipelineError(f"unsupported env schema {record.get('schema')!r}") + core = envhash.env_hash_core(record) + if record.get("env_hash_core") != core: + raise PipelineError( + f"env_hash_core cross-check failed: binary says {record.get('env_hash_core')}, " + f"recomputed {core}") + files = fonts.font_files(envhash.font_file_list(record), self.cfg.font_resolver, self._font_cache) + platform = envhash.platform_tag(record, under_xvfb=bool(self.xvfb)) + return EnvInfo(record, core, envhash.final_env_hash(core, platform, files), platform, files) + + def probe(self, entry: Entry) -> RunOutcome: + key = entry.probe_key() + if key not in self._probe_cache: + self._probe_cache[key] = self._run(entry, probe=True, dest_dir=None, stem="env") + return self._probe_cache[key] + + def render(self, entry: Entry, dest_dir: Path) -> RunOutcome: + return self._run(entry, probe=False, dest_dir=dest_dir, stem=entry.id) + + +def check_geometry(geometry: dict[str, Any], png: Path) -> str: + if geometry.get("schema") != 1: + return f"unsupported geometry schema {geometry.get('schema')!r}" + if geometry.get("error"): + return f"record carries an error: {geometry['error']}" + if geometry.get("capture_mode") not in CAPTURE_MODES: + return f"unknown capture_mode {geometry.get('capture_mode')!r}" + if geometry.get("dpr") != 2: + return f"dpr-mismatch: expected 2, got {geometry.get('dpr')!r}" + if not png.exists(): + return f"{png.name} was not written" + with Image.open(png) as img: + actual = list(img.size) + if list(geometry.get("image_size") or []) != actual: + return f"image_size {geometry.get('image_size')} does not match the PNG {actual}" + if not isinstance(geometry.get("widgets"), dict) or not isinstance(geometry.get("missing", []), list): + return "widgets/missing malformed" + return "" + + +def comparable_json(path: Path) -> bytes: + """A render JSON without the fields that legitimately differ per run.""" + data = load_json(path) + env = data.get("env") if isinstance(data, dict) else None + if isinstance(env, dict): + data["env"] = envhash.strip_volatile(env) + return canonical_json(data) + + +def freshness_warnings(binary: Path, gpgfrontend: Path, source_paths: list[str]) -> list[str]: + """A heuristic, not proof that the binary matches the source.""" + warnings: list[str] = [] + binary = Path(binary) + for candidate in (binary.parent / "BUILD-INFO.json", binary.parent.parent / "BUILD-INFO.json"): + if candidate.exists(): + info = load_json(candidate, {}) or {} + head = gitutil.head(gpgfrontend) + if info.get("commit") and info.get("commit") != head: + warnings.append(f"{candidate}: commit {info.get('commit')} differs from GpgFrontend HEAD {head}") + return warnings + try: + bin_mtime = binary.stat().st_mtime + except OSError: + return warnings + newest: tuple[float, str] = (0.0, "") + for rel in source_paths: + path = Path(gpgfrontend) / rel + files = [path] if path.is_file() else (p for p in path.rglob("*") if p.is_file()) if path.is_dir() else [] + for f in files: + m = f.stat().st_mtime + if m > newest[0]: + newest = (m, str(f)) + if newest[0] > bin_mtime: + warnings.append(f"binary {binary} is older than {newest[1]}; it may not contain the latest source") + return warnings + + +def main(argv: Optional[list[str]] = None) -> int: + parser = argparse.ArgumentParser(description="Render or probe manifest entries (debug helper).") + parser.add_argument("--gpgfrontend", required=True, type=Path) + parser.add_argument("--binary", type=Path, default=os.environ.get("GF_SCREENSHOT_BINARY")) + parser.add_argument("--manifest", type=Path, default=DEFAULT_MANIFEST) + parser.add_argument("--only", required=True, help="comma separated entry ids") + parser.add_argument("--probe", action="store_true", help="run --screenshot-env instead") + parser.add_argument("--out-dir", type=Path, default=DEFAULT_CACHE / "render") + parser.add_argument("--work-root", type=Path) + parser.add_argument("--keep-work", action="store_true") + args = parser.parse_args(argv) + try: + if not args.binary: + raise UsageError("--binary or GF_SCREENSHOT_BINARY is required") + manifest: Manifest = load_manifest(args.manifest) + by_id = manifest.by_id() + renderer = Renderer(RenderConfig(args.binary, args.gpgfrontend, args.work_root, args.keep_work)) + rc = 0 + for entry_id in split_ids(args.only): + if entry_id not in by_id or not by_id[entry_id].renderable: + raise UsageError(f"unknown or non-renderable id: {entry_id}") + entry = by_id[entry_id] + outcome = renderer.probe(entry) if args.probe else renderer.render(entry, args.out_dir) + if outcome.ok and outcome.env: + print(f"{entry_id}: ok platform={outcome.env.platform} env_hash={outcome.env.env_hash}" + + (f" png={outcome.png}" if outcome.png else "")) + else: + rc = 1 + print(f"{entry_id}: failed:{outcome.failure}") + return rc + except UsageError as exc: + print(f"error: {exc}", file=sys.stderr) + return 2 + except PipelineError as exc: + print(f"error: {exc}", file=sys.stderr) + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/screenshots/requirements.txt b/scripts/screenshots/requirements.txt new file mode 100644 index 0000000..953febb --- /dev/null +++ b/scripts/screenshots/requirements.txt @@ -0,0 +1,2 @@ +Pillow==13.0.0 +PyYAML==6.0.3 diff --git a/scripts/screenshots/run.py b/scripts/screenshots/run.py new file mode 100644 index 0000000..44d6faa --- /dev/null +++ b/scripts/screenshots/run.py @@ -0,0 +1,455 @@ +"""Selects, renders, post-processes, diffs and (optionally) publishes screenshots. + +Default is a dry run: previews, cache and last-run.json only. + --apply local lock, doc and cache writes; no network + --upload implies --apply and allows the CDN upload + --recover completes pending transactions without rendering or network + +Exit codes: 0 all fine, 1 refused/failed/baseline-unavailable, 2 usage, +3 a conflict, upload-unknown or artifact-corrupt is pending. +""" + +from __future__ import annotations + +import argparse +import os +import sys +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any, Optional + +import diff as diffmod +import docrefs +import gitutil +import txn as txnmod +import upload +from apply import Conflict, apply_doc, apply_lock, doc_sha256, read_lock +from check import (DIRTY_SOURCES, ORPHAN, PLATFORM_MISMATCH, STALE, CheckContext, entry_row) +from common import (DEFAULT_CACHE, DEFAULT_LOCK, DEFAULT_MANIFEST, MANUAL_ROOT, PipelineError, + UsageError, atomic_write_json, dedupe, split_ids, utc_now_iso) +from manifest import Entry, Manifest, inputs_hash, load_manifest +from postprocess import PostprocessError, process, processing_descriptor +from render import RenderConfig, Renderer, comparable_json, freshness_warnings + +OK_STATUSES = ("updated", "unchanged", "lock-advanced", "unchanged-lock-not-advanced", "would-replace") +PENDING_STATUSES = ("conflict", "upload-unknown", "artifact-corrupt") +PREVIEW_STATUSES = ("preview-only-dirty", "preview-only-platform") + + +@dataclass +class EntryResult: + id: str + status: str + details: dict[str, Any] = field(default_factory=dict) + # Printed to the terminal only: capability URLs never reach a file. + del_url: Optional[str] = None + + def as_dict(self) -> dict[str, Any]: + return {"id": self.id, "status": self.status, "details": self.details} + + +def build_parser() -> argparse.ArgumentParser: + p = argparse.ArgumentParser(description="Render and publish manual screenshots.") + p.add_argument("--gpgfrontend", type=Path, help="GpgFrontend repository") + p.add_argument("--binary", type=Path, help="screenshot-mode binary (else $GF_SCREENSHOT_BINARY)") + p.add_argument("--only", help="comma separated entry ids (default: every renderable entry)") + p.add_argument("--force", action="store_true", help="select every candidate, stale or not") + p.add_argument("--accept-baseline", default="", help="ids whose baseline is unavailable: replace anyway") + p.add_argument("--force-replace", default="", help="ids to replace whatever the diff says") + p.add_argument("--allow-dirty", action="store_true", help="render dirty entries as previews only") + p.add_argument("--apply", action="store_true", help="write lock, docs and cache (no network)") + p.add_argument("--upload", action="store_true", help="implies --apply; upload replacements") + p.add_argument("--repeat", type=int, default=None, help="render N times and require identical bytes") + p.add_argument("--recover", action="store_true", help="complete pending transactions only") + p.add_argument("--resolve", metavar="TXN_ID", help="with --recover: decide an upload-unknown transaction") + p.add_argument("--url", help="with --resolve: the URL found in the EasyImages admin") + p.add_argument("--retry-upload", action="store_true", help="with --resolve: upload again (may duplicate)") + p.add_argument("--manifest", type=Path, default=DEFAULT_MANIFEST) + p.add_argument("--lock", type=Path, default=DEFAULT_LOCK) + p.add_argument("--cache", type=Path, default=DEFAULT_CACHE) + p.add_argument("--manual-root", type=Path, default=MANUAL_ROOT) + p.add_argument("--work-root", type=Path, help="throwaway directory root (default /tmp/gf-shot-<uid>)") + p.add_argument("--keep-work", action="store_true", help="keep throwaway directories for debugging") + return p + + +def validate_flags(args: argparse.Namespace) -> None: + if args.recover: + clashes = [name for name, on in ( + ("--only", args.only is not None), ("--force", args.force), + ("--accept-baseline", bool(args.accept_baseline)), ("--force-replace", bool(args.force_replace)), + ("--repeat", args.repeat is not None)) if on] + if clashes: + raise UsageError(f"--recover cannot be combined with {', '.join(clashes)}") + else: + if args.resolve: + raise UsageError("--resolve needs --recover") + if not args.gpgfrontend: + raise UsageError("--gpgfrontend is required") + if (args.url or args.retry_upload) and not args.resolve: + raise UsageError("--url and --retry-upload need --resolve") + if args.resolve and bool(args.url) == bool(args.retry_upload): + raise UsageError("--resolve needs exactly one of --url or --retry-upload") + if args.repeat is not None and args.repeat < 1: + raise UsageError("--repeat must be at least 1") + both = set(split_ids(args.accept_baseline)) & set(split_ids(args.force_replace)) + if both: + raise UsageError(f"id(s) in both --accept-baseline and --force-replace: {', '.join(sorted(both))}") + + +def candidates(manifest: Manifest, only: Optional[str]) -> list[Entry]: + by_id = manifest.by_id() + if only is None: + return [e for e in manifest.entries if e.renderable] + ids = split_ids(only) + if not ids: + raise UsageError("--only needs at least one id") + out = [] + for entry_id in dedupe(ids): + if entry_id not in by_id: + raise UsageError(f"unknown id: {entry_id}") + if not by_id[entry_id].renderable: + raise UsageError(f"{entry_id} is {by_id[entry_id].status} and is never rendered") + out.append(by_id[entry_id]) + return out + + +def validate_overrides(args: argparse.Namespace, selected: set[str], lock: dict[str, Any], + cache: Path) -> tuple[set[str], set[str]]: + accept = set(split_ids(args.accept_baseline)) + force = set(split_ids(args.force_replace)) + for entry_id in sorted(accept | force): + if entry_id not in selected: + raise UsageError(f"{entry_id} is not selected in this run; overrides apply to selected ids only") + for entry_id in sorted(accept): + record = lock.get(entry_id) + if not record: + raise UsageError(f"--accept-baseline {entry_id}: no lock record, so the result is new, " + "not baseline-unavailable") + if diffmod.baseline_valid(cache, entry_id, record): + raise UsageError(f"--accept-baseline {entry_id}: the baseline is valid; use --force-replace " + "to replace it anyway") + return accept, force + + +class Runner: + def __init__(self, args: argparse.Namespace, manifest: Manifest, renderer: Renderer, + creds: Optional[upload.Credentials]) -> None: + self.args = args + self.manifest = manifest + self.renderer = renderer + self.creds = creds + self.cache: Path = args.cache + self.apply = args.apply or args.upload + self.paths = txnmod.Paths(Path(args.manual_root), Path(args.lock), self.cache) + self.warnings: list[str] = [] + self.stop_txn = False + self.head = gitutil.head(args.gpgfrontend) + self.processing = processing_descriptor() + + # Rendering ------------------------------------------------------------- + + def render_repeated(self, entry: Entry) -> tuple[Any, Optional[str]]: + repeat = self.args.repeat or 1 + outcomes = [] + for i in range(repeat): + outcome = self.renderer.render(entry, self.cache / "render" / str(i)) + if not outcome.ok: + return outcome, f"failed:{outcome.reason}" + outcomes.append(outcome) + first = outcomes[0] + for other in outcomes[1:]: + if first.png.read_bytes() != other.png.read_bytes(): + return first, "failed:repeat-mismatch" + if comparable_json(first.json_path) != comparable_json(other.json_path): + return first, "failed:repeat-mismatch" + return first, None + + def proposed_record(self, entry: Entry, processed: Any, env: Any) -> dict[str, Any]: + return { + "webp_sha256": processed.webp_sha256, + "pixels_sha256": processed.pixels_sha256, + "gpgfrontend_commit": self.head, + "inputs_hash": inputs_hash(entry, self.processing), + "env_hash": env.env_hash, + "env": env.lock_env(), + "generated_at": utc_now_iso(), + } + + # Per entry ------------------------------------------------------------- + + def run_entry(self, entry: Entry, row: Any, selected: bool, accept: set[str], force: set[str]) -> EntryResult: + res = EntryResult(entry.id, "", {"statuses": list(row.statuses)}) + if ORPHAN in row.statuses: + res.status = "failed:orphan" + return res + if row.details.get("probe_failed"): + res.status = "failed:env-probe" + res.details["probe"] = row.details["probe_failed"] + return res + if not selected: + res.status = "unchanged" + res.details["note"] = "not stale; not rendered" + return res + dirty = DIRTY_SOURCES in row.statuses + if dirty and not self.args.allow_dirty: + res.status = "refused-dirty" + res.details["dirty_files"] = row.details.get("dirty_files") + return res + preview = "preview-only-dirty" if dirty else ( + "preview-only-platform" if PLATFORM_MISMATCH in row.statuses else None) + if self.apply and not preview: + pending = txnmod.pending_for(self.paths, entry.id) + if pending: + states = {t.state for t in pending} + res.status = "upload-unknown" if states & {txnmod.UPLOADING, txnmod.UPLOAD_UNKNOWN} \ + else "failed:pending-transaction" + res.details["pending"] = [t.id for t in pending] + res.details["hint"] = "run run.py --recover first" + return res + self.warnings.extend(freshness_warnings(self.renderer.binary, self.args.gpgfrontend, + self.manifest.relevant_sources(entry))) + outcome, failure = self.render_repeated(entry) + if failure: + res.status = failure + res.details["reason"] = outcome.failure + return res + env = outcome.env + probe = self.renderer.probe(entry) + if probe.env and probe.env.env_hash != env.env_hash: + res.status = "failed:probe-render-env-mismatch" + res.details.update({"probe_env_hash": probe.env.env_hash, "render_env_hash": env.env_hash}) + return res + if env.platform != entry.platform: + preview = preview or "preview-only-platform" + preview_dir = self.cache / "preview" + try: + processed = process(outcome.png, outcome.geometry, entry, preview_dir, entry.id) + except PostprocessError as exc: + res.status = f"failed:{exc.reason}" + res.details["reason"] = str(exc) + return res + lock_record = read_lock(self.args.lock).get(entry.id) + result = diffmod.compare(processed.png, self.cache, entry.id, lock_record, entry.diff) + decision = result.result + res.details.update({"diff": decision, "diff_detail": result.detail, "platform": env.platform, + "env_hash": env.env_hash, "preview": str(processed.png)}) + if entry.id in force: + decision = diffmod.REPLACE + elif entry.id in accept and decision == diffmod.BASELINE_UNAVAILABLE: + decision = diffmod.REPLACE + elif decision == diffmod.NEW: + decision = diffmod.REPLACE + res.details["decision"] = decision + proposed = self.proposed_record(entry, processed, env) + atomic_write_json(preview_dir / f"{entry.id}.json", { + "entry": entry.settings(), "geometry": outcome.geometry, "platform": env.platform, + "env_hash": env.env_hash, "font_files": env.font_files, "proposed_lock_record": proposed, + "diff": result.result, "decision": decision, + }) + if preview: + res.status = preview + elif decision == diffmod.BASELINE_UNAVAILABLE: + res.status = "baseline-unavailable" + elif decision == diffmod.REPLACE: + self.replace(entry, res, processed, outcome, proposed) + else: + self.unchanged(entry, row, res, lock_record, proposed) + return res + + def replace(self, entry: Entry, res: EntryResult, processed: Any, outcome: Any, + proposed: dict[str, Any]) -> None: + if not self.args.upload: + res.status = "would-replace" + return + if self.stop_txn: + res.status = "failed:stopped-after-conflict" + return + assert self.creds is not None + try: + txn = txnmod.prepare(self.paths, entry.id, entry.doc, entry.ref, processed.png, processed.webp, + outcome.geometry, proposed) + except PipelineError as exc: + res.status = "failed:prepare" + res.details["reason"] = str(exc) + return + out = txnmod.execute(txn, self.creds) + res.details.update({"txn": out.txn_id, "txn_state": out.state, "url": out.url}) + if out.detail: + res.details["reason"] = out.detail + res.status = { + txnmod.COMMITTED: "updated", + txnmod.CONFLICT: "conflict", + txnmod.UPLOAD_UNKNOWN: "upload-unknown", + txnmod.ARTIFACT_CORRUPT: "artifact-corrupt", + }.get(out.state, f"failed:{out.state}") + if out.state == txnmod.CONFLICT: + res.del_url = out.del_url + self.stop_txn = True + if out.state == txnmod.COMMITTED: + superseded = superseded_del(self.paths, txn.data["doc"]["old_url"]) + if superseded: + res.del_url = superseded + res.details["superseded_url"] = txn.data["doc"]["old_url"] + + def restore_doc_url(self, entry: Entry, row: Any, res: EntryResult, + lock_record: Optional[dict[str, Any]]) -> bool: + """An emptied or TODO link whose image is unchanged gets the locked URL back. + + No upload: the image on the CDN is still the right one. Returns False + when the entry is not in that state, or it could not be restored. + """ + if row.details.get("missing") != "empty or TODO url" or not lock_record or not lock_record.get("url"): + return True + if not self.apply: + res.status = "would-restore-doc" + return False + path = Path(self.args.manual_root) / entry.doc + try: + apply_doc(path, entry.ref, row.details.get("url") or "", lock_record["url"], doc_sha256(path)) + except Conflict as exc: + res.status = "conflict" + res.details["reason"] = str(exc) + return False + res.details["restored_url"] = lock_record["url"] + return True + + def unchanged(self, entry: Entry, row: Any, res: EntryResult, lock_record: Optional[dict[str, Any]], + proposed: dict[str, Any]) -> None: + if not self.restore_doc_url(entry, row, res, lock_record): + return + restored = "restored_url" in res.details + if not set(row.statuses) & set(STALE): + res.status = "unchanged" + return + if not self.apply: + res.status = "unchanged-lock-not-advanced" + return + assert lock_record is not None + target = dict(lock_record) + for key in ("gpgfrontend_commit", "inputs_hash", "env_hash", "env", "generated_at"): + target[key] = proposed[key] + try: + apply_lock(self.args.lock, entry.id, lock_record, target) + res.status = "doc-restored" if restored else "lock-advanced" + except Conflict as exc: + res.status = "conflict" + res.details["reason"] = str(exc) + + +def superseded_del(paths: txnmod.Paths, old_url: str) -> Optional[str]: + """The del link of an earlier upload of ours that has just been replaced.""" + if not old_url: + return None + for t in txnmod.list_txns(paths): + up = t.data.get("upload") or {} + if t.state == txnmod.COMMITTED and up.get("url") == old_url and up.get("del"): + return up["del"] + return None + + +def exit_code(results: list[EntryResult], applying: bool) -> int: + statuses = [r.status for r in results] + if any(s in PENDING_STATUSES for s in statuses): + return 3 + bad = [s for s in statuses if s.startswith("failed:") or s in ("refused-dirty", "baseline-unavailable")] + # A preview-only entry withheld from --apply/--upload counts as refused; + # in a dry run nothing was withheld. + if applying: + bad += [s for s in statuses if s in PREVIEW_STATUSES] + return 1 if bad else 0 + + +def print_summary(results: list[EntryResult], warnings: list[str]) -> None: + width = max([len(r.id) for r in results] + [8]) + print("Summary") + for r in results: + extra = r.details.get("reason") or r.details.get("diff_detail") or r.details.get("note") or "" + print(f" {r.id:<{width}} {r.status:<28} {extra}".rstrip()) + if r.details.get("url"): + print(f" {'':<{width}} url: {r.details['url']}") + if r.del_url: + print(f" {'':<{width}} del: {r.del_url} (delete link, from the local cache only)") + for w in warnings: + print(f"warning: {w}") + + +def write_last_run(cache: Path, argv: list[str], results: list[EntryResult], warnings: list[str], + code: int) -> None: + atomic_write_json(cache / "last-run.json", { + "generated_at": utc_now_iso(), + "argv": argv, + "exit_code": code, + "warnings": warnings, + "entries": [r.as_dict() for r in results], + }) + + +def do_run(args: argparse.Namespace, argv: list[str]) -> int: + manifest = load_manifest(args.manifest) + cands = candidates(manifest, args.only) + binary = args.binary or (Path(os.environ["GF_SCREENSHOT_BINARY"]) if os.environ.get("GF_SCREENSHOT_BINARY") else None) + if not binary: + raise UsageError("--binary or GF_SCREENSHOT_BINARY is required to render") + creds = upload.credentials_from_env() if args.upload else None + renderer = Renderer(RenderConfig(binary, args.gpgfrontend, args.work_root, args.keep_work)) + # A manifest naming a scenario this binary does not have is a usage error, + # found before anything is rendered rather than as one failed render. + known = renderer.list_scenarios() + unknown = sorted({e.scenario for e in cands if e.scenario} - set(known)) + if unknown: + raise UsageError(f"scenario(s) not in this binary's --screenshot-list: {', '.join(unknown)}") + lock = read_lock(args.lock) + ctx = CheckContext(manifest, lock, args.manual_root, args.gpgfrontend, renderer) + pages = {p.doc: p for p in docrefs.scan_docs(args.manual_root)} + rows = {e.id: entry_row(ctx, e, pages) for e in cands} + selected = {e.id for e in cands if args.force or set(rows[e.id].statuses) & set(STALE)} + accept, force = validate_overrides(args, selected, lock, args.cache) + runner = Runner(args, manifest, renderer, creds) + results = [runner.run_entry(e, rows[e.id], e.id in selected, accept, force) for e in cands] + warnings = dedupe(runner.warnings) + code = exit_code(results, runner.apply) + print_summary(results, warnings) + write_last_run(args.cache, argv, results, warnings, code) + return code + + +def do_recover(args: argparse.Namespace) -> int: + creds = upload.credentials_from_env() if args.retry_upload else None + paths = txnmod.Paths(Path(args.manual_root), Path(args.lock), Path(args.cache)) + outcomes = txnmod.recover(paths, args.resolve, args.url, args.retry_upload, creds) + if not outcomes: + print("nothing to recover") + return 0 + for o in outcomes: + print(f" {o.txn_id} {o.entry_id} {o.state} {o.detail}".rstrip()) + if o.state in (txnmod.CONFLICT, txnmod.UPLOAD_UNKNOWN, txnmod.UPLOADED) and (o.url or o.del_url): + print(f" url: {o.url} del: {o.del_url}") + if o.state == txnmod.UPLOAD_UNKNOWN: + print(f" decide with: run.py --recover --resolve {o.txn_id} (--url <url> | --retry-upload)") + states = {o.state for o in outcomes} + if states & {txnmod.CONFLICT, txnmod.UPLOAD_UNKNOWN, txnmod.ARTIFACT_CORRUPT, txnmod.UPLOADING}: + return 3 + if txnmod.UPLOAD_FAILED in states: + return 1 + return 0 + + +def main(argv: Optional[list[str]] = None) -> int: + argv = list(sys.argv[1:] if argv is None else argv) + args = build_parser().parse_args(argv) + try: + validate_flags(args) + if args.recover: + return do_recover(args) + return do_run(args, argv) + except UsageError as exc: + print(f"error: {exc}", file=sys.stderr) + return 2 + except PipelineError as exc: + print(f"error: {exc}", file=sys.stderr) + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/screenshots/screenshots.lock.json b/scripts/screenshots/screenshots.lock.json new file mode 100644 index 0000000..0967ef4 --- /dev/null +++ b/scripts/screenshots/screenshots.lock.json @@ -0,0 +1 @@ +{} diff --git a/scripts/screenshots/screenshots.yaml b/scripts/screenshots/screenshots.yaml new file mode 100644 index 0000000..2ffd312 --- /dev/null +++ b/scripts/screenshots/screenshots.yaml @@ -0,0 +1,61 @@ +# Manual screenshot manifest. Paths in `doc` are relative to the Manual repo; +# paths in `sources` and `global_sources` are relative to the GpgFrontend repo. +# Frame sizes are logical pixels and are scaled by the capture DPR (2). +# +# Phase 1: four entries, dry run only. Nothing here has been applied yet. + +defaults: + platform: windows11 + locale: en_US + theme: light + size: 1100x680 + fonts: ["Noto Sans", "Noto Sans CJK SC"] + clock: "2026-01-15T10:00:00Z" + timeout_ms: 30000 + frame: {radius: 8, shadow: 12, padding: 24} + diff: {tolerance: 16, threshold: 0.001} + preview_platforms: [linux-xvfb] + +global_sources: + - src/screenshot/ + - resource/screenshot/ + - scripts/screenshot/ + +entries: + - id: main-window + doc: src/content/docs/guides/understand-interface.mdx + ref: img-1 # the "Windows" tab of the window tour + scenario: main-window + size: 1100x680 + sources: + - src/ui/main_window/ + - src/ui/widgets/ + - src/core/utils/GpgUtils.cpp + + - id: key-generate-dialog + doc: src/content/docs/guides/generate-key.md + ref: img-1 # "Step 1: Open the Generate Key Dialog" + scenario: key-generate-dialog + size: auto # natural dialog size + sources: + - src/ui/dialog/key_generate/ + - src/core/model/GpgKeyGenerateInfo.cpp + + - id: text-decrypt-verify-result + doc: src/content/docs/guides/text-operations.md + ref: img-1 # page header image; img-3/img-5 are animated GIFs + scenario: text-decrypt-verify-result + size: 1100x680 + sources: + - src/ui/main_window/ + - src/ui/widgets/ + highlight: InfoBoardWidget + + - id: main-window-menu-popup + doc: src/content/docs/guides/key-server-operations.md + ref: img-1 # "click Keys in the menu bar, then Import Key" + scenario: main-window-menu-popup + size: 1100x680 + sources: + - src/ui/main_window/ + highlight: {name: importKeyFromKeyServer, style: arrow} diff --git a/scripts/screenshots/tests/fakegf.py b/scripts/screenshots/tests/fakegf.py new file mode 100644 index 0000000..982a2e7 --- /dev/null +++ b/scripts/screenshots/tests/fakegf.py @@ -0,0 +1,153 @@ +"""A stand-in for the GpgFrontend screenshot-mode binary, following the contract. + +Behaviour is driven by the JSON file named in $FAKEGF_CONFIG: + platform_name, product, build env record values ("windows" by default) + exit {scenario-or-"env": code} forced exit codes + variant int; shifts every pixel's red channel (a changed render) + tweak_pixels int; recolours this many pixels by a small amount + nondeterministic bool; every run differs + write_appdata path; a file is created there (footprint test) + write_appdata_render_only bool; only renders write it, probes do not + missing list of widget names reported as missing + gate_failures list; written into the env record with exit 6 + log path; one line per invocation is appended +""" + +from __future__ import annotations + +import argparse +import json +import os +import random +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from PIL import Image, ImageDraw # noqa: E402 + +import envhash # noqa: E402 + +SCENARIOS = { + "main-window": ("1100x680", "window"), + "key-generate-dialog": ("auto", "dialog"), + "text-decrypt-verify-result": ("1100x680", "window"), + "main-window-menu-popup": ("1100x680", "composite"), + "demo": ("320x200", "window"), +} + + +def config() -> dict: + path = os.environ.get("FAKEGF_CONFIG") + if path and Path(path).exists(): + return json.loads(Path(path).read_text(encoding="utf-8")) + return {} + + +def env_record(args: argparse.Namespace, cfg: dict) -> dict: + fonts = [f for f in args.font_family.split(",") if f] + record = { + "schema": 1, + "os": {"product": cfg.get("product", "Windows 11 Pro"), "kernel": "10.0.26100", + "build": cfg.get("build", "26100")}, + "qt_version": "6.8.2", + "platform_name": cfg.get("platform_name", "windows"), + "screen": {"name": "FAKE-1", "size": [2560, 1600], "geometry": [0, 0, 2560, 1600], + "dpr": 2, "logical_dpi": 96, "physical_dpi": 96}, + "style": "windows11", + "color_scheme": args.theme, + "locale": args.locale, + "clock": args.clock, + "font_requested": fonts, + "font_resolved": fonts, + "system_font": "Segoe UI", + "gnupg_version": "2.4.7", + "gpgme_version": "1.24.2", + "gpg_clock": {"gpg_conf": "applied", "agent_conf": "applied"}, + "ui_clock_probe": "ok", + "gpg_clock_probe": "ok", + "tz_offset_check": 0, + "build_commit": "fake", + "display": os.environ.get("DISPLAY", ""), + "profile_path": os.path.join(os.environ.get("TMPDIR", "/tmp"), f".tmp-{random.getrandbits(64):016x}"), + "gate_failures": list(cfg.get("gate_failures", [])), + "color_scheme_method": "hint", + } + record["env_hash_core"] = envhash.env_hash_core(record) + return record + + +def render(args: argparse.Namespace, cfg: dict, out: Path) -> dict: + size = args.size or "120x90" # the natural size of an "auto" scenario + w, h = (int(v) * 2 for v in size.split("x")) + base = (240, 240, 240) if args.theme == "light" else (32, 32, 36) + red = (base[0] + int(cfg.get("variant", 0))) % 256 + if cfg.get("nondeterministic"): + red = random.randrange(256) + img = Image.new("RGBA", (w, h), (red, base[1], base[2], 255)) + draw = ImageDraw.Draw(img) + draw.rectangle((4, 4, w // 3, h // 3), fill=(30, 120, 220, 255)) + for i in range(int(cfg.get("tweak_pixels", 0))): + x, y = w // 4 + i % (w // 2), h // 2 + i // (w // 2) + r, g, b, a = img.getpixel((x, y)) + img.putpixel((x, y), (min(255, r + 30), g, b, a)) + img.save(out, format="PNG") + mode = SCENARIOS.get(args.screenshot, ("", "window"))[1] + widgets = { + "InfoBoardWidget": {"rect": [10, h // 2, w - 20, h // 2 - 10], "clipped": False}, + "importKeyMenu": {"rect": [w // 2, 10, w // 4, 20], "clipped": False}, + "nameEdit": {"rect": [10, 10, w // 3, 20], "clipped": False}, + } + missing = list(cfg.get("missing", [])) + for name in missing: + widgets.pop(name, None) + return {"schema": 1, "scenario": args.screenshot, "capture_mode": mode, "dpr": 2, + "image_size": [w, h], "origin": [0, 0], "layers": [{"name": "window", "rect": [0, 0, w, h]}], + "widgets": widgets, "missing": missing, "error": None} + + +def main() -> int: + cfg = config() + if cfg.get("log"): + with open(cfg["log"], "a", encoding="utf-8") as handle: + handle.write(" ".join(sys.argv[1:]) + "\n") + if "--screenshot-list" in sys.argv: + for name, (size, mode) in SCENARIOS.items(): + print(f"{name}\t{size}\t{mode}") + return 0 + p = argparse.ArgumentParser() + p.add_argument("--temp-profile", action="store_true") + p.add_argument("--screenshot") + p.add_argument("--screenshot-env", action="store_true") + for opt in ("locale", "theme", "size", "font-family", "clock", "demo-gnupghome", "fixtures", "out", "timeout"): + p.add_argument(f"--{opt}") + args = p.parse_args() + if not args.temp_profile: + return 2 + if cfg.get("write_appdata") and not (cfg.get("write_appdata_render_only") and args.screenshot_env): + Path(cfg["write_appdata"]).mkdir(parents=True, exist_ok=True) + (Path(cfg["write_appdata"]) / "leak.txt").write_text("x", encoding="utf-8") + out = Path(args.out) + record = env_record(args, cfg) + key = "env" if args.screenshot_env else args.screenshot + forced = cfg.get("exit", {}).get(key) + if record["gate_failures"]: + forced = 6 + if args.screenshot_env: + if forced: + record["error"] = {"code": forced, "reason": "forced", "detail": "fake"} + out.write_text(json.dumps(record), encoding="utf-8") + return forced or 0 + if forced: + partial = {"schema": 1, "scenario": args.screenshot, "env": record, + "error": {"code": forced, "reason": "forced-failure", "detail": "fake"}} + out.with_suffix(".json").write_text(json.dumps(partial), encoding="utf-8") + return forced + geometry = render(args, cfg, out) + geometry["env"] = record + out.with_suffix(".json").write_text(json.dumps(geometry), encoding="utf-8") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/screenshots/tests/helpers.py b/scripts/screenshots/tests/helpers.py new file mode 100644 index 0000000..ad72f0c --- /dev/null +++ b/scripts/screenshots/tests/helpers.py @@ -0,0 +1,281 @@ +"""Shared fixtures: a temp GpgFrontend git repo, a temp manual tree, a fake +binary and a localhost EasyImages mock. Nothing here touches the network +beyond 127.0.0.1.""" + +from __future__ import annotations + +import contextlib +import io +import json +import os +import shutil +import subprocess +import sys +import tempfile +import threading +import unittest +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path +from typing import Any, Optional +from unittest import mock + +PIPELINE_DIR = Path(__file__).resolve().parents[1] +if str(PIPELINE_DIR) not in sys.path: + sys.path.insert(0, str(PIPELINE_DIR)) + +FAKE_SCRIPT = Path(__file__).resolve().parent / "fakegf.py" + +PAGE_A = """--- +title: Page A +--- + +Intro text. + +![Main window][img-1] + +Some steps. + + ![][img-2] + +[img-1]: https://cdn.example.org/i/a1.webp +[img-2]: + https://cdn.example.org/i/a2.webp +[link]: https://example.org/ +""" + +PAGE_B = "# B\r\n\r\n![][img-1]\r\n\r\nText line.\r\n\r\n[img-1]: https://cdn.example.org/i/b1.webp\r\n" + +MANIFEST = """ +defaults: + platform: windows11 + size: 160x100 + fonts: ["Noto Sans"] + frame: {radius: 4, shadow: 4, padding: 8} +global_sources: [src/screenshot/, resource/screenshot/] +entries: + - id: a-main + doc: src/content/docs/guides/a.md + ref: img-1 + scenario: main-window + sources: [src/ui/main/] + highlight: InfoBoardWidget + - id: a-dialog + doc: src/content/docs/guides/a.md + ref: img-2 + scenario: key-generate-dialog + sources: [src/ui/dialog/] + highlight: {name: nameEdit, style: arrow} + - id: b-main + doc: src/content/docs/guides/b.md + ref: img-1 + scenario: main-window + sources: [src/ui/other/] + - id: manual-shot + doc: src/content/docs/guides/b.md + ref: img-9 + status: manual-only +""" + + +def git(repo: Path, *args: str) -> str: + env = dict(os.environ, GIT_AUTHOR_NAME="t", GIT_AUTHOR_EMAIL="[email protected]", + GIT_COMMITTER_NAME="t", GIT_COMMITTER_EMAIL="[email protected]") + out = subprocess.run(["git", "-C", str(repo), *args], capture_output=True, check=True, env=env) + return out.stdout.decode().strip() + + +def write(path: Path, text: str, newline_exact: bool = True) -> Path: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(text.encode("utf-8")) + return path + + +def make_gf_repo(root: Path) -> Path: + root.mkdir(parents=True) + git(root, "init", "-q", "-b", "main") + for rel in ("src/screenshot/Mode.cpp", "resource/screenshot/demo-gnupghome/pubring.kbx", + "resource/screenshot/fixtures/decrypt-verify.asc", "src/ui/main/MainWindow.cpp", + "src/ui/dialog/Dialog.cpp", "src/ui/other/Other.cpp", "README"): + write(root / rel, f"// {rel}\n") + git(root, "add", "-A") + git(root, "commit", "-q", "-m", "init") + return root + + +def commit_change(repo: Path, rel: str, text: str = "change\n") -> str: + write(repo / rel, text) + git(repo, "add", "-A") + git(repo, "commit", "-q", "-m", f"edit {rel}") + return git(repo, "rev-parse", "HEAD") + + +def make_fake_binary(directory: Path) -> Path: + directory.mkdir(parents=True, exist_ok=True) + if os.name == "nt": + launcher = directory / "fakegf.cmd" + launcher.write_text(f'@"{sys.executable}" "{FAKE_SCRIPT}" %*\r\n', encoding="utf-8") + else: + launcher = directory / "fakegf" + launcher.write_text(f'#!/bin/sh\nexec "{sys.executable}" "{FAKE_SCRIPT}" "$@"\n', encoding="utf-8") + launcher.chmod(0o755) + return launcher + + +class MockCdn: + """EasyImages-like endpoint on 127.0.0.1. Modes: success, error, http500, drop.""" + + def __init__(self) -> None: + self.mode = "success" + self.requests: list[dict[str, Any]] = [] + outer = self + + class Handler(BaseHTTPRequestHandler): + def log_message(self, *args: Any) -> None: + pass + + def do_POST(self) -> None: # noqa: N802 + length = int(self.headers.get("Content-Length", "0")) + body = self.rfile.read(length) + n = len(outer.requests) + 1 + outer.requests.append({ + "has_image": b'name="image"' in body and b"RIFF" in body, + "has_token": b'name="token"\r\n\r\nsecret-token' in body, + "content_type": self.headers.get("Content-Type", ""), + }) + if outer.mode == "drop": + self.close_connection = True + self.connection.shutdown(2) + return + if outer.mode == "http500": + self.send_response(500) + self.end_headers() + self.wfile.write(b"internal error") + return + if outer.mode == "error": + payload = {"result": "failed", "code": 202, "message": "token invalid"} + else: + payload = {"result": "success", "code": 200, + "url": f"https://img.example.org/i/up{n}.webp", "srcName": "x.webp", + "thumb": f"https://img.example.org/thumb/up{n}.webp", + "del": f"https://img.example.org/del/secret{n}"} + data = json.dumps(payload).encode() + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(data))) + self.end_headers() + self.wfile.write(data) + + self.server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + self.thread = threading.Thread(target=self.server.serve_forever, daemon=True) + self.thread.start() + + @property + def url(self) -> str: + return f"http://127.0.0.1:{self.server.server_address[1]}/api/index.php" + + def close(self) -> None: + self.server.shutdown() + self.server.server_close() + + +def short_tempdir(prefix: str) -> Path: + base = "/tmp" if os.name != "nt" else None + return Path(tempfile.mkdtemp(prefix=prefix, dir=base)) + + +class World(unittest.TestCase): + """A complete temp setup; subclasses get self.gf, self.manual, ... ready.""" + + crlf_b = True + + def setUp(self) -> None: + self.tmp = short_tempdir("gfst") + self.addCleanup(shutil.rmtree, self.tmp, True) + self.gf = make_gf_repo(self.tmp / "gf") + self.manual = self.tmp / "manual" + docs = self.manual / "src" / "content" / "docs" / "guides" + write(docs / "a.md", PAGE_A) + write(docs / "b.md", PAGE_B) + self.scripts = self.manual / "scripts" / "screenshots" + self.manifest = write(self.scripts / "screenshots.yaml", MANIFEST) + self.lock = write(self.scripts / "screenshots.lock.json", "{}\n") + self.cache = self.scripts / ".screenshots-cache" + self.binary = make_fake_binary(self.tmp / "bin") + self.fake_config_path = self.tmp / "fake.json" + self.appdata = self.tmp / "appdata" + self.appdata.mkdir() + self.work_root = self.tmp / "w" + self.cdn: Optional[MockCdn] = None + self.set_fake({}) + env = { + "FAKEGF_CONFIG": str(self.fake_config_path), + "GF_SHOTS_NO_XVFB": "1", + "GF_SHOTS_APPDATA_ROOTS": str(self.appdata), + "GF_MANUAL_CDN_API": "", + "GF_MANUAL_CDN_TOKEN": "", + "GF_SCREENSHOT_BINARY": "", + } + patcher = mock.patch.dict(os.environ, env) + patcher.start() + self.addCleanup(patcher.stop) + + # Helpers --------------------------------------------------------------- + + def set_fake(self, cfg: dict[str, Any]) -> None: + self.fake_config_path.write_text(json.dumps(cfg), encoding="utf-8") + + def start_cdn(self) -> MockCdn: + self.cdn = MockCdn() + self.addCleanup(self.cdn.close) + os.environ["GF_MANUAL_CDN_API"] = self.cdn.url + os.environ["GF_MANUAL_CDN_TOKEN"] = "secret-token" + return self.cdn + + def common_args(self, binary: bool = True) -> list[str]: + args = ["--gpgfrontend", str(self.gf), "--manifest", str(self.manifest), "--lock", str(self.lock), + "--cache", str(self.cache), "--manual-root", str(self.manual), "--work-root", str(self.work_root)] + if binary: + args += ["--binary", str(self.binary)] + return args + + def run_pipeline(self, *args: str, binary: bool = True) -> tuple[int, str, str]: + import run + + out, err = io.StringIO(), io.StringIO() + with contextlib.redirect_stdout(out), contextlib.redirect_stderr(err): + code = run.main(self.common_args(binary) + list(args)) + return code, out.getvalue(), err.getvalue() + + def recover(self, *args: str) -> tuple[int, str, str]: + import run + + out, err = io.StringIO(), io.StringIO() + base = ["--recover", "--manifest", str(self.manifest), "--lock", str(self.lock), "--cache", + str(self.cache), "--manual-root", str(self.manual)] + with contextlib.redirect_stdout(out), contextlib.redirect_stderr(err): + code = run.main(base + list(args)) + return code, out.getvalue(), err.getvalue() + + def last_run(self) -> dict[str, Any]: + return json.loads((self.cache / "last-run.json").read_text(encoding="utf-8")) + + def statuses(self) -> dict[str, str]: + return {e["id"]: e["status"] for e in self.last_run()["entries"]} + + def read_lock(self) -> dict[str, Any]: + return json.loads(self.lock.read_text(encoding="utf-8")) + + def doc(self, name: str) -> bytes: + return (self.manual / "src" / "content" / "docs" / "guides" / name).read_bytes() + + def check_rows(self, binary: bool = False, pipeline_dir: Optional[Path] = None) -> dict[str, list[str]]: + import check + from apply import read_lock + from manifest import load_manifest + from render import RenderConfig, Renderer + + renderer = Renderer(RenderConfig(self.binary, self.gf, self.work_root)) if binary else None + ctx = check.CheckContext(load_manifest(self.manifest), read_lock(self.lock), self.manual, self.gf, + renderer, pipeline_dir or PIPELINE_DIR) + rows, _ = check.compute(ctx) + return {(r.id or f"{r.doc}#{r.ref}"): r.statuses for r in rows} diff --git a/scripts/screenshots/tests/test_check.py b/scripts/screenshots/tests/test_check.py new file mode 100644 index 0000000..b630113 --- /dev/null +++ b/scripts/screenshots/tests/test_check.py @@ -0,0 +1,138 @@ +from __future__ import annotations + +import contextlib +import io +import json +import shutil + +from helpers import PIPELINE_DIR, World, commit_change, git, write + +import check +from manifest import load_manifest, inputs_hash +from postprocess import processing_descriptor +from render import RenderConfig, Renderer + + +class CheckBaseline(World): + def write_record(self, entry_id: str, commit: str, env_hash: str = "x", inputs: str | None = None, + pipeline_dir=PIPELINE_DIR) -> None: + entry = load_manifest(self.manifest).by_id()[entry_id] + lock = self.read_lock() + lock[entry_id] = { + "url": "https://cdn.example.org/i/a1.webp", "webp_sha256": "w", "pixels_sha256": "p", + "gpgfrontend_commit": commit, + "inputs_hash": inputs or inputs_hash(entry, processing_descriptor(), pipeline_dir), + "env_hash": env_hash, "env": {}, "generated_at": "2026-01-01T00:00:00Z", + } + self.lock.write_text(json.dumps(lock), encoding="utf-8") + + def head(self) -> str: + return git(self.gf, "rev-parse", "HEAD") + + +class BaselineStatusTest(CheckBaseline): + def test_missing_commit_is_unknown_baseline(self) -> None: + self.write_record("a-main", "0" * 40) + self.assertIn("unknown-baseline", self.check_rows()["a-main"]) + + def test_non_ancestor_commit_is_unknown_baseline(self) -> None: + git(self.gf, "checkout", "-q", "-b", "side") + side = commit_change(self.gf, "src/ui/main/MainWindow.cpp", "side\n") + git(self.gf, "checkout", "-q", "main") + self.write_record("a-main", side) + self.assertIn("unknown-baseline", self.check_rows()["a-main"]) + + def test_outdated_from_entry_sources(self) -> None: + self.write_record("a-main", self.head()) + commit_change(self.gf, "src/ui/main/MainWindow.cpp") + rows = self.check_rows() + self.assertIn("outdated", rows["a-main"]) + self.assertNotIn("unknown-baseline", rows["a-main"]) + + def test_outdated_from_global_sources(self) -> None: + self.write_record("b-main", self.head()) + commit_change(self.gf, "resource/screenshot/fixtures/decrypt-verify.asc") + self.assertIn("outdated", self.check_rows()["b-main"]) + + def test_unrelated_change_is_not_outdated(self) -> None: + self.write_record("b-main", self.head()) + commit_change(self.gf, "src/ui/main/MainWindow.cpp") + self.assertNotIn("outdated", self.check_rows()["b-main"]) + + def test_inputs_changed_from_manifest_edit(self) -> None: + self.write_record("a-main", self.head()) + self.assertNotIn("inputs-changed", self.check_rows()["a-main"]) + text = self.manifest.read_text(encoding="utf-8").replace( + " scenario: main-window\n sources: [src/ui/main/]", + " scenario: main-window\n theme: dark\n sources: [src/ui/main/]") + self.manifest.write_text(text, encoding="utf-8") + self.assertIn("inputs-changed", self.check_rows()["a-main"]) + + def test_inputs_changed_from_script_edit(self) -> None: + pipe = self.tmp / "pipeline" + pipe.mkdir() + for f in list(PIPELINE_DIR.glob("*.py")) + [PIPELINE_DIR / "requirements.txt"]: + shutil.copy(f, pipe / f.name) + self.write_record("a-main", self.head(), pipeline_dir=pipe) + self.assertNotIn("inputs-changed", self.check_rows(pipeline_dir=pipe)["a-main"]) + with (pipe / "postprocess.py").open("a", encoding="utf-8") as handle: + handle.write("\n# tweak\n") + self.assertIn("inputs-changed", self.check_rows(pipeline_dir=pipe)["a-main"]) + + def test_dirty_sources(self) -> None: + self.write_record("a-main", self.head()) + write(self.gf / "src/ui/main/MainWindow.cpp", "uncommitted\n") + rows = self.check_rows() + self.assertIn("dirty-sources", rows["a-main"]) + self.assertNotIn("dirty-sources", rows["b-main"]) + write(self.gf / "src/screenshot/New.cpp", "untracked global source\n") + self.assertIn("dirty-sources", self.check_rows()["b-main"]) + + +class EnvStatusTest(CheckBaseline): + def probe_hash(self, entry_id: str) -> str: + entry = load_manifest(self.manifest).by_id()[entry_id] + outcome = Renderer(RenderConfig(self.binary, self.gf, self.work_root)).probe(entry) + self.assertTrue(outcome.ok, outcome.failure) + return outcome.env.env_hash + + def test_env_unchecked_without_binary(self) -> None: + self.write_record("a-main", self.head()) + rows = self.check_rows(binary=False) + self.assertIn("env-unchecked", rows["a-main"]) + self.assertNotIn("ok", rows["a-main"]) + + def test_ok_then_env_changed(self) -> None: + self.write_record("a-main", self.head(), env_hash=self.probe_hash("a-main")) + self.assertEqual(self.check_rows(binary=True)["a-main"], ["ok"]) + self.set_fake({"build": "26200", "product": "Windows 11 Pro 25H2"}) + self.assertEqual(self.check_rows(binary=True)["a-main"], ["env-changed"]) + + def test_platform_mismatch(self) -> None: + self.set_fake({"platform_name": "xcb", "product": "Debian GNU/Linux 13"}) + rows = self.check_rows(binary=True) + self.assertIn("platform-mismatch", rows["a-main"]) + + def test_probe_runs_once_per_tuple(self) -> None: + log = self.tmp / "calls.log" + self.set_fake({"log": str(log)}) + self.check_rows(binary=True) + probes = [line for line in log.read_text().splitlines() if "--screenshot-env" in line] + # a-main and b-main share the default tuple; a-dialog has the same tuple too. + self.assertEqual(len(probes), 1) + + def test_cli_json_and_inventory(self) -> None: + out = io.StringIO() + with contextlib.redirect_stdout(out): + code = check.main(["--gpgfrontend", str(self.gf), "--manifest", str(self.manifest), "--lock", + str(self.lock), "--manual-root", str(self.manual), "--json", "--inventory"]) + self.assertEqual(code, 0) + data = json.loads(out.getvalue()) + ids = {row["id"] for row in data["entries"]} + self.assertEqual(ids, {"a-main", "a-dialog", "b-main", "manual-shot"}) + self.assertEqual(data["inventory"]["total"]["references"], 3) + out = io.StringIO() + with contextlib.redirect_stdout(out): + check.main(["--gpgfrontend", str(self.gf), "--manifest", str(self.manifest), "--lock", + str(self.lock), "--manual-root", str(self.manual), "--json"]) + self.assertIsInstance(json.loads(out.getvalue()), list) diff --git a/scripts/screenshots/tests/test_docrefs.py b/scripts/screenshots/tests/test_docrefs.py new file mode 100644 index 0000000..1958b73 --- /dev/null +++ b/scripts/screenshots/tests/test_docrefs.py @@ -0,0 +1,125 @@ +from __future__ import annotations + +import shutil +import unittest +from pathlib import Path + +from helpers import World, short_tempdir, write + +import docrefs +from common import PipelineError + +TREE = { + "src/content/docs/one.md": ( + "![A][img-1]\n" + "![again][img-1]\n" + "Text  more\n" + "<img src=\"https://cdn.example.org/i/shared.webp\" alt=\"h\">\n" + "```md\n![not an image][img-2]\n```\n" + "`![code span][img-2]`\n" + "\n" + "[img-1]:\n" + " https://cdn.example.org/i/one.gif\n" + "[other]: https://example.org/\n" + ), + "src/content/docs/sub/two.mdx": ( + "<Tabs>\n ![][img-1]\n</Tabs>\n![shot][]\n![missing][img-7]\n![empty][img-3]\n" + "[img-1]: https://cdn.example.org/i/shared.webp\n" + "[shot]: <https://cdn.example.org/i/shot.WEBP>\n" + "[img-3]: TODO\n" + ), + "src/content/docs/none.md": "No images here. [link][x]\n\n[x]: https://example.org/\n", +} + + +class ParseTest(unittest.TestCase): + def test_reference_inline_html_and_code_skipping(self) -> None: + page = docrefs.scan_text("one.md", TREE["src/content/docs/one.md"]) + kinds = [(r.kind, r.label, r.line) for r in page.refs] + self.assertEqual(kinds, [("reference", "img-1", 1), ("reference", "img-1", 2), + ("inline", None, 3), ("html", None, 4)]) + self.assertEqual(page.refs[2].url, "https://cdn.example.org/i/x.png") + self.assertEqual(page.refs[3].url, "https://cdn.example.org/i/shared.webp") + + def test_wrapped_definition(self) -> None: + page = docrefs.scan_text("one.md", TREE["src/content/docs/one.md"]) + d = page.definition_for("IMG-1") + self.assertEqual(d.url, "https://cdn.example.org/i/one.gif") + self.assertEqual((d.line_start, d.line_end), (10, 11)) + self.assertTrue(d.text.startswith("[img-1]:\n https://")) + + def test_collapsed_angle_and_todo(self) -> None: + page = docrefs.scan_text("two.mdx", TREE["src/content/docs/sub/two.mdx"]) + self.assertEqual(page.definition_for("shot").url, "https://cdn.example.org/i/shot.WEBP") + self.assertIsNone(page.definition_for("img-7")) + self.assertTrue(docrefs.is_missing_url(page.definition_for("img-3").url)) + + +class InventoryTest(unittest.TestCase): + def setUp(self) -> None: + self.root = short_tempdir("gfinv") + self.addCleanup(shutil.rmtree, self.root, True) + for rel, text in TREE.items(): + write(self.root / rel, text) + + def test_counts_match_fixture(self) -> None: + pages = docrefs.scan_docs(self.root) + inv = docrefs.inventory(pages, {("src/content/docs/one.md", "img-1"): "managed-one"}) + by_doc = {p["doc"]: p for p in inv["pages"]} + one = by_doc["src/content/docs/one.md"] + # img-1 twice, one inline, one <img>; one definition is used. + self.assertEqual((one["references"], one["definitions"], one["unique_urls"]), (4, 1, 3)) + self.assertEqual(one["formats_by_reference"], {"gif": 2, "png": 1, "webp": 1}) + two = by_doc["src/content/docs/sub/two.mdx"] + # img-1, shot, img-7 (no definition), img-3 (TODO): 3 definitions, 2 real URLs. + self.assertEqual((two["references"], two["definitions"], two["unique_urls"]), (4, 3, 2)) + self.assertEqual(two["formats_by_reference"], {"(none)": 2, "webp": 2}) + total = inv["total"] + self.assertEqual(total["pages_with_images"], 2) + self.assertEqual(total["references"], 8) + self.assertEqual(total["definitions"], 4) + # shared.webp appears on both pages and is counted once. + self.assertEqual(total["unique_urls"], 4) + self.assertEqual(total["formats_by_unique_url"], {"gif": 1, "png": 1, "webp": 2}) + managed = [r for r in inv["refs"] if r["managed_id"]] + self.assertEqual(len(managed), 2) + self.assertEqual(len(inv["refs"]), 8) + + +class RewriteTest(unittest.TestCase): + def test_only_the_url_token_changes_and_crlf_survives(self) -> None: + text = "a\r\n![][img-1]\r\n\r\n[img-1]: https://old.example/x.webp \"title\"\r\n[img-2]: https://keep/y\r\n" + out = docrefs.replace_definition_url(text, "img-1", "https://new.example/z.webp", "https://old.example/x.webp") + self.assertEqual(out, text.replace("https://old.example/x.webp", "https://new.example/z.webp")) + self.assertEqual(out.count("\r\n"), text.count("\r\n")) + + def test_wrapped_and_empty_definitions(self) -> None: + wrapped = "[img-1]:\n https://old/a.webp\n" + self.assertEqual(docrefs.replace_definition_url(wrapped, "img-1", "https://n/b.webp", "https://old/a.webp"), + "[img-1]:\n https://n/b.webp\n") + empty = "[img-1]:\n\nText\n" + self.assertEqual(docrefs.replace_definition_url(empty, "img-1", "https://n/b.webp", ""), + "[img-1]: https://n/b.webp\n\nText\n") + + def test_refuses_duplicates_and_unexpected_url(self) -> None: + dup = "[img-1]: https://a/1\n[img-1]: https://a/2\n" + with self.assertRaises(PipelineError): + docrefs.replace_definition_url(dup, "img-1", "https://n/x", "https://a/1") + with self.assertRaises(PipelineError): + docrefs.replace_definition_url("[img-1]: https://a/1\n", "img-1", "https://n/x", "https://a/9") + + +class CheckRefStatusTest(World): + def test_unmanaged_orphan_missing_and_manual_only(self) -> None: + write(self.manual / "src/content/docs/guides/c.md", "![x][img-1]\n\n") + rows = self.check_rows() + self.assertEqual(rows["src/content/docs/guides/c.md#img-1"], ["unmanaged", "missing"]) + self.assertIn("unmanaged", rows["src/content/docs/guides/c.md#None"]) + # manual-shot points at img-9, which b.md does not use. + self.assertEqual(rows["manual-shot"], ["orphan", "manual-only"]) + self.assertIn("missing", rows["a-main"]) # no lock record yet + self.assertIn("env-unchecked", rows["a-main"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/screenshots/tests/test_envhash.py b/scripts/screenshots/tests/test_envhash.py new file mode 100644 index 0000000..20ab1f9 --- /dev/null +++ b/scripts/screenshots/tests/test_envhash.py @@ -0,0 +1,105 @@ +from __future__ import annotations + +import copy +import json +import os +import unittest +from pathlib import Path + +import helpers # noqa: F401 (sets sys.path) + +import envhash + +# The fixture is owned by the C++ side; its own test checks the same values. +GPGFRONTEND = Path(os.environ.get("GF_SHOTS_GPGFRONTEND", + str(helpers.PIPELINE_DIR.parents[2] / "GpgFrontend"))) +CROSS_CHECK_FIXTURE = GPGFRONTEND / "resource" / "screenshot" / "env-hash-fixture.json" + + +def sample_env() -> dict: + return { + "schema": 1, + "os": {"product": "Windows 11 Pro", "kernel": "10.0.26100", "build": "26100"}, + "qt_version": "6.8.2", "platform_name": "windows", + "screen": {"name": "DISPLAY1", "size": [2560, 1600], "geometry": [0, 0, 2560, 1600], + "dpr": 2, "logical_dpi": 96, "physical_dpi": 96.4}, + "style": "windows11", "color_scheme": "light", "color_scheme_method": "hint", + "locale": "en_US", "clock": "2026-01-15T10:00:00Z", + "font_requested": ["Noto Sans", "思源黑体"], "font_resolved": ["Noto Sans", "思源黑体"], + "system_font": "Segoe UI", "gnupg_version": "2.4.7", "gpgme_version": "1.24.2", + "gpg_clock": {"gpg_conf": "applied", "agent_conf": "applied"}, + "ui_clock_probe": "ok", "gpg_clock_probe": "ok", "tz_offset_check": 0, + "build_commit": "abc", "display": ":99", "profile_path": "/tmp/x/.tmp-1", "gate_failures": [], + } + + +FONTS = [{"family": "Noto Sans", "sha256": "aa"}, {"family": "思源黑体", "sha256": "bb"}, + {"family": "Segoe UI", "sha256": "cc"}] + + +def final(env: dict, fonts: list = FONTS, platform: str = "windows11") -> str: + return envhash.final_env_hash(envhash.env_hash_core(env), platform, fonts) + + +class CrossCheckTest(unittest.TestCase): + def test_cpp_fixture_reproduces_byte_for_byte(self) -> None: + if not CROSS_CHECK_FIXTURE.exists(): + self.skipTest(f"C++ fixture not found at {CROSS_CHECK_FIXTURE} (set GF_SHOTS_GPGFRONTEND)") + fixture = json.loads(CROSS_CHECK_FIXTURE.read_text(encoding="utf-8")) + self.assertEqual(envhash.canonical_core(fixture["env"]).decode("utf-8"), fixture["canonical"]) + self.assertEqual(envhash.env_hash_core(fixture["env"]), fixture["env_hash_core"]) + self.assertEqual(envhash.platform_tag(fixture["env"], under_xvfb=False), "windows11") + + def test_canonical_form_is_compact_sorted_utf8(self) -> None: + text = envhash.canonical_core(sample_env()).decode("utf-8") + self.assertTrue(text.startswith('{"clock":"2026-01-15T10:00:00Z","color_scheme":"light"')) + self.assertIn('"思源黑体"', text) # ensure_ascii=False + self.assertIn('"physical_dpi":96,', text) # DPI hashed as an integer + self.assertNotIn(" ", text.replace("Noto Sans", "").replace("Segoe UI", "").replace("Windows 11 Pro", "")) + + +class HashSensitivityTest(unittest.TestCase): + def test_key_order_and_volatile_fields_do_not_matter(self) -> None: + base = final(sample_env()) + reordered = dict(reversed(list(sample_env().items()))) + self.assertEqual(final(reordered), base) + env = sample_env() + env.update(display=":42", profile_path="/tmp/other/.tmp-2", build_commit="zzz", + color_scheme_method="palette") + env["screen"]["name"] = "OTHER" + env["screen"]["geometry"] = [100, 50, 2560, 1600] + env["os"]["kernel"] = "6.1" + self.assertEqual(final(env), base) + + def test_render_relevant_changes_do(self) -> None: + base = final(sample_env()) + for key, value in (("color_scheme", "dark"), ("locale", "zh_CN"), ("clock", "2026-02-01T10:00:00Z")): + env = sample_env() + env[key] = value + self.assertNotEqual(final(env), base, key) + env = sample_env() + env["font_requested"] = ["Noto Sans"] + self.assertNotEqual(final(env), base) + fonts = copy.deepcopy(FONTS) + fonts[1]["sha256"] = "changed" + self.assertNotEqual(final(sample_env(), fonts), base) + self.assertNotEqual(final(sample_env(), platform="linux-xvfb"), base) + + def test_platform_tags(self) -> None: + env = sample_env() + self.assertEqual(envhash.platform_tag(env, False), "windows11") + env["os"] = {"product": "Windows 10", "build": "19045"} + self.assertEqual(envhash.platform_tag(env, False), "windows-other") + env["platform_name"] = "xcb" + self.assertEqual(envhash.platform_tag(env, True), "linux-xvfb") + self.assertEqual(envhash.platform_tag(env, False), "linux-xcb") + + def test_lock_env_drops_run_volatile_fields(self) -> None: + stripped = envhash.strip_volatile(sample_env()) + self.assertNotIn("profile_path", stripped) + self.assertNotIn("display", stripped) + self.assertIn("build_commit", stripped) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/screenshots/tests/test_run.py b/scripts/screenshots/tests/test_run.py new file mode 100644 index 0000000..9197b5b --- /dev/null +++ b/scripts/screenshots/tests/test_run.py @@ -0,0 +1,288 @@ +from __future__ import annotations + +import json +import os + +from PIL import Image + +from helpers import World, commit_change, write + + +class RunWorld(World): + def publish_all(self) -> None: + """Brings every renderable entry to a committed lock + baseline.""" + self.start_cdn() + code, out, err = self.run_pipeline("--upload") + self.assertEqual(code, 0, out + err) + self.assertEqual(set(self.statuses().values()), {"updated"}) + + def rendered_ids(self, log) -> list[str]: + ids = [] + for line in log.read_text().splitlines(): + if "--screenshot " in line: + out = line.split("--out ")[1].split(" ")[0] + ids.append(os.path.basename(out).rsplit(".", 1)[0]) + return ids + + +class SelectionTest(RunWorld): + def test_unknown_or_manual_only_id_is_usage_error(self) -> None: + self.assertEqual(self.run_pipeline("--only", "nope")[0], 2) + self.assertEqual(self.run_pipeline("--only", "manual-shot")[0], 2) + self.assertFalse((self.cache / "last-run.json").exists()) + + def test_replace_without_upload_is_would_replace(self) -> None: + code, out, err = self.run_pipeline() + self.assertEqual(code, 0, out + err) + self.assertEqual(self.statuses(), {"a-main": "would-replace", "a-dialog": "would-replace", + "b-main": "would-replace"}) + self.assertTrue((self.cache / "preview" / "a-main.webp").exists()) + self.assertEqual(self.read_lock(), {}) + + def test_force_with_and_without_only(self) -> None: + self.publish_all() + log = self.tmp / "calls.log" + self.set_fake({"log": str(log)}) + self.assertEqual(self.run_pipeline()[0], 0) + self.assertEqual(set(self.statuses().values()), {"unchanged"}) + self.assertEqual(self.rendered_ids(log), []) + self.assertEqual(self.run_pipeline("--force")[0], 0) + self.assertEqual(sorted(self.rendered_ids(log)), ["a-dialog", "a-main", "b-main"]) + log.write_text("") + self.assertEqual(self.run_pipeline("--force", "--only", "a-main")[0], 0) + self.assertEqual(self.rendered_ids(log), ["a-main"]) + self.assertEqual(list(self.statuses()), ["a-main"]) + + def test_override_validation(self) -> None: + self.publish_all() + # Valid baseline: accept-baseline does not apply. + code, _, err = self.run_pipeline("--force", "--accept-baseline", "a-main") + self.assertEqual(code, 2) + self.assertIn("--force-replace", err) + code, _, err = self.run_pipeline("--force", "--accept-baseline", "a-main", "--force-replace", "a-main") + self.assertEqual(code, 2) + # Not stale, so not selected without --force. + code, _, err = self.run_pipeline("--force-replace", "a-main") + self.assertEqual(code, 2) + self.assertIn("not selected", err) + code, out, err = self.run_pipeline("--force", "--force-replace", "a-main") + self.assertEqual(code, 0, out + err) + self.assertEqual(self.statuses()["a-main"], "would-replace") + self.assertEqual(self.statuses()["b-main"], "unchanged") + + def test_scenario_missing_from_binary_is_usage_error(self) -> None: + log = self.tmp / "calls.log" + self.set_fake({"log": str(log)}) + self.manifest.write_text(self.manifest.read_text().replace( + "scenario: key-generate-dialog", "scenario: no-such-scenario")) + code, _, err = self.run_pipeline() + self.assertEqual(code, 2) + self.assertIn("no-such-scenario", err) + self.assertEqual(self.rendered_ids(log), []) + + def test_recover_rejects_selection_flags(self) -> None: + for extra in (["--only", "a-main"], ["--force"], ["--accept-baseline", "a-main"], + ["--force-replace", "a-main"], ["--repeat", "2"]): + self.assertEqual(self.recover(*extra)[0], 2, extra) + self.assertEqual(self.run_pipeline("--resolve", "x", "--url", "https://a/b")[0], 2) + + +class DirtyRuleTest(RunWorld): + def test_dirty_without_allow_is_refused_and_not_rendered(self) -> None: + log = self.tmp / "calls.log" + self.set_fake({"log": str(log)}) + write(self.gf / "src/ui/main/MainWindow.cpp", "dirty\n") + code, _, _ = self.run_pipeline() + self.assertEqual(code, 1) + self.assertEqual(self.statuses()["a-main"], "refused-dirty") + self.assertEqual(self.statuses()["b-main"], "would-replace") + self.assertNotIn("a-main", self.rendered_ids(log)) + + def test_dirty_with_allow_is_preview_only(self) -> None: + write(self.gf / "src/ui/main/MainWindow.cpp", "dirty\n") + code, _, _ = self.run_pipeline("--allow-dirty") + self.assertEqual(code, 0) + self.assertEqual(self.statuses()["a-main"], "preview-only-dirty") + self.assertTrue((self.cache / "preview" / "a-main.png").exists()) + + def test_dirty_with_upload_never_publishes_but_clean_entries_do(self) -> None: + cdn = self.start_cdn() + before = self.doc("a.md") + write(self.gf / "src/ui/main/MainWindow.cpp", "dirty\n") + code, out, err = self.run_pipeline("--allow-dirty", "--upload") + self.assertEqual(code, 1, out + err) + st = self.statuses() + self.assertEqual(st["a-main"], "preview-only-dirty") + self.assertEqual(st["a-dialog"], "updated") + self.assertEqual(st["b-main"], "updated") + self.assertEqual(len(cdn.requests), 2) + lock = self.read_lock() + self.assertNotIn("a-main", lock) + self.assertIn("https://cdn.example.org/i/a1.webp", self.doc("a.md").decode()) + self.assertNotEqual(self.doc("a.md"), before) # img-2 was updated + self.assertFalse((self.cache / "baseline" / "a-main.png").exists()) + + def test_platform_mismatch_is_preview_only_under_upload(self) -> None: + cdn = self.start_cdn() + self.set_fake({"platform_name": "xcb", "product": "Debian GNU/Linux 13"}) + code, _, _ = self.run_pipeline("--upload") + self.assertEqual(code, 1) + self.assertEqual(set(self.statuses().values()), {"preview-only-platform"}) + self.assertEqual(cdn.requests, []) + self.assertEqual(self.read_lock(), {}) + + +class DiffTest(RunWorld): + def test_missing_baseline_then_accept_baseline(self) -> None: + self.publish_all() + (self.cache / "baseline" / "a-main.png").unlink() + commit_change(self.gf, "src/ui/main/MainWindow.cpp") + code, _, _ = self.run_pipeline("--only", "a-main") + self.assertEqual(code, 1) + self.assertEqual(self.statuses()["a-main"], "baseline-unavailable") + code, out, err = self.run_pipeline("--only", "a-main", "--accept-baseline", "a-main", "--upload") + self.assertEqual(code, 0, out + err) + self.assertEqual(self.statuses()["a-main"], "updated") + + def test_baseline_sha_mismatch_is_unavailable(self) -> None: + self.publish_all() + Image.new("RGBA", (10, 10), (1, 2, 3, 255)).save(self.cache / "baseline" / "a-main.png") + commit_change(self.gf, "src/ui/main/MainWindow.cpp") + self.run_pipeline("--only", "a-main") + self.assertEqual(self.statuses()["a-main"], "baseline-unavailable") + + def test_sub_threshold_advances_lock_then_ok(self) -> None: + self.publish_all() + old = self.read_lock()["a-main"] + self.set_fake({"tweak_pixels": 5}) + new_head = commit_change(self.gf, "src/ui/main/MainWindow.cpp") + self.assertEqual(self.check_rows(binary=True)["a-main"], ["outdated"]) + self.run_pipeline("--only", "a-main") + self.assertEqual(self.statuses()["a-main"], "unchanged-lock-not-advanced") + self.assertEqual(self.read_lock()["a-main"], old) + code, out, err = self.run_pipeline("--only", "a-main", "--apply") + self.assertEqual(code, 0, out + err) + self.assertEqual(self.statuses()["a-main"], "lock-advanced") + new = self.read_lock()["a-main"] + self.assertEqual(new["gpgfrontend_commit"], new_head) + for key in ("url", "webp_sha256", "pixels_sha256"): + self.assertEqual(new[key], old[key]) + self.assertEqual(self.check_rows(binary=True)["a-main"], ["ok"]) + + def test_emptied_link_is_restored_from_the_lock(self) -> None: + self.publish_all() + url = self.read_lock()["a-main"]["url"] + page = self.manual / "src" / "content" / "docs" / "guides" / "a.md" + page.write_bytes(page.read_bytes().replace(f"[img-1]: {url}".encode(), b"[img-1]: TODO")) + self.assertEqual(self.check_rows(binary=True)["a-main"], ["missing"]) + + self.run_pipeline("--only", "a-main") + self.assertEqual(self.statuses()["a-main"], "would-restore-doc") + self.assertIn(b"[img-1]: TODO", self.doc("a.md")) + + code, out, err = self.run_pipeline("--only", "a-main", "--apply") + self.assertEqual(code, 0, out + err) + self.assertEqual(self.statuses()["a-main"], "doc-restored") + self.assertIn(f"[img-1]: {url}".encode(), self.doc("a.md")) + self.assertEqual(self.check_rows(binary=True)["a-main"], ["ok"]) + + def test_large_change_is_replace(self) -> None: + self.publish_all() + self.set_fake({"variant": 40}) + commit_change(self.gf, "src/ui/main/MainWindow.cpp") + self.run_pipeline("--only", "a-main") + self.assertEqual(self.statuses()["a-main"], "would-replace") + + +class UploadOutcomeTest(RunWorld): + def test_success_request_shape(self) -> None: + cdn = self.start_cdn() + self.run_pipeline("--only", "a-main", "--upload") + self.assertEqual(len(cdn.requests), 1) + req = cdn.requests[0] + self.assertTrue(req["has_image"] and req["has_token"]) + self.assertTrue(req["content_type"].startswith("multipart/form-data")) + self.assertIn("https://img.example.org/i/up1.webp", self.doc("a.md").decode()) + + def _expect_failed(self, mode: str) -> None: + cdn = self.start_cdn() + cdn.mode = mode + before = self.doc("a.md") + code, _, _ = self.run_pipeline("--only", "a-main", "--upload") + self.assertEqual(code, 1) + self.assertEqual(self.statuses()["a-main"], "failed:upload-failed") + self.assertEqual(self.doc("a.md"), before) + self.assertEqual(self.read_lock(), {}) + + def test_error_json_is_upload_failed(self) -> None: + self._expect_failed("error") + + def test_http_500_is_upload_failed(self) -> None: + self._expect_failed("http500") + + def test_dropped_connection_is_upload_unknown(self) -> None: + cdn = self.start_cdn() + cdn.mode = "drop" + code, _, _ = self.run_pipeline("--only", "a-main", "--upload") + self.assertEqual(code, 3) + self.assertEqual(self.statuses()["a-main"], "upload-unknown") + # A later normal run does not upload the entry again. + cdn.mode = "success" + code, _, _ = self.run_pipeline("--only", "a-main", "--upload") + self.assertEqual(code, 3) + self.assertEqual(len(cdn.requests), 1) + + def test_upload_needs_credentials(self) -> None: + self.assertEqual(self.run_pipeline("--upload")[0], 2) + + +class MiscRunTest(RunWorld): + def test_stale_binary_mtime_warns_without_blocking(self) -> None: + os.utime(self.binary, (946684800, 946684800)) + code, out, _ = self.run_pipeline("--only", "a-main") + self.assertEqual(code, 0) + self.assertEqual(self.statuses()["a-main"], "would-replace") + self.assertTrue(any("older than" in w for w in self.last_run()["warnings"])) + self.assertIn("warning:", out) + + def test_repeat_identical_and_mismatch(self) -> None: + self.assertEqual(self.run_pipeline("--only", "a-main", "--repeat", "2")[0], 0) + self.set_fake({"nondeterministic": True}) + self.assertEqual(self.run_pipeline("--only", "a-main", "--repeat", "2")[0], 1) + self.assertEqual(self.statuses()["a-main"], "failed:repeat-mismatch") + + def test_footprint_outside_throwaway_fails(self) -> None: + self.set_fake({"write_appdata": str(self.appdata / "GpgFrontendLeak"), "write_appdata_render_only": True}) + self.assertEqual(self.run_pipeline("--only", "a-main")[0], 1) + self.assertEqual(self.statuses()["a-main"], "failed:footprint") + + def test_footprint_in_probe_fails(self) -> None: + self.set_fake({"write_appdata": str(self.appdata / "GpgFrontendLeak")}) + self.assertEqual(self.run_pipeline("--only", "a-main")[0], 1) + self.assertEqual(self.statuses()["a-main"], "failed:env-probe") + self.assertIn("footprint", json.dumps(self.last_run())) + + def test_gate_failure_in_probe(self) -> None: + self.set_fake({"gate_failures": ["font:Noto Sans"]}) + self.assertEqual(self.run_pipeline("--only", "a-main")[0], 1) + self.assertEqual(self.statuses()["a-main"], "failed:env-probe") + self.assertIn("font:Noto Sans", json.dumps(self.last_run())) + + def test_missing_highlight_fails_entry(self) -> None: + self.set_fake({"missing": ["InfoBoardWidget"]}) + self.run_pipeline("--only", "a-main") + self.assertEqual(self.statuses()["a-main"], "failed:highlight-missing") + + def test_command_line_follows_contract(self) -> None: + log = self.tmp / "calls.log" + self.set_fake({"log": str(log)}) + self.run_pipeline("--only", "a-main") + lines = log.read_text().splitlines() + probe = next(l for l in lines if "--screenshot-env" in l) + render = next(l for l in lines if "--screenshot main-window" in l) + for line in (probe, render): + self.assertTrue(line.startswith("--temp-profile")) + for flag in ("--locale en_US", "--theme light", "--size 160x100", "--clock 2026-01-15T10:00:00Z", + "--demo-gnupghome", "--fixtures", "--out", "--timeout 30000"): + self.assertIn(flag, line) + self.assertIn("demo-gnupghome", render) diff --git a/scripts/screenshots/tests/test_txn.py b/scripts/screenshots/tests/test_txn.py new file mode 100644 index 0000000..a82f945 --- /dev/null +++ b/scripts/screenshots/tests/test_txn.py @@ -0,0 +1,194 @@ +from __future__ import annotations + +import json +import os + +from helpers import World + +import txn as txnmod + + +class TxnWorld(World): + def setUp(self) -> None: + super().setUp() + self.cdn_server = self.start_cdn() + + def crash_at(self, state: str, entry: str = "a-main") -> dict: + os.environ[txnmod.CRASH_ENV] = state + try: + with self.assertRaises(txnmod.InjectedCrash): + self.run_pipeline("--only", entry, "--upload") + finally: + del os.environ[txnmod.CRASH_ENV] + return self.txn_data() + + def txn_dirs(self) -> list: + root = self.cache / "txn" + return sorted(p for p in root.iterdir() if p.is_dir()) if root.exists() else [] + + def txn_data(self, index: int = -1) -> dict: + return json.loads((self.txn_dirs()[index] / "txn.json").read_text(encoding="utf-8")) + + def a_url(self) -> str: + return "https://img.example.org/i/up1.webp" + + +class CrashRecoveryTest(TxnWorld): + def test_prepared_becomes_abandoned(self) -> None: + before = self.doc("a.md") + self.assertEqual(self.crash_at("prepared")["state"], "prepared") + code, out, _ = self.recover() + self.assertEqual(code, 0, out) + self.assertEqual(self.txn_data()["state"], "abandoned") + self.assertEqual(self.cdn_server.requests, []) + self.assertEqual(self.doc("a.md"), before) + self.assertEqual(self.read_lock(), {}) + # The entry is still stale and the next run picks it up. + code, _, _ = self.run_pipeline("--only", "a-main", "--upload") + self.assertEqual(code, 0) + self.assertEqual(self.statuses()["a-main"], "updated") + + def test_uploading_becomes_unknown_then_resolve_url(self) -> None: + data = self.crash_at("uploading") + code, out, _ = self.recover() + self.assertEqual(code, 3) + self.assertEqual(self.txn_data()["state"], "upload-unknown") + # Never retried automatically. + self.assertEqual(self.recover()[0], 3) + self.assertEqual(self.cdn_server.requests, []) + manual_url = "https://img.example.org/i/found-in-admin.webp" + code, out, err = self.recover("--resolve", data["txn_id"], "--url", manual_url) + self.assertEqual(code, 0, out + err) + final = self.txn_data() + self.assertEqual(final["state"], "committed") + self.assertEqual(final["upload"]["del"], txnmod.DEL_UNKNOWN) + self.assertIn(manual_url, self.doc("a.md").decode()) + self.assertEqual(self.read_lock()["a-main"]["url"], manual_url) + self.assertTrue((self.cache / "baseline" / "a-main.png").exists()) + + def test_uploading_resolve_retry_upload(self) -> None: + data = self.crash_at("uploading") + self.recover() + code, out, err = self.recover("--resolve", data["txn_id"], "--retry-upload") + self.assertEqual(code, 0, out + err) + self.assertEqual(len(self.cdn_server.requests), 1) + self.assertEqual(self.txn_data()["state"], "committed") + self.assertIn(self.a_url(), self.doc("a.md").decode()) + + def test_resolve_validation(self) -> None: + data = self.crash_at("uploaded") + self.assertEqual(self.recover("--resolve", data["txn_id"], "--url", "https://x/y")[0], 2) + self.assertEqual(self.recover("--resolve", "nope", "--url", "https://x/y")[0], 2) + self.assertEqual(self.recover("--resolve", data["txn_id"])[0], 2) + + def test_uploaded_completes_doc_lock_baseline(self) -> None: + self.crash_at("uploaded") + self.assertNotIn(self.a_url(), self.doc("a.md").decode()) + code, out, _ = self.recover() + self.assertEqual(code, 0, out) + self.assertEqual(self.txn_data()["state"], "committed") + self.assertIn(self.a_url(), self.doc("a.md").decode()) + self.assertEqual(self.read_lock()["a-main"]["url"], self.a_url()) + self.assertTrue((self.cache / "baseline" / "a-main.png").exists()) + self.assertEqual(len(self.cdn_server.requests), 1) + + def test_doc_applied_completes_lock_only(self) -> None: + self.crash_at("doc-applied") + doc_after_crash = self.doc("a.md") + self.assertIn(self.a_url(), doc_after_crash.decode()) + self.assertEqual(self.read_lock(), {}) + self.recover() + self.assertEqual(self.doc("a.md"), doc_after_crash) + self.assertEqual(self.read_lock()["a-main"]["url"], self.a_url()) + self.assertEqual(self.txn_data()["state"], "committed") + + def test_lock_applied_completes_baseline_only(self) -> None: + self.crash_at("lock-applied") + lock = self.read_lock() + self.assertFalse((self.cache / "baseline" / "a-main.png").exists()) + self.recover() + self.assertEqual(self.read_lock(), lock) + self.assertTrue((self.cache / "baseline" / "a-main.png").exists()) + self.assertEqual(self.txn_data()["state"], "committed") + + def test_tampered_webp_is_artifact_corrupt(self) -> None: + self.crash_at("uploaded") + before_doc, before_lock = self.doc("a.md"), self.read_lock() + webp = self.txn_dirs()[-1] / "final.webp" + webp.write_bytes(webp.read_bytes() + b"x") + code, _, _ = self.recover() + self.assertEqual(code, 3) + self.assertEqual(self.txn_data()["state"], "artifact-corrupt") + self.assertEqual(self.doc("a.md"), before_doc) + self.assertEqual(self.read_lock(), before_lock) + + def test_lock_changed_after_prepare_is_conflict(self) -> None: + self.crash_at("uploaded") + foreign = {"a-main": {"url": "https://other/x.webp", "note": "newer data"}} + self.lock.write_text(json.dumps(foreign), encoding="utf-8") + before_doc = self.doc("a.md") + code, out, _ = self.recover() + self.assertEqual(code, 3) + self.assertEqual(self.txn_data()["state"], "conflict") + self.assertEqual(self.read_lock(), foreign) + self.assertEqual(self.doc("a.md"), before_doc) + self.assertIn("del:", out) + + +class DocEditTest(TxnWorld): + def test_concurrent_unrelated_edit_is_kept(self) -> None: + self.crash_at("uploaded") + path = self.manual / "src/content/docs/guides/a.md" + path.write_bytes(self.doc("a.md").replace(b"Some steps.", b"Some edited steps.")) + self.assertEqual(self.recover()[0], 0) + text = self.doc("a.md").decode() + self.assertIn("Some edited steps.", text) + self.assertIn(f"[img-1]: {self.a_url()}", text) + + def test_edit_to_same_definition_is_conflict(self) -> None: + self.crash_at("uploaded") + path = self.manual / "src/content/docs/guides/a.md" + edited = self.doc("a.md").replace(b"https://cdn.example.org/i/a1.webp", b"https://elsewhere/x.webp") + path.write_bytes(edited) + self.assertEqual(self.recover()[0], 3) + self.assertEqual(self.txn_data()["state"], "conflict") + self.assertEqual(self.doc("a.md"), edited) + self.assertEqual(self.read_lock(), {}) + + def test_crlf_doc_keeps_crlf(self) -> None: + code, _, _ = self.run_pipeline("--only", "b-main", "--upload") + self.assertEqual(code, 0) + raw = self.doc("b.md") + self.assertIn(b"https://img.example.org/i/up1.webp\r\n", raw) + self.assertEqual(raw.count(b"\n"), raw.count(b"\r\n")) + + def test_del_never_in_committed_files(self) -> None: + code, _, _ = self.run_pipeline("--upload") + self.assertEqual(code, 0) + for path in (self.lock, self.manual / "src/content/docs/guides/a.md", + self.manual / "src/content/docs/guides/b.md", self.cache / "last-run.json"): + text = path.read_text(encoding="utf-8") + self.assertNotIn("/del/", text, path) + self.assertNotIn('"del"', text, path) + # It is kept in the ignored transaction cache. + self.assertIn("/del/secret", (self.txn_dirs()[0] / "txn.json").read_text(encoding="utf-8")) + + def test_conflict_stops_remaining_transactions(self) -> None: + import apply + + original = apply.apply_doc + + def conflicting(*args, **kwargs): + raise apply.Conflict("simulated concurrent edit") + + txnmod.apply_doc = conflicting + try: + code, out, _ = self.run_pipeline("--upload") + finally: + txnmod.apply_doc = original + self.assertEqual(code, 3) + st = self.statuses() + self.assertEqual(st["a-main"], "conflict") + self.assertEqual(st["a-dialog"], "failed:stopped-after-conflict") + self.assertEqual(len(self.cdn_server.requests), 1) + self.assertIn("del: https://img.example.org/del/secret1", out) diff --git a/scripts/screenshots/tests/test_units.py b/scripts/screenshots/tests/test_units.py new file mode 100644 index 0000000..48c5c18 --- /dev/null +++ b/scripts/screenshots/tests/test_units.py @@ -0,0 +1,227 @@ +"""Post-processing, manifest, render runner, upload parsing and font lookup.""" + +from __future__ import annotations + +import shutil +import unittest +from pathlib import Path + +from PIL import Image + +from helpers import MANIFEST, World, short_tempdir, write + +import fonts +import postprocess +import upload +from common import UsageError +from manifest import Highlight, load_manifest, parse_manifest +from render import RenderConfig, Renderer + + +def geometry(w: int, h: int, **extra) -> dict: + g = {"schema": 1, "capture_mode": "window", "dpr": 2, "image_size": [w, h], "origin": [0, 0], + "layers": [], "widgets": {"box": {"rect": [20, 20, 60, 30], "clipped": False}}, "missing": ["gone"], + "error": None} + g.update(extra) + return g + + +class PostprocessTest(unittest.TestCase): + def setUp(self) -> None: + self.tmp = short_tempdir("gfpp") + self.addCleanup(shutil.rmtree, self.tmp, True) + self.raw = self.tmp / "raw.png" + Image.new("RGBA", (200, 120), (240, 240, 240, 255)).save(self.raw) + manifest = parse_manifest(__import__("yaml").safe_load(MANIFEST), Path("m.yaml")) + self.entry = manifest.by_id()["a-main"] + + def entry_with(self, *highlights: Highlight): + from dataclasses import replace + return replace(self.entry, highlight=tuple(highlights)) + + def test_frame_size_shadow_and_corners(self) -> None: + out = postprocess.process(self.raw, geometry(200, 120), self.entry_with(), self.tmp, "a") + pad = self.entry.frame["padding"] * 2 + with Image.open(out.png) as img: + self.assertEqual(img.size, (200 + 2 * pad, 120 + 2 * pad)) + self.assertEqual(img.getpixel((0, 0))[3], 0) # transparent padding + self.assertEqual(img.getpixel((pad, pad))[3] < 255, True) # rounded corner + self.assertEqual(img.getpixel((pad + 100, pad + 60)), (240, 240, 240, 255)) + below = img.getpixel((pad + 100, pad + 120 + 4)) + self.assertGreater(below[3], 0) # shadow under the content + + def test_highlights_box_and_arrow_draw(self) -> None: + plain = postprocess.process(self.raw, geometry(200, 120), self.entry_with(), self.tmp, "p") + boxed = postprocess.process(self.raw, geometry(200, 120), self.entry_with(Highlight("box")), self.tmp, "b") + arrow = postprocess.process(self.raw, geometry(200, 120), self.entry_with(Highlight("box", "arrow")), + self.tmp, "r") + self.assertEqual(len({plain.pixels_sha256, boxed.pixels_sha256, arrow.pixels_sha256}), 3) + + def test_unknown_or_missing_highlight_fails(self) -> None: + for name, reason in (("gone", "highlight-missing"), ("nowhere", "highlight-unknown")): + with self.assertRaises(postprocess.PostprocessError) as ctx: + postprocess.process(self.raw, geometry(200, 120), self.entry_with(Highlight(name)), self.tmp, "x") + self.assertEqual(ctx.exception.reason, reason) + + def test_deterministic_and_webp_lossless(self) -> None: + a = postprocess.process(self.raw, geometry(200, 120), self.entry_with(Highlight("box")), self.tmp / "1", "a") + b = postprocess.process(self.raw, geometry(200, 120), self.entry_with(Highlight("box")), self.tmp / "2", "a") + self.assertEqual((a.png_sha256, a.webp_sha256, a.pixels_sha256), (b.png_sha256, b.webp_sha256, b.pixels_sha256)) + with Image.open(a.webp) as webp: + self.assertEqual(postprocess.pixels_sha256(webp), a.pixels_sha256) + + def test_dpr_and_size_mismatch(self) -> None: + with self.assertRaises(postprocess.PostprocessError): + postprocess.process(self.raw, geometry(200, 120, dpr=1), self.entry, self.tmp, "x") + with self.assertRaises(postprocess.PostprocessError): + postprocess.process(self.raw, geometry(201, 120), self.entry, self.tmp, "x") + + def test_optimizer_descriptor(self) -> None: + desc = postprocess.processing_descriptor() + self.assertIn(desc["optimizer"], ("none",) if not shutil.which("oxipng") else (desc["optimizer"],)) + self.assertTrue(desc["pillow"]) + + +class ManifestTest(unittest.TestCase): + def load(self, text: str): + return parse_manifest(__import__("yaml").safe_load(text), Path("m.yaml")) + + def test_defaults_resolution_and_auto_size(self) -> None: + m = self.load("entries:\n - {id: x, doc: d.md, ref: img-1, scenario: s, size: auto}\n" + " - {id: y, doc: d.md, ref: img-2, scenario: s, highlight: [a, {name: b, style: arrow}]}\n") + x, y = m.entries + self.assertIsNone(x.size) + self.assertEqual(x.size_str, "auto") + self.assertEqual(y.size, (1100, 680)) + self.assertEqual(y.fonts, ("Noto Sans", "Noto Sans CJK SC")) + self.assertEqual(y.highlight, (Highlight("a"), Highlight("b", "arrow"))) + self.assertEqual(y.diff, {"tolerance": 16, "threshold": 0.001}) + + def test_rejects_bad_input(self) -> None: + bad = [ + "entries:\n - {id: x, doc: d.md, ref: r, scenario: s, clock: '2026-01-15 10:00'}\n", + "entries:\n - {id: x, doc: d.md, ref: r, scenario: s, size: big}\n", + "entries:\n - {id: x, doc: d.md, ref: r, scenario: s, typo: 1}\n", + "entries:\n - {id: x, doc: d.md, ref: r}\n", + "entries:\n - {id: x, doc: d.md, ref: r, scenario: s}\n - {id: x, doc: e.md, ref: r, scenario: s}\n", + "entries:\n - {id: x, doc: d.md, ref: r, scenario: s, platform: macos}\n", + "entries:\n - {id: x, doc: d.md, ref: r, scenario: s, highlight: {name: a, style: circle}}\n", + ] + for text in bad: + with self.assertRaises(UsageError, msg=text): + self.load(text) + + def test_real_manifest_loads(self) -> None: + m = load_manifest(Path(__file__).resolve().parents[1] / "screenshots.yaml") + self.assertEqual([e.id for e in m.entries], ["main-window", "key-generate-dialog", + "text-decrypt-verify-result", "main-window-menu-popup"]) + + +class RendererTest(World): + def renderer(self, **kw) -> Renderer: + return Renderer(RenderConfig(self.binary, self.gf, self.work_root, **kw)) + + def entry(self, entry_id: str = "a-main"): + return load_manifest(self.manifest).by_id()[entry_id] + + def test_exit_codes_map_to_reasons_with_partial_record(self) -> None: + for code, reason in ((3, "timeout"), (5, "unexpected-modal"), (4, "output-unwritable")): + self.set_fake({"exit": {"main-window": code}}) + outcome = self.renderer().render(self.entry(), self.tmp / "out") + self.assertFalse(outcome.ok) + self.assertEqual((outcome.exit_code, outcome.reason), (code, reason)) + self.assertIn("forced-failure", outcome.detail) + + def test_auto_size_omits_size_flag(self) -> None: + from dataclasses import replace + r = self.renderer() + cmd = r.command(replace(self.entry(), size=None), self.tmp / "x.png", probe=False) + self.assertNotIn("--size", cmd) + self.assertIn("--size", r.command(self.entry(), self.tmp / "x.png", probe=True)) + + def test_isolated_environment(self) -> None: + import os + os.environ["GNUPGHOME"] = "/real/gnupg" + os.environ["QT_QPA_PLATFORM"] = "offscreen" + r = self.renderer() + work = r._new_workdir() + env = r.subprocess_env(work) + self.assertNotIn("GNUPGHOME", env) + self.assertNotIn("QT_QPA_PLATFORM", env) + if os.name != "nt": + for var in ("HOME", "XDG_DATA_HOME", "XDG_CONFIG_HOME", "TMPDIR"): + self.assertTrue(env[var].startswith(str(work)), var) + + def test_long_work_root_is_rejected(self) -> None: + import os + if os.name == "nt": + self.skipTest("Unix socket budget only") + r = Renderer(RenderConfig(self.binary, self.gf, self.tmp / ("x" * 60))) + with self.assertRaises(UsageError): + r._new_workdir() + + def test_env_hash_crosscheck_failure_is_reported(self) -> None: + import envhash + real = envhash.env_hash_core + envhash.env_hash_core = lambda env: "0" * 64 + try: + outcome = self.renderer().probe(self.entry()) + finally: + envhash.env_hash_core = real + self.assertEqual(outcome.reason, "env-invalid") + + def test_screenshot_list(self) -> None: + listed = self.renderer().list_scenarios() + self.assertEqual(listed["key-generate-dialog"], {"size": "auto", "capture_mode": "dialog"}) + + +class UploadParseTest(unittest.TestCase): + def test_response_validation(self) -> None: + ok = upload.parse_response('{"result":"success","code":200,"url":"https://i.example/x.webp","del":"https://i/d"}') + self.assertEqual((ok.url, ok.del_url), ("https://i.example/x.webp", "https://i/d")) + for raw in ('{"result":"success","url":"http://i.example/x.webp"}', '{"result":"failed"}', "<html>"): + with self.assertRaises(upload.UploadFailed): + upload.parse_response(raw) + + def test_credentials_require_https_except_loopback(self) -> None: + import os + from unittest import mock + with mock.patch.dict(os.environ, {"GF_MANUAL_CDN_API": "http://cdn.example.org/api", "GF_MANUAL_CDN_TOKEN": "t"}): + with self.assertRaises(UsageError): + upload.credentials_from_env() + with mock.patch.dict(os.environ, {"GF_MANUAL_CDN_API": "http://127.0.0.1:9/api", "GF_MANUAL_CDN_TOKEN": "t"}): + self.assertEqual(upload.credentials_from_env().host, "127.0.0.1") + + +class FontLookupTest(unittest.TestCase): + def test_registry_matching_prefers_regular(self) -> None: + values = [ + ("Noto Sans Bold (TrueType)", "NotoSans-Bold.ttf"), + ("Noto Sans Regular (TrueType)", "NotoSans-Regular.ttf"), + ("Noto Sans CJK JP Regular & Noto Sans CJK SC Regular (OpenType)", "NotoSansCJK-Regular.ttc"), + ("Noto Sans CJK SC Bold (OpenType)", "NotoSansCJK-Bold.ttc"), + ] + self.assertEqual(fonts.match_registry("Noto Sans", values), "NotoSans-Regular.ttf") + self.assertEqual(fonts.match_registry("Noto Sans CJK SC", values), "NotoSansCJK-Regular.ttc") + self.assertIsNone(fonts.match_registry("Noto Serif", values)) + + def test_font_files_hash_and_cache(self) -> None: + tmp = short_tempdir("gffont") + self.addCleanup(shutil.rmtree, tmp, True) + f = write(tmp / "A.ttf", "font-bytes") + calls = [] + + def resolver(family): + calls.append(family) + return f if family == "A" else None + + cache: dict = {} + out = fonts.font_files(["A", "B", "A"], resolver, cache) + self.assertEqual(out[0]["file"], "A.ttf") + self.assertIsNotNone(out[0]["sha256"]) + self.assertIsNone(out[1]["sha256"]) + self.assertEqual(calls, ["A", "B"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/screenshots/txn.py b/scripts/screenshots/txn.py new file mode 100644 index 0000000..180955d --- /dev/null +++ b/scripts/screenshots/txn.py @@ -0,0 +1,338 @@ +"""Durable upload/apply transaction, one per replaced image. + +States: prepared -> uploading -> uploaded -> doc-applied -> lock-applied +-> committed, with the terminal side exits abandoned, upload-failed, +conflict and artifact-corrupt, and upload-unknown, which waits for a +decision (`run.py --recover --resolve`). + +Every state is persisted to txn.json (temp file, fsync, os.replace, dir +fsync) before the step that depends on it. Recovery reads only txn.json; +journal.jsonl is an audit log. +""" + +from __future__ import annotations + +import json +import os +import secrets +import shutil +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Callable, Optional + +import docrefs +import upload +from apply import Conflict, apply_doc, apply_lock, doc_sha256, lock_state, read_lock +from common import (PipelineError, UsageError, append_line_durable, atomic_copy, atomic_write_json, + load_json, read_text_exact, sha256_file, utc_now_iso, utc_stamp_compact) + +SCHEMA = 1 +PREPARED = "prepared" +UPLOADING = "uploading" +UPLOADED = "uploaded" +DOC_APPLIED = "doc-applied" +LOCK_APPLIED = "lock-applied" +COMMITTED = "committed" +ABANDONED = "abandoned" +UPLOAD_FAILED = "upload-failed" +UPLOAD_UNKNOWN = "upload-unknown" +CONFLICT = "conflict" +ARTIFACT_CORRUPT = "artifact-corrupt" + +TERMINAL = (COMMITTED, ABANDONED, UPLOAD_FAILED, CONFLICT, ARTIFACT_CORRUPT) +DEL_UNKNOWN = "unknown" + +# Test hook: raise InjectedCrash right after this state has been persisted. +CRASH_ENV = "GF_SHOTS_TEST_CRASH_AFTER" +crash_hook: Optional[Callable[[str], None]] = None + +_PLACEHOLDER_URL = "https://placeholder.invalid/prepare-check.webp" + +Uploader = Callable[[upload.Credentials, Path, str], upload.Uploaded] + + +class InjectedCrash(BaseException): + """Simulates the process dying; BaseException so nothing swallows it.""" + + +@dataclass +class Paths: + manual_root: Path + lock: Path + cache: Path + + @property + def txn_root(self) -> Path: + return self.cache / "txn" + + @property + def journal(self) -> Path: + return self.txn_root / "journal.jsonl" + + def baseline(self, entry_id: str) -> Path: + return self.cache / "baseline" / f"{entry_id}.png" + + +@dataclass +class Outcome: + txn_id: str + entry_id: str + state: str + detail: str = "" + url: Optional[str] = None + del_url: Optional[str] = None + + +class Txn: + def __init__(self, directory: Path, data: dict[str, Any], paths: Paths) -> None: + self.dir = directory + self.data = data + self.paths = paths + + @property + def id(self) -> str: + return self.data["txn_id"] + + @property + def entry_id(self) -> str: + return self.data["entry_id"] + + @property + def state(self) -> str: + return self.data["state"] + + @classmethod + def load(cls, directory: Path, paths: Paths) -> "Txn": + data = load_json(directory / "txn.json") + if not isinstance(data, dict) or data.get("schema") != SCHEMA: + raise PipelineError(f"{directory}: unreadable txn.json") + return cls(directory, data, paths) + + def persist(self, state: str, detail: str = "") -> None: + at = utc_now_iso() + self.data["state"] = state + self.data.setdefault("state_history", []).append({"state": state, "at": at}) + self.data["detail"] = detail + atomic_write_json(self.dir / "txn.json", self.data) + append_line_durable(self.paths.journal, json.dumps( + {"txn_id": self.id, "entry_id": self.entry_id, "state": state, "at": at}, sort_keys=True)) + if os.environ.get(CRASH_ENV) == state: + raise InjectedCrash(state) + if crash_hook is not None: + crash_hook(state) + + def outcome(self) -> Outcome: + up = self.data.get("upload") or {} + return Outcome(self.id, self.entry_id, self.state, self.data.get("detail", ""), + up.get("url"), up.get("del")) + + def artifacts_intact(self) -> bool: + arts = self.data["artifacts"] + for kind in ("png", "webp"): + path = self.dir / arts[kind]["file"] + if not path.is_file() or sha256_file(path) != arts[kind]["sha256"]: + return False + return True + + def target_record(self) -> dict[str, Any]: + record = dict(self.data["proposed_lock_record"]) + record["url"] = self.data["upload"]["url"] + return record + + +def prepare(paths: Paths, entry_id: str, doc_rel: str, ref: str, final_png: Path, final_webp: Path, + render_record: dict[str, Any], proposed: dict[str, Any]) -> Txn: + """Validates everything that can be checked offline. On failure no + transaction exists and nothing has been uploaded.""" + if "url" in proposed: + raise PipelineError("proposed lock record must not carry a url") + doc_path = paths.manual_root / doc_rel + if not doc_path.is_file(): + raise PipelineError(f"doc not found: {doc_rel}") + text = read_text_exact(doc_path) + defs = docrefs.find_definitions(text, ref) + if len(defs) != 1: + raise PipelineError(f"{doc_rel}: [{ref}] has {len(defs)} definitions, expected exactly one") + definition = defs[0] + rewritten = docrefs.replace_url(text, definition, _PLACEHOLDER_URL) + check = docrefs.find_definitions(rewritten, ref) + if len(check) != 1 or check[0].url != _PLACEHOLDER_URL: + raise PipelineError(f"{doc_rel}: cannot rewrite [{ref}] safely") + + txn_id = f"{utc_stamp_compact()}-{entry_id}-{secrets.token_hex(4)}" + directory = paths.txn_root / txn_id + directory.mkdir(parents=True) + try: + atomic_copy(final_png, directory / "final.png") + atomic_copy(final_webp, directory / "final.webp") + atomic_write_json(directory / "render.json", render_record) + webp_size = (directory / "final.webp").stat().st_size + data = { + "schema": SCHEMA, + "txn_id": txn_id, + "entry_id": entry_id, + "state": None, + "state_history": [], + "artifacts": { + "png": {"file": "final.png", "sha256": sha256_file(directory / "final.png")}, + "webp": {"file": "final.webp", "sha256": sha256_file(directory / "final.webp"), + "bytes": webp_size}, + }, + "proposed_lock_record": proposed, + "lock_record_before": read_lock(paths.lock).get(entry_id), + "doc": { + "path": doc_rel, + "ref": ref, + "old_url": definition.url, + "sha256_before": doc_sha256(doc_path), + "definition": { + "line_start": definition.line_start, + "line_end": definition.line_end, + "text_before": definition.text, + }, + }, + "upload": None, + } + if data["artifacts"]["webp"]["sha256"] != proposed.get("webp_sha256"): + raise PipelineError("final.webp does not match the proposed lock record") + txn = Txn(directory, data, paths) + except BaseException: + shutil.rmtree(directory, ignore_errors=True) + raise + txn.persist(PREPARED) + return txn + + +def _lock_precheck(txn: Txn) -> None: + state = lock_state(txn.paths.lock, txn.entry_id, txn.data["lock_record_before"], _maybe_target(txn)) + if state == "other": + raise Conflict(f"lock record for {txn.entry_id} changed since the transaction was prepared") + + +def _maybe_target(txn: Txn) -> dict[str, Any]: + if txn.data.get("upload") and txn.data["upload"].get("url"): + return txn.target_record() + return {"__no_target__": True} + + +def _do_upload(txn: Txn, creds: upload.Credentials, uploader: Uploader) -> None: + txn.data["upload"] = {"endpoint_host": creds.host, "attempted_at": utc_now_iso(), + "response_raw": None, "url": None, "del": None} + txn.persist(UPLOADING) + try: + result = uploader(creds, txn.dir / "final.webp", f"{txn.entry_id}.webp") + except upload.UploadFailed as exc: + txn.data["upload"]["response_raw"] = exc.raw + txn.persist(UPLOAD_FAILED, exc.reason) + return + except upload.UploadUnknown as exc: + txn.persist(UPLOAD_UNKNOWN, f"no response after sending: {exc}") + return + txn.data["upload"].update({"response_raw": result.raw, "url": result.url, "del": result.del_url}) + txn.persist(UPLOADED) + + +def _advance(txn: Txn) -> None: + """Runs the local steps from the current state to committed.""" + try: + while txn.state not in TERMINAL and txn.state != UPLOAD_UNKNOWN: + if txn.state == UPLOADED: + _lock_precheck(txn) + doc = txn.data["doc"] + apply_doc(txn.paths.manual_root / doc["path"], doc["ref"], doc["old_url"], + txn.data["upload"]["url"], doc["sha256_before"]) + txn.persist(DOC_APPLIED) + elif txn.state == DOC_APPLIED: + apply_lock(txn.paths.lock, txn.entry_id, txn.data["lock_record_before"], txn.target_record()) + txn.persist(LOCK_APPLIED) + elif txn.state == LOCK_APPLIED: + baseline = txn.paths.baseline(txn.entry_id) + atomic_copy(txn.dir / "final.png", baseline) + if sha256_file(baseline) != txn.data["artifacts"]["png"]["sha256"]: + raise PipelineError(f"baseline copy for {txn.entry_id} does not match final.png") + txn.persist(COMMITTED) + else: + raise PipelineError(f"cannot advance a transaction in state {txn.state}") + except Conflict as exc: + txn.persist(CONFLICT, str(exc)) + + +def execute(txn: Txn, creds: upload.Credentials, uploader: Uploader = upload.upload_file) -> Outcome: + if txn.state != PREPARED: + raise PipelineError(f"{txn.id} is not prepared") + if not txn.artifacts_intact(): + txn.persist(ARTIFACT_CORRUPT, "artifacts changed before upload") + return txn.outcome() + try: + _lock_precheck(txn) + except Conflict as exc: + txn.persist(CONFLICT, str(exc)) + return txn.outcome() + _do_upload(txn, creds, uploader) + _advance(txn) + return txn.outcome() + + +def list_txns(paths: Paths) -> list[Txn]: + if not paths.txn_root.is_dir(): + return [] + out = [] + for directory in sorted(p for p in paths.txn_root.iterdir() if p.is_dir()): + if (directory / "txn.json").is_file(): + out.append(Txn.load(directory, paths)) + return out + + +def pending_for(paths: Paths, entry_id: str) -> list[Txn]: + """Transactions for an entry that still need --recover.""" + return [t for t in list_txns(paths) if t.entry_id == entry_id and t.state not in TERMINAL] + + +def recover(paths: Paths, resolve: Optional[str] = None, url: Optional[str] = None, + retry_upload: bool = False, creds: Optional[upload.Credentials] = None, + uploader: Uploader = upload.upload_file) -> list[Outcome]: + """Completes every non-terminal transaction without rendering. + + Only an explicit --resolve moves an upload-unknown transaction on. + """ + txns = list_txns(paths) + if resolve is not None: + match = [t for t in txns if t.id == resolve] + if not match: + raise UsageError(f"no transaction {resolve}") + if match[0].state not in (UPLOADING, UPLOAD_UNKNOWN): + raise UsageError(f"{resolve} is {match[0].state}; only upload-unknown can be resolved") + if bool(url) == bool(retry_upload): + raise UsageError("--resolve needs exactly one of --url or --retry-upload") + if url and not upload.is_https_url(url): + raise UsageError(f"--url must be an https URL, got {url!r}") + if retry_upload and creds is None: + raise UsageError("--retry-upload needs CDN credentials") + outcomes: list[Outcome] = [] + stopped = False + for txn in txns: + if txn.state in TERMINAL: + continue + if stopped: + outcomes.append(Outcome(txn.id, txn.entry_id, txn.state, "not processed: an earlier conflict stopped the run")) + continue + if not txn.artifacts_intact(): + txn.persist(ARTIFACT_CORRUPT, "final.png or final.webp does not match txn.json") + elif txn.state == PREPARED: + txn.persist(ABANDONED, "nothing was uploaded") + else: + if txn.state == UPLOADING: + txn.persist(UPLOAD_UNKNOWN, "interrupted after the request may have been sent") + if txn.state == UPLOAD_UNKNOWN and txn.id == resolve: + if url: + txn.data["upload"].update({"url": url, "del": DEL_UNKNOWN, + "response_raw": "resolved by hand"}) + txn.persist(UPLOADED, "url supplied with --resolve; del unknown") + else: + assert creds is not None + _do_upload(txn, creds, uploader) + _advance(txn) + outcomes.append(txn.outcome()) + if txn.state == CONFLICT: + stopped = True + return outcomes diff --git a/scripts/screenshots/upload.py b/scripts/screenshots/upload.py new file mode 100644 index 0000000..0cbae4a --- /dev/null +++ b/scripts/screenshots/upload.py @@ -0,0 +1,130 @@ +"""EasyImages 2.0 upload: POST multipart `image` + `token` to $GF_MANUAL_CDN_API. + +A successful response is JSON {result, code, url, srcName, thumb, del}. +The token is read from $GF_MANUAL_CDN_TOKEN and is never stored. +""" + +from __future__ import annotations + +import ipaddress +import json +import os +import secrets +import socket +import urllib.error +import urllib.request +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Optional +from urllib.parse import urlparse + +from common import UsageError + +API_ENV = "GF_MANUAL_CDN_API" +TOKEN_ENV = "GF_MANUAL_CDN_TOKEN" +TIMEOUT_S = 120 + + +class UploadFailed(Exception): + """The server answered and did not store the image; a retry is safe.""" + + def __init__(self, reason: str, raw: str = "") -> None: + super().__init__(reason) + self.reason = reason + self.raw = raw + + +class UploadUnknown(Exception): + """The request may have reached the server; the outcome is unknown.""" + + +@dataclass +class Uploaded: + url: str + del_url: Optional[str] + raw: str + + +@dataclass +class Credentials: + api: str + token: str + + @property + def host(self) -> str: + return urlparse(self.api).hostname or "" + + +def _is_loopback(host: str) -> bool: + if host == "localhost": + return True + try: + return ipaddress.ip_address(host).is_loopback + except ValueError: + return False + + +def is_https_url(url: str) -> bool: + parsed = urlparse(url) + return parsed.scheme == "https" and bool(parsed.hostname) and not any(c.isspace() for c in url) + + +def credentials_from_env() -> Credentials: + api = os.environ.get(API_ENV, "").strip() + token = os.environ.get(TOKEN_ENV, "").strip() + if not api or not token: + raise UsageError(f"--upload needs {API_ENV} and {TOKEN_ENV}") + parsed = urlparse(api) + if parsed.scheme != "https" and not (parsed.scheme == "http" and _is_loopback(parsed.hostname or "")): + raise UsageError(f"{API_ENV} must be an https URL") + return Credentials(api, token) + + +def _multipart(fields: dict[str, str], file_field: str, filename: str, data: bytes, + content_type: str) -> tuple[bytes, str]: + boundary = "----gfshot" + secrets.token_hex(16) + parts: list[bytes] = [] + for name, value in fields.items(): + parts.append( + f"--{boundary}\r\nContent-Disposition: form-data; name=\"{name}\"\r\n\r\n{value}\r\n".encode("utf-8")) + parts.append( + (f"--{boundary}\r\nContent-Disposition: form-data; name=\"{file_field}\"; filename=\"{filename}\"\r\n" + f"Content-Type: {content_type}\r\n\r\n").encode("utf-8") + data + b"\r\n") + parts.append(f"--{boundary}--\r\n".encode("utf-8")) + return b"".join(parts), f"multipart/form-data; boundary={boundary}" + + +def parse_response(raw: str) -> Uploaded: + try: + data: Any = json.loads(raw) + except ValueError as exc: + raise UploadFailed("response is not JSON", raw) from exc + if not isinstance(data, dict) or data.get("result") != "success": + message = data.get("message") or data.get("result") if isinstance(data, dict) else None + raise UploadFailed(f"server rejected the upload: {message or 'no success result'}", raw) + url = data.get("url") + if not isinstance(url, str) or not is_https_url(url): + raise UploadFailed(f"response url is not https: {url!r}", raw) + del_url = data.get("del") if isinstance(data.get("del"), str) else None + return Uploaded(url=url, del_url=del_url, raw=raw) + + +def upload_file(creds: Credentials, path: Path, filename: str) -> Uploaded: + body, content_type = _multipart({"token": creds.token}, "image", filename, Path(path).read_bytes(), "image/webp") + request = urllib.request.Request(creds.api, data=body, method="POST", + headers={"Content-Type": content_type, "Accept": "application/json"}) + handlers = [urllib.request.ProxyHandler({})] if _is_loopback(creds.host) else [] + opener = urllib.request.build_opener(*handlers) + try: + with opener.open(request, timeout=TIMEOUT_S) as response: + raw = response.read().decode("utf-8", "replace") + except urllib.error.HTTPError as exc: + raw = exc.read().decode("utf-8", "replace") if exc.fp else "" + raise UploadFailed(f"HTTP {exc.code}", raw) from exc + except urllib.error.URLError as exc: + if isinstance(exc.reason, (ConnectionRefusedError, socket.gaierror)): + raise UploadFailed(f"connection failed before sending: {exc.reason}") from exc + raise UploadUnknown(str(exc.reason)) from exc + except (OSError, TimeoutError) as exc: + raise UploadUnknown(str(exc)) from exc + return parse_response(raw) |
