linux-wallpaperengine-kde/tools/regression/regress.py

#!/usr/bin/env python3
"""Wallpaper regression runs: render every installed scene wallpaper headlessly with a build,
then compare two runs (usually the last good build against the one being tested).

    tools/regression/regress.py run /path/to/old/linux-wallpaperengine runs/base --repeat 2
    tools/regression/regress.py run build/output/linux-wallpaperengine runs/new
    tools/regression/regress.py compare runs/base runs/new --report runs/report

Runs are reproducible: animation time advances by a fixed step per frame (LWE_FIXED_TIMESTEP),
particles are seeded per object, the wall clock is pinned (fixed_clock.c) and audio input is
off. Each worker gets its own Xvfb display, nothing touches the real screen.

--gpu renders on the GPU instead (the engine's headless EGL driver, XDG_SESSION_TYPE=headless),
no Xvfb and many times faster than llvmpipe. GPU and llvmpipe pixels differ slightly, so compare
runs made the same way; the binary needs the headless driver for this.

compare looks at four things per wallpaper: whether it still renders (crash/timeout), error
lines that appear or disappear in the log, how much of the screenshot changed, and whether gif
and video textures still move (wallpapers shipping them get a second render a few frames later,
a hidden or frozen one shows up as less motion). Video textures play in real time and never
render the same twice, --repeat 2 on the base run marks those pixels so compare ignores them.
It writes an HTML report with side-by-sides of everything flagged and exits 1 if anything
regressed. Both builds need LWE_FIXED_TIMESTEP support, older ones render every run differently.

Needs Xvfb, gcc and Pillow (tools/requirements.txt).
"""

import argparse
import atexit
import concurrent.futures
import html
import json
import os
import queue
import re
import shutil
import signal
import subprocess
import sys
import tempfile
import time
from pathlib import Path

HERE = Path (__file__).resolve ().parent
TOOLS = HERE.parent

DEFAULT_WORKSHOP = Path.home () / '.local/share/Steam/steamapps/workshop/content/431960'
DEFAULT_ASSETS = Path.home () / '.local/share/Steam/steamapps/common/wallpaper_engine/assets'

# lines worth tracking between builds; everything else in the log is chatter
ERROR_LINE = re.compile (
    r'error|exception|failed|cannot|can\'t|unsupported|not supported|typeerror|referenceerror|'
    r'syntaxerror|abort|segmentation|falling back',
    re.IGNORECASE
)
# driver/X noise that shows up on some machines and says nothing about the engine
IGNORED_LINE = re.compile (r'amdgpu|_XSERVTrans|MESA-LOADER|libEGL warning|ALSA lib|Puppet draw result')
# mpv announcing a video stream, those textures play in real time and never render the same twice
VIDEO_LINE = re.compile (r'^\W*Video\s+--vid=', re.MULTILINE)


def env_path (name, fallback):
    value = os.environ.get (name)
    return Path (value) if value else fallback


def read_project (folder):
    try:
        with open (folder / 'project.json', encoding = 'utf-8-sig') as f:
            return json.load (f)
    except (OSError, ValueError):
        return None


def read_sized (f):
    size = int.from_bytes (f.read (4), 'little')
    return f.read (size).decode ('utf-8', 'replace')


def texture_kinds (header):
    """'gif' for animated (sprite sheet) textures, 'video' for mp4 ones, from a .tex header."""
    if header[:8] != b'TEXV0005' or len (header) < 26:
        return set ()

    flags = int.from_bytes (header[22:26], 'little')
    return ({'gif'} if flags & 4 else set ()) | ({'video'} if flags & 32 else set ())


def animated_textures (folder):
    """What kind of animated textures a wallpaper ships, loose or inside its .pkg files."""
    kinds = set ()

    for path in folder.rglob ('*'):
        if path.suffix == '.mp4':
            kinds.add ('video')
        elif path.suffix == '.tex':
            with open (path, 'rb') as f:
                kinds |= texture_kinds (f.read (64))
        elif path.suffix == '.pkg':
            try:
                with open (path, 'rb') as f:
                    read_sized (f)
                    entries = []
                    for _ in range (int.from_bytes (f.read (4), 'little')):
                        name = read_sized (f)
                        offset = int.from_bytes (f.read (4), 'little')
                        f.read (4)
                        entries.append ((name, offset))
                    base = f.tell ()
                    for name, offset in entries:
                        if name.endswith ('.mp4'):
                            kinds.add ('video')
                        elif name.endswith ('.tex'):
                            f.seek (base + offset)
                            kinds |= texture_kinds (f.read (64))
            except (OSError, ValueError):
                pass

    return kinds


def find_wallpapers (workshop, types, ids, variants):
    """(name, folder, title, properties) per render; variants add renders of the same wallpaper
    with user properties set, named <id>@<variant>."""
    found = []

    for folder in sorted (workshop.iterdir ()):
        if not folder.is_dir () or (ids and folder.name not in ids):
            continue

        project = read_project (folder)
        kind = str ((project or {}).get ('type', '')).lower ()

        if kind in types:
            title = (project or {}).get ('title', '')
            found.append ((folder.name, folder, title, []))

            for variant, properties in variants.get (folder.name, {}).items ():
                found.append ((f'{folder.name}@{variant}', folder, f'{title} [{variant}]', properties))

    return found


def build_shims (out):
    shims = out / '.shims'
    shims.mkdir (parents = True, exist_ok = True)
    built = []

    for source in (TOOLS / 'dbus_noop_shim.c', HERE / 'fixed_clock.c'):
        target = shims / (source.stem + '.so')

        if not target.exists () or source.stat ().st_mtime > target.stat ().st_mtime:
            subprocess.run (['gcc', '-shared', '-fPIC', '-O2', '-o', str (target), str (source), '-ldl'], check = True)

        built.append (str (target))

    return ':'.join (built)


def x_sockets ():
    """Display numbers with a listening X socket, abstract ones included (some sandboxes can't
    write /tmp/.X11-unix, Xvfb then only has the abstract socket)."""
    numbers = set ()

    with open ('/proc/net/unix') as f:
        for line in f:
            match = re.search (r'/tmp/\.X11-unix/X(\d+)$', line.strip ())
            if match:
                numbers.add (int (match.group (1)))

    return numbers


class Display:
    """One Xvfb server per worker so renders can run side by side."""

    process = None

    def __init__ (self, first, width, height):
        number = first

        while True:
            while number in x_sockets () or Path (f'/tmp/.X11-unix/X{number}').exists ():
                number += 1

            self.name = f':{number}'
            self.number = number
            self.process = subprocess.Popen (
                ['Xvfb', self.name, '-screen', '0', f'{width}x{height}x24', '-nolisten', 'tcp'],
                stdout = subprocess.DEVNULL, stderr = subprocess.DEVNULL
            )

            for _ in range (100):
                if number in x_sockets ():
                    return
                if self.process.poll () is not None:
                    break
                time.sleep (0.05)

            self.close ()
            number += 1

            if number > first + 50:
                raise RuntimeError ('could not start Xvfb')

    def close (self):
        if self.process is None:
            return

        self.process.terminate ()
        try:
            self.process.wait (5)
        except subprocess.TimeoutExpired:
            self.process.kill ()


def stop (process):
    if process.poll () is not None:
        return

    os.killpg (process.pid, signal.SIGTERM)
    try:
        process.wait (5)
    except subprocess.TimeoutExpired:
        os.killpg (process.pid, signal.SIGKILL)
        process.wait ()


def render_once (args, env, folder, shot, log_path, properties):
    command = [
        str (args.binary), '--window', f'0x0x{args.width}x{args.height}', '--fps', '1000',
        '--silent', '--no-audio-processing', '--screenshot', str (shot), '--screenshot-delay', str (args.frame),
        '--assets-dir', str (args.assets),
    ]

    for value in properties:
        command += ['--set-property', value]

    command.append (str (folder))
    # a private session bus per render, the engine needs one and must not use the desktop's
    if shutil.which ('dbus-run-session'):
        command = ['dbus-run-session', '--'] + command

    started = time.monotonic ()
    status = 'timeout'

    with open (log_path, 'wb') as log:
        process = subprocess.Popen (
            command, env = env, stdin = subprocess.DEVNULL, stdout = log, stderr = subprocess.STDOUT, start_new_session = True
        )
        last_size = -1

        while time.monotonic () - started < args.timeout:
            code = process.poll ()
            size = shot.stat ().st_size if shot.exists () else 0

            # the engine keeps running after the screenshot, stop it once the file is complete
            if size > 0 and size == last_size:
                status = 'ok'
                break

            if code is not None:
                status = 'ok' if size > 0 else ('crash' if code < 0 else 'noshot')
                break

            last_size = size
            time.sleep (0.25)

        stop (process)

    return status, process.returncode, time.monotonic () - started


def render (args, preload, display, wallpaper):
    wallpaper_id, folder, title, properties = wallpaper
    target = args.out / wallpaper_id
    shutil.rmtree (target, ignore_errors = True)
    target.mkdir (parents = True)

    env = dict (os.environ)
    env.pop ('WAYLAND_DISPLAY', None)
    env.pop ('WAYLAND_SOCKET', None)
    env.pop ('DISPLAY', None)
    # nothing may reach the desktop session: libwayland and D-Bus fall back to sockets in XDG_RUNTIME_DIR, so a
    # build without the headless driver would open a window on the real desktop
    for name in ('DBUS_SESSION_BUS_ADDRESS', 'KDE_FULL_SESSION', 'KDE_SESSION_VERSION', 'DESKTOP_SESSION',
                 'XDG_CURRENT_DESKTOP', 'XDG_SESSION_DESKTOP'):
        env.pop (name, None)
    runtime_dir = tempfile.mkdtemp (prefix = 'lwe-regress-run-')
    atexit.register (shutil.rmtree, runtime_dir, True)
    env.update ({
        'XDG_RUNTIME_DIR': runtime_dir,
        'XDG_SESSION_TYPE': 'x11',
        'LD_LIBRARY_PATH': ':'.join (filter (None, [str (args.binary.parent), os.environ.get ('LD_LIBRARY_PATH')])),
        'LD_PRELOAD': preload,
        'LWE_FIXED_TIMESTEP': str (args.step),
        'LWE_FIXED_CLOCK': str (args.clock),
        'TZ': 'UTC',
        # no audio server: whatever the machine is playing would otherwise feed the audio visualizers
        'PULSE_SERVER': 'unix:/nonexistent-pulse',
        'PIPEWIRE_REMOTE': '/nonexistent-pipewire',
    })
    if args.gpu:
        env['XDG_SESSION_TYPE'] = 'headless'
        if args.gpu != 'auto':
            env['LWE_HEADLESS_DEVICE'] = args.gpu
    else:
        env['DISPLAY'] = display.name
        # software rendering (llvmpipe) starts a thread per core in every instance, which just
        # thrashes when several renders run at once
        env.setdefault ('LP_NUM_THREADS', str (max (1, (os.cpu_count () or 1) // args.jobs)))

    status, code, seconds = render_once (args, env, folder, target / 'shot.png', target / 'log.txt', properties)

    # extra renders of the same build show what isn't reproducible (video textures play in real
    # time), compare ignores those pixels
    if status == 'ok':
        for repeat in range (2, args.repeat + 1):
            render_once (args, env, folder, target / f'shot{repeat}.png', target / f'log{repeat}.txt', properties)

    # animated textures have to keep moving: a second, later frame shows whether a gif or video
    # is still drawn and still playing (a hidden or frozen one leaves its area static)
    animated = sorted (animated_textures (folder))
    motion = None

    if status == 'ok' and animated and args.motion_gap > 0:
        later = target / 'shot_motion.png'
        args_later = argparse.Namespace (**{**vars (args), 'frame': args.frame + args.motion_gap})

        if render_once (args_later, env, folder, later, target / 'log_motion.txt', properties)[0] == 'ok':
            motion = changed_fraction (target / 'shot.png', later, 16)

    log = (target / 'log.txt').read_text (errors = 'replace')
    result = {
        'id': wallpaper_id,
        'title': title,
        'status': status,
        'exit': code,
        'seconds': round (seconds, 1),
        'video': bool (VIDEO_LINE.search (log)),
        'animated': animated,
        'motion': motion,
    }
    (target / 'result.json').write_text (json.dumps (result, ensure_ascii = False, indent = 1))

    return result


def cmd_run (args):
    if args.jobs is None:
        cores = os.cpu_count () or 2
        args.jobs = max (1, cores // 2 if args.gpu else min (8, cores // 4))

    args.binary = args.binary.resolve ()
    args.out = args.out.resolve ()
    args.out.mkdir (parents = True, exist_ok = True)

    variants = {}
    if args.variants.exists ():
        variants = {k: v for k, v in json.loads (args.variants.read_text ()).items () if not k.startswith ('_')}

    wallpapers = find_wallpapers (args.workshop, set (args.types.split (',')), set (args.ids or []), variants)

    if not wallpapers:
        sys.exit (f'no wallpapers found in {args.workshop}')

    preload = build_shims (args.out)
    displays = queue.Queue ()
    opened = []

    try:
        for _ in range (args.jobs):
            if args.gpu:
                displays.put (None)
                continue

            display = Display (opened[-1].number + 1 if opened else 90, args.width, args.height)
            opened.append (display)
            displays.put (display)

        def job (wallpaper):
            display = displays.get ()
            try:
                return render (args, preload, display, wallpaper)
            finally:
                displays.put (display)

        (args.out / 'run.json').write_text (json.dumps ({
            'binary': str (args.binary),
            'frame': args.frame,
            'step': args.step,
            'clock': args.clock,
            'size': [args.width, args.height],
            'repeat': args.repeat,
            'renderer': 'gpu' if args.gpu else 'llvmpipe',
            'started': time.strftime ('%Y-%m-%d %H:%M:%S'),
        }, indent = 1))

        done = 0

        with concurrent.futures.ThreadPoolExecutor (args.jobs) as pool:
            for result in pool.map (job, wallpapers):
                done += 1
                print (f'[{done}/{len (wallpapers)}] {result["id"]} {result["status"]} {result["seconds"]}s', flush = True)
    finally:
        for display in opened:
            display.close ()


def error_lines (log):
    if not log.exists ():
        return set ()

    lines = set ()

    for line in log.read_text (errors = 'replace').splitlines ():
        if ERROR_LINE.search (line) and not IGNORED_LINE.search (line):
            # numbers, pointers and temp paths change between runs, the message doesn't
            line = re.sub (r'0x[0-9a-fA-F]+', 'X', line)
            line = re.sub (r'\d+', 'N', line)
            lines.add (line.strip ()[:300])

    return lines


def strongest_difference (a, b):
    from PIL import ImageChops

    if a.size != b.size:
        b = b.resize (a.size)

    # per pixel, the largest channel difference
    channels = ImageChops.difference (a, b).split ()
    return ImageChops.lighter (ImageChops.lighter (channels[0], channels[1]), channels[2])


def changed_fraction (first, second, threshold):
    from PIL import Image

    a = Image.open (first).convert ('RGB')
    histogram = strongest_difference (a, Image.open (second).convert ('RGB')).histogram ()
    return sum (histogram[threshold + 1:]) / (a.size[0] * a.size[1])


def noise_mask (shots, threshold):
    """Pixels that already differ between repeated renders of the same build."""
    from PIL import Image, ImageChops

    images = [Image.open (shot).convert ('RGB') for shot in shots]
    mask = None

    for other in images[1:]:
        changed = strongest_difference (images[0], other).point (lambda v: 255 if v > threshold else 0)
        mask = changed if mask is None else ImageChops.lighter (mask, changed)

    return mask


def image_diff (base, new, threshold, heatmap, noise):
    from PIL import Image, ImageChops

    a = Image.open (base).convert ('RGB')
    b = Image.open (new).convert ('RGB')
    changed_mask = strongest_difference (a, b).point (lambda v: 255 if v > threshold else 0)
    total = a.size[0] * a.size[1]
    ignored = 0

    if noise is not None:
        noise = noise.resize (a.size)
        ignored = noise.histogram ()[255] / total
        changed_mask = ImageChops.subtract (changed_mask, noise)

    changed = changed_mask.histogram ()[255] / total

    if changed > 0 or ignored > 0:
        dim = a.convert ('L').convert ('RGB').point (lambda v: v // 3)
        if noise is not None:
            dim = Image.composite (Image.new ('RGB', a.size, (60, 60, 110)), dim, noise)
        Image.composite (Image.new ('RGB', a.size, (255, 40, 40)), dim, changed_mask).save (heatmap, quality = 85)

    return changed, ignored


def thumbnail (source, target, width = 640):
    from PIL import Image

    image = Image.open (source).convert ('RGB')
    image.thumbnail ((width, width))
    image.save (target, quality = 85)


def load_result (folder):
    try:
        return json.loads ((folder / 'result.json').read_text ())
    except (OSError, ValueError):
        return None


def renderer (folder):
    try:
        return json.loads ((folder / 'run.json').read_text ()).get ('renderer', 'llvmpipe')
    except (OSError, ValueError):
        return '?'


def cmd_compare (args):
    if renderer (args.base) != renderer (args.new):
        print (f'warning: base was rendered with {renderer (args.base)} and new with {renderer (args.new)}, '
               'expect small differences everywhere', file = sys.stderr)

    report = args.report.resolve ()
    shutil.rmtree (report, ignore_errors = True)
    (report / 'img').mkdir (parents = True)

    ids = sorted ({p.name for p in args.base.iterdir () if (p / 'result.json').exists ()}
                  | {p.name for p in args.new.iterdir () if (p / 'result.json').exists ()})
    rows = []

    for wallpaper_id in ids:
        base = load_result (args.base / wallpaper_id)
        new = load_result (args.new / wallpaper_id)
        row = {
            'id': wallpaper_id,
            'title': (new or base or {}).get ('title', ''),
            'base': base['status'] if base else 'missing',
            'new': new['status'] if new else 'missing',
            'added_errors': [],
            'fixed_errors': [],
            'changed': None,
            'verdict': 'same',
        }

        if base and new:
            base_errors = error_lines (args.base / wallpaper_id / 'log.txt')
            new_errors = error_lines (args.new / wallpaper_id / 'log.txt')
            row['added_errors'] = sorted (new_errors - base_errors)
            row['fixed_errors'] = sorted (base_errors - new_errors)

        base_shot = args.base / wallpaper_id / 'shot.png'
        new_shot = args.new / wallpaper_id / 'shot.png'

        row['video'] = bool ((base or {}).get ('video') or (new or {}).get ('video'))
        row['ignored'] = 0

        if base and new and base_shot.exists () and new_shot.exists ():
            heatmap = report / 'img' / f'{wallpaper_id}_diff.jpg'
            repeats = sorted ((args.base / wallpaper_id).glob ('shot[0-9]*.png'))
            repeats += sorted ((args.new / wallpaper_id).glob ('shot[0-9]*.png'))
            noise = None

            if repeats:
                noise = noise_mask ([base_shot] + [r for r in repeats if r.parent == base_shot.parent], args.threshold)
                new_repeats = [r for r in repeats if r.parent == new_shot.parent]
                if new_repeats:
                    from PIL import ImageChops
                    extra = noise_mask ([new_shot] + new_repeats, args.threshold)
                    noise = extra if noise is None else ImageChops.lighter (noise, extra.resize (noise.size))

            row['changed'], row['ignored'] = image_diff (base_shot, new_shot, args.threshold, heatmap, noise)

        row['animated'] = (new or base or {}).get ('animated', [])
        row['motion'] = [(base or {}).get ('motion'), (new or {}).get ('motion')]
        base_motion, new_motion = row['motion']
        # the second frame moved noticeably less than with the base build
        # runs made before the motion check have no "motion" key at all, None means the later render failed
        less_motion = ('motion' in (new or {}) and base_motion is not None and base_motion * 100 >= args.motion_min
                       and (new_motion or 0) < base_motion * 0.5)

        if row['new'] == 'missing':
            row['verdict'] = 'not run'
        elif row['base'] == 'missing':
            row['verdict'] = 'new wallpaper'
        elif row['new'] != 'ok' and row['base'] == 'ok':
            row['verdict'] = 'broken'
        elif row['new'] == 'ok' and row['base'] not in ('ok', 'missing'):
            row['verdict'] = 'fixed'
        elif row['added_errors']:
            row['verdict'] = 'new errors'
        elif less_motion:
            row['verdict'] = 'less motion'
        elif row['changed'] is not None and row['changed'] * 100 > (args.video_tolerance if row['video'] else args.tolerance):
            row['verdict'] = 'changed'
        elif row['fixed_errors']:
            row['verdict'] = 'fewer errors'

        if row['verdict'] not in ('same', 'not run'):
            for side, shot in (('base', base_shot), ('new', new_shot),
                               ('base_motion', args.base / wallpaper_id / 'shot_motion.png'),
                               ('new_motion', args.new / wallpaper_id / 'shot_motion.png')):
                if shot.exists ():
                    thumbnail (shot, report / 'img' / f'{wallpaper_id}_{side}.jpg')

        rows.append (row)

    order = {'broken': 0, 'new errors': 1, 'less motion': 2, 'changed': 3, 'fixed': 4, 'fewer errors': 5,
             'new wallpaper': 6, 'same': 7, 'not run': 8}
    rows.sort (key = lambda r: (order[r['verdict']], -(r['changed'] or 0)))

    (report / 'summary.json').write_text (json.dumps (rows, ensure_ascii = False, indent = 1))
    write_html (report, args, rows)

    counts = {}
    for row in rows:
        counts[row['verdict']] = counts.get (row['verdict'], 0) + 1

    print (', '.join (f'{count} {verdict}' for verdict, count in sorted (counts.items (), key = lambda c: order[c[0]])))

    for row in rows:
        if row['verdict'] not in ('same', 'not run'):
            changed = f' {row["changed"] * 100:.2f}% of pixels' if row['changed'] else ''
            print (f'  {row["verdict"]:12} {row["id"]} {row["title"][:50]}{changed}')

    print (f'report: {report / "index.html"}')

    regressed = any (r['verdict'] in ('broken', 'new errors', 'less motion', 'changed') for r in rows)
    sys.exit (1 if regressed else 0)


def write_html (report, args, rows):
    def esc (value):
        return html.escape (str (value))

    def run_info (folder):
        try:
            return json.loads ((folder / 'run.json').read_text ())
        except (OSError, ValueError):
            return {}

    base_info = run_info (args.base)
    new_info = run_info (args.new)
    parts = [f'''<!doctype html><html><head><meta charset="utf-8"><title>Wallpaper regressions</title>
<style>
body {{ font: 14px system-ui, sans-serif; margin: 24px; background: #16161a; color: #ddd; }}
a {{ color: #8ab4f8; }} h2 {{ margin-top: 40px; }}
table {{ border-collapse: collapse; }} td, th {{ padding: 4px 10px; border-bottom: 1px solid #333; text-align: left; }}
.broken, .new-errors {{ color: #ff6b6b; }} .changed {{ color: #ffb86b; }} .fixed, .fewer-errors {{ color: #7ee787; }}
.pair {{ display: flex; gap: 8px; flex-wrap: wrap; }} .pair figure {{ margin: 0; }} .pair img {{ max-width: 420px; display: block; }}
figcaption {{ color: #999; font-size: 12px; }} pre {{ background: #222; padding: 8px; overflow-x: auto; }}
</style></head><body>
<h1>Wallpaper regressions</h1>
<p>base: {esc (args.base)} ({esc (base_info.get ('binary', '?'))}, {esc (base_info.get ('started', ''))})<br>
new: {esc (args.new)} ({esc (new_info.get ('binary', '?'))}, {esc (new_info.get ('started', ''))})<br>
frame {esc (new_info.get ('frame', '?'))} at {esc (new_info.get ('step', '?'))}s per frame, pixel threshold {args.threshold},
changed above {args.tolerance}% of pixels ({args.video_tolerance}% with video textures)</p>
<p>Heatmaps: red is changed, blue is ignored because it already differs between repeated renders of one build
(<code>run --repeat 2</code>), grey is unchanged.</p>
<table><tr><th>verdict</th><th>id</th><th>title</th><th>base</th><th>new</th><th>pixels changed</th><th>ignored</th><th>motion base / new</th></tr>''']

    for row in rows:
        css = row['verdict'].replace (' ', '-')
        changed = f'{row["changed"] * 100:.2f}%' if row['changed'] is not None else '-'
        ignored = f'{row["ignored"] * 100:.1f}%' if row['ignored'] else ''
        title = esc (row['title']) + (' <small>(video texture)</small>' if row['video'] else '')
        detailed = row['verdict'] not in ('same', 'not run')
        link = f'<a href="#w{esc (row["id"])}">{esc (row["id"])}</a>' if detailed else esc (row['id'])
        motion = ' / '.join ('-' if m is None else f'{m * 100:.2f}%' for m in row['motion']) if row['animated'] else ''
        if row['animated']:
            title += f' <small>({", ".join (row["animated"])})</small>'

        parts.append (f'<tr><td class="{css}">{esc (row["verdict"])}</td><td>{link}</td><td>{title}</td>'
                      f'<td>{esc (row["base"])}</td><td>{esc (row["new"])}</td><td>{changed}</td><td>{ignored}</td><td>{motion}</td></tr>')

    parts.append ('</table>')

    for row in rows:
        if row['verdict'] in ('same', 'not run'):
            continue

        parts.append (f'<h2 id="w{esc (row["id"])}" class="{row["verdict"].replace (" ", "-")}">'
                      f'{esc (row["id"])} - {esc (row["title"])} ({esc (row["verdict"])})</h2><div class="pair">')

        for name, caption in (('base', 'base'), ('new', 'new'), ('diff', 'changed pixels'),
                              ('base_motion', 'base, a few frames later'), ('new_motion', 'new, a few frames later')):
            image = report / 'img' / f'{row["id"]}_{name}.jpg'
            if image.exists ():
                parts.append (f'<figure><img src="img/{image.name}"><figcaption>{caption}</figcaption></figure>')

        parts.append ('</div>')

        if row['added_errors']:
            parts.append ('<p>new in the log:</p><pre>' + esc ('\n'.join (row['added_errors'])) + '</pre>')
        if row['fixed_errors']:
            parts.append ('<p>gone from the log:</p><pre>' + esc ('\n'.join (row['fixed_errors'])) + '</pre>')

    parts.append ('</body></html>')
    (report / 'index.html').write_text ('\n'.join (parts))


def main ():
    parser = argparse.ArgumentParser (description = __doc__, formatter_class = argparse.RawDescriptionHelpFormatter)
    commands = parser.add_subparsers (dest = 'command', required = True)

    run = commands.add_parser ('run', help = 'render every wallpaper with one build')
    run.add_argument ('binary', type = Path, help = 'the linux-wallpaperengine executable to test')
    run.add_argument ('out', type = Path, help = 'folder for the screenshots and logs')
    run.add_argument ('--workshop', type = Path, default = env_path ('LWE_WORKSHOP_DIR', DEFAULT_WORKSHOP))
    run.add_argument ('--assets', type = Path, default = env_path ('LWE_ASSETS_DIR', DEFAULT_ASSETS))
    run.add_argument ('--ids', nargs = '*', help = 'only these workshop ids')
    run.add_argument ('--variants', type = Path, default = HERE / 'variants.json',
                      help = 'extra renders with user properties set (default: variants.json next to this script)')
    run.add_argument ('--types', default = 'scene', help = 'project types to include, comma separated (default: scene)')
    run.add_argument ('--jobs', type = int, help = 'renders at once (default: cores/4 up to 8, cores/2 with --gpu)')
    run.add_argument ('--gpu', nargs = '?', const = 'auto', metavar = 'RENDER_NODE',
                      help = 'render on the GPU without Xvfb, optionally on this /dev/dri/renderD* node')
    run.add_argument ('--frame', type = int, default = 30, help = 'frame to screenshot (default: 30)')
    run.add_argument ('--step', type = float, default = 0.1, help = 'animation seconds per frame (default: 0.1)')
    run.add_argument ('--clock', type = int, default = 1767268800, help = 'pinned wall clock, unix seconds')
    run.add_argument ('--width', type = int, default = 1280)
    run.add_argument ('--height', type = int, default = 720)
    run.add_argument ('--repeat', type = int, default = 1,
                      help = 'renders per wallpaper; use 2+ for the base run so compare can ignore noise')
    run.add_argument ('--motion-gap', type = int, default = 5,
                      help = 'frames between the two renders of wallpapers with gif/video textures (0 disables)')
    run.add_argument ('--timeout', type = float, default = 300, help = 'seconds per wallpaper')
    run.set_defaults (func = cmd_run)

    compare = commands.add_parser ('compare', help = 'compare two runs and write a report')
    compare.add_argument ('base', type = Path)
    compare.add_argument ('new', type = Path)
    compare.add_argument ('--report', type = Path, default = Path ('regression-report'))
    compare.add_argument ('--threshold', type = int, default = 16, help = 'per-channel difference that counts as changed')
    compare.add_argument ('--video-tolerance', type = float, default = 5,
                          help = 'percent of changed pixels allowed with video textures, the motion check covers those')
    compare.add_argument ('--motion-min', type = float, default = 0.05,
                          help = 'percent of pixels that must move in the base run for the motion check to apply')
    compare.add_argument ('--tolerance', type = float, default = 0.1, help = 'percent of changed pixels allowed')
    compare.set_defaults (func = cmd_compare)

    args = parser.parse_args ()
    args.func (args)


if __name__ == '__main__':
    main ()