|
|
@@ -0,0 +1,681 @@
|
|
|
+#!/usr/bin/env python3
|
|
|
+"""Wallpaper regression runs: render every installed scene wallpaper headlessly with a build,
|
|
|
+then compare two runs (usually the last good build against the one being tested).
|
|
|
+
|
|
|
+ tools/regression/regress.py run /path/to/old/linux-wallpaperengine runs/base --repeat 2
|
|
|
+ tools/regression/regress.py run build/output/linux-wallpaperengine runs/new
|
|
|
+ tools/regression/regress.py compare runs/base runs/new --report runs/report
|
|
|
+
|
|
|
+Runs are reproducible: animation time advances by a fixed step per frame (LWE_FIXED_TIMESTEP),
|
|
|
+particles are seeded per object, the wall clock is pinned (fixed_clock.c) and audio input is
|
|
|
+off. Each worker gets its own Xvfb display, nothing touches the real screen.
|
|
|
+
|
|
|
+compare looks at four things per wallpaper: whether it still renders (crash/timeout), error
|
|
|
+lines that appear or disappear in the log, how much of the screenshot changed, and whether gif
|
|
|
+and video textures still move (wallpapers shipping them get a second render a few frames later,
|
|
|
+a hidden or frozen one shows up as less motion). Video textures play in real time and never
|
|
|
+render the same twice, --repeat 2 on the base run marks those pixels so compare ignores them.
|
|
|
+It writes an HTML report with side-by-sides of everything flagged and exits 1 if anything
|
|
|
+regressed. Both builds need LWE_FIXED_TIMESTEP support, older ones render every run differently.
|
|
|
+
|
|
|
+Needs Xvfb, gcc and Pillow (tools/requirements.txt).
|
|
|
+"""
|
|
|
+
|
|
|
+import argparse
|
|
|
+import concurrent.futures
|
|
|
+import html
|
|
|
+import json
|
|
|
+import os
|
|
|
+import queue
|
|
|
+import re
|
|
|
+import shutil
|
|
|
+import signal
|
|
|
+import subprocess
|
|
|
+import sys
|
|
|
+import time
|
|
|
+from pathlib import Path
|
|
|
+
|
|
|
+HERE = Path (__file__).resolve ().parent
|
|
|
+TOOLS = HERE.parent
|
|
|
+
|
|
|
+DEFAULT_WORKSHOP = Path.home () / '.local/share/Steam/steamapps/workshop/content/431960'
|
|
|
+DEFAULT_ASSETS = Path.home () / '.local/share/Steam/steamapps/common/wallpaper_engine/assets'
|
|
|
+
|
|
|
+# lines worth tracking between builds; everything else in the log is chatter
|
|
|
+ERROR_LINE = re.compile (
|
|
|
+ r'error|exception|failed|cannot|can\'t|unsupported|not supported|typeerror|referenceerror|'
|
|
|
+ r'syntaxerror|abort|segmentation|falling back',
|
|
|
+ re.IGNORECASE
|
|
|
+)
|
|
|
+# driver/X noise that shows up on some machines and says nothing about the engine
|
|
|
+IGNORED_LINE = re.compile (r'amdgpu|_XSERVTrans|MESA-LOADER|libEGL warning|ALSA lib|Puppet draw result')
|
|
|
+# mpv announcing a video stream, those textures play in real time and never render the same twice
|
|
|
+VIDEO_LINE = re.compile (r'^\W*Video\s+--vid=', re.MULTILINE)
|
|
|
+
|
|
|
+
|
|
|
+def env_path (name, fallback):
|
|
|
+ value = os.environ.get (name)
|
|
|
+ return Path (value) if value else fallback
|
|
|
+
|
|
|
+
|
|
|
+def read_project (folder):
|
|
|
+ try:
|
|
|
+ with open (folder / 'project.json', encoding = 'utf-8-sig') as f:
|
|
|
+ return json.load (f)
|
|
|
+ except (OSError, ValueError):
|
|
|
+ return None
|
|
|
+
|
|
|
+
|
|
|
+def read_sized (f):
|
|
|
+ size = int.from_bytes (f.read (4), 'little')
|
|
|
+ return f.read (size).decode ('utf-8', 'replace')
|
|
|
+
|
|
|
+
|
|
|
+def texture_kinds (header):
|
|
|
+ """'gif' for animated (sprite sheet) textures, 'video' for mp4 ones, from a .tex header."""
|
|
|
+ if header[:8] != b'TEXV0005' or len (header) < 26:
|
|
|
+ return set ()
|
|
|
+
|
|
|
+ flags = int.from_bytes (header[22:26], 'little')
|
|
|
+ return ({'gif'} if flags & 4 else set ()) | ({'video'} if flags & 32 else set ())
|
|
|
+
|
|
|
+
|
|
|
+def animated_textures (folder):
|
|
|
+ """What kind of animated textures a wallpaper ships, loose or inside its .pkg files."""
|
|
|
+ kinds = set ()
|
|
|
+
|
|
|
+ for path in folder.rglob ('*'):
|
|
|
+ if path.suffix == '.mp4':
|
|
|
+ kinds.add ('video')
|
|
|
+ elif path.suffix == '.tex':
|
|
|
+ with open (path, 'rb') as f:
|
|
|
+ kinds |= texture_kinds (f.read (64))
|
|
|
+ elif path.suffix == '.pkg':
|
|
|
+ try:
|
|
|
+ with open (path, 'rb') as f:
|
|
|
+ read_sized (f)
|
|
|
+ entries = []
|
|
|
+ for _ in range (int.from_bytes (f.read (4), 'little')):
|
|
|
+ name = read_sized (f)
|
|
|
+ offset = int.from_bytes (f.read (4), 'little')
|
|
|
+ f.read (4)
|
|
|
+ entries.append ((name, offset))
|
|
|
+ base = f.tell ()
|
|
|
+ for name, offset in entries:
|
|
|
+ if name.endswith ('.mp4'):
|
|
|
+ kinds.add ('video')
|
|
|
+ elif name.endswith ('.tex'):
|
|
|
+ f.seek (base + offset)
|
|
|
+ kinds |= texture_kinds (f.read (64))
|
|
|
+ except (OSError, ValueError):
|
|
|
+ pass
|
|
|
+
|
|
|
+ return kinds
|
|
|
+
|
|
|
+
|
|
|
+def find_wallpapers (workshop, types, ids, variants):
|
|
|
+ """(name, folder, title, properties) per render; variants add renders of the same wallpaper
|
|
|
+ with user properties set, named <id>@<variant>."""
|
|
|
+ found = []
|
|
|
+
|
|
|
+ for folder in sorted (workshop.iterdir ()):
|
|
|
+ if not folder.is_dir () or (ids and folder.name not in ids):
|
|
|
+ continue
|
|
|
+
|
|
|
+ project = read_project (folder)
|
|
|
+ kind = str ((project or {}).get ('type', '')).lower ()
|
|
|
+
|
|
|
+ if kind in types:
|
|
|
+ title = (project or {}).get ('title', '')
|
|
|
+ found.append ((folder.name, folder, title, []))
|
|
|
+
|
|
|
+ for variant, properties in variants.get (folder.name, {}).items ():
|
|
|
+ found.append ((f'{folder.name}@{variant}', folder, f'{title} [{variant}]', properties))
|
|
|
+
|
|
|
+ return found
|
|
|
+
|
|
|
+
|
|
|
+def build_shims (out):
|
|
|
+ shims = out / '.shims'
|
|
|
+ shims.mkdir (parents = True, exist_ok = True)
|
|
|
+ built = []
|
|
|
+
|
|
|
+ for source in (TOOLS / 'dbus_noop_shim.c', HERE / 'fixed_clock.c'):
|
|
|
+ target = shims / (source.stem + '.so')
|
|
|
+
|
|
|
+ if not target.exists () or source.stat ().st_mtime > target.stat ().st_mtime:
|
|
|
+ subprocess.run (['gcc', '-shared', '-fPIC', '-O2', '-o', str (target), str (source), '-ldl'], check = True)
|
|
|
+
|
|
|
+ built.append (str (target))
|
|
|
+
|
|
|
+ return ':'.join (built)
|
|
|
+
|
|
|
+
|
|
|
+def x_sockets ():
|
|
|
+ """Display numbers with a listening X socket, abstract ones included (some sandboxes can't
|
|
|
+ write /tmp/.X11-unix, Xvfb then only has the abstract socket)."""
|
|
|
+ numbers = set ()
|
|
|
+
|
|
|
+ with open ('/proc/net/unix') as f:
|
|
|
+ for line in f:
|
|
|
+ match = re.search (r'/tmp/\.X11-unix/X(\d+)$', line.strip ())
|
|
|
+ if match:
|
|
|
+ numbers.add (int (match.group (1)))
|
|
|
+
|
|
|
+ return numbers
|
|
|
+
|
|
|
+
|
|
|
+class Display:
|
|
|
+ """One Xvfb server per worker so renders can run side by side."""
|
|
|
+
|
|
|
+ def __init__ (self, first, width, height):
|
|
|
+ number = first
|
|
|
+
|
|
|
+ while True:
|
|
|
+ while number in x_sockets () or Path (f'/tmp/.X11-unix/X{number}').exists ():
|
|
|
+ number += 1
|
|
|
+
|
|
|
+ self.name = f':{number}'
|
|
|
+ self.number = number
|
|
|
+ self.process = subprocess.Popen (
|
|
|
+ ['Xvfb', self.name, '-screen', '0', f'{width}x{height}x24', '-nolisten', 'tcp'],
|
|
|
+ stdout = subprocess.DEVNULL, stderr = subprocess.DEVNULL
|
|
|
+ )
|
|
|
+
|
|
|
+ for _ in range (100):
|
|
|
+ if number in x_sockets ():
|
|
|
+ return
|
|
|
+ if self.process.poll () is not None:
|
|
|
+ break
|
|
|
+ time.sleep (0.05)
|
|
|
+
|
|
|
+ self.close ()
|
|
|
+ number += 1
|
|
|
+
|
|
|
+ if number > first + 50:
|
|
|
+ raise RuntimeError ('could not start Xvfb')
|
|
|
+
|
|
|
+ def close (self):
|
|
|
+ self.process.terminate ()
|
|
|
+ try:
|
|
|
+ self.process.wait (5)
|
|
|
+ except subprocess.TimeoutExpired:
|
|
|
+ self.process.kill ()
|
|
|
+
|
|
|
+
|
|
|
+def stop (process):
|
|
|
+ if process.poll () is not None:
|
|
|
+ return
|
|
|
+
|
|
|
+ os.killpg (process.pid, signal.SIGTERM)
|
|
|
+ try:
|
|
|
+ process.wait (5)
|
|
|
+ except subprocess.TimeoutExpired:
|
|
|
+ os.killpg (process.pid, signal.SIGKILL)
|
|
|
+ process.wait ()
|
|
|
+
|
|
|
+
|
|
|
+def render_once (args, env, folder, shot, log_path, properties):
|
|
|
+ command = [
|
|
|
+ str (args.binary), '--window', f'0x0x{args.width}x{args.height}', '--fps', '1000',
|
|
|
+ '--silent', '--no-audio-processing', '--screenshot', str (shot), '--screenshot-delay', str (args.frame),
|
|
|
+ '--assets-dir', str (args.assets),
|
|
|
+ ]
|
|
|
+
|
|
|
+ for value in properties:
|
|
|
+ command += ['--set-property', value]
|
|
|
+
|
|
|
+ command.append (str (folder))
|
|
|
+
|
|
|
+ started = time.monotonic ()
|
|
|
+ status = 'timeout'
|
|
|
+
|
|
|
+ with open (log_path, 'wb') as log:
|
|
|
+ process = subprocess.Popen (
|
|
|
+ command, env = env, stdin = subprocess.DEVNULL, stdout = log, stderr = subprocess.STDOUT, start_new_session = True
|
|
|
+ )
|
|
|
+ last_size = -1
|
|
|
+
|
|
|
+ while time.monotonic () - started < args.timeout:
|
|
|
+ code = process.poll ()
|
|
|
+ size = shot.stat ().st_size if shot.exists () else 0
|
|
|
+
|
|
|
+ # the engine keeps running after the screenshot, stop it once the file is complete
|
|
|
+ if size > 0 and size == last_size:
|
|
|
+ status = 'ok'
|
|
|
+ break
|
|
|
+
|
|
|
+ if code is not None:
|
|
|
+ status = 'ok' if size > 0 else ('crash' if code < 0 else 'noshot')
|
|
|
+ break
|
|
|
+
|
|
|
+ last_size = size
|
|
|
+ time.sleep (0.25)
|
|
|
+
|
|
|
+ stop (process)
|
|
|
+
|
|
|
+ return status, process.returncode, time.monotonic () - started
|
|
|
+
|
|
|
+
|
|
|
+def render (args, preload, display, wallpaper):
|
|
|
+ wallpaper_id, folder, title, properties = wallpaper
|
|
|
+ target = args.out / wallpaper_id
|
|
|
+ shutil.rmtree (target, ignore_errors = True)
|
|
|
+ target.mkdir (parents = True)
|
|
|
+
|
|
|
+ env = dict (os.environ)
|
|
|
+ env.pop ('WAYLAND_DISPLAY', None)
|
|
|
+ env.pop ('WAYLAND_SOCKET', None)
|
|
|
+ env.update ({
|
|
|
+ 'DISPLAY': display.name,
|
|
|
+ 'XDG_SESSION_TYPE': 'x11',
|
|
|
+ 'LD_LIBRARY_PATH': ':'.join (filter (None, [str (args.binary.parent), os.environ.get ('LD_LIBRARY_PATH')])),
|
|
|
+ 'LD_PRELOAD': preload,
|
|
|
+ 'LWE_FIXED_TIMESTEP': str (args.step),
|
|
|
+ 'LWE_FIXED_CLOCK': str (args.clock),
|
|
|
+ 'TZ': 'UTC',
|
|
|
+ })
|
|
|
+ # software rendering (llvmpipe) starts a thread per core in every instance, which just
|
|
|
+ # thrashes when several renders run at once
|
|
|
+ env.setdefault ('LP_NUM_THREADS', str (max (1, (os.cpu_count () or 1) // args.jobs)))
|
|
|
+
|
|
|
+ status, code, seconds = render_once (args, env, folder, target / 'shot.png', target / 'log.txt', properties)
|
|
|
+
|
|
|
+ # extra renders of the same build show what isn't reproducible (video textures play in real
|
|
|
+ # time), compare ignores those pixels
|
|
|
+ if status == 'ok':
|
|
|
+ for repeat in range (2, args.repeat + 1):
|
|
|
+ render_once (args, env, folder, target / f'shot{repeat}.png', target / f'log{repeat}.txt', properties)
|
|
|
+
|
|
|
+ # animated textures have to keep moving: a second, later frame shows whether a gif or video
|
|
|
+ # is still drawn and still playing (a hidden or frozen one leaves its area static)
|
|
|
+ animated = sorted (animated_textures (folder))
|
|
|
+ motion = None
|
|
|
+
|
|
|
+ if status == 'ok' and animated and args.motion_gap > 0:
|
|
|
+ later = target / 'shot_motion.png'
|
|
|
+ args_later = argparse.Namespace (**{**vars (args), 'frame': args.frame + args.motion_gap})
|
|
|
+
|
|
|
+ if render_once (args_later, env, folder, later, target / 'log_motion.txt', properties)[0] == 'ok':
|
|
|
+ motion = changed_fraction (target / 'shot.png', later, 16)
|
|
|
+
|
|
|
+ log = (target / 'log.txt').read_text (errors = 'replace')
|
|
|
+ result = {
|
|
|
+ 'id': wallpaper_id,
|
|
|
+ 'title': title,
|
|
|
+ 'status': status,
|
|
|
+ 'exit': code,
|
|
|
+ 'seconds': round (seconds, 1),
|
|
|
+ 'video': bool (VIDEO_LINE.search (log)),
|
|
|
+ 'animated': animated,
|
|
|
+ 'motion': motion,
|
|
|
+ }
|
|
|
+ (target / 'result.json').write_text (json.dumps (result, ensure_ascii = False, indent = 1))
|
|
|
+
|
|
|
+ return result
|
|
|
+
|
|
|
+
|
|
|
+def cmd_run (args):
|
|
|
+ args.binary = args.binary.resolve ()
|
|
|
+ args.out = args.out.resolve ()
|
|
|
+ args.out.mkdir (parents = True, exist_ok = True)
|
|
|
+
|
|
|
+ variants = {}
|
|
|
+ if args.variants.exists ():
|
|
|
+ variants = {k: v for k, v in json.loads (args.variants.read_text ()).items () if not k.startswith ('_')}
|
|
|
+
|
|
|
+ wallpapers = find_wallpapers (args.workshop, set (args.types.split (',')), set (args.ids or []), variants)
|
|
|
+
|
|
|
+ if not wallpapers:
|
|
|
+ sys.exit (f'no wallpapers found in {args.workshop}')
|
|
|
+
|
|
|
+ preload = build_shims (args.out)
|
|
|
+ displays = queue.Queue ()
|
|
|
+ opened = []
|
|
|
+
|
|
|
+ try:
|
|
|
+ for _ in range (args.jobs):
|
|
|
+ display = Display (opened[-1].number + 1 if opened else 90, args.width, args.height)
|
|
|
+ opened.append (display)
|
|
|
+ displays.put (display)
|
|
|
+
|
|
|
+ def job (wallpaper):
|
|
|
+ display = displays.get ()
|
|
|
+ try:
|
|
|
+ return render (args, preload, display, wallpaper)
|
|
|
+ finally:
|
|
|
+ displays.put (display)
|
|
|
+
|
|
|
+ (args.out / 'run.json').write_text (json.dumps ({
|
|
|
+ 'binary': str (args.binary),
|
|
|
+ 'frame': args.frame,
|
|
|
+ 'step': args.step,
|
|
|
+ 'clock': args.clock,
|
|
|
+ 'size': [args.width, args.height],
|
|
|
+ 'repeat': args.repeat,
|
|
|
+ 'started': time.strftime ('%Y-%m-%d %H:%M:%S'),
|
|
|
+ }, indent = 1))
|
|
|
+
|
|
|
+ done = 0
|
|
|
+
|
|
|
+ with concurrent.futures.ThreadPoolExecutor (args.jobs) as pool:
|
|
|
+ for result in pool.map (job, wallpapers):
|
|
|
+ done += 1
|
|
|
+ print (f'[{done}/{len (wallpapers)}] {result["id"]} {result["status"]} {result["seconds"]}s', flush = True)
|
|
|
+ finally:
|
|
|
+ for display in opened:
|
|
|
+ display.close ()
|
|
|
+
|
|
|
+
|
|
|
+def error_lines (log):
|
|
|
+ if not log.exists ():
|
|
|
+ return set ()
|
|
|
+
|
|
|
+ lines = set ()
|
|
|
+
|
|
|
+ for line in log.read_text (errors = 'replace').splitlines ():
|
|
|
+ if ERROR_LINE.search (line) and not IGNORED_LINE.search (line):
|
|
|
+ # numbers, pointers and temp paths change between runs, the message doesn't
|
|
|
+ line = re.sub (r'0x[0-9a-fA-F]+', 'X', line)
|
|
|
+ line = re.sub (r'\d+', 'N', line)
|
|
|
+ lines.add (line.strip ()[:300])
|
|
|
+
|
|
|
+ return lines
|
|
|
+
|
|
|
+
|
|
|
+def strongest_difference (a, b):
|
|
|
+ from PIL import ImageChops
|
|
|
+
|
|
|
+ if a.size != b.size:
|
|
|
+ b = b.resize (a.size)
|
|
|
+
|
|
|
+ # per pixel, the largest channel difference
|
|
|
+ channels = ImageChops.difference (a, b).split ()
|
|
|
+ return ImageChops.lighter (ImageChops.lighter (channels[0], channels[1]), channels[2])
|
|
|
+
|
|
|
+
|
|
|
+def changed_fraction (first, second, threshold):
|
|
|
+ from PIL import Image
|
|
|
+
|
|
|
+ a = Image.open (first).convert ('RGB')
|
|
|
+ histogram = strongest_difference (a, Image.open (second).convert ('RGB')).histogram ()
|
|
|
+ return sum (histogram[threshold + 1:]) / (a.size[0] * a.size[1])
|
|
|
+
|
|
|
+
|
|
|
+def noise_mask (shots, threshold):
|
|
|
+ """Pixels that already differ between repeated renders of the same build."""
|
|
|
+ from PIL import Image, ImageChops
|
|
|
+
|
|
|
+ images = [Image.open (shot).convert ('RGB') for shot in shots]
|
|
|
+ mask = None
|
|
|
+
|
|
|
+ for other in images[1:]:
|
|
|
+ changed = strongest_difference (images[0], other).point (lambda v: 255 if v > threshold else 0)
|
|
|
+ mask = changed if mask is None else ImageChops.lighter (mask, changed)
|
|
|
+
|
|
|
+ return mask
|
|
|
+
|
|
|
+
|
|
|
+def image_diff (base, new, threshold, heatmap, noise):
|
|
|
+ from PIL import Image, ImageChops
|
|
|
+
|
|
|
+ a = Image.open (base).convert ('RGB')
|
|
|
+ b = Image.open (new).convert ('RGB')
|
|
|
+ changed_mask = strongest_difference (a, b).point (lambda v: 255 if v > threshold else 0)
|
|
|
+ total = a.size[0] * a.size[1]
|
|
|
+ ignored = 0
|
|
|
+
|
|
|
+ if noise is not None:
|
|
|
+ noise = noise.resize (a.size)
|
|
|
+ ignored = noise.histogram ()[255] / total
|
|
|
+ changed_mask = ImageChops.subtract (changed_mask, noise)
|
|
|
+
|
|
|
+ changed = changed_mask.histogram ()[255] / total
|
|
|
+
|
|
|
+ if changed > 0 or ignored > 0:
|
|
|
+ dim = a.convert ('L').convert ('RGB').point (lambda v: v // 3)
|
|
|
+ if noise is not None:
|
|
|
+ dim = Image.composite (Image.new ('RGB', a.size, (60, 60, 110)), dim, noise)
|
|
|
+ Image.composite (Image.new ('RGB', a.size, (255, 40, 40)), dim, changed_mask).save (heatmap, quality = 85)
|
|
|
+
|
|
|
+ return changed, ignored
|
|
|
+
|
|
|
+
|
|
|
+def thumbnail (source, target, width = 640):
|
|
|
+ from PIL import Image
|
|
|
+
|
|
|
+ image = Image.open (source).convert ('RGB')
|
|
|
+ image.thumbnail ((width, width))
|
|
|
+ image.save (target, quality = 85)
|
|
|
+
|
|
|
+
|
|
|
+def load_result (folder):
|
|
|
+ try:
|
|
|
+ return json.loads ((folder / 'result.json').read_text ())
|
|
|
+ except (OSError, ValueError):
|
|
|
+ return None
|
|
|
+
|
|
|
+
|
|
|
+def cmd_compare (args):
|
|
|
+ report = args.report.resolve ()
|
|
|
+ shutil.rmtree (report, ignore_errors = True)
|
|
|
+ (report / 'img').mkdir (parents = True)
|
|
|
+
|
|
|
+ ids = sorted ({p.name for p in args.base.iterdir () if (p / 'result.json').exists ()}
|
|
|
+ | {p.name for p in args.new.iterdir () if (p / 'result.json').exists ()})
|
|
|
+ rows = []
|
|
|
+
|
|
|
+ for wallpaper_id in ids:
|
|
|
+ base = load_result (args.base / wallpaper_id)
|
|
|
+ new = load_result (args.new / wallpaper_id)
|
|
|
+ row = {
|
|
|
+ 'id': wallpaper_id,
|
|
|
+ 'title': (new or base or {}).get ('title', ''),
|
|
|
+ 'base': base['status'] if base else 'missing',
|
|
|
+ 'new': new['status'] if new else 'missing',
|
|
|
+ 'added_errors': [],
|
|
|
+ 'fixed_errors': [],
|
|
|
+ 'changed': None,
|
|
|
+ 'verdict': 'same',
|
|
|
+ }
|
|
|
+
|
|
|
+ if base and new:
|
|
|
+ base_errors = error_lines (args.base / wallpaper_id / 'log.txt')
|
|
|
+ new_errors = error_lines (args.new / wallpaper_id / 'log.txt')
|
|
|
+ row['added_errors'] = sorted (new_errors - base_errors)
|
|
|
+ row['fixed_errors'] = sorted (base_errors - new_errors)
|
|
|
+
|
|
|
+ base_shot = args.base / wallpaper_id / 'shot.png'
|
|
|
+ new_shot = args.new / wallpaper_id / 'shot.png'
|
|
|
+
|
|
|
+ row['video'] = bool ((base or {}).get ('video') or (new or {}).get ('video'))
|
|
|
+ row['ignored'] = 0
|
|
|
+
|
|
|
+ if base and new and base_shot.exists () and new_shot.exists ():
|
|
|
+ heatmap = report / 'img' / f'{wallpaper_id}_diff.jpg'
|
|
|
+ repeats = sorted ((args.base / wallpaper_id).glob ('shot[0-9]*.png'))
|
|
|
+ repeats += sorted ((args.new / wallpaper_id).glob ('shot[0-9]*.png'))
|
|
|
+ noise = None
|
|
|
+
|
|
|
+ if repeats:
|
|
|
+ noise = noise_mask ([base_shot] + [r for r in repeats if r.parent == base_shot.parent], args.threshold)
|
|
|
+ new_repeats = [r for r in repeats if r.parent == new_shot.parent]
|
|
|
+ if new_repeats:
|
|
|
+ from PIL import ImageChops
|
|
|
+ extra = noise_mask ([new_shot] + new_repeats, args.threshold)
|
|
|
+ noise = extra if noise is None else ImageChops.lighter (noise, extra.resize (noise.size))
|
|
|
+
|
|
|
+ row['changed'], row['ignored'] = image_diff (base_shot, new_shot, args.threshold, heatmap, noise)
|
|
|
+
|
|
|
+ row['animated'] = (new or base or {}).get ('animated', [])
|
|
|
+ row['motion'] = [(base or {}).get ('motion'), (new or {}).get ('motion')]
|
|
|
+ base_motion, new_motion = row['motion']
|
|
|
+ # the second frame moved noticeably less than with the base build
|
|
|
+ # runs made before the motion check have no "motion" key at all, None means the later render failed
|
|
|
+ less_motion = ('motion' in (new or {}) and base_motion is not None and base_motion * 100 >= args.motion_min
|
|
|
+ and (new_motion or 0) < base_motion * 0.5)
|
|
|
+
|
|
|
+ if row['new'] == 'missing':
|
|
|
+ row['verdict'] = 'not run'
|
|
|
+ elif row['base'] == 'missing':
|
|
|
+ row['verdict'] = 'new wallpaper'
|
|
|
+ elif row['new'] != 'ok' and row['base'] == 'ok':
|
|
|
+ row['verdict'] = 'broken'
|
|
|
+ elif row['new'] == 'ok' and row['base'] not in ('ok', 'missing'):
|
|
|
+ row['verdict'] = 'fixed'
|
|
|
+ elif row['added_errors']:
|
|
|
+ row['verdict'] = 'new errors'
|
|
|
+ elif less_motion:
|
|
|
+ row['verdict'] = 'less motion'
|
|
|
+ elif row['changed'] is not None and row['changed'] * 100 > (args.video_tolerance if row['video'] else args.tolerance):
|
|
|
+ row['verdict'] = 'changed'
|
|
|
+ elif row['fixed_errors']:
|
|
|
+ row['verdict'] = 'fewer errors'
|
|
|
+
|
|
|
+ if row['verdict'] not in ('same', 'not run'):
|
|
|
+ for side, shot in (('base', base_shot), ('new', new_shot),
|
|
|
+ ('base_motion', args.base / wallpaper_id / 'shot_motion.png'),
|
|
|
+ ('new_motion', args.new / wallpaper_id / 'shot_motion.png')):
|
|
|
+ if shot.exists ():
|
|
|
+ thumbnail (shot, report / 'img' / f'{wallpaper_id}_{side}.jpg')
|
|
|
+
|
|
|
+ rows.append (row)
|
|
|
+
|
|
|
+ order = {'broken': 0, 'new errors': 1, 'less motion': 2, 'changed': 3, 'fixed': 4, 'fewer errors': 5,
|
|
|
+ 'new wallpaper': 6, 'same': 7, 'not run': 8}
|
|
|
+ rows.sort (key = lambda r: (order[r['verdict']], -(r['changed'] or 0)))
|
|
|
+
|
|
|
+ (report / 'summary.json').write_text (json.dumps (rows, ensure_ascii = False, indent = 1))
|
|
|
+ write_html (report, args, rows)
|
|
|
+
|
|
|
+ counts = {}
|
|
|
+ for row in rows:
|
|
|
+ counts[row['verdict']] = counts.get (row['verdict'], 0) + 1
|
|
|
+
|
|
|
+ print (', '.join (f'{count} {verdict}' for verdict, count in sorted (counts.items (), key = lambda c: order[c[0]])))
|
|
|
+
|
|
|
+ for row in rows:
|
|
|
+ if row['verdict'] not in ('same', 'not run'):
|
|
|
+ changed = f' {row["changed"] * 100:.2f}% of pixels' if row['changed'] else ''
|
|
|
+ print (f' {row["verdict"]:12} {row["id"]} {row["title"][:50]}{changed}')
|
|
|
+
|
|
|
+ print (f'report: {report / "index.html"}')
|
|
|
+
|
|
|
+ regressed = any (r['verdict'] in ('broken', 'new errors', 'less motion', 'changed') for r in rows)
|
|
|
+ sys.exit (1 if regressed else 0)
|
|
|
+
|
|
|
+
|
|
|
+def write_html (report, args, rows):
|
|
|
+ def esc (value):
|
|
|
+ return html.escape (str (value))
|
|
|
+
|
|
|
+ def run_info (folder):
|
|
|
+ try:
|
|
|
+ return json.loads ((folder / 'run.json').read_text ())
|
|
|
+ except (OSError, ValueError):
|
|
|
+ return {}
|
|
|
+
|
|
|
+ base_info = run_info (args.base)
|
|
|
+ new_info = run_info (args.new)
|
|
|
+ parts = [f'''<!doctype html><html><head><meta charset="utf-8"><title>Wallpaper regressions</title>
|
|
|
+<style>
|
|
|
+body {{ font: 14px system-ui, sans-serif; margin: 24px; background: #16161a; color: #ddd; }}
|
|
|
+a {{ color: #8ab4f8; }} h2 {{ margin-top: 40px; }}
|
|
|
+table {{ border-collapse: collapse; }} td, th {{ padding: 4px 10px; border-bottom: 1px solid #333; text-align: left; }}
|
|
|
+.broken, .new-errors {{ color: #ff6b6b; }} .changed {{ color: #ffb86b; }} .fixed, .fewer-errors {{ color: #7ee787; }}
|
|
|
+.pair {{ display: flex; gap: 8px; flex-wrap: wrap; }} .pair figure {{ margin: 0; }} .pair img {{ max-width: 420px; display: block; }}
|
|
|
+figcaption {{ color: #999; font-size: 12px; }} pre {{ background: #222; padding: 8px; overflow-x: auto; }}
|
|
|
+</style></head><body>
|
|
|
+<h1>Wallpaper regressions</h1>
|
|
|
+<p>base: {esc (args.base)} ({esc (base_info.get ('binary', '?'))}, {esc (base_info.get ('started', ''))})<br>
|
|
|
+new: {esc (args.new)} ({esc (new_info.get ('binary', '?'))}, {esc (new_info.get ('started', ''))})<br>
|
|
|
+frame {esc (new_info.get ('frame', '?'))} at {esc (new_info.get ('step', '?'))}s per frame, pixel threshold {args.threshold},
|
|
|
+changed above {args.tolerance}% of pixels ({args.video_tolerance}% with video textures)</p>
|
|
|
+<p>Heatmaps: red is changed, blue is ignored because it already differs between repeated renders of one build
|
|
|
+(<code>run --repeat 2</code>), grey is unchanged.</p>
|
|
|
+<table><tr><th>verdict</th><th>id</th><th>title</th><th>base</th><th>new</th><th>pixels changed</th><th>ignored</th><th>motion base / new</th></tr>''']
|
|
|
+
|
|
|
+ for row in rows:
|
|
|
+ css = row['verdict'].replace (' ', '-')
|
|
|
+ changed = f'{row["changed"] * 100:.2f}%' if row['changed'] is not None else '-'
|
|
|
+ ignored = f'{row["ignored"] * 100:.1f}%' if row['ignored'] else ''
|
|
|
+ title = esc (row['title']) + (' <small>(video texture)</small>' if row['video'] else '')
|
|
|
+ detailed = row['verdict'] not in ('same', 'not run')
|
|
|
+ link = f'<a href="#w{esc (row["id"])}">{esc (row["id"])}</a>' if detailed else esc (row['id'])
|
|
|
+ motion = ' / '.join ('-' if m is None else f'{m * 100:.2f}%' for m in row['motion']) if row['animated'] else ''
|
|
|
+ if row['animated']:
|
|
|
+ title += f' <small>({", ".join (row["animated"])})</small>'
|
|
|
+
|
|
|
+ parts.append (f'<tr><td class="{css}">{esc (row["verdict"])}</td><td>{link}</td><td>{title}</td>'
|
|
|
+ f'<td>{esc (row["base"])}</td><td>{esc (row["new"])}</td><td>{changed}</td><td>{ignored}</td><td>{motion}</td></tr>')
|
|
|
+
|
|
|
+ parts.append ('</table>')
|
|
|
+
|
|
|
+ for row in rows:
|
|
|
+ if row['verdict'] in ('same', 'not run'):
|
|
|
+ continue
|
|
|
+
|
|
|
+ parts.append (f'<h2 id="w{esc (row["id"])}" class="{row["verdict"].replace (" ", "-")}">'
|
|
|
+ f'{esc (row["id"])} - {esc (row["title"])} ({esc (row["verdict"])})</h2><div class="pair">')
|
|
|
+
|
|
|
+ for name, caption in (('base', 'base'), ('new', 'new'), ('diff', 'changed pixels'),
|
|
|
+ ('base_motion', 'base, a few frames later'), ('new_motion', 'new, a few frames later')):
|
|
|
+ image = report / 'img' / f'{row["id"]}_{name}.jpg'
|
|
|
+ if image.exists ():
|
|
|
+ parts.append (f'<figure><img src="img/{image.name}"><figcaption>{caption}</figcaption></figure>')
|
|
|
+
|
|
|
+ parts.append ('</div>')
|
|
|
+
|
|
|
+ if row['added_errors']:
|
|
|
+ parts.append ('<p>new in the log:</p><pre>' + esc ('\n'.join (row['added_errors'])) + '</pre>')
|
|
|
+ if row['fixed_errors']:
|
|
|
+ parts.append ('<p>gone from the log:</p><pre>' + esc ('\n'.join (row['fixed_errors'])) + '</pre>')
|
|
|
+
|
|
|
+ parts.append ('</body></html>')
|
|
|
+ (report / 'index.html').write_text ('\n'.join (parts))
|
|
|
+
|
|
|
+
|
|
|
+def main ():
|
|
|
+ parser = argparse.ArgumentParser (description = __doc__, formatter_class = argparse.RawDescriptionHelpFormatter)
|
|
|
+ commands = parser.add_subparsers (dest = 'command', required = True)
|
|
|
+
|
|
|
+ run = commands.add_parser ('run', help = 'render every wallpaper with one build')
|
|
|
+ run.add_argument ('binary', type = Path, help = 'the linux-wallpaperengine executable to test')
|
|
|
+ run.add_argument ('out', type = Path, help = 'folder for the screenshots and logs')
|
|
|
+ run.add_argument ('--workshop', type = Path, default = env_path ('LWE_WORKSHOP_DIR', DEFAULT_WORKSHOP))
|
|
|
+ run.add_argument ('--assets', type = Path, default = env_path ('LWE_ASSETS_DIR', DEFAULT_ASSETS))
|
|
|
+ run.add_argument ('--ids', nargs = '*', help = 'only these workshop ids')
|
|
|
+ run.add_argument ('--variants', type = Path, default = HERE / 'variants.json',
|
|
|
+ help = 'extra renders with user properties set (default: variants.json next to this script)')
|
|
|
+ run.add_argument ('--types', default = 'scene', help = 'project types to include, comma separated (default: scene)')
|
|
|
+ run.add_argument ('--jobs', type = int, default = max (1, min (8, (os.cpu_count () or 2) // 4)))
|
|
|
+ run.add_argument ('--frame', type = int, default = 30, help = 'frame to screenshot (default: 30)')
|
|
|
+ run.add_argument ('--step', type = float, default = 0.1, help = 'animation seconds per frame (default: 0.1)')
|
|
|
+ run.add_argument ('--clock', type = int, default = 1767268800, help = 'pinned wall clock, unix seconds')
|
|
|
+ run.add_argument ('--width', type = int, default = 1280)
|
|
|
+ run.add_argument ('--height', type = int, default = 720)
|
|
|
+ run.add_argument ('--repeat', type = int, default = 1,
|
|
|
+ help = 'renders per wallpaper; use 2+ for the base run so compare can ignore noise')
|
|
|
+ run.add_argument ('--motion-gap', type = int, default = 5,
|
|
|
+ help = 'frames between the two renders of wallpapers with gif/video textures (0 disables)')
|
|
|
+ run.add_argument ('--timeout', type = float, default = 300, help = 'seconds per wallpaper')
|
|
|
+ run.set_defaults (func = cmd_run)
|
|
|
+
|
|
|
+ compare = commands.add_parser ('compare', help = 'compare two runs and write a report')
|
|
|
+ compare.add_argument ('base', type = Path)
|
|
|
+ compare.add_argument ('new', type = Path)
|
|
|
+ compare.add_argument ('--report', type = Path, default = Path ('regression-report'))
|
|
|
+ compare.add_argument ('--threshold', type = int, default = 16, help = 'per-channel difference that counts as changed')
|
|
|
+ compare.add_argument ('--video-tolerance', type = float, default = 5,
|
|
|
+ help = 'percent of changed pixels allowed with video textures, the motion check covers those')
|
|
|
+ compare.add_argument ('--motion-min', type = float, default = 0.05,
|
|
|
+ help = 'percent of pixels that must move in the base run for the motion check to apply')
|
|
|
+ compare.add_argument ('--tolerance', type = float, default = 0.1, help = 'percent of changed pixels allowed')
|
|
|
+ compare.set_defaults (func = cmd_compare)
|
|
|
+
|
|
|
+ args = parser.parse_args ()
|
|
|
+ args.func (args)
|
|
|
+
|
|
|
+
|
|
|
+if __name__ == '__main__':
|
|
|
+ main ()
|
UwU