#!/usr/bin/env python3 """Compare matched normal-startup reports using raw RSS/PSS and swap samples. Requires schema-2 reports, at least three fresh process rounds, matching build features/profile/tool versions, fixtures, display, declared geometry and timing. Artifact manifests must have been created for each independently verified build. This reports observed ranges, not statistical significance or whole-app savings. """ from __future__ import annotations import argparse import json import math import re import statistics from pathlib import Path COUNTERS = ('rss_kib', 'pss_kib', 'swap_kib', 'swap_pss_kib') def require(condition: bool, message: str) -> None: if not condition: raise ValueError(message) def number(value) -> bool: return isinstance(value, (int, float)) and not isinstance(value, bool) and math.isfinite(value) def validate(report: dict, label: str) -> dict: require(isinstance(report, dict) and report.get('schema_version') == 2, f'{label}: requires schema-2 normal-startup report') identity = report.get('identity', {}) config = report.get('configuration', {}) require(isinstance(identity, dict) and isinstance(config, dict), f'{label}: invalid identity or configuration') artifact = identity.get('artifact') require(isinstance(artifact, dict), f'{label}: artifact provenance is missing') digest = identity.get('binary_sha256') require(isinstance(digest, str) and re.fullmatch(r'[0-9a-f]{64}', digest) is not None, f'{label}: invalid executable SHA-256') require(artifact.get('binary_sha256') == digest, f'{label}: manifest/executable SHA-256 mismatch') commit = artifact.get('source_commit') require(isinstance(commit, str) and re.fullmatch(r'[0-9a-f]{40}', commit) is not None, f'{label}: invalid artifact source commit') for key in ('build_profile', 'profile_settings', 'rust_version', 'slint_version'): require(isinstance(artifact.get(key), str) and bool(artifact[key]), f'{label}: missing {key}') features = artifact.get('build_features') require(isinstance(features, list) and all(isinstance(f, str) and f for f in features), f'{label}: missing build features') require(config.get('forced_snapshots') is False and config.get('provider_sync_disabled') is True, f'{label}: requires unsnapshotted, sync-disabled runs') for key in ('include_mappings', 'include_thread_stacks', 'profile_data_supplied', 'profile_cache_supplied'): require(isinstance(config.get(key), bool), f'{label}: missing {key}') fixtures = config.get('fixture_fingerprints') require(isinstance(fixtures, dict) and all(isinstance(fixtures.get(k), str) and (fixtures[k] == 'empty' or re.fullmatch(r'[0-9a-f]{64}', fixtures[k])) for k in ('data', 'cache')), f'{label}: missing fixture fingerprints') window = config.get('declared_window') require(isinstance(window, dict) and all(number(window.get(k)) and window[k] > 0 for k in ('width', 'height', 'scale')), f'{label}: record observed window width, height and scale') duration = config.get('sample_seconds') interval = config.get('sample_interval') require(number(duration) and duration > 0, f'{label}: invalid sample duration') require(interval is None or (number(interval) and 0 < interval <= duration), f'{label}: invalid sample interval') require(interval is None or duration / interval <= 10_000, f'{label}: sample interval produces more than 10000 samples') require(isinstance(identity.get('platform'), str) and bool(identity['platform']), f'{label}: missing platform') require(bool(identity.get('display') or identity.get('wayland_display')), f'{label}: missing display context') runs = report.get('runs') require(isinstance(runs, list) and len(runs) >= 3, f'{label}: requires at least three raw process rounds') require(config.get('rounds') == len(runs), f'{label}: round count does not match raw runs') require(all(isinstance(run, dict) for run in runs), f'{label}: invalid raw run') require([run.get('round') for run in runs] == list(range(1, len(runs) + 1)), f'{label}: rounds must be unique and ordered') for run in runs: renderer = run.get('renderer_selected', {}) require(isinstance(renderer, dict) and renderer.get('active') == 'cpu' and renderer.get('wgpu_initialized') is False, f'{label}: runtime CPU/WGPU gate failed') samples = run.get('resource_samples') require(isinstance(samples, list) and bool(samples), f'{label}: raw procfs samples missing') expected_samples = 1 if interval is None else math.ceil(duration / interval) require(len(samples) == expected_samples, f'{label}: sample count disagrees with interval') previous = -1.0 for sample in samples: require(isinstance(sample, dict), f'{label}: invalid procfs sample') elapsed = sample.get('elapsed_seconds') require(number(elapsed) and elapsed >= 0 and elapsed >= previous, f'{label}: nonmonotonic sample times') previous = elapsed resources = sample.get('resources', {}) require(isinstance(resources, dict) and all(number(resources.get(key)) and resources[key] >= 0 for key in COUNTERS), f'{label}: missing or invalid RAM/swap counters') require(resources['swap_kib'] == 0 and resources['swap_pss_kib'] == 0, f'{label}: swapped samples cannot support RAM-saving claims') require(previous >= duration, f'{label}: final sample is earlier than declared duration') require(run.get('idle') == samples[-1]['resources'], f'{label}: final counters disagree with raw samples') return {'identity': identity, 'configuration': config, 'artifact': artifact, 'runs': runs} def compare(baseline: dict, candidate: dict) -> dict: left, right = validate(baseline, 'baseline'), validate(candidate, 'candidate') for key in ('platform', 'display', 'wayland_display', 'slint_backend', 'winit_backend'): require(left['identity'].get(key) == right['identity'].get(key), f'incompatible display/platform context: {key}') for key in ('build_profile', 'profile_settings', 'rust_version', 'slint_version'): require(left['artifact'][key] == right['artifact'][key], f'incompatible artifact settings: {key}') require(sorted(set(left['artifact']['build_features'])) == sorted(set(right['artifact']['build_features'])), 'incompatible artifact settings: build_features') for key in ('sample_seconds', 'sample_interval', 'fixture_fingerprints', 'declared_window', 'forced_snapshots', 'provider_sync_disabled', 'include_mappings', 'include_thread_stacks', 'profile_data_supplied', 'profile_cache_supplied'): require(left['configuration'][key] == right['configuration'][key], f'incompatible measurement settings: {key}') rows = {} for phase in ('timed_end', 'sampled_peak'): rows[phase] = {} for counter in ('rss_kib', 'pss_kib'): def values(side): return [run['idle'][counter] if phase == 'timed_end' else max(sample['resources'][counter] for sample in run['resource_samples']) for run in side['runs']] old, new = values(left), values(right) old_median, new_median = statistics.median(old), statistics.median(new) rows[phase][counter.removesuffix('_kib')] = { 'baseline_median_mib': old_median / 1024, 'candidate_median_mib': new_median / 1024, 'median_reduction_mib': (old_median - new_median) / 1024, 'median_reduction_percent': (old_median - new_median) / old_median * 100 if old_median else None, 'baseline_range_mib': [min(old) / 1024, max(old) / 1024], 'candidate_range_mib': [min(new) / 1024, max(new) / 1024], 'observed_ranges': 'lower' if max(new) < min(old) else 'higher' if min(new) > max(old) else 'overlap', } same_executable = left['identity']['binary_sha256'] == right['identity']['binary_sha256'] outcome = 'same_executable_control' if same_executable else 'observed_lower_ranges' if all(row['observed_ranges'] == 'lower' for row in rows['timed_end'].values()) else 'mixed_or_overlapping_ranges' return {'schema_version': 1, 'scope': 'matched_timed_startup', 'outcome': outcome, 'baseline_artifact': left['artifact'], 'candidate_artifact': right['artifact'], 'rounds': {'baseline': len(left['runs']), 'candidate': len(right['runs'])}, 'measurements': rows, 'limitations': ['Timed procfs samples are not frame or settled-state events.', 'Sampled peaks are lower bounds.', 'Geometry and build provenance are declared; verify them independently.', 'Observed ranges do not establish statistical significance or memory savings in other app scenarios.']} def main() -> None: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument('baseline', type=Path) parser.add_argument('candidate', type=Path) parser.add_argument('--output', required=True, type=Path) args = parser.parse_args() try: report = compare(json.loads(args.baseline.read_text()), json.loads(args.candidate.read_text())) except (ValueError, OSError) as error: parser.error(str(error)) args.output.parent.mkdir(parents=True, exist_ok=True) args.output.write_text(json.dumps(report, indent=2, sort_keys=True) + '\n') for name, row in report['measurements']['timed_end'].items(): print(f"{name.upper()}: {row['baseline_median_mib']:.2f} -> {row['candidate_median_mib']:.2f} MiB; ranges={row['observed_ranges']}") print(f"outcome={report['outcome']}; report={args.output}") if __name__ == '__main__': main()