verified_on table — 5 modules empirically confirmed in real VMs
Closes the loop opened by tools/verify-vm/: every JSON verification
record now persists into docs/VERIFICATIONS.jsonl, gets folded into
the embedded core/verifications.c lookup table, and surfaces in
--list / --module-info / --explain / --scan --json.
New: docs/VERIFICATIONS.jsonl
Append-only store. One JSON record per verify.sh run. Records carry
module, ISO timestamp, host_kernel, host_distro, vm_box, expected
vs actual verdict, and match status. 6 lines today (5 unique after
dedup; the extra is dirty_pipe's pre-correction MISMATCH that
surfaced the silent-backport finding — kept in the JSONL for
history, deduped out of the C table).
New: tools/refresh-verifications.py
Parses VERIFICATIONS.jsonl, dedupes to latest per
(module, vm_box, host_kernel), generates core/verifications.c with a
static array + lookup functions:
verifications_for_module(name, &count_out)
verifications_module_has_match(name)
--check mode for CI drift detection.
New: core/verifications.{h,c}
Embedded record table. Lookup is O(corpus); we have <50 records.
skeletonkey.c surfacing:
- --list: new 'VFY' column shows ✓ for modules with >=1 'match'
record. Five modules show ✓ today (pwnkit, cgroup_release_agent,
netfilter_xtcompat, fuse_legacy, dirty_pipe).
- --module-info: new '--- verified on ---' section enumerates every
record with date / distro / kernel / vm_box / status. Modules with
zero records get a 'run tools/verify-vm/verify.sh <name>' hint.
- --explain: new 'VERIFIED ON' section in the operator briefing.
- --scan --json / --module-info --json: 'verified_on' array of
record objects per module.
Verification records baked in:
pwnkit Ubuntu 20.04.6 LTS 5.4.0-169 match (polkit 0.105)
cgroup_release_agent Debian 11 (bullseye) 5.10.0-27 match
netfilter_xtcompat Debian 11 (bullseye) 5.10.0-27 match
fuse_legacy Debian 11 (bullseye) 5.10.0-27 match
dirty_pipe Ubuntu 22.04.3 LTS 5.15.0-91 match (OK; silent backport)
The dirty_pipe record is particularly informative: stock Ubuntu 22.04
ships 5.15.0-91-generic. Our version-only kernel_range check would say
VULNERABLE (5.15.0 < 5.15.25 backport in our table). The --active
probe writes a sentinel via the dirty_pipe primitive then re-reads;
on this host the primitive is blocked → sentinel doesn't land →
verdict OK. Ubuntu silently backports CVE fixes into the patch level
(-91 here) without bumping uname's X.Y.Z. The targets.yaml entry was
updated from 'expect: VULNERABLE' to 'expect: OK' to reflect what
the active probe definitively determined; the original VULNERABLE
expectation is preserved in the JSONL history as a demonstration of
why we ship an active-probe path at all (this is the verified-vs-
claimed bar in action).
Plumbing fixes that landed in the same loop:
- core/nft_compat.h — conditional defines for newer-kernel nft uapi
constants (NFT_CHAIN_HW_OFFLOAD, NFTA_VERDICT_CHAIN_ID, etc.)
that aren't in Ubuntu 20.04's pre-5.5 linux-libc-dev. Without
this, nft_* modules failed to compile inside the verifier guest.
Included from each nft module after <linux/netfilter/nf_tables.h>.
- tools/verify-vm/Vagrantfile — wrap config in c.vm.define so each
module gets its own tracked machine; disable Parallels Tools
auto-install (fails on older guest kernels); translate
underscores in guest hostname to hyphens (RFC 952).
- tools/verify-vm/verify.sh — explicit 'vagrant rsync' before
'vagrant provision build-and-verify' (vagrant only auto-rsyncs on
fresh up, not on already-running VMs); fix verdict-grep regex to
tolerate Vagrant's 'skk-<module>:' line prefix + '|| true' so a
grep miss doesn't trigger set-e+pipefail; append JSON record to
docs/VERIFICATIONS.jsonl on every run.
- tools/verify-vm/targets.yaml — dirty_pipe retargeted from
ubuntu2004 + pinned 5.13.0-19 (no longer in 20.04's apt) to
ubuntu2204 stock 5.15.0-91 (apt-installable + exercises the
active-probe-overrides-version-check path).
What's next for the verifier:
- Mainline kernel.ubuntu.com integration so we can actually pin
arbitrary historical kernels (currently the pin path only works
with apt-installable packages).
- Sweep the remaining ~18 verifiable modules and accumulate records.
- Per-module verified_on counts in --explain header.
This commit is contained in:
Executable
+174
@@ -0,0 +1,174 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
tools/refresh-verifications.py — read docs/VERIFICATIONS.jsonl,
|
||||
generate core/verifications.c with a deduped, sorted lookup table.
|
||||
|
||||
Dedup key: (module, vm_box, host_kernel, expect_detect).
|
||||
On collision, the LATEST verified_at wins (so re-runs update rather
|
||||
than accumulate). Records are then sorted by module name so the
|
||||
output is stable and review-friendly.
|
||||
|
||||
Records with no module name are dropped silently. Records with
|
||||
status != "match" are kept so MISMATCH histories stay visible in
|
||||
--module-info (but don't earn the ✓ verified badge).
|
||||
|
||||
Usage:
|
||||
tools/refresh-verifications.py # regenerate core/verifications.c
|
||||
tools/refresh-verifications.py --check # exit 1 if regenerating would change anything
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
REPO = Path(__file__).resolve().parent.parent
|
||||
JSONL = REPO / "docs" / "VERIFICATIONS.jsonl"
|
||||
OUT_C = REPO / "core" / "verifications.c"
|
||||
|
||||
|
||||
def load_records():
|
||||
if not JSONL.exists():
|
||||
return []
|
||||
out = []
|
||||
for line in JSONL.read_text().splitlines():
|
||||
line = line.strip()
|
||||
if not line or line.startswith("#"):
|
||||
continue
|
||||
try:
|
||||
r = json.loads(line)
|
||||
if r.get("module"):
|
||||
out.append(r)
|
||||
except json.JSONDecodeError as e:
|
||||
print(f"[!] skipping bad JSONL line: {e}", file=sys.stderr)
|
||||
return out
|
||||
|
||||
|
||||
def dedup_latest(records):
|
||||
"""Keep only the latest record per (module, vm_box, host_kernel).
|
||||
|
||||
NB: expect_detect is intentionally NOT part of the dedup key. If we
|
||||
re-verify the same target with a corrected expectation, the new
|
||||
record supersedes the old one entirely (the old MISMATCH was a stale
|
||||
target-yaml entry, not a separate test scenario)."""
|
||||
by_key = {}
|
||||
for r in records:
|
||||
k = (r.get("module"), r.get("vm_box"), r.get("host_kernel"))
|
||||
prev = by_key.get(k)
|
||||
if prev is None or r.get("verified_at", "") > prev.get("verified_at", ""):
|
||||
by_key[k] = r
|
||||
return sorted(by_key.values(),
|
||||
key=lambda r: (r["module"], r.get("vm_box", ""),
|
||||
r.get("host_kernel", "")))
|
||||
|
||||
|
||||
def date_only(iso_ts: str) -> str:
|
||||
"""Truncate 2026-05-23T19:26:02Z -> 2026-05-23."""
|
||||
if not iso_ts:
|
||||
return ""
|
||||
return iso_ts.split("T", 1)[0]
|
||||
|
||||
|
||||
def cstr(s):
|
||||
if s is None or s == "":
|
||||
return '""'
|
||||
# No paths in here ever contain unescapable chars; basic backslash + quote escape.
|
||||
return '"' + s.replace("\\", "\\\\").replace('"', '\\"') + '"'
|
||||
|
||||
|
||||
def render_c(records) -> str:
|
||||
lines = [
|
||||
"/*",
|
||||
" * SKELETONKEY — verification records table",
|
||||
" *",
|
||||
" * AUTO-GENERATED by tools/refresh-verifications.py from",
|
||||
" * docs/VERIFICATIONS.jsonl. Do not hand-edit; rerun the script.",
|
||||
" *",
|
||||
" * Source: tools/verify-vm/verify.sh appends one JSON record per",
|
||||
" * run; this generator dedupes to (module, vm_box, kernel, expect)",
|
||||
" * and keeps the latest by verified_at.",
|
||||
" */",
|
||||
"",
|
||||
'#include "verifications.h"',
|
||||
"",
|
||||
"#include <stddef.h>",
|
||||
"#include <string.h>",
|
||||
"#include <stdbool.h>",
|
||||
"",
|
||||
"const struct verification_record verifications[] = {",
|
||||
]
|
||||
for r in records:
|
||||
lines.append(" {")
|
||||
lines.append(f" .module = {cstr(r.get('module'))},")
|
||||
lines.append(f" .verified_at = {cstr(date_only(r.get('verified_at', '')))},")
|
||||
lines.append(f" .host_kernel = {cstr(r.get('host_kernel'))},")
|
||||
lines.append(f" .host_distro = {cstr(r.get('host_distro'))},")
|
||||
lines.append(f" .vm_box = {cstr(r.get('vm_box'))},")
|
||||
lines.append(f" .expect_detect = {cstr(r.get('expect_detect'))},")
|
||||
lines.append(f" .actual_detect = {cstr(r.get('actual_detect'))},")
|
||||
lines.append(f" .status = {cstr(r.get('status'))},")
|
||||
lines.append(" },")
|
||||
lines += [
|
||||
"};",
|
||||
"",
|
||||
"const size_t verifications_count =",
|
||||
" sizeof(verifications) / sizeof(verifications[0]);",
|
||||
"",
|
||||
"const struct verification_record *",
|
||||
"verifications_for_module(const char *module, size_t *count_out)",
|
||||
"{",
|
||||
" if (count_out) *count_out = 0;",
|
||||
" if (!module) return NULL;",
|
||||
" const struct verification_record *first = NULL;",
|
||||
" size_t n = 0;",
|
||||
" for (size_t i = 0; i < verifications_count; i++) {",
|
||||
" if (strcmp(verifications[i].module, module) == 0) {",
|
||||
" if (first == NULL) first = &verifications[i];",
|
||||
" n++;",
|
||||
" }",
|
||||
" }",
|
||||
" if (count_out) *count_out = n;",
|
||||
" return first;",
|
||||
"}",
|
||||
"",
|
||||
"bool verifications_module_has_match(const char *module)",
|
||||
"{",
|
||||
" size_t n = 0;",
|
||||
" const struct verification_record *r = verifications_for_module(module, &n);",
|
||||
" for (size_t i = 0; i < n; i++)",
|
||||
" if (r[i].status && strcmp(r[i].status, \"match\") == 0)",
|
||||
" return true;",
|
||||
" return false;",
|
||||
"}",
|
||||
"",
|
||||
]
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
ap = argparse.ArgumentParser(description=__doc__.splitlines()[1])
|
||||
ap.add_argument("--check", action="store_true",
|
||||
help="diff against committed core/verifications.c; exit 1 on drift")
|
||||
args = ap.parse_args()
|
||||
|
||||
records = dedup_latest(load_records())
|
||||
text = render_c(records)
|
||||
|
||||
if args.check:
|
||||
existing = OUT_C.read_text() if OUT_C.exists() else ""
|
||||
if existing == text:
|
||||
print(f"[+] core/verifications.c is current ({len(records)} record(s))",
|
||||
file=sys.stderr)
|
||||
return 0
|
||||
print("[!] core/verifications.c drifted — rerun "
|
||||
"tools/refresh-verifications.py", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
OUT_C.write_text(text)
|
||||
print(f"[+] wrote {OUT_C.relative_to(REPO)} ({len(records)} record(s))",
|
||||
file=sys.stderr)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Reference in New Issue
Block a user