~james/scope

feat(capture): persist bounded table snapshots with atomic writes

commit
1f6e0491a866c52f42183ae59fb1a1b75b6c41dd browse tree
parent
149423fc8891152959ae927d11751f26a4042733
author
james <j@cpl0.zip>
signature
Verified ~james signed with a key registered by ~james
date
2026-10-18 13:16 UTC

3 files changed, +350 −0

format.md added

@@ -0,0 +1,111 @@
1+A version 1 capture contains a fixed header, three metadata strings and the table frames below the selected root. Integers are little-endian and no OCaml values are marshalled, so the reader can check lengths and tags before constructing a paging context.
2+
3+## Header
4+
5+```text
6+offset bytes encoding field
7+0 8 bytes 53 43 4f 50 45 00 01 0a
8+8 4 u32 domid
9+12 4 u32 vcpu
10+16 4 u32 physical-address width
11+20 4 u32 feature flags
12+24 56 7 * u64 cr0 cr2 cr3 cr4 efer rflags rip
13+80 4 u32 root-override tag
14+84 8 u64 root-override value
15+
16+feature flags
17+ bit 0 = nx
18+ bit 1 = 1 gib pages
19+ other bits = rejected
20+
21+root override
22+ tag 0 = use saved cr3, value must be zero
23+ tag 1 = select value as the raw root, retain saved cr3
24+ other tags = rejected
25+```
26+
27+## Payload
28+
29+```text
30+string = u32 byte-length, byte-length bytes
31+metadata = string source, string consistency, string domain-identity
32+frame = u64 guest-physical-address, u32 tag, frame-payload
33+tag 0 = 4096 bytes of table contents
34+tag 1 = string read-error
35+frames = u32 count, count frame records
36+file = header, metadata, frames, 16-byte digest
37+digest = raw md5 of every preceding byte
38+
39+bounds
40+ runtime = 64-bit
41+ physical width = 32..52
42+ string length <= 4096
43+ frame count <= 65536
44+ file length <= 280000000
45+ frame alignment = 4096
46+ duplicate frames = rejected
47+ unknown frame tags = rejected
48+ out-of-width addresses = rejected
49+ trailing payload bytes = rejected
50+ checksum mismatch = rejected
51+```
52+
53+MD5 detects accidental corruption and does not authenticate a capture. Writes go through a mode-0600 temporary file beside the destination, followed by `fsync` and rename, but the directory itself is not synchronised and the rename may therefore be lost after a host crash.
54+
55+## Import
56+
57+```text
58+command
59+ scope import TRANSCRIPT OUTPUT [--source TEXT]
60+
61+records
62+ context CR0 CR2 CR3 CR4 EFER RFLAGS RIP PHYS_BITS NX GB_PAGES
63+ page ADDRESS HEX
64+ missing ADDRESS REASON
65+
66+grammar
67+ addresses and registers = unsigned decimal or 0x-prefixed hex
68+ phys_bits = decimal
69+ nx and gb_pages = 0 or 1
70+ hex = 8192 hex digits in ascending byte order
71+ field separator = ordinary spaces
72+ record separator = newline
73+ reason = rest of line, may be empty
74+ context count = exactly one, may follow frame records
75+ duplicate frames = rejected
76+ unknown records = rejected
77+ line length <= 16384 bytes
78+ input length <= 840000000 bytes
79+ imported root override = none
80+ imported domid and vcpu = 0
81+```
82+
83+## Reports
84+
85+```text
86+schema = scope.report.v1
87+addresses and sizes = hexadecimal strings
88+complete = completeness of the requested analysis
89+info = checks the selected table graph
90+walk = structural fault bits, no assumed access or privilege
91+explain = error code for the explicit request
92+ac = captured rflags unless --ac overrides it
93+error_code_scope = structural_bits_only or request
94+
95+diff
96+ intervals = disjoint virtual ranges
97+ mapping states = values and paths at the interval start
98+ leaf sizes = original sizes retained
99+ huge-page split = potentially one record per smaller leaf
100+ unreadable subtree = incomplete, never treated as absent
101+
102+limits
103+ traversal budget = abort before emitting a partial report
104+ alias budget = abort before emitting a partial report
105+ error stream = stderr, structured when --json is selected
106+ result stream = stdout
107+ exit 0 = complete, no fault or finding
108+ exit 1 = predicted fault, invalid table, alias or diff
109+ exit 2 = incomplete or operational failure
110+ exit 64 = usage error
111+```

lib/capture.ml added

@@ -0,0 +1,228 @@
1+module Frames = Map.Make (struct
2+ type t = int64
3+ let compare = Address.compare
4+end)
5+
6+type t = {
7+ context : Model.context;
8+ frames : (string, string) result Frames.t;
9+}
10+
11+let maximum_pages = 65536
12+let maximum_bytes = 280_000_000
13+let context capture = capture.context
14+let pages capture = Frames.cardinal capture.frames
15+
16+let make context frames =
17+ Model.validate context;
18+ if List.length frames > maximum_pages then invalid_arg "too many capture pages";
19+ let frames = List.fold_left (fun map (address, page) ->
20+ let raw = Address.Physical.to_int64 address in
21+ if Int64.logand raw 0xfffL <> 0L || Address.compare raw (Address.mask context.features.phys_bits) > 0 then
22+ invalid_arg "unaligned or out-of-range table frame";
23+ if Frames.mem raw map then invalid_arg "duplicate table frame";
24+ (match page with
25+ | Ok bytes when String.length bytes <> 4096 -> invalid_arg "table must contain 4096 bytes"
26+ | Error reason when String.length reason > 4096 -> invalid_arg "table error is too long"
27+ | _ -> ());
28+ Frames.add raw page map) Frames.empty frames
29+ in
30+ { context; frames }
31+
32+let read capture address =
33+ let raw = Address.Physical.to_int64 address in
34+ match Frames.find_opt raw capture.frames with
35+ | Some page -> page
36+ | None -> Error ("table not captured at " ^ Address.hex raw)
37+
38+let with_root capture root =
39+ let context = { capture.context with root_override = Some root } in
40+ Model.validate context;
41+ { capture with context }
42+
43+let collect ?(max_pages = maximum_pages) context reader =
44+ Model.validate context;
45+ if max_pages < 1 || max_pages > maximum_pages then invalid_arg "invalid capture page budget";
46+ let frames = ref Frames.empty and visited = Hashtbl.create 64 in
47+ let rec go level address =
48+ let raw = Address.Physical.to_int64 address in
49+ if not (Hashtbl.mem visited (level, raw)) then begin
50+ Hashtbl.add visited (level, raw) ();
51+ let page = match Frames.find_opt raw !frames with
52+ | Some page -> page
53+ | None ->
54+ if Frames.cardinal !frames >= max_pages then raise (Model.Limit "capture page budget exceeded");
55+ let page = reader address in
56+ (match page with Ok bytes when String.length bytes <> 4096 -> invalid_arg "short table read" | _ -> ());
57+ frames := Frames.add raw page !frames;
58+ page
59+ in
60+ match page with
61+ | Error _ -> ()
62+ | Ok bytes ->
63+ for index = 0 to 511 do
64+ let entry = String.get_int64_le bytes (index * 8) in
65+ match Paging.decode context.features ~nxe:(Address.bit context.registers.efer 11) ~level entry with
66+ | Paging.Table next -> go (level - 1) next
67+ | _ -> ()
68+ done
69+ end
70+ in
71+ go 4 (Model.root context);
72+ { context; frames = !frames }
73+
74+let complete capture =
75+ let closure = collect capture.context (read capture) in
76+ Frames.for_all (fun _ -> function Ok _ -> true | Error _ -> false) closure.frames
77+
78+let add_u32 buffer n =
79+ let bytes = Bytes.create 4 in
80+ Bytes.set_int32_le bytes 0 (Int32.of_int n);
81+ Buffer.add_bytes buffer bytes
82+
83+let add_u64 buffer n =
84+ let bytes = Bytes.create 8 in
85+ Bytes.set_int64_le bytes 0 n;
86+ Buffer.add_bytes buffer bytes
87+
88+let add_string buffer s =
89+ if String.length s > 4096 then invalid_arg "capture string exceeds 4096 bytes";
90+ add_u32 buffer (String.length s);
91+ Buffer.add_string buffer s
92+
93+let encode capture =
94+ let buffer = Buffer.create (128 + pages capture * 4112) in
95+ Buffer.add_string buffer "SCOPE\000\001\n";
96+ let c = capture.context in
97+ add_u32 buffer c.domid;
98+ add_u32 buffer c.vcpu;
99+ add_u32 buffer c.features.phys_bits;
100+ add_u32 buffer ((if c.features.nx then 1 else 0) lor (if c.features.gb_pages then 2 else 0));
101+ let r = c.registers in
102+ List.iter (add_u64 buffer) [r.cr0; r.cr2; r.cr3; r.cr4; r.efer; r.rflags; r.rip];
103+ add_u32 buffer (if c.root_override = None then 0 else 1);
104+ add_u64 buffer (Option.value c.root_override ~default:0L);
105+ List.iter (add_string buffer) [c.source; c.consistency; c.identity];
106+ add_u32 buffer (pages capture);
107+ Frames.iter (fun address page ->
108+ add_u64 buffer address;
109+ match page with
110+ | Ok bytes -> add_u32 buffer 0; Buffer.add_string buffer bytes
111+ | Error reason -> add_u32 buffer 1; add_string buffer reason) capture.frames;
112+ let payload = Buffer.contents buffer in
113+ payload ^ Digest.string payload
114+
115+let save path capture =
116+ let bytes = encode capture in
117+ let temporary, channel = Filename.open_temp_file ~temp_dir:(Filename.dirname path) ".scope" ".tmp" in
118+ Fun.protect ~finally:(fun () -> close_out_noerr channel; if Sys.file_exists temporary then Sys.remove temporary)
119+ (fun () ->
120+ output_string channel bytes;
121+ flush channel;
122+ Unix.fsync (Unix.descr_of_out_channel channel);
123+ close_out channel;
124+ Unix.rename temporary path)
125+
126+let load path =
127+ let channel = open_in_bin path in
128+ let bytes = Fun.protect ~finally:(fun () -> close_in_noerr channel) (fun () ->
129+ let length = in_channel_length channel in
130+ if length > maximum_bytes || length < 24 then invalid_arg "capture size outside format bounds";
131+ really_input_string channel length)
132+ in
133+ let limit = String.length bytes - 16 in
134+ if Digest.string (String.sub bytes 0 limit) <> String.sub bytes limit 16 then
135+ invalid_arg "capture checksum mismatch";
136+ let position = ref 0 in
137+ let take n =
138+ if n < 0 || n > limit - !position then invalid_arg "truncated capture";
139+ let s = String.sub bytes !position n in
140+ position := !position + n;
141+ s
142+ in
143+ let u32 () =
144+ let n = Int64.logand (Int64.of_int32 (String.get_int32_le (take 4) 0)) 0xffffffffL in
145+ if n > Int64.of_int max_int then invalid_arg "capture integer overflow";
146+ Int64.to_int n
147+ in
148+ let u64 () = String.get_int64_le (take 8) 0 in
149+ let string () = let n = u32 () in if n > 4096 then invalid_arg "capture string too long"; take n in
150+ if take 8 <> "SCOPE\000\001\n" then invalid_arg "unsupported capture format";
151+ let domid = u32 () in
152+ let vcpu = u32 () in
153+ let phys_bits = u32 () in
154+ let flags = u32 () in
155+ if flags land (lnot 3) <> 0 then invalid_arg "unsupported feature flags";
156+ let cr0 = u64 () in
157+ let cr2 = u64 () in
158+ let cr3 = u64 () in
159+ let cr4 = u64 () in
160+ let efer = u64 () in
161+ let rflags = u64 () in
162+ let rip = u64 () in
163+ let root_tag = u32 () in
164+ let root_value = u64 () in
165+ let root_override = match root_tag, root_value with
166+ | 0, 0L -> None | 1, value -> Some value | _ -> invalid_arg "invalid root override record"
167+ in
168+ let source = string () in
169+ let consistency = string () in
170+ let identity = string () in
171+ let count = u32 () in
172+ if count > maximum_pages then invalid_arg "too many capture pages";
173+ let frames = List.init count (fun _ ->
174+ let address = Address.Physical.of_int64 (u64 ()) in
175+ let page = match u32 () with
176+ | 0 -> Ok (take 4096)
177+ | 1 -> Error (string ())
178+ | _ -> invalid_arg "invalid table record tag"
179+ in
180+ address, page)
181+ in
182+ if !position <> limit then invalid_arg "trailing capture data";
183+ make { registers = { cr0; cr2; cr3; cr4; efer; rflags; rip }; root_override;
184+ features = { phys_bits; nx = flags land 1 <> 0; gb_pages = flags land 2 <> 0 };
185+ source; consistency; identity; domid; vcpu } frames
186+
187+let import ~source path =
188+ let channel = open_in path in
189+ Fun.protect ~finally:(fun () -> close_in_noerr channel) (fun () ->
190+ if in_channel_length channel > maximum_bytes * 3 then invalid_arg "import exceeds format size budget";
191+ let context = ref None and frames = ref [] and count = ref 0 in
192+ let number = ref 0 in
193+ (try while true do
194+ let line = input_line channel in
195+ incr number;
196+ if String.length line > 16384 then invalid_arg "import line exceeds 16384 bytes";
197+ let fields = String.split_on_char ' ' line |> List.filter ((<>) "") in
198+ match fields with
199+ | [] -> ()
200+ | ["context"; cr0; cr2; cr3; cr4; efer; rflags; rip; phys_bits; nx; gb_pages] ->
201+ if !context <> None then invalid_arg "duplicate import context";
202+ let boolean = function "0" -> false | "1" -> true | _ -> invalid_arg "invalid feature boolean" in
203+ let open Model in
204+ context := Some { registers = { cr0 = Address.parse cr0; cr2 = Address.parse cr2;
205+ cr3 = Address.parse cr3; cr4 = Address.parse cr4; efer = Address.parse efer;
206+ rflags = Address.parse rflags; rip = Address.parse rip };
207+ features = { phys_bits = int_of_string phys_bits; nx = boolean nx; gb_pages = boolean gb_pages };
208+ root_override = None; source; consistency = "offline import; producer defines consistency";
209+ identity = "import"; domid = 0; vcpu = 0 }
210+ | ["page"; address; hex] ->
211+ incr count;
212+ if !count > maximum_pages then invalid_arg "too many imported pages";
213+ if String.length hex <> 8192 then invalid_arg "page hex must contain 8192 digits";
214+ let digit = function
215+ | '0' .. '9' as c -> Char.code c - Char.code '0'
216+ | 'a' .. 'f' as c -> Char.code c - Char.code 'a' + 10
217+ | 'A' .. 'F' as c -> Char.code c - Char.code 'A' + 10
218+ | _ -> invalid_arg "invalid page hex"
219+ in
220+ let bytes = String.init 4096 (fun i -> Char.chr ((digit hex.[2*i] lsl 4) lor digit hex.[2*i+1])) in
221+ frames := (Address.Physical.of_int64 (Address.parse address), Ok bytes) :: !frames
222+ | "missing" :: address :: reason ->
223+ incr count;
224+ if !count > maximum_pages then invalid_arg "too many imported pages";
225+ frames := (Address.Physical.of_int64 (Address.parse address), Error (String.concat " " reason)) :: !frames
226+ | _ -> invalid_arg (Printf.sprintf "invalid import record on line %d" !number)
227+ done with End_of_file -> ());
228+ match !context with None -> invalid_arg "missing import context" | Some context -> make context !frames)

lib/capture.mli added

@@ -0,0 +1,11 @@
1+type t
2+val make : Model.context -> (Address.Physical.t * (string, string) result) list -> t
3+val context : t -> Model.context
4+val read : t -> Model.reader
5+val pages : t -> int
6+val with_root : t -> int64 -> t
7+val collect : ?max_pages:int -> Model.context -> Model.reader -> t
8+val complete : t -> bool
9+val save : string -> t -> unit
10+val load : string -> t
11+val import : source:string -> string -> t