~james/remy

feat(allocator): retain intermediates in registers and spill nested values

commit
edf26390ae3eb8281b7adc9bb0e40f56de1941a8 browse tree
parent
175b8b22df89336d3a03eb1209d760f6f426315b
author
james <j@cpl0.zip>
signature
Verified ~james signed with a key registered by ~james
date
2026-11-01 02:59 UTC

16 files changed, +276 −17

Makefile modified

@@ -1,10 +1,10 @@
11 ISABELLE ?= isabelle
22
3-.PHONY: all check payload clean
3+.PHONY: all check measure payload clean
44
55 all: .build/ready
66
7-.build/ready: ROOT $(wildcard src/*.thy) cli/driver.ML Makefile
7+.build/ready: ROOT $(wildcard src/*.thy src/core/*.thy) cli/driver.ML Makefile
88 @test "$$('$(ISABELLE)' version)" = Isabelle2026
99 "$(ISABELLE)" build -D . -j 1 -o threads=1 -o parallel_proofs=0
1010 "$(ISABELLE)" export -n -d . -O .build -x 'Remy.Export:code/remy.ML' Remy
@@ -15,6 +15,9 @@ check: all
1515 python3 tests/native.py
1616 python3 tests/cli.py
1717
18+measure: all
19+ python3 tests/measure.py
20+
1821 payload: all
1922 @mkdir -p .build/payload
2023 @set -e; for name in Rings Weave Cells; do .build/remy examples/$$name.remy .build/payload/$$name.BIN; .build/remy vectors examples/$$name.remy .build/payload/$$name.DAT; cp examples/$$name.remy .build/payload/$$name.TXT; done

README.md modified

@@ -23,9 +23,9 @@ Show(0);
2323
2424 ## Correctness
2525
26-Each `let` pushes a binding onto the evaluation stack. [Compilation](src/Compiler.thy) adjusts variable offsets as intermediate values accumulate above those bindings and proves that execution leaves the result followed by the original stack whereas accepted programs stay within 512 evaluation words at every instruction boundary
26+Each `let` pushes a binding onto the evaluation stack. [Compilation](src/core/Compiler.thy) adjusts variable offsets as intermediate values accumulate above those bindings and proves that execution leaves the result followed by the original stack whereas accepted programs stay within 512 evaluation words at every instruction boundary
2727
28-[Native lowering](src/Native.thy) puts the result in `rax` and preserves `r8`, `r9` and `rdx` which carry `x`, `y` and `t`. The temporary registers are `rax`, `rcx` and `rbx` while the [Encoding](src/Encoding.thy) proves decoder inversion and byte bounds for the emitted blocks including their immediate operands and stack displacements
28+[Native lowering](src/core/Native.thy) puts the result in `rax` and preserves `r8`, `r9` and `rdx` which carry `x`, `y` and `t`. The temporary registers are `rax`, `rcx` and `rbx` while the [Encoding](src/core/Encoding.thy) proves decoder inversion and byte bounds for the emitted blocks including their immediate operands and stack displacements
2929
3030 ![Source eval](media/correspondence.svg)
3131

ROOT modified

@@ -1,19 +1,20 @@
1-session Remybase = HOL +
1+session Remybase in "src/core" = HOL +
22 options [document = false]
33 sessions "HOL-Library"
44 theories
55 "HOL-Library.Word"
66 "HOL-Library.Code_Target_Numeral"
7-
8-session Remy in src = Remybase +
9- options [document = false]
10- sessions "HOL-Library"
11- theories
127 Model
138 Compiler
149 Native
1510 Encoding
1611 Checker
12+
13+session Remy in src = Remybase +
14+ options [document = false]
15+ sessions "HOL-Library"
16+ theories
17+ Allocation
1718 Frontend
1819 Examples
1920 Export

cli/driver.ML modified

@@ -71,8 +71,9 @@ fun main () =
7171 emit destination (Remy.configured (chars depth) (chars nodes) (chars stack) (read source))
7272 | ["check", binary] => output (Remy.validate (read binary))
7373 | ["disassemble", binary] => output (Remy.disassemble (read binary))
74+ | ["baseline", source, destination] => emit destination (Remy.baseline (read source))
7475 | [source, destination] => emit destination (Remy.artifact (read source))
75- | _ => raise Fail "usage: remy check binary | remy disassemble binary | remy limits depth nodes stack source binary | remy source binary | remy vectors source data | remy eval source x y t (source - reads standard input)")
76+ | _ => raise Fail "usage: remy check binary | remy disassemble binary | remy baseline source binary | remy limits depth nodes stack source binary | remy source binary | remy vectors source data | remy eval source x y t (source - reads standard input)")
7677 handle exn =>
7778 (TextIO.output (TextIO.stdErr, "remy: " ^ General.exnMessage exn ^ "\n");
7879 OS.Process.exit OS.Process.failure);

src/Allocation.thy added

@@ -0,0 +1,123 @@
1+theory Allocation
2+ imports "Remybase.Checker"
3+begin
4+
5+fun allocate :: "bool \<Rightarrow> expression \<Rightarrow> nat list \<Rightarrow> opcode list" where
6+ "allocate free (Literal v) slots = [Immediate v]"
7+| "allocate free (Input i) slots = [Move Rax (input_register i)]"
8+| "allocate free (Variable i) slots = [Peek (slots ! i)]"
9+| "allocate free (Binary op a b) slots = (if free then
10+ allocate True a slots @ [Move Rbx Rax] @ allocate False b slots @
11+ [Move Rcx Rbx, Arithmetic op]
12+ else allocate False a slots @ [Push Rax] @ allocate False b (map Suc slots) @
13+ [Pop Rcx, Arithmetic op])"
14+| "allocate free (Cond c a b) slots = [Push Rbx] @
15+ allocate True c (map Suc slots) @ [Push Rax] @
16+ allocate True a (map (\<lambda>i. Suc (Suc i)) slots) @ [Push Rax] @
17+ allocate True b (map (\<lambda>i. Suc (Suc (Suc i))) slots) @
18+ [Pop Rcx, Pop Rbx, Conditional, Pop Rbx]"
19+| "allocate free (Let a b) slots = allocate free a slots @ [Push Rax] @
20+ allocate free b (0 # map Suc slots) @ [Pop Rcx]"
21+
22+definition allocated_result :: "bool \<Rightarrow> expression \<Rightarrow> pixel list \<Rightarrow> machine \<Rightarrow> machine \<Rightarrow> bool" where
23+ "allocated_result free e env s q \<longleftrightarrow>
24+ registers q Rax = evaluate e env (inputs s) \<and> stack q = stack s \<and>
25+ inputs q = inputs s \<and> (\<not> free \<longrightarrow> registers q Rbx = registers s Rbx)"
26+
27+theorem allocate_correct:
28+ assumes "scoped (length env) e" "represents slots (stack s) env"
29+ shows "\<exists>q. run (allocate free e slots) s = Some q \<and> allocated_result free e env s q"
30+ using assms
31+proof (induction e arbitrary: free slots env s)
32+ case (Literal v)
33+ then show ?case by (auto simp: allocated_result_def)
34+next
35+ case (Input i)
36+ then show ?case by (auto simp: allocated_result_def inputs_def fun_eq_iff)
37+next
38+ case (Variable i)
39+ then show ?case by (auto simp: allocated_result_def represents_def)
40+next
41+ case (Binary op a b)
42+ obtain p where a: "run (allocate free a slots) s = Some p"
43+ and p: "allocated_result free a env s p"
44+ using Binary.IH(1)[of env slots s free] Binary.prems by auto
45+ show ?case
46+ proof (cases free)
47+ case True
48+ let ?p = "p\<lparr>registers := (registers p)(Rbx := registers p Rax)\<rparr>"
49+ obtain q where b: "run (allocate False b slots) ?p = Some q"
50+ and q: "allocated_result False b env ?p q"
51+ using Binary.IH(2)[of env slots ?p False] Binary.prems p
52+ by (auto simp: allocated_result_def)
53+ show ?thesis using a b p q True
54+ by (auto simp: allocated_result_def run_append; auto simp: inputs_def fun_eq_iff)
55+ next
56+ case False
57+ let ?p = "p\<lparr>stack := registers p Rax # stack p\<rparr>"
58+ obtain q where b: "run (allocate False b (map Suc slots)) ?p = Some q"
59+ and q: "allocated_result False b env ?p q"
60+ using Binary.IH(2)[of env "map Suc slots" ?p False] Binary.prems p
61+ represents_push[OF Binary.prems(2), of "registers p Rax"]
62+ by (auto simp: allocated_result_def)
63+ show ?thesis using a b p q False
64+ by (auto simp: allocated_result_def run_append; auto simp: inputs_def fun_eq_iff)
65+ qed
66+next
67+ case (Cond c a b)
68+ have sc: "scoped (length env) c" and sa: "scoped (length env) a"
69+ and sb: "scoped (length env) b" using Cond.prems by auto
70+ let ?s = "s\<lparr>stack := registers s Rbx # stack s\<rparr>"
71+ have repc: "represents (map Suc slots) (stack ?s) env"
72+ using represents_push[OF Cond.prems(2)] by simp
73+ obtain p where c: "run (allocate True c (map Suc slots)) ?s = Some p"
74+ and p: "allocated_result True c env ?s p"
75+ using Cond.IH(1)[of env "map Suc slots" ?s True] sc repc by blast
76+ let ?p = "p\<lparr>stack := registers p Rax # stack p\<rparr>"
77+ have repa: "represents (map (\<lambda>i. Suc (Suc i)) slots) (stack ?p) env"
78+ using Cond.prems(2) p by (auto simp: allocated_result_def represents_def)
79+ obtain q where a: "run (allocate True a (map (\<lambda>i. Suc (Suc i)) slots)) ?p = Some q"
80+ and q: "allocated_result True a env ?p q"
81+ using Cond.IH(2)[of env "map (\<lambda>i. Suc (Suc i)) slots" ?p True] sa repa by blast
82+ let ?q = "q\<lparr>stack := registers q Rax # stack q\<rparr>"
83+ have repb: "represents (map (\<lambda>i. Suc (Suc (Suc i))) slots) (stack ?q) env"
84+ using Cond.prems(2) p q by (auto simp: allocated_result_def represents_def)
85+ obtain r where b: "run (allocate True b (map (\<lambda>i. Suc (Suc (Suc i))) slots)) ?q = Some r"
86+ and r: "allocated_result True b env ?q r"
87+ using Cond.IH(3)[of env "map (\<lambda>i. Suc (Suc (Suc i))) slots" ?q True] sb repb by blast
88+ show ?case using c a b p q r
89+ by (auto simp: allocated_result_def run_append; auto simp: inputs_def fun_eq_iff)
90+next
91+ case (Let a b)
92+ obtain p where a: "run (allocate free a slots) s = Some p"
93+ and p: "allocated_result free a env s p"
94+ using Let.IH(1)[of env slots s free] Let.prems by auto
95+ let ?p = "p\<lparr>stack := registers p Rax # stack p\<rparr>"
96+ obtain q where b: "run (allocate free b (0 # map Suc slots)) ?p = Some q"
97+ and q: "allocated_result free b (registers p Rax # env) ?p q"
98+ using Let.IH(2)[of "registers p Rax # env" "0 # map Suc slots" ?p free]
99+ Let.prems p represents_bind[OF Let.prems(2), of "registers p Rax"]
100+ by (auto simp: allocated_result_def)
101+ show ?case using a b p q
102+ by (auto simp: allocated_result_def run_append; auto simp: inputs_def fun_eq_iff)
103+qed
104+
105+definition allocated :: "expression \<Rightarrow> opcode list" where
106+ "allocated e = transfers (allocate True (optimise e) [])"
107+
108+theorem allocated_return:
109+ "scoped 0 e \<Longrightarrow> \<exists>q. run (allocated e) s = Some q \<and>
110+ registers q Rax = evaluate e [] (inputs s) \<and> stack q = stack s \<and> inputs q = inputs s"
111+ using allocate_correct[where env="[]" and slots="[]" and free=True and e="optimise e"]
112+ optimise_scoped
113+ by (auto simp: allocated_def transfers_correct allocated_result_def represents_def optimise_correct)
114+
115+definition allocated_bytes :: "expression \<Rightarrow> nat list" where
116+ "allocated_bytes e = concat (map encode (allocated e)) @ [195]"
117+
118+lemma allocated_add:
119+ "allocated (Binary Add (Input X) (Input Y)) =
120+ [Move Rax R8, Move Rbx Rax, Move Rax R9, Move Rcx Rbx, Arithmetic Add]"
121+ by eval
122+
123+end

src/Export.thy modified

@@ -6,6 +6,6 @@ definition ascii :: "integer \<Rightarrow> char" where "ascii = char_of"
66
77 definition ordinal :: "char \<Rightarrow> integer" where "ordinal = of_char"
88
9-export_code validate disassemble configured artifact vectors reference ascii ordinal in SML module_name Remy file_prefix remy
9+export_code validate disassemble configured artifact baseline vectors reference ascii ordinal in SML module_name Remy file_prefix remy
1010
1111 end

src/Frontend.thy modified

@@ -1,5 +1,5 @@
11 theory Frontend
2- imports Checker
2+ imports Allocation
33 begin
44
55 fun finish :: "string \<Rightarrow> string list \<Rightarrow> string list" where
@@ -35,8 +35,11 @@ definition number :: "string \<Rightarrow> nat option" where
3535 if a = CHR ''0'' \<and> (b = CHR ''x'' \<or> b = CHR ''X'') then parse_digits 16 rest
3636 else parse_digits 10 s | _ \<Rightarrow> parse_digits 10 s)"
3737
38+lemma parse_digits_bound: "parse_digits base s = Some n \<Longrightarrow> n < 2 ^ 64"
39+ by (auto simp only: parse_digits_def Let_def option.inject option.distinct split: if_splits)
40+
3841 theorem number_bound: "number s = Some n \<Longrightarrow> n < 2 ^ 64"
39- by (auto simp: number_def parse_digits_def Let_def split: list.splits if_splits)
42+ using parse_digits_bound by (auto simp: number_def split: list.splits if_splits)
4043
4144 definition valid_name :: "string \<Rightarrow> bool" where
4245 "valid_name s = (case s of [] \<Rightarrow> False | c # cs \<Rightarrow>
@@ -269,9 +272,11 @@ definition source_error_with :: "limits \<Rightarrow> string \<Rightarrow> strin
269272
270273 definition artifact_with :: "limits \<Rightarrow> string \<Rightarrow> (bool \<times> string) \<times> string" where
271274 "artifact_with lim s = (case prepare_with lim s of None \<Rightarrow> ((False,[]), source_error_with lim s)
272- | Some e \<Rightarrow> let opt = optimise e; bytes = executable e in
273- ((True, map char_of_nat bytes), decimal (nodes e) @ '' nodes -> '' @ decimal (nodes opt) @
274- '' nodes, '' @ decimal (length bytes) @ '' bytes, '' @ decimal (demand opt) @ '' stack words'' @ newline))"
275+ | Some e \<Rightarrow> let opt = optimise e; code = allocated e; bytes = allocated_bytes e in
276+ if audit code 0 = Some 0 \<and> list_all encodable code then
277+ ((True, map char_of_nat bytes), decimal (nodes e) @ '' nodes -> '' @ decimal (nodes opt) @
278+ '' nodes, '' @ decimal (length bytes) @ '' bytes, '' @ decimal (demand opt) @ '' stack words'' @ newline)
279+ else ((False,[]),''allocated code exceeds native limits'' @ newline))"
275280
276281 definition source_error :: "string \<Rightarrow> string" where
277282 "source_error = source_error_with defaults"
@@ -279,6 +284,10 @@ definition source_error :: "string \<Rightarrow> string" where
279284 definition artifact :: "string \<Rightarrow> (bool \<times> string) \<times> string" where
280285 "artifact = artifact_with defaults"
281286
287+definition baseline :: "string \<Rightarrow> (bool \<times> string) \<times> string" where
288+ "baseline s = (case prepare s of None \<Rightarrow> ((False,[]),source_error s)
289+ | Some e \<Rightarrow> ((True,map char_of_nat (executable e)),''stack backend'' @ newline))"
290+
282291 definition configured :: "string \<Rightarrow> string \<Rightarrow> string \<Rightarrow> string \<Rightarrow> (bool \<times> string) \<times> string" where
283292 "configured depth count stacktext source = (case (number depth, number count, number stacktext) of
284293 (Some d,Some n,Some k) \<Rightarrow> artifact_with \<lparr>depth_budget=d, node_budget=n, stack_budget=k\<rparr> source

src/Checker.thy → src/core/Checker.thy renamed

src/Compiler.thy → src/core/Compiler.thy renamed

src/Encoding.thy → src/core/Encoding.thy renamed

src/Model.thy → src/core/Model.thy renamed

src/Native.thy → src/core/Native.thy renamed

tests/cli.py modified

@@ -53,6 +53,11 @@ def main():
5353 invoke(source,binary)
5454 listing = invoke('disassemble',binary)
5555 assert 'mov rax r9' in listing and 'arithmetic add' in listing and 'ret' in listing
56+ assert 'push' not in listing and 'pop' not in listing
57+ source.write_text('(- x (- y (- t x)))')
58+ invoke(source,binary)
59+ listing = invoke('disassemble',binary)
60+ assert 'push rax' in listing and 'pop rcx' in listing
5661 for malformed in [b'',bytes([195,0]),bytes([72,199,192,0]),bytes([80])]:
5762 binary.write_bytes(malformed)
5863 assert 'invalid native' in invoke('disassemble',binary,ok=False)

tests/measure.c added

@@ -0,0 +1,72 @@
1+#define _GNU_SOURCE
2+#include <inttypes.h>
3+#include <stdint.h>
4+#include <stdio.h>
5+#include <stdlib.h>
6+#include <string.h>
7+#include <sys/mman.h>
8+#include <time.h>
9+#include <unistd.h>
10+
11+static uint64_t ticks(void)
12+{
13+ struct timespec now;
14+ if (clock_gettime(CLOCK_MONOTONIC, &now)) {
15+ perror("clock_gettime");
16+ exit(1);
17+ }
18+ return (uint64_t)now.tv_sec * UINT64_C(1000000000) + now.tv_nsec;
19+}
20+
21+int main(int argc, char **argv)
22+{
23+ if (argc != 2) return 1;
24+ FILE *file = fopen(argv[1], "rb");
25+ if (!file) return 1;
26+ if (fseek(file, 0, SEEK_END)) {
27+ fclose(file);
28+ return 1;
29+ }
30+ long size = ftell(file);
31+ long page = sysconf(_SC_PAGESIZE);
32+ if (size <= 0 || size > 65536 || page <= 0 || fseek(file, 0, SEEK_SET)) {
33+ fclose(file);
34+ return 1;
35+ }
36+ size_t offset = ((size_t)size + page - 1) / page * page;
37+ size_t capacity = offset + page;
38+ unsigned char *image = mmap(NULL, capacity, PROT_READ | PROT_WRITE,
39+ MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
40+ if (image == MAP_FAILED) {
41+ fclose(file);
42+ return 1;
43+ }
44+ int failed = fread(image, 1, size, file) != (size_t)size;
45+ failed |= fclose(file) != 0;
46+ if (failed) {
47+ munmap(image, capacity);
48+ return 1;
49+ }
50+ unsigned char bridge[] = {0x53,0x49,0x89,0xf8,0x49,0x89,0xf1,0x48,0xb8,
51+ 0,0,0,0,0,0,0,0,0xff,0xd0,0x5b,0xc3};
52+ uint64_t address = (uintptr_t)image;
53+ memcpy(bridge + 9, &address, sizeof(address));
54+ memcpy(image + offset, bridge, sizeof(bridge));
55+ if (mprotect(image, capacity, PROT_READ | PROT_EXEC)) {
56+ munmap(image, capacity);
57+ return 1;
58+ }
59+ uint64_t (*pixel)(uint64_t, uint64_t, uint64_t) = (void *)(image + offset);
60+ uint64_t checksum = 0;
61+ uint64_t begin = ticks();
62+ for (uint64_t t = 0; t < 64; t++) {
63+ for (uint64_t y = 0; y < 112; y++) {
64+ for (uint64_t x = 0; x < 160; x++) {
65+ checksum += pixel(x, y, 2*t) & 15;
66+ }
67+ }
68+ }
69+ uint64_t elapsed = ticks() - begin;
70+ printf("%" PRIu64 " %" PRIu64 "\n", elapsed / 64, checksum);
71+ return munmap(image, capacity) != 0;
72+}

tests/measure.py added

@@ -0,0 +1,42 @@
1+from pathlib import Path
2+import statistics
3+import subprocess
4+import tempfile
5+
6+
7+ROOT = Path(__file__).resolve().parents[1]
8+
9+
10+def main():
11+ with tempfile.TemporaryDirectory() as directory:
12+ directory = Path(directory)
13+ runner = directory / 'measure'
14+ subprocess.run(['cc', '-std=c11', '-O2', '-Wall', '-Wextra', '-Werror',
15+ ROOT / 'tests/measure.c', '-o', runner], check=True)
16+ print('example backend bytes stack_instructions median_frame_ns checksum')
17+ for source in sorted((ROOT / 'examples').glob('*.remy')):
18+ checksums = set()
19+ for backend in ('baseline', 'allocated'):
20+ binary = directory / backend
21+ args = ['baseline'] if backend == 'baseline' else []
22+ subprocess.run([ROOT / '.build/remy', *args, source, binary],
23+ check=True, capture_output=True)
24+ subprocess.run([ROOT / '.build/remy', 'check', binary],
25+ check=True, capture_output=True)
26+ listing = subprocess.run([ROOT / '.build/remy', 'disassemble', binary],
27+ check=True, capture_output=True, text=True).stdout
28+ traffic = sum(' push ' in row or ' pop ' in row for row in listing.splitlines())
29+ samples = []
30+ for _ in range(9):
31+ result = subprocess.run([runner, binary], check=True, capture_output=True,
32+ text=True, timeout=30)
33+ elapsed, checksum = map(int, result.stdout.split())
34+ samples.append(elapsed)
35+ checksums.add(checksum)
36+ print(source.stem, backend, binary.stat().st_size, traffic,
37+ int(statistics.median(samples)), checksum)
38+ assert len(checksums) == 1, (source, checksums)
39+
40+
41+if __name__ == '__main__':
42+ main()

tests/native.py modified

@@ -122,6 +122,9 @@ def main():
122122 cases += [("let", "x1", ("+", "x", 1), ("*", "x1", "x1"))]
123123 cases += [("let", "a", ("+", "x", 3), ("+", "y", ("let", "b", "t", ("*", "a", "b")))),
124124 ("let", "x", "y", ("if", "t", ("+", "x", 1), ("-", "x", 1)))]
125+ cases += [("-", "x", ("if", "t", ("+", "x", "y"), ("*", "y", "t"))),
126+ ("shl", "x", ("let", "a", ("+", "y", "t"), ("-", "a", "x"))),
127+ ("let", "a", "x", ("+", ("if", "y", "a", "t"), ("if", "t", "y", "a")))]
125128 cases += [generate(rng, 5, ["x", "y", "t"]) for _ in range(117)]
126129 atoms = (0,1,MASK,'x','y','t')
127130 exhaustive = list(atoms)