nx_qplan.nx source
↩ module page · 156 lines · 7844 B
1// nx_qplan.nx -- COST-BASED QUERY PLANNER, the optimizer that makes the engine intelligent (the capstone
2// over colframe/vecexec/distexec/parexec). What separates a query engine from a bag of operators is a
3// planner that ESTIMATES the cost of each execution strategy and CHOOSES the cheapest -- and ours is
4// grounded in the numbers we MEASURED, not guessed (that is the whole point of having benched parexec).
5//
6// The decision the bench proved and this encodes: a job splits into a SCAN cost (bandwidth-bound, barely
7// parallelises -- 4 workers fight one memory bus) and a per-row COMPUTE cost (CPU-bound, parallelises by
8// ~ncores x efficiency). Parallel execution also pays a fixed FORK/COW cost. So:
9// cost_seq(Mrows, cw) = scan_us*Mrows + comp_us*cw*Mrows
10// cost_par(Mrows, cw) = fork_us + scan_us*Mrows*bw_penalty + (comp_us*cw*Mrows)*1000/(ncores*eff_permille)
11// Pure scan (cw=0): parallel adds fork+penalty and divides nothing -> SEQUENTIAL wins (we measured 0.66x).
12// Heavy compute (cw high): the /ncores term dominates -> PARALLEL wins (we measured 3.10x). Small data:
13// fork_us dwarfs the work -> SEQUENTIAL. All constants live in analyst_qplan.conf (rule 11), seeded FROM
14// the measured bench. license_tier: ORIGINAL No hardware writes (Rule 26).
15import "nx_syscalls.nx"
16import "nx_estate_path.nx" // ep_anchor: the CWD must not decide this organ's verdict
17const QP_MAGIC_9050: i64 = 9050
18const QP_MAGIC_5273: i64 = 5273
19const QP_MAGIC_50000: i64 = 50000
20const QP_MAGIC_1200: i64 = 1200
21
22const QP_OUT: i64 = 4096
23const QP_CONF_CAP: i64 = 4096
24const QP_TAB: i64 = 9
25const QP_NL: i64 = 10
26const QP_HASH: i64 = 35
27const QP_ZERO: i64 = 48
28const QP_SEQ: i64 = 0
29const QP_PAR: i64 = 1
30
31func qp_atoi_rng(buf: *u8, a: i64, b: i64) -> i64 {
32 var v: i64 = 0
33 var i: i64 = a
34 while i < b { let c: i64 = buf[i] as i64; if c >= QP_ZERO { if c <= (QP_ZERO + 9) { v = v * 10 + (c - QP_ZERO) } } i = i + 1 }
35 return v
36}
37// one calibration constant from analyst_qplan.conf ("key <TAB> value"), else dflt.
38func qp_conf(key: *u8, dflt: i64) -> i64 {
39 let fd: i64 = sys_openat_rd("knowledge/registry/analyst_qplan.conf" as *u8)
40 if fd < 0 { return dflt }
41 let cb: *u8 = sys_mmap(QP_CONF_CAP)
42 var n: i64 = 0
43 var r: i64 = 1
44 while r > 0 { if n >= QP_CONF_CAP { r = 0 } else { r = sys_read(fd, (((cb as i64) + n) as *u8), QP_CONF_CAP - n); if r > 0 { n = n + r } } }
45 sys_close(fd)
46 var out: i64 = dflt
47 var found: i64 = 0
48 var ls: i64 = 0
49 while ls < n {
50 var le: i64 = ls
51 var e: i64 = 0
52 while e == 0 { if le >= n { e = 1 } else { if cb[le] == (QP_NL as u8) { e = 1 } else { le = le + 1 } } }
53 if le > ls { if cb[ls] != (QP_HASH as u8) { if found == 0 {
54 var tb: i64 = ls
55 var f: i64 = 0
56 while f == 0 { if tb >= le { f = 1 } else { if cb[tb] == (QP_TAB as u8) { f = 1 } else { tb = tb + 1 } } }
57 if tb < le {
58 var same: i64 = 1
59 var j: i64 = 0
60 while key[j] != (0 as u8) { if ls + j >= tb { same = 0 } else { if cb[ls + j] != key[j] { same = 0 } } j = j + 1 }
61 if ls + j != tb { same = 0 }
62 if same == 1 { out = qp_atoi_rng(cb, tb + 1, le); found = 1 }
63 }
64 } } }
65 ls = le + 1
66 }
67 return out
68}
69
70// estimated microseconds for SEQUENTIAL execution of a job: Mrows million rows, cw = per-row compute
71// weight (0 = pure bandwidth-bound scan; higher = heavier per-row arithmetic).
72func qp_cost_seq(mrows: i64, cw: i64) -> i64 {
73 let scan_us: i64 = qp_conf("scan_us_per_mrow" as *u8, QP_MAGIC_9050)
74 let comp_us: i64 = qp_conf("compute_us_per_mrow_per_op" as *u8, QP_MAGIC_5273)
75 return scan_us * mrows + comp_us * cw * mrows
76}
77// estimated microseconds for PARALLEL (fork-per-partition) execution.
78func qp_cost_par(mrows: i64, cw: i64, ncores: i64) -> i64 {
79 let scan_us: i64 = qp_conf("scan_us_per_mrow" as *u8, QP_MAGIC_9050)
80 let comp_us: i64 = qp_conf("compute_us_per_mrow_per_op" as *u8, QP_MAGIC_5273)
81 let fork_us: i64 = qp_conf("fork_us" as *u8, QP_MAGIC_50000)
82 let bw_pen: i64 = qp_conf("bw_parallel_penalty_permille" as *u8, QP_MAGIC_1200)
83 let eff: i64 = qp_conf("parallel_eff_permille" as *u8, 770)
84 var nc: i64 = ncores
85 if nc < 1 { nc = 1 }
86 let scan_par: i64 = scan_us * mrows * bw_pen / 1000
87 let comp_seq: i64 = comp_us * cw * mrows
88 var comp_par: i64 = comp_seq
89 let eff_cores: i64 = nc * eff
90 if eff_cores > 0 { comp_par = comp_seq * 1000 / eff_cores }
91 return fork_us + scan_par + comp_par
92}
93// THE CHOICE: min-cost strategy. Returns QP_SEQ or QP_PAR.
94func qp_choose(mrows: i64, cw: i64, ncores: i64) -> i64 {
95 let cs: i64 = qp_cost_seq(mrows, cw)
96 let cp: i64 = qp_cost_par(mrows, cw, ncores)
97 if cp < cs { return QP_PAR }
98 return QP_SEQ
99}
100
101func qp_raw(out: *u8, o: i64, s: *u8) -> i64 { var i: i64 = 0; while s[i] != (0 as u8) { out[o] = s[i]; o = o + 1; i = i + 1 } return o }
102func qp_num(out: *u8, o: i64, v: i64) -> i64 {
103 var oo: i64 = o
104 var m: i64 = v
105 if m < 0 { out[oo] = 45 as u8; oo = oo + 1; m = 0 - m }
106 let t: *u8 = sys_mmap(24)
107 var k: i64 = 0
108 if m == 0 { t[0] = QP_ZERO as u8; k = 1 }
109 while m > 0 { t[k] = (QP_ZERO + (m % 10)) as u8; m = m / 10; k = k + 1 }
110 var i: i64 = k - 1
111 while i >= 0 { out[oo] = t[i]; oo = oo + 1; i = i - 1 }
112 return oo
113}
114func qp_atoi(s: *u8) -> i64 { var v: i64 = 0; var i: i64 = 0; var go: i64 = 1; while go == 1 { let c: i64 = s[i] as i64; if c < 48 { go = 0 } else { if c > 57 { go = 0 } else { v = v * 10 + (c - 48); i = i + 1 } } } return v }
115
116// plan a job -> JSON with the estimates, the choice, and the reason
117func qp_plan_json(mrows: i64, cw: i64, ncores: i64, out: *u8) -> i64 {
118 let cs: i64 = qp_cost_seq(mrows, cw)
119 let cp: i64 = qp_cost_par(mrows, cw, ncores)
120 let ch: i64 = qp_choose(mrows, cw, ncores)
121 var o: i64 = 0
122 o = qp_raw(out, o, "{\"tool\":\"nx_qplan\",\"verb\":\"plan\",\"m_rows\":" as *u8)
123 o = qp_num(out, o, mrows)
124 o = qp_raw(out, o, ",\"compute_weight\":" as *u8)
125 o = qp_num(out, o, cw)
126 o = qp_raw(out, o, ",\"ncores\":" as *u8)
127 o = qp_num(out, o, ncores)
128 o = qp_raw(out, o, ",\"est_sequential_us\":" as *u8)
129 o = qp_num(out, o, cs)
130 o = qp_raw(out, o, ",\"est_parallel_us\":" as *u8)
131 o = qp_num(out, o, cp)
132 o = qp_raw(out, o, ",\"chosen\":\"" as *u8)
133 if ch == QP_PAR { o = qp_raw(out, o, "parallel" as *u8) } else { o = qp_raw(out, o, "sequential" as *u8) }
134 o = qp_raw(out, o, "\",\"reason\":\"" as *u8)
135 if cw == 0 { o = qp_raw(out, o, "bandwidth-bound scan -- parallel workers contend for one memory bus and pay fork cost; sequential wins" as *u8) } else {
136 if ch == QP_PAR { o = qp_raw(out, o, "compute-bound -- the per-row work parallelises across cores past the fork cost; parallel wins" as *u8) } else { o = qp_raw(out, o, "too small / too light -- fork overhead exceeds the parallel saving; sequential wins" as *u8) } }
137 o = qp_raw(out, o, "\",\"grounded_in\":\"MEASURED constants (analyst_qplan.conf, seeded from the nx_parexec bench)\"}\n" as *u8)
138 out[o] = 0 as u8
139 return o
140}
141func main(argc: i64, argv: *i64) -> i64 {
142 // ANCHOR FIRST (2026-08-04, nx_cwdguard finding): this organ reads a RELATIVE
143 // knowledge/ path, so its answer depended on where it was launched. No-op when
144 // already at the estate root, so the cron/MCP context is unchanged.
145 ep_anchor()
146 let out: *u8 = sys_mmap(QP_OUT)
147 var mrows: i64 = 40
148 var cw: i64 = 0
149 var ncores: i64 = 4
150 if argc >= 2 { mrows = qp_atoi(argv[1] as *u8) }
151 if argc >= 3 { cw = qp_atoi(argv[2] as *u8) }
152 if argc >= 4 { ncores = qp_atoi(argv[3] as *u8) }
153 let n: i64 = qp_plan_json(mrows, cw, ncores, out)
154 sys_write(1, out, n)
155 return 0
156}