nx_field_discover.nx source
↩ module page · 280 lines · 16972 B
1// nx_field_discover.nx -- DISCOVER THE FIELD FOR ONE /compare DOMAIN (/compare/fieldwatch FW1, 2026-09-05): the thin
2// program over nx_field_lib. Reads buildroot/knowledge/compare/<dom>.seeds (seed|key|url|kind|why[|Display Name]), mirrors
3// every seed through the sovereign fetcher (provenance-pinned into knowledge/fetched/cmp_field_<dom>_<key>.<ext>), runs the
4// kind's extractor over the mirror, and writes buildroot/knowledge/compare/<dom>.field -- the population of rivals with
5// evidence counts, which the compare emitter renders and against which a matrix's columns are measured as a subset.
6// KINDS: wiki-raw gh-topic md-list are DISCOVERY seeds (the industry's own lists, read mechanically); named is a DECLARED
7// member of the population (operator correction 2026-09-05: Nexus Mods, LoversLab and the other major modding communities
8// were absent from every discovered list, so the population is ENUMERATED as data, every member's home page mirrored and
9// pinned, and the discovery seeds CORROBORATE it -- a member also named by a public list carries seeds_hit >= 2).
10//
11// Usage: nx_field_discover <dom> [--nofetch] --nofetch reuses the mirrors already on disk (no network)
12// Receipt: FIELD-OK dom= seeds= fetched= reused= failed= named= candidates= mentions= capped= bytes= (partition printed)
13// Exits: 0 ok | 2 usage | 3 refused (no seeds file, no seed rows, output unwritable) | 4 every seed failed
14// license_tier: ORIGINAL No hw writes (Rule 26). expect_exit: 0
15import "nx_syscalls.nx"
16import "nx_tool_run.nx"
17import "nx_field_lib.nx"
18
19const FD_ROOT: *u8 = "/volume1/homes/elderwesto/nishihost/"
20const FD_FETCHER: *u8 = "/volume1/homes/elderwesto/nishihost/_offc/nx_research_fetch.elf"
21const FD_SEEDS_DIR: *u8 = "buildroot/knowledge/compare/"
22const FD_MIRROR_DIR: *u8 = "knowledge/fetched/"
23const FD_MIRROR_PFX: *u8 = "cmp_field_"
24const FD_PATH_CAP: i64 = 1024
25const FD_KEY_CAP: i64 = 96
26// FD_MAX_SEEDS (62) REMOVED 2026-09-06: a fixed seed cap silently dropped 16 of 78 rows while the receipt still read
27// capped=0 (measured on modding.seeds the night the non-US community population landed: seeds=62 of 78 rows). A cap
28// that has to be guessed is a defect generator in both directions; the seed table is now SIZED FROM THE FILE by
29// fd_count_seed_rows, so every declared row is walked and seeds= is the population, never a prefix of it.
30const FD_SEED_F: i64 = 6 // seed|key|url|kind|why|Display Name -- the sixth field is read only for kind named
31const FD_SEED_MIN: i64 = 5 // a row needs at least seed|key|url|kind|why
32const FD_F_NAME: i64 = 5 // index of the Display Name field
33const FD_CAPCAP: i64 = 262144
34const FD_ARGVN: i64 = 8
35const FD_MODE: i64 = 0x1a4
36const FD_EXIT_USAGE: i64 = 2
37const FD_EXIT_REFUSE: i64 = 3
38const FD_EXIT_ALLFAIL: i64 = 4
39const FD_CH_NL: i64 = 10
40const FD_CH_PIPE: i64 = 124
41const FD_CH_HASH: i64 = 35
42const FD_CH_MINUS: i64 = 45
43const FD_OUTFD: i64 = 1
44const FD_ERRFD: i64 = 2
45
46func fd_slen(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } return n }
47// the population the seed table is sized from: every row that begins with seed| (comment and blank rows excluded)
48func fd_count_seed_rows(sb: *u8, sn: i64) -> i64 {
49 var n: i64 = 0
50 var p: i64 = 0
51 while p < sn {
52 var le: i64 = p
53 while le < sn { if (sb[le] as i64) == FD_CH_NL { break } le = le + 1 }
54 if le - p >= 5 { if (sb[p] as i64) == 115 { if (sb[p+1] as i64) == 101 { if (sb[p+2] as i64) == 101 { if (sb[p+3] as i64) == 100 { if (sb[p+4] as i64) == FD_CH_PIPE { n = n + 1 } } } } } }
55 p = le + 1
56 }
57 return n
58}
59func fd_out(s: *u8) -> i64 { sys_write(FD_OUTFD, s, fd_slen(s)); return 0 }
60func fd_err(s: *u8) -> i64 { sys_write(FD_ERRFD, s, fd_slen(s)); return 0 }
61func fd_cat(dst: *u8, off: i64, s: *u8) -> i64 { var i: i64 = 0; while s[i] != (0 as u8) { if off + i < FD_PATH_CAP - 1 { dst[off + i] = s[i] } i = i + 1 } var o: i64 = off + i; if o > FD_PATH_CAP - 1 { o = FD_PATH_CAP - 1 } dst[o] = 0 as u8; return o }
62func fd_catn(dst: *u8, off: i64, b: *u8, s: i64, e: i64) -> i64 { var i: i64 = 0; while s + i < e { if off + i < FD_PATH_CAP - 1 { dst[off + i] = b[s + i] } i = i + 1 } var o: i64 = off + i; if o > FD_PATH_CAP - 1 { o = FD_PATH_CAP - 1 } dst[o] = 0 as u8; return o }
63func fd_span_eq(b: *u8, s: i64, e: i64, lit: *u8) -> i64 {
64 var i: i64 = 0
65 while s + i < e { if lit[i] == (0 as u8) { return 0 } if b[s + i] != lit[i] { return 0 } i = i + 1 }
66 if lit[i] != (0 as u8) { return 0 }
67 return 1
68}
69func fd_streq(a: *u8, b: *u8) -> i64 { var i: i64 = 0; while a[i] != (0 as u8) { if a[i] != b[i] { return 0 } i = i + 1 } if b[i] != (0 as u8) { return 0 } return 1 }
70func fd_kind_code(b: *u8, s: i64, e: i64) -> i64 {
71 if fd_span_eq(b, s, e, "wiki-raw" as *u8) == 1 { return FL_KIND_WIKI }
72 if fd_span_eq(b, s, e, "gh-topic" as *u8) == 1 { return FL_KIND_GH }
73 if fd_span_eq(b, s, e, "md-list" as *u8) == 1 { return FL_KIND_MD }
74 if fd_span_eq(b, s, e, "named" as *u8) == 1 { return FL_KIND_NAMED }
75 return 0
76}
77func fd_ext(kind: i64) -> *u8 {
78 if kind == FL_KIND_WIKI { return ".txt" as *u8 }
79 if kind == FL_KIND_GH { return ".html" as *u8 }
80 if kind == FL_KIND_NAMED { return ".html" as *u8 }
81 return ".md" as *u8
82}
83func fd_exists(path: *u8, lenout: *i64) -> *u8 {
84 let lp: *i64 = sys_mmap(16) as *i64
85 let b: *u8 = sys_read_file(path, lp)
86 if (b as i64) == 0 { lenout[0] = 0; return 0 as *u8 }
87 lenout[0] = lp[0]
88 return b
89}
90// PURE: are two mirrors the same bytes? (FW5 drift: a seed page that changed since the last beat is DRIFTED; a dynamic page
91// that carries a per-request nonce reads DRIFTED every beat, so this is a COUNT the stamp publishes, never an alarm --
92// the alarm is the .field diff FW9 raises when a rival appears or vanishes)
93func fd_same(a: *u8, an: i64, b: *u8, bn: i64) -> i64 {
94 if an != bn { return 0 }
95 var i: i64 = 0
96 while i < an { if a[i] != b[i] { return 0 } i = i + 1 }
97 return 1
98}
99// fork the sovereign fetcher: <url> <dest-name>; success = the SAVED line in its receipt
100func fd_fetch(url: *u8, dest: *u8, cap: *u8, olen: *i64) -> i64 {
101 let av: *i64 = sys_mmap(8 * FD_ARGVN) as *i64
102 av[0] = FD_FETCHER as i64
103 av[1] = url as i64
104 av[2] = dest as i64
105 av[3] = 0
106 let rc: i64 = tr_run_capture(FD_FETCHER, av, cap, FD_CAPCAP, olen)
107 // the receipt is the witness, not the rc: a fetch that saved nothing prints no SAVED line
108 let key: *u8 = "SAVED knowledge/fetched/" as *u8
109 let kl: i64 = fd_slen(key)
110 var i: i64 = 0
111 while i + kl <= olen[0] {
112 var m: i64 = 0
113 var j: i64 = 0
114 while j < kl { if cap[i + j] == key[j] { m = m + 1 } j = j + 1 }
115 if m == kl { return 1 }
116 i = i + 1
117 }
118 return 0
119}
120
121func main(argc: i64, argv: *i64) -> i64 {
122 if argc < 2 { fd_err("usage: nx_field_discover <dom> [--nofetch]\n" as *u8); return FD_EXIT_USAGE }
123 // CWD ANCHOR (2026-09-05): the mirror dir and the fetcher's provenance journal are SERVING-ROOT-relative
124 // (knowledge/fetched/, knowledge/status/), and the seeds path is built from FD_ROOT. A caller that starts in
125 // buildroot (nx_sov_build_run anchors there; the build queue starts in the serving root) would otherwise fork
126 // the fetcher into the WRONG tree and read mirrors that are not there. Announced, never silent.
127 sys_chdir(FD_ROOT)
128 fd_out("[nx_field_discover] CWD anchored to " as *u8); fd_out(FD_ROOT); fd_out("\n" as *u8)
129 let dom: *u8 = argv[1] as *u8
130 var nofetch: i64 = 0
131 if argc >= 3 { if fd_streq(argv[2] as *u8, "--nofetch" as *u8) == 1 { nofetch = 1 } }
132 let spath: *u8 = sys_mmap(FD_PATH_CAP)
133 var o: i64 = fd_cat(spath, 0, FD_ROOT)
134 o = fd_cat(spath, o, FD_SEEDS_DIR)
135 o = fd_cat(spath, o, dom)
136 o = fd_cat(spath, o, ".seeds" as *u8)
137 let slen: *i64 = sys_mmap(16) as *i64
138 let sb: *u8 = fd_exists(spath, slen)
139 if (sb as i64) == 0 { fd_err("FIELD-REFUSE no seeds file: " as *u8); fd_err(spath); fd_err("\n" as *u8); return FD_EXIT_REFUSE }
140 let sn: i64 = slen[0]
141 let t: *i64 = fl_tbl_new()
142 // the seed table is SIZED FROM THE FILE: every seed| row gets a slot, so no declared member can be dropped in silence
143 let nrows: i64 = fd_count_seed_rows(sb, sn)
144 let seedkeys: *i64 = sys_mmap(8 * (nrows + 1)) as *i64
145 let keyarena: *u8 = sys_mmap(FD_KEY_CAP * (nrows + 1))
146 let cap: *u8 = sys_mmap(FD_CAPCAP)
147 let olen: *i64 = sys_mmap(16) as *i64
148 let url: *u8 = sys_mmap(FD_PATH_CAP)
149 let dest: *u8 = sys_mmap(FD_PATH_CAP)
150 let mpath: *u8 = sys_mmap(FD_PATH_CAP)
151 let nbuf: *u8 = sys_mmap(FD_PATH_CAP)
152 var nseeds: i64 = 0
153 var fetched: i64 = 0
154 var reused: i64 = 0
155 var failed: i64 = 0
156 var named: i64 = 0
157 var drifted: i64 = 0 // FW5: a mirror that changed since the previous fetch
158 var same: i64 = 0 // re-fetched and byte-identical
159 var fresh: i64 = 0 // fetched for the first time (no previous mirror to compare)
160 var p: i64 = 0
161 while p < sn {
162 var le: i64 = p
163 while le < sn { if (sb[le] as i64) == FD_CH_NL { break } le = le + 1 }
164 if le > p { if (sb[p] as i64) != FD_CH_HASH {
165 // split the row into up to FD_SEED_F fields (start/end pairs)
166 let fs: *i64 = sys_mmap(8 * 2 * FD_SEED_F) as *i64
167 var nf: i64 = 0
168 var f0: i64 = p
169 var q: i64 = p
170 while q <= le {
171 var atend: i64 = 0
172 if q == le { atend = 1 } else { if (sb[q] as i64) == FD_CH_PIPE { atend = 1 } }
173 if atend == 1 { if nf < FD_SEED_F { fs[nf*2] = f0; fs[nf*2 + 1] = q; nf = nf + 1 } f0 = q + 1 }
174 q = q + 1
175 }
176 if nf >= FD_SEED_MIN { if fd_span_eq(sb, fs[0], fs[1], "seed" as *u8) == 1 { if nseeds <= nrows {
177 let kind: i64 = fd_kind_code(sb, fs[6], fs[7])
178 let key: *u8 = (keyarena as i64 + nseeds * FD_KEY_CAP) as *u8
179 var kk: i64 = 0
180 while fs[2] + kk < fs[3] { if kk < FD_KEY_CAP - 1 { key[kk] = sb[fs[2] + kk] } kk = kk + 1 }
181 if kk > FD_KEY_CAP - 1 { kk = FD_KEY_CAP - 1 }
182 key[kk] = 0 as u8
183 seedkeys[nseeds] = key as i64
184 fd_catn(url, 0, sb, fs[4], fs[5])
185 var d: i64 = fd_cat(dest, 0, FD_MIRROR_PFX)
186 d = fd_cat(dest, d, dom)
187 d = fd_cat(dest, d, "_" as *u8)
188 d = fd_cat(dest, d, key)
189 d = fd_cat(dest, d, fd_ext(kind))
190 var m: i64 = fd_cat(mpath, 0, FD_ROOT)
191 m = fd_cat(mpath, m, FD_MIRROR_DIR)
192 m = fd_cat(mpath, m, dest)
193 fd_out("FIELD-SEED key=" as *u8); fd_out(key); fd_out(" kind=" as *u8); fd_out(fl_kind_name(kind)); fd_out(" url=" as *u8); fd_out(url); fd_out("\n" as *u8)
194 var have: i64 = 0
195 if kind == 0 {
196 fd_out("FIELD-SEED-SKIP key=" as *u8); fd_out(key); fd_out(" unknown kind -- declare wiki-raw, gh-topic, md-list or named\n" as *u8)
197 failed = failed + 1
198 } else {
199 if kind == FL_KIND_NAMED {
200 // a DECLARED member: its name is the sixth field (or the key), its link is the url. It is IN the
201 // population whether or not its home page can be mirrored today -- the mirror is evidence about
202 // the member, not the membership -- and the fetch outcome is still counted below by name.
203 if nf > FD_F_NAME { fd_catn(nbuf, 0, sb, fs[FD_F_NAME*2], fs[FD_F_NAME*2 + 1]) } else { fd_cat(nbuf, 0, key) }
204 if fl_named_add(t, nbuf, url, nseeds) >= 0 { named = named + 1 }
205 fd_out("FIELD-SEED-NAMED key=" as *u8); fd_out(key); fd_out(" name=" as *u8); fd_out(nbuf); fd_out("\n" as *u8)
206 }
207 // FW5: snapshot the previous mirror BEFORE the fetcher overwrites it, so drift is measured, never inferred
208 let ol: *i64 = sys_mmap(16) as *i64
209 let ob: *u8 = fd_exists(mpath, ol)
210 if nofetch == 0 {
211 if fd_fetch(url, dest, cap, olen) == 1 { fetched = fetched + 1; have = 1 } else { fd_out("FIELD-SEED-FETCH-FAILED key=" as *u8); fd_out(key); fd_out(" (the fetcher printed no SAVED line; a mirror already on disk is reused if present)\n" as *u8) }
212 }
213 let ml: *i64 = sys_mmap(16) as *i64
214 let mb: *u8 = fd_exists(mpath, ml)
215 if (mb as i64) != 0 {
216 if have == 0 { reused = reused + 1 }
217 if have == 1 { if (ob as i64) != 0 {
218 if fd_same(ob, ol[0], mb, ml[0]) == 0 { drifted = drifted + 1; fd_out("FIELD-SEED-DRIFTED key=" as *u8); fd_out(key); fd_out(" old_bytes=" as *u8); fl_putn(FD_OUTFD, ol[0]); fd_out(" new_bytes=" as *u8); fl_putn(FD_OUTFD, ml[0]); fd_out("\n" as *u8) } else { same = same + 1 }
219 } else { fresh = fresh + 1 } }
220 var added: i64 = 0
221 if kind == FL_KIND_WIKI { added = fl_wiki_scan(t, mb, ml[0], nseeds) }
222 if kind == FL_KIND_GH { added = fl_gh_scan(t, mb, ml[0], nseeds) }
223 if kind == FL_KIND_MD { added = fl_md_scan(t, mb, ml[0], nseeds) }
224 fd_out("FIELD-SEED-SCANNED key=" as *u8); fd_out(key); fd_out(" mirror_bytes=" as *u8); fl_putn(FD_OUTFD, ml[0]); fd_out(" occurrences=" as *u8); fl_putn(FD_OUTFD, added); fd_out("\n" as *u8)
225 } else {
226 if have == 0 { failed = failed + 1 }
227 fd_out("FIELD-SEED-NO-MIRROR key=" as *u8); fd_out(key); fd_out(" " as *u8); fd_out(mpath); fd_out("\n" as *u8)
228 }
229 }
230 nseeds = nseeds + 1
231 } } }
232 } }
233 p = le + 1
234 }
235 if nseeds < 1 { fd_err("FIELD-REFUSE the seeds file declares no seed rows\n" as *u8); return FD_EXIT_REFUSE }
236 let scanned: i64 = fetched + reused
237 // the .field file: header carries the partition so a reader never mistakes a prefix for the population
238 let opath: *u8 = sys_mmap(FD_PATH_CAP)
239 var oo: i64 = fd_cat(opath, 0, FD_ROOT)
240 oo = fd_cat(opath, oo, FD_SEEDS_DIR)
241 oo = fd_cat(opath, oo, dom)
242 oo = fd_cat(opath, oo, ".field" as *u8)
243 let fd: i64 = sys_openat_wr(opath, FD_MODE)
244 if fd < 0 { fd_err("FIELD-REFUSE cannot open output " as *u8); fd_err(opath); fd_err("\n" as *u8); return FD_EXIT_REFUSE }
245 fl_puts(fd, "# " as *u8); fl_puts(fd, dom); fl_puts(fd, ".field -- THE FIELD, written by nx_field_discover from " as *u8); fl_puts(fd, dom); fl_puts(fd, ".seeds: DECLARED members (kind named) plus what the industry's own lists name (wiki-raw gh-topic md-list). NEVER hand-edit: add a seed row.\n" as *u8)
246 fl_puts(fd, "# Row: rival|<name>|<seeds_hit>|<mentions>|<first_seed_key>|<link>|<kind> rows sorted by seeds_hit then mentions, descending; a declared member also named by a public list carries seeds_hit of 2 or more\n" as *u8)
247 fl_puts(fd, "field|seeds=" as *u8); fl_putn(fd, nseeds)
248 fl_puts(fd, "|scanned=" as *u8); fl_putn(fd, scanned)
249 fl_puts(fd, "|fetched=" as *u8); fl_putn(fd, fetched)
250 fl_puts(fd, "|reused=" as *u8); fl_putn(fd, reused)
251 fl_puts(fd, "|failed=" as *u8); fl_putn(fd, failed)
252 fl_puts(fd, "|named=" as *u8); fl_putn(fd, named)
253 fl_puts(fd, "|candidates=" as *u8); fl_putn(fd, fl_count(t))
254 fl_puts(fd, "|mentions=" as *u8); fl_putn(fd, fl_mentions_total(t))
255 fl_puts(fd, "|capped=" as *u8); fl_putn(fd, fl_capped(t))
256 fl_puts(fd, "\n" as *u8)
257 let rows: i64 = fl_emit(t, fd, seedkeys, nseeds)
258 sys_close(fd)
259 let wl: *i64 = sys_mmap(16) as *i64
260 fd_exists(opath, wl)
261 fd_out("FIELD-OK dom=" as *u8); fd_out(dom)
262 fd_out(" seeds=" as *u8); fl_putn(FD_OUTFD, nseeds)
263 fd_out(" scanned=" as *u8); fl_putn(FD_OUTFD, scanned)
264 fd_out(" fetched=" as *u8); fl_putn(FD_OUTFD, fetched)
265 fd_out(" reused=" as *u8); fl_putn(FD_OUTFD, reused)
266 fd_out(" failed=" as *u8); fl_putn(FD_OUTFD, failed)
267 fd_out(" sum=" as *u8); fl_putn(FD_OUTFD, scanned + failed)
268 fd_out(" named=" as *u8); fl_putn(FD_OUTFD, named)
269 fd_out(" drifted=" as *u8); fl_putn(FD_OUTFD, drifted)
270 fd_out(" same=" as *u8); fl_putn(FD_OUTFD, same)
271 fd_out(" fresh=" as *u8); fl_putn(FD_OUTFD, fresh)
272 fd_out(" candidates=" as *u8); fl_putn(FD_OUTFD, rows)
273 fd_out(" mentions=" as *u8); fl_putn(FD_OUTFD, fl_mentions_total(t))
274 fd_out(" capped=" as *u8); fl_putn(FD_OUTFD, fl_capped(t))
275 fd_out(" bytes=" as *u8); fl_putn(FD_OUTFD, wl[0])
276 fd_out(" out=" as *u8); fd_out(opath)
277 fd_out("\n" as *u8)
278 if scanned < 1 { if named < 1 { return FD_EXIT_ALLFAIL } }
279 return 0
280}