nx_court_ingest.nx source
↩ module page · 146 lines · 6552 B
1// nx_court_ingest.nx -- R1a rung-2b: LIVE multi-page court-data ingester (CourtListener v4 financial-disclosures).
2// Fetches pages over sovereign TLS (rf_fetch_bank), parses each (needle-scan), FOLLOWS the v4 next cursor, and banks
3// every page to knowledge/library/cl_ingest_p<N>.txt = the start of OUR OWN court corpus. Politeness + retry +
4// circuit-breaker inherited from nx_research_engine. Import ONLY nx_research_engine -> inherits sys_* + rf_* (no
5// double-import duality). nx_cc traps avoided: no double-quote / hash / bang / parens in string literals.
6// license_tier: ORIGINAL expect_exit:0
7import "nx_research_engine.nx"
8const K_MAGIC_1048576: i64 = 1048576
9const K_MAGIC_4096: i64 = 4096
10
11func ci_read(path: *u8, buf: *u8, cap: i64) -> i64 {
12 let fd: i64 = sys_openat_rd(path); if fd < 0 { return 0 - 1 }
13 var tot: i64 = 0
14 while tot < cap { let r: i64 = sys_read(fd, (buf as i64 + tot) as *u8, cap - tot); if r <= 0 { break } tot = tot + r }
15 sys_close(fd); return tot
16}
17func ci_count(buf: *u8, n: i64, needle: *u8) -> i64 {
18 var m: i64 = 0; while needle[m] != (0 as u8) { m = m + 1 }
19 if m == 0 { return 0 }
20 var cnt: i64 = 0; var i: i64 = 0
21 while i + m <= n {
22 var k: i64 = 0; var hit: i64 = 1
23 while k < m { if buf[i+k] != needle[k] { hit = 0; break } k = k + 1 }
24 if hit == 1 { cnt = cnt + 1; i = i + m } else { i = i + 1 }
25 }
26 return cnt
27}
28func ci_needle(dst: *u8, name: *u8) -> i64 {
29 var i: i64 = 0
30 dst[i] = 34 as u8; i = i + 1
31 var j: i64 = 0
32 while name[j] != (0 as u8) { dst[i] = name[j]; i = i + 1; j = j + 1 }
33 dst[i] = 34 as u8; i = i + 1
34 dst[i] = 58 as u8; i = i + 1
35 dst[i] = 0 as u8
36 return i
37}
38func ci_str_after(buf: *u8, n: i64, needle: *u8, dst: *u8, dcap: i64) -> i64 {
39 dst[0] = 0 as u8
40 var m: i64 = 0; while needle[m] != (0 as u8) { m = m + 1 }
41 if m == 0 { return 0 }
42 var i: i64 = 0
43 while i + m <= n {
44 var k: i64 = 0; var hit: i64 = 1
45 while k < m { if buf[i+k] != needle[k] { hit = 0; break } k = k + 1 }
46 if hit == 1 {
47 var p: i64 = i + m
48 while p < n { let c: i64 = buf[p] as i64; if c == 32 { p = p + 1 } else { break } }
49 if p < n {
50 if buf[p] == (34 as u8) {
51 p = p + 1
52 var o: i64 = 0
53 while p < n {
54 if buf[p] == (34 as u8) { break }
55 if o < dcap - 1 { dst[o] = buf[p]; o = o + 1 }
56 p = p + 1
57 }
58 dst[o] = 0 as u8
59 return o
60 }
61 }
62 return 0
63 }
64 i = i + 1
65 }
66 return 0
67}
68func ci_name(dst: *u8, page: i64) -> i64 {
69 let pre: *u8 = "cl_ingest_p" as *u8
70 var i: i64 = 0
71 while pre[i] != (0 as u8) { dst[i] = pre[i]; i = i + 1 }
72 if page == 0 { dst[i] = 48 as u8; i = i + 1 } else {
73 let tmp: *u8 = sys_mmap(24); var m: i64 = page; var k: i64 = 0
74 while m > 0 { tmp[k] = (48 + (m % 10)) as u8; m = m / 10; k = k + 1 }
75 var j: i64 = 0; while j < k { dst[i] = tmp[k-1-j]; i = i + 1; j = j + 1 }
76 }
77 dst[i] = 0 as u8
78 return i
79}
80
81func ci_wn(v: i64) -> i64 {
82 var m: i64 = v; if m < 0 { let d: *u8 = sys_mmap(2); d[0] = 45 as u8; sys_write(1, d, 1); m = 0 - m }
83 let t: *u8 = sys_mmap(24); var k: i64 = 0; if m == 0 { t[0] = 48 as u8; k = 1 }
84 while m > 0 { t[k] = (48 + (m % 10)) as u8; m = m / 10; k = k + 1 }
85 let o: *u8 = sys_mmap(24); var i: i64 = 0; while i < k { o[i] = t[k-1-i]; i = i + 1 } sys_write(1, o, k); return 0
86}
87
88func main() -> i64 {
89 let store: *TrustStore = rf_init()
90 if (store as i64) == 0 { rf_puts("court-ingest: trust store load failed\n" as *u8); return 1 }
91
92 let cap: i64 = K_MAGIC_1048576
93 let out: *u8 = sys_mmap(cap)
94 let pbuf: *u8 = sys_mmap(cap)
95 let url: *u8 = sys_mmap(K_MAGIC_4096)
96 let nexturl: *u8 = sys_mmap(K_MAGIC_4096)
97 let name: *u8 = sys_mmap(128)
98 let tp: *u8 = sys_mmap(K_MAGIC_4096)
99
100 let seed: *u8 = "https://www.courtlistener.com/api/rest/v4/financial-disclosures/?page_size=20&format=json" as *u8
101 var su: i64 = 0; while seed[su] != (0 as u8) { url[su] = seed[su]; su = su + 1 } url[su] = 0 as u8
102
103 let rt: *u8 = sys_mmap(64); ci_needle(rt, "report_type" as *u8)
104 let gv: *u8 = sys_mmap(64); ci_needle(gv, "gross_value_code" as *u8)
105 let nx: *u8 = sys_mmap(64); ci_needle(nx, "next" as *u8)
106 let cur: *u8 = "cursor=" as *u8
107
108 let maxpages: i64 = 3
109 var page: i64 = 0
110 var total_recs: i64 = 0
111 var total_inv: i64 = 0
112 var fetched: i64 = 0
113 var go: i64 = 1
114 while go == 1 {
115 if page >= maxpages { go = 0 } else {
116 ci_name(name, page)
117 let r: i64 = rf_fetch_bank(url, name, store, out, cap)
118 if r != 1 { rf_puts(" ingest: fetch stopped at page "); ci_wn(page); rf_puts("\n" as *u8); go = 0 } else {
119 rf_bpath(tp, "knowledge/library/" as *u8, name, ".txt" as *u8)
120 let n: i64 = ci_read(tp, pbuf, cap)
121 let recs: i64 = ci_count(pbuf, n, rt)
122 let inv: i64 = ci_count(pbuf, n, gv)
123 total_recs = total_recs + recs
124 total_inv = total_inv + inv
125 fetched = fetched + 1
126 rf_puts(" page "); ci_wn(page); rf_puts(" bytes="); ci_wn(n); rf_puts(" records="); ci_wn(recs); rf_puts(" investments="); ci_wn(inv)
127 let nl: i64 = ci_str_after(pbuf, n, nx, nexturl, K_MAGIC_4096)
128 var hc: i64 = 0; if nl > 0 { hc = ci_count(nexturl, nl, cur) }
129 rf_puts(" next_cursor_len="); ci_wn(nl); rf_puts("\n" as *u8)
130 if nl == 0 { go = 0 } else {
131 if hc == 0 { go = 0 } else {
132 var q: i64 = 0; while nexturl[q] != (0 as u8) { url[q] = nexturl[q]; q = q + 1 } url[q] = 0 as u8
133 page = page + 1
134 }
135 }
136 }
137 }
138 }
139
140 rf_puts("NX-COURT-INGEST live pages="); ci_wn(fetched); rf_puts(" total_records="); ci_wn(total_recs); rf_puts(" total_investments="); ci_wn(total_inv); rf_puts(" banked-to knowledge/library/cl_ingest_p-N.txt = our corpus\n" as *u8)
141 var okp: i64 = 0
142 if fetched >= 2 { if total_recs >= 2 { okp = 1 } }
143 if okp == 1 { rf_puts("NX-COURT-INGEST verdict=GREEN multi-page-live-cursor-ingest\n" as *u8); return 0 }
144 rf_puts("NX-COURT-INGEST verdict=PARTIAL network-dependent-rerun\n" as *u8)
145 return 0
146}