nx_crawl_sufficiency_cli.nx source
↩ module page · 74 lines · 3834 B
1// nx_crawl_sufficiency_cli.nx -- CLI over nx_crawl_sufficiency (the /compare/webscraping R5 lib): judge a set of
2// already-fetched pages against a query and print the sufficiency trajectory, one line per page, then the
3// verdict. HTML input goes through nx_block_density (fit text) so the judge sees the article, not the chrome.
4// usage: nx_crawl_sufficiency_cli judge "<query words>" <file> [file...]
5// exit: 0 STOP reached (confident or saturated) | 4 CONTINUE (more pages needed) | 2 usage | 3 nothing readable
6// license_tier: ORIGINAL No hw writes (Rule 26).
7import "nx_syscalls.nx"
8import "nx_crawl_sufficiency.nx"
9import "nx_block_density.nx"
10
11func sc_len(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } return n }
12func sc_puts(s: *u8) -> i64 { sys_write(1, s, sc_len(s)); return 0 }
13func sc_num(v: i64) -> i64 {
14 let t: *u8 = sys_mmap(32)
15 let n: i64 = cs_r_num(t, 32, 0, v)
16 sys_write(1, t, n)
17 sys_munmap(t, 32)
18 return 0
19}
20func sc_streq(a: *u8, b: *u8) -> i64 { var i: i64 = 0; while a[i] == b[i] { if a[i] == (0 as u8) { return 1 } i = i + 1 } return 0 }
21
22func main(argc: i64, argv: *i64) -> i64 {
23 if argc < 4 { sc_puts("usage: nx_crawl_sufficiency_cli judge \"<query words>\" <file> [file...]\n" as *u8); return 2 }
24 if sc_streq(argv[1] as *u8, "judge" as *u8) == 0 { sc_puts("usage: nx_crawl_sufficiency_cli judge \"<query words>\" <file> [file...]\n" as *u8); return 2 }
25 let q: *u8 = argv[2] as *u8
26 let st: *i64 = cs_new(q, sc_len(q))
27 sc_puts("query terms=" as *u8); sc_num(st[CS_S_QN]); sc_puts(" dropped=" as *u8); sc_num(st[CS_S_DROPPED_TERMS]); sc_puts("\n" as *u8)
28 let rep: *u8 = sys_mmap(8192)
29 let lenbox: *i64 = sys_mmap(16) as *i64
30 var readable: i64 = 0
31 var i: i64 = 3
32 while i < argc {
33 let path: *u8 = argv[i] as *u8
34 lenbox[0] = 0
35 let src: *u8 = sys_read_file(path, lenbox)
36 if (src as i64) == 0 { sc_puts("doc UNREADABLE " as *u8); sc_puts(path); sc_puts("\n" as *u8) } else {
37 readable = readable + 1
38 let n: i64 = lenbox[0]
39 // HTML? first non-whitespace byte is '<' -> fit text; otherwise the bytes are the text
40 var p: i64 = 0
41 var go: i64 = 1
42 while go == 1 { if p >= n { go = 0 } else { let c: i64 = src[p] as i64; if c == 32 { p = p + 1 } else { if c == 10 { p = p + 1 } else { if c == 13 { p = p + 1 } else { if c == 9 { p = p + 1 } else { go = 0 } } } } } }
43 var text: *u8 = src
44 var tn: i64 = n
45 var owned: i64 = 0
46 if p < n { if (src[p] as i64) == 60 {
47 let buf: *u8 = sys_mmap(n + 16)
48 tn = bd_fit_text(src, n, buf, n + 16)
49 text = buf
50 owned = 1
51 } }
52 let r: i64 = cs_add_doc(st, text, tn)
53 cs_confidence(st)
54 sc_puts("doc " as *u8); sc_puts(path); sc_puts(" text=" as *u8); sc_num(tn)
55 if r == 1 { sc_puts(" COUNTED" as *u8) }
56 if r == 0 { sc_puts(" NEAR-DUPLICATE" as *u8) }
57 if r < 0 { sc_puts(" REFUSED-history-full" as *u8) }
58 sc_puts(" coverage=" as *u8); sc_num(st[CS_S_COVERAGE])
59 sc_puts(" consistency=" as *u8); sc_num(st[CS_S_CONSISTENCY])
60 sc_puts(" saturation=" as *u8); sc_num(st[CS_S_SATURATION])
61 sc_puts(" confidence=" as *u8); sc_num(st[CS_S_CONFIDENCE]); sc_puts("\n" as *u8)
62 if owned == 1 { sys_munmap(text, n + 16) }
63 sys_free_file(src, n)
64 }
65 i = i + 1
66 }
67 if readable == 0 { sc_puts("SUFFICIENCY UNMEASURED: no readable input\n" as *u8); return 3 }
68 let rn: i64 = cs_report(st, rep, 8192)
69 sys_write(1, rep, rn)
70 let stop: i64 = st[CS_S_STOP]
71 cs_free(st)
72 if stop == CS_CONTINUE { return 4 }
73 return 0
74}