nx_doc_research_fetch.nx source
↩ module page · 108 lines · 8578 B
1// nx_doc_research_fetch.nx -- the Nishi RESEARCHER's sovereign web-fetch for THE NISHI DOCUMENT-INTELLIGENCE
2// arc: scanning / reading / extracting / classifying / cataloging / managing / making documents -> structured
3// data. GROWS the researcher team with a new domain (operator: "really nishi research this ... goes both ways
4// in nishi researcher and librarian and the team consuming, making, managing documents"). Mirrors the proven
5// nx_fin_research_fetch / nx_pub_research_fetch pattern: own TLS-1.3 (nx_https_fetch_follow) + Mozilla CA store,
6// each cited source -> knowledge/fetched/doc_*.raw so research COMPOUNDS to the Library and is grep-citable.
7// 100% sovereign (own TLS, nx_cc->nxasm, no curl/wget/gcc). Idempotent have-skip = re-run fills gaps (also the
8// per-fetch mmap-leak workaround -- run multiple passes; each pass is a fresh process). expect_exit: 0
9// license_tier: ORIGINAL
10import "nx_syscalls.nx"
11import "nx_x509_trust_store.nx"
12import "nx_trust_store_load_from_certdata.nx"
13import "nx_https_fetch_follow.nx"
14const K_MAGIC_4194304: i64 = 4194304
15const K_MAGIC_8388608: i64 = 8388608
16
17func ff_puts(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } sys_write(1, s, n); return 0 }
18func ff_putn(v: i64) -> i64 {
19 if v == 0 { sys_write(1, "0" as *u8, 1); return 0 }
20 var m: i64 = v
21 if m < 0 { sys_write(1, "-" as *u8, 1); m = 0 - m }
22 let d: *u8 = sys_mmap(24); var k: i64 = 0
23 while m > 0 { d[k] = (48 + (m % 10)) as u8; m = m / 10; k = k + 1 }
24 var j: i64 = k - 1
25 while j >= 0 { sys_write(1, ((d as i64)+j) as *u8, 1); j = j - 1 }
26 return 0
27}
28func have_file(path: *u8) -> i64 {
29 let fd: i64 = sys_openat_rd(path)
30 if fd < 0 { return 0 }
31 sys_close(fd)
32 return 1
33}
34func fetch_save(url: *u8, opath: *u8, store: *TrustStore, out: *u8, cap: i64) -> i64 {
35 if have_file(opath) == 1 { ff_puts(opath); ff_puts(" [have-skip]\n"); return 1 }
36 let status: *i64 = sys_mmap(8) as *i64
37 let n: i64 = nx_https_fetch_follow(url, store, out, cap, 6, status)
38 ff_puts(url); ff_puts(" status="); ff_putn(status[0]); ff_puts(" bytes="); ff_putn(n)
39 if n <= 0 { ff_puts(" FETCH-FAIL\n"); return 0 }
40 var gz: i64 = 0
41 if n >= 2 { if out[0] == 0x1f as u8 { if out[1] == 0x8b as u8 { gz = 1 } } }
42 if gz == 1 { ff_puts(" [GZIP-skip]\n"); return 0 }
43 let fd: i64 = sys_openat_wr(opath, 0x1a4)
44 if fd < 0 { ff_puts(" SAVE-FAIL\n"); return 0 }
45 sys_write(fd, out, n)
46 sys_close(fd)
47 ff_puts(" SAVED\n")
48 return 1
49}
50
51func main() -> i64 {
52 let r: i64 = nx_trust_store_load_from_certdata("data/mozilla_certdata.txt" as *u8, 512, K_MAGIC_4194304)
53 if r <= 0 { ff_puts("DOC: certdata load failed\n"); return 1 }
54 let store: *TrustStore = r as *TrustStore
55 ff_puts("CA roots="); ff_putn(trust_store_count(store)); ff_puts("\n")
56 let cap: i64 = K_MAGIC_8388608
57 let out: *u8 = sys_mmap(cap)
58 var ok: i64 = 0
59
60 ff_puts("== SCAN / CAPTURE (paper + image -> pixels/text) ==\n")
61 ok = ok + fetch_save("https://en.wikipedia.org/wiki/Optical_character_recognition" as *u8, "knowledge/fetched/doc_scan_ocr.raw" as *u8, store, out, cap)
62 ok = ok + fetch_save("https://en.wikipedia.org/wiki/Image_scanner" as *u8, "knowledge/fetched/doc_scan_scanner.raw" as *u8, store, out, cap)
63 ok = ok + fetch_save("https://en.wikipedia.org/wiki/Document_imaging" as *u8, "knowledge/fetched/doc_scan_imaging.raw" as *u8, store, out, cap)
64 ok = ok + fetch_save("https://en.wikipedia.org/wiki/Optical_mark_recognition" as *u8, "knowledge/fetched/doc_scan_omr.raw" as *u8, store, out, cap)
65 ok = ok + fetch_save("https://en.wikipedia.org/wiki/Handwriting_recognition" as *u8, "knowledge/fetched/doc_scan_handwriting.raw" as *u8, store, out, cap)
66
67 ff_puts("== READ / EXTRACT / STRUCTURE (layout, fields, entities, parsing) ==\n")
68 ok = ok + fetch_save("https://en.wikipedia.org/wiki/Document_layout_analysis" as *u8, "knowledge/fetched/doc_read_layout.raw" as *u8, store, out, cap)
69 ok = ok + fetch_save("https://en.wikipedia.org/wiki/Information_extraction" as *u8, "knowledge/fetched/doc_read_ie.raw" as *u8, store, out, cap)
70 ok = ok + fetch_save("https://en.wikipedia.org/wiki/Named-entity_recognition" as *u8, "knowledge/fetched/doc_read_ner.raw" as *u8, store, out, cap)
71 ok = ok + fetch_save("https://en.wikipedia.org/wiki/Data_extraction" as *u8, "knowledge/fetched/doc_read_extraction.raw" as *u8, store, out, cap)
72 ok = ok + fetch_save("https://en.wikipedia.org/wiki/Parsing" as *u8, "knowledge/fetched/doc_read_parsing.raw" as *u8, store, out, cap)
73 ok = ok + fetch_save("https://en.wikipedia.org/wiki/Regular_expression" as *u8, "knowledge/fetched/doc_read_regex.raw" as *u8, store, out, cap)
74 ok = ok + fetch_save("https://en.wikipedia.org/wiki/Semi-structured_data" as *u8, "knowledge/fetched/doc_read_semistructured.raw" as *u8, store, out, cap)
75 ok = ok + fetch_save("https://en.wikipedia.org/wiki/Electronic_data_interchange" as *u8, "knowledge/fetched/doc_read_edi.raw" as *u8, store, out, cap)
76
77 ff_puts("== CLASSIFY (what kind of document is this) ==\n")
78 ok = ok + fetch_save("https://en.wikipedia.org/wiki/Document_classification" as *u8, "knowledge/fetched/doc_cls_docclass.raw" as *u8, store, out, cap)
79 ok = ok + fetch_save("https://en.wikipedia.org/wiki/Statistical_classification" as *u8, "knowledge/fetched/doc_cls_statclass.raw" as *u8, store, out, cap)
80
81 ff_puts("== CATALOG / ORGANIZE (the LIBRARIAN: metadata, catalogs, vocab) ==\n")
82 ok = ok + fetch_save("https://en.wikipedia.org/wiki/Metadata" as *u8, "knowledge/fetched/doc_cat_metadata.raw" as *u8, store, out, cap)
83 ok = ok + fetch_save("https://en.wikipedia.org/wiki/Dublin_Core" as *u8, "knowledge/fetched/doc_cat_dublincore.raw" as *u8, store, out, cap)
84 ok = ok + fetch_save("https://en.wikipedia.org/wiki/Library_catalog" as *u8, "knowledge/fetched/doc_cat_libcatalog.raw" as *u8, store, out, cap)
85 ok = ok + fetch_save("https://en.wikipedia.org/wiki/MARC_standards" as *u8, "knowledge/fetched/doc_cat_marc.raw" as *u8, store, out, cap)
86 ok = ok + fetch_save("https://en.wikipedia.org/wiki/Records_management" as *u8, "knowledge/fetched/doc_cat_records.raw" as *u8, store, out, cap)
87 ok = ok + fetch_save("https://en.wikipedia.org/wiki/Controlled_vocabulary" as *u8, "knowledge/fetched/doc_cat_vocab.raw" as *u8, store, out, cap)
88 ok = ok + fetch_save("https://en.wikipedia.org/wiki/Knowledge_organization" as *u8, "knowledge/fetched/doc_cat_ko.raw" as *u8, store, out, cap)
89 ok = ok + fetch_save("https://en.wikipedia.org/wiki/Finding_aid" as *u8, "knowledge/fetched/doc_cat_findingaid.raw" as *u8, store, out, cap)
90
91 ff_puts("== MANAGE / MAKE (lifecycle, DMS, automation, templates) ==\n")
92 ok = ok + fetch_save("https://en.wikipedia.org/wiki/Document_management_system" as *u8, "knowledge/fetched/doc_mng_dms.raw" as *u8, store, out, cap)
93 ok = ok + fetch_save("https://en.wikipedia.org/wiki/Enterprise_content_management" as *u8, "knowledge/fetched/doc_mng_ecm.raw" as *u8, store, out, cap)
94 ok = ok + fetch_save("https://en.wikipedia.org/wiki/Content_management_system" as *u8, "knowledge/fetched/doc_mng_cms.raw" as *u8, store, out, cap)
95 ok = ok + fetch_save("https://en.wikipedia.org/wiki/Document_automation" as *u8, "knowledge/fetched/doc_mng_automation.raw" as *u8, store, out, cap)
96 ok = ok + fetch_save("https://en.wikipedia.org/wiki/Template_processor" as *u8, "knowledge/fetched/doc_mng_template.raw" as *u8, store, out, cap)
97 ok = ok + fetch_save("https://en.wikipedia.org/wiki/Digital_preservation" as *u8, "knowledge/fetched/doc_mng_preservation.raw" as *u8, store, out, cap)
98
99 ff_puts("== FORMATS / VALIDATION (PDF, archival, structured formats, QA) ==\n")
100 ok = ok + fetch_save("https://en.wikipedia.org/wiki/PDF" as *u8, "knowledge/fetched/doc_fmt_pdf.raw" as *u8, store, out, cap)
101 ok = ok + fetch_save("https://en.wikipedia.org/wiki/PDF/A" as *u8, "knowledge/fetched/doc_fmt_pdfa.raw" as *u8, store, out, cap)
102 ok = ok + fetch_save("https://en.wikipedia.org/wiki/TIFF" as *u8, "knowledge/fetched/doc_fmt_tiff.raw" as *u8, store, out, cap)
103 ok = ok + fetch_save("https://en.wikipedia.org/wiki/Office_Open_XML" as *u8, "knowledge/fetched/doc_fmt_ooxml.raw" as *u8, store, out, cap)
104 ok = ok + fetch_save("https://en.wikipedia.org/wiki/Data_validation" as *u8, "knowledge/fetched/doc_fmt_validation.raw" as *u8, store, out, cap)
105
106 ff_puts("DOC-INTEL READABLE SOURCES SAVED THIS PASS+HAVE: "); ff_putn(ok); ff_puts(" / 34\n")
107 return 0
108}