code wiki / _hdl_build / nx_site_ingest.nx
`nx_site_ingest.nx` source

↩ module page · 137 lines · 6720 B
1// nx_site_ingest.nx -- FLAG ingestion-friendly sites + AUTO-INGEST the full site to the local Nishi
2// library (operator: "flag good sites of research that allow ingestion and ... auto ingest the full
3// site for local use via the nishi library", so "bot-blocking cant starve us"). The discipline:
4//   1. PERMISSION FIRST -- never bulk-ingest a site that doesn't allow it. si_robots_allows parses
5//      robots.txt Disallow rules; si_is_open_access reads license/repository signals (CC / arXiv /
6//      PMC / DOI / MIT / public-domain). Only INGEST_OK sites get flagged + mirrored.
7//   2. WHOLE-SITE MIRROR -- enumerate the site's URLs from its sitemap, fetch each POLITELY (via
8//      nx_polite_crawl's per-host pacing), and write each page into the library on disk.
9//   3. LOCAL-FIRST / STARVE-PROOF -- once mirrored, the team reads the LOCAL copy; if the live site
10//      goes dark or bot-blocks us, the corpus is already ours.
11// Composes nx_polite_crawl (the fetch governor that keeps it polite) + nx_research_synth/claim_extract
12// (which then corroborate/extract the mirrored text). LAWS: struct-free, integer-only. license_tier: ORIGINAL
13import "nx_polite_crawl.nx"
14import "nx_syscalls.nx"
15
16const SI_INGEST_OK:       i64 = 2   // allowed + clearly licensed/repository -> flag + auto-ingest
17const SI_INGEST_PARTIAL:  i64 = 1   // allowed by robots but unclear license -> ingest cautiously, flag-for-review
18const SI_INGEST_FORBIDDEN: i64 = 0  // robots disallows -> DO NOT bulk-ingest (respect the site)
19
20func si_strlen(s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} return n }
21
22func si_match_at(buf: *u8, pos: i64, n: i64, needle: *u8) -> i64 {
23    var k: i64 = 0
24    while needle[k] != (0 as u8) {
25        if pos + k >= n { return 0 }
26        if (buf[pos+k] & 0xff) != (needle[k] & 0xff) { return 0 }
27        k = k + 1
28    }
29    return 1
30}
31
32func si_count_sub(buf: *u8, n: i64, needle: *u8) -> i64 {
33    let nl: i64 = si_strlen(needle)
34    if nl == 0 { return 0 }
35    var c: i64 = 0; var i: i64 = 0
36    while i + nl <= n { if si_match_at(buf, i, n, needle) == 1 { c = c + 1; i = i + nl } else { i = i + 1 } }
37    return c
38}
39
40// is the disallow VALUE buf[vs,vs+vl) a prefix of `path`[0,pathlen)?
41func si_is_prefix(buf: *u8, vs: i64, vl: i64, path: *u8, pathlen: i64) -> i64 {
42    if vl <= 0 { return 0 }
43    if vl > pathlen { return 0 }
44    var k: i64 = 0
45    while k < vl { if (buf[vs+k] & 0xff) != (path[k] & 0xff) { return 0 } k = k + 1 }
46    return 1
47}
48
49// does robots.txt DISALLOW `path`? scans every "Disallow:" rule; a non-empty value that prefixes the
50// path disallows it. ("Disallow: /" disallows everything; "Disallow:" with empty value allows.)
51// CONSERVATIVE by design: honors all Disallow lines (safe -- never ingest where told not to).
52func si_robots_disallows(robots: *u8, n: i64, path: *u8, pathlen: i64) -> i64 {
53    let needle: *u8 = "Disallow:" as *u8
54    let dl: i64 = si_strlen(needle)
55    var i: i64 = 0
56    while i + dl <= n {
57        if si_match_at(robots, i, n, needle) == 1 {
58            var p: i64 = i + dl
59            while p < n { if (robots[p] & 0xff) == 32 { p = p + 1 } else { break } }   // skip spaces
60            let vs: i64 = p
61            while p < n { let c: i64 = robots[p] & 0xff; if c == 10 { break } if c == 13 { break } if c == 32 { break } p = p + 1 }
62            let vl: i64 = p - vs
63            if si_is_prefix(robots, vs, vl, path, pathlen) == 1 { return 1 }
64            i = p
65        } else { i = i + 1 }
66    }
67    return 0
68}
69
70func si_robots_allows(robots: *u8, n: i64, path: *u8, pathlen: i64) -> i64 {
71    if si_robots_disallows(robots, n, path, pathlen) == 1 { return 0 }
72    return 1
73}
74
75// count open-access / permissive-license / repository signals in a page or about-text
76func si_open_signals(buf: *u8, n: i64) -> i64 {
77    var c: i64 = 0
78    c = c + si_count_sub(buf, n, "creativecommons" as *u8)
79    c = c + si_count_sub(buf, n, "CC BY" as *u8)
80    c = c + si_count_sub(buf, n, "open access" as *u8)
81    c = c + si_count_sub(buf, n, "public domain" as *u8)
82    c = c + si_count_sub(buf, n, "MIT License" as *u8)
83    c = c + si_count_sub(buf, n, "arxiv" as *u8)
84    c = c + si_count_sub(buf, n, "arXiv" as *u8)
85    c = c + si_count_sub(buf, n, "PMC" as *u8)
86    return c
87}
88
89// the INGEST VERDICT: FORBIDDEN if robots disallows; OK if allowed AND (licensed OR repository);
90// PARTIAL if allowed but no clear license (flag for human review before bulk pull).
91func si_ingest_verdict(robots_allows: i64, open_signals: i64, is_repository: i64) -> i64 {
92    if robots_allows == 0 { return SI_INGEST_FORBIDDEN }
93    if open_signals >= 1 { return SI_INGEST_OK }
94    if is_repository == 1 { return SI_INGEST_OK }
95    return SI_INGEST_PARTIAL
96}
97
98// flag a GOOD ingestion target (the researcher surfaces these to the operator/library)
99func si_flag_good_site(verdict: i64) -> i64 { if verdict == SI_INGEST_OK { return 1 } return 0 }
100
101// ---- whole-site enumeration from a sitemap.xml ----
102func si_sitemap_url_count(sitemap: *u8, n: i64) -> i64 { return si_count_sub(sitemap, n, "<loc>" as *u8) }
103
104// ---- ingest progress + the STARVE-PROOF property ----
105func si_ingest_progress_permil(ingested: i64, total: i64) -> i64 {
106    if total <= 0 { return 0 }
107    return ingested * 1000 / total
108}
109func si_fully_mirrored(ingested: i64, total: i64) -> i64 {
110    if total <= 0 { return 0 }
111    if ingested >= total { return 1 }
112    return 0
113}
114// once the site is local, bot-blocking / the site going dark CANNOT starve the team's research.
115func si_starve_proof(fully_mirrored: i64) -> i64 { return fully_mirrored }
116
117// write one mirrored page to the local Nishi library; returns bytes written (0 on fail).
118// path e.g. "knowledge/library/mirror/<id>.txt". The local-first corpus the team reads from.
119func si_library_write(path: *u8, data: *u8, len: i64) -> i64 {
120    let fd: i64 = sys_openat_wr(path, 0x1a4)
121    if fd < 0 { return 0 }
122    let w: i64 = sys_write(fd, data, len)
123    sys_close(fd)
124    return w
125}
126
127// the END-TO-END gate for one site. NOTE the two politeness MODES are different:
128//   * BROAD RESEARCH crawl = many hosts, FEW requests each (pc_research_gate floors per-host count).
129//   * FULL-SITE INGEST = ONE host, MANY requests -- legitimately, BECAUSE it is permitted AND PACED.
130// So bulk ingest is admissible iff (a) verdict OK (robots+license permit it) AND (b) it is PACED
131// through the governor (per-host min-interval + robots crawl-delay honored), NOT gated on a low total
132// count. Pacing is the politeness for a permitted bulk pull, not scarcity.
133func si_ingest_admissible(verdict: i64, paced_through_governor: i64) -> i64 {
134    if verdict != SI_INGEST_OK { return 0 }
135    if paced_through_governor != 1 { return 0 }
136    return 1
137}
nx_site_ingest.nx source

`nx_site_ingest.nx` source
