code wiki / _hdl_build / nx_outlink_harvest.nx

nx_outlink_harvest.nx

buildroot/runtime/_hdl_build/nx_outlink_harvest.nx

19339 B374 linesdepth 6pulls 17 transitivereach 9 importersview sourcekind library

docs dependencies structs consts functions

about

nx_outlink_harvest.nx — shared outlink -> link-graph-edge harvester (P1 PageRank feed). LIB (no main). Extracted (by copy) from nx_web_crawl_step.wc_harvest so BOTH the live crawler AND nx_cc_ingest emit the same row shape: out:<ci_hash(page_url)> = packed i64 ci_hash(target_url). Node identity = ci_hash(url) == dss_urlcid(url) at serve time, so an outlink resolves to a graph node with zero extra index; serve-time authority = content_cid -> url:<cid> -> ci_hash(url) -> pr:<cid>. ABSOLUTE cross-site links (the authority- bearing ones) produce byte-identical cids to the crawler. Three intentional improvements over the crawler's inline copy (all to backport when the crawler adopts this lib): (1) root-relative "/path" resolves HOST-based (scheme://host + path) not base+path — the crawler's concatenation makes ".../Trust_law" + "/about" a phantom node; (2) edge capture is DECOUPLED from the frontier cap (the crawler stops recording edges once discbox hits WC_MAXDISC — a graph must not depend on frontier state); (3) one REUSED url buffer instead of an mmap per outlink (bounds address-space growth for CC-scale ingests of thousands of pages). license_tier: ORIGINAL

dependencies 3 imports · 6 importers

imports: nx_corpus_ingest.nxnx_store_seed_lib.nxnx_url_canon.nx

imported by: nx_cc_ingest.nxnx_cc_warc_ingest.nxnx_crawl_callee_probe.nxnx_outlink_harvest_gate.nxnx_page_ingest.nxnx_web_crawl_step.nx

structs

none

consts

17	const OLH_MAGIC_8192: i64 = 8192
19	const OLH_MAXURL: i64 = 1500 // == the crawler's WC_MAXURL (measured junk/tracking-monster threshold) so
21	const OLH_MAXEDGE: i64 = 1024
75	const OLH_PATHPOL_PLANE: *u8 = "knowledge/store/urlpolicy-"
76	const OLH_PSLOT: i64 = 64 // bytes per rule: [kind byte][NUL-terminated pattern]
77	const OLH_PMAXRULE: i64 = 128
78	const OLH_PPLANECAP: i64 = 65536 // ONE-TIME read buffer (per process, not per call); bounded and named
79	const OLH_PKIND_SUB: i64 = 1
80	const OLH_PKIND_END: i64 = 2

functions

23	func olh_len(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } return n } called by 6: olh_hassub olh_endswith olh_prule_addc olh_pname_is olh_pname_pref main
26	func olh_hassub(u: u8, ul: i64, lit: u8) -> i64 called by 2: olh_path_denied olh_urlok_q calls 1: olh_len
40	func olh_endswith(u: u8, ul: i64, lit: u8) -> i64 called by 1: olh_path_denied calls 1: olh_len
85	func olh_pathpol_set(p: *u8) -> i64 { olh_ppath_g = p; olh_ploaded_g = 0; return 0 } called by 1: main
87	func olh_prule_add(kind: i64, pat: *u8, pl: i64) -> i64 called by 2: olh_prule_addc olh_load_pathpolicy
98	func olh_prule_addc(kind: i64, pat: *u8) -> i64 { return olh_prule_add(kind, pat, olh_len(pat)) } called by 1: olh_prule_defaults calls 2: olh_prule_add olh_len
100	func olh_prule_defaults() -> i64 called by 1: olh_load_pathpolicy calls 1: olh_prule_addc
108	func olh_pname_is(b: u8, s: i64, nl: i64, lit: u8) -> i64 called by 1: olh_load_pathpolicy calls 1: olh_len
116	func olh_pname_pref(b: u8, s: i64, nl: i64, lit: u8) -> i64 called by 1: olh_load_pathpolicy calls 1: olh_len
123	func olh_load_pathpolicy() -> i64 called by 1: olh_path_denied calls 5: sts_load olh_pname_is olh_pname_pref olh_prule_add olh_prule_defaults
171	func olh_path_denied(u: *u8, ul: i64) -> i64 calls 3: olh_load_pathpolicy olh_hassub olh_endswith
184	func olh_urlok_q(u: *u8, ul: i64, allowq: i64) -> i64 calls 1: olh_hassub
244	func olh_urlok(u: *u8, ul: i64) -> i64 { return olh_urlok_q(u, ul, 0) }
246	func olh_outkey(cid: i64, out: *u8) -> i64
265	func olh_resolve_root(base: u8, hp: u8, clen: i64, scratch: *u8) -> i64
296	func olh_scratch() -> i64
303	func olh_scan(hh: u8, hlen: i64, base: u8, blen: i64, edgebuf: *i64, maxedge: i64) -> i64