nx_hf_tree_parse.nx source

1// nx_hf_tree_parse.nx -- LIB: parse the HuggingFace repo tree JSON (from /api/models/<repo>/tree/main?recursive=1, 2// after chunked_decode) into the per-file facts a VERIFIED mirror needs: path, size, and the LFS sha256 (lfs.oid). 3// A .safetensors weight shard entry looks like: 4// {"type":"file","oid":"<gitsha>","size":N,"path":"model-00001-of-00002.safetensors", 5// "lfs":{"oid":"<sha256-64hex>","size":N,"pointerSize":135}} 6// The lfs.oid IS the sha256 the mirror-orchestrator verifies each downloaded shard against (fail-closed). Pure 7// parsing (no crypto/TLS) -> nx_syscalls only; targeted field extraction over the structured HF shape (a full JSON 8// parser is overkill for this fixed schema). No 'break' (nx) -> flag-driven. license_tier: ORIGINAL 9import "nx_syscalls.nx" 10 11func hft_is_hex(c: i64) -> i64 { if c>=48 { if c<=57 {return 1} } if c>=97 { if c<=102 {return 1} } if c>=65 { if c<=70 {return 1} } return 0 } 12 13// find needle (NUL-terminated) in json[from..n); return start index or -1. 14func hft_find(json: *u8, n: i64, from: i64, needle: *u8) -> i64 { 15 var m: i64=0; while needle[m]!=(0 as u8){m=m+1} 16 if m==0 { return 0-1 } 17 var i: i64=from 18 while i+m<=n { 19 var j: i64=0; var ok: i64=1 20 while j<m { if json[i+j]!=needle[j] {ok=0;j=m} else {j=j+1} } 21 if ok==1 { return i } 22 i=i+1 23 } 24 return 0-1 25} 26 27// count non-overlapping occurrences of needle in json[0..n). 28func hft_count(json: *u8, n: i64, needle: *u8) -> i64 { 29 var m: i64=0; while needle[m]!=(0 as u8){m=m+1} 30 if m==0 { return 0 } 31 var c: i64=0; var i: i64=0 32 while i+m<=n { 33 var j: i64=0; var ok: i64=1 34 while j<m { if json[i+j]!=needle[j] {ok=0;j=m} else {j=j+1} } 35 if ok==1 { c=c+1; i=i+m } else { i=i+1 } 36 } 37 return c 38} 39 40// copy the hex run starting at pos into out (NUL-terminated); return length. 41func hft_copy_hex(json: *u8, pos: i64, n: i64, out: *u8) -> i64 { 42 var o: i64=0; var i: i64=pos; var st: i64=0 43 while st==0 { if i>=n { st=1 } else { if hft_is_hex(json[i] as i64)==1 { out[o]=json[i]; o=o+1; i=i+1 } else { st=1 } } } 44 out[o]=0 as u8; return o 45} 46 47// copy from pos until terminator byte `term` (exclusive) into out (NUL-terminated); return length. 48func hft_copy_until(json: *u8, pos: i64, n: i64, term: i64, out: *u8) -> i64 { 49 var o: i64=0; var i: i64=pos; var st: i64=0 50 while st==0 { if i>=n { st=1 } else { if (json[i] as i64)==term { st=1 } else { out[o]=json[i]; o=o+1; i=i+1 } } } 51 out[o]=0 as u8; return o 52} 53 54// parse an unsigned decimal starting at pos; return value (0 if none). 55func hft_parse_uint(json: *u8, pos: i64, n: i64) -> i64 { 56 var v: i64=0; var i: i64=pos; var st: i64=0 57 while st==0 { if i>=n { st=1 } else { let c: i64=json[i] as i64; if c>=48 { if c<=57 { v=(v*10)+(c-48); i=i+1 } else { st=1 } } else { st=1 } } } 58 return v 59} 60 61// find `marker` at/after `from` and parse the unsigned decimal that follows it; -1 if marker absent. 62func hft_num_field(json: *u8, n: i64, from: i64, marker: *u8) -> i64 { 63 let p: i64 = hft_find(json, n, from, marker) 64 if p<0 { return 0-1 } 65 var ml: i64=0; while marker[ml]!=(0 as u8){ml=ml+1} 66 return hft_parse_uint(json, p+ml, n) 67} 68 69// extract the "path":"..." value at/after `from` into out; return the value-start index or -1. 70func hft_path(json: *u8, n: i64, from: i64, out: *u8) -> i64 { 71 let p: i64 = hft_find(json, n, from, "\"path\":\"" as *u8) 72 if p<0 { out[0]=0 as u8; return 0-1 } 73 var ml: i64=0; let mk: *u8="\"path\":\"" as *u8; while mk[ml]!=(0 as u8){ml=ml+1} 74 let start: i64 = p+ml 75 hft_copy_until(json, start, n, 34, out) // 34 = double-quote 76 return start 77} 78 79// extract the LFS sha256 (lfs.oid, 64 hex) at/after `from` into out; return the hex-start index or -1. 80func hft_lfs_oid(json: *u8, n: i64, from: i64, out: *u8) -> i64 { 81 let p: i64 = hft_find(json, n, from, "\"lfs\":{\"oid\":\"" as *u8) 82 if p<0 { out[0]=0 as u8; return 0-1 } 83 var ml: i64=0; let mk: *u8="\"lfs\":{\"oid\":\"" as *u8; while mk[ml]!=(0 as u8){ml=ml+1} 84 let start: i64 = p+ml 85 hft_copy_hex(json, start, n, out) 86 return start 87}