nx_research_relation_clean.nx source

1// nx_research_relation_clean.nx -- Researcher EXTRACT, rung 3: CLAIM CLEANUP. 2// The measured noise from rung 2 (pronoun subjects, "[1]"/"^" citation markers, 3// "Wikipedia(R) trademark" boilerplate, junk) is fixed here -- by IMPROVING the 4// claims, not by stripping the feature. Inspectable + data-driven (stopword / 5// pronoun / boilerplate tables). nx_claim_clean(subj,obj -> clean) -> 1 good / 0 reject. 6// license_tier: ORIGINAL. 7 8import "nx_syscalls.nx" 9import "nx_research_extract.nx" 10 11func cl_is_alpha(c: u8) -> i64 { 12 if c >= (65 as u8) { if c <= (90 as u8) { return 1 } } 13 if c >= (97 as u8) { if c <= (122 as u8) { return 1 } } 14 return 0 15} 16func cl_is_alnum(c: u8) -> i64 { 17 if cl_is_alpha(c) == 1 { return 1 } 18 if c >= (48 as u8) { if c <= (57 as u8) { return 1 } } 19 return 0 20} 21func cl_strlen(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } return n } 22// a real entity subject is short; an over-long "subject" is a cue-mismatch/fragment (rung R3c) 23func cl_max_subj_words() -> i64 { return 6 } 24func cl_word_count(s: *u8) -> i64 { 25 var n: i64 = 0; var i: i64 = 0; var in_word: i64 = 0 26 while s[i] != (0 as u8) { 27 if s[i] == (32 as u8) { in_word = 0 } else { if in_word == 0 { n = n + 1; in_word = 1 } } 28 i = i + 1 29 } 30 return n 31} 32func cl_streq(a: *u8, b: *u8) -> i64 { 33 var i: i64 = 0 34 while i < 256 { if a[i] != b[i] { return 0 } if a[i] == (0 as u8) { return 1 } i = i + 1 } 35 return 1 36} 37func cl_lc(c: u8) -> u8 { if c >= (65 as u8) { if c <= (90 as u8) { return (c + 32) as u8 } } return c } 38// case-insensitive equality -- leading stopwords/pronouns are often Capitalized 39func cl_streq_ci(a: *u8, b: *u8) -> i64 { 40 var i: i64 = 0 41 while i < 256 { if cl_lc(a[i]) != cl_lc(b[i]) { return 0 } if a[i] == (0 as u8) { return 1 } i = i + 1 } 42 return 1 43} 44 45// first whitespace-delimited token of s into tok; returns token length 46func cl_first_token(s: *u8, tok: *u8) -> i64 { 47 var i: i64 = 0 48 while s[i] != (0 as u8) { if s[i] == (32 as u8) { break } tok[i] = s[i]; i = i + 1 } 49 tok[i] = 0 as u8 50 return i 51} 52 53// leading-stopword table (dropped from the front of a subject, repeatedly) 54func cl_is_stopword(t: *u8) -> i64 { 55 if cl_streq_ci(t,"the" as *u8) == 1 { return 1 } 56 if cl_streq_ci(t,"a" as *u8) == 1 { return 1 } 57 if cl_streq_ci(t,"an" as *u8) == 1 { return 1 } 58 if cl_streq_ci(t,"and" as *u8) == 1 { return 1 } 59 if cl_streq_ci(t,"but" as *u8) == 1 { return 1 } 60 if cl_streq_ci(t,"which" as *u8) == 1 { return 1 } 61 if cl_streq_ci(t,"that" as *u8) == 1 { return 1 } 62 if cl_streq_ci(t,"this" as *u8) == 1 { return 1 } 63 if cl_streq_ci(t,"these" as *u8) == 1 { return 1 } 64 if cl_streq_ci(t,"they" as *u8) == 1 { return 1 } 65 if cl_streq_ci(t,"what" as *u8) == 1 { return 1 } 66 return 0 67} 68// bare-pronoun subject -> reject entirely (no real entity) 69func cl_is_pronoun(t: *u8) -> i64 { 70 if cl_streq_ci(t,"it" as *u8) == 1 { return 1 } 71 if cl_streq_ci(t,"they" as *u8) == 1 { return 1 } 72 if cl_streq_ci(t,"which" as *u8) == 1 { return 1 } 73 if cl_streq_ci(t,"what" as *u8) == 1 { return 1 } 74 if cl_streq_ci(t,"this" as *u8) == 1 { return 1 } 75 if cl_streq_ci(t,"that" as *u8) == 1 { return 1 } 76 return 0 77} 78// boilerplate terms -> reject the claim 79func cl_boiler(s: *u8) -> i64 { 80 let n: i64 = cl_strlen(s) 81 if re_find(s, n, "Wikipedia" as *u8) >= 0 { return 1 } 82 if re_find(s, n, "trademark" as *u8) >= 0 { return 1 } 83 if re_find(s, n, "Foundation" as *u8) >= 0 { return 1 } 84 if re_find(s, n, "Retrieved" as *u8) >= 0 { return 1 } 85 if re_find(s, n, "ISBN" as *u8) >= 0 { return 1 } 86 if re_find(s, n, "edit]" as *u8) >= 0 { return 1 } 87 return 0 88} 89 90// shift out[] left, deleting the first token (+ following space) 91func cl_drop_first_token(out: *u8, tl: i64) -> i64 { 92 var src: i64 = tl 93 if out[src] == (32 as u8) { src = src + 1 } 94 var d: i64 = 0 95 while out[src] != (0 as u8) { out[d] = out[src]; d = d + 1; src = src + 1 } 96 out[d] = 0 as u8 97 return d 98} 99 100// copy src into out starting at first alphanumeric char; first drops a leading 101// "[..]" citation block, then leading markers/quotes/spaces -- but KEEPS leading 102// digits so "3D"/"2D" survive (the old version ate the "3" of "3D"). 103func cl_copy_from_alpha(src: *u8, out: *u8) -> i64 { 104 var p: i64 = 0 105 if src[p] == (91 as u8) { // '[' -> drop the whole [..] block 106 while src[p] != (0 as u8) { if src[p] == (93 as u8) { p = p + 1; break } p = p + 1 } 107 } 108 while src[p] != (0 as u8) { if cl_is_alnum(src[p]) == 1 { break } p = p + 1 } 109 var k: i64 = 0 110 while src[p] != (0 as u8) { out[k] = src[p]; k = k + 1; p = p + 1 } 111 out[k] = 0 as u8 112 return k 113} 114 115// clean a subject: strip leading non-alpha, then drop leading stopwords 116func cl_clean_subject(src: *u8, out: *u8) -> i64 { 117 cl_copy_from_alpha(src, out) 118 let tok: *u8 = sys_mmap(256) 119 var go: i64 = 1 120 while go == 1 { 121 let tl: i64 = cl_first_token(out, tok) 122 if tl == 0 { go = 0 } 123 else { if cl_is_stopword(tok) == 1 { cl_drop_first_token(out, tl) } else { go = 0 } } 124 } 125 return cl_strlen(out) 126} 127 128// verb-led subject -> clause fragment, not an entity -> reject (rung R3b) 129func cl_is_verb(t: *u8) -> i64 { 130 if cl_streq_ci(t, "is" as *u8) == 1 { return 1 } 131 if cl_streq_ci(t, "are" as *u8) == 1 { return 1 } 132 if cl_streq_ci(t, "was" as *u8) == 1 { return 1 } 133 if cl_streq_ci(t, "were" as *u8) == 1 { return 1 } 134 if cl_streq_ci(t, "be" as *u8) == 1 { return 1 } 135 if cl_streq_ci(t, "been" as *u8) == 1 { return 1 } 136 if cl_streq_ci(t, "have" as *u8) == 1 { return 1 } 137 if cl_streq_ci(t, "has" as *u8) == 1 { return 1 } 138 if cl_streq_ci(t, "had" as *u8) == 1 { return 1 } 139 if cl_streq_ci(t, "provide" as *u8) == 1 { return 1 } 140 if cl_streq_ci(t, "provides" as *u8) == 1 { return 1 } 141 if cl_streq_ci(t, "provided" as *u8) == 1 { return 1 } 142 if cl_streq_ci(t, "remains" as *u8) == 1 { return 1 } 143 if cl_streq_ci(t, "remain" as *u8) == 1 { return 1 } 144 if cl_streq_ci(t, "found" as *u8) == 1 { return 1 } 145 if cl_streq_ci(t, "find" as *u8) == 1 { return 1 } 146 if cl_streq_ci(t, "use" as *u8) == 1 { return 1 } 147 if cl_streq_ci(t, "uses" as *u8) == 1 { return 1 } 148 if cl_streq_ci(t, "used" as *u8) == 1 { return 1 } 149 if cl_streq_ci(t, "make" as *u8) == 1 { return 1 } 150 if cl_streq_ci(t, "makes" as *u8) == 1 { return 1 } 151 if cl_streq_ci(t, "include" as *u8) == 1 { return 1 } 152 if cl_streq_ci(t, "includes" as *u8) == 1 { return 1 } 153 if cl_streq_ci(t, "allow" as *u8) == 1 { return 1 } 154 if cl_streq_ci(t, "allows" as *u8) == 1 { return 1 } 155 if cl_streq_ci(t, "enable" as *u8) == 1 { return 1 } 156 if cl_streq_ci(t, "enables" as *u8) == 1 { return 1 } 157 if cl_streq_ci(t, "require" as *u8) == 1 { return 1 } 158 if cl_streq_ci(t, "requires" as *u8) == 1 { return 1 } 159 if cl_streq_ci(t, "can" as *u8) == 1 { return 1 } 160 if cl_streq_ci(t, "will" as *u8) == 1 { return 1 } 161 if cl_streq_ci(t, "do" as *u8) == 1 { return 1 } 162 if cl_streq_ci(t, "does" as *u8) == 1 { return 1 } 163 return 0 164} 165 166// MAIN: subj_in,obj_in -> cleaned subj_out,obj_out; returns 1 GOOD | 0 REJECT. 167func nx_claim_clean(subj_in: *u8, obj_in: *u8, subj_out: *u8, obj_out: *u8) -> i64 { 168 cl_clean_subject(subj_in, subj_out) 169 cl_copy_from_alpha(obj_in, obj_out) 170 if cl_strlen(subj_out) < 2 { return 0 } 171 if cl_strlen(obj_out) < 2 { return 0 } 172 let tok: *u8 = sys_mmap(256) 173 cl_first_token(subj_out, tok) 174 if cl_is_pronoun(tok) == 1 { return 0 } 175 if cl_is_verb(tok) == 1 { return 0 } 176 if cl_word_count(subj_out) > cl_max_subj_words() { return 0 } 177 if cl_boiler(subj_out) == 1 { return 0 } 178 if cl_boiler(obj_out) == 1 { return 0 } 179 return 1 180}