code wiki / (root) / nx_licdetect_lib.nx
`nx_licdetect_lib.nx` source

↩ module page · 228 lines · 9446 B
1// nx_licdetect_lib.nx -- WHICH LICENCE IS THIS TEXT? Read from the bytes, never from a declaration.
2//
3// THE MEASUREMENT THAT JUSTIFIES A TEXT DETECTOR AT ALL. Across 124,278 licence chains surveyed in the
4// field, 96.5% of datasets and 95.8% of models LACK the required licence text, and only 2.3% / 3.2%
5// satisfy both the licence-text and copyright requirements. The survey's conclusion is this lib's
6// design rule: LICENCE FILES AND NOTICES, NOT METADATA, ARE THE SOURCE OF LEGAL TRUTH.
7// It is also what the librarian measured directly: of 26 NAVER releases, 2 carried any licence string
8// on the listing, and the one we then FETCHED turned out to be 483 bytes off the canonical Apache text
9// because it adds a copyright preamble and drops the appendix. A declaration is a claim; bytes are not.
10//
11// THREE OUTCOMES, AND ONLY ONE OF THEM IS AN ANSWER:
12//   DETECTED  exactly one licence's REQUIRE markers all matched and none of its EXCLUDE markers did
13//   UNKNOWN   nothing matched -- fail-closed, the artifact does NOT advance to evidence=READ
14//   AMBIGUOUS two or more matched -- ALSO fail-closed. A text matching two licences is not a decision,
15//             and taking the first would be a coin flip wearing a verdict.
16//
17// MARKERS ARE DATA (knowledge/license_markers.conf), so a new licence costs a row and never a rebuild,
18// and every marker set is auditable by a human reading the conf rather than the binary.
19//
20// NO GUESSED CAPS: every array here is sized from the CONF'S OWN LINE COUNT, counted before allocation.
21// A marker table cannot overflow a bound derived from the file that defines it.
22// license_tier: ORIGINAL
23import "nx_syscalls.nx"
24
25const LD_DETECTED: i64 = 0
26const LD_UNKNOWN: i64 = 1
27const LD_AMBIGUOUS: i64 = 2
28
29const LD_TAB: i64 = 9
30const LD_NL: i64 = 10
31const LD_HASH: i64 = 35
32const LD_M: i64 = 77          // 'M' marker row
33const LD_V: i64 = 86          // 'V' verified-against row
34const LD_WORD: i64 = 8
35const LD_ROW_STRIDE: i64 = 5  // lic_off, lic_len, kind, mk_off, mk_len
36const LD_KIND_REQUIRE: i64 = 1
37const LD_KIND_EXCLUDE: i64 = 2
38const LD_CTX_SLOTS: i64 = 8   // buf, n, rows, nrows, vrows, nv
39
40func ld_streq_n(a: *u8, ao: i64, an: i64, b: *u8, bo: i64, bn: i64) -> i64 {
  if an != bn { return 0 }
  var i: i64 = 0
  while i < an { if a[ao + i] != b[bo + i] { return 0 } i = i + 1 }
  return 1
45}
46
47// literal substring search. Returns 1 if needle occurs in text[0,n).
48func ld_contains(text: *u8, n: i64, buf: *u8, mo: i64, mn: i64) -> i64 {
  if mn <= 0 { return 0 }
  if mn > n { return 0 }
  var i: i64 = 0
  let last: i64 = n - mn
  while i <= last {
      var j: i64 = 0
      var ok: i64 = 1
      while j < mn {
          if text[i + j] != buf[mo + j] { ok = 0; j = mn } else { j = j + 1 }
      }
      if ok == 1 { return 1 }
      i = i + 1
  }
  return 0
63}
64
65// TAB field [idx] of row [s,e) -> box[0]=off box[1]=len ; len -1 when the row has too few fields.
66func ld_field(buf: *u8, s: i64, e: i64, idx: i64, box: *i64) -> i64 {
  var f: i64 = 0
  var i: i64 = s
  while f < idx {
      if i >= e { box[0] = 0 - 1; box[1] = 0 - 1; return 0 - 1 }
      if buf[i] == (LD_TAB as u8) { f = f + 1 }
      i = i + 1
  }
  var j: i64 = i
  while j < e { if buf[j] == (LD_TAB as u8) { break } j = j + 1 }
  box[0] = i
  box[1] = j - i
  return j - i
79}
80
81// ctx[0]=buf ctx[1]=n ctx[2]=rows ctx[3]=nrows ctx[4]=vrows ctx[5]=nv ; ctx[3] = -1 means unreadable.
82func ld_ctx() -> *i64 {
  let ctx: *i64 = sys_mmap(LD_CTX_SLOTS * LD_WORD) as *i64
  let lp: *i64 = sys_mmap(LD_WORD * 2) as *i64
  let buf: *u8 = sys_read_file("knowledge/license_markers.conf" as *u8, lp)
  ctx[3] = 0 - 1
  ctx[5] = 0
  if (buf as i64) == 0 { return ctx }
  let n: i64 = lp[0]
  ctx[0] = buf as i64
  ctx[1] = n
  if n <= 0 { return ctx }
93
  // CAP DERIVED FROM THE DATA: a marker row cannot outnumber the conf's own lines.
  var lines: i64 = 1
  var c: i64 = 0
  while c < n { if buf[c] == (LD_NL as u8) { lines = lines + 1 } c = c + 1 }
  let rows: *i64 = sys_mmap(lines * LD_ROW_STRIDE * LD_WORD) as *i64
  let vrows: *i64 = sys_mmap(lines * 2 * LD_WORD) as *i64
  let box: *i64 = sys_mmap(32) as *i64
101
  var nrows: i64 = 0
  var nv: i64 = 0
  var p: i64 = 0
  while p < n {
      var q: i64 = p
      var scan: i64 = 1
      while scan == 1 {
          if q >= n { scan = 0 }
          else { if buf[q] == (LD_NL as u8) { scan = 0 } else { q = q + 1 } }
      }
      if q > p + 1 {
          if buf[p + 1] == (LD_TAB as u8) {
              if buf[p] == (LD_M as u8) {
                  if ld_field(buf, p, q, 1, box) >= 0 {
                      let lo: i64 = box[0]
                      let ll: i64 = box[1]
                      if ld_field(buf, p, q, 2, box) >= 0 {
                          // REQUIRE and EXCLUDE are both 7 characters, so the LENGTH admits the
                          // field and the FIRST BYTE discriminates ('R'=82, 'E'=69). A row whose
                          // kind word is neither keeps kind 0 and is therefore matched by no branch
                          // in ld_detect -- an unrecognised kind is INERT, never silently REQUIRE.
                          var kind: i64 = 0
                          let ko: i64 = box[0]
                          let kl: i64 = box[1]
                          if kl == 7 {
                              if buf[ko] == (82 as u8) { kind = LD_KIND_REQUIRE }
                              if buf[ko] == (69 as u8) { kind = LD_KIND_EXCLUDE }
                          }
                          if ld_field(buf, p, q, 3, box) >= 0 {
                              rows[nrows * LD_ROW_STRIDE + 0] = lo
                              rows[nrows * LD_ROW_STRIDE + 1] = ll
                              rows[nrows * LD_ROW_STRIDE + 2] = kind
                              rows[nrows * LD_ROW_STRIDE + 3] = box[0]
                              rows[nrows * LD_ROW_STRIDE + 4] = box[1]
                              nrows = nrows + 1
                          }
                      }
                  }
              }
              if buf[p] == (LD_V as u8) {
                  if ld_field(buf, p, q, 1, box) >= 0 {
                      vrows[nv * 2 + 0] = box[0]
                      vrows[nv * 2 + 1] = box[1]
                      nv = nv + 1
                  }
              }
          }
      }
      p = q + 1
  }
  ctx[2] = rows as i64
  ctx[3] = nrows
  ctx[4] = vrows as i64
  ctx[5] = nv
  return ctx
157}
158
159// is this licence's marker set PROVEN against a licence text we actually hold?
160func ld_marker_set_verified(ctx: *i64, lo: i64, ll: i64) -> i64 {
  let buf: *u8 = ctx[0] as *u8
  let vrows: *i64 = ctx[4] as *i64
  let nv: i64 = ctx[5]
  var i: i64 = 0
  while i < nv {
      if ld_streq_n(buf, lo, ll, buf, vrows[i * 2 + 0], vrows[i * 2 + 1]) == 1 { return 1 }
      i = i + 1
  }
  return 0
170}
171
172// THE DETECTION. Fills out_lo/out_ll with the winning licence id's offset+len in the conf buffer.
173// Returns LD_DETECTED / LD_UNKNOWN / LD_AMBIGUOUS.
174func ld_detect(ctx: *i64, text: *u8, n: i64, out: *i64) -> i64 {
  out[0] = 0 - 1
  out[1] = 0 - 1
  if ctx[3] < 0 { return LD_UNKNOWN }
  let buf: *u8 = ctx[0] as *u8
  let rows: *i64 = ctx[2] as *i64
  let nrows: i64 = ctx[3]
  var hits: i64 = 0
  var i: i64 = 0
  while i < nrows {
      let lo: i64 = rows[i * LD_ROW_STRIDE + 0]
      let ll: i64 = rows[i * LD_ROW_STRIDE + 1]
      // only evaluate a licence the FIRST time its id appears, so a licence with k markers is not
      // scored k times -- that would make the ambiguity count meaningless.
      var first: i64 = 1
      var b: i64 = 0
      while b < i {
          if ld_streq_n(buf, lo, ll, buf, rows[b * LD_ROW_STRIDE + 0], rows[b * LD_ROW_STRIDE + 1]) == 1 { first = 0; b = i }
          else { b = b + 1 }
      }
      if first == 1 {
          var all_req: i64 = 1
          var any_exc: i64 = 0
          var j: i64 = 0
          while j < nrows {
              if ld_streq_n(buf, lo, ll, buf, rows[j * LD_ROW_STRIDE + 0], rows[j * LD_ROW_STRIDE + 1]) == 1 {
                  let kind: i64 = rows[j * LD_ROW_STRIDE + 2]
                  let present: i64 = ld_contains(text, n, buf, rows[j * LD_ROW_STRIDE + 3], rows[j * LD_ROW_STRIDE + 4])
                  if kind == LD_KIND_REQUIRE { if present == 0 { all_req = 0 } }
                  if kind == LD_KIND_EXCLUDE { if present == 1 { any_exc = 1 } }
              }
              j = j + 1
          }
          if all_req == 1 {
              if any_exc == 0 {
                  hits = hits + 1
                  out[0] = lo
                  out[1] = ll
              }
          }
      }
      i = i + 1
  }
  if hits == 0 { out[0] = 0 - 1; out[1] = 0 - 1; return LD_UNKNOWN }
  // TWO MATCHES IS NOT A TIE TO BREAK, IT IS A REFUSAL. Clearing the output makes it impossible for a
  // caller to read a winner out of an ambiguous result by ignoring the return code.
  if hits > 1 { out[0] = 0 - 1; out[1] = 0 - 1; return LD_AMBIGUOUS }
  return LD_DETECTED
222}
223
224func ld_result_name(r: i64) -> *u8 {
  if r == LD_DETECTED { return "DETECTED" as *u8 }
  if r == LD_AMBIGUOUS { return "AMBIGUOUS" as *u8 }
  return "UNKNOWN" as *u8
228}
nx_licdetect_lib.nx source

`nx_licdetect_lib.nx` source
