nx_legalsem_lib.nx source
↩ module page · 171 lines · 7871 B
1// nx_legalsem_lib.nx -- SEMANTIC clause matching: legal concept normalisation + POLARITY.
2// nx_clause matches WORDS. Two clauses can say the same thing in different words -- "indemnify and hold
3// harmless" versus "defend against claims" -- and word overlap scores them as unrelated. This layer
4// matches at the CONCEPT level, so paraphrase works: surface terms normalise to a legal concept and
5// clauses are compared on the concepts they invoke, not the vocabulary they happen to use.
6//
7// u2605u2605THE REASON THIS IS NOT JUST A THESAURUS -- POLARITY. "Supplier shall indemnify Customer" and
8// "Supplier shall NOT indemnify Customer" invoke IDENTICAL concepts. Every similarity measure that works
9// on concept overlap alone -- including dense vector embeddings, which are notoriously weak on negation --
10// scores those two as near-identical. They are opposites, and it is the single most dangerous redline in
11// contracting: the obligation survives the concept check while its direction has been inverted.
12//
13// So equivalence here requires BOTH concept overlap AND matching polarity, and the specific case of
14// "same concepts, opposite polarity" gets its own predicate -- sem_reversal -- because that is not a
15// weak match, it is an ALARM. A reversal must never be reported as a low similarity score; it must be
16// reported as a reversal.
17//
18// u2605FAIL-CLOSED: text invoking NO known concept yields zero overlap and is never "equivalent" by
19// vacuous agreement. Two clauses that both mention nothing recognisable are not the same clause.
20//
21// The ontology is an explicit, greppable table (extend it by adding rows, not by retraining anything) --
22// a deliberate trade: narrower than an embedder, but auditable, deterministic, and incapable of
23// hallucinating a match. license_tier: ORIGINAL LIB.
24
25import "nx_clause_lib.nx"
26
27const SEM_NONE: i64 = 0
28const SEM_INDEMNITY: i64 = 1
29const SEM_CONFIDENTIALITY: i64 = 2
30const SEM_TERMINATION: i64 = 3
31const SEM_LIABILITY: i64 = 4
32const SEM_UNCAPPED: i64 = 5
33const SEM_WARRANTY: i64 = 6
34const SEM_ASSIGNMENT: i64 = 7
35const SEM_PAYMENT: i64 = 8
36const SEM_MAXC: i64 = 9
37
38const SEM_EQUIV_FLOOR: i64 = 600
39
40// u2605THE ONTOLOGY. Surface term -> legal concept. Extend by adding a row.
41func sem_concept_of(w: *u8) -> i64 {
42 if mt_streq(w, "indemnify" as *u8) == 1 { return SEM_INDEMNITY }
43 if mt_streq(w, "indemnity" as *u8) == 1 { return SEM_INDEMNITY }
44 if mt_streq(w, "indemnification" as *u8) == 1 { return SEM_INDEMNITY }
45 if mt_streq(w, "harmless" as *u8) == 1 { return SEM_INDEMNITY }
46 if mt_streq(w, "defend" as *u8) == 1 { return SEM_INDEMNITY }
47
48 if mt_streq(w, "confidential" as *u8) == 1 { return SEM_CONFIDENTIALITY }
49 if mt_streq(w, "confidentiality" as *u8) == 1 { return SEM_CONFIDENTIALITY }
50 if mt_streq(w, "proprietary" as *u8) == 1 { return SEM_CONFIDENTIALITY }
51 if mt_streq(w, "nondisclosure" as *u8) == 1 { return SEM_CONFIDENTIALITY }
52 if mt_streq(w, "secret" as *u8) == 1 { return SEM_CONFIDENTIALITY }
53
54 if mt_streq(w, "terminate" as *u8) == 1 { return SEM_TERMINATION }
55 if mt_streq(w, "termination" as *u8) == 1 { return SEM_TERMINATION }
56 if mt_streq(w, "cancel" as *u8) == 1 { return SEM_TERMINATION }
57 if mt_streq(w, "rescind" as *u8) == 1 { return SEM_TERMINATION }
58 if mt_streq(w, "expire" as *u8) == 1 { return SEM_TERMINATION }
59
60 if mt_streq(w, "liable" as *u8) == 1 { return SEM_LIABILITY }
61 if mt_streq(w, "liability" as *u8) == 1 { return SEM_LIABILITY }
62 if mt_streq(w, "damages" as *u8) == 1 { return SEM_LIABILITY }
63 if mt_streq(w, "losses" as *u8) == 1 { return SEM_LIABILITY }
64
65 if mt_streq(w, "unlimited" as *u8) == 1 { return SEM_UNCAPPED }
66 if mt_streq(w, "uncapped" as *u8) == 1 { return SEM_UNCAPPED }
67 if mt_streq(w, "unrestricted" as *u8) == 1 { return SEM_UNCAPPED }
68
69 if mt_streq(w, "warrant" as *u8) == 1 { return SEM_WARRANTY }
70 if mt_streq(w, "warranty" as *u8) == 1 { return SEM_WARRANTY }
71 if mt_streq(w, "warranties" as *u8) == 1 { return SEM_WARRANTY }
72 if mt_streq(w, "represents" as *u8) == 1 { return SEM_WARRANTY }
73
74 if mt_streq(w, "assign" as *u8) == 1 { return SEM_ASSIGNMENT }
75 if mt_streq(w, "assignment" as *u8) == 1 { return SEM_ASSIGNMENT }
76 if mt_streq(w, "transfer" as *u8) == 1 { return SEM_ASSIGNMENT }
77 if mt_streq(w, "delegate" as *u8) == 1 { return SEM_ASSIGNMENT }
78
79 if mt_streq(w, "pay" as *u8) == 1 { return SEM_PAYMENT }
80 if mt_streq(w, "payment" as *u8) == 1 { return SEM_PAYMENT }
81 if mt_streq(w, "invoice" as *u8) == 1 { return SEM_PAYMENT }
82 if mt_streq(w, "fees" as *u8) == 1 { return SEM_PAYMENT }
83 return SEM_NONE
84}
85
86// 1 if the text invokes `concept` anywhere
87func sem_has_concept(text: *u8, concept: i64) -> i64 {
88 let w: *u8 = sys_mmap(CL_WORDBUF)
89 var i: i64 = 0
90 while 1 == 1 {
91 let nx: i64 = cl_word_at(text, i, w)
92 if w[0] == (0 as u8) { return 0 }
93 if sem_concept_of(w) == concept { return 1 }
94 i = nx
95 }
96 return 0
97}
98
99// how many distinct legal concepts the text invokes
100func sem_concept_count(text: *u8) -> i64 {
101 var n: i64 = 0
102 var c: i64 = 1
103 while c < SEM_MAXC {
104 n = n + sem_has_concept(text, c)
105 c = c + 1
106 }
107 return n
108}
109
110// u2605CONCEPT-LEVEL similarity (Jaccard, per mille): paraphrases score high even with different words.
111// Two texts invoking no known concept score 0 -- never 1000 by vacuous agreement.
112func sem_overlap_permille(a: *u8, b: *u8) -> i64 {
113 var both: i64 = 0
114 var either: i64 = 0
115 var c: i64 = 1
116 while c < SEM_MAXC {
117 let ina: i64 = sem_has_concept(a, c)
118 let inb: i64 = sem_has_concept(b, c)
119 if ina + inb > 0 { either = either + 1 }
120 if ina == 1 {
121 if inb == 1 { both = both + 1 }
122 }
123 c = c + 1
124 }
125 if either == 0 { return 0 }
126 return (both * 1000) / either
127}
128
129// u2605POLARITY. Negation markers invert the obligation. Detected as whole words so "nothing" does not
130// register as "not" and "another" does not register as "no".
131func sem_negated(text: *u8) -> i64 {
132 if cl_has_word(text, "not" as *u8) == 1 { return 1 }
133 if cl_has_word(text, "no" as *u8) == 1 { return 1 }
134 if cl_has_word(text, "never" as *u8) == 1 { return 1 }
135 if cl_has_word(text, "neither" as *u8) == 1 { return 1 }
136 if cl_has_word(text, "nor" as *u8) == 1 { return 1 }
137 if cl_has_word(text, "without" as *u8) == 1 { return 1 }
138 if cl_has_word(text, "excluding" as *u8) == 1 { return 1 }
139 return 0
140}
141
142// u2605EQUIVALENCE requires concept agreement AND matching polarity. Either alone is not enough.
143func sem_equivalent(a: *u8, b: *u8) -> i64 {
144 if sem_overlap_permille(a, b) < SEM_EQUIV_FLOOR { return 0 }
145 if sem_negated(a) != sem_negated(b) { return 0 }
146 return 1
147}
148
149// u2605u2605THE ALARM: same concepts, OPPOSITE polarity. This is an obligation that has been INVERTED while
150// still looking like the same clause. It is not a weak match and must never be reported as one.
151func sem_reversal(a: *u8, b: *u8) -> i64 {
152 if sem_overlap_permille(a, b) < SEM_EQUIV_FLOOR { return 0 }
153 if sem_negated(a) == sem_negated(b) { return 0 }
154 return 1
155}
156
157// u2605the honest verdict for a drafted clause against a standard: 2 = REVERSED (alarm), 1 = equivalent,
158// 0 = materially different. A reversal can never be mistaken for either of the others.
159func sem_verdict(standard: *u8, draft: *u8) -> i64 {
160 if sem_reversal(standard, draft) == 1 { return 2 }
161 if sem_equivalent(standard, draft) == 1 { return 1 }
162 return 0
163}
164
165func sem_verdict_label(v: i64, out: *u8) -> i64 {
166 if v == 2 { mt_catcopy(out, 0, "REVERSED-ALARM" as *u8); out[14] = 0 as u8; return 14 }
167 if v == 1 { mt_catcopy(out, 0, "EQUIVALENT" as *u8); out[10] = 0 as u8; return 10 }
168 mt_catcopy(out, 0, "DIFFERENT" as *u8)
169 out[9] = 0 as u8
170 return 9
171}