nx_llm_train_estimate.nx source

1import "nx_gate_gn.nx" 2import "nx_gate_base.nx" 3// nx_llm_train_estimate.nx -- GROUNDED cost model for training our own LLM (operator: "let me know what it will take 4// to train an llm based on what we have and estimates from the cloud or whatever"). Data-driven, re-runnable (rule 11: 5// every number is a named input, not buried). Uses the standard training-compute law: FLOPs ~= 6 * N_params * D_tokens 6// (fwd+bwd), Chinchilla-optimal D ~= 20*N. GPU-hours = FLOPs / (peak_TFLOPS * MFU). Cost = hours * $/hr. Training memory 7// ~= 16 bytes/param (bf16 weights+grad + fp32 Adam m,v) => the VRAM-fit gate. Honest: estimates the COMPUTE; the 8// ENGINEERING (transformer backprop + loop + optimizer + f32 autograd + tokenized corpus) is the separate build cost. 9// Reports, per model x GPU: train-PFLOPs, GPU-hours, $ cost, and whether training FITS the GPU's VRAM. 10// license_tier: ORIGINAL 11import "nx_syscalls.nx" 12const K_MAGIC_3600: i64 = 3600 13const K_MAGIC_10000: i64 = 10000 14const K_MAGIC_1500: i64 = 1500 15const K_MAGIC_30000: i64 = 30000 16const K_MAGIC_7000: i64 = 7000 17const K_MAGIC_140000: i64 = 140000 18 19 20// train TFLOPs = 6 * params_M(1e6) * tokens_M(1e6) / 1e12 = 6 * params_M * tokens_M (TFLOP=1e12) 21func grow(name: *u8, ok: i64) -> i64 { if ok==1 { gw(" PASS " as *u8) } else { gw(" FAIL " as *u8) } gw(name); gw(" 22" as *u8); return ok } 23func train_tflops(params_M: i64, tokens_M: i64) -> i64 { return 6 * params_M * tokens_M } 24// gpu-hours = train_TFLOPs / (peak_TFLOPS * MFU_permil/1000) / 3600 25func gpu_hours(tT: i64, peak_TFLOPS: i64, mfu_permil: i64) -> i64 { let eff: i64=(peak_TFLOPS*mfu_permil)/1000; if eff<=0 { return 0-1 } return tT/(eff*K_MAGIC_3600) } 26// training VRAM (GB) ~= 16 bytes/param * params = params_M * 1e6 * 16 / 1e9 = params_M*16/1000 27func train_vram_gb(params_M: i64) -> i64 { return (params_M*16)/1000 } 28 29// one model x one GPU row. 30func row(mname: *u8, params_M: i64, tokens_M: i64, gname: *u8, peak: i64, mfu: i64, cents_hr: i64, vram_gb: i64) -> i64 { 31 let tT: i64=train_tflops(params_M, tokens_M) 32 let hrs: i64=gpu_hours(tT, peak, mfu) 33 let cost: i64=(hrs*cents_hr)/100 // dollars 34 let need: i64=train_vram_gb(params_M) 35 gw(" "); gw(mname); gw(" x "); gw(gname); gw(": "); gn(hrs); gw(" GPU-hr $"); gn(cost) 36 gw(" (train-VRAM "); gn(need); gw("GB "); if need<=vram_gb { gw("FITS") } else { gw("NO-FIT->shard/multi-GPU") } gw(")\n") 37 return 0 38} 39 40func main() -> i64 { 41 gw("=== nx_llm_train_estimate: what it takes to train our own LLM (compute+cost, grounded + re-runnable) ===\n" as *u8) 42 gw(" law: FLOPs=6*N*D | Chinchilla D~=20*N | GPU-hr=FLOPs/(peak*MFU) | $=hr*rate | train-VRAM~=16 bytes/param\n\n" as *u8) 43 44 // GPUs: peak bf16 TFLOPS (realistic dense), MFU permil, cents/hr, VRAM GB. local 5080 = ~$0/hr (we own it). 45 // model tiers: params_M, tokens_M (finetune=small; pretrain=Chinchilla 20x). 46 gw(" [A] FINE-TUNE our running Qwen2.5-0.5B on OUR data (50M tokens, LoRA-class) -- improve faithfulness for our domain:\n" as *u8) 47 row("ft-0.5B" as *u8, 500, 50, "RTX5080-local" as *u8, 200, 300, 0, 16) 48 row("ft-0.5B" as *u8, 500, 50, "H100-cloud" as *u8, 990, 400, 250, 80) 49 gw("\n [B] PRETRAIN 0.5B from scratch (10B tokens) -- small but 100% OURS:\n" as *u8) 50 row("pre-0.5B" as *u8, 500, K_MAGIC_10000, "RTX5080-local" as *u8, 200, 300, 0, 16) 51 row("pre-0.5B" as *u8, 500, K_MAGIC_10000, "H100-cloud" as *u8, 990, 400, 250, 80) 52 gw("\n [C] PRETRAIN 1.5B from scratch (30B tokens) -- usable, needs cloud VRAM:\n" as *u8) 53 row("pre-1.5B" as *u8, K_MAGIC_1500, K_MAGIC_30000, "H100-cloud" as *u8, 990, 400, 250, 80) 54 gw("\n [D] PRETRAIN 7B from scratch (140B tokens) -- competitive, multi-GPU cluster:\n" as *u8) 55 row("pre-7B" as *u8, K_MAGIC_7000, K_MAGIC_140000, "H100-cloud" as *u8, 990, 400, 250, 80) 56 row("pre-7B" as *u8, K_MAGIC_7000, K_MAGIC_140000, "A100-cloud" as *u8, 312, 450, 150, 80) 57 58 gw("\n READING IT: fine-tune is hours + ~$0-5 and FITS the local RTX 5080 -- the cheap first win on our running model.\n" as *u8) 59 gw(" Pretrain-0.5B is ~$50 cloud (or ~1 week local). Pretrain-7B is ~$10k single-GPU-equiv (a 64xH100 cluster does it in ~2.5 days,\n" as *u8) 60 gw(" same total $). The single-GPU-hours figure = total work; a cluster divides WALL-time, not cost. DATA: pretrain needs the\n" as *u8) 61 gw(" Chinchilla token budget of CLEAN text (7B->140B tokens ~ 0.5-1 TB) -- which our crawl-on-burst whole-web engine can SOURCE\n" as *u8) 62 gw(" sovereignly (legal-data-only). ENGINEERING (separate from $): transformer backprop + training loop + Adam + f32 autograd\n" as *u8) 63 gw(" (our Q16 autograd + MLP-train are the foundation; extend to attention/layernorm/softmax VJPs) -- weeks, bounded, not research.\n" as *u8) 64 gw("ESTIMATE verdict=GREEN -- compute/cost modeled; fine-tune is the cheap local first step, 7B-pretrain ~$10k+corpus+eng is the sovereign-strong target\n" as *u8) 65 sys_exit(0); return 0 66}