{"$schema":"https://json.schemastore.org/sarif-2.1.0.json","version":"2.1.0","runs":[{"tool":{"driver":{"name":"codehealth","informationUri":"https://codehealth.canine.dev","rules":[{"id":"D1","name":"Cyclomatic Complexity","shortDescription":{"text":"Cyclomatic Complexity"},"helpUri":"https://codehealth.canine.dev/dimensions/D1"},{"id":"D2","name":"Cognitive Complexity","shortDescription":{"text":"Cognitive Complexity"},"helpUri":"https://codehealth.canine.dev/dimensions/D2"},{"id":"D3","name":"God Classes","shortDescription":{"text":"God Classes"},"helpUri":"https://codehealth.canine.dev/dimensions/D3"},{"id":"D4","name":"Code Duplication","shortDescription":{"text":"Code Duplication"},"helpUri":"https://codehealth.canine.dev/dimensions/D4"},{"id":"D5","name":"Coupling","shortDescription":{"text":"Coupling"},"helpUri":"https://codehealth.canine.dev/dimensions/D5"},{"id":"D6","name":"Cohesion (LCOM4)","shortDescription":{"text":"Cohesion (LCOM4)"},"helpUri":"https://codehealth.canine.dev/dimensions/D6"},{"id":"D10","name":"Test Quality","shortDescription":{"text":"Test Quality"},"helpUri":"https://codehealth.canine.dev/dimensions/D10"},{"id":"D12","name":"Dependency Hygiene","shortDescription":{"text":"Dependency Hygiene"},"helpUri":"https://codehealth.canine.dev/dimensions/D12"},{"id":"D13","name":"Secret Scanning","shortDescription":{"text":"Secret Scanning"},"helpUri":"https://codehealth.canine.dev/dimensions/D13","relationships":[{"target":{"id":"CWE-798","toolComponent":{"name":"CWE","guid":"c3a2b1d0-7f3e-4b2a-9c1d-5e6f7a8b9c0d"}},"kinds":["relevant"]},{"target":{"id":"CWE-259","toolComponent":{"name":"CWE","guid":"c3a2b1d0-7f3e-4b2a-9c1d-5e6f7a8b9c0d"}},"kinds":["relevant"]},{"target":{"id":"A07:2025","toolComponent":{"name":"OWASP Top 10","guid":"7d1f0c2e-4b6a-4e39-9a51-0b2e8c3d5f61"}},"kinds":["relevant"]}],"properties":{"cwe":["CWE-798","CWE-259"],"owasp":"A07:2025"}},{"id":"D14","name":"License Compliance","shortDescription":{"text":"License Compliance"},"helpUri":"https://codehealth.canine.dev/dimensions/D14"},{"id":"D15","name":"Churn \u00D7 Complexity Hotspots","shortDescription":{"text":"Churn \u00D7 Complexity Hotspots"},"helpUri":"https://codehealth.canine.dev/dimensions/D15"},{"id":"D16","name":"Bus Factor","shortDescription":{"text":"Bus Factor"},"helpUri":"https://codehealth.canine.dev/dimensions/D16"},{"id":"D17","name":"Explicit Debt","shortDescription":{"text":"Explicit Debt"},"helpUri":"https://codehealth.canine.dev/dimensions/D17"},{"id":"D19","name":"Documentation Quality","shortDescription":{"text":"Documentation Quality"},"helpUri":"https://codehealth.canine.dev/dimensions/D19"},{"id":"D20","name":"ADR Quality","shortDescription":{"text":"ADR Quality"},"helpUri":"https://codehealth.canine.dev/dimensions/D20"},{"id":"D26","name":"Project Cohesion","shortDescription":{"text":"Project Cohesion"},"helpUri":"https://codehealth.canine.dev/dimensions/D26"},{"id":"D28","name":"Secrets (history)","shortDescription":{"text":"Secrets (history)"},"helpUri":"https://codehealth.canine.dev/dimensions/D28","relationships":[{"target":{"id":"CWE-798","toolComponent":{"name":"CWE","guid":"c3a2b1d0-7f3e-4b2a-9c1d-5e6f7a8b9c0d"}},"kinds":["relevant"]},{"target":{"id":"CWE-259","toolComponent":{"name":"CWE","guid":"c3a2b1d0-7f3e-4b2a-9c1d-5e6f7a8b9c0d"}},"kinds":["relevant"]},{"target":{"id":"A07:2025","toolComponent":{"name":"OWASP Top 10","guid":"7d1f0c2e-4b6a-4e39-9a51-0b2e8c3d5f61"}},"kinds":["relevant"]}],"properties":{"cwe":["CWE-798","CWE-259"],"owasp":"A07:2025"}},{"id":"D29","name":"Static Analysis (SAST)","shortDescription":{"text":"Static Analysis (SAST)"},"helpUri":"https://codehealth.canine.dev/dimensions/D29","relationships":[{"target":{"id":"CWE-79","toolComponent":{"name":"CWE","guid":"c3a2b1d0-7f3e-4b2a-9c1d-5e6f7a8b9c0d"}},"kinds":["relevant"]},{"target":{"id":"CWE-89","toolComponent":{"name":"CWE","guid":"c3a2b1d0-7f3e-4b2a-9c1d-5e6f7a8b9c0d"}},"kinds":["relevant"]},{"target":{"id":"CWE-78","toolComponent":{"name":"CWE","guid":"c3a2b1d0-7f3e-4b2a-9c1d-5e6f7a8b9c0d"}},"kinds":["relevant"]},{"target":{"id":"CWE-94","toolComponent":{"name":"CWE","guid":"c3a2b1d0-7f3e-4b2a-9c1d-5e6f7a8b9c0d"}},"kinds":["relevant"]},{"target":{"id":"CWE-77","toolComponent":{"name":"CWE","guid":"c3a2b1d0-7f3e-4b2a-9c1d-5e6f7a8b9c0d"}},"kinds":["relevant"]},{"target":{"id":"A05:2025","toolComponent":{"name":"OWASP Top 10","guid":"7d1f0c2e-4b6a-4e39-9a51-0b2e8c3d5f61"}},"kinds":["relevant"]}],"properties":{"cwe":["CWE-79","CWE-89","CWE-78","CWE-94","CWE-77"],"owasp":"A05:2025"}},{"id":"D30","name":"Dependency Vulnerabilities","shortDescription":{"text":"Dependency Vulnerabilities"},"helpUri":"https://codehealth.canine.dev/dimensions/D30","relationships":[{"target":{"id":"CWE-1395","toolComponent":{"name":"CWE","guid":"c3a2b1d0-7f3e-4b2a-9c1d-5e6f7a8b9c0d"}},"kinds":["relevant"]},{"target":{"id":"CWE-937","toolComponent":{"name":"CWE","guid":"c3a2b1d0-7f3e-4b2a-9c1d-5e6f7a8b9c0d"}},"kinds":["relevant"]},{"target":{"id":"A03:2025","toolComponent":{"name":"OWASP Top 10","guid":"7d1f0c2e-4b6a-4e39-9a51-0b2e8c3d5f61"}},"kinds":["relevant"]}],"properties":{"cwe":["CWE-1395","CWE-937"],"owasp":"A03:2025"}},{"id":"D34","name":"Knowledge Freshness","shortDescription":{"text":"Knowledge Freshness"},"helpUri":"https://codehealth.canine.dev/dimensions/D34"},{"id":"D35","name":"Change Coupling","shortDescription":{"text":"Change Coupling"},"helpUri":"https://codehealth.canine.dev/dimensions/D35"},{"id":"D36","name":"Supply-chain Provenance \u0026 Signing","shortDescription":{"text":"Supply-chain Provenance \u0026 Signing"},"helpUri":"https://codehealth.canine.dev/dimensions/D36","relationships":[{"target":{"id":"CWE-1357","toolComponent":{"name":"CWE","guid":"c3a2b1d0-7f3e-4b2a-9c1d-5e6f7a8b9c0d"}},"kinds":["relevant"]},{"target":{"id":"CWE-494","toolComponent":{"name":"CWE","guid":"c3a2b1d0-7f3e-4b2a-9c1d-5e6f7a8b9c0d"}},"kinds":["relevant"]},{"target":{"id":"A03:2025","toolComponent":{"name":"OWASP Top 10","guid":"7d1f0c2e-4b6a-4e39-9a51-0b2e8c3d5f61"}},"kinds":["relevant"]}],"properties":{"cwe":["CWE-1357","CWE-494"],"owasp":"A03:2025"}},{"id":"D43","name":"Malicious Dependencies","shortDescription":{"text":"Malicious Dependencies"},"helpUri":"https://codehealth.canine.dev/dimensions/D43","relationships":[{"target":{"id":"CWE-506","toolComponent":{"name":"CWE","guid":"c3a2b1d0-7f3e-4b2a-9c1d-5e6f7a8b9c0d"}},"kinds":["relevant"]},{"target":{"id":"A08:2025","toolComponent":{"name":"OWASP Top 10","guid":"7d1f0c2e-4b6a-4e39-9a51-0b2e8c3d5f61"}},"kinds":["relevant"]}],"properties":{"cwe":["CWE-506"],"owasp":"A08:2025"}},{"id":"D44","name":"Platform End-of-Life","shortDescription":{"text":"Platform End-of-Life"},"helpUri":"https://codehealth.canine.dev/dimensions/D44"},{"id":"AX10","name":"Code composition","shortDescription":{"text":"Code composition"},"helpUri":"https://codehealth.canine.dev/dimensions/AX10"},{"id":"AX3","name":"Project dependency cycles","shortDescription":{"text":"Project dependency cycles"},"helpUri":"https://codehealth.canine.dev/dimensions/AX3"},{"id":"AX4","name":"Dependency direction","shortDescription":{"text":"Dependency direction"},"helpUri":"https://codehealth.canine.dev/dimensions/AX4"},{"id":"AX9","name":"CQS / query purity","shortDescription":{"text":"CQS / query purity"},"helpUri":"https://codehealth.canine.dev/dimensions/AX9"},{"id":"AXB2","name":"Runtime readiness","shortDescription":{"text":"Runtime readiness"},"helpUri":"https://codehealth.canine.dev/dimensions/AXB2"},{"id":"ED5","name":"Idempotency","shortDescription":{"text":"Idempotency"},"helpUri":"https://codehealth.canine.dev/dimensions/ED5"},{"id":"M1","name":"Documentation (README)","shortDescription":{"text":"Documentation (README)"},"helpUri":"https://codehealth.canine.dev/dimensions/M1"},{"id":"M2","name":"Architecture documentation","shortDescription":{"text":"Architecture documentation"},"helpUri":"https://codehealth.canine.dev/dimensions/M2"},{"id":"M3","name":"Folder \u0026 project structure","shortDescription":{"text":"Folder \u0026 project structure"},"helpUri":"https://codehealth.canine.dev/dimensions/M3"},{"id":"M4","name":"Documentation accuracy","shortDescription":{"text":"Documentation accuracy"},"helpUri":"https://codehealth.canine.dev/dimensions/M4"},{"id":"P1","name":"CI/CD gates","shortDescription":{"text":"CI/CD gates"},"helpUri":"https://codehealth.canine.dev/dimensions/P1"},{"id":"P10","name":"Library API \u0026 versioning","shortDescription":{"text":"Library API \u0026 versioning"},"helpUri":"https://codehealth.canine.dev/dimensions/P10"},{"id":"P12","name":"CI test-gate honesty","shortDescription":{"text":"CI test-gate honesty"},"helpUri":"https://codehealth.canine.dev/dimensions/P12"},{"id":"P2","name":"Observability","shortDescription":{"text":"Observability"},"helpUri":"https://codehealth.canine.dev/dimensions/P2"},{"id":"P3","name":"Security \u0026 performance tooling","shortDescription":{"text":"Security \u0026 performance tooling"},"helpUri":"https://codehealth.canine.dev/dimensions/P3"},{"id":"P4","name":"Deployment \u0026 Rollback","shortDescription":{"text":"Deployment \u0026 Rollback"},"helpUri":"https://codehealth.canine.dev/dimensions/P4"},{"id":"P6","name":"Release Hygiene","shortDescription":{"text":"Release Hygiene"},"helpUri":"https://codehealth.canine.dev/dimensions/P6"},{"id":"PF1","name":"Benchmark discipline","shortDescription":{"text":"Benchmark discipline"},"helpUri":"https://codehealth.canine.dev/dimensions/PF1"},{"id":"PF3","name":"Async \u0026 latency hygiene","shortDescription":{"text":"Async \u0026 latency hygiene"},"helpUri":"https://codehealth.canine.dev/dimensions/PF3"},{"id":"SC1","name":"Supply-chain hygiene","shortDescription":{"text":"Supply-chain hygiene"},"helpUri":"https://codehealth.canine.dev/dimensions/SC1"},{"id":"X1","name":"Async correctness","shortDescription":{"text":"Async correctness"},"helpUri":"https://codehealth.canine.dev/dimensions/X1"},{"id":"X10","name":"Duplicated predicate","shortDescription":{"text":"Duplicated predicate"},"helpUri":"https://codehealth.canine.dev/dimensions/X10"},{"id":"X12","name":"Unreachable branch","shortDescription":{"text":"Unreachable branch"},"helpUri":"https://codehealth.canine.dev/dimensions/X12"},{"id":"X13","name":"Undrained process stream","shortDescription":{"text":"Undrained process stream"},"helpUri":"https://codehealth.canine.dev/dimensions/X13"},{"id":"X14","name":"Bypassable address classification","shortDescription":{"text":"Bypassable address classification"},"helpUri":"https://codehealth.canine.dev/dimensions/X14"},{"id":"X15","name":"Unvalidated length from an untrusted reader","shortDescription":{"text":"Unvalidated length from an untrusted reader"},"helpUri":"https://codehealth.canine.dev/dimensions/X15"},{"id":"X16","name":"Unfloored truncation loop","shortDescription":{"text":"Unfloored truncation loop"},"helpUri":"https://codehealth.canine.dev/dimensions/X16"},{"id":"X17","name":"Uncapped recursion over a caller-supplied document","shortDescription":{"text":"Uncapped recursion over a caller-supplied document"},"helpUri":"https://codehealth.canine.dev/dimensions/X17"},{"id":"X18","name":"Disposal-pattern correctness","shortDescription":{"text":"Disposal-pattern correctness"},"helpUri":"https://codehealth.canine.dev/dimensions/X18"},{"id":"X19","name":"Unrestored process-global state","shortDescription":{"text":"Unrestored process-global state"},"helpUri":"https://codehealth.canine.dev/dimensions/X19"},{"id":"X20","name":"Mistyped argument guard","shortDescription":{"text":"Mistyped argument guard"},"helpUri":"https://codehealth.canine.dev/dimensions/X20"},{"id":"X21","name":"Side-effecting pattern guard","shortDescription":{"text":"Side-effecting pattern guard"},"helpUri":"https://codehealth.canine.dev/dimensions/X21"},{"id":"X22","name":"Contradicted release guard","shortDescription":{"text":"Contradicted release guard"},"helpUri":"https://codehealth.canine.dev/dimensions/X22"},{"id":"X23","name":"Unguarded diagnostic materialisation","shortDescription":{"text":"Unguarded diagnostic materialisation"},"helpUri":"https://codehealth.canine.dev/dimensions/X23"},{"id":"X24","name":"Document value interpolated into markup unescaped","shortDescription":{"text":"Document value interpolated into markup unescaped"},"helpUri":"https://codehealth.canine.dev/dimensions/X24"},{"id":"X25","name":"Inert configuration knob","shortDescription":{"text":"Inert configuration knob"},"helpUri":"https://codehealth.canine.dev/dimensions/X25"},{"id":"X26","name":"Unsynchronised callback handoff","shortDescription":{"text":"Unsynchronised callback handoff"},"helpUri":"https://codehealth.canine.dev/dimensions/X26"},{"id":"X27","name":"Collection changed while being enumerated","shortDescription":{"text":"Collection changed while being enumerated"},"helpUri":"https://codehealth.canine.dev/dimensions/X27"},{"id":"X28","name":"Index access outside its own emptiness guard","shortDescription":{"text":"Index access outside its own emptiness guard"},"helpUri":"https://codehealth.canine.dev/dimensions/X28"},{"id":"X29","name":"Per-element action decided by a fixed element","shortDescription":{"text":"Per-element action decided by a fixed element"},"helpUri":"https://codehealth.canine.dev/dimensions/X29"},{"id":"X30","name":"Support guard that admits what it rejects","shortDescription":{"text":"Support guard that admits what it rejects"},"helpUri":"https://codehealth.canine.dev/dimensions/X30"},{"id":"X32","name":"Type resolved by simple name across every loaded assembly","shortDescription":{"text":"Type resolved by simple name across every loaded assembly"},"helpUri":"https://codehealth.canine.dev/dimensions/X32"},{"id":"X4","name":"Structured logging","shortDescription":{"text":"Structured logging"},"helpUri":"https://codehealth.canine.dev/dimensions/X4"},{"id":"X6","name":"Hand-rolled structured-format parsing","shortDescription":{"text":"Hand-rolled structured-format parsing"},"helpUri":"https://codehealth.canine.dev/dimensions/X6"},{"id":"X7","name":"Silent fallback defaults","shortDescription":{"text":"Silent fallback defaults"},"helpUri":"https://codehealth.canine.dev/dimensions/X7"},{"id":"X9","name":"Subsumed condition operand","shortDescription":{"text":"Subsumed condition operand"},"helpUri":"https://codehealth.canine.dev/dimensions/X9"}]}},"results":[{"ruleId":"D1","level":"warning","message":{"text":"bitmap_gen::generate_tables (cyclomatic 41): bitmap_gen::generate_tables has cyclomatic complexity 41 (threshold 15). To reduce it, separate the cases: extract each independent branch into its own named function, and where the body has guards that only reject input, fold those into early returns at the top."},"locations":[{"physicalLocation":{"artifactLocation":{"uri":"tokenizers/bitmap_gen/src/lib.rs"},"region":{"startLine":134}}}],"partialFingerprints":{"codehealthFindingId/v1":"cf42a12ac222da2f150eb219d93f83b6518e7c624dcc52b92ccefeed1ec0d7ed"}},{"ruleId":"D1","level":"warning","message":{"text":"ParityBpeTrainer::do_train (cyclomatic 32): ParityBpeTrainer::do_train has cyclomatic complexity 32 (threshold 15). To reduce it, separate the cases: extract each independent branch into its own named function, and where the body has guards that only reject input, fold those into early returns at the top."},"locations":[{"physicalLocation":{"artifactLocation":{"uri":"tokenizers/tk-train/src/trainers/bpe/parity_trainer.rs"},"region":{"startLine":1173}}}],"partialFingerprints":{"codehealthFindingId/v1":"afa37f8b1a98818fd3f0293ee3961027328727e0fed7edc17b38de047e676c37"}},{"ruleId":"D1","level":"warning","message":{"text":"bitmap_gen::atom (cyclomatic 27): bitmap_gen::atom has cyclomatic complexity 27 (threshold 15). To reduce it, split the body: these branches sit side by side rather than nested inside one another, so extracting each one on its own would leave a function per branch. Group the statements between the checks into named steps and move each step into its own function, so the body reads as a short sequence of named stages."},"locations":[{"physicalLocation":{"artifactLocation":{"uri":"tokenizers/bitmap_gen/src/lib.rs"},"region":{"startLine":41}}}],"partialFingerprints":{"codehealthFindingId/v1":"25b309a89156113a98f57acf4925c0d1093e8454c1aa54e93201d0d8b8268aae"}},{"ruleId":"D1","level":"warning","message":{"text":"bitcannon::models::family_o200k::run (cyclomatic 24): bitcannon::models::family_o200k::run has cyclomatic complexity 24 (threshold 15). To reduce it, separate the cases: extract each independent branch into its own named function, and where the body has guards that only reject input, fold those into early returns at the top."},"locations":[{"physicalLocation":{"artifactLocation":{"uri":"tokenizers/bitcannon/src/models/family_o200k.rs"},"region":{"startLine":189}}}],"partialFingerprints":{"codehealthFindingId/v1":"83b4679ef5de4f61454b72b2be1d739a6833660ad002fb11248ca78fc0d31135"}},{"ruleId":"D1","level":"warning","message":{"text":"UnigramTrainer::prune_sentence_pieces (cyclomatic 21): UnigramTrainer::prune_sentence_pieces has cyclomatic complexity 21 (threshold 15). To reduce it, separate the cases: extract each independent branch into its own named function, and where the body has guards that only reject input, fold those into early returns at the top."},"locations":[{"physicalLocation":{"artifactLocation":{"uri":"tokenizers/tk-train/src/trainers/unigram.rs"},"region":{"startLine":352}}}],"partialFingerprints":{"codehealthFindingId/v1":"531ff23cab052ee3df8cb2ccce7db8c32ebcb6eea83d667037b47492191e3737"}},{"ruleId":"D1","level":"warning","message":{"text":"PipelineBPE::from_merge_map (cyclomatic 20): PipelineBPE::from_merge_map has cyclomatic complexity 20 (threshold 15). To reduce it, separate the cases: extract each independent branch into its own named function, and where the body has guards that only reject input, fold those into early returns at the top."},"locations":[{"physicalLocation":{"artifactLocation":{"uri":"tokenizers/tk-encode/src/models/bpe/serialization.rs"},"region":{"startLine":193}}}],"partialFingerprints":{"codehealthFindingId/v1":"dd06f1e63793117924ecc69bd8f3030d3e9adfd1b0ab038d2a59d87ba8be9263"}},{"ruleId":"D1","level":"warning","message":{"text":"bitcannon::classify::wasm::classify_wasm (cyclomatic 19): bitcannon::classify::wasm::classify_wasm has cyclomatic complexity 19 (threshold 15). To reduce it, separate the cases: extract each independent branch into its own named function, and where the body has guards that only reject input, fold those into early returns at the top."},"locations":[{"physicalLocation":{"artifactLocation":{"uri":"tokenizers/bitcannon/src/classify/wasm.rs"},"region":{"startLine":149}}}],"partialFingerprints":{"codehealthFindingId/v1":"da80e82ecb86506b517a87b8c671623a36cbad6367116320ea1172526f4c18a1"}},{"ruleId":"D1","level":"warning","message":{"text":"tk_convert::convert::lower_template_node (cyclomatic 19): tk_convert::convert::lower_template_node has cyclomatic complexity 19 (threshold 15). To reduce it, separate the cases: extract each independent branch into its own named function, and where the body has guards that only reject input, fold those into early returns at the top."},"locations":[{"physicalLocation":{"artifactLocation":{"uri":"tokenizers/tk-convert/src/convert.rs"},"region":{"startLine":740}}}],"partialFingerprints":{"codehealthFindingId/v1":"45c7c6d9d245bf00f8055b9f6b3821de24c7c7e66639df686fc4f76b0557d11c"}},{"ruleId":"D1","level":"warning","message":{"text":"ParityBpeTrainer::init_lengths (cyclomatic 18): ParityBpeTrainer::init_lengths has cyclomatic complexity 18 (threshold 15). To reduce it, separate the cases: extract each independent branch into its own named function, and where the body has guards that only reject input, fold those into early returns at the top."},"locations":[{"physicalLocation":{"artifactLocation":{"uri":"tokenizers/tk-train/src/trainers/bpe/parity_trainer.rs"},"region":{"startLine":903}}}],"partialFingerprints":{"codehealthFindingId/v1":"a386be17bc2718cb67dea2fd65eca8586d30b09bf48a22625bc225754b33acaf"}},{"ruleId":"D1","level":"warning","message":{"text":"bitcannon::classify::neon::classify_neon (cyclomatic 17): bitcannon::classify::neon::classify_neon has cyclomatic complexity 17 (threshold 15). To reduce it, separate the cases: extract each independent branch into its own named function, and where the body has guards that only reject input, fold those into early returns at the top."},"locations":[{"physicalLocation":{"artifactLocation":{"uri":"tokenizers/bitcannon/src/classify/neon.rs"},"region":{"startLine":159}}}],"partialFingerprints":{"codehealthFindingId/v1":"663e6667fbf46c5180d8690dd80dd2c93288ed08af1f7d1b9926b4bb0a80f478"}},{"ruleId":"D1","level":"warning","message":{"text":"BpeTables::build (cyclomatic 16): BpeTables::build has cyclomatic complexity 16 (threshold 15). To reduce it, separate the cases: extract each independent branch into its own named function, and where the body has guards that only reject input, fold those into early returns at the top."},"locations":[{"physicalLocation":{"artifactLocation":{"uri":"tokenizers/tk-encode/src/models/bpe/tables.rs"},"region":{"startLine":79}}}],"partialFingerprints":{"codehealthFindingId/v1":"fac9bbea4aef608da5382063a7ca44b1400d7d6f27e20af99062be13d9cf65bb"}},{"ruleId":"D1","level":"warning","message":{"text":"tk_encode::tokenizer::pipeline::pre_tokenizer::split_matches (cyclomatic 16): tk_encode::tokenizer::pipeline::pre_tokenizer::split_matches has cyclomatic complexity 16 (threshold 15). To reduce it, separate the cases: extract each independent branch into its own named function, and where the body has guards that only reject input, fold those into early returns at the top."},"locations":[{"physicalLocation":{"artifactLocation":{"uri":"tokenizers/tk-encode/src/tokenizer/pipeline/pre_tokenizer.rs"},"region":{"startLine":342}}}],"partialFingerprints":{"codehealthFindingId/v1":"8c87eeb7534e8d2eb1cedc179e01663212384a4dc746d5872b48ecd29ac082f0"}},{"ruleId":"D2","level":"warning","message":{"text":"bitmap_gen::generate_tables (cognitive 79): bitmap_gen::generate_tables has cognitive complexity 79 (threshold 15). Drivers by points: if/else 23 (47 pts), loops 20 (32 pts) (nesting depth added 36). To reduce it, split the body into named stages: move each independent step or branch into its own named function so the body reads as a short sequence of named calls rather than one long body."},"locations":[{"physicalLocation":{"artifactLocation":{"uri":"tokenizers/bitmap_gen/src/lib.rs"},"region":{"startLine":134}}}],"partialFingerprints":{"codehealthFindingId/v1":"d0a76d718b2c5689af59e5da6658fbbcc05c31b54ec78b46753394deb16166f4"}},{"ruleId":"D2","level":"warning","message":{"text":"ParityBpeTrainer::do_train (cognitive 63): ParityBpeTrainer::do_train has cognitive complexity 63 (threshold 15). Drivers by points: if/else 21 (40 pts), loops 10 (19 pts), boolean chains 2, match/switch 1 (2 pts) (nesting depth added 29). To reduce it, split the body into named stages: move each independent step or branch into its own named function so the body reads as a short sequence of named calls rather than one long body."},"locations":[{"physicalLocation":{"artifactLocation":{"uri":"tokenizers/tk-train/src/trainers/bpe/parity_trainer.rs"},"region":{"startLine":1173}}}],"partialFingerprints":{"codehealthFindingId/v1":"b5e535ed7a554d70691e87c71f97215af045e5b50acf7760228460a4fa45bd85"}},{"ruleId":"D2","level":"warning","message":{"text":"ParityBpeTrainer::init_lengths (cognitive 53): ParityBpeTrainer::init_lengths has cognitive complexity 53 (threshold 15). Drivers by points: if/else 10 (32 pts), loops 7 (18 pts), boolean chains 3 (nesting depth added 33). To reduce it, split the body into named stages: move each independent step or branch into its own named function so the body reads as a short sequence of named calls rather than one long body."},"locations":[{"physicalLocation":{"artifactLocation":{"uri":"tokenizers/tk-train/src/trainers/bpe/parity_trainer.rs"},"region":{"startLine":903}}}],"partialFingerprints":{"codehealthFindingId/v1":"5bf54ff98870eb2d09a63c5e63d3d6526e4ed52c443fbbff476decfc8830217a"}},{"ruleId":"D2","level":"warning","message":{"text":"bitcannon::models::family_o200k::run (cognitive 51): bitcannon::models::family_o200k::run has cognitive complexity 51 (threshold 15). Drivers by points: if/else 23 (42 pts), loops 3 (5 pts), boolean chains 4 (nesting depth added 21). To reduce it, split the body into named stages: move each independent step or branch into its own named function so the body reads as a short sequence of named calls rather than one long body."},"locations":[{"physicalLocation":{"artifactLocation":{"uri":"tokenizers/bitcannon/src/models/family_o200k.rs"},"region":{"startLine":189}}}],"partialFingerprints":{"codehealthFindingId/v1":"8b543d7f09fada287969aff026cb3dd45df34a471f0a3e4023995718331f2a08"}},{"ruleId":"D2","level":"warning","message":{"text":"tk_encode::utils::truncation::truncate_pair (cognitive 51): tk_encode::utils::truncation::truncate_pair has cognitive complexity 51 (threshold 15). Drivers by points: if/else 19 (48 pts), match/switch 1 (3 pts) (nesting depth added 31). To reduce it, flatten the nesting: this score is depth rather than breadth \u2014 most of its points come from checks stacked inside one another, so the work sits several levels in. Invert each enclosing check into an early exit (a return, or the language\u0027s equivalent) so the happy path stays at one level, and where a level cannot be exited early, lift the block it encloses into its own named function."},"locations":[{"physicalLocation":{"artifactLocation":{"uri":"tokenizers/tk-encode/src/utils/truncation.rs"},"region":{"startLine":69}}}],"partialFingerprints":{"codehealthFindingId/v1":"da3281df73edd32a97ca563e7c2d09daa89dbb21b403464189f722ac751274ea"}},{"ruleId":"D2","level":"warning","message":{"text":"PipelineBPE::from_merge_map (cognitive 49): PipelineBPE::from_merge_map has cognitive complexity 49 (threshold 15). Drivers by points: if/else 17 (33 pts), loops 3 (8 pts), match/switch 1 (5 pts), boolean chains 3 (nesting depth added 25). To reduce it, split the body into named stages: move each independent step or branch into its own named function so the body reads as a short sequence of named calls rather than one long body."},"locations":[{"physicalLocation":{"artifactLocation":{"uri":"tokenizers/tk-encode/src/models/bpe/serialization.rs"},"region":{"startLine":193}}}],"partialFingerprints":{"codehealthFindingId/v1":"89e3e7edebffcd01b57746365ca24e14c60e22e8528ce629a0176fe952d46fc2"}},{"ruleId":"D2","level":"warning","message":{"text":"bitcannon::classify::neon::classify_neon (cognitive 43): bitcannon::classify::neon::classify_neon has cognitive complexity 43 (threshold 15). Drivers by points: if/else 12 (30 pts), loops 6 (13 pts) (nesting depth added 25). To reduce it, split the body into named stages: move each independent step or branch into its own named function so the body reads as a short sequence of named calls rather than one long body."},"locations":[{"physicalLocation":{"artifactLocation":{"uri":"tokenizers/bitcannon/src/classify/neon.rs"},"region":{"startLine":159}}}],"partialFingerprints":{"codehealthFindingId/v1":"155fefbdc7059a5c479fc34a774694138e027b8e64299209ef1ea145525ed45f"}},{"ruleId":"D2","level":"warning","message":{"text":"bitcannon::classify::wasm::classify_wasm (cognitive 43): bitcannon::classify::wasm::classify_wasm has cognitive complexity 43 (threshold 15). Drivers by points: if/else 13 (30 pts), loops 6 (12 pts), boolean chains 1 (nesting depth added 23). To reduce it, split the body into named stages: move each independent step or branch into its own named function so the body reads as a short sequence of named calls rather than one long body."},"locations":[{"physicalLocation":{"artifactLocation":{"uri":"tokenizers/bitcannon/src/classify/wasm.rs"},"region":{"startLine":149}}}],"partialFingerprints":{"codehealthFindingId/v1":"c3ee7610f4c656f469cb9163d07259a7cafe3c75b908d90e2bc76b1de3290acf"}},{"ruleId":"D2","level":"warning","message":{"text":"UnigramTrainer::prune_sentence_pieces (cognitive 37): UnigramTrainer::prune_sentence_pieces has cognitive complexity 37 (threshold 15). Drivers by points: if/else 11 (20 pts), loops 8 (15 pts), boolean chains 2 (nesting depth added 16). To reduce it, split the body into named stages: move each independent step or branch into its own named function so the body reads as a short sequence of named calls rather than one long body."},"locations":[{"physicalLocation":{"artifactLocation":{"uri":"tokenizers/tk-train/src/trainers/unigram.rs"},"region":{"startLine":352}}}],"partialFingerprints":{"codehealthFindingId/v1":"8bc9343a95b0b95e8c65f612718fe1eea92de58be8e159b3a4a84c8b6c5ef56f"}},{"ruleId":"D2","level":"warning","message":{"text":"ParityBpeTrainer::select_next_language (cognitive 35): ParityBpeTrainer::select_next_language has cognitive complexity 35 (threshold 15). Drivers by points: if/else 7 (17 pts), loops 4 (12 pts), match/switch 2 (4 pts), boolean chains 2 (nesting depth added 20). To reduce it, split the body into named stages: move each independent step or branch into its own named function so the body reads as a short sequence of named calls rather than one long body."},"locations":[{"physicalLocation":{"artifactLocation":{"uri":"tokenizers/tk-train/src/trainers/bpe/parity_trainer.rs"},"region":{"startLine":1042}}}],"partialFingerprints":{"codehealthFindingId/v1":"e80fffe60650a04a0feac1461619518a0593327fe5c97d223f91c228bfd8a88b"}},{"ruleId":"D2","level":"warning","message":{"text":"bitmap_gen::atom (cognitive 31): bitmap_gen::atom has cognitive complexity 31 (threshold 15). Drivers by points: if/else 22 (27 pts), boolean chains 3, match/switch 1 (nesting depth added 5). To reduce it, split the body: most of this score is breadth rather than depth \u2014 checks laid out side by side rather than stacked \u2014 so group the statements between the checks into named steps and move each step into its own function. Some of it IS depth: where a check sits inside another whose only job is to reach it, merge the two into one condition, and where an else follows a branch that already returns, drop the trailing else and let the rest of the body continue at one level."},"locations":[{"physicalLocation":{"artifactLocation":{"uri":"tokenizers/bitmap_gen/src/lib.rs"},"region":{"startLine":41}}}],"partialFingerprints":{"codehealthFindingId/v1":"ec76978f715eb5d7b71d6dd2136eda8b55b9e3a29e0c9b309e36285e9e52ffd6"}},{"ruleId":"D2","level":"warning","message":{"text":"tk_convert::convert::lower_template_node (cognitive 30): tk_convert::convert::lower_template_node has cognitive complexity 30 (threshold 15). Drivers by points: if/else 11 (22 pts), loops 3 (5 pts), boolean chains 2, match/switch 1 (nesting depth added 13). To reduce it, split the body into named stages: move each independent step or branch into its own named function so the body reads as a short sequence of named calls rather than one long body."},"locations":[{"physicalLocation":{"artifactLocation":{"uri":"tokenizers/tk-convert/src/convert.rs"},"region":{"startLine":740}}}],"partialFingerprints":{"codehealthFindingId/v1":"25073a8c81e16a40c38c344ee2865c561cfd067a00bff00897da7b15954c378c"}},{"ruleId":"D2","level":"warning","message":{"text":"tk_serialize::vendored::f64_from_parts (cognitive 30): tk_serialize::vendored::f64_from_parts has cognitive complexity 30 (threshold 15). Drivers by points: if/else 11 (27 pts), match/switch 1 (2 pts), loops 1 (nesting depth added 17). To reduce it, split the body into named stages: move each independent step or branch into its own named function so the body reads as a short sequence of named calls rather than one long body."},"locations":[{"physicalLocation":{"artifactLocation":{"uri":"tokenizers/tk-serialize/src/vendored.rs"},"region":{"startLine":62}}}],"partialFingerprints":{"codehealthFindingId/v1":"305fbdbfd942c241c81ab73ab8914ff87994d70c0534d2602d65c9791d620b50"}},{"ruleId":"D2","level":"warning","message":{"text":"ByteFallback::decode_chain (cognitive 28): ByteFallback::decode_chain has cognitive complexity 28 (threshold 15). Drivers by points: if/else 10 (18 pts), loops 3 (9 pts), boolean chains 1 (nesting depth added 14). To reduce it, split the body into named stages: move each independent step or branch into its own named function so the body reads as a short sequence of named calls rather than one long body."},"locations":[{"physicalLocation":{"artifactLocation":{"uri":"tokenizers/tk-encode/src/decoders/byte_fallback.rs"},"region":{"startLine":17}}}],"partialFingerprints":{"codehealthFindingId/v1":"915c5ce1441519056acc8579eee7b29299d3001a6611724b6ff3f95ceefaf53b"}},{"ruleId":"D2","level":"warning","message":{"text":"Lattice::nbest (cognitive 28): Lattice::nbest has cognitive complexity 28 (threshold 15). Drivers by points: loops 4 (15 pts), if/else 4 (12 pts), match/switch 1 (nesting depth added 19). The drivers above price the dispatch low by construction \u2014 a dispatch is charged once however many cases it lists, while each branch inside an arm is charged in full \u2014 so most of this count is what the case bodies hold, and the arms are where it can be reduced. To reduce it, keep the dispatch but shrink the arms: move each non-trivial case body into its own named function (or onto the value being matched) so the dispatch reads one line per case, and group related cases into a sub-dispatch. Keep every case explicit, and make the behaviour for cases you do not list a deliberate choice rather than an accident."},"locations":[{"physicalLocation":{"artifactLocation":{"uri":"tokenizers/tk-encode/src/models/unigram/lattice.rs"},"region":{"startLine":250}}}],"partialFingerprints":{"codehealthFindingId/v1":"ab8f2c94caf70b42afb0f7724421c4391862b7d7bb1a5adc8e60266fb0c5b20f"}},{"ruleId":"D2","level":"warning","message":{"text":"tk_encode::tokenizer::pipeline::pre_tokenizer::split_matches (cognitive 28): tk_encode::tokenizer::pipeline::pre_tokenizer::split_matches has cognitive complexity 28 (threshold 15). Drivers by points: if/else 13 (23 pts), boolean chains 3, loops 1, match/switch 1 (nesting depth added 10). To reduce it, split the body into named stages: move each independent step or branch into its own named function so the body reads as a short sequence of named calls rather than one long body."},"locations":[{"physicalLocation":{"artifactLocation":{"uri":"tokenizers/tk-encode/src/tokenizer/pipeline/pre_tokenizer.rs"},"region":{"startLine":342}}}],"partialFingerprints":{"codehealthFindingId/v1":"d710936118aa6484d4f2c7d59b0641a614042e982a1176a377c42ef3369dc55f"}},{"ruleId":"D2","level":"warning","message":{"text":"Unigram::encode_optimized (cognitive 26): Unigram::encode_optimized has cognitive complexity 26 (threshold 15). Drivers by points: if/else 8 (18 pts), boolean chains 4, loops 3 (4 pts) (nesting depth added 11). To reduce it, split the body into named stages: move each independent step or branch into its own named function so the body reads as a short sequence of named calls rather than one long body."},"locations":[{"physicalLocation":{"artifactLocation":{"uri":"tokenizers/tk-encode/src/models/unigram/model.rs"},"region":{"startLine":269}}}],"partialFingerprints":{"codehealthFindingId/v1":"374ffc56420a3886ed21cdf58de092cfba40f9f1130bc11f835785ef1fd85022"}},{"ruleId":"D2","level":"warning","message":{"text":"BpeTrainer::do_train (cognitive 25): BpeTrainer::do_train has cognitive complexity 25 (threshold 15). Drivers by points: if/else 10 (20 pts), loops 2 (3 pts), boolean chains 2 (nesting depth added 11). To reduce it, split the body into named stages: move each independent step or branch into its own named function so the body reads as a short sequence of named calls rather than one long body."},"locations":[{"physicalLocation":{"artifactLocation":{"uri":"tokenizers/tk-train/src/trainers/bpe/mod.rs"},"region":{"startLine":503}}}],"partialFingerprints":{"codehealthFindingId/v1":"1c44255dd173902e09303c7980d003dc0d5a27379195b4b12ef863693583a025"}},{"ruleId":"D2","level":"warning","message":{"text":"tk_serialize::vendored::number (cognitive 24): tk_serialize::vendored::number has cognitive complexity 24 (threshold 15). Drivers by points: if/else 9 (13 pts), match/switch 3 (6 pts), loops 3 (5 pts) (nesting depth added 9). To reduce it, split the body into named stages: move each independent step or branch into its own named function so the body reads as a short sequence of named calls rather than one long body."},"locations":[{"physicalLocation":{"artifactLocation":{"uri":"tokenizers/tk-serialize/src/vendored.rs"},"region":{"startLine":100}}}],"partialFingerprints":{"codehealthFindingId/v1":"338727dcfcf98260c63a584b3c04a80b752f406c4c8798b3753b6ffda3c3130c"}},{"ruleId":"D2","level":"warning","message":{"text":"BpeTables::build (cognitive 23): BpeTables::build has cognitive complexity 23 (threshold 15). Drivers by points: if/else 8 (14 pts), loops 6, boolean chains 3 (nesting depth added 6). To reduce it, split the body into named stages: move each independent step or branch into its own named function so the body reads as a short sequence of named calls rather than one long body."},"locations":[{"physicalLocation":{"artifactLocation":{"uri":"tokenizers/tk-encode/src/models/bpe/tables.rs"},"region":{"startLine":79}}}],"partialFingerprints":{"codehealthFindingId/v1":"56246f5e4adf5562ebfd136a33bc25cbb6d9f2e57637da87f792c4dc380c6854"}},{"ruleId":"D2","level":"warning","message":{"text":"BpeTrainer::tokenize_words (cognitive 22): BpeTrainer::tokenize_words has cognitive complexity 22 (threshold 15). Drivers by points: if/else 5 (17 pts), loops 2 (3 pts), boolean chains 2 (nesting depth added 13). To reduce it, split the body into named stages: move each independent step or branch into its own named function so the body reads as a short sequence of named calls rather than one long body."},"locations":[{"physicalLocation":{"artifactLocation":{"uri":"tokenizers/tk-train/src/trainers/bpe/mod.rs"},"region":{"startLine":386}}}],"partialFingerprints":{"codehealthFindingId/v1":"699a4d7d4498a2909b346e1816914b7c5d195756aff453c3558391eb928a66c2"}},{"ruleId":"D2","level":"warning","message":{"text":"UnigramTrainer::finalize (cognitive 22): UnigramTrainer::finalize has cognitive complexity 22 (threshold 15). Drivers by points: if/else 13 (18 pts), loops 2, match/switch 1 (2 pts) (nesting depth added 6). To reduce it, split the body into named stages: move each independent step or branch into its own named function so the body reads as a short sequence of named calls rather than one long body."},"locations":[{"physicalLocation":{"artifactLocation":{"uri":"tokenizers/tk-train/src/trainers/unigram.rs"},"region":{"startLine":190}}}],"partialFingerprints":{"codehealthFindingId/v1":"0f895a30877720082baf536da5cfaf6d66b52d686ee45bc71eb543532b81a104"}},{"ruleId":"D2","level":"warning","message":{"text":"Lattice::viterbi (cognitive 21): Lattice::viterbi has cognitive complexity 21 (threshold 15). Drivers by points: if/else 5 (10 pts), loops 4 (7 pts), match/switch 1 (3 pts), boolean chains 1 (nesting depth added 10). To reduce it, split the body into named stages: move each independent step or branch into its own named function so the body reads as a short sequence of named calls rather than one long body."},"locations":[{"physicalLocation":{"artifactLocation":{"uri":"tokenizers/tk-encode/src/models/unigram/lattice.rs"},"region":{"startLine":189}}}],"partialFingerprints":{"codehealthFindingId/v1":"bb2ae321d68464a3afb0b7ac2fd614e39e3b922b027547a825c174f11554ab83"}},{"ruleId":"D2","level":"warning","message":{"text":"tk_encode::tokenizer::pipeline::pre_tokenizer::split_delimiter (cognitive 21): tk_encode::tokenizer::pipeline::pre_tokenizer::split_delimiter has cognitive complexity 21 (threshold 15). Drivers by points: if/else 7 (16 pts), loops 2 (4 pts), match/switch 1 (nesting depth added 11). To reduce it, split the body into named stages: move each independent step or branch into its own named function so the body reads as a short sequence of named calls rather than one long body."},"locations":[{"physicalLocation":{"artifactLocation":{"uri":"tokenizers/tk-encode/src/tokenizer/pipeline/pre_tokenizer.rs"},"region":{"startLine":278}}}],"partialFingerprints":{"codehealthFindingId/v1":"03ebd5ddbc2c656e9683a8e8f5abaadfa9c89dbe28189431bb49897bae50be3e"}},{"ruleId":"D2","level":"warning","message":{"text":"bitcannon::digit_groups (cognitive 20): bitcannon::digit_groups has cognitive complexity 20 (threshold 15). Drivers by points: loops 5 (11 pts), if/else 5 (7 pts), boolean chains 2 (nesting depth added 8). To reduce it, break up the iteration: give each loop body a named function, and split a multi-phase loop into one function per phase so no single body carries the whole pipeline."},"locations":[{"physicalLocation":{"artifactLocation":{"uri":"tokenizers/bitcannon/src/lib.rs"},"region":{"startLine":646}}}],"partialFingerprints":{"codehealthFindingId/v1":"be534a7e5793e34b938912626153e20fb1607be51a2787ba282a52a8bd5dc984"}},{"ruleId":"D2","level":"warning","message":{"text":"PipelineTokenizer::encode_sequence_into (cognitive 20): PipelineTokenizer::encode_sequence_into has cognitive complexity 20 (threshold 15). Drivers by points: loops 3 (9 pts), match/switch 2 (6 pts), if/else 1 (5 pts) (nesting depth added 14). To reduce it, break up the iteration: give each loop body a named function, and split a multi-phase loop into one function per phase so no single body carries the whole pipeline."},"locations":[{"physicalLocation":{"artifactLocation":{"uri":"tokenizers/tk-encode/src/tokenizer/pipeline/mod.rs"},"region":{"startLine":763}}}],"partialFingerprints":{"codehealthFindingId/v1":"d8a4970793eaa1bde96f4f7e77c5e6c1f218ac8233539382ad7d4a4dc6e08ae9"}},{"ruleId":"D2","level":"warning","message":{"text":"ParityBpeTrainer::tokenize_words (cognitive 20): ParityBpeTrainer::tokenize_words has cognitive complexity 20 (threshold 15). Drivers by points: if/else 4 (15 pts), loops 2 (3 pts), boolean chains 2 (nesting depth added 12). To reduce it, split the body into named stages: move each independent step or branch into its own named function so the body reads as a short sequence of named calls rather than one long body."},"locations":[{"physicalLocation":{"artifactLocation":{"uri":"tokenizers/tk-train/src/trainers/bpe/parity_trainer.rs"},"region":{"startLine":586}}}],"partialFingerprints":{"codehealthFindingId/v1":"860110ef641cdc9ebbde002770a57742aa15e4799747bf3bbb7952d1d200d600"}},{"ruleId":"D2","level":"warning","message":{"text":"Word::merge (cognitive 20): Word::merge has cognitive complexity 20 (threshold 15). Drivers by points: if/else 6 (18 pts), boolean chains 1, loops 1 (nesting depth added 12). To reduce it, split the body into named stages: move each independent step or branch into its own named function so the body reads as a short sequence of named calls rather than one long body."},"locations":[{"physicalLocation":{"artifactLocation":{"uri":"tokenizers/tk-train/src/trainers/bpe/word.rs"},"region":{"startLine":99}}}],"partialFingerprints":{"codehealthFindingId/v1":"51ba0b1d1c9cc6a0d34de84f7305b8122a2280814c26c6d7fb0241807347e3cc"}},{"ruleId":"D2","level":"warning","message":{"text":"tk_convert::convert::lower_byte_level_pre_tokenizer (cognitive 19): tk_convert::convert::lower_byte_level_pre_tokenizer has cognitive complexity 19 (threshold 15). Drivers by points: if/else 10 (14 pts), match/switch 3 (5 pts) (nesting depth added 6). To reduce it, split the body into named stages: move each independent step or branch into its own named function so the body reads as a short sequence of named calls rather than one long body."},"locations":[{"physicalLocation":{"artifactLocation":{"uri":"tokenizers/tk-convert/src/convert.rs"},"region":{"startLine":511}}}],"partialFingerprints":{"codehealthFindingId/v1":"cc9e25ed1332e2b14475f7c980ddea98c976b5cd737f8f7a83c0b1dcf143e3fd"}},{"ruleId":"D2","level":"warning","message":{"text":"PipelineBPE::tokenize_spans (cognitive 19): PipelineBPE::tokenize_spans has cognitive complexity 19 (threshold 15). Drivers by points: if/else 6 (13 pts), match/switch 1 (3 pts), boolean chains 2, loops 1 (nesting depth added 9). To reduce it, split the body into named stages: move each independent step or branch into its own named function so the body reads as a short sequence of named calls rather than one long body."},"locations":[{"physicalLocation":{"artifactLocation":{"uri":"tokenizers/tk-encode/src/models/bpe/model.rs"},"region":{"startLine":276}}}],"partialFingerprints":{"codehealthFindingId/v1":"3c42484c92078681d666539ca8cae6d04c25bd5ccb9eae50300772f62469c338"}},{"ruleId":"D2","level":"warning","message":{"text":"MetaspaceNormalizer::normalize (cognitive 19): MetaspaceNormalizer::normalize has cognitive complexity 19 (threshold 15). Drivers by points: if/else 5 (8 pts), loops 2 (4 pts), match/switch 2 (4 pts), boolean chains 3 (nesting depth added 7). To reduce it, split the body into named stages: move each independent step or branch into its own named function so the body reads as a short sequence of named calls rather than one long body."},"locations":[{"physicalLocation":{"artifactLocation":{"uri":"tokenizers/tk-encode/src/normalizers/metaspace.rs"},"region":{"startLine":70}}}],"partialFingerprints":{"codehealthFindingId/v1":"a581638d50265592cf6bd226ea37ac10c56c1d2deb23bb80f3a6791c16cea285"}},{"ruleId":"D2","level":"warning","message":{"text":"Buckets::from_tokens (cognitive 18): Buckets::from_tokens has cognitive complexity 18 (threshold 15). Drivers by points: if/else 5 (9 pts), loops 4 (6 pts), match/switch 1 (3 pts) (nesting depth added 8). To reduce it, split the body into named stages: move each independent step or branch into its own named function so the body reads as a short sequence of named calls rather than one long body."},"locations":[{"physicalLocation":{"artifactLocation":{"uri":"tokenizers/tk-encode/src/vocab/buckets.rs"},"region":{"startLine":117}}}],"partialFingerprints":{"codehealthFindingId/v1":"4ea2c742de27de857b3fbbeb0a62c2e497d9a9e4ef32648ffe427b947c0016de"}},{"ruleId":"D2","level":"warning","message":{"text":"bitcannon::models::family_gpt::contractions (cognitive 17): bitcannon::models::family_gpt::contractions has cognitive complexity 17 (threshold 15). Drivers by points: if/else 8 (16 pts), loops 1 (nesting depth added 8). To reduce it, split the body into named stages: move each independent step or branch into its own named function so the body reads as a short sequence of named calls rather than one long body."},"locations":[{"physicalLocation":{"artifactLocation":{"uri":"tokenizers/bitcannon/src/models/family_gpt.rs"},"region":{"startLine":75}}}],"partialFingerprints":{"codehealthFindingId/v1":"f88971b7f4613fb40e1537588dd313849565b0b36f13d362a84d3edfb81a6eee"}},{"ruleId":"D2","level":"warning","message":{"text":"Unigram::tokenize_pipeline (cognitive 17): Unigram::tokenize_pipeline has cognitive complexity 17 (threshold 15). Drivers by points: if/else 5 (10 pts), match/switch 2 (4 pts), boolean chains 2, loops 1 (nesting depth added 7). To reduce it, split the body into named stages: move each independent step or branch into its own named function so the body reads as a short sequence of named calls rather than one long body."},"locations":[{"physicalLocation":{"artifactLocation":{"uri":"tokenizers/tk-encode/src/models/unigram/model.rs"},"region":{"startLine":528}}}],"partialFingerprints":{"codehealthFindingId/v1":"7243301952c3404c52e7c1cee640886bcf1a034864c44ea0190d302ce3ce5af1"}},{"ruleId":"D2","level":"warning","message":{"text":"tk_serialize::from_json::post_processors::read_template (cognitive 17): tk_serialize::from_json::post_processors::read_template has cognitive complexity 17 (threshold 15). Drivers by points: if/else 5 (6 pts), match/switch 2 (6 pts), loops 2 (4 pts), boolean chains 1 (nesting depth added 7). To reduce it, split the body into named stages: move each independent step or branch into its own named function so the body reads as a short sequence of named calls rather than one long body."},"locations":[{"physicalLocation":{"artifactLocation":{"uri":"tokenizers/tk-serialize/src/from_json/post_processors.rs"},"region":{"startLine":21}}}],"partialFingerprints":{"codehealthFindingId/v1":"523974df99dcbf48373a179928c3ffdcba35416adcd9747a533146632a32f440"}},{"ruleId":"D2","level":"warning","message":{"text":"ParityBpeTrainer::validate_train_config (cognitive 17): ParityBpeTrainer::validate_train_config has cognitive complexity 17 (threshold 15). Drivers by points: if/else 7 (11 pts), boolean chains 4, loops 1 (2 pts) (nesting depth added 5). To reduce it, split the body into named stages: move each independent step or branch into its own named function so the body reads as a short sequence of named calls rather than one long body."},"locations":[{"physicalLocation":{"artifactLocation":{"uri":"tokenizers/tk-train/src/trainers/bpe/parity_trainer.rs"},"region":{"startLine":797}}}],"partialFingerprints":{"codehealthFindingId/v1":"8c65a91b638402e523013fe7df98dcfce435153f4466da009fcaf1e2c379549b"}},{"ruleId":"D2","level":"warning","message":{"text":"bitcannon::blocks (cognitive 16): bitcannon::blocks has cognitive complexity 16 (threshold 15). Drivers by points: if/else 9 (15 pts), loops 1 (nesting depth added 6). To reduce it, split the body into named stages: move each independent step or branch into its own named function so the body reads as a short sequence of named calls rather than one long body."},"locations":[{"physicalLocation":{"artifactLocation":{"uri":"tokenizers/bitcannon/src/lib.rs"},"region":{"startLine":131}}}],"partialFingerprints":{"codehealthFindingId/v1":"f26b450b7feb2410d908f45860d942cdb9d56ee5835cb3898b63764dd84f814c"}},{"ruleId":"D2","level":"warning","message":{"text":"ByteLevelFold::fold (cognitive 16): ByteLevelFold::fold has cognitive complexity 16 (threshold 15). Drivers by points: if/else 7 (11 pts), loops 2 (3 pts), boolean chains 2 (nesting depth added 5). To reduce it, split the body into named stages: move each independent step or branch into its own named function so the body reads as a short sequence of named calls rather than one long body."},"locations":[{"physicalLocation":{"artifactLocation":{"uri":"tokenizers/tk-encode/src/models/bpe/fold/byte_level.rs"},"region":{"startLine":98}}}],"partialFingerprints":{"codehealthFindingId/v1":"af587acf8c69d1f2d9e8332744fb0b955b92681291cd018becb1630e5369bf34"}},{"ruleId":"D2","level":"warning","message":{"text":"AddedVocabulary::add_tokens (cognitive 16): AddedVocabulary::add_tokens has cognitive complexity 16 (threshold 15). Drivers by points: if/else 6 (10 pts), loops 3, match/switch 1 (2 pts), boolean chains 1 (nesting depth added 5). To reduce it, split the body into named stages: move each independent step or branch into its own named function so the body reads as a short sequence of named calls rather than one long body."},"locations":[{"physicalLocation":{"artifactLocation":{"uri":"tokenizers/tk-encode/src/vocab/bucket_added_vocabulary.rs"},"region":{"startLine":347}}}],"partialFingerprints":{"codehealthFindingId/v1":"7f5a264d1e2c9d00d2bf733ce09c38ddf8ac06157f3d2dab2fc91c8e042332b7"}},{"ruleId":"D2","level":"warning","message":{"text":"Buckets::build_nibble_table (cognitive 16): Buckets::build_nibble_table has cognitive complexity 16 (threshold 15). Drivers by points: if/else 4 (11 pts), loops 3 (5 pts) (nesting depth added 9). To reduce it, split the body into named stages: move each independent step or branch into its own named function so the body reads as a short sequence of named calls rather than one long body."},"locations":[{"physicalLocation":{"artifactLocation":{"uri":"tokenizers/tk-encode/src/vocab/buckets.rs"},"region":{"startLine":195}}}],"partialFingerprints":{"codehealthFindingId/v1":"56eeada9283aaf24128f74c746d2852b3d11df65b43e163f7ab2cb9f2a4d849f"}},{"ruleId":"D3","level":"warning","message":{"text":"MethodTooLong: ParityBpeTrainer.do_train: MethodTooLong \u2014 do_train runs 185 significant lines (blank, comment-only and punctuation-only lines excluded, and inline test code \u2014 #[cfg(test)] modules and bare #[test] functions \u2014 not counted) in one body. The bar is 100 significant lines; this is 85 over it, 1.85\u00D7 the bar. This is length, not branching: a long straight-line body scores low on complexity and is still read whole to change any part of it, so the complexity numbers beside this row neither confirm nor excuse it. To reduce it, extract each cohesive step of the body \u2014 the runs of statements that work on the same values and would earn the same name \u2014 into its own named unit, and have this one call them in order."},"locations":[{"physicalLocation":{"artifactLocation":{"uri":"tokenizers/tk-train/src/trainers/bpe/parity_trainer.rs"},"region":{"startLine":1173}}}],"partialFingerprints":{"codehealthFindingId/v1":"01c1b1ab1b611c05f2c6d157af21a3806331919843650e45f598bcf350e980c8"}},{"ruleId":"D3","level":"warning","message":{"text":"ClassTooLong: ParityBpeTrainer: ClassTooLong \u2014 736 significant lines (blank, comment-only and punctuation-only lines excluded, and inline test code \u2014 #[cfg(test)] modules and bare #[test] functions \u2014 not counted), 23 methods, 3 blocks, lines 321-1533. The bar is 400 significant lines; this is 336 over it, 1.84\u00D7 the bar. To reduce it, group the members that share the same data into a smaller type of their own and delegate to it, so no single type carries every responsibility."},"locations":[{"physicalLocation":{"artifactLocation":{"uri":"tokenizers/tk-train/src/trainers/bpe/parity_trainer.rs"},"region":{"startLine":321}}}],"partialFingerprints":{"codehealthFindingId/v1":"c8794336087b723d643da7e7d3f319c8e4a668ee1d8d4868b6ad34a036526216"}},{"ruleId":"D3","level":"warning","message":{"text":"FunctionTooLong: bitcannon::models::family_o200k::run: FunctionTooLong \u2014 bitcannon::models::family_o200k::run runs 183 significant lines (blank, comment-only and punctuation-only lines excluded, and inline test code \u2014 #[cfg(test)] modules and bare #[test] functions \u2014 not counted) in one body. The bar is 100 significant lines; this is 83 over it, 1.83\u00D7 the bar. This is length, not branching: a long straight-line body scores low on complexity and is still read whole to change any part of it, so the complexity numbers beside this row neither confirm nor excuse it. To reduce it, extract each cohesive step of the body \u2014 the runs of statements that work on the same values and would earn the same name \u2014 into its own named unit, and have this one call them in order."},"locations":[{"physicalLocation":{"artifactLocation":{"uri":"tokenizers/bitcannon/src/models/family_o200k.rs"},"region":{"startLine":189}}}],"partialFingerprints":{"codehealthFindingId/v1":"d3939ce9a86ef524264197ff1a4c6bd2b1abf96b85ddcedd6116d712a1f02eac"}},{"ruleId":"D3","level":"warning","message":{"text":"FileTooLong: bpe/parity_trainer.rs: FileTooLong \u2014 898 significant lines (blank, comment-only and punctuation-only lines excluded, and inline test code \u2014 #[cfg(test)] modules and bare #[test] functions \u2014 not counted), about 82% of them inside a single declaration: ParityBpeTrainer (3 blocks, 321-1533). The bar is 500 significant lines; this is 398 over it, 1.80\u00D7 the bar. Moving the declarations that sit BESIDE it into sibling files will not shorten this file. Extract from INSIDE that declaration instead: lift each cohesive group of its body \u2014 the parts that share the same inputs and are named together \u2014 into its own unit in a sibling file, and have the original call them."},"locations":[{"physicalLocation":{"artifactLocation":{"uri":"tokenizers/tk-train/src/trainers/bpe/parity_trainer.rs"},"region":{"startLine":1}}}],"partialFingerprints":{"codehealthFindingId/v1":"29d014650b9acfee5d7d949c6fa17d5e0ba820b1821d9a13504eb3c08439de57"}},{"ruleId":"D3","level":"warning","message":{"text":"FunctionTooLong: bitmap_gen::generate_tables: FunctionTooLong \u2014 bitmap_gen::generate_tables runs 156 significant lines (blank, comment-only and punctuation-only lines excluded, and inline test code \u2014 #[cfg(test)] modules and bare #[test] functions \u2014 not counted) in one body. The bar is 100 significant lines; this is 56 over it, 1.56\u00D7 the bar. This is length, not branching: a long straight-line body scores low on complexity and is still read whole to change any part of it, so the complexity numbers beside this row neither confirm nor excuse it. To reduce it, extract each cohesive step of the body \u2014 the runs of statements that work on the same values and would earn the same name \u2014 into its own named unit, and have this one call them in order."},"locations":[{"physicalLocation":{"artifactLocation":{"uri":"tokenizers/bitmap_gen/src/lib.rs"},"region":{"startLine":134}}}],"partialFingerprints":{"codehealthFindingId/v1":"267c49b610336985ac6e3adc4f64aac0a2696a42a855432fa36b1d1655fd4b15"}},{"ruleId":"D3","level":"warning","message":{"text":"FileTooLong: pipeline/mod.rs: FileTooLong \u2014 724 significant lines (blank, comment-only and punctuation-only lines excluded, and inline test code \u2014 #[cfg(test)] modules and bare #[test] functions \u2014 not counted). The bar is 500 significant lines; this is 224 over it, 1.45\u00D7 the bar. To reduce it, split the file along the responsibilities already in it: move each cohesive group of declarations into its own sibling file in the same module or package, so no one file has to be read whole to change one of them."},"locations":[{"physicalLocation":{"artifactLocation":{"uri":"tokenizers/tk-encode/src/tokenizer/pipeline/mod.rs"},"region":{"startLine":1}}}],"partialFingerprints":{"codehealthFindingId/v1":"0f741427c31451013cad6bc4a72d2c12f32b81ab538d68e11ecaa1ff9395a7dd"}},{"ruleId":"D3","level":"warning","message":{"text":"FunctionTooLong: bitcannon::simd::x86::build64: FunctionTooLong \u2014 bitcannon::simd::x86::build64 runs 115 significant lines (blank, comment-only and punctuation-only lines excluded, and inline test code \u2014 #[cfg(test)] modules and bare #[test] functions \u2014 not counted) in one body. The bar is 100 significant lines; this is 15 over it, 1.15\u00D7 the bar. This is length, not branching: a long straight-line body scores low on complexity and is still read whole to change any part of it, so the complexity numbers beside this row neither confirm nor excuse it. To reduce it, extract each cohesive step of the body \u2014 the runs of statements that work on the same values and would earn the same name \u2014 into its own named unit, and have this one call them in order."},"locations":[{"physicalLocation":{"artifactLocation":{"uri":"tokenizers/bitcannon/src/simd/x86.rs"},"region":{"startLine":87}}}],"partialFingerprints":{"codehealthFindingId/v1":"a441d7c733230be87ffb1ed5b64c4fef9c55e9455a73e415876a86baf5d01bfb"}},{"ruleId":"D3","level":"warning","message":{"text":"FunctionTooLong: bitcannon::classify::wasm::classify_wasm: FunctionTooLong \u2014 bitcannon::classify::wasm::classify_wasm runs 114 significant lines (blank, comment-only and punctuation-only lines excluded, and inline test code \u2014 #[cfg(test)] modules and bare #[test] functions \u2014 not counted) in one body. The bar is 100 significant lines; this is 14 over it, 1.14\u00D7 the bar. This is length, not branching: a long straight-line body scores low on complexity and is still read whole to change any part of it, so the complexity numbers beside this row neither confirm nor excuse it. To reduce it, extract each cohesive step of the body \u2014 the runs of statements that work on the same values and would earn the same name \u2014 into its own named unit, and have this one call them in order."},"locations":[{"physicalLocation":{"artifactLocation":{"uri":"tokenizers/bitcannon/src/classify/wasm.rs"},"region":{"startLine":149}}}],"partialFingerprints":{"codehealthFindingId/v1":"0ffaa58d9bd8b37300fed0b95028c38b28e04c64cf04d609d750435632b38b10"}},{"ruleId":"D3","level":"warning","message":{"text":"FunctionTooLong: bitcannon::classify::neon::classify_neon: FunctionTooLong \u2014 bitcannon::classify::neon::classify_neon runs 110 significant lines (blank, comment-only and punctuation-only lines excluded, and inline test code \u2014 #[cfg(test)] modules and bare #[test] functions \u2014 not counted) in one body. The bar is 100 significant lines; this is 10 over it, 1.10\u00D7 the bar. This is length, not branching: a long straight-line body scores low on complexity and is still read whole to change any part of it, so the complexity numbers beside this row neither confirm nor excuse it. To reduce it, extract each cohesive step of the body \u2014 the runs of statements that work on the same values and would earn the same name \u2014 into its own named unit, and have this one call them in order."},"locations":[{"physicalLocation":{"artifactLocation":{"uri":"tokenizers/bitcannon/src/classify/neon.rs"},"region":{"startLine":159}}}],"partialFingerprints":{"codehealthFindingId/v1":"f0d7ca5a06d6945351123b36e6951df60f7ca1b4f0071b2dd5ad433081b1f452"}},{"ruleId":"D3","level":"warning","message":{"text":"TooManyMethods: Tokenizer: TooManyMethods \u2014 32 methods. The bar is 30 methods; this is 2 over it, 1.07\u00D7 the bar. This type is on a published API surface \u2014 a cargo-semver-checks gate in CI (.github/workflows/rust.yml) declares its compatibility contractual \u2014 so moving members onto a smaller type is a breaking change for every consumer, not a local refactor. To reduce it, treat the split as an API migration: move each cohesive group onto its own published type and keep the old members as deprecated forwards for a deprecation period, removing them at the next compatibility break. Where the surface has to stay as it is, that is a decision to record rather than a change to make."},"locations":[{"physicalLocation":{"artifactLocation":{"uri":"bindings/python/src/tokenizer.rs"},"region":{"startLine":107}}}],"partialFingerprints":{"codehealthFindingId/v1":"e220b78f32254f9872ec11debffb521b6399b92f24fdc77ec17000ce3628cdb7"}},{"ruleId":"D3","level":"warning","message":{"text":"TooManyMethods: Encoding: TooManyMethods \u2014 32 methods. The bar is 30 methods; this is 2 over it, 1.07\u00D7 the bar. This type is on a published API surface \u2014 a cargo-semver-checks gate in CI (.github/workflows/rust.yml) declares its compatibility contractual \u2014 so moving members onto a smaller type is a breaking change for every consumer, not a local refactor. To reduce it, treat the split as an API migration: move each cohesive group onto its own published type and keep the old members as deprecated forwards for a deprecation period, removing them at the next compatibility break. Where the surface has to stay as it is, that is a decision to record rather than a change to make."},"locations":[{"physicalLocation":{"artifactLocation":{"uri":"tokenizers/tk-encode/src/tokenizer/encoding.rs"},"region":{"startLine":12}}}],"partialFingerprints":{"codehealthFindingId/v1":"1802f3c37f2182ed623e9d3fbbc0b810a498ae9311d581325c3a3d52ed5f978e"}},{"ruleId":"D3","level":"warning","message":{"text":"TooManyMethods: PipelineTokenizer: TooManyMethods \u2014 31 methods, declared across 2 files: pipeline/mod.rs (28), pipeline/parallel.rs (3). The bar is 30 methods; this is 1 over it, 1.03\u00D7 the bar. That list is where to read them, not a suggestion to split the file: the members belong to the type wherever they are declared, so moving them between files leaves the count unchanged. This type is on a published API surface \u2014 a cargo-semver-checks gate in CI (.github/workflows/rust.yml) declares its compatibility contractual \u2014 so moving members onto a smaller type is a breaking change for every consumer, not a local refactor. To reduce it, treat the split as an API migration: move each cohesive group onto its own published type and keep the old members as deprecated forwards for a deprecation period, removing them at the next compatibility break. Where the surface has to stay as it is, that is a decision to record rather than a change to make."},"locations":[{"physicalLocation":{"artifactLocation":{"uri":"tokenizers/tk-encode/src/tokenizer/pipeline/mod.rs"},"region":{"startLine":248}}}],"partialFingerprints":{"codehealthFindingId/v1":"e8b4132944aa1da8b9579ea91fa2d3b47688197b90749ae73793c7fab8cc7d10"}},{"ruleId":"D3","level":"warning","message":{"text":"MethodTooLong: BpeTables.build: MethodTooLong \u2014 build runs 103 significant lines (blank, comment-only and punctuation-only lines excluded, and inline test code \u2014 #[cfg(test)] modules and bare #[test] functions \u2014 not counted) in one body. The bar is 100 significant lines; this is 3 over it, 1.03\u00D7 the bar. This is length, not branching: a long straight-line body scores low on complexity and is still read whole to change any part of it, so the complexity numbers beside this row neither confirm nor excuse it. To reduce it, extract each cohesive step of the body \u2014 the runs of statements that work on the same values and would earn the same name \u2014 into its own named unit, and have this one call them in order."},"locations":[{"physicalLocation":{"artifactLocation":{"uri":"tokenizers/tk-encode/src/models/bpe/tables.rs"},"region":{"startLine":79}}}],"partialFingerprints":{"codehealthFindingId/v1":"b2df23d886d5c574a911cd05380ce3721c0379d816ae8d5c201809debb6b6a21"}},{"ruleId":"D3","level":"warning","message":{"text":"MethodTooLong: PipelineBPE.from_merge_map: MethodTooLong \u2014 from_merge_map runs 102 significant lines (blank, comment-only and punctuation-only lines excluded, and inline test code \u2014 #[cfg(test)] modules and bare #[test] functions \u2014 not counted) in one body. The bar is 100 significant lines; this is 2 over it, 1.02\u00D7 the bar. This is length, not branching: a long straight-line body scores low on complexity and is still read whole to change any part of it, so the complexity numbers beside this row neither confirm nor excuse it. To reduce it, extract each cohesive step of the body \u2014 the runs of statements that work on the same values and would earn the same name \u2014 into its own named unit, and have this one call them in order."},"locations":[{"physicalLocation":{"artifactLocation":{"uri":"tokenizers/tk-encode/src/models/bpe/serialization.rs"},"region":{"startLine":193}}}],"partialFingerprints":{"codehealthFindingId/v1":"b04a70154b9737d6f923fe2dcdf46786fe315ed48421f884f9cdbdaf63953697"}},{"ruleId":"D3","level":"warning","message":{"text":"FileTooLong: src/convert.rs: FileTooLong \u2014 506 significant lines (blank, comment-only and punctuation-only lines excluded, and inline test code \u2014 #[cfg(test)] modules and bare #[test] functions \u2014 not counted). The bar is 500 significant lines; this is 6 over it, 1.01\u00D7 the bar. To reduce it, split the file along the responsibilities already in it: move each cohesive group of declarations into its own sibling file in the same module or package, so no one file has to be read whole to change one of them."},"locations":[{"physicalLocation":{"artifactLocation":{"uri":"tokenizers/tk-convert/src/convert.rs"},"region":{"startLine":1}}}],"partialFingerprints":{"codehealthFindingId/v1":"d501650366b13c5f1697242611abb94c34d539e3a1cf280536b8a1592254ffa3"}},{"ruleId":"D3","level":"warning","message":{"text":"FunctionTooLong: bitcannon::simd::neon::build64: FunctionTooLong \u2014 bitcannon::simd::neon::build64 runs 101 significant lines (blank, comment-only and punctuation-only lines excluded, and inline test code \u2014 #[cfg(test)] modules and bare #[test] functions \u2014 not counted) in one body. The bar is 100 significant lines; this is 1 over it, 1.01\u00D7 the bar. This is length, not branching: a long straight-line body scores low on complexity and is still read whole to change any part of it, so the complexity numbers beside this row neither confirm nor excuse it. To reduce it, extract each cohesive step of the body \u2014 the runs of statements that work on the same values and would earn the same name \u2014 into its own named unit, and have this one call them in order."},"locations":[{"physicalLocation":{"artifactLocation":{"uri":"tokenizers/bitcannon/src/simd/neon.rs"},"region":{"startLine":78}}}],"partialFingerprints":{"codehealthFindingId/v1":"140721c8388623534ce95f1d65b5c98986709377557f30fbefdbb8f9b61c3395"}},{"ruleId":"D4","level":"warning","message":{"text":"Near-duplicate member family (3 members, 9 shared lines): tokenizers/bitcannon/src/classify/avx.rs:41-54 | tokenizers/bitcannon/src/classify/neon.rs:100-113 | tokenizers/bitcannon/src/classify/wasm.rs:30-43 \u2014 These 3 members are variants of one another: a block of 9 lines reported below appears in every one of them, and the pairwise near-duplicate rows they would otherwise produce are collapsed into this row. Read them as one construct written 3 times. The repair is at the members\u0027 grain \u2014 factor the shared implementation out once and have all of them call it with their differences as parameters or as an injected step, or, where the difference is systematic, generate them from one template. Extracting the individual blocks below is not the same fix: it leaves every body in place and the next edit still has to be made 3 times."},"locations":[{"physicalLocation":{"artifactLocation":{"uri":"tokenizers/bitcannon/src/classify/avx.rs"},"region":{"startLine":41}}}],"partialFingerprints":{"codehealthFindingId/v1":"bcb6f7395f31ef62926c85507bf6478f3a1bfad4f7e017e2540b9070abf6c03f"}},{"ruleId":"D4","level":"warning","message":{"text":"Edited copy of a member (32 corresponding lines): tokenizers/tk-train/src/trainers/bpe/mod.rs:337-383 | tokenizers/tk-train/src/trainers/bpe/parity_trainer.rs:548-582 \u2014 These two members are one piece of code written twice and then edited apart: 32 consecutive lines correspond almost exactly, broken only by small local edits. Most of that correspondence is NOT reported as duplicated blocks below \u2014 the edits cut it into fragments and only the largest of them clear the block floor, so the rows below understate it. The repair is at the members\u0027 grain \u2014 factor the shared implementation into one the two call with their differences as parameters or as an injected step, or, where the difference is systematic (an extra return value, one transport against another), generate one from the other. Left alone, the next edit has to be made twice and the two will drift further apart."},"locations":[{"physicalLocation":{"artifactLocation":{"uri":"tokenizers/tk-train/src/trainers/bpe/mod.rs"},"region":{"startLine":337}}}],"partialFingerprints":{"codehealthFindingId/v1":"0323d2c1ddb7c50eede5b01085655cd55c736157d73bd17d9b3ced9d8e7c36ae"}},{"ruleId":"D4","level":"warning","message":{"text":"Members sharing a duplicated core (5 members, 50\u002B identical tokens): tokenizers/tk-train/src/trainers/bpe/mod.rs:706-730 | tokenizers/tk-train/src/trainers/bpe/parity_trainer.rs:418-445 | tokenizers/tk-train/src/trainers/bpe/parity_trainer.rs:459-487 | tokenizers/tk-train/src/trainers/unigram.rs:708-732 | tokenizers/tk-train/src/trainers/wordlevel.rs:143-167 \u2014 These 5 members share a duplicated core: a run of at least 50 identical tokens appears in every one of them. That run is NOT broken out as duplicated-block rows below \u2014 it is what admitted this row, and the blocks below cover only the part of it that clears the block floor, so they understate the correspondence. Read the members as one construct written 5 times. The repair is at the members\u0027 grain \u2014 factor the shared implementation out once and have all of them call it with their differences as parameters or as an injected step, or, where the difference is systematic, generate them from one template. Extracting the individual blocks below is not the same fix: it leaves every body in place and the next edit still has to be made 5 times."},"locations":[{"physicalLocation":{"artifactLocation":{"uri":"tokenizers/tk-train/src/trainers/bpe/mod.rs"},"region":{"startLine":706}}}],"partialFingerprints":{"codehealthFindingId/v1":"8d50ac32902d9de1b43d9c3bf24233569f2207a361019e1e003d3a629e78086f"}},{"ruleId":"D4","level":"warning","message":{"text":"Edited copy of a member (30 corresponding lines): tokenizers/tk-train/src/trainers/bpe/mod.rs:392-430 | tokenizers/tk-train/src/trainers/bpe/parity_trainer.rs:591-620 \u2014 These two members are one piece of code written twice and then edited apart: 30 consecutive lines correspond almost exactly, broken only by small local edits. Most of that correspondence is NOT reported as duplicated blocks below \u2014 the edits cut it into fragments and only the largest of them clear the block floor, so the rows below understate it. The repair is at the members\u0027 grain \u2014 factor the shared implementation into one the two call with their differences as parameters or as an injected step, or, where the difference is systematic (an extra return value, one transport against another), generate one from the other. Left alone, the next edit has to be made twice and the two will drift further apart."},"locations":[{"physicalLocation":{"artifactLocation":{"uri":"tokenizers/tk-train/src/trainers/bpe/mod.rs"},"region":{"startLine":392}}}],"partialFingerprints":{"codehealthFindingId/v1":"f9f18859aab176207a5777eda311ac9c571bf6fb452a7794471a3d942e4ac8bf"}},{"ruleId":"D4","level":"warning","message":{"text":"Edited copy of a member (1 corresponding lines): bindings/python/src/components/decoders.rs:6-6 | bindings/python/src/components/pre_tokenizers.rs:6-6 \u2014 These two members are one piece of code written twice and then edited apart: 1 consecutive lines correspond almost exactly, broken only by small local edits. Most of that correspondence is NOT reported as duplicated blocks below \u2014 the edits cut it into fragments and only the largest of them clear the block floor, so the rows below understate it. The repair is at the members\u0027 grain \u2014 factor the shared implementation into one the two call with their differences as parameters or as an injected step, or, where the difference is systematic (an extra return value, one transport against another), generate one from the other. Left alone, the next edit has to be made twice and the two will drift further apart."},"locations":[{"physicalLocation":{"artifactLocation":{"uri":"bindings/python/src/components/decoders.rs"},"region":{"startLine":6}}}],"partialFingerprints":{"codehealthFindingId/v1":"250df70f89ec27063919165cce63715bc498b35f57d0a746093cbe4e3415aa4a"}},{"ruleId":"D4","level":"warning","message":{"text":"Edited copy of a member (29 corresponding lines): tokenizers/tk-train/src/trainers/bpe/mod.rs:437-472 | tokenizers/tk-train/src/trainers/bpe/parity_trainer.rs:627-655 \u2014 These two members are one piece of code written twice and then edited apart: 29 consecutive lines correspond almost exactly, broken only by small local edits. Most of that correspondence is NOT reported as duplicated blocks below \u2014 the edits cut it into fragments and only the largest of them clear the block floor, so the rows below understate it. The repair is at the members\u0027 grain \u2014 factor the shared implementation into one the two call with their differences as parameters or as an injected step, or, where the difference is systematic (an extra return value, one transport against another), generate one from the other. Left alone, the next edit has to be made twice and the two will drift further apart."},"locations":[{"physicalLocation":{"artifactLocation":{"uri":"tokenizers/tk-train/src/trainers/bpe/mod.rs"},"region":{"startLine":437}}}],"partialFingerprints":{"codehealthFindingId/v1":"4e39d7865f926f70d6465bc1d8fe89f5ac1c22c0fa51c0a249c4353da37132be"}},{"ruleId":"D4","level":"warning","message":{"text":"Edited copy of a member (19 corresponding lines): tokenizers/bitcannon/src/classify/wasm.rs:69-87 | tokenizers/bitcannon/src/classify/wasm.rs:90-108 \u2014 These two members are one piece of code written twice and then edited apart: 19 consecutive lines correspond almost exactly, broken only by small local edits. Most of that correspondence is NOT reported as duplicated blocks below \u2014 the edits cut it into fragments and only the largest of them clear the block floor, so the rows below understate it. The repair is at the members\u0027 grain \u2014 factor the shared implementation into one the two call with their differences as parameters or as an injected step, or, where the difference is systematic (an extra return value, one transport against another), generate one from the other. Left alone, the next edit has to be made twice and the two will drift further apart."},"locations":[{"physicalLocation":{"artifactLocation":{"uri":"tokenizers/bitcannon/src/classify/wasm.rs"},"region":{"startLine":69}}}],"partialFingerprints":{"codehealthFindingId/v1":"20d91db769b97225e61c87530503df5b1443e94e3eff9e993cb2bca6da2e124b"}},{"ruleId":"D4","level":"warning","message":{"text":"Edited copy of a member (20 corresponding lines): tokenizers/tk-encode/src/tokenizer/pipeline/mod.rs:1151-1170 | tokenizers/tk-encode/src/tokenizer/pipeline/mod.rs:1178-1197 \u2014 These two members are one piece of code written twice and then edited apart: 20 consecutive lines correspond almost exactly, broken only by small local edits. Most of that correspondence is NOT reported as duplicated blocks below \u2014 the edits cut it into fragments and only the largest of them clear the block floor, so the rows below understate it. The repair is at the members\u0027 grain \u2014 factor the shared implementation into one the two call with their differences as parameters or as an injected step, or, where the difference is systematic (an extra return value, one transport against another), generate one from the other. Left alone, the next edit has to be made twice and the two will drift further apart."},"locations":[{"physicalLocation":{"artifactLocation":{"uri":"tokenizers/tk-encode/src/tokenizer/pipeline/mod.rs"},"region":{"startLine":1151}}}],"partialFingerprints":{"codehealthFindingId/v1":"2556b74c561109a9ff684b4efcef878d89e9721997298a53a75f16d686377f01"}},{"ruleId":"D4","level":"warning","message":{"text":"Edited copy of a member (27 corresponding lines): tokenizers/tk-train/src/trainers/bpe/parity_trainer.rs:663-690 | tokenizers/tk-train/src/trainers/bpe/parity_trainer.rs:698-724 \u2014 These two members are one piece of code written twice and then edited apart: 27 consecutive lines correspond almost exactly, broken only by small local edits. Most of that correspondence is NOT reported as duplicated blocks below \u2014 the edits cut it into fragments and only the largest of them clear the block floor, so the rows below understate it. The repair is at the members\u0027 grain \u2014 factor the shared implementation into one the two call with their differences as parameters or as an injected step, or, where the difference is systematic (an extra return value, one transport against another), generate one from the other. Left alone, the next edit has to be made twice and the two will drift further apart."},"locations":[{"physicalLocation":{"artifactLocation":{"uri":"tokenizers/tk-train/src/trainers/bpe/parity_trainer.rs"},"region":{"startLine":663}}}],"partialFingerprints":{"codehealthFindingId/v1":"55a778a9d9dc17fae2ed174c29fa54478dea8140229d69d9530b8d3820d3ab0c"}},{"ruleId":"D4","level":"warning","message":{"text":"Edited copies of one member (3 members, 19\u002B corresponding lines): tokenizers/tk-encode/src/tokenizer/pattern.rs:54-73 | tokenizers/tk-encode/src/tokenizer/pattern.rs:96-114 | tokenizers/tk-encode/src/utils/fancy.rs:41-62 \u2014 These 3 members are one piece of code written 3 times and then edited apart: every two of them correspond over at least 19 lines, broken only by small local edits. Most of that correspondence is NOT reported as duplicated blocks below: the edits cut it into fragments and only the largest of them clear the block floor, so the rows below understate it. The repair is at the members\u0027 grain. Factor the shared implementation out once and have all of them call it with their differences as parameters or as an injected step, or, where the difference is systematic, generate them from one template. Left alone, the next edit has to be made 3 times and the copies will drift further apart."},"locations":[{"physicalLocation":{"artifactLocation":{"uri":"tokenizers/tk-encode/src/tokenizer/pattern.rs"},"region":{"startLine":54}}}],"partialFingerprints":{"codehealthFindingId/v1":"e33e278baee3e3b111ae41bce6e36a2b477853ecf7bae31142fd10a98759ec50"}},{"ruleId":"D4","level":"warning","message":{"text":"Edited copy of a member (19 corresponding lines): tokenizers/bitcannon/src/literal.rs:94-112 | tokenizers/bitcannon/src/literal.rs:117-140 \u2014 These two members are one piece of code written twice and then edited apart: 19 consecutive lines correspond almost exactly, broken only by small local edits. Most of that correspondence is NOT reported as duplicated blocks below \u2014 the edits cut it into fragments and only the largest of them clear the block floor, so the rows below understate it. The repair is at the members\u0027 grain \u2014 factor the shared implementation into one the two call with their differences as parameters or as an injected step, or, where the difference is systematic (an extra return value, one transport against another), generate one from the other. Left alone, the next edit has to be made twice and the two will drift further apart."},"locations":[{"physicalLocation":{"artifactLocation":{"uri":"tokenizers/bitcannon/src/literal.rs"},"region":{"startLine":94}}}],"partialFingerprints":{"codehealthFindingId/v1":"ac3148b4d53425077136511512b259331c59aa1807cc8790d08ec35233276cab"}},{"ruleId":"D4","level":"warning","message":{"text":"Edited copy of a member (15 corresponding lines): bindings/node/src/pipeline.rs:87-101 | bindings/wasm/src/options.rs:129-143 \u2014 These two members are one piece of code written twice and then edited apart: 15 consecutive lines correspond almost exactly, broken only by small local edits. Most of that correspondence is NOT reported as duplicated blocks below \u2014 the edits cut it into fragments and only the largest of them clear the block floor, so the rows below understate it. The repair is at the members\u0027 grain \u2014 factor the shared implementation into one the two call with their differences as parameters or as an injected step, or, where the difference is systematic (an extra return value, one transport against another), generate one from the other. Left alone, the next edit has to be made twice and the two will drift further apart."},"locations":[{"physicalLocation":{"artifactLocation":{"uri":"bindings/node/src/pipeline.rs"},"region":{"startLine":87}}}],"partialFingerprints":{"codehealthFindingId/v1":"6ffecc19565c833c217178694074c64495521e9f8d6839b80a0f4e29e59923b8"}},{"ruleId":"D4","level":"warning","message":{"text":"Edited copy of a member (12 corresponding lines): tokenizers/bitcannon/src/classify/avx.rs:100-116 | tokenizers/bitcannon/src/classify/avx.rs:120-131 \u2014 These two members are one piece of code written twice and then edited apart: 12 consecutive lines correspond almost exactly, broken only by small local edits. Most of that correspondence is NOT reported as duplicated blocks below \u2014 the edits cut it into fragments and only the largest of them clear the block floor, so the rows below understate it. The repair is at the members\u0027 grain \u2014 factor the shared implementation into one the two call with their differences as parameters or as an injected step, or, where the difference is systematic (an extra return value, one transport against another), generate one from the other. Left alone, the next edit has to be made twice and the two will drift further apart."},"locations":[{"physicalLocation":{"artifactLocation":{"uri":"tokenizers/bitcannon/src/classify/avx.rs"},"region":{"startLine":100}}}],"partialFingerprints":{"codehealthFindingId/v1":"05709845a7a6f0ef2adfcb29c3453734a9a01bb9107ff0330fd5a4d55a2e2da2"}},{"ruleId":"D4","level":"warning","message":{"text":"Edited copy of a member (21 corresponding lines): tokenizers/bitcannon/src/simd/rvv.rs:9-31 | tokenizers/bitcannon/src/simd/rvv.rs:38-58 \u2014 These two members are one piece of code written twice and then edited apart: 21 consecutive lines correspond almost exactly, broken only by small local edits. Most of that correspondence is NOT reported as duplicated blocks below \u2014 the edits cut it into fragments and only the largest of them clear the block floor, so the rows below understate it. The repair is at the members\u0027 grain \u2014 factor the shared implementation into one the two call with their differences as parameters or as an injected step, or, where the difference is systematic (an extra return value, one transport against another), generate one from the other. Left alone, the next edit has to be made twice and the two will drift further apart."},"locations":[{"physicalLocation":{"artifactLocation":{"uri":"tokenizers/bitcannon/src/simd/rvv.rs"},"region":{"startLine":9}}}],"partialFingerprints":{"codehealthFindingId/v1":"ed20bd39f945f93a382e6fa365f160f875a9f6f1845ef288c43c9a176f500345"}},{"ruleId":"D4","level":"warning","message":{"text":"Edited copy of a member (12 corresponding lines): tokenizers/tk-encode/src/utils/unicode.rs:22-33 | tokenizers/tk-encode/src/vocab/bucket_added_vocabulary.rs:192-212 \u2014 These two members are one piece of code written twice and then edited apart: 12 consecutive lines correspond almost exactly, broken only by small local edits. Most of that correspondence is NOT reported as duplicated blocks below \u2014 the edits cut it into fragments and only the largest of them clear the block floor, so the rows below understate it. The repair is at the members\u0027 grain \u2014 factor the shared implementation into one the two call with their differences as parameters or as an injected step, or, where the difference is systematic (an extra return value, one transport against another), generate one from the other. Left alone, the next edit has to be made twice and the two will drift further apart."},"locations":[{"physicalLocation":{"artifactLocation":{"uri":"tokenizers/tk-encode/src/utils/unicode.rs"},"region":{"startLine":22}}}],"partialFingerprints":{"codehealthFindingId/v1":"810d95a654d536a75846cfe37bf4c6ce30a4fc9092145da44d496d858d795d21"}},{"ruleId":"D4","level":"warning","message":{"text":"Edited copy of a member (12 corresponding lines): tokenizers/tk-train/src/trainers/mod.rs:45-56 | tokenizers/tk-train/src/trainers/mod.rs:90-101 \u2014 These two members are one piece of code written twice and then edited apart: 12 consecutive lines correspond almost exactly, broken only by small local edits. Most of that correspondence is NOT reported as duplicated blocks below \u2014 the edits cut it into fragments and only the largest of them clear the block floor, so the rows below understate it. The repair is at the members\u0027 grain \u2014 factor the shared implementation into one the two call with their differences as parameters or as an injected step, or, where the difference is systematic (an extra return value, one transport against another), generate one from the other. Left alone, the next edit has to be made twice and the two will drift further apart."},"locations":[{"physicalLocation":{"artifactLocation":{"uri":"tokenizers/tk-train/src/trainers/mod.rs"},"region":{"startLine":45}}}],"partialFingerprints":{"codehealthFindingId/v1":"41da27690193c0528a426e6f8fecbdc5add3367bfb6c7c18136ad694a177696b"}},{"ruleId":"D4","level":"warning","message":{"text":"Edited copies of one member (4 members, 11\u002B corresponding lines): tokenizers/tk-encode/src/tokenizer/pipeline/mod.rs:1090-1100 | tokenizers/tk-encode/src/tokenizer/pipeline/mod.rs:1102-1112 | tokenizers/tk-encode/src/tokenizer/pipeline/mod.rs:1114-1124 | tokenizers/tk-encode/src/tokenizer/pipeline/mod.rs:1126-1136 \u2014 These 4 members are one piece of code written 4 times and then edited apart: every two of them correspond over at least 11 lines, broken only by small local edits. Most of that correspondence is NOT reported as duplicated blocks below: the edits cut it into fragments and only the largest of them clear the block floor, so the rows below understate it. The repair is at the members\u0027 grain. Factor the shared implementation out once and have all of them call it with their differences as parameters or as an injected step, or, where the difference is systematic, generate them from one template. Left alone, the next edit has to be made 4 times and the copies will drift further apart."},"locations":[{"physicalLocation":{"artifactLocation":{"uri":"tokenizers/tk-encode/src/tokenizer/pipeline/mod.rs"},"region":{"startLine":1090}}}],"partialFingerprints":{"codehealthFindingId/v1":"d809dbd3b2656a571986f74d239d90924a01645c3f26c697f1fcd217954bdde5"}},{"ruleId":"D4","level":"warning","message":{"text":"Edited copy of a member (10 corresponding lines): tokenizers/tk-encode/src/vocab/bucket_vocab_store.rs:428-437 | tokenizers/tk-encode/src/vocab/bucket_vocab_store.rs:445-454 \u2014 These two members are one piece of code written twice and then edited apart: 10 consecutive lines correspond almost exactly, broken only by small local edits. Most of that correspondence is NOT reported as duplicated blocks below \u2014 the edits cut it into fragments and only the largest of them clear the block floor, so the rows below understate it. The repair is at the members\u0027 grain \u2014 factor the shared implementation into one the two call with their differences as parameters or as an injected step, or, where the difference is systematic (an extra return value, one transport against another), generate one from the other. Left alone, the next edit has to be made twice and the two will drift further apart."},"locations":[{"physicalLocation":{"artifactLocation":{"uri":"tokenizers/tk-encode/src/vocab/bucket_vocab_store.rs"},"region":{"startLine":428}}}],"partialFingerprints":{"codehealthFindingId/v1":"9858877e967db74926f7b80144c7319aebdae2487ad5332c4253331a1d9fdada"}},{"ruleId":"D4","level":"warning","message":{"text":"Edited copy of a member (9 corresponding lines): bindings/node/src/pipeline.rs:189-200 | bindings/wasm/src/lib.rs:74-82 \u2014 These two members are one piece of code written twice and then edited apart: 9 consecutive lines correspond almost exactly, broken only by small local edits. Most of that correspondence is NOT reported as duplicated blocks below \u2014 the edits cut it into fragments and only the largest of them clear the block floor, so the rows below understate it. The repair is at the members\u0027 grain \u2014 factor the shared implementation into one the two call with their differences as parameters or as an injected step, or, where the difference is systematic (an extra return value, one transport against another), generate one from the other. Left alone, the next edit has to be made twice and the two will drift further apart."},"locations":[{"physicalLocation":{"artifactLocation":{"uri":"bindings/node/src/pipeline.rs"},"region":{"startLine":189}}}],"partialFingerprints":{"codehealthFindingId/v1":"230c2b56e95bf9b4646bcb82eb2e46f0432dc288dac56a2d8d15532f808d1662"}},{"ruleId":"D4","level":"warning","message":{"text":"Edited copy of a member (7 corresponding lines): tokenizers/bitcannon/src/classify/avx.rs:79-85 | tokenizers/bitcannon/src/classify/avx.rs:88-94 \u2014 These two members are one piece of code written twice and then edited apart: 7 consecutive lines correspond almost exactly, broken only by small local edits. Most of that correspondence is NOT reported as duplicated blocks below \u2014 the edits cut it into fragments and only the largest of them clear the block floor, so the rows below understate it. The repair is at the members\u0027 grain \u2014 factor the shared implementation into one the two call with their differences as parameters or as an injected step, or, where the difference is systematic (an extra return value, one transport against another), generate one from the other. Left alone, the next edit has to be made twice and the two will drift further apart."},"locations":[{"physicalLocation":{"artifactLocation":{"uri":"tokenizers/bitcannon/src/classify/avx.rs"},"region":{"startLine":79}}}],"partialFingerprints":{"codehealthFindingId/v1":"d64e3cbb4c3009e3e99f0264680cdbf2664092d0dc8484458519f281917391a7"}},{"ruleId":"D4","level":"warning","message":{"text":"Edited copy of a member (8 corresponding lines): tokenizers/tk-train/src/trainers/bpe/mod.rs:321-329 | tokenizers/tk-train/src/trainers/bpe/parity_trainer.rs:532-539 \u2014 These two members are one piece of code written twice and then edited apart: 8 consecutive lines correspond almost exactly, broken only by small local edits. Most of that correspondence is NOT reported as duplicated blocks below \u2014 the edits cut it into fragments and only the largest of them clear the block floor, so the rows below understate it. The repair is at the members\u0027 grain \u2014 factor the shared implementation into one the two call with their differences as parameters or as an injected step, or, where the difference is systematic (an extra return value, one transport against another), generate one from the other. Left alone, the next edit has to be made twice and the two will drift further apart."},"locations":[{"physicalLocation":{"artifactLocation":{"uri":"tokenizers/tk-train/src/trainers/bpe/mod.rs"},"region":{"startLine":321}}}],"partialFingerprints":{"codehealthFindingId/v1":"2facd436ef180bd11482742c192674743ad06863b94d40d92c6e293031df9c28"}},{"ruleId":"D4","level":"warning","message":{"text":"Edited copy of a member (11 corresponding lines): tokenizers/bitcannon/src/simd/x86.rs:51-61 | tokenizers/bitcannon/src/simd/x86.rs:70-80 \u2014 These two members are one piece of code written twice and then edited apart: 11 consecutive lines correspond almost exactly, broken only by small local edits. Most of that correspondence is NOT reported as duplicated blocks below \u2014 the edits cut it into fragments and only the largest of them clear the block floor, so the rows below understate it. The repair is at the members\u0027 grain \u2014 factor the shared implementation into one the two call with their differences as parameters or as an injected step, or, where the difference is systematic (an extra return value, one transport against another), generate one from the other. Left alone, the next edit has to be made twice and the two will drift further apart."},"locations":[{"physicalLocation":{"artifactLocation":{"uri":"tokenizers/bitcannon/src/simd/x86.rs"},"region":{"startLine":51}}}],"partialFingerprints":{"codehealthFindingId/v1":"4b2792e91dc156896328f2f6953a84ad262ea9675ddca50f8b0254a272036a48"}},{"ruleId":"D4","level":"warning","message":{"text":"Edited copy of a member (14 corresponding lines): tokenizers/tk-encode/src/models/wordlevel/mod.rs:61-74 | tokenizers/tk-encode/src/models/wordpiece/mod.rs:86-101 \u2014 These two members are one piece of code written twice and then edited apart: 14 consecutive lines correspond almost exactly, broken only by small local edits. Most of that correspondence is NOT reported as duplicated blocks below \u2014 the edits cut it into fragments and only the largest of them clear the block floor, so the rows below understate it. The repair is at the members\u0027 grain \u2014 factor the shared implementation into one the two call with their differences as parameters or as an injected step, or, where the difference is systematic (an extra return value, one transport against another), generate one from the other. Left alone, the next edit has to be made twice and the two will drift further apart."},"locations":[{"physicalLocation":{"artifactLocation":{"uri":"tokenizers/tk-encode/src/models/wordlevel/mod.rs"},"region":{"startLine":61}}}],"partialFingerprints":{"codehealthFindingId/v1":"f5b0fb90b2ccf94ece3c282ee5c4067971f3d26137658ee59e1bc1490158c1f2"}},{"ruleId":"D4","level":"warning","message":{"text":"Edited copy of a member (9 corresponding lines): bindings/node/src/pipeline.rs:161-169 | bindings/wasm/src/options.rs:107-118 \u2014 These two members are one piece of code written twice and then edited apart: 9 consecutive lines correspond almost exactly, broken only by small local edits. Most of that correspondence is NOT reported as duplicated blocks below \u2014 the edits cut it into fragments and only the largest of them clear the block floor, so the rows below understate it. The repair is at the members\u0027 grain \u2014 factor the shared implementation into one the two call with their differences as parameters or as an injected step, or, where the difference is systematic (an extra return value, one transport against another), generate one from the other. Left alone, the next edit has to be made twice and the two will drift further apart."},"locations":[{"physicalLocation":{"artifactLocation":{"uri":"bindings/node/src/pipeline.rs"},"region":{"startLine":161}}}],"partialFingerprints":{"codehealthFindingId/v1":"2af3b1657fcc6ab95500edb68cbd674f0079b4c2980f97dffd61ca05195fb2d5"}},{"ruleId":"D4","level":"warning","message":{"text":"Edited copies of one member (3 members, 12\u002B corresponding lines): tokenizers/tk-encode/src/pre_tokenizers/bert.rs:19-31 | tokenizers/tk-encode/src/pre_tokenizers/whitespace.rs:26-37 | tokenizers/tk-encode/src/pre_tokenizers/whitespace.rs:63-76 \u2014 These 3 members are one piece of code written 3 times and then edited apart: every two of them correspond over at least 12 lines, broken only by small local edits. Most of that correspondence is NOT reported as duplicated blocks below: the edits cut it into fragments and only the largest of them clear the block floor, so the rows below understate it. The repair is at the members\u0027 grain. Factor the shared implementation out once and have all of them call it with their differences as parameters or as an injected step, or, where the difference is systematic, generate them from one template. Left alone, the next edit has to be made 3 times and the copies will drift further apart."},"locations":[{"physicalLocation":{"artifactLocation":{"uri":"tokenizers/tk-encode/src/pre_tokenizers/bert.rs"},"region":{"startLine":19}}}],"partialFingerprints":{"codehealthFindingId/v1":"e42b75e5f57d1846c362b281179b5ddc74c659f5b6ed880faa4713f8a0409a3a"}},{"ruleId":"D4","level":"warning","message":{"text":"Edited copy of a member (9 corresponding lines): tokenizers/tk-encode/src/vocab/bucket_vocab_store.rs:318-326 | tokenizers/tk-encode/src/vocab/bucket_vocab_store.rs:354-362 \u2014 These two members are one piece of code written twice and then edited apart: 9 consecutive lines correspond almost exactly, broken only by small local edits. Most of that correspondence is NOT reported as duplicated blocks below \u2014 the edits cut it into fragments and only the largest of them clear the block floor, so the rows below understate it. The repair is at the members\u0027 grain \u2014 factor the shared implementation into one the two call with their differences as parameters or as an injected step, or, where the difference is systematic (an extra return value, one transport against another), generate one from the other. Left alone, the next edit has to be made twice and the two will drift further apart."},"locations":[{"physicalLocation":{"artifactLocation":{"uri":"tokenizers/tk-encode/src/vocab/bucket_vocab_store.rs"},"region":{"startLine":318}}}],"partialFingerprints":{"codehealthFindingId/v1":"fadb7a773c485241ad28fe9383f5063bf800070452f6525869ba29a7f27b18b8"}},{"ruleId":"D4","level":"warning","message":{"text":"Edited copy of a member (7 corresponding lines): bindings/python/src/compiled.rs:358-364 | bindings/python/src/tokenizer.rs:454-460 \u2014 These two members are one piece of code written twice and then edited apart: 7 consecutive lines correspond almost exactly, broken only by small local edits. Most of that correspondence is NOT reported as duplicated blocks below \u2014 the edits cut it into fragments and only the largest of them clear the block floor, so the rows below understate it. The repair is at the members\u0027 grain \u2014 factor the shared implementation into one the two call with their differences as parameters or as an injected step, or, where the difference is systematic (an extra return value, one transport against another), generate one from the other. Left alone, the next edit has to be made twice and the two will drift further apart."},"locations":[{"physicalLocation":{"artifactLocation":{"uri":"bindings/python/src/compiled.rs"},"region":{"startLine":358}}}],"partialFingerprints":{"codehealthFindingId/v1":"85e78fc6f7eead4096fac0adda3a2bd2a01671a1abd60ee4b7a56d396b250d5e"}},{"ruleId":"D4","level":"warning","message":{"text":"Edited copy of a member (9 corresponding lines): bindings/python/src/options/padding.rs:66-74 | bindings/python/src/options/truncation.rs:99-107 \u2014 These two members are one piece of code written twice and then edited apart: 9 consecutive lines correspond almost exactly, broken only by small local edits. Most of that correspondence is NOT reported as duplicated blocks below \u2014 the edits cut it into fragments and only the largest of them clear the block floor, so the rows below understate it. The repair is at the members\u0027 grain \u2014 factor the shared implementation into one the two call with their differences as parameters or as an injected step, or, where the difference is systematic (an extra return value, one transport against another), generate one from the other. Left alone, the next edit has to be made twice and the two will drift further apart."},"locations":[{"physicalLocation":{"artifactLocation":{"uri":"bindings/python/src/options/padding.rs"},"region":{"startLine":66}}}],"partialFingerprints":{"codehealthFindingId/v1":"6d8b3b6332210c2f75deca18631bacee1414add8ac401eb0c4b521571d031e63"}},{"ruleId":"D4","level":"warning","message":{"text":"Edited copies of one member (3 members, 13\u002B corresponding lines): tokenizers/tk-train/src/trainers/bpe/mod.rs:271-283 | tokenizers/tk-train/src/trainers/bpe/parity_trainer.rs:494-506 | tokenizers/tk-train/src/trainers/unigram.rs:163-175 \u2014 These 3 members are one piece of code written 3 times and then edited apart: every two of them correspond over at least 13 lines, broken only by small local edits. Most of that correspondence is NOT reported as duplicated blocks below: the edits cut it into fragments and only the largest of them clear the block floor, so the rows below understate it. The repair is at the members\u0027 grain. Factor the shared implementation out once and have all of them call it with their differences as parameters or as an injected step, or, where the difference is systematic, generate them from one template. Left alone, the next edit has to be made 3 times and the copies will drift further apart."},"locations":[{"physicalLocation":{"artifactLocation":{"uri":"tokenizers/tk-train/src/trainers/bpe/mod.rs"},"region":{"startLine":271}}}],"partialFingerprints":{"codehealthFindingId/v1":"4e326895b2fdcea47c075e94a3727b4740958449169953c2b2de2c73a03818a5"}},{"ruleId":"D4","level":"warning","message":{"text":"Duplicated block (25\u201330 lines \u00D7 2): tokenizers/tk-train/src/trainers/bpe/mod.rs:393-422 | tokenizers/tk-train/src/trainers/bpe/parity_trainer.rs:592-616 \u2014 before extracting anything, compare \u0060tokenizers/tk-train/src/trainers/bpe/mod.rs\u0060 and \u0060tokenizers/tk-train/src/trainers/bpe/parity_trainer.rs\u0060 as WHOLE FILES: this scan already matched 10 separate duplicated blocks between them, totalling at least 150 lines, which is the signature of one file having been copied from the other rather than of a helper waiting to be extracted. If that is what happened, the fix is to keep one copy and have the other call it (or delete it), which resolves this row and its siblings together \u2014 extracting one helper per block leaves the fork in place. Read the line range as the matched WINDOW rather than a finished unit: at \u0060tokenizers/tk-train/src/trainers/bpe/mod.rs:393\u0060 it does not close everything it opens, so those exact lines cannot be lifted as they stand \u2014 widen the region to the smallest complete statement or declaration that contains it, and extract that. \u2605 These copies have DRIFTED, and that is worth reading before extracting anything: just after the matched lines, \u0060tokenizers/tk-train/src/trainers/bpe/mod.rs:424\u0060 calls \u0060Some\u0060, \u0060inc\u0060 and \u0060tokenizers/tk-train/src/trainers/bpe/parity_trainer.rs:617\u0060 does not \u2014 after which the two agree again for 2 more lines. One of those two behaviours is the intended one and the other is what a copy-paste left behind, so decide which BEFORE unifying them: extracting the shared part will silently settle it, and if the copy that skips the call is the wrong one, that bug is already live."},"locations":[{"physicalLocation":{"artifactLocation":{"uri":"tokenizers/tk-train/src/trainers/bpe/mod.rs"},"region":{"startLine":393}}}],"partialFingerprints":{"codehealthFindingId/v1":"b9da1cfa45ae8c341a3091da6c96ec4973e4e7e69c933fa3d9e775ec1927892a"}},{"ruleId":"D4","level":"warning","message":{"text":"Duplicated block (19\u201324 lines \u00D7 2): tokenizers/tk-train/src/trainers/bpe/mod.rs:347-370 | tokenizers/tk-train/src/trainers/bpe/parity_trainer.rs:558-576 \u2014 before extracting anything, compare \u0060tokenizers/tk-train/src/trainers/bpe/mod.rs\u0060 and \u0060tokenizers/tk-train/src/trainers/bpe/parity_trainer.rs\u0060 as WHOLE FILES: this scan already matched 10 separate duplicated blocks between them, totalling at least 150 lines, which is the signature of one file having been copied from the other rather than of a helper waiting to be extracted. If that is what happened, the fix is to keep one copy and have the other call it (or delete it), which resolves this row and its siblings together \u2014 extracting one helper per block leaves the fork in place. Read the line range as the matched WINDOW rather than a finished unit: at \u0060tokenizers/tk-train/src/trainers/bpe/mod.rs:347\u0060 it does not close everything it opens, so those exact lines cannot be lifted as they stand \u2014 widen the region to the smallest complete statement or declaration that contains it, and extract that."},"locations":[{"physicalLocation":{"artifactLocation":{"uri":"tokenizers/tk-train/src/trainers/bpe/mod.rs"},"region":{"startLine":347}}}],"partialFingerprints":{"codehealthFindingId/v1":"778abb673fb3039d1611518e8d030f20f37015d8df38e72aca3489e445983175"}},{"ruleId":"D4","level":"warning","message":{"text":"Duplicated block (20\u201321 lines \u00D7 2): tokenizers/tk-train/src/trainers/bpe/parity_trainer.rs:670-690 | tokenizers/tk-train/src/trainers/bpe/parity_trainer.rs:705-724 \u2014 both copies are in the same file, so extract the block into one function there and call it from each site \u2014 the copies drift apart the first time only one of them is edited. Read the line range as the matched WINDOW rather than a finished unit: at \u0060tokenizers/tk-train/src/trainers/bpe/parity_trainer.rs:670\u0060 it runs out through the closing brace of the declaration holding it \u2014 the window is that declaration\u0027s tail, not a fragment that begins part-way through something, so those exact lines cannot be lifted as they stand \u2014 widen the region to the smallest complete statement or declaration that contains it, and extract that."},"locations":[{"physicalLocation":{"artifactLocation":{"uri":"tokenizers/tk-train/src/trainers/bpe/parity_trainer.rs"},"region":{"startLine":670}}}],"partialFingerprints":{"codehealthFindingId/v1":"834b67f270374fc39ba632599f410d770467e3eac65a25202e42e82b9d17cffd"}},{"ruleId":"D4","level":"warning","message":{"text":"Duplicated block (12\u201320 lines \u00D7 2): tokenizers/bitcannon/src/classify/neon.rs:200-219 | tokenizers/bitcannon/src/classify/neon.rs:280-291 \u2014 both copies are in the same file, so extract the block into one function there and call it from each site \u2014 the copies drift apart the first time only one of them is edited. Read the line range as the matched WINDOW rather than a finished unit: at \u0060tokenizers/bitcannon/src/classify/neon.rs:200\u0060 it does not close everything it opens, so those exact lines cannot be lifted as they stand \u2014 widen the region to the smallest complete statement or declaration that contains it, and extract that."},"locations":[{"physicalLocation":{"artifactLocation":{"uri":"tokenizers/bitcannon/src/classify/neon.rs"},"region":{"startLine":200}}}],"partialFingerprints":{"codehealthFindingId/v1":"231dd357ca463667d6810cb81b14fc0372ab1878f13abe94c172f68c25f1b1a6"}},{"ruleId":"D4","level":"warning","message":{"text":"Duplicated block (16\u201320 lines \u00D7 2): tokenizers/tk-train/src/trainers/bpe/mod.rs:660-679 | tokenizers/tk-train/src/trainers/bpe/parity_trainer.rs:1453-1468 \u2014 before extracting anything, compare \u0060tokenizers/tk-train/src/trainers/bpe/mod.rs\u0060 and \u0060tokenizers/tk-train/src/trainers/bpe/parity_trainer.rs\u0060 as WHOLE FILES: this scan already matched 10 separate duplicated blocks between them, totalling at least 150 lines, which is the signature of one file having been copied from the other rather than of a helper waiting to be extracted. If that is what happened, the fix is to keep one copy and have the other call it (or delete it), which resolves this row and its siblings together \u2014 extracting one helper per block leaves the fork in place. Read the line range as the matched WINDOW rather than a finished unit: at \u0060tokenizers/tk-train/src/trainers/bpe/mod.rs:660\u0060 it runs out through the closing brace of the declaration holding it \u2014 the window is that declaration\u0027s tail, not a fragment that begins part-way through something, so those exact lines cannot be lifted as they stand \u2014 widen the region to the smallest complete statement or declaration that contains it, and extract that."},"locations":[{"physicalLocation":{"artifactLocation":{"uri":"tokenizers/tk-train/src/trainers/bpe/mod.rs"},"region":{"startLine":660}}}],"partialFingerprints":{"codehealthFindingId/v1":"97f37c9e778108ff8350f340be4d43e77cdc9755598e342213b399d8495ab38c"}},{"ruleId":"D4","level":"warning","message":{"text":"Duplicated block (16\u201317 lines \u00D7 2): tokenizers/bitcannon/src/classify/mod.rs:171-187 | tokenizers/bitcannon/src/classify/wasm.rs:291-306 \u2014 the copies sit in sibling files of one directory, so a shared home is within easy reach: extract the block into a single shared function the call sites can all reach \u2014 a file they already depend on, or a new one alongside them \u2014 and call it from both call sites, so a change lands once."},"locations":[{"physicalLocation":{"artifactLocation":{"uri":"tokenizers/bitcannon/src/classify/mod.rs"},"region":{"startLine":171}}}],"partialFingerprints":{"codehealthFindingId/v1":"3ea4e910bf0b518d7286c53c681f39a1fbfd8b89db2100bdda3ada53a74047c9"}},{"ruleId":"D4","level":"warning","message":{"text":"Duplicated block (14\u201316 lines \u00D7 2): tokenizers/tk-train/src/trainers/bpe/mod.rs:579-594 | tokenizers/tk-train/src/trainers/bpe/parity_trainer.rs:1337-1350 \u2014 before extracting anything, compare \u0060tokenizers/tk-train/src/trainers/bpe/mod.rs\u0060 and \u0060tokenizers/tk-train/src/trainers/bpe/parity_trainer.rs\u0060 as WHOLE FILES: this scan already matched 10 separate duplicated blocks between them, totalling at least 150 lines, which is the signature of one file having been copied from the other rather than of a helper waiting to be extracted. If that is what happened, the fix is to keep one copy and have the other call it (or delete it), which resolves this row and its siblings together \u2014 extracting one helper per block leaves the fork in place."},"locations":[{"physicalLocation":{"artifactLocation":{"uri":"tokenizers/tk-train/src/trainers/bpe/mod.rs"},"region":{"startLine":579}}}],"partialFingerprints":{"codehealthFindingId/v1":"5fa9614dfb2c8b672dbce9c4407458cf53b7b43e2f4ff9a3383958feec031e7e"}},{"ruleId":"D4","level":"warning","message":{"text":"Duplicated block (15 lines \u00D7 3): tokenizers/tk-train/src/trainers/bpe/mod.rs:707-721 | tokenizers/tk-train/src/trainers/bpe/parity_trainer.rs:419-433 | tokenizers/tk-train/src/trainers/bpe/parity_trainer.rs:460-474 \u2014 before extracting anything, compare \u0060tokenizers/tk-train/src/trainers/bpe/mod.rs\u0060 and \u0060tokenizers/tk-train/src/trainers/bpe/parity_trainer.rs\u0060 as WHOLE FILES: this scan already matched 10 separate duplicated blocks between them, totalling at least 150 lines, which is the signature of one file having been copied from the other rather than of a helper waiting to be extracted. If that is what happened, the fix is to keep one copy and have the other call it (or delete it), which resolves this row and its siblings together \u2014 extracting one helper per block leaves the fork in place. Read the line range as the matched WINDOW rather than a finished unit: at \u0060tokenizers/tk-train/src/trainers/bpe/mod.rs:707\u0060 it does not close everything it opens, so those exact lines cannot be lifted as they stand \u2014 widen the region to the smallest complete statement or declaration that contains it, and extract that."},"locations":[{"physicalLocation":{"artifactLocation":{"uri":"tokenizers/tk-train/src/trainers/bpe/mod.rs"},"region":{"startLine":707}}}],"partialFingerprints":{"codehealthFindingId/v1":"33b46dc0be811e8169217bc7ea2a9281181edb68068510bf9a6111390f24ccc1"}},{"ruleId":"D4","level":"warning","message":{"text":"Duplicated block (15 lines \u00D7 2): tokenizers/tk-train/src/trainers/bpe/mod.rs:458-472 | tokenizers/tk-train/src/trainers/bpe/parity_trainer.rs:641-655 \u2014 before extracting anything, compare \u0060tokenizers/tk-train/src/trainers/bpe/mod.rs\u0060 and \u0060tokenizers/tk-train/src/trainers/bpe/parity_trainer.rs\u0060 as WHOLE FILES: this scan already matched 10 separate duplicated blocks between them, totalling at least 150 lines, which is the signature of one file having been copied from the other rather than of a helper waiting to be extracted. If that is what happened, the fix is to keep one copy and have the other call it (or delete it), which resolves this row and its siblings together \u2014 extracting one helper per block leaves the fork in place. Read the line range as the matched WINDOW rather than a finished unit: at \u0060tokenizers/tk-train/src/trainers/bpe/mod.rs:458\u0060 it runs out through the closing brace of the declaration holding it \u2014 the window is that declaration\u0027s tail, not a fragment that begins part-way through something, so those exact lines cannot be lifted as they stand \u2014 widen the region to the smallest complete statement or declaration that contains it, and extract that."},"locations":[{"physicalLocation":{"artifactLocation":{"uri":"tokenizers/tk-train/src/trainers/bpe/mod.rs"},"region":{"startLine":458}}}],"partialFingerprints":{"codehealthFindingId/v1":"8887535ea6aff6c1dc22b3dd9b1644f1554ca2dbd554441b9e81d7c30eb62b95"}},{"ruleId":"D4","level":"warning","message":{"text":"Duplicated block (14 lines \u00D7 2): tokenizers/tk-train/src/trainers/unigram.rs:710-723 | tokenizers/tk-train/src/trainers/wordlevel.rs:145-158 \u2014 before extracting anything, compare \u0060tokenizers/tk-train/src/trainers/unigram.rs\u0060 and \u0060tokenizers/tk-train/src/trainers/wordlevel.rs\u0060 as WHOLE FILES: this scan already matched 3 separate duplicated blocks between them, totalling at least 39 lines, which is the signature of one file having been copied from the other rather than of a helper waiting to be extracted. If that is what happened, the fix is to keep one copy and have the other call it (or delete it), which resolves this row and its siblings together \u2014 extracting one helper per block leaves the fork in place. Read the line range as the matched WINDOW rather than a finished unit: at \u0060tokenizers/tk-train/src/trainers/unigram.rs:710\u0060 it begins part-way through the construct above it, so those exact lines cannot be lifted as they stand \u2014 widen the region to the smallest complete statement or declaration that contains it, and extract that."},"locations":[{"physicalLocation":{"artifactLocation":{"uri":"tokenizers/tk-train/src/trainers/unigram.rs"},"region":{"startLine":710}}}],"partialFingerprints":{"codehealthFindingId/v1":"f46787c7afeeaf5ba0b74646a4f3e2c05899424107a7db52c91825e1d08f06b1"}},{"ruleId":"D4","level":"warning","message":{"text":"Duplicated block (13 lines \u00D7 3): tokenizers/tk-train/src/trainers/bpe/mod.rs:718-730 | tokenizers/tk-train/src/trainers/unigram.rs:720-732 | tokenizers/tk-train/src/trainers/wordlevel.rs:155-167 \u2014 before extracting anything, compare \u0060tokenizers/tk-train/src/trainers/unigram.rs\u0060 and \u0060tokenizers/tk-train/src/trainers/wordlevel.rs\u0060 as WHOLE FILES: this scan already matched 3 separate duplicated blocks between them, totalling at least 39 lines, which is the signature of one file having been copied from the other rather than of a helper waiting to be extracted. If that is what happened, the fix is to keep one copy and have the other call it (or delete it), which resolves this row and its siblings together \u2014 extracting one helper per block leaves the fork in place. Read the line range as the matched WINDOW rather than a finished unit: at \u0060tokenizers/tk-train/src/trainers/bpe/mod.rs:718\u0060 it runs out through the closing brace of the declaration holding it \u2014 the window is that declaration\u0027s tail, not a fragment that begins part-way through something, so those exact lines cannot be lifted as they stand \u2014 widen the region to the smallest complete statement or declaration that contains it, and extract that."},"locations":[{"physicalLocation":{"artifactLocation":{"uri":"tokenizers/tk-train/src/trainers/bpe/mod.rs"},"region":{"startLine":718}}}],"partialFingerprints":{"codehealthFindingId/v1":"1a0b89f68b35ad9e212da67eb35e28153b1cdd013c49d86994008d7365d106ab"}},{"ruleId":"D4","level":"warning","message":{"text":"Duplicated block (13 lines \u00D7 2): tokenizers/bitcannon/src/classify/neon.rs:337-349 | tokenizers/bitcannon/src/classify/wasm.rs:287-299 \u2014 the copies sit in sibling files of one directory, so a shared home is within easy reach: extract the block into a single shared function the call sites can all reach \u2014 a file they already depend on, or a new one alongside them \u2014 and call it from both call sites, so a change lands once. Read the line range as the matched WINDOW rather than a finished unit: at \u0060tokenizers/bitcannon/src/classify/neon.rs:337\u0060 it begins part-way through the construct above it, so those exact lines cannot be lifted as they stand \u2014 widen the region to the smallest complete statement or declaration that contains it, and extract that."},"locations":[{"physicalLocation":{"artifactLocation":{"uri":"tokenizers/bitcannon/src/classify/neon.rs"},"region":{"startLine":337}}}],"partialFingerprints":{"codehealthFindingId/v1":"967ada90664f066a2493e2713dc14ede39cce1b5c1c8d94f248885b4afd3791d"}},{"ruleId":"D4","level":"warning","message":{"text":"Duplicated block (12 lines \u00D7 5): tokenizers/tk-train/src/trainers/bpe/mod.rs:715-726 | tokenizers/tk-train/src/trainers/bpe/parity_trainer.rs:427-438 | tokenizers/tk-train/src/trainers/bpe/parity_trainer.rs:468-479 | tokenizers/tk-train/src/trainers/unigram.rs:717-728 | tokenizers/tk-train/src/trainers/wordlevel.rs:152-163 \u2014 before extracting anything, compare \u0060tokenizers/tk-train/src/trainers/bpe/mod.rs\u0060 and \u0060tokenizers/tk-train/src/trainers/bpe/parity_trainer.rs\u0060 as WHOLE FILES: this scan already matched 10 separate duplicated blocks between them, totalling at least 150 lines, which is the signature of one file having been copied from the other rather than of a helper waiting to be extracted. If that is what happened, the fix is to keep one copy and have the other call it (or delete it), which resolves this row and its siblings together \u2014 extracting one helper per block leaves the fork in place. Read the line range as the matched WINDOW rather than a finished unit: at \u0060tokenizers/tk-train/src/trainers/bpe/mod.rs:715\u0060 it runs out through the closing brace of the declaration holding it and carries on into the declaration that follows \u2014 the window is the tail of one member plus the head of the next, so no call can be substituted for those exact lines, and the smallest declaration that contains all of them is the type they sit in. The repeated unit is the member each site sits in: where those members\u0027 bodies are the same, move one whole member to the shared location and have the others delegate to it; where the copies are a run of near-identical overloads or wrappers that differ only in their signatures, the repetition IS the run \u2014 a one-line delegation has no helper inside it to lift \u2014 so generate the run from the set it enumerates, or accept it and keep each member\u0027s own documentation with it."},"locations":[{"physicalLocation":{"artifactLocation":{"uri":"tokenizers/tk-train/src/trainers/bpe/mod.rs"},"region":{"startLine":715}}}],"partialFingerprints":{"codehealthFindingId/v1":"5a22447ec0a816d1bd055eb2b2666331e42ed3f79292abca6e877b6cd2453eb5"}},{"ruleId":"D4","level":"warning","message":{"text":"Duplicated block (12 lines \u00D7 2): tokenizers/tk-encode/src/tokenizer/pipeline/pre_tokenizer.rs:359-370 | tokenizers/tk-encode/src/tokenizer/pipeline/pre_tokenizer.rs:377-388 \u2014 both copies are in the same file, so extract the block into one function there and call it from each site \u2014 the copies drift apart the first time only one of them is edited. Read the line range as the matched WINDOW rather than a finished unit: at \u0060tokenizers/tk-encode/src/tokenizer/pipeline/pre_tokenizer.rs:359\u0060 it runs out through the closing brace of the declaration holding it \u2014 the window is that declaration\u0027s tail, not a fragment that begins part-way through something, so those exact lines cannot be lifted as they stand \u2014 widen the region to the smallest complete statement or declaration that contains it, and extract that."},"locations":[{"physicalLocation":{"artifactLocation":{"uri":"tokenizers/tk-encode/src/tokenizer/pipeline/pre_tokenizer.rs"},"region":{"startLine":359}}}],"partialFingerprints":{"codehealthFindingId/v1":"366c0c3e68f32a7fb02af8cda32e9fd870490d88f0baf94417f6377ce866ce24"}},{"ruleId":"D4","level":"warning","message":{"text":"Duplicated block (11 lines \u00D7 3): tokenizers/tk-encode/src/tokenizer/pattern.rs:63-73 | tokenizers/tk-encode/src/tokenizer/pattern.rs:104-114 | tokenizers/tk-encode/src/utils/fancy.rs:52-62 \u2014 there are 3 copies across 2 file(s) \u2014 more copies than files, so at least one file holds the block twice. Extract it once into a single shared function every call site can reach and call it from all 3 sites; resolving a subset leaves the remainder to drift apart. Read the line range as the matched WINDOW rather than a finished unit: at \u0060tokenizers/tk-encode/src/tokenizer/pattern.rs:63\u0060 it runs out through the closing brace of the declaration holding it \u2014 the window is that declaration\u0027s tail, not a fragment that begins part-way through something, so those exact lines cannot be lifted as they stand \u2014 widen the region to the smallest complete statement or declaration that contains it, and extract that."},"locations":[{"physicalLocation":{"artifactLocation":{"uri":"tokenizers/tk-encode/src/tokenizer/pattern.rs"},"region":{"startLine":63}}}],"partialFingerprints":{"codehealthFindingId/v1":"43ca9f49cc363ea1688f1df68a97094d89c3b8d7630e7b2acd4694018a3a7efc"}},{"ruleId":"D4","level":"warning","message":{"text":"Duplicated block (11 lines \u00D7 2): tokenizers/tk-encode/src/vocab/buckets.rs:397-407 | tokenizers/tk-encode/src/vocab/buckets.rs:415-425 \u2014 both copies are in the same file, so extract the block into one function there and call it from each site \u2014 the copies drift apart the first time only one of them is edited. Read the line range as the matched WINDOW rather than a finished unit: at \u0060tokenizers/tk-encode/src/vocab/buckets.rs:397\u0060 it begins part-way through the construct above it, so those exact lines cannot be lifted as they stand \u2014 widen the region to the smallest complete statement or declaration that contains it, and extract that."},"locations":[{"physicalLocation":{"artifactLocation":{"uri":"tokenizers/tk-encode/src/vocab/buckets.rs"},"region":{"startLine":397}}}],"partialFingerprints":{"codehealthFindingId/v1":"096b80f0c0173c86c7d3c0712b38faa7ac545c4cf12d555bbb7600801b5eee87"}},{"ruleId":"D4","level":"warning","message":{"text":"Duplicated block (9 lines \u00D7 2): tokenizers/bitmap_gen/src/lib.rs:332-340 | tokenizers/bitmap_gen/src/lib.rs:346-354 \u2014 both copies are in the same file, so extract the block into one function there and call it from each site \u2014 the copies drift apart the first time only one of them is edited. Read the line range as the matched WINDOW rather than a finished unit: at \u0060tokenizers/bitmap_gen/src/lib.rs:332\u0060 it runs out through the closing brace of the declaration holding it and carries on into the declaration that follows \u2014 the window is the tail of one member plus the head of the next, so no call can be substituted for those exact lines, and the smallest declaration that contains all of them is the type they sit in. The repeated unit is the member each site sits in: where those members\u0027 bodies are the same, move one whole member to the shared location and have the others delegate to it; where the copies are a run of near-identical overloads or wrappers that differ only in their signatures, the repetition IS the run \u2014 a one-line delegation has no helper inside it to lift \u2014 so generate the run from the set it enumerates, or accept it and keep each member\u0027s own documentation with it."},"locations":[{"physicalLocation":{"artifactLocation":{"uri":"tokenizers/bitmap_gen/src/lib.rs"},"region":{"startLine":332}}}],"partialFingerprints":{"codehealthFindingId/v1":"4dbd0c2283d0849f9f54b0166c3ae2f13bdd2affe6da4f9ac0051bf345ed4eec"}},{"ruleId":"D4","level":"warning","message":{"text":"Duplicated block (9 lines \u00D7 3): tokenizers/bitcannon/src/classify/avx.rs:46-54 | tokenizers/bitcannon/src/classify/neon.rs:105-113 | tokenizers/bitcannon/src/classify/wasm.rs:35-43 \u2014 the copies sit in sibling files of one directory, so a shared home is within easy reach: extract the block into a single shared function the call sites can all reach \u2014 a file they already depend on, or a new one alongside them \u2014 and call it from all 3 call sites, so a change lands once. Read the line range as the matched WINDOW rather than a finished unit: at \u0060tokenizers/bitcannon/src/classify/avx.rs:46\u0060 it runs out through the closing brace of the declaration holding it \u2014 the window is that declaration\u0027s tail, not a fragment that begins part-way through something, so those exact lines cannot be lifted as they stand \u2014 widen the region to the smallest complete statement or declaration that contains it, and extract that."},"locations":[{"physicalLocation":{"artifactLocation":{"uri":"tokenizers/bitcannon/src/classify/avx.rs"},"region":{"startLine":46}}}],"partialFingerprints":{"codehealthFindingId/v1":"65e6329e28b320f1a7ff83809aa6e72eaf7ac8b88484a08bd5ec3394078b4083"}},{"ruleId":"D4","level":"warning","message":{"text":"Duplicated block (8 lines \u00D7 2): tokenizers/tk-encode/src/decoders/byte_fallback.rs:30-37 | tokenizers/tk-encode/src/decoders/byte_fallback.rs:43-50 \u2014 both copies are in the same file, so extract the block into one function there and call it from each site \u2014 the copies drift apart the first time only one of them is edited. Read the line range as the matched WINDOW rather than a finished unit: at \u0060tokenizers/tk-encode/src/decoders/byte_fallback.rs:30\u0060 it does not close everything it opens, so those exact lines cannot be lifted as they stand \u2014 widen the region to the smallest complete statement or declaration that contains it, and extract that."},"locations":[{"physicalLocation":{"artifactLocation":{"uri":"tokenizers/tk-encode/src/decoders/byte_fallback.rs"},"region":{"startLine":30}}}],"partialFingerprints":{"codehealthFindingId/v1":"17579101feb8303c2b73177ec67b6d37be7f91d04a866af0a3595875670b0763"}},{"ruleId":"D4","level":"warning","message":{"text":"Duplicated block (8 lines \u00D7 2): tokenizers/tk-encode/src/models/bpe/model.rs:253-260 | tokenizers/tk-encode/src/models/bpe/model.rs:384-391 \u2014 both copies are in the same file, so extract the block into one function there and call it from each site \u2014 the copies drift apart the first time only one of them is edited."},"locations":[{"physicalLocation":{"artifactLocation":{"uri":"tokenizers/tk-encode/src/models/bpe/model.rs"},"region":{"startLine":253}}}],"partialFingerprints":{"codehealthFindingId/v1":"6f180f40051855ea2aad3190d9ced8fd232fd9f8ed736d48cb9ccceb8d6841fa"}},{"ruleId":"D4","level":"warning","message":{"text":"Duplicated block (7 lines \u00D7 2): tokenizers/tk-encode/src/models/unigram/lattice.rs:349-355 | tokenizers/tk-encode/src/models/unigram/lattice.rs:399-405 \u2014 both copies are in the same file, so extract the block into one function there and call it from each site \u2014 the copies drift apart the first time only one of them is edited. Read the line range as the matched WINDOW rather than a finished unit: at \u0060tokenizers/tk-encode/src/models/unigram/lattice.rs:349\u0060 it does not close everything it opens, so those exact lines cannot be lifted as they stand \u2014 widen the region to the smallest complete statement or declaration that contains it, and extract that."},"locations":[{"physicalLocation":{"artifactLocation":{"uri":"tokenizers/tk-encode/src/models/unigram/lattice.rs"},"region":{"startLine":349}}}],"partialFingerprints":{"codehealthFindingId/v1":"047d2128f2d18fa46e2fea54b370269416a6311b5a4d225fce8a761394ff41bd"}},{"ruleId":"D4","level":"warning","message":{"text":"Duplicated block (6 lines \u00D7 2): tokenizers/bitcannon/src/models/cl100k.rs:47-52 | tokenizers/bitcannon/src/models/gpt2.rs:23-28 \u2014 the copies sit in sibling files of one directory, so a shared home is within easy reach: extract the block into a single shared function the call sites can all reach \u2014 a file they already depend on, or a new one alongside them \u2014 and call it from both call sites, so a change lands once."},"locations":[{"physicalLocation":{"artifactLocation":{"uri":"tokenizers/bitcannon/src/models/cl100k.rs"},"region":{"startLine":47}}}],"partialFingerprints":{"codehealthFindingId/v1":"cc6e6facb89e0881c7c30ac4204bc27e654370ab8aebdf064620ebec606d8560"}},{"ruleId":"D4","level":"warning","message":{"text":"Duplicated block (6 lines \u00D7 2): tokenizers/tk-encode/src/pre_tokenizers/split.rs:83-88 | tokenizers/tk-encode/src/pre_tokenizers/split.rs:120-125 \u2014 both copies are in the same file, so extract the block into one function there and call it from each site \u2014 the copies drift apart the first time only one of them is edited. Read the line range as the matched WINDOW rather than a finished unit: at \u0060tokenizers/tk-encode/src/pre_tokenizers/split.rs:83\u0060 it does not close everything it opens, so those exact lines cannot be lifted as they stand \u2014 widen the region to the smallest complete statement or declaration that contains it, and extract that. \u2605 These copies have DRIFTED, and that is worth reading before extracting anything: just before the matched lines, \u0060tokenizers/tk-encode/src/pre_tokenizers/split.rs:82\u0060 calls \u0060into\u0060 and \u0060tokenizers/tk-encode/src/pre_tokenizers/split.rs:119\u0060 does not \u2014 after which the two agree again for 4 more lines. One of those two behaviours is the intended one and the other is what a copy-paste left behind, so decide which BEFORE unifying them: extracting the shared part will silently settle it, and if the copy that skips the call is the wrong one, that bug is already live."},"locations":[{"physicalLocation":{"artifactLocation":{"uri":"tokenizers/tk-encode/src/pre_tokenizers/split.rs"},"region":{"startLine":83}}}],"partialFingerprints":{"codehealthFindingId/v1":"1248c94f17e295880b8631f1753d091cf56f5b65e8a3f5a9202afc96dd1f01f7"}},{"ruleId":"D4","level":"warning","message":{"text":"Duplicated block (6 lines \u00D7 2): tokenizers/tk-train/src/trainers/bpe/mod.rs:378-383 | tokenizers/tk-train/src/trainers/bpe/parity_trainer.rs:577-582 \u2014 before extracting anything, compare \u0060tokenizers/tk-train/src/trainers/bpe/mod.rs\u0060 and \u0060tokenizers/tk-train/src/trainers/bpe/parity_trainer.rs\u0060 as WHOLE FILES: this scan already matched 10 separate duplicated blocks between them, totalling at least 150 lines, which is the signature of one file having been copied from the other rather than of a helper waiting to be extracted. If that is what happened, the fix is to keep one copy and have the other call it (or delete it), which resolves this row and its siblings together \u2014 extracting one helper per block leaves the fork in place. Read the line range as the matched WINDOW rather than a finished unit: at \u0060tokenizers/tk-train/src/trainers/bpe/mod.rs:378\u0060 it runs out through the closing brace of the declaration holding it \u2014 the window is that declaration\u0027s tail, not a fragment that begins part-way through something, so those exact lines cannot be lifted as they stand \u2014 widen the region to the smallest complete statement or declaration that contains it, and extract that."},"locations":[{"physicalLocation":{"artifactLocation":{"uri":"tokenizers/tk-train/src/trainers/bpe/mod.rs"},"region":{"startLine":378}}}],"partialFingerprints":{"codehealthFindingId/v1":"553d5a8dd59f5049afa382af033e8b02bb7f7912d34b2c2ac8f10205070187bd"}},{"ruleId":"D4","level":"warning","message":{"text":"Duplicated block (6 lines \u00D7 2): tokenizers/tk-train/src/trainers/bpe/mod.rs:324-329 | tokenizers/tk-train/src/trainers/bpe/parity_trainer.rs:534-539 \u2014 before extracting anything, compare \u0060tokenizers/tk-train/src/trainers/bpe/mod.rs\u0060 and \u0060tokenizers/tk-train/src/trainers/bpe/parity_trainer.rs\u0060 as WHOLE FILES: this scan already matched 10 separate duplicated blocks between them, totalling at least 150 lines, which is the signature of one file having been copied from the other rather than of a helper waiting to be extracted. If that is what happened, the fix is to keep one copy and have the other call it (or delete it), which resolves this row and its siblings together \u2014 extracting one helper per block leaves the fork in place. Read the line range as the matched WINDOW rather than a finished unit: at \u0060tokenizers/tk-train/src/trainers/bpe/mod.rs:324\u0060 it runs out through the closing brace of the declaration holding it \u2014 the window is that declaration\u0027s tail, not a fragment that begins part-way through something, so those exact lines cannot be lifted as they stand \u2014 widen the region to the smallest complete statement or declaration that contains it, and extract that."},"locations":[{"physicalLocation":{"artifactLocation":{"uri":"tokenizers/tk-train/src/trainers/bpe/mod.rs"},"region":{"startLine":324}}}],"partialFingerprints":{"codehealthFindingId/v1":"a2681b71820c3cbe10c7c6acb54942b49dbb91ac0e82e96c047a256a5ec4b710"}},{"ruleId":"D4","level":"warning","message":{"text":"Duplicated block (9 lines \u00D7 2): tokenizers/tk-encode/src/decoders/replace.rs:16-24 | tokenizers/tk-encode/src/normalizers/replace.rs:37-46 \u2014 the copies span different directories, so extracting a shared function means choosing where it lives: put it somewhere both call sites can already reach \u2014 a location they all depend on today, or a new shared one if there is none \u2014 and call it from each site; until then, every change has to be made twice."},"locations":[{"physicalLocation":{"artifactLocation":{"uri":"tokenizers/tk-encode/src/decoders/replace.rs"},"region":{"startLine":16}}}],"partialFingerprints":{"codehealthFindingId/v1":"bad83e81f6808bfa8dc6d507870e84f0b7ac734416058d68ec4d9d0f42ba4063"}},{"ruleId":"D4","level":"warning","message":{"text":"Duplicated block (6 lines \u00D7 2): tokenizers/tk-train/src/trainers/bpe/mod.rs:148-153 | tokenizers/tk-train/src/trainers/bpe/parity_trainer.rs:167-172 \u2014 before extracting anything, compare \u0060tokenizers/tk-train/src/trainers/bpe/mod.rs\u0060 and \u0060tokenizers/tk-train/src/trainers/bpe/parity_trainer.rs\u0060 as WHOLE FILES: this scan already matched 10 separate duplicated blocks between them, totalling at least 150 lines, which is the signature of one file having been copied from the other rather than of a helper waiting to be extracted. If that is what happened, the fix is to keep one copy and have the other call it (or delete it), which resolves this row and its siblings together \u2014 extracting one helper per block leaves the fork in place."},"locations":[{"physicalLocation":{"artifactLocation":{"uri":"tokenizers/tk-train/src/trainers/bpe/mod.rs"},"region":{"startLine":148}}}],"partialFingerprints":{"codehealthFindingId/v1":"4d13e4f6c52e077287ed24999b0d8b3d81158c7104d9652841b57e09132332da"}},{"ruleId":"D4","level":"warning","message":{"text":"Duplicated block (7 lines \u00D7 2): tokenizers/tk-train/src/trainers/bpe/parity_trainer.rs:509-515 | tokenizers/tk-train/src/trainers/unigram.rs:518-524 \u2014 the copies span different directories, so extracting a shared function means choosing where it lives: put it somewhere both call sites can already reach \u2014 a location they all depend on today, or a new shared one if there is none \u2014 and call it from each site; until then, every change has to be made twice."},"locations":[{"physicalLocation":{"artifactLocation":{"uri":"tokenizers/tk-train/src/trainers/bpe/parity_trainer.rs"},"region":{"startLine":509}}}],"partialFingerprints":{"codehealthFindingId/v1":"10dc7c9298af50edf6a14293296a533830ba43500bb51af6112f69bddb2083b6"}},{"ruleId":"D4","level":"warning","message":{"text":"Duplicated block (7 lines \u00D7 2): tokenizers/tk-train/src/trainers/bpe/parity_trainer.rs:518-524 | tokenizers/tk-train/src/trainers/unigram.rs:510-516 \u2014 the copies span different directories, so extracting a shared function means choosing where it lives: put it somewhere both call sites can already reach \u2014 a location they all depend on today, or a new shared one if there is none \u2014 and call it from each site; until then, every change has to be made twice."},"locations":[{"physicalLocation":{"artifactLocation":{"uri":"tokenizers/tk-train/src/trainers/bpe/parity_trainer.rs"},"region":{"startLine":518}}}],"partialFingerprints":{"codehealthFindingId/v1":"a0eba683aaccc9ff5311fe9d7c26a582ca9471334e4ae5329f3c2a34e8ff2d1e"}},{"ruleId":"D4","level":"warning","message":{"text":"Duplicated block (12 lines \u00D7 3): tokenizers/bitmap_gen/src/lib.rs:299-310 | tokenizers/bitmap_gen/src/lib.rs:327-338 | tokenizers/bitmap_gen/src/lib.rs:341-352 \u2014 all 3 copies are in the same file, so extract the block into one function there and call it from every one of those sites \u2014 resolving only two of them leaves the rest to drift apart the first time one is edited. Read the line range as the matched WINDOW rather than a finished unit: at \u0060tokenizers/bitmap_gen/src/lib.rs:299\u0060 it does not close everything it opens, so those exact lines cannot be lifted as they stand \u2014 widen the region to the smallest complete statement or declaration that contains it, and extract that."},"locations":[{"physicalLocation":{"artifactLocation":{"uri":"tokenizers/bitmap_gen/src/lib.rs"},"region":{"startLine":299}}}],"partialFingerprints":{"codehealthFindingId/v1":"3f603d10b6e7595d04bca2848038f84e93e5e9b470c3bc4db7e290725bfdf680"}},{"ruleId":"D5","level":"warning","message":{"text":"Off the main sequence: bitcannon: bitcannon: abstractness 0.07, instability 0.00, distance 0.93 \u2014 zone of pain \u2014 concrete and depended on by 3 project(s), so it\u0027s rigid to change."},"locations":[],"partialFingerprints":{"codehealthFindingId/v1":"54381f13bdb20836afd3e1f99bed5f4cae08156f6b4bfeb3d9e39ae5a6ddb9ad"}},{"ruleId":"D15","level":"warning","message":{"text":"Hotspot: tokenizers/tk-convert/src/convert.rs: tokenizers/tk-convert/src/convert.rs changed 7 times in last 90 days, and the most complex body those changes touched has cyclomatic complexity 19 in tk_convert::convert::lower_template_node at line 740. 1 of those changes was a fix/bug commit, and the other 6 changed it for other reasons \u2014 this file is under both repair and feature pressure. Before the next change lands here, make sure the area it touches is under test, then split that area out of the file so the following change is smaller than this one \u2014 a file this often edited pays the complexity back every time. Counted over 2026-08-14..2026-11-12, the 90 days ending at the analysed commit. Reproduce with \u0060git log --since=\u00272026-08-14 16:57:08 \u002B01:00\u0027 --until=\u00272026-11-12 16:57:08 \u002B01:00\u0027 --full-history --no-merges -- tokenizers/tk-convert/src/convert.rs\u0060: merges are excluded because a merge re-states changes already counted at their own commits, and history is NOT path-simplified because a change that reached the file through a merged branch is still a change to it. That command counts raw commits and can read HIGHER than this row, which counts a cherry-picked re-land, and a revert together with the commit it undoes, once each \u2014 a difference of several commits on a file whose history was re-landed or reverted inside the window."},"locations":[{"physicalLocation":{"artifactLocation":{"uri":"tokenizers/tk-convert/src/convert.rs"},"region":{"startLine":740}}}],"partialFingerprints":{"codehealthFindingId/v1":"0c074f27ed89fed18b4fc1d70e4e02cc7c189bb58ab48312a04482349f2a62e7"}},{"ruleId":"D15","level":"warning","message":{"text":"Hotspot: tokenizers/bitmap_gen/src/lib.rs: tokenizers/bitmap_gen/src/lib.rs changed 4 times in last 90 days, and the most complex body those changes touched has cyclomatic complexity 27 in bitmap_gen::atom at line 41. 1 of those changes was a fix/bug commit, and the other 3 changed it for other reasons \u2014 this file is under both repair and feature pressure. Before the next change lands here, make sure the area it touches is under test, then split that area out of the file so the following change is smaller than this one \u2014 a file this often edited pays the complexity back every time. Counted over 2026-08-14..2026-11-12, the 90 days ending at the analysed commit. Reproduce with \u0060git log --since=\u00272026-08-14 16:57:08 \u002B01:00\u0027 --until=\u00272026-11-12 16:57:08 \u002B01:00\u0027 --full-history --no-merges -- tokenizers/bitmap_gen/src/lib.rs\u0060: merges are excluded because a merge re-states changes already counted at their own commits, and history is NOT path-simplified because a change that reached the file through a merged branch is still a change to it. That command counts raw commits and can read HIGHER than this row, which counts a cherry-picked re-land, and a revert together with the commit it undoes, once each \u2014 a difference of several commits on a file whose history was re-landed or reverted inside the window."},"locations":[{"physicalLocation":{"artifactLocation":{"uri":"tokenizers/bitmap_gen/src/lib.rs"},"region":{"startLine":41}}}],"partialFingerprints":{"codehealthFindingId/v1":"b974b447701b69649cb013b1b52dc15b056e9778b0146a26c77b7e18c7477481"}},{"ruleId":"D15","level":"warning","message":{"text":"Hotspot: tokenizers/tk-encode/src/models/bpe/serialization.rs: tokenizers/tk-encode/src/models/bpe/serialization.rs changed 5 times in last 90 days, and the most complex body those changes touched has cyclomatic complexity 20 in PipelineBPE::from_merge_map at line 193. 3 of those changes were fix/bug commits, so the churn is repair rather than feature work. Before the next change lands here, make sure the area it touches is under test, then split that area out of the file so the following change is smaller than this one \u2014 a file this often edited pays the complexity back every time. Counted over 2026-08-14..2026-11-12, the 90 days ending at the analysed commit. Reproduce with \u0060git log --since=\u00272026-08-14 16:57:08 \u002B01:00\u0027 --until=\u00272026-11-12 16:57:08 \u002B01:00\u0027 --full-history --no-merges -- tokenizers/tk-encode/src/models/bpe/serialization.rs\u0060: merges are excluded because a merge re-states changes already counted at their own commits, and history is NOT path-simplified because a change that reached the file through a merged branch is still a change to it. That command counts raw commits and can read HIGHER than this row, which counts a cherry-picked re-land, and a revert together with the commit it undoes, once each \u2014 a difference of several commits on a file whose history was re-landed or reverted inside the window."},"locations":[{"physicalLocation":{"artifactLocation":{"uri":"tokenizers/tk-encode/src/models/bpe/serialization.rs"},"region":{"startLine":193}}}],"partialFingerprints":{"codehealthFindingId/v1":"5d38891c6d99a06c2b558c98ceaf2534d27940c598b375d9b87cc97b38492744"}},{"ruleId":"D15","level":"warning","message":{"text":"Hotspot: tokenizers/tk-encode/src/models/bpe/tables.rs: tokenizers/tk-encode/src/models/bpe/tables.rs changed 3 times in last 90 days, and the most complex body those changes touched has cyclomatic complexity 16 in BpeTables::build at line 79. 1 of those changes was a fix/bug commit, and the other 2 changed it for other reasons \u2014 this file is under both repair and feature pressure. Before the next change lands here, make sure the area it touches is under test, then split that area out of the file so the following change is smaller than this one \u2014 a file this often edited pays the complexity back every time. Counted over 2026-08-14..2026-11-12, the 90 days ending at the analysed commit. Reproduce with \u0060git log --since=\u00272026-08-14 16:57:08 \u002B01:00\u0027 --until=\u00272026-11-12 16:57:08 \u002B01:00\u0027 --full-history --no-merges -- tokenizers/tk-encode/src/models/bpe/tables.rs\u0060: merges are excluded because a merge re-states changes already counted at their own commits, and history is NOT path-simplified because a change that reached the file through a merged branch is still a change to it. That command counts raw commits and can read HIGHER than this row, which counts a cherry-picked re-land, and a revert together with the commit it undoes, once each \u2014 a difference of several commits on a file whose history was re-landed or reverted inside the window."},"locations":[{"physicalLocation":{"artifactLocation":{"uri":"tokenizers/tk-encode/src/models/bpe/tables.rs"},"region":{"startLine":79}}}],"partialFingerprints":{"codehealthFindingId/v1":"73fa8cd57fad76f4cd17de2172140f8f354a63207be59c8ed2c11ec378b5e5f2"}},{"ruleId":"D16","level":"note","message":{"text":"Off-boarding risk: anonymized user #1: If anonymized user #1 becomes unavailable, 9 significant file(s) lose their only recent owner: tokenizers/tk-train/src/trainers/bpe/parity_trainer.rs, tokenizers/tk-train/src/trainers/bpe/mod.rs, bindings/wasm/src/options.rs, tokenizers/tk-encode/src/models/wordlevel/mod.rs, bindings/python/src/options/padding.rs, bindings/python/src/repr.rs, bindings/python/src/options/truncation.rs, bindings/python/src/type_hints.rs (\u002B1 more). Pair on, review, or document these before any departure."},"locations":[],"partialFingerprints":{"codehealthFindingId/v1":"5f65ade1dbb8379f5ea0a2a9bb59a1c8a042057a866619268f123d7bf2e8671c"}},{"ruleId":"D16","level":"note","message":{"text":"Further sole-owners (lower concentration): 2 other contributor(s) are each the sole owner of a small amount of code below the off-boarding threshold \u2014 folded into the bus-factor score and metrics (12 single-owned of 130 analysed files in total, counted over production source files of roughly 2,400 bytes or more, excluding vendored, generated, example/demo and scaffolding-template trees and test files identified by path convention, largest first; 130 of the 184 production source files in this repository met that bar). They are anonymized user #2 (2 file(s)), anonymized user #3 (1 file(s)) \u2014 spread or document their files in the same way, at lower priority than the named off-boarding risks above. The files folded here: tokenizers/tk-serialize/src/from_json/post_processors.rs, tokenizers/tk-serialize/src/to_json/post_processors.rs, tokenizers/tk-encode/src/utils/parallelism.rs."},"locations":[],"partialFingerprints":{"codehealthFindingId/v1":"6eac528f2429e724e1a97ed868f79eefbcf1888dab4af9a9e673429369bb5b2f"}},{"ruleId":"D17","level":"warning","message":{"text":"XxxComment: // XXX \u2014 source code is not a task system: move the work to your tracker and leave a reference instead (e.g. \u0060// REF: #123\u0060), so the task is planned where tasks live and the ticket links back to the code."},"locations":[{"physicalLocation":{"artifactLocation":{"uri":"tokenizers/tk-train/src/trainers/unigram.rs"},"region":{"startLine":291}}}],"partialFingerprints":{"codehealthFindingId/v1":"818b115af810f213cbe84968e6df16ea1b29bf45562ffb711375ad38afb91dae"}},{"ruleId":"D17","level":"warning","message":{"text":"TodoComment: // TODO: Temporary hack to avoid Nans. \u2014 source code is not a task system: move the work to your tracker and leave a reference instead (e.g. \u0060// REF: #123\u0060), so the task is planned where tasks live and the ticket links back to the code."},"locations":[{"physicalLocation":{"artifactLocation":{"uri":"tokenizers/tk-train/src/trainers/unigram.rs"},"region":{"startLine":464}}}],"partialFingerprints":{"codehealthFindingId/v1":"cca0f83c0a35a7bb45e0b186887985f164624faf53c0f6b3b2c06919429c7868"}},{"ruleId":"D17","level":"warning","message":{"text":"TodoComment: // TODO Should be able to upgrade to u64 when needed \u2014 source code is not a task system: move the work to your tracker and leave a reference instead (e.g. \u0060// REF: #123\u0060), so the task is planned where tasks live and the ticket links back to the code."},"locations":[{"physicalLocation":{"artifactLocation":{"uri":"tokenizers/tk-train/src/trainers/unigram.rs"},"region":{"startLine":612}}}],"partialFingerprints":{"codehealthFindingId/v1":"cebc101e196bc877556b80b0792496aeeecdac07acd297fe3bcd24b09b0459f3"}},{"ruleId":"D17","level":"warning","message":{"text":"FixmeComment: // FIXME: doesn\u0027t look great \u2014 source code is not a task system: move the work to your tracker and leave a reference instead (e.g. \u0060// REF: #123\u0060), so the task is planned where tasks live and the ticket links back to the code."},"locations":[{"physicalLocation":{"artifactLocation":{"uri":"tokenizers/tk-train/src/trainers/bpe/mod.rs"},"region":{"startLine":603}}}],"partialFingerprints":{"codehealthFindingId/v1":"5c1038e8c0b8a9c3f2e0f762b26e2363a2d24660c4d824da4df9b5b39d5e1f35"}},{"ruleId":"D17","level":"warning","message":{"text":"FixmeComment: // FIXME: doesn\u0027t look great \u2014 source code is not a task system: move the work to your tracker and leave a reference instead (e.g. \u0060// REF: #123\u0060), so the task is planned where tasks live and the ticket links back to the code."},"locations":[{"physicalLocation":{"artifactLocation":{"uri":"tokenizers/tk-train/src/trainers/bpe/parity_trainer.rs"},"region":{"startLine":1493}}}],"partialFingerprints":{"codehealthFindingId/v1":"e78a5390a1acca06aedbbd3137debff9ccb3a554db16188e70e4670ed1745c62"}},{"ruleId":"D17","level":"warning","message":{"text":"TodoComment: // TODO: this will be fixed for v1! \u2014 source code is not a task system: move the work to your tracker and leave a reference instead (e.g. \u0060// REF: #123\u0060), so the task is planned where tasks live and the ticket links back to the code."},"locations":[{"physicalLocation":{"artifactLocation":{"uri":"tokenizers/tk-serialize/src/to_json/normalizers.rs"},"region":{"startLine":104}}}],"partialFingerprints":{"codehealthFindingId/v1":"8099aa1a826dffb877c8a421088532d22bc05d77cfe67bf0d6d76cfad6633206"}},{"ruleId":"D17","level":"warning","message":{"text":"TodoComment: // TODO(pr3): every fixture in \u0060data/\u0060 is a 1.0 file, so this has no input until tk-convert can \u2014 source code is not a task system: move the work to your tracker and leave a reference instead (e.g. \u0060// REF: #123\u0060), so the task is planned where tasks live and the ticket links back to the code."},"locations":[{"physicalLocation":{"artifactLocation":{"uri":"tokenizers/tk-serialize/src/to_json/tests.rs"},"region":{"startLine":110}}}],"partialFingerprints":{"codehealthFindingId/v1":"4ac937d90353ade7d36078af7816d974ad571fb11a5021839f327003e79bd321"}},{"ruleId":"D17","level":"warning","message":{"text":"TodoComment: //! ## TODO(pr3): port to tk-convert \u2014 source code is not a task system: move the work to your tracker and leave a reference instead (e.g. \u0060// REF: #123\u0060), so the task is planned where tasks live and the ticket links back to the code."},"locations":[{"physicalLocation":{"artifactLocation":{"uri":"tokenizers/tk-serialize/src/from_json/mod.rs"},"region":{"startLine":6}}}],"partialFingerprints":{"codehealthFindingId/v1":"471b9d6d86b8b42b38017eb1af8ac1ba4ea2dc095bfabb9ae89b368e0d5ba85a"}},{"ruleId":"D17","level":"warning","message":{"text":"TodoComment: // TODO: this has nothing to do here. \u2014 source code is not a task system: move the work to your tracker and leave a reference instead (e.g. \u0060// REF: #123\u0060), so the task is planned where tasks live and the ticket links back to the code."},"locations":[{"physicalLocation":{"artifactLocation":{"uri":"tokenizers/tk-serialize/src/from_json/mod.rs"},"region":{"startLine":213}}}],"partialFingerprints":{"codehealthFindingId/v1":"d598fe0608c4357f8afa18636cfbb2a58f159f786789676e5ebda0827d2cff2b"}},{"ruleId":"D17","level":"warning","message":{"text":"FixmeComment: // FIXME: we use chr conversion for now, this can be inproved by using bitmap. \u2014 source code is not a task system: move the work to your tracker and leave a reference instead (e.g. \u0060// REF: #123\u0060), so the task is planned where tasks live and the ticket links back to the code."},"locations":[{"physicalLocation":{"artifactLocation":{"uri":"tokenizers/tk-encode/src/vocab/bucket_added_vocabulary.rs"},"region":{"startLine":42}}}],"partialFingerprints":{"codehealthFindingId/v1":"ab45cdeaf2615e96fad6da8d736511b6da97f589e612142a4c662eecbb79017e"}},{"ruleId":"D17","level":"warning","message":{"text":"TodoComment: // TODO: remove this once we have the final saving format \u2014 source code is not a task system: move the work to your tracker and leave a reference instead (e.g. \u0060// REF: #123\u0060), so the task is planned where tasks live and the ticket links back to the code."},"locations":[{"physicalLocation":{"artifactLocation":{"uri":"tokenizers/tk-encode/src/vocab/bucket_added_vocabulary.rs"},"region":{"startLine":130}}}],"partialFingerprints":{"codehealthFindingId/v1":"3c89967f6d38b5faf25248872114341dfa83e887965bb51c38e8c69a3dfe52f4"}},{"ruleId":"D17","level":"warning","message":{"text":"TodoComment: // TODO: make it clean. \u2014 source code is not a task system: move the work to your tracker and leave a reference instead (e.g. \u0060// REF: #123\u0060), so the task is planned where tasks live and the ticket links back to the code."},"locations":[{"physicalLocation":{"artifactLocation":{"uri":"tokenizers/tk-encode/src/vocab/bucket_added_vocabulary.rs"},"region":{"startLine":233}}}],"partialFingerprints":{"codehealthFindingId/v1":"2fcb84da66608922b49449d99b1134552698ca802c77d1c1ba72bb282f13e76f"}},{"ruleId":"D17","level":"warning","message":{"text":"TodoComment: /// TODO: this will be slow because we rebuild the added tokens \u2014 source code is not a task system: move the work to your tracker and leave a reference instead (e.g. \u0060// REF: #123\u0060), so the task is planned where tasks live and the ticket links back to the code."},"locations":[{"physicalLocation":{"artifactLocation":{"uri":"tokenizers/tk-encode/src/vocab/bucket_added_vocabulary.rs"},"region":{"startLine":294}}}],"partialFingerprints":{"codehealthFindingId/v1":"4dd13d639fde49a057241a3d19ae9c2e960655507738f63c2e97d13ab6a95d43"}},{"ruleId":"D17","level":"warning","message":{"text":"TodoComment: // TODO: implem from / to \u2014 source code is not a task system: move the work to your tracker and leave a reference instead (e.g. \u0060// REF: #123\u0060), so the task is planned where tasks live and the ticket links back to the code."},"locations":[{"physicalLocation":{"artifactLocation":{"uri":"tokenizers/tk-encode/src/vocab/bucket_added_vocabulary.rs"},"region":{"startLine":308}}}],"partialFingerprints":{"codehealthFindingId/v1":"b9fa19504b7885c660b00d9478bf5ab1382f50c780c505cafcc175431b3a63af"}},{"ruleId":"D17","level":"warning","message":{"text":"TodoComment: // \u003E8 high nibbles -\u003E fallback to the general implementation TODO: \u2014 source code is not a task system: move the work to your tracker and leave a reference instead (e.g. \u0060// REF: #123\u0060), so the task is planned where tasks live and the ticket links back to the code."},"locations":[{"physicalLocation":{"artifactLocation":{"uri":"tokenizers/tk-encode/src/vocab/buckets.rs"},"region":{"startLine":207}}}],"partialFingerprints":{"codehealthFindingId/v1":"db4633c08b6d02d66bff9db759b1271d8316cdf440853404acfe011a5d65aeec"}},{"ruleId":"D17","level":"warning","message":{"text":"TodoComment: /// TODO: deprecate \u2014 source code is not a task system: move the work to your tracker and leave a reference instead (e.g. \u0060// REF: #123\u0060), so the task is planned where tasks live and the ticket links back to the code."},"locations":[{"physicalLocation":{"artifactLocation":{"uri":"tokenizers/tk-encode/src/utils/parallelism.rs"},"region":{"startLine":20}}}],"partialFingerprints":{"codehealthFindingId/v1":"13b90b39e21e5fe7894261fdf3803a805ed14604aa7cff850cdf8c240fed7cb9"}},{"ruleId":"D17","level":"warning","message":{"text":"TodoComment: /// TODO: deprecate \u2014 source code is not a task system: move the work to your tracker and leave a reference instead (e.g. \u0060// REF: #123\u0060), so the task is planned where tasks live and the ticket links back to the code."},"locations":[{"physicalLocation":{"artifactLocation":{"uri":"tokenizers/tk-encode/src/utils/parallelism.rs"},"region":{"startLine":23}}}],"partialFingerprints":{"codehealthFindingId/v1":"7d17326720a3a0f74df0ccecd3d823ef893988e53845c4aff467cc961274a2d4"}},{"ruleId":"D17","level":"warning","message":{"text":"TodoComment: /// TODO: deprecate \u2014 source code is not a task system: move the work to your tracker and leave a reference instead (e.g. \u0060// REF: #123\u0060), so the task is planned where tasks live and the ticket links back to the code."},"locations":[{"physicalLocation":{"artifactLocation":{"uri":"tokenizers/tk-encode/src/utils/parallelism.rs"},"region":{"startLine":26}}}],"partialFingerprints":{"codehealthFindingId/v1":"cdfab37878284ac7c78d5b20085c5eaf0ef24a896cead19cd00c836a0a7d0360"}},{"ruleId":"D17","level":"warning","message":{"text":"XxxComment: // XXX: maybe we should error out when instantiating the PipelineTokenizer to avoid this \u2014 source code is not a task system: move the work to your tracker and leave a reference instead (e.g. \u0060// REF: #123\u0060), so the task is planned where tasks live and the ticket links back to the code."},"locations":[{"physicalLocation":{"artifactLocation":{"uri":"tokenizers/tk-encode/src/utils/truncation.rs"},"region":{"startLine":83}}}],"partialFingerprints":{"codehealthFindingId/v1":"0c3c9705e3a78f8579928edc0d9b4ff2d94e03afa4877fac29a510db85955d77"}},{"ruleId":"D17","level":"warning","message":{"text":"XxxComment: // XXX: this algorithm is ported from the legacy truncation code, verbatim \u2014 source code is not a task system: move the work to your tracker and leave a reference instead (e.g. \u0060// REF: #123\u0060), so the task is planned where tasks live and the ticket links back to the code."},"locations":[{"physicalLocation":{"artifactLocation":{"uri":"tokenizers/tk-encode/src/utils/truncation.rs"},"region":{"startLine":95}}}],"partialFingerprints":{"codehealthFindingId/v1":"9f353a685ac3dd17ce2d848eb29ca2bff141a4ed72310cb0ae8e02af56c0731f"}},{"ruleId":"D17","level":"warning","message":{"text":"TodoComment: // TODO: the mutable \u0060words\u0060 / \u0060offsets\u0060 accessors and \u0060get_sequence_ranges\u0060 were dropped \u2014 source code is not a task system: move the work to your tracker and leave a reference instead (e.g. \u0060// REF: #123\u0060), so the task is planned where tasks live and the ticket links back to the code."},"locations":[{"physicalLocation":{"artifactLocation":{"uri":"tokenizers/tk-encode/src/tokenizer/encoding.rs"},"region":{"startLine":168}}}],"partialFingerprints":{"codehealthFindingId/v1":"2118bc1789e6250265d2a06e4780a28797793f3c8ebf22489fd4bd577302acfb"}},{"ruleId":"D17","level":"warning","message":{"text":"TodoComment: // TODO this is suboptimal as we\u0027re doing this iteratively instead of preallocating \u2014 source code is not a task system: move the work to your tracker and leave a reference instead (e.g. \u0060// REF: #123\u0060), so the task is planned where tasks live and the ticket links back to the code."},"locations":[{"physicalLocation":{"artifactLocation":{"uri":"tokenizers/tk-encode/src/tokenizer/encoding.rs"},"region":{"startLine":396}}}],"partialFingerprints":{"codehealthFindingId/v1":"6ea686f0385914fe1225fc933a0894767be0561726fb8a7ca810ad1f6f5b8fd2"}},{"ruleId":"D17","level":"warning","message":{"text":"TodoComment: /// TODO: For RC0 this is abolutely fine. For v1, we need an enum, Token or Encoding, which can \u2014 source code is not a task system: move the work to your tracker and leave a reference instead (e.g. \u0060// REF: #123\u0060), so the task is planned where tasks live and the ticket links back to the code."},"locations":[{"physicalLocation":{"artifactLocation":{"uri":"tokenizers/tk-encode/src/tokenizer/pipeline/mod.rs"},"region":{"startLine":53}}}],"partialFingerprints":{"codehealthFindingId/v1":"eb0c0917d8b34163b2f215bc29f7a0d4b09fb9e8ae5cd39c40f3ae29903dfbbb"}},{"ruleId":"D17","level":"warning","message":{"text":"XxxComment: // XXX: \u0060Vec::new\u0060 does not allocate anything when capacity == 0, so creating empty \u2014 source code is not a task system: move the work to your tracker and leave a reference instead (e.g. \u0060// REF: #123\u0060), so the task is planned where tasks live and the ticket links back to the code."},"locations":[{"physicalLocation":{"artifactLocation":{"uri":"tokenizers/tk-encode/src/tokenizer/pipeline/mod.rs"},"region":{"startLine":475}}}],"partialFingerprints":{"codehealthFindingId/v1":"4afedf1ec31c838c9f98767aff07fe0760435e72c0bdc158f5251058e5c127e0"}},{"ruleId":"D17","level":"warning","message":{"text":"TodoComment: /// TODO: benchmark with and without to validate usefulness \u2014 source code is not a task system: move the work to your tracker and leave a reference instead (e.g. \u0060// REF: #123\u0060), so the task is planned where tasks live and the ticket links back to the code."},"locations":[{"physicalLocation":{"artifactLocation":{"uri":"tokenizers/tk-encode/src/tokenizer/pipeline/parallel.rs"},"region":{"startLine":50}}}],"partialFingerprints":{"codehealthFindingId/v1":"97a38dab3e2cf1a84f6e2367b7c6f9e8dd32ff667d3d3680a6e7d432520d52f7"}},{"ruleId":"D17","level":"warning","message":{"text":"TodoComment: // TODO: safety doc \u2014 source code is not a task system: move the work to your tracker and leave a reference instead (e.g. \u0060// REF: #123\u0060), so the task is planned where tasks live and the ticket links back to the code."},"locations":[{"physicalLocation":{"artifactLocation":{"uri":"tokenizers/tk-encode/src/tokenizer/pipeline/parallel.rs"},"region":{"startLine":67}}}],"partialFingerprints":{"codehealthFindingId/v1":"4782c446b884632d37b8b7917776e37c8de226e77b40fc51dba1263148be4d45"}},{"ruleId":"D17","level":"warning","message":{"text":"FixmeComment: // FIXME: do we want to add an \u0060offset\u0060 param to \u0060pre_tokenize\u0060 so we don\u0027t have to \u2014 source code is not a task system: move the work to your tracker and leave a reference instead (e.g. \u0060// REF: #123\u0060), so the task is planned where tasks live and the ticket links back to the code."},"locations":[{"physicalLocation":{"artifactLocation":{"uri":"tokenizers/tk-encode/src/pre_tokenizers/sequence.rs"},"region":{"startLine":117}}}],"partialFingerprints":{"codehealthFindingId/v1":"3ec63644b6f48fec5bbef4413dd37de1f4192016ff4fdf7e9440549e556768e8"}},{"ruleId":"D17","level":"warning","message":{"text":"TodoComment: // TODO: add xnli test: \u2014 source code is not a task system: move the work to your tracker and leave a reference instead (e.g. \u0060// REF: #123\u0060), so the task is planned where tasks live and the ticket links back to the code."},"locations":[{"physicalLocation":{"artifactLocation":{"uri":"tokenizers/tk-encode/src/pre_tokenizers/whitespace.rs"},"region":{"startLine":212}}}],"partialFingerprints":{"codehealthFindingId/v1":"e7c198cc40f46599d1f2be4d8e1035c8bf36b15dd343658bc54270a24ad69161"}},{"ruleId":"D17","level":"warning","message":{"text":"TodoComment: // TODO Maybe use Ordered Floats (https://docs.rs/ordered-float/1.0.2/ordered_float/)"},"locations":[{"physicalLocation":{"artifactLocation":{"uri":"tokenizers/tk-encode/src/models/unigram/lattice.rs"},"region":{"startLine":39}}}],"partialFingerprints":{"codehealthFindingId/v1":"3888303876a88eaac3edef369f14c43ba87834cbb1608e3ec2101afc8e0fafab"}},{"ruleId":"D17","level":"warning","message":{"text":"TodoComment: // TODO can we remove this clone ? \u2014 source code is not a task system: move the work to your tracker and leave a reference instead (e.g. \u0060// REF: #123\u0060), so the task is planned where tasks live and the ticket links back to the code."},"locations":[{"physicalLocation":{"artifactLocation":{"uri":"tokenizers/tk-encode/src/models/unigram/lattice.rs"},"region":{"startLine":203}}}],"partialFingerprints":{"codehealthFindingId/v1":"c147adb655bafd56cd25ba0058b910da8af0b29262ac03b0b500a8816a75ae50"}},{"ruleId":"D17","level":"warning","message":{"text":"XxxComment: // XXX: In sentence piece this is not tested, still incomplete ? \u2014 source code is not a task system: move the work to your tracker and leave a reference instead (e.g. \u0060// REF: #123\u0060), so the task is planned where tasks live and the ticket links back to the code."},"locations":[{"physicalLocation":{"artifactLocation":{"uri":"tokenizers/tk-encode/src/models/unigram/lattice.rs"},"region":{"startLine":608}}}],"partialFingerprints":{"codehealthFindingId/v1":"fcf481403cfe5e8929d3a130cacf3602a808ddbe3958cbe39d6d857fe3ac04fd"}},{"ruleId":"D17","level":"warning","message":{"text":"TodoComment: // TODO: check for collisions. \u2014 source code is not a task system: move the work to your tracker and leave a reference instead (e.g. \u0060// REF: #123\u0060), so the task is planned where tasks live and the ticket links back to the code."},"locations":[{"physicalLocation":{"artifactLocation":{"uri":"tokenizers/tk-encode/src/models/bpe/pair_map.rs"},"region":{"startLine":48}}}],"partialFingerprints":{"codehealthFindingId/v1":"011af6c9246ee380216a52af7ec94a10f7a9577635fb82f31c2a336a7c665ed8"}},{"ruleId":"D17","level":"warning","message":{"text":"TodoComment: // TODO: For now we keep serializing the old \u0022printable\u0022 chars, but this costs nothing to \u2014 source code is not a task system: move the work to your tracker and leave a reference instead (e.g. \u0060// REF: #123\u0060), so the task is planned where tasks live and the ticket links back to the code."},"locations":[{"physicalLocation":{"artifactLocation":{"uri":"tokenizers/tk-encode/src/models/bpe/serialization.rs"},"region":{"startLine":71}}}],"partialFingerprints":{"codehealthFindingId/v1":"6d417c8da7d6ab332498339c9de1a35670f9ffe8bb9202c8a5a681fe2aea9d27"}},{"ruleId":"D17","level":"warning","message":{"text":"TodoComment: // TODO: depending on how we implement the byte level post processor we might have to re implement that \u2014 source code is not a task system: move the work to your tracker and leave a reference instead (e.g. \u0060// REF: #123\u0060), so the task is planned where tasks live and the ticket links back to the code."},"locations":[{"physicalLocation":{"artifactLocation":{"uri":"tokenizers/tk-convert/src/convert.rs"},"region":{"startLine":764}}}],"partialFingerprints":{"codehealthFindingId/v1":"d20f4fa8009ee545c8922b0789bcb23a07aa53f1426f6f3d2efbeb8b3c15b1d3"}},{"ruleId":"D17","level":"warning","message":{"text":"XxxComment: // | xxx | U\u002B range  | scripts                                            | \u2014 source code is not a task system: move the work to your tracker and leave a reference instead (e.g. \u0060// REF: #123\u0060), so the task is planned where tasks live and the ticket links back to the code."},"locations":[{"physicalLocation":{"artifactLocation":{"uri":"tokenizers/bitcannon/src/classify/neon.rs"},"region":{"startLine":128}}}],"partialFingerprints":{"codehealthFindingId/v1":"374d3f25c4c0c88ca910b83dac72d64ec21f657c05b47196c236ead72de31a10"}},{"ruleId":"D17","level":"warning","message":{"text":"TodoComment: // TODO: bind trainers \u2014 source code is not a task system: move the work to your tracker and leave a reference instead (e.g. \u0060// REF: #123\u0060), so the task is planned where tasks live and the ticket links back to the code."},"locations":[{"physicalLocation":{"artifactLocation":{"uri":"bindings/python/src/lib.rs"},"region":{"startLine":54}}}],"partialFingerprints":{"codehealthFindingId/v1":"a07a7c1b799eda5ac79f4107deb767d3c02b611f3b9c91d7bb47574cac9c68fc"}},{"ruleId":"D17","level":"warning","message":{"text":"TodoComment: // TODO: Stride is not implemented yet \u2014 source code is not a task system: move the work to your tracker and leave a reference instead (e.g. \u0060// REF: #123\u0060), so the task is planned where tasks live and the ticket links back to the code."},"locations":[{"physicalLocation":{"artifactLocation":{"uri":"bindings/python/src/options/truncation.rs"},"region":{"startLine":142}}}],"partialFingerprints":{"codehealthFindingId/v1":"d1b0bf964e36d0203aa63fd56e714a49761a339dccafa85d261c1d0a0e4de5d8"}},{"ruleId":"D20","level":"note","message":{"text":"No ADRs found: No ADRs found. No recognised ADR directory (\u0060docs/adr/\u0060, \u0060docs/decisions/\u0060, \u0060adr/\u0060, \u0060docs/rfcs/\u0060, an \u0060ADR0001/\u0060 folder, or their siblings) exists anywhere in this tree. What was searched, so you can tell an empty log from a search that missed one: every directory under the tree (build output, dependencies and VCS metadata excepted), for a document that is either any non-index page inside a recognised ADR directory, whatever its name and however deeply nested (\u0060docs/adr/use-postgres.md\u0060, \u0060docs/adr/2024/0001-x.md\u0060); or a file anywhere whose name is ADR-shaped (\u00600001-use-postgres.md\u0060, \u0060adr-012-caching.md\u0060); or, when neither turned anything up, a document carrying the decision-record signature (an \u0022Architecture Decision Record\u0022 heading, or Status / Context / Decision / Consequences as section headings). A decision log that clears none of these \u2014 unnumbered files outside any recognised directory, without those headings \u2014 is not seen by this check and this row is then wrong. If that is your case, say so rather than renaming anything; otherwise, consider recording architectural decisions in \u0060docs/adr/\u0060. ADRs are expected here because \u0060tokenizers/bitmap_gen/src/main.rs\u0060 makes this repository a deployable product (line 1)."},"locations":[{"physicalLocation":{"artifactLocation":{"uri":"tokenizers/bitmap_gen/src/main.rs"},"region":{"startLine":1}}}],"partialFingerprints":{"codehealthFindingId/v1":"75fe48f12eb7e9ff188e96313af867b2fa906ccbe9af70e882ed924c412a8414"}},{"ruleId":"D26","level":"note","message":{"text":"Projects may be oversized for their cohesion: 1 of 11 project(s) overshoot their size bounds, lowering Project Cohesion to 8.2/10. The most over is \u0060tokenizers/tk-encode\u0060 (20573 LoC, 168 module-visible types across 13 directories). Review these for cohesion \u2014 draw the boundary inside the module first (group each responsibility into its own package or directory and keep the cross-boundary members non-public), since splitting a published package moves types between packages and breaks consumers."},"locations":[],"partialFingerprints":{"codehealthFindingId/v1":"d5a94650dc74886a0f1f08eb9fb6775f1fc395279ef7e901c970c9d26f767a72"}},{"ruleId":"D29","level":"error","message":{"text":"A security finding was recorded here. Its details are withheld on the public artifact \u2014 ask the repository owner for the full report."},"partialFingerprints":{"codehealthFindingId/v1":"30d3d3364e0adb72c6d5a0890cf6df7a20b3f5e559458708c7fc3c186b4467a9"},"taxa":[{"id":"CWE-1357","toolComponent":{"name":"CWE","guid":"c3a2b1d0-7f3e-4b2a-9c1d-5e6f7a8b9c0d"}},{"id":"CWE-829","toolComponent":{"name":"CWE","guid":"c3a2b1d0-7f3e-4b2a-9c1d-5e6f7a8b9c0d"}},{"id":"A03:2025","toolComponent":{"name":"OWASP Top 10","guid":"7d1f0c2e-4b6a-4e39-9a51-0b2e8c3d5f61"}}]},{"ruleId":"D29","level":"error","message":{"text":"A security finding was recorded here. Its details are withheld on the public artifact \u2014 ask the repository owner for the full report."},"partialFingerprints":{"codehealthFindingId/v1":"51ad6aceba9104f00cff51dc5f69d79881f20548d1705d9966a5378552e9d34f"},"taxa":[{"id":"CWE-1357","toolComponent":{"name":"CWE","guid":"c3a2b1d0-7f3e-4b2a-9c1d-5e6f7a8b9c0d"}},{"id":"CWE-829","toolComponent":{"name":"CWE","guid":"c3a2b1d0-7f3e-4b2a-9c1d-5e6f7a8b9c0d"}},{"id":"A03:2025","toolComponent":{"name":"OWASP Top 10","guid":"7d1f0c2e-4b6a-4e39-9a51-0b2e8c3d5f61"}}]},{"ruleId":"D29","level":"error","message":{"text":"A security finding was recorded here. Its details are withheld on the public artifact \u2014 ask the repository owner for the full report."},"partialFingerprints":{"codehealthFindingId/v1":"440e5cace23c87b0c5536014492969a0e288526595b54ccfaa8fe21accd48044"},"taxa":[{"id":"CWE-1357","toolComponent":{"name":"CWE","guid":"c3a2b1d0-7f3e-4b2a-9c1d-5e6f7a8b9c0d"}},{"id":"CWE-829","toolComponent":{"name":"CWE","guid":"c3a2b1d0-7f3e-4b2a-9c1d-5e6f7a8b9c0d"}},{"id":"A03:2025","toolComponent":{"name":"OWASP Top 10","guid":"7d1f0c2e-4b6a-4e39-9a51-0b2e8c3d5f61"}}]},{"ruleId":"D29","level":"error","message":{"text":"A security finding was recorded here. Its details are withheld on the public artifact \u2014 ask the repository owner for the full report."},"partialFingerprints":{"codehealthFindingId/v1":"abf5ef05ea2b2ec3796d96d1f73facde48814ccf8e55842f4a6fec85062837cb"},"taxa":[{"id":"CWE-1357","toolComponent":{"name":"CWE","guid":"c3a2b1d0-7f3e-4b2a-9c1d-5e6f7a8b9c0d"}},{"id":"CWE-829","toolComponent":{"name":"CWE","guid":"c3a2b1d0-7f3e-4b2a-9c1d-5e6f7a8b9c0d"}},{"id":"A03:2025","toolComponent":{"name":"OWASP Top 10","guid":"7d1f0c2e-4b6a-4e39-9a51-0b2e8c3d5f61"}}]},{"ruleId":"D29","level":"error","message":{"text":"A security finding was recorded here. Its details are withheld on the public artifact \u2014 ask the repository owner for the full report."},"partialFingerprints":{"codehealthFindingId/v1":"c5b4b8ff37d3729b7bf429899534d10940deeb1a73086c19491c5cbbee8b2f78"},"taxa":[{"id":"CWE-1357","toolComponent":{"name":"CWE","guid":"c3a2b1d0-7f3e-4b2a-9c1d-5e6f7a8b9c0d"}},{"id":"CWE-829","toolComponent":{"name":"CWE","guid":"c3a2b1d0-7f3e-4b2a-9c1d-5e6f7a8b9c0d"}},{"id":"A03:2025","toolComponent":{"name":"OWASP Top 10","guid":"7d1f0c2e-4b6a-4e39-9a51-0b2e8c3d5f61"}}]},{"ruleId":"D29","level":"error","message":{"text":"A security finding was recorded here. Its details are withheld on the public artifact \u2014 ask the repository owner for the full report."},"partialFingerprints":{"codehealthFindingId/v1":"af863c25c980c87beb30026456c1bb295645782bb5e0803416fe5d9040d0ab9d"},"taxa":[{"id":"CWE-1357","toolComponent":{"name":"CWE","guid":"c3a2b1d0-7f3e-4b2a-9c1d-5e6f7a8b9c0d"}},{"id":"CWE-829","toolComponent":{"name":"CWE","guid":"c3a2b1d0-7f3e-4b2a-9c1d-5e6f7a8b9c0d"}},{"id":"A03:2025","toolComponent":{"name":"OWASP Top 10","guid":"7d1f0c2e-4b6a-4e39-9a51-0b2e8c3d5f61"}}]},{"ruleId":"D29","level":"error","message":{"text":"A security finding was recorded here. Its details are withheld on the public artifact \u2014 ask the repository owner for the full report."},"partialFingerprints":{"codehealthFindingId/v1":"4f1e445640c7500fcb9d12ef5e70943d6e3cb86e79ad86253ba3836fa946c132"},"taxa":[{"id":"CWE-1357","toolComponent":{"name":"CWE","guid":"c3a2b1d0-7f3e-4b2a-9c1d-5e6f7a8b9c0d"}},{"id":"CWE-829","toolComponent":{"name":"CWE","guid":"c3a2b1d0-7f3e-4b2a-9c1d-5e6f7a8b9c0d"}},{"id":"A03:2025","toolComponent":{"name":"OWASP Top 10","guid":"7d1f0c2e-4b6a-4e39-9a51-0b2e8c3d5f61"}}]},{"ruleId":"D29","level":"error","message":{"text":"A security finding was recorded here. Its details are withheld on the public artifact \u2014 ask the repository owner for the full report."},"partialFingerprints":{"codehealthFindingId/v1":"785b728bc912cb44d81311b6180adf536195891799dbf704a746b3173fd3d446"},"taxa":[{"id":"CWE-829","toolComponent":{"name":"CWE","guid":"c3a2b1d0-7f3e-4b2a-9c1d-5e6f7a8b9c0d"}},{"id":"A08:2025","toolComponent":{"name":"OWASP Top 10","guid":"7d1f0c2e-4b6a-4e39-9a51-0b2e8c3d5f61"}}]},{"ruleId":"D29","level":"error","message":{"text":"A security finding was recorded here. Its details are withheld on the public artifact \u2014 ask the repository owner for the full report."},"partialFingerprints":{"codehealthFindingId/v1":"9469c6a6e483b017aea907e18f31461648588f7698fca6492010d5f9b5f91f82"},"taxa":[{"id":"CWE-829","toolComponent":{"name":"CWE","guid":"c3a2b1d0-7f3e-4b2a-9c1d-5e6f7a8b9c0d"}},{"id":"A08:2025","toolComponent":{"name":"OWASP Top 10","guid":"7d1f0c2e-4b6a-4e39-9a51-0b2e8c3d5f61"}}]},{"ruleId":"D29","level":"warning","message":{"text":"A security finding was recorded here. Its details are withheld on the public artifact \u2014 ask the repository owner for the full report."},"partialFingerprints":{"codehealthFindingId/v1":"d06c9590f131d6fb78af609bbc6f97bf12eb106f494890740c07f13585c5d5bf"},"taxa":[{"id":"CWE-1104","toolComponent":{"name":"CWE","guid":"c3a2b1d0-7f3e-4b2a-9c1d-5e6f7a8b9c0d"}},{"id":"CWE-494","toolComponent":{"name":"CWE","guid":"c3a2b1d0-7f3e-4b2a-9c1d-5e6f7a8b9c0d"}},{"id":"A03:2025","toolComponent":{"name":"OWASP Top 10","guid":"7d1f0c2e-4b6a-4e39-9a51-0b2e8c3d5f61"}},{"id":"CICD-SEC-3","toolComponent":{"name":"OWASP Top 10 CI/CD Security Risks","guid":"3c9b2a6e-5d14-4f7b-8e02-6a1d9c4b7f35"}}]},{"ruleId":"D29","level":"warning","message":{"text":"A security finding was recorded here. Its details are withheld on the public artifact \u2014 ask the repository owner for the full report."},"partialFingerprints":{"codehealthFindingId/v1":"56a1dbda958a23d3e4fc73603f9319d71d8ae91fee177e037bae5ba0c7f371c9"},"taxa":[{"id":"CWE-1104","toolComponent":{"name":"CWE","guid":"c3a2b1d0-7f3e-4b2a-9c1d-5e6f7a8b9c0d"}},{"id":"CWE-494","toolComponent":{"name":"CWE","guid":"c3a2b1d0-7f3e-4b2a-9c1d-5e6f7a8b9c0d"}},{"id":"A03:2025","toolComponent":{"name":"OWASP Top 10","guid":"7d1f0c2e-4b6a-4e39-9a51-0b2e8c3d5f61"}},{"id":"CICD-SEC-3","toolComponent":{"name":"OWASP Top 10 CI/CD Security Risks","guid":"3c9b2a6e-5d14-4f7b-8e02-6a1d9c4b7f35"}}]},{"ruleId":"D29","level":"warning","message":{"text":"A security finding was recorded here. Its details are withheld on the public artifact \u2014 ask the repository owner for the full report."},"partialFingerprints":{"codehealthFindingId/v1":"56b1681e2810cbe59edd5cf3f07baacfbdad9705945f427ff434a3501db4ebe1"},"taxa":[{"id":"CWE-1104","toolComponent":{"name":"CWE","guid":"c3a2b1d0-7f3e-4b2a-9c1d-5e6f7a8b9c0d"}},{"id":"CWE-494","toolComponent":{"name":"CWE","guid":"c3a2b1d0-7f3e-4b2a-9c1d-5e6f7a8b9c0d"}},{"id":"A03:2025","toolComponent":{"name":"OWASP Top 10","guid":"7d1f0c2e-4b6a-4e39-9a51-0b2e8c3d5f61"}},{"id":"CICD-SEC-3","toolComponent":{"name":"OWASP Top 10 CI/CD Security Risks","guid":"3c9b2a6e-5d14-4f7b-8e02-6a1d9c4b7f35"}}]},{"ruleId":"D29","level":"warning","message":{"text":"A security finding was recorded here. Its details are withheld on the public artifact \u2014 ask the repository owner for the full report."},"partialFingerprints":{"codehealthFindingId/v1":"c5c7437e9cb4985d66f310538dd251c7d8dca49cadd5c23c1007e09c66b9544e"},"taxa":[{"id":"CWE-1104","toolComponent":{"name":"CWE","guid":"c3a2b1d0-7f3e-4b2a-9c1d-5e6f7a8b9c0d"}},{"id":"CWE-494","toolComponent":{"name":"CWE","guid":"c3a2b1d0-7f3e-4b2a-9c1d-5e6f7a8b9c0d"}},{"id":"A03:2025","toolComponent":{"name":"OWASP Top 10","guid":"7d1f0c2e-4b6a-4e39-9a51-0b2e8c3d5f61"}},{"id":"CICD-SEC-3","toolComponent":{"name":"OWASP Top 10 CI/CD Security Risks","guid":"3c9b2a6e-5d14-4f7b-8e02-6a1d9c4b7f35"}}]},{"ruleId":"D29","level":"warning","message":{"text":"A security finding was recorded here. Its details are withheld on the public artifact \u2014 ask the repository owner for the full report."},"partialFingerprints":{"codehealthFindingId/v1":"4bcde0de4e53537dedc11a6c284ca25b982b6ea42852c13755839ab6fc28ef8d"},"taxa":[{"id":"CWE-1104","toolComponent":{"name":"CWE","guid":"c3a2b1d0-7f3e-4b2a-9c1d-5e6f7a8b9c0d"}},{"id":"CWE-494","toolComponent":{"name":"CWE","guid":"c3a2b1d0-7f3e-4b2a-9c1d-5e6f7a8b9c0d"}},{"id":"A03:2025","toolComponent":{"name":"OWASP Top 10","guid":"7d1f0c2e-4b6a-4e39-9a51-0b2e8c3d5f61"}},{"id":"CICD-SEC-3","toolComponent":{"name":"OWASP Top 10 CI/CD Security Risks","guid":"3c9b2a6e-5d14-4f7b-8e02-6a1d9c4b7f35"}}]},{"ruleId":"D30","level":"warning","message":{"text":"A security finding was recorded here. Its details are withheld on the public artifact \u2014 ask the repository owner for the full report."},"partialFingerprints":{"codehealthFindingId/v1":"ea213ca06ecfe49655146ef510aa1fe6804b000c75cc8c1073079f0151317840"},"properties":{"dependency":{"package":"quinn-udp","version":"0.5.15","advisory":"RUSTSEC-2026-0538","aliases":["[GHSA redacted]"],"reachability":{"kind":"unknown"}}}},{"ruleId":"D30","level":"note","message":{"text":"A security finding was recorded here. Its details are withheld on the public artifact \u2014 ask the repository owner for the full report."},"partialFingerprints":{"codehealthFindingId/v1":"7a49ee4ec0e87d2575df84b84ab1daf2f33dc318eaee136fb793dc5abe1a9671"},"properties":{"dependency":{"package":"fxhash","version":"0.2.1","advisory":"RUSTSEC-2025-0057","reachability":{"kind":"unknown"}}}},{"ruleId":"D30","level":"note","message":{"text":"A security finding was recorded here. Its details are withheld on the public artifact \u2014 ask the repository owner for the full report."},"partialFingerprints":{"codehealthFindingId/v1":"3d14ed4afa973c37d3f0deafa50c7bbd0c50fc7834b73277beceeef3acd854d5"},"properties":{"dependency":{"package":"fxhash","version":"0.2.1","advisory":"RUSTSEC-2025-0057","reachability":{"kind":"unknown"}}}},{"ruleId":"D30","level":"note","message":{"text":"A security finding was recorded here. Its details are withheld on the public artifact \u2014 ask the repository owner for the full report."},"partialFingerprints":{"codehealthFindingId/v1":"00f206ee3522b4e36296b9e0b74ad767dd38c516c4864f56c67c6d186c942945"},"properties":{"dependency":{"package":"fxhash","version":"0.2.1","advisory":"RUSTSEC-2025-0057","reachability":{"kind":"unknown"}}}},{"ruleId":"D30","level":"note","message":{"text":"A security finding was recorded here. Its details are withheld on the public artifact \u2014 ask the repository owner for the full report."},"partialFingerprints":{"codehealthFindingId/v1":"6f87666fef4813246b99a0bc53884d988375e0def9d2fda78ede7845b7c37c22"},"properties":{"dependency":{"package":"fxhash","version":"0.2.1","advisory":"RUSTSEC-2025-0057","reachability":{"kind":"unknown"}}}},{"ruleId":"D30","level":"note","message":{"text":"A security finding was recorded here. Its details are withheld on the public artifact \u2014 ask the repository owner for the full report."},"partialFingerprints":{"codehealthFindingId/v1":"fadbaedd898c698ac1a459242fc24ff5ecf62ed1840c73399a24a2f975ff6cfb"},"properties":{"dependency":{"package":"paste","version":"1.0.15","advisory":"RUSTSEC-2024-0436","reachability":{"kind":"unknown"}}}},{"ruleId":"D34","level":"note","message":{"text":"Orphaned files with no living knowledge: 3 of 130 analysed file(s) have no living knowledge left \u2014 their last meaningful change has decayed away, so if one breaks, no one currently understands it (counted over production source files of roughly 2,400 bytes or more, excluding vendored, generated, example/demo and scaffolding-template trees and test files identified by path convention, largest first; 130 of the 184 production source files in this repository met that bar). None is large enough to earn a read-through of its own, so this row stands in for the per-file rows rather than raising one each \u2014 most significant first: tokenizers/tk-encode/src/utils/cache.rs, tokenizers/tk-encode/src/decoders/byte_fallback.rs, tokenizers/tk-encode/src/utils/iter.rs. Attach the read to the next change that touches one of them: have a second person review that change, and leave behind a short comment or test recording what the file is for, so the knowledge comes back at the cost of a change you were making anyway."},"locations":[],"partialFingerprints":{"codehealthFindingId/v1":"ce861fd78f6935114d3a342cb90ad25870f984e1042954fcbbe27c0e23be3658"}},{"ruleId":"D34","level":"note","message":{"text":"Orphaned file: No one still active in this repository holds living knowledge of this file: everyone who made a meaningful change to it has gone quiet, or what the current team did to it (moves, reformatting, log-level or import edits, renames) never needed understanding it. If it breaks, no one currently understands it. Attach the read to the next change that touches it: have a second person review that change, and leave behind a short comment or test recording what the file is for."},"locations":[{"physicalLocation":{"artifactLocation":{"uri":"tokenizers/tk-encode/src/utils/cache.rs"},"region":{"startLine":1}}}],"partialFingerprints":{"codehealthFindingId/v1":"6b7086425c5c5ccb04bdd24316b744536c8506b3632969913fc421024d72982e"}},{"ruleId":"D34","level":"note","message":{"text":"Orphaned file: No one still active in this repository holds living knowledge of this file: everyone who made a meaningful change to it has gone quiet, or what the current team did to it (moves, reformatting, log-level or import edits, renames) never needed understanding it. If it breaks, no one currently understands it. Attach the read to the next change that touches it: have a second person review that change, and leave behind a short comment or test recording what the file is for."},"locations":[{"physicalLocation":{"artifactLocation":{"uri":"tokenizers/tk-encode/src/decoders/byte_fallback.rs"},"region":{"startLine":1}}}],"partialFingerprints":{"codehealthFindingId/v1":"9e9ab8c0c348cfa6226fbf6ce737589abbbef1daf00ffd596426a39ce67481b7"}},{"ruleId":"D34","level":"note","message":{"text":"Orphaned file: No one still active in this repository holds living knowledge of this file: everyone who made a meaningful change to it has gone quiet, or what the current team did to it (moves, reformatting, log-level or import edits, renames) never needed understanding it. If it breaks, no one currently understands it. Attach the read to the next change that touches it: have a second person review that change, and leave behind a short comment or test recording what the file is for."},"locations":[{"physicalLocation":{"artifactLocation":{"uri":"tokenizers/tk-encode/src/utils/iter.rs"},"region":{"startLine":1}}}],"partialFingerprints":{"codehealthFindingId/v1":"8672436f834279c07a6ba4f1ec01552bbb367a95cebec1549fb60d6f29d3bff0"}},{"ruleId":"D35","level":"warning","message":{"text":"Change coupling: model.rs \u2194 cache.rs: \u0060tokenizers/tk-encode/src/models/bpe/model.rs\u0060 and \u0060tokenizers/tk-encode/src/utils/cache.rs\u0060 change together 76% of the time (13 of the 17 commits that touched whichever of the two files changed less often, counting a file under its earlier names as well \u2014 a repo-wide or module-wide sweep is evidence about the sweep rather than about any pair inside it and is left out of BOTH sides of this ratio, while a dependency bump, a formatter/rename sweep, or a commit whose edit to one of the two files was a tool directive such as //go:generate or whitespace only is left out of the shared count ONLY, so the two sides are not taken over identical commit sets) with no explicit dependency \u2014 the edge is real but nothing declares it. Read the pair before acting: if one registers itself into the other through a hook or an initialiser, the missing dependency is DELIBERATE \u2014 the registration is the link, and it is meant not to be an import \u2014 and the thing to add is a comment on each side naming the other, not a merge; if they simply belong together, co-locate them; if neither holds, the coupling is hidden and worth breaking. You can check this without leaving the row: of the 13 shared commits counted here, the most recent 3 are \u0060af5a3e3a\u0060 perf(bpe): WordCache (#2262); \u00605aa9f6cf\u0060 Disable caching for long strings. (#1676) (at that commit the files were still \u0060tokenizers/src/models/bpe/model.rs\u0060 and \u0060tokenizers/src/utils/cache.rs\u0060); \u0060c6b5c3ea\u0060 More cache options. (#1675) (at that commit the files were still \u0060tokenizers/src/models/bpe/model.rs\u0060 and \u0060tokenizers/src/utils/cache.rs\u0060) \u2014 run \u0060git show\u0060 on any of them."},"locations":[{"physicalLocation":{"artifactLocation":{"uri":"tokenizers/tk-encode/src/models/bpe/model.rs"},"region":{"startLine":1}}}],"partialFingerprints":{"codehealthFindingId/v1":"3c37b5c881d658c64f623db93933ae1b735d64ef729d34a1fb957783c6fcbb11"}},{"ruleId":"D35","level":"warning","message":{"text":"Change coupling: unigram.rs \u2194 wordlevel.rs: \u0060tokenizers/tk-train/src/trainers/unigram.rs\u0060 and \u0060tokenizers/tk-train/src/trainers/wordlevel.rs\u0060 change together 54% of the time (7 of the 13 commits that touched whichever of the two files changed less often, counting a file under its earlier names as well \u2014 a repo-wide or module-wide sweep is evidence about the sweep rather than about any pair inside it and is left out of BOTH sides of this ratio, while a dependency bump, a formatter/rename sweep, or a commit whose edit to one of the two files was a tool directive such as //go:generate or whitespace only is left out of the shared count ONLY, so the two sides are not taken over identical commit sets) with no explicit dependency between them. They sit in the same directory, but in this ecosystem each file is its own module \u2014 a sibling reference still needs an import \u2014 so the missing import edge is real: the coupling runs through shared behaviour, not a declared dependency. If they duplicate structure, extract the common part into one unit; otherwise the coupling is hidden and worth breaking. You can check this without leaving the row: of the 7 shared commits counted here, the most recent 3 are \u006015c2e22b\u0060 tk-train: fmt, dead code, stale docs, drop unused tk-convert dep (#2473); \u0060abbd2d32\u0060 ci: update CI after encode/train split (#2125); \u006076abe0f7\u0060 Bpe clones (#1707) (at that commit the files were still \u0060tokenizers/src/models/unigram/trainer.rs\u0060 and \u0060tokenizers/src/models/wordlevel/trainer.rs\u0060) \u2014 run \u0060git show\u0060 on any of them."},"locations":[{"physicalLocation":{"artifactLocation":{"uri":"tokenizers/tk-train/src/trainers/unigram.rs"},"region":{"startLine":1}}}],"partialFingerprints":{"codehealthFindingId/v1":"3e05ebcd7730d9456f29317ff49104a06f77f31e805a203782b636cd1bab127b"}},{"ruleId":"D35","level":"warning","message":{"text":"Change coupling: model.rs \u2194 word.rs: \u0060tokenizers/tk-encode/src/models/bpe/model.rs\u0060 and \u0060tokenizers/tk-train/src/trainers/bpe/word.rs\u0060 change together 52% of the time (11 of the 21 commits that touched whichever of the two files changed less often, counting a file under its earlier names as well \u2014 a repo-wide or module-wide sweep is evidence about the sweep rather than about any pair inside it and is left out of BOTH sides of this ratio, while a dependency bump, a formatter/rename sweep, or a commit whose edit to one of the two files was a tool directive such as //go:generate or whitespace only is left out of the shared count ONLY, so the two sides are not taken over identical commit sets) with no explicit dependency \u2014 the edge is real but nothing declares it. Read the pair before acting: if one registers itself into the other through a hook or an initialiser, the missing dependency is DELIBERATE \u2014 the registration is the link, and it is meant not to be an import \u2014 and the thing to add is a comment on each side naming the other, not a merge; if they simply belong together, co-locate them; if neither holds, the coupling is hidden and worth breaking. You can check this without leaving the row: of the 11 shared commits counted here, the most recent 3 are \u0060af5a3e3a\u0060 perf(bpe): WordCache (#2262) (at that commit the file was still \u0060tokenizers/tk-encode/src/models/bpe/word.rs\u0060); \u0060077130b7\u0060 perf(scratch): reuse Scratch buffers across encode() calls (#2261) (at that commit the file was still \u0060tokenizers/tk-encode/src/models/bpe/word.rs\u0060); \u00602c4f6100\u0060 no alloc model, follow-ups (#2175) (at that commit the file was still \u0060tokenizers/tk-encode/src/models/bpe/word.rs\u0060) \u2014 run \u0060git show\u0060 on any of them."},"locations":[{"physicalLocation":{"artifactLocation":{"uri":"tokenizers/tk-encode/src/models/bpe/model.rs"},"region":{"startLine":1}}}],"partialFingerprints":{"codehealthFindingId/v1":"6122200a9cd0d04a9a29d557b314afbed7d52789f9b58251daf438565322a55b"}},{"ruleId":"D36","level":"note","message":{"text":"A security finding was recorded here. Its details are withheld on the public artifact \u2014 ask the repository owner for the full report."},"partialFingerprints":{"codehealthFindingId/v1":"eb00976a698a5d68999b7ee6d27206fd374e916853e7ff1ead5eed42386f043f"},"taxa":[{"id":"CICD-SEC-9","toolComponent":{"name":"OWASP Top 10 CI/CD Security Risks","guid":"3c9b2a6e-5d14-4f7b-8e02-6a1d9c4b7f35"}}]},{"ruleId":"D36","level":"warning","message":{"text":"A security finding was recorded here. Its details are withheld on the public artifact \u2014 ask the repository owner for the full report."},"partialFingerprints":{"codehealthFindingId/v1":"c4f4744d19c0bd2de165e075b0f17889675cdbc3674c6d77b4ae7b4441578703"}},{"ruleId":"D36","level":"warning","message":{"text":"A security finding was recorded here. Its details are withheld on the public artifact \u2014 ask the repository owner for the full report."},"partialFingerprints":{"codehealthFindingId/v1":"9db5c60d4084fc4489fda2c0b08e0f9f9eadaf1bef38325f94036ff8f23fd5e4"}},{"ruleId":"D36","level":"warning","message":{"text":"A security finding was recorded here. Its details are withheld on the public artifact \u2014 ask the repository owner for the full report."},"partialFingerprints":{"codehealthFindingId/v1":"f75dd61fbc28fd905f770db3d1f7eb0358d9be28694424c2916fc17880d890c5"},"taxa":[{"id":"CICD-SEC-6","toolComponent":{"name":"OWASP Top 10 CI/CD Security Risks","guid":"3c9b2a6e-5d14-4f7b-8e02-6a1d9c4b7f35"}}]},{"ruleId":"D36","level":"warning","message":{"text":"A security finding was recorded here. Its details are withheld on the public artifact \u2014 ask the repository owner for the full report."},"partialFingerprints":{"codehealthFindingId/v1":"14f87399f93ab64f19861601492d66dc37c75b7d0db6e3fbad8a564d5778afce"},"taxa":[{"id":"CICD-SEC-4","toolComponent":{"name":"OWASP Top 10 CI/CD Security Risks","guid":"3c9b2a6e-5d14-4f7b-8e02-6a1d9c4b7f35"}}]},{"ruleId":"D36","level":"note","message":{"text":"A security finding was recorded here. Its details are withheld on the public artifact \u2014 ask the repository owner for the full report."},"partialFingerprints":{"codehealthFindingId/v1":"c593d4dbd23724f337bfb8d2c1653812f143ae255bd9b59d4b3e9c37779fa6f4"},"taxa":[{"id":"CICD-SEC-3","toolComponent":{"name":"OWASP Top 10 CI/CD Security Risks","guid":"3c9b2a6e-5d14-4f7b-8e02-6a1d9c4b7f35"}}]},{"ruleId":"D44","level":"warning","message":{"text":"End-of-life runtime: Node.js 10: bindings/node/package.json declares Node.js 10 as this project\u0027s engines.node, and Node.js 10, support ended 2021-04-30. An unsupported runtime receives no security patches, so every vulnerability disclosed in it since 2021-04-30 is present and unfixable without moving off it. This is a migration rather than an upgrade: there is no newer release of a runtime that has ended."},"locations":[{"physicalLocation":{"artifactLocation":{"uri":"bindings/node/package.json"},"region":{"startLine":39}}}],"partialFingerprints":{"codehealthFindingId/v1":"b1a2cd25bd36558cc709766f11572178ef5de07cbca7425ec24d79124100be77"}},{"ruleId":"M2","level":"note","message":{"text":"No ADRs: No Architecture Decision Records found \u2014 no conventional ADR directory, no numbered \u0060NNNN-title\u0060 documents in any markup this check reads, and nothing ADR-shaped by content. Design rationale recorded elsewhere (a design-notes tree, a mailing list, pull-request discussion) is not visible to this check and is not re-findable per decision, so a future maintainer cannot ask why one choice was made and get an answer."},"locations":[],"partialFingerprints":{"codehealthFindingId/v1":"670b3d6e36a756d63097d0dfbf90afd5fc761308800b9354894a07c3f4e4aa14"}},{"ruleId":"P12","level":"warning","message":{"text":"Coverage collected but not gated: CI collects a coverage report but no step enforces a minimum \u2014 coverage could halve and CI stays green. Add a step that fails the build when coverage drops below a floor (your coverage tool\u0027s minimum-threshold flag, or a coverage-gate action) so the number guards something. What was searched, so you can tell an absence from a miss: this repository\u0027s CI files AND its coverage configuration \u2014 the well-known coverage and test-runner config files, read at the repository root and inside workspace package directories two levels down, so a floor declared beside the tests rather than in the pipeline is credited \u2014 matched against the threshold settings this check knows by name. A floor set in your coverage service\u0027s web UI rather than in a committed file, or under a setting whose name is not one of those, is not seen here."},"locations":[],"partialFingerprints":{"codehealthFindingId/v1":"7f63c06044cf998d9a0698692df30d8873e29bc9b0c98ee829255017c1193d33"}},{"ruleId":"P2","level":"note","message":{"text":"Logging is not universal: Only 2/3 runnable modules use logging (modules with no entry point or server are excluded \u2014 they are libraries a runnable module hosts). Silent: \u0060tokenizers/bitmap_gen\u0060."},"locations":[],"partialFingerprints":{"codehealthFindingId/v1":"91a94e21d6d4d0e6a2003f94505b0b02b157176f0b24c4820f3f82b4447a97fe"}},{"ruleId":"P3","level":"note","message":{"text":"Dependabot does not watch npm/cargo/pip: \u0060.github/dependabot.yml\u0060 schedules updates, but no \u0060package-ecosystem\u0060 entry covers \u0060npm\u0060, \u0060cargo\u0060, \u0060pip\u0060, which this tree depends on \u2014 so those dependencies get no update pull requests and no advisory pressure, however complete the dep-scan credit looks. Add the missing entries to the \u0060updates:\u0060 list."},"locations":[{"physicalLocation":{"artifactLocation":{"uri":".github/dependabot.yml"},"region":{"startLine":2}}}],"partialFingerprints":{"codehealthFindingId/v1":"d5cf01fbc37d3eb57109302421c9bbf92c95bcbcbaa2935183208c17149ff79d"}},{"ruleId":"P6","level":"note","message":{"text":"No changelog: No CHANGELOG/HISTORY/RELEASES file \u2014 what shipped when isn\u0027t easy to reconstruct for support or audit. (Versioning/tagging makes releases traceable, but a changelog records the what.)"},"locations":[],"partialFingerprints":{"codehealthFindingId/v1":"dda5aa5aed8cbb292c3ef2b733bc138f614293ae6426c85e98bf31ef330d9415"}},{"ruleId":"X10","level":"note","message":{"text":"Duplicated predicate: \u0060self.count == other.count \u0026\u0026 self.pair == other.pair\u0060 appears character-identically in 2 files \u2014 tokenizers/tk-train/src/trainers/bpe/mod.rs, tokenizers/tk-train/src/trainers/bpe/parity_trainer.rs. It is one line, so the duplication detector\u0027s token window never sees it; the copies drift when only one is corrected. Give the condition a name and one home."},"locations":[{"physicalLocation":{"artifactLocation":{"uri":"tokenizers/tk-train/src/trainers/bpe/mod.rs"},"region":{"startLine":36}}}],"partialFingerprints":{"codehealthFindingId/v1":"ba7f96aa644cb73c6905780918572b0d85af75f04ff69b75046a33de916ac2a7"}},{"ruleId":"X10","level":"note","message":{"text":"Duplicated predicate: \u0060self.pattern == other.pattern \u0026\u0026 self.content == other.content\u0060 appears character-identically in 2 files \u2014 tokenizers/tk-encode/src/decoders/replace.rs, tokenizers/tk-encode/src/normalizers/replace.rs. It is one line, so the duplication detector\u0027s token window never sees it; the copies drift when only one is corrected. Give the condition a name and one home."},"locations":[{"physicalLocation":{"artifactLocation":{"uri":"tokenizers/tk-encode/src/decoders/replace.rs"},"region":{"startLine":45}}}],"partialFingerprints":{"codehealthFindingId/v1":"c69be7d108eee272ee1d339b0cef28f2b302f38097d40568e57c42f2b8230ce9"}},{"ruleId":"X10","level":"note","message":{"text":"Duplicated predicate: \u0060tags.len() \u003E= ntext \u0026\u0026 starts.len() \u003E= nblk \u0026\u0026 out.len() \u003E= ntext\u0060 appears character-identically in 2 files \u2014 tokenizers/bitcannon/src/models/cl100k.rs, tokenizers/bitcannon/src/models/gpt2.rs. It is one line, so the duplication detector\u0027s token window never sees it; the copies drift when only one is corrected. Give the condition a name and one home."},"locations":[{"physicalLocation":{"artifactLocation":{"uri":"tokenizers/bitcannon/src/models/cl100k.rs"},"region":{"startLine":52}}}],"partialFingerprints":{"codehealthFindingId/v1":"414074543fb2fc958061b488541386a97bdaea1570565e2c829120c3e7ad01df"}},{"ruleId":"X7","level":"note","message":{"text":"Silent fallback default on Err: \u0060pre_tokenize\u0060 turns every \u0060Err\u0060 into \u00600\u0060 \u2014 a failure becomes a plausible value and a silent behavior change with no trail. Log the error or propagate it with \u0060?\u0060. If that constant is the correct answer rather than a stand-in for a value that could not be read, say so in a comment on the arm or in the doc comment, and the row stops firing."},"locations":[{"physicalLocation":{"artifactLocation":{"uri":"tokenizers/bitcannon/src/classes.rs"},"region":{"startLine":113}}}],"partialFingerprints":{"codehealthFindingId/v1":"0c46c97b0362cbdf4926f2c81cfd9f6ae1e37d240939bd908b4cf6fbf8803c14"}}],"taxonomies":[{"name":"CWE","guid":"c3a2b1d0-7f3e-4b2a-9c1d-5e6f7a8b9c0d","organization":"MITRE","informationUri":"https://cwe.mitre.org/","isComprehensive":false,"shortDescription":{"text":"The MITRE Common Weakness Enumeration (CWE)."},"taxa":[{"id":"CWE-1104","guid":"4c918cb5-b2a6-6c55-9963-a44ee464305e","name":"Use of Unmaintained Third Party Components","shortDescription":{"text":"Use of Unmaintained Third Party Components"},"helpUri":"https://cwe.mitre.org/data/definitions/1104.html"},{"id":"CWE-1357","guid":"e4d2e772-757e-0a5c-bd7d-77052949d866","name":"Reliance on Insufficiently Trustworthy Component","shortDescription":{"text":"Reliance on Insufficiently Trustworthy Component"},"helpUri":"https://cwe.mitre.org/data/definitions/1357.html"},{"id":"CWE-1395","guid":"800e09e7-c11a-8654-9fa6-86f398995fed","name":"Dependency on Vulnerable Third-Party Component","shortDescription":{"text":"Dependency on Vulnerable Third-Party Component"},"helpUri":"https://cwe.mitre.org/data/definitions/1395.html"},{"id":"CWE-259","guid":"ae9ad959-fbb6-9d5e-892d-3dca66da0b69","name":"Use of Hard-coded Password","shortDescription":{"text":"Use of Hard-coded Password"},"helpUri":"https://cwe.mitre.org/data/definitions/259.html"},{"id":"CWE-494","guid":"b8a65e0d-e459-4a55-a931-fc1136482375","name":"Download of Code Without Integrity Check","shortDescription":{"text":"Download of Code Without Integrity Check"},"helpUri":"https://cwe.mitre.org/data/definitions/494.html"},{"id":"CWE-506","guid":"401d6455-56e3-0552-9a39-f77461673e3f","name":"Embedded Malicious Code","shortDescription":{"text":"Embedded Malicious Code"},"helpUri":"https://cwe.mitre.org/data/definitions/506.html"},{"id":"CWE-77","guid":"332c8ade-6612-9f56-a06b-d8d90b1a8750","name":"Improper Neutralization of Special Elements used in a Command (\u0027Command Injection\u0027)","shortDescription":{"text":"Improper Neutralization of Special Elements used in a Command (\u0027Command Injection\u0027)"},"helpUri":"https://cwe.mitre.org/data/definitions/77.html"},{"id":"CWE-78","guid":"2e31ceaf-c7ae-2e5e-9661-cfb1362789cf","name":"Improper Neutralization of Special Elements used in an OS Command (\u0027OS Command Injection\u0027)","shortDescription":{"text":"Improper Neutralization of Special Elements used in an OS Command (\u0027OS Command Injection\u0027)"},"helpUri":"https://cwe.mitre.org/data/definitions/78.html"},{"id":"CWE-79","guid":"fd45580b-e8c4-fc5e-8c2f-aa8fab0b4dbf","name":"Improper Neutralization of Input During Web Page Generation (\u0027Cross-site Scripting\u0027)","shortDescription":{"text":"Improper Neutralization of Input During Web Page Generation (\u0027Cross-site Scripting\u0027)"},"helpUri":"https://cwe.mitre.org/data/definitions/79.html"},{"id":"CWE-798","guid":"5e8f057d-fee3-995a-a0cb-9fc5b0d174d1","name":"Use of Hard-coded Credentials","shortDescription":{"text":"Use of Hard-coded Credentials"},"helpUri":"https://cwe.mitre.org/data/definitions/798.html"},{"id":"CWE-829","guid":"13c33925-97fb-5a5e-b40c-56d328b8a4d7","name":"Inclusion of Functionality from Untrusted Control Sphere","shortDescription":{"text":"Inclusion of Functionality from Untrusted Control Sphere"},"helpUri":"https://cwe.mitre.org/data/definitions/829.html"},{"id":"CWE-89","guid":"6d08fdad-37eb-c150-bbf0-d7d946863407","name":"Improper Neutralization of Special Elements used in an SQL Command (\u0027SQL Injection\u0027)","shortDescription":{"text":"Improper Neutralization of Special Elements used in an SQL Command (\u0027SQL Injection\u0027)"},"helpUri":"https://cwe.mitre.org/data/definitions/89.html"},{"id":"CWE-937","guid":"16f316ae-415c-b354-a59b-1f7905f756e9","name":"Using Components with Known Vulnerabilities","shortDescription":{"text":"Using Components with Known Vulnerabilities"},"helpUri":"https://cwe.mitre.org/data/definitions/937.html"},{"id":"CWE-94","guid":"75e7f50c-6c2f-dd52-bf40-bf6c52b861fd","name":"Improper Control of Generation of Code (\u0027Code Injection\u0027)","shortDescription":{"text":"Improper Control of Generation of Code (\u0027Code Injection\u0027)"},"helpUri":"https://cwe.mitre.org/data/definitions/94.html"}]},{"name":"OWASP Top 10","guid":"7d1f0c2e-4b6a-4e39-9a51-0b2e8c3d5f61","version":"2025","organization":"OWASP Foundation","informationUri":"https://top10.owasp.org/2025/","isComprehensive":false,"shortDescription":{"text":"The OWASP Top 10:2025 web application security risks."},"taxa":[{"id":"A03:2025","guid":"168e227c-7735-2e57-bc6e-997affe644c5","name":"Software Supply Chain Failures","shortDescription":{"text":"Software Supply Chain Failures"},"helpUri":"https://top10.owasp.org/2025/A03_2025-Software_Supply_Chain_Failures/"},{"id":"A05:2025","guid":"87e86033-3231-8458-8e4a-5cb82743553a","name":"Injection","shortDescription":{"text":"Injection"},"helpUri":"https://top10.owasp.org/2025/A05_2025-Injection/"},{"id":"A07:2025","guid":"334ddd78-539d-d35c-83dd-253647eb7eaf","name":"Authentication Failures","shortDescription":{"text":"Authentication Failures"},"helpUri":"https://top10.owasp.org/2025/A07_2025-Authentication_Failures/"},{"id":"A08:2025","guid":"84b0c297-710e-9a5f-9335-936ecb260c36","name":"Software or Data Integrity Failures","shortDescription":{"text":"Software or Data Integrity Failures"},"helpUri":"https://top10.owasp.org/2025/A08_2025-Software_or_Data_Integrity_Failures/"}]},{"name":"OWASP Top 10 CI/CD Security Risks","guid":"3c9b2a6e-5d14-4f7b-8e02-6a1d9c4b7f35","organization":"OWASP Foundation","informationUri":"https://owasp.org/www-project-top-10-ci-cd-security-risks/","isComprehensive":false,"shortDescription":{"text":"The OWASP Top 10 CI/CD Security Risks."},"taxa":[{"id":"CICD-SEC-3","guid":"458c83a6-ebf7-cf55-8cc3-b4d949bbf271","name":"Dependency Chain Abuse","shortDescription":{"text":"Dependency Chain Abuse"},"helpUri":"https://owasp.github.io/www-project-top-10-ci-cd-security-risks/CICD-SEC-03-Dependency-Chain-Abuse"},{"id":"CICD-SEC-4","guid":"c859d8ea-8b24-a157-9406-94929c10397a","name":"Poisoned Pipeline Execution (PPE)","shortDescription":{"text":"Poisoned Pipeline Execution (PPE)"},"helpUri":"https://owasp.github.io/www-project-top-10-ci-cd-security-risks/CICD-SEC-04-Poisoned-Pipeline-Execution"},{"id":"CICD-SEC-6","guid":"f2ee2486-b260-2952-a41f-6b67fa713940","name":"Insufficient Credential Hygiene","shortDescription":{"text":"Insufficient Credential Hygiene"},"helpUri":"https://owasp.github.io/www-project-top-10-ci-cd-security-risks/CICD-SEC-06-Insufficient-Credential-Hygiene"},{"id":"CICD-SEC-9","guid":"b3d36727-0395-5b51-94c4-5e173d62eab3","name":"Improper Artifact Integrity Validation","shortDescription":{"text":"Improper Artifact Integrity Validation"},"helpUri":"https://owasp.github.io/www-project-top-10-ci-cd-security-risks/CICD-SEC-09-Improper-Artifact-Integrity-Validation"}]}],"properties":{"codehealthPublication":{"public":true,"notice":"This is the PUBLIC form of this artifact. Findings are listed in full, but the details of SECURITY findings \u2014 which rule fired, in which file, on which line, and how to fix it \u2014 are deliberately withheld, and any secret-scanner results are excluded entirely. Where detail is absent here it was REMOVED FOR PUBLICATION; it is not missing from the analysis. The complete artifact is available from the repository owner.","securityFindingsRedacted":26,"secretScannerRunsExcluded":0}},"redactionTokens":["A security finding was recorded here. Its details are withheld on the public artifact \u2014 ask the repository owner for the full report."]}]}