Download scripts/pad_tokenizer.py from cwenzi/neuroflow-cpp: direct link, hf CLI and curl.
- Browser
- Download file 856 Bytes
-
https://huggingface.co/cwenzi/neuroflow-cpp/resolve/main/scripts/pad_tokenizer.py
- Command line
-
hf download hf://cwenzi/neuroflow-cpp/scripts/pad_tokenizer.py
-
curl -L -o pad_tokenizer.py https://huggingface.co/cwenzi/neuroflow-cpp/resolve/main/scripts/pad_tokenizer.py
856 Bytes
| import json | |
| path = r'D:\neuroflow-C++\configs\tokenizer_128k.json' | |
| d = json.load(open(path, 'r', encoding='utf-8')) | |
| vocab = d['vocab'] | |
| expected = set(range(128000)) | |
| actual = set(vocab.values()) | |
| missing = sorted(list(expected - actual)) | |
| print('Missing IDs:', len(missing)) | |
| print('Range:', missing[0], '-', missing[-1]) | |
| for mid in missing: | |
| token = '<unused_' + str(mid) + '>' | |
| if token in vocab: | |
| print('Collision at', token, '-> using alt') | |
| token = '<fill_' + str(mid) + '>' | |
| vocab[token] = mid | |
| d['vocab'] = vocab | |
| d['vocab_size'] = 128000 | |
| with open(path, 'w', encoding='utf-8') as f: | |
| json.dump(d, f, ensure_ascii=False, indent=2) | |
| # Verify | |
| all_ids = set(vocab.values()) | |
| still_missing = set(range(128000)) - all_ids | |
| print('After fix - tokens:', len(vocab), 'missing IDs:', len(still_missing)) | |