marauder-actual
300 lines
7.0 KiB
JSON
300 lines
7.0 KiB
JSON
{
|
|
"add_prefix_space": false,
|
|
"audio_bos_token": "<|audio_start|>",
|
|
"audio_eos_token": "<|audio_end|>",
|
|
"audio_token": "<|audio_pad|>",
|
|
"backend": "tokenizers",
|
|
"bos_token": null,
|
|
"clean_up_tokenization_spaces": false,
|
|
"eos_token": "<|im_end|>",
|
|
"errors": "replace",
|
|
"image_token": "<|image_pad|>",
|
|
"is_local": false,
|
|
"model_max_length": 262144,
|
|
"model_specific_special_tokens": {
|
|
"audio_bos_token": "<|audio_start|>",
|
|
"audio_eos_token": "<|audio_end|>",
|
|
"audio_token": "<|audio_pad|>",
|
|
"image_token": "<|image_pad|>",
|
|
"video_token": "<|video_pad|>",
|
|
"vision_bos_token": "<|vision_start|>",
|
|
"vision_eos_token": "<|vision_end|>"
|
|
},
|
|
"pad_token": "<|endoftext|>",
|
|
"padding_side": "right",
|
|
"pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+",
|
|
"processor_class": "Qwen3VLProcessor",
|
|
"split_special_tokens": false,
|
|
"tokenizer_class": "TokenizersBackend",
|
|
"unk_token": null,
|
|
"video_token": "<|video_pad|>",
|
|
"vision_bos_token": "<|vision_start|>",
|
|
"vision_eos_token": "<|vision_end|>",
|
|
"added_tokens_decoder": {
|
|
"248044": {
|
|
"content": "<|endoftext|>",
|
|
"single_word": false,
|
|
"lstrip": false,
|
|
"rstrip": false,
|
|
"normalized": false,
|
|
"special": true
|
|
},
|
|
"248045": {
|
|
"content": "<|im_start|>",
|
|
"single_word": false,
|
|
"lstrip": false,
|
|
"rstrip": false,
|
|
"normalized": false,
|
|
"special": true
|
|
},
|
|
"248046": {
|
|
"content": "<|im_end|>",
|
|
"single_word": false,
|
|
"lstrip": false,
|
|
"rstrip": false,
|
|
"normalized": false,
|
|
"special": true
|
|
},
|
|
"248047": {
|
|
"content": "<|object_ref_start|>",
|
|
"single_word": false,
|
|
"lstrip": false,
|
|
"rstrip": false,
|
|
"normalized": false,
|
|
"special": true
|
|
},
|
|
"248048": {
|
|
"content": "<|object_ref_end|>",
|
|
"single_word": false,
|
|
"lstrip": false,
|
|
"rstrip": false,
|
|
"normalized": false,
|
|
"special": true
|
|
},
|
|
"248049": {
|
|
"content": "<|box_start|>",
|
|
"single_word": false,
|
|
"lstrip": false,
|
|
"rstrip": false,
|
|
"normalized": false,
|
|
"special": true
|
|
},
|
|
"248050": {
|
|
"content": "<|box_end|>",
|
|
"single_word": false,
|
|
"lstrip": false,
|
|
"rstrip": false,
|
|
"normalized": false,
|
|
"special": true
|
|
},
|
|
"248051": {
|
|
"content": "<|quad_start|>",
|
|
"single_word": false,
|
|
"lstrip": false,
|
|
"rstrip": false,
|
|
"normalized": false,
|
|
"special": true
|
|
},
|
|
"248052": {
|
|
"content": "<|quad_end|>",
|
|
"single_word": false,
|
|
"lstrip": false,
|
|
"rstrip": false,
|
|
"normalized": false,
|
|
"special": true
|
|
},
|
|
"248053": {
|
|
"content": "<|vision_start|>",
|
|
"single_word": false,
|
|
"lstrip": false,
|
|
"rstrip": false,
|
|
"normalized": false,
|
|
"special": true
|
|
},
|
|
"248054": {
|
|
"content": "<|vision_end|>",
|
|
"single_word": false,
|
|
"lstrip": false,
|
|
"rstrip": false,
|
|
"normalized": false,
|
|
"special": true
|
|
},
|
|
"248055": {
|
|
"content": "<|vision_pad|>",
|
|
"single_word": false,
|
|
"lstrip": false,
|
|
"rstrip": false,
|
|
"normalized": false,
|
|
"special": true
|
|
},
|
|
"248056": {
|
|
"content": "<|image_pad|>",
|
|
"single_word": false,
|
|
"lstrip": false,
|
|
"rstrip": false,
|
|
"normalized": false,
|
|
"special": true
|
|
},
|
|
"248057": {
|
|
"content": "<|video_pad|>",
|
|
"single_word": false,
|
|
"lstrip": false,
|
|
"rstrip": false,
|
|
"normalized": false,
|
|
"special": true
|
|
},
|
|
"248058": {
|
|
"content": "<tool_call>",
|
|
"single_word": false,
|
|
"lstrip": false,
|
|
"rstrip": false,
|
|
"normalized": false,
|
|
"special": false
|
|
},
|
|
"248059": {
|
|
"content": "</tool_call>",
|
|
"single_word": false,
|
|
"lstrip": false,
|
|
"rstrip": false,
|
|
"normalized": false,
|
|
"special": false
|
|
},
|
|
"248060": {
|
|
"content": "<|fim_prefix|>",
|
|
"single_word": false,
|
|
"lstrip": false,
|
|
"rstrip": false,
|
|
"normalized": false,
|
|
"special": false
|
|
},
|
|
"248061": {
|
|
"content": "<|fim_middle|>",
|
|
"single_word": false,
|
|
"lstrip": false,
|
|
"rstrip": false,
|
|
"normalized": false,
|
|
"special": false
|
|
},
|
|
"248062": {
|
|
"content": "<|fim_suffix|>",
|
|
"single_word": false,
|
|
"lstrip": false,
|
|
"rstrip": false,
|
|
"normalized": false,
|
|
"special": false
|
|
},
|
|
"248063": {
|
|
"content": "<|fim_pad|>",
|
|
"single_word": false,
|
|
"lstrip": false,
|
|
"rstrip": false,
|
|
"normalized": false,
|
|
"special": false
|
|
},
|
|
"248064": {
|
|
"content": "<|repo_name|>",
|
|
"single_word": false,
|
|
"lstrip": false,
|
|
"rstrip": false,
|
|
"normalized": false,
|
|
"special": false
|
|
},
|
|
"248065": {
|
|
"content": "<|file_sep|>",
|
|
"single_word": false,
|
|
"lstrip": false,
|
|
"rstrip": false,
|
|
"normalized": false,
|
|
"special": false
|
|
},
|
|
"248066": {
|
|
"content": "<tool_response>",
|
|
"single_word": false,
|
|
"lstrip": false,
|
|
"rstrip": false,
|
|
"normalized": false,
|
|
"special": false
|
|
},
|
|
"248067": {
|
|
"content": "</tool_response>",
|
|
"single_word": false,
|
|
"lstrip": false,
|
|
"rstrip": false,
|
|
"normalized": false,
|
|
"special": false
|
|
},
|
|
"248068": {
|
|
"content": "<think>",
|
|
"single_word": false,
|
|
"lstrip": false,
|
|
"rstrip": false,
|
|
"normalized": false,
|
|
"special": false
|
|
},
|
|
"248069": {
|
|
"content": "</think>",
|
|
"single_word": false,
|
|
"lstrip": false,
|
|
"rstrip": false,
|
|
"normalized": false,
|
|
"special": false
|
|
},
|
|
"248070": {
|
|
"content": "<|audio_start|>",
|
|
"single_word": false,
|
|
"lstrip": false,
|
|
"rstrip": false,
|
|
"normalized": false,
|
|
"special": true
|
|
},
|
|
"248071": {
|
|
"content": "<|audio_end|>",
|
|
"single_word": false,
|
|
"lstrip": false,
|
|
"rstrip": false,
|
|
"normalized": false,
|
|
"special": true
|
|
},
|
|
"248072": {
|
|
"content": "<tts_pad>",
|
|
"single_word": false,
|
|
"lstrip": false,
|
|
"rstrip": false,
|
|
"normalized": false,
|
|
"special": true
|
|
},
|
|
"248073": {
|
|
"content": "<tts_text_bos>",
|
|
"single_word": false,
|
|
"lstrip": false,
|
|
"rstrip": false,
|
|
"normalized": false,
|
|
"special": true
|
|
},
|
|
"248074": {
|
|
"content": "<tts_text_eod>",
|
|
"single_word": false,
|
|
"lstrip": false,
|
|
"rstrip": false,
|
|
"normalized": false,
|
|
"special": true
|
|
},
|
|
"248075": {
|
|
"content": "<tts_text_bos_single>",
|
|
"single_word": false,
|
|
"lstrip": false,
|
|
"rstrip": false,
|
|
"normalized": false,
|
|
"special": true
|
|
},
|
|
"248076": {
|
|
"content": "<|audio_pad|>",
|
|
"single_word": false,
|
|
"lstrip": false,
|
|
"rstrip": false,
|
|
"normalized": false,
|
|
"special": true
|
|
}
|
|
}
|
|
}
|