Upload folder using huggingface_hub

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

added_tokens.json

@@ -0,0 +1,28 @@
+{
+  "</img>": 151653,
+  "</think>": 151668,
+  "</tool_call>": 151658,
+  "</tool_response>": 151666,
+  "<img>": 151652,
+  "<think>": 151667,
+  "<tool_call>": 151657,
+  "<tool_response>": 151665,
+  "<|box_end|>": 151649,
+  "<|box_start|>": 151648,
+  "<|endoftext|>": 151643,
+  "<|file_sep|>": 151664,
+  "<|fim_middle|>": 151660,
+  "<|fim_pad|>": 151662,
+  "<|fim_prefix|>": 151659,
+  "<|fim_suffix|>": 151661,
+  "<|im_end|>": 151645,
+  "<|im_start|>": 151644,
+  "<|image_pad|>": 151655,
+  "<|object_ref_end|>": 151647,
+  "<|object_ref_start|>": 151646,
+  "<|quad_end|>": 151651,
+  "<|quad_start|>": 151650,
+  "<|repo_name|>": 151663,
+  "<|video_pad|>": 151656,
+  "<|vision_pad|>": 151654
+}

chat_template.jinja

@@ -0,0 +1,85 @@
+{%- if tools %}
+    {{- '<|im_start|>system\n' }}
+    {%- if messages[0].role == 'system' %}
+        {{- messages[0].content + '\n\n' }}
+    {%- endif %}
+    {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
+    {%- for tool in tools %}
+        {{- "\n" }}
+        {{- tool | tojson }}
+    {%- endfor %}
+    {{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
+{%- else %}
+    {%- if messages[0].role == 'system' %}
+        {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }}
+    {%- endif %}
+{%- endif %}
+{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}
+{%- for message in messages[::-1] %}
+    {%- set index = (messages|length - 1) - loop.index0 %}
+    {%- if ns.multi_step_tool and message.role == "user" and not(message.content.startswith('<tool_response>') and message.content.endswith('</tool_response>')) %}
+        {%- set ns.multi_step_tool = false %}
+        {%- set ns.last_query_index = index %}
+    {%- endif %}
+{%- endfor %}
+{%- for message in messages %}
+    {%- if (message.role == "user") or (message.role == "system" and not loop.first) %}
+        {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }}
+    {%- elif message.role == "assistant" %}
+        {%- set content = message.content %}
+        {%- set reasoning_content = '' %}
+        {%- if message.reasoning_content is defined and message.reasoning_content is not none %}
+            {%- set reasoning_content = message.reasoning_content %}
+        {%- else %}
+            {%- if '</think>' in message.content %}
+                {%- set content = message.content.split('</think>')[-1].lstrip('\n') %}
+                {%- set reasoning_content = message.content.split('</think>')[0].rstrip('\n').split('<think>')[-1].lstrip('\n') %}
+            {%- endif %}
+        {%- endif %}
+        {%- if loop.index0 > ns.last_query_index %}
+            {%- if loop.last or (not loop.last and reasoning_content) %}
+                {{- '<|im_start|>' + message.role + '\n<think>\n' + reasoning_content.strip('\n') + '\n</think>\n\n' + content.lstrip('\n') }}
+            {%- else %}
+                {{- '<|im_start|>' + message.role + '\n' + content }}
+            {%- endif %}
+        {%- else %}
+            {{- '<|im_start|>' + message.role + '\n' + content }}
+        {%- endif %}
+        {%- if message.tool_calls %}
+            {%- for tool_call in message.tool_calls %}
+                {%- if (loop.first and content) or (not loop.first) %}
+                    {{- '\n' }}
+                {%- endif %}
+                {%- if tool_call.function %}
+                    {%- set tool_call = tool_call.function %}
+                {%- endif %}
+                {{- '<tool_call>\n{"name": "' }}
+                {{- tool_call.name }}
+                {{- '", "arguments": ' }}
+                {%- if tool_call.arguments is string %}
+                    {{- tool_call.arguments }}
+                {%- else %}
+                    {{- tool_call.arguments | tojson }}
+                {%- endif %}
+                {{- '}\n</tool_call>' }}
+            {%- endfor %}
+        {%- endif %}
+        {{- '<|im_end|>\n' }}
+    {%- elif message.role == "tool" %}
+        {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %}
+            {{- '<|im_start|>user' }}
+        {%- endif %}
+        {{- '\n<tool_response>\n' }}
+        {{- message.content }}
+        {{- '\n</tool_response>' }}
+        {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+            {{- '<|im_end|>\n' }}
+        {%- endif %}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|im_start|>assistant\n' }}
+    {%- if enable_thinking is defined and enable_thinking is false %}
+        {{- '<think>\n\n</think>\n\n' }}
+    {%- endif %}
+{%- endif %}

config.json

@@ -0,0 +1,60 @@
+{
+  "architectures": [
+    "AndesVLForConditionalGeneration"
+  ],
+  "auto_map": {
+    "AutoConfig": "configuration_andesvl.AndesVLConfig",
+    "AutoModel": "modeling_andesvl.AndesVLForConditionalGeneration",
+    "AutoModelForCausalLM": "modeling_andesvl.AndesVLForConditionalGeneration"
+  },
+  "model_type": "andesvl-siglip2-qwen3",
+  "text_config": {
+    "vocab_size": 151936,
+    "max_position_embeddings": 40960,
+    "hidden_size": 1024,
+    "intermediate_size": 3072,
+    "num_hidden_layers": 28,
+    "num_attention_heads": 16,
+    "use_sliding_window": false,
+    "sliding_window": null,
+    "max_window_layers": 28,
+    "num_key_value_heads": 8,
+    "head_dim": 128,
+    "hidden_act": "silu",
+    "initializer_range": 0.02,
+    "rms_norm_eps": 1e-06,
+    "use_cache": true,
+    "rope_theta": 1000000,
+    "rope_scaling": null,
+    "attention_bias": false,
+    "attention_dropout": 0.0,
+    "tie_word_embeddings": true,
+    "architectures": [
+      "Qwen3ForCausalLM"
+    ],
+    "bos_token_id": 151643,
+    "eos_token_id": 151645,
+    "model_type": "qwen3"
+  },
+  "vision_config": {
+    "architectures": [
+      "Siglip2VisionModel"
+    ],
+    "disable_rope": false,
+    "hidden_act": "gelu_pytorch_tanh",
+    "hidden_size": 768,
+    "intermediate_size": 3072,
+    "layer_norm_eps": 1e-06,
+    "model_type": "siglip2_navit_rope_model",
+    "num_attention_heads": 12,
+    "num_channels": 3,
+    "num_hidden_layers": 12,
+    "num_patches": 1024,
+    "patch_size": 16,
+    "preserve_original_pe": true,
+    "rope_theta": 10000.0
+  },
+  "tie_word_embeddings": true,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.51.0"
+}

configuration_andesvl.py

@@ -0,0 +1,36 @@
+import copy
+
+from transformers import  Qwen3Config 
+from transformers.configuration_utils import PretrainedConfig
+from transformers.utils import logging
+from .configuration_siglip2_navit_rope import Siglip2VisionConfig
+
+
+logger = logging.get_logger(__name__)
+
+
+class AndesVLConfig(PretrainedConfig):
+    model_type = 'andesvl-siglip2-qwen3'
+    is_composition = True
+
+    def __init__(
+            self,
+            vision_config=None,
+            text_config=None,
+            **kwargs):
+        super().__init__(**kwargs)
+
+        self.vision_config = Siglip2VisionConfig(**vision_config) if vision_config is not None else Siglip2VisionConfig()
+        self.text_config = Qwen3Config(**text_config) if text_config is not None else Qwen3Config()
+
+    def to_dict(self):
+        """
+        Serializes this instance to a Python dictionary. Override the default [`~PretrainedConfig.to_dict`].
+        Returns:
+            `Dict[str, any]`: Dictionary of all the attributes that make up this configuration instance,
+        """
+        output = copy.deepcopy(self.__dict__)
+        output['vision_config'] = self.vision_config.to_dict()
+        output['text_config'] = self.text_config.to_dict()
+        output['model_type'] = self.__class__.model_type
+        return output

configuration_siglip2_navit_rope.py

@@ -0,0 +1,41 @@
+import os
+from typing import Union
+from transformers.configuration_utils import PretrainedConfig
+from transformers.utils import logging
+logger = logging.get_logger(__name__)
+
+
+class Siglip2VisionConfig(PretrainedConfig):
+    model_type = "siglip2_navit_rope_model"
+    base_config_key = "vision_config"
+
+    def __init__(
+        self,
+        hidden_size=768,
+        intermediate_size=3072,
+        num_hidden_layers=12,
+        num_attention_heads=12,
+        num_channels=3,
+        patch_size=16,
+        hidden_act="gelu_pytorch_tanh",
+        layer_norm_eps=1e-6,
+        preserve_original_pe=True,
+        disable_rope=False,
+        num_patches=1369,
+        rope_theta=10000.0,
+        **kwargs,
+    ):
+        super().__init__(**kwargs)
+
+        self.hidden_size = hidden_size
+        self.intermediate_size = intermediate_size
+        self.num_hidden_layers = num_hidden_layers
+        self.num_attention_heads = num_attention_heads
+        self.num_channels = num_channels
+        self.patch_size = patch_size
+        self.hidden_act = hidden_act
+        self.layer_norm_eps = layer_norm_eps
+        self.preserve_original_pe = preserve_original_pe
+        self.disable_rope = disable_rope
+        self.num_patches = num_patches
+        self.rope_theta = rope_theta

generation_config.json

@@ -0,0 +1,12 @@
+{
+  "do_sample": true,
+  "temperature": 0.6,
+  "top_k": 20,
+  "top_p": 0.95,
+  "pad_token_id": 151643,
+  "bos_token_id": 151643,
+  "eos_token_id": [
+    151645,
+    151643
+  ]
+}

modeling_andesvl.py

@@ -0,0 +1,290 @@
+from torch import nn
+import torch.utils.checkpoint
+from transformers import Qwen3ForCausalLM
+from transformers.modeling_utils import PreTrainedModel
+from transformers.utils import  logging
+from .configuration_andesvl import AndesVLConfig
+from .modeling_siglip2_navit_rope import Siglip2VisionModel
+
+logger = logging.get_logger(__name__)
+
+class AndesVLForConditionalGeneration(PreTrainedModel):
+    config_class = AndesVLConfig
+    main_input_name = 'pixel_values'
+    _supports_flash_attn_2 = True
+    _no_split_modules = ['Siglip2VisionModel','Qwen3DecoderLayer']
+
+
+    def __init__(self, config: AndesVLConfig):
+        super().__init__(config)
+        
+        self.config = config
+        self.vision_encoder = Siglip2VisionModel(config.vision_config)
+        self.language_model = Qwen3ForCausalLM(config.text_config)
+        
+        vit_hidden_size = self.vision_encoder.config.hidden_size
+        llm_hidden_size = self.language_model.config.hidden_size
+        self.patch_size = self.vision_encoder.config.patch_size
+        self.mlp = nn.Sequential(
+            nn.Linear(vit_hidden_size * 4, vit_hidden_size * 4),
+            nn.GELU(),
+            nn.Linear(vit_hidden_size * 4, llm_hidden_size),
+        )
+
+    def get_input_embeddings(self):
+        return self.language_model.model.embed_tokens
+
+    def set_input_embeddings(self, value):
+        self.language_model.model.embed_tokens = value
+
+    def get_output_embeddings(self):
+        return self.language_model.lm_head
+
+    def set_output_embeddings(self, new_embeddings):
+        self.language_model.lm_head = new_embeddings
+
+    def get_flated_pixel_values(self, pixel_values):
+        flated_pixel_values = []
+        image_grid_hw = []
+        for pv in pixel_values:
+            c, h, w = pv.shape
+            assert c==3 and h%self.patch_size==0 and w%self.patch_size==0, f"{c}, {w}, {h}, {self.patch_size}"
+            image_grid_hw.append((h//self.patch_size, w//self.patch_size))
+            fpv = pv.reshape(c, h//(2*self.patch_size), 2, self.patch_size, w//(2*self.patch_size), 2, self.patch_size)
+            flated_pixel_values.append(fpv.permute(1, 4, 2, 5, 0, 3, 6).reshape(-1, c*self.patch_size*self.patch_size))
+        flated_pixel_values = torch.cat(flated_pixel_values, dim=0) # (Len_img, C, H, W)
+        image_grid_hw = torch.tensor(image_grid_hw, device=flated_pixel_values.device) # (N_img, 2)
+        return flated_pixel_values, image_grid_hw
+
+
+    def get_vit_embeds_and_merge(self, pixel_values, image_grid_hw, input_embeds, image_flags):
+        """
+        Args:
+            pixel_values: (Len_img, H_vit0)， 拉平后的初始patch特征，按照序列维度拼接在一起
+            image_grid_hw: (N_img, 2)， 每个图片的宽高
+            input_embeds: (Bt, Lt, Ht)， 每个token的embedding
+            image_flags: (Bt, Lt)， 每个token是否是图片
+        """
+        vit_embeds = self.vision_encoder(pixel_values, image_grid_hw)  # (Len_img, H_vit)
+        vit_embeds = vit_embeds.view(-1, vit_embeds.shape[-1]*4) # (Len_img//4, H_vit*4)
+        vit_embeds = self.mlp(vit_embeds) # (Len_img//4, H_llm)
+        vit_embeds = vit_embeds[:image_flags.sum()]
+        Bt, Lt, Ht = input_embeds.shape
+        input_embeds = input_embeds.reshape(-1, Ht)
+        image_flags = image_flags.view(-1)
+        input_embeds[image_flags == 1] = vit_embeds
+        input_embeds = input_embeds.view(Bt, Lt, Ht)
+        return input_embeds 
+
+    @torch.inference_mode()
+    @torch.autocast(device_type="cuda", dtype=torch.bfloat16)
+    def generate(
+        self,
+        pixel_values=None,
+        input_ids=None,
+        attention_mask=None,
+        image_flags=None,  # (Bt, Lt)
+        generation_config=None,
+        **generate_kwargs,
+    ) -> torch.LongTensor:
+
+        input_embeds = self.language_model.get_input_embeddings()(input_ids)  # (Bt, Lt, Ht)
+        if image_flags != None and (image_flags == 1).sum() > 0:
+            flated_pixel_values, image_grid_hw = self.get_flated_pixel_values(pixel_values)
+            input_embeds = self.get_vit_embeds_and_merge(flated_pixel_values, image_grid_hw, input_embeds, image_flags)
+        outputs = self.language_model.generate(
+            input_ids=input_ids,
+            inputs_embeds=input_embeds,
+            attention_mask=attention_mask,
+            generation_config=generation_config,
+            use_cache=True,
+            **generate_kwargs,
+        )
+        return outputs
+    
+    #NOTE: completion和chat接口暂不支持batch推理，需要手动构建self.generate函数的输入来实现。
+    def completion(self, prompt, images, tokenizer, image_processor, **kwargs):
+        """输入一段文字和一组图片（其中文字中的图片用占位符标记为<image>），输出补全的文本"""
+        assert prompt.count("<image>") == len(images), "图片数量和占位符数量不匹配"
+        def replacement(m):
+            token_count = image_tokens.pop(0)
+            return f"<img>{'<|vision_pad|>' * token_count}</img>"
+        #首先对所有的图像进行处理，获取对应的size
+        max_size = kwargs.get("max_size", 733) # max_size**2为支持的最大的面积
+        base = self.patch_size*2
+        image_token_id = tokenizer.vocab['<|vision_pad|>'] # 图像token的占位符
+        background_color = tuple(int(x*255) for x in image_processor.image_mean)
+        transform = T.Compose([T.ToTensor(),T.Normalize(mean=image_processor.image_mean, std=image_processor.image_std)])
+        pixel_values = []
+        image_tokens = []
+        for image in images:
+            if isinstance(image, (tuple, list)):
+                image, detail = image
+            else:
+                detail = "low"
+            image = load_image(image)
+            if detail=="low":
+                image = native_preprocess(image, max_size, base, background_color, min_tokens=4)
+                pixel_values.append(transform(image))
+                image_tokens.append(image.size[0]*image.size[1]//(base*base))
+            else:
+                raise NotImplementedError("暂未实现")
+        new_prompt = re.sub(r"<image>", replacement, prompt)
+        input_ids = tokenizer(new_prompt, return_tensors="pt", add_special_tokens=False).input_ids.to(self.device)
+        image_flags = (input_ids == image_token_id).int()
+        input_ids = input_ids.to(self.vision_encoder.device)
+        pixel_values = [pv.to(self.vision_encoder.device) for pv in pixel_values]
+        image_flags = image_flags.to(self.vision_encoder.device)
+        output_ids = self.generate(pixel_values=pixel_values, input_ids=input_ids, image_flags=image_flags, **kwargs)[0][input_ids.shape[1]:]
+        return tokenizer.decode(output_ids, skip_special_tokens=True)
+            
+    def chat(self, messages, tokenizer, image_processor, **kwargs):
+        """输入是一组对话信息（openai格式），输出是回复"""
+        prompt = ""
+        images = []
+        for message in messages:
+            role = message["role"]
+            assert role in ["user", "assistant", "system"], f"非法的角色{role}"
+            content = message['content']
+            if isinstance(content, str):
+                prompt += f"<|im_start|>{role}\n{content}{tokenizer.eos_token}\n"
+            elif isinstance(content, list):
+                temp = ""
+                for sub_content in content:
+                    if sub_content['type']=='text':
+                        temp += f"{sub_content['text']}"
+                    elif sub_content['type']=='image_url':
+                        temp += "<image>"
+                        images.append([load_image(sub_content['image_url']['url']), sub_content['image_url'].get("detail",'low')])
+                prompt += f"<|im_start|>{role}\n{temp}{tokenizer.eos_token}\n"
+            else:
+                raise ValueError(f"非法的内容{content}")
+        prompt += f"<|im_start|>assistant\n"
+        thinking = 'thinking' in kwargs and kwargs['thinking']
+        if 'thinking' in kwargs:
+            kwargs.pop('thinking')
+        prompt += f"<|im_start|>assistant\n" + ('<think>' if thinking else '')
+        return ('<think>' if thinking else '') + self.completion(prompt, images, tokenizer, image_processor, **kwargs)
+        # return self.completion(prompt, images, tokenizer, image_processor, **kwargs)
+
+########################
+###下面是图像处理的代码###
+########################
+
+import os
+import math
+import re
+from typing import Union
+import requests
+import base64
+from io import BytesIO
+from PIL import Image
+import torchvision.transforms as T
+
+def load_image(source: Union[str, Image.Image]) -> Image.Image:
+    """加载图像"""
+    if isinstance(source, Image.Image):
+        img = source
+    elif isinstance(source, str):
+        if source.startswith('http'):
+            response = requests.get(source)
+            response.raise_for_status()
+            img = Image.open(BytesIO(response.content))
+        elif os.path.exists(source):
+            img = Image.open(source)
+        elif source.startswith('data:image'):
+            img = Image.open(BytesIO(base64.b64decode(source.split(',')[1])))
+        else:
+            raise ValueError("Unsupported image source")
+    else:
+        raise ValueError("Unsupported image source")
+    return img.convert('RGB')
+
+def get_scaled_img_size(image_size, max_area, base, max_resolution=4172, upper=True):
+    """计算缩放后的图片大小和包裹矩形的大小"""
+    # 计算原始图片的宽高比
+    aspect_ratio = image_size[0] / image_size[1]
+    # 计算包裹矩形的最大可能宽度和高度
+    max_width = math.floor(math.sqrt(max_area * aspect_ratio))
+    max_height = math.floor(math.sqrt(max_area / aspect_ratio))
+    max_width, max_height = min(max_width, max_resolution), min(
+        max_height, max_resolution
+    )
+    max_width, max_height = max(max_width, base), max(max_height, base)
+    # 确保包裹矩形的宽度和高度都是base的整数倍
+    if not upper:
+        # 向��取整, 保证面积不会超过max_area
+        max_width = max_width - max_width % base
+        max_height = max_height - max_height % base
+    else:
+        # 向上取整，同时不超过max_resolution（单边最大长度）
+        max_width = min(max_width + (base - max_width % base), max_resolution)
+        max_height = min(max_height + (base - max_height % base), max_resolution)
+    # 计算缩放因子
+    scale_factor = min(max_width / image_size[0], max_height / image_size[1])
+    # 计算缩放后的图片大小
+    new_image_size = (
+        round(image_size[0] * scale_factor),
+        round(image_size[1] * scale_factor),
+    )
+    # 计算包裹矩形的大小
+    bounding_box_size = (max_width, max_height)
+    return new_image_size, bounding_box_size
+
+
+def max_preprocess(
+    img, max_size, base, background_color, max_resolution=4172, upper=True, force_resize=False
+):
+    """对图片进行预处理，使其面积接近max_size**2"""
+    # 首先把图片resize到长度和宽度都低于max_resolution
+    w, h = img.size
+    if max(w, h) > max_resolution:
+        scale = max_resolution / max(w, h)
+        w, h = int(w * scale), int(h * scale)
+    # 获取缩放后的图片大小和包裹矩形的大小
+    new_image_size, bounding_box_size = get_scaled_img_size(
+        (w, h), max_size**2, base, max_resolution, upper
+    )
+    if force_resize:
+        return img.resize(bounding_box_size)
+    # 创建一个新的画布
+    canvas = Image.new("RGB", bounding_box_size, background_color)
+    # 计算将图像粘贴到画布上的位置
+    paste_width = (bounding_box_size[0] - new_image_size[0]) // 2
+    paste_height = (bounding_box_size[1] - new_image_size[1]) // 2
+    # 将图像粘贴到画布上
+    canvas.paste(img.resize(new_image_size), (paste_width, paste_height))
+    return canvas
+
+def native_preprocess(
+    img, max_size, base, background_color, max_resolution=4172, min_tokens=64
+):
+    # 对图片进行处理，使其宽度和高度都是base的整数倍
+    # 如果图片的最长边超过max_resolution，就把图片resize到max_resolution以内
+    w, h = img.size
+    # 首先保证图片的最长边不超过max_resolution(ViT在极限长度)
+    if max(w, h) > max_resolution:
+        scale = max_resolution / max(w, h)
+        w, h = int(w * scale), int(h * scale)
+        img = img.resize((w, h))
+    if w * h > max_size**2:
+        return max_preprocess(img, max_size, base, background_color, max_resolution)
+    if w * h < (base * base * min_tokens):
+        return max_preprocess(
+            img,
+            int(base * (min_tokens**0.5)),
+            base,
+            background_color,
+            max_resolution,
+        )  
+    w1, h1 = w + base - w % base, h + base - h % base
+    if w1 == w and h1 == h:
+        return img
+    else:
+        # 创建一个新的(w1, h1)的画布，并把图片resize保证只有一侧存在白边的情况
+        scale = min(w1 / w, h1 / h)
+        new_w, new_h = int(w * scale), int(h * scale)
+        img = img.resize((new_w, new_h))
+        canvas = Image.new("RGB", (w1, h1), background_color)
+        canvas.paste(img, ((w1 - new_w) // 2, (h1 - new_h) // 2))
+        return canvas

modeling_siglip2_navit_rope.py

@@ -0,0 +1,520 @@
+import math
+
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+import torch.utils.checkpoint
+from transformers.activations import ACT2FN
+from transformers.modeling_utils import PreTrainedModel
+from transformers.utils import (
+    is_flash_attn_2_available,
+)
+try:
+    from .configuration_siglip2_navit_rope import Siglip2VisionConfig
+except:
+    from configuration_siglip2_navit_rope import Siglip2VisionConfig
+
+if is_flash_attn_2_available():
+    from flash_attn import flash_attn_varlen_func
+else:
+    flash_attn_varlen_func = None
+
+
+# Copied from transformers.models.llama.modeling_llama.rotate_half
+def rotate_half(x):
+    """Rotates half the hidden dims of the input."""
+    x1 = x[..., : x.shape[-1] // 2]
+    x2 = x[..., x.shape[-1] // 2 :]
+    return torch.cat((-x2, x1), dim=-1)
+
+
+def apply_rotary_pos_emb_vision(
+    tensor: torch.Tensor, freqs: torch.Tensor
+) -> torch.Tensor:
+    orig_dtype = tensor.dtype
+    tensor = tensor.float()
+    cos = freqs.cos()
+    sin = freqs.sin()
+    cos = cos.unsqueeze(1).repeat(1, 1, 2).unsqueeze(0).float()
+    sin = sin.unsqueeze(1).repeat(1, 1, 2).unsqueeze(0).float()
+    output = (tensor * cos) + (rotate_half(tensor) * sin)
+    output = output.to(orig_dtype)
+    return output
+
+
+class VisionRotaryEmbedding(nn.Module):
+    def __init__(self, dim: int, theta: float = 10000.0) -> None:
+        super().__init__()
+        inv_freq = 1.0 / (theta ** (torch.arange(0, dim, 2, dtype=torch.float) / dim))
+        self.register_buffer("inv_freq", inv_freq, persistent=False)
+
+    def forward(self, seqlen: int) -> torch.Tensor:
+        seq = torch.arange(
+            seqlen, device=self.inv_freq.device, dtype=self.inv_freq.dtype
+        )
+        freqs = torch.outer(seq, self.inv_freq)
+        return freqs
+
+
+class PatchEmbed(nn.Module):
+    def __init__(
+        self,
+        patch_size,
+        num_channels,
+        embed_dim,
+        num_patches,
+        preserve_original_pe=False
+    ):
+        super().__init__()
+        self.patch_size = patch_size
+        self.num_patches = num_patches
+        self.embed_dim = embed_dim
+        self.preserve_original_pe = preserve_original_pe
+
+        self.proj = nn.Linear(
+            num_channels * patch_size * patch_size, embed_dim
+        )  # NOTE: bias默认为True
+
+        if preserve_original_pe:
+            assert num_patches**0.5 == int(num_patches**0.5), "num_patches must be a perfect square"
+            self.pos_embed = nn.Embedding(num_patches, embed_dim)
+            self.original_grid_size = int(num_patches**0.5)
+        else:
+            self.pos_embed = None
+            self.original_grid_size = 0
+
+    def get_patch_coordinates(self, grid_hw: torch.Tensor, device: torch.device):
+        """
+        生成与2x2分块扫描顺序匹配的patch坐标。
+        """
+        all_h_coords, all_w_coords, all_target_sizes = [], [], []
+        
+        for h, w in grid_hw:
+            h, w = h.item(), w.item()
+            
+            # 生成标准网格坐标
+            h_grid, w_grid = torch.meshgrid(
+                torch.arange(h, device=device, dtype=torch.float32),
+                torch.arange(w, device=device, dtype=torch.float32),
+                indexing='ij'
+            )
+            
+            # 重排列为分块扫描顺序
+            h_coords = h_grid.reshape(
+                h//2, 2, w//2, 2
+            ).permute(0, 2, 1, 3).flatten()
+            
+            w_coords = w_grid.reshape(
+                h//2, 2, w//2, 2
+            ).permute(0, 2, 1, 3).flatten()
+            
+            all_h_coords.append(h_coords)
+            all_w_coords.append(w_coords)
+            
+            target_size = torch.tensor([h, w], device=device, dtype=torch.float32)
+            all_target_sizes.append(target_size.expand(h * w, -1))
+
+        return torch.cat(all_h_coords), torch.cat(all_w_coords), torch.cat(all_target_sizes)
+
+    def abs_pos_embed(self, grid_hw: torch.Tensor, mode='bicubic') -> torch.Tensor:
+        pos_embed_weight = self.pos_embed.weight
+        pos_embed_2d = pos_embed_weight.transpose(0, 1).reshape(
+            self.embed_dim, self.original_grid_size, self.original_grid_size
+        ).unsqueeze(0).to(torch.float32)
+
+        if grid_hw.numel() == 0:
+            return torch.empty(0, self.embed_dim, device=pos_embed_2d.device, dtype=pos_embed_weight.dtype)
+        
+        h_coords, w_coords, target_sizes = self.get_patch_coordinates(grid_hw, pos_embed_2d.device)
+        
+        if h_coords.shape[0] == 0:
+            return torch.empty(0, self.embed_dim, device=pos_embed_2d.device, dtype=pos_embed_weight.dtype)
+
+        target_h = target_sizes[:, 0]
+        target_w = target_sizes[:, 1]
+        
+        # 这个归一化公式对于 align_corners=False 是正确的。
+        norm_w = (2.0 * (w_coords + 0.5) / target_w) - 1.0
+        norm_h = (2.0 * (h_coords + 0.5) / target_h) - 1.0
+
+        grid = torch.stack((norm_w, norm_h), dim=-1).unsqueeze(0).unsqueeze(0)
+
+        interpolated_embed = F.grid_sample(
+            pos_embed_2d, grid, mode=mode, align_corners=False,
+            padding_mode='border'  
+        )
+        
+        adapted_pos_embed = interpolated_embed.squeeze(0).squeeze(1).permute(1, 0)
+        
+        return adapted_pos_embed.to(pos_embed_weight.dtype)
+            
+        
+    def forward(self, hidden_states: torch.Tensor, grid_hw: torch.Tensor) -> torch.Tensor:
+        """
+        Args:
+            hidden_states (torch.Tensor): input tensor of shape [seq_len, num_channels*patch_size*patch_size]
+            grid_hw (torch.Tensor): 形状为 [num_images, 2] 的张量，表示每个图像的patch网格高度和宽度
+        Returns:
+            torch.Tensor: output tensor of shape [seq_len, embed_dim]
+        """
+        target_dtype = self.proj.weight.dtype
+        hidden_states = self.proj(hidden_states.to(dtype=target_dtype))
+        
+        if self.preserve_original_pe:
+            pos_emb = self.abs_pos_embed(grid_hw)
+            hidden_states = hidden_states + pos_emb
+            
+        return hidden_states
+
+
+class Siglip2MLP(nn.Module):
+    def __init__(self, config):
+        super().__init__()
+        self.config = config
+        self.activation_fn = ACT2FN[config.hidden_act]
+        self.fc1 = nn.Linear(config.hidden_size, config.intermediate_size)
+        self.fc2 = nn.Linear(config.intermediate_size, config.hidden_size)
+
+    def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
+        hidden_states = self.fc1(hidden_states)
+        hidden_states = self.activation_fn(hidden_states)
+        hidden_states = self.fc2(hidden_states)
+        return hidden_states
+
+
+class Siglip2Attention(nn.Module):
+
+    def __init__(self, config):
+        super().__init__()
+        self.config = config
+        self.embed_dim = config.hidden_size
+        self.num_heads = config.num_attention_heads
+        self.head_dim = self.embed_dim // self.num_heads
+        if self.head_dim * self.num_heads != self.embed_dim:
+            raise ValueError(
+                f"embed_dim must be divisible by num_heads (got `embed_dim`: {self.embed_dim} and `num_heads`:"
+                f" {self.num_heads})."
+            )
+        self.scale = self.head_dim**-0.5
+
+        self.k_proj = nn.Linear(self.embed_dim, self.embed_dim)
+        self.v_proj = nn.Linear(self.embed_dim, self.embed_dim)
+        self.q_proj = nn.Linear(self.embed_dim, self.embed_dim)
+        self.out_proj = nn.Linear(self.embed_dim, self.embed_dim)
+
+    def forward(
+        self,
+        hidden_states: torch.Tensor,
+        cu_seqlens: torch.Tensor,
+        rotary_pos_emb: torch.Tensor = None,
+    ) -> torch.Tensor:
+        seq_length = hidden_states.shape[0]
+        q = self.q_proj(hidden_states)
+        k = self.k_proj(hidden_states)
+        v = self.v_proj(hidden_states)
+        
+        q = q.reshape(seq_length, self.num_heads, -1)
+        k = k.reshape(seq_length, self.num_heads, -1)
+        v = v.reshape(seq_length, self.num_heads, -1)    
+    
+        q = apply_rotary_pos_emb_vision(q.unsqueeze(0), rotary_pos_emb).squeeze(0)
+        k = apply_rotary_pos_emb_vision(k.unsqueeze(0), rotary_pos_emb).squeeze(0)
+
+        attention_mask = torch.full(
+            [1, seq_length, seq_length],
+            torch.finfo(q.dtype).min,
+            device=q.device,
+            dtype=q.dtype,
+        )
+        for i in range(1, len(cu_seqlens)):
+            attention_mask[
+                ...,
+                cu_seqlens[i - 1] : cu_seqlens[i],
+                cu_seqlens[i - 1] : cu_seqlens[i],
+            ] = 0
+
+        q = q.transpose(0, 1)
+        k = k.transpose(0, 1)
+        v = v.transpose(0, 1)
+        attn_weights = torch.matmul(q, k.transpose(1, 2)) / math.sqrt(self.head_dim)
+        attn_weights = attn_weights + attention_mask
+        attn_weights = nn.functional.softmax(
+            attn_weights, dim=-1, dtype=torch.float32
+        ).to(q.dtype)
+        attn_output = torch.matmul(attn_weights, v)
+        attn_output = attn_output.transpose(0, 1)
+        attn_output = attn_output.reshape(seq_length, -1)
+        attn_output = self.out_proj(attn_output)
+        return attn_output
+
+
+class Siglip2FlashAttention2(nn.Module):
+
+    def __init__(self, config):
+        super().__init__()
+        self.config = config
+        self.embed_dim = config.hidden_size
+        self.num_heads = config.num_attention_heads
+        self.head_dim = self.embed_dim // self.num_heads
+        if self.head_dim * self.num_heads != self.embed_dim:
+            raise ValueError(
+                f"embed_dim must be divisible by num_heads (got `embed_dim`: {self.embed_dim} and `num_heads`:"
+                f" {self.num_heads})."
+            )
+        self.scale = self.head_dim**-0.5
+
+        self.k_proj = nn.Linear(self.embed_dim, self.embed_dim)
+        self.v_proj = nn.Linear(self.embed_dim, self.embed_dim)
+        self.q_proj = nn.Linear(self.embed_dim, self.embed_dim)
+        self.out_proj = nn.Linear(self.embed_dim, self.embed_dim)
+
+    def forward(
+        self,
+        hidden_states: torch.Tensor,
+        cu_seqlens: torch.Tensor,
+        rotary_pos_emb: torch.Tensor = None,
+    ) -> torch.Tensor:
+        seq_length = hidden_states.shape[0]
+        q = self.q_proj(hidden_states)
+        k = self.k_proj(hidden_states)
+        v = self.v_proj(hidden_states)
+        
+        # 将 q, k, v 重塑为多头注意力的形状
+        q = q.reshape(seq_length, self.num_heads, -1)
+        k = k.reshape(seq_length, self.num_heads, -1)
+        v = v.reshape(seq_length, self.num_heads, -1)
+
+        q = apply_rotary_pos_emb_vision(q.unsqueeze(0), rotary_pos_emb).squeeze(0)
+        k = apply_rotary_pos_emb_vision(k.unsqueeze(0), rotary_pos_emb).squeeze(0)
+
+        max_seqlen = (cu_seqlens[1:] - cu_seqlens[:-1]).max().item()
+        attn_output = flash_attn_varlen_func(
+            q, k, v, cu_seqlens, cu_seqlens, max_seqlen, max_seqlen
+        ).reshape(seq_length, -1)
+        attn_output = self.out_proj(attn_output)
+        return attn_output
+
+
+class Siglip2SdpaAttention(nn.Module):
+
+    def __init__(self, config):
+        super().__init__()
+        self.config = config
+        self.embed_dim = config.hidden_size
+        self.num_heads = config.num_attention_heads
+        self.head_dim = self.embed_dim // self.num_heads
+        if self.head_dim * self.num_heads != self.embed_dim:
+            raise ValueError(
+                f"embed_dim must be divisible by num_heads (got `embed_dim`: {self.embed_dim} and `num_heads`:"
+                f" {self.num_heads})."
+            )
+        self.scale = self.head_dim**-0.5
+
+        self.k_proj = nn.Linear(self.embed_dim, self.embed_dim)
+        self.v_proj = nn.Linear(self.embed_dim, self.embed_dim)
+        self.q_proj = nn.Linear(self.embed_dim, self.embed_dim)
+        self.out_proj = nn.Linear(self.embed_dim, self.embed_dim)
+
+    def forward(
+        self,
+        hidden_states: torch.Tensor,
+        cu_seqlens: torch.Tensor,
+        rotary_pos_emb: torch.Tensor = None,
+    ) -> torch.Tensor:
+        seq_length = hidden_states.shape[0]
+        q = self.q_proj(hidden_states)
+        k = self.k_proj(hidden_states)
+        v = self.v_proj(hidden_states)
+        
+        q = q.reshape(seq_length, self.num_heads, -1)
+        k = k.reshape(seq_length, self.num_heads, -1)
+        v = v.reshape(seq_length, self.num_heads, -1)
+        
+        q = apply_rotary_pos_emb_vision(q.unsqueeze(0), rotary_pos_emb).squeeze(0)
+        k = apply_rotary_pos_emb_vision(k.unsqueeze(0), rotary_pos_emb).squeeze(0)
+
+        attention_mask = torch.zeros(
+            [1, seq_length, seq_length], device=q.device, dtype=torch.bool
+        )
+        for i in range(1, len(cu_seqlens)):
+            attention_mask[
+                ...,
+                cu_seqlens[i - 1] : cu_seqlens[i],
+                cu_seqlens[i - 1] : cu_seqlens[i],
+            ] = True
+        q = q.transpose(0, 1)
+        k = k.transpose(0, 1)
+        v = v.transpose(0, 1)
+        attn_output = F.scaled_dot_product_attention(
+            q.unsqueeze(0), k.unsqueeze(0), v.unsqueeze(0), attention_mask, dropout_p=0.0
+        )
+        attn_output = attn_output.squeeze(0).transpose(0, 1)
+        attn_output = attn_output.reshape(seq_length, -1)
+        attn_output = self.out_proj(attn_output)
+        return attn_output
+
+
+VISION_ATTENTION_CLASSES = {
+    "eager": Siglip2Attention,
+    "flash_attention_2": Siglip2FlashAttention2,
+    "sdpa": Siglip2SdpaAttention,
+}
+
+
+class Siglip2EncoderLayer(nn.Module):
+    def __init__(self, config):
+        super().__init__()
+        self.embed_dim = config.hidden_size
+        self.self_attn = VISION_ATTENTION_CLASSES[config._attn_implementation](
+            config=config
+        )
+        self.layer_norm1 = nn.LayerNorm(self.embed_dim, eps=config.layer_norm_eps)
+        self.mlp = Siglip2MLP(config)
+        self.layer_norm2 = nn.LayerNorm(self.embed_dim, eps=config.layer_norm_eps)
+
+    # Ignore copy
+    def forward(self, hidden_states, cu_seqlens, rotary_pos_emb):
+        residual = hidden_states
+
+        hidden_states = self.layer_norm1(hidden_states)
+        hidden_states = self.self_attn(
+            hidden_states=hidden_states,
+            cu_seqlens=cu_seqlens,
+            rotary_pos_emb=rotary_pos_emb,
+        )
+        hidden_states = residual + hidden_states
+
+        residual = hidden_states
+        hidden_states = self.layer_norm2(hidden_states)
+        hidden_states = self.mlp(hidden_states)
+        hidden_states = residual + hidden_states
+
+        return hidden_states
+
+
+class Siglip2Encoder(nn.Module):
+    """
+    Transformer encoder consisting of `config.num_hidden_layers` self attention layers. Each layer is a
+    [`Siglip2EncoderLayer`].
+
+    Args:
+        config: Siglip2Config
+    """
+
+    def __init__(self, config):
+        super().__init__()
+        self.config = config
+        self.layers = nn.ModuleList(
+            [Siglip2EncoderLayer(config) for _ in range(config.num_hidden_layers)]
+        )
+        self.gradient_checkpointing = True
+
+    # Ignore copy
+    def forward(
+        self,
+        hidden_states,
+        cu_seqlens,
+        rotary_pos_emb,
+    ):
+        for encoder_layer in self.layers:
+            if self.gradient_checkpointing and self.training:
+                hidden_states = torch.utils.checkpoint.checkpoint(
+                    encoder_layer,
+                    hidden_states,
+                    cu_seqlens,
+                    rotary_pos_emb,
+                    use_reentrant=False,
+                )
+            else:
+                hidden_states  = encoder_layer(
+                    hidden_states,
+                    cu_seqlens,
+                    rotary_pos_emb,
+                )
+        return hidden_states
+
+
+class Siglip2VisionTransformer(nn.Module):
+    def __init__(self, config: Siglip2VisionConfig):
+        super().__init__()
+        self.config = config
+        embed_dim = config.hidden_size
+
+        self.embeddings = PatchEmbed(
+            patch_size=config.patch_size,
+            num_channels=config.num_channels,
+            embed_dim=embed_dim,
+            num_patches=config.num_patches,
+            preserve_original_pe=config.preserve_original_pe,
+        )
+        head_dim = config.hidden_size // config.num_attention_heads
+        self.rotary_pos_emb = VisionRotaryEmbedding(head_dim // 2, config.rope_theta)
+        self.encoder = Siglip2Encoder(config)
+        self.post_layernorm = nn.LayerNorm(embed_dim, eps=config.layer_norm_eps)
+
+
+    def rot_pos_emb(self, grid_hw):
+        pos_ids = []
+        for h, w in grid_hw:
+            hpos_ids = torch.arange(h).unsqueeze(1).expand(-1, w)
+            hpos_ids = hpos_ids.reshape(
+                h // 2,
+                2,
+                w // 2,
+                2,
+            )
+            hpos_ids = hpos_ids.permute(0, 2, 1, 3)
+            hpos_ids = hpos_ids.flatten()
+
+            wpos_ids = torch.arange(w).unsqueeze(0).expand(h, -1)
+            wpos_ids = wpos_ids.reshape(
+                h // 2,
+                2,
+                w // 2,
+                2,
+            )
+            wpos_ids = wpos_ids.permute(0, 2, 1, 3)
+            wpos_ids = wpos_ids.flatten()
+            pos_ids.append(torch.stack([hpos_ids, wpos_ids], dim=-1))
+        pos_ids = torch.cat(pos_ids, dim=0)
+        max_grid_size = grid_hw.max()
+        rotary_pos_emb_full = self.rotary_pos_emb(max_grid_size)
+        rotary_pos_emb = rotary_pos_emb_full[pos_ids].flatten(1)
+        return rotary_pos_emb
+
+    def forward(
+        self,
+        hidden_states: torch.Tensor,
+        grid_hw: torch.Tensor,
+    ):
+        hidden_states = self.embeddings(hidden_states, grid_hw)
+        rotary_pos_emb = self.rot_pos_emb(grid_hw)
+        cu_seqlens = (grid_hw[:, 0] * grid_hw[:, 1]).cumsum(dim=0, dtype=torch.int32)
+        cu_seqlens = F.pad(cu_seqlens, (1, 0), value=0)
+        hidden_states = self.encoder(
+            hidden_states, cu_seqlens=cu_seqlens, rotary_pos_emb=rotary_pos_emb
+        )
+        hidden_states = self.post_layernorm(hidden_states)
+        return hidden_states
+
+
+class Siglip2VisionModel(PreTrainedModel):
+    supports_gradient_checkpointing = True
+    _supports_flash_attn_2 = True
+    _supports_sdpa = True
+    config_class = Siglip2VisionConfig
+    main_input_name = "pixel_values"
+
+    def __init__(self, config):
+        super().__init__(config)
+        self.vision_model = Siglip2VisionTransformer(config)
+        # Initialize weights and apply final processing
+        self.post_init()
+
+    def get_input_embeddings(self) -> nn.Module:
+        return self.vision_model.embeddings.patch_embedding
+
+    def forward(
+        self, hidden_states: torch.Tensor, grid_hw: torch.Tensor
+    ) -> torch.Tensor:
+        return self.vision_model(hidden_states, grid_hw)

preprocessor_config.json

@@ -0,0 +1,28 @@
+{
+  "crop_size": {
+    "height": 448,
+    "width": 448
+  },
+  "do_center_crop": false,
+  "do_convert_rgb": true,
+  "do_normalize": true,
+  "do_rescale": true,
+  "do_resize": false,
+  "image_mean": [
+    0.5,
+    0.5,
+    0.5
+  ],
+  "image_processor_type": "CLIPImageProcessor",
+  "image_std": [
+    0.5,
+    0.5,
+    0.5
+  ],
+  "patch_size": 16,
+  "resample": 2,
+  "rescale_factor": 0.00392156862745098,
+  "size": {
+    "shortest_edge": 733
+  }
+}

pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:38dd9c9cce255a4fb0434fa344aa51f238b4d85e699f59843c95cdb55a7d86ca
+size 1701479779

special_tokens_map.json

@@ -0,0 +1,31 @@
+{
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<img>",
+    "</img>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "eos_token": {
+    "content": "<|im_end|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0e0d3ee707b399f44f189e1abfb2b3cd844b96407e9b2a5a21cb3e0b5f57bb05
+size 11422629

tokenizer_config.json

@@ -0,0 +1,239 @@
+{
+  "add_bos_token": false,
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "151643": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151644": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151645": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151646": {
+      "content": "<|object_ref_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151647": {
+      "content": "<|object_ref_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151648": {
+      "content": "<|box_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151649": {
+      "content": "<|box_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151650": {
+      "content": "<|quad_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151651": {
+      "content": "<|quad_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151652": {
+      "content": "<img>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151653": {
+      "content": "</img>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151654": {
+      "content": "<|vision_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151655": {
+      "content": "<|image_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151656": {
+      "content": "<|video_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151657": {
+      "content": "<tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151658": {
+      "content": "</tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151659": {
+      "content": "<|fim_prefix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151660": {
+      "content": "<|fim_middle|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151661": {
+      "content": "<|fim_suffix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151662": {
+      "content": "<|fim_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151663": {
+      "content": "<|repo_name|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151664": {
+      "content": "<|file_sep|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151665": {
+      "content": "<tool_response>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151666": {
+      "content": "</tool_response>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151667": {
+      "content": "<think>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151668": {
+      "content": "</think>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    }
+  },
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<img>",
+    "</img>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "bos_token": null,
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "extra_special_tokens": {},
+  "model_max_length": 131072,
+  "pad_token": "<|endoftext|>",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}