Update modeling_dots_ocr_vllm.py

modeling_dots_ocr_vllm.py

@@ -99,6 +99,7 @@ class DotsOCRProcessingInfo(Qwen2_5_VLProcessingInfo):
         size: Optional[dict[str, int]] = None,
         **kwargs: object,
     ) -> Qwen2VLProcessor:
+        self.get_tokenizer().image_token = "<|imgpad|>" # Ensure image token is set
         processor = self.ctx.get_hf_processor(
             Qwen2VLProcessor,
             image_processor=self.get_image_processor(min_pixels=min_pixels, max_pixels=max_pixels, size=size),
@@ -166,6 +167,11 @@ class DotsOCRForCausalLM(nn.Module, SupportsMultiModal):
     )
     _tp_plan = {}
 
+    @classmethod
+    def get_placeholder_str(cls, modality: str, i: int) -> Optional[str]:
+        if modality in ("image",):
+            return "<|img|><|imgpad|><|endofimg|>"
+
     def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
         super().__init__()
 
@@ -409,6 +415,10 @@ class DotsOCRForCausalLM(nn.Module, SupportsMultiModal):
 
 
 def patch_vllm_chat_placeholder():
+    import vllm
+    # return when vllm version > 0.9.1
+    if not (vllm.__version_tuple__[0]==0 and vllm.__version_tuple__[1] <= 9 and vllm.__version_tuple__[2] <= 1):
+        return
     from vllm.entrypoints.chat_utils import BaseMultiModalItemTracker
 
     ori = BaseMultiModalItemTracker._placeholder_str
@@ -426,4 +436,5 @@ ModelRegistry.register_model(
     "DotsOCRForCausalLM", DotsOCRForCausalLM,
 )
 
+
 patch_vllm_chat_placeholder()