Skip to content

fix(vllm_plugin): fix multimodal registration for offline inference #348

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
baotran152 wants to merge 1 commit into
base: main
Choose a base branch
from
+9 −4
Open

fix(vllm_plugin): fix multimodal registration for offline inference #348

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 9 additions & 4 deletions vllm_plugin/model.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -90,7 +90,7 @@ def load_file(self, filepath) -> tuple[np.ndarray, int]:
from transformers.models.whisper import WhisperFeatureExtractor
from vllm.config import VllmConfig
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.multimodal.parse import MultiModalDataParser
from vllm.multimodal.parse import MultiModalDataParser, MultiModalDataItems, AudioProcessorItems
from vllm.sequence import IntermediateTensors
from vllm.model_executor.models.interfaces import SupportsMultiModal, SupportsPP, MultiModalEmbeddings
from vllm.model_executor.models.utils import (
Expand Down Expand Up @@ -553,6 +553,9 @@ def get_audio_token_info(self) -> dict:

return tokens

def get_data_parser(self) -> MultiModalDataParser:
return MultiModalDataParser(target_sr=24000)

def get_supported_mm_limits(self) -> Mapping[str, int | None]:
return {"audio": 1}

Expand Down Expand Up @@ -670,9 +673,12 @@ def get_dummy_processor_inputs(
mm_options: Mapping[str, Any] | None = None,
) -> ProcessorInputs:
"""Build ProcessorInputs for dummy profiling."""
dummy_mm_data = self.get_dummy_mm_data(seq_len, mm_counts, mm_options)
return ProcessorInputs(
prompt=self.get_dummy_text(mm_counts),
mm_data=self.get_dummy_mm_data(seq_len, mm_counts, mm_options),
mm_data_items=MultiModalDataItems({
"audio": AudioProcessorItems(dummy_mm_data.get("audio", [])),
}),
)


Expand Down Expand Up @@ -707,9 +713,8 @@ class VibeVoiceMultiModalProcessor(BaseMultiModalProcessor[VibeVoiceProcessingIn
and manages the prompt token replacement for audio placeholders.
"""

def _get_data_parser(self) -> MultiModalDataParser:
def get_data_parser(self) -> MultiModalDataParser:
"""Create a data parser with the correct target sample rate (24kHz)."""
# VibeVoice requires 24kHz, not 16kHz (Whisper default)
target_sr = 24000
return MultiModalDataParser(target_sr=target_sr)

Expand Down