#                🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨
#           This file was automatically generated from src/transformers/models/qwen3_omni_moe/modular_qwen3_omni_moe.py.
#               Do NOT edit this file manually as any edits will be overwritten by the generation of
#             the file from the modular. If any change should be done, please apply the change to the
#                          modular_qwen3_omni_moe.py file directly. One of our CI enforces this.
#                🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨
# Copyright 2025 The Qwen team, Alibaba Group and the HuggingFace Inc. team. All rights reserved.
#
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import numpy as np

from ...audio_utils import AudioInput
from ...feature_extraction_utils import BatchFeature
from ...image_utils import ImageInput
from ...processing_utils import (
    MultiModalData,
    OmniModalProcessingKwargs,
    OmniModalProcessorMixin,
    ProcessingKwargs,
    Unpack,
    VideosKwargs,
)
from ...tokenization_utils_base import PreTokenizedInput, TextInput
from ...utils import auto_docstring, logging
from ...utils.deprecation import deprecate_attr
from ...video_utils import VideoInput


logger = logging.get_logger(__name__)


class DeprecatedQwen3OmniMoeVideosKwargs(VideosKwargs, total=False):
    """
    min_pixels (`int`, *optional*):
        Minimum number of pixels (height × width) for video frames after resizing. Frames smaller than this
        threshold will be upscaled to meet the minimum requirement.
    max_pixels (`int`, *optional*):
        Maximum number of pixels (height × width) for video frames after resizing. Frames larger than this
        threshold will be downscaled to fit within the limit.
    patch_size (`int`, *optional*):
        The spatial patch size used by the vision encoder. Video frames are divided into patches of this size
        in both height and width dimensions.
    temporal_patch_size (`int`, *optional*):
        The temporal patch size used by the vision encoder. This determines how many consecutive frames are
        grouped together as a single temporal patch.
    merge_size (`int`, *optional*):
        The merge size used for combining spatial patches. Multiple patches are merged together to reduce the
        sequence length while maintaining spatial information.
    min_frames (`int`, *optional*):
        Minimum number of frames to extract from the video. Videos with fewer frames will be padded or repeated
        to meet this requirement.
    max_frames (`int`, *optional*):
        Maximum number of frames to extract from the video. Longer videos will be truncated or sampled to fit
        within this limit.
    load_audio_from_video (`bool`, *optional*, defaults to `False`):
        Deprecated, use [`Qwen3OmniMoeProcessorKwargs.load_audio_from_video`] instead.
        Whether to incorporate audio information when processing videos. When enabled, audio tokens are
        interleaved with video tokens based on temporal alignment, creating a unified multimodal representation.
    use_audio_in_video (`bool`, *optional*, defaults to `False`):
        Deprecated, use [`Qwen3OmniMoeProcessorKwargs.load_audio_from_video`] instead.
        Whether to incorporate audio information when processing videos. When enabled, audio tokens are
        interleaved with video tokens based on temporal alignment, creating a unified multimodal representation.
    seconds_per_chunk (`float`, *optional*, defaults to `2.0`):
        Deprecated, use [`Qwen3OmniMoeProcessorKwargs.seconds_per_chunk`] instead.
        The duration (in seconds) of each video chunk when splitting long videos. This parameter controls how
        videos are divided into temporal segments for processing.
    position_id_per_seconds (`int` or `float`, *optional*, defaults to `25`):
        Deprecated, use [`Qwen3OmniMoeProcessorKwargs.position_id_per_seconds`] instead.
        The number of position IDs allocated per second of video. This parameter controls the temporal resolution
        of position embeddings and is used to align video tokens with audio tokens when `use_audio_in_video=True`.
    """

    min_pixels: int
    max_pixels: int
    patch_size: int
    temporal_patch_size: int
    merge_size: int
    min_frames: int
    max_frames: int
    use_audio_in_video: bool
    load_audio_from_video: bool
    seconds_per_chunk: float
    position_id_per_seconds: int | float


class Qwen3OmniMoeProcessorKwargs(OmniModalProcessingKwargs, total=False):
    """
    seconds_per_chunk (`float`, *optional*, defaults to `2.0`):
        The duration (in seconds) of each video chunk when splitting long videos. This parameter controls how
        videos are divided into temporal segments for processing.
    position_id_per_seconds (`int` or `float`, *optional*, defaults to `13.0`):
        The number of position IDs allocated per second of video. This parameter controls the temporal resolution
        of position embeddings and is used to align video tokens with audio tokens when `load_audio_from_video=True`.
    n_window (`int`, *optional*, defaults to `50`):
        The audio encoder window size. This should match `n_window` in the model configuration.
    """

    videos_kwargs: DeprecatedQwen3OmniMoeVideosKwargs

    seconds_per_chunk: float
    position_id_per_seconds: int | float
    n_window: int


def _get_feat_extract_output_lengths(input_lengths, n_window=50):
    """
    Computes the output length of the convolutional layers and the output length of the audio encoder
    """
    chunk_len = n_window * 2
    input_lengths_leave = input_lengths % chunk_len
    feat_lengths = (input_lengths_leave - 1) // 2 + 1
    return ((feat_lengths - 1) // 2 + 1 - 1) // 2 + 1 + (input_lengths // chunk_len) * 13


@auto_docstring
class Qwen3OmniMoeProcessor(OmniModalProcessorMixin):
    valid_processor_kwargs = Qwen3OmniMoeProcessorKwargs
    text_kwargs = {
        "padding_side": "left",
    }
    videos_kwargs = {
        "size": {
            "shortest_edge": 128 * 32 * 32,
            "longest_edge": 768 * 32 * 32,
        },
    }
    audio_kwargs = {
        "sampling_rate": 16000,
        "padding": True,
        "truncation": False,
        "return_attention_mask": True,
    }
    seconds_per_chunk = 2.0
    position_id_per_seconds = 13.0
    n_window = 50

    def __init__(
        self, image_processor=None, video_processor=None, feature_extractor=None, tokenizer=None, chat_template=None
    ):
        super().__init__(image_processor, video_processor, feature_extractor, tokenizer, chat_template=chat_template)
        self.image_token = self.tokenizer.image_token
        self.audio_token = self.tokenizer.audio_token
        self.video_token = self.tokenizer.video_token
        self.image_token_id = self.tokenizer.image_token_id
        self.audio_token_id = self.tokenizer.audio_token_id
        self.video_token_id = self.tokenizer.video_token_id
        self.vision_bos_token = self.tokenizer.vision_bos_token
        self.vision_eos_token = self.tokenizer.vision_eos_token
        self.audio_bos_token = self.tokenizer.audio_bos_token
        self.audio_eos_token = self.tokenizer.audio_eos_token

    @auto_docstring
    def __call__(
        self,
        text: TextInput | PreTokenizedInput | list[TextInput] | list[PreTokenizedInput] = None,
        images: ImageInput | None = None,
        videos: VideoInput | None = None,
        audio: AudioInput | None = None,
        **kwargs: Unpack[Qwen3OmniMoeProcessorKwargs],
    ) -> BatchFeature:
        audio_kwargs = dict(kwargs.pop("audio_kwargs", {}))
        if "n_window" in audio_kwargs:
            logger.warning_once(
                "Passing `n_window` in `audio_kwargs` is deprecated "
                "and will be removed in v5.29.0. Pass it directly to the processor instead."
            )
        # "audio_kwargs" has priority for backwards compatibility. The omni hooks read it from "audio_kwargs"
        audio_kwargs.setdefault("n_window", kwargs.pop("n_window", self.n_window))
        videos_kwargs = dict(kwargs.pop("videos_kwargs", {}))
        for key in ("seconds_per_chunk", "position_id_per_seconds"):
            if key in videos_kwargs:
                logger.warning_once(
                    f"Passing `{key}` in `videos_kwargs` is deprecated "
                    "and will be removed in v5.29.0. "
                    "Pass it directly to the processor instead."
                )
                # "videos_kwargs" has priority for backwards compatibility
                kwargs[key] = videos_kwargs.pop(key)

        if kwargs.pop("use_audio_in_video", False) or videos_kwargs.pop("use_audio_in_video", False):
            logger.warning(
                "`use_audio_in_video` is depreacted and will be removed in v5.25, use `load_audio_from_video` instead"
            )
            kwargs["load_audio_from_video"] = True
        return super().__call__(
            images=images, text=text, videos=videos, audio=audio, videos_kwargs=videos_kwargs, **kwargs
        )

    def validate_inputs(
        self,
        images: ImageInput | None = None,
        text: TextInput | PreTokenizedInput | list[TextInput] | list[PreTokenizedInput] | None = None,
        **kwargs: Unpack[ProcessingKwargs],
    ):
        super().validate_inputs(images=images, text=text, **kwargs)

        if text is None:
            raise ValueError("You need to specify either a `text` input to process.")

    def _process_audio(self, audio: AudioInput, **kwargs):
        n_window = kwargs.pop("n_window", self.n_window)
        audio_inputs = self.feature_extractor(audio, **kwargs)
        audio_inputs["feature_attention_mask"] = audio_inputs.pop(
            "attention_mask"
        )  # rename feature_attention_mask to prevent conflicts later on
        audio_inputs["num_audio_tokens"] = _get_feat_extract_output_lengths(
            audio_inputs["feature_attention_mask"].sum(-1), n_window
        )

        audio_replacements = []
        for idx in range(len(audio)):
            replacement_text = self.replace_audio_token(audio_inputs, audio_idx=idx)
            audio_replacements.append(replacement_text)

        return audio_inputs, audio_replacements

    def _process_videos(
        self,
        videos: VideoInput,
        audio: AudioInput | None = None,
        processed_audio: dict | None = None,
        video_to_audio_indices: list | None = None,
        **kwargs: Unpack[Qwen3OmniMoeProcessorKwargs],
    ):
        processed_videos = self.video_processor(videos, **kwargs["videos_kwargs"])

        fps = kwargs["videos_kwargs"].get("fps")
        fps = fps if fps is not None else 1.0
        video_grid_thw = processed_videos["video_grid_thw"]
        processed_videos["video_second_per_grid"] = [self.video_processor.temporal_patch_size / fps] * len(
            video_grid_thw
        )

        video_replacements = []
        for idx in range(len(video_grid_thw)):
            replacement_text = self.replace_video_token(
                processed_videos,
                video_idx=idx,
                processed_audio=processed_audio,
                video_to_audio_indices=video_to_audio_indices,
                **kwargs,
            )
            video_replacements.append(replacement_text)
        return processed_videos, video_replacements

    def replace_image_token(self, image_inputs: dict, image_idx: int, **kwargs) -> str:
        image_seq_length = image_inputs["image_grid_thw"][image_idx].prod() // self.image_processor.merge_size**2
        return self.image_token * image_seq_length

    def replace_audio_token(self, audio_inputs: dict, audio_idx: int, **kwargs) -> str:
        num_audio_tokens = audio_inputs["num_audio_tokens"][audio_idx]
        return self.audio_token * num_audio_tokens

    def replace_video_token(
        self,
        video_inputs: dict,
        video_idx: int,
        processed_audio: dict | None = None,
        video_to_audio_indices: list | None = None,
        **kwargs: Unpack[Qwen3OmniMoeProcessorKwargs],
    ) -> str:
        # When loading audio from video and if current video has an associated audio, i.e. no silent
        if kwargs["load_audio_from_video"] and video_to_audio_indices[video_idx] is not None:
            return self.replace_video_with_audio_token(
                video_inputs,
                video_idx,
                processed_audio,
                video_to_audio_indices[video_idx],
                **kwargs,
            )

        video_seq_length = video_inputs["video_grid_thw"][video_idx].prod() // self.video_processor.merge_size**2
        return self.video_token * video_seq_length

    def replace_video_with_audio_token(
        self,
        video_inputs: dict,
        video_idx: int,
        audio_inputs: dict,
        audio_idx: int,
        **kwargs: Unpack[Qwen3OmniMoeProcessorKwargs],
    ) -> str:
        # Each video grid (`temporal_patch_size` frames) goes at its start time, followed by the audio tokens up to
        # the next grid. Times are counted in position ids, one per audio token
        audio_token_indices = np.arange(audio_inputs["num_audio_tokens"][audio_idx])
        curr_video_grid_thw = video_inputs["video_grid_thw"][video_idx]
        height = curr_video_grid_thw[1] // self.video_processor.merge_size
        width = curr_video_grid_thw[2] // self.video_processor.merge_size
        video_token_indices = np.arange(curr_video_grid_thw[0]).reshape(-1, 1, 1)
        video_token_indices = np.broadcast_to(
            video_token_indices, (video_token_indices.shape[0], height, width)
        ).reshape(-1)
        video_token_indices = (
            video_token_indices * video_inputs["video_second_per_grid"][video_idx] * kwargs["position_id_per_seconds"]
        )
        video_data_index, audio_data_index = 0, 0
        placeholder_string = self.audio_bos_token
        while video_data_index < len(video_token_indices) and audio_data_index < len(audio_token_indices):
            if video_token_indices[video_data_index] <= audio_token_indices[audio_data_index]:
                placeholder_string += self.video_token
                video_data_index += 1
            else:
                placeholder_string += self.audio_token
                audio_data_index += 1
        if video_data_index < len(video_token_indices):
            placeholder_string += self.video_token * (len(video_token_indices) - video_data_index)
        if audio_data_index < len(audio_token_indices):
            placeholder_string += self.audio_token * (len(audio_token_indices) - audio_data_index)
        placeholder_string += self.audio_eos_token
        return placeholder_string

    def get_chunked_index(self, token_indices: np.ndarray, tokens_per_chunk: int) -> list[tuple[int, int]]:
        """
        Splits token index list into chunks based on token value ranges.

        Given a list of token indices, returns a list of (start, end) index tuples representing
        slices of the list where the token values fall within successive ranges of `t_ntoken_per_chunk`.

        For example, if `t_ntoken_per_chunk` is 1000, the function will create chunks such that:
        - the first chunk contains token values < 1000,
        - the second chunk contains values >= 1000 and < 2000, and so on.

        Parameters:
            token_indices (`np.ndarray`): A monotonically increasing list of token index values.
            t_ntoken_per_chunk (`int`): Number of tokens per chunk (used as the chunk size threshold).

        Returns:
            `list[tuple[int, int]]`: A list of tuples, each representing the start (inclusive)
                                and end (exclusive) indices of a chunk in `token_indices`.
        """

        def _iter():
            i, start_idx = 0, 0  # skip bos token
            current_chunk = 1
            while i < len(token_indices):  # skip eos token
                if token_indices[i] >= current_chunk * tokens_per_chunk:
                    yield (start_idx, i)
                    start_idx = i
                    current_chunk += 1
                i += 1
            yield (start_idx, len(token_indices))

        return list(_iter())

    def post_process_image_text_to_text(self, generated_outputs, skip_special_tokens=True, **kwargs):
        """
        Post-process the output of a vlm to decode the text.

        Args:
            generated_outputs (`torch.Tensor` or `np.ndarray`):
                The output of the model `generate` function. The output is expected to be a tensor of shape `(batch_size, sequence_length)`
                or `(sequence_length,)`.
            skip_special_tokens (`bool`, *optional*, defaults to `True`):
                Whether or not to remove special tokens in the output. Argument passed to the tokenizer's `batch_decode` method.
            **kwargs:
                Additional arguments to be passed to the tokenizer's `batch_decode method`.

        Returns:
            `list[str]`: The decoded text.
        """
        if isinstance(generated_outputs, (tuple, list)):
            generated_outputs = generated_outputs[0]
        return self.tokenizer.batch_decode(generated_outputs, skip_special_tokens=skip_special_tokens, **kwargs)

    def post_process_multimodal_output(
        self, generated_outputs, skip_special_tokens=True, generation_mode=None, **kwargs
    ):
        """
        Post-process the output of a multimodal model to return the requested modality output.
        If the model cannot generated the requested modality, an error will be raised.

        Args:
            generated_outputs (`torch.Tensor` or `np.ndarray`):
                The output of the model `generate` function. The output is expected to be a tensor of shape `(batch_size, sequence_length)`
                or `(sequence_length,)`.
            skip_special_tokens (`bool`, *optional*, defaults to `True`):
                Whether or not to remove special tokens in the output. Argument passed to the tokenizer's `batch_decode` method.
            generation_mode (`str`, *optional*):
                Generation mode indicated which modality to output and can be one of `["text", "image", "audio"]`.
            **kwargs:
                Additional arguments to be passed to the tokenizer's `batch_decode method`.

        Returns:
            `list[Union[str, np.ndarray]]`: The decoded text or generated audio.
        """
        if generation_mode is None or generation_mode == "text":
            return self.post_process_image_text_to_text(
                generated_outputs, skip_special_tokens=skip_special_tokens, **kwargs
            )

        elif generation_mode == "audio":
            # Batched generation returns one waveform per sample, while a single sample comes back as a lone
            # `(num_samples,)` tensor that return as a list
            audio_outputs = generated_outputs[1]
            if not isinstance(audio_outputs, (list, tuple)):
                audio_outputs = [audio_outputs]
            return [audio.reshape(-1).detach().cpu().numpy() for audio in audio_outputs]

        else:
            raise ValueError(
                f"{self.__class__.__name__} got an unexpected generation_mode={generation_mode}. Supported options are only `text` and `audio"
            )

    def _get_num_multimodal_tokens(self, image_sizes=None, video_sizes=None, audio_lengths=None, **kwargs):
        """
        Computes the number of placeholder tokens needed for multimodal inputs with the given sizes.
        Args:
            image_sizes (`list[list[int]]`, *optional*):
                The input sizes formatted as (height, width) per each image.
            video_sizes (`list[list[int]]`, *optional*):
                The input sizes formatted as (num_frames, height, width) per each video.
            audio_lengths (`list[int]`, *optional*):
                The lengths of audio inputs in number of samples. Used to dynamically
                compute per-audio token counts.
        Returns:
            `MultiModalData`: A `MultiModalData` object holding number of tokens per each of the provided
            input modalities, along with other useful data.
        """

        vision_data = {}
        merged_kwargs = self._merge_kwargs(**kwargs)
        if image_sizes is not None:
            images_kwargs = merged_kwargs["images_kwargs"]
            merge_size = images_kwargs.get("merge_size", None) or self.image_processor.merge_size

            num_image_patches = [
                self.image_processor.get_number_of_image_patches(*image_size, images_kwargs)
                for image_size in image_sizes
            ]
            num_image_tokens = [(num_patches // merge_size**2) for num_patches in num_image_patches]
            vision_data.update({"num_image_tokens": num_image_tokens, "num_image_patches": num_image_patches})

        if video_sizes is not None:
            videos_kwargs = merged_kwargs["videos_kwargs"]
            merge_size = videos_kwargs.get("merge_size", None) or self.video_processor.merge_size
            num_video_patches = [
                self.video_processor.get_num_of_video_patches(*video_size, videos_kwargs) for video_size in video_sizes
            ]
            num_video_tokens = [(num_patches // merge_size**2) for num_patches in num_video_patches]
            vision_data["num_video_tokens"] = num_video_tokens

        if audio_lengths is not None:
            vision_data["num_audio_tokens"] = [
                self._get_num_audio_tokens(length, **merged_kwargs["audio_kwargs"]) for length in audio_lengths
            ]

        return MultiModalData(**vision_data)

    def _get_num_audio_tokens(self, audio_length: int, **kwargs) -> int:
        """
        Computes the output length of the convolutional layers and the output length of the audio encoder
        """
        n_window = kwargs.get("n_window", self.n_window)
        input_lengths = audio_length // self.feature_extractor.hop_length
        return int(_get_feat_extract_output_lengths(input_lengths, n_window))

    @property
    def unused_input_names(self) -> list[str]:
        "Input names returned always by subprocessors but not used in model's `forward`"
        return ["num_audio_tokens"]

    @property
    def model_input_names(self):
        tokenizer_input_names = self.tokenizer.model_input_names
        feature_extractor_input_names = self.feature_extractor.model_input_names
        image_processor_input_names = self.image_processor.model_input_names
        video_processor_input_names = self.video_processor.model_input_names
        return list(
            dict.fromkeys(
                tokenizer_input_names
                + feature_extractor_input_names
                + image_processor_input_names
                + video_processor_input_names
                + ["feature_attention_mask"]
                + ["video_second_per_grid"]
            )
        )

    def apply_chat_template(self, conversations, chat_template=None, **kwargs):
        return super().apply_chat_template(conversations, chat_template, **kwargs)


__getattr__ = deprecate_attr(
    __name__, remove={"Qwen3OmniMoeVideosKwargs": (DeprecatedQwen3OmniMoeVideosKwargs, "5.29.0")}
)

__all__ = ["Qwen3OmniMoeProcessor"]
