# Copyright 2026 NAVER Corp. and The HuggingFace Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""PyTorch HyperCLOVAX Vision V2 model."""

import json
import re

import torch
from huggingface_hub.dataclasses import strict
from torch import nn

from ...cache_utils import Cache
from ...configuration_utils import PreTrainedConfig, SubConfigSpec
from ...modeling_outputs import (
    BaseModelOutputWithPast,
    BaseModelOutputWithPooling,
    CausalLMOutputWithPast,
)
from ...modeling_utils import slice_logits_to_keep
from ...processing_utils import ProcessingKwargs, Unpack
from ...utils import TransformersKwargs, auto_docstring, can_return_tuple, logging
from ...utils.deprecation import deprecate_attr
from ...utils.generic import accepts_precomputed_kwargs
from ..auto import AutoConfig
from ..exaone4_5.modeling_exaone4_5 import Exaone4_5_ForConditionalGeneration
from ..exaone4_5.processing_exaone4_5 import Exaone4_5_Processor
from ..video_llama_3.modeling_video_llama_3 import VideoLlama3Model, VideoLlama3PreTrainedModel


logger = logging.get_logger(__name__)


@auto_docstring(checkpoint="naver-hyperclovax/HyperCLOVAX-SEED-Think-32B")
@strict
class HyperCLOVAXVisionV2Config(PreTrainedConfig):
    r"""
    ```python
    >>> from transformers import HyperCLOVAXVisionV2Config, HyperCLOVAXVisionV2ForConditionalGeneration

    >>> # Initializing a HyperCLOVAX Vision V2 configuration with defaults
    >>> configuration = HyperCLOVAXVisionV2Config()

    >>> # Initializing a model from the configuration
    >>> model = HyperCLOVAXVisionV2ForConditionalGeneration(configuration)

    >>> # Accessing the model configuration
    >>> configuration = model.config
    ```
    """

    model_type = "hyperclovax_vision_v2"
    sub_configs_defaults = {
        "vision_config": SubConfigSpec(config_class=AutoConfig, model_type="qwen2_5_vl_vision"),
        "text_config": SubConfigSpec(config_class=AutoConfig, model_type="hyperclovax"),
    }
    keys_to_ignore_at_inference = ["past_key_values"]

    text_config: dict | PreTrainedConfig | None = None
    vision_config: dict | PreTrainedConfig | None = None
    image_token_id: int = 128060
    video_token_id: int = 128061
    tie_word_embeddings: bool = True

    def __post_init__(self, **kwargs):
        # The Hub config uses the full Qwen2.5-VL type for the vision transformer.
        if isinstance(self.vision_config, dict) and self.vision_config.get("model_type") == "qwen2_5_vl":
            self.vision_config["model_type"] = "qwen2_5_vl_vision"

        # This is necessary to properly find the weight conversion mapping.
        if kwargs.get("model_type") == "vlm":
            kwargs["model_type"] = "hyperclovax_vision_v2"

        super().__post_init__(**kwargs)


@auto_docstring
class HyperCLOVAXVisionV2PreTrainedModel(VideoLlama3PreTrainedModel):
    config: HyperCLOVAXVisionV2Config
    # Both towers are built with `AutoModel.from_config` and register their own layers in `post_init`
    _no_split_modules = []
    input_modalities = ("image", "video", "text")


@auto_docstring
class HyperCLOVAXVisionV2Model(HyperCLOVAXVisionV2PreTrainedModel, VideoLlama3Model):
    def __init__(self, config: HyperCLOVAXVisionV2Config):
        super().__init__(config)
        self.projector = nn.Linear(config.vision_config.out_hidden_size, config.text_config.hidden_size)

    @accepts_precomputed_kwargs(modality="video")
    @can_return_tuple
    @auto_docstring
    def get_video_features(
        self,
        pixel_values_videos: torch.Tensor,
        video_grid_thw: torch.LongTensor,
        **kwargs: Unpack[TransformersKwargs],
    ) -> tuple | BaseModelOutputWithPooling:
        r"""
        pixel_values_videos (`torch.Tensor` of shape `(batch_size, num_channels, image_size, image_size)`):
            The tensors corresponding to the input videos.
        """
        return self.get_image_features(pixel_values=pixel_values_videos, image_grid_thw=video_grid_thw, **kwargs)

    @accepts_precomputed_kwargs(modality="image")
    @can_return_tuple
    @auto_docstring
    def get_image_features(
        self,
        pixel_values: torch.Tensor,
        image_grid_thw: torch.LongTensor,
        **kwargs: Unpack[TransformersKwargs],
    ) -> tuple | BaseModelOutputWithPooling:
        r"""
        pixel_values (`torch.Tensor` of shape `(batch_size, num_channels, image_size, image_size)`):
            The tensors corresponding to the input images.
        """
        vision_outputs = self.vision_model(pixel_values, grid_thw=image_grid_thw, return_dict=True, **kwargs)
        image_embeds = self.projector(vision_outputs.pooler_output)

        split_sizes = (image_grid_thw.prod(-1) // self.vision_model.spatial_merge_size**2).tolist()
        vision_outputs.pooler_output = torch.split(image_embeds, split_sizes)

        return vision_outputs

    @can_return_tuple
    @auto_docstring
    def forward(
        self,
        input_ids: torch.LongTensor | None = None,
        attention_mask: torch.Tensor | None = None,
        position_ids: torch.LongTensor | None = None,
        past_key_values: Cache | None = None,
        inputs_embeds: torch.Tensor | None = None,
        use_cache: bool | None = None,
        pixel_values: torch.Tensor | None = None,
        pixel_values_videos: torch.Tensor | None = None,
        image_grid_thw: torch.LongTensor | None = None,
        video_grid_thw: torch.LongTensor | None = None,
        mm_encoder_outputs: dict[str, BaseModelOutputWithPooling] | None = None,
        **kwargs: Unpack[TransformersKwargs],
    ) -> tuple | CausalLMOutputWithPast:
        r"""
        pixel_values (`torch.Tensor`, *optional*):
            Pixel values of input images after preprocessing by [`Qwen2VLImageProcessor`].
            A 2D tensor of shape `(total_num_patches, channels * patch_size^2 * temporal_patch_size)`.
            In the input token sequence, each image position should contain `config.image_token_id`.
        pixel_values_videos (`torch.Tensor`, *optional*):
            Pixel values of input videos, with the same format as `pixel_values`.
        image_grid_thw (`torch.LongTensor` of shape `(num_images, 3)`, *optional*):
            The temporal, height and width dimensions of the feature grid for each image.
            Each row contains `[temporal, height, width]` grid counts.
        video_grid_thw (`torch.LongTensor` of shape `(num_videos, 3)`, *optional*):
            The temporal, height and width dimensions of the feature grid for each video.
        """
        if (input_ids is None) ^ (inputs_embeds is not None):
            raise ValueError("You must specify exactly one of input_ids or inputs_embeds")

        if (pixel_values is not None or pixel_values_videos is not None) and mm_encoder_outputs is not None:
            raise ValueError(
                "You cannot specify both pixel_values/pixel_values_videos and mm_encoder_outputs at the same time"
            )

        if inputs_embeds is None:
            inputs_embeds = self.get_input_embeddings()(input_ids)

        mm_encoder_outputs = mm_encoder_outputs if mm_encoder_outputs is not None else {}
        if mm_encoder_outputs.get("image") is None and pixel_values is not None:
            mm_encoder_outputs["image"] = self.get_image_features(
                pixel_values, image_grid_thw, return_dict=True, **kwargs
            )

        if mm_encoder_outputs.get("video") is None and pixel_values_videos is not None:
            mm_encoder_outputs["video"] = self.get_video_features(
                pixel_values_videos, video_grid_thw, return_dict=True, **kwargs
            )

        if mm_encoder_outputs.get("image") is not None:
            image_embeds = torch.cat(mm_encoder_outputs["image"].pooler_output, dim=0).to(
                inputs_embeds.device, inputs_embeds.dtype
            )
            image_mask, _ = self.get_placeholder_mask(
                input_ids, inputs_embeds=inputs_embeds, image_features=image_embeds
            )
            inputs_embeds = inputs_embeds.masked_scatter(image_mask, image_embeds)

        if mm_encoder_outputs.get("video") is not None:
            video_embeds = torch.cat(mm_encoder_outputs["video"].pooler_output, dim=0).to(
                inputs_embeds.device, inputs_embeds.dtype
            )
            _, video_mask = self.get_placeholder_mask(
                input_ids, inputs_embeds=inputs_embeds, video_features=video_embeds
            )
            inputs_embeds = inputs_embeds.masked_scatter(video_mask, video_embeds)

        outputs = self.language_model(
            input_ids=None,
            position_ids=position_ids,
            attention_mask=attention_mask,
            past_key_values=past_key_values,
            inputs_embeds=inputs_embeds,
            use_cache=use_cache,
            **kwargs,
        )
        return BaseModelOutputWithPast(
            last_hidden_state=outputs.last_hidden_state,
            past_key_values=outputs.past_key_values,
            hidden_states=outputs.hidden_states,
            attentions=outputs.attentions,
        )


@auto_docstring
class HyperCLOVAXVisionV2ForConditionalGeneration(
    HyperCLOVAXVisionV2PreTrainedModel, Exaone4_5_ForConditionalGeneration
):
    def __init__(self, config: HyperCLOVAXVisionV2Config):
        super().__init__(config)
        self.model = HyperCLOVAXVisionV2Model(config)

    @can_return_tuple
    @auto_docstring
    def forward(
        self,
        input_ids: torch.LongTensor | None = None,
        pixel_values: torch.Tensor | None = None,
        pixel_values_videos: torch.Tensor | None = None,
        image_grid_thw: torch.LongTensor | None = None,
        video_grid_thw: torch.LongTensor | None = None,
        attention_mask: torch.Tensor | None = None,
        position_ids: torch.LongTensor | None = None,
        past_key_values: Cache | None = None,
        inputs_embeds: torch.Tensor | None = None,
        labels: torch.LongTensor | None = None,
        use_cache: bool | None = None,
        logits_to_keep: int | torch.Tensor = 0,
        mm_encoder_outputs: dict[str, BaseModelOutputWithPooling] | None = None,
        **kwargs: Unpack[TransformersKwargs],
    ) -> tuple | CausalLMOutputWithPast:
        r"""
        pixel_values (`torch.Tensor`, *optional*):
            Pixel values of input images after preprocessing.
        pixel_values_videos (`torch.Tensor`, *optional*):
            Pixel values of input videos, same format as `pixel_values`.
        image_grid_thw (`torch.LongTensor` of shape `(num_images, 3)`, *optional*):
            `[temporal, height, width]` grid counts per image.
        video_grid_thw (`torch.LongTensor` of shape `(num_videos, 3)`, *optional*):
            `[temporal, height, width]` grid counts per video.
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the masked language modeling loss.
        logits_to_keep (`int` or `torch.Tensor`, *optional*, defaults to 0):
            If an `int`, compute logits for the last `logits_to_keep` tokens.

        Example:

        ```python
        >>> from PIL import Image
        >>> import requests
        >>> from transformers import AutoProcessor, HyperCLOVAXVisionV2ForConditionalGeneration

        >>> model = HyperCLOVAXVisionV2ForConditionalGeneration.from_pretrained(
        ...     "naver-hyperclovax/HyperCLOVAX-SEED-Think-32B", device_map="auto"
        ... )
        >>> processor = AutoProcessor.from_pretrained("naver-hyperclovax/HyperCLOVAX-SEED-Think-32B")

        >>> messages = [
        ...     {"role": "user", "content": [
        ...         {"type": "image", "url": "http://images.cocodataset.org/val2017/000000039769.jpg"},
        ...         {"type": "text", "text": "Describe this image in detail."},
        ...     ]}
        ... ]
        >>> inputs = processor.apply_chat_template(
        ...     messages, tokenize=True, return_dict=True, add_generation_prompt=True, return_tensors="pt"
        ... ).to(model.device)
        >>> output = model.generate(**inputs, max_new_tokens=200)
        >>> processor.decode(output[0], skip_special_tokens=True)
        ```
        """
        outputs = self.model(
            input_ids=input_ids,
            pixel_values=pixel_values,
            pixel_values_videos=pixel_values_videos,
            image_grid_thw=image_grid_thw,
            video_grid_thw=video_grid_thw,
            past_key_values=past_key_values,
            attention_mask=attention_mask,
            position_ids=position_ids,
            inputs_embeds=inputs_embeds,
            use_cache=use_cache,
            mm_encoder_outputs=mm_encoder_outputs,
            **kwargs,
        )
        hidden_states = outputs.last_hidden_state
        hidden_states = slice_logits_to_keep(hidden_states, logits_to_keep)
        # Key difference: additional logits scaling applied
        logits = self.lm_head(hidden_states) * self.config.text_config.logits_scaling

        loss = None
        if labels is not None:
            loss = self.loss_function(
                logits=logits, labels=labels, vocab_size=self.config.text_config.vocab_size, **kwargs
            )

        return CausalLMOutputWithPast(
            loss=loss,
            logits=logits,
            past_key_values=outputs.past_key_values,
            hidden_states=outputs.hidden_states,
            attentions=outputs.attentions,
        )


@auto_docstring
class HyperCLOVAXVisionV2Processor(Exaone4_5_Processor):
    # Stopgap for `video_duration`: the hub template renders it as text, leaving the literal
    # `<|video_duration|>` placeholder when missing. These three overrides fill it in after the fact,
    # instead of the single `replace_video_token` hook other video models use (qwen3_vl, glm4v). Remove
    # once the hub template is simplified to emit one placeholder token per video.
    # See https://github.com/huggingface/transformers/pull/44314#issuecomment-5569986614
    video_duration_token = "<|video_duration|>"

    def __init__(self, image_processor=None, tokenizer=None, video_processor=None, chat_template=None, **kwargs):
        super().__init__(image_processor, tokenizer, video_processor, chat_template=chat_template, **kwargs)
        self._pending_video_durations = []
        self._pending_needs_placeholder = None

    def apply_chat_template(self, conversation, **kwargs):
        is_batched = isinstance(conversation[0], (list, tuple)) if conversation else False
        conversations = conversation if is_batched else [conversation]
        self._pending_needs_placeholder = [
            "video_duration" not in block
            for conv in conversations
            for message in conv
            for block in (message.get("content") or [])
            if isinstance(block, dict) and block.get("type") == "video"
        ]
        result = super().apply_chat_template(conversation, **kwargs)
        return result

    def _process_videos(self, videos, **kwargs):
        processed_videos, video_replacements = super()._process_videos(videos, **kwargs)
        all_durations = [
            round(float(m.duration if m.duration is not None else m.total_num_frames / (m.fps or 24)), 2)
            for m in processed_videos.get("video_metadata", [])
        ]
        needs = self._pending_needs_placeholder
        self._pending_video_durations = (
            [d for d, n in zip(all_durations, needs) if n]
            if needs is not None and len(needs) == len(all_durations)
            else all_durations
        )
        self._pending_needs_placeholder = None
        return processed_videos, video_replacements

    def get_text_with_replacements(
        self,
        text: list[str],
        images_replacements: list[str] = [],
        videos_replacements: list[str] = [],
        audio_replacements: list[str] = [],
    ):
        durations = iter(self._pending_video_durations)
        pattern = re.escape(self.video_duration_token)
        text = [re.sub(pattern, lambda _: json.dumps(next(durations)), sample) for sample in text]
        self._pending_video_durations = []
        result = super().get_text_with_replacements(text, images_replacements, videos_replacements, audio_replacements)
        return result


__getattr__ = deprecate_attr(__name__, rename={"HyperCLOVAXVisionV2_ProcessorKwargs": (ProcessingKwargs, "5.29.0")})

__all__ = [
    "HyperCLOVAXVisionV2Config",
    "HyperCLOVAXVisionV2ForConditionalGeneration",
    "HyperCLOVAXVisionV2Model",
    "HyperCLOVAXVisionV2PreTrainedModel",
    "HyperCLOVAXVisionV2Processor",
]
