#                🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨
#           This file was automatically generated from src/transformers/models/granite_docling/modular_granite_docling.py.
#               Do NOT edit this file manually as any edits will be overwritten by the generation of
#             the file from the modular. If any change should be done, please apply the change to the
#                          modular_granite_docling.py file directly. One of our CI enforces this.
#                🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨
# Copyright 2026 IBM and The HuggingFace Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from collections.abc import Callable
from dataclasses import dataclass

import torch
from torch import nn

from ... import initialization as init
from ...activations import ACT2FN
from ...cache_utils import Cache, DynamicCache
from ...generation import GenerationMixin
from ...integrations import use_kernel_forward_from_hub, use_kernelized_func
from ...masking_utils import create_causal_mask
from ...modeling_layers import GradientCheckpointingLayer
from ...modeling_outputs import BaseModelOutput, BaseModelOutputWithPast, BaseModelOutputWithPooling, ModelOutput
from ...modeling_rope_utils import ROPE_INIT_FUNCTIONS, dynamic_rope_update
from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel, slice_logits_to_keep
from ...processing_utils import Unpack
from ...utils import TransformersKwargs, auto_docstring, can_return_tuple, torch_int
from ...utils.generic import merge_with_config_defaults
from ...utils.import_utils import torch_compilable_check
from ...utils.output_capturing import capture_outputs
from .configuration_granite_docling import GraniteDoclingConfig, GraniteDoclingTextConfig, GraniteDoclingVisionConfig


@auto_docstring(
    custom_intro="""
    Base class for GraniteDocling model outputs, with potential hidden states and attentions.
    """
)
@dataclass
class GraniteDoclingBaseModelOutputWithPast(ModelOutput):
    r"""
    router_logits (`torch.FloatTensor` of shape `(batch_size,)`, *optional*):
        Logits of the density router, one per sample. Returned when the model has a router and `pixel_values` are
        given.
    """

    last_hidden_state: torch.Tensor | None = None
    past_key_values: Cache | None = None
    hidden_states: tuple[torch.Tensor] | None = None
    attentions: tuple[torch.Tensor] | None = None
    image_hidden_states: tuple[torch.Tensor] | None = None

    router_logits: torch.FloatTensor | None = None


@auto_docstring(
    custom_intro="""
    Base class for GraniteDocling causal language model (or autoregressive) outputs.
    """
)
@dataclass
class GraniteDoclingCausalLMOutputWithPast(ModelOutput):
    r"""
    loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `labels` is provided):
        Language modeling loss (for next-token prediction).
    logits (`torch.FloatTensor` of shape `(batch_size, sequence_length, config.vocab_size)`):
        Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax).
    router_logits (`torch.FloatTensor` of shape `(batch_size,)`, *optional*):
        Logits of the density router, one per sample. Returned when the model has a router and `pixel_values` are
        given.
    """

    loss: torch.Tensor | None = None
    logits: torch.Tensor | None = None
    past_key_values: Cache | None = None
    hidden_states: tuple[torch.Tensor] | None = None
    attentions: tuple[torch.Tensor] | None = None
    image_hidden_states: tuple[torch.Tensor] | None = None

    router_logits: torch.FloatTensor | None = None


@auto_docstring(
    custom_intro="""
    Base class for GraniteDocling image features.
    """
)
@dataclass
class GraniteDoclingImageFeaturesOutput(BaseModelOutputWithPooling):
    r"""
    pooler_output (`tuple[torch.FloatTensor]` of length `batch_size`):
        Image tokens of each sample, in tile order, projected to the text hidden size. Each tensor has shape
        `(num_image_tokens, hidden_size)`. A tile yields `image_seq_len` tokens on the coarse connector path and four
        times as many on the fine path. Generation repeats these per-sample tensors across beams.
    deepstack_features (`list[tuple[torch.FloatTensor]]`, *optional*):
        One projected intermediate vision encoder hidden state per entry of `config.deepstack_visual_indexes`. Each
        entry is split per sample, with the same layout as `pooler_output`.
    router_logits (`torch.FloatTensor` of shape `(batch_size,)`, *optional*):
        Logits of the density router, one per sample. Returned when the model has a router.
    """

    deepstack_features: list[torch.FloatTensor] | None = None
    router_logits: torch.FloatTensor | None = None


def rotate_half(x):
    """Rotates half the hidden dims of the input."""
    x1 = x[..., : x.shape[-1] // 2]
    x2 = x[..., x.shape[-1] // 2 :]
    return torch.cat((-x2, x1), dim=-1)


@use_kernel_forward_from_hub("rotary_pos_emb")
def apply_rotary_pos_emb(q, k, cos, sin, unsqueeze_dim=1):
    """Applies Rotary Position Embedding to the query and key tensors.

    Args:
        q (`torch.Tensor`): The query tensor.
        k (`torch.Tensor`): The key tensor.
        cos (`torch.Tensor`): The cosine part of the rotary embedding.
        sin (`torch.Tensor`): The sine part of the rotary embedding.
        unsqueeze_dim (`int`, *optional*, defaults to 1):
            The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
            sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
            that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
            k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
            cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
            the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
    Returns:
        `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
    """
    cos = cos.unsqueeze(unsqueeze_dim)
    sin = sin.unsqueeze(unsqueeze_dim)
    q_embed = (q * cos) + (rotate_half(q) * sin)
    k_embed = (k * cos) + (rotate_half(k) * sin)
    return q_embed, k_embed


def repeat_kv(hidden_states: torch.Tensor, n_rep: int) -> torch.Tensor:
    """
    This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,
    num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)
    """
    batch, num_key_value_heads, slen, head_dim = hidden_states.shape
    if n_rep == 1:
        return hidden_states
    hidden_states = hidden_states[:, :, None, :, :].expand(batch, num_key_value_heads, n_rep, slen, head_dim)
    return hidden_states.reshape(batch, num_key_value_heads * n_rep, slen, head_dim)


def eager_attention_forward(
    module: nn.Module,
    query: torch.Tensor,
    key: torch.Tensor,
    value: torch.Tensor,
    attention_mask: torch.Tensor | None,
    scaling: float,
    dropout: float = 0.0,
    **kwargs: Unpack[TransformersKwargs],
):
    key_states = repeat_kv(key, module.num_key_value_groups)
    value_states = repeat_kv(value, module.num_key_value_groups)

    attn_weights = torch.matmul(query, key_states.transpose(2, 3)) * scaling
    if attention_mask is not None:
        attn_weights = attn_weights + attention_mask

    attn_weights = nn.functional.softmax(attn_weights, dim=-1, dtype=torch.float32).to(query.dtype)
    attn_weights = nn.functional.dropout(attn_weights, p=dropout, training=module.training)
    attn_output = torch.matmul(attn_weights, value_states)
    attn_output = attn_output.transpose(1, 2).contiguous()

    return attn_output, attn_weights


@use_kernelized_func(apply_rotary_pos_emb)
class GraniteDoclingTextAttention(nn.Module):
    """Multi-headed attention from 'Attention Is All You Need' paper"""

    def __init__(self, config: GraniteDoclingTextConfig, layer_idx: int | None = None):
        super().__init__()
        self.config = config
        self.layer_idx = layer_idx
        self.head_dim = getattr(config, "head_dim", config.hidden_size // config.num_attention_heads)
        self.num_key_value_groups = config.num_attention_heads // config.num_key_value_heads
        self.scaling = config.attention_multiplier
        self.attention_dropout = config.attention_dropout
        self.is_causal = True

        self.q_proj = nn.Linear(
            config.hidden_size, config.num_attention_heads * self.head_dim, bias=config.attention_bias
        )
        self.k_proj = nn.Linear(
            config.hidden_size, config.num_key_value_heads * self.head_dim, bias=config.attention_bias
        )
        self.v_proj = nn.Linear(
            config.hidden_size, config.num_key_value_heads * self.head_dim, bias=config.attention_bias
        )
        self.o_proj = nn.Linear(
            config.num_attention_heads * self.head_dim, config.hidden_size, bias=config.attention_bias
        )

    def forward(
        self,
        hidden_states: torch.Tensor,
        position_embeddings: tuple[torch.Tensor, torch.Tensor] | None = None,
        attention_mask: torch.Tensor | None = None,
        past_key_values: Cache | None = None,
        **kwargs: Unpack[TransformersKwargs],
    ) -> tuple[torch.Tensor, torch.Tensor]:
        input_shape = hidden_states.shape[:-1]
        hidden_shape = (*input_shape, -1, self.head_dim)

        query_states = self.q_proj(hidden_states).view(hidden_shape).transpose(1, 2)
        key_states = self.k_proj(hidden_states).view(hidden_shape).transpose(1, 2)
        value_states = self.v_proj(hidden_states).view(hidden_shape).transpose(1, 2)

        cos, sin = position_embeddings
        query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin)

        if past_key_values is not None:
            key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx)

        attention_interface: Callable = ALL_ATTENTION_FUNCTIONS.get_interface(
            self.config._attn_implementation, eager_attention_forward
        )

        attn_output, attn_weights = attention_interface(
            self,
            query_states,
            key_states,
            value_states,
            attention_mask,
            dropout=0.0 if not self.training else self.attention_dropout,
            scaling=self.scaling,
            **kwargs,
        )

        attn_output = attn_output.reshape(*input_shape, -1).contiguous()
        attn_output = self.o_proj(attn_output)
        return attn_output, attn_weights


@use_kernel_forward_from_hub("RMSNorm")
class GraniteDoclingTextRMSNorm(nn.Module):
    def __init__(self, hidden_size, eps: float = 1e-6) -> None:
        """
        GraniteDoclingTextRMSNorm is equivalent to T5LayerNorm
        """
        super().__init__()
        self.weight = nn.Parameter(torch.ones(hidden_size))
        self.variance_epsilon = eps

    def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
        input_dtype = hidden_states.dtype
        hidden_states = hidden_states.to(torch.float32)
        variance = hidden_states.pow(2).mean(-1, keepdim=True)
        hidden_states = hidden_states * torch.rsqrt(variance + self.variance_epsilon)
        return self.weight * hidden_states.to(input_dtype)

    def extra_repr(self):
        return f"{tuple(self.weight.shape)}, eps={self.variance_epsilon}"


class GraniteDoclingTextMLP(nn.Module):
    def __init__(self, config):
        super().__init__()
        self.config = config
        self.hidden_size = config.hidden_size
        self.intermediate_size = config.intermediate_size
        self.gate_proj = nn.Linear(self.hidden_size, self.intermediate_size, bias=config.mlp_bias)
        self.up_proj = nn.Linear(self.hidden_size, self.intermediate_size, bias=config.mlp_bias)
        self.down_proj = nn.Linear(self.intermediate_size, self.hidden_size, bias=config.mlp_bias)
        self.act_fn = ACT2FN[config.hidden_act]

    def forward(self, x):
        down_proj = self.down_proj(self.act_fn(self.gate_proj(x)) * self.up_proj(x))
        return down_proj


class GraniteDoclingTextDecoderLayer(GradientCheckpointingLayer):
    def __init__(self, config: GraniteDoclingTextConfig, layer_idx: int):
        super().__init__()
        self.hidden_size = config.hidden_size
        self.self_attn = GraniteDoclingTextAttention(config=config, layer_idx=layer_idx)

        self.mlp = GraniteDoclingTextMLP(config)
        self.input_layernorm = GraniteDoclingTextRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
        self.post_attention_layernorm = GraniteDoclingTextRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
        self.residual_multiplier = config.residual_multiplier

    def forward(
        self,
        hidden_states: torch.Tensor,
        attention_mask: torch.Tensor | None = None,
        position_ids: torch.LongTensor | None = None,
        past_key_values: Cache | None = None,
        use_cache: bool | None = False,
        position_embeddings: tuple[torch.Tensor, torch.Tensor] | None = None,
        **kwargs: Unpack[TransformersKwargs],
    ) -> torch.Tensor:
        """
        Args:
            hidden_states (`torch.Tensor`): input to the layer of shape `(batch, seq_len, embed_dim)`
            attention_mask (`torch.Tensor`, *optional*):
                attention mask of size `(batch_size, sequence_length)` if flash attention is used or `(batch_size, 1,
                query_sequence_length, key_sequence_length)` if default attention is used.
            output_attentions (`bool`, *optional*):
                Whether or not to return the attentions tensors of all attention layers. See `attentions` under
                returned tensors for more detail.
            use_cache (`bool`, *optional*):
                If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding
                (see `past_key_values`).
            past_key_values (`Cache`, *optional*): cached past key and value projection states
            position_embeddings (`tuple[torch.Tensor, torch.Tensor]`, *optional*):
                Tuple containing the cosine and sine positional embeddings of shape `(batch_size, seq_len, head_dim)`,
                with `head_dim` being the embedding dimension of each attention head.
            kwargs (`dict`, *optional*):
                Arbitrary kwargs to be ignored, used for FSDP and other methods that injects code
                into the model
        """
        residual = hidden_states

        hidden_states = self.input_layernorm(hidden_states)

        hidden_states, _ = self.self_attn(
            hidden_states=hidden_states,
            attention_mask=attention_mask,
            position_ids=position_ids,
            past_key_values=past_key_values,
            use_cache=use_cache,
            position_embeddings=position_embeddings,
            **kwargs,
        )
        hidden_states = residual + hidden_states * self.residual_multiplier

        residual = hidden_states
        hidden_states = self.post_attention_layernorm(hidden_states)
        hidden_states = self.mlp(hidden_states)
        hidden_states = residual + hidden_states * self.residual_multiplier

        return hidden_states


@auto_docstring
class GraniteDoclingTextPreTrainedModel(PreTrainedModel):
    config: GraniteDoclingTextConfig
    base_model_prefix = "model"
    supports_gradient_checkpointing = True
    _no_split_modules = ["GraniteDoclingTextDecoderLayer"]
    _skip_keys_device_placement = ["past_key_values"]
    _supports_flash_attn = True
    _supports_sdpa = True
    _supports_flex_attn = True

    _can_compile_fullgraph = True
    _supports_attention_backend = True
    _can_record_outputs = {
        "hidden_states": GraniteDoclingTextDecoderLayer,
        "attentions": GraniteDoclingTextAttention,
    }


class GraniteDoclingTextRotaryEmbedding(nn.Module):
    def __init__(self, config: GraniteDoclingTextConfig):
        super().__init__()
        self.max_seq_len_cached = config.max_position_embeddings
        self.original_max_seq_len = config.max_position_embeddings

        self.config = config

        self.rope_type = self.config.rope_parameters["rope_type"]
        rope_init_fn: Callable = self.compute_default_rope_parameters
        if self.rope_type != "default":
            rope_init_fn = ROPE_INIT_FUNCTIONS[self.rope_type]
        inv_freq, self.attention_scaling = rope_init_fn(self.config)

        self.inv_freq = nn.Buffer(inv_freq, persistent=False)
        self.original_inv_freq = nn.Buffer(inv_freq.clone(), persistent=False)

    @staticmethod
    def compute_default_rope_parameters(config: GraniteDoclingTextConfig, **kwargs) -> tuple[torch.Tensor, float]:
        """
        Computes the inverse frequencies according to the original RoPE implementation
        Args:
            config ([`~transformers.PreTrainedConfig`]):
                The model configuration.
        Returns:
            Tuple of (`torch.Tensor`, `float`), containing the inverse frequencies for the RoPE embeddings and the
            post-processing scaling factor applied to the computed cos/sin (unused in this type of RoPE).
        """
        base = config.rope_parameters["rope_theta"]
        dim = getattr(config, "head_dim", None) or config.hidden_size // config.num_attention_heads

        attention_factor = 1.0  # Unused in this type of RoPE
        # Compute the inverse frequencies
        inv_freq = 1.0 / (base ** (torch.arange(0, dim, 2, dtype=torch.float) / dim))
        return inv_freq, attention_factor

    @torch.no_grad()
    @dynamic_rope_update  # power user: used with advanced RoPE types (e.g. dynamic rope)
    def forward(self, x, position_ids):
        freqs = position_ids[..., None].float() * self.inv_freq.to(device=x.device, dtype=torch.float)
        emb = torch.cat((freqs, freqs), dim=-1)
        cos = emb.cos() * self.attention_scaling
        sin = emb.sin() * self.attention_scaling

        return cos.to(dtype=x.dtype), sin.to(dtype=x.dtype)


@auto_docstring
class GraniteDoclingTextModel(GraniteDoclingTextPreTrainedModel):
    def __init__(self, config: GraniteDoclingTextConfig):
        super().__init__(config)
        self.padding_idx = config.pad_token_id
        self.vocab_size = config.vocab_size

        self.embed_tokens = nn.Embedding(config.vocab_size, config.hidden_size, self.padding_idx)
        self.layers = nn.ModuleList(
            [GraniteDoclingTextDecoderLayer(config, layer_idx) for layer_idx in range(config.num_hidden_layers)]
        )
        self.norm = GraniteDoclingTextRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
        self.rotary_emb = GraniteDoclingTextRotaryEmbedding(config=config)
        self.gradient_checkpointing = False
        self.embedding_multiplier = config.embedding_multiplier

        # Initialize weights and apply final processing
        self.post_init()

    @merge_with_config_defaults
    @capture_outputs
    @auto_docstring
    def forward(
        self,
        input_ids: torch.LongTensor | None = None,
        attention_mask: torch.Tensor | None = None,
        position_ids: torch.LongTensor | None = None,
        past_key_values: Cache | None = None,
        inputs_embeds: torch.FloatTensor | None = None,
        use_cache: bool | None = None,
        visual_pos_masks: torch.BoolTensor | None = None,
        deepstack_visual_embeds: list[torch.Tensor] | None = None,
        **kwargs: Unpack[TransformersKwargs],
    ) -> BaseModelOutputWithPast:
        r"""
        visual_pos_masks (`torch.BoolTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Positions of the image tokens that receive the DeepStack features.
        deepstack_visual_embeds (`list[torch.Tensor]`, *optional*):
            DeepStack features, each of shape `(num_image_tokens, hidden_size)`. Entry `i` is added to the hidden
            states at `visual_pos_masks` after decoder layer `i`.
        """
        if (input_ids is None) ^ (inputs_embeds is not None):
            raise ValueError("You must specify exactly one of input_ids or inputs_embeds")

        if inputs_embeds is None:
            inputs_embeds = self.embed_tokens(input_ids)

        inputs_embeds = inputs_embeds * self.embedding_multiplier

        if use_cache and past_key_values is None:
            past_key_values = DynamicCache(config=self.config)

        if position_ids is None:
            past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0
            position_ids = torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device) + past_seen_tokens
            position_ids = position_ids.unsqueeze(0)

        causal_mask = create_causal_mask(
            config=self.config,
            inputs_embeds=inputs_embeds,
            attention_mask=attention_mask,
            past_key_values=past_key_values,
            position_ids=position_ids,
        )

        hidden_states = inputs_embeds
        position_embeddings = self.rotary_emb(hidden_states, position_ids=position_ids)

        for layer_idx, decoder_layer in enumerate(self.layers[: self.config.num_hidden_layers]):
            hidden_states = decoder_layer(
                hidden_states,
                attention_mask=causal_mask,
                position_ids=position_ids,
                past_key_values=past_key_values,
                use_cache=use_cache,
                position_embeddings=position_embeddings,
                **kwargs,
            )
            if deepstack_visual_embeds is not None and layer_idx < len(deepstack_visual_embeds):
                hidden_states = self._deepstack_process(
                    hidden_states,
                    visual_pos_masks,
                    deepstack_visual_embeds[layer_idx],
                )

        hidden_states = self.norm(hidden_states)
        return BaseModelOutputWithPast(
            last_hidden_state=hidden_states,
            past_key_values=past_key_values,
        )

    def _deepstack_process(
        self, hidden_states: torch.Tensor, visual_pos_masks: torch.Tensor, visual_embeds: torch.Tensor
    ):
        visual_pos_masks = visual_pos_masks.to(hidden_states.device)
        visual_embeds = visual_embeds.to(hidden_states.device, hidden_states.dtype)
        hidden_states = hidden_states.clone()
        local_this = hidden_states[visual_pos_masks, :] + visual_embeds
        hidden_states[visual_pos_masks, :] = local_this
        return hidden_states


def build_2d_sinusoidal_position_embedding(
    height: int,
    width: int,
    embed_dim: int = 256,
    temperature: float = 10000.0,
    cls_token: bool = False,
    device: torch.device | None = None,
    dtype: torch.dtype = torch.float32,
) -> torch.Tensor:
    """2D sinusoidal position embeddings for an image patch grid.

    Each (h, w) position gets an ``embed_dim``-dimensional vector laid out as
    ``[sin_h | cos_h | sin_w | cos_w]``, with row-major (H-outer) patch ordering.

    Args:
        height: Grid height in patches.
        width: Grid width in patches.
        embed_dim: Total embedding dimension; must be divisible by 4.
        temperature: Base for the frequency decay.
        cls_token: If `True`, prepend a zero row for a CLS token.
        device: Target device; defaults to CPU.
        dtype: Output dtype; frequency arithmetic uses float64 internally.

    Returns:
        Tensor of shape ``(height * width [+1], embed_dim)``.
    """
    if embed_dim % 4 != 0:
        raise ValueError(f"`embed_dim` must be divisible by 4, got {embed_dim}")

    pos_dim = embed_dim // 4
    # mps doesn't support float64
    compute_dtype = torch.float64 if device is None or device.type != "mps" else torch.float32
    omega = torch.arange(pos_dim, dtype=compute_dtype, device=device) / pos_dim
    omega = 1.0 / temperature**omega  # (D/4,)

    grid_h = torch.arange(height, dtype=compute_dtype, device=device)
    grid_w = torch.arange(width, dtype=compute_dtype, device=device)
    grid_h, grid_w = torch.meshgrid(grid_h, grid_w, indexing="ij")  # (H, W) each

    emb_h = grid_h.flatten().outer(omega)  # (H*W, D/4)
    emb_w = grid_w.flatten().outer(omega)  # (H*W, D/4)

    pos_embed = torch.cat([emb_h.sin(), emb_h.cos(), emb_w.sin(), emb_w.cos()], dim=1)

    if cls_token:
        pos_embed = torch.cat([torch.zeros(1, embed_dim, dtype=compute_dtype, device=device), pos_embed], dim=0)

    return pos_embed.to(dtype)


class GraniteDoclingPositionEmbedding(nn.Module):
    """
    Fixed 2D sine-cosine position embedding of the `grid_size x grid_size` image tokens of a tile, in row-major order.
    """

    def __init__(self, embed_dim: int, grid_size: int):
        super().__init__()
        self.embed_dim = embed_dim
        self.grid_size = grid_size
        self.pos_embed = nn.Buffer(self.build(embed_dim, grid_size), persistent=False)

    @staticmethod
    def build(embed_dim: int, grid_size: int) -> torch.Tensor:
        # vit_mae's layout is [rows | columns]; the checkpoints were trained with [columns | rows]
        return build_2d_sinusoidal_position_embedding(grid_size, grid_size, embed_dim).roll(embed_dim // 2, -1)

    def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
        return hidden_states + self.pos_embed.to(hidden_states.dtype)


class GraniteDoclingDeepStackMerger(nn.Module):
    def __init__(self, config: GraniteDoclingVisionConfig, scale_factor: int):
        super().__init__()
        merged_dim = config.hidden_size * (scale_factor**2)
        self.layer_norm = nn.LayerNorm(merged_dim)
        self.linear_1 = nn.Linear(merged_dim, config.out_hidden_size)
        self.act = ACT2FN["gelu"]
        self.linear_2 = nn.Linear(config.out_hidden_size, config.out_hidden_size)

    def forward(self, image_features):
        hidden_states = self.layer_norm(image_features)
        hidden_states = self.linear_1(hidden_states)
        hidden_states = self.act(hidden_states)
        hidden_states = self.linear_2(hidden_states)
        return hidden_states


class GraniteDoclingConnectorPath(nn.Module):
    """Pixel shuffle, projection, fixed 2D position embedding and DeepStack mergers for one pixel-shuffle factor."""

    def __init__(self, config: GraniteDoclingVisionConfig, scale_factor: int):
        super().__init__()
        self.scale_factor = scale_factor
        tokens_per_side = config.image_size // config.patch_size
        self.projection = nn.Linear(config.hidden_size * (scale_factor**2), config.out_hidden_size, bias=False)
        self.pos_embed = GraniteDoclingPositionEmbedding(config.out_hidden_size, tokens_per_side // scale_factor)
        self.deepstack_mergers = nn.ModuleList(
            [GraniteDoclingDeepStackMerger(config, scale_factor) for _ in config.deepstack_visual_indexes]
        )

    def pixel_shuffle(self, x, scale_factor=2):
        bsz, seq, embed_dim = x.size()
        height = width = int(seq**0.5)
        x = x.view(bsz, height, width, embed_dim)
        x = x.view(bsz, height, int(width / scale_factor), embed_dim * scale_factor)
        x = x.permute(0, 2, 1, 3)
        x = x.reshape(bsz, int(width / scale_factor), int(height / scale_factor), embed_dim * (scale_factor**2))
        x = x.permute(0, 2, 1, 3)
        x = x.reshape(bsz, int(seq / (scale_factor**2)), embed_dim * (scale_factor**2))
        return x

    def forward(
        self, image_hidden_states: torch.Tensor, deepstack_hidden_states: list[torch.Tensor]
    ) -> tuple[torch.Tensor, list[torch.Tensor]]:
        hidden_states = self.pixel_shuffle(image_hidden_states, self.scale_factor)
        hidden_states = self.projection(hidden_states)
        hidden_states = self.pos_embed(hidden_states)
        deepstack_features = []
        for merger, deepstack_hidden in zip(self.deepstack_mergers, deepstack_hidden_states):
            deepstack_hidden = self.pixel_shuffle(deepstack_hidden.contiguous(), self.scale_factor)
            deepstack_features.append(merger(deepstack_hidden).flatten(0, 1))
        return hidden_states, deepstack_features


class GraniteDoclingConnector(nn.Module):
    """
    Turns the vision encoder output of each tile into image tokens, on the coarse path (`scale_factor`) or the fine
    path (half the factor, four times as many tokens). The layer norms and the output projection are shared.
    """

    def __init__(self, config: GraniteDoclingVisionConfig):
        super().__init__()
        self.ln_in = nn.LayerNorm(config.hidden_size)
        self.coarse = GraniteDoclingConnectorPath(config, config.scale_factor)
        self.fine = GraniteDoclingConnectorPath(config, config.scale_factor // 2) if config.use_fine_route else None
        self.ln_mid = nn.LayerNorm(config.out_hidden_size)
        self.mlp_fc2 = nn.Linear(config.out_hidden_size, config.out_hidden_size, bias=False)
        self.ln_out = nn.LayerNorm(config.out_hidden_size)

    def forward(
        self, image_hidden_states: torch.Tensor, deepstack_hidden_states: list[torch.Tensor], fine: bool = False
    ) -> tuple[torch.Tensor, list[torch.Tensor]]:
        path = self.fine if fine else self.coarse
        hidden_states, deepstack_features = path(self.ln_in(image_hidden_states), deepstack_hidden_states)
        hidden_states = nn.functional.gelu(self.ln_mid(hidden_states))
        image_features = self.ln_out(self.mlp_fc2(hidden_states))
        return image_features.flatten(0, 1), deepstack_features


class GraniteDoclingDensityRouter(nn.Module):
    """
    Predicts from the vision encoder features of a sample's tiles whether the sample needs the fine connector path.
    """

    def __init__(self, config: GraniteDoclingConfig):
        super().__init__()
        vision_hidden_size = config.vision_config.hidden_size
        self.att = nn.Linear(vision_hidden_size, 1)
        self.norm = nn.LayerNorm(vision_hidden_size)
        self.fc1 = nn.Linear(vision_hidden_size, config.density_router_hidden_size)
        self.act = nn.GELU()
        self.fc2 = nn.Linear(config.density_router_hidden_size, 1)

    def forward(
        self, image_hidden_states: torch.Tensor, tile_sample_index: torch.Tensor, num_samples: int
    ) -> torch.Tensor:
        # Attention-pool the patches of every tile, then average the tiles of every sample
        weights = torch.softmax(self.att(image_hidden_states), dim=1)
        tile_features = (image_hidden_states * weights).sum(dim=1)
        sample_features = tile_features.new_zeros(num_samples, tile_features.shape[-1])
        sample_features = sample_features.index_add(0, tile_sample_index, tile_features)
        num_tiles = tile_features.new_zeros(num_samples).index_add(
            0, tile_sample_index, tile_features.new_ones(tile_features.shape[0])
        )
        sample_features = sample_features / num_tiles.clamp(min=1).unsqueeze(-1)
        hidden = self.norm(sample_features)
        hidden = self.fc1(hidden)
        hidden = self.act(hidden)
        return self.fc2(hidden).squeeze(-1)


class GraniteDoclingVisionEmbeddings(nn.Module):
    """
    Patch embeddings with learned position embeddings for square tiles of `config.image_size` pixels.
    Tiles are always that size, so the positions are an arange and no patch mask is required.
    """

    def __init__(self, config: GraniteDoclingVisionConfig):
        super().__init__()
        self.config = config
        self.embed_dim = config.hidden_size
        self.image_size = config.image_size
        self.patch_size = config.patch_size

        self.patch_embedding = nn.Conv2d(
            in_channels=config.num_channels,
            out_channels=self.embed_dim,
            kernel_size=self.patch_size,
            stride=self.patch_size,
            padding="valid",
        )

        self.num_patches = (self.image_size // self.patch_size) ** 2
        self.num_positions = self.num_patches
        self.position_embedding = nn.Embedding(self.num_positions, self.embed_dim)
        self.position_ids = nn.Buffer(torch.arange(self.num_positions).expand((1, -1)), persistent=False)

    def interpolate_pos_encoding(self, embeddings: torch.Tensor, height: int, width: int) -> torch.Tensor:
        """
        This method allows to interpolate the pre-trained position encodings, to be able to use the model on higher resolution
        images. This method is also adapted to support torch.jit tracing and no class embeddings.

        Adapted from:
        - https://github.com/facebookresearch/dino/blob/de9ee3df6cf39fac952ab558447af1fa1365362a/vision_transformer.py#L174-L194, and
        - https://github.com/facebookresearch/dinov2/blob/e1277af2ba9496fbadf7aec6eba56e8d882d1e35/dinov2/models/vision_transformer.py#L179-L211
        """

        num_patches = embeddings.shape[1]
        num_positions = self.position_embedding.weight.shape[0]

        # always interpolate when tracing to ensure the exported model works for dynamic input shapes
        if not torch.jit.is_tracing() and num_patches == num_positions and height == width:
            return self.position_embedding(self.position_ids)

        patch_pos_embed = self.position_embedding.weight.unsqueeze(0)

        dim = embeddings.shape[-1]

        new_height = height // self.patch_size
        new_width = width // self.patch_size

        sqrt_num_positions = torch_int(num_positions**0.5)
        patch_pos_embed = patch_pos_embed.reshape(1, sqrt_num_positions, sqrt_num_positions, dim)
        patch_pos_embed = patch_pos_embed.permute(0, 3, 1, 2)

        patch_pos_embed = nn.functional.interpolate(
            patch_pos_embed,
            size=(new_height, new_width),
            mode="bicubic",
            align_corners=False,
        )

        patch_pos_embed = patch_pos_embed.permute(0, 2, 3, 1).view(1, -1, dim)
        return patch_pos_embed

    def forward(self, pixel_values: torch.Tensor, interpolate_pos_encoding=False) -> torch.Tensor:
        _, _, height, width = pixel_values.shape
        target_dtype = self.patch_embedding.weight.dtype
        patch_embeds = self.patch_embedding(pixel_values.to(dtype=target_dtype))  # shape = [*, width, grid, grid]
        embeddings = patch_embeds.flatten(2).transpose(1, 2)

        if interpolate_pos_encoding:
            embeddings = embeddings + self.interpolate_pos_encoding(embeddings, height, width)
        else:
            embeddings = embeddings + self.position_embedding(self.position_ids)
        return embeddings


class GraniteDoclingVisionAttention(nn.Module):
    """Multi-headed attention from 'Attention Is All You Need' paper"""

    def __init__(self, config):
        super().__init__()
        self.config = config
        self.embed_dim = config.hidden_size
        self.num_heads = config.num_attention_heads
        self.head_dim = self.embed_dim // self.num_heads
        if self.head_dim * self.num_heads != self.embed_dim:
            raise ValueError(
                f"embed_dim must be divisible by num_heads (got `embed_dim`: {self.embed_dim} and `num_heads`:"
                f" {self.num_heads})."
            )
        self.scale = self.head_dim**-0.5
        self.dropout = config.attention_dropout

        self.k_proj = nn.Linear(self.embed_dim, self.embed_dim)
        self.v_proj = nn.Linear(self.embed_dim, self.embed_dim)
        self.q_proj = nn.Linear(self.embed_dim, self.embed_dim)
        self.out_proj = nn.Linear(self.embed_dim, self.embed_dim)

        # Ignore copy
        self.is_causal = False
        self.num_key_value_groups = 1  # needed for eager attention

    def forward(
        self,
        hidden_states: torch.Tensor,
        attention_mask: torch.Tensor | None = None,
        **kwargs,
    ) -> tuple[torch.Tensor, torch.Tensor | None]:
        """Input shape: Batch x Time x Channel"""
        input_shape = hidden_states.shape[:-1]
        hidden_shape = (*input_shape, -1, self.head_dim)
        queries = self.q_proj(hidden_states).view(hidden_shape).transpose(1, 2)
        keys = self.k_proj(hidden_states).view(hidden_shape).transpose(1, 2)
        values = self.v_proj(hidden_states).view(hidden_shape).transpose(1, 2)

        attention_interface: Callable = ALL_ATTENTION_FUNCTIONS.get_interface(
            self.config._attn_implementation, eager_attention_forward
        )

        attn_output, attn_weights = attention_interface(
            self,
            queries,
            keys,
            values,
            attention_mask,
            is_causal=self.is_causal,
            scaling=self.scale,
            dropout=0.0 if not self.training else self.dropout,
        )

        attn_output = attn_output.reshape(*input_shape, -1).contiguous()
        attn_output = self.out_proj(attn_output)

        return attn_output, attn_weights


@auto_docstring
class GraniteDoclingPreTrainedModel(PreTrainedModel):
    config: GraniteDoclingConfig
    base_model_prefix = "model"
    input_modalities = ("image", "text")
    supports_gradient_checkpointing = True
    _no_split_modules = ["GraniteDoclingEncoderLayer", "GraniteDoclingTextDecoderLayer"]
    _skip_keys_device_placement = ["past_key_values"]
    _supports_flash_attn = True
    _supports_sdpa = True
    _supports_flex_attn = True
    _supports_attention_backend = True
    # Some checkpoints carry extra prediction heads under `mtp.*` for serving engines; they are not part of this model.
    _keys_to_ignore_on_load_unexpected = [r"^mtp\."]

    def _init_weights(self, module):
        # Idefics3 has no `_init_weights` of its own, the chain starts at PreTrainedModel
        super()._init_weights(module)
        if isinstance(module, GraniteDoclingVisionEmbeddings):
            init.copy_(module.position_ids, torch.arange(module.position_ids.shape[-1]).expand((1, -1)))
        if isinstance(module, GraniteDoclingPositionEmbedding):
            init.copy_(module.pos_embed, module.build(module.embed_dim, module.grid_size))


class GraniteDoclingVisionMLP(nn.Module):
    def __init__(self, config):
        super().__init__()
        self.config = config
        self.activation_fn = ACT2FN[config.hidden_act]
        self.fc1 = nn.Linear(config.hidden_size, config.intermediate_size)
        self.fc2 = nn.Linear(config.intermediate_size, config.hidden_size)

    def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
        hidden_states = self.fc1(hidden_states)
        hidden_states = self.activation_fn(hidden_states)
        hidden_states = self.fc2(hidden_states)
        return hidden_states


class GraniteDoclingEncoderLayer(GradientCheckpointingLayer):
    def __init__(self, config: GraniteDoclingVisionConfig):
        super().__init__()
        self.embed_dim = config.hidden_size
        self.self_attn = GraniteDoclingVisionAttention(config)
        self.layer_norm1 = nn.LayerNorm(self.embed_dim, eps=config.layer_norm_eps)
        self.mlp = GraniteDoclingVisionMLP(config)
        self.layer_norm2 = nn.LayerNorm(self.embed_dim, eps=config.layer_norm_eps)

    @auto_docstring
    def forward(
        self,
        hidden_states: torch.Tensor,
        attention_mask: torch.Tensor,
        **kwargs: Unpack[TransformersKwargs],
    ) -> torch.Tensor:
        residual = hidden_states

        hidden_states = self.layer_norm1(hidden_states)
        hidden_states, _ = self.self_attn(
            hidden_states=hidden_states,
            attention_mask=attention_mask,
            **kwargs,
        )
        hidden_states = residual + hidden_states

        residual = hidden_states
        hidden_states = self.layer_norm2(hidden_states)
        hidden_states = self.mlp(hidden_states)
        hidden_states = residual + hidden_states

        return hidden_states


class GraniteDoclingEncoder(nn.Module):
    """
    Transformer encoder consisting of `config.num_hidden_layers` self attention layers. Each layer is a
    [`GraniteDoclingEncoderLayer`].

    Args:
        config: GraniteDoclingConfig
    """

    def __init__(self, config: GraniteDoclingConfig):
        super().__init__()
        self.config = config
        self.layers = nn.ModuleList([GraniteDoclingEncoderLayer(config) for _ in range(config.num_hidden_layers)])
        self.gradient_checkpointing = False

    # Ignore copy
    @auto_docstring
    def forward(
        self,
        inputs_embeds,
        attention_mask: torch.Tensor | None = None,
    ) -> tuple | BaseModelOutput:
        hidden_states = inputs_embeds
        for encoder_layer in self.layers:
            layer_outputs = encoder_layer(
                hidden_states,
                attention_mask,
            )

            hidden_states = layer_outputs

        return BaseModelOutput(last_hidden_state=hidden_states)


@auto_docstring(
    custom_intro="""
    The vision encoder of GraniteDocling with its connector on top: embeds the tiles of a page and returns the
    image tokens of every tile together with the projected DeepStack features.
    """
)
class GraniteDoclingVisionModel(GraniteDoclingPreTrainedModel):
    config: GraniteDoclingVisionConfig
    input_modalities = ("image",)
    _can_record_outputs = {
        "hidden_states": GraniteDoclingEncoderLayer,
        "attentions": GraniteDoclingVisionAttention,
    }

    def __init__(self, config: GraniteDoclingVisionConfig):
        super().__init__(config)
        embed_dim = config.hidden_size

        self.embeddings = GraniteDoclingVisionEmbeddings(config)
        self.encoder = GraniteDoclingEncoder(config)
        self.patch_size = config.patch_size
        self.post_layernorm = nn.LayerNorm(embed_dim, eps=config.layer_norm_eps)
        self.connector = GraniteDoclingConnector(config)

        self.post_init()

    def get_input_embeddings(self):
        return self.embeddings

    def set_input_embeddings(self, value):
        self.embeddings = value

    @merge_with_config_defaults
    @capture_outputs(tie_last_hidden_states=False)
    @auto_docstring
    def forward(
        self,
        pixel_values: torch.FloatTensor,
        fine: bool = False,
        **kwargs: Unpack[TransformersKwargs],
    ) -> tuple | GraniteDoclingImageFeaturesOutput:
        r"""
        pixel_values (`torch.FloatTensor` of shape `(num_tiles, num_channels, image_size, image_size)`):
            Tiles flattened across the batch, without padding tiles.
        fine (`bool`, *optional*, defaults to `False`):
            Whether the tiles take the fine connector path, which yields four times as many image tokens per tile.

        Returns:
            Image tokens of every tile in tile order (`pooler_output`) and one projected DeepStack feature per
            tapped vision layer (`deepstack_features`), with the same tile order.
        """
        last_hidden_state, deepstack_hidden_states = self.encode(pixel_values)
        image_features, deepstack_features = self.connector(last_hidden_state, deepstack_hidden_states, fine=fine)
        return GraniteDoclingImageFeaturesOutput(
            last_hidden_state=last_hidden_state,
            pooler_output=image_features,
            deepstack_features=deepstack_features,
        )

    def encode(self, pixel_values: torch.FloatTensor) -> tuple[torch.Tensor, list[torch.Tensor]]:
        """Vision tokens and DeepStack taps, before the connector. The route does not change this."""
        hidden_states = self.embeddings(pixel_values)
        deepstack_hidden_states = []
        for layer_idx, encoder_layer in enumerate(self.encoder.layers):
            hidden_states = encoder_layer(hidden_states, attention_mask=None)
            if layer_idx in self.config.deepstack_visual_indexes:
                deepstack_hidden_states.append(hidden_states)
        return self.post_layernorm(hidden_states), deepstack_hidden_states


@auto_docstring(
    custom_intro="""
    GraniteDocling model consisting of a vision encoder, a connector and a text decoder, outputting raw hidden
    states without any specific head on top.
    """
)
class GraniteDoclingModel(GraniteDoclingPreTrainedModel):
    def __init__(self, config: GraniteDoclingConfig):
        super().__init__(config)
        self.padding_idx = self.config.text_config.pad_token_id
        self.vocab_size = self.config.text_config.vocab_size

        # The connector lives in the vision model, which returns image tokens and DeepStack features
        self.vision_model = GraniteDoclingVisionModel._from_config(config.vision_config)
        self.text_model = GraniteDoclingTextModel._from_config(config.text_config)
        self.density_router = None
        if config.density_router_hidden_size is not None:
            self.density_router = GraniteDoclingDensityRouter(config)

        self.image_seq_len = int(
            ((config.vision_config.image_size // config.vision_config.patch_size) ** 2)
            / (config.vision_config.scale_factor**2)
        )
        self.image_token_id = self.config.image_token_id

        self.post_init()

    def get_input_embeddings(self):
        return self.text_model.get_input_embeddings()

    def set_input_embeddings(self, value):
        self.text_model.set_input_embeddings(value)

    @can_return_tuple
    @auto_docstring
    def get_image_features(
        self,
        pixel_values: torch.FloatTensor,
        pixel_attention_mask: torch.BoolTensor,
        tile_fine_mask: torch.BoolTensor | None = None,
        **kwargs: Unpack[TransformersKwargs],
    ) -> tuple | GraniteDoclingImageFeaturesOutput:
        r"""
        pixel_values (`torch.FloatTensor` of shape `(batch_size, num_tiles, num_channels, image_size, image_size)`):
            The tensors corresponding to the input image tiles.
        pixel_attention_mask (`torch.BoolTensor` of shape `(batch_size, num_tiles)`):
            Real tiles, as returned by the processor. Padding tiles are false.
        tile_fine_mask (`torch.BoolTensor` of shape `(batch_size,)`, *optional*):
            Requires processor called with `fine_route=True`: the batch takes the fine connector
            path, which yields four times as many image tokens per tile.
        """
        batch_size = pixel_values.shape[0]
        fine = tile_fine_mask is not None
        tiles_per_sample = pixel_attention_mask.sum(-1)
        image_outputs = self.vision_model(
            pixel_values=pixel_values[pixel_attention_mask].to(self.dtype), fine=fine, return_dict=True, **kwargs
        )
        tokens_per_tile = self.image_seq_len * 4 if fine else self.image_seq_len
        split_sizes = (tiles_per_sample * tokens_per_tile).tolist()
        image_outputs.pooler_output = torch.split(image_outputs.pooler_output, split_sizes)
        image_outputs.deepstack_features = [
            torch.split(features, split_sizes) for features in image_outputs.deepstack_features
        ]
        if self.density_router is not None:
            tile_sample_index = torch.arange(batch_size, device=pixel_values.device).repeat_interleave(
                tiles_per_sample
            )
            image_outputs.router_logits = self.density_router(
                image_outputs.last_hidden_state, tile_sample_index, batch_size
            )
        return image_outputs

    @can_return_tuple
    @auto_docstring(
        custom_intro="""
        Inputs fed to the model can have an arbitrary number of images. To account for this, pixel_values fed to
        the model have image padding -> (batch_size, max_num_tiles, 3, image_size, image_size) where max_num_tiles
        is the maximum number of tiles among the batch_size samples in the batch. Padding tiles are false in
        `pixel_attention_mask` and are discarded before the vision encoder.
        """
    )
    def forward(
        self,
        input_ids: torch.LongTensor | None = None,
        attention_mask: torch.Tensor | None = None,
        position_ids: torch.LongTensor | None = None,
        past_key_values: Cache | None = None,
        inputs_embeds: torch.FloatTensor | None = None,
        pixel_values: torch.FloatTensor | None = None,
        tile_fine_mask: torch.BoolTensor | None = None,
        pixel_attention_mask: torch.BoolTensor | None = None,
        mm_encoder_outputs: dict | None = None,
        **kwargs: Unpack[TransformersKwargs],
    ) -> tuple | GraniteDoclingBaseModelOutputWithPast:
        r"""
        tile_fine_mask (`torch.BoolTensor` of shape `(batch_size,)`, *optional*):
            All True when the processor is called with `fine_route=True`: the batch takes the fine connector path,
            which yields four times as many image tokens per tile.
        pixel_attention_mask (`torch.BoolTensor` of shape `(batch_size, num_tiles)`, *optional*):
            Real tiles, as returned by the processor. Padding tiles are false. Required with `pixel_values`.
        """
        if (input_ids is None) ^ (inputs_embeds is not None):
            raise ValueError("You must specify exactly one of input_ids or inputs_embeds")

        if inputs_embeds is None:
            inputs_embeds = self.text_model.get_input_embeddings()(input_ids).to(self.device)

        if pixel_values is not None and mm_encoder_outputs is not None:
            raise ValueError("You cannot specify both pixel_values and mm_encoder_outputs at the same time")
        if mm_encoder_outputs is not None and not isinstance(mm_encoder_outputs, dict):
            raise ValueError("mm_encoder_outputs must be a dict of cached encoder outputs.")

        image_hidden_states = None
        deepstack_visual_embeds = None
        visual_pos_masks = None
        router_logits = None
        mm_encoder_outputs = {} if mm_encoder_outputs is None else mm_encoder_outputs
        if mm_encoder_outputs.get("image") is not None or pixel_values is not None:
            if mm_encoder_outputs.get("image") is None:
                mm_encoder_outputs["image"] = self.get_image_features(
                    pixel_values, pixel_attention_mask, tile_fine_mask=tile_fine_mask, return_dict=True
                )
            image_outputs = mm_encoder_outputs["image"]
            image_hidden_states = torch.cat(image_outputs.pooler_output, dim=0).to(
                inputs_embeds.device, inputs_embeds.dtype
            )
            router_logits = image_outputs.router_logits
            deepstack_visual_embeds = image_outputs.deepstack_features
            if deepstack_visual_embeds is not None:
                deepstack_visual_embeds = [
                    torch.cat(features, dim=0).to(inputs_embeds.device, inputs_embeds.dtype)
                    for features in deepstack_visual_embeds
                ]
            special_image_mask = self.get_placeholder_mask(input_ids, inputs_embeds, image_hidden_states)
            inputs_embeds = inputs_embeds.masked_scatter(special_image_mask.unsqueeze(-1), image_hidden_states)
            visual_pos_masks = special_image_mask

        outputs = self.text_model(
            inputs_embeds=inputs_embeds,
            attention_mask=attention_mask,
            position_ids=position_ids,
            past_key_values=past_key_values,
            visual_pos_masks=visual_pos_masks,
            deepstack_visual_embeds=deepstack_visual_embeds,
            **kwargs,
        )

        return GraniteDoclingBaseModelOutputWithPast(
            last_hidden_state=outputs.last_hidden_state,
            past_key_values=outputs.past_key_values,
            hidden_states=outputs.hidden_states,
            attentions=outputs.attentions,
            image_hidden_states=image_hidden_states,
            router_logits=router_logits,
        )

    def get_placeholder_mask(
        self, input_ids: torch.LongTensor, inputs_embeds: torch.FloatTensor, image_features: torch.FloatTensor
    ):
        """
        Obtains multimodal placeholder mask from `input_ids` or `inputs_embeds`, and checks that the placeholder token count is
        equal to the length of multimodal features. If the lengths are different, an error is raised.
        """
        if input_ids is None:
            special_image_mask = inputs_embeds == self.get_input_embeddings()(
                torch.full((), self.config.image_token_id, dtype=torch.long, device=inputs_embeds.device)
            )
            special_image_mask = special_image_mask.all(-1)
        else:
            special_image_mask = input_ids == self.config.image_token_id

        n_image_tokens = special_image_mask.sum()
        n_image_features = image_features.numel() // image_features.shape[-1]
        torch_compilable_check(
            n_image_tokens == n_image_features,
            f"Image features and image tokens do not match: tokens: {n_image_tokens}, features {n_image_features}",
        )
        return special_image_mask.to(inputs_embeds.device)

    def predict_fine_route(
        self, pixel_values: torch.FloatTensor, pixel_attention_mask: torch.BoolTensor
    ) -> torch.BoolTensor:
        r"""
        Predicts with the density router which samples need the fine connector path. Route them by calling the
        processor again with `fine_route=True`.

        Args:
            pixel_values (`torch.FloatTensor` of shape `(batch_size, num_tiles, num_channels, image_size, image_size)`):
                The tensors corresponding to the input image tiles.
            pixel_attention_mask (`torch.BoolTensor` of shape `(batch_size, num_tiles)`):
                Real tiles, as returned by the processor.

        Returns:
            `torch.BoolTensor` of shape `(batch_size,)`: `True` for the samples that need the fine connector path.
        """
        if self.density_router is None:
            raise ValueError("This model has no density router, set `density_router_hidden_size` in the config.")
        batch_size = pixel_values.shape[0]
        last_hidden_state, _ = self.vision_model.encode(pixel_values[pixel_attention_mask].to(self.dtype))
        tile_sample_index = torch.arange(batch_size, device=pixel_values.device).repeat_interleave(
            pixel_attention_mask.sum(-1)
        )
        router_logits = self.density_router(last_hidden_state, tile_sample_index, batch_size)
        return torch.sigmoid(router_logits) >= self.config.density_router_threshold


@auto_docstring(
    custom_intro="""
    The GraniteDocling Model with a language modeling head. It is made up of a vision encoder, a connector and a
    text decoder, with a language modeling head on top.
    """
)
class GraniteDoclingForConditionalGeneration(GraniteDoclingPreTrainedModel, GenerationMixin):
    _tied_weights_keys = {"lm_head.weight": "model.text_model.embed_tokens.weight"}
    _tp_plan = {"lm_head": "colwise_gather_output"}

    def __init__(self, config: GraniteDoclingConfig):
        super().__init__(config)
        self.model = GraniteDoclingModel(config)
        self.image_token_id = self.config.image_token_id

        self.lm_head = nn.Linear(config.text_config.hidden_size, config.text_config.vocab_size, bias=False)
        self.vocab_size = config.text_config.vocab_size

        # Initialize weights and apply final processing
        self.post_init()

    def get_input_embeddings(self):
        return self.model.text_model.get_input_embeddings()

    def set_input_embeddings(self, value):
        self.model.text_model.set_input_embeddings(value)

    @auto_docstring
    def get_image_features(
        self,
        pixel_values: torch.Tensor,
        pixel_attention_mask: torch.LongTensor | None = None,
        **kwargs: Unpack[TransformersKwargs],
    ) -> tuple | BaseModelOutputWithPooling:
        r"""
        pixel_values (`torch.Tensor` of shape `(batch_size, num_channels, image_size, image_size)`):
            The tensors corresponding to the input images.
        pixel_attention_mask (`torch.LongTensor`, *optional*):
            The attention mask indicating padded regions in the image.
        """
        return self.model.get_image_features(
            pixel_values=pixel_values, pixel_attention_mask=pixel_attention_mask, **kwargs
        )

    @can_return_tuple
    @auto_docstring
    def forward(
        self,
        input_ids: torch.LongTensor | None = None,
        attention_mask: torch.Tensor | None = None,
        position_ids: torch.LongTensor | None = None,
        past_key_values: Cache | None = None,
        inputs_embeds: torch.FloatTensor | None = None,
        pixel_values: torch.FloatTensor | None = None,
        tile_fine_mask: torch.BoolTensor | None = None,
        pixel_attention_mask: torch.BoolTensor | None = None,
        mm_encoder_outputs: dict | None = None,
        labels: torch.LongTensor | None = None,
        router_labels: torch.Tensor | None = None,
        use_cache: bool | None = None,
        logits_to_keep: int | torch.Tensor = 0,
        **kwargs: Unpack[TransformersKwargs],
    ) -> tuple | GraniteDoclingCausalLMOutputWithPast:
        r"""
        tile_fine_mask (`torch.BoolTensor` of shape `(batch_size,)`, *optional*):
            All True when the processor is called with `fine_route=True`: the batch takes the fine connector path,
            which yields four times as many image tokens per tile.
        pixel_attention_mask (`torch.BoolTensor` of shape `(batch_size, num_tiles)`, *optional*):
            Real tiles, as returned by the processor. Padding tiles are false. Required with `pixel_values`.
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
            config.vocab_size]` or `model.image_token_id` (where `model` is your instance of `GraniteDoclingForConditionalGeneration`).
            Tokens with indices set to `model.image_token_id` are ignored (masked), the loss is only
            computed for the tokens with labels in `[0, ..., config.vocab_size]`.
        router_labels (`torch.Tensor` of shape `(batch_size,)`, *optional*):
            Targets of the density router, `1` (or `True`) for the samples that need the fine connector path and `0`
            otherwise. Given as float, bool or int; the router loss is a binary cross-entropy on the logits.

        Example:

        ```python
        >>> from transformers import AutoProcessor, AutoModelForImageTextToText
        >>> from transformers.image_utils import load_image

        >>> image = load_image("https://huggingface.co/ibm-granite/granite-for-docling-500m/resolve/main/docling_technical_report_p1.png")

        >>> processor = AutoProcessor.from_pretrained("ibm-granite/granite-for-docling-500m")
        >>> model = AutoModelForImageTextToText.from_pretrained("ibm-granite/granite-for-docling-500m", device_map="auto")

        >>> messages = [
        ...     {
        ...         "role": "user",
        ...         "content": [
        ...             {"type": "image"},
        ...             {"type": "text", "text": "<doclang>"},
        ...         ],
        ...     }
        ... ]
        >>> prompt = processor.apply_chat_template(messages, add_generation_prompt=True)
        >>> inputs = processor(text=prompt, images=image, return_tensors="pt").to(model.device)

        >>> generated_ids = model.generate(**inputs, max_new_tokens=4096)
        >>> print(processor.decode(generated_ids[0, inputs["input_ids"].shape[1] :], skip_special_tokens=False))
        ```"""
        outputs = self.model(
            input_ids=input_ids,
            attention_mask=attention_mask,
            position_ids=position_ids,
            past_key_values=past_key_values,
            inputs_embeds=inputs_embeds,
            pixel_values=pixel_values,
            tile_fine_mask=tile_fine_mask,
            pixel_attention_mask=pixel_attention_mask,
            mm_encoder_outputs=mm_encoder_outputs,
            use_cache=use_cache,
            return_dict=True,
            **kwargs,
        )
        router_logits = outputs.router_logits

        hidden_states = outputs[0]
        hidden_states = slice_logits_to_keep(hidden_states, logits_to_keep)
        logits = self.lm_head(hidden_states)
        logits = logits / self.config.text_config.logits_scaling

        loss = None
        if labels is not None:
            loss = self.loss_function(
                logits=logits, labels=labels, vocab_size=self.config.text_config.vocab_size, **kwargs
            )
        if router_labels is not None and router_logits is not None:
            router_labels = router_labels.to(router_logits.device, router_logits.dtype).reshape(-1)
            router_loss = self.config.density_router_loss_weight * nn.functional.binary_cross_entropy_with_logits(
                router_logits.reshape(-1), router_labels
            )
            loss = router_loss if loss is None else loss + router_loss

        return GraniteDoclingCausalLMOutputWithPast(
            loss=loss,
            logits=logits,
            past_key_values=outputs.past_key_values,
            hidden_states=outputs.hidden_states,
            attentions=outputs.attentions,
            image_hidden_states=outputs.image_hidden_states,
            router_logits=router_logits,
        )


__all__ = [
    "GraniteDoclingForConditionalGeneration",
    "GraniteDoclingModel",
    "GraniteDoclingPreTrainedModel",
    "GraniteDoclingTextModel",
    "GraniteDoclingTextPreTrainedModel",
    "GraniteDoclingVisionModel",
]
