#                🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨
#           This file was automatically generated from src/transformers/models/unlimited_ocr/modular_unlimited_ocr.py.
#               Do NOT edit this file manually as any edits will be overwritten by the generation of
#             the file from the modular. If any change should be done, please apply the change to the
#                          modular_unlimited_ocr.py file directly. One of our CI enforces this.
#                🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨
# Copyright 2026 the HuggingFace Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import collections
import math
from collections.abc import Callable
from dataclasses import dataclass

import torch
import torch.nn.functional as F
from torch import nn

from ... import initialization as init
from ...activations import ACT2FN
from ...cache_utils import Cache, DynamicCache, DynamicLayer, DynamicSlidingWindowLayer, StaticSlidingWindowLayer
from ...configuration_utils import PreTrainedConfig
from ...integrations import use_experts_implementation, use_kernel_forward_from_hub, use_kernelized_func
from ...masking_utils import (
    BlockMask,
    and_masks,
    causal_mask_function,
    create_causal_mask,
    create_sliding_window_causal_mask,
    or_masks,
)
from ...modeling_layers import GradientCheckpointingLayer
from ...modeling_outputs import (
    BaseModelOutput,
    BaseModelOutputWithPast,
    BaseModelOutputWithPooling,
    MoeModelOutputWithPast,
)
from ...modeling_rope_utils import ROPE_INIT_FUNCTIONS, dynamic_rope_update
from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel, slice_logits_to_keep
from ...processing_utils import Unpack
from ...utils import (
    ModelOutput,
    TransformersKwargs,
    auto_docstring,
    can_return_tuple,
    is_torchdynamo_compiling,
    logging,
    torch_compilable_check,
    torch_int,
)
from ...utils.generic import merge_with_config_defaults
from ...utils.output_capturing import OutputRecorder, capture_outputs
from .configuration_unlimited_ocr import (
    UnlimitedOcrConfig,
    UnlimitedOcrSamVisionConfig,
    UnlimitedOcrTextConfig,
    UnlimitedOcrVisionConfig,
    UnlimitedOcrVisionEncoderConfig,
)
from .generation_unlimited_ocr import UnlimitedOcrGenerationMixin


logger = logging.get_logger(__name__)


@auto_docstring
@dataclass
class UnlimitedOcrModelOutputWithPooling(BaseModelOutputWithPooling):
    r"""
    local_last_hidden_state (`torch.Tensor` of shape `(total_local_patches, sequence_length, hidden_size)`, *optional*):
        Last hidden state from the vision encoder for local (cropped) patches.
    local_hidden_states (`torch.Tensor`, *optional*):
        Hidden states from all layers of the vision encoder for local patches.
    local_attentions (`torch.Tensor`, *optional*):
        Attention weights from all layers of the vision encoder for local patches.
    """

    local_last_hidden_state: torch.Tensor | None = None
    local_hidden_states: torch.Tensor | None = None
    local_attentions: torch.Tensor | None = None


@auto_docstring(
    custom_intro="""
    Base class for UnlimitedOcr outputs, with hidden states and attentions.
    """
)
@dataclass
class UnlimitedOcrModelOutputWithPast(BaseModelOutputWithPast):
    r"""
    past_key_values (`Cache`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
        It is a [`~cache_utils.Cache`] instance. For more details, see our [kv cache guide](https://huggingface.co/docs/transformers/en/kv_cache).

        Contains pre-computed hidden-states (key and values in the self-attention blocks) that can be used (see
        `past_key_values` input) to speed up sequential decoding.
    image_hidden_states (`torch.Tensor`, *optional*):
        A `torch.Tensor` of size `(batch_size, num_images, sequence_length, hidden_size)`.
        image_hidden_states of the model produced by the vision encoder and after projecting the last hidden state.
    """

    image_hidden_states: torch.Tensor | None = None
    router_logits: tuple[torch.Tensor] | None = None


@auto_docstring(
    custom_intro="""
    Base class for UnlimitedOcr causal language model (or autoregressive) outputs.
    """
)
@dataclass
class UnlimitedOcrCausalLMOutputWithPast(ModelOutput):
    r"""
    loss (`torch.Tensor` of shape `(1,)`, *optional*, returned when `labels` is provided):
        Language modeling loss (for next-token prediction).
    logits (`torch.Tensor` of shape `(batch_size, sequence_length, config.vocab_size)`):
        Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax).
    past_key_values (`Cache`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
        It is a [`~cache_utils.Cache`] instance. For more details, see our [kv cache guide](https://huggingface.co/docs/transformers/en/kv_cache).

        Contains pre-computed hidden-states (key and values in the self-attention blocks) that can be used (see
        `past_key_values` input) to speed up sequential decoding.
    image_hidden_states (`torch.Tensor`, *optional*):
        A `torch.Tensor` of size (batch_size * num_patches, num_images, sequence_length, hidden_size)`.
        image_hidden_states of the model produced by the vision encoder and after projecting the last hidden state.
    """

    loss: torch.Tensor | None = None
    logits: torch.Tensor | None = None
    past_key_values: Cache | None = None
    hidden_states: tuple[torch.Tensor] | None = None
    attentions: tuple[torch.Tensor] | None = None
    image_hidden_states: torch.Tensor | None = None
    router_logits: tuple[torch.Tensor] | None = None


class UnlimitedOcrSamVisionAttention(nn.Module):
    """Multi-head Attention block with relative position embeddings."""

    def __init__(self, config, window_size):
        super().__init__()
        input_size = (
            (config.image_size // config.patch_size, config.image_size // config.patch_size)
            if window_size == 0
            else (window_size, window_size)
        )

        self.num_attention_heads = config.num_attention_heads
        head_dim = config.hidden_size // config.num_attention_heads
        self.scale = head_dim**-0.5
        self.dropout = config.attention_dropout

        self.qkv = nn.Linear(config.hidden_size, config.hidden_size * 3, bias=config.qkv_bias)
        self.proj = nn.Linear(config.hidden_size, config.hidden_size)

        self.use_rel_pos = config.use_rel_pos
        if self.use_rel_pos:
            if input_size is None:
                raise ValueError("Input size must be provided if using relative positional encoding.")

            # initialize relative positional embeddings
            self.rel_pos_h = nn.Parameter(torch.zeros(2 * input_size[0] - 1, head_dim))
            self.rel_pos_w = nn.Parameter(torch.zeros(2 * input_size[1] - 1, head_dim))

    def get_rel_pos(self, q_size: int, k_size: int, rel_pos: torch.Tensor) -> torch.Tensor:
        """
        Get relative positional embeddings according to the relative positions of
            query and key sizes.

        Args:
            q_size (int):
                size of the query.
            k_size (int):
                size of key k.
            rel_pos (`torch.Tensor`):
                relative position embeddings (L, channel).

        Returns:
            Extracted positional embeddings according to relative positions.
        """
        max_rel_dist = int(2 * max(q_size, k_size) - 1)
        # Interpolate rel pos.
        rel_pos_resized = F.interpolate(
            rel_pos.reshape(1, rel_pos.shape[0], -1).transpose(1, 2),
            size=max_rel_dist,
            mode="linear",
        )
        rel_pos_resized = rel_pos_resized.reshape(-1, max_rel_dist).permute(1, 0)

        # Scale the coords with short length if shapes for q and k are different.
        q_coords = torch.arange(q_size)[:, None] * max(k_size / q_size, 1.0)
        k_coords = torch.arange(k_size)[None, :] * max(q_size / k_size, 1.0)
        relative_coords = (q_coords - k_coords) + (k_size - 1) * max(q_size / k_size, 1.0)

        return rel_pos_resized[relative_coords.long()]

    def get_decomposed_rel_pos(
        self,
        query: torch.Tensor,
        rel_pos_h: torch.Tensor,
        rel_pos_w: torch.Tensor,
        q_size: tuple[int, int],
        k_size: tuple[int, int],
    ) -> torch.Tensor:
        """
        Calculate decomposed Relative Positional Embeddings from :paper:`mvitv2`.
        https://github.com/facebookresearch/mvit/blob/19786631e330df9f3622e5402b4a419a263a2c80/mvit/models/attention.py

        Args:
            query (`torch.Tensor`):
                query q in the attention layer with shape (batch_size, query_height * query_width, channel).
            rel_pos_h (`torch.Tensor`):
                relative position embeddings (Lh, channel) for height axis.
            rel_pos_w (`torch.Tensor`):
                relative position embeddings (Lw, channel) for width axis.
            q_size (tuple):
                spatial sequence size of query q with (query_height, query_width).
            k_size (tuple):
                spatial sequence size of key k with (key_height, key_width).

        Returns:
            decomposed_rel_pos (`torch.Tensor`):
                decomposed relative position embeddings.
        """
        query_height, query_width = q_size
        key_height, key_width = k_size
        relative_position_height = self.get_rel_pos(query_height, key_height, rel_pos_h)
        relative_position_width = self.get_rel_pos(query_width, key_width, rel_pos_w)

        batch_size, _, dim = query.shape
        reshaped_query = query.reshape(batch_size, query_height, query_width, dim)
        rel_h = torch.einsum("bhwc,hkc->bhwk", reshaped_query, relative_position_height)
        rel_w = torch.einsum("bhwc,wkc->bhwk", reshaped_query, relative_position_width)

        decomposed_rel_pos = rel_h[:, :, :, :, None] + rel_w[:, :, :, None, :]

        return decomposed_rel_pos

    def forward(self, hidden_states: torch.Tensor, output_attentions=None) -> tuple[torch.Tensor, torch.Tensor]:
        batch_size, height, width, _ = hidden_states.shape
        # qkv with shape (3, batch_size, nHead, height * width, channel)
        qkv = (
            self.qkv(hidden_states)
            .reshape(batch_size, height * width, 3, self.num_attention_heads, -1)
            .permute(2, 0, 3, 1, 4)
        )
        # q, k, v with shape (batch_size * nHead, height * width, channel)
        query, key, value = qkv.reshape(3, batch_size * self.num_attention_heads, height * width, -1).unbind(0)

        attn_weights = (query * self.scale) @ key.transpose(-2, -1)

        if self.use_rel_pos:
            decomposed_rel_pos = self.get_decomposed_rel_pos(
                query, self.rel_pos_h, self.rel_pos_w, (height, width), (height, width)
            )
            decomposed_rel_pos = decomposed_rel_pos.reshape_as(attn_weights)
            attn_weights = attn_weights + decomposed_rel_pos

        attn_weights = torch.nn.functional.softmax(attn_weights, dtype=torch.float32, dim=-1).to(query.dtype)

        attn_probs = nn.functional.dropout(attn_weights, p=self.dropout, training=self.training)

        attn_output = (attn_probs @ value).reshape(batch_size, self.num_attention_heads, height, width, -1)
        attn_output = attn_output.permute(0, 2, 3, 1, 4).reshape(batch_size, height, width, -1)

        attn_output = self.proj(attn_output)
        return attn_output, attn_weights


@auto_docstring
class UnlimitedOcrPreTrainedModel(PreTrainedModel):
    config: UnlimitedOcrConfig
    base_model_prefix = "model"
    input_modalities = ("image", "text")
    supports_gradient_checkpointing = True
    _skip_keys_device_placement = ["past_key_values"]
    # SAM uses rel-pos bias, incompatible with flash attention.
    _supports_flash_attn = False
    _supports_sdpa = True

    _can_compile_fullgraph = True
    # SAM doesn't support flex attention
    _supports_flex_attn = False
    _supports_attention_backend = True
    _no_split_modules = [
        "UnlimitedOcrEncoderLayer",
        "UnlimitedOcrSamVisionLayer",
        "UnlimitedOcrTextDecoderLayer",
    ]

    @torch.no_grad()
    def _init_weights(self, module):
        super()._init_weights(module)
        if isinstance(module, UnlimitedOcrSamVisionAttention):
            if module.use_rel_pos:
                init.zeros_(module.rel_pos_h)
                init.zeros_(module.rel_pos_w)
        elif isinstance(module, UnlimitedOcrSamVisionEncoder):
            if module.pos_embed is not None:
                init.zeros_(module.pos_embed)
        elif isinstance(module, UnlimitedOcrModel):
            embed_std = 1 / math.sqrt(self.config.text_config.hidden_size)
            init.normal_(module.view_separator, mean=0.0, std=embed_std)
        if isinstance(module, UnlimitedOcrModel):
            embed_std = 1 / math.sqrt(self.config.text_config.hidden_size)
            init.normal_(module.image_newline, mean=0.0, std=embed_std)
        elif isinstance(module, UnlimitedOcrVisionEmbeddings):
            factor = module.config.initializer_factor
            init.normal_(module.class_embedding, mean=0.0, std=module.embed_dim**-0.5 * factor)
            init.normal_(module.position_embedding.weight, std=module.config.initializer_range * factor)
            init.copy_(module.position_ids, torch.arange(module.num_positions).expand((1, -1)))


class UnlimitedOcrSamMLPBlock(nn.Module):
    def __init__(self, config):
        super().__init__()
        self.lin1 = nn.Linear(config.hidden_size, config.mlp_dim)
        self.lin2 = nn.Linear(config.mlp_dim, config.hidden_size)
        self.act = ACT2FN[config.hidden_act]

    def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
        hidden_states = self.lin1(hidden_states)
        hidden_states = self.act(hidden_states)
        hidden_states = self.lin2(hidden_states)
        return hidden_states


class UnlimitedOcrSamVisionSdpaAttention(UnlimitedOcrSamVisionAttention):
    """
    Multi-head Attention block with relative position embeddings.
    Using SDPA instead of the default attention.
    """

    def __init__(self, config, window_size):
        super().__init__(config, window_size)

    def forward(self, hidden_states: torch.Tensor, output_attentions=False) -> torch.Tensor:
        if output_attentions:
            logger.warning_once(
                f"{self.__class__.__name__} does not support `output_attentions=True`. The returned attention weights will "
                "be `None`. If you want to get attention weights, please set `attn_implementation='eager'` when loading the model."
            )
        batch_size, height, width, _ = hidden_states.shape
        # qkv with shape (3, B, nHead, H * W, C)
        qkv = (
            self.qkv(hidden_states)
            .reshape(batch_size, height * width, 3, self.num_attention_heads, -1)
            .permute(2, 0, 3, 1, 4)
        )
        # q, k, v with shape (B * nHead, H * W, C)
        query, key, value = qkv.reshape(3, batch_size * self.num_attention_heads, height * width, -1).unbind(0)

        attn_bias = None
        if self.use_rel_pos:
            decomposed_rel_pos = self.get_decomposed_rel_pos(
                query, self.rel_pos_h, self.rel_pos_w, (height, width), (height, width)
            )
            decomposed_rel_pos = decomposed_rel_pos.reshape(
                batch_size, self.num_attention_heads, height * width, height * width
            )
            attn_bias = decomposed_rel_pos

        query = query.view(batch_size, self.num_attention_heads, height * width, -1)
        key = key.view(batch_size, self.num_attention_heads, height * width, -1)
        value = value.view(batch_size, self.num_attention_heads, height * width, -1)

        attn_output = torch.nn.functional.scaled_dot_product_attention(query, key, value, attn_mask=attn_bias)

        attn_output = (
            attn_output.view(batch_size, self.num_attention_heads, height, width, -1)
            .permute(0, 2, 3, 1, 4)
            .reshape(batch_size, height, width, -1)
        )

        attn_output = self.proj(attn_output)
        return attn_output, None


UNLIMITED_OCR_SAM_VISION_ATTENTION_CLASSES = {
    "eager": UnlimitedOcrSamVisionAttention,
    "sdpa": UnlimitedOcrSamVisionSdpaAttention,
}


class UnlimitedOcrSamVisionLayer(GradientCheckpointingLayer):
    def __init__(self, config, window_size):
        super().__init__()
        self.layer_norm1 = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps)
        self.attn = UNLIMITED_OCR_SAM_VISION_ATTENTION_CLASSES[config._attn_implementation](config, window_size)
        self.layer_norm2 = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps)
        self.mlp = UnlimitedOcrSamMLPBlock(config)
        self.window_size = window_size

    def window_partition(self, hidden_states: torch.Tensor, window_size: int) -> tuple[torch.Tensor, tuple[int, int]]:
        """
        Args:
        Partition into non-overlapping windows with padding if needed.
            hidden_states (tensor): input tokens with [batch_size, height, width, channel]. window_size (int): window
            size.

        Returns:
            windows: windows after partition with [batch_size * num_windows, window_size, window_size, channel].
            (pad_height, pad_width): padded height and width before partition
        """
        batch_size, height, width, channel = hidden_states.shape

        pad_h = (window_size - height % window_size) % window_size
        pad_w = (window_size - width % window_size) % window_size
        hidden_states = F.pad(hidden_states, (0, 0, 0, pad_w, 0, pad_h))
        pad_height, pad_width = height + pad_h, width + pad_w

        hidden_states = hidden_states.reshape(
            batch_size, pad_height // window_size, window_size, pad_width // window_size, window_size, channel
        )
        windows = hidden_states.permute(0, 1, 3, 2, 4, 5).contiguous().reshape(-1, window_size, window_size, channel)
        return windows, (pad_height, pad_width)

    def window_unpartition(
        self, windows: torch.Tensor, window_size: int, padding_shape: tuple[int, int], original_shape: tuple[int, int]
    ) -> torch.Tensor:
        """
        Args:
        Window unpartition into original sequences and removing padding.
            hidden_states (tensor):
                input tokens with [batch_size * num_windows, window_size, window_size, channel].
            window_size (int):
                window size.
            padding_shape (Tuple):
                padded height and width (pad_height, pad_width).
            original_shape (Tuple): original height and width (height, width) before padding.

        Returns:
            hidden_states: unpartitioned sequences with [batch_size, height, width, channel].
        """
        pad_height, pad_width = padding_shape
        height, width = original_shape
        batch_size = windows.shape[0] // (pad_height * pad_width // window_size // window_size)
        hidden_states = windows.reshape(
            batch_size, pad_height // window_size, pad_width // window_size, window_size, window_size, -1
        )
        hidden_states = (
            hidden_states.permute(0, 1, 3, 2, 4, 5).contiguous().reshape(batch_size, pad_height, pad_width, -1)
        )

        hidden_states = hidden_states[:, :height, :width, :].contiguous()
        return hidden_states

    def forward(self, hidden_states: torch.Tensor) -> tuple[torch.Tensor]:
        residual = hidden_states
        hidden_states = self.layer_norm1(hidden_states)
        # Window partition
        if self.window_size > 0:
            height, width = hidden_states.shape[1], hidden_states.shape[2]
            hidden_states, padding_shape = self.window_partition(hidden_states, self.window_size)

        hidden_states, attn_weights = self.attn(
            hidden_states=hidden_states,
        )
        # Reverse window partition
        if self.window_size > 0:
            hidden_states = self.window_unpartition(hidden_states, self.window_size, padding_shape, (height, width))

        hidden_states = residual + hidden_states
        layernorm_output = self.layer_norm2(hidden_states)
        hidden_states = hidden_states + self.mlp(layernorm_output)
        return hidden_states


class UnlimitedOcrSamLayerNorm(nn.LayerNorm):
    r"""LayerNorm that supports two data formats: channels_last (default) or channels_first.
    The ordering of the dimensions in the inputs. channels_last corresponds to inputs with shape (batch_size, height,
    width, channels) while channels_first corresponds to inputs with shape (batch_size, channels, height, width).
    """

    def __init__(self, normalized_shape, *, eps=1e-6, data_format="channels_last", **kwargs):
        super().__init__(normalized_shape, eps=eps, **kwargs)
        if data_format not in ["channels_last", "channels_first"]:
            raise NotImplementedError(f"Unsupported data format: {data_format}")
        self.data_format = data_format

    def forward(self, features: torch.Tensor) -> torch.Tensor:
        """
        Args:
            features: Tensor of shape (batch_size, channels, height, width) OR (batch_size, height, width, channels)
        """
        if self.data_format == "channels_first":
            features = features.permute(0, 2, 3, 1)
            features = super().forward(features)
            features = features.permute(0, 3, 1, 2)
        else:
            features = super().forward(features)
        return features


class UnlimitedOcrSamVisionNeck(nn.Module):
    def __init__(self, config: UnlimitedOcrSamVisionConfig):
        super().__init__()
        self.config = config

        self.conv1 = nn.Conv2d(config.hidden_size, config.output_channels, kernel_size=1, bias=False)
        self.layer_norm1 = UnlimitedOcrSamLayerNorm(config.output_channels, data_format="channels_first")
        self.conv2 = nn.Conv2d(config.output_channels, config.output_channels, kernel_size=3, padding=1, bias=False)
        self.layer_norm2 = UnlimitedOcrSamLayerNorm(config.output_channels, data_format="channels_first")

    def forward(self, hidden_states):
        hidden_states = hidden_states.permute(0, 3, 1, 2)
        hidden_states = self.conv1(hidden_states)
        hidden_states = self.layer_norm1(hidden_states)

        hidden_states = self.conv2(hidden_states)
        hidden_states = self.layer_norm2(hidden_states)
        return hidden_states


class UnlimitedOcrSamPatchEmbeddings(nn.Module):
    """
    This class turns `pixel_values` of shape `(batch_size, num_channels, height, width)` into the initial
    `hidden_states` (patch embeddings) of shape `(batch_size, seq_length, hidden_size)` to be consumed by a
    Transformer.
    """

    def __init__(self, config):
        super().__init__()
        image_size, patch_size = config.image_size, config.patch_size
        num_channels, hidden_size = config.num_channels, config.hidden_size
        image_size = image_size if isinstance(image_size, collections.abc.Iterable) else (image_size, image_size)
        patch_size = patch_size if isinstance(patch_size, collections.abc.Iterable) else (patch_size, patch_size)
        num_patches = (image_size[1] // patch_size[1]) * (image_size[0] // patch_size[0])
        self.image_size = image_size
        self.patch_size = patch_size
        self.num_channels = num_channels
        self.num_patches = num_patches

        self.projection = nn.Conv2d(num_channels, hidden_size, kernel_size=patch_size, stride=patch_size)

    def forward(self, pixel_values):
        embeddings = self.projection(pixel_values).permute(0, 2, 3, 1)
        return embeddings


class UnlimitedOcrSamVisionProj(nn.Module):
    """Neck and multi-scale downsampling for SAM ViT-B output."""

    def __init__(self, config: UnlimitedOcrSamVisionConfig):
        super().__init__()
        self.conv1 = nn.Conv2d(
            config.output_channels,
            config.downsample_channels[0],
            kernel_size=3,
            stride=2,
            padding=1,
            bias=False,
        )
        self.conv2 = nn.Conv2d(
            config.downsample_channels[0],
            config.downsample_channels[1],
            kernel_size=3,
            stride=2,
            padding=1,
            bias=False,
        )

    def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
        hidden_states = self.conv1(hidden_states)
        hidden_states = self.conv2(hidden_states)
        return hidden_states


class UnlimitedOcrSamVisionEncoder(UnlimitedOcrPreTrainedModel):
    config: UnlimitedOcrSamVisionConfig
    _can_record_outputs = {"hidden_states": UnlimitedOcrSamVisionLayer, "attentions": UnlimitedOcrSamVisionAttention}
    _input_embed_layer = "patch_embed"

    def __init__(self, config: UnlimitedOcrSamVisionConfig):
        super().__init__(config)
        self.config = config
        self.image_size = config.image_size
        self.patch_embed = UnlimitedOcrSamPatchEmbeddings(config)

        self.pos_embed = None
        if config.use_abs_pos:
            # Initialize absolute positional embedding with pretrain image size.
            self.pos_embed = nn.Parameter(
                torch.zeros(
                    1,
                    config.image_size // config.patch_size,
                    config.image_size // config.patch_size,
                    config.hidden_size,
                )
            )

        self.layers = nn.ModuleList()
        for i in range(config.num_hidden_layers):
            layer = UnlimitedOcrSamVisionLayer(
                config,
                window_size=config.window_size if i not in config.global_attn_indexes else 0,
            )
            self.layers.append(layer)

        self.neck = UnlimitedOcrSamVisionNeck(config)

        self.gradient_checkpointing = False
        self.proj = UnlimitedOcrSamVisionProj(config)
        self.post_init()

    @merge_with_config_defaults
    @capture_outputs
    @auto_docstring
    def forward(self, pixel_values: torch.Tensor, **kwargs) -> BaseModelOutput:
        hidden_states = self.patch_embed(pixel_values)
        if self.pos_embed is not None:
            hidden_states = hidden_states + self.interpolate_pos_encoding(
                hidden_states.shape[1], hidden_states.shape[2]
            )

        for layer_module in self.layers:
            hidden_states = layer_module(hidden_states)

        hidden_states = self.neck(hidden_states)
        hidden_states = self.proj(hidden_states)
        return BaseModelOutput(last_hidden_state=hidden_states)

    def interpolate_pos_encoding(self, height: int, width: int) -> torch.Tensor:
        """Interpolate the positional encoding to match the target spatial size using bicubic interpolation."""
        if not torch.jit.is_tracing() and self.pos_embed.shape[1] == height and self.pos_embed.shape[2] == width:
            return self.pos_embed

        target_dtype = self.pos_embed.dtype
        pos_embed = self.pos_embed.permute(0, 3, 1, 2)
        pos_embed = torch.nn.functional.interpolate(
            pos_embed.to(torch.float32),
            size=(height, width),
            mode="bicubic",
            align_corners=False,
            antialias=True,
        ).to(dtype=target_dtype)
        pos_embed = pos_embed.permute(0, 2, 3, 1)
        return pos_embed


def eager_attention_forward(
    module: nn.Module,
    query: torch.Tensor,
    key: torch.Tensor,
    value: torch.Tensor,
    attention_mask: torch.Tensor | None,
    scaling: float,
    dropout: float = 0.0,
    **kwargs: Unpack[TransformersKwargs],
):
    attn_weights = torch.matmul(query, key.transpose(-1, -2)) * scaling
    if attention_mask is not None:
        attn_weights = attn_weights + attention_mask
    attn_weights = nn.functional.softmax(attn_weights, dim=-1, dtype=torch.float32).to(query.dtype)
    attn_weights = nn.functional.dropout(attn_weights, p=dropout, training=module.training)

    attn_output = torch.matmul(attn_weights, value)
    attn_output = attn_output.transpose(1, 2).contiguous()
    return attn_output, attn_weights


class UnlimitedOcrAttention(nn.Module):
    """Multi-headed attention from 'Attention Is All You Need' paper"""

    def __init__(self, config: UnlimitedOcrVisionEncoderConfig):
        super().__init__()
        self.config = config
        self.embed_dim = config.hidden_size
        self.num_heads = config.num_attention_heads
        self.head_dim = self.embed_dim // self.num_heads
        self.scale = self.head_dim**-0.5
        self.dropout = config.attention_dropout
        self.is_causal = False

        self.k_proj = nn.Linear(self.embed_dim, self.embed_dim)
        self.v_proj = nn.Linear(self.embed_dim, self.embed_dim)
        self.q_proj = nn.Linear(self.embed_dim, self.embed_dim)
        self.out_proj = nn.Linear(self.embed_dim, self.embed_dim)
        # Required for GQA compatibility
        self.num_key_value_groups = 1

    def forward(
        self,
        hidden_states: torch.Tensor,
        attention_mask: torch.Tensor | None = None,
        **kwargs: Unpack[TransformersKwargs],
    ) -> tuple[torch.Tensor, torch.Tensor | None]:
        """Input shape: Batch x Time x Channel"""

        input_shape = hidden_states.shape[:-1]

        hidden_shape = (*input_shape, -1, self.head_dim)
        queries = self.q_proj(hidden_states)
        keys = self.k_proj(hidden_states)
        values = self.v_proj(hidden_states)

        queries = queries.view(hidden_shape).transpose(1, 2)
        keys = keys.view(hidden_shape).transpose(1, 2)
        values = values.view(hidden_shape).transpose(1, 2)

        attention_interface: Callable = ALL_ATTENTION_FUNCTIONS.get_interface(
            self.config._attn_implementation, eager_attention_forward
        )

        attn_output, attn_weights = attention_interface(
            self,
            queries,
            keys,
            values,
            attention_mask,
            scaling=self.scale,
            dropout=0.0 if not self.training else self.dropout,
            **kwargs,
        )

        attn_output = attn_output.reshape(*input_shape, -1).contiguous()
        attn_output = self.out_proj(attn_output)

        return attn_output, attn_weights


class UnlimitedOcrMLP(nn.Module):
    def __init__(self, config):
        super().__init__()
        self.config = config
        self.activation_fn = ACT2FN[config.hidden_act]
        self.fc1 = nn.Linear(config.hidden_size, config.intermediate_size)
        self.fc2 = nn.Linear(config.intermediate_size, config.hidden_size)

    def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
        hidden_states = self.fc1(hidden_states)
        hidden_states = self.activation_fn(hidden_states)
        hidden_states = self.fc2(hidden_states)
        return hidden_states


class UnlimitedOcrEncoderLayer(GradientCheckpointingLayer):
    def __init__(self, config: UnlimitedOcrVisionConfig | UnlimitedOcrTextConfig):
        super().__init__()
        self.embed_dim = config.hidden_size
        self.self_attn = UnlimitedOcrAttention(config)
        self.layer_norm1 = nn.LayerNorm(self.embed_dim, eps=config.layer_norm_eps)
        self.mlp = UnlimitedOcrMLP(config)
        self.layer_norm2 = nn.LayerNorm(self.embed_dim, eps=config.layer_norm_eps)

    def forward(
        self,
        hidden_states: torch.Tensor,
        attention_mask: torch.Tensor,
        **kwargs: Unpack[TransformersKwargs],
    ) -> torch.Tensor:
        residual = hidden_states

        hidden_states = self.layer_norm1(hidden_states)
        hidden_states, _ = self.self_attn(
            hidden_states=hidden_states,
            attention_mask=attention_mask,
            **kwargs,
        )
        hidden_states = residual + hidden_states

        residual = hidden_states
        hidden_states = self.layer_norm2(hidden_states)
        hidden_states = self.mlp(hidden_states)
        hidden_states = residual + hidden_states

        return hidden_states


class UnlimitedOcrVisionEmbeddings(nn.Module):
    def __init__(self, config: UnlimitedOcrVisionConfig):
        super().__init__()
        self.config = config
        self.embed_dim = config.hidden_size
        self.image_size = config.image_size
        self.patch_size = config.patch_size

        self.class_embedding = nn.Parameter(torch.randn(self.embed_dim))

        self.num_patches = (self.image_size // self.patch_size) ** 2
        self.num_positions = self.num_patches + 1
        self.position_embedding = nn.Embedding(self.num_positions, self.embed_dim)
        self.position_ids = nn.Buffer(torch.arange(self.num_positions).expand((1, -1)), persistent=False)

    def interpolate_pos_encoding(self, embeddings: torch.Tensor, height: int, width: int) -> torch.Tensor:
        """
        This method allows to interpolate the pre-trained position encodings, to be able to use the model on higher resolution
        images. This method is also adapted to support torch.jit tracing.

        Adapted from:
        - https://github.com/facebookresearch/dino/blob/de9ee3df6cf39fac952ab558447af1fa1365362a/vision_transformer.py#L174-L194, and
        - https://github.com/facebookresearch/dinov2/blob/e1277af2ba9496fbadf7aec6eba56e8d882d1e35/dinov2/models/vision_transformer.py#L179-L211
        """
        num_patches = embeddings.shape[1] - 1
        position_embedding = self.position_embedding.weight.unsqueeze(0)
        num_positions = position_embedding.shape[1] - 1

        # always interpolate when tracing to ensure the exported model works for dynamic input shapes
        if not torch.jit.is_tracing() and num_patches == num_positions and height == width:
            return self.position_embedding(self.position_ids)

        class_pos_embed = position_embedding[:, :1]
        patch_pos_embed = position_embedding[:, 1:]

        dim = embeddings.shape[-1]

        new_height = height // self.patch_size
        new_width = width // self.patch_size

        sqrt_num_positions = torch_int(num_positions**0.5)
        patch_pos_embed = patch_pos_embed.reshape(1, sqrt_num_positions, sqrt_num_positions, dim)
        patch_pos_embed = patch_pos_embed.permute(0, 3, 1, 2)

        target_dtype = patch_pos_embed.dtype
        patch_pos_embed = nn.functional.interpolate(
            patch_pos_embed.to(torch.float32),
            size=(new_height, new_width),
            mode="bicubic",
            antialias=True,  # different from CLIP
            align_corners=False,
        ).to(target_dtype)

        patch_pos_embed = patch_pos_embed.permute(0, 2, 3, 1).view(1, -1, dim)

        return torch.cat((class_pos_embed, patch_pos_embed), dim=1)

    def forward(self, patch_embeds: torch.Tensor) -> torch.Tensor:
        r"""
        patch_embeds (`torch.Tensor` of shape `(batch_size, hidden_size, grid_height, grid_width)`):
            The SAM feature map, injected directly as patch embeddings.
        """
        batch_size, _, grid_height, grid_width = patch_embeds.shape
        patch_embeds = patch_embeds.flatten(2).transpose(1, 2)

        class_embeds = self.class_embedding.expand(batch_size, 1, -1)
        embeddings = torch.cat([class_embeds, patch_embeds], dim=1)
        # always interpolate compared to clip where this can be optional
        embeddings = embeddings + self.interpolate_pos_encoding(
            embeddings, grid_height * self.patch_size, grid_width * self.patch_size
        )
        return embeddings


class UnlimitedOcrEncoder(nn.Module):
    """
    Transformer encoder consisting of `config.num_hidden_layers` self attention layers. Each layer is a
    [`UnlimitedOcrEncoderLayer`].

    Args:
        config: UnlimitedOcrConfig
    """

    def __init__(self, config: UnlimitedOcrConfig):
        super().__init__()
        self.config = config
        self.layers = nn.ModuleList([UnlimitedOcrEncoderLayer(config) for _ in range(config.num_hidden_layers)])
        self.gradient_checkpointing = False

    def forward(
        self,
        inputs_embeds,
        attention_mask: torch.Tensor | None = None,
        **kwargs: Unpack[TransformersKwargs],
    ) -> BaseModelOutput:
        hidden_states = inputs_embeds
        for encoder_layer in self.layers:
            hidden_states = encoder_layer(
                hidden_states,
                attention_mask,
                **kwargs,
            )

        return BaseModelOutput(
            last_hidden_state=hidden_states,
        )


@auto_docstring(
    custom_intro="""
    The vision encoder from Unlimited OCR without any head or projection on top.
    """
)
class UnlimitedOcrVisionEncoder(UnlimitedOcrPreTrainedModel):
    config: UnlimitedOcrVisionEncoderConfig
    main_input_name = "patch_embeds"
    input_modalities = ("image",)
    _can_record_outputs = {
        "hidden_states": UnlimitedOcrEncoderLayer,
        "attentions": UnlimitedOcrAttention,
    }
    _keys_to_ignore_on_load_unexpected = {"patch_embedding"}  # unused

    def __init__(self, config: UnlimitedOcrVisionEncoderConfig):
        super().__init__(config)
        embed_dim = config.hidden_size

        self.embeddings = UnlimitedOcrVisionEmbeddings(config)
        self.pre_layrnorm = nn.LayerNorm(embed_dim, eps=config.layer_norm_eps)
        self.encoder = UnlimitedOcrEncoder(config)
        self.post_init()

    @merge_with_config_defaults
    @capture_outputs
    @auto_docstring
    def forward(self, patch_embeds: torch.Tensor, **kwargs: Unpack[TransformersKwargs]) -> BaseModelOutput:
        r"""
        patch_embeds (`torch.Tensor` of shape `(batch_size, hidden_size, grid_height, grid_width)`):
            Patch embeddings.
        """
        hidden_states = self.embeddings(patch_embeds)
        hidden_states = self.pre_layrnorm(hidden_states)
        encoder_outputs = self.encoder(inputs_embeds=hidden_states, **kwargs)
        return BaseModelOutput(last_hidden_state=encoder_outputs.last_hidden_state)


@auto_docstring(custom_intro="Vision model encoding images first with SAM followed by an additional (CLIP) model.")
class UnlimitedOcrVisionModel(UnlimitedOcrPreTrainedModel):
    config: UnlimitedOcrVisionConfig

    def __init__(self, config: UnlimitedOcrVisionConfig):
        super().__init__(config)
        self.sam_encoder = UnlimitedOcrSamVisionEncoder(config.sam_config)
        self.vision_encoder = UnlimitedOcrVisionEncoder(config.encoder_config)
        self.post_init()

    @can_return_tuple
    @auto_docstring
    def forward(self, pixel_values: torch.Tensor, **kwargs: Unpack[TransformersKwargs]) -> BaseModelOutput:
        r"""
        Example:

        ```python
        >>> import torch
        >>> from transformers import AutoConfig, AutoImageProcessor, AutoModel
        >>> from transformers.image_utils import load_image

        >>> model_id = "baidu/Unlimited-OCR"
        >>> config = AutoConfig.from_pretrained(model_id)
        >>> model = AutoModel.from_pretrained(model_id, config=config.vision_config, device_map="auto")
        >>> image_processor = AutoImageProcessor.from_pretrained(model_id)

        >>> image = load_image("https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/transformers/model_doc/ocr_suggestion_form.jpg")
        >>> inputs = image_processor(images=image, return_tensors="pt").to(model.device, model.dtype)

        >>> with torch.no_grad():
        ...     outputs = model(**inputs)

        >>> # Patch tokens without class tokens (batch_size, height * width, vision_hidden_size + sam_hidden_size)
        >>> patch_tokens = outputs.last_hidden_state
        ```"""

        sam_outputs = self.sam_encoder(pixel_values, **kwargs)
        residual = sam_outputs.last_hidden_state

        vision_encoder_outputs = self.vision_encoder(residual, **kwargs)
        hidden_states = vision_encoder_outputs.last_hidden_state

        residual = residual.flatten(2).transpose(1, 2)
        hidden_states = torch.cat([hidden_states[:, 1:], residual], dim=-1)

        return BaseModelOutput(
            last_hidden_state=hidden_states,
            hidden_states=vision_encoder_outputs.hidden_states,
            attentions=vision_encoder_outputs.attentions,
        )


def rotate_half(x):
    """Rotates half the hidden dims of the input."""
    x1 = x[..., : x.shape[-1] // 2]
    x2 = x[..., x.shape[-1] // 2 :]
    return torch.cat((-x2, x1), dim=-1)


@use_kernel_forward_from_hub("rotary_pos_emb")
def apply_rotary_pos_emb(q, k, cos, sin, unsqueeze_dim=1):
    """Applies Rotary Position Embedding to the query and key tensors.

    Args:
        q (`torch.Tensor`): The query tensor.
        k (`torch.Tensor`): The key tensor.
        cos (`torch.Tensor`): The cosine part of the rotary embedding.
        sin (`torch.Tensor`): The sine part of the rotary embedding.
        unsqueeze_dim (`int`, *optional*, defaults to 1):
            The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
            sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
            that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
            k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
            cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
            the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
    Returns:
        `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
    """
    cos = cos.unsqueeze(unsqueeze_dim)
    sin = sin.unsqueeze(unsqueeze_dim)
    q_embed = (q * cos) + (rotate_half(q) * sin)
    k_embed = (k * cos) + (rotate_half(k) * sin)
    return q_embed, k_embed


def repeat_kv(hidden_states: torch.Tensor, n_rep: int) -> torch.Tensor:
    """
    This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,
    num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)
    """
    batch, num_key_value_heads, slen, head_dim = hidden_states.shape
    if n_rep == 1:
        return hidden_states
    hidden_states = hidden_states[:, :, None, :, :].expand(batch, num_key_value_heads, n_rep, slen, head_dim)
    return hidden_states.reshape(batch, num_key_value_heads * n_rep, slen, head_dim)


@use_kernelized_func(apply_rotary_pos_emb)
class UnlimitedOcrTextAttention(nn.Module):
    """Multi-headed attention from 'Attention Is All You Need' paper"""

    def __init__(self, config: UnlimitedOcrTextConfig, layer_idx: int):
        super().__init__()
        self.config = config
        self.layer_idx = layer_idx
        self.head_dim = getattr(config, "head_dim", config.hidden_size // config.num_attention_heads)
        self.num_key_value_groups = config.num_attention_heads // config.num_key_value_heads
        self.scaling = self.head_dim**-0.5
        self.attention_dropout = config.attention_dropout
        self.is_causal = True

        self.q_proj = nn.Linear(
            config.hidden_size, config.num_attention_heads * self.head_dim, bias=config.attention_bias
        )
        self.k_proj = nn.Linear(
            config.hidden_size, config.num_key_value_heads * self.head_dim, bias=config.attention_bias
        )
        self.v_proj = nn.Linear(
            config.hidden_size, config.num_key_value_heads * self.head_dim, bias=config.attention_bias
        )
        self.o_proj = nn.Linear(
            config.num_attention_heads * self.head_dim, config.hidden_size, bias=config.attention_bias
        )

    def forward(
        self,
        hidden_states: torch.Tensor,
        position_embeddings: tuple[torch.Tensor, torch.Tensor] | None = None,
        attention_mask: torch.Tensor | None = None,
        past_key_values: Cache | None = None,
        **kwargs: Unpack[TransformersKwargs],
    ) -> tuple[torch.Tensor, torch.Tensor]:
        input_shape = hidden_states.shape[:-1]
        hidden_shape = (*input_shape, -1, self.head_dim)

        query_states = self.q_proj(hidden_states).view(hidden_shape).transpose(1, 2)
        key_states = self.k_proj(hidden_states).view(hidden_shape).transpose(1, 2)
        value_states = self.v_proj(hidden_states).view(hidden_shape).transpose(1, 2)

        cos, sin = position_embeddings
        query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin)

        if past_key_values is not None:
            key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx)

        attention_interface: Callable = ALL_ATTENTION_FUNCTIONS.get_interface(
            self.config._attn_implementation, eager_attention_forward
        )

        attn_output, attn_weights = attention_interface(
            self,
            query_states,
            key_states,
            value_states,
            attention_mask,
            dropout=0.0 if not self.training else self.attention_dropout,
            scaling=self.scaling,
            **kwargs,
        )

        attn_output = attn_output.reshape(*input_shape, -1).contiguous()
        attn_output = self.o_proj(attn_output)
        return attn_output, attn_weights


class UnlimitedOcrTextMLP(nn.Module):
    def __init__(self, config: UnlimitedOcrTextConfig, hidden_size=None, intermediate_size=None):
        super().__init__()
        self.config = config
        self.hidden_size = config.hidden_size if hidden_size is None else hidden_size
        self.intermediate_size = config.intermediate_size if intermediate_size is None else intermediate_size
        self.gate_proj = nn.Linear(self.hidden_size, self.intermediate_size, bias=config.mlp_bias)
        self.up_proj = nn.Linear(self.hidden_size, self.intermediate_size, bias=config.mlp_bias)
        self.down_proj = nn.Linear(self.intermediate_size, self.hidden_size, bias=config.mlp_bias)
        self.act_fn = ACT2FN[config.hidden_act]

    def forward(self, x):
        down_proj = self.down_proj(self.act_fn(self.gate_proj(x)) * self.up_proj(x))
        return down_proj


@use_experts_implementation
class UnlimitedOcrTextExperts(nn.Module):
    """Collection of expert weights stored as 3D tensors."""

    def __init__(self, config):
        super().__init__()
        self.num_experts = config.num_experts
        self.hidden_dim = config.hidden_size
        self.intermediate_dim = config.moe_intermediate_size
        self.gate_up_proj = nn.Parameter(torch.empty(self.num_experts, 2 * self.intermediate_dim, self.hidden_dim))
        self.down_proj = nn.Parameter(torch.empty(self.num_experts, self.hidden_dim, self.intermediate_dim))
        self.act_fn = ACT2FN[config.hidden_act]

    def forward(
        self,
        hidden_states: torch.Tensor,
        top_k_index: torch.Tensor,
        top_k_weights: torch.Tensor,
    ) -> torch.Tensor:
        final_hidden_states = torch.zeros_like(hidden_states)
        with torch.no_grad():
            expert_mask = torch.nn.functional.one_hot(top_k_index, num_classes=self.num_experts + 1)
            expert_mask = expert_mask.permute(2, 1, 0)
            expert_hit = torch.greater(expert_mask.sum(dim=(-1, -2)), 0).nonzero()

        for expert_idx in expert_hit:
            expert_idx = expert_idx[0]
            if expert_idx == self.num_experts:
                continue
            top_k_pos, token_idx = torch.where(expert_mask[expert_idx])
            current_state = hidden_states[token_idx]
            gate, up = nn.functional.linear(current_state, self.gate_up_proj[expert_idx]).chunk(2, dim=-1)
            current_hidden_states = self.act_fn(gate) * up
            current_hidden_states = nn.functional.linear(current_hidden_states, self.down_proj[expert_idx])
            current_hidden_states = current_hidden_states * top_k_weights[token_idx, top_k_pos, None]
            final_hidden_states.index_add_(0, token_idx, current_hidden_states.to(final_hidden_states.dtype))

        return final_hidden_states


class UnlimitedOcrTextTopkRouter(nn.Module):
    def __init__(self, config: UnlimitedOcrTextConfig):
        super().__init__()
        self.top_k = config.num_experts_per_tok
        self.num_experts = config.num_experts
        self.hidden_dim = config.hidden_size
        self.weight = nn.Parameter(torch.zeros(self.num_experts, self.hidden_dim))
        self.routed_scaling_factor = config.routed_scaling_factor
        self.topk_method = config.topk_method
        self.num_group = config.n_group
        self.topk_group = config.topk_group

    def forward(self, hidden_states):
        hidden_states = hidden_states.view(-1, self.hidden_dim)
        router_logits = F.linear(hidden_states.type(torch.float32), self.weight.type(torch.float32))
        scores = router_logits.softmax(dim=-1, dtype=torch.float32)
        if self.topk_method == "greedy":
            topk_weights, topk_indices = torch.topk(scores, k=self.top_k, dim=-1, sorted=False)
        elif self.topk_method == "group_limited_greedy":
            group_scores = scores.view(-1, self.num_group, self.num_experts // self.num_group).max(dim=-1).values
            group_idx = torch.topk(group_scores, k=self.topk_group, dim=-1, sorted=False)[1]
            group_mask = torch.zeros_like(group_scores)
            group_mask.scatter_(1, group_idx, 1)
            score_mask = (
                group_mask.unsqueeze(-1)
                .expand(-1, self.num_group, self.num_experts // self.num_group)
                .reshape(-1, self.num_experts)
            )
            scores = scores.masked_fill(~score_mask.bool(), 0.0)
            topk_weights, topk_indices = torch.topk(scores, k=self.top_k, dim=-1, sorted=False)
        topk_weights = topk_weights * self.routed_scaling_factor
        return router_logits, topk_weights, topk_indices


class UnlimitedOcrTextMoe(nn.Module):
    def __init__(self, config: UnlimitedOcrTextConfig):
        super().__init__()
        self.config = config
        self.experts = UnlimitedOcrTextExperts(config)
        self.gate = UnlimitedOcrTextTopkRouter(config)
        self.shared_experts = UnlimitedOcrTextMLP(
            config=config, intermediate_size=config.moe_intermediate_size * config.n_shared_experts
        )

    def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
        residuals = hidden_states
        orig_shape = hidden_states.shape
        _, topk_weights, topk_indices = self.gate(hidden_states)
        hidden_states = hidden_states.view(-1, hidden_states.shape[-1])
        hidden_states = self.experts(hidden_states, topk_indices, topk_weights).view(*orig_shape)
        hidden_states = hidden_states + self.shared_experts(residuals)
        return hidden_states


@use_kernel_forward_from_hub("RMSNorm")
class UnlimitedOcrTextRMSNorm(nn.Module):
    def __init__(self, hidden_size, eps: float = 1e-6) -> None:
        """
        UnlimitedOcrTextRMSNorm is equivalent to T5LayerNorm
        """
        super().__init__()
        self.weight = nn.Parameter(torch.ones(hidden_size))
        self.variance_epsilon = eps

    def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
        input_dtype = hidden_states.dtype
        hidden_states = hidden_states.to(torch.float32)
        variance = hidden_states.pow(2).mean(-1, keepdim=True)
        hidden_states = hidden_states * torch.rsqrt(variance + self.variance_epsilon)
        return self.weight * hidden_states.to(input_dtype)

    def extra_repr(self):
        return f"{tuple(self.weight.shape)}, eps={self.variance_epsilon}"


class UnlimitedOcrTextDecoderLayer(GradientCheckpointingLayer):
    def __init__(self, config, layer_idx: int):
        super().__init__()
        self.hidden_size = config.hidden_size
        self.self_attn = UnlimitedOcrTextAttention(config=config, layer_idx=layer_idx)
        self.mlp = (
            UnlimitedOcrTextMoe(config)
            if config.mlp_layer_types[layer_idx] == "sparse"
            else UnlimitedOcrTextMLP(config)
        )

        self.input_layernorm = UnlimitedOcrTextRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
        self.post_attention_layernorm = UnlimitedOcrTextRMSNorm(config.hidden_size, eps=config.rms_norm_eps)

    def forward(
        self,
        hidden_states: torch.Tensor,
        attention_mask: torch.Tensor | None = None,
        position_ids: torch.LongTensor | None = None,
        past_key_values: Cache | None = None,
        use_cache: bool | None = False,
        position_embeddings: tuple[torch.Tensor, torch.Tensor] | None = None,
        **kwargs: Unpack[TransformersKwargs],
    ) -> torch.Tensor:
        residual = hidden_states
        hidden_states = self.input_layernorm(hidden_states)
        # Self Attention
        hidden_states, _ = self.self_attn(
            hidden_states=hidden_states,
            attention_mask=attention_mask,
            position_ids=position_ids,
            past_key_values=past_key_values,
            use_cache=use_cache,
            position_embeddings=position_embeddings,
            **kwargs,
        )
        hidden_states = residual + hidden_states

        # Fully Connected
        residual = hidden_states
        hidden_states = self.post_attention_layernorm(hidden_states)
        hidden_states = self.mlp(hidden_states)
        hidden_states = residual + hidden_states
        return hidden_states


@auto_docstring
class UnlimitedOcrTextPreTrainedModel(PreTrainedModel):
    config: UnlimitedOcrTextConfig
    base_model_prefix = "model"
    supports_gradient_checkpointing = True
    _no_split_modules = ["UnlimitedOcrTextDecoderLayer"]
    _skip_keys_device_placement = ["past_key_values"]
    _supports_flash_attn = True
    _supports_sdpa = True
    _supports_flex_attn = True

    _can_compile_fullgraph = True
    _supports_attention_backend = True
    _can_record_outputs = {
        "hidden_states": UnlimitedOcrTextDecoderLayer,
        "attentions": UnlimitedOcrTextAttention,
        "router_logits": OutputRecorder(UnlimitedOcrTextTopkRouter, index=0),
    }

    @torch.no_grad()
    def _init_weights(self, module):
        super()._init_weights(module)
        if isinstance(module, UnlimitedOcrTextTopkRouter):
            init.normal_(module.weight, mean=0.0, std=self.config.initializer_range)
        elif isinstance(module, UnlimitedOcrTextExperts):
            init.normal_(module.gate_up_proj, mean=0.0, std=self.config.initializer_range)
            init.normal_(module.down_proj, mean=0.0, std=self.config.initializer_range)


class UnlimitedOcrDynamicReferenceSlidingWindowLayer(DynamicSlidingWindowLayer):
    """Reference sliding-window attention (R-SWA) cache layer that keeps all prefill tokens and applies a
    sliding window to all decoded tokens.

    Once ``sliding_window`` decode tokens have accumulated, the oldest decode tokens are evicted and
    replaced by the most recent ones. The prefill tokens always remain in the cache.

    The states of the first ``update`` are the prefill states, unless their length is declared upfront with
    ``set_prefill_length``, which is required when the prefill spans several ``update`` calls, e.g. for chunked
    prefill.
    """

    _layer_type = "reference_sliding_attention"

    def __init__(self, sliding_window: int | None = None, **kwargs):
        super().__init__(sliding_window=sliding_window, **kwargs)
        self.prefill_length: int | None = None
        self.prefill_cumulative_length: int = 0
        self.prefill_keys: torch.Tensor | None = None
        self.prefill_values: torch.Tensor | None = None

    def update(
        self, key_states: torch.Tensor, value_states: torch.Tensor, *args, **kwargs
    ) -> tuple[torch.Tensor, torch.Tensor]:
        # Lazy initialization
        if not self.is_initialized:
            self.lazy_initialization(key_states, value_states)

        # Prefill
        if self.prefill_length is None or self.prefill_cumulative_length < self.prefill_length:
            kv_length = key_states.shape[-2]

            if self.prefill_length is None:
                self.prefill_length = self.prefill_cumulative_length + kv_length

            prefill_kv_length = min(kv_length, self.prefill_length - self.prefill_cumulative_length)
            self.prefill_cumulative_length += prefill_kv_length

            self.prefill_keys = torch.cat([self.prefill_keys, key_states[..., :prefill_kv_length, :]], dim=-2)
            self.prefill_values = torch.cat([self.prefill_values, value_states[..., :prefill_kv_length, :]], dim=-2)

            if prefill_kv_length == kv_length:
                return self.prefill_keys, self.prefill_values

            # Old speculative methods can draft tokens before prefill is complete
            key_states = key_states[..., prefill_kv_length:, :]
            value_states = value_states[..., prefill_kv_length:, :]

        sliding_key_states, sliding_value_states = super().update(key_states=key_states, value_states=value_states)
        full_key_states = torch.cat([self.prefill_keys, sliding_key_states], dim=-2)
        full_value_states = torch.cat([self.prefill_values, sliding_value_states], dim=-2)
        return full_key_states, full_value_states

    def lazy_initialization(self, key_states, value_states):
        super().lazy_initialization(key_states, value_states)
        self.prefill_keys = torch.empty_like(self.keys)
        self.prefill_values = torch.empty_like(self.values)

    def get_mask_sizes(self, query_length: int) -> tuple[int, int]:
        """Return the length and offset of the cache, used to generate the attention mask"""
        # Prefill
        if self.prefill_length is None or self.prefill_cumulative_length < self.prefill_length:
            kv_offset = 0
            kv_length = self.prefill_cumulative_length + query_length
        # Decode
        else:
            kv_length, kv_offset = super().get_mask_sizes(query_length)
            kv_length += self.prefill_length

        # Returned kv_offset is with respect to sliding window keys.
        # Remove kv_offset from kv_idx to retrieve the prefill indices.
        return kv_length, kv_offset

    def get_seq_length(self) -> int:
        return self.prefill_cumulative_length + super().get_seq_length()

    def get_max_length(self) -> int:
        """Return the maximum cache shape of the cache"""
        if self.prefill_length is None:
            return -1
        return self.prefill_length + self.sliding_window

    def set_prefill_length(self, prefill_length: int) -> None:  # trf-ignore: TRF033
        """Declare how many leading tokens are prefill states, before they are cached."""
        if self.prefill_length is None:
            self.prefill_length = prefill_length

    def reset(self) -> None:
        super().reset()
        if self.is_initialized:
            self.prefill_keys.zero_()
            self.prefill_values.zero_()
        self.prefill_length = None
        self.prefill_cumulative_length = 0

    def reorder_cache(self, beam_idx: torch.LongTensor) -> None:
        """Reorders this layer's cache for beam search."""
        # Sliding window
        if self.cumulative_length > 0:
            super().reorder_cache(beam_idx)

        # Prefill
        if self.prefill_cumulative_length > 0:
            self.prefill_keys = self.prefill_keys.index_select(0, beam_idx.to(self.prefill_keys.device))
            self.prefill_values = self.prefill_values.index_select(0, beam_idx.to(self.prefill_values.device))

    def crop(self, tokens_to_remove: int) -> None:
        """
        Remove `tokens_to_remove` tokens from the current cache layer. This will also restrict the size of the cached states back to their
        minimal working size, i.e. `sliding_window - 1` if they reached the sliding window length. This means that `crop(0)` will not
        necessarily always be a no-op, as it may still remove useless states (i.e. states that are not needed for the next `forward`).
        """
        if tokens_to_remove > 0:
            raise ValueError(
                f"Calling `crop` with a positive value (`{tokens_to_remove}`) is not supported. "
                "Please use a negative integer to remove that number of tokens from the cache instead."
            )

        tokens_to_remove = abs(tokens_to_remove)
        sliding_tokens_to_remove = min(tokens_to_remove, self.cumulative_length)
        prefill_tokens_to_remove = tokens_to_remove - sliding_tokens_to_remove

        # Only call `super().crop` if we already exceeded the sliding window length
        # as super checks `self.get_seq_length()` which includes the prefill tokens.
        if self.cumulative_length >= self.sliding_window:
            super().crop(-sliding_tokens_to_remove)
        elif sliding_tokens_to_remove > 0:
            DynamicLayer.crop(self, -sliding_tokens_to_remove)
            self.cumulative_length -= sliding_tokens_to_remove

        if prefill_tokens_to_remove > 0:
            new_prefill_length = max(self.prefill_cumulative_length - prefill_tokens_to_remove, 0)
            self.prefill_keys = self.prefill_keys[..., :new_prefill_length, :]
            self.prefill_values = self.prefill_values[..., :new_prefill_length, :]
            self.prefill_cumulative_length = new_prefill_length


class UnlimitedOcrStaticReferenceSlidingWindowLayer(StaticSlidingWindowLayer):
    """
    A static cache layer that stores the key and value states as static tensors of shape
    `[batch_size, num_heads, min(max_cache_len, prefill_length + sliding_window), head_dim]`.
    It lazily allocates its full backing tensors, and then mutates them in-place.
    Built for `torch.compile` support.

    The backing buffer is split in two regions: slots ``[0, prefill_length)`` hold the prefill (reference) slots
    that are never eviced from the cache. Slots ``[prefill_length, prefill_length + sliding_window)`` hold the
    sliding window decode slots where the oldest entries are always replaced by the newest ones.

    The states of the first ``update`` are the prefill states, unless their length is declared upfront with
    ``set_prefill_length``, which is required when the prefill spans several ``update`` calls, e.g. for chunked
    prefill.

    Args:
        max_cache_len (`int`):
            Maximum number of tokens that can be stored, used for tensor preallocation.
        sliding_window (`int`):
            The size of the sliding window.
    """

    _layer_type = "reference_sliding_attention"

    def __init__(self, max_cache_len: int, sliding_window: int, **kwargs):
        super().__init__(max_cache_len=max_cache_len, sliding_window=sliding_window, **kwargs)
        # Effective sliding window length is determined by max_cache_len in parent
        self.sliding_window = self.max_cache_len
        self.prefill_max_cache_len = max_cache_len - self.sliding_window
        self.prefill_length: int | None = None
        self.prefill_cumulative_length: int = 0
        self.prefill_keys: torch.Tensor | None = None
        self.prefill_values: torch.Tensor | None = None

        # all_keys and all_values are contiguous buffers that hold the full cache to keep dynamo addresses static.
        # prefill_keys, prefill_values, keys, and values are views of the buffers:
        # all_keys = [prefill_keys, keys]
        # all_values = [prefill_values, values]
        self.all_keys: torch.Tensor | None = None
        self.all_values: torch.Tensor | None = None

    def update(
        self, key_states: torch.Tensor, value_states: torch.Tensor, *args, **kwargs
    ) -> tuple[torch.Tensor, torch.Tensor]:
        """
        Update the key and value caches in-place, and return the necessary keys and value states.

        Args:
            key_states (`torch.Tensor`): The new key states to cache.
            value_states (`torch.Tensor`): The new value states to cache.

        Returns:
            tuple[`torch.Tensor`, `torch.Tensor`]: The key and value states.
        """
        # Lazy initialization
        if not self.is_initialized:
            self.lazy_initialization(key_states, value_states)

        # Prefill
        if self.prefill_length is None or self.prefill_cumulative_length < self.prefill_length:
            kv_length = key_states.shape[-2]

            if self.prefill_length is None:
                self.prefill_length = min(self.prefill_max_cache_len, self.prefill_cumulative_length + kv_length)

            # Resize buffer if necessary
            if self.prefill_keys.shape[-2] != self.prefill_length:
                self._allocate_key_value_buffers(self.prefill_length, copy_existing=True)

            prefill_kv_length = min(kv_length, self.prefill_length - self.prefill_cumulative_length)
            prefill_key_states = key_states[..., :prefill_kv_length, :]
            prefill_value_states = value_states[..., :prefill_kv_length, :]

            cache_position = torch.arange(prefill_kv_length, device=self.device) + self.prefill_cumulative_length
            try:
                self.prefill_keys.index_copy_(2, cache_position, prefill_key_states)
                self.prefill_values.index_copy_(2, cache_position, prefill_value_states)
            except NotImplementedError:
                # Fallback for devices like MPS where index_copy_ might not be supported.
                self.prefill_keys[:, :, cache_position] = prefill_key_states
                self.prefill_values[:, :, cache_position] = prefill_value_states

            self.prefill_cumulative_length += prefill_kv_length

            if prefill_kv_length == kv_length:
                return self.all_keys, self.all_values

            # Old speculative methods can draft tokens before prefill is complete
            key_states = key_states[..., prefill_kv_length:, :]
            value_states = value_states[..., prefill_kv_length:, :]

        # Decode
        sliding_keys, sliding_values = super().update(key_states, value_states, *args, **kwargs)

        # The parent returned the original buffers
        if sliding_keys is self.keys:
            return self.all_keys, self.all_values

        # The parent returned concatenated full_key_states/full_value_states
        return (
            torch.cat((self.prefill_keys, sliding_keys), dim=-2),
            torch.cat((self.prefill_values, sliding_values), dim=-2),
        )

    def get_mask_sizes(self, query_length: int) -> tuple[int, int]:
        """Return the length and offset of the cache, used to generate the attention mask"""
        # Prefill
        if self.prefill_length is None or self.prefill_cumulative_length < self.prefill_length:
            kv_offset = 0
            prefill_length = (
                min(self.prefill_max_cache_len, self.prefill_cumulative_length + query_length)
                if self.prefill_length is None
                else self.prefill_length
            )
            kv_length = max(
                prefill_length + self.sliding_window,
                # Query exceeds prefill and maybe also sliding window
                self.prefill_cumulative_length + super().get_seq_length() + query_length,
            )
        # Decode
        else:
            kv_length, kv_offset = super().get_mask_sizes(query_length)
            kv_length += self.prefill_length

        # Returned kv_offset is with respect to sliding window keys.
        # Remove kv_offset from kv_idx to retrieve the prefill indices.
        return kv_length, kv_offset

    def get_max_length(self) -> int:
        """Return the maximum cache shape of the cache"""
        max_cache_len = self.prefill_max_cache_len + self.max_cache_len
        if self.prefill_length is None:
            return max_cache_len
        return min(max_cache_len, self.prefill_length + self.sliding_window)

    def get_seq_length(self) -> int:
        return self.prefill_cumulative_length + super().get_seq_length()

    def set_prefill_length(self, prefill_length: int) -> None:  # trf-ignore: TRF033
        """Declare how many leading tokens are prefill states, before they are cached."""
        if self.prefill_length is None:
            self.prefill_length = min(prefill_length, self.prefill_max_cache_len)

    def reset(self) -> None:
        super().reset()
        if self.is_initialized:
            # This also zeros the self.prefill_keys, self.keys, etc. views
            self.all_keys.zero_()
            self.all_values.zero_()
        self.prefill_length = None
        self.prefill_cumulative_length = 0

    def lazy_initialization(self, key_states: torch.Tensor, value_states: torch.Tensor) -> None:
        self.dtype, self.device = key_states.dtype, key_states.device
        self.batch_size, self.num_heads = key_states.shape[:2]
        self.v_head_dim = value_states.shape[-1]
        self.k_head_dim = key_states.shape[-1]

        self.cumulative_length = self.cumulative_length.to(self.device)
        if not is_torchdynamo_compiling():
            torch._dynamo.mark_static_address(self.cumulative_length)

        prefill_length = key_states.shape[-2] if self.prefill_length is None else self.prefill_length
        self._allocate_key_value_buffers(min(self.prefill_max_cache_len, prefill_length))

        self.is_initialized = True

    def _allocate_key_value_buffers(self, prefill_length: int, copy_existing: bool = False) -> None:
        """(Re)allocate static key value buffers."""
        total_length = prefill_length + self.sliding_window
        new_all_keys = torch.zeros(
            (self.batch_size, self.num_heads, total_length, self.k_head_dim), dtype=self.dtype, device=self.device
        )
        new_all_values = torch.zeros(
            (self.batch_size, self.num_heads, total_length, self.v_head_dim), dtype=self.dtype, device=self.device
        )
        if copy_existing:
            old_length = self.all_keys.shape[-2]
            new_all_keys[:, :, :old_length, :].copy_(self.all_keys)
            new_all_values[:, :, :old_length, :].copy_(self.all_values)
        self.all_keys = new_all_keys
        self.all_values = new_all_values

        # Views into all_keys/all_values
        self.prefill_keys = self.all_keys[:, :, :prefill_length, :]
        self.prefill_values = self.all_values[:, :, :prefill_length, :]
        self.keys = self.all_keys[:, :, prefill_length:, :]
        self.values = self.all_values[:, :, prefill_length:, :]

        if not is_torchdynamo_compiling():
            torch._dynamo.mark_static_address(self.all_keys)
            torch._dynamo.mark_static_address(self.all_values)
            torch._dynamo.mark_static_address(self.prefill_keys)
            torch._dynamo.mark_static_address(self.prefill_values)
            torch._dynamo.mark_static_address(self.keys)
            torch._dynamo.mark_static_address(self.values)


class UnlimitedOcrTextRotaryEmbedding(nn.Module):
    def __init__(self, config: UnlimitedOcrTextConfig):
        super().__init__()
        self.max_seq_len_cached = config.max_position_embeddings
        self.original_max_seq_len = config.max_position_embeddings

        self.config = config

        self.rope_type = self.config.rope_parameters["rope_type"]
        rope_init_fn: Callable = self.compute_default_rope_parameters
        if self.rope_type != "default":
            rope_init_fn = ROPE_INIT_FUNCTIONS[self.rope_type]
        inv_freq, self.attention_scaling = rope_init_fn(self.config)

        self.inv_freq = nn.Buffer(inv_freq, persistent=False)
        self.original_inv_freq = nn.Buffer(inv_freq.clone(), persistent=False)

    @staticmethod
    def compute_default_rope_parameters(config: UnlimitedOcrTextConfig, **kwargs) -> tuple[torch.Tensor, float]:
        """
        Computes the inverse frequencies according to the original RoPE implementation
        Args:
            config ([`~transformers.PreTrainedConfig`]):
                The model configuration.
        Returns:
            Tuple of (`torch.Tensor`, `float`), containing the inverse frequencies for the RoPE embeddings and the
            post-processing scaling factor applied to the computed cos/sin (unused in this type of RoPE).
        """
        base = config.rope_parameters["rope_theta"]
        dim = getattr(config, "head_dim", None) or config.hidden_size // config.num_attention_heads

        attention_factor = 1.0  # Unused in this type of RoPE
        # Compute the inverse frequencies
        inv_freq = 1.0 / (base ** (torch.arange(0, dim, 2, dtype=torch.float) / dim))
        return inv_freq, attention_factor

    @torch.no_grad()
    @dynamic_rope_update  # power user: used with advanced RoPE types (e.g. dynamic rope)
    def forward(self, x, position_ids):
        freqs = position_ids[..., None].float() * self.inv_freq.to(device=x.device, dtype=torch.float)
        emb = torch.cat((freqs, freqs), dim=-1)
        cos = emb.cos() * self.attention_scaling
        sin = emb.sin() * self.attention_scaling

        return cos.to(dtype=x.dtype), sin.to(dtype=x.dtype)


def create_reference_sliding_window_causal_mask(
    config: PreTrainedConfig,
    inputs_embeds: torch.Tensor,
    attention_mask: torch.Tensor | None,
    past_key_values: Cache | None,
    position_ids: torch.Tensor | None = None,
    or_mask_function: Callable | None = None,
    and_mask_function: Callable | None = None,
    block_sequence_ids: torch.Tensor | None = None,
    layer_idx: int | None = None,
    allow_is_causal_skip: bool = False,
) -> torch.Tensor | BlockMask | None:
    layer = None
    if past_key_values is not None:
        layer = next(
            (layer for layer in past_key_values.layers if layer._layer_type == "reference_sliding_attention"), None
        )

    if layer is None:
        prefill_length = float("inf")
        kv_offset = 0
    else:
        # A layer that is still prefilling has no reference states yet, all its states are prefill states
        prefill_length = float("inf") if layer.prefill_length is None else layer.prefill_length
        _, kv_offset = layer.get_mask_sizes(query_length=inputs_embeds.shape[1])

    def prefill_overlay(batch_idx, head_idx, q_idx, kv_idx):
        # Remove kv_offset to retrieve the kv_index with respect to prefill
        return kv_idx - kv_offset < prefill_length

    prefill_mask_function = and_masks(causal_mask_function, prefill_overlay)

    # Workaround in case an or_mask_function is passed as prefill_mask_function
    # is also an or mask.
    if or_mask_function is not None:
        prefill_mask_function = or_masks(prefill_mask_function, or_mask_function)

    return create_sliding_window_causal_mask(
        config=config,
        inputs_embeds=inputs_embeds,
        attention_mask=attention_mask,
        past_key_values=past_key_values,
        position_ids=position_ids,
        or_mask_function=prefill_mask_function,
        and_mask_function=and_mask_function,
        block_sequence_ids=block_sequence_ids,
        layer_idx=layer_idx,
        allow_is_causal_skip=allow_is_causal_skip,
    )


@auto_docstring
class UnlimitedOcrTextModel(UnlimitedOcrTextPreTrainedModel):
    def __init__(self, config: UnlimitedOcrTextConfig):
        super().__init__(config)
        self.padding_idx = config.pad_token_id
        self.vocab_size = config.vocab_size
        self.embed_tokens = nn.Embedding(config.vocab_size, config.hidden_size, self.padding_idx)
        self.layers = nn.ModuleList(
            [UnlimitedOcrTextDecoderLayer(config, layer_idx) for layer_idx in range(config.num_hidden_layers)]
        )
        self.norm = UnlimitedOcrTextRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
        # Use (cos/sin) RoPE instead of complex RoPE to match LlamaAttention (MHA)
        self.rotary_emb = UnlimitedOcrTextRotaryEmbedding(config=config)
        self.gradient_checkpointing = False

        # Initialize weights and apply final processing
        self.post_init()

    @merge_with_config_defaults
    @capture_outputs
    @auto_docstring
    def forward(
        self,
        input_ids: torch.LongTensor | None = None,
        attention_mask: torch.Tensor | None = None,
        position_ids: torch.LongTensor | None = None,
        past_key_values: Cache | None = None,
        inputs_embeds: torch.Tensor | None = None,
        use_cache: bool | None = None,
        **kwargs: Unpack[TransformersKwargs],
    ) -> MoeModelOutputWithPast:
        if (input_ids is None) ^ (inputs_embeds is not None):
            raise ValueError("You must specify exactly one of input_ids or inputs_embeds")

        if inputs_embeds is None:
            inputs_embeds = self.embed_tokens(input_ids)

        if use_cache and past_key_values is None:
            past_key_values = DynamicCache(config=self.config)

        # It may already have been prepared by, e.g., `generate`
        if position_ids is None:
            past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0
            position_ids = torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device) + past_seen_tokens
            position_ids = position_ids.unsqueeze(0)

        if not isinstance(causal_mask_mapping := attention_mask, dict):
            mask_kwargs = {
                "config": self.config,
                "inputs_embeds": inputs_embeds,
                "attention_mask": attention_mask,
                "past_key_values": past_key_values,
                "position_ids": position_ids,
            }

            causal_mask_mapping = {
                "full_attention": create_causal_mask(**mask_kwargs),
                "reference_sliding_attention": create_reference_sliding_window_causal_mask(**mask_kwargs),
            }

        hidden_states = inputs_embeds
        position_embeddings = self.rotary_emb(hidden_states, position_ids=position_ids)

        for layer_idx, decoder_layer in enumerate(self.layers[: self.config.num_hidden_layers]):
            hidden_states = decoder_layer(
                hidden_states,
                attention_mask=causal_mask_mapping[self.config.layer_types[layer_idx]],
                position_embeddings=position_embeddings,
                position_ids=position_ids,
                past_key_values=past_key_values,
                use_cache=use_cache,
                **kwargs,
            )

        hidden_states = self.norm(hidden_states)
        return MoeModelOutputWithPast(
            last_hidden_state=hidden_states,
            past_key_values=past_key_values,
        )


@auto_docstring(
    custom_intro="""
    The UnlimitedOcr model which consists of a vision backbone and a language model.
    """
)
class UnlimitedOcrModel(UnlimitedOcrPreTrainedModel):
    base_model_prefix = "model"
    _keys_to_ignore_on_load_unexpected = {"lm_head"}  # unused and untied weight in original checkpoint

    def __init__(self, config: UnlimitedOcrConfig):
        super().__init__(config)

        self.vision_tower = UnlimitedOcrVisionModel(config.vision_config)
        self.multi_modal_projector = nn.Linear(
            # trf-ignore: TRF030
            config.vision_config.sam_config.downsample_channels[-1] + config.vision_config.encoder_config.hidden_size,
            config.text_config.hidden_size,
        )

        self.vocab_size = config.text_config.vocab_size

        self.language_model = UnlimitedOcrTextModel(config.text_config)

        # Learnable separator between local and global views (initialized in `_init_weights`).
        self.view_separator = nn.Parameter(torch.empty(config.text_config.hidden_size))
        self.image_newline = nn.Parameter(torch.empty(config.text_config.hidden_size))
        self.post_init()

    @can_return_tuple
    @auto_docstring
    def get_image_features(
        self,
        pixel_values: torch.Tensor,
        pixel_values_local: torch.Tensor | None = None,
        num_local_patches: list[int] | None = None,
        patches_grid: list[list[int]] | None = None,
        **kwargs: Unpack[TransformersKwargs],
    ) -> "UnlimitedOcrModelOutputWithPooling":
        r"""
        pixel_values_local (`torch.Tensor` of shape `(total_patches, 3, height, width)`, *optional*):
            All local patches flattened across the batch, or `None` if no local views.
        num_local_patches (`list[int]`, *optional*):
            Number of local patches per image, e.g. `[6, 0, 4]`.
        patches_grid (`list[list[int]]`, *optional*):
            The patches grid `(num_columns, num_rows)` per image. Required if `pixel_values_local` is passed.
        """
        batch_size = pixel_values.shape[0]

        global_vision_outputs = self.vision_tower(pixel_values, **kwargs)
        global_features = self.multi_modal_projector(global_vision_outputs.last_hidden_state)

        local_outputs = {}
        if pixel_values_local is not None:
            local_vision_outputs = self.vision_tower(pixel_values_local, **kwargs)
            all_local_features = self.multi_modal_projector(local_vision_outputs.last_hidden_state)
            per_image_local = torch.split(all_local_features, num_local_patches, dim=0)
            local_outputs = {
                "local_last_hidden_state": local_vision_outputs.last_hidden_state,
                "local_hidden_states": local_vision_outputs.hidden_states,
                "local_attentions": local_vision_outputs.attentions,
            }
        else:
            per_image_local = [None] * batch_size

        hidden_size = global_features.shape[-1]
        newline = self.image_newline[None, None, :].to(global_features.device)
        num_queries_global = int(global_features.shape[1] ** 0.5)

        # Different from DeepseekOcr2: each row of image patches gets a newline token appended.
        all_features = []
        view_separator = self.view_separator.to(global_features.device).unsqueeze(0)
        for idx in range(batch_size):
            global_grid = global_features[idx].reshape(num_queries_global, num_queries_global, hidden_size)
            global_grid = torch.cat([global_grid, newline.expand(num_queries_global, 1, hidden_size)], dim=1)
            global_flat = global_grid.reshape(-1, hidden_size)

            local_features = per_image_local[idx]
            if local_features is not None and local_features.shape[0] > 0:
                num_columns, num_rows = patches_grid[idx]
                num_queries_local = int(local_features.shape[1] ** 0.5)
                local_grid_shape = (num_rows * num_queries_local, -1, hidden_size)
                local_grid = local_features.reshape(
                    num_rows, num_columns, num_queries_local, num_queries_local, hidden_size
                )
                local_grid = local_grid.transpose(1, 2).reshape(local_grid_shape)
                local_grid = torch.cat([local_grid, newline.expand(local_grid_shape)], dim=1)
                local_flat = local_grid.reshape(-1, hidden_size)
                all_features.append(torch.cat([local_flat, global_flat, view_separator], dim=0))
            else:
                all_features.append(torch.cat([global_flat, view_separator], dim=0))

        return UnlimitedOcrModelOutputWithPooling(
            last_hidden_state=global_vision_outputs.last_hidden_state,
            pooler_output=all_features,
            hidden_states=global_vision_outputs.hidden_states,
            attentions=global_vision_outputs.attentions,
            **local_outputs,
        )

    def get_placeholder_mask(
        self, input_ids: torch.LongTensor, inputs_embeds: torch.Tensor, image_features: torch.Tensor
    ):
        """
        Obtains multimodal placeholder mask from `input_ids` or `inputs_embeds`, and checks that the placeholder token count is
        equal to the length of multimodal features. If the lengths are different, an error is raised.
        """
        if input_ids is None:
            special_image_mask = inputs_embeds == self.get_input_embeddings()(
                torch.full((), self.config.image_token_id, dtype=torch.long, device=inputs_embeds.device)
            )
            special_image_mask = special_image_mask.all(-1)
        else:
            special_image_mask = input_ids == self.config.image_token_id

        n_image_tokens = special_image_mask.sum()
        special_image_mask = special_image_mask.unsqueeze(-1).to(inputs_embeds.device)
        torch_compilable_check(
            n_image_tokens * inputs_embeds.shape[-1] == image_features.numel(),
            f"Image features and image tokens do not match, tokens: {n_image_tokens}, features: {image_features.shape[0]}",
        )
        return special_image_mask

    @can_return_tuple
    @auto_docstring
    def forward(
        self,
        input_ids: torch.LongTensor | None = None,
        pixel_values: torch.Tensor | None = None,
        pixel_values_local: torch.Tensor | None = None,
        num_local_patches: list[int] | None = None,
        patches_grid: list[list[int]] | None = None,
        attention_mask: torch.Tensor | None = None,
        position_ids: torch.LongTensor | None = None,
        past_key_values: Cache | None = None,
        inputs_embeds: torch.Tensor | None = None,
        use_cache: bool | None = None,
        mm_encoder_outputs: dict[str, BaseModelOutputWithPooling] | None = None,
        **kwargs: Unpack[TransformersKwargs],
    ) -> tuple | UnlimitedOcrModelOutputWithPast:
        r"""
        pixel_values_local (`torch.Tensor`, *optional*):
            Local patch pixel values of shape `(total_patches, 3, H, W)`.
        num_local_patches (`list[int]`, *optional*):
            Number of local patches per image in the batch.
        patches_grid (`list[list[int]]`, *optional*):
            The patches grid `(num_columns, num_rows)` per image.
        """
        if pixel_values is not None and mm_encoder_outputs is not None:
            raise ValueError("You cannot specify both pixel_values and mm_encoder_outputs at the same time")

        if inputs_embeds is None:
            inputs_embeds = self.get_input_embeddings()(input_ids)

        mm_encoder_outputs = mm_encoder_outputs if mm_encoder_outputs is not None else {}
        if mm_encoder_outputs.get("image") is None and pixel_values is not None:
            mm_encoder_outputs["image"] = self.get_image_features(
                pixel_values, pixel_values_local, num_local_patches, patches_grid, return_dict=True
            )

        if mm_encoder_outputs.get("image") is not None:
            image_features = mm_encoder_outputs["image"].pooler_output
            image_features = torch.cat(image_features, dim=0).to(inputs_embeds.device, inputs_embeds.dtype)
            special_image_mask = self.get_placeholder_mask(input_ids, inputs_embeds, image_features)
            inputs_embeds = inputs_embeds.masked_scatter(special_image_mask, image_features)

        outputs = self.language_model(
            input_ids=None,
            inputs_embeds=inputs_embeds,
            attention_mask=attention_mask,
            position_ids=position_ids,
            past_key_values=past_key_values,
            use_cache=use_cache,
            **kwargs,
        )

        return UnlimitedOcrModelOutputWithPast(
            last_hidden_state=outputs.last_hidden_state,
            past_key_values=outputs.past_key_values,
            hidden_states=outputs.hidden_states,
            attentions=outputs.attentions,
            image_hidden_states=mm_encoder_outputs["image"].pooler_output if mm_encoder_outputs.get("image") else None,
            router_logits=outputs.router_logits,
        )


# Cannot inherit from XYZForConditionalGeneration as this results in MRO clashes between GenerationMixin
# and UnlimitedOcrGenerationMixin. Cannot change the inheritance order as this breaks modular generation.
@auto_docstring
class UnlimitedOcrForConditionalGeneration(UnlimitedOcrPreTrainedModel, UnlimitedOcrGenerationMixin):
    _tied_weights_keys = {"lm_head.weight": "model.language_model.embed_tokens.weight"}
    _tp_plan = {"lm_head": "colwise_gather_output"}

    def __init__(self, config: UnlimitedOcrConfig):
        super().__init__(config)
        self.model = UnlimitedOcrModel(config)
        self.lm_head = nn.Linear(config.text_config.hidden_size, config.text_config.vocab_size, bias=False)
        self.post_init()

    @can_return_tuple
    @auto_docstring
    def get_image_features(
        self,
        pixel_values: torch.Tensor,
        pixel_values_local: torch.Tensor | None = None,
        num_local_patches: list[int] | None = None,
        patches_grid: list[list[int]] | None = None,
        **kwargs: Unpack[TransformersKwargs],
    ) -> tuple | BaseModelOutputWithPooling:
        r"""
        pixel_values (`torch.Tensor` of shape `(batch_size, 3, height, width)`):
            The tensors corresponding to the global view input images.
        pixel_values_local (`torch.Tensor` of shape `(total_patches, 3, height, width)`, *optional*):
            All local patches flattened across the batch, or `None` if no local views.
        num_local_patches (`list[int]`, *optional*):
            Number of local patches per image, e.g. `[6, 0, 4]`.
        patches_grid (`list[list[int]]`, *optional*):
            The patches grid `(num_columns, num_rows)` per image. Required if `pixel_values_local` is passed.
        """
        return self.model.get_image_features(
            pixel_values=pixel_values,
            pixel_values_local=pixel_values_local,
            num_local_patches=num_local_patches,
            patches_grid=patches_grid,
            **kwargs,
        )

    @can_return_tuple
    @auto_docstring
    def forward(
        self,
        input_ids: torch.LongTensor | None = None,
        pixel_values: torch.Tensor | None = None,
        pixel_values_local: torch.Tensor | None = None,
        num_local_patches: list[int] | None = None,
        patches_grid: list[list[int]] | None = None,
        attention_mask: torch.Tensor | None = None,
        position_ids: torch.LongTensor | None = None,
        past_key_values: Cache | None = None,
        inputs_embeds: torch.Tensor | None = None,
        labels: torch.LongTensor | None = None,
        use_cache: bool | None = None,
        logits_to_keep: int | torch.Tensor = 0,
        mm_encoder_outputs: dict[str, BaseModelOutputWithPooling] | None = None,
        **kwargs: Unpack[TransformersKwargs],
    ) -> tuple | UnlimitedOcrCausalLMOutputWithPast:
        r"""
        pixel_values_local (`torch.Tensor`, *optional*):
            Local patch pixel values of shape `(total_patches, 3, H, W)`.
        num_local_patches (`list[int]`, *optional*):
            Number of local patches per image in the batch.
        patches_grid (`list[list[int]]`, *optional*):
            The patches grid `(num_columns, num_rows)` per image.

        Example single-page OCR:

        ```python
        >>> from transformers import AutoProcessor, AutoModelForImageTextToText

        >>> model = AutoModelForImageTextToText.from_pretrained("baidu/Unlimited-OCR", device_map="auto")
        >>> processor = AutoProcessor.from_pretrained("baidu/Unlimited-OCR")

        >>> image = "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/transformers/model_doc/ocr_suggestion_form.jpg"
        >>> messages = [
        ...     {
        ...         "role": "user",
        ...         "content": [{"type": "image", "url": image}, {"type": "text", "text": "document parsing."}],
        ...     }
        ... ]
        >>> inputs = processor.apply_chat_template(
        ...     messages, add_generation_prompt=True, tokenize=True, return_dict=True, return_tensors="pt"
        ... ).to(model.device, model.dtype)

        >>> output = model.generate(**inputs)
        >>> processor.decode(output[0, inputs["input_ids"].shape[1]:], skip_special_tokens=True)
        >>> # image [383, 87, 497, 171]\ntext [333, 201, 558, 230]R&D QUALITY IMPROVEMENT\nSUGGESTION/SOLUTION FORM...
        ```

        Example multi-page OCR:

        ```python
        >>> from transformers import AutoProcessor, AutoModelForImageTextToText

        >>> model = AutoModelForImageTextToText.from_pretrained("baidu/Unlimited-OCR", device_map="auto")
        >>> processor = AutoProcessor.from_pretrained("baidu/Unlimited-OCR")

        >>> page1 = "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/transformers/model_doc/ocr_suggestion_form.jpg"
        >>> page2 = "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/transformers/model_doc/ocr_receipt.jpeg"

        >>> messages = [
        ...     {
        ...         "role": "user",
        ...         "content": [{"type": "image", "url": page} for page in [page1, page2]]
        ...         + [{"type": "text", "text": "Multi page parsing."}],
        ...     }
        ... ]
        >>> inputs = processor.apply_chat_template(
        ...     messages,
        ...     add_generation_prompt=True,
        ...     tokenize=True,
        ...     return_dict=True,
        ...     return_tensors="pt",
        ...     processor_kwargs={"crop_to_patches": False},
        ... ).to(model.device, model.dtype)

        >>> output = model.generate(
        ...     **inputs,
        ...     no_repeat_ngram_window_size=1024,
        ... )
        >>> processor.decode(output[0, inputs["input_ids"].shape[1]:], skip_special_tokens=True)
        >>> # <PAGE>image [382, 87, 489, 174]\ntitle [333, 201, 556, 230]R&D QUALITY IMPROVEMENT\nSUGGESTION/SCLUTION FORM...
        ```"""
        outputs = self.model(
            input_ids=input_ids,
            pixel_values=pixel_values,
            pixel_values_local=pixel_values_local,
            num_local_patches=num_local_patches,
            patches_grid=patches_grid,
            attention_mask=attention_mask,
            position_ids=position_ids,
            past_key_values=past_key_values,
            inputs_embeds=inputs_embeds,
            use_cache=use_cache,
            mm_encoder_outputs=mm_encoder_outputs,
            **kwargs,
        )

        hidden_states = outputs[0]
        hidden_states = slice_logits_to_keep(hidden_states, logits_to_keep)
        logits = self.lm_head(hidden_states)

        loss = None
        if labels is not None:
            loss = self.loss_function(
                logits=logits,
                labels=labels,
                vocab_size=self.config.text_config.vocab_size,
                **kwargs,
            )

        return UnlimitedOcrCausalLMOutputWithPast(
            loss=loss,
            logits=logits,
            past_key_values=outputs.past_key_values,
            hidden_states=outputs.hidden_states,
            attentions=outputs.attentions,
            image_hidden_states=outputs.image_hidden_states,
            router_logits=outputs.router_logits,
        )


__all__ = [
    "UnlimitedOcrForConditionalGeneration",
    "UnlimitedOcrModel",
    "UnlimitedOcrPreTrainedModel",
    "UnlimitedOcrTextModel",
    "UnlimitedOcrTextPreTrainedModel",
    "UnlimitedOcrVisionModel",
]
