#                🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨
#           This file was automatically generated from src/transformers/models/nemotron3_5_transcribe/modular_nemotron3_5_transcribe.py.
#               Do NOT edit this file manually as any edits will be overwritten by the generation of
#             the file from the modular. If any change should be done, please apply the change to the
#                          modular_nemotron3_5_transcribe.py file directly. One of our CI enforces this.
#                🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨
# Copyright 2026 The HuggingFace Inc. team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import math
from collections.abc import Callable
from dataclasses import dataclass

import torch
from torch import nn

from ... import initialization as init
from ...activations import ACT2FN
from ...cache_utils import Cache
from ...generation import GenerationMixin
from ...masking_utils import create_bidirectional_mask, create_causal_mask, create_recurrent_attention_mask
from ...modeling_layers import GradientCheckpointingLayer
from ...modeling_outputs import BaseModelOutput, BaseModelOutputWithPast, BaseModelOutputWithPooling, ModelOutput
from ...modeling_rope_utils import ROPE_INIT_FUNCTIONS, dynamic_rope_update
from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel, slice_logits_to_keep
from ...processing_utils import Unpack
from ...utils import TransformersKwargs, auto_docstring, can_return_tuple, torch_compilable_check
from ...utils.generic import merge_with_config_defaults
from ...utils.output_capturing import capture_outputs
from ..auto import AutoModel, AutoModelForAudioFrameClassification
from .configuration_nemotron3_5_transcribe import (
    Nemotron3_5TranscribeAsrAudioConfig,
    Nemotron3_5TranscribeAudioConfig,
    Nemotron3_5TranscribeConfig,
    Nemotron3_5TranscribeWordTimestampConfig,
)
from .generation_nemotron3_5_transcribe import Nemotron3_5TranscribeGenerateOutput


@auto_docstring
@dataclass
class Nemotron3_5TranscribeModelOutputWithPast(BaseModelOutputWithPast):
    r"""
    audio_hidden_states (`torch.Tensor`, *optional*):
        Projected audio hidden states.
    """

    audio_hidden_states: torch.Tensor | None = None


@auto_docstring(
    custom_intro="""
    Base class for Nemotron3_5Transcribe causal language model (or autoregressive) outputs.
    """
)
@dataclass
class Nemotron3_5TranscribeCausalLMOutputWithPast(ModelOutput):
    r"""
    loss (`torch.Tensor` of shape `(1,)`, *optional*, returned when `labels` is provided):
        Language modeling loss (for next-token prediction).
    logits (`torch.Tensor` of shape `(batch_size, sequence_length, config.vocab_size)`):
        Prediction scores of the language modeling head.
    past_key_values (`Cache`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
        It is a [`~cache_utils.Cache`] instance.
    audio_hidden_states (`torch.Tensor`, *optional*):
        Hidden states of the audio encoder after projection.
    """

    loss: torch.Tensor | None = None
    logits: torch.Tensor | None = None
    past_key_values: Cache | None = None
    hidden_states: tuple[torch.Tensor] | None = None
    attentions: tuple[torch.Tensor] | None = None
    audio_hidden_states: torch.Tensor | None = None


@auto_docstring
@dataclass
class Nemotron3_5TranscribeAudioModelOutput(BaseModelOutputWithPooling):
    r"""
    pooler_output (`torch.Tensor` of shape `(num_audio_tokens, text_config.hidden_size)`, *optional*):
        Projected audio embeddings of the valid encoder frames of every recording, in batch order: the embeddings that
        replace the audio tokens of the prompt. Returned by `get_audio_features`.
    asr_last_hidden_state (`torch.Tensor` of shape `(batch_size, num_frames, audio_config.hidden_size)`):
        Output of the speech recognition encoder, before its normalization and the speaker embeddings: the input of
        [`~Nemotron3_5TranscribeForConditionalGeneration.get_word_timestamp_logits`].
    speaker_logits (`torch.Tensor` of shape `(batch_size, feature_sequence_length, audio_config.num_speakers)`):
        Logits of the diarization model, one frame per spectrogram frame.
    """

    asr_last_hidden_state: torch.Tensor | None = None
    speaker_logits: torch.Tensor | None = None


@auto_docstring
class Nemotron3_5TranscribePreTrainedModel(PreTrainedModel):
    config: Nemotron3_5TranscribeConfig
    base_model_prefix = "model"
    input_modalities = ("audio", "text")
    supports_gradient_checkpointing = True
    _no_split_modules = None
    _skip_keys_device_placement = ["past_key_values"]
    _supports_flash_attn = True
    _supports_sdpa = True
    _supports_attention_backend = True
    _is_stateful = True
    _keys_to_ignore_on_load_unexpected = [
        r"^llm\.mtp\.",
        r"^perception\.preprocessor\.",
        r"^perception\.encoder\.diarization_model\.preprocessor\.",
        r"^perception\.encoder\.diarization_model\.sortformer_modules\.hidden_to_spks\.",
        r"\._extra_state$",
        r"^ctc_timestamp\.decoder\.residual_scale$",
        r"^ctc_timestamp\.decoder\.upsample_factor$",
        r"^ctc_timestamp\.tokenizer\.model_proto$",
    ]

    @torch.no_grad()
    def _init_weights(self, module):
        super()._init_weights(module)
        if isinstance(module, Nemotron3_5TranscribeSpeakerEmbedding):
            init.copy_(module.weight, module.compute_embeddings(module.config))


class Nemotron3_5TranscribeFeatureStacking(nn.Module):
    """Stacks `subsampling_factor` consecutive spectrogram frames and projects them to the encoder hidden size."""

    def __init__(self, config: Nemotron3_5TranscribeAsrAudioConfig):
        super().__init__()
        self.subsampling_factor = config.subsampling_factor
        self.projection = nn.Linear(config.subsampling_factor * config.num_mel_bins, config.hidden_size, bias=False)

    def forward(self, input_features: torch.Tensor) -> torch.Tensor:
        batch_size, num_frames, num_mel_bins = input_features.shape
        # The original zero-pads the last incomplete group of frames.
        padding = -num_frames % self.subsampling_factor
        input_features = nn.functional.pad(input_features, (0, 0, 0, padding))
        stacked = input_features.reshape(
            batch_size, (num_frames + padding) // self.subsampling_factor, num_mel_bins * self.subsampling_factor
        )
        return self.projection(stacked)


class Nemotron3_5TranscribeRotaryEmbedding(nn.Module):
    def __init__(self, config: Nemotron3_5TranscribeAsrAudioConfig):
        super().__init__()
        self.max_seq_len_cached = config.max_position_embeddings
        self.original_max_seq_len = config.max_position_embeddings

        self.config = config

        self.rope_type = self.config.rope_parameters["rope_type"]
        rope_init_fn: Callable = self.compute_default_rope_parameters
        if self.rope_type != "default":
            rope_init_fn = ROPE_INIT_FUNCTIONS[self.rope_type]
        inv_freq, self.attention_scaling = rope_init_fn(self.config)

        self.inv_freq = nn.Buffer(inv_freq, persistent=False)
        self.original_inv_freq = nn.Buffer(inv_freq.clone(), persistent=False)

    @staticmethod
    def compute_default_rope_parameters(
        config: Nemotron3_5TranscribeConfig, device=None, **kwargs
    ) -> tuple[torch.Tensor, float]:
        """
        Computes the inverse frequencies according to the original RoPE implementation
        Args:
            config ([`~transformers.PreTrainedConfig`]):
                The model configuration.
        Returns:
            Tuple of (`torch.Tensor`, `float`), containing the inverse frequencies for the RoPE embeddings and the
            post-processing scaling factor applied to the computed cos/sin (unused in this type of RoPE).
        """
        base = config.rope_parameters["rope_theta"]
        partial_rotary_factor = config.rope_parameters.get("partial_rotary_factor", 1.0)
        head_dim = getattr(config, "head_dim", None) or config.hidden_size // config.num_attention_heads
        dim = int(head_dim * partial_rotary_factor)

        attention_factor = 1.0  # Unused in this type of RoPE
        # Compute the inverse frequencies
        inv_freq = 1.0 / (base ** (torch.arange(0, dim, 2, dtype=torch.float) / dim))
        return inv_freq.to(device), attention_factor

    @torch.no_grad()
    @dynamic_rope_update  # power user: used with advanced RoPE types (e.g. dynamic rope)
    def forward(self, x, position_ids):
        freqs = position_ids[..., None].float() * self.inv_freq.to(device=x.device, dtype=torch.float)
        emb = torch.cat((freqs, freqs), dim=-1)
        cos = emb.cos() * self.attention_scaling
        sin = emb.sin() * self.attention_scaling

        return cos.to(dtype=x.dtype), sin.to(dtype=x.dtype)


def rotate_half(x):
    """Rotates half the hidden dims of the input."""
    x1 = x[..., : x.shape[-1] // 2]
    x2 = x[..., x.shape[-1] // 2 :]
    return torch.cat((-x2, x1), dim=-1)


def repeat_kv(hidden_states: torch.Tensor, n_rep: int) -> torch.Tensor:
    """
    This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,
    num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)
    """
    batch, num_key_value_heads, slen, head_dim = hidden_states.shape
    if n_rep == 1:
        return hidden_states
    hidden_states = hidden_states[:, :, None, :, :].expand(batch, num_key_value_heads, n_rep, slen, head_dim)
    return hidden_states.reshape(batch, num_key_value_heads * n_rep, slen, head_dim)


def eager_attention_forward(
    module: nn.Module,
    query: torch.Tensor,
    key: torch.Tensor,
    value: torch.Tensor,
    attention_mask: torch.Tensor | None,
    scaling: float,
    dropout: float = 0.0,
    **kwargs: Unpack[TransformersKwargs],
):
    key_states = repeat_kv(key, module.num_key_value_groups)
    value_states = repeat_kv(value, module.num_key_value_groups)

    attn_weights = torch.matmul(query, key_states.transpose(2, 3)) * scaling
    if attention_mask is not None:
        attn_weights = attn_weights + attention_mask

    attn_weights = nn.functional.softmax(attn_weights, dim=-1, dtype=torch.float32).to(query.dtype)
    attn_weights = nn.functional.dropout(attn_weights, p=dropout, training=module.training)
    attn_output = torch.matmul(attn_weights, value_states)
    attn_output = attn_output.transpose(1, 2).contiguous()

    return attn_output, attn_weights


def apply_rotary_pos_emb(q, k, cos, sin, position_ids=None, unsqueeze_dim=1):
    cos = cos.unsqueeze(unsqueeze_dim)
    sin = sin.unsqueeze(unsqueeze_dim)

    rotary_dim = cos.shape[-1]
    q_rot, q_pass = q[..., :rotary_dim], q[..., rotary_dim:]
    k_rot, k_pass = k[..., :rotary_dim], k[..., rotary_dim:]

    # Apply rotary embeddings on the first half or full tensor
    q_embed = (q_rot * cos) + (rotate_half(q_rot) * sin)
    k_embed = (k_rot * cos) + (rotate_half(k_rot) * sin)

    # Concatenate back to full shape
    q_embed = torch.cat([q_embed, q_pass], dim=-1)
    k_embed = torch.cat([k_embed, k_pass], dim=-1)
    return q_embed, k_embed


class Nemotron3_5TranscribeAttention(nn.Module):
    """Multi-headed attention from 'Attention Is All You Need' paper"""

    def __init__(self, config: Nemotron3_5TranscribeAsrAudioConfig, layer_idx: int):
        super().__init__()
        self.config = config
        self.layer_idx = layer_idx
        self.head_dim = getattr(config, "head_dim", config.hidden_size // config.num_attention_heads)
        self.num_key_value_groups = config.num_attention_heads // config.num_key_value_heads
        self.scaling = self.head_dim**-0.5
        self.attention_dropout = config.attention_dropout
        self.is_causal = False
        # The original fused query/key/value projection has no bias while the output projection has one.
        self.q_proj = nn.Linear(config.hidden_size, config.num_attention_heads * self.head_dim, bias=False)
        self.k_proj = nn.Linear(config.hidden_size, config.num_key_value_heads * self.head_dim, bias=False)
        self.v_proj = nn.Linear(config.hidden_size, config.num_attention_heads * self.head_dim, bias=False)
        self.o_proj = nn.Linear(config.num_attention_heads * self.head_dim, config.hidden_size, bias=True)
        self.q_norm = nn.LayerNorm(self.head_dim)
        self.k_norm = nn.LayerNorm(self.head_dim)

    def forward(
        self,
        hidden_states: torch.Tensor,
        position_embeddings: tuple[torch.Tensor, torch.Tensor] | None = None,
        attention_mask: torch.Tensor | None = None,
        **kwargs: Unpack[TransformersKwargs],
    ) -> tuple[torch.Tensor, torch.Tensor]:
        input_shape = hidden_states.shape[:-1]
        hidden_shape = (*input_shape, -1, self.head_dim)

        query_states = self.q_norm(self.q_proj(hidden_states).view(hidden_shape)).transpose(1, 2)
        key_states = self.k_norm(self.k_proj(hidden_states).view(hidden_shape)).transpose(1, 2)
        value_states = self.v_proj(hidden_states).view(hidden_shape).transpose(1, 2)

        cos, sin = position_embeddings
        query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin)

        attention_interface: Callable = ALL_ATTENTION_FUNCTIONS.get_interface(
            self.config._attn_implementation, eager_attention_forward
        )

        attn_output, attn_weights = attention_interface(
            self,
            query_states,
            key_states,
            value_states,
            attention_mask=attention_mask,
            dropout=0.0 if not self.training else self.attention_dropout,
            scaling=self.scaling,
            **kwargs,
        )

        attn_output = attn_output.reshape(*input_shape, -1).contiguous()
        attn_output = self.o_proj(attn_output)
        return attn_output, attn_weights


class Nemotron3_5TranscribeMLP(nn.Module):
    def __init__(self, config):
        super().__init__()
        self.config = config
        self.activation_fn = ACT2FN[config.hidden_act]
        self.fc1 = nn.Linear(config.hidden_size, config.intermediate_size)
        self.fc2 = nn.Linear(config.intermediate_size, config.hidden_size)

    def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
        hidden_states = self.fc1(hidden_states)
        hidden_states = self.activation_fn(hidden_states)
        hidden_states = self.fc2(hidden_states)
        return hidden_states


class Nemotron3_5TranscribeAsrAudioLayer(GradientCheckpointingLayer):
    def __init__(self, config: Nemotron3_5TranscribeAsrAudioConfig, layer_idx: int):
        super().__init__()
        self.embed_dim = config.hidden_size
        self.self_attn = Nemotron3_5TranscribeAttention(config, layer_idx)
        self.layer_norm1 = nn.LayerNorm(config.hidden_size)
        self.mlp = Nemotron3_5TranscribeMLP(config)
        self.layer_norm2 = nn.LayerNorm(config.hidden_size)

    def forward(
        self,
        hidden_states: torch.Tensor,
        attention_mask: torch.Tensor,
        **kwargs: Unpack[TransformersKwargs],
    ) -> torch.Tensor:
        residual = hidden_states

        hidden_states = self.layer_norm1(hidden_states)
        hidden_states, _ = self.self_attn(
            hidden_states=hidden_states,
            attention_mask=attention_mask,
            **kwargs,
        )
        hidden_states = residual + hidden_states

        residual = hidden_states
        hidden_states = self.layer_norm2(hidden_states)
        hidden_states = self.mlp(hidden_states)
        hidden_states = residual + hidden_states

        return hidden_states


@auto_docstring(
    custom_intro="""
    The Nemotron 3.5 Transcribe speech recognition encoder: the Nemotron 3 Diarization audio encoder with the queries
    and keys of each attention head normalized, and rotary position embeddings on part of each head.
    """
)
class Nemotron3_5TranscribeAsrAudioModel(Nemotron3_5TranscribePreTrainedModel):
    config: Nemotron3_5TranscribeAsrAudioConfig
    main_input_name = "input_features"
    input_modalities = "audio"
    _no_split_modules = ["Nemotron3_5TranscribeAsrAudioLayer"]
    _can_record_outputs = {
        "hidden_states": Nemotron3_5TranscribeAsrAudioLayer,
        "attentions": Nemotron3_5TranscribeAttention,
    }

    def __init__(self, config: Nemotron3_5TranscribeAsrAudioConfig):
        super().__init__(config)
        self.embedder = Nemotron3_5TranscribeFeatureStacking(config)
        self.input_layer_norm = nn.LayerNorm(config.hidden_size)
        self.layers = nn.ModuleList(
            [Nemotron3_5TranscribeAsrAudioLayer(config, layer_idx) for layer_idx in range(config.num_hidden_layers)]
        )
        self.layer_norm = nn.LayerNorm(config.hidden_size)
        self.rotary_emb = Nemotron3_5TranscribeRotaryEmbedding(config)
        self.post_init()

    @merge_with_config_defaults
    @capture_outputs
    @auto_docstring
    def forward(
        self,
        input_features: torch.Tensor | None = None,
        attention_mask: torch.Tensor | None = None,
        inputs_embeds: torch.Tensor | None = None,
        position_ids: torch.Tensor | None = None,
        **kwargs: Unpack[TransformersKwargs],
    ) -> BaseModelOutput:
        if (input_features is None) == (inputs_embeds is None):
            raise ValueError("Provide exactly one of `input_features` and `inputs_embeds`.")

        if inputs_embeds is None:
            inputs_embeds = self.embedder(input_features)
            # if inputs_embeds is provided, we expect attention_mask already downsampled
            if attention_mask is not None:
                attention_mask = attention_mask[:, :: self.config.subsampling_factor].bool()

        if position_ids is None:
            position_ids = torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device)[None, :]
        position_embeddings = self.rotary_emb(inputs_embeds, position_ids)

        attention_mask = create_bidirectional_mask(
            config=self.config, inputs_embeds=inputs_embeds, attention_mask=attention_mask
        )
        hidden_states = self.input_layer_norm(inputs_embeds)
        for encoder_layer in self.layers:
            hidden_states = encoder_layer(
                hidden_states,
                attention_mask=attention_mask,
                position_embeddings=position_embeddings,
                **kwargs,
            )
        return BaseModelOutput(last_hidden_state=self.layer_norm(hidden_states))


@auto_docstring(
    custom_intro="""
    The transformer of the Nemotron 3.5 Transcribe word timestamp head.
    [`Nemotron3_5TranscribeForConditionalGeneration.word_timestamp_head`] classifies its output.
    """
)
class Nemotron3_5TranscribeWordTimestampModel(Nemotron3_5TranscribePreTrainedModel):
    config: Nemotron3_5TranscribeWordTimestampConfig
    main_input_name = "inputs_embeds"
    input_modalities = "audio"
    _no_split_modules = ["Nemotron3_5TranscribeAsrAudioLayer"]
    _can_record_outputs = {
        "hidden_states": Nemotron3_5TranscribeAsrAudioLayer,
        "attentions": Nemotron3_5TranscribeAttention,
    }

    def __init__(self, config: Nemotron3_5TranscribeWordTimestampConfig):
        super().__init__(config)
        self.upsampler = nn.ConvTranspose1d(
            config.hidden_size,
            config.hidden_size,
            kernel_size=config.upsampling_factor,
            stride=config.upsampling_factor,
        )
        self.input_layer_norm = nn.LayerNorm(config.hidden_size)
        self.layers = nn.ModuleList(
            [Nemotron3_5TranscribeAsrAudioLayer(config, layer_idx) for layer_idx in range(config.num_hidden_layers)]
        )
        self.layer_norm = nn.LayerNorm(config.hidden_size)
        self.rotary_emb = Nemotron3_5TranscribeRotaryEmbedding(config)
        self.post_init()

    @merge_with_config_defaults
    @capture_outputs
    @auto_docstring
    def forward(
        self,
        inputs_embeds: torch.Tensor,
        attention_mask: torch.Tensor | None = None,
        **kwargs: Unpack[TransformersKwargs],
    ) -> BaseModelOutput:
        r"""
        inputs_embeds (`torch.Tensor` of shape `(batch_size, num_frames, hidden_size)`):
            Speech recognition encoder states, before the audio model's `asr_norm` and speaker embeddings.
        attention_mask (`torch.Tensor` of shape `(batch_size, num_frames)`, *optional*):
            Mask of the valid encoder frames.

        Returns `num_frames * upsampling_factor` frames.
        """
        inputs_embeds = self.upsampler(inputs_embeds.transpose(1, 2)).transpose(1, 2)
        if attention_mask is not None:
            attention_mask = attention_mask.repeat_interleave(self.config.upsampling_factor, dim=1)
        position_ids = torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device)[None, :]
        position_embeddings = self.rotary_emb(inputs_embeds, position_ids)
        attention_mask = create_bidirectional_mask(
            config=self.config, inputs_embeds=inputs_embeds, attention_mask=attention_mask
        )
        hidden_states = self.input_layer_norm(inputs_embeds)
        for layer in self.layers:
            hidden_states = layer(
                hidden_states, attention_mask=attention_mask, position_embeddings=position_embeddings, **kwargs
            )
        # the original casts its residual scale to the dtype of the states before the product
        residual_scale = inputs_embeds.new_tensor(self.config.residual_scale)
        return BaseModelOutput(last_hidden_state=inputs_embeds + residual_scale * self.layer_norm(hidden_states))


class Nemotron3_5TranscribeSpeakerEmbedding(nn.Module):
    """Fixed sinusoidal embedding of each speaker slot, which the speaker activities select."""

    def __init__(self, config: Nemotron3_5TranscribeAudioConfig):
        super().__init__()
        self.config = config
        self.weight = nn.Buffer(self.compute_embeddings(config), persistent=False)

    @staticmethod
    def compute_embeddings(config: Nemotron3_5TranscribeAudioConfig) -> torch.Tensor:
        position = torch.arange(config.num_speakers, dtype=torch.float32)[:, None]
        inv_freq = torch.exp(
            torch.arange(0, config.hidden_size, 2, dtype=torch.float32) * -(math.log(10000.0) / config.hidden_size)
        )
        # float32 whatever the default dtype of the loading context: the buffer is a fixed table, not a weight
        embeddings = torch.zeros(config.num_speakers, config.hidden_size, dtype=torch.float32)
        embeddings[:, 0::2] = torch.sin(position * inv_freq)
        embeddings[:, 1::2] = torch.cos(position * inv_freq)
        return embeddings

    def forward(self, speaker_activity: torch.Tensor) -> torch.Tensor:
        return speaker_activity @ self.weight.to(speaker_activity.dtype)


def normalize_per_feature(input_features: torch.Tensor, input_features_mask: torch.Tensor) -> torch.Tensor:
    """Normalizes each mel bin to zero mean and unit variance over the valid frames of each sample."""
    mask = input_features_mask.bool()[..., None]
    input_features = input_features.float().masked_fill(~mask, 0.0)
    num_frames = mask.sum(1)
    mean = input_features.sum(1, keepdim=True) / num_frames[:, None].clamp(min=1)
    centered = (input_features - mean).masked_fill(~mask, 0.0)
    # the original uses the unbiased variance (0 for a single frame) and adds its epsilon to the standard deviation
    std = (centered.pow(2).sum(1, keepdim=True) / (num_frames[:, None] - 1).clamp(min=1)).sqrt() + 1e-5
    return centered / std


@auto_docstring(
    custom_intro="""
    The Nemotron 3.5 Transcribe audio model: a speech recognition encoder and a speaker diarization model run in
    parallel on the same spectrogram, and the speaker activities are added to the speech recognition encoder outputs.
    """
)
class Nemotron3_5TranscribeAudioModel(Nemotron3_5TranscribePreTrainedModel):
    config: Nemotron3_5TranscribeAudioConfig
    main_input_name = "input_features"
    input_modalities = "audio"

    def __init__(self, config: Nemotron3_5TranscribeAudioConfig):
        super().__init__(config)
        self.asr_encoder = Nemotron3_5TranscribeAsrAudioModel(config.asr_config)
        self.diarization_model = AutoModelForAudioFrameClassification.from_config(config.diarization_config)
        self.asr_norm = nn.LayerNorm(config.hidden_size)
        self.speaker_norm = nn.LayerNorm(config.num_speakers)
        self.speaker_embedding = Nemotron3_5TranscribeSpeakerEmbedding(config)
        self.post_init()

    @can_return_tuple
    @auto_docstring
    def forward(
        self,
        input_features: torch.Tensor,
        input_features_mask: torch.Tensor | None = None,
        **kwargs: Unpack[TransformersKwargs],
    ) -> Nemotron3_5TranscribeAudioModelOutput:
        r"""
        input_features (`torch.Tensor` of shape `(batch_size, feature_sequence_length, num_mel_bins)`):
            Log mel spectrogram, padded to a multiple of `config.subsampling_factor` frames. As in the original, audio
            longer than `config.diarization_config.chunk_length` encoder frames is encoded in chunks with context.
        input_features_mask (`torch.Tensor` of shape `(batch_size, feature_sequence_length)`, *optional*):
            Mask of the valid spectrogram frames. Padding must be on the right.
        """
        subsampling_factor = self.config.subsampling_factor
        if input_features.shape[1] % subsampling_factor != 0:
            raise ValueError(
                f"The number of spectrogram frames ({input_features.shape[1]}) must be a multiple of "
                f"`subsampling_factor` ({subsampling_factor}): pad `input_features` and `input_features_mask` on the "
                "right, as the processor does."
            )
        if input_features_mask is None:
            input_features_mask = torch.ones(input_features.shape[:2], dtype=torch.long, device=input_features.device)

        num_frames = input_features.shape[1] // subsampling_factor
        asr_input = normalize_per_feature(input_features, input_features_mask).to(self.asr_encoder.dtype)
        hidden_states = asr_input.new_empty(asr_input.shape[0], num_frames, self.config.asr_config.hidden_size)
        for context, chunk in self._get_chunks(num_frames):
            spectrogram_context = slice(context.start * subsampling_factor, context.stop * subsampling_factor)
            chunk_states = self.asr_encoder(
                asr_input[:, spectrogram_context], input_features_mask[:, spectrogram_context]
            )
            hidden_states[:, context.start + chunk.start : context.start + chunk.stop] = (
                chunk_states.last_hidden_state[:, chunk]
            )

        speaker_logits = self.diarization_model(
            input_features.to(self.diarization_model.dtype), attention_mask=input_features_mask
        ).logits
        speaker_probabilities = speaker_logits.sigmoid().unflatten(1, (num_frames, -1)).mean(2)
        speaker_probs = speaker_probabilities.to(hidden_states.dtype)
        speaker_activity = (speaker_probs > self.config.speaker_activity_threshold).to(hidden_states.dtype)
        speaker_states = self.speaker_embedding(self.speaker_norm(speaker_activity))
        last_hidden_state = self.asr_norm(hidden_states) + speaker_states

        return Nemotron3_5TranscribeAudioModelOutput(
            last_hidden_state=last_hidden_state,
            asr_last_hidden_state=hidden_states,
            speaker_logits=speaker_logits,
        )

    def _get_feat_extract_output_lengths(self, input_lengths: torch.LongTensor):
        return input_lengths, (input_lengths + self.config.subsampling_factor - 1) // self.config.subsampling_factor

    def _get_chunks(self, num_frames: int) -> list[tuple[slice, slice]]:
        """
        The chunks of the original's online inference, which the speech recognition encoder and the word timestamp
        encoder run on (NeMo `ParallelExpertEncoder._forward_online`, `TimestampAligner._iter_ctc_windows`):
        `chunk_length` encoder frames each, seeing `chunk_left_context` and `chunk_right_context` more frames, as the
        diarization model's chunks. Per chunk, `context`, the encoder frames of the sequence the encoder runs on (the
        chunk with its context), and `chunk`, the chunk's frames in the output of that run.
        """
        config = self.config.diarization_config
        chunks = []
        for start in range(0, num_frames, config.chunk_length):
            end = min(start + config.chunk_length, num_frames)
            context = slice(
                max(start - config.chunk_left_context, 0), min(end + config.chunk_right_context, num_frames)
            )
            chunks.append((context, slice(start - context.start, end - context.start)))
        return chunks


@auto_docstring(
    custom_intro="""
    The Nemotron 3.5 Transcribe model (speech recognition encoder and speaker diarization model in parallel, linear
    projector and Nemotron-H language model), without a language modeling head.
    """
)
class Nemotron3_5TranscribeModel(Nemotron3_5TranscribePreTrainedModel):
    _tp_plan = None
    _pp_plan = None
    _keep_in_fp32_modules_strict = ["audio_tower", "multi_modal_projector"]

    def __init__(self, config: Nemotron3_5TranscribeConfig):
        super().__init__(config)
        self.audio_tower = AutoModel.from_config(config.audio_config)
        self.language_model = AutoModel.from_config(config.text_config)
        self.multi_modal_projector = nn.Linear(config.audio_config.hidden_size, config.text_config.hidden_size)
        self.word_timestamp_encoder = Nemotron3_5TranscribeWordTimestampModel(config.word_timestamp_config)
        self.post_init()

    @can_return_tuple
    @auto_docstring(
        custom_intro="This method is used to get the audio embeddings from input features (a log mel spectrogram), meaning inferring the audio encoder and the multi-modal projector."
    )
    def get_audio_features(
        self,
        input_features: torch.Tensor,
        input_features_mask: torch.Tensor,
        **kwargs: Unpack[TransformersKwargs],
    ) -> tuple | BaseModelOutputWithPooling:
        r"""
        input_features (`torch.Tensor`):
            Float values of mel features extracted from the raw speech waveform.
        input_features_mask (`torch.Tensor` of shape `(batch_size, feature_sequence_length)`):
            Mask to avoid performing attention on padded feature indices.
        """

        audio_output = self.audio_tower(
            input_features, input_features_mask=input_features_mask, return_dict=True, **kwargs
        )
        audio_embeds = self.multi_modal_projector(audio_output.last_hidden_state)

        # Mask according to the audio tower output lengths, accounting for both conv downsampling and final avg pooling
        input_lengths = input_features_mask.sum(-1).to(torch.long)
        _, post_lengths = self.audio_tower._get_feat_extract_output_lengths(input_lengths)
        valid_mask = torch.arange(audio_embeds.shape[1], device=post_lengths.device)[None, :] < post_lengths[:, None]
        audio_output.pooler_output = audio_embeds[valid_mask.to(audio_embeds.device)]

        return audio_output

    def get_placeholder_mask(
        self, input_ids: torch.LongTensor, inputs_embeds: torch.Tensor, audio_features: torch.Tensor
    ):
        """
        Obtains multimodal placeholder mask from `input_ids` or `inputs_embeds`, and checks that the placeholder token count is
        equal to the length of multimodal features. If the lengths are different, an error is raised.
        """
        if input_ids is None:
            special_audio_mask = inputs_embeds == self.get_input_embeddings()(
                torch.full((), self.config.audio_token_id, dtype=torch.long, device=inputs_embeds.device)
            )
            special_audio_mask = special_audio_mask.all(-1)
        else:
            special_audio_mask = input_ids == self.config.audio_token_id

        n_audio_tokens = special_audio_mask.sum()
        n_audio_features = audio_features.shape[0]
        special_audio_mask = special_audio_mask.unsqueeze(-1).expand_as(inputs_embeds).to(inputs_embeds.device)
        torch_compilable_check(
            inputs_embeds[special_audio_mask].numel() == audio_features.numel(),
            f"Audio features and audio tokens do not match, tokens: {n_audio_tokens}, features: {n_audio_features}",
        )
        return special_audio_mask

    @can_return_tuple
    @auto_docstring
    def forward(
        self,
        input_ids: torch.LongTensor | None = None,
        input_features: torch.Tensor | None = None,
        input_features_mask: torch.Tensor | None = None,
        attention_mask: torch.Tensor | None = None,
        position_ids: torch.LongTensor | None = None,
        past_key_values: Cache | None = None,
        inputs_embeds: torch.Tensor | None = None,
        use_cache: bool | None = None,
        **kwargs: Unpack[TransformersKwargs],
    ) -> tuple | Nemotron3_5TranscribeModelOutputWithPast:
        r"""
        input_features_mask (`torch.Tensor` of shape `(batch_size, feature_sequence_length)`):
            Mask to avoid performing attention on padding feature indices.
        """
        if inputs_embeds is None and input_ids is not None:
            inputs_embeds = self._embed_input_ids(input_ids)
        if inputs_embeds is None:
            inputs_embeds = self.get_input_embeddings()(input_ids)

        audio_embeds = None
        if input_features is not None and input_ids is not None:
            audio_embeds = self.get_audio_features(input_features, input_features_mask, return_dict=True).pooler_output

            # replace text-audio token placeholders with audio embeddings
            special_audio_mask = self.get_placeholder_mask(
                input_ids, inputs_embeds=inputs_embeds, audio_features=audio_embeds
            )
            inputs_embeds = inputs_embeds.masked_scatter(
                special_audio_mask, audio_embeds.to(inputs_embeds.device, inputs_embeds.dtype)
            )

        outputs = self.language_model(
            inputs_embeds=inputs_embeds,
            attention_mask=attention_mask,
            position_ids=position_ids,
            past_key_values=past_key_values,
            use_cache=use_cache,
            **kwargs,
        )

        return Nemotron3_5TranscribeModelOutputWithPast(
            last_hidden_state=outputs.last_hidden_state,
            past_key_values=outputs.past_key_values,
            hidden_states=outputs.hidden_states,
            attentions=outputs.attentions,
            audio_hidden_states=audio_embeds,
        )

    def _embed_input_ids(self, input_ids: torch.LongTensor) -> torch.Tensor:
        # the audio token can be past the language model's vocabulary (NVIDIA's tokenizer appends it): it is looked up
        # as 0, its positions take the audio embeddings
        if self.config.audio_token_id >= self.config.text_config.vocab_size:
            input_ids = input_ids.masked_fill(input_ids == self.config.audio_token_id, 0)
        return self.get_input_embeddings()(input_ids)

    @auto_docstring(
        custom_intro="""
        Runs the word timestamp encoder on the speech recognition encoder states, in the chunks of the encoder: the
        states the word timestamp head classifies.
        """
    )
    def get_word_timestamp_hidden_states(
        self, asr_last_hidden_state: torch.Tensor, input_features_mask: torch.Tensor
    ) -> torch.Tensor:
        r"""
        asr_last_hidden_state (`torch.Tensor` of shape `(batch_size, num_frames, audio_config.hidden_size)`):
            Output of the speech recognition encoder before its normalization and the speaker embeddings, as returned
            by [`~Nemotron3_5TranscribeModel.get_audio_features`] for `input_features_mask`.
        input_features_mask (`torch.Tensor` of shape `(batch_size, feature_sequence_length)`):
            Mask of the valid spectrogram frames. Padding must be on the right.

        Returns:
            `torch.Tensor` of shape `(batch_size, num_frames * word_timestamp_config.upsampling_factor,
            audio_config.hidden_size)`: `upsampling_factor` frames per encoder frame, in the dtype of the word timestamp
            encoder.
        """
        states = asr_last_hidden_state.to(self.word_timestamp_encoder.dtype)
        frame_mask = input_features_mask[:, :: self.config.audio_config.subsampling_factor].bool()
        factor = self.config.word_timestamp_config.upsampling_factor
        hidden_states = states.new_empty(
            states.shape[0], states.shape[1] * factor, self.config.word_timestamp_config.hidden_size
        )
        for context, chunk in self.audio_tower._get_chunks(states.shape[1]):
            chunk_states = self.word_timestamp_encoder(states[:, context], frame_mask[:, context])
            hidden_states[:, (context.start + chunk.start) * factor : (context.start + chunk.stop) * factor] = (
                chunk_states.last_hidden_state[:, chunk.start * factor : chunk.stop * factor]
            )
        return hidden_states


@auto_docstring(
    custom_intro="""
    The Nemotron 3.5 Transcribe model (speech recognition encoder and speaker diarization model in parallel, linear
    projector and Nemotron-H language model) with a language modeling head.
    """
)
class Nemotron3_5TranscribeForConditionalGeneration(Nemotron3_5TranscribePreTrainedModel, GenerationMixin):
    _tied_weights_keys = None

    def __init__(self, config: Nemotron3_5TranscribeConfig):
        super().__init__(config)
        self.model = Nemotron3_5TranscribeModel(config)
        self.lm_head = nn.Linear(config.text_config.hidden_size, config.text_config.vocab_size, bias=False)
        self.word_timestamp_head = nn.Conv1d(
            config.word_timestamp_config.hidden_size, config.word_timestamp_config.vocab_size, kernel_size=1
        )
        self.post_init()

    @auto_docstring
    def get_audio_features(self, input_features, input_features_mask, **kwargs):
        r"""
        input_features_mask (`torch.Tensor` of shape `(batch_size, feature_sequence_length)`):
            Mask of the valid spectrogram frames. Padding must be on the right.
        """
        return self.model.get_audio_features(input_features, input_features_mask, **kwargs)

    @can_return_tuple
    @auto_docstring
    def forward(
        self,
        input_ids: torch.LongTensor | None = None,
        input_features: torch.Tensor | None = None,
        input_features_mask: torch.Tensor | None = None,
        attention_mask: torch.Tensor | None = None,
        position_ids: torch.LongTensor | None = None,
        past_key_values: Cache | None = None,
        inputs_embeds: torch.Tensor | None = None,
        labels: torch.LongTensor | None = None,
        use_cache: bool | None = None,
        logits_to_keep: int | torch.Tensor = 0,
        **kwargs: Unpack[TransformersKwargs],
    ) -> tuple | Nemotron3_5TranscribeCausalLMOutputWithPast:
        r"""
        input_features_mask (`torch.Tensor` of shape `(batch_size, feature_sequence_length)`):
            Mask to avoid performing attention on padding feature indices.
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the masked language modeling loss.

        Example:

        ```python
        >>> from transformers import Nemotron3_5TranscribeForConditionalGeneration, AutoProcessor

        >>> model_id = "nvidia/audio-flamingo-3-hf"
        >>> processor = AutoProcessor.from_pretrained(model_id)
        >>> model = Nemotron3_5TranscribeForConditionalGeneration.from_pretrained(model_id, device_map="auto")
        ```"""
        outputs = self.model(
            input_ids=input_ids,
            input_features=input_features,
            input_features_mask=input_features_mask,
            attention_mask=attention_mask,
            position_ids=position_ids,
            past_key_values=past_key_values,
            inputs_embeds=inputs_embeds,
            use_cache=use_cache,
            **kwargs,
        )

        hidden_states = outputs.last_hidden_state
        hidden_states = slice_logits_to_keep(hidden_states, logits_to_keep)
        logits = self.lm_head(hidden_states)

        loss = None
        if labels is not None:
            loss = self.loss_function(
                logits=logits, labels=labels, vocab_size=self.config.text_config.vocab_size, **kwargs
            )

        return Nemotron3_5TranscribeCausalLMOutputWithPast(
            loss=loss,
            logits=logits,
            past_key_values=outputs.past_key_values,
            hidden_states=outputs.hidden_states,
            attentions=outputs.attentions,
            audio_hidden_states=outputs.audio_hidden_states,
        )

    @auto_docstring(
        custom_intro="""
        Logits of the word timestamp head on the speech recognition encoder states, which
        [`~Nemotron3_5TranscribeProcessor.align_word_timestamps`] turns into word times.
        """
    )
    def get_word_timestamp_logits(
        self, asr_last_hidden_state: torch.Tensor, input_features_mask: torch.Tensor
    ) -> torch.Tensor:
        r"""
        asr_last_hidden_state (`torch.Tensor` of shape `(batch_size, num_frames, audio_config.hidden_size)`):
            Output of the speech recognition encoder before its normalization and the speaker embeddings, as returned
            by [`~Nemotron3_5TranscribeForConditionalGeneration.get_audio_features`] for `input_features_mask`.
        input_features_mask (`torch.Tensor` of shape `(batch_size, feature_sequence_length)`):
            Mask of the valid spectrogram frames. Padding must be on the right.

        Returns:
            `torch.Tensor` of shape `(batch_size, num_frames * word_timestamp_config.upsampling_factor,
            word_timestamp_config.vocab_size)`: raw logits over the pieces of the alignment tokenizer and the blank,
            `upsampling_factor` frames per encoder frame (the processor applies the log-softmax).
        """
        hidden_states = self.model.get_word_timestamp_hidden_states(asr_last_hidden_state, input_features_mask)
        return self.word_timestamp_head(hidden_states.transpose(1, 2)).transpose(1, 2)

    @torch.no_grad()
    def generate(self, input_ids=None, input_features=None, input_features_mask=None, **kwargs):
        r"""
        return_word_timestamp_logits (`bool`, *optional*, defaults to `False`):
            Whether to also return the logits of the word timestamp head and of the diarization model, for
            [`~Nemotron3_5TranscribeProcessor.align_word_timestamps`], in a [`Nemotron3_5TranscribeGenerateOutput`]. The
            audio model runs once. Greedy or sampled decoding with the cache, one sequence per recording.
        """
        if not kwargs.pop("return_word_timestamp_logits", False):
            return super().generate(
                input_ids=input_ids,
                input_features=input_features,
                input_features_mask=input_features_mask,
                **kwargs,
            )
        generation_config = kwargs.get("generation_config", self.generation_config)
        if kwargs.get("use_cache") is False or kwargs.get(
            "num_return_sequences", generation_config.num_return_sequences
        ) not in (None, 1):
            raise ValueError("`return_word_timestamp_logits=True` needs the cache and one sequence per recording.")

        # the audio model runs once: its embeddings fill the prompt, its states feed the word timestamp head
        audio_outputs = self.get_audio_features(input_features, input_features_mask)
        inputs_embeds = self.model._embed_input_ids(input_ids)
        audio_embeds = audio_outputs.pooler_output.to(inputs_embeds.device, inputs_embeds.dtype)
        special_audio_mask = self.model.get_placeholder_mask(input_ids, inputs_embeds, audio_embeds)
        inputs_embeds = inputs_embeds.masked_scatter(special_audio_mask, audio_embeds)

        outputs = super().generate(
            input_ids=input_ids, inputs_embeds=inputs_embeds, return_dict_in_generate=True, **kwargs
        )
        return Nemotron3_5TranscribeGenerateOutput(
            **outputs,
            word_timestamp_logits=self.get_word_timestamp_logits(
                audio_outputs.asr_last_hidden_state, input_features_mask
            ),
            speaker_logits=audio_outputs.speaker_logits,
        )

    @staticmethod
    def create_masks_for_generate(config, inputs_embeds, attention_mask, past_key_values, position_ids=None, **_):
        # TODO: @eustlb, remove once `masking_utils.create_masks_for_generate` skips the layers without attention
        # (Nemotron-H `moe` / `mlp`), which also removes `NemotronHForCausalLM.create_masks_for_generate`
        # as `NemotronHForCausalLM`: the default dispatch does not know the Nemotron-H `moe` layer type, so both masks
        # that the language model needs are returned
        mask_kwargs = {
            "config": config.get_text_config(),
            "inputs_embeds": inputs_embeds,
            "attention_mask": attention_mask,
            "past_key_values": past_key_values,
            "position_ids": position_ids,
        }
        return {
            "full_attention": create_causal_mask(**mask_kwargs),
            "linear_attention": create_recurrent_attention_mask(**mask_kwargs),
        }


__all__ = [
    "Nemotron3_5TranscribeAsrAudioModel",
    "Nemotron3_5TranscribeAudioModel",
    "Nemotron3_5TranscribeForConditionalGeneration",
    "Nemotron3_5TranscribeModel",
    "Nemotron3_5TranscribePreTrainedModel",
    "Nemotron3_5TranscribeWordTimestampModel",
]
