content: # coding=utf-8
  # Copyright 2018 Mesh TensorFlow authors, T5 Authors and HuggingFace Inc. team.
  #
  # Licensed under the Apache License, Version 2.0 (the "License");
  # you may not use this file except in compliance with the License.
  # You may obtain a copy of the License at
  #
  #     http://www.apache.org/licenses/LICENSE-2.0
  #
  # Unless required by applicable law or agreed to in writing, software
  # distributed under the License is distributed on an "AS IS" BASIS,
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  # See the License for the specific language governing permissions and
  # limitations under the License.
  """ PyTorch T5 model."""
  
  
  import copy
  import math
  import os
  import warnings
  from typing import Optional, Tuple, Union
  
  import torch
  from torch import nn
  from torch.nn import CrossEntropyLoss
  from torch.utils.checkpoint import checkpoint
  
  from transformers.activations import ACT2FN
  from transformers.modeling_outputs import (
      BaseModelOutput,
      BaseModelOutputWithPastAndCrossAttentions,
      Seq2SeqLMOutput,
      Seq2SeqModelOutput,
  )
  from transformers.modeling_utils import PreTrainedModel
  from transformers.pytorch_utils import (
      ALL_LAYERNORM_LAYERS,
      find_pruneable_heads_and_indices,
      prune_linear_layer,
  )
  from transformers.utils import (
      DUMMY_INPUTS,
      DUMMY_MASK,
      add_start_docstrings,
      add_start_docstrings_to_model_forward,
      is_torch_fx_proxy,
      logging,
      replace_return_docstrings,
  )
  from transformers.utils.model_parallel_utils import assert_device_map, get_device_map
  from transformers.models.t5.configuration_t5 import T5Config
  
  
  logger = logging.get_logger(__name__)
  
  _CONFIG_FOR_DOC = "T5Config"
  _TOKENIZER_FOR_DOC = "T5Tokenizer"
  _CHECKPOINT_FOR_DOC = "t5-small"
  
  ####################################################
  # This dict contains ids and associated url
  # for the pretrained weights provided with the models
  ####################################################
  T5_PRETRAINED_MODEL_ARCHIVE_LIST = [
      "t5-small",
      "t5-base",
      "t5-large",
      "t5-3b",
      "t5-11b",
      # See all T5 models at https://huggingface.co/models?filter=t5
  ]
  
  
  ####################################################
  # This is a conversion method from TF 1.0 to PyTorch
  # More details: https://medium.com/huggingface/from-tensorflow-to-pytorch-265f40ef2a28
  ####################################################
  def load_tf_weights_in_t5(model, config, tf_checkpoint_path):
      """Load tf checkpoints in a pytorch model."""
      try:
          import re
  
          import numpy as np
          import tensorflow as tf
      except ImportError:
          logger.error(
              "Loading a TensorFlow model in PyTorch, requires TensorFlow to be installed. Please see "
              "https://www.tensorflow.org/install/ for installation instructions."
          )
          raise
      tf_path = os.path.abspath(tf_checkpoint_path)
      logger.info(f"Converting TensorFlow checkpoint from {tf_path}")
      # Load weights from TF model
      init_vars = tf.train.list_variables(tf_path)
      names = []
      tf_weights = {}
      for name, shape in init_vars:
          logger.info(f"Loading TF weight {name} with shape {shape}")
          array = tf.train.load_variable(tf_path, name)
          names.append(name)
          tf_weights[name] = array
  
      for txt_name in names:
          name = txt_name.split("/")
          # adam_v and adam_m are variables used in AdamWeightDecayOptimizer to calculated m and v
          # which are not required for using pretrained model
          if any(
              n
              in [
                  "adam_v",
                  "adam_m",
                  "AdamWeightDecayOptimizer",
                  "AdamWeightDecayOptimizer_1",
                  "global_step",
              ]
              for n in name
          ):
              logger.info(f"Skipping {'/'.join(name)}")
              tf_weights.pop(txt_name, None)
              continue
          if "_slot_" in name[-1]:
              logger.info(f"Skipping {'/'.join(name)}")
              tf_weights.pop(txt_name, None)
              continue
          pointer = model
          array = tf_weights[txt_name]
  
          for m_name in name:
              if re.fullmatch(r"[A-Za-z]+_\d+", m_name):
                  scope_names = re.split(r"_(\d+)", m_name)
              else:
                  scope_names = [m_name]
              if scope_names[0] in ["kernel", "scale", "embedding"]:
                  pointer = getattr(pointer, "weight")
              elif scope_names[0] == "self_attention":
                  pointer = getattr(pointer, "layer")
                  pointer = pointer[0]
              elif scope_names[0] == "enc_dec_attention":
                  pointer = getattr(pointer, "layer")
                  pointer = pointer[1]
              elif scope_names[0] == "dense_relu_dense":
                  pointer = getattr(pointer, "layer")
                  pointer = pointer[2]
              elif scope_names[0] == "rms_norm":
                  if hasattr(pointer, "layer_norm"):
                      pointer = getattr(pointer, "layer_norm")
                  elif hasattr(pointer, "final_layer_norm"):
                      pointer = getattr(pointer, "final_layer_norm")
              elif scope_names[0] == "scale":
                  pointer = getattr(pointer, "weight")
              elif scope_names[0] == "output_bias" or scope_names[0] == "beta":
                  pointer = getattr(pointer, "bias")
              elif scope_names[0] == "squad":
                  pointer = getattr(pointer, "classifier")
              elif scope_names[0] == "decoder" and name[1] == "logits":
                  continue
              elif scope_names[0] == "logits":
                  pointer = getattr(pointer, "lm_head")
              elif (
                  scope_names[0] == "wi"
                  and len(scope_names) > 1
                  and scope_names[1].isdigit()
              ):
                  pointer = getattr(pointer, f"wi_{scope_names[1]}")
                  continue
              else:
                  try:
                      pointer = getattr(pointer, scope_names[0])
                  except AttributeError:
                      logger.info(f"Skipping {'/'.join(name)}")
                      continue
              if len(scope_names) >= 2:
                  num = int(scope_names[1])
                  pointer = pointer[num]
          if scope_names[0] not in ["kernel", "scale", "embedding"]:
              pointer = getattr(pointer, "weight")
          if scope_names[0] != "embedding":
              logger.info(f"Transposing numpy weight of shape {array.shape} for {name}")
              array = np.transpose(array)
          try:
              assert (
                  pointer.shape == array.shape
              ), f"Pointer shape {pointer.shape} and array shape {array.shape} mismatched"
          except AssertionError as e:
              e.args += (pointer.shape, array.shape)
              raise
          logger.info(f"Initialize PyTorch weight {name}")
          pointer.data = torch.from_numpy(array.astype(np.float32))
          tf_weights.pop(txt_name, None)
  
      logger.info(f"Weights not copied to PyTorch model: {', '.join(tf_weights.keys())}.")
      return model
  
  
  ####################################################
  # PyTorch Models are constructed by sub-classing
  # - torch.nn.Module for the layers and
  # - PreTrainedModel for the models (it-self a sub-class of nn.Module)
  ####################################################
  PARALLELIZE_DOCSTRING = r"""
      This is an experimental feature and is a subject to change at a moment's notice.
  
      Uses a device map to distribute attention modules of the model across several devices. If no device map is given,
      it will evenly distribute blocks across all devices.
  
      Args:
          device_map (`Dict[int, list]`, optional, defaults to None):
              A dictionary that maps attention modules to devices. Note that the embedding module and LMHead are always
              automatically mapped to the first device (for esoteric reasons). That means that the first device should
              have fewer attention modules mapped to it than other devices. For reference, the t5 models have the
              following number of attention modules:
  
                  - t5-small: 6
                  - t5-base: 12
                  - t5-large: 24
                  - t5-3b: 24
                  - t5-11b: 24
  
      Example:
  
      ```python
      # Here is an example of a device map on a machine with 4 GPUs using t5-3b, which has a total of 24 attention modules:
      model = T5ForConditionalGeneration.from_pretrained("t5-3b")
      device_map = {
          0: [0, 1, 2],
          1: [3, 4, 5, 6, 7, 8, 9],
          2: [10, 11, 12, 13, 14, 15, 16],
          3: [17, 18, 19, 20, 21, 22, 23],
      }
      model.parallelize(device_map)
      ```
  """
  DEPARALLELIZE_DOCSTRING = r"""
      Moves the model to cpu from a model parallel state.
  
      Example:
  
      ```python
      # On a 4 GPU machine with t5-3b:
      model = T5ForConditionalGeneration.from_pretrained("t5-3b")
      device_map = {
          0: [0, 1, 2],
          1: [3, 4, 5, 6, 7, 8, 9],
          2: [10, 11, 12, 13, 14, 15, 16],
          3: [17, 18, 19, 20, 21, 22, 23],
      }
      model.parallelize(device_map)  # Splits the model across several devices
      model.deparallelize()  # Put the model back on cpu and cleans memory by calling torch.cuda.empty_cache()
      ```
  """
  
  
  class T5LayerNorm(nn.Module):
      def __init__(self, hidden_size, eps=1e-6):
          """
          Construct a layernorm module in the T5 style. No bias and no subtraction of mean.
          """
          super().__init__()
          self.weight = nn.Parameter(torch.ones(hidden_size))
          self.variance_epsilon = eps
  
      def forward(self, hidden_states):
  
          # T5 uses a layer_norm which only scales and doesn't shift, which is also known as Root Mean
          # Square Layer Normalization https://arxiv.org/abs/1910.07467 thus varience is calculated
          # w/o mean and there is no bias. Additionally we want to make sure that the accumulation for
          # half-precision inputs is done in fp32
  
          variance = hidden_states.to(torch.float32).pow(2).mean(-1, keepdim=True)
          hidden_states = hidden_states * torch.rsqrt(variance + self.variance_epsilon)
  
          # convert into half-precision if necessary
          if self.weight.dtype in [torch.float16, torch.bfloat16]:
              hidden_states = hidden_states.to(self.weight.dtype)
  
          return self.weight * hidden_states
  
  
  try:
      from apex.normalization import FusedRMSNorm
  
      T5LayerNorm = FusedRMSNorm  # noqa
  
      logger.info(
          "Discovered apex.normalization.FusedRMSNorm - will use it instead of T5LayerNorm"
      )
  except ImportError:
      # using the normal T5LayerNorm
      pass
  except Exception:
      logger.warning("discovered apex but it failed to load, falling back to T5LayerNorm")
      pass
  
  ALL_LAYERNORM_LAYERS.append(T5LayerNorm)
  
  
  class T5DenseActDense(nn.Module):
      def __init__(self, config: T5Config):
          super().__init__()
          self.wi = nn.Linear(config.d_model, config.d_ff, bias=False)
          self.wo = nn.Linear(config.d_ff, config.d_model, bias=False)
          self.dropout = nn.Dropout(config.dropout_rate)
          self.act = ACT2FN[config.dense_act_fn]
  
      def forward(self, hidden_states):
          hidden_states = self.wi(hidden_states)
          hidden_states = self.act(hidden_states)
          hidden_states = self.dropout(hidden_states)
          hidden_states = self.wo(hidden_states)
          return hidden_states
  
  
  class T5DenseGatedActDense(nn.Module):
      def __init__(self, config: T5Config):
          super().__init__()
          self.wi_0 = nn.Linear(config.d_model, config.d_ff, bias=False)
          self.wi_1 = nn.Linear(config.d_model, config.d_ff, bias=False)
          self.wo = nn.Linear(config.d_ff, config.d_model, bias=False)
          self.dropout = nn.Dropout(config.dropout_rate)
          self.act = ACT2FN[config.dense_act_fn]
  
      def forward(self, hidden_states):
          hidden_gelu = self.act(self.wi_0(hidden_states))
          hidden_linear = self.wi_1(hidden_states)
          hidden_states = hidden_gelu * hidden_linear
          hidden_states = self.dropout(hidden_states)
          hidden_states = self.wo(hidden_states)
          return hidden_states
  
  
  class T5LayerFF(nn.Module):
      def __init__(self, config: T5Config):
          super().__init__()
          if config.is_gated_act:
              self.DenseReluDense = T5DenseGatedActDense(config)
          else:
              self.DenseReluDense = T5DenseActDense(config)
  
          self.layer_norm = T5LayerNorm(config.d_model, eps=config.layer_norm_epsilon)
          self.dropout = nn.Dropout(config.dropout_rate)
  
      def forward(self, hidden_states):
          forwarded_states = self.layer_norm(hidden_states)
          forwarded_states = self.DenseReluDense(forwarded_states)
          hidden_states = hidden_states + self.dropout(forwarded_states)
          return hidden_states
  
  
  class T5Attention(nn.Module):
      def __init__(self, config: T5Config, has_relative_attention_bias=False):
          super().__init__()
          self.is_decoder = config.is_decoder
          self.has_relative_attention_bias = has_relative_attention_bias
          self.relative_attention_num_buckets = config.relative_attention_num_buckets
          self.relative_attention_max_distance = config.relative_attention_max_distance
          self.d_model = config.d_model
          self.key_value_proj_dim = config.d_kv
          self.n_heads = config.num_heads
          self.dropout = config.dropout_rate
          self.inner_dim = self.n_heads * self.key_value_proj_dim
  
          # Mesh TensorFlow initialization to avoid scaling before softmax
          self.q = nn.Linear(self.d_model, self.inner_dim, bias=False)
          self.k = nn.Linear(self.d_model, self.inner_dim, bias=False)
          self.v = nn.Linear(self.d_model, self.inner_dim, bias=False)
          self.o = nn.Linear(self.inner_dim, self.d_model, bias=False)
  
          if self.has_relative_attention_bias:
              self.relative_attention_bias = nn.Embedding(
                  self.relative_attention_num_buckets, self.n_heads
              )
          self.pruned_heads = set()
          self.gradient_checkpointing = False
  
      def prune_heads(self, heads):
          if len(heads) == 0:
              return
          heads, index = find_pruneable_heads_and_indices(
              heads, self.n_heads, self.key_value_proj_dim, self.pruned_heads
          )
          # Prune linear layers
          self.q = prune_linear_layer(self.q, index)
          self.k = prune_linear_layer(self.k, index)
          self.v = prune_linear_layer(self.v, index)
          self.o = prune_linear_layer(self.o, index, dim=1)
          # Update hyper params
          self.n_heads = self.n_heads - len(heads)
          self.inner_dim = self.key_value_proj_dim * self.n_heads
          self.pruned_heads = self.pruned_heads.union(heads)
  
      @staticmethod
      def _relative_position_bucket(
          relative_position, bidirectional=True, num_buckets=32, max_distance=128
      ):
          """
          Adapted from Mesh Tensorflow:
          https://github.com/tensorflow/mesh/blob/0cb87fe07da627bf0b7e60475d59f95ed6b5be3d/mesh_tensorflow/transformer/transformer_layers.py#L593
  
          Translate relative position to a bucket number for relative attention. The relative position is defined as
          memory_position - query_position, i.e. the distance in tokens from the attending position to the attended-to
          position. If bidirectional=False, then positive relative positions are invalid. We use smaller buckets for
          small absolute relative_position and larger buckets for larger absolute relative_positions. All relative
          positions >=max_distance map to the same bucket. All relative positions <=-max_distance map to the same bucket.
          This should allow for more graceful generalization to longer sequences than the model has been trained on
  
          Args:
              relative_position: an int32 Tensor
              bidirectional: a boolean - whether the attention is bidirectional
              num_buckets: an integer
              max_distance: an integer
  
          Returns:
              a Tensor with the same shape as relative_position, containing int32 values in the range [0, num_buckets)
          """
          relative_buckets = 0
          if bidirectional:
              num_buckets //= 2
              relative_buckets += (relative_position > 0).to(torch.long) * num_buckets
              relative_position = torch.abs(relative_position)
          else:
              relative_position = -torch.min(
                  relative_position, torch.zeros_like(relative_position)
              )
          # now relative_position is in the range [0, inf)
  
          # half of the buckets are for exact increments in positions
          max_exact = num_buckets // 2
          is_small = relative_position < max_exact
  
          # The other half of the buckets are for logarithmically bigger bins in positions up to max_distance
          relative_position_if_large = max_exact + (
              torch.log(relative_position.float() / max_exact)
              / math.log(max_distance / max_exact)
              * (num_buckets - max_exact)
          ).to(torch.long)
          relative_position_if_large = torch.min(
              relative_position_if_large,
              torch.full_like(relative_position_if_large, num_buckets - 1),
          )
  
          relative_buckets += torch.where(
              is_small, relative_position, relative_position_if_large
          )
          return relative_buckets
  
      def compute_bias(self, query_length, key_length, device=None):
          """Compute binned relative position bias"""
          if device is None:
              device = self.relative_attention_bias.weight.device
          context_position = torch.arange(query_length, dtype=torch.long, device=device)[
              :, None
          ]
          memory_position = torch.arange(key_length, dtype=torch.long, device=device)[
              None, :
          ]
          relative_position = (
              memory_position - context_position
          )  # shape (query_length, key_length)
          relative_position_bucket = self._relative_position_bucket(
              relative_position,  # shape (query_length, key_length)
              bidirectional=(not self.is_decoder),
              num_buckets=self.relative_attention_num_buckets,
              max_distance=self.relative_attention_max_distance,
          )
          values = self.relative_attention_bias(
              relative_position_bucket
          )  # shape (query_length, key_length, num_heads)
          values = values.permute([2, 0, 1]).unsqueeze(
              0
          )  # shape (1, num_heads, query_length, key_length)
          return values
  
      def forward(
          self,
          hidden_states,
          mask=None,
          key_value_states=None,
          position_bias=None,
          past_key_value=None,
          layer_head_mask=None,
          query_length=None,
          use_cache=False,
          output_attentions=False,
      ):
          """
          Self-attention (if key_value_states is None) or attention over source sentence (provided by key_value_states).
          """
          # Input is (batch_size, seq_length, dim)
          # Mask is (batch_size, key_length) (non-causal) or (batch_size, key_length, key_length)
          # past_key_value[0] is (batch_size, n_heads, q_len - 1, dim_per_head)
          batch_size, seq_length = hidden_states.shape[:2]
  
          real_seq_length = seq_length
  
          if past_key_value is not None:
              assert (
                  len(past_key_value) == 2
              ), f"past_key_value should have 2 past states: keys and values. Got { len(past_key_value)} past states"
              real_seq_length += (
                  past_key_value[0].shape[2] if query_length is None else query_length
              )
  
          key_length = (
              real_seq_length if key_value_states is None else key_value_states.shape[1]
          )
  
          def shape(states):
              """projection"""
              return states.view(
                  batch_size, -1, self.n_heads, self.key_value_proj_dim
              ).transpose(1, 2)
  
          def unshape(states):
              """reshape"""
              return (
                  states.transpose(1, 2).contiguous().view(batch_size, -1, self.inner_dim)
              )
  
          def project(hidden_states, proj_layer, key_value_states, past_key_value):
              """projects hidden states correctly to key/query states"""
              if key_value_states is None:
                  # self-attn
                  # (batch_size, n_heads, seq_length, dim_per_head)
                  hidden_states = shape(proj_layer(hidden_states))
              elif past_key_value is None:
                  # cross-attn
                  # (batch_size, n_heads, seq_length, dim_per_head)
                  hidden_states = shape(proj_layer(key_value_states))
  
              if past_key_value is not None:
                  if key_value_states is None:
                      # self-attn
                      # (batch_size, n_heads, key_length, dim_per_head)
                      hidden_states = torch.cat([past_key_value, hidden_states], dim=2)
                  else:
                      # cross-attn
                      hidden_states = past_key_value
              return hidden_states
  
          # get query states
          query_states = shape(
              self.q(hidden_states)
          )  # (batch_size, n_heads, seq_length, dim_per_head)
  
          # get key/value states
          key_states = project(
              hidden_states,
              self.k,
              key_value_states,
              past_key_value[0] if past_key_value is not None else None,
          )
          value_states = project(
              hidden_states,
              self.v,
              key_value_states,
              past_key_value[1] if past_key_value is not None else None,
          )
  
          # compute scores
          scores = torch.matmul(
              query_states, key_states.transpose(3, 2)
          )  # equivalent of torch.einsum("bnqd,bnkd->bnqk", query_states, key_states), compatible with onnx op>9
  
          if position_bias is None:
              if not self.has_relative_attention_bias:
                  position_bias = torch.zeros(
                      (1, self.n_heads, real_seq_length, key_length),
                      device=scores.device,
                      dtype=scores.dtype,
                  )
                  if self.gradient_checkpointing and self.training:
                      position_bias.requires_grad = True
              else:
                  position_bias = self.compute_bias(
                      real_seq_length, key_length, device=scores.device
                  )
  
              # if key and values are already calculated
              # we want only the last query position bias
              if past_key_value is not None:
                  position_bias = position_bias[:, :, -hidden_states.size(1) :, :]
  
              if mask is not None:
                  position_bias = (
                      position_bias + mask
                  )  # (batch_size, n_heads, seq_length, key_length)
  
          if self.pruned_heads:
              mask = torch.ones(position_bias.shape[1])
              mask[list(self.pruned_heads)] = 0
              position_bias_masked = position_bias[:, mask.bool()]
          else:
              position_bias_masked = position_bias
  
          scores += position_bias_masked
          attn_weights = nn.functional.softmax(scores.float(), dim=-1).type_as(
              scores
          )  # (batch_size, n_heads, seq_length, key_length)
          attn_weights = nn.functional.dropout(
              attn_weights, p=self.dropout, training=self.training
          )  # (batch_size, n_heads, seq_length, key_length)
  
          # Mask heads if we want to
          if layer_head_mask is not None:
              attn_weights = attn_weights * layer_head_mask
  
          attn_output = unshape(
              torch.matmul(attn_weights, value_states)
          )  # (batch_size, seq_length, dim)
          attn_output = self.o(attn_output)
  
          present_key_value_state = (
              (key_states, value_states) if (self.is_decoder and use_cache) else None
          )
          outputs = (attn_output,) + (present_key_value_state,) + (position_bias,)
  
          if output_attentions:
              outputs = outputs + (attn_weights,)
          return outputs
  
  
  class T5LayerSelfAttention(nn.Module):
      def __init__(self, config, has_relative_attention_bias=False):
          super().__init__()
          self.SelfAttention = T5Attention(
              config, has_relative_attention_bias=has_relative_attention_bias
          )
          self.layer_norm = T5LayerNorm(config.d_model, eps=config.layer_norm_epsilon)
          self.dropout = nn.Dropout(config.dropout_rate)
  
      def forward(
          self,
          hidden_states,
          attention_mask=None,
          position_bias=None,
          layer_head_mask=None,
          past_key_value=None,
          use_cache=False,
          output_attentions=False,
      ):
          normed_hidden_states = self.layer_norm(hidden_states)
          attention_output = self.SelfAttention(
              normed_hidden_states,
              mask=attention_mask,
              position_bias=position_bias,
              layer_head_mask=layer_head_mask,
              past_key_value=past_key_value,
              use_cache=use_cache,
              output_attentions=output_attentions,
          )
          hidden_states = hidden_states + self.dropout(attention_output[0])
          outputs = (hidden_states,) + attention_output[
              1:
          ]  # add attentions if we output them
          return outputs
  
  
  class T5LayerCrossAttention(nn.Module):
      def __init__(self, config):
          super().__init__()
          self.EncDecAttention = T5Attention(config, has_relative_attention_bias=False)
          self.layer_norm = T5LayerNorm(config.d_model, eps=config.layer_norm_epsilon)
          self.dropout = nn.Dropout(config.dropout_rate)
  
      def forward(
          self,
          hidden_states,
          key_value_states,
          attention_mask=None,
          position_bias=None,
          layer_head_mask=None,
          past_key_value=None,
          use_cache=False,
          query_length=None,
          output_attentions=False,
      ):
          normed_hidden_states = self.layer_norm(hidden_states)
          attention_output = self.EncDecAttention(
              normed_hidden_states,
              mask=attention_mask,
              key_value_states=key_value_states,
              position_bias=position_bias,
              layer_head_mask=layer_head_mask,
              past_key_value=past_key_value,
              use_cache=use_cache,
              query_length=query_length,
              output_attentions=output_attentions,
          )
          layer_output = hidden_states + self.dropout(attention_output[0])
          outputs = (layer_output,) + attention_output[
              1:
          ]  # add attentions if we output them
          return outputs
  
  
  class T5Block(nn.Module):
      def __init__(self, config, has_relative_attention_bias=False):
          super().__init__()
          self.is_decoder = config.is_decoder
          self.layer = nn.ModuleList()
          self.layer.append(
              T5LayerSelfAttention(
                  config, has_relative_attention_bias=has_relative_attention_bias
              )
          )
          if self.is_decoder:
              self.layer.append(T5LayerCrossAttention(config))
  
          self.layer.append(T5LayerFF(config))
  
      def forward(
          self,
          hidden_states,
          attention_mask=None,
          position_bias=None,
          encoder_hidden_states=None,
          encoder_attention_mask=None,
          encoder_decoder_position_bias=None,
          layer_head_mask=None,
          cross_attn_layer_head_mask=None,
          past_key_value=None,
          use_cache=False,
          output_attentions=False,
          return_dict=True,
      ):
  
          if past_key_value is not None:
              if not self.is_decoder:
                  logger.warning(
                      "`past_key_values` is passed to the encoder. Please make sure this is intended."
                  )
              expected_num_past_key_values = 2 if encoder_hidden_states is None else 4
  
              if len(past_key_value) != expected_num_past_key_values:
                  raise ValueError(
                      f"There should be {expected_num_past_key_values} past states. "
                      f"{'2 (past / key) for cross attention. ' if expected_num_past_key_values == 4 else ''}"
                      f"Got {len(past_key_value)} past key / value states"
                  )
  
              self_attn_past_key_value = past_key_value[:2]
              cross_attn_past_key_value = past_key_value[2:]
          else:
              self_attn_past_key_value, cross_attn_past_key_value = None, None
  
          self_attention_outputs = self.layer[0](
              hidden_states,
              attention_mask=attention_mask,
              position_bias=position_bias,
              layer_head_mask=layer_head_mask,
              past_key_value=self_attn_past_key_value,
              use_cache=use_cache,
              output_attentions=output_attentions,
          )
          hidden_states, present_key_value_state = self_attention_outputs[:2]
          attention_outputs = self_attention_outputs[
              2:
          ]  # Keep self-attention outputs and relative position weights
  
          # clamp inf values to enable fp16 training
          if hidden_states.dtype == torch.float16 and torch.isinf(hidden_states).any():
              clamp_value = torch.finfo(hidden_states.dtype).max - 1000
              hidden_states = torch.clamp(
                  hidden_states, min=-clamp_value, max=clamp_value
              )
  
          do_cross_attention = self.is_decoder and encoder_hidden_states is not None
          if do_cross_attention:
              # the actual query length is unknown for cross attention
              # if using past key value states. Need to inject it here
              if present_key_value_state is not None:
                  query_length = present_key_value_state[0].shape[2]
              else:
                  query_length = None
  
              cross_attention_outputs = self.layer[1](
                  hidden_states,
                  key_value_states=encoder_hidden_states,
                  attention_mask=encoder_attention_mask,
                  position_bias=encoder_decoder_position_bias,
                  layer_head_mask=cross_attn_layer_head_mask,
                  past_key_value=cross_attn_past_key_value,
                  query_length=query_length,
                  use_cache=use_cache,
                  output_attentions=output_attentions,
              )
              hidden_states = cross_attention_outputs[0]
  
              # clamp inf values to enable fp16 training
              if (
                  hidden_states.dtype == torch.float16
                  and torch.isinf(hidden_states).any()
              ):
                  clamp_value = torch.finfo(hidden_states.dtype).max - 1000
                  hidden_states = torch.clamp(
                      hidden_states, min=-clamp_value, max=clamp_value
                  )
  
              # Combine self attn and cross attn key value states
              if present_key_value_state is not None:
                  present_key_value_state = (
                      present_key_value_state + cross_attention_outputs[1]
                  )
  
              # Keep cross-attention outputs and relative position weights
              attention_outputs = attention_outputs + cross_attention_outputs[2:]
  
          # Apply Feed Forward layer
          hidden_states = self.layer[-1](hidden_states)
  
          # clamp inf values to enable fp16 training
          if hidden_states.dtype == torch.float16 and torch.isinf(hidden_states).any():
              clamp_value = torch.finfo(hidden_states.dtype).max - 1000
              hidden_states = torch.clamp(
                  hidden_states, min=-clamp_value, max=clamp_value
              )
  
          outputs = (hidden_states,)
  
          if use_cache:
              outputs = outputs + (present_key_value_state,) + attention_outputs
          else:
              outputs = outputs + attention_outputs
  
          return outputs  # hidden-states, present_key_value_states, (self-attention position bias), (self-attention weights), (cross-attention position bias), (cross-attention weights)
  
  
  class T5PreTrainedModel(PreTrainedModel):
      """
      An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained
      models.
      """
  
      config_class = T5Config
      load_tf_weights = load_tf_weights_in_t5
      base_model_prefix = "transformer"
      is_parallelizable = True
      supports_gradient_checkpointing = True
      _no_split_modules = ["T5Block"]
  
      @property
      def dummy_inputs(self):
          input_ids = torch.tensor(DUMMY_INPUTS)
          input_mask = torch.tensor(DUMMY_MASK)
          dummy_inputs = {
              "decoder_input_ids": input_ids,
              "input_ids": input_ids,
              "decoder_attention_mask": input_mask,
          }
  