content: import numpy as np
  
  
  # --------------------------------------------------------
  # 3D sine-cosine position embedding
  # References:
  # MVD: https://github.com/ruiwang2021/mvd/blob/main/modeling_finetune.py
  # --------------------------------------------------------
  def get_3d_sincos_pos_embed(embed_dim, grid_size, t_size, cls_token=False):
      """
      grid_size: int of the grid height and width
      t_size: int of the temporal size
      return:
      pos_embed: [t_size*grid_size*grid_size, embed_dim] or [1+t_size*grid_size*grid_size, embed_dim] (w/ or w/o cls_token)
      """
      assert embed_dim % 4 == 0
      embed_dim_spatial = embed_dim // 4 * 3
      embed_dim_temporal = embed_dim // 4
  
      # spatial
      grid_h = np.arange(grid_size, dtype=np.float32)
      grid_w = np.arange(grid_size, dtype=np.float32)
      grid = np.meshgrid(grid_w, grid_h)  # here w goes first
      grid = np.stack(grid, axis=0)
  
      grid = grid.reshape([2, 1, grid_size, grid_size])
      pos_embed_spatial = get_2d_sincos_pos_embed_from_grid(
          embed_dim_spatial, grid
      )
  
      # temporal
      grid_t = np.arange(t_size, dtype=np.float32)
      pos_embed_temporal = get_1d_sincos_pos_embed_from_grid(
          embed_dim_temporal, grid_t
      )
  
      # concate: [T, H, W] order
      pos_embed_temporal = pos_embed_temporal[:, np.newaxis, :]
      pos_embed_temporal = np.repeat(
          pos_embed_temporal, grid_size**2, axis=1
      )  # [T, H*W, D // 4]
      pos_embed_spatial = pos_embed_spatial[np.newaxis, :, :]
      pos_embed_spatial = np.repeat(
          pos_embed_spatial, t_size, axis=0
      )  # [T, H*W, D // 4 * 3]
  
      pos_embed = np.concatenate([pos_embed_temporal, pos_embed_spatial], axis=-1)
      pos_embed = pos_embed.reshape([-1, embed_dim])  # [T*H*W, D]
  
      if cls_token:
          pos_embed = np.concatenate(
              [np.zeros([1, embed_dim]), pos_embed], axis=0
          )
      return pos_embed
  
  
  # --------------------------------------------------------
  # 2D sine-cosine position embedding
  # References:
  # Transformer: https://github.com/tensorflow/models/blob/master/official/nlp/transformer/model_utils.py
  # MoCo v3: https://github.com/facebookresearch/moco-v3
  # --------------------------------------------------------
  def get_2d_sincos_pos_embed(embed_dim, grid_size, cls_token=False):
      """
      grid_size: int of the grid height and width
      return:
      pos_embed: [grid_size*grid_size, embed_dim] or [1+grid_size*grid_size, embed_dim] (w/ or w/o cls_token)
      """
      grid_h = np.arange(grid_size, dtype=np.float32)
      grid_w = np.arange(grid_size, dtype=np.float32)
      grid = np.meshgrid(grid_w, grid_h)  # here w goes first
      grid = np.stack(grid, axis=0)
  
      grid = grid.reshape([2, 1, grid_size, grid_size])
      pos_embed = get_2d_sincos_pos_embed_from_grid(embed_dim, grid)
      if cls_token:
          pos_embed = np.concatenate(
              [np.zeros([1, embed_dim]), pos_embed], axis=0
          )
      return pos_embed
  
  
  def get_1d_sincos_pos_embed(embed_dim, t_size, cls_token=False):
      """
      t_size: int of the temporal size
      return:
      pos_embed: [t_size, embed_dim] or [1+t_size, embed_dim] (w/ or w/o cls_token)
      """
      grid_t = np.arange(t_size, dtype=np.float32)
      pos_embed = get_1d_sincos_pos_embed_from_grid(embed_dim, grid_t)
      if cls_token:
          pos_embed = np.concatenate(
              [np.zeros([1, embed_dim]), pos_embed], axis=0
          )
      return pos_embed
  
  
  def get_2d_sincos_pos_embed_from_grid(embed_dim, grid):
      assert embed_dim % 2 == 0
  
      # use half of dimensions to encode grid_h
      emb_h = get_1d_sincos_pos_embed_from_grid(
          embed_dim // 2, grid[0]
      )  # (H*W, D/2)
      emb_w = get_1d_sincos_pos_embed_from_grid(
          embed_dim // 2, grid[1]
      )  # (H*W, D/2)
  
      emb = np.concatenate([emb_h, emb_w], axis=1)  # (H*W, D)
      return emb
  
  
  def get_1d_sincos_pos_embed_from_grid(embed_dim, pos):
      """
      embed_dim: output dimension for each position
      pos: a list of positions to be encoded: size (M,)
      out: (M, D)
      """
      assert embed_dim % 2 == 0
      omega = np.arange(embed_dim // 2, dtype=np.float32)
      omega /= embed_dim / 2.0
      omega = 1.0 / 10000**omega  # (D/2,)
  
      pos = pos.reshape(-1)  # (M,)
      out = np.einsum("m,d->md", pos, omega)  # (M, D/2), outer product
  
      emb_sin = np.sin(out)  # (M, D/2)
      emb_cos = np.cos(out)  # (M, D/2)
  
      emb = np.concatenate([emb_sin, emb_cos], axis=1)  # (M, D)
      return emb