content: from typing import *
  import torch
  import torch.nn as nn
  import torch.nn.functional as F
  from ..modules.norm import GroupNorm32, ChannelLayerNorm32
  from ..modules.spatial import pixel_shuffle_3d
  from ..modules.utils import zero_module, convert_module_to_f16, convert_module_to_f32
  
  
  def norm_layer(norm_type: str, *args, **kwargs) -> nn.Module:
      """
      Return a normalization layer.
      """
      if norm_type == "group":
          return GroupNorm32(32, *args, **kwargs)
      elif norm_type == "layer":
          return ChannelLayerNorm32(*args, **kwargs)
      else:
          raise ValueError(f"Invalid norm type {norm_type}")
  
  
  class ResBlock3d(nn.Module):
      def __init__(
          self,
          channels: int,
          out_channels: Optional[int] = None,
          norm_type: Literal["group", "layer"] = "layer",
      ):
          super().__init__()
          self.channels = channels
          self.out_channels = out_channels or channels
  
          self.norm1 = norm_layer(norm_type, channels)
          self.norm2 = norm_layer(norm_type, self.out_channels)
          self.conv1 = nn.Conv3d(channels, self.out_channels, 3, padding=1)
          self.conv2 = zero_module(nn.Conv3d(self.out_channels, self.out_channels, 3, padding=1))
          self.skip_connection = nn.Conv3d(channels, self.out_channels, 1) if channels != self.out_channels else nn.Identity()
      
      def forward(self, x: torch.Tensor) -> torch.Tensor:
          h = self.norm1(x)
          h = F.silu(h)
          h = self.conv1(h)
          h = self.norm2(h)
          h = F.silu(h)
          h = self.conv2(h)
          h = h + self.skip_connection(x)
          return h
  
  
  class DownsampleBlock3d(nn.Module):
      def __init__(
          self,
          in_channels: int,
          out_channels: int,
          mode: Literal["conv", "avgpool"] = "conv",
      ):
          assert mode in ["conv", "avgpool"], f"Invalid mode {mode}"
  
          super().__init__()
          self.in_channels = in_channels
          self.out_channels = out_channels
  
          if mode == "conv":
              self.conv = nn.Conv3d(in_channels, out_channels, 2, stride=2)
          elif mode == "avgpool":
              assert in_channels == out_channels, "Pooling mode requires in_channels to be equal to out_channels"
  
      def forward(self, x: torch.Tensor) -> torch.Tensor:
          if hasattr(self, "conv"):
              return self.conv(x)
          else:
              return F.avg_pool3d(x, 2)
  
  
  class UpsampleBlock3d(nn.Module):
      def __init__(
          self,
          in_channels: int,
          out_channels: int,
          mode: Literal["conv", "nearest"] = "conv",
      ):
          assert mode in ["conv", "nearest"], f"Invalid mode {mode}"
  
          super().__init__()
          self.in_channels = in_channels
          self.out_channels = out_channels
  
          if mode == "conv":
              self.conv = nn.Conv3d(in_channels, out_channels*8, 3, padding=1)
          elif mode == "nearest":
              assert in_channels == out_channels, "Nearest mode requires in_channels to be equal to out_channels"
  
      def forward(self, x: torch.Tensor) -> torch.Tensor:
          if hasattr(self, "conv"):
              x = self.conv(x)
              return pixel_shuffle_3d(x, 2)
          else:
              return F.interpolate(x, scale_factor=2, mode="nearest")
          
  
  class SparseStructureEncoder(nn.Module):
      """
      Encoder for Sparse Structure (\mathcal{E}_S in the paper Sec. 3.3).
      
      Args:
          in_channels (int): Channels of the input.
          latent_channels (int): Channels of the latent representation.
          num_res_blocks (int): Number of residual blocks at each resolution.
          channels (List[int]): Channels of the encoder blocks.
          num_res_blocks_middle (int): Number of residual blocks in the middle.
          norm_type (Literal["group", "layer"]): Type of normalization layer.
          use_fp16 (bool): Whether to use FP16.
      """
      def __init__(
          self,
          in_channels: int,
          latent_channels: int,
          num_res_blocks: int,
          channels: List[int],
          num_res_blocks_middle: int = 2,
          norm_type: Literal["group", "layer"] = "layer",
          use_fp16: bool = False,
      ):
          super().__init__()
          self.in_channels = in_channels
          self.latent_channels = latent_channels
          self.num_res_blocks = num_res_blocks
          self.channels = channels
          self.num_res_blocks_middle = num_res_blocks_middle
          self.norm_type = norm_type
          self.use_fp16 = use_fp16
          self.dtype = torch.float16 if use_fp16 else torch.float32
  
          self.input_layer = nn.Conv3d(in_channels, channels[0], 3, padding=1)
  
          self.blocks = nn.ModuleList([])
          for i, ch in enumerate(channels):
              self.blocks.extend([
                  ResBlock3d(ch, ch)
                  for _ in range(num_res_blocks)
              ])
              if i < len(channels) - 1:
                  self.blocks.append(
                      DownsampleBlock3d(ch, channels[i+1])
                  )
          
          self.middle_block = nn.Sequential(*[
              ResBlock3d(channels[-1], channels[-1])
              for _ in range(num_res_blocks_middle)
          ])
  
          self.out_layer = nn.Sequential(
              norm_layer(norm_type, channels[-1]),
              nn.SiLU(),
              nn.Conv3d(channels[-1], latent_channels*2, 3, padding=1)
          )
  
          if use_fp16:
              self.convert_to_fp16()
  
      @property
      def device(self) -> torch.device:
          """
          Return the device of the model.
          """
          return next(self.parameters()).device
  
      def convert_to_fp16(self) -> None:
          """
          Convert the torso of the model to float16.
          """
          self.use_fp16 = True
          self.dtype = torch.float16
          self.blocks.apply(convert_module_to_f16)
          self.middle_block.apply(convert_module_to_f16)
  
      def convert_to_fp32(self) -> None:
          """
          Convert the torso of the model to float32.
          """
          self.use_fp16 = False
          self.dtype = torch.float32
          self.blocks.apply(convert_module_to_f32)
          self.middle_block.apply(convert_module_to_f32)
  
      def forward(self, x: torch.Tensor, sample_posterior: bool = False, return_raw: bool = False) -> torch.Tensor:
          h = self.input_layer(x)
          h = h.type(self.dtype)
  
          for block in self.blocks:
              h = block(h)
          h = self.middle_block(h)
  
          h = h.type(x.dtype)
          h = self.out_layer(h)
  
          mean, logvar = h.chunk(2, dim=1)
  
          if sample_posterior:
              std = torch.exp(0.5 * logvar)
              z = mean + std * torch.randn_like(std)
          else:
              z = mean
              
          if return_raw:
              return z, mean, logvar
          return z
          
  
  class SparseStructureDecoder(nn.Module):
      """
      Decoder for Sparse Structure (\mathcal{D}_S in the paper Sec. 3.3).
      
      Args:
          out_channels (int): Channels of the output.
          latent_channels (int): Channels of the latent representation.
          num_res_blocks (int): Number of residual blocks at each resolution.
          channels (List[int]): Channels of the decoder blocks.
          num_res_blocks_middle (int): Number of residual blocks in the middle.
          norm_type (Literal["group", "layer"]): Type of normalization layer.
          use_fp16 (bool): Whether to use FP16.
      """ 
      def __init__(
          self,
          out_channels: int,
          latent_channels: int,
          num_res_blocks: int,
          channels: List[int],
          num_res_blocks_middle: int = 2,
          norm_type: Literal["group", "layer"] = "layer",
          use_fp16: bool = False,
      ):
          super().__init__()
          self.out_channels = out_channels
          self.latent_channels = latent_channels
          self.num_res_blocks = num_res_blocks
          self.channels = channels
          self.num_res_blocks_middle = num_res_blocks_middle
          self.norm_type = norm_type
          self.use_fp16 = use_fp16
          self.dtype = torch.float16 if use_fp16 else torch.float32
  
          self.input_layer = nn.Conv3d(latent_channels, channels[0], 3, padding=1)
  
          self.middle_block = nn.Sequential(*[
              ResBlock3d(channels[0], channels[0])
              for _ in range(num_res_blocks_middle)
          ])
  
          self.blocks = nn.ModuleList([])
          for i, ch in enumerate(channels):
              self.blocks.extend([
                  ResBlock3d(ch, ch)
                  for _ in range(num_res_blocks)
              ])
              if i < len(channels) - 1:
                  self.blocks.append(
                      UpsampleBlock3d(ch, channels[i+1])
                  )
  
          self.out_layer = nn.Sequential(
              norm_layer(norm_type, channels[-1]),
              nn.SiLU(),
              nn.Conv3d(channels[-1], out_channels, 3, padding=1)
          )
  
          if use_fp16:
              self.convert_to_fp16()
  
      @property
      def device(self) -> torch.device:
          """
          Return the device of the model.
          """
          return next(self.parameters()).device
      
      def convert_to_fp16(self) -> None:
          """
          Convert the torso of the model to float16.
          """
          self.use_fp16 = True
          self.dtype = torch.float16
          self.blocks.apply(convert_module_to_f16)
          self.middle_block.apply(convert_module_to_f16)
  
      def convert_to_fp32(self) -> None:
          """
          Convert the torso of the model to float32.
          """
          self.use_fp16 = False
          self.dtype = torch.float32
          self.blocks.apply(convert_module_to_f32)
          self.middle_block.apply(convert_module_to_f32)
      
      def forward(self, x: torch.Tensor) -> torch.Tensor:
          h = self.input_layer(x)
          
          h = h.type(self.dtype)
                  
          h = self.middle_block(h)
          for block in self.blocks:
              h = block(h)
  
          h = h.type(x.dtype)
          h = self.out_layer(h)
          return h
  