content: from collections import namedtuple
  import copy
  import json
  import logging
  import os
  import warnings
  
  import numpy as np
  import typing
  from tqdm import tqdm
  from typing import (
      Any, List, Optional, Text, Dict, Tuple, Union)
  
  from rasa.core import utils
  from rasa.core.actions.action import ACTION_LISTEN_NAME
  from rasa.core.domain import Domain
  from rasa.core.featurizers import (
      TrackerFeaturizer,
      FullDialogueTrackerFeaturizer,
      LabelTokenizerSingleStateFeaturizer)
  from rasa.core.policies.policy import Policy
  
  import tensorflow as tf
  from rasa.core.policies.tf_utils import (
      TimeAttentionWrapper,
      ChronoBiasLayerNormBasicLSTMCell)
  from rasa.core.trackers import DialogueStateTracker
  
  if typing.TYPE_CHECKING:
      from rasa.core.policies.tf_utils import TimeAttentionWrapperState
  
  try:
      import cPickle as pickle
  except ImportError:
      import pickle
  
  logger = logging.getLogger(__name__)
  
  # namedtuple for all tf session related data
  SessionData = namedtuple("SessionData", ("X", "Y", "slots",
                                           "previous_actions",
                                           "actions_for_Y",
                                           "x_for_no_intent",
                                           "y_for_no_action",
                                           "y_for_action_listen",
                                           "all_Y_d"))
  
  
  class EmbeddingPolicy(Policy):
      """Recurrent Embedding Dialogue Policy (REDP)
  
      The policy that is used in our paper https://arxiv.org/abs/1811.11707
      """
  
      SUPPORTS_ONLINE_TRAINING = True
  
      # default properties (DOC MARKER - don't remove)
      defaults = {
          # nn architecture
          # a list of hidden layers sizes before user embed layer
          # number of hidden layers is equal to the length of this list
          "hidden_layers_sizes_a": [],
          # a list of hidden layers sizes before bot embed layer
          # number of hidden layers is equal to the length of this list
          "hidden_layers_sizes_b": [],
          # number of units in rnn cell
          "rnn_size": 64,
  
          # training parameters
          # flag if to turn on layer normalization for lstm cell
          "layer_norm": True,
          # initial and final batch sizes - batch size will be
          # linearly increased for each epoch
          "batch_size": [8, 32],
          # number of epochs
          "epochs": 1,
          # set random seed to any int to get reproducible results
          "random_seed": None,
  
          # embedding parameters
          # dimension size of embedding vectors
          "embed_dim": 20,
          # how similar the algorithm should try
          # to make embedding vectors for correct actions
          "mu_pos": 0.8,  # should be 0.0 < ... < 1.0 for 'cosine'
          # maximum negative similarity for incorrect actions
          "mu_neg": -0.2,  # should be -1.0 < ... < 1.0 for 'cosine'
          # the type of the similarity
          "similarity_type": 'cosine',  # string 'cosine' or 'inner'
          # the number of incorrect actions, the algorithm will minimize
          # their similarity to the user input during training
          "num_neg": 20,
          # flag if minimize only maximum similarity over incorrect actions
          "use_max_sim_neg": True,  # flag which loss function to use
  
          # regularization
          # the scale of L2 regularization
          "C2": 0.001,
          # the scale of how important is to minimize the maximum similarity
          # between embeddings of different actions
          "C_emb": 0.8,
          # scale loss with inverse frequency of bot actions
          "scale_loss_by_action_counts": True,
  
          # dropout rate for user nn
          "droprate_a": 0.0,
          # dropout rate for bot nn
          "droprate_b": 0.0,
          # dropout rate for rnn
          "droprate_rnn": 0.1,
  
          # attention parameters
          # flag to use attention over user input
          # as an input to rnn
          "attn_before_rnn": True,
          # flag to use attention over prev bot actions
          # and copy it to output bypassing rnn
          "attn_after_rnn": True,
  
          # flag to use `sparsemax` instead of `softmax` for attention
          "sparse_attention": False,  # flag to use sparsemax for probs
          # the range of allowed location-based attention shifts
          "attn_shift_range": None,  # if None, set to mean dialogue length / 2
  
          # visualization of accuracy
          # how often calculate train accuracy
          "evaluate_every_num_epochs": 20,  # small values may hurt performance
          # how many examples to use for calculation of train accuracy
          "evaluate_on_num_examples": 100  # large values may hurt performance
      }
  
      # end default properties (DOC MARKER - don't remove)
  
      @classmethod
      def _standard_featurizer(cls):
          return FullDialogueTrackerFeaturizer(
              LabelTokenizerSingleStateFeaturizer())
  
      def __init__(
          self,
          featurizer: Optional[FullDialogueTrackerFeaturizer] = None,
          priority: int = 1,
          encoded_all_actions: Optional[np.ndarray] = None,
          graph: Optional[tf.Graph] = None,
          session: Optional[tf.Session] = None,
          intent_placeholder: Optional[tf.Tensor] = None,
          action_placeholder: Optional[tf.Tensor] = None,
          slots_placeholder: Optional[tf.Tensor] = None,
          prev_act_placeholder: Optional[tf.Tensor] = None,
          dialogue_len: Optional[tf.Tensor] = None,
          x_for_no_intent: Optional[tf.Tensor] = None,
          y_for_no_action: Optional[tf.Tensor] = None,
          y_for_action_listen: Optional[tf.Tensor] = None,
          similarity_op: Optional[tf.Tensor] = None,
          alignment_history: Optional[tf.Tensor] = None,
          user_embed: Optional[tf.Tensor] = None,
          bot_embed: Optional[tf.Tensor] = None,
          slot_embed: Optional[tf.Tensor] = None,
          dial_embed: Optional[tf.Tensor] = None,
          rnn_embed: Optional[tf.Tensor] = None,
          attn_embed: Optional[tf.Tensor] = None,
          copy_attn_debug: Optional[tf.Tensor] = None,
          all_time_masks: Optional[tf.Tensor] = None,
          **kwargs: Any
      ) -> None:
          if featurizer:
              if not isinstance(featurizer, FullDialogueTrackerFeaturizer):
                  raise TypeError("Passed tracker featurizer of type {}, "
                                  "should be FullDialogueTrackerFeaturizer."
                                  "".format(type(featurizer).__name__))
          super(EmbeddingPolicy, self).__init__(featurizer, priority)
  
          # flag if to use the same embeddings for user and bot
          try:
              self.share_embedding = \
                  self.featurizer.state_featurizer.use_shared_vocab
          except AttributeError:
              self.share_embedding = False
  
          self._load_params(**kwargs)
  
          # chrono initialization for forget bias
          self.characteristic_time = None
  
          # encode all actions with numbers
          # persist this array for prediction time
          self.encoded_all_actions = encoded_all_actions
  
          # tf related instances
          self.graph = graph
          self.session = session
          self.a_in = intent_placeholder
          self.b_in = action_placeholder
          self.c_in = slots_placeholder
          self.b_prev_in = prev_act_placeholder
          self._dialogue_len = dialogue_len
          self._x_for_no_intent_in = x_for_no_intent
          self._y_for_no_action_in = y_for_no_action
          self._y_for_action_listen_in = y_for_action_listen
          self.sim_op = similarity_op
  
          # store attention probability distribution as
          # concatenated tensor of each attention types
          self.alignment_history = alignment_history
  
          # persisted embeddings
          self.user_embed = user_embed
          self.bot_embed = bot_embed
          self.slot_embed = slot_embed
          self.dial_embed = dial_embed
  
          self.rnn_embed = rnn_embed
          self.attn_embed = attn_embed
          self.copy_attn_debug = copy_attn_debug
  
          self.all_time_masks = all_time_masks
  
          # internal tf instances
          self._train_op = None
          self._is_training = None
          self._loss_scales = None
  
      # init helpers
      def _load_nn_architecture_params(self, config: Dict[Text, Any]) -> None:
          self.hidden_layer_sizes = {'a': config['hidden_layers_sizes_a'],
                                     'b': config['hidden_layers_sizes_b']}
  
          if self.share_embedding:
              if self.hidden_layer_sizes['a'] != self.hidden_layer_sizes['b']:
                  raise ValueError("Due to sharing vocabulary "
                                   "in the featurizer, embedding weights "
                                   "are shared as well. "
                                   "So hidden_layers_sizes_a={} should be "
                                   "equal to hidden_layers_sizes_b={}"
                                   "".format(self.hidden_layer_sizes['a'],
                                             self.hidden_layer_sizes['b']))
  
          self.rnn_size = config['rnn_size']
          self.layer_norm = config['layer_norm']
  
          self.batch_size = config['batch_size']
  
          self.epochs = config['epochs']
  
          self.random_seed = config['random_seed']
  
      def _load_embedding_params(self, config: Dict[Text, Any]) -> None:
          self.embed_dim = config['embed_dim']
          self.mu_pos = config['mu_pos']
          self.mu_neg = config['mu_neg']
          self.similarity_type = config['similarity_type']
          self.num_neg = config['num_neg']
          self.use_max_sim_neg = config['use_max_sim_neg']
  
      def _load_regularization_params(self, config: Dict[Text, Any]) -> None:
          self.C2 = config['C2']
          self.C_emb = config['C_emb']
          self.scale_loss_by_action_counts = \
              config['scale_loss_by_action_counts']
          self.droprate = {"a": config['droprate_a'],
                           "b": config['droprate_b'],
                           "rnn": config['droprate_rnn']}
  
      def _load_attn_params(self, config: Dict[Text, Any]) -> None:
          self.sparse_attention = config['sparse_attention']
          self.attn_shift_range = config['attn_shift_range']
          self.attn_after_rnn = config['attn_after_rnn']
          self.attn_before_rnn = config['attn_before_rnn']
  
      def is_using_attention(self):
          return self.attn_after_rnn or self.attn_before_rnn
  
      def _load_visual_params(self, config: Dict[Text, Any]) -> None:
          self.evaluate_every_num_epochs = config['evaluate_every_num_epochs']
          if self.evaluate_every_num_epochs < 1:
              self.evaluate_every_num_epochs = self.epochs
          self.evaluate_on_num_examples = config['evaluate_on_num_examples']
  
      def _load_params(self, **kwargs: Dict[Text, Any]) -> None:
          config = copy.deepcopy(self.defaults)
          config.update(kwargs)
  
          self._tf_config = self._load_tf_config(config)
          self._load_nn_architecture_params(config)
          self._load_embedding_params(config)
          self._load_regularization_params(config)
          self._load_attn_params(config)
          self._load_visual_params(config)
  
      # data helpers
      # noinspection PyPep8Naming
      def _create_X_slots_previous_actions(
          self,
          data_X: np.ndarray
      ) -> Tuple[np.ndarray, np.ndarray, np.ndarray]:
          """Extract feature vectors
  
          for user input (X), slots and
          previously executed actions from training data.
          """
  
          featurizer = self.featurizer.state_featurizer
          slot_start = featurizer.user_feature_len
          previous_start = slot_start + featurizer.slot_feature_len
  
          X = data_X[:, :, :slot_start]
          slots = data_X[:, :, slot_start:previous_start]
          previous_actions = data_X[:, :, previous_start:]
  
          return X, slots, previous_actions
  
      # noinspection PyPep8Naming
      @staticmethod
      def _actions_for_Y(data_Y: np.ndarray) -> np.ndarray:
          """Prepare Y data for training: extract actions indices."""
          return data_Y.argmax(axis=-1)
  
      # noinspection PyPep8Naming
      def _action_features_for_Y(self, actions_for_Y: np.ndarray) -> np.ndarray:
          """Prepare Y data for training: features for action labels."""
  
          return np.stack([np.stack([self.encoded_all_actions[action_idx]
                                     for action_idx in action_ids])
                           for action_ids in actions_for_Y])
  
      # noinspection PyPep8Naming
      @staticmethod
      def _create_zero_vector(X: np.ndarray) -> np.ndarray:
          """Create zero vector of shape (1, X.shape[-1])."""
  
          return np.zeros((1, X.shape[-1]), X.dtype)
  
      def _create_y_for_action_listen(self, domain: 'Domain') -> np.ndarray:
          """Extract feature vector for action_listen"""
          action_listen_idx = domain.index_for_action(ACTION_LISTEN_NAME)
          return self.encoded_all_actions[action_listen_idx:
                                          action_listen_idx + 1]
  
      # noinspection PyPep8Naming
      def _create_all_Y_d(self, dialogue_len: int) -> np.ndarray:
          """Stack encoded_all_intents on top of each other
  
          to create candidates for training examples and
          to calculate training accuracy.
          """
  
          return np.stack([self.encoded_all_actions] * dialogue_len)
  
      # noinspection PyPep8Naming
      def _create_tf_session_data(self,
                                  domain: 'Domain',
                                  data_X: np.ndarray,
                                  data_Y: Optional[np.ndarray] = None
                                  ) -> SessionData:
          """Combine all tf session related data into a named tuple"""
  
          X, slots, previous_actions = \
              self._create_X_slots_previous_actions(data_X)
  
          if data_Y is not None:
              # training time
              actions_for_Y = self._actions_for_Y(data_Y)
              Y = self._action_features_for_Y(actions_for_Y)
          else:
              # prediction time
              actions_for_Y = None
              Y = None
  
          x_for_no_intent = self._create_zero_vector(X)
          y_for_no_action = self._create_zero_vector(previous_actions)
          y_for_action_listen = self._create_y_for_action_listen(domain)
  
          # is needed to calculate train accuracy
          all_Y_d = self._create_all_Y_d(X.shape[1])
  
          return SessionData(
              X=X, Y=Y, slots=slots,
              previous_actions=previous_actions,
              actions_for_Y=actions_for_Y,
              x_for_no_intent=x_for_no_intent,
              y_for_no_action=y_for_no_action,
              y_for_action_listen=y_for_action_listen,
              all_Y_d=all_Y_d
          )
  
          # tf helpers:
  
      def _create_tf_nn(self,
                        x_in: tf.Tensor,
                        layer_sizes: List,
                        droprate: float,
                        layer_name_suffix: Text) -> tf.Tensor:
          """Create nn with hidden layers and name suffix."""
  
          reg = tf.contrib.layers.l2_regularizer(self.C2)
          x = tf.nn.relu(x_in)
          for i, layer_size in enumerate(layer_sizes):
              x = tf.layers.dense(inputs=x,
                                  units=layer_size,
                                  activation=tf.nn.relu,
                                  kernel_regularizer=reg,
                                  name='hidden_layer_{}_{}'
                                       ''.format(layer_name_suffix, i),
                                  reuse=tf.AUTO_REUSE)
              x = tf.layers.dropout(x, rate=droprate,
                                    training=self._is_training)
          return x
  
      def _create_embed(self,
                        x: tf.Tensor,
                        layer_name_suffix: Text) -> tf.Tensor:
          """Create dense embedding layer with a name."""
  
          reg = tf.contrib.layers.l2_regularizer(self.C2)
          embed_x = tf.layers.dense(inputs=x,
                                    units=self.embed_dim,
                                    activation=None,
                                    kernel_regularizer=reg,
                                    name='embed_layer_{}'
                                         ''.format(layer_name_suffix),
                                    reuse=tf.AUTO_REUSE)
          return embed_x
  
      def _create_tf_user_embed(self, a_in: tf.Tensor) -> tf.Tensor:
          """Create embedding user vector."""
  
          layer_name_suffix = 'a_and_b' if self.share_embedding else 'a'
  
          a = self._create_tf_nn(
              a_in,
              self.hidden_layer_sizes['a'],
              self.droprate['a'],
              layer_name_suffix=layer_name_suffix
          )
          return self._create_embed(a, layer_name_suffix=layer_name_suffix)
  
      def _create_tf_bot_embed(self, b_in: tf.Tensor) -> tf.Tensor:
          """Create embedding bot vector."""
  
          layer_name_suffix = 'a_and_b' if self.share_embedding else 'b'
  
          b = self._create_tf_nn(
              b_in,
              self.hidden_layer_sizes['b'],
              self.droprate['b'],
              layer_name_suffix=layer_name_suffix
          )
          return self._create_embed(b, layer_name_suffix=layer_name_suffix)
  
      def _create_tf_no_intent_embed(self,
                                     x_for_no_intent_i: tf.Tensor) -> tf.Tensor:
          """Create embedding user vector for empty intent."""
  
          layer_name_suffix = 'a_and_b' if self.share_embedding else 'a'
  
          x_for_no_intent = self._create_tf_nn(
              x_for_no_intent_i,
              self.hidden_layer_sizes['a'],
              droprate=0,
              layer_name_suffix=layer_name_suffix
          )
          return tf.stop_gradient(
              self._create_embed(x_for_no_intent,
                                 layer_name_suffix=layer_name_suffix))
  
      def _create_tf_no_action_embed(self,
                                     y_for_no_action_in: tf.Tensor) -> tf.Tensor:
          """Create embedding bot vector for empty action and action_listen."""
  
          layer_name_suffix = 'a_and_b' if self.share_embedding else 'b'
  
          y_for_no_action = self._create_tf_nn(
              y_for_no_action_in,
              self.hidden_layer_sizes['b'],
              droprate=0,
              layer_name_suffix=layer_name_suffix
          )
          return tf.stop_gradient(
              self._create_embed(y_for_no_action,
                                 layer_name_suffix=layer_name_suffix))
  
      def _create_rnn_cell(self):
          # type: () -> tf.contrib.rnn.RNNCell
          """Create one rnn cell."""
  
          # chrono initialization for forget bias
          # assuming that characteristic time is max dialogue length
          # left border that initializes forget gate close to 0
          bias_0 = -1.0
  
          # right border that initializes forget gate close to 1
          bias_1 = np.log(self.characteristic_time - 1.0)
          fbias = (bias_1 - bias_0) * np.random.random(self.rnn_size) + bias_0
  
          if self.attn_after_rnn:
              # since attention is copied to rnn output,
              # embedding should be performed inside the cell
              embed_layer_size = self.embed_dim
          else:
              embed_layer_size = None
  
          keep_prob = 1.0 - (self.droprate['rnn'] *
                             tf.cast(self._is_training, tf.float32))
  
          return ChronoBiasLayerNormBasicLSTMCell(
              num_units=self.rnn_size,
              layer_norm=self.layer_norm,
              forget_bias=fbias,
              input_bias=-fbias,
              dropout_keep_prob=keep_prob,
              out_layer_size=embed_layer_size
          )
  
      @staticmethod
      def _num_units(memory: tf.Tensor) -> int:
          return memory.shape[-1].value
  
      def _create_attn_mech(self,
                            memory: tf.Tensor,
                            real_length: tf.Tensor
                            ) -> tf.contrib.seq2seq.AttentionMechanism:
  
          return tf.contrib.seq2seq.BahdanauAttention(
              num_units=self._num_units(memory),
              memory=memory,
              memory_sequence_length=real_length,
              normalize=True,
              probability_fn=tf.identity,
              # we only attend to memory up to a current time step
              # it does not affect alignments, but
              # is important for interpolation gate
              score_mask_value=0
          )
  
      def cell_input_fn(self, rnn_inputs: tf.Tensor, attention: tf.Tensor,
                        num_cell_input_memory_units: int) -> tf.Tensor:
          """Combine rnn inputs and attention into cell input.
  
          Args:
            rnn_inputs: Tensor, first output from `rnn_and_attn_inputs_fn`.
  
            attention: Tensor, concatenated all attentions for one time step.
  
            num_cell_input_memory_units: int, number of the first units in
                                         `attention` that are responsible for
                                         enhancing cell input.
  
          Returns:
            A Tensor `cell_inputs` to feed to an rnn cell.
          """
  
          if num_cell_input_memory_units:
              if num_cell_input_memory_units == self.embed_dim:
                  # since attention can contain additional
                  # attention mechanisms, only attention
                  # from previous user input is used as an input
                  # for rnn cell and only if memory before rnn
                  # is the same size as embed_utter
                  return tf.concat([rnn_inputs[:, :self.embed_dim] +
                                    attention[:, :num_cell_input_memory_units],
                                    rnn_inputs[:, self.embed_dim:]], -1)
              else:
                  # in current implementation it cannot fall here,
                  # but this Exception exists in case
                  # attention before rnn is changed
                  raise ValueError("Number of memory units {} is not "
                                   "equal to number of utter units {}. "
                                   "Please modify cell input function "
                                   "accordingly."
                                   "".format(num_cell_input_memory_units,
                                             self.embed_dim))
          else:
              return rnn_inputs
  
      def rnn_and_attn_inputs_fn(self,
                                 inputs: tf.Tensor,
                                 cell_state: tf.Tensor
                                 ) -> Tuple[tf.Tensor, tf.Tensor]:
          """Construct rnn input and attention mechanism input.
  
          Args:
            inputs: Tensor, concatenated all embeddings for one time step:
                    [embed_utter, embed_slots, embed_prev_action].
  
            cell_state: Tensor, state of an rnn cell.
  
          Returns:
            Tuple of Tensors `rnn_inputs, attn_inputs` to feed to
            rnn and attention mechanisms.
          """
  
          # the hidden state c and slots are not included,
          # in hope that algorithm would learn correct attention
          # regardless of the hidden state c of an lstm and slots
          if isinstance(cell_state, tf.contrib.rnn.LSTMStateTuple):
              attn_inputs = tf.concat([inputs[:, :self.embed_dim],
                                       cell_state.h], -1)
          else:
              attn_inputs = tf.concat([inputs[:, :self.embed_dim],
                                       cell_state], -1)
  
          # include slots in inputs but exclude previous action, since
          # rnn should get previous action from its hidden state
          rnn_inputs = inputs[:, :(self.embed_dim +
                                   self.embed_dim)]
  
          return rnn_inputs, attn_inputs
  
      def _create_attn_cell(self,
                            cell: tf.contrib.rnn.RNNCell,
                            embed_utter: tf.Tensor,
                            embed_prev_action: tf.Tensor,
                            real_length: tf.Tensor,
                            embed_for_no_intent: tf.Tensor,
                            embed_for_no_action: tf.Tensor,
                            embed_for_action_listen: tf.Tensor
                            ) -> tf.contrib.rnn.RNNCell:
          """Wrap cell in attention wrapper with given memory."""
  
          if self.attn_before_rnn:
              # create attention over previous user input
              num_memory_units_before_rnn = self._num_units(embed_utter)
              attn_mech = self._create_attn_mech(embed_utter, real_length)
  
              # create mask for empty user input not to pay attention to it
              ignore_mask = tf.reduce_all(tf.equal(tf.expand_dims(
                  embed_for_no_intent, 0), embed_utter), -1)
  
              # do not use attention by location before rnn
              attn_shift_range = 0
          else:
              attn_mech = None
              ignore_mask = None
              num_memory_units_before_rnn = None
              attn_shift_range = None
  
          if self.attn_after_rnn:
              # create attention over previous bot actions
              attn_mech_after_rnn = self._create_attn_mech(embed_prev_action,
                                                           real_length)
  
              # create mask for empty bot action or action_listen
              # not to pay attention to them
              ignore_mask_listen = tf.logical_or(
                  tf.reduce_all(tf.equal(
                      tf.expand_dims(embed_for_no_action, 0),
                      embed_prev_action), -1),
                  tf.reduce_all(tf.equal(
                      tf.expand_dims(embed_for_action_listen, 0),
                      embed_prev_action), -1)
              )
  
              if attn_mech is not None:
                  # if there is another attention mechanism,
                  # create a list of attention mechanisms
                  attn_mech = [attn_mech, attn_mech_after_rnn]
                  ignore_mask = [ignore_mask, ignore_mask_listen]
                  attn_shift_range = [attn_shift_range, self.attn_shift_range]
              else:
                  attn_mech = attn_mech_after_rnn
                  ignore_mask = ignore_mask_listen
                  attn_shift_range = self.attn_shift_range
  
              # this particular attention mechanism is unusual
              # in the sense that its calculated attention vector is directly
              # added to cell output, therefore enabling copy mechanism
  
              # `index_of_attn_to_copy` is used by `TimeAttentionWrapper`,
              # to know which attention to copy
              index_of_attn_to_copy = -1
          else:
              index_of_attn_to_copy = None
  
          return TimeAttentionWrapper(
              cell=cell,
              attention_mechanism=attn_mech,
              sequence_len=self._dialogue_len,
              attn_shift_range=attn_shift_range,
              sparse_attention=self.sparse_attention,
              rnn_and_attn_inputs_fn=self.rnn_and_attn_inputs_fn,
              ignore_mask=ignore_mask,
              cell_input_fn=lambda inputs, attention: (
                  self.cell_input_fn(inputs, attention,
                                     num_memory_units_before_rnn)),
              index_of_attn_to_copy=index_of_attn_to_copy,
              likelihood_fn=lambda emb_1, emb_2: (
                  self._tf_sim(emb_1, emb_2, None)),
              tensor_not_to_copy=embed_for_action_listen,
              output_attention=True,
              alignment_history=True
          )
  
      def _create_tf_dial_embed(
              self,
              embed_utter: tf.Tensor,
              embed_slots: tf.Tensor,
              embed_prev_action: tf.Tensor,
              mask: tf.Tensor,
              embed_for_no_intent: tf.Tensor,
              embed_for_no_action: tf.Tensor,
              embed_for_action_listen: tf.Tensor
      ) -> Tuple[tf.Tensor, Union[tf.Tensor, 'TimeAttentionWrapperState']]:
          """Create rnn for dialogue level embedding."""
  
          cell_input = tf.concat([embed_utter, embed_slots,
                                  embed_prev_action], -1)
  
          cell = self._create_rnn_cell()
  
          real_length = tf.cast(tf.reduce_sum(mask, 1), tf.int32)
  
          if self.is_using_attention():
              cell = self._create_attn_cell(cell, embed_utter,
                                            embed_prev_action,
                                            real_length, embed_for_no_intent,
                                            embed_for_no_action,
                                            embed_for_action_listen)
  
          return tf.nn.dynamic_rnn(
              cell, cell_input,
              dtype=tf.float32,
              sequence_length=real_length,
              scope='rnn_decoder'
          )
  
      @staticmethod
      def _alignments_history_from(
          final_state: 'TimeAttentionWrapperState'
      ) -> tf.Tensor:
          """Extract alignments history form final rnn cell state."""
  
          alignments_from_state = final_state.alignment_history
          if not isinstance(alignments_from_state, tuple):
              alignments_from_state = [alignments_from_state]
  
          alignment_history = []
          for alignments in alignments_from_state:
              # reshape to (batch, time, memory_time)
              alignment_history.append(
                  tf.transpose(alignments.stack(), [1, 0, 2]))
  
          return tf.concat(alignment_history, -1)
  
      @staticmethod
      def _all_time_masks_from(
          final_state: 'TimeAttentionWrapperState'
      ) -> tf.Tensor:
          """Extract all time masks form final rnn cell state."""
  
          # reshape to (batch, time, memory_time) and ignore last time
          # because time_mask is created for the next time step
          return tf.transpose(final_state.all_time_masks.stack(),
                              [1, 0, 2])[:, :-1, :]
  
      def _sims_rnn_to_max_from(self, cell_output: tf.Tensor) -> List[tf.Tensor]:
          """Save intermediate tensors for debug purposes."""
  
          if self.attn_after_rnn:
              # extract additional debug tensors
              num_add = TimeAttentionWrapper.additional_output_size()
              self.copy_attn_debug = cell_output[:, :, -num_add:]
  
              # extract additional similarity to maximize
              sim_attn_to_max = cell_output[:, :, -num_add]
              sim_state_to_max = cell_output[:, :, -num_add + 1]
              return [sim_attn_to_max, sim_state_to_max]
          else:
              return []
  
      def _embed_dialogue_from(self,
                               cell_output: tf.Tensor) -> tf.Tensor:
          """Extract or calculate dialogue level embedding from cell_output."""
  
          if self.attn_after_rnn:
              # embedding layer is inside rnn cell
              embed_dialogue = cell_output[:, :, :self.embed_dim]
  
              # extract additional debug tensors
              num_add = TimeAttentionWrapper.additional_output_size()
              self.rnn_embed = cell_output[
                  :,
                  :,
                  self.embed_dim:(self.embed_dim + self.embed_dim)]
              self.attn_embed = cell_output[
                  :,
                  :,
                  (self.embed_dim + self.embed_dim):-num_add]
          else:
              # add embedding layer to rnn cell output
              embed_dialogue = self._create_embed(
                  cell_output[:, :, :self.rnn_size],
                  layer_name_suffix='out'
              )
              if self.attn_before_rnn:
                  # extract additional debug tensors
                  self.attn_embed = cell_output[:, :, self.rnn_size:]
  
          return embed_dialogue
  
      def _tf_sim(self,
                  embed_dialogue: tf.Tensor,
                  embed_action: tf.Tensor,
                  mask: Optional[tf.Tensor]
                  ) -> Tuple[tf.Tensor, tf.Tensor]:
          """Define similarity.
  
          This method has two roles:
          - calculate similarity between
              two embedding vectors of the same size
              and output binary mask and similarity;
          - calculate similarity with several embedded actions for the loss
              and output similarities between user input and bot actions
              and similarities between bot actions.
  
          They are kept in the same helper method,
          because it is necessary for them to be mathematically identical.
          """
  
          if self.similarity_type == 'cosine':
              # normalize embedding vectors for cosine similarity
              embed_dialogue = tf.nn.l2_normalize(embed_dialogue, -1)
              embed_action = tf.nn.l2_normalize(embed_action, -1)
  
          if self.similarity_type in {'cosine', 'inner'}:
  
              if len(embed_dialogue.shape) == len(embed_action.shape):
                  # calculate similarity between
                  # two embedding vectors of the same size
                  sim = tf.reduce_sum(embed_dialogue * embed_action, -1,
                                      keepdims=True)
                  bin_sim = tf.where(sim > (self.mu_pos - self.mu_neg) / 2.0,
                                     tf.ones_like(sim),
  