content: # Copyright 2024 The Qwen Team and The HuggingFace Inc. team.
  # SPDX-License-Identifier: Apache-2.0
  
  """Tokenization classes for Qwen2."""
  
  import json
  import os
  import unicodedata
  from functools import lru_cache
  from typing import Optional, Tuple
  
  import regex as re
  
  from transformers.tokenization_utils import AddedToken, PreTrainedTokenizer
  from transformers.utils import logging
  
  
  logger = logging.get_logger(__name__)
  
  VOCAB_FILES_NAMES = {
      "vocab_file": "vocab.json",
      "merges_file": "merges.txt",
  }
  
  
  MAX_MODEL_INPUT_SIZES = {"qwen/qwen-tokenizer": 32768}
  
  PRETOKENIZE_REGEX = r"""(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\r\n\p{L}\p{N}]?\p{L}+|\p{N}| ?[^\s\p{L}\p{N}]+[\r\n]*|\s*[\r\n]+|\s+(?!\S)|\s+"""
  
  
  @lru_cache()
  # Copied from transformers.models.gpt2.tokenization_gpt2.bytes_to_unicode
  def bytes_to_unicode():
      """
      Returns list of utf-8 byte and a mapping to unicode strings. We specifically avoids mapping to whitespace/control
      characters the bpe code barfs on.
  
      The reversible bpe codes work on unicode strings. This means you need a large # of unicode characters in your vocab
      if you want to avoid UNKs. When you're at something like a 10B token dataset you end up needing around 5K for
      decent coverage. This is a significant percentage of your normal, say, 32K bpe vocab. To avoid that, we want lookup
      tables between utf-8 bytes and unicode strings.
      """
      bs = (
          list(range(ord("!"), ord("~") + 1)) + list(range(ord("¡"), ord("¬") + 1)) + list(range(ord("®"), ord("ÿ") + 1))
      )
      cs = bs[:]
      n = 0
      for b in range(2**8):
          if b not in bs:
              bs.append(b)
              cs.append(2**8 + n)
              n += 1
      cs = [chr(n) for n in cs]
      return dict(zip(bs, cs))
  
  
  # Copied from transformers.models.gpt2.tokenization_gpt2.get_pairs
  def get_pairs(word):
      """
      Return set of symbol pairs in a word.
  
      Word is represented as tuple of symbols (symbols being variable-length strings).
      """
      pairs = set()
      prev_char = word[0]
      for char in word[1:]:
          pairs.add((prev_char, char))
          prev_char = char
      return pairs
  
  
  class Qwen2Tokenizer(PreTrainedTokenizer):
      """
      Construct a Qwen2 tokenizer. Based on byte-level Byte-Pair-Encoding.
  
      Same with GPT2Tokenizer, this tokenizer has been trained to treat spaces like parts of the tokens so a word will
      be encoded differently whether it is at the beginning of the sentence (without space) or not:
  
      ```python
      >>> from transformers import Qwen2Tokenizer
  
      >>> tokenizer = Qwen2Tokenizer.from_pretrained("Qwen/Qwen-tokenizer")
      >>> tokenizer("Hello world")["input_ids"]
      [9707, 1879]
  
      >>> tokenizer(" Hello world")["input_ids"]
      [21927, 1879]
      ```
      This is expected.
  
      You should not use GPT2Tokenizer instead, because of the different pretokenization rules.
  
      This tokenizer inherits from [`PreTrainedTokenizer`] which contains most of the main methods. Users should refer to
      this superclass for more information regarding those methods.
  
      Args:
          vocab_file (`str`):
              Path to the vocabulary file.
          merges_file (`str`):
              Path to the merges file.
          errors (`str`, *optional*, defaults to `"replace"`):
              Paradigm to follow when decoding bytes to UTF-8. See
              [bytes.decode](https://docs.python.org/3/library/stdtypes.html#bytes.decode) for more information.
          unk_token (`str`, *optional*, defaults to `"<|endoftext|>"`):
              The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this
              token instead.
          bos_token (`str`, *optional*):
              The beginning of sequence token. Not applicable for this tokenizer.
          eos_token (`str`, *optional*, defaults to `"<|endoftext|>"`):
              The end of sequence token.
          pad_token (`str`, *optional*, defaults to `"<|endoftext|>"`):
              The token used for padding, for example when batching sequences of different lengths.
          clean_up_tokenization_spaces (`bool`, *optional*, defaults to `False`):
              Whether or not the model should cleanup the spaces that were added when splitting the input text during the
              tokenization process. Not applicable to this tokenizer, since tokenization does not add spaces.
          split_special_tokens (`bool`, *optional*, defaults to `False`):
              Whether or not the special tokens should be split during the tokenization process. The default behavior is
              to not split special tokens. This means that if `<|endoftext|>` is the `eos_token`, then `tokenizer.tokenize("<|endoftext|>") =
              ['<|endoftext|>`]. Otherwise, if `split_special_tokens=True`, then `tokenizer.tokenize("<|endoftext|>")` will be give `['<',
              '|', 'endo', 'ft', 'ext', '|', '>']`. This argument is only supported for `slow` tokenizers for the moment.
      """
  
      vocab_files_names = VOCAB_FILES_NAMES
      model_input_names = ["input_ids", "attention_mask"]
  
      def __init__(
          self,
          vocab_file,
          merges_file,
          errors="replace",
          unk_token="<|endoftext|>",
          bos_token=None,
          eos_token="<|endoftext|>",
          pad_token="<|endoftext|>",
          clean_up_tokenization_spaces=False,
          split_special_tokens=False,
          **kwargs,
      ):
          # Qwen vocab does not contain control tokens; added tokens need to be special
          bos_token = (
              AddedToken(bos_token, lstrip=False, rstrip=False, special=True, normalized=False)
              if isinstance(bos_token, str)
              else bos_token
          )
          eos_token = (
              AddedToken(eos_token, lstrip=False, rstrip=False, special=True, normalized=False)
              if isinstance(eos_token, str)
              else eos_token
          )
          unk_token = (
              AddedToken(unk_token, lstrip=False, rstrip=False, special=True, normalized=False)
              if isinstance(unk_token, str)
              else unk_token
          )
          pad_token = (
              AddedToken(pad_token, lstrip=False, rstrip=False, special=True, normalized=False)
              if isinstance(pad_token, str)
              else pad_token
          )
  
          with open(vocab_file, encoding="utf-8") as vocab_handle:
              self.encoder = json.load(vocab_handle)
          self.decoder = {v: k for k, v in self.encoder.items()}
          self.errors = errors  # how to handle errors in decoding
          self.byte_encoder = bytes_to_unicode()
          self.byte_decoder = {v: k for k, v in self.byte_encoder.items()}
          bpe_merges = []
          with open(merges_file, encoding="utf-8") as merges_handle:
              for i, line in enumerate(merges_handle):
                  line = line.strip()
                  if (i == 0 and line.startswith("#version:")) or not line:
                      continue
                  bpe_merges.append(tuple(line.split()))
          self.bpe_ranks = dict(zip(bpe_merges, range(len(bpe_merges))))
          # NOTE: the cache can grow without bound and will get really large for long running processes
          # (esp. for texts of language that do not use space between word, e.g. Chinese); technically
          # not a memory leak but appears as one.
          # GPT2Tokenizer has the same problem, so let's be consistent.
          self.cache = {}
  
          self.pat = re.compile(PRETOKENIZE_REGEX)
  
          if kwargs.get("add_prefix_space", False):
              logger.warning_once(
                  f"{self.__class__.__name} does not support `add_prefix_space`, setting it to True has no effect."
              )
  
          super().__init__(
              errors=errors,
              bos_token=bos_token,
              eos_token=eos_token,
              pad_token=pad_token,
              unk_token=unk_token,
              clean_up_tokenization_spaces=clean_up_tokenization_spaces,
              split_special_tokens=split_special_tokens,
              **kwargs,
          )
  
      @property
      def vocab_size(self) -> int:
          return len(self.encoder)
  
      # Copied from transformers.models.gpt2.tokenization_gpt2.GPT2Tokenizer.get_vocab
      def get_vocab(self):
          return dict(self.encoder, **self.added_tokens_encoder)
  
      # Copied from transformers.models.gpt2.tokenization_gpt2.GPT2Tokenizer.bpe
      def bpe(self, token):
          if token in self.cache:
              return self.cache[token]
          word = tuple(token)
          pairs = get_pairs(word)
  
          if not pairs:
              return token
  
          while True:
              bigram = min(pairs, key=lambda pair: self.bpe_ranks.get(pair, float("inf")))
              if bigram not in self.bpe_ranks:
                  break
              first, second = bigram
              new_word = []
              i = 0
              while i < len(word):
                  try:
                      j = word.index(first, i)
                  except ValueError:
                      new_word.extend(word[i:])
                      break
                  else:
                      new_word.extend(word[i:j])
                      i = j
  
                  if word[i] == first and i < len(word) - 1 and word[i + 1] == second:
                      new_word.append(first + second)
                      i += 2
                  else:
                      new_word.append(word[i])
                      i += 1
              new_word = tuple(new_word)
              word = new_word
              if len(word) == 1:
                  break
              else:
                  pairs = get_pairs(word)
          word = " ".join(word)
          self.cache[token] = word
          return word
  
      # Copied from transformers.models.gpt2.tokenization_gpt2.GPT2Tokenizer._tokenize
      def _tokenize(self, text):
          """Tokenize a string."""
          bpe_tokens = []
          for token in re.findall(self.pat, text):
              token = "".join(
                  self.byte_encoder[b] for b in token.encode("utf-8")
              )  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
              bpe_tokens.extend(bpe_token for bpe_token in self.bpe(token).split(" "))
          return bpe_tokens
  
      # Copied from transformers.models.gpt2.tokenization_gpt2.GPT2Tokenizer._convert_token_to_id
      def _convert_token_to_id(self, token):
          """Converts a token (str) in an id using the vocab."""
          return self.encoder.get(token, self.encoder.get(self.unk_token))
  
      # Copied from transformers.models.gpt2.tokenization_gpt2.GPT2Tokenizer._convert_id_to_token
      def _convert_id_to_token(self, index):
          """Converts an index (integer) in a token (str) using the vocab."""
          return self.decoder.get(index)
  
      # Copied from transformers.models.gpt2.tokenization_gpt2.GPT2Tokenizer.convert_tokens_to_string
      def convert_tokens_to_string(self, tokens):
          """Converts a sequence of tokens (string) in a single string."""
          text = "".join(tokens)
          text = bytearray([self.byte_decoder[c] for c in text]).decode("utf-8", errors=self.errors)
          return text
  
      def decode(
          self,
          token_ids,
          skip_special_tokens: bool = False,
          clean_up_tokenization_spaces: Optional[bool] = False,
          spaces_between_special_tokens: bool = False,
          **kwargs,
      ) -> str:
          # `spaces_between_special_tokens` defaults to True for _decode in slow tokenizers
          # and cannot be configured elsewhere, but it should default to False for Qwen2Tokenizer
          return super().decode(
              token_ids,
              skip_special_tokens=skip_special_tokens,
              clean_up_tokenization_spaces=clean_up_tokenization_spaces,
              spaces_between_special_tokens=spaces_between_special_tokens,
              **kwargs,
          )
  
      # Copied from transformers.models.gpt2.tokenization_gpt2.GPT2Tokenizer.save_vocabulary
      def save_vocabulary(self, save_directory: str, filename_prefix: Optional[str] = None) -> Tuple[str]:
          if not os.path.isdir(save_directory):
              logger.error(f"Vocabulary path ({save_directory}) should be a directory")
              return
          vocab_file = os.path.join(
              save_directory, (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]
          )
          merge_file = os.path.join(
              save_directory, (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["merges_file"]
          )
  
          with open(vocab_file, "w", encoding="utf-8") as f:
              f.write(json.dumps(self.encoder, indent=2, sort_keys=True, ensure_ascii=False) + "\n")
  
          index = 0
          with open(merge_file, "w", encoding="utf-8") as writer:
              writer.write("#version: 0.2\n")
              for bpe_tokens, token_index in sorted(self.bpe_ranks.items(), key=lambda kv: kv[1]):
                  if index != token_index:
                      logger.warning(
                          f"Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."
                          " Please check that the tokenizer is not corrupted!"
                      )
                      index = token_index
                  writer.write(" ".join(bpe_tokens) + "\n")
                  index += 1
  
          return vocab_file, merge_file
  
      def prepare_for_tokenization(self, text, **kwargs):
          text = unicodedata.normalize("NFC", text)
          return (text, kwargs)
  