add tokenizer

Files changed (2) hide show

tokenizer.json CHANGED Viewed

The diff for this file is too large to render. See raw diff

tokenizer_config.json CHANGED Viewed

@@ -104,7 +104,7 @@
   "eos_token": "</s>",
   "extra_ids": 100,
   "model_max_length": 512,
-  "name_or_path": "models/t5/upload_model_tokenizer/tokenizer",
   "pad_token": "<pad>",
   "sp_model_kwargs": {},
   "special_tokens_map_file": null,

   "eos_token": "</s>",
   "extra_ids": 100,
   "model_max_length": 512,
+  "name_or_path": "models/one_shot_70000/voca_12000/tokenizer",
   "pad_token": "<pad>",
   "sp_model_kwargs": {},
   "special_tokens_map_file": null,