Training in progress, epoch 3

README.md

@@ -1,5 +1,5 @@
 ---
-base_model: google/gemma-3-270m-it
+base_model: google/gemma-3-1b-it
 library_name: transformers
 model_name: notepad
 tags:
@@ -11,7 +11,7 @@ licence: license
 
 # Model Card for notepad
 
-This model is a fine-tuned version of [google/gemma-3-270m-it](https://huggingface.co/google/gemma-3-270m-it).
+This model is a fine-tuned version of [google/gemma-3-1b-it](https://huggingface.co/google/gemma-3-1b-it).
 It has been trained using [TRL](https://github.com/huggingface/trl).
 
 ## Quick start

config.json

@@ -7,13 +7,17 @@
   "attention_dropout": 0.0,
   "attn_logit_softcapping": null,
   "bos_token_id": 2,
-  "eos_token_id": 1,
+  "cache_implementation": "hybrid",
+  "eos_token_id": [
+    1,
+    106
+  ],
   "final_logit_softcapping": null,
   "head_dim": 256,
   "hidden_activation": "gelu_pytorch_tanh",
-  "hidden_size": 640,
+  "hidden_size": 1152,
   "initializer_range": 0.02,
-  "intermediate_size": 2048,
+  "intermediate_size": 6912,
   "layer_types": [
     "sliding_attention",
     "sliding_attention",
@@ -32,23 +36,30 @@
     "sliding_attention",
     "sliding_attention",
     "sliding_attention",
-    "full_attention"
+    "full_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "sliding_attention"
   ],
   "max_position_embeddings": 32768,
   "model_type": "gemma3_text",
   "num_attention_heads": 4,
-  "num_hidden_layers": 18,
+  "num_hidden_layers": 26,
   "num_key_value_heads": 1,
   "pad_token_id": 0,
   "query_pre_attn_scalar": 256,
   "rms_norm_eps": 1e-06,
-  "rope_local_base_freq": 10000.0,
+  "rope_local_base_freq": 10000,
   "rope_scaling": null,
-  "rope_theta": 1000000.0,
+  "rope_theta": 1000000,
   "sliding_window": 512,
   "torch_dtype": "bfloat16",
   "transformers_version": "4.55.0",
-  "use_bidirectional_attention": false,
   "use_cache": true,
   "vocab_size": 262144
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c3094db18e5684404e0e9e9288d613e1d3f592b2f7dc5ef46e29639c8e9915a2
-size 536223056
+oid sha256:193d49eade16d67885fa1f33388e9aa66205e0c35cda2df110f6998f4e7fbe7d
+size 1999811208

runs/Aug15_08-48-48_3bc57af93f5f/events.out.tfevents.1755247734.3bc57af93f5f.2587.1

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:beda25e5a36efcf7789b3b136876a7ebe4a32da5fae1a15f3089561bc2e5e052
+size 37665

tokenizer_config.json

@@ -51336,7 +51336,7 @@
   "image_token": "<image_soft_token>",
   "model_max_length": 1000000000000000019884624838656,
   "pad_token": "<pad>",
-  "padding_side": "left",
+  "processor_class": "Gemma3Processor",
   "sp_model_kwargs": null,
   "spaces_between_special_tokens": false,
   "tokenizer_class": "GemmaTokenizer",

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ec94e0bd75870e927e1511e56566993ccc502c015c7b57de2589afbca5ad691c
+oid sha256:d45e7271a22a1a44f138257cc052b21e95a7cc23ad61f23f7a3d5590b3dabf9e
 size 5752