Make the model larger and add PReLU alpha constraint

chatbot_constructor.py

@@ -4,14 +4,24 @@ from keras.models import Model
 from keras.saving import load_model
 from keras.layers import *
 from keras.regularizers import L1
+from keras.constraints import Constraint
 from tensorflow.keras.optimizers import RMSprop
 from keras.preprocessing.text import Tokenizer
+import keras.backend as K
 import os
 import hashlib
 import keras
 
 os.mkdir("cache")
 
+class ValueConstraint(Constraint):
+    def __init__(self, min_value: float = -1, max_value: float = 1):
+        self.min_value = min_value
+        self.max_value
+
+    def __call__(self, w):
+        return K.clip(w, self.min_value, self.max_value)
+
 def todset(text: str):
     lines = [x.rstrip("\n").lower().split("→") for x in text.split("\n")]
     lines = [(x[0].replace("\\n", "\n"), x[1].replace("\\n", "\n")) for x in lines]
@@ -33,8 +43,8 @@ def hash_str(data: str):
 def train(message: str = "", regularization: float = 0.0001, dropout: float = 0.1, learning_rate: float = 0.001, epochs: int = 16, emb_size: int = 128, input_len: int = 16, kernels_count: int = 8, kernel_size: int = 8, left_padding: bool = True, end_activation: str = "softmax", data: str = ""):
     data_hash = None
     if "→" not in data or "\n" not in data:
-            if data in os.listdir("cache"):
-                data_hash = data
+            if data in os.listdir("cache"): # data = filename
+                data_hash = data # set the hash to the file name
             else:
                 return "Dataset example:\nquestion→answer\nquestion→answer\netc."
     dset, responses = todset(data)
@@ -68,14 +78,14 @@ def train(message: str = "", regularization: float = 0.0001, dropout: float = 0.
         conv3_flatten_layer = Flatten()(conv3_layer)
         concat1_layer = Concatenate()([flatten_layer, attn_flatten_layer, conv1_flatten_layer, conv2_flatten_layer, conv3_flatten_layer])
         dropout2_layer = Dropout(dropout)(concat1_layer)
-        dense1_layer = Dense(512, activation="linear", kernel_regularizer=L1(regularization))(dropout2_layer)
-        prelu1_layer = PReLU()(dense1_layer)
+        dense1_layer = Dense(2048, activation="linear", kernel_regularizer=L1(regularization))(dropout2_layer)
+        prelu1_layer = PReLU(alpha_constraint=ValueConstraint())(dense1_layer)
         dropout3_layer = Dropout(dropout)(prelu1_layer)
-        dense2_layer = Dense(256, activation="relu", kernel_regularizer=L1(regularization))(dropout3_layer)
+        dense2_layer = Dense(1024, activation="relu", kernel_regularizer=L1(regularization))(dropout3_layer)
         dropout4_layer = Dropout(dropout)(dense2_layer)
-        dense3_layer = Dense(256, activation="relu", kernel_regularizer=L1(regularization))(dropout4_layer)
+        dense3_layer = Dense(512, activation="relu", kernel_regularizer=L1(regularization))(dropout4_layer)
         dropout5_layer = Dropout(dropout)(dense3_layer)
-        dense4_layer = Dense(100, activation="relu", kernel_regularizer=L1(regularization))(dropout5_layer)
+        dense4_layer = Dense(256, activation="relu", kernel_regularizer=L1(regularization))(dropout5_layer)
         concat2_layer = Concatenate()([dense4_layer, prelu1_layer, attn_flatten_layer, conv1_flatten_layer])
         dense4_layer = Dense(resps_len, activation=end_activation, kernel_regularizer=L1(regularization))(concat2_layer)
         model = Model(inputs=input_layer, outputs=dense4_layer)
@@ -103,7 +113,7 @@ def train(message: str = "", regularization: float = 0.0001, dropout: float = 0.
         model.save(f"cache/{data_hash}")
     tokens = tokenizer.texts_to_sequences([message,])[0]
     prediction = model.predict(np.array([(list(tokens)+[0,]*inp_len)[:inp_len],]))[0]
-    keras.backend.clear_session()
+    K.clear_session()
     return responses[np.argmax(prediction)]
 
 if __name__ == "__main__":