remove tokenizer filter

levanpon98 · Sep 29, 2020 · e44ff41 · e44ff41
1 parent 9b2ebaf
commit e44ff41
Showing 1 changed file with 1 addition and 2 deletions.
diff --git a/src/loader.py b/src/loader.py
@@ -47,8 +47,7 @@ def load_data(data_path):
     train_images, valid_images, train_texts, valid_texts, all_text = load_csv(data_path)
 
     tokenizer = tf.keras.preprocessing.text.Tokenizer(num_words=config.top_k,
-                                                      oov_token="<unk>",
-                                                      filters='!"#$%&()*+.,-/:;=?@[\]^_`{|}~ ')
+                                                      oov_token="<unk>")
 
     tokenizer.fit_on_texts(all_text)
     tokenizer.word_index['<pad>'] = 0