End of training

README.md

@@ -18,11 +18,11 @@ should probably proofread and complete it, then remove this comment. -->
 
 This model is a fine-tuned version of [riotu-lab/ArabianGPT-01B](https://huggingface.co/riotu-lab/ArabianGPT-01B) on an unknown dataset.
 It achieves the following results on the evaluation set:
-- Loss: 1.1581
-- Bleu: 0.3740
-- Rouge1: 0.5622
-- Rouge2: 0.3102
-- Rougel: 0.5600
+- Loss: 1.0817
+- Bleu: 0.3500
+- Rouge1: 0.5071
+- Rouge2: 0.2486
+- Rougel: 0.5048
 
 ## Model description
 

all_results.json

@@ -1,19 +1,19 @@
 {
-    "epoch": 13.0,
-    "eval_bleu": 0.23046108076912605,
-    "eval_loss": 0.5856799483299255,
-    "eval_rouge1": 0.552383162149263,
-    "eval_rouge2": 0.28740939784429775,
-    "eval_rougeL": 0.5497007000948397,
-    "eval_runtime": 7.0212,
-    "eval_samples": 2117,
-    "eval_samples_per_second": 301.517,
-    "eval_steps_per_second": 37.743,
-    "perplexity": 1.796211901795729,
-    "total_flos": 7192716410880000.0,
-    "train_loss": 0.055393123952289666,
-    "train_runtime": 1209.9717,
-    "train_samples": 8470,
-    "train_samples_per_second": 140.003,
-    "train_steps_per_second": 17.505
+    "epoch": 7.0,
+    "eval_bleu": 0.34996698643265867,
+    "eval_loss": 1.0816909074783325,
+    "eval_rouge1": 0.5070888923629823,
+    "eval_rouge2": 0.24860353236577704,
+    "eval_rougeL": 0.5048066808619132,
+    "eval_runtime": 7.2625,
+    "eval_samples": 2113,
+    "eval_samples_per_second": 290.948,
+    "eval_steps_per_second": 36.489,
+    "perplexity": 2.9496629437803548,
+    "total_flos": 3864770445312000.0,
+    "train_loss": 0.5907337587255773,
+    "train_runtime": 1774.4968,
+    "train_samples": 8452,
+    "train_samples_per_second": 95.261,
+    "train_steps_per_second": 11.913
 }

egy_training_log.txt

@@ -158,3 +158,5 @@ INFO:root:Epoch 6.0: Train Loss = 0.3357, Eval Loss = 1.1360507011413574
 INFO:absl:Using default tokenizer.
 INFO:root:Epoch 7.0: Train Loss = 0.2832, Eval Loss = 1.1256883144378662
 INFO:absl:Using default tokenizer.
+INFO:__main__:*** Evaluate ***
+INFO:absl:Using default tokenizer.

eval_results.json

@@ -1,13 +1,13 @@
 {
-    "epoch": 13.0,
-    "eval_bleu": 0.23046108076912605,
-    "eval_loss": 0.5856799483299255,
-    "eval_rouge1": 0.552383162149263,
-    "eval_rouge2": 0.28740939784429775,
-    "eval_rougeL": 0.5497007000948397,
-    "eval_runtime": 7.0212,
-    "eval_samples": 2117,
-    "eval_samples_per_second": 301.517,
-    "eval_steps_per_second": 37.743,
-    "perplexity": 1.796211901795729
+    "epoch": 7.0,
+    "eval_bleu": 0.34996698643265867,
+    "eval_loss": 1.0816909074783325,
+    "eval_rouge1": 0.5070888923629823,
+    "eval_rouge2": 0.24860353236577704,
+    "eval_rougeL": 0.5048066808619132,
+    "eval_runtime": 7.2625,
+    "eval_samples": 2113,
+    "eval_samples_per_second": 290.948,
+    "eval_steps_per_second": 36.489,
+    "perplexity": 2.9496629437803548
 }

train_results.json

@@ -1,9 +1,9 @@
 {
-    "epoch": 13.0,
-    "total_flos": 7192716410880000.0,
-    "train_loss": 0.055393123952289666,
-    "train_runtime": 1209.9717,
-    "train_samples": 8470,
-    "train_samples_per_second": 140.003,
-    "train_steps_per_second": 17.505
+    "epoch": 7.0,
+    "total_flos": 3864770445312000.0,
+    "train_loss": 0.5907337587255773,
+    "train_runtime": 1774.4968,
+    "train_samples": 8452,
+    "train_samples_per_second": 95.261,
+    "train_steps_per_second": 11.913
 }

trainer_state.json

@@ -1,272 +1,158 @@
 {
-  "best_metric": 0.5856799483299255,
-  "best_model_checkpoint": "/home/iais_marenpielka/Bouthaina/res_nw_irq/checkpoint-3177",
-  "epoch": 13.0,
+  "best_metric": 1.0816909074783325,
+  "best_model_checkpoint": "/home/iais_marenpielka/Bouthaina/res_nw_irq/checkpoint-2114",
+  "epoch": 7.0,
   "eval_steps": 500,
-  "global_step": 13767,
+  "global_step": 7399,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
       "epoch": 1.0,
-      "grad_norm": 0.9141271114349365,
-      "learning_rate": 4.864845261121857e-05,
-      "loss": 1.8421,
-      "step": 1059
+      "grad_norm": 1.3965729475021362,
+      "learning_rate": 4.865067829457365e-05,
+      "loss": 1.7995,
+      "step": 1057
     },
     {
       "epoch": 1.0,
-      "eval_bleu": 0.19309252278222033,
-      "eval_loss": 0.6647776365280151,
-      "eval_rouge1": 0.48057112484546394,
-      "eval_rouge2": 0.21418316037711305,
-      "eval_rougeL": 0.4779283146059673,
-      "eval_runtime": 7.1769,
-      "eval_samples_per_second": 294.974,
-      "eval_steps_per_second": 36.924,
-      "step": 1059
+      "eval_bleu": 0.3157163520866108,
+      "eval_loss": 1.1331889629364014,
+      "eval_rouge1": 0.45516049357610144,
+      "eval_rouge2": 0.2017132883532014,
+      "eval_rougeL": 0.453144508008762,
+      "eval_runtime": 20.354,
+      "eval_samples_per_second": 103.812,
+      "eval_steps_per_second": 13.02,
+      "step": 1057
     },
     {
       "epoch": 2.0,
-      "grad_norm": 1.218380093574524,
-      "learning_rate": 4.608800773694391e-05,
-      "loss": 0.591,
-      "step": 2118
+      "grad_norm": 1.2999993562698364,
+      "learning_rate": 4.609011627906977e-05,
+      "loss": 0.5906,
+      "step": 2114
     },
     {
       "epoch": 2.0,
-      "eval_bleu": 0.22252011703416463,
-      "eval_loss": 0.6045418977737427,
-      "eval_rouge1": 0.5287371210505303,
-      "eval_rouge2": 0.2614938115780706,
-      "eval_rougeL": 0.5264748671435004,
-      "eval_runtime": 9.5179,
-      "eval_samples_per_second": 222.424,
-      "eval_steps_per_second": 27.842,
-      "step": 2118
+      "eval_bleu": 0.34996698643265867,
+      "eval_loss": 1.0816909074783325,
+      "eval_rouge1": 0.5070888923629823,
+      "eval_rouge2": 0.24860353236577704,
+      "eval_rougeL": 0.5048066808619132,
+      "eval_runtime": 18.5352,
+      "eval_samples_per_second": 113.999,
+      "eval_steps_per_second": 14.297,
+      "step": 2114
     },
     {
       "epoch": 3.0,
-      "grad_norm": 1.0960309505462646,
-      "learning_rate": 4.3527562862669246e-05,
-      "loss": 0.4845,
-      "step": 3177
+      "grad_norm": 1.494644284248352,
+      "learning_rate": 4.3529554263565894e-05,
+      "loss": 0.484,
+      "step": 3171
     },
     {
       "epoch": 3.0,
-      "eval_bleu": 0.23046108076912605,
-      "eval_loss": 0.5856799483299255,
-      "eval_rouge1": 0.552383162149263,
-      "eval_rouge2": 0.28740939784429775,
-      "eval_rougeL": 0.5497007000948397,
-      "eval_runtime": 10.8015,
-      "eval_samples_per_second": 195.992,
-      "eval_steps_per_second": 24.534,
-      "step": 3177
+      "eval_bleu": 0.3592334262176334,
+      "eval_loss": 1.1038055419921875,
+      "eval_rouge1": 0.5329660042734197,
+      "eval_rouge2": 0.2737288088550716,
+      "eval_rougeL": 0.530301732660456,
+      "eval_runtime": 11.0163,
+      "eval_samples_per_second": 191.807,
+      "eval_steps_per_second": 24.055,
+      "step": 3171
     },
     {
       "epoch": 4.0,
-      "grad_norm": 1.3864747285842896,
-      "learning_rate": 4.096711798839459e-05,
-      "loss": 0.4018,
-      "step": 4236
+      "grad_norm": 2.169060230255127,
+      "learning_rate": 4.096899224806201e-05,
+      "loss": 0.4017,
+      "step": 4228
     },
     {
       "epoch": 4.0,
-      "eval_bleu": 0.2415877566741423,
-      "eval_loss": 0.5871986746788025,
-      "eval_rouge1": 0.5715504611231429,
-      "eval_rouge2": 0.3055462547357787,
-      "eval_rougeL": 0.5693046274769865,
-      "eval_runtime": 13.9766,
-      "eval_samples_per_second": 151.468,
-      "eval_steps_per_second": 18.96,
-      "step": 4236
+      "eval_bleu": 0.3664876357280857,
+      "eval_loss": 1.1117204427719116,
+      "eval_rouge1": 0.5470701344699284,
+      "eval_rouge2": 0.29154846104100973,
+      "eval_rougeL": 0.5448816975696349,
+      "eval_runtime": 11.4837,
+      "eval_samples_per_second": 184.001,
+      "eval_steps_per_second": 23.076,
+      "step": 4228
     },
     {
       "epoch": 5.0,
-      "grad_norm": 1.157429575920105,
-      "learning_rate": 3.840667311411992e-05,
-      "loss": 0.3351,
-      "step": 5295
+      "grad_norm": 1.5656139850616455,
+      "learning_rate": 3.840843023255814e-05,
+      "loss": 0.3357,
+      "step": 5285
     },
     {
       "epoch": 5.0,
-      "eval_bleu": 0.24406004584501667,
-      "eval_loss": 0.5902981162071228,
-      "eval_rouge1": 0.5779981254861084,
-      "eval_rouge2": 0.317524337500259,
-      "eval_rougeL": 0.5758829591426086,
-      "eval_runtime": 9.689,
-      "eval_samples_per_second": 218.496,
-      "eval_steps_per_second": 27.351,
-      "step": 5295
+      "eval_bleu": 0.36951226807058296,
+      "eval_loss": 1.1360507011413574,
+      "eval_rouge1": 0.5552416771197572,
+      "eval_rouge2": 0.300977455987009,
+      "eval_rougeL": 0.5533455418855034,
+      "eval_runtime": 7.2292,
+      "eval_samples_per_second": 292.287,
+      "eval_steps_per_second": 36.657,
+      "step": 5285
     },
     {
       "epoch": 6.0,
-      "grad_norm": 1.5503590106964111,
-      "learning_rate": 3.5846228239845266e-05,
-      "loss": 0.2824,
-      "step": 6354
+      "grad_norm": 1.9493229389190674,
+      "learning_rate": 3.5847868217054265e-05,
+      "loss": 0.2832,
+      "step": 6342
     },
     {
       "epoch": 6.0,
-      "eval_bleu": 0.25233205020508037,
-      "eval_loss": 0.5987895131111145,
-      "eval_rouge1": 0.5857067061237097,
-      "eval_rouge2": 0.3285041572893456,
-      "eval_rougeL": 0.5837783686526783,
-      "eval_runtime": 49.8895,
-      "eval_samples_per_second": 42.434,
-      "eval_steps_per_second": 5.312,
-      "step": 6354
+      "eval_bleu": 0.37209002173126987,
+      "eval_loss": 1.1256883144378662,
+      "eval_rouge1": 0.5606510219414631,
+      "eval_rouge2": 0.3089222128154705,
+      "eval_rougeL": 0.5584217258440263,
+      "eval_runtime": 7.0098,
+      "eval_samples_per_second": 301.433,
+      "eval_steps_per_second": 37.804,
+      "step": 6342
     },
     {
       "epoch": 7.0,
-      "grad_norm": 1.2503788471221924,
-      "learning_rate": 3.3285783365570597e-05,
-      "loss": 0.2395,
-      "step": 7413
+      "grad_norm": 1.3939344882965088,
+      "learning_rate": 3.328730620155039e-05,
+      "loss": 0.2404,
+      "step": 7399
     },
     {
       "epoch": 7.0,
-      "eval_bleu": 0.25714921193093904,
-      "eval_loss": 0.6094574332237244,
-      "eval_rouge1": 0.5876661421958642,
-      "eval_rouge2": 0.33302273991869746,
-      "eval_rougeL": 0.5860531719581312,
-      "eval_runtime": 23.7105,
-      "eval_samples_per_second": 89.285,
-      "eval_steps_per_second": 11.176,
-      "step": 7413
+      "eval_bleu": 0.37399333863753975,
+      "eval_loss": 1.1580852270126343,
+      "eval_rouge1": 0.5622192023319419,
+      "eval_rouge2": 0.3102334124697639,
+      "eval_rougeL": 0.5600347361241877,
+      "eval_runtime": 7.2059,
+      "eval_samples_per_second": 293.233,
+      "eval_steps_per_second": 36.776,
+      "step": 7399
     },
     {
-      "epoch": 8.0,
-      "grad_norm": 1.2774244546890259,
-      "learning_rate": 3.072533849129594e-05,
-      "loss": 0.206,
-      "step": 8472
-    },
-    {
-      "epoch": 8.0,
-      "eval_bleu": 0.25980262957911116,
-      "eval_loss": 0.6177937388420105,
-      "eval_rouge1": 0.5921019219570411,
-      "eval_rouge2": 0.33750981785667133,
-      "eval_rougeL": 0.5897405302098737,
-      "eval_runtime": 8.9943,
-      "eval_samples_per_second": 235.37,
-      "eval_steps_per_second": 29.463,
-      "step": 8472
-    },
-    {
-      "epoch": 9.0,
-      "grad_norm": 1.7185566425323486,
-      "learning_rate": 2.8164893617021275e-05,
-      "loss": 0.1782,
-      "step": 9531
-    },
-    {
-      "epoch": 9.0,
-      "eval_bleu": 0.2614863236371811,
-      "eval_loss": 0.629348874092102,
-      "eval_rouge1": 0.5946629982518206,
-      "eval_rouge2": 0.3436011935268504,
-      "eval_rougeL": 0.5923928551586273,
-      "eval_runtime": 7.5598,
-      "eval_samples_per_second": 280.035,
-      "eval_steps_per_second": 35.054,
-      "step": 9531
-    },
-    {
-      "epoch": 10.0,
-      "grad_norm": 1.2046067714691162,
-      "learning_rate": 2.5604448742746616e-05,
-      "loss": 0.1569,
-      "step": 10590
-    },
-    {
-      "epoch": 10.0,
-      "eval_bleu": 0.2622683837036058,
-      "eval_loss": 0.6383969187736511,
-      "eval_rouge1": 0.5956308067865153,
-      "eval_rouge2": 0.3446481345143322,
-      "eval_rougeL": 0.5936155042273001,
-      "eval_runtime": 6.9925,
-      "eval_samples_per_second": 302.752,
-      "eval_steps_per_second": 37.898,
-      "step": 10590
-    },
-    {
-      "epoch": 11.0,
-      "grad_norm": 1.339342474937439,
-      "learning_rate": 2.3044003868471954e-05,
-      "loss": 0.14,
-      "step": 11649
-    },
-    {
-      "epoch": 11.0,
-      "eval_bleu": 0.26262679811001677,
-      "eval_loss": 0.6476752161979675,
-      "eval_rouge1": 0.5934893482225323,
-      "eval_rouge2": 0.34369593409547294,
-      "eval_rougeL": 0.5911903400030549,
-      "eval_runtime": 20.541,
-      "eval_samples_per_second": 103.062,
-      "eval_steps_per_second": 12.901,
-      "step": 11649
-    },
-    {
-      "epoch": 12.0,
-      "grad_norm": 0.8063332438468933,
-      "learning_rate": 2.0483558994197295e-05,
-      "loss": 0.1273,
-      "step": 12708
-    },
-    {
-      "epoch": 12.0,
-      "eval_bleu": 0.26337346373364584,
-      "eval_loss": 0.6561474800109863,
-      "eval_rouge1": 0.5959828528791276,
-      "eval_rouge2": 0.34512909829817506,
-      "eval_rougeL": 0.5936516463492448,
-      "eval_runtime": 7.0944,
-      "eval_samples_per_second": 298.404,
-      "eval_steps_per_second": 37.353,
-      "step": 12708
-    },
-    {
-      "epoch": 13.0,
-      "grad_norm": 1.136061668395996,
-      "learning_rate": 1.7923114119922633e-05,
-      "loss": 0.1177,
-      "step": 13767
-    },
-    {
-      "epoch": 13.0,
-      "eval_bleu": 0.2637881399114867,
-      "eval_loss": 0.6630375385284424,
-      "eval_rouge1": 0.5976052885688896,
-      "eval_rouge2": 0.3469216711027344,
-      "eval_rougeL": 0.5951145249152312,
-      "eval_runtime": 7.126,
-      "eval_samples_per_second": 297.08,
-      "eval_steps_per_second": 37.188,
-      "step": 13767
-    },
-    {
-      "epoch": 13.0,
-      "step": 13767,
-      "total_flos": 7192716410880000.0,
-      "train_loss": 0.055393123952289666,
-      "train_runtime": 1209.9717,
-      "train_samples_per_second": 140.003,
-      "train_steps_per_second": 17.505
+      "epoch": 7.0,
+      "step": 7399,
+      "total_flos": 3864770445312000.0,
+      "train_loss": 0.5907337587255773,
+      "train_runtime": 1774.4968,
+      "train_samples_per_second": 95.261,
+      "train_steps_per_second": 11.913
     }
   ],
   "logging_steps": 500,
-  "max_steps": 21180,
+  "max_steps": 21140,
   "num_input_tokens_seen": 0,
   "num_train_epochs": 20,
   "save_steps": 500,
@@ -291,7 +177,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 7192716410880000.0,
+  "total_flos": 3864770445312000.0,
   "train_batch_size": 8,
   "trial_name": null,
   "trial_params": null