ray_results

ray_results/test_experiment/train_model_016f7353_1_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=1,drop_ra_2024-06-25_13-42-37/params.json

+{
+  "batch_size": 1024,
+  "decoder_int_ff": 512,
+  "decoder_int_num_layer": 1,
+  "decoder_rt_ff": 512,
+  "decoder_rt_num_layer": 1,
+  "drop_rate": 0.2,
+  "embedding_dim": 1024,
+  "encoder_ff": 512,
+  "encoder_num_layer": 1,
+  "lr": 0.0001,
+  "n_head": 1
+}
\ No newline at end of file

ray_results/test_experiment/train_model_016f7353_1_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=1,drop_ra_2024-06-25_13-42-37/progress.csv

+loss,timestamp,checkpoint_dir_name,should_checkpoint,done,training_iteration,trial_id,date,time_this_iter_s,time_total_s,pid,hostname,node_ip,time_since_restore,iterations_since_restore
+220.77590359665277,1719315958,checkpoint_000000,True,False,1,016f7353,2024-06-25_13-45-58,188.1737368106842,188.1737368106842,339920,r9i1n2,10.159.32.15,188.1737368106842,1
+180.26498923714706,1719316230,checkpoint_000001,True,False,2,016f7353,2024-06-25_13-50-30,187.0150330066681,375.1887698173523,347039,r9i1n2,10.159.32.15,187.0150330066681,1
+147.3930386821116,1719316493,checkpoint_000002,True,False,3,016f7353,2024-06-25_13-54-53,185.07583117485046,560.2646009922028,353741,r9i1n2,10.159.32.15,185.07583117485046,1
+134.21845143235575,1719316663,checkpoint_000003,True,False,4,016f7353,2024-06-25_13-57-43,170.05932211875916,730.3239231109619,353741,r9i1n2,10.159.32.15,355.1351532936096,2
+125.97545107143132,1719316969,checkpoint_000004,True,False,5,016f7353,2024-06-25_14-02-49,187.42582607269287,917.7497491836548,366320,r9i1n2,10.159.32.15,187.42582607269287,1
+119.25108247291385,1719317141,checkpoint_000005,True,False,6,016f7353,2024-06-25_14-05-41,172.101571559906,1089.8513207435608,366320,r9i1n2,10.159.32.15,359.5273976325989,2
+114.85140582707923,1719317314,checkpoint_000006,True,False,7,016f7353,2024-06-25_14-08-34,172.7140347957611,1262.565355539322,366320,r9i1n2,10.159.32.15,532.24143242836,3
+110.36972532497616,1719317486,checkpoint_000007,True,False,8,016f7353,2024-06-25_14-11-26,172.59792017936707,1435.163275718689,366320,r9i1n2,10.159.32.15,704.839352607727,4
+108.50565037764902,1719317880,checkpoint_000008,True,False,9,016f7353,2024-06-25_14-18-00,187.18606781959534,1622.3493435382843,366320,r9i1n2,10.159.32.15,187.18606781959534,1
+107.58732394721564,1719318052,checkpoint_000009,True,False,10,016f7353,2024-06-25_14-20-52,172.15624952316284,1794.5055930614471,366320,r9i1n2,10.159.32.15,359.3423173427582,2
+104.85794463871032,1719318224,checkpoint_000010,True,False,11,016f7353,2024-06-25_14-23-44,172.26081562042236,1966.7664086818695,366320,r9i1n2,10.159.32.15,531.6031329631805,3
+104.4458197045514,1719318397,checkpoint_000011,True,False,12,016f7353,2024-06-25_14-26-37,172.6046712398529,2139.3710799217224,366320,r9i1n2,10.159.32.15,704.2078042030334,4
+99.18223199318713,1719318569,checkpoint_000012,True,False,13,016f7353,2024-06-25_14-29-29,172.24528121948242,2311.616361141205,366320,r9i1n2,10.159.32.15,876.4530854225159,5
+99.00881489430826,1719318741,checkpoint_000013,True,False,14,016f7353,2024-06-25_14-32-22,172.6691517829895,2484.2855129241943,366320,r9i1n2,10.159.32.15,1049.1222372055054,6
+97.12115343349187,1719318914,checkpoint_000014,True,False,15,016f7353,2024-06-25_14-35-15,172.88193345069885,2657.167446374893,366320,r9i1n2,10.159.32.15,1222.0041706562042,7
+97.0790608623835,1719319087,checkpoint_000015,True,False,16,016f7353,2024-06-25_14-38-07,172.1524109840393,2829.3198573589325,366320,r9i1n2,10.159.32.15,1394.1565816402435,8
+97.36735979215366,1719319259,checkpoint_000016,True,False,17,016f7353,2024-06-25_14-40-59,172.8087797164917,3002.128637075424,366320,r9i1n2,10.159.32.15,1566.9653613567352,9
+94.57831074872355,1719319432,checkpoint_000017,True,False,18,016f7353,2024-06-25_14-43-52,172.65869641304016,3174.7873334884644,366320,r9i1n2,10.159.32.15,1739.6240577697754,10

ray_results/test_experiment/train_model_016f7353_1_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=1,drop_ra_2024-06-25_13-42-37/result.json

+{"loss": 220.77590359665277, "timestamp": 1719315958, "checkpoint_dir_name": "checkpoint_000000", "should_checkpoint": true, "done": false, "training_iteration": 1, "trial_id": "016f7353", "date": "2024-06-25_13-45-58", "time_this_iter_s": 188.1737368106842, "time_total_s": 188.1737368106842, "pid": 339920, "hostname": "r9i1n2", "node_ip": "10.159.32.15", "config": {"encoder_num_layer": 1, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 1024, "encoder_ff": 512, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 1, "drop_rate": 0.2, "lr": 0.0001, "batch_size": 1024}, "time_since_restore": 188.1737368106842, "iterations_since_restore": 1}
+{"loss": 147.3930386821116, "timestamp": 1719316493, "checkpoint_dir_name": "checkpoint_000002", "should_checkpoint": true, "done": false, "training_iteration": 3, "trial_id": "016f7353", "date": "2024-06-25_13-54-53", "time_this_iter_s": 185.07583117485046, "time_total_s": 560.2646009922028, "pid": 353741, "hostname": "r9i1n2", "node_ip": "10.159.32.15", "config": {"encoder_num_layer": 1, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 1024, "encoder_ff": 512, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 1, "drop_rate": 0.2, "lr": 0.0001, "batch_size": 1024}, "time_since_restore": 185.07583117485046, "iterations_since_restore": 1}
+{"loss": 125.97545107143132, "timestamp": 1719316969, "checkpoint_dir_name": "checkpoint_000004", "should_checkpoint": true, "done": false, "training_iteration": 5, "trial_id": "016f7353", "date": "2024-06-25_14-02-49", "time_this_iter_s": 187.42582607269287, "time_total_s": 917.7497491836548, "pid": 366320, "hostname": "r9i1n2", "node_ip": "10.159.32.15", "config": {"encoder_num_layer": 1, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 1024, "encoder_ff": 512, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 1, "drop_rate": 0.2, "lr": 0.0001, "batch_size": 1024}, "time_since_restore": 187.42582607269287, "iterations_since_restore": 1}
+{"loss": 119.25108247291385, "timestamp": 1719317141, "checkpoint_dir_name": "checkpoint_000005", "should_checkpoint": true, "done": false, "training_iteration": 6, "trial_id": "016f7353", "date": "2024-06-25_14-05-41", "time_this_iter_s": 172.101571559906, "time_total_s": 1089.8513207435608, "pid": 366320, "hostname": "r9i1n2", "node_ip": "10.159.32.15", "config": {"encoder_num_layer": 1, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 1024, "encoder_ff": 512, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 1, "drop_rate": 0.2, "lr": 0.0001, "batch_size": 1024}, "time_since_restore": 359.5273976325989, "iterations_since_restore": 2}
+{"loss": 114.85140582707923, "timestamp": 1719317314, "checkpoint_dir_name": "checkpoint_000006", "should_checkpoint": true, "done": false, "training_iteration": 7, "trial_id": "016f7353", "date": "2024-06-25_14-08-34", "time_this_iter_s": 172.7140347957611, "time_total_s": 1262.565355539322, "pid": 366320, "hostname": "r9i1n2", "node_ip": "10.159.32.15", "config": {"encoder_num_layer": 1, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 1024, "encoder_ff": 512, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 1, "drop_rate": 0.2, "lr": 0.0001, "batch_size": 1024}, "time_since_restore": 532.24143242836, "iterations_since_restore": 3}
+{"loss": 110.36972532497616, "timestamp": 1719317486, "checkpoint_dir_name": "checkpoint_000007", "should_checkpoint": true, "done": false, "training_iteration": 8, "trial_id": "016f7353", "date": "2024-06-25_14-11-26", "time_this_iter_s": 172.59792017936707, "time_total_s": 1435.163275718689, "pid": 366320, "hostname": "r9i1n2", "node_ip": "10.159.32.15", "config": {"encoder_num_layer": 1, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 1024, "encoder_ff": 512, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 1, "drop_rate": 0.2, "lr": 0.0001, "batch_size": 1024}, "time_since_restore": 704.839352607727, "iterations_since_restore": 4}
+{"loss": 108.50565037764902, "timestamp": 1719317880, "checkpoint_dir_name": "checkpoint_000008", "should_checkpoint": true, "done": false, "training_iteration": 9, "trial_id": "016f7353", "date": "2024-06-25_14-18-00", "time_this_iter_s": 187.18606781959534, "time_total_s": 1622.3493435382843, "pid": 366320, "hostname": "r9i1n2", "node_ip": "10.159.32.15", "config": {"encoder_num_layer": 1, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 1024, "encoder_ff": 512, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 1, "drop_rate": 0.2, "lr": 0.0001, "batch_size": 1024}, "time_since_restore": 187.18606781959534, "iterations_since_restore": 1}
+{"loss": 107.58732394721564, "timestamp": 1719318052, "checkpoint_dir_name": "checkpoint_000009", "should_checkpoint": true, "done": false, "training_iteration": 10, "trial_id": "016f7353", "date": "2024-06-25_14-20-52", "time_this_iter_s": 172.15624952316284, "time_total_s": 1794.5055930614471, "pid": 366320, "hostname": "r9i1n2", "node_ip": "10.159.32.15", "config": {"encoder_num_layer": 1, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 1024, "encoder_ff": 512, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 1, "drop_rate": 0.2, "lr": 0.0001, "batch_size": 1024}, "time_since_restore": 359.3423173427582, "iterations_since_restore": 2}
+{"loss": 104.85794463871032, "timestamp": 1719318224, "checkpoint_dir_name": "checkpoint_000010", "should_checkpoint": true, "done": false, "training_iteration": 11, "trial_id": "016f7353", "date": "2024-06-25_14-23-44", "time_this_iter_s": 172.26081562042236, "time_total_s": 1966.7664086818695, "pid": 366320, "hostname": "r9i1n2", "node_ip": "10.159.32.15", "config": {"encoder_num_layer": 1, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 1024, "encoder_ff": 512, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 1, "drop_rate": 0.2, "lr": 0.0001, "batch_size": 1024}, "time_since_restore": 531.6031329631805, "iterations_since_restore": 3}
+{"loss": 104.4458197045514, "timestamp": 1719318397, "checkpoint_dir_name": "checkpoint_000011", "should_checkpoint": true, "done": false, "training_iteration": 12, "trial_id": "016f7353", "date": "2024-06-25_14-26-37", "time_this_iter_s": 172.6046712398529, "time_total_s": 2139.3710799217224, "pid": 366320, "hostname": "r9i1n2", "node_ip": "10.159.32.15", "config": {"encoder_num_layer": 1, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 1024, "encoder_ff": 512, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 1, "drop_rate": 0.2, "lr": 0.0001, "batch_size": 1024}, "time_since_restore": 704.2078042030334, "iterations_since_restore": 4}
+{"loss": 99.18223199318713, "timestamp": 1719318569, "checkpoint_dir_name": "checkpoint_000012", "should_checkpoint": true, "done": false, "training_iteration": 13, "trial_id": "016f7353", "date": "2024-06-25_14-29-29", "time_this_iter_s": 172.24528121948242, "time_total_s": 2311.616361141205, "pid": 366320, "hostname": "r9i1n2", "node_ip": "10.159.32.15", "config": {"encoder_num_layer": 1, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 1024, "encoder_ff": 512, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 1, "drop_rate": 0.2, "lr": 0.0001, "batch_size": 1024}, "time_since_restore": 876.4530854225159, "iterations_since_restore": 5}
+{"loss": 99.00881489430826, "timestamp": 1719318741, "checkpoint_dir_name": "checkpoint_000013", "should_checkpoint": true, "done": false, "training_iteration": 14, "trial_id": "016f7353", "date": "2024-06-25_14-32-22", "time_this_iter_s": 172.6691517829895, "time_total_s": 2484.2855129241943, "pid": 366320, "hostname": "r9i1n2", "node_ip": "10.159.32.15", "config": {"encoder_num_layer": 1, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 1024, "encoder_ff": 512, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 1, "drop_rate": 0.2, "lr": 0.0001, "batch_size": 1024}, "time_since_restore": 1049.1222372055054, "iterations_since_restore": 6}
+{"loss": 97.12115343349187, "timestamp": 1719318914, "checkpoint_dir_name": "checkpoint_000014", "should_checkpoint": true, "done": false, "training_iteration": 15, "trial_id": "016f7353", "date": "2024-06-25_14-35-15", "time_this_iter_s": 172.88193345069885, "time_total_s": 2657.167446374893, "pid": 366320, "hostname": "r9i1n2", "node_ip": "10.159.32.15", "config": {"encoder_num_layer": 1, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 1024, "encoder_ff": 512, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 1, "drop_rate": 0.2, "lr": 0.0001, "batch_size": 1024}, "time_since_restore": 1222.0041706562042, "iterations_since_restore": 7}
+{"loss": 97.0790608623835, "timestamp": 1719319087, "checkpoint_dir_name": "checkpoint_000015", "should_checkpoint": true, "done": false, "training_iteration": 16, "trial_id": "016f7353", "date": "2024-06-25_14-38-07", "time_this_iter_s": 172.1524109840393, "time_total_s": 2829.3198573589325, "pid": 366320, "hostname": "r9i1n2", "node_ip": "10.159.32.15", "config": {"encoder_num_layer": 1, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 1024, "encoder_ff": 512, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 1, "drop_rate": 0.2, "lr": 0.0001, "batch_size": 1024}, "time_since_restore": 1394.1565816402435, "iterations_since_restore": 8}
+{"loss": 97.36735979215366, "timestamp": 1719319259, "checkpoint_dir_name": "checkpoint_000016", "should_checkpoint": true, "done": false, "training_iteration": 17, "trial_id": "016f7353", "date": "2024-06-25_14-40-59", "time_this_iter_s": 172.8087797164917, "time_total_s": 3002.128637075424, "pid": 366320, "hostname": "r9i1n2", "node_ip": "10.159.32.15", "config": {"encoder_num_layer": 1, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 1024, "encoder_ff": 512, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 1, "drop_rate": 0.2, "lr": 0.0001, "batch_size": 1024}, "time_since_restore": 1566.9653613567352, "iterations_since_restore": 9}
+{"loss": 94.57831074872355, "timestamp": 1719319432, "checkpoint_dir_name": "checkpoint_000017", "should_checkpoint": true, "done": false, "training_iteration": 18, "trial_id": "016f7353", "date": "2024-06-25_14-43-52", "time_this_iter_s": 172.65869641304016, "time_total_s": 3174.7873334884644, "pid": 366320, "hostname": "r9i1n2", "node_ip": "10.159.32.15", "config": {"encoder_num_layer": 1, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 1024, "encoder_ff": 512, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 1, "drop_rate": 0.2, "lr": 0.0001, "batch_size": 1024}, "time_since_restore": 1739.6240577697754, "iterations_since_restore": 10}

ray_results/test_experiment/train_model_58bc48cb_2_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=1,drop_ra_2024-06-25_13-42-50/params.json

+{
+  "batch_size": 1024,
+  "decoder_int_ff": 512,
+  "decoder_int_num_layer": 1,
+  "decoder_rt_ff": 512,
+  "decoder_rt_num_layer": 1,
+  "drop_rate": 0.2,
+  "embedding_dim": 64,
+  "encoder_ff": 512,
+  "encoder_num_layer": 1,
+  "lr": 0.0001,
+  "n_head": 1
+}
\ No newline at end of file

ray_results/test_experiment/train_model_58bc48cb_2_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=1,drop_ra_2024-06-25_13-42-50/progress.csv

+loss,timestamp,checkpoint_dir_name,should_checkpoint,done,training_iteration,trial_id,date,time_this_iter_s,time_total_s,pid,hostname,node_ip,time_since_restore,iterations_since_restore
+396.9987843430887,1719316035,checkpoint_000000,True,False,1,58bc48cb,2024-06-25_13-47-15,76.29459309577942,76.29459309577942,339920,r9i1n2,10.159.32.15,76.29459309577942,1
+298.5339005838229,1719316294,checkpoint_000001,True,False,2,58bc48cb,2024-06-25_13-51-34,63.463342905044556,139.75793600082397,347039,r9i1n2,10.159.32.15,63.463342905044556,1
+250.0197828397976,1719316725,checkpoint_000002,True,False,3,58bc48cb,2024-06-25_13-58-45,61.615302085876465,201.37323808670044,353741,r9i1n2,10.159.32.15,61.615302085876465,1
+219.07703201414094,1719316773,checkpoint_000003,True,False,4,58bc48cb,2024-06-25_13-59-33,48.90440893173218,250.27764701843262,353741,r9i1n2,10.159.32.15,110.51971101760864,2
+196.89034751832017,1719317549,checkpoint_000004,True,False,5,58bc48cb,2024-06-25_14-12-29,62.07963538169861,312.3572824001312,366320,r9i1n2,10.159.32.15,62.07963538169861,1
+177.75854870653527,1719317597,checkpoint_000005,True,False,6,58bc48cb,2024-06-25_14-13-17,48.146273374557495,360.5035557746887,366320,r9i1n2,10.159.32.15,110.2259087562561,2
+165.82053633562222,1719317644,checkpoint_000006,True,False,7,58bc48cb,2024-06-25_14-14-04,47.530285120010376,408.0338408946991,366320,r9i1n2,10.159.32.15,157.75619387626648,3
+156.16600364024245,1719317692,checkpoint_000007,True,True,8,58bc48cb,2024-06-25_14-14-52,47.988035917282104,456.0218768119812,366320,r9i1n2,10.159.32.15,205.74422979354858,4

ray_results/test_experiment/train_model_58bc48cb_2_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=1,drop_ra_2024-06-25_13-42-50/result.json

+{"loss": 396.9987843430887, "timestamp": 1719316035, "checkpoint_dir_name": "checkpoint_000000", "should_checkpoint": true, "done": false, "training_iteration": 1, "trial_id": "58bc48cb", "date": "2024-06-25_13-47-15", "time_this_iter_s": 76.29459309577942, "time_total_s": 76.29459309577942, "pid": 339920, "hostname": "r9i1n2", "node_ip": "10.159.32.15", "config": {"encoder_num_layer": 1, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 64, "encoder_ff": 512, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 1, "drop_rate": 0.2, "lr": 0.0001, "batch_size": 1024}, "time_since_restore": 76.29459309577942, "iterations_since_restore": 1}
+{"loss": 298.5339005838229, "timestamp": 1719316294, "checkpoint_dir_name": "checkpoint_000001", "should_checkpoint": true, "done": false, "training_iteration": 2, "trial_id": "58bc48cb", "date": "2024-06-25_13-51-34", "time_this_iter_s": 63.463342905044556, "time_total_s": 139.75793600082397, "pid": 347039, "hostname": "r9i1n2", "node_ip": "10.159.32.15", "config": {"encoder_num_layer": 1, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 64, "encoder_ff": 512, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 1, "drop_rate": 0.2, "lr": 0.0001, "batch_size": 1024}, "time_since_restore": 63.463342905044556, "iterations_since_restore": 1}
+{"loss": 250.0197828397976, "timestamp": 1719316725, "checkpoint_dir_name": "checkpoint_000002", "should_checkpoint": true, "done": false, "training_iteration": 3, "trial_id": "58bc48cb", "date": "2024-06-25_13-58-45", "time_this_iter_s": 61.615302085876465, "time_total_s": 201.37323808670044, "pid": 353741, "hostname": "r9i1n2", "node_ip": "10.159.32.15", "config": {"encoder_num_layer": 1, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 64, "encoder_ff": 512, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 1, "drop_rate": 0.2, "lr": 0.0001, "batch_size": 1024}, "time_since_restore": 61.615302085876465, "iterations_since_restore": 1}
+{"loss": 219.07703201414094, "timestamp": 1719316773, "checkpoint_dir_name": "checkpoint_000003", "should_checkpoint": true, "done": false, "training_iteration": 4, "trial_id": "58bc48cb", "date": "2024-06-25_13-59-33", "time_this_iter_s": 48.90440893173218, "time_total_s": 250.27764701843262, "pid": 353741, "hostname": "r9i1n2", "node_ip": "10.159.32.15", "config": {"encoder_num_layer": 1, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 64, "encoder_ff": 512, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 1, "drop_rate": 0.2, "lr": 0.0001, "batch_size": 1024}, "time_since_restore": 110.51971101760864, "iterations_since_restore": 2}
+{"loss": 196.89034751832017, "timestamp": 1719317549, "checkpoint_dir_name": "checkpoint_000004", "should_checkpoint": true, "done": false, "training_iteration": 5, "trial_id": "58bc48cb", "date": "2024-06-25_14-12-29", "time_this_iter_s": 62.07963538169861, "time_total_s": 312.3572824001312, "pid": 366320, "hostname": "r9i1n2", "node_ip": "10.159.32.15", "config": {"encoder_num_layer": 1, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 64, "encoder_ff": 512, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 1, "drop_rate": 0.2, "lr": 0.0001, "batch_size": 1024}, "time_since_restore": 62.07963538169861, "iterations_since_restore": 1}
+{"loss": 177.75854870653527, "timestamp": 1719317597, "checkpoint_dir_name": "checkpoint_000005", "should_checkpoint": true, "done": false, "training_iteration": 6, "trial_id": "58bc48cb", "date": "2024-06-25_14-13-17", "time_this_iter_s": 48.146273374557495, "time_total_s": 360.5035557746887, "pid": 366320, "hostname": "r9i1n2", "node_ip": "10.159.32.15", "config": {"encoder_num_layer": 1, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 64, "encoder_ff": 512, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 1, "drop_rate": 0.2, "lr": 0.0001, "batch_size": 1024}, "time_since_restore": 110.2259087562561, "iterations_since_restore": 2}
+{"loss": 165.82053633562222, "timestamp": 1719317644, "checkpoint_dir_name": "checkpoint_000006", "should_checkpoint": true, "done": false, "training_iteration": 7, "trial_id": "58bc48cb", "date": "2024-06-25_14-14-04", "time_this_iter_s": 47.530285120010376, "time_total_s": 408.0338408946991, "pid": 366320, "hostname": "r9i1n2", "node_ip": "10.159.32.15", "config": {"encoder_num_layer": 1, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 64, "encoder_ff": 512, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 1, "drop_rate": 0.2, "lr": 0.0001, "batch_size": 1024}, "time_since_restore": 157.75619387626648, "iterations_since_restore": 3}
+{"loss": 156.16600364024245, "timestamp": 1719317692, "checkpoint_dir_name": "checkpoint_000007", "should_checkpoint": true, "done": true, "training_iteration": 8, "trial_id": "58bc48cb", "date": "2024-06-25_14-14-52", "time_this_iter_s": 47.988035917282104, "time_total_s": 456.0218768119812, "pid": 366320, "hostname": "r9i1n2", "node_ip": "10.159.32.15", "config": {"encoder_num_layer": 1, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 64, "encoder_ff": 512, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 1, "drop_rate": 0.2, "lr": 0.0001, "batch_size": 1024}, "time_since_restore": 205.74422979354858, "iterations_since_restore": 4}