res

ray_results/test_experiment/train_model_c04d8657_16_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-27_15-02-12/params.json

+{
+  "batch_size": 1024,
+  "decoder_int_ff": 512,
+  "decoder_int_num_layer": 1,
+  "decoder_rt_ff": 512,
+  "decoder_rt_num_layer": 2,
+  "drop_rate": 0.37340656523341487,
+  "embedding_dim": 64,
+  "encoder_ff": 512,
+  "encoder_num_layer": 1,
+  "lr": 0.0007457945749126396,
+  "n_head": 16
+}
\ No newline at end of file

ray_results/test_experiment/train_model_c04d8657_16_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-27_15-02-12/progress.csv

+loss,timestamp,checkpoint_dir_name,should_checkpoint,done,training_iteration,trial_id,date,time_this_iter_s,time_total_s,pid,hostname,node_ip,time_since_restore,iterations_since_restore
+248.8490186075526,1719493632,checkpoint_000000,True,False,1,c04d8657,2024-06-27_15-07-12,69.30736112594604,69.30736112594604,69992,r8i6n8,10.159.28.66,69.30736112594604,1
+176.55898278904712,1719494854,checkpoint_000001,True,False,2,c04d8657,2024-06-27_15-27-34,70.9636754989624,140.27103662490845,69992,r8i6n8,10.159.28.66,70.9636754989624,1
+153.5841199018824,1719494911,checkpoint_000002,True,False,3,c04d8657,2024-06-27_15-28-31,56.80309820175171,197.07413482666016,69992,r8i6n8,10.159.28.66,127.76677370071411,2
+141.87842869195413,1719494968,checkpoint_000003,True,False,4,c04d8657,2024-06-27_15-29-28,56.76813840866089,253.84227323532104,69992,r8i6n8,10.159.28.66,184.534912109375,3
+132.76247937660517,1719496213,checkpoint_000004,True,False,5,c04d8657,2024-06-27_15-50-13,70.93559050559998,324.777863740921,69992,r8i6n8,10.159.28.66,70.93559050559998,1
+127.65858931053342,1719496269,checkpoint_000005,True,False,6,c04d8657,2024-06-27_15-51-09,56.18658232688904,380.96444606781006,69992,r8i6n8,10.159.28.66,127.12217283248901,2
+128.54880821047806,1719496325,checkpoint_000006,True,False,7,c04d8657,2024-06-27_15-52-05,55.70376706123352,436.6682131290436,69992,r8i6n8,10.159.28.66,182.82593989372253,3
+121.30412758053757,1719496382,checkpoint_000007,True,False,8,c04d8657,2024-06-27_15-53-02,56.52959203720093,493.1978051662445,69992,r8i6n8,10.159.28.66,239.35553193092346,4
+124.07332433865766,1719496438,checkpoint_000008,True,False,9,c04d8657,2024-06-27_15-53-58,56.19808506965637,549.3958902359009,69992,r8i6n8,10.159.28.66,295.55361700057983,5
+115.79139270932656,1719496495,checkpoint_000009,True,False,10,c04d8657,2024-06-27_15-54-55,57.11151361465454,606.5074038505554,69992,r8i6n8,10.159.28.66,352.6651306152344,6
+113.53236323078787,1719496551,checkpoint_000010,True,False,11,c04d8657,2024-06-27_15-55-51,56.11752891540527,662.6249327659607,69992,r8i6n8,10.159.28.66,408.78265953063965,7
+112.30559909249854,1719496607,checkpoint_000011,True,False,12,c04d8657,2024-06-27_15-56-47,55.55129075050354,718.1762235164642,69992,r8i6n8,10.159.28.66,464.3339502811432,8
+113.81121531809409,1719496663,checkpoint_000012,True,False,13,c04d8657,2024-06-27_15-57-43,56.53476905822754,774.7109925746918,69992,r8i6n8,10.159.28.66,520.8687193393707,9
+110.44758017232098,1719496719,checkpoint_000013,True,False,14,c04d8657,2024-06-27_15-58-39,56.147799015045166,830.8587915897369,69992,r8i6n8,10.159.28.66,577.0165183544159,10

ray_results/test_experiment/train_model_c04d8657_16_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-27_15-02-12/result.json

+{"loss": 248.8490186075526, "timestamp": 1719493632, "checkpoint_dir_name": "checkpoint_000000", "should_checkpoint": true, "done": false, "training_iteration": 1, "trial_id": "c04d8657", "date": "2024-06-27_15-07-12", "time_this_iter_s": 69.30736112594604, "time_total_s": 69.30736112594604, "pid": 69992, "hostname": "r8i6n8", "node_ip": "10.159.28.66", "config": {"encoder_num_layer": 1, "decoder_rt_num_layer": 2, "decoder_int_num_layer": 1, "embedding_dim": 64, "encoder_ff": 512, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 16, "drop_rate": 0.37340656523341487, "lr": 0.0007457945749126396, "batch_size": 1024}, "time_since_restore": 69.30736112594604, "iterations_since_restore": 1}
+{"loss": 176.55898278904712, "timestamp": 1719494854, "checkpoint_dir_name": "checkpoint_000001", "should_checkpoint": true, "done": false, "training_iteration": 2, "trial_id": "c04d8657", "date": "2024-06-27_15-27-34", "time_this_iter_s": 70.9636754989624, "time_total_s": 140.27103662490845, "pid": 69992, "hostname": "r8i6n8", "node_ip": "10.159.28.66", "config": {"encoder_num_layer": 1, "decoder_rt_num_layer": 2, "decoder_int_num_layer": 1, "embedding_dim": 64, "encoder_ff": 512, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 16, "drop_rate": 0.37340656523341487, "lr": 0.0007457945749126396, "batch_size": 1024}, "time_since_restore": 70.9636754989624, "iterations_since_restore": 1}
+{"loss": 153.5841199018824, "timestamp": 1719494911, "checkpoint_dir_name": "checkpoint_000002", "should_checkpoint": true, "done": false, "training_iteration": 3, "trial_id": "c04d8657", "date": "2024-06-27_15-28-31", "time_this_iter_s": 56.80309820175171, "time_total_s": 197.07413482666016, "pid": 69992, "hostname": "r8i6n8", "node_ip": "10.159.28.66", "config": {"encoder_num_layer": 1, "decoder_rt_num_layer": 2, "decoder_int_num_layer": 1, "embedding_dim": 64, "encoder_ff": 512, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 16, "drop_rate": 0.37340656523341487, "lr": 0.0007457945749126396, "batch_size": 1024}, "time_since_restore": 127.76677370071411, "iterations_since_restore": 2}
+{"loss": 141.87842869195413, "timestamp": 1719494968, "checkpoint_dir_name": "checkpoint_000003", "should_checkpoint": true, "done": false, "training_iteration": 4, "trial_id": "c04d8657", "date": "2024-06-27_15-29-28", "time_this_iter_s": 56.76813840866089, "time_total_s": 253.84227323532104, "pid": 69992, "hostname": "r8i6n8", "node_ip": "10.159.28.66", "config": {"encoder_num_layer": 1, "decoder_rt_num_layer": 2, "decoder_int_num_layer": 1, "embedding_dim": 64, "encoder_ff": 512, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 16, "drop_rate": 0.37340656523341487, "lr": 0.0007457945749126396, "batch_size": 1024}, "time_since_restore": 184.534912109375, "iterations_since_restore": 3}
+{"loss": 132.76247937660517, "timestamp": 1719496213, "checkpoint_dir_name": "checkpoint_000004", "should_checkpoint": true, "done": false, "training_iteration": 5, "trial_id": "c04d8657", "date": "2024-06-27_15-50-13", "time_this_iter_s": 70.93559050559998, "time_total_s": 324.777863740921, "pid": 69992, "hostname": "r8i6n8", "node_ip": "10.159.28.66", "config": {"encoder_num_layer": 1, "decoder_rt_num_layer": 2, "decoder_int_num_layer": 1, "embedding_dim": 64, "encoder_ff": 512, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 16, "drop_rate": 0.37340656523341487, "lr": 0.0007457945749126396, "batch_size": 1024}, "time_since_restore": 70.93559050559998, "iterations_since_restore": 1}
+{"loss": 127.65858931053342, "timestamp": 1719496269, "checkpoint_dir_name": "checkpoint_000005", "should_checkpoint": true, "done": false, "training_iteration": 6, "trial_id": "c04d8657", "date": "2024-06-27_15-51-09", "time_this_iter_s": 56.18658232688904, "time_total_s": 380.96444606781006, "pid": 69992, "hostname": "r8i6n8", "node_ip": "10.159.28.66", "config": {"encoder_num_layer": 1, "decoder_rt_num_layer": 2, "decoder_int_num_layer": 1, "embedding_dim": 64, "encoder_ff": 512, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 16, "drop_rate": 0.37340656523341487, "lr": 0.0007457945749126396, "batch_size": 1024}, "time_since_restore": 127.12217283248901, "iterations_since_restore": 2}
+{"loss": 128.54880821047806, "timestamp": 1719496325, "checkpoint_dir_name": "checkpoint_000006", "should_checkpoint": true, "done": false, "training_iteration": 7, "trial_id": "c04d8657", "date": "2024-06-27_15-52-05", "time_this_iter_s": 55.70376706123352, "time_total_s": 436.6682131290436, "pid": 69992, "hostname": "r8i6n8", "node_ip": "10.159.28.66", "config": {"encoder_num_layer": 1, "decoder_rt_num_layer": 2, "decoder_int_num_layer": 1, "embedding_dim": 64, "encoder_ff": 512, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 16, "drop_rate": 0.37340656523341487, "lr": 0.0007457945749126396, "batch_size": 1024}, "time_since_restore": 182.82593989372253, "iterations_since_restore": 3}
+{"loss": 121.30412758053757, "timestamp": 1719496382, "checkpoint_dir_name": "checkpoint_000007", "should_checkpoint": true, "done": false, "training_iteration": 8, "trial_id": "c04d8657", "date": "2024-06-27_15-53-02", "time_this_iter_s": 56.52959203720093, "time_total_s": 493.1978051662445, "pid": 69992, "hostname": "r8i6n8", "node_ip": "10.159.28.66", "config": {"encoder_num_layer": 1, "decoder_rt_num_layer": 2, "decoder_int_num_layer": 1, "embedding_dim": 64, "encoder_ff": 512, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 16, "drop_rate": 0.37340656523341487, "lr": 0.0007457945749126396, "batch_size": 1024}, "time_since_restore": 239.35553193092346, "iterations_since_restore": 4}
+{"loss": 124.07332433865766, "timestamp": 1719496438, "checkpoint_dir_name": "checkpoint_000008", "should_checkpoint": true, "done": false, "training_iteration": 9, "trial_id": "c04d8657", "date": "2024-06-27_15-53-58", "time_this_iter_s": 56.19808506965637, "time_total_s": 549.3958902359009, "pid": 69992, "hostname": "r8i6n8", "node_ip": "10.159.28.66", "config": {"encoder_num_layer": 1, "decoder_rt_num_layer": 2, "decoder_int_num_layer": 1, "embedding_dim": 64, "encoder_ff": 512, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 16, "drop_rate": 0.37340656523341487, "lr": 0.0007457945749126396, "batch_size": 1024}, "time_since_restore": 295.55361700057983, "iterations_since_restore": 5}
+{"loss": 115.79139270932656, "timestamp": 1719496495, "checkpoint_dir_name": "checkpoint_000009", "should_checkpoint": true, "done": false, "training_iteration": 10, "trial_id": "c04d8657", "date": "2024-06-27_15-54-55", "time_this_iter_s": 57.11151361465454, "time_total_s": 606.5074038505554, "pid": 69992, "hostname": "r8i6n8", "node_ip": "10.159.28.66", "config": {"encoder_num_layer": 1, "decoder_rt_num_layer": 2, "decoder_int_num_layer": 1, "embedding_dim": 64, "encoder_ff": 512, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 16, "drop_rate": 0.37340656523341487, "lr": 0.0007457945749126396, "batch_size": 1024}, "time_since_restore": 352.6651306152344, "iterations_since_restore": 6}
+{"loss": 113.53236323078787, "timestamp": 1719496551, "checkpoint_dir_name": "checkpoint_000010", "should_checkpoint": true, "done": false, "training_iteration": 11, "trial_id": "c04d8657", "date": "2024-06-27_15-55-51", "time_this_iter_s": 56.11752891540527, "time_total_s": 662.6249327659607, "pid": 69992, "hostname": "r8i6n8", "node_ip": "10.159.28.66", "config": {"encoder_num_layer": 1, "decoder_rt_num_layer": 2, "decoder_int_num_layer": 1, "embedding_dim": 64, "encoder_ff": 512, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 16, "drop_rate": 0.37340656523341487, "lr": 0.0007457945749126396, "batch_size": 1024}, "time_since_restore": 408.78265953063965, "iterations_since_restore": 7}
+{"loss": 112.30559909249854, "timestamp": 1719496607, "checkpoint_dir_name": "checkpoint_000011", "should_checkpoint": true, "done": false, "training_iteration": 12, "trial_id": "c04d8657", "date": "2024-06-27_15-56-47", "time_this_iter_s": 55.55129075050354, "time_total_s": 718.1762235164642, "pid": 69992, "hostname": "r8i6n8", "node_ip": "10.159.28.66", "config": {"encoder_num_layer": 1, "decoder_rt_num_layer": 2, "decoder_int_num_layer": 1, "embedding_dim": 64, "encoder_ff": 512, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 16, "drop_rate": 0.37340656523341487, "lr": 0.0007457945749126396, "batch_size": 1024}, "time_since_restore": 464.3339502811432, "iterations_since_restore": 8}
+{"loss": 113.81121531809409, "timestamp": 1719496663, "checkpoint_dir_name": "checkpoint_000012", "should_checkpoint": true, "done": false, "training_iteration": 13, "trial_id": "c04d8657", "date": "2024-06-27_15-57-43", "time_this_iter_s": 56.53476905822754, "time_total_s": 774.7109925746918, "pid": 69992, "hostname": "r8i6n8", "node_ip": "10.159.28.66", "config": {"encoder_num_layer": 1, "decoder_rt_num_layer": 2, "decoder_int_num_layer": 1, "embedding_dim": 64, "encoder_ff": 512, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 16, "drop_rate": 0.37340656523341487, "lr": 0.0007457945749126396, "batch_size": 1024}, "time_since_restore": 520.8687193393707, "iterations_since_restore": 9}
+{"loss": 110.44758017232098, "timestamp": 1719496719, "checkpoint_dir_name": "checkpoint_000013", "should_checkpoint": true, "done": false, "training_iteration": 14, "trial_id": "c04d8657", "date": "2024-06-27_15-58-39", "time_this_iter_s": 56.147799015045166, "time_total_s": 830.8587915897369, "pid": 69992, "hostname": "r8i6n8", "node_ip": "10.159.28.66", "config": {"encoder_num_layer": 1, "decoder_rt_num_layer": 2, "decoder_int_num_layer": 1, "embedding_dim": 64, "encoder_ff": 512, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 16, "drop_rate": 0.37340656523341487, "lr": 0.0007457945749126396, "batch_size": 1024}, "time_since_restore": 577.0165183544159, "iterations_since_restore": 10}

ray_results/test_experiment/train_model_c590a939_3_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=8,drop_ra_2024-06-27_14-16-53/params.json

+{
+  "batch_size": 1024,
+  "decoder_int_ff": 512,
+  "decoder_int_num_layer": 1,
+  "decoder_rt_ff": 512,
+  "decoder_rt_num_layer": 8,
+  "drop_rate": 0.8825060164596403,
+  "embedding_dim": 256,
+  "encoder_ff": 2048,
+  "encoder_num_layer": 4,
+  "lr": 0.00022475452906788255,
+  "n_head": 1
+}
\ No newline at end of file

ray_results/test_experiment/train_model_c590a939_3_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=8,drop_ra_2024-06-27_14-16-53/progress.csv

+loss,timestamp,checkpoint_dir_name,should_checkpoint,done,training_iteration,trial_id,date,time_this_iter_s,time_total_s,pid,hostname,node_ip,time_since_restore,iterations_since_restore
+2052.7300347756213,1719490908,checkpoint_000000,True,False,1,c590a939,2024-06-27_14-21-48,205.80420541763306,205.80420541763306,35196,r8i6n8,10.159.28.66,205.80420541763306,1

ray_results/test_experiment/train_model_c590a939_3_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=8,drop_ra_2024-06-27_14-16-53/result.json

+{"loss": 2052.7300347756213, "timestamp": 1719490908, "checkpoint_dir_name": "checkpoint_000000", "should_checkpoint": true, "done": false, "training_iteration": 1, "trial_id": "c590a939", "date": "2024-06-27_14-21-48", "time_this_iter_s": 205.80420541763306, "time_total_s": 205.80420541763306, "pid": 35196, "hostname": "r8i6n8", "node_ip": "10.159.28.66", "config": {"encoder_num_layer": 4, "decoder_rt_num_layer": 8, "decoder_int_num_layer": 1, "embedding_dim": 256, "encoder_ff": 2048, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 1, "drop_rate": 0.8825060164596403, "lr": 0.00022475452906788255, "batch_size": 1024}, "time_since_restore": 205.80420541763306, "iterations_since_restore": 1}

ray_results/test_experiment/train_model_cd09d91f_62_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop__2024-06-28_09-50-58/params.json

+{
+  "batch_size": 2048,
+  "decoder_int_ff": 512,
+  "decoder_int_num_layer": 1,
+  "decoder_rt_ff": 1024,
+  "decoder_rt_num_layer": 2,
+  "drop_rate": 0.8804057782771979,
+  "embedding_dim": 256,
+  "encoder_ff": 1024,
+  "encoder_num_layer": 4,
+  "lr": 0.00020339747661453456,
+  "n_head": 1
+}
\ No newline at end of file

ray_results/test_experiment/train_model_cd09d91f_62_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop__2024-06-28_09-50-58/progress.csv

+loss,timestamp,checkpoint_dir_name,should_checkpoint,done,training_iteration,trial_id,date,time_this_iter_s,time_total_s,pid,hostname,node_ip,time_since_restore,iterations_since_restore
+2046.8147607037401,1719562168,checkpoint_000000,True,False,1,cd09d91f,2024-06-28_10-09-28,113.24074220657349,113.24074220657349,242786,r8i6n8,10.159.28.66,113.24074220657349,1
+2046.7734077033097,1719562267,checkpoint_000001,True,False,2,cd09d91f,2024-06-28_10-11-07,99.61961531639099,212.86035752296448,242786,r8i6n8,10.159.28.66,212.86035752296448,2
+2044.4744459737942,1719562367,checkpoint_000002,True,False,3,cd09d91f,2024-06-28_10-12-47,99.62483620643616,312.48519372940063,242786,r8i6n8,10.159.28.66,312.48519372940063,3

ray_results/test_experiment/train_model_cd09d91f_62_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop__2024-06-28_09-50-58/result.json

+{"loss": 2046.8147607037401, "timestamp": 1719562168, "checkpoint_dir_name": "checkpoint_000000", "should_checkpoint": true, "done": false, "training_iteration": 1, "trial_id": "cd09d91f", "date": "2024-06-28_10-09-28", "time_this_iter_s": 113.24074220657349, "time_total_s": 113.24074220657349, "pid": 242786, "hostname": "r8i6n8", "node_ip": "10.159.28.66", "config": {"encoder_num_layer": 4, "decoder_rt_num_layer": 2, "decoder_int_num_layer": 1, "embedding_dim": 256, "encoder_ff": 1024, "decoder_rt_ff": 1024, "decoder_int_ff": 512, "n_head": 1, "drop_rate": 0.8804057782771979, "lr": 0.00020339747661453456, "batch_size": 2048}, "time_since_restore": 113.24074220657349, "iterations_since_restore": 1}
+{"loss": 2046.7734077033097, "timestamp": 1719562267, "checkpoint_dir_name": "checkpoint_000001", "should_checkpoint": true, "done": false, "training_iteration": 2, "trial_id": "cd09d91f", "date": "2024-06-28_10-11-07", "time_this_iter_s": 99.61961531639099, "time_total_s": 212.86035752296448, "pid": 242786, "hostname": "r8i6n8", "node_ip": "10.159.28.66", "config": {"encoder_num_layer": 4, "decoder_rt_num_layer": 2, "decoder_int_num_layer": 1, "embedding_dim": 256, "encoder_ff": 1024, "decoder_rt_ff": 1024, "decoder_int_ff": 512, "n_head": 1, "drop_rate": 0.8804057782771979, "lr": 0.00020339747661453456, "batch_size": 2048}, "time_since_restore": 212.86035752296448, "iterations_since_restore": 2}
+{"loss": 2044.4744459737942, "timestamp": 1719562367, "checkpoint_dir_name": "checkpoint_000002", "should_checkpoint": true, "done": false, "training_iteration": 3, "trial_id": "cd09d91f", "date": "2024-06-28_10-12-47", "time_this_iter_s": 99.62483620643616, "time_total_s": 312.48519372940063, "pid": 242786, "hostname": "r8i6n8", "node_ip": "10.159.28.66", "config": {"encoder_num_layer": 4, "decoder_rt_num_layer": 2, "decoder_int_num_layer": 1, "embedding_dim": 256, "encoder_ff": 1024, "decoder_rt_ff": 1024, "decoder_int_ff": 512, "n_head": 1, "drop_rate": 0.8804057782771979, "lr": 0.00020339747661453456, "batch_size": 2048}, "time_since_restore": 312.48519372940063, "iterations_since_restore": 3}

ray_results/test_experiment/train_model_ce13f5b5_19_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=8,drop__2024-06-27_15-58-40/params.json

+{
+  "batch_size": 2048,
+  "decoder_int_ff": 512,
+  "decoder_int_num_layer": 1,
+  "decoder_rt_ff": 1024,
+  "decoder_rt_num_layer": 8,
+  "drop_rate": 0.5002709319790926,
+  "embedding_dim": 1024,
+  "encoder_ff": 2048,
+  "encoder_num_layer": 2,
+  "lr": 0.0010079586534441948,
+  "n_head": 16
+}
\ No newline at end of file

ray_results/test_experiment/train_model_ce13f5b5_19_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=8,drop__2024-06-27_15-58-40/progress.csv

+loss,timestamp,checkpoint_dir_name,should_checkpoint,done,training_iteration,trial_id,date,time_this_iter_s,time_total_s,pid,hostname,node_ip,time_since_restore,iterations_since_restore
+1988.827157088152,1719497844,checkpoint_000000,True,False,1,ce13f5b5,2024-06-27_16-17-25,980.2613203525543,980.2613203525543,69992,r8i6n8,10.159.28.66,980.2613203525543,1
+1978.954319751169,1719498811,checkpoint_000001,True,False,2,ce13f5b5,2024-06-27_16-33-32,966.9953911304474,1947.2567114830017,69992,r8i6n8,10.159.28.66,1947.2567114830017,2
+1980.5698492095225,1719503758,checkpoint_000002,True,False,3,ce13f5b5,2024-06-27_17-55-58,980.9022901058197,2928.1590015888214,242786,r8i6n8,10.159.28.66,980.9022901058197,1
+1991.1704995463213,1719504722,checkpoint_000003,True,False,4,ce13f5b5,2024-06-27_18-12-02,964.2863810062408,3892.4453825950623,242786,r8i6n8,10.159.28.66,1945.1886711120605,2