del raysesult

ray_results/test_experiment/train_model_40101b9a_29_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=1,drop_r_2024-06-25_22-15-47/progress.csv

-loss,timestamp,checkpoint_dir_name,should_checkpoint,done,training_iteration,trial_id,date,time_this_iter_s,time_total_s,pid,hostname,node_ip,time_since_restore,iterations_since_restore
-4106.712271622785,1719347137,checkpoint_000000,True,False,1,40101b9a,2024-06-25_22-25-37,313.700572013855,313.700572013855,2195253,r8i6n2,10.159.28.60,313.700572013855,1
-3130.0891709215057,1719347437,checkpoint_000001,True,False,2,40101b9a,2024-06-25_22-30-38,300.49900484085083,614.1995768547058,2195253,r8i6n2,10.159.28.60,614.1995768547058,2
-2512.6716385488435,1719347737,checkpoint_000002,True,False,3,40101b9a,2024-06-25_22-35-38,300.07938170433044,914.2789585590363,2195253,r8i6n2,10.159.28.60,914.2789585590363,3
-2181.0040408156988,1719348038,checkpoint_000003,True,False,4,40101b9a,2024-06-25_22-40-38,300.6292908191681,1214.9082493782043,2195253,r8i6n2,10.159.28.60,1214.9082493782043,4
-2031.5720868448573,1719348338,checkpoint_000004,True,False,5,40101b9a,2024-06-25_22-45-38,300.0205228328705,1514.9287722110748,2195253,r8i6n2,10.159.28.60,1514.9287722110748,5

ray_results/test_experiment/train_model_40101b9a_29_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=1,drop_r_2024-06-25_22-15-47/result.json

-{"loss": 4106.712271622785, "timestamp": 1719347137, "checkpoint_dir_name": "checkpoint_000000", "should_checkpoint": true, "done": false, "training_iteration": 1, "trial_id": "40101b9a", "date": "2024-06-25_22-25-37", "time_this_iter_s": 313.700572013855, "time_total_s": 313.700572013855, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 2, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 1024, "encoder_ff": 2048, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 1, "drop_rate": 0.8983976265900149, "lr": 0.039339947723305395, "batch_size": 2048}, "time_since_restore": 313.700572013855, "iterations_since_restore": 1}
-{"loss": 3130.0891709215057, "timestamp": 1719347437, "checkpoint_dir_name": "checkpoint_000001", "should_checkpoint": true, "done": false, "training_iteration": 2, "trial_id": "40101b9a", "date": "2024-06-25_22-30-38", "time_this_iter_s": 300.49900484085083, "time_total_s": 614.1995768547058, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 2, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 1024, "encoder_ff": 2048, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 1, "drop_rate": 0.8983976265900149, "lr": 0.039339947723305395, "batch_size": 2048}, "time_since_restore": 614.1995768547058, "iterations_since_restore": 2}
-{"loss": 2512.6716385488435, "timestamp": 1719347737, "checkpoint_dir_name": "checkpoint_000002", "should_checkpoint": true, "done": false, "training_iteration": 3, "trial_id": "40101b9a", "date": "2024-06-25_22-35-38", "time_this_iter_s": 300.07938170433044, "time_total_s": 914.2789585590363, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 2, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 1024, "encoder_ff": 2048, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 1, "drop_rate": 0.8983976265900149, "lr": 0.039339947723305395, "batch_size": 2048}, "time_since_restore": 914.2789585590363, "iterations_since_restore": 3}
-{"loss": 2181.0040408156988, "timestamp": 1719348038, "checkpoint_dir_name": "checkpoint_000003", "should_checkpoint": true, "done": false, "training_iteration": 4, "trial_id": "40101b9a", "date": "2024-06-25_22-40-38", "time_this_iter_s": 300.6292908191681, "time_total_s": 1214.9082493782043, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 2, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 1024, "encoder_ff": 2048, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 1, "drop_rate": 0.8983976265900149, "lr": 0.039339947723305395, "batch_size": 2048}, "time_since_restore": 1214.9082493782043, "iterations_since_restore": 4}
-{"loss": 2031.5720868448573, "timestamp": 1719348338, "checkpoint_dir_name": "checkpoint_000004", "should_checkpoint": true, "done": false, "training_iteration": 5, "trial_id": "40101b9a", "date": "2024-06-25_22-45-38", "time_this_iter_s": 300.0205228328705, "time_total_s": 1514.9287722110748, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 2, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 1024, "encoder_ff": 2048, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 1, "drop_rate": 0.8983976265900149, "lr": 0.039339947723305395, "batch_size": 2048}, "time_since_restore": 1514.9287722110748, "iterations_since_restore": 5}

ray_results/test_experiment/train_model_41e4bd86_12_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=8,drop_r_2024-06-27_14-42-34/params.json

-{
-  "batch_size": 2048,
-  "decoder_int_ff": 512,
-  "decoder_int_num_layer": 1,
-  "decoder_rt_ff": 512,
-  "decoder_rt_num_layer": 8,
-  "drop_rate": 0.9072626058636482,
-  "embedding_dim": 1024,
-  "encoder_ff": 512,
-  "encoder_num_layer": 1,
-  "lr": 0.003236929771052812,
-  "n_head": 4
-}
\ No newline at end of file

ray_results/test_experiment/train_model_41e4bd86_12_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=8,drop_r_2024-06-27_14-42-34/progress.csv

-loss,timestamp,checkpoint_dir_name,should_checkpoint,done,training_iteration,trial_id,date,time_this_iter_s,time_total_s,pid,hostname,node_ip,time_since_restore,iterations_since_restore
-2134.4770325187624,1719493181,checkpoint_000000,True,False,1,41e4bd86,2024-06-27_14-59-41,720.7119467258453,720.7119467258453,69992,r8i6n8,10.159.28.66,720.7119467258453,1

ray_results/test_experiment/train_model_41e4bd86_12_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=8,drop_r_2024-06-27_14-42-34/result.json

-{"loss": 2134.4770325187624, "timestamp": 1719493181, "checkpoint_dir_name": "checkpoint_000000", "should_checkpoint": true, "done": false, "training_iteration": 1, "trial_id": "41e4bd86", "date": "2024-06-27_14-59-41", "time_this_iter_s": 720.7119467258453, "time_total_s": 720.7119467258453, "pid": 69992, "hostname": "r8i6n8", "node_ip": "10.159.28.66", "config": {"encoder_num_layer": 1, "decoder_rt_num_layer": 8, "decoder_int_num_layer": 1, "embedding_dim": 1024, "encoder_ff": 512, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 4, "drop_rate": 0.9072626058636482, "lr": 0.003236929771052812, "batch_size": 2048}, "time_since_restore": 720.7119467258453, "iterations_since_restore": 1}

ray_results/test_experiment/train_model_43bba37c_19_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_18-28-52/params.json

-{
-  "batch_size": 1024,
-  "decoder_int_ff": 512,
-  "decoder_int_num_layer": 1,
-  "decoder_rt_ff": 1024,
-  "decoder_rt_num_layer": 1,
-  "drop_rate": 0.4036073673456261,
-  "embedding_dim": 256,
-  "encoder_ff": 1024,
-  "encoder_num_layer": 8,
-  "lr": 0.0007661776725543005,
-  "n_head": 1
-}
\ No newline at end of file

ray_results/test_experiment/train_model_43bba37c_19_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_18-28-52/progress.csv

-loss,timestamp,checkpoint_dir_name,should_checkpoint,done,training_iteration,trial_id,date,time_this_iter_s,time_total_s,pid,hostname,node_ip,time_since_restore,iterations_since_restore
-1961.9788803941622,1719333991,checkpoint_000000,True,False,1,43bba37c,2024-06-25_18-46-31,161.8538646697998,161.8538646697998,2195253,r8i6n2,10.159.28.60,161.8538646697998,1
-1966.8054059847134,1719334139,checkpoint_000001,True,False,2,43bba37c,2024-06-25_18-48-59,147.8709270954132,309.724791765213,2195253,r8i6n2,10.159.28.60,309.724791765213,2
-1967.6592320720042,1719338350,checkpoint_000002,True,False,3,43bba37c,2024-06-25_19-59-10,161.5647487640381,471.2895405292511,2195253,r8i6n2,10.159.28.60,161.5647487640381,1
-1972.7257733682948,1719338498,checkpoint_000003,True,False,4,43bba37c,2024-06-25_20-01-38,147.5350902080536,618.8246307373047,2195253,r8i6n2,10.159.28.60,309.0998389720917,2
-1965.584105484129,1719340228,checkpoint_000004,True,False,5,43bba37c,2024-06-25_20-30-28,160.8864643573761,779.7110950946808,2195253,r8i6n2,10.159.28.60,160.8864643573761,1
-1965.0863916592334,1719340375,checkpoint_000005,True,False,6,43bba37c,2024-06-25_20-32-55,147.0123507976532,926.723445892334,2195253,r8i6n2,10.159.28.60,307.8988151550293,2
-1965.4208993986836,1719340522,checkpoint_000006,True,False,7,43bba37c,2024-06-25_20-35-23,147.50104451179504,1074.224490404129,2195253,r8i6n2,10.159.28.60,455.39985966682434,3
-1962.8120477631337,1719340670,checkpoint_000007,True,False,8,43bba37c,2024-06-25_20-37-50,147.67359066009521,1221.8980810642242,2195253,r8i6n2,10.159.28.60,603.0734503269196,4

ray_results/test_experiment/train_model_43bba37c_19_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_18-28-52/result.json

-{"loss": 1961.9788803941622, "timestamp": 1719333991, "checkpoint_dir_name": "checkpoint_000000", "should_checkpoint": true, "done": false, "training_iteration": 1, "trial_id": "43bba37c", "date": "2024-06-25_18-46-31", "time_this_iter_s": 161.8538646697998, "time_total_s": 161.8538646697998, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 8, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 256, "encoder_ff": 1024, "decoder_rt_ff": 1024, "decoder_int_ff": 512, "n_head": 1, "drop_rate": 0.4036073673456261, "lr": 0.0007661776725543005, "batch_size": 1024}, "time_since_restore": 161.8538646697998, "iterations_since_restore": 1}
-{"loss": 1966.8054059847134, "timestamp": 1719334139, "checkpoint_dir_name": "checkpoint_000001", "should_checkpoint": true, "done": false, "training_iteration": 2, "trial_id": "43bba37c", "date": "2024-06-25_18-48-59", "time_this_iter_s": 147.8709270954132, "time_total_s": 309.724791765213, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 8, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 256, "encoder_ff": 1024, "decoder_rt_ff": 1024, "decoder_int_ff": 512, "n_head": 1, "drop_rate": 0.4036073673456261, "lr": 0.0007661776725543005, "batch_size": 1024}, "time_since_restore": 309.724791765213, "iterations_since_restore": 2}
-{"loss": 1967.6592320720042, "timestamp": 1719338350, "checkpoint_dir_name": "checkpoint_000002", "should_checkpoint": true, "done": false, "training_iteration": 3, "trial_id": "43bba37c", "date": "2024-06-25_19-59-10", "time_this_iter_s": 161.5647487640381, "time_total_s": 471.2895405292511, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 8, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 256, "encoder_ff": 1024, "decoder_rt_ff": 1024, "decoder_int_ff": 512, "n_head": 1, "drop_rate": 0.4036073673456261, "lr": 0.0007661776725543005, "batch_size": 1024}, "time_since_restore": 161.5647487640381, "iterations_since_restore": 1}
-{"loss": 1972.7257733682948, "timestamp": 1719338498, "checkpoint_dir_name": "checkpoint_000003", "should_checkpoint": true, "done": false, "training_iteration": 4, "trial_id": "43bba37c", "date": "2024-06-25_20-01-38", "time_this_iter_s": 147.5350902080536, "time_total_s": 618.8246307373047, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 8, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 256, "encoder_ff": 1024, "decoder_rt_ff": 1024, "decoder_int_ff": 512, "n_head": 1, "drop_rate": 0.4036073673456261, "lr": 0.0007661776725543005, "batch_size": 1024}, "time_since_restore": 309.0998389720917, "iterations_since_restore": 2}
-{"loss": 1965.584105484129, "timestamp": 1719340228, "checkpoint_dir_name": "checkpoint_000004", "should_checkpoint": true, "done": false, "training_iteration": 5, "trial_id": "43bba37c", "date": "2024-06-25_20-30-28", "time_this_iter_s": 160.8864643573761, "time_total_s": 779.7110950946808, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 8, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 256, "encoder_ff": 1024, "decoder_rt_ff": 1024, "decoder_int_ff": 512, "n_head": 1, "drop_rate": 0.4036073673456261, "lr": 0.0007661776725543005, "batch_size": 1024}, "time_since_restore": 160.8864643573761, "iterations_since_restore": 1}
-{"loss": 1965.0863916592334, "timestamp": 1719340375, "checkpoint_dir_name": "checkpoint_000005", "should_checkpoint": true, "done": false, "training_iteration": 6, "trial_id": "43bba37c", "date": "2024-06-25_20-32-55", "time_this_iter_s": 147.0123507976532, "time_total_s": 926.723445892334, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 8, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 256, "encoder_ff": 1024, "decoder_rt_ff": 1024, "decoder_int_ff": 512, "n_head": 1, "drop_rate": 0.4036073673456261, "lr": 0.0007661776725543005, "batch_size": 1024}, "time_since_restore": 307.8988151550293, "iterations_since_restore": 2}
-{"loss": 1965.4208993986836, "timestamp": 1719340522, "checkpoint_dir_name": "checkpoint_000006", "should_checkpoint": true, "done": false, "training_iteration": 7, "trial_id": "43bba37c", "date": "2024-06-25_20-35-23", "time_this_iter_s": 147.50104451179504, "time_total_s": 1074.224490404129, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 8, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 256, "encoder_ff": 1024, "decoder_rt_ff": 1024, "decoder_int_ff": 512, "n_head": 1, "drop_rate": 0.4036073673456261, "lr": 0.0007661776725543005, "batch_size": 1024}, "time_since_restore": 455.39985966682434, "iterations_since_restore": 3}
-{"loss": 1962.8120477631337, "timestamp": 1719340670, "checkpoint_dir_name": "checkpoint_000007", "should_checkpoint": true, "done": false, "training_iteration": 8, "trial_id": "43bba37c", "date": "2024-06-25_20-37-50", "time_this_iter_s": 147.67359066009521, "time_total_s": 1221.8980810642242, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 8, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 256, "encoder_ff": 1024, "decoder_rt_ff": 1024, "decoder_int_ff": 512, "n_head": 1, "drop_rate": 0.4036073673456261, "lr": 0.0007661776725543005, "batch_size": 1024}, "time_since_restore": 603.0734503269196, "iterations_since_restore": 4}

ray_results/test_experiment/train_model_45b0a7c2_1_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop_r_2024-06-27_14-14-55/params.json

-{
-  "batch_size": 2048,
-  "decoder_int_ff": 512,
-  "decoder_int_num_layer": 1,
-  "decoder_rt_ff": 1024,
-  "decoder_rt_num_layer": 1,
-  "drop_rate": 0.584365221617647,
-  "embedding_dim": 256,
-  "encoder_ff": 1024,
-  "encoder_num_layer": 4,
-  "lr": 0.0009124143820205934,
-  "n_head": 4
-}
\ No newline at end of file

ray_results/test_experiment/train_model_45b0a7c2_1_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop_r_2024-06-27_14-14-55/progress.csv

-loss,timestamp,checkpoint_dir_name,should_checkpoint,done,training_iteration,trial_id,date,time_this_iter_s,time_total_s,pid,hostname,node_ip,time_since_restore,iterations_since_restore
-1976.3005707507998,1719490613,checkpoint_000000,True,False,1,45b0a7c2,2024-06-27_14-16-53,105.02805352210999,105.02805352210999,35196,r8i6n8,10.159.28.66,105.02805352210999,1
-1972.5077211875616,1719493830,checkpoint_000001,True,False,2,45b0a7c2,2024-06-27_15-10-30,103.25011920928955,208.27817273139954,69992,r8i6n8,10.159.28.66,103.25011920928955,1

ray_results/test_experiment/train_model_45b0a7c2_1_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop_r_2024-06-27_14-14-55/result.json

-{"loss": 1976.3005707507998, "timestamp": 1719490613, "checkpoint_dir_name": "checkpoint_000000", "should_checkpoint": true, "done": false, "training_iteration": 1, "trial_id": "45b0a7c2", "date": "2024-06-27_14-16-53", "time_this_iter_s": 105.02805352210999, "time_total_s": 105.02805352210999, "pid": 35196, "hostname": "r8i6n8", "node_ip": "10.159.28.66", "config": {"encoder_num_layer": 4, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 256, "encoder_ff": 1024, "decoder_rt_ff": 1024, "decoder_int_ff": 512, "n_head": 4, "drop_rate": 0.584365221617647, "lr": 0.0009124143820205934, "batch_size": 2048}, "time_since_restore": 105.02805352210999, "iterations_since_restore": 1}
-{"loss": 1972.5077211875616, "timestamp": 1719493830, "checkpoint_dir_name": "checkpoint_000001", "should_checkpoint": true, "done": false, "training_iteration": 2, "trial_id": "45b0a7c2", "date": "2024-06-27_15-10-30", "time_this_iter_s": 103.25011920928955, "time_total_s": 208.27817273139954, "pid": 69992, "hostname": "r8i6n8", "node_ip": "10.159.28.66", "config": {"encoder_num_layer": 4, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 256, "encoder_ff": 1024, "decoder_rt_ff": 1024, "decoder_int_ff": 512, "n_head": 4, "drop_rate": 0.584365221617647, "lr": 0.0009124143820205934, "batch_size": 2048}, "time_since_restore": 103.25011920928955, "iterations_since_restore": 1}