del raysesult

ray_results/test_experiment/train_model_57284f74_48_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=8,drop__2024-06-28_01-55-17/progress.csv

-loss,timestamp,checkpoint_dir_name,should_checkpoint,done,training_iteration,trial_id,date,time_this_iter_s,time_total_s,pid,hostname,node_ip,time_since_restore,iterations_since_restore
-1961.4776693028728,1719533395,checkpoint_000000,True,False,1,57284f74,2024-06-28_02-09-55,103.31512880325317,103.31512880325317,242786,r8i6n8,10.159.28.66,103.31512880325317,1
-1959.5523080900898,1719533483,checkpoint_000001,True,False,2,57284f74,2024-06-28_02-11-23,88.52644205093384,191.841570854187,242786,r8i6n8,10.159.28.66,191.841570854187,2
-1990.1104097141056,1719533572,checkpoint_000002,True,False,3,57284f74,2024-06-28_02-12-52,88.94331359863281,280.7848844528198,242786,r8i6n8,10.159.28.66,280.7848844528198,3
-1962.9307063545768,1719533662,checkpoint_000003,True,False,4,57284f74,2024-06-28_02-14-22,89.54130935668945,370.3261938095093,242786,r8i6n8,10.159.28.66,370.3261938095093,4
-1954.3513726662463,1719533751,checkpoint_000004,True,False,5,57284f74,2024-06-28_02-15-51,89.19881558418274,459.525009393692,242786,r8i6n8,10.159.28.66,459.525009393692,5
-1957.0877286655696,1719533840,checkpoint_000005,True,False,6,57284f74,2024-06-28_02-17-20,89.5766806602478,549.1016900539398,242786,r8i6n8,10.159.28.66,549.1016900539398,6
-1960.7224668968381,1719533930,checkpoint_000006,True,False,7,57284f74,2024-06-28_02-18-50,89.54632711410522,638.648017168045,242786,r8i6n8,10.159.28.66,638.648017168045,7
-1960.303706612174,1719534019,checkpoint_000007,True,False,8,57284f74,2024-06-28_02-20-19,89.50325727462769,728.1512744426727,242786,r8i6n8,10.159.28.66,728.1512744426727,8
-1958.0356551042692,1719534109,checkpoint_000008,True,False,9,57284f74,2024-06-28_02-21-49,89.23209404945374,817.3833684921265,242786,r8i6n8,10.159.28.66,817.3833684921265,9
-1957.8637003260335,1719534198,checkpoint_000009,True,False,10,57284f74,2024-06-28_02-23-18,89.53279662132263,906.9161651134491,242786,r8i6n8,10.159.28.66,906.9161651134491,10

ray_results/test_experiment/train_model_57284f74_48_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=8,drop__2024-06-28_01-55-17/result.json

-{"loss": 1961.4776693028728, "timestamp": 1719533395, "checkpoint_dir_name": "checkpoint_000000", "should_checkpoint": true, "done": false, "training_iteration": 1, "trial_id": "57284f74", "date": "2024-06-28_02-09-55", "time_this_iter_s": 103.31512880325317, "time_total_s": 103.31512880325317, "pid": 242786, "hostname": "r8i6n8", "node_ip": "10.159.28.66", "config": {"encoder_num_layer": 4, "decoder_rt_num_layer": 8, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 2048, "decoder_rt_ff": 2048, "decoder_int_ff": 512, "n_head": 2, "drop_rate": 0.3048722403961074, "lr": 0.0003667007874849629, "batch_size": 1024}, "time_since_restore": 103.31512880325317, "iterations_since_restore": 1}
-{"loss": 1959.5523080900898, "timestamp": 1719533483, "checkpoint_dir_name": "checkpoint_000001", "should_checkpoint": true, "done": false, "training_iteration": 2, "trial_id": "57284f74", "date": "2024-06-28_02-11-23", "time_this_iter_s": 88.52644205093384, "time_total_s": 191.841570854187, "pid": 242786, "hostname": "r8i6n8", "node_ip": "10.159.28.66", "config": {"encoder_num_layer": 4, "decoder_rt_num_layer": 8, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 2048, "decoder_rt_ff": 2048, "decoder_int_ff": 512, "n_head": 2, "drop_rate": 0.3048722403961074, "lr": 0.0003667007874849629, "batch_size": 1024}, "time_since_restore": 191.841570854187, "iterations_since_restore": 2}
-{"loss": 1990.1104097141056, "timestamp": 1719533572, "checkpoint_dir_name": "checkpoint_000002", "should_checkpoint": true, "done": false, "training_iteration": 3, "trial_id": "57284f74", "date": "2024-06-28_02-12-52", "time_this_iter_s": 88.94331359863281, "time_total_s": 280.7848844528198, "pid": 242786, "hostname": "r8i6n8", "node_ip": "10.159.28.66", "config": {"encoder_num_layer": 4, "decoder_rt_num_layer": 8, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 2048, "decoder_rt_ff": 2048, "decoder_int_ff": 512, "n_head": 2, "drop_rate": 0.3048722403961074, "lr": 0.0003667007874849629, "batch_size": 1024}, "time_since_restore": 280.7848844528198, "iterations_since_restore": 3}
-{"loss": 1962.9307063545768, "timestamp": 1719533662, "checkpoint_dir_name": "checkpoint_000003", "should_checkpoint": true, "done": false, "training_iteration": 4, "trial_id": "57284f74", "date": "2024-06-28_02-14-22", "time_this_iter_s": 89.54130935668945, "time_total_s": 370.3261938095093, "pid": 242786, "hostname": "r8i6n8", "node_ip": "10.159.28.66", "config": {"encoder_num_layer": 4, "decoder_rt_num_layer": 8, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 2048, "decoder_rt_ff": 2048, "decoder_int_ff": 512, "n_head": 2, "drop_rate": 0.3048722403961074, "lr": 0.0003667007874849629, "batch_size": 1024}, "time_since_restore": 370.3261938095093, "iterations_since_restore": 4}
-{"loss": 1954.3513726662463, "timestamp": 1719533751, "checkpoint_dir_name": "checkpoint_000004", "should_checkpoint": true, "done": false, "training_iteration": 5, "trial_id": "57284f74", "date": "2024-06-28_02-15-51", "time_this_iter_s": 89.19881558418274, "time_total_s": 459.525009393692, "pid": 242786, "hostname": "r8i6n8", "node_ip": "10.159.28.66", "config": {"encoder_num_layer": 4, "decoder_rt_num_layer": 8, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 2048, "decoder_rt_ff": 2048, "decoder_int_ff": 512, "n_head": 2, "drop_rate": 0.3048722403961074, "lr": 0.0003667007874849629, "batch_size": 1024}, "time_since_restore": 459.525009393692, "iterations_since_restore": 5}
-{"loss": 1957.0877286655696, "timestamp": 1719533840, "checkpoint_dir_name": "checkpoint_000005", "should_checkpoint": true, "done": false, "training_iteration": 6, "trial_id": "57284f74", "date": "2024-06-28_02-17-20", "time_this_iter_s": 89.5766806602478, "time_total_s": 549.1016900539398, "pid": 242786, "hostname": "r8i6n8", "node_ip": "10.159.28.66", "config": {"encoder_num_layer": 4, "decoder_rt_num_layer": 8, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 2048, "decoder_rt_ff": 2048, "decoder_int_ff": 512, "n_head": 2, "drop_rate": 0.3048722403961074, "lr": 0.0003667007874849629, "batch_size": 1024}, "time_since_restore": 549.1016900539398, "iterations_since_restore": 6}
-{"loss": 1960.7224668968381, "timestamp": 1719533930, "checkpoint_dir_name": "checkpoint_000006", "should_checkpoint": true, "done": false, "training_iteration": 7, "trial_id": "57284f74", "date": "2024-06-28_02-18-50", "time_this_iter_s": 89.54632711410522, "time_total_s": 638.648017168045, "pid": 242786, "hostname": "r8i6n8", "node_ip": "10.159.28.66", "config": {"encoder_num_layer": 4, "decoder_rt_num_layer": 8, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 2048, "decoder_rt_ff": 2048, "decoder_int_ff": 512, "n_head": 2, "drop_rate": 0.3048722403961074, "lr": 0.0003667007874849629, "batch_size": 1024}, "time_since_restore": 638.648017168045, "iterations_since_restore": 7}
-{"loss": 1960.303706612174, "timestamp": 1719534019, "checkpoint_dir_name": "checkpoint_000007", "should_checkpoint": true, "done": false, "training_iteration": 8, "trial_id": "57284f74", "date": "2024-06-28_02-20-19", "time_this_iter_s": 89.50325727462769, "time_total_s": 728.1512744426727, "pid": 242786, "hostname": "r8i6n8", "node_ip": "10.159.28.66", "config": {"encoder_num_layer": 4, "decoder_rt_num_layer": 8, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 2048, "decoder_rt_ff": 2048, "decoder_int_ff": 512, "n_head": 2, "drop_rate": 0.3048722403961074, "lr": 0.0003667007874849629, "batch_size": 1024}, "time_since_restore": 728.1512744426727, "iterations_since_restore": 8}
-{"loss": 1958.0356551042692, "timestamp": 1719534109, "checkpoint_dir_name": "checkpoint_000008", "should_checkpoint": true, "done": false, "training_iteration": 9, "trial_id": "57284f74", "date": "2024-06-28_02-21-49", "time_this_iter_s": 89.23209404945374, "time_total_s": 817.3833684921265, "pid": 242786, "hostname": "r8i6n8", "node_ip": "10.159.28.66", "config": {"encoder_num_layer": 4, "decoder_rt_num_layer": 8, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 2048, "decoder_rt_ff": 2048, "decoder_int_ff": 512, "n_head": 2, "drop_rate": 0.3048722403961074, "lr": 0.0003667007874849629, "batch_size": 1024}, "time_since_restore": 817.3833684921265, "iterations_since_restore": 9}
-{"loss": 1957.8637003260335, "timestamp": 1719534198, "checkpoint_dir_name": "checkpoint_000009", "should_checkpoint": true, "done": false, "training_iteration": 10, "trial_id": "57284f74", "date": "2024-06-28_02-23-18", "time_this_iter_s": 89.53279662132263, "time_total_s": 906.9161651134491, "pid": 242786, "hostname": "r8i6n8", "node_ip": "10.159.28.66", "config": {"encoder_num_layer": 4, "decoder_rt_num_layer": 8, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 2048, "decoder_rt_ff": 2048, "decoder_int_ff": 512, "n_head": 2, "drop_rate": 0.3048722403961074, "lr": 0.0003667007874849629, "batch_size": 1024}, "time_since_restore": 906.9161651134491, "iterations_since_restore": 10}

ray_results/test_experiment/train_model_5763f130_1_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=8,drop_r_2024-06-25_17-10-00/params.json

-{
-  "batch_size": 2048,
-  "decoder_int_ff": 512,
-  "decoder_int_num_layer": 1,
-  "decoder_rt_ff": 2048,
-  "decoder_rt_num_layer": 8,
-  "drop_rate": 0.7224904221599877,
-  "embedding_dim": 256,
-  "encoder_ff": 512,
-  "encoder_num_layer": 4,
-  "lr": 0.04561252173503944,
-  "n_head": 2
-}
\ No newline at end of file

ray_results/test_experiment/train_model_5763f130_1_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=8,drop_r_2024-06-25_17-10-00/progress.csv

-loss,timestamp,checkpoint_dir_name,should_checkpoint,done,training_iteration,trial_id,date,time_this_iter_s,time_total_s,pid,hostname,node_ip,time_since_restore,iterations_since_restore
-4538.754975086122,1719328455,checkpoint_000000,True,False,1,5763f130,2024-06-25_17-14-16,242.80822491645813,242.80822491645813,2195253,r8i6n2,10.159.28.60,242.80822491645813,1

ray_results/test_experiment/train_model_5763f130_1_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=8,drop_r_2024-06-25_17-10-00/result.json

-{"loss": 4538.754975086122, "timestamp": 1719328455, "checkpoint_dir_name": "checkpoint_000000", "should_checkpoint": true, "done": false, "training_iteration": 1, "trial_id": "5763f130", "date": "2024-06-25_17-14-16", "time_this_iter_s": 242.80822491645813, "time_total_s": 242.80822491645813, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 4, "decoder_rt_num_layer": 8, "decoder_int_num_layer": 1, "embedding_dim": 256, "encoder_ff": 512, "decoder_rt_ff": 2048, "decoder_int_ff": 512, "n_head": 2, "drop_rate": 0.7224904221599877, "lr": 0.04561252173503944, "batch_size": 2048}, "time_since_restore": 242.80822491645813, "iterations_since_restore": 1}

ray_results/test_experiment/train_model_58bc48cb_2_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=1,drop_ra_2024-06-25_13-42-50/params.json

-{
-  "batch_size": 1024,
-  "decoder_int_ff": 512,
-  "decoder_int_num_layer": 1,
-  "decoder_rt_ff": 512,
-  "decoder_rt_num_layer": 1,
-  "drop_rate": 0.2,
-  "embedding_dim": 64,
-  "encoder_ff": 512,
-  "encoder_num_layer": 1,
-  "lr": 0.0001,
-  "n_head": 1
-}
\ No newline at end of file

ray_results/test_experiment/train_model_58bc48cb_2_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=1,drop_ra_2024-06-25_13-42-50/progress.csv

-loss,timestamp,checkpoint_dir_name,should_checkpoint,done,training_iteration,trial_id,date,time_this_iter_s,time_total_s,pid,hostname,node_ip,time_since_restore,iterations_since_restore
-396.9987843430887,1719316035,checkpoint_000000,True,False,1,58bc48cb,2024-06-25_13-47-15,76.29459309577942,76.29459309577942,339920,r9i1n2,10.159.32.15,76.29459309577942,1
-298.5339005838229,1719316294,checkpoint_000001,True,False,2,58bc48cb,2024-06-25_13-51-34,63.463342905044556,139.75793600082397,347039,r9i1n2,10.159.32.15,63.463342905044556,1
-250.0197828397976,1719316725,checkpoint_000002,True,False,3,58bc48cb,2024-06-25_13-58-45,61.615302085876465,201.37323808670044,353741,r9i1n2,10.159.32.15,61.615302085876465,1
-219.07703201414094,1719316773,checkpoint_000003,True,False,4,58bc48cb,2024-06-25_13-59-33,48.90440893173218,250.27764701843262,353741,r9i1n2,10.159.32.15,110.51971101760864,2
-196.89034751832017,1719317549,checkpoint_000004,True,False,5,58bc48cb,2024-06-25_14-12-29,62.07963538169861,312.3572824001312,366320,r9i1n2,10.159.32.15,62.07963538169861,1
-177.75854870653527,1719317597,checkpoint_000005,True,False,6,58bc48cb,2024-06-25_14-13-17,48.146273374557495,360.5035557746887,366320,r9i1n2,10.159.32.15,110.2259087562561,2
-165.82053633562222,1719317644,checkpoint_000006,True,False,7,58bc48cb,2024-06-25_14-14-04,47.530285120010376,408.0338408946991,366320,r9i1n2,10.159.32.15,157.75619387626648,3
-156.16600364024245,1719317692,checkpoint_000007,True,True,8,58bc48cb,2024-06-25_14-14-52,47.988035917282104,456.0218768119812,366320,r9i1n2,10.159.32.15,205.74422979354858,4

ray_results/test_experiment/train_model_58bc48cb_2_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=1,drop_ra_2024-06-25_13-42-50/result.json

-{"loss": 396.9987843430887, "timestamp": 1719316035, "checkpoint_dir_name": "checkpoint_000000", "should_checkpoint": true, "done": false, "training_iteration": 1, "trial_id": "58bc48cb", "date": "2024-06-25_13-47-15", "time_this_iter_s": 76.29459309577942, "time_total_s": 76.29459309577942, "pid": 339920, "hostname": "r9i1n2", "node_ip": "10.159.32.15", "config": {"encoder_num_layer": 1, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 64, "encoder_ff": 512, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 1, "drop_rate": 0.2, "lr": 0.0001, "batch_size": 1024}, "time_since_restore": 76.29459309577942, "iterations_since_restore": 1}
-{"loss": 298.5339005838229, "timestamp": 1719316294, "checkpoint_dir_name": "checkpoint_000001", "should_checkpoint": true, "done": false, "training_iteration": 2, "trial_id": "58bc48cb", "date": "2024-06-25_13-51-34", "time_this_iter_s": 63.463342905044556, "time_total_s": 139.75793600082397, "pid": 347039, "hostname": "r9i1n2", "node_ip": "10.159.32.15", "config": {"encoder_num_layer": 1, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 64, "encoder_ff": 512, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 1, "drop_rate": 0.2, "lr": 0.0001, "batch_size": 1024}, "time_since_restore": 63.463342905044556, "iterations_since_restore": 1}
-{"loss": 250.0197828397976, "timestamp": 1719316725, "checkpoint_dir_name": "checkpoint_000002", "should_checkpoint": true, "done": false, "training_iteration": 3, "trial_id": "58bc48cb", "date": "2024-06-25_13-58-45", "time_this_iter_s": 61.615302085876465, "time_total_s": 201.37323808670044, "pid": 353741, "hostname": "r9i1n2", "node_ip": "10.159.32.15", "config": {"encoder_num_layer": 1, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 64, "encoder_ff": 512, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 1, "drop_rate": 0.2, "lr": 0.0001, "batch_size": 1024}, "time_since_restore": 61.615302085876465, "iterations_since_restore": 1}
-{"loss": 219.07703201414094, "timestamp": 1719316773, "checkpoint_dir_name": "checkpoint_000003", "should_checkpoint": true, "done": false, "training_iteration": 4, "trial_id": "58bc48cb", "date": "2024-06-25_13-59-33", "time_this_iter_s": 48.90440893173218, "time_total_s": 250.27764701843262, "pid": 353741, "hostname": "r9i1n2", "node_ip": "10.159.32.15", "config": {"encoder_num_layer": 1, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 64, "encoder_ff": 512, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 1, "drop_rate": 0.2, "lr": 0.0001, "batch_size": 1024}, "time_since_restore": 110.51971101760864, "iterations_since_restore": 2}
-{"loss": 196.89034751832017, "timestamp": 1719317549, "checkpoint_dir_name": "checkpoint_000004", "should_checkpoint": true, "done": false, "training_iteration": 5, "trial_id": "58bc48cb", "date": "2024-06-25_14-12-29", "time_this_iter_s": 62.07963538169861, "time_total_s": 312.3572824001312, "pid": 366320, "hostname": "r9i1n2", "node_ip": "10.159.32.15", "config": {"encoder_num_layer": 1, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 64, "encoder_ff": 512, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 1, "drop_rate": 0.2, "lr": 0.0001, "batch_size": 1024}, "time_since_restore": 62.07963538169861, "iterations_since_restore": 1}
-{"loss": 177.75854870653527, "timestamp": 1719317597, "checkpoint_dir_name": "checkpoint_000005", "should_checkpoint": true, "done": false, "training_iteration": 6, "trial_id": "58bc48cb", "date": "2024-06-25_14-13-17", "time_this_iter_s": 48.146273374557495, "time_total_s": 360.5035557746887, "pid": 366320, "hostname": "r9i1n2", "node_ip": "10.159.32.15", "config": {"encoder_num_layer": 1, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 64, "encoder_ff": 512, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 1, "drop_rate": 0.2, "lr": 0.0001, "batch_size": 1024}, "time_since_restore": 110.2259087562561, "iterations_since_restore": 2}
-{"loss": 165.82053633562222, "timestamp": 1719317644, "checkpoint_dir_name": "checkpoint_000006", "should_checkpoint": true, "done": false, "training_iteration": 7, "trial_id": "58bc48cb", "date": "2024-06-25_14-14-04", "time_this_iter_s": 47.530285120010376, "time_total_s": 408.0338408946991, "pid": 366320, "hostname": "r9i1n2", "node_ip": "10.159.32.15", "config": {"encoder_num_layer": 1, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 64, "encoder_ff": 512, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 1, "drop_rate": 0.2, "lr": 0.0001, "batch_size": 1024}, "time_since_restore": 157.75619387626648, "iterations_since_restore": 3}
-{"loss": 156.16600364024245, "timestamp": 1719317692, "checkpoint_dir_name": "checkpoint_000007", "should_checkpoint": true, "done": true, "training_iteration": 8, "trial_id": "58bc48cb", "date": "2024-06-25_14-14-52", "time_this_iter_s": 47.988035917282104, "time_total_s": 456.0218768119812, "pid": 366320, "hostname": "r9i1n2", "node_ip": "10.159.32.15", "config": {"encoder_num_layer": 1, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 64, "encoder_ff": 512, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 1, "drop_rate": 0.2, "lr": 0.0001, "batch_size": 1024}, "time_since_restore": 205.74422979354858, "iterations_since_restore": 4}

ray_results/test_experiment/train_model_5b0a0ae2_6_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop_r_2024-06-25_17-18-52/params.json

-{
-  "batch_size": 2048,
-  "decoder_int_ff": 512,
-  "decoder_int_num_layer": 1,
-  "decoder_rt_ff": 2048,
-  "decoder_rt_num_layer": 1,
-  "drop_rate": 0.0063231604639778505,
-  "embedding_dim": 16,
-  "encoder_ff": 512,
-  "encoder_num_layer": 8,
-  "lr": 0.0021079084039053323,
-  "n_head": 16
-}
\ No newline at end of file