del raysesult

ray_results/test_experiment/train_model_3ab2f301_28_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop__2024-06-25_22-10-34/progress.csv

-loss,timestamp,checkpoint_dir_name,should_checkpoint,done,training_iteration,trial_id,date,time_this_iter_s,time_total_s,pid,hostname,node_ip,time_since_restore,iterations_since_restore
-217.46355960875985,1719346612,checkpoint_000000,True,False,1,3ab2f301,2024-06-25_22-16-52,65.41611504554749,65.41611504554749,2195253,r8i6n2,10.159.28.60,65.41611504554749,1
-163.8450069878045,1719346664,checkpoint_000001,True,False,2,3ab2f301,2024-06-25_22-17-45,52.40175247192383,117.81786751747131,2195253,r8i6n2,10.159.28.60,117.81786751747131,2
-139.51932639775313,1719346717,checkpoint_000002,True,False,3,3ab2f301,2024-06-25_22-18-37,52.207762479782104,170.02562999725342,2195253,r8i6n2,10.159.28.60,170.02562999725342,3
-125.82796093798059,1719346770,checkpoint_000003,True,False,4,3ab2f301,2024-06-25_22-19-30,53.092029094696045,223.11765909194946,2195253,r8i6n2,10.159.28.60,223.11765909194946,4
-117.45965227742833,1719346823,checkpoint_000004,True,False,5,3ab2f301,2024-06-25_22-20-23,53.195783615112305,276.31344270706177,2195253,r8i6n2,10.159.28.60,276.31344270706177,5
-114.47784069391686,1719355017,checkpoint_000005,True,False,6,3ab2f301,2024-06-26_00-36-57,65.25149631500244,341.5649390220642,2850562,r8i6n2,10.159.28.60,65.25149631500244,1
-109.24919566957969,1719355069,checkpoint_000006,True,False,7,3ab2f301,2024-06-26_00-37-49,51.495158195495605,393.0600972175598,2850562,r8i6n2,10.159.28.60,116.74665451049805,2
-105.2013188159372,1719355120,checkpoint_000007,True,False,8,3ab2f301,2024-06-26_00-38-40,51.496910572052,444.5570077896118,2850562,r8i6n2,10.159.28.60,168.24356508255005,3
-118.58170366662694,1719355171,checkpoint_000008,True,False,9,3ab2f301,2024-06-26_00-39-31,51.06172227859497,495.6187300682068,2850562,r8i6n2,10.159.28.60,219.30528736114502,4
-99.85706461508443,1719355223,checkpoint_000009,True,False,10,3ab2f301,2024-06-26_00-40-23,51.387521266937256,547.006251335144,2850562,r8i6n2,10.159.28.60,270.6928086280823,5
-98.49699257677935,1719357936,checkpoint_000010,True,False,11,3ab2f301,2024-06-26_01-25-36,64.502512216568,611.508763551712,2850562,r8i6n2,10.159.28.60,64.502512216568,1
-93.49606281190407,1719357987,checkpoint_000011,True,False,12,3ab2f301,2024-06-26_01-26-27,50.864012002944946,662.372775554657,2850562,r8i6n2,10.159.28.60,115.36652421951294,2
-94.78943622018409,1719358039,checkpoint_000012,True,False,13,3ab2f301,2024-06-26_01-27-19,51.64423751831055,714.0170130729675,2850562,r8i6n2,10.159.28.60,167.0107617378235,3
-91.46543307567205,1719358090,checkpoint_000013,True,False,14,3ab2f301,2024-06-26_01-28-10,51.174827098846436,765.191840171814,2850562,r8i6n2,10.159.28.60,218.18558883666992,4
-88.73407943605439,1719358141,checkpoint_000014,True,False,15,3ab2f301,2024-06-26_01-29-01,50.784724712371826,815.9765648841858,2850562,r8i6n2,10.159.28.60,268.97031354904175,5
-94.00536857064314,1719358192,checkpoint_000015,True,False,16,3ab2f301,2024-06-26_01-29-52,51.60023093223572,867.5767958164215,2850562,r8i6n2,10.159.28.60,320.57054448127747,6
-87.47035950187623,1719358244,checkpoint_000016,True,False,17,3ab2f301,2024-06-26_01-30-44,51.84270000457764,919.4194958209991,2850562,r8i6n2,10.159.28.60,372.4132444858551,7
-88.06384884090875,1719358295,checkpoint_000017,True,False,18,3ab2f301,2024-06-26_01-31-35,51.09972786903381,970.519223690033,2850562,r8i6n2,10.159.28.60,423.5129723548889,8
-87.50748780017763,1719358346,checkpoint_000018,True,False,19,3ab2f301,2024-06-26_01-32-26,51.09087514877319,1021.6100988388062,2850562,r8i6n2,10.159.28.60,474.6038475036621,9
-84.81797225832001,1719358398,checkpoint_000019,True,False,20,3ab2f301,2024-06-26_01-33-18,51.372178077697754,1072.982276916504,2850562,r8i6n2,10.159.28.60,525.9760255813599,10

ray_results/test_experiment/train_model_3ab2f301_28_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop__2024-06-25_22-10-34/result.json

-{"loss": 217.46355960875985, "timestamp": 1719346612, "checkpoint_dir_name": "checkpoint_000000", "should_checkpoint": true, "done": false, "training_iteration": 1, "trial_id": "3ab2f301", "date": "2024-06-25_22-16-52", "time_this_iter_s": 65.41611504554749, "time_total_s": 65.41611504554749, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 2, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 2048, "decoder_rt_ff": 2048, "decoder_int_ff": 512, "n_head": 1, "drop_rate": 0.034481700426631456, "lr": 0.0010186128360476897, "batch_size": 2048}, "time_since_restore": 65.41611504554749, "iterations_since_restore": 1}
-{"loss": 163.8450069878045, "timestamp": 1719346664, "checkpoint_dir_name": "checkpoint_000001", "should_checkpoint": true, "done": false, "training_iteration": 2, "trial_id": "3ab2f301", "date": "2024-06-25_22-17-45", "time_this_iter_s": 52.40175247192383, "time_total_s": 117.81786751747131, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 2, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 2048, "decoder_rt_ff": 2048, "decoder_int_ff": 512, "n_head": 1, "drop_rate": 0.034481700426631456, "lr": 0.0010186128360476897, "batch_size": 2048}, "time_since_restore": 117.81786751747131, "iterations_since_restore": 2}
-{"loss": 139.51932639775313, "timestamp": 1719346717, "checkpoint_dir_name": "checkpoint_000002", "should_checkpoint": true, "done": false, "training_iteration": 3, "trial_id": "3ab2f301", "date": "2024-06-25_22-18-37", "time_this_iter_s": 52.207762479782104, "time_total_s": 170.02562999725342, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 2, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 2048, "decoder_rt_ff": 2048, "decoder_int_ff": 512, "n_head": 1, "drop_rate": 0.034481700426631456, "lr": 0.0010186128360476897, "batch_size": 2048}, "time_since_restore": 170.02562999725342, "iterations_since_restore": 3}
-{"loss": 125.82796093798059, "timestamp": 1719346770, "checkpoint_dir_name": "checkpoint_000003", "should_checkpoint": true, "done": false, "training_iteration": 4, "trial_id": "3ab2f301", "date": "2024-06-25_22-19-30", "time_this_iter_s": 53.092029094696045, "time_total_s": 223.11765909194946, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 2, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 2048, "decoder_rt_ff": 2048, "decoder_int_ff": 512, "n_head": 1, "drop_rate": 0.034481700426631456, "lr": 0.0010186128360476897, "batch_size": 2048}, "time_since_restore": 223.11765909194946, "iterations_since_restore": 4}
-{"loss": 117.45965227742833, "timestamp": 1719346823, "checkpoint_dir_name": "checkpoint_000004", "should_checkpoint": true, "done": false, "training_iteration": 5, "trial_id": "3ab2f301", "date": "2024-06-25_22-20-23", "time_this_iter_s": 53.195783615112305, "time_total_s": 276.31344270706177, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 2, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 2048, "decoder_rt_ff": 2048, "decoder_int_ff": 512, "n_head": 1, "drop_rate": 0.034481700426631456, "lr": 0.0010186128360476897, "batch_size": 2048}, "time_since_restore": 276.31344270706177, "iterations_since_restore": 5}
-{"loss": 114.47784069391686, "timestamp": 1719355017, "checkpoint_dir_name": "checkpoint_000005", "should_checkpoint": true, "done": false, "training_iteration": 6, "trial_id": "3ab2f301", "date": "2024-06-26_00-36-57", "time_this_iter_s": 65.25149631500244, "time_total_s": 341.5649390220642, "pid": 2850562, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 2, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 2048, "decoder_rt_ff": 2048, "decoder_int_ff": 512, "n_head": 1, "drop_rate": 0.034481700426631456, "lr": 0.0010186128360476897, "batch_size": 2048}, "time_since_restore": 65.25149631500244, "iterations_since_restore": 1}
-{"loss": 109.24919566957969, "timestamp": 1719355069, "checkpoint_dir_name": "checkpoint_000006", "should_checkpoint": true, "done": false, "training_iteration": 7, "trial_id": "3ab2f301", "date": "2024-06-26_00-37-49", "time_this_iter_s": 51.495158195495605, "time_total_s": 393.0600972175598, "pid": 2850562, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 2, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 2048, "decoder_rt_ff": 2048, "decoder_int_ff": 512, "n_head": 1, "drop_rate": 0.034481700426631456, "lr": 0.0010186128360476897, "batch_size": 2048}, "time_since_restore": 116.74665451049805, "iterations_since_restore": 2}
-{"loss": 105.2013188159372, "timestamp": 1719355120, "checkpoint_dir_name": "checkpoint_000007", "should_checkpoint": true, "done": false, "training_iteration": 8, "trial_id": "3ab2f301", "date": "2024-06-26_00-38-40", "time_this_iter_s": 51.496910572052, "time_total_s": 444.5570077896118, "pid": 2850562, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 2, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 2048, "decoder_rt_ff": 2048, "decoder_int_ff": 512, "n_head": 1, "drop_rate": 0.034481700426631456, "lr": 0.0010186128360476897, "batch_size": 2048}, "time_since_restore": 168.24356508255005, "iterations_since_restore": 3}
-{"loss": 118.58170366662694, "timestamp": 1719355171, "checkpoint_dir_name": "checkpoint_000008", "should_checkpoint": true, "done": false, "training_iteration": 9, "trial_id": "3ab2f301", "date": "2024-06-26_00-39-31", "time_this_iter_s": 51.06172227859497, "time_total_s": 495.6187300682068, "pid": 2850562, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 2, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 2048, "decoder_rt_ff": 2048, "decoder_int_ff": 512, "n_head": 1, "drop_rate": 0.034481700426631456, "lr": 0.0010186128360476897, "batch_size": 2048}, "time_since_restore": 219.30528736114502, "iterations_since_restore": 4}
-{"loss": 99.85706461508443, "timestamp": 1719355223, "checkpoint_dir_name": "checkpoint_000009", "should_checkpoint": true, "done": false, "training_iteration": 10, "trial_id": "3ab2f301", "date": "2024-06-26_00-40-23", "time_this_iter_s": 51.387521266937256, "time_total_s": 547.006251335144, "pid": 2850562, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 2, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 2048, "decoder_rt_ff": 2048, "decoder_int_ff": 512, "n_head": 1, "drop_rate": 0.034481700426631456, "lr": 0.0010186128360476897, "batch_size": 2048}, "time_since_restore": 270.6928086280823, "iterations_since_restore": 5}
-{"loss": 98.49699257677935, "timestamp": 1719357936, "checkpoint_dir_name": "checkpoint_000010", "should_checkpoint": true, "done": false, "training_iteration": 11, "trial_id": "3ab2f301", "date": "2024-06-26_01-25-36", "time_this_iter_s": 64.502512216568, "time_total_s": 611.508763551712, "pid": 2850562, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 2, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 2048, "decoder_rt_ff": 2048, "decoder_int_ff": 512, "n_head": 1, "drop_rate": 0.034481700426631456, "lr": 0.0010186128360476897, "batch_size": 2048}, "time_since_restore": 64.502512216568, "iterations_since_restore": 1}
-{"loss": 93.49606281190407, "timestamp": 1719357987, "checkpoint_dir_name": "checkpoint_000011", "should_checkpoint": true, "done": false, "training_iteration": 12, "trial_id": "3ab2f301", "date": "2024-06-26_01-26-27", "time_this_iter_s": 50.864012002944946, "time_total_s": 662.372775554657, "pid": 2850562, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 2, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 2048, "decoder_rt_ff": 2048, "decoder_int_ff": 512, "n_head": 1, "drop_rate": 0.034481700426631456, "lr": 0.0010186128360476897, "batch_size": 2048}, "time_since_restore": 115.36652421951294, "iterations_since_restore": 2}
-{"loss": 94.78943622018409, "timestamp": 1719358039, "checkpoint_dir_name": "checkpoint_000012", "should_checkpoint": true, "done": false, "training_iteration": 13, "trial_id": "3ab2f301", "date": "2024-06-26_01-27-19", "time_this_iter_s": 51.64423751831055, "time_total_s": 714.0170130729675, "pid": 2850562, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 2, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 2048, "decoder_rt_ff": 2048, "decoder_int_ff": 512, "n_head": 1, "drop_rate": 0.034481700426631456, "lr": 0.0010186128360476897, "batch_size": 2048}, "time_since_restore": 167.0107617378235, "iterations_since_restore": 3}
-{"loss": 91.46543307567205, "timestamp": 1719358090, "checkpoint_dir_name": "checkpoint_000013", "should_checkpoint": true, "done": false, "training_iteration": 14, "trial_id": "3ab2f301", "date": "2024-06-26_01-28-10", "time_this_iter_s": 51.174827098846436, "time_total_s": 765.191840171814, "pid": 2850562, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 2, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 2048, "decoder_rt_ff": 2048, "decoder_int_ff": 512, "n_head": 1, "drop_rate": 0.034481700426631456, "lr": 0.0010186128360476897, "batch_size": 2048}, "time_since_restore": 218.18558883666992, "iterations_since_restore": 4}
-{"loss": 88.73407943605439, "timestamp": 1719358141, "checkpoint_dir_name": "checkpoint_000014", "should_checkpoint": true, "done": false, "training_iteration": 15, "trial_id": "3ab2f301", "date": "2024-06-26_01-29-01", "time_this_iter_s": 50.784724712371826, "time_total_s": 815.9765648841858, "pid": 2850562, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 2, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 2048, "decoder_rt_ff": 2048, "decoder_int_ff": 512, "n_head": 1, "drop_rate": 0.034481700426631456, "lr": 0.0010186128360476897, "batch_size": 2048}, "time_since_restore": 268.97031354904175, "iterations_since_restore": 5}
-{"loss": 94.00536857064314, "timestamp": 1719358192, "checkpoint_dir_name": "checkpoint_000015", "should_checkpoint": true, "done": false, "training_iteration": 16, "trial_id": "3ab2f301", "date": "2024-06-26_01-29-52", "time_this_iter_s": 51.60023093223572, "time_total_s": 867.5767958164215, "pid": 2850562, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 2, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 2048, "decoder_rt_ff": 2048, "decoder_int_ff": 512, "n_head": 1, "drop_rate": 0.034481700426631456, "lr": 0.0010186128360476897, "batch_size": 2048}, "time_since_restore": 320.57054448127747, "iterations_since_restore": 6}
-{"loss": 87.47035950187623, "timestamp": 1719358244, "checkpoint_dir_name": "checkpoint_000016", "should_checkpoint": true, "done": false, "training_iteration": 17, "trial_id": "3ab2f301", "date": "2024-06-26_01-30-44", "time_this_iter_s": 51.84270000457764, "time_total_s": 919.4194958209991, "pid": 2850562, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 2, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 2048, "decoder_rt_ff": 2048, "decoder_int_ff": 512, "n_head": 1, "drop_rate": 0.034481700426631456, "lr": 0.0010186128360476897, "batch_size": 2048}, "time_since_restore": 372.4132444858551, "iterations_since_restore": 7}
-{"loss": 88.06384884090875, "timestamp": 1719358295, "checkpoint_dir_name": "checkpoint_000017", "should_checkpoint": true, "done": false, "training_iteration": 18, "trial_id": "3ab2f301", "date": "2024-06-26_01-31-35", "time_this_iter_s": 51.09972786903381, "time_total_s": 970.519223690033, "pid": 2850562, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 2, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 2048, "decoder_rt_ff": 2048, "decoder_int_ff": 512, "n_head": 1, "drop_rate": 0.034481700426631456, "lr": 0.0010186128360476897, "batch_size": 2048}, "time_since_restore": 423.5129723548889, "iterations_since_restore": 8}
-{"loss": 87.50748780017763, "timestamp": 1719358346, "checkpoint_dir_name": "checkpoint_000018", "should_checkpoint": true, "done": false, "training_iteration": 19, "trial_id": "3ab2f301", "date": "2024-06-26_01-32-26", "time_this_iter_s": 51.09087514877319, "time_total_s": 1021.6100988388062, "pid": 2850562, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 2, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 2048, "decoder_rt_ff": 2048, "decoder_int_ff": 512, "n_head": 1, "drop_rate": 0.034481700426631456, "lr": 0.0010186128360476897, "batch_size": 2048}, "time_since_restore": 474.6038475036621, "iterations_since_restore": 9}
-{"loss": 84.81797225832001, "timestamp": 1719358398, "checkpoint_dir_name": "checkpoint_000019", "should_checkpoint": true, "done": false, "training_iteration": 20, "trial_id": "3ab2f301", "date": "2024-06-26_01-33-18", "time_this_iter_s": 51.372178077697754, "time_total_s": 1072.982276916504, "pid": 2850562, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 2, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 2048, "decoder_rt_ff": 2048, "decoder_int_ff": 512, "n_head": 1, "drop_rate": 0.034481700426631456, "lr": 0.0010186128360476897, "batch_size": 2048}, "time_since_restore": 525.9760255813599, "iterations_since_restore": 10}

ray_results/test_experiment/train_model_3c9a1c0c_10_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-27_14-37-40/params.json

-{
-  "batch_size": 1024,
-  "decoder_int_ff": 512,
-  "decoder_int_num_layer": 1,
-  "decoder_rt_ff": 512,
-  "decoder_rt_num_layer": 2,
-  "drop_rate": 0.3031602001882847,
-  "embedding_dim": 1024,
-  "encoder_ff": 1024,
-  "encoder_num_layer": 1,
-  "lr": 0.019401170360006482,
-  "n_head": 2
-}
\ No newline at end of file

ray_results/test_experiment/train_model_3c9a1c0c_10_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-27_14-37-40/progress.csv

-loss,timestamp,checkpoint_dir_name,should_checkpoint,done,training_iteration,trial_id,date,time_this_iter_s,time_total_s,pid,hostname,node_ip,time_since_restore,iterations_since_restore
-4021.740253598671,1719492153,checkpoint_000000,True,False,1,3c9a1c0c,2024-06-27_14-42-33,270.95838141441345,270.95838141441345,69992,r8i6n8,10.159.28.66,270.95838141441345,1

ray_results/test_experiment/train_model_3c9a1c0c_10_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-27_14-37-40/result.json

-{"loss": 4021.740253598671, "timestamp": 1719492153, "checkpoint_dir_name": "checkpoint_000000", "should_checkpoint": true, "done": false, "training_iteration": 1, "trial_id": "3c9a1c0c", "date": "2024-06-27_14-42-33", "time_this_iter_s": 270.95838141441345, "time_total_s": 270.95838141441345, "pid": 69992, "hostname": "r8i6n8", "node_ip": "10.159.28.66", "config": {"encoder_num_layer": 1, "decoder_rt_num_layer": 2, "decoder_int_num_layer": 1, "embedding_dim": 1024, "encoder_ff": 1024, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 2, "drop_rate": 0.3031602001882847, "lr": 0.019401170360006482, "batch_size": 1024}, "time_since_restore": 270.95838141441345, "iterations_since_restore": 1}

ray_results/test_experiment/train_model_3d8a1340_55_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=8,drop__2024-06-28_06-10-06/params.json

-{
-  "batch_size": 1024,
-  "decoder_int_ff": 512,
-  "decoder_int_num_layer": 1,
-  "decoder_rt_ff": 2048,
-  "decoder_rt_num_layer": 8,
-  "drop_rate": 0.9833194838066408,
-  "embedding_dim": 16,
-  "encoder_ff": 512,
-  "encoder_num_layer": 2,
-  "lr": 0.0004542749659626088,
-  "n_head": 2
-}
\ No newline at end of file

ray_results/test_experiment/train_model_3d8a1340_55_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=8,drop__2024-06-28_06-10-06/progress.csv

-loss,timestamp,checkpoint_dir_name,should_checkpoint,done,training_iteration,trial_id,date,time_this_iter_s,time_total_s,pid,hostname,node_ip,time_since_restore,iterations_since_restore
-nan,1719548436,checkpoint_000000,True,False,1,3d8a1340,2024-06-28_06-20-36,91.73844981193542,91.73844981193542,242786,r8i6n8,10.159.28.66,91.73844981193542,1
-nan,1719548515,checkpoint_000001,True,False,2,3d8a1340,2024-06-28_06-21-55,78.92051219940186,170.65896201133728,242786,r8i6n8,10.159.28.66,170.65896201133728,2
-nan,1719548594,checkpoint_000002,True,False,3,3d8a1340,2024-06-28_06-23-14,78.98558306694031,249.6445450782776,242786,r8i6n8,10.159.28.66,249.6445450782776,3
-nan,1719548672,checkpoint_000003,True,False,4,3d8a1340,2024-06-28_06-24-32,78.50712847709656,328.15167355537415,242786,r8i6n8,10.159.28.66,328.15167355537415,4
-nan,1719548751,checkpoint_000004,True,False,5,3d8a1340,2024-06-28_06-25-51,78.95465588569641,407.10632944107056,242786,r8i6n8,10.159.28.66,407.10632944107056,5
-nan,1719548830,checkpoint_000005,True,False,6,3d8a1340,2024-06-28_06-27-10,79.0093162059784,486.11564564704895,242786,r8i6n8,10.159.28.66,486.11564564704895,6
-nan,1719548909,checkpoint_000006,True,False,7,3d8a1340,2024-06-28_06-28-29,78.55737948417664,564.6730251312256,242786,r8i6n8,10.159.28.66,564.6730251312256,7
-nan,1719548988,checkpoint_000007,True,False,8,3d8a1340,2024-06-28_06-29-48,79.00080251693726,643.6738276481628,242786,r8i6n8,10.159.28.66,643.6738276481628,8
-nan,1719549067,checkpoint_000008,True,False,9,3d8a1340,2024-06-28_06-31-07,78.99766731262207,722.6714949607849,242786,r8i6n8,10.159.28.66,722.6714949607849,9
-nan,1719549146,checkpoint_000009,True,False,10,3d8a1340,2024-06-28_06-32-26,79.03019595146179,801.7016909122467,242786,r8i6n8,10.159.28.66,801.7016909122467,10

ray_results/test_experiment/train_model_3d8a1340_55_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=8,drop__2024-06-28_06-10-06/result.json

-{"loss": NaN, "timestamp": 1719548436, "checkpoint_dir_name": "checkpoint_000000", "should_checkpoint": true, "done": false, "training_iteration": 1, "trial_id": "3d8a1340", "date": "2024-06-28_06-20-36", "time_this_iter_s": 91.73844981193542, "time_total_s": 91.73844981193542, "pid": 242786, "hostname": "r8i6n8", "node_ip": "10.159.28.66", "config": {"encoder_num_layer": 2, "decoder_rt_num_layer": 8, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 512, "decoder_rt_ff": 2048, "decoder_int_ff": 512, "n_head": 2, "drop_rate": 0.9833194838066408, "lr": 0.0004542749659626088, "batch_size": 1024}, "time_since_restore": 91.73844981193542, "iterations_since_restore": 1}
-{"loss": NaN, "timestamp": 1719548515, "checkpoint_dir_name": "checkpoint_000001", "should_checkpoint": true, "done": false, "training_iteration": 2, "trial_id": "3d8a1340", "date": "2024-06-28_06-21-55", "time_this_iter_s": 78.92051219940186, "time_total_s": 170.65896201133728, "pid": 242786, "hostname": "r8i6n8", "node_ip": "10.159.28.66", "config": {"encoder_num_layer": 2, "decoder_rt_num_layer": 8, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 512, "decoder_rt_ff": 2048, "decoder_int_ff": 512, "n_head": 2, "drop_rate": 0.9833194838066408, "lr": 0.0004542749659626088, "batch_size": 1024}, "time_since_restore": 170.65896201133728, "iterations_since_restore": 2}
-{"loss": NaN, "timestamp": 1719548594, "checkpoint_dir_name": "checkpoint_000002", "should_checkpoint": true, "done": false, "training_iteration": 3, "trial_id": "3d8a1340", "date": "2024-06-28_06-23-14", "time_this_iter_s": 78.98558306694031, "time_total_s": 249.6445450782776, "pid": 242786, "hostname": "r8i6n8", "node_ip": "10.159.28.66", "config": {"encoder_num_layer": 2, "decoder_rt_num_layer": 8, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 512, "decoder_rt_ff": 2048, "decoder_int_ff": 512, "n_head": 2, "drop_rate": 0.9833194838066408, "lr": 0.0004542749659626088, "batch_size": 1024}, "time_since_restore": 249.6445450782776, "iterations_since_restore": 3}
-{"loss": NaN, "timestamp": 1719548672, "checkpoint_dir_name": "checkpoint_000003", "should_checkpoint": true, "done": false, "training_iteration": 4, "trial_id": "3d8a1340", "date": "2024-06-28_06-24-32", "time_this_iter_s": 78.50712847709656, "time_total_s": 328.15167355537415, "pid": 242786, "hostname": "r8i6n8", "node_ip": "10.159.28.66", "config": {"encoder_num_layer": 2, "decoder_rt_num_layer": 8, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 512, "decoder_rt_ff": 2048, "decoder_int_ff": 512, "n_head": 2, "drop_rate": 0.9833194838066408, "lr": 0.0004542749659626088, "batch_size": 1024}, "time_since_restore": 328.15167355537415, "iterations_since_restore": 4}
-{"loss": NaN, "timestamp": 1719548751, "checkpoint_dir_name": "checkpoint_000004", "should_checkpoint": true, "done": false, "training_iteration": 5, "trial_id": "3d8a1340", "date": "2024-06-28_06-25-51", "time_this_iter_s": 78.95465588569641, "time_total_s": 407.10632944107056, "pid": 242786, "hostname": "r8i6n8", "node_ip": "10.159.28.66", "config": {"encoder_num_layer": 2, "decoder_rt_num_layer": 8, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 512, "decoder_rt_ff": 2048, "decoder_int_ff": 512, "n_head": 2, "drop_rate": 0.9833194838066408, "lr": 0.0004542749659626088, "batch_size": 1024}, "time_since_restore": 407.10632944107056, "iterations_since_restore": 5}
-{"loss": NaN, "timestamp": 1719548830, "checkpoint_dir_name": "checkpoint_000005", "should_checkpoint": true, "done": false, "training_iteration": 6, "trial_id": "3d8a1340", "date": "2024-06-28_06-27-10", "time_this_iter_s": 79.0093162059784, "time_total_s": 486.11564564704895, "pid": 242786, "hostname": "r8i6n8", "node_ip": "10.159.28.66", "config": {"encoder_num_layer": 2, "decoder_rt_num_layer": 8, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 512, "decoder_rt_ff": 2048, "decoder_int_ff": 512, "n_head": 2, "drop_rate": 0.9833194838066408, "lr": 0.0004542749659626088, "batch_size": 1024}, "time_since_restore": 486.11564564704895, "iterations_since_restore": 6}
-{"loss": NaN, "timestamp": 1719548909, "checkpoint_dir_name": "checkpoint_000006", "should_checkpoint": true, "done": false, "training_iteration": 7, "trial_id": "3d8a1340", "date": "2024-06-28_06-28-29", "time_this_iter_s": 78.55737948417664, "time_total_s": 564.6730251312256, "pid": 242786, "hostname": "r8i6n8", "node_ip": "10.159.28.66", "config": {"encoder_num_layer": 2, "decoder_rt_num_layer": 8, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 512, "decoder_rt_ff": 2048, "decoder_int_ff": 512, "n_head": 2, "drop_rate": 0.9833194838066408, "lr": 0.0004542749659626088, "batch_size": 1024}, "time_since_restore": 564.6730251312256, "iterations_since_restore": 7}
-{"loss": NaN, "timestamp": 1719548988, "checkpoint_dir_name": "checkpoint_000007", "should_checkpoint": true, "done": false, "training_iteration": 8, "trial_id": "3d8a1340", "date": "2024-06-28_06-29-48", "time_this_iter_s": 79.00080251693726, "time_total_s": 643.6738276481628, "pid": 242786, "hostname": "r8i6n8", "node_ip": "10.159.28.66", "config": {"encoder_num_layer": 2, "decoder_rt_num_layer": 8, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 512, "decoder_rt_ff": 2048, "decoder_int_ff": 512, "n_head": 2, "drop_rate": 0.9833194838066408, "lr": 0.0004542749659626088, "batch_size": 1024}, "time_since_restore": 643.6738276481628, "iterations_since_restore": 8}
-{"loss": NaN, "timestamp": 1719549067, "checkpoint_dir_name": "checkpoint_000008", "should_checkpoint": true, "done": false, "training_iteration": 9, "trial_id": "3d8a1340", "date": "2024-06-28_06-31-07", "time_this_iter_s": 78.99766731262207, "time_total_s": 722.6714949607849, "pid": 242786, "hostname": "r8i6n8", "node_ip": "10.159.28.66", "config": {"encoder_num_layer": 2, "decoder_rt_num_layer": 8, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 512, "decoder_rt_ff": 2048, "decoder_int_ff": 512, "n_head": 2, "drop_rate": 0.9833194838066408, "lr": 0.0004542749659626088, "batch_size": 1024}, "time_since_restore": 722.6714949607849, "iterations_since_restore": 9}
-{"loss": NaN, "timestamp": 1719549146, "checkpoint_dir_name": "checkpoint_000009", "should_checkpoint": true, "done": false, "training_iteration": 10, "trial_id": "3d8a1340", "date": "2024-06-28_06-32-26", "time_this_iter_s": 79.03019595146179, "time_total_s": 801.7016909122467, "pid": 242786, "hostname": "r8i6n8", "node_ip": "10.159.28.66", "config": {"encoder_num_layer": 2, "decoder_rt_num_layer": 8, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 512, "decoder_rt_ff": 2048, "decoder_int_ff": 512, "n_head": 2, "drop_rate": 0.9833194838066408, "lr": 0.0004542749659626088, "batch_size": 1024}, "time_since_restore": 801.7016909122467, "iterations_since_restore": 10}

ray_results/test_experiment/train_model_3f37f45c_7_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=2,drop_r_2024-06-25_17-29-36/params.json

-{
-  "batch_size": 1024,
-  "decoder_int_ff": 512,
-  "decoder_int_num_layer": 1,
-  "decoder_rt_ff": 2048,
-  "decoder_rt_num_layer": 2,
-  "drop_rate": 0.6691316467787182,
-  "embedding_dim": 256,
-  "encoder_ff": 512,
-  "encoder_num_layer": 1,
-  "lr": 0.004731694425031267,
-  "n_head": 4
-}
\ No newline at end of file

ray_results/test_experiment/train_model_3f37f45c_7_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=2,drop_r_2024-06-25_17-29-36/progress.csv

-loss,timestamp,checkpoint_dir_name,should_checkpoint,done,training_iteration,trial_id,date,time_this_iter_s,time_total_s,pid,hostname,node_ip,time_since_restore,iterations_since_restore
-2052.4382574126475,1719329544,checkpoint_000000,True,False,1,3f37f45c,2024-06-25_17-32-25,83.68853616714478,83.68853616714478,2195253,r8i6n2,10.159.28.60,83.68853616714478,1

ray_results/test_experiment/train_model_3f37f45c_7_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=2,drop_r_2024-06-25_17-29-36/result.json

-{"loss": 2052.4382574126475, "timestamp": 1719329544, "checkpoint_dir_name": "checkpoint_000000", "should_checkpoint": true, "done": false, "training_iteration": 1, "trial_id": "3f37f45c", "date": "2024-06-25_17-32-25", "time_this_iter_s": 83.68853616714478, "time_total_s": 83.68853616714478, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 1, "decoder_rt_num_layer": 2, "decoder_int_num_layer": 1, "embedding_dim": 256, "encoder_ff": 512, "decoder_rt_ff": 2048, "decoder_int_ff": 512, "n_head": 4, "drop_rate": 0.6691316467787182, "lr": 0.004731694425031267, "batch_size": 1024}, "time_since_restore": 83.68853616714478, "iterations_since_restore": 1}

ray_results/test_experiment/train_model_40101b9a_29_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=1,drop_r_2024-06-25_22-15-47/params.json

-{
-  "batch_size": 2048,
-  "decoder_int_ff": 512,
-  "decoder_int_num_layer": 1,
-  "decoder_rt_ff": 512,
-  "decoder_rt_num_layer": 1,
-  "drop_rate": 0.8983976265900149,
-  "embedding_dim": 1024,
-  "encoder_ff": 2048,
-  "encoder_num_layer": 2,
-  "lr": 0.039339947723305395,
-  "n_head": 1
-}
\ No newline at end of file