del raysesult

ray_results/test_experiment/train_model_a01c9b2d_16_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_17-44-50/result.json

-{"loss": 983.9757466954509, "timestamp": 1719330446, "checkpoint_dir_name": "checkpoint_000000", "should_checkpoint": true, "done": false, "training_iteration": 1, "trial_id": "a01c9b2d", "date": "2024-06-25_17-47-26", "time_this_iter_s": 83.82870721817017, "time_total_s": 83.82870721817017, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 8, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 1024, "decoder_rt_ff": 1024, "decoder_int_ff": 512, "n_head": 4, "drop_rate": 0.23338023981015343, "lr": 0.00010269515886648154, "batch_size": 1024}, "time_since_restore": 83.82870721817017, "iterations_since_restore": 1}
-{"loss": 501.4262033297321, "timestamp": 1719330517, "checkpoint_dir_name": "checkpoint_000001", "should_checkpoint": true, "done": false, "training_iteration": 2, "trial_id": "a01c9b2d", "date": "2024-06-25_17-48-37", "time_this_iter_s": 70.83729195594788, "time_total_s": 154.66599917411804, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 8, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 1024, "decoder_rt_ff": 1024, "decoder_int_ff": 512, "n_head": 4, "drop_rate": 0.23338023981015343, "lr": 0.00010269515886648154, "batch_size": 1024}, "time_since_restore": 154.66599917411804, "iterations_since_restore": 2}
-{"loss": 362.6629641074834, "timestamp": 1719331736, "checkpoint_dir_name": "checkpoint_000002", "should_checkpoint": true, "done": false, "training_iteration": 3, "trial_id": "a01c9b2d", "date": "2024-06-25_18-08-56", "time_this_iter_s": 101.81216025352478, "time_total_s": 256.4781594276428, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 8, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 1024, "decoder_rt_ff": 1024, "decoder_int_ff": 512, "n_head": 4, "drop_rate": 0.23338023981015343, "lr": 0.00010269515886648154, "batch_size": 1024}, "time_since_restore": 101.81216025352478, "iterations_since_restore": 1}
-{"loss": 290.2177711246521, "timestamp": 1719331824, "checkpoint_dir_name": "checkpoint_000003", "should_checkpoint": true, "done": true, "training_iteration": 4, "trial_id": "a01c9b2d", "date": "2024-06-25_18-10-24", "time_this_iter_s": 87.97965884208679, "time_total_s": 344.4578182697296, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 8, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 1024, "decoder_rt_ff": 1024, "decoder_int_ff": 512, "n_head": 4, "drop_rate": 0.23338023981015343, "lr": 0.00010269515886648154, "batch_size": 1024}, "time_since_restore": 189.79181909561157, "iterations_since_restore": 2}

ray_results/test_experiment/train_model_a0e8b2af_1_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=8,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop_r_2024-06-25_16-07-48/params.json

-{
-  "batch_size": 1024,
-  "decoder_int_ff": 512,
-  "decoder_int_num_layer": 8,
-  "decoder_rt_ff": 1024,
-  "decoder_rt_num_layer": 1,
-  "drop_rate": 0.4534009171604666,
-  "embedding_dim": 256,
-  "encoder_ff": 2048,
-  "encoder_num_layer": 2,
-  "lr": 0.019472817375775914,
-  "n_head": 8
-}
\ No newline at end of file

ray_results/test_experiment/train_model_a0e8b2af_1_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=8,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop_r_2024-06-25_16-07-48/progress.csv

-loss,timestamp,checkpoint_dir_name,should_checkpoint,done,training_iteration,trial_id,date,time_this_iter_s,time_total_s,pid,hostname,node_ip,time_since_restore,iterations_since_restore
-2773.484795037217,1719324574,checkpoint_000000,True,False,1,a0e8b2af,2024-06-25_16-09-34,89.29933595657349,89.29933595657349,69318,r3i5n6,10.159.8.159,89.29933595657349,1

ray_results/test_experiment/train_model_a0e8b2af_1_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=8,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop_r_2024-06-25_16-07-48/result.json

-{"loss": 2773.484795037217, "timestamp": 1719324574, "checkpoint_dir_name": "checkpoint_000000", "should_checkpoint": true, "done": false, "training_iteration": 1, "trial_id": "a0e8b2af", "date": "2024-06-25_16-09-34", "time_this_iter_s": 89.29933595657349, "time_total_s": 89.29933595657349, "pid": 69318, "hostname": "r3i5n6", "node_ip": "10.159.8.159", "config": {"encoder_num_layer": 2, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 8, "embedding_dim": 256, "encoder_ff": 2048, "decoder_rt_ff": 1024, "decoder_int_ff": 512, "n_head": 8, "drop_rate": 0.4534009171604666, "lr": 0.019472817375775914, "batch_size": 1024}, "time_since_restore": 89.29933595657349, "iterations_since_restore": 1}

ray_results/test_experiment/train_model_a22bbb57_47_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=4,drop__2024-06-28_01-43-12/params.json

-{
-  "batch_size": 1024,
-  "decoder_int_ff": 512,
-  "decoder_int_num_layer": 1,
-  "decoder_rt_ff": 2048,
-  "decoder_rt_num_layer": 4,
-  "drop_rate": 0.9881911141872612,
-  "embedding_dim": 16,
-  "encoder_ff": 2048,
-  "encoder_num_layer": 4,
-  "lr": 0.005037349286937313,
-  "n_head": 4
-}
\ No newline at end of file

ray_results/test_experiment/train_model_a22bbb57_47_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=4,drop__2024-06-28_01-43-12/progress.csv

-loss,timestamp,checkpoint_dir_name,should_checkpoint,done,training_iteration,trial_id,date,time_this_iter_s,time_total_s,pid,hostname,node_ip,time_since_restore,iterations_since_restore
-nan,1719532607,checkpoint_000000,True,False,1,a22bbb57,2024-06-28_01-56-47,89.77370476722717,89.77370476722717,242786,r8i6n8,10.159.28.66,89.77370476722717,1
-nan,1719532684,checkpoint_000001,True,False,2,a22bbb57,2024-06-28_01-58-04,76.8076319694519,166.58133673667908,242786,r8i6n8,10.159.28.66,166.58133673667908,2
-nan,1719532761,checkpoint_000002,True,False,3,a22bbb57,2024-06-28_01-59-21,76.79015970230103,243.3714964389801,242786,r8i6n8,10.159.28.66,243.3714964389801,3
-nan,1719532838,checkpoint_000003,True,False,4,a22bbb57,2024-06-28_02-00-38,76.78578019142151,320.1572766304016,242786,r8i6n8,10.159.28.66,320.1572766304016,4
-nan,1719532914,checkpoint_000004,True,False,5,a22bbb57,2024-06-28_02-01-54,76.73958778381348,396.8968644142151,242786,r8i6n8,10.159.28.66,396.8968644142151,5
-nan,1719532991,checkpoint_000005,True,False,6,a22bbb57,2024-06-28_02-03-11,76.74712491035461,473.6439893245697,242786,r8i6n8,10.159.28.66,473.6439893245697,6
-nan,1719533067,checkpoint_000006,True,False,7,a22bbb57,2024-06-28_02-04-27,75.64315581321716,549.2871451377869,242786,r8i6n8,10.159.28.66,549.2871451377869,7
-nan,1719533142,checkpoint_000007,True,False,8,a22bbb57,2024-06-28_02-05-42,75.23933148384094,624.5264766216278,242786,r8i6n8,10.159.28.66,624.5264766216278,8
-nan,1719533216,checkpoint_000008,True,False,9,a22bbb57,2024-06-28_02-06-56,74.07769989967346,698.6041765213013,242786,r8i6n8,10.159.28.66,698.6041765213013,9
-nan,1719533291,checkpoint_000009,True,False,10,a22bbb57,2024-06-28_02-08-11,75.01777005195618,773.6219465732574,242786,r8i6n8,10.159.28.66,773.6219465732574,10

ray_results/test_experiment/train_model_a22bbb57_47_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=4,drop__2024-06-28_01-43-12/result.json

-{"loss": NaN, "timestamp": 1719532607, "checkpoint_dir_name": "checkpoint_000000", "should_checkpoint": true, "done": false, "training_iteration": 1, "trial_id": "a22bbb57", "date": "2024-06-28_01-56-47", "time_this_iter_s": 89.77370476722717, "time_total_s": 89.77370476722717, "pid": 242786, "hostname": "r8i6n8", "node_ip": "10.159.28.66", "config": {"encoder_num_layer": 4, "decoder_rt_num_layer": 4, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 2048, "decoder_rt_ff": 2048, "decoder_int_ff": 512, "n_head": 4, "drop_rate": 0.9881911141872612, "lr": 0.005037349286937313, "batch_size": 1024}, "time_since_restore": 89.77370476722717, "iterations_since_restore": 1}
-{"loss": NaN, "timestamp": 1719532684, "checkpoint_dir_name": "checkpoint_000001", "should_checkpoint": true, "done": false, "training_iteration": 2, "trial_id": "a22bbb57", "date": "2024-06-28_01-58-04", "time_this_iter_s": 76.8076319694519, "time_total_s": 166.58133673667908, "pid": 242786, "hostname": "r8i6n8", "node_ip": "10.159.28.66", "config": {"encoder_num_layer": 4, "decoder_rt_num_layer": 4, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 2048, "decoder_rt_ff": 2048, "decoder_int_ff": 512, "n_head": 4, "drop_rate": 0.9881911141872612, "lr": 0.005037349286937313, "batch_size": 1024}, "time_since_restore": 166.58133673667908, "iterations_since_restore": 2}
-{"loss": NaN, "timestamp": 1719532761, "checkpoint_dir_name": "checkpoint_000002", "should_checkpoint": true, "done": false, "training_iteration": 3, "trial_id": "a22bbb57", "date": "2024-06-28_01-59-21", "time_this_iter_s": 76.79015970230103, "time_total_s": 243.3714964389801, "pid": 242786, "hostname": "r8i6n8", "node_ip": "10.159.28.66", "config": {"encoder_num_layer": 4, "decoder_rt_num_layer": 4, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 2048, "decoder_rt_ff": 2048, "decoder_int_ff": 512, "n_head": 4, "drop_rate": 0.9881911141872612, "lr": 0.005037349286937313, "batch_size": 1024}, "time_since_restore": 243.3714964389801, "iterations_since_restore": 3}
-{"loss": NaN, "timestamp": 1719532838, "checkpoint_dir_name": "checkpoint_000003", "should_checkpoint": true, "done": false, "training_iteration": 4, "trial_id": "a22bbb57", "date": "2024-06-28_02-00-38", "time_this_iter_s": 76.78578019142151, "time_total_s": 320.1572766304016, "pid": 242786, "hostname": "r8i6n8", "node_ip": "10.159.28.66", "config": {"encoder_num_layer": 4, "decoder_rt_num_layer": 4, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 2048, "decoder_rt_ff": 2048, "decoder_int_ff": 512, "n_head": 4, "drop_rate": 0.9881911141872612, "lr": 0.005037349286937313, "batch_size": 1024}, "time_since_restore": 320.1572766304016, "iterations_since_restore": 4}
-{"loss": NaN, "timestamp": 1719532914, "checkpoint_dir_name": "checkpoint_000004", "should_checkpoint": true, "done": false, "training_iteration": 5, "trial_id": "a22bbb57", "date": "2024-06-28_02-01-54", "time_this_iter_s": 76.73958778381348, "time_total_s": 396.8968644142151, "pid": 242786, "hostname": "r8i6n8", "node_ip": "10.159.28.66", "config": {"encoder_num_layer": 4, "decoder_rt_num_layer": 4, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 2048, "decoder_rt_ff": 2048, "decoder_int_ff": 512, "n_head": 4, "drop_rate": 0.9881911141872612, "lr": 0.005037349286937313, "batch_size": 1024}, "time_since_restore": 396.8968644142151, "iterations_since_restore": 5}
-{"loss": NaN, "timestamp": 1719532991, "checkpoint_dir_name": "checkpoint_000005", "should_checkpoint": true, "done": false, "training_iteration": 6, "trial_id": "a22bbb57", "date": "2024-06-28_02-03-11", "time_this_iter_s": 76.74712491035461, "time_total_s": 473.6439893245697, "pid": 242786, "hostname": "r8i6n8", "node_ip": "10.159.28.66", "config": {"encoder_num_layer": 4, "decoder_rt_num_layer": 4, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 2048, "decoder_rt_ff": 2048, "decoder_int_ff": 512, "n_head": 4, "drop_rate": 0.9881911141872612, "lr": 0.005037349286937313, "batch_size": 1024}, "time_since_restore": 473.6439893245697, "iterations_since_restore": 6}
-{"loss": NaN, "timestamp": 1719533067, "checkpoint_dir_name": "checkpoint_000006", "should_checkpoint": true, "done": false, "training_iteration": 7, "trial_id": "a22bbb57", "date": "2024-06-28_02-04-27", "time_this_iter_s": 75.64315581321716, "time_total_s": 549.2871451377869, "pid": 242786, "hostname": "r8i6n8", "node_ip": "10.159.28.66", "config": {"encoder_num_layer": 4, "decoder_rt_num_layer": 4, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 2048, "decoder_rt_ff": 2048, "decoder_int_ff": 512, "n_head": 4, "drop_rate": 0.9881911141872612, "lr": 0.005037349286937313, "batch_size": 1024}, "time_since_restore": 549.2871451377869, "iterations_since_restore": 7}
-{"loss": NaN, "timestamp": 1719533142, "checkpoint_dir_name": "checkpoint_000007", "should_checkpoint": true, "done": false, "training_iteration": 8, "trial_id": "a22bbb57", "date": "2024-06-28_02-05-42", "time_this_iter_s": 75.23933148384094, "time_total_s": 624.5264766216278, "pid": 242786, "hostname": "r8i6n8", "node_ip": "10.159.28.66", "config": {"encoder_num_layer": 4, "decoder_rt_num_layer": 4, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 2048, "decoder_rt_ff": 2048, "decoder_int_ff": 512, "n_head": 4, "drop_rate": 0.9881911141872612, "lr": 0.005037349286937313, "batch_size": 1024}, "time_since_restore": 624.5264766216278, "iterations_since_restore": 8}
-{"loss": NaN, "timestamp": 1719533216, "checkpoint_dir_name": "checkpoint_000008", "should_checkpoint": true, "done": false, "training_iteration": 9, "trial_id": "a22bbb57", "date": "2024-06-28_02-06-56", "time_this_iter_s": 74.07769989967346, "time_total_s": 698.6041765213013, "pid": 242786, "hostname": "r8i6n8", "node_ip": "10.159.28.66", "config": {"encoder_num_layer": 4, "decoder_rt_num_layer": 4, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 2048, "decoder_rt_ff": 2048, "decoder_int_ff": 512, "n_head": 4, "drop_rate": 0.9881911141872612, "lr": 0.005037349286937313, "batch_size": 1024}, "time_since_restore": 698.6041765213013, "iterations_since_restore": 9}
-{"loss": NaN, "timestamp": 1719533291, "checkpoint_dir_name": "checkpoint_000009", "should_checkpoint": true, "done": false, "training_iteration": 10, "trial_id": "a22bbb57", "date": "2024-06-28_02-08-11", "time_this_iter_s": 75.01777005195618, "time_total_s": 773.6219465732574, "pid": 242786, "hostname": "r8i6n8", "node_ip": "10.159.28.66", "config": {"encoder_num_layer": 4, "decoder_rt_num_layer": 4, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 2048, "decoder_rt_ff": 2048, "decoder_int_ff": 512, "n_head": 4, "drop_rate": 0.9881911141872612, "lr": 0.005037349286937313, "batch_size": 1024}, "time_since_restore": 773.6219465732574, "iterations_since_restore": 10}

ray_results/test_experiment/train_model_a33ea899_14_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-27_14-59-41/params.json

-{
-  "batch_size": 1024,
-  "decoder_int_ff": 512,
-  "decoder_int_num_layer": 1,
-  "decoder_rt_ff": 512,
-  "decoder_rt_num_layer": 2,
-  "drop_rate": 0.8014267791772675,
-  "embedding_dim": 64,
-  "encoder_ff": 512,
-  "encoder_num_layer": 1,
-  "lr": 0.06443936649826909,
-  "n_head": 16
-}
\ No newline at end of file

ray_results/test_experiment/train_model_a33ea899_14_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-27_14-59-41/progress.csv

-loss,timestamp,checkpoint_dir_name,should_checkpoint,done,training_iteration,trial_id,date,time_this_iter_s,time_total_s,pid,hostname,node_ip,time_since_restore,iterations_since_restore
-2313.215312807579,1719493332,checkpoint_000000,True,False,1,a33ea899,2024-06-27_15-02-12,69.87668585777283,69.87668585777283,69992,r8i6n8,10.159.28.66,69.87668585777283,1

ray_results/test_experiment/train_model_a33ea899_14_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-27_14-59-41/result.json

-{"loss": 2313.215312807579, "timestamp": 1719493332, "checkpoint_dir_name": "checkpoint_000000", "should_checkpoint": true, "done": false, "training_iteration": 1, "trial_id": "a33ea899", "date": "2024-06-27_15-02-12", "time_this_iter_s": 69.87668585777283, "time_total_s": 69.87668585777283, "pid": 69992, "hostname": "r8i6n8", "node_ip": "10.159.28.66", "config": {"encoder_num_layer": 1, "decoder_rt_num_layer": 2, "decoder_int_num_layer": 1, "embedding_dim": 64, "encoder_ff": 512, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 16, "drop_rate": 0.8014267791772675, "lr": 0.06443936649826909, "batch_size": 1024}, "time_since_restore": 69.87668585777283, "iterations_since_restore": 1}

ray_results/test_experiment/train_model_aaac0ba4_53_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=8,drop_r_2024-06-28_03-39-37/params.json

-{
-  "batch_size": 2048,
-  "decoder_int_ff": 512,
-  "decoder_int_num_layer": 1,
-  "decoder_rt_ff": 512,
-  "decoder_rt_num_layer": 8,
-  "drop_rate": 0.620662951741282,
-  "embedding_dim": 1024,
-  "encoder_ff": 2048,
-  "encoder_num_layer": 2,
-  "lr": 0.00347738554616066,
-  "n_head": 4
-}
\ No newline at end of file

ray_results/test_experiment/train_model_aaac0ba4_53_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=8,drop_r_2024-06-28_03-39-37/progress.csv

-loss,timestamp,checkpoint_dir_name,should_checkpoint,done,training_iteration,trial_id,date,time_this_iter_s,time_total_s,pid,hostname,node_ip,time_since_restore,iterations_since_restore
-2050.3518681563733,1719540226,checkpoint_000000,True,False,1,aaac0ba4,2024-06-28_04-03-47,859.009360074997,859.009360074997,242786,r8i6n8,10.159.28.66,859.009360074997,1
-2052.4004069651205,1719541069,checkpoint_000001,True,False,2,aaac0ba4,2024-06-28_04-17-50,843.1840569972992,1702.1934170722961,242786,r8i6n8,10.159.28.66,1702.1934170722961,2
-2052.2435062438485,1719541912,checkpoint_000002,True,False,3,aaac0ba4,2024-06-28_04-31-53,842.8456587791443,2545.0390758514404,242786,r8i6n8,10.159.28.66,2545.0390758514404,3
-2050.0776482529527,1719542755,checkpoint_000003,True,False,4,aaac0ba4,2024-06-28_04-45-55,842.6302690505981,3387.6693449020386,242786,r8i6n8,10.159.28.66,3387.6693449020386,4
-2056.0218419352855,1719543597,checkpoint_000004,True,False,5,aaac0ba4,2024-06-28_04-59-57,841.9567799568176,4229.626124858856,242786,r8i6n8,10.159.28.66,4229.626124858856,5
-2051.6352116141734,1719544438,checkpoint_000005,True,False,6,aaac0ba4,2024-06-28_05-13-59,841.3759407997131,5071.002065658569,242786,r8i6n8,10.159.28.66,5071.002065658569,6
-2049.202928918553,1719545280,checkpoint_000006,True,False,7,aaac0ba4,2024-06-28_05-28-01,842.0156388282776,5913.017704486847,242786,r8i6n8,10.159.28.66,5913.017704486847,7
-2061.2424547090304,1719546122,checkpoint_000007,True,False,8,aaac0ba4,2024-06-28_05-42-02,841.3607151508331,6754.37841963768,242786,r8i6n8,10.159.28.66,6754.37841963768,8
-2040.9990743802289,1719546963,checkpoint_000008,True,False,9,aaac0ba4,2024-06-28_05-56-04,841.6030082702637,7595.981427907944,242786,r8i6n8,10.159.28.66,7595.981427907944,9
-2036.6472908080093,1719547805,checkpoint_000009,True,False,10,aaac0ba4,2024-06-28_06-10-05,841.7590441703796,8437.740472078323,242786,r8i6n8,10.159.28.66,8437.740472078323,10