diff --git a/ray_results/test_experiment/experiment_state-2024-06-25_15-19-46.json b/ray_results/test_experiment/experiment_state-2024-06-25_15-19-46.json
new file mode 100644
index 0000000000000000000000000000000000000000..86a72d474345b2837371dfe89ced31dd50c72253
--- /dev/null
+++ b/ray_results/test_experiment/experiment_state-2024-06-25_15-19-46.json
@@ -0,0 +1,52 @@
+{
+ "trial_data": [
+ [
+ "{\n \"stub\": false,\n \"trainable_name\": \"train_model\",\n \"trial_id\": \"dd950274\",\n \"storage\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059582030000000000008c1b7261792e747261696e2e5f696e7465726e616c2e73746f72616765948c0e53746f72616765436f6e746578749493942981947d94288c12637573746f6d5f66735f70726f766964656494898c1273746f726167655f6c6f63616c5f70617468948c2b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c7473948c136578706572696d656e745f6469725f6e616d65948c0f746573745f6578706572696d656e74948c0e747269616c5f6469725f6e616d65948c96747261696e5f6d6f64656c5f64643935303237345f315f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d313032342c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d313032342c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f5f323032342d30362d32355f31352d31392d3436948c1863757272656e745f636865636b706f696e745f696e646578944b008c0b73796e635f636f6e666967948c097261792e747261696e948c0a53796e63436f6e6669679493942981947d94288c0b73796e635f706572696f64944d2c018c0c73796e635f74696d656f7574944d08078c0e73796e635f61727469666163747394898c1c73796e635f6172746966616374735f6f6e5f636865636b706f696e7494888c0a75706c6f61645f646972948c0a44455052454341544544948c0673796e6365729468188c1273796e635f6f6e5f636865636b706f696e7494681875628c1273746f726167655f66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c0f73746f726167655f66735f70617468948c452f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f7465737494681968008c115f46696c6573797374656d53796e6365729493942981947d9428681b682268134d2c0168144d08078c116c6173745f73796e635f75705f74696d659447fff00000000000008c136c6173745f73796e635f646f776e5f74696d659447fff00000000000008c0d5f73796e635f70726f63657373944e8c0c5f63757272656e745f636d64944e756275622e\"\n },\n \"config\": {\n \"encoder_num_layer\": 8,\n \"decoder_rt_num_layer\": 1,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 16,\n \"encoder_ff\": 1024,\n \"decoder_rt_ff\": 1024,\n \"decoder_int_ff\": 1024,\n \"n_head\": 16,\n \"drop_rate\": 0.7776595209025089,\n \"lr\": 0.0006106789085082022,\n \"batch_size\": 2048\n },\n \"_Trial__unresolved_config\": {\n \"encoder_num_layer\": 8,\n \"decoder_rt_num_layer\": 1,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 16,\n \"encoder_ff\": 1024,\n \"decoder_rt_ff\": 1024,\n \"decoder_int_ff\": 1024,\n \"n_head\": 16,\n \"drop_rate\": 0.7776595209025089,\n \"lr\": 0.0006106789085082022,\n \"batch_size\": 2048\n },\n \"evaluated_params\": {\n \"batch_size\": 2048,\n \"decoder_int_ff\": 1024,\n \"decoder_int_num_layer\": 1,\n \"decoder_rt_ff\": 1024,\n \"decoder_rt_num_layer\": 1,\n \"drop_rate\": 0.7776595209025089,\n \"embedding_dim\": 16,\n \"encoder_ff\": 1024,\n \"encoder_num_layer\": 8,\n \"lr\": 0.0006106789085082022,\n \"n_head\": 16\n },\n \"experiment_tag\": \"1_batch_size=2048,decoder_int_ff=1024,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop_rate=0.7777,embedding_dim=16,encoder_ff=1024,encoder_num_layer=8,lr=0.0006,n_head=16\",\n \"stopping_criterion\": {},\n \"_setup_default_resource\": true,\n \"_default_placement_group_factory\": \"80054e2e\",\n \"placement_group_factory\": \"800595ba000000000000008c237261792e74756e652e657865637574696f6e2e706c6163656d656e745f67726f757073948c15506c6163656d656e7447726f7570466163746f72799493942981947d94288c085f62756e646c6573945d947d94288c03435055944740540000000000008c0347505594473ff000000000000075618c155f686561645f62756e646c655f69735f656d70747994898c095f7374726174656779948c045041434b948c055f6172677394298c075f6b7761726773947d9475622e\",\n \"log_to_file\": [\n null,\n null\n ],\n \"max_failures\": 0,\n \"_default_result_or_future\": null,\n \"export_formats\": [],\n \"status\": \"PAUSED\",\n \"relative_logdir\": \"train_model_dd950274_1_batch_size=2048,decoder_int_ff=1024,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_15-19-46\",\n \"trial_name_creator\": null,\n \"trial_dirname_creator\": null,\n \"custom_trial_name\": null,\n \"custom_dirname\": null,\n \"restore_path\": null,\n \"_restore_checkpoint_result\": null,\n \"_state_json\": null,\n \"results\": \"80054e2e\",\n \"extra_arg\": \"80054e2e\",\n \"_resources\": \"80054e2e\"\n}",
+ "{\n \"start_time\": 1719321603.1018882,\n \"num_failures\": 0,\n \"num_failures_after_restore\": 0,\n \"error_filename\": null,\n \"pickled_error_filename\": null,\n \"last_result\": {\n \"loss\": 2001.7704808993603,\n \"timestamp\": 1719321692,\n \"checkpoint_dir_name\": \"checkpoint_000000\",\n \"should_checkpoint\": true,\n \"done\": false,\n \"training_iteration\": 1,\n \"trial_id\": \"dd950274\",\n \"date\": \"2024-06-25_15-21-32\",\n \"time_this_iter_s\": 89.34920597076416,\n \"time_total_s\": 89.34920597076416,\n \"pid\": 355319,\n \"hostname\": \"r7i5n7\",\n \"node_ip\": \"10.159.24.56\",\n \"config\": {\n \"encoder_num_layer\": 8,\n \"decoder_rt_num_layer\": 1,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 16,\n \"encoder_ff\": 1024,\n \"decoder_rt_ff\": 1024,\n \"decoder_int_ff\": 1024,\n \"n_head\": 16,\n \"drop_rate\": 0.7776595209025089,\n \"lr\": 0.0006106789085082022,\n \"batch_size\": 2048\n },\n \"time_since_restore\": 89.34920597076416,\n \"iterations_since_restore\": 1,\n \"experiment_tag\": \"1_batch_size=2048,decoder_int_ff=1024,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop_rate=0.7777,embedding_dim=16,encoder_ff=1024,encoder_num_layer=8,lr=0.0006,n_head=16\"\n },\n \"last_result_time\": 1719321692.456364,\n \"metric_analysis\": {\n \"loss\": {\n \"max\": 2001.7704808993603,\n \"min\": 2001.7704808993603,\n \"avg\": 2001.7704808993603,\n \"last\": 2001.7704808993603,\n \"last-5-avg\": 2001.7704808993603,\n \"last-10-avg\": 2001.7704808993603\n },\n \"should_checkpoint\": {\n \"max\": true,\n \"min\": true,\n \"avg\": true,\n \"last\": true,\n \"last-5-avg\": true,\n \"last-10-avg\": true\n },\n \"done\": {\n \"max\": false,\n \"min\": false,\n \"avg\": false,\n \"last\": false,\n \"last-5-avg\": false,\n \"last-10-avg\": false\n },\n \"training_iteration\": {\n \"max\": 1,\n \"min\": 1,\n \"avg\": 1,\n \"last\": 1,\n \"last-5-avg\": 1,\n \"last-10-avg\": 1\n },\n \"time_this_iter_s\": {\n \"max\": 89.34920597076416,\n \"min\": 89.34920597076416,\n \"avg\": 89.34920597076416,\n \"last\": 89.34920597076416,\n \"last-5-avg\": 89.34920597076416,\n \"last-10-avg\": 89.34920597076416\n },\n \"time_total_s\": {\n \"max\": 89.34920597076416,\n \"min\": 89.34920597076416,\n \"avg\": 89.34920597076416,\n \"last\": 89.34920597076416,\n \"last-5-avg\": 89.34920597076416,\n \"last-10-avg\": 89.34920597076416\n },\n \"time_since_restore\": {\n \"max\": 89.34920597076416,\n \"min\": 89.34920597076416,\n \"avg\": 89.34920597076416,\n \"last\": 89.34920597076416,\n \"last-5-avg\": 89.34920597076416,\n \"last-10-avg\": 89.34920597076416\n },\n \"iterations_since_restore\": {\n \"max\": 1,\n \"min\": 1,\n \"avg\": 1,\n \"last\": 1,\n \"last-5-avg\": 1,\n \"last-10-avg\": 1\n }\n },\n \"_n_steps\": [\n 5,\n 10\n ],\n \"metric_n_steps\": {\n \"loss\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059589000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452948c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b007494624308c8e3f1f814479f409486945294612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059589000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452948c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b007494624308c8e3f1f814479f409486945294612e\"\n }\n },\n \"should_checkpoint\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059522000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b058694529488612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059522000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a8694529488612e\"\n }\n },\n \"done\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059522000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b058694529489612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059522000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a8694529489612e\"\n }\n },\n \"training_iteration\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059523000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452944b01612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059523000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452944b01612e\"\n }\n },\n \"time_this_iter_s\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294474056565964000000612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294474056565964000000612e\"\n }\n },\n \"time_total_s\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294474056565964000000612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294474056565964000000612e\"\n }\n },\n \"time_since_restore\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294474056565964000000612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294474056565964000000612e\"\n }\n },\n \"iterations_since_restore\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059523000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452944b01612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059523000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452944b01612e\"\n }\n }\n },\n \"checkpoint_manager\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059517060000000000008c267261792e747261696e2e5f696e7465726e616c2e636865636b706f696e745f6d616e61676572948c125f436865636b706f696e744d616e616765729493942981947d94288c125f636865636b706f696e745f636f6e666967948c097261792e747261696e948c10436865636b706f696e74436f6e6669679493942981947d94288c0b6e756d5f746f5f6b656570944e8c1a636865636b706f696e745f73636f72655f617474726962757465944e8c16636865636b706f696e745f73636f72655f6f72646572948c036d6178948c14636865636b706f696e745f6672657175656e6379944b008c11636865636b706f696e745f61745f656e6494898c1a5f636865636b706f696e745f6b6565705f616c6c5f72616e6b73948c0a44455052454341544544948c1f5f636865636b706f696e745f75706c6f61645f66726f6d5f776f726b65727394681275628c135f636865636b706f696e745f726573756c7473945d948c1b7261792e747261696e2e5f696e7465726e616c2e73657373696f6e948c0f5f547261696e696e67526573756c749493942981947d94288c0a636865636b706f696e749468068c0a436865636b706f696e749493942981947d94288c0470617468948cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f64643935303237345f315f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d313032342c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d313032342c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f5f323032342d30362d32355f31352d31392d34362f636865636b706f696e745f303030303030948c0a66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c055f75756964948c0475756964948c04555549449493942981947d948c03696e74948a1050959366a0fa3fa36746f99b4aeaad34736275628c076d657472696373947d94288c046c6f7373948c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b007494624308c8e3f1f814479f4094869452948c0974696d657374616d70944a5cc47a668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303030948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b018c08747269616c5f6964948c086464393530323734948c0464617465948c13323032342d30362d32355f31352d32312d3332948c1074696d655f746869735f697465725f73944740565659640000008c0c74696d655f746f74616c5f73944740565659640000008c03706964944af76b05008c08686f73746e616d65948c06723769356e37948c076e6f64655f6970948c0c31302e3135392e32342e3536948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b088c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b108c0a656e636f6465725f6666944d00048c0d6465636f6465725f72745f6666944d00048c0e6465636f6465725f696e745f6666944d00048c066e5f68656164944b108c0964726f705f7261746594473fe8e296383660b38c026c7294473f4402bef85f047c8c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f7265944740565659640000008c18697465726174696f6e735f73696e63655f726573746f7265944b01757562618c195f6c61746573745f636865636b706f696e745f726573756c7494681975622e\"\n }\n}"
+ ],
+ [
+ "{\n \"stub\": false,\n \"trainable_name\": \"train_model\",\n \"trial_id\": \"f74eb513\",\n \"storage\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059575040000000000008c1b7261792e747261696e2e5f696e7465726e616c2e73746f72616765948c0e53746f72616765436f6e746578749493942981947d94288c12637573746f6d5f66735f70726f766964656494898c1273746f726167655f6c6f63616c5f70617468948c2b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c7473948c136578706572696d656e745f6469725f6e616d65948c0f746573745f6578706572696d656e74948c0e747269616c5f6469725f6e616d65948c96747261696e5f6d6f64656c5f66373465623531335f325f62617463685f73697a653d313032342c6465636f6465725f696e745f66663d313032342c6465636f6465725f696e745f6e756d5f6c617965723d382c6465636f6465725f72745f66663d313032342c6465636f6465725f72745f6e756d5f6c617965723d322c64726f705f5f323032342d30362d32355f31352d32302d3033948c1863757272656e745f636865636b706f696e745f696e646578944affffffff8c0b73796e635f636f6e666967948c097261792e747261696e948c0a53796e63436f6e6669679493942981947d94288c0b73796e635f706572696f64944d2c018c0c73796e635f74696d656f7574944d08078c0e73796e635f61727469666163747394898c1c73796e635f6172746966616374735f6f6e5f636865636b706f696e7494888c0a75706c6f61645f646972948c0a44455052454341544544948c0673796e6365729468188c1273796e635f6f6e5f636865636b706f696e7494681875628c1273746f726167655f66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c0f73746f726167655f66735f70617468948c452f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f7465737494681968008c115f46696c6573797374656d53796e6365729493942981947d9428681b682268134d2c0168144d08078c116c6173745f73796e635f75705f74696d65944741d99eb0fc98aa368c136c6173745f73796e635f646f776e5f74696d659447fff00000000000008c0d5f73796e635f70726f63657373944e8c0c5f63757272656e745f636d649468008c125f75706c6f61645f746f5f66735f706174689493947d94288c0a6c6f63616c5f70617468948c3b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c74732f746573745f6578706572696d656e74948c0266739468228c0766735f70617468948c552f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e74948c076578636c756465945d948c0e2a2f636865636b706f696e745f2a9461758694756275622e\"\n },\n \"config\": {\n \"encoder_num_layer\": 8,\n \"decoder_rt_num_layer\": 2,\n \"decoder_int_num_layer\": 8,\n \"embedding_dim\": 1024,\n \"encoder_ff\": 2048,\n \"decoder_rt_ff\": 1024,\n \"decoder_int_ff\": 1024,\n \"n_head\": 8,\n \"drop_rate\": 0.5887719140440579,\n \"lr\": 0.009580025887992769,\n \"batch_size\": 1024\n },\n \"_Trial__unresolved_config\": {\n \"encoder_num_layer\": 8,\n \"decoder_rt_num_layer\": 2,\n \"decoder_int_num_layer\": 8,\n \"embedding_dim\": 1024,\n \"encoder_ff\": 2048,\n \"decoder_rt_ff\": 1024,\n \"decoder_int_ff\": 1024,\n \"n_head\": 8,\n \"drop_rate\": 0.5887719140440579,\n \"lr\": 0.009580025887992769,\n \"batch_size\": 1024\n },\n \"evaluated_params\": {\n \"batch_size\": 1024,\n \"decoder_int_ff\": 1024,\n \"decoder_int_num_layer\": 8,\n \"decoder_rt_ff\": 1024,\n \"decoder_rt_num_layer\": 2,\n \"drop_rate\": 0.5887719140440579,\n \"embedding_dim\": 1024,\n \"encoder_ff\": 2048,\n \"encoder_num_layer\": 8,\n \"lr\": 0.009580025887992769,\n \"n_head\": 8\n },\n \"experiment_tag\": \"2_batch_size=1024,decoder_int_ff=1024,decoder_int_num_layer=8,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop_rate=0.5888,embedding_dim=1024,encoder_ff=2048,encoder_num_layer=8,lr=0.0096,n_head=8\",\n \"stopping_criterion\": {},\n \"_setup_default_resource\": true,\n \"_default_placement_group_factory\": \"80054e2e\",\n \"placement_group_factory\": \"800595ba000000000000008c237261792e74756e652e657865637574696f6e2e706c6163656d656e745f67726f757073948c15506c6163656d656e7447726f7570466163746f72799493942981947d94288c085f62756e646c6573945d947d94288c03435055944740540000000000008c0347505594473ff000000000000075618c155f686561645f62756e646c655f69735f656d70747994898c095f7374726174656779948c045041434b948c055f6172677394298c075f6b7761726773947d9475622e\",\n \"log_to_file\": [\n null,\n null\n ],\n \"max_failures\": 0,\n \"_default_result_or_future\": null,\n \"export_formats\": [],\n \"status\": \"RUNNING\",\n \"relative_logdir\": \"train_model_f74eb513_2_batch_size=1024,decoder_int_ff=1024,decoder_int_num_layer=8,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop__2024-06-25_15-20-03\",\n \"trial_name_creator\": null,\n \"trial_dirname_creator\": null,\n \"custom_trial_name\": null,\n \"custom_dirname\": null,\n \"restore_path\": null,\n \"_restore_checkpoint_result\": null,\n \"_state_json\": null,\n \"results\": \"80054e2e\",\n \"extra_arg\": \"80054e2e\",\n \"_resources\": \"80054e2e\"\n}",
+ "{\n \"start_time\": 1719321692.6103284,\n \"num_failures\": 0,\n \"num_failures_after_restore\": 0,\n \"error_filename\": null,\n \"pickled_error_filename\": null,\n \"last_result\": {\n \"trial_id\": \"f74eb513\"\n },\n \"last_result_time\": -Infinity,\n \"metric_analysis\": {},\n \"_n_steps\": [\n 5,\n 10\n ],\n \"metric_n_steps\": {},\n \"checkpoint_manager\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059584010000000000008c267261792e747261696e2e5f696e7465726e616c2e636865636b706f696e745f6d616e61676572948c125f436865636b706f696e744d616e616765729493942981947d94288c125f636865636b706f696e745f636f6e666967948c097261792e747261696e948c10436865636b706f696e74436f6e6669679493942981947d94288c0b6e756d5f746f5f6b656570944e8c1a636865636b706f696e745f73636f72655f617474726962757465944e8c16636865636b706f696e745f73636f72655f6f72646572948c036d6178948c14636865636b706f696e745f6672657175656e6379944b008c11636865636b706f696e745f61745f656e6494898c1a5f636865636b706f696e745f6b6565705f616c6c5f72616e6b73948c0a44455052454341544544948c1f5f636865636b706f696e745f75706c6f61645f66726f6d5f776f726b65727394681275628c135f636865636b706f696e745f726573756c7473945d948c195f6c61746573745f636865636b706f696e745f726573756c74944e75622e\"\n }\n}"
+ ],
+ [
+ "{\n \"stub\": false,\n \"trainable_name\": \"train_model\",\n \"trial_id\": \"53fe56f2\",\n \"storage\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059575040000000000008c1b7261792e747261696e2e5f696e7465726e616c2e73746f72616765948c0e53746f72616765436f6e746578749493942981947d94288c12637573746f6d5f66735f70726f766964656494898c1273746f726167655f6c6f63616c5f70617468948c2b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c7473948c136578706572696d656e745f6469725f6e616d65948c0f746573745f6578706572696d656e74948c0e747269616c5f6469725f6e616d65948c96747261696e5f6d6f64656c5f35336665353666325f335f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d323034382c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f725f323032342d30362d32355f31352d32312d3332948c1863757272656e745f636865636b706f696e745f696e646578944affffffff8c0b73796e635f636f6e666967948c097261792e747261696e948c0a53796e63436f6e6669679493942981947d94288c0b73796e635f706572696f64944d2c018c0c73796e635f74696d656f7574944d08078c0e73796e635f61727469666163747394898c1c73796e635f6172746966616374735f6f6e5f636865636b706f696e7494888c0a75706c6f61645f646972948c0a44455052454341544544948c0673796e6365729468188c1273796e635f6f6e5f636865636b706f696e7494681875628c1273746f726167655f66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c0f73746f726167655f66735f70617468948c452f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f7465737494681968008c115f46696c6573797374656d53796e6365729493942981947d9428681b682268134d2c0168144d08078c116c6173745f73796e635f75705f74696d65944741d99eb0fc98aa368c136c6173745f73796e635f646f776e5f74696d659447fff00000000000008c0d5f73796e635f70726f63657373944e8c0c5f63757272656e745f636d649468008c125f75706c6f61645f746f5f66735f706174689493947d94288c0a6c6f63616c5f70617468948c3b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c74732f746573745f6578706572696d656e74948c0266739468228c0766735f70617468948c552f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e74948c076578636c756465945d948c0e2a2f636865636b706f696e745f2a9461758694756275622e\"\n },\n \"config\": {\n \"encoder_num_layer\": 1,\n \"decoder_rt_num_layer\": 1,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 16,\n \"encoder_ff\": 2048,\n \"decoder_rt_ff\": 2048,\n \"decoder_int_ff\": 512,\n \"n_head\": 1,\n \"drop_rate\": 0.21039295796266755,\n \"lr\": 0.00010129411733818616,\n \"batch_size\": 2048\n },\n \"_Trial__unresolved_config\": {\n \"encoder_num_layer\": 1,\n \"decoder_rt_num_layer\": 1,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 16,\n \"encoder_ff\": 2048,\n \"decoder_rt_ff\": 2048,\n \"decoder_int_ff\": 512,\n \"n_head\": 1,\n \"drop_rate\": 0.21039295796266755,\n \"lr\": 0.00010129411733818616,\n \"batch_size\": 2048\n },\n \"evaluated_params\": {\n \"batch_size\": 2048,\n \"decoder_int_ff\": 512,\n \"decoder_int_num_layer\": 1,\n \"decoder_rt_ff\": 2048,\n \"decoder_rt_num_layer\": 1,\n \"drop_rate\": 0.21039295796266755,\n \"embedding_dim\": 16,\n \"encoder_ff\": 2048,\n \"encoder_num_layer\": 1,\n \"lr\": 0.00010129411733818616,\n \"n_head\": 1\n },\n \"experiment_tag\": \"3_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop_rate=0.2104,embedding_dim=16,encoder_ff=2048,encoder_num_layer=1,lr=0.0001,n_head=1\",\n \"stopping_criterion\": {},\n \"_setup_default_resource\": true,\n \"_default_placement_group_factory\": \"80054e2e\",\n \"placement_group_factory\": \"800595ba000000000000008c237261792e74756e652e657865637574696f6e2e706c6163656d656e745f67726f757073948c15506c6163656d656e7447726f7570466163746f72799493942981947d94288c085f62756e646c6573945d947d94288c03435055944740540000000000008c0347505594473ff000000000000075618c155f686561645f62756e646c655f69735f656d70747994898c095f7374726174656779948c045041434b948c055f6172677394298c075f6b7761726773947d9475622e\",\n \"log_to_file\": [\n null,\n null\n ],\n \"max_failures\": 0,\n \"_default_result_or_future\": null,\n \"export_formats\": [],\n \"status\": \"PENDING\",\n \"relative_logdir\": \"train_model_53fe56f2_3_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop_r_2024-06-25_15-21-32\",\n \"trial_name_creator\": null,\n \"trial_dirname_creator\": null,\n \"custom_trial_name\": null,\n \"custom_dirname\": null,\n \"restore_path\": null,\n \"_restore_checkpoint_result\": null,\n \"_state_json\": null,\n \"results\": \"80054e2e\",\n \"extra_arg\": \"80054e2e\",\n \"_resources\": \"80054e2e\"\n}",
+ "{\n \"start_time\": null,\n \"num_failures\": 0,\n \"num_failures_after_restore\": 0,\n \"error_filename\": null,\n \"pickled_error_filename\": null,\n \"last_result\": {},\n \"last_result_time\": -Infinity,\n \"metric_analysis\": {},\n \"_n_steps\": [\n 5,\n 10\n ],\n \"metric_n_steps\": {},\n \"checkpoint_manager\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059584010000000000008c267261792e747261696e2e5f696e7465726e616c2e636865636b706f696e745f6d616e61676572948c125f436865636b706f696e744d616e616765729493942981947d94288c125f636865636b706f696e745f636f6e666967948c097261792e747261696e948c10436865636b706f696e74436f6e6669679493942981947d94288c0b6e756d5f746f5f6b656570944e8c1a636865636b706f696e745f73636f72655f617474726962757465944e8c16636865636b706f696e745f73636f72655f6f72646572948c036d6178948c14636865636b706f696e745f6672657175656e6379944b008c11636865636b706f696e745f61745f656e6494898c1a5f636865636b706f696e745f6b6565705f616c6c5f72616e6b73948c0a44455052454341544544948c1f5f636865636b706f696e745f75706c6f61645f66726f6d5f776f726b65727394681275628c135f636865636b706f696e745f726573756c7473945d948c195f6c61746573745f636865636b706f696e745f726573756c74944e75622e\"\n }\n}"
+ ]
+ ],
+ "runner_data": {
+ "_earliest_stopping_actor": Infinity,
+ "_actor_cleanup_timeout": 600,
+ "_actor_force_cleanup_timeout": 10,
+ "_reuse_actors": true,
+ "_buffer_length": 1,
+ "_buffer_min_time_s": 0.0,
+ "_buffer_max_time_s": 100.0,
+ "_max_pending_trials": 1,
+ "_metric": "loss",
+ "_total_time": 89.34920597076416,
+ "_iteration": 4001,
+ "_has_errored": false,
+ "_fail_fast": false,
+ "_print_trial_errors": true,
+ "_cached_trial_decisions": {},
+ "_queued_trial_decisions": {},
+ "_should_stop_experiment": false,
+ "_stopper": {
+ "_type": "CLOUDPICKLE_FALLBACK",
+ "value": "8005952c000000000000008c157261792e74756e652e73746f707065722e6e6f6f70948c0b4e6f6f7053746f707065729493942981942e"
+ },
+ "_start_time": 1719321586.2168348,
+ "_last_checkpoint_time": -Infinity,
+ "_session_str": "2024-06-25_15-19-46",
+ "_checkpoint_period": "auto",
+ "_trial_checkpoint_config": {
+ "_type": "CLOUDPICKLE_FALLBACK",
+ "value": "800595f2000000000000008c097261792e747261696e948c10436865636b706f696e74436f6e6669679493942981947d94288c0b6e756d5f746f5f6b656570944e8c1a636865636b706f696e745f73636f72655f617474726962757465944e8c16636865636b706f696e745f73636f72655f6f72646572948c036d6178948c14636865636b706f696e745f6672657175656e6379944b008c11636865636b706f696e745f61745f656e6494898c1a5f636865636b706f696e745f6b6565705f616c6c5f72616e6b73948c0a44455052454341544544948c1f5f636865636b706f696e745f75706c6f61645f66726f6d5f776f726b65727394680c75622e"
+ },
+ "_resumed": false
+ },
+ "stats": {
+ "start_time": 1719321586.2168348,
+ "timestamp": -Infinity
+ }
+}
\ No newline at end of file
diff --git a/ray_results/test_experiment/experiment_state-2024-06-25_16-07-48.json b/ray_results/test_experiment/experiment_state-2024-06-25_16-07-48.json
new file mode 100644
index 0000000000000000000000000000000000000000..f1a806f7b6b3cad6a3c8aed77edbd2ce859ce6be
--- /dev/null
+++ b/ray_results/test_experiment/experiment_state-2024-06-25_16-07-48.json
@@ -0,0 +1,116 @@
+{
+ "trial_data": [
+ [
+ "{\n \"stub\": false,\n \"trainable_name\": \"train_model\",\n \"trial_id\": \"a0e8b2af\",\n \"storage\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059582030000000000008c1b7261792e747261696e2e5f696e7465726e616c2e73746f72616765948c0e53746f72616765436f6e746578749493942981947d94288c12637573746f6d5f66735f70726f766964656494898c1273746f726167655f6c6f63616c5f70617468948c2b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c7473948c136578706572696d656e745f6469725f6e616d65948c0f746573745f6578706572696d656e74948c0e747269616c5f6469725f6e616d65948c96747261696e5f6d6f64656c5f61306538623261665f315f62617463685f73697a653d313032342c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d382c6465636f6465725f72745f66663d313032342c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f725f323032342d30362d32355f31362d30372d3438948c1863757272656e745f636865636b706f696e745f696e646578944b008c0b73796e635f636f6e666967948c097261792e747261696e948c0a53796e63436f6e6669679493942981947d94288c0b73796e635f706572696f64944d2c018c0c73796e635f74696d656f7574944d08078c0e73796e635f61727469666163747394898c1c73796e635f6172746966616374735f6f6e5f636865636b706f696e7494888c0a75706c6f61645f646972948c0a44455052454341544544948c0673796e6365729468188c1273796e635f6f6e5f636865636b706f696e7494681875628c1273746f726167655f66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c0f73746f726167655f66735f70617468948c452f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f7465737494681968008c115f46696c6573797374656d53796e6365729493942981947d9428681b682268134d2c0168144d08078c116c6173745f73796e635f75705f74696d659447fff00000000000008c136c6173745f73796e635f646f776e5f74696d659447fff00000000000008c0d5f73796e635f70726f63657373944e8c0c5f63757272656e745f636d64944e756275622e\"\n },\n \"config\": {\n \"encoder_num_layer\": 2,\n \"decoder_rt_num_layer\": 1,\n \"decoder_int_num_layer\": 8,\n \"embedding_dim\": 256,\n \"encoder_ff\": 2048,\n \"decoder_rt_ff\": 1024,\n \"decoder_int_ff\": 512,\n \"n_head\": 8,\n \"drop_rate\": 0.4534009171604666,\n \"lr\": 0.019472817375775914,\n \"batch_size\": 1024\n },\n \"_Trial__unresolved_config\": {\n \"encoder_num_layer\": 2,\n \"decoder_rt_num_layer\": 1,\n \"decoder_int_num_layer\": 8,\n \"embedding_dim\": 256,\n \"encoder_ff\": 2048,\n \"decoder_rt_ff\": 1024,\n \"decoder_int_ff\": 512,\n \"n_head\": 8,\n \"drop_rate\": 0.4534009171604666,\n \"lr\": 0.019472817375775914,\n \"batch_size\": 1024\n },\n \"evaluated_params\": {\n \"batch_size\": 1024,\n \"decoder_int_ff\": 512,\n \"decoder_int_num_layer\": 8,\n \"decoder_rt_ff\": 1024,\n \"decoder_rt_num_layer\": 1,\n \"drop_rate\": 0.4534009171604666,\n \"embedding_dim\": 256,\n \"encoder_ff\": 2048,\n \"encoder_num_layer\": 2,\n \"lr\": 0.019472817375775914,\n \"n_head\": 8\n },\n \"experiment_tag\": \"1_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=8,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop_rate=0.4534,embedding_dim=256,encoder_ff=2048,encoder_num_layer=2,lr=0.0195,n_head=8\",\n \"stopping_criterion\": {},\n \"_setup_default_resource\": true,\n \"_default_placement_group_factory\": \"80054e2e\",\n \"placement_group_factory\": \"800595ba000000000000008c237261792e74756e652e657865637574696f6e2e706c6163656d656e745f67726f757073948c15506c6163656d656e7447726f7570466163746f72799493942981947d94288c085f62756e646c6573945d947d94288c03435055944740540000000000008c034750559447401000000000000075618c155f686561645f62756e646c655f69735f656d70747994898c095f7374726174656779948c045041434b948c055f6172677394298c075f6b7761726773947d9475622e\",\n \"log_to_file\": [\n null,\n null\n ],\n \"max_failures\": 0,\n \"_default_result_or_future\": null,\n \"export_formats\": [],\n \"status\": \"PAUSED\",\n \"relative_logdir\": \"train_model_a0e8b2af_1_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=8,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop_r_2024-06-25_16-07-48\",\n \"trial_name_creator\": null,\n \"trial_dirname_creator\": null,\n \"custom_trial_name\": null,\n \"custom_dirname\": null,\n \"restore_path\": null,\n \"_restore_checkpoint_result\": null,\n \"_state_json\": null,\n \"results\": \"80054e2e\",\n \"extra_arg\": \"80054e2e\",\n \"_resources\": \"80054e2e\"\n}",
+ "{\n \"start_time\": 1719324484.8937905,\n \"num_failures\": 0,\n \"num_failures_after_restore\": 0,\n \"error_filename\": null,\n \"pickled_error_filename\": null,\n \"last_result\": {\n \"loss\": 2773.484795037217,\n \"timestamp\": 1719324574,\n \"checkpoint_dir_name\": \"checkpoint_000000\",\n \"should_checkpoint\": true,\n \"done\": false,\n \"training_iteration\": 1,\n \"trial_id\": \"a0e8b2af\",\n \"date\": \"2024-06-25_16-09-34\",\n \"time_this_iter_s\": 89.29933595657349,\n \"time_total_s\": 89.29933595657349,\n \"pid\": 69318,\n \"hostname\": \"r3i5n6\",\n \"node_ip\": \"10.159.8.159\",\n \"config\": {\n \"encoder_num_layer\": 2,\n \"decoder_rt_num_layer\": 1,\n \"decoder_int_num_layer\": 8,\n \"embedding_dim\": 256,\n \"encoder_ff\": 2048,\n \"decoder_rt_ff\": 1024,\n \"decoder_int_ff\": 512,\n \"n_head\": 8,\n \"drop_rate\": 0.4534009171604666,\n \"lr\": 0.019472817375775914,\n \"batch_size\": 1024\n },\n \"time_since_restore\": 89.29933595657349,\n \"iterations_since_restore\": 1,\n \"experiment_tag\": \"1_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=8,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop_rate=0.4534,embedding_dim=256,encoder_ff=2048,encoder_num_layer=2,lr=0.0195,n_head=8\"\n },\n \"last_result_time\": 1719324574.1991315,\n \"metric_analysis\": {\n \"loss\": {\n \"max\": 2773.484795037217,\n \"min\": 2773.484795037217,\n \"avg\": 2773.484795037217,\n \"last\": 2773.484795037217,\n \"last-5-avg\": 2773.484795037217,\n \"last-10-avg\": 2773.484795037217\n },\n \"should_checkpoint\": {\n \"max\": true,\n \"min\": true,\n \"avg\": true,\n \"last\": true,\n \"last-5-avg\": true,\n \"last-10-avg\": true\n },\n \"done\": {\n \"max\": false,\n \"min\": false,\n \"avg\": false,\n \"last\": false,\n \"last-5-avg\": false,\n \"last-10-avg\": false\n },\n \"training_iteration\": {\n \"max\": 1,\n \"min\": 1,\n \"avg\": 1,\n \"last\": 1,\n \"last-5-avg\": 1,\n \"last-10-avg\": 1\n },\n \"time_this_iter_s\": {\n \"max\": 89.29933595657349,\n \"min\": 89.29933595657349,\n \"avg\": 89.29933595657349,\n \"last\": 89.29933595657349,\n \"last-5-avg\": 89.29933595657349,\n \"last-10-avg\": 89.29933595657349\n },\n \"time_total_s\": {\n \"max\": 89.29933595657349,\n \"min\": 89.29933595657349,\n \"avg\": 89.29933595657349,\n \"last\": 89.29933595657349,\n \"last-5-avg\": 89.29933595657349,\n \"last-10-avg\": 89.29933595657349\n },\n \"time_since_restore\": {\n \"max\": 89.29933595657349,\n \"min\": 89.29933595657349,\n \"avg\": 89.29933595657349,\n \"last\": 89.29933595657349,\n \"last-5-avg\": 89.29933595657349,\n \"last-10-avg\": 89.29933595657349\n },\n \"iterations_since_restore\": {\n \"max\": 1,\n \"min\": 1,\n \"avg\": 1,\n \"last\": 1,\n \"last-5-avg\": 1,\n \"last-10-avg\": 1\n }\n },\n \"_n_steps\": [\n 5,\n 10\n ],\n \"metric_n_steps\": {\n \"loss\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059589000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452948c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b007494624308381c0e37f8aaa5409486945294612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059589000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452948c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b007494624308381c0e37f8aaa5409486945294612e\"\n }\n },\n \"should_checkpoint\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059522000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b058694529488612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059522000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a8694529488612e\"\n }\n },\n \"done\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059522000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b058694529489612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059522000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a8694529489612e\"\n }\n },\n \"training_iteration\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059523000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452944b01612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059523000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452944b01612e\"\n }\n },\n \"time_this_iter_s\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294474056532852000000612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294474056532852000000612e\"\n }\n },\n \"time_total_s\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294474056532852000000612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294474056532852000000612e\"\n }\n },\n \"time_since_restore\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294474056532852000000612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294474056532852000000612e\"\n }\n },\n \"iterations_since_restore\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059523000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452944b01612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059523000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452944b01612e\"\n }\n }\n },\n \"checkpoint_manager\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059519060000000000008c267261792e747261696e2e5f696e7465726e616c2e636865636b706f696e745f6d616e61676572948c125f436865636b706f696e744d616e616765729493942981947d94288c125f636865636b706f696e745f636f6e666967948c097261792e747261696e948c10436865636b706f696e74436f6e6669679493942981947d94288c0b6e756d5f746f5f6b656570944e8c1a636865636b706f696e745f73636f72655f617474726962757465944e8c16636865636b706f696e745f73636f72655f6f72646572948c036d6178948c14636865636b706f696e745f6672657175656e6379944b008c11636865636b706f696e745f61745f656e6494898c1a5f636865636b706f696e745f6b6565705f616c6c5f72616e6b73948c0a44455052454341544544948c1f5f636865636b706f696e745f75706c6f61645f66726f6d5f776f726b65727394681275628c135f636865636b706f696e745f726573756c7473945d948c1b7261792e747261696e2e5f696e7465726e616c2e73657373696f6e948c0f5f547261696e696e67526573756c749493942981947d94288c0a636865636b706f696e749468068c0a436865636b706f696e749493942981947d94288c0470617468948cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f61306538623261665f315f62617463685f73697a653d313032342c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d382c6465636f6465725f72745f66663d313032342c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f725f323032342d30362d32355f31362d30372d34382f636865636b706f696e745f303030303030948c0a66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c055f75756964948c0475756964948c04555549449493942981947d948c03696e74948a117618d482644e63b29e4dd84919f39dde00736275628c076d657472696373947d94288c046c6f7373948c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b007494624308381c0e37f8aaa54094869452948c0974696d657374616d70944a9ecf7a668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303030948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b018c08747269616c5f6964948c086130653862326166948c0464617465948c13323032342d30362d32355f31362d30392d3334948c1074696d655f746869735f697465725f73944740565328520000008c0c74696d655f746f74616c5f73944740565328520000008c03706964944ac60e01008c08686f73746e616d65948c06723369356e36948c076e6f64655f6970948c0c31302e3135392e382e313539948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b028c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b088c0d656d62656464696e675f64696d944d00018c0a656e636f6465725f6666944d00088c0d6465636f6465725f72745f6666944d00048c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b088c0964726f705f7261746594473fdd048547cb8f1a8c026c7294473f93f0aea728e5888c0a62617463685f73697a65944d0004758c1274696d655f73696e63655f726573746f7265944740565328520000008c18697465726174696f6e735f73696e63655f726573746f7265944b01757562618c195f6c61746573745f636865636b706f696e745f726573756c7494681975622e\"\n }\n}"
+ ],
+ [
+ "{\n \"stub\": false,\n \"trainable_name\": \"train_model\",\n \"trial_id\": \"52d8dfa1\",\n \"storage\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059572040000000000008c1b7261792e747261696e2e5f696e7465726e616c2e73746f72616765948c0e53746f72616765436f6e746578749493942981947d94288c12637573746f6d5f66735f70726f766964656494898c1273746f726167655f6c6f63616c5f70617468948c2b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c7473948c136578706572696d656e745f6469725f6e616d65948c0f746573745f6578706572696d656e74948c0e747269616c5f6469725f6e616d65948c96747261696e5f6d6f64656c5f35326438646661315f325f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d313032342c6465636f6465725f72745f6e756d5f6c617965723d342c64726f705f725f323032342d30362d32355f31362d30382d3034948c1863757272656e745f636865636b706f696e745f696e646578944b008c0b73796e635f636f6e666967948c097261792e747261696e948c0a53796e63436f6e6669679493942981947d94288c0b73796e635f706572696f64944d2c018c0c73796e635f74696d656f7574944d08078c0e73796e635f61727469666163747394898c1c73796e635f6172746966616374735f6f6e5f636865636b706f696e7494888c0a75706c6f61645f646972948c0a44455052454341544544948c0673796e6365729468188c1273796e635f6f6e5f636865636b706f696e7494681875628c1273746f726167655f66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c0f73746f726167655f66735f70617468948c452f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f7465737494681968008c115f46696c6573797374656d53796e6365729493942981947d9428681b682268134d2c0168144d08078c116c6173745f73796e635f75705f74696d65944741d99eb3cd22d0378c136c6173745f73796e635f646f776e5f74696d659447fff00000000000008c0d5f73796e635f70726f63657373944e8c0c5f63757272656e745f636d649468008c125f75706c6f61645f746f5f66735f706174689493947d94288c0a6c6f63616c5f70617468948c3b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c74732f746573745f6578706572696d656e74948c0266739468228c0766735f70617468948c552f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e74948c076578636c756465945d948c0e2a2f636865636b706f696e745f2a9461758694756275622e\"\n },\n \"config\": {\n \"encoder_num_layer\": 8,\n \"decoder_rt_num_layer\": 4,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 16,\n \"encoder_ff\": 1024,\n \"decoder_rt_ff\": 1024,\n \"decoder_int_ff\": 512,\n \"n_head\": 8,\n \"drop_rate\": 0.982125481793317,\n \"lr\": 0.002789281371289888,\n \"batch_size\": 2048\n },\n \"_Trial__unresolved_config\": {\n \"encoder_num_layer\": 8,\n \"decoder_rt_num_layer\": 4,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 16,\n \"encoder_ff\": 1024,\n \"decoder_rt_ff\": 1024,\n \"decoder_int_ff\": 512,\n \"n_head\": 8,\n \"drop_rate\": 0.982125481793317,\n \"lr\": 0.002789281371289888,\n \"batch_size\": 2048\n },\n \"evaluated_params\": {\n \"batch_size\": 2048,\n \"decoder_int_ff\": 512,\n \"decoder_int_num_layer\": 1,\n \"decoder_rt_ff\": 1024,\n \"decoder_rt_num_layer\": 4,\n \"drop_rate\": 0.982125481793317,\n \"embedding_dim\": 16,\n \"encoder_ff\": 1024,\n \"encoder_num_layer\": 8,\n \"lr\": 0.002789281371289888,\n \"n_head\": 8\n },\n \"experiment_tag\": \"2_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=4,drop_rate=0.9821,embedding_dim=16,encoder_ff=1024,encoder_num_layer=8,lr=0.0028,n_head=8\",\n \"stopping_criterion\": {},\n \"_setup_default_resource\": true,\n \"_default_placement_group_factory\": \"80054e2e\",\n \"placement_group_factory\": \"800595ba000000000000008c237261792e74756e652e657865637574696f6e2e706c6163656d656e745f67726f757073948c15506c6163656d656e7447726f7570466163746f72799493942981947d94288c085f62756e646c6573945d947d94288c03435055944740540000000000008c034750559447401000000000000075618c155f686561645f62756e646c655f69735f656d70747994898c095f7374726174656779948c045041434b948c055f6172677394298c075f6b7761726773947d9475622e\",\n \"log_to_file\": [\n null,\n null\n ],\n \"max_failures\": 0,\n \"_default_result_or_future\": null,\n \"export_formats\": [],\n \"status\": \"PAUSED\",\n \"relative_logdir\": \"train_model_52d8dfa1_2_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=4,drop_r_2024-06-25_16-08-04\",\n \"trial_name_creator\": null,\n \"trial_dirname_creator\": null,\n \"custom_trial_name\": null,\n \"custom_dirname\": null,\n \"restore_path\": null,\n \"_restore_checkpoint_result\": null,\n \"_state_json\": null,\n \"results\": \"80054e2e\",\n \"extra_arg\": \"80054e2e\",\n \"_resources\": \"80054e2e\"\n}",
+ "{\n \"start_time\": 1719324574.3448324,\n \"num_failures\": 0,\n \"num_failures_after_restore\": 0,\n \"error_filename\": null,\n \"pickled_error_filename\": null,\n \"last_result\": {\n \"loss\": NaN,\n \"timestamp\": 1719324663,\n \"checkpoint_dir_name\": \"checkpoint_000000\",\n \"should_checkpoint\": true,\n \"done\": false,\n \"training_iteration\": 1,\n \"trial_id\": \"52d8dfa1\",\n \"date\": \"2024-06-25_16-11-03\",\n \"time_this_iter_s\": 89.4755928516388,\n \"time_total_s\": 89.4755928516388,\n \"pid\": 69318,\n \"hostname\": \"r3i5n6\",\n \"node_ip\": \"10.159.8.159\",\n \"config\": {\n \"encoder_num_layer\": 8,\n \"decoder_rt_num_layer\": 4,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 16,\n \"encoder_ff\": 1024,\n \"decoder_rt_ff\": 1024,\n \"decoder_int_ff\": 512,\n \"n_head\": 8,\n \"drop_rate\": 0.982125481793317,\n \"lr\": 0.002789281371289888,\n \"batch_size\": 2048\n },\n \"time_since_restore\": 89.4755928516388,\n \"iterations_since_restore\": 1,\n \"experiment_tag\": \"2_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=4,drop_rate=0.9821,embedding_dim=16,encoder_ff=1024,encoder_num_layer=8,lr=0.0028,n_head=8\"\n },\n \"last_result_time\": 1719324663.8246083,\n \"metric_analysis\": {\n \"loss\": {\n \"max\": NaN,\n \"min\": NaN,\n \"avg\": NaN,\n \"last\": NaN,\n \"last-5-avg\": NaN,\n \"last-10-avg\": NaN\n },\n \"should_checkpoint\": {\n \"max\": true,\n \"min\": true,\n \"avg\": true,\n \"last\": true,\n \"last-5-avg\": true,\n \"last-10-avg\": true\n },\n \"done\": {\n \"max\": false,\n \"min\": false,\n \"avg\": false,\n \"last\": false,\n \"last-5-avg\": false,\n \"last-10-avg\": false\n },\n \"training_iteration\": {\n \"max\": 1,\n \"min\": 1,\n \"avg\": 1,\n \"last\": 1,\n \"last-5-avg\": 1,\n \"last-10-avg\": 1\n },\n \"time_this_iter_s\": {\n \"max\": 89.4755928516388,\n \"min\": 89.4755928516388,\n \"avg\": 89.4755928516388,\n \"last\": 89.4755928516388,\n \"last-5-avg\": 89.4755928516388,\n \"last-10-avg\": 89.4755928516388\n },\n \"time_total_s\": {\n \"max\": 89.4755928516388,\n \"min\": 89.4755928516388,\n \"avg\": 89.4755928516388,\n \"last\": 89.4755928516388,\n \"last-5-avg\": 89.4755928516388,\n \"last-10-avg\": 89.4755928516388\n },\n \"time_since_restore\": {\n \"max\": 89.4755928516388,\n \"min\": 89.4755928516388,\n \"avg\": 89.4755928516388,\n \"last\": 89.4755928516388,\n \"last-5-avg\": 89.4755928516388,\n \"last-10-avg\": 89.4755928516388\n },\n \"iterations_since_restore\": {\n \"max\": 1,\n \"min\": 1,\n \"avg\": 1,\n \"last\": 1,\n \"last-5-avg\": 1,\n \"last-10-avg\": 1\n }\n },\n \"_n_steps\": [\n 5,\n 10\n ],\n \"metric_n_steps\": {\n \"loss\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059589000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452948c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b007494624308000000e0ffffff7f9486945294612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059589000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452948c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b007494624308000000e0ffffff7f9486945294612e\"\n }\n },\n \"should_checkpoint\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059522000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b058694529488612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059522000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a8694529488612e\"\n }\n },\n \"done\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059522000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b058694529489612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059522000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a8694529489612e\"\n }\n },\n \"training_iteration\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059523000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452944b01612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059523000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452944b01612e\"\n }\n },\n \"time_this_iter_s\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452944740565e701d000000612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452944740565e701d000000612e\"\n }\n },\n \"time_total_s\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452944740565e701d000000612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452944740565e701d000000612e\"\n }\n },\n \"time_since_restore\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452944740565e701d000000612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452944740565e701d000000612e\"\n }\n },\n \"iterations_since_restore\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059523000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452944b01612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059523000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452944b01612e\"\n }\n }\n },\n \"checkpoint_manager\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059518060000000000008c267261792e747261696e2e5f696e7465726e616c2e636865636b706f696e745f6d616e61676572948c125f436865636b706f696e744d616e616765729493942981947d94288c125f636865636b706f696e745f636f6e666967948c097261792e747261696e948c10436865636b706f696e74436f6e6669679493942981947d94288c0b6e756d5f746f5f6b656570944e8c1a636865636b706f696e745f73636f72655f617474726962757465944e8c16636865636b706f696e745f73636f72655f6f72646572948c036d6178948c14636865636b706f696e745f6672657175656e6379944b008c11636865636b706f696e745f61745f656e6494898c1a5f636865636b706f696e745f6b6565705f616c6c5f72616e6b73948c0a44455052454341544544948c1f5f636865636b706f696e745f75706c6f61645f66726f6d5f776f726b65727394681275628c135f636865636b706f696e745f726573756c7473945d948c1b7261792e747261696e2e5f696e7465726e616c2e73657373696f6e948c0f5f547261696e696e67526573756c749493942981947d94288c0a636865636b706f696e749468068c0a436865636b706f696e749493942981947d94288c0470617468948cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f35326438646661315f325f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d313032342c6465636f6465725f72745f6e756d5f6c617965723d342c64726f705f725f323032342d30362d32355f31362d30382d30342f636865636b706f696e745f303030303030948c0a66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c055f75756964948c0475756964948c04555549449493942981947d948c03696e74948a11c21e3bc53b8b4899854129ba1f17e9de00736275628c076d657472696373947d94288c046c6f7373948c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b007494624308000000e0ffffff7f94869452948c0974696d657374616d70944af7cf7a668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303030948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b018c08747269616c5f6964948c083532643864666131948c0464617465948c13323032342d30362d32355f31362d31312d3033948c1074696d655f746869735f697465725f73944740565e701d0000008c0c74696d655f746f74616c5f73944740565e701d0000008c03706964944ac60e01008c08686f73746e616d65948c06723369356e36948c076e6f64655f6970948c0c31302e3135392e382e313539948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b088c146465636f6465725f72745f6e756d5f6c61796572944b048c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b108c0a656e636f6465725f6666944d00048c0d6465636f6465725f72745f6666944d00048c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b088c0964726f705f7261746594473fef6d926b1bdb778c026c7294473f66d98c089be9368c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f7265944740565e701d0000008c18697465726174696f6e735f73696e63655f726573746f7265944b01757562618c195f6c61746573745f636865636b706f696e745f726573756c7494681975622e\"\n }\n}"
+ ],
+ [
+ "{\n \"stub\": false,\n \"trainable_name\": \"train_model\",\n \"trial_id\": \"6a62b85a\",\n \"storage\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059572040000000000008c1b7261792e747261696e2e5f696e7465726e616c2e73746f72616765948c0e53746f72616765436f6e746578749493942981947d94288c12637573746f6d5f66735f70726f766964656494898c1273746f726167655f6c6f63616c5f70617468948c2b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c7473948c136578706572696d656e745f6469725f6e616d65948c0f746573745f6578706572696d656e74948c0e747269616c5f6469725f6e616d65948c96747261696e5f6d6f64656c5f36613632623835615f335f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d313032342c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d313032342c6465636f6465725f72745f6e756d5f6c617965723d322c64726f705f5f323032342d30362d32355f31362d30392d3334948c1863757272656e745f636865636b706f696e745f696e646578944b018c0b73796e635f636f6e666967948c097261792e747261696e948c0a53796e63436f6e6669679493942981947d94288c0b73796e635f706572696f64944d2c018c0c73796e635f74696d656f7574944d08078c0e73796e635f61727469666163747394898c1c73796e635f6172746966616374735f6f6e5f636865636b706f696e7494888c0a75706c6f61645f646972948c0a44455052454341544544948c0673796e6365729468188c1273796e635f6f6e5f636865636b706f696e7494681875628c1273746f726167655f66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c0f73746f726167655f66735f70617468948c452f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f7465737494681968008c115f46696c6573797374656d53796e6365729493942981947d9428681b682268134d2c0168144d08078c116c6173745f73796e635f75705f74696d65944741d99eb3cd22d0378c136c6173745f73796e635f646f776e5f74696d659447fff00000000000008c0d5f73796e635f70726f63657373944e8c0c5f63757272656e745f636d649468008c125f75706c6f61645f746f5f66735f706174689493947d94288c0a6c6f63616c5f70617468948c3b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c74732f746573745f6578706572696d656e74948c0266739468228c0766735f70617468948c552f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e74948c076578636c756465945d948c0e2a2f636865636b706f696e745f2a9461758694756275622e\"\n },\n \"config\": {\n \"encoder_num_layer\": 1,\n \"decoder_rt_num_layer\": 2,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 16,\n \"encoder_ff\": 512,\n \"decoder_rt_ff\": 1024,\n \"decoder_int_ff\": 1024,\n \"n_head\": 4,\n \"drop_rate\": 0.06860896590384413,\n \"lr\": 0.00016561766448063182,\n \"batch_size\": 2048\n },\n \"_Trial__unresolved_config\": {\n \"encoder_num_layer\": 1,\n \"decoder_rt_num_layer\": 2,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 16,\n \"encoder_ff\": 512,\n \"decoder_rt_ff\": 1024,\n \"decoder_int_ff\": 1024,\n \"n_head\": 4,\n \"drop_rate\": 0.06860896590384413,\n \"lr\": 0.00016561766448063182,\n \"batch_size\": 2048\n },\n \"evaluated_params\": {\n \"batch_size\": 2048,\n \"decoder_int_ff\": 1024,\n \"decoder_int_num_layer\": 1,\n \"decoder_rt_ff\": 1024,\n \"decoder_rt_num_layer\": 2,\n \"drop_rate\": 0.06860896590384413,\n \"embedding_dim\": 16,\n \"encoder_ff\": 512,\n \"encoder_num_layer\": 1,\n \"lr\": 0.00016561766448063182,\n \"n_head\": 4\n },\n \"experiment_tag\": \"3_batch_size=2048,decoder_int_ff=1024,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop_rate=0.0686,embedding_dim=16,encoder_ff=512,encoder_num_layer=1,lr=0.0002,n_head=4\",\n \"stopping_criterion\": {},\n \"_setup_default_resource\": true,\n \"_default_placement_group_factory\": \"80054e2e\",\n \"placement_group_factory\": \"800595ba000000000000008c237261792e74756e652e657865637574696f6e2e706c6163656d656e745f67726f757073948c15506c6163656d656e7447726f7570466163746f72799493942981947d94288c085f62756e646c6573945d947d94288c03435055944740540000000000008c034750559447401000000000000075618c155f686561645f62756e646c655f69735f656d70747994898c095f7374726174656779948c045041434b948c055f6172677394298c075f6b7761726773947d9475622e\",\n \"log_to_file\": [\n null,\n null\n ],\n \"max_failures\": 0,\n \"_default_result_or_future\": null,\n \"export_formats\": [],\n \"status\": \"PAUSED\",\n \"relative_logdir\": \"train_model_6a62b85a_3_batch_size=2048,decoder_int_ff=1024,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop__2024-06-25_16-09-34\",\n \"trial_name_creator\": null,\n \"trial_dirname_creator\": null,\n \"custom_trial_name\": null,\n \"custom_dirname\": null,\n \"restore_path\": null,\n \"_restore_checkpoint_result\": null,\n \"_state_json\": null,\n \"results\": \"80054e2e\",\n \"extra_arg\": \"80054e2e\",\n \"_resources\": \"80054e2e\"\n}",
+ "{\n \"start_time\": 1719324663.967162,\n \"num_failures\": 0,\n \"num_failures_after_restore\": 0,\n \"error_filename\": null,\n \"pickled_error_filename\": null,\n \"last_result\": {\n \"loss\": 310.75900604968933,\n \"timestamp\": 1719327245,\n \"checkpoint_dir_name\": \"checkpoint_000001\",\n \"should_checkpoint\": true,\n \"done\": false,\n \"training_iteration\": 2,\n \"trial_id\": \"6a62b85a\",\n \"date\": \"2024-06-25_16-54-05\",\n \"time_this_iter_s\": 63.18879699707031,\n \"time_total_s\": 127.51110315322876,\n \"pid\": 130020,\n \"hostname\": \"r3i5n6\",\n \"node_ip\": \"10.159.8.159\",\n \"config\": {\n \"encoder_num_layer\": 1,\n \"decoder_rt_num_layer\": 2,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 16,\n \"encoder_ff\": 512,\n \"decoder_rt_ff\": 1024,\n \"decoder_int_ff\": 1024,\n \"n_head\": 4,\n \"drop_rate\": 0.06860896590384413,\n \"lr\": 0.00016561766448063182,\n \"batch_size\": 2048\n },\n \"time_since_restore\": 63.18879699707031,\n \"iterations_since_restore\": 1,\n \"experiment_tag\": \"3_batch_size=2048,decoder_int_ff=1024,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop_rate=0.0686,embedding_dim=16,encoder_ff=512,encoder_num_layer=1,lr=0.0002,n_head=4\"\n },\n \"last_result_time\": 1719327245.5672662,\n \"metric_analysis\": {\n \"loss\": {\n \"max\": 510.42659056656004,\n \"min\": 310.75900604968933,\n \"avg\": 410.59279830812466,\n \"last\": 310.75900604968933,\n \"last-5-avg\": 410.59279830812466,\n \"last-10-avg\": 410.59279830812466\n },\n \"should_checkpoint\": {\n \"max\": true,\n \"min\": true,\n \"avg\": 1.0,\n \"last\": true,\n \"last-5-avg\": 1.0,\n \"last-10-avg\": 1.0\n },\n \"done\": {\n \"max\": false,\n \"min\": false,\n \"avg\": 0.0,\n \"last\": false,\n \"last-5-avg\": 0.0,\n \"last-10-avg\": 0.0\n },\n \"training_iteration\": {\n \"max\": 2,\n \"min\": 1,\n \"avg\": 1.5,\n \"last\": 2,\n \"last-5-avg\": 1.5,\n \"last-10-avg\": 1.5\n },\n \"time_this_iter_s\": {\n \"max\": 64.32230615615845,\n \"min\": 63.18879699707031,\n \"avg\": 63.75555157661438,\n \"last\": 63.18879699707031,\n \"last-5-avg\": 63.75555157661438,\n \"last-10-avg\": 63.75555157661438\n },\n \"time_total_s\": {\n \"max\": 127.51110315322876,\n \"min\": 64.32230615615845,\n \"avg\": 95.9167046546936,\n \"last\": 127.51110315322876,\n \"last-5-avg\": 95.9167046546936,\n \"last-10-avg\": 95.9167046546936\n },\n \"time_since_restore\": {\n \"max\": 64.32230615615845,\n \"min\": 63.18879699707031,\n \"avg\": 63.75555157661438,\n \"last\": 63.18879699707031,\n \"last-5-avg\": 63.75555157661438,\n \"last-10-avg\": 63.75555157661438\n },\n \"iterations_since_restore\": {\n \"max\": 1,\n \"min\": 1,\n \"avg\": 1.0,\n \"last\": 1,\n \"last-5-avg\": 1.0,\n \"last-10-avg\": 1.0\n }\n },\n \"_n_steps\": [\n 5,\n 10\n ],\n \"metric_n_steps\": {\n \"loss\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005959d000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294288c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b0074946243088542a150d3e67f4094869452946807680d43081c0e87e3246c73409486945294652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005959d000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294288c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b0074946243088542a150d3e67f4094869452946807680d43081c0e87e3246c73409486945294652e\"\n }\n },\n \"should_checkpoint\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059524000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294288888652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059524000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294288888652e\"\n }\n },\n \"done\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059524000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294288989652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059524000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294288989652e\"\n }\n },\n \"training_iteration\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059526000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294284b014b02652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059526000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294284b014b02652e\"\n }\n },\n \"time_this_iter_s\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059534000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452942847405014a0aa00000047404f982a80000000652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059534000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452942847405014a0aa00000047404f982a80000000652e\"\n }\n },\n \"time_total_s\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059534000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452942847405014a0aa00000047405fe0b5ea000000652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059534000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452942847405014a0aa00000047405fe0b5ea000000652e\"\n }\n },\n \"time_since_restore\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059534000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452942847405014a0aa00000047404f982a80000000652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059534000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452942847405014a0aa00000047404f982a80000000652e\"\n }\n },\n \"iterations_since_restore\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059526000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294284b014b01652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059526000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294284b014b01652e\"\n }\n }\n },\n \"checkpoint_manager\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"800595a5090000000000008c267261792e747261696e2e5f696e7465726e616c2e636865636b706f696e745f6d616e61676572948c125f436865636b706f696e744d616e616765729493942981947d94288c125f636865636b706f696e745f636f6e666967948c097261792e747261696e948c10436865636b706f696e74436f6e6669679493942981947d94288c0b6e756d5f746f5f6b656570944e8c1a636865636b706f696e745f73636f72655f617474726962757465944e8c16636865636b706f696e745f73636f72655f6f72646572948c036d6178948c14636865636b706f696e745f6672657175656e6379944b008c11636865636b706f696e745f61745f656e6494898c1a5f636865636b706f696e745f6b6565705f616c6c5f72616e6b73948c0a44455052454341544544948c1f5f636865636b706f696e745f75706c6f61645f66726f6d5f776f726b65727394681275628c135f636865636b706f696e745f726573756c7473945d94288c1b7261792e747261696e2e5f696e7465726e616c2e73657373696f6e948c0f5f547261696e696e67526573756c749493942981947d94288c0a636865636b706f696e749468068c0a436865636b706f696e749493942981947d94288c0470617468948cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f36613632623835615f335f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d313032342c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d313032342c6465636f6465725f72745f6e756d5f6c617965723d322c64726f705f5f323032342d30362d32355f31362d30392d33342f636865636b706f696e745f303030303030948c0a66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c055f75756964948c0475756964948c04555549449493942981947d948c03696e74948a111770d9a5a2b89096d44582c2ceb22a9d00736275628c076d657472696373947d94288c046c6f7373948c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b0074946243088542a150d3e67f4094869452948c0974696d657374616d70944a38d07a668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303030948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b018c08747269616c5f6964948c083661363262383561948c0464617465948c13323032342d30362d32355f31362d31322d3038948c1074696d655f746869735f697465725f739447405014a0aa0000008c0c74696d655f746f74616c5f739447405014a0aa0000008c03706964944ac60e01008c08686f73746e616d65948c06723369356e36948c076e6f64655f6970948c0c31302e3135392e382e313539948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b018c146465636f6465725f72745f6e756d5f6c61796572944b028c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b108c0a656e636f6465725f6666944d00028c0d6465636f6465725f72745f6666944d00048c0e6465636f6465725f696e745f6666944d00048c066e5f68656164944b048c0964726f705f7261746594473fb1905b70c4f6b88c026c7294473f25b534e7b91e408c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f72659447405014a0aa0000008c18697465726174696f6e735f73696e63655f726573746f7265944b0175756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f36613632623835615f335f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d313032342c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d313032342c6465636f6465725f72745f6e756d5f6c617965723d322c64726f705f5f323032342d30362d32355f31362d30392d33342f636865636b706f696e745f30303030303194682268257d946827897385945294682a682d2981947d9468308a103744df4a18a121ad344e9c027800e32b7362756268317d94288c046c6f7373946836683c43081c0e87e3246c734094869452948c0974696d657374616d70944a0dda7a668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303031948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b028c08747269616c5f6964948c083661363262383561948c0464617465948c13323032342d30362d32355f31362d35342d3035948c1074696d655f746869735f697465725f739447404f982a800000008c0c74696d655f746f74616c5f739447405fe0b5ea0000008c03706964944ae4fb01008c08686f73746e616d65948c06723369356e36948c076e6f64655f6970948c0c31302e3135392e382e313539948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b018c146465636f6465725f72745f6e756d5f6c61796572944b028c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b108c0a656e636f6465725f6666944d00028c0d6465636f6465725f72745f6666944d00048c0e6465636f6465725f696e745f6666944d00048c066e5f68656164944b048c0964726f705f7261746594473fb1905b70c4f6b88c026c7294473f25b534e7b91e408c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f72659447404f982a800000008c18697465726174696f6e735f73696e63655f726573746f7265944b01757562658c195f6c61746573745f636865636b706f696e745f726573756c7494686275622e\"\n }\n}"
+ ],
+ [
+ "{\n \"stub\": false,\n \"trainable_name\": \"train_model\",\n \"trial_id\": \"975aa6a0\",\n \"storage\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059572040000000000008c1b7261792e747261696e2e5f696e7465726e616c2e73746f72616765948c0e53746f72616765436f6e746578749493942981947d94288c12637573746f6d5f66735f70726f766964656494898c1273746f726167655f6c6f63616c5f70617468948c2b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c7473948c136578706572696d656e745f6469725f6e616d65948c0f746573745f6578706572696d656e74948c0e747269616c5f6469725f6e616d65948c96747261696e5f6d6f64656c5f39373561613661305f345f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d322c6465636f6465725f72745f66663d313032342c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f725f323032342d30362d32355f31362d31312d3033948c1863757272656e745f636865636b706f696e745f696e646578944b008c0b73796e635f636f6e666967948c097261792e747261696e948c0a53796e63436f6e6669679493942981947d94288c0b73796e635f706572696f64944d2c018c0c73796e635f74696d656f7574944d08078c0e73796e635f61727469666163747394898c1c73796e635f6172746966616374735f6f6e5f636865636b706f696e7494888c0a75706c6f61645f646972948c0a44455052454341544544948c0673796e6365729468188c1273796e635f6f6e5f636865636b706f696e7494681875628c1273746f726167655f66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c0f73746f726167655f66735f70617468948c452f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f7465737494681968008c115f46696c6573797374656d53796e6365729493942981947d9428681b682268134d2c0168144d08078c116c6173745f73796e635f75705f74696d65944741d99eb3cd22d0378c136c6173745f73796e635f646f776e5f74696d659447fff00000000000008c0d5f73796e635f70726f63657373944e8c0c5f63757272656e745f636d649468008c125f75706c6f61645f746f5f66735f706174689493947d94288c0a6c6f63616c5f70617468948c3b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c74732f746573745f6578706572696d656e74948c0266739468228c0766735f70617468948c552f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e74948c076578636c756465945d948c0e2a2f636865636b706f696e745f2a9461758694756275622e\"\n },\n \"config\": {\n \"encoder_num_layer\": 1,\n \"decoder_rt_num_layer\": 1,\n \"decoder_int_num_layer\": 2,\n \"embedding_dim\": 64,\n \"encoder_ff\": 1024,\n \"decoder_rt_ff\": 1024,\n \"decoder_int_ff\": 512,\n \"n_head\": 16,\n \"drop_rate\": 0.8156275326399022,\n \"lr\": 0.00010404955423531405,\n \"batch_size\": 2048\n },\n \"_Trial__unresolved_config\": {\n \"encoder_num_layer\": 1,\n \"decoder_rt_num_layer\": 1,\n \"decoder_int_num_layer\": 2,\n \"embedding_dim\": 64,\n \"encoder_ff\": 1024,\n \"decoder_rt_ff\": 1024,\n \"decoder_int_ff\": 512,\n \"n_head\": 16,\n \"drop_rate\": 0.8156275326399022,\n \"lr\": 0.00010404955423531405,\n \"batch_size\": 2048\n },\n \"evaluated_params\": {\n \"batch_size\": 2048,\n \"decoder_int_ff\": 512,\n \"decoder_int_num_layer\": 2,\n \"decoder_rt_ff\": 1024,\n \"decoder_rt_num_layer\": 1,\n \"drop_rate\": 0.8156275326399022,\n \"embedding_dim\": 64,\n \"encoder_ff\": 1024,\n \"encoder_num_layer\": 1,\n \"lr\": 0.00010404955423531405,\n \"n_head\": 16\n },\n \"experiment_tag\": \"4_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=2,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop_rate=0.8156,embedding_dim=64,encoder_ff=1024,encoder_num_layer=1,lr=0.0001,n_head=16\",\n \"stopping_criterion\": {},\n \"_setup_default_resource\": true,\n \"_default_placement_group_factory\": \"80054e2e\",\n \"placement_group_factory\": \"800595ba000000000000008c237261792e74756e652e657865637574696f6e2e706c6163656d656e745f67726f757073948c15506c6163656d656e7447726f7570466163746f72799493942981947d94288c085f62756e646c6573945d947d94288c03435055944740540000000000008c034750559447401000000000000075618c155f686561645f62756e646c655f69735f656d70747994898c095f7374726174656779948c045041434b948c055f6172677394298c075f6b7761726773947d9475622e\",\n \"log_to_file\": [\n null,\n null\n ],\n \"max_failures\": 0,\n \"_default_result_or_future\": null,\n \"export_formats\": [],\n \"status\": \"PAUSED\",\n \"relative_logdir\": \"train_model_975aa6a0_4_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=2,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop_r_2024-06-25_16-11-03\",\n \"trial_name_creator\": null,\n \"trial_dirname_creator\": null,\n \"custom_trial_name\": null,\n \"custom_dirname\": null,\n \"restore_path\": null,\n \"_restore_checkpoint_result\": null,\n \"_state_json\": null,\n \"results\": \"80054e2e\",\n \"extra_arg\": \"80054e2e\",\n \"_resources\": \"80054e2e\"\n}",
+ "{\n \"start_time\": 1719324728.438826,\n \"num_failures\": 0,\n \"num_failures_after_restore\": 0,\n \"error_filename\": null,\n \"pickled_error_filename\": null,\n \"last_result\": {\n \"loss\": 2002.7163720318651,\n \"timestamp\": 1719324792,\n \"checkpoint_dir_name\": \"checkpoint_000000\",\n \"should_checkpoint\": true,\n \"done\": false,\n \"training_iteration\": 1,\n \"trial_id\": \"975aa6a0\",\n \"date\": \"2024-06-25_16-13-12\",\n \"time_this_iter_s\": 64.14145565032959,\n \"time_total_s\": 64.14145565032959,\n \"pid\": 69318,\n \"hostname\": \"r3i5n6\",\n \"node_ip\": \"10.159.8.159\",\n \"config\": {\n \"encoder_num_layer\": 1,\n \"decoder_rt_num_layer\": 1,\n \"decoder_int_num_layer\": 2,\n \"embedding_dim\": 64,\n \"encoder_ff\": 1024,\n \"decoder_rt_ff\": 1024,\n \"decoder_int_ff\": 512,\n \"n_head\": 16,\n \"drop_rate\": 0.8156275326399022,\n \"lr\": 0.00010404955423531405,\n \"batch_size\": 2048\n },\n \"time_since_restore\": 64.14145565032959,\n \"iterations_since_restore\": 1,\n \"experiment_tag\": \"4_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=2,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop_rate=0.8156,embedding_dim=64,encoder_ff=1024,encoder_num_layer=1,lr=0.0001,n_head=16\"\n },\n \"last_result_time\": 1719324792.5855737,\n \"metric_analysis\": {\n \"loss\": {\n \"max\": 2002.7163720318651,\n \"min\": 2002.7163720318651,\n \"avg\": 2002.7163720318651,\n \"last\": 2002.7163720318651,\n \"last-5-avg\": 2002.7163720318651,\n \"last-10-avg\": 2002.7163720318651\n },\n \"should_checkpoint\": {\n \"max\": true,\n \"min\": true,\n \"avg\": true,\n \"last\": true,\n \"last-5-avg\": true,\n \"last-10-avg\": true\n },\n \"done\": {\n \"max\": false,\n \"min\": false,\n \"avg\": false,\n \"last\": false,\n \"last-5-avg\": false,\n \"last-10-avg\": false\n },\n \"training_iteration\": {\n \"max\": 1,\n \"min\": 1,\n \"avg\": 1,\n \"last\": 1,\n \"last-5-avg\": 1,\n \"last-10-avg\": 1\n },\n \"time_this_iter_s\": {\n \"max\": 64.14145565032959,\n \"min\": 64.14145565032959,\n \"avg\": 64.14145565032959,\n \"last\": 64.14145565032959,\n \"last-5-avg\": 64.14145565032959,\n \"last-10-avg\": 64.14145565032959\n },\n \"time_total_s\": {\n \"max\": 64.14145565032959,\n \"min\": 64.14145565032959,\n \"avg\": 64.14145565032959,\n \"last\": 64.14145565032959,\n \"last-5-avg\": 64.14145565032959,\n \"last-10-avg\": 64.14145565032959\n },\n \"time_since_restore\": {\n \"max\": 64.14145565032959,\n \"min\": 64.14145565032959,\n \"avg\": 64.14145565032959,\n \"last\": 64.14145565032959,\n \"last-5-avg\": 64.14145565032959,\n \"last-10-avg\": 64.14145565032959\n },\n \"iterations_since_restore\": {\n \"max\": 1,\n \"min\": 1,\n \"avg\": 1,\n \"last\": 1,\n \"last-5-avg\": 1,\n \"last-10-avg\": 1\n }\n },\n \"_n_steps\": [\n 5,\n 10\n ],\n \"metric_n_steps\": {\n \"loss\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059589000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452948c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b0074946243088542a190dd4a9f409486945294612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059589000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452948c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b0074946243088542a190dd4a9f409486945294612e\"\n }\n },\n \"should_checkpoint\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059522000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b058694529488612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059522000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a8694529488612e\"\n }\n },\n \"done\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059522000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b058694529489612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059522000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a8694529489612e\"\n }\n },\n \"training_iteration\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059523000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452944b01612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059523000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452944b01612e\"\n }\n },\n \"time_this_iter_s\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294474050090d9c000000612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294474050090d9c000000612e\"\n }\n },\n \"time_total_s\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294474050090d9c000000612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294474050090d9c000000612e\"\n }\n },\n \"time_since_restore\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294474050090d9c000000612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294474050090d9c000000612e\"\n }\n },\n \"iterations_since_restore\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059523000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452944b01612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059523000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452944b01612e\"\n }\n }\n },\n \"checkpoint_manager\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059517060000000000008c267261792e747261696e2e5f696e7465726e616c2e636865636b706f696e745f6d616e61676572948c125f436865636b706f696e744d616e616765729493942981947d94288c125f636865636b706f696e745f636f6e666967948c097261792e747261696e948c10436865636b706f696e74436f6e6669679493942981947d94288c0b6e756d5f746f5f6b656570944e8c1a636865636b706f696e745f73636f72655f617474726962757465944e8c16636865636b706f696e745f73636f72655f6f72646572948c036d6178948c14636865636b706f696e745f6672657175656e6379944b008c11636865636b706f696e745f61745f656e6494898c1a5f636865636b706f696e745f6b6565705f616c6c5f72616e6b73948c0a44455052454341544544948c1f5f636865636b706f696e745f75706c6f61645f66726f6d5f776f726b65727394681275628c135f636865636b706f696e745f726573756c7473945d948c1b7261792e747261696e2e5f696e7465726e616c2e73657373696f6e948c0f5f547261696e696e67526573756c749493942981947d94288c0a636865636b706f696e749468068c0a436865636b706f696e749493942981947d94288c0470617468948cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f39373561613661305f345f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d322c6465636f6465725f72745f66663d313032342c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f725f323032342d30362d32355f31362d31312d30332f636865636b706f696e745f303030303030948c0a66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c055f75756964948c0475756964948c04555549449493942981947d948c03696e74948a104d5f716ab4722ebf3b45fadaa52f804f736275628c076d657472696373947d94288c046c6f7373948c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b0074946243088542a190dd4a9f4094869452948c0974696d657374616d70944a78d07a668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303030948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b018c08747269616c5f6964948c083937356161366130948c0464617465948c13323032342d30362d32355f31362d31332d3132948c1074696d655f746869735f697465725f7394474050090d9c0000008c0c74696d655f746f74616c5f7394474050090d9c0000008c03706964944ac60e01008c08686f73746e616d65948c06723369356e36948c076e6f64655f6970948c0c31302e3135392e382e313539948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b018c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b028c0d656d62656464696e675f64696d944b408c0a656e636f6465725f6666944d00048c0d6465636f6465725f72745f6666944d00048c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b108c0964726f705f7261746594473fea199ee94cfa498c026c7294473f1b46a5bafc8e808c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f726594474050090d9c0000008c18697465726174696f6e735f73696e63655f726573746f7265944b01757562618c195f6c61746573745f636865636b706f696e745f726573756c7494681975622e\"\n }\n}"
+ ],
+ [
+ "{\n \"stub\": false,\n \"trainable_name\": \"train_model\",\n \"trial_id\": \"af6d9c38\",\n \"storage\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059572040000000000008c1b7261792e747261696e2e5f696e7465726e616c2e73746f72616765948c0e53746f72616765436f6e746578749493942981947d94288c12637573746f6d5f66735f70726f766964656494898c1273746f726167655f6c6f63616c5f70617468948c2b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c7473948c136578706572696d656e745f6469725f6e616d65948c0f746573745f6578706572696d656e74948c0e747269616c5f6469725f6e616d65948c96747261696e5f6d6f64656c5f61663664396333385f355f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d313032342c6465636f6465725f696e745f6e756d5f6c617965723d382c6465636f6465725f72745f66663d313032342c6465636f6465725f72745f6e756d5f6c617965723d382c64726f705f5f323032342d30362d32355f31362d31322d3038948c1863757272656e745f636865636b706f696e745f696e646578944b018c0b73796e635f636f6e666967948c097261792e747261696e948c0a53796e63436f6e6669679493942981947d94288c0b73796e635f706572696f64944d2c018c0c73796e635f74696d656f7574944d08078c0e73796e635f61727469666163747394898c1c73796e635f6172746966616374735f6f6e5f636865636b706f696e7494888c0a75706c6f61645f646972948c0a44455052454341544544948c0673796e6365729468188c1273796e635f6f6e5f636865636b706f696e7494681875628c1273746f726167655f66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c0f73746f726167655f66735f70617468948c452f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f7465737494681968008c115f46696c6573797374656d53796e6365729493942981947d9428681b682268134d2c0168144d08078c116c6173745f73796e635f75705f74696d65944741d99eb3cd22d0378c136c6173745f73796e635f646f776e5f74696d659447fff00000000000008c0d5f73796e635f70726f63657373944e8c0c5f63757272656e745f636d649468008c125f75706c6f61645f746f5f66735f706174689493947d94288c0a6c6f63616c5f70617468948c3b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c74732f746573745f6578706572696d656e74948c0266739468228c0766735f70617468948c552f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e74948c076578636c756465945d948c0e2a2f636865636b706f696e745f2a9461758694756275622e\"\n },\n \"config\": {\n \"encoder_num_layer\": 8,\n \"decoder_rt_num_layer\": 8,\n \"decoder_int_num_layer\": 8,\n \"embedding_dim\": 16,\n \"encoder_ff\": 512,\n \"decoder_rt_ff\": 1024,\n \"decoder_int_ff\": 1024,\n \"n_head\": 16,\n \"drop_rate\": 0.5912116123671598,\n \"lr\": 0.009202977404873457,\n \"batch_size\": 2048\n },\n \"_Trial__unresolved_config\": {\n \"encoder_num_layer\": 8,\n \"decoder_rt_num_layer\": 8,\n \"decoder_int_num_layer\": 8,\n \"embedding_dim\": 16,\n \"encoder_ff\": 512,\n \"decoder_rt_ff\": 1024,\n \"decoder_int_ff\": 1024,\n \"n_head\": 16,\n \"drop_rate\": 0.5912116123671598,\n \"lr\": 0.009202977404873457,\n \"batch_size\": 2048\n },\n \"evaluated_params\": {\n \"batch_size\": 2048,\n \"decoder_int_ff\": 1024,\n \"decoder_int_num_layer\": 8,\n \"decoder_rt_ff\": 1024,\n \"decoder_rt_num_layer\": 8,\n \"drop_rate\": 0.5912116123671598,\n \"embedding_dim\": 16,\n \"encoder_ff\": 512,\n \"encoder_num_layer\": 8,\n \"lr\": 0.009202977404873457,\n \"n_head\": 16\n },\n \"experiment_tag\": \"5_batch_size=2048,decoder_int_ff=1024,decoder_int_num_layer=8,decoder_rt_ff=1024,decoder_rt_num_layer=8,drop_rate=0.5912,embedding_dim=16,encoder_ff=512,encoder_num_layer=8,lr=0.0092,n_head=16\",\n \"stopping_criterion\": {},\n \"_setup_default_resource\": true,\n \"_default_placement_group_factory\": \"80054e2e\",\n \"placement_group_factory\": \"800595ba000000000000008c237261792e74756e652e657865637574696f6e2e706c6163656d656e745f67726f757073948c15506c6163656d656e7447726f7570466163746f72799493942981947d94288c085f62756e646c6573945d947d94288c03435055944740540000000000008c034750559447401000000000000075618c155f686561645f62756e646c655f69735f656d70747994898c095f7374726174656779948c045041434b948c055f6172677394298c075f6b7761726773947d9475622e\",\n \"log_to_file\": [\n null,\n null\n ],\n \"max_failures\": 0,\n \"_default_result_or_future\": null,\n \"export_formats\": [],\n \"status\": \"PAUSED\",\n \"relative_logdir\": \"train_model_af6d9c38_5_batch_size=2048,decoder_int_ff=1024,decoder_int_num_layer=8,decoder_rt_ff=1024,decoder_rt_num_layer=8,drop__2024-06-25_16-12-08\",\n \"trial_name_creator\": null,\n \"trial_dirname_creator\": null,\n \"custom_trial_name\": null,\n \"custom_dirname\": null,\n \"restore_path\": null,\n \"_restore_checkpoint_result\": null,\n \"_state_json\": null,\n \"results\": \"80054e2e\",\n \"extra_arg\": \"80054e2e\",\n \"_resources\": \"80054e2e\"\n}",
+ "{\n \"start_time\": 1719324792.730206,\n \"num_failures\": 0,\n \"num_failures_after_restore\": 0,\n \"error_filename\": null,\n \"pickled_error_filename\": null,\n \"last_result\": {\n \"loss\": 1982.2825418307086,\n \"timestamp\": 1719327372,\n \"checkpoint_dir_name\": \"checkpoint_000001\",\n \"should_checkpoint\": true,\n \"done\": false,\n \"training_iteration\": 2,\n \"trial_id\": \"af6d9c38\",\n \"date\": \"2024-06-25_16-56-12\",\n \"time_this_iter_s\": 126.36091089248657,\n \"time_total_s\": 252.29485964775085,\n \"pid\": 130020,\n \"hostname\": \"r3i5n6\",\n \"node_ip\": \"10.159.8.159\",\n \"config\": {\n \"encoder_num_layer\": 8,\n \"decoder_rt_num_layer\": 8,\n \"decoder_int_num_layer\": 8,\n \"embedding_dim\": 16,\n \"encoder_ff\": 512,\n \"decoder_rt_ff\": 1024,\n \"decoder_int_ff\": 1024,\n \"n_head\": 16,\n \"drop_rate\": 0.5912116123671598,\n \"lr\": 0.009202977404873457,\n \"batch_size\": 2048\n },\n \"time_since_restore\": 126.36091089248657,\n \"iterations_since_restore\": 1,\n \"experiment_tag\": \"5_batch_size=2048,decoder_int_ff=1024,decoder_int_num_layer=8,decoder_rt_ff=1024,decoder_rt_num_layer=8,drop_rate=0.5912,embedding_dim=16,encoder_ff=512,encoder_num_layer=8,lr=0.0092,n_head=16\"\n },\n \"last_result_time\": 1719327372.0857606,\n \"metric_analysis\": {\n \"loss\": {\n \"max\": 1987.8132929764395,\n \"min\": 1982.2825418307086,\n \"avg\": 1985.047917403574,\n \"last\": 1982.2825418307086,\n \"last-5-avg\": 1985.047917403574,\n \"last-10-avg\": 1985.047917403574\n },\n \"should_checkpoint\": {\n \"max\": true,\n \"min\": true,\n \"avg\": 1.0,\n \"last\": true,\n \"last-5-avg\": 1.0,\n \"last-10-avg\": 1.0\n },\n \"done\": {\n \"max\": false,\n \"min\": false,\n \"avg\": 0.0,\n \"last\": false,\n \"last-5-avg\": 0.0,\n \"last-10-avg\": 0.0\n },\n \"training_iteration\": {\n \"max\": 2,\n \"min\": 1,\n \"avg\": 1.5,\n \"last\": 2,\n \"last-5-avg\": 1.5,\n \"last-10-avg\": 1.5\n },\n \"time_this_iter_s\": {\n \"max\": 126.36091089248657,\n \"min\": 125.93394875526428,\n \"avg\": 126.14742982387543,\n \"last\": 126.36091089248657,\n \"last-5-avg\": 126.14742982387543,\n \"last-10-avg\": 126.14742982387543\n },\n \"time_total_s\": {\n \"max\": 252.29485964775085,\n \"min\": 125.93394875526428,\n \"avg\": 189.11440420150757,\n \"last\": 252.29485964775085,\n \"last-5-avg\": 189.11440420150757,\n \"last-10-avg\": 189.11440420150757\n },\n \"time_since_restore\": {\n \"max\": 126.36091089248657,\n \"min\": 125.93394875526428,\n \"avg\": 126.14742982387543,\n \"last\": 126.36091089248657,\n \"last-5-avg\": 126.14742982387543,\n \"last-10-avg\": 126.14742982387543\n },\n \"iterations_since_restore\": {\n \"max\": 1,\n \"min\": 1,\n \"avg\": 1.0,\n \"last\": 1,\n \"last-5-avg\": 1.0,\n \"last-10-avg\": 1.0\n }\n },\n \"_n_steps\": [\n 5,\n 10\n ],\n \"metric_n_steps\": {\n \"loss\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005959d000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294288c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b0074946243087fbfdfcf400f9f4094869452946807680d4308954aa55221f99e409486945294652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005959d000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294288c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b0074946243087fbfdfcf400f9f4094869452946807680d4308954aa55221f99e409486945294652e\"\n }\n },\n \"should_checkpoint\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059524000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294288888652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059524000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294288888652e\"\n }\n },\n \"done\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059524000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294288989652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059524000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294288989652e\"\n }\n },\n \"training_iteration\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059526000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294284b014b02652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059526000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294284b014b02652e\"\n }\n },\n \"time_this_iter_s\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059534000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452942847405f7bc5d100000047405f97192a000000652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059534000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452942847405f7bc5d100000047405f97192a000000652e\"\n }\n },\n \"time_total_s\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059534000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452942847405f7bc5d100000047406f896f7d800000652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059534000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452942847405f7bc5d100000047406f896f7d800000652e\"\n }\n },\n \"time_since_restore\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059534000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452942847405f7bc5d100000047405f97192a000000652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059534000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452942847405f7bc5d100000047405f97192a000000652e\"\n }\n },\n \"iterations_since_restore\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059526000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294284b014b01652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059526000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294284b014b01652e\"\n }\n }\n },\n \"checkpoint_manager\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"800595a5090000000000008c267261792e747261696e2e5f696e7465726e616c2e636865636b706f696e745f6d616e61676572948c125f436865636b706f696e744d616e616765729493942981947d94288c125f636865636b706f696e745f636f6e666967948c097261792e747261696e948c10436865636b706f696e74436f6e6669679493942981947d94288c0b6e756d5f746f5f6b656570944e8c1a636865636b706f696e745f73636f72655f617474726962757465944e8c16636865636b706f696e745f73636f72655f6f72646572948c036d6178948c14636865636b706f696e745f6672657175656e6379944b008c11636865636b706f696e745f61745f656e6494898c1a5f636865636b706f696e745f6b6565705f616c6c5f72616e6b73948c0a44455052454341544544948c1f5f636865636b706f696e745f75706c6f61645f66726f6d5f776f726b65727394681275628c135f636865636b706f696e745f726573756c7473945d94288c1b7261792e747261696e2e5f696e7465726e616c2e73657373696f6e948c0f5f547261696e696e67526573756c749493942981947d94288c0a636865636b706f696e749468068c0a436865636b706f696e749493942981947d94288c0470617468948cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f61663664396333385f355f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d313032342c6465636f6465725f696e745f6e756d5f6c617965723d382c6465636f6465725f72745f66663d313032342c6465636f6465725f72745f6e756d5f6c617965723d382c64726f705f5f323032342d30362d32355f31362d31322d30382f636865636b706f696e745f303030303030948c0a66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c055f75756964948c0475756964948c04555549449493942981947d948c03696e74948a115d4407c8c02494bf934f054d85e6a9e100736275628c076d657472696373947d94288c046c6f7373948c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b0074946243087fbfdfcf400f9f4094869452948c0974696d657374616d70944af6d07a668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303030948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b018c08747269616c5f6964948c086166366439633338948c0464617465948c13323032342d30362d32355f31362d31352d3138948c1074696d655f746869735f697465725f739447405f7bc5d10000008c0c74696d655f746f74616c5f739447405f7bc5d10000008c03706964944ac60e01008c08686f73746e616d65948c06723369356e36948c076e6f64655f6970948c0c31302e3135392e382e313539948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b088c146465636f6465725f72745f6e756d5f6c61796572944b088c156465636f6465725f696e745f6e756d5f6c61796572944b088c0d656d62656464696e675f64696d944b108c0a656e636f6465725f6666944d00028c0d6465636f6465725f72745f6666944d00048c0e6465636f6465725f696e745f6666944d00048c066e5f68656164944b108c0964726f705f7261746594473fe2eb349d843c768c026c7294473f82d902b7d68b998c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f72659447405f7bc5d10000008c18697465726174696f6e735f73696e63655f726573746f7265944b0175756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f61663664396333385f355f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d313032342c6465636f6465725f696e745f6e756d5f6c617965723d382c6465636f6465725f72745f66663d313032342c6465636f6465725f72745f6e756d5f6c617965723d382c64726f705f5f323032342d30362d32355f31362d31322d30382f636865636b706f696e745f30303030303194682268257d946827897385945294682a682d2981947d9468308a10d273959b2f684f801f41e4e3e4159f7b7362756268317d94288c046c6f7373946836683c4308954aa55221f99e4094869452948c0974696d657374616d70944a8cda7a668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303031948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b028c08747269616c5f6964948c086166366439633338948c0464617465948c13323032342d30362d32355f31362d35362d3132948c1074696d655f746869735f697465725f739447405f97192a0000008c0c74696d655f746f74616c5f739447406f896f7d8000008c03706964944ae4fb01008c08686f73746e616d65948c06723369356e36948c076e6f64655f6970948c0c31302e3135392e382e313539948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b088c146465636f6465725f72745f6e756d5f6c61796572944b088c156465636f6465725f696e745f6e756d5f6c61796572944b088c0d656d62656464696e675f64696d944b108c0a656e636f6465725f6666944d00028c0d6465636f6465725f72745f6666944d00048c0e6465636f6465725f696e745f6666944d00048c066e5f68656164944b108c0964726f705f7261746594473fe2eb349d843c768c026c7294473f82d902b7d68b998c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f72659447405f97192a0000008c18697465726174696f6e735f73696e63655f726573746f7265944b01757562658c195f6c61746573745f636865636b706f696e745f726573756c7494686275622e\"\n }\n}"
+ ],
+ [
+ "{\n \"stub\": false,\n \"trainable_name\": \"train_model\",\n \"trial_id\": \"b577349d\",\n \"storage\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059572040000000000008c1b7261792e747261696e2e5f696e7465726e616c2e73746f72616765948c0e53746f72616765436f6e746578749493942981947d94288c12637573746f6d5f66735f70726f766964656494898c1273746f726167655f6c6f63616c5f70617468948c2b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c7473948c136578706572696d656e745f6469725f6e616d65948c0f746573745f6578706572696d656e74948c0e747269616c5f6469725f6e616d65948c96747261696e5f6d6f64656c5f62353737333439645f365f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d322c6465636f6465725f72745f66663d313032342c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f725f323032342d30362d32355f31362d31332d3132948c1863757272656e745f636865636b706f696e745f696e646578944b018c0b73796e635f636f6e666967948c097261792e747261696e948c0a53796e63436f6e6669679493942981947d94288c0b73796e635f706572696f64944d2c018c0c73796e635f74696d656f7574944d08078c0e73796e635f61727469666163747394898c1c73796e635f6172746966616374735f6f6e5f636865636b706f696e7494888c0a75706c6f61645f646972948c0a44455052454341544544948c0673796e6365729468188c1273796e635f6f6e5f636865636b706f696e7494681875628c1273746f726167655f66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c0f73746f726167655f66735f70617468948c452f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f7465737494681968008c115f46696c6573797374656d53796e6365729493942981947d9428681b682268134d2c0168144d08078c116c6173745f73796e635f75705f74696d65944741d99eb418846a298c136c6173745f73796e635f646f776e5f74696d659447fff00000000000008c0d5f73796e635f70726f63657373944e8c0c5f63757272656e745f636d649468008c125f75706c6f61645f746f5f66735f706174689493947d94288c0a6c6f63616c5f70617468948c3b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c74732f746573745f6578706572696d656e74948c0266739468228c0766735f70617468948c552f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e74948c076578636c756465945d948c0e2a2f636865636b706f696e745f2a9461758694756275622e\"\n },\n \"config\": {\n \"encoder_num_layer\": 8,\n \"decoder_rt_num_layer\": 1,\n \"decoder_int_num_layer\": 2,\n \"embedding_dim\": 64,\n \"encoder_ff\": 2048,\n \"decoder_rt_ff\": 1024,\n \"decoder_int_ff\": 512,\n \"n_head\": 2,\n \"drop_rate\": 0.2945022566291742,\n \"lr\": 0.000496318687335349,\n \"batch_size\": 2048\n },\n \"_Trial__unresolved_config\": {\n \"encoder_num_layer\": 8,\n \"decoder_rt_num_layer\": 1,\n \"decoder_int_num_layer\": 2,\n \"embedding_dim\": 64,\n \"encoder_ff\": 2048,\n \"decoder_rt_ff\": 1024,\n \"decoder_int_ff\": 512,\n \"n_head\": 2,\n \"drop_rate\": 0.2945022566291742,\n \"lr\": 0.000496318687335349,\n \"batch_size\": 2048\n },\n \"evaluated_params\": {\n \"batch_size\": 2048,\n \"decoder_int_ff\": 512,\n \"decoder_int_num_layer\": 2,\n \"decoder_rt_ff\": 1024,\n \"decoder_rt_num_layer\": 1,\n \"drop_rate\": 0.2945022566291742,\n \"embedding_dim\": 64,\n \"encoder_ff\": 2048,\n \"encoder_num_layer\": 8,\n \"lr\": 0.000496318687335349,\n \"n_head\": 2\n },\n \"experiment_tag\": \"6_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=2,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop_rate=0.2945,embedding_dim=64,encoder_ff=2048,encoder_num_layer=8,lr=0.0005,n_head=2\",\n \"stopping_criterion\": {},\n \"_setup_default_resource\": true,\n \"_default_placement_group_factory\": \"80054e2e\",\n \"placement_group_factory\": \"800595ba000000000000008c237261792e74756e652e657865637574696f6e2e706c6163656d656e745f67726f757073948c15506c6163656d656e7447726f7570466163746f72799493942981947d94288c085f62756e646c6573945d947d94288c03435055944740540000000000008c034750559447401000000000000075618c155f686561645f62756e646c655f69735f656d70747994898c095f7374726174656779948c045041434b948c055f6172677394298c075f6b7761726773947d9475622e\",\n \"log_to_file\": [\n null,\n null\n ],\n \"max_failures\": 0,\n \"_default_result_or_future\": null,\n \"export_formats\": [],\n \"status\": \"PAUSED\",\n \"relative_logdir\": \"train_model_b577349d_6_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=2,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop_r_2024-06-25_16-13-12\",\n \"trial_name_creator\": null,\n \"trial_dirname_creator\": null,\n \"custom_trial_name\": null,\n \"custom_dirname\": null,\n \"restore_path\": null,\n \"_restore_checkpoint_result\": null,\n \"_state_json\": null,\n \"results\": \"80054e2e\",\n \"extra_arg\": \"80054e2e\",\n \"_resources\": \"80054e2e\"\n}",
+ "{\n \"start_time\": 1719324918.8126864,\n \"num_failures\": 0,\n \"num_failures_after_restore\": 0,\n \"error_filename\": null,\n \"pickled_error_filename\": null,\n \"last_result\": {\n \"loss\": 1958.5695377860482,\n \"timestamp\": 1719327468,\n \"checkpoint_dir_name\": \"checkpoint_000001\",\n \"should_checkpoint\": true,\n \"done\": false,\n \"training_iteration\": 2,\n \"trial_id\": \"b577349d\",\n \"date\": \"2024-06-25_16-57-48\",\n \"time_this_iter_s\": 96.6448974609375,\n \"time_total_s\": 192.2383737564087,\n \"pid\": 130020,\n \"hostname\": \"r3i5n6\",\n \"node_ip\": \"10.159.8.159\",\n \"config\": {\n \"encoder_num_layer\": 8,\n \"decoder_rt_num_layer\": 1,\n \"decoder_int_num_layer\": 2,\n \"embedding_dim\": 64,\n \"encoder_ff\": 2048,\n \"decoder_rt_ff\": 1024,\n \"decoder_int_ff\": 512,\n \"n_head\": 2,\n \"drop_rate\": 0.2945022566291742,\n \"lr\": 0.000496318687335349,\n \"batch_size\": 2048\n },\n \"time_since_restore\": 96.6448974609375,\n \"iterations_since_restore\": 1,\n \"experiment_tag\": \"6_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=2,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop_rate=0.2945,embedding_dim=64,encoder_ff=2048,encoder_num_layer=8,lr=0.0005,n_head=2\"\n },\n \"last_result_time\": 1719327468.8863273,\n \"metric_analysis\": {\n \"loss\": {\n \"max\": 1958.5695377860482,\n \"min\": 1958.5306944359006,\n \"avg\": 1958.5501161109744,\n \"last\": 1958.5695377860482,\n \"last-5-avg\": 1958.5501161109744,\n \"last-10-avg\": 1958.5501161109744\n },\n \"should_checkpoint\": {\n \"max\": true,\n \"min\": true,\n \"avg\": 1.0,\n \"last\": true,\n \"last-5-avg\": 1.0,\n \"last-10-avg\": 1.0\n },\n \"done\": {\n \"max\": false,\n \"min\": false,\n \"avg\": 0.0,\n \"last\": false,\n \"last-5-avg\": 0.0,\n \"last-10-avg\": 0.0\n },\n \"training_iteration\": {\n \"max\": 2,\n \"min\": 1,\n \"avg\": 1.5,\n \"last\": 2,\n \"last-5-avg\": 1.5,\n \"last-10-avg\": 1.5\n },\n \"time_this_iter_s\": {\n \"max\": 96.6448974609375,\n \"min\": 95.59347629547119,\n \"avg\": 96.11918687820435,\n \"last\": 96.6448974609375,\n \"last-5-avg\": 96.11918687820435,\n \"last-10-avg\": 96.11918687820435\n },\n \"time_total_s\": {\n \"max\": 192.2383737564087,\n \"min\": 95.59347629547119,\n \"avg\": 143.91592502593994,\n \"last\": 192.2383737564087,\n \"last-5-avg\": 143.91592502593994,\n \"last-10-avg\": 143.91592502593994\n },\n \"time_since_restore\": {\n \"max\": 96.6448974609375,\n \"min\": 95.59347629547119,\n \"avg\": 96.11918687820435,\n \"last\": 96.6448974609375,\n \"last-5-avg\": 96.11918687820435,\n \"last-10-avg\": 96.11918687820435\n },\n \"iterations_since_restore\": {\n \"max\": 1,\n \"min\": 1,\n \"avg\": 1.0,\n \"last\": 1,\n \"last-5-avg\": 1.0,\n \"last-10-avg\": 1.0\n }\n },\n \"_n_steps\": [\n 5,\n 10\n ],\n \"metric_n_steps\": {\n \"loss\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005959d000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294288c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b00749462430873b95c6e1f9a9e4094869452946807680d4308a7d3e934479a9e409486945294652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005959d000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294288c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b00749462430873b95c6e1f9a9e4094869452946807680d4308a7d3e934479a9e409486945294652e\"\n }\n },\n \"should_checkpoint\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059524000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294288888652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059524000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294288888652e\"\n }\n },\n \"done\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059524000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294288989652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059524000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294288989652e\"\n }\n },\n \"training_iteration\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059526000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294284b014b02652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059526000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294284b014b02652e\"\n }\n },\n \"time_this_iter_s\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059534000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b058694529428474057e5fb84000000474058294600000000652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059534000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a8694529428474057e5fb84000000474058294600000000652e\"\n }\n },\n \"time_total_s\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059534000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b058694529428474057e5fb8400000047406807a0c2000000652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059534000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a8694529428474057e5fb8400000047406807a0c2000000652e\"\n }\n },\n \"time_since_restore\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059534000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b058694529428474057e5fb84000000474058294600000000652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059534000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a8694529428474057e5fb84000000474058294600000000652e\"\n }\n },\n \"iterations_since_restore\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059526000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294284b014b01652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059526000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294284b014b01652e\"\n }\n }\n },\n \"checkpoint_manager\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"800595a6090000000000008c267261792e747261696e2e5f696e7465726e616c2e636865636b706f696e745f6d616e61676572948c125f436865636b706f696e744d616e616765729493942981947d94288c125f636865636b706f696e745f636f6e666967948c097261792e747261696e948c10436865636b706f696e74436f6e6669679493942981947d94288c0b6e756d5f746f5f6b656570944e8c1a636865636b706f696e745f73636f72655f617474726962757465944e8c16636865636b706f696e745f73636f72655f6f72646572948c036d6178948c14636865636b706f696e745f6672657175656e6379944b008c11636865636b706f696e745f61745f656e6494898c1a5f636865636b706f696e745f6b6565705f616c6c5f72616e6b73948c0a44455052454341544544948c1f5f636865636b706f696e745f75706c6f61645f66726f6d5f776f726b65727394681275628c135f636865636b706f696e745f726573756c7473945d94288c1b7261792e747261696e2e5f696e7465726e616c2e73657373696f6e948c0f5f547261696e696e67526573756c749493942981947d94288c0a636865636b706f696e749468068c0a436865636b706f696e749493942981947d94288c0470617468948cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f62353737333439645f365f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d322c6465636f6465725f72745f66663d313032342c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f725f323032342d30362d32355f31362d31332d31322f636865636b706f696e745f303030303030948c0a66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c055f75756964948c0475756964948c04555549449493942981947d948c03696e74948a11539cca0b2b6fcbbd0f44024aa04e0da500736275628c076d657472696373947d94288c046c6f7373948c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b00749462430873b95c6e1f9a9e4094869452948c0974696d657374616d70944a56d17a668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303030948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b018c08747269616c5f6964948c086235373733343964948c0464617465948c13323032342d30362d32355f31362d31362d3534948c1074696d655f746869735f697465725f7394474057e5fb840000008c0c74696d655f746f74616c5f7394474057e5fb840000008c03706964944ac60e01008c08686f73746e616d65948c06723369356e36948c076e6f64655f6970948c0c31302e3135392e382e313539948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b088c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b028c0d656d62656464696e675f64696d944b408c0a656e636f6465725f6666944d00088c0d6465636f6465725f72745f6666944d00048c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b028c0964726f705f7261746594473fd2d91ffe34831c8c026c7294473f40436c43e763ef8c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f726594474057e5fb840000008c18697465726174696f6e735f73696e63655f726573746f7265944b0175756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f62353737333439645f365f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d322c6465636f6465725f72745f66663d313032342c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f725f323032342d30362d32355f31362d31332d31322f636865636b706f696e745f30303030303194682268257d946827897385945294682a682d2981947d9468308a11a771b6bee3cbf999f74d2cbec6d3de8c007362756268317d94288c046c6f7373946836683c4308a7d3e934479a9e4094869452948c0974696d657374616d70944aecda7a668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303031948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b028c08747269616c5f6964948c086235373733343964948c0464617465948c13323032342d30362d32355f31362d35372d3438948c1074696d655f746869735f697465725f73944740582946000000008c0c74696d655f746f74616c5f739447406807a0c20000008c03706964944ae4fb01008c08686f73746e616d65948c06723369356e36948c076e6f64655f6970948c0c31302e3135392e382e313539948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b088c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b028c0d656d62656464696e675f64696d944b408c0a656e636f6465725f6666944d00088c0d6465636f6465725f72745f6666944d00048c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b028c0964726f705f7261746594473fd2d91ffe34831c8c026c7294473f40436c43e763ef8c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f7265944740582946000000008c18697465726174696f6e735f73696e63655f726573746f7265944b01757562658c195f6c61746573745f636865636b706f696e745f726573756c7494686275622e\"\n }\n}"
+ ],
+ [
+ "{\n \"stub\": false,\n \"trainable_name\": \"train_model\",\n \"trial_id\": \"86ae258f\",\n \"storage\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059572040000000000008c1b7261792e747261696e2e5f696e7465726e616c2e73746f72616765948c0e53746f72616765436f6e746578749493942981947d94288c12637573746f6d5f66735f70726f766964656494898c1273746f726167655f6c6f63616c5f70617468948c2b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c7473948c136578706572696d656e745f6469725f6e616d65948c0f746573745f6578706572696d656e74948c0e747269616c5f6469725f6e616d65948c96747261696e5f6d6f64656c5f38366165323538665f375f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d313032342c6465636f6465725f696e745f6e756d5f6c617965723d342c6465636f6465725f72745f66663d313032342c6465636f6465725f72745f6e756d5f6c617965723d342c64726f705f5f323032342d30362d32355f31362d31352d3138948c1863757272656e745f636865636b706f696e745f696e646578944b018c0b73796e635f636f6e666967948c097261792e747261696e948c0a53796e63436f6e6669679493942981947d94288c0b73796e635f706572696f64944d2c018c0c73796e635f74696d656f7574944d08078c0e73796e635f61727469666163747394898c1c73796e635f6172746966616374735f6f6e5f636865636b706f696e7494888c0a75706c6f61645f646972948c0a44455052454341544544948c0673796e6365729468188c1273796e635f6f6e5f636865636b706f696e7494681875628c1273746f726167655f66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c0f73746f726167655f66735f70617468948c452f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f7465737494681968008c115f46696c6573797374656d53796e6365729493942981947d9428681b682268134d2c0168144d08078c116c6173745f73796e635f75705f74696d65944741d99eb418846a298c136c6173745f73796e635f646f776e5f74696d659447fff00000000000008c0d5f73796e635f70726f63657373944e8c0c5f63757272656e745f636d649468008c125f75706c6f61645f746f5f66735f706174689493947d94288c0a6c6f63616c5f70617468948c3b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c74732f746573745f6578706572696d656e74948c0266739468228c0766735f70617468948c552f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e74948c076578636c756465945d948c0e2a2f636865636b706f696e745f2a9461758694756275622e\"\n },\n \"config\": {\n \"encoder_num_layer\": 2,\n \"decoder_rt_num_layer\": 4,\n \"decoder_int_num_layer\": 4,\n \"embedding_dim\": 256,\n \"encoder_ff\": 2048,\n \"decoder_rt_ff\": 1024,\n \"decoder_int_ff\": 1024,\n \"n_head\": 16,\n \"drop_rate\": 0.7568171644612052,\n \"lr\": 0.0002629821336450532,\n \"batch_size\": 2048\n },\n \"_Trial__unresolved_config\": {\n \"encoder_num_layer\": 2,\n \"decoder_rt_num_layer\": 4,\n \"decoder_int_num_layer\": 4,\n \"embedding_dim\": 256,\n \"encoder_ff\": 2048,\n \"decoder_rt_ff\": 1024,\n \"decoder_int_ff\": 1024,\n \"n_head\": 16,\n \"drop_rate\": 0.7568171644612052,\n \"lr\": 0.0002629821336450532,\n \"batch_size\": 2048\n },\n \"evaluated_params\": {\n \"batch_size\": 2048,\n \"decoder_int_ff\": 1024,\n \"decoder_int_num_layer\": 4,\n \"decoder_rt_ff\": 1024,\n \"decoder_rt_num_layer\": 4,\n \"drop_rate\": 0.7568171644612052,\n \"embedding_dim\": 256,\n \"encoder_ff\": 2048,\n \"encoder_num_layer\": 2,\n \"lr\": 0.0002629821336450532,\n \"n_head\": 16\n },\n \"experiment_tag\": \"7_batch_size=2048,decoder_int_ff=1024,decoder_int_num_layer=4,decoder_rt_ff=1024,decoder_rt_num_layer=4,drop_rate=0.7568,embedding_dim=256,encoder_ff=2048,encoder_num_layer=2,lr=0.0003,n_head=16\",\n \"stopping_criterion\": {},\n \"_setup_default_resource\": true,\n \"_default_placement_group_factory\": \"80054e2e\",\n \"placement_group_factory\": \"800595ba000000000000008c237261792e74756e652e657865637574696f6e2e706c6163656d656e745f67726f757073948c15506c6163656d656e7447726f7570466163746f72799493942981947d94288c085f62756e646c6573945d947d94288c03435055944740540000000000008c034750559447401000000000000075618c155f686561645f62756e646c655f69735f656d70747994898c095f7374726174656779948c045041434b948c055f6172677394298c075f6b7761726773947d9475622e\",\n \"log_to_file\": [\n null,\n null\n ],\n \"max_failures\": 0,\n \"_default_result_or_future\": null,\n \"export_formats\": [],\n \"status\": \"PAUSED\",\n \"relative_logdir\": \"train_model_86ae258f_7_batch_size=2048,decoder_int_ff=1024,decoder_int_num_layer=4,decoder_rt_ff=1024,decoder_rt_num_layer=4,drop__2024-06-25_16-15-18\",\n \"trial_name_creator\": null,\n \"trial_dirname_creator\": null,\n \"custom_trial_name\": null,\n \"custom_dirname\": null,\n \"restore_path\": null,\n \"_restore_checkpoint_result\": null,\n \"_state_json\": null,\n \"results\": \"80054e2e\",\n \"extra_arg\": \"80054e2e\",\n \"_resources\": \"80054e2e\"\n}",
+ "{\n \"start_time\": 1719325014.5539198,\n \"num_failures\": 0,\n \"num_failures_after_restore\": 0,\n \"error_filename\": null,\n \"pickled_error_filename\": null,\n \"last_result\": {\n \"loss\": 1995.102591927596,\n \"timestamp\": 1719327610,\n \"checkpoint_dir_name\": \"checkpoint_000001\",\n \"should_checkpoint\": true,\n \"done\": false,\n \"training_iteration\": 2,\n \"trial_id\": \"86ae258f\",\n \"date\": \"2024-06-25_17-00-10\",\n \"time_this_iter_s\": 141.5520534515381,\n \"time_total_s\": 283.12366557121277,\n \"pid\": 130020,\n \"hostname\": \"r3i5n6\",\n \"node_ip\": \"10.159.8.159\",\n \"config\": {\n \"encoder_num_layer\": 2,\n \"decoder_rt_num_layer\": 4,\n \"decoder_int_num_layer\": 4,\n \"embedding_dim\": 256,\n \"encoder_ff\": 2048,\n \"decoder_rt_ff\": 1024,\n \"decoder_int_ff\": 1024,\n \"n_head\": 16,\n \"drop_rate\": 0.7568171644612052,\n \"lr\": 0.0002629821336450532,\n \"batch_size\": 2048\n },\n \"time_since_restore\": 141.5520534515381,\n \"iterations_since_restore\": 1,\n \"experiment_tag\": \"7_batch_size=2048,decoder_int_ff=1024,decoder_int_num_layer=4,decoder_rt_ff=1024,decoder_rt_num_layer=4,drop_rate=0.7568,embedding_dim=256,encoder_ff=2048,encoder_num_layer=2,lr=0.0003,n_head=16\"\n },\n \"last_result_time\": 1719327610.6072223,\n \"metric_analysis\": {\n \"loss\": {\n \"max\": 1995.102591927596,\n \"min\": 1993.6307767132137,\n \"avg\": 1994.3666843204048,\n \"last\": 1995.102591927596,\n \"last-5-avg\": 1994.3666843204048,\n \"last-10-avg\": 1994.3666843204048\n },\n \"should_checkpoint\": {\n \"max\": true,\n \"min\": true,\n \"avg\": 1.0,\n \"last\": true,\n \"last-5-avg\": 1.0,\n \"last-10-avg\": 1.0\n },\n \"done\": {\n \"max\": false,\n \"min\": false,\n \"avg\": 0.0,\n \"last\": false,\n \"last-5-avg\": 0.0,\n \"last-10-avg\": 0.0\n },\n \"training_iteration\": {\n \"max\": 2,\n \"min\": 1,\n \"avg\": 1.5,\n \"last\": 2,\n \"last-5-avg\": 1.5,\n \"last-10-avg\": 1.5\n },\n \"time_this_iter_s\": {\n \"max\": 141.57161211967468,\n \"min\": 141.5520534515381,\n \"avg\": 141.56183278560638,\n \"last\": 141.5520534515381,\n \"last-5-avg\": 141.56183278560638,\n \"last-10-avg\": 141.56183278560638\n },\n \"time_total_s\": {\n \"max\": 283.12366557121277,\n \"min\": 141.57161211967468,\n \"avg\": 212.34763884544373,\n \"last\": 283.12366557121277,\n \"last-5-avg\": 212.34763884544373,\n \"last-10-avg\": 212.34763884544373\n },\n \"time_since_restore\": {\n \"max\": 141.57161211967468,\n \"min\": 141.5520534515381,\n \"avg\": 141.56183278560638,\n \"last\": 141.5520534515381,\n \"last-5-avg\": 141.56183278560638,\n \"last-10-avg\": 141.56183278560638\n },\n \"iterations_since_restore\": {\n \"max\": 1,\n \"min\": 1,\n \"avg\": 1.0,\n \"last\": 1,\n \"last-5-avg\": 1.0,\n \"last-10-avg\": 1.0\n }\n },\n \"_n_steps\": [\n 5,\n 10\n ],\n \"metric_n_steps\": {\n \"loss\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005959d000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294288c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b00749462430853a954ea85269f4094869452946807680d43086fb7db0d692c9f409486945294652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005959d000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294288c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b00749462430853a954ea85269f4094869452946807680d43086fb7db0d692c9f409486945294652e\"\n }\n },\n \"should_checkpoint\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059524000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294288888652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059524000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294288888652e\"\n }\n },\n \"done\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059524000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294288989652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059524000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294288989652e\"\n }\n },\n \"training_iteration\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059526000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294284b014b02652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059526000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294284b014b02652e\"\n }\n },\n \"time_this_iter_s\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059534000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b058694529428474061b24aa5800000474061b1aa6c000000652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059534000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a8694529428474061b24aa5800000474061b1aa6c000000652e\"\n }\n },\n \"time_total_s\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059534000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b058694529428474061b24aa5800000474071b1fa88c00000652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059534000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a8694529428474061b24aa5800000474071b1fa88c00000652e\"\n }\n },\n \"time_since_restore\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059534000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b058694529428474061b24aa5800000474061b1aa6c000000652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059534000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a8694529428474061b24aa5800000474061b1aa6c000000652e\"\n }\n },\n \"iterations_since_restore\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059526000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294284b014b01652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059526000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294284b014b01652e\"\n }\n }\n },\n \"checkpoint_manager\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"800595a7090000000000008c267261792e747261696e2e5f696e7465726e616c2e636865636b706f696e745f6d616e61676572948c125f436865636b706f696e744d616e616765729493942981947d94288c125f636865636b706f696e745f636f6e666967948c097261792e747261696e948c10436865636b706f696e74436f6e6669679493942981947d94288c0b6e756d5f746f5f6b656570944e8c1a636865636b706f696e745f73636f72655f617474726962757465944e8c16636865636b706f696e745f73636f72655f6f72646572948c036d6178948c14636865636b706f696e745f6672657175656e6379944b008c11636865636b706f696e745f61745f656e6494898c1a5f636865636b706f696e745f6b6565705f616c6c5f72616e6b73948c0a44455052454341544544948c1f5f636865636b706f696e745f75706c6f61645f66726f6d5f776f726b65727394681275628c135f636865636b706f696e745f726573756c7473945d94288c1b7261792e747261696e2e5f696e7465726e616c2e73657373696f6e948c0f5f547261696e696e67526573756c749493942981947d94288c0a636865636b706f696e749468068c0a436865636b706f696e749493942981947d94288c0470617468948cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f38366165323538665f375f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d313032342c6465636f6465725f696e745f6e756d5f6c617965723d342c6465636f6465725f72745f66663d313032342c6465636f6465725f72745f6e756d5f6c617965723d342c64726f705f5f323032342d30362d32355f31362d31352d31382f636865636b706f696e745f303030303030948c0a66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c055f75756964948c0475756964948c04555549449493942981947d948c03696e74948a11f44260e4487af3908249f6c6bea7609700736275628c076d657472696373947d94288c046c6f7373948c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b00749462430853a954ea85269f4094869452948c0974696d657374616d70944ae4d17a668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303030948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b018c08747269616c5f6964948c083836616532353866948c0464617465948c13323032342d30362d32355f31362d31392d3136948c1074696d655f746869735f697465725f7394474061b24aa58000008c0c74696d655f746f74616c5f7394474061b24aa58000008c03706964944ac60e01008c08686f73746e616d65948c06723369356e36948c076e6f64655f6970948c0c31302e3135392e382e313539948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b028c146465636f6465725f72745f6e756d5f6c61796572944b048c156465636f6465725f696e745f6e756d5f6c61796572944b048c0d656d62656464696e675f64696d944d00018c0a656e636f6465725f6666944d00088c0d6465636f6465725f72745f6666944d00048c0e6465636f6465725f696e745f6666944d00048c066e5f68656164944b108c0964726f705f7261746594473fe837d8a14d31ce8c026c7294473f313c1ba9d70f5a8c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f726594474061b24aa58000008c18697465726174696f6e735f73696e63655f726573746f7265944b0175756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f38366165323538665f375f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d313032342c6465636f6465725f696e745f6e756d5f6c617965723d342c6465636f6465725f72745f66663d313032342c6465636f6465725f72745f6e756d5f6c617965723d342c64726f705f5f323032342d30362d32355f31362d31352d31382f636865636b706f696e745f30303030303194682268257d946827897385945294682a682d2981947d9468308a104c28418a6a14e6b2a24745a6d15899067362756268317d94288c046c6f7373946836683c43086fb7db0d692c9f4094869452948c0974696d657374616d70944a7adb7a668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303031948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b028c08747269616c5f6964948c083836616532353866948c0464617465948c13323032342d30362d32355f31372d30302d3130948c1074696d655f746869735f697465725f7394474061b1aa6c0000008c0c74696d655f746f74616c5f7394474071b1fa88c000008c03706964944ae4fb01008c08686f73746e616d65948c06723369356e36948c076e6f64655f6970948c0c31302e3135392e382e313539948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b028c146465636f6465725f72745f6e756d5f6c61796572944b048c156465636f6465725f696e745f6e756d5f6c61796572944b048c0d656d62656464696e675f64696d944d00018c0a656e636f6465725f6666944d00088c0d6465636f6465725f72745f6666944d00048c0e6465636f6465725f696e745f6666944d00048c066e5f68656164944b108c0964726f705f7261746594473fe837d8a14d31ce8c026c7294473f313c1ba9d70f5a8c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f726594474061b1aa6c0000008c18697465726174696f6e735f73696e63655f726573746f7265944b01757562658c195f6c61746573745f636865636b706f696e745f726573756c7494686275622e\"\n }\n}"
+ ],
+ [
+ "{\n \"stub\": false,\n \"trainable_name\": \"train_model\",\n \"trial_id\": \"c4172973\",\n \"storage\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059572040000000000008c1b7261792e747261696e2e5f696e7465726e616c2e73746f72616765948c0e53746f72616765436f6e746578749493942981947d94288c12637573746f6d5f66735f70726f766964656494898c1273746f726167655f6c6f63616c5f70617468948c2b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c7473948c136578706572696d656e745f6469725f6e616d65948c0f746573745f6578706572696d656e74948c0e747269616c5f6469725f6e616d65948c96747261696e5f6d6f64656c5f63343137323937335f385f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d323034382c6465636f6465725f696e745f6e756d5f6c617965723d382c6465636f6465725f72745f66663d3531322c6465636f6465725f72745f6e756d5f6c617965723d322c64726f705f725f323032342d30362d32355f31362d31362d3534948c1863757272656e745f636865636b706f696e745f696e646578944b018c0b73796e635f636f6e666967948c097261792e747261696e948c0a53796e63436f6e6669679493942981947d94288c0b73796e635f706572696f64944d2c018c0c73796e635f74696d656f7574944d08078c0e73796e635f61727469666163747394898c1c73796e635f6172746966616374735f6f6e5f636865636b706f696e7494888c0a75706c6f61645f646972948c0a44455052454341544544948c0673796e6365729468188c1273796e635f6f6e5f636865636b706f696e7494681875628c1273746f726167655f66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c0f73746f726167655f66735f70617468948c452f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f7465737494681968008c115f46696c6573797374656d53796e6365729493942981947d9428681b682268134d2c0168144d08078c116c6173745f73796e635f75705f74696d65944741d99eb418846a298c136c6173745f73796e635f646f776e5f74696d659447fff00000000000008c0d5f73796e635f70726f63657373944e8c0c5f63757272656e745f636d649468008c125f75706c6f61645f746f5f66735f706174689493947d94288c0a6c6f63616c5f70617468948c3b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c74732f746573745f6578706572696d656e74948c0266739468228c0766735f70617468948c552f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e74948c076578636c756465945d948c0e2a2f636865636b706f696e745f2a9461758694756275622e\"\n },\n \"config\": {\n \"encoder_num_layer\": 2,\n \"decoder_rt_num_layer\": 2,\n \"decoder_int_num_layer\": 8,\n \"embedding_dim\": 256,\n \"encoder_ff\": 1024,\n \"decoder_rt_ff\": 512,\n \"decoder_int_ff\": 2048,\n \"n_head\": 2,\n \"drop_rate\": 0.4176788223006639,\n \"lr\": 0.00038150746043137153,\n \"batch_size\": 2048\n },\n \"_Trial__unresolved_config\": {\n \"encoder_num_layer\": 2,\n \"decoder_rt_num_layer\": 2,\n \"decoder_int_num_layer\": 8,\n \"embedding_dim\": 256,\n \"encoder_ff\": 1024,\n \"decoder_rt_ff\": 512,\n \"decoder_int_ff\": 2048,\n \"n_head\": 2,\n \"drop_rate\": 0.4176788223006639,\n \"lr\": 0.00038150746043137153,\n \"batch_size\": 2048\n },\n \"evaluated_params\": {\n \"batch_size\": 2048,\n \"decoder_int_ff\": 2048,\n \"decoder_int_num_layer\": 8,\n \"decoder_rt_ff\": 512,\n \"decoder_rt_num_layer\": 2,\n \"drop_rate\": 0.4176788223006639,\n \"embedding_dim\": 256,\n \"encoder_ff\": 1024,\n \"encoder_num_layer\": 2,\n \"lr\": 0.00038150746043137153,\n \"n_head\": 2\n },\n \"experiment_tag\": \"8_batch_size=2048,decoder_int_ff=2048,decoder_int_num_layer=8,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_rate=0.4177,embedding_dim=256,encoder_ff=1024,encoder_num_layer=2,lr=0.0004,n_head=2\",\n \"stopping_criterion\": {},\n \"_setup_default_resource\": true,\n \"_default_placement_group_factory\": \"80054e2e\",\n \"placement_group_factory\": \"800595ba000000000000008c237261792e74756e652e657865637574696f6e2e706c6163656d656e745f67726f757073948c15506c6163656d656e7447726f7570466163746f72799493942981947d94288c085f62756e646c6573945d947d94288c03435055944740540000000000008c034750559447401000000000000075618c155f686561645f62756e646c655f69735f656d70747994898c095f7374726174656779948c045041434b948c055f6172677394298c075f6b7761726773947d9475622e\",\n \"log_to_file\": [\n null,\n null\n ],\n \"max_failures\": 0,\n \"_default_result_or_future\": null,\n \"export_formats\": [],\n \"status\": \"PAUSED\",\n \"relative_logdir\": \"train_model_c4172973_8_batch_size=2048,decoder_int_ff=2048,decoder_int_num_layer=8,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_16-16-54\",\n \"trial_name_creator\": null,\n \"trial_dirname_creator\": null,\n \"custom_trial_name\": null,\n \"custom_dirname\": null,\n \"restore_path\": null,\n \"_restore_checkpoint_result\": null,\n \"_state_json\": null,\n \"results\": \"80054e2e\",\n \"extra_arg\": \"80054e2e\",\n \"_resources\": \"80054e2e\"\n}",
+ "{\n \"start_time\": 1719325156.2771058,\n \"num_failures\": 0,\n \"num_failures_after_restore\": 0,\n \"error_filename\": null,\n \"pickled_error_filename\": null,\n \"last_result\": {\n \"loss\": 261.25574487400803,\n \"timestamp\": 1719327690,\n \"checkpoint_dir_name\": \"checkpoint_000001\",\n \"should_checkpoint\": true,\n \"done\": false,\n \"training_iteration\": 2,\n \"trial_id\": \"c4172973\",\n \"date\": \"2024-06-25_17-01-30\",\n \"time_this_iter_s\": 79.34292578697205,\n \"time_total_s\": 157.75713872909546,\n \"pid\": 130020,\n \"hostname\": \"r3i5n6\",\n \"node_ip\": \"10.159.8.159\",\n \"config\": {\n \"encoder_num_layer\": 2,\n \"decoder_rt_num_layer\": 2,\n \"decoder_int_num_layer\": 8,\n \"embedding_dim\": 256,\n \"encoder_ff\": 1024,\n \"decoder_rt_ff\": 512,\n \"decoder_int_ff\": 2048,\n \"n_head\": 2,\n \"drop_rate\": 0.4176788223006639,\n \"lr\": 0.00038150746043137153,\n \"batch_size\": 2048\n },\n \"time_since_restore\": 79.34292578697205,\n \"iterations_since_restore\": 1,\n \"experiment_tag\": \"8_batch_size=2048,decoder_int_ff=2048,decoder_int_num_layer=8,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_rate=0.4177,embedding_dim=256,encoder_ff=1024,encoder_num_layer=2,lr=0.0004,n_head=2\"\n },\n \"last_result_time\": 1719327690.1038792,\n \"metric_analysis\": {\n \"loss\": {\n \"max\": 357.444681242695,\n \"min\": 261.25574487400803,\n \"avg\": 309.3502130583515,\n \"last\": 261.25574487400803,\n \"last-5-avg\": 309.3502130583515,\n \"last-10-avg\": 309.3502130583515\n },\n \"should_checkpoint\": {\n \"max\": true,\n \"min\": true,\n \"avg\": 1.0,\n \"last\": true,\n \"last-5-avg\": 1.0,\n \"last-10-avg\": 1.0\n },\n \"done\": {\n \"max\": false,\n \"min\": false,\n \"avg\": 0.0,\n \"last\": false,\n \"last-5-avg\": 0.0,\n \"last-10-avg\": 0.0\n },\n \"training_iteration\": {\n \"max\": 2,\n \"min\": 1,\n \"avg\": 1.5,\n \"last\": 2,\n \"last-5-avg\": 1.5,\n \"last-10-avg\": 1.5\n },\n \"time_this_iter_s\": {\n \"max\": 79.34292578697205,\n \"min\": 78.41421294212341,\n \"avg\": 78.87856936454773,\n \"last\": 79.34292578697205,\n \"last-5-avg\": 78.87856936454773,\n \"last-10-avg\": 78.87856936454773\n },\n \"time_total_s\": {\n \"max\": 157.75713872909546,\n \"min\": 78.41421294212341,\n \"avg\": 118.08567583560944,\n \"last\": 157.75713872909546,\n \"last-5-avg\": 118.08567583560944,\n \"last-10-avg\": 118.08567583560944\n },\n \"time_since_restore\": {\n \"max\": 79.34292578697205,\n \"min\": 78.41421294212341,\n \"avg\": 78.87856936454773,\n \"last\": 79.34292578697205,\n \"last-5-avg\": 78.87856936454773,\n \"last-10-avg\": 78.87856936454773\n },\n \"iterations_since_restore\": {\n \"max\": 1,\n \"min\": 1,\n \"avg\": 1.0,\n \"last\": 1,\n \"last-5-avg\": 1.0,\n \"last-10-avg\": 1.0\n }\n },\n \"_n_steps\": [\n 5,\n 10\n ],\n \"metric_n_steps\": {\n \"loss\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005959d000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294288c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b0074946243085128146a1d57764094869452946807680d4308bfdfef87175470409486945294652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005959d000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294288c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b0074946243085128146a1d57764094869452946807680d4308bfdfef87175470409486945294652e\"\n }\n },\n \"should_checkpoint\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059524000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294288888652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059524000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294288888652e\"\n }\n },\n \"done\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059524000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294288989652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059524000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294288989652e\"\n }\n },\n \"training_iteration\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059526000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294284b014b02652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059526000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294284b014b02652e\"\n }\n },\n \"time_this_iter_s\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059534000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294284740539a8277000000474053d5f27f000000652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059534000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294284740539a8277000000474053d5f27f000000652e\"\n }\n },\n \"time_total_s\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059534000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294284740539a8277000000474063b83a7b000000652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059534000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294284740539a8277000000474063b83a7b000000652e\"\n }\n },\n \"time_since_restore\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059534000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294284740539a8277000000474053d5f27f000000652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059534000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294284740539a8277000000474053d5f27f000000652e\"\n }\n },\n \"iterations_since_restore\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059526000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294284b014b01652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059526000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294284b014b01652e\"\n }\n }\n },\n \"checkpoint_manager\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"800595a8090000000000008c267261792e747261696e2e5f696e7465726e616c2e636865636b706f696e745f6d616e61676572948c125f436865636b706f696e744d616e616765729493942981947d94288c125f636865636b706f696e745f636f6e666967948c097261792e747261696e948c10436865636b706f696e74436f6e6669679493942981947d94288c0b6e756d5f746f5f6b656570944e8c1a636865636b706f696e745f73636f72655f617474726962757465944e8c16636865636b706f696e745f73636f72655f6f72646572948c036d6178948c14636865636b706f696e745f6672657175656e6379944b008c11636865636b706f696e745f61745f656e6494898c1a5f636865636b706f696e745f6b6565705f616c6c5f72616e6b73948c0a44455052454341544544948c1f5f636865636b706f696e745f75706c6f61645f66726f6d5f776f726b65727394681275628c135f636865636b706f696e745f726573756c7473945d94288c1b7261792e747261696e2e5f696e7465726e616c2e73657373696f6e948c0f5f547261696e696e67526573756c749493942981947d94288c0a636865636b706f696e749468068c0a436865636b706f696e749493942981947d94288c0470617468948cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f63343137323937335f385f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d323034382c6465636f6465725f696e745f6e756d5f6c617965723d382c6465636f6465725f72745f66663d3531322c6465636f6465725f72745f6e756d5f6c617965723d322c64726f705f725f323032342d30362d32355f31362d31362d35342f636865636b706f696e745f303030303030948c0a66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c055f75756964948c0475756964948c04555549449493942981947d948c03696e74948a11767c713a35b871b5de447f9814b0d9b300736275628c076d657472696373947d94288c046c6f7373948c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b0074946243085128146a1d57764094869452948c0974696d657374616d70944a32d27a668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303030948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b018c08747269616c5f6964948c086334313732393733948c0464617465948c13323032342d30362d32355f31362d32302d3334948c1074696d655f746869735f697465725f73944740539a82770000008c0c74696d655f746f74616c5f73944740539a82770000008c03706964944ac60e01008c08686f73746e616d65948c06723369356e36948c076e6f64655f6970948c0c31302e3135392e382e313539948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b028c146465636f6465725f72745f6e756d5f6c61796572944b028c156465636f6465725f696e745f6e756d5f6c61796572944b088c0d656d62656464696e675f64696d944d00018c0a656e636f6465725f6666944d00048c0d6465636f6465725f72745f6666944d00028c0e6465636f6465725f696e745f6666944d00088c066e5f68656164944b028c0964726f705f7261746594473fdabb3ff480d7668c026c7294473f3900a210d3dca08c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f7265944740539a82770000008c18697465726174696f6e735f73696e63655f726573746f7265944b0175756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f63343137323937335f385f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d323034382c6465636f6465725f696e745f6e756d5f6c617965723d382c6465636f6465725f72745f66663d3531322c6465636f6465725f72745f6e756d5f6c617965723d322c64726f705f725f323032342d30362d32355f31362d31362d35342f636865636b706f696e745f30303030303194682268257d946827897385945294682a682d2981947d9468308a116354e19f53c12892fe4f978c6a8c7a8b007362756268317d94288c046c6f7373946836683c4308bfdfef871754704094869452948c0974696d657374616d70944acadb7a668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303031948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b028c08747269616c5f6964948c086334313732393733948c0464617465948c13323032342d30362d32355f31372d30312d3330948c1074696d655f746869735f697465725f7394474053d5f27f0000008c0c74696d655f746f74616c5f7394474063b83a7b0000008c03706964944ae4fb01008c08686f73746e616d65948c06723369356e36948c076e6f64655f6970948c0c31302e3135392e382e313539948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b028c146465636f6465725f72745f6e756d5f6c61796572944b028c156465636f6465725f696e745f6e756d5f6c61796572944b088c0d656d62656464696e675f64696d944d00018c0a656e636f6465725f6666944d00048c0d6465636f6465725f72745f6666944d00028c0e6465636f6465725f696e745f6666944d00088c066e5f68656164944b028c0964726f705f7261746594473fdabb3ff480d7668c026c7294473f3900a210d3dca08c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f726594474053d5f27f0000008c18697465726174696f6e735f73696e63655f726573746f7265944b01757562658c195f6c61746573745f636865636b706f696e745f726573756c7494686275622e\"\n }\n}"
+ ],
+ [
+ "{\n \"stub\": false,\n \"trainable_name\": \"train_model\",\n \"trial_id\": \"1920f493\",\n \"storage\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059572040000000000008c1b7261792e747261696e2e5f696e7465726e616c2e73746f72616765948c0e53746f72616765436f6e746578749493942981947d94288c12637573746f6d5f66735f70726f766964656494898c1273746f726167655f6c6f63616c5f70617468948c2b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c7473948c136578706572696d656e745f6469725f6e616d65948c0f746573745f6578706572696d656e74948c0e747269616c5f6469725f6e616d65948c96747261696e5f6d6f64656c5f31393230663439335f395f62617463685f73697a653d313032342c6465636f6465725f696e745f66663d313032342c6465636f6465725f696e745f6e756d5f6c617965723d322c6465636f6465725f72745f66663d323034382c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f5f323032342d30362d32355f31362d31392d3136948c1863757272656e745f636865636b706f696e745f696e646578944b008c0b73796e635f636f6e666967948c097261792e747261696e948c0a53796e63436f6e6669679493942981947d94288c0b73796e635f706572696f64944d2c018c0c73796e635f74696d656f7574944d08078c0e73796e635f61727469666163747394898c1c73796e635f6172746966616374735f6f6e5f636865636b706f696e7494888c0a75706c6f61645f646972948c0a44455052454341544544948c0673796e6365729468188c1273796e635f6f6e5f636865636b706f696e7494681875628c1273746f726167655f66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c0f73746f726167655f66735f70617468948c452f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f7465737494681968008c115f46696c6573797374656d53796e6365729493942981947d9428681b682268134d2c0168144d08078c116c6173745f73796e635f75705f74696d65944741d99eb463df15148c136c6173745f73796e635f646f776e5f74696d659447fff00000000000008c0d5f73796e635f70726f63657373944e8c0c5f63757272656e745f636d649468008c125f75706c6f61645f746f5f66735f706174689493947d94288c0a6c6f63616c5f70617468948c3b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c74732f746573745f6578706572696d656e74948c0266739468228c0766735f70617468948c552f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e74948c076578636c756465945d948c0e2a2f636865636b706f696e745f2a9461758694756275622e\"\n },\n \"config\": {\n \"encoder_num_layer\": 4,\n \"decoder_rt_num_layer\": 1,\n \"decoder_int_num_layer\": 2,\n \"embedding_dim\": 64,\n \"encoder_ff\": 2048,\n \"decoder_rt_ff\": 2048,\n \"decoder_int_ff\": 1024,\n \"n_head\": 1,\n \"drop_rate\": 0.922082156004742,\n \"lr\": 0.0016525285062566051,\n \"batch_size\": 1024\n },\n \"_Trial__unresolved_config\": {\n \"encoder_num_layer\": 4,\n \"decoder_rt_num_layer\": 1,\n \"decoder_int_num_layer\": 2,\n \"embedding_dim\": 64,\n \"encoder_ff\": 2048,\n \"decoder_rt_ff\": 2048,\n \"decoder_int_ff\": 1024,\n \"n_head\": 1,\n \"drop_rate\": 0.922082156004742,\n \"lr\": 0.0016525285062566051,\n \"batch_size\": 1024\n },\n \"evaluated_params\": {\n \"batch_size\": 1024,\n \"decoder_int_ff\": 1024,\n \"decoder_int_num_layer\": 2,\n \"decoder_rt_ff\": 2048,\n \"decoder_rt_num_layer\": 1,\n \"drop_rate\": 0.922082156004742,\n \"embedding_dim\": 64,\n \"encoder_ff\": 2048,\n \"encoder_num_layer\": 4,\n \"lr\": 0.0016525285062566051,\n \"n_head\": 1\n },\n \"experiment_tag\": \"9_batch_size=1024,decoder_int_ff=1024,decoder_int_num_layer=2,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop_rate=0.9221,embedding_dim=64,encoder_ff=2048,encoder_num_layer=4,lr=0.0017,n_head=1\",\n \"stopping_criterion\": {},\n \"_setup_default_resource\": true,\n \"_default_placement_group_factory\": \"80054e2e\",\n \"placement_group_factory\": \"800595ba000000000000008c237261792e74756e652e657865637574696f6e2e706c6163656d656e745f67726f757073948c15506c6163656d656e7447726f7570466163746f72799493942981947d94288c085f62756e646c6573945d947d94288c03435055944740540000000000008c034750559447401000000000000075618c155f686561645f62756e646c655f69735f656d70747994898c095f7374726174656779948c045041434b948c055f6172677394298c075f6b7761726773947d9475622e\",\n \"log_to_file\": [\n null,\n null\n ],\n \"max_failures\": 0,\n \"_default_result_or_future\": null,\n \"export_formats\": [],\n \"status\": \"PAUSED\",\n \"relative_logdir\": \"train_model_1920f493_9_batch_size=1024,decoder_int_ff=1024,decoder_int_num_layer=2,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop__2024-06-25_16-19-16\",\n \"trial_name_creator\": null,\n \"trial_dirname_creator\": null,\n \"custom_trial_name\": null,\n \"custom_dirname\": null,\n \"restore_path\": null,\n \"_restore_checkpoint_result\": null,\n \"_state_json\": null,\n \"results\": \"80054e2e\",\n \"extra_arg\": \"80054e2e\",\n \"_resources\": \"80054e2e\"\n}",
+ "{\n \"start_time\": 1719325234.8395333,\n \"num_failures\": 0,\n \"num_failures_after_restore\": 0,\n \"error_filename\": null,\n \"pickled_error_filename\": null,\n \"last_result\": {\n \"loss\": 2102.196185254675,\n \"timestamp\": 1719325314,\n \"checkpoint_dir_name\": \"checkpoint_000000\",\n \"should_checkpoint\": true,\n \"done\": false,\n \"training_iteration\": 1,\n \"trial_id\": \"1920f493\",\n \"date\": \"2024-06-25_16-21-54\",\n \"time_this_iter_s\": 79.38143539428711,\n \"time_total_s\": 79.38143539428711,\n \"pid\": 69318,\n \"hostname\": \"r3i5n6\",\n \"node_ip\": \"10.159.8.159\",\n \"config\": {\n \"encoder_num_layer\": 4,\n \"decoder_rt_num_layer\": 1,\n \"decoder_int_num_layer\": 2,\n \"embedding_dim\": 64,\n \"encoder_ff\": 2048,\n \"decoder_rt_ff\": 2048,\n \"decoder_int_ff\": 1024,\n \"n_head\": 1,\n \"drop_rate\": 0.922082156004742,\n \"lr\": 0.0016525285062566051,\n \"batch_size\": 1024\n },\n \"time_since_restore\": 79.38143539428711,\n \"iterations_since_restore\": 1,\n \"experiment_tag\": \"9_batch_size=1024,decoder_int_ff=1024,decoder_int_num_layer=2,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop_rate=0.9221,embedding_dim=64,encoder_ff=2048,encoder_num_layer=4,lr=0.0017,n_head=1\"\n },\n \"last_result_time\": 1719325314.225798,\n \"metric_analysis\": {\n \"loss\": {\n \"max\": 2102.196185254675,\n \"min\": 2102.196185254675,\n \"avg\": 2102.196185254675,\n \"last\": 2102.196185254675,\n \"last-5-avg\": 2102.196185254675,\n \"last-10-avg\": 2102.196185254675\n },\n \"should_checkpoint\": {\n \"max\": true,\n \"min\": true,\n \"avg\": true,\n \"last\": true,\n \"last-5-avg\": true,\n \"last-10-avg\": true\n },\n \"done\": {\n \"max\": false,\n \"min\": false,\n \"avg\": false,\n \"last\": false,\n \"last-5-avg\": false,\n \"last-10-avg\": false\n },\n \"training_iteration\": {\n \"max\": 1,\n \"min\": 1,\n \"avg\": 1,\n \"last\": 1,\n \"last-5-avg\": 1,\n \"last-10-avg\": 1\n },\n \"time_this_iter_s\": {\n \"max\": 79.38143539428711,\n \"min\": 79.38143539428711,\n \"avg\": 79.38143539428711,\n \"last\": 79.38143539428711,\n \"last-5-avg\": 79.38143539428711,\n \"last-10-avg\": 79.38143539428711\n },\n \"time_total_s\": {\n \"max\": 79.38143539428711,\n \"min\": 79.38143539428711,\n \"avg\": 79.38143539428711,\n \"last\": 79.38143539428711,\n \"last-5-avg\": 79.38143539428711,\n \"last-10-avg\": 79.38143539428711\n },\n \"time_since_restore\": {\n \"max\": 79.38143539428711,\n \"min\": 79.38143539428711,\n \"avg\": 79.38143539428711,\n \"last\": 79.38143539428711,\n \"last-5-avg\": 79.38143539428711,\n \"last-10-avg\": 79.38143539428711\n },\n \"iterations_since_restore\": {\n \"max\": 1,\n \"min\": 1,\n \"avg\": 1,\n \"last\": 1,\n \"last-5-avg\": 1,\n \"last-10-avg\": 1\n }\n },\n \"_n_steps\": [\n 5,\n 10\n ],\n \"metric_n_steps\": {\n \"loss\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059589000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452948c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b00749462430893c96472646ca0409486945294612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059589000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452948c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b00749462430893c96472646ca0409486945294612e\"\n }\n },\n \"should_checkpoint\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059522000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b058694529488612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059522000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a8694529488612e\"\n }\n },\n \"done\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059522000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b058694529489612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059522000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a8694529489612e\"\n }\n },\n \"training_iteration\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059523000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452944b01612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059523000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452944b01612e\"\n }\n },\n \"time_this_iter_s\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294474053d86970000000612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294474053d86970000000612e\"\n }\n },\n \"time_total_s\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294474053d86970000000612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294474053d86970000000612e\"\n }\n },\n \"time_since_restore\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294474053d86970000000612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294474053d86970000000612e\"\n }\n },\n \"iterations_since_restore\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059523000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452944b01612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059523000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452944b01612e\"\n }\n }\n },\n \"checkpoint_manager\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059518060000000000008c267261792e747261696e2e5f696e7465726e616c2e636865636b706f696e745f6d616e61676572948c125f436865636b706f696e744d616e616765729493942981947d94288c125f636865636b706f696e745f636f6e666967948c097261792e747261696e948c10436865636b706f696e74436f6e6669679493942981947d94288c0b6e756d5f746f5f6b656570944e8c1a636865636b706f696e745f73636f72655f617474726962757465944e8c16636865636b706f696e745f73636f72655f6f72646572948c036d6178948c14636865636b706f696e745f6672657175656e6379944b008c11636865636b706f696e745f61745f656e6494898c1a5f636865636b706f696e745f6b6565705f616c6c5f72616e6b73948c0a44455052454341544544948c1f5f636865636b706f696e745f75706c6f61645f66726f6d5f776f726b65727394681275628c135f636865636b706f696e745f726573756c7473945d948c1b7261792e747261696e2e5f696e7465726e616c2e73657373696f6e948c0f5f547261696e696e67526573756c749493942981947d94288c0a636865636b706f696e749468068c0a436865636b706f696e749493942981947d94288c0470617468948cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f31393230663439335f395f62617463685f73697a653d313032342c6465636f6465725f696e745f66663d313032342c6465636f6465725f696e745f6e756d5f6c617965723d322c6465636f6465725f72745f66663d323034382c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f5f323032342d30362d32355f31362d31392d31362f636865636b706f696e745f303030303030948c0a66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c055f75756964948c0475756964948c04555549449493942981947d948c03696e74948a11fdcf99429d7f868e6d47eef0e42d098000736275628c076d657472696373947d94288c046c6f7373948c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b00749462430893c96472646ca04094869452948c0974696d657374616d70944a82d27a668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303030948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b018c08747269616c5f6964948c083139323066343933948c0464617465948c13323032342d30362d32355f31362d32312d3534948c1074696d655f746869735f697465725f7394474053d869700000008c0c74696d655f746f74616c5f7394474053d869700000008c03706964944ac60e01008c08686f73746e616d65948c06723369356e36948c076e6f64655f6970948c0c31302e3135392e382e313539948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b048c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b028c0d656d62656464696e675f64696d944b408c0a656e636f6465725f6666944d00088c0d6465636f6465725f72745f6666944d00088c0e6465636f6465725f696e745f6666944d00048c066e5f68656164944b018c0964726f705f7261746594473fed81b270087f478c026c7294473f5b1334f8f7117a8c0a62617463685f73697a65944d0004758c1274696d655f73696e63655f726573746f726594474053d869700000008c18697465726174696f6e735f73696e63655f726573746f7265944b01757562618c195f6c61746573745f636865636b706f696e745f726573756c7494681975622e\"\n }\n}"
+ ],
+ [
+ "{\n \"stub\": false,\n \"trainable_name\": \"train_model\",\n \"trial_id\": \"1de460a8\",\n \"storage\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059572040000000000008c1b7261792e747261696e2e5f696e7465726e616c2e73746f72616765948c0e53746f72616765436f6e746578749493942981947d94288c12637573746f6d5f66735f70726f766964656494898c1273746f726167655f6c6f63616c5f70617468948c2b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c7473948c136578706572696d656e745f6469725f6e616d65948c0f746573745f6578706572696d656e74948c0e747269616c5f6469725f6e616d65948c96747261696e5f6d6f64656c5f31646534363061385f31305f62617463685f73697a653d313032342c6465636f6465725f696e745f66663d313032342c6465636f6465725f696e745f6e756d5f6c617965723d382c6465636f6465725f72745f66663d3531322c6465636f6465725f72745f6e756d5f6c617965723d342c64726f705f5f323032342d30362d32355f31362d32302d3334948c1863757272656e745f636865636b706f696e745f696e646578944b018c0b73796e635f636f6e666967948c097261792e747261696e948c0a53796e63436f6e6669679493942981947d94288c0b73796e635f706572696f64944d2c018c0c73796e635f74696d656f7574944d08078c0e73796e635f61727469666163747394898c1c73796e635f6172746966616374735f6f6e5f636865636b706f696e7494888c0a75706c6f61645f646972948c0a44455052454341544544948c0673796e6365729468188c1273796e635f6f6e5f636865636b706f696e7494681875628c1273746f726167655f66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c0f73746f726167655f66735f70617468948c452f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f7465737494681968008c115f46696c6573797374656d53796e6365729493942981947d9428681b682268134d2c0168144d08078c116c6173745f73796e635f75705f74696d65944741d99eb463df15148c136c6173745f73796e635f646f776e5f74696d659447fff00000000000008c0d5f73796e635f70726f63657373944e8c0c5f63757272656e745f636d649468008c125f75706c6f61645f746f5f66735f706174689493947d94288c0a6c6f63616c5f70617468948c3b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c74732f746573745f6578706572696d656e74948c0266739468228c0766735f70617468948c552f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e74948c076578636c756465945d948c0e2a2f636865636b706f696e745f2a9461758694756275622e\"\n },\n \"config\": {\n \"encoder_num_layer\": 2,\n \"decoder_rt_num_layer\": 4,\n \"decoder_int_num_layer\": 8,\n \"embedding_dim\": 16,\n \"encoder_ff\": 2048,\n \"decoder_rt_ff\": 512,\n \"decoder_int_ff\": 1024,\n \"n_head\": 1,\n \"drop_rate\": 0.03175079062321118,\n \"lr\": 0.003436670599863372,\n \"batch_size\": 1024\n },\n \"_Trial__unresolved_config\": {\n \"encoder_num_layer\": 2,\n \"decoder_rt_num_layer\": 4,\n \"decoder_int_num_layer\": 8,\n \"embedding_dim\": 16,\n \"encoder_ff\": 2048,\n \"decoder_rt_ff\": 512,\n \"decoder_int_ff\": 1024,\n \"n_head\": 1,\n \"drop_rate\": 0.03175079062321118,\n \"lr\": 0.003436670599863372,\n \"batch_size\": 1024\n },\n \"evaluated_params\": {\n \"batch_size\": 1024,\n \"decoder_int_ff\": 1024,\n \"decoder_int_num_layer\": 8,\n \"decoder_rt_ff\": 512,\n \"decoder_rt_num_layer\": 4,\n \"drop_rate\": 0.03175079062321118,\n \"embedding_dim\": 16,\n \"encoder_ff\": 2048,\n \"encoder_num_layer\": 2,\n \"lr\": 0.003436670599863372,\n \"n_head\": 1\n },\n \"experiment_tag\": \"10_batch_size=1024,decoder_int_ff=1024,decoder_int_num_layer=8,decoder_rt_ff=512,decoder_rt_num_layer=4,drop_rate=0.0318,embedding_dim=16,encoder_ff=2048,encoder_num_layer=2,lr=0.0034,n_head=1\",\n \"stopping_criterion\": {},\n \"_setup_default_resource\": true,\n \"_default_placement_group_factory\": \"80054e2e\",\n \"placement_group_factory\": \"800595ba000000000000008c237261792e74756e652e657865637574696f6e2e706c6163656d656e745f67726f757073948c15506c6163656d656e7447726f7570466163746f72799493942981947d94288c085f62756e646c6573945d947d94288c03435055944740540000000000008c034750559447401000000000000075618c155f686561645f62756e646c655f69735f656d70747994898c095f7374726174656779948c045041434b948c055f6172677394298c075f6b7761726773947d9475622e\",\n \"log_to_file\": [\n null,\n null\n ],\n \"max_failures\": 0,\n \"_default_result_or_future\": null,\n \"export_formats\": [],\n \"status\": \"PAUSED\",\n \"relative_logdir\": \"train_model_1de460a8_10_batch_size=1024,decoder_int_ff=1024,decoder_int_num_layer=8,decoder_rt_ff=512,decoder_rt_num_layer=4,drop__2024-06-25_16-20-34\",\n \"trial_name_creator\": null,\n \"trial_dirname_creator\": null,\n \"custom_trial_name\": null,\n \"custom_dirname\": null,\n \"restore_path\": null,\n \"_restore_checkpoint_result\": null,\n \"_state_json\": null,\n \"results\": \"80054e2e\",\n \"extra_arg\": \"80054e2e\",\n \"_resources\": \"80054e2e\"\n}",
+ "{\n \"start_time\": 1719325314.396891,\n \"num_failures\": 0,\n \"num_failures_after_restore\": 0,\n \"error_filename\": null,\n \"pickled_error_filename\": null,\n \"last_result\": {\n \"loss\": 1955.3101080947035,\n \"timestamp\": 1719327763,\n \"checkpoint_dir_name\": \"checkpoint_000001\",\n \"should_checkpoint\": true,\n \"done\": false,\n \"training_iteration\": 2,\n \"trial_id\": \"1de460a8\",\n \"date\": \"2024-06-25_17-02-43\",\n \"time_this_iter_s\": 72.80412983894348,\n \"time_total_s\": 146.37021279335022,\n \"pid\": 130020,\n \"hostname\": \"r3i5n6\",\n \"node_ip\": \"10.159.8.159\",\n \"config\": {\n \"encoder_num_layer\": 2,\n \"decoder_rt_num_layer\": 4,\n \"decoder_int_num_layer\": 8,\n \"embedding_dim\": 16,\n \"encoder_ff\": 2048,\n \"decoder_rt_ff\": 512,\n \"decoder_int_ff\": 1024,\n \"n_head\": 1,\n \"drop_rate\": 0.03175079062321118,\n \"lr\": 0.003436670599863372,\n \"batch_size\": 1024\n },\n \"time_since_restore\": 72.80412983894348,\n \"iterations_since_restore\": 1,\n \"experiment_tag\": \"10_batch_size=1024,decoder_int_ff=1024,decoder_int_num_layer=8,decoder_rt_ff=512,decoder_rt_num_layer=4,drop_rate=0.0318,embedding_dim=16,encoder_ff=2048,encoder_num_layer=2,lr=0.0034,n_head=1\"\n },\n \"last_result_time\": 1719327763.0620155,\n \"metric_analysis\": {\n \"loss\": {\n \"max\": 1961.4144263079786,\n \"min\": 1955.3101080947035,\n \"avg\": 1958.3622672013412,\n \"last\": 1955.3101080947035,\n \"last-5-avg\": 1958.3622672013412,\n \"last-10-avg\": 1958.3622672013412\n },\n \"should_checkpoint\": {\n \"max\": true,\n \"min\": true,\n \"avg\": 1.0,\n \"last\": true,\n \"last-5-avg\": 1.0,\n \"last-10-avg\": 1.0\n },\n \"done\": {\n \"max\": false,\n \"min\": false,\n \"avg\": 0.0,\n \"last\": false,\n \"last-5-avg\": 0.0,\n \"last-10-avg\": 0.0\n },\n \"training_iteration\": {\n \"max\": 2,\n \"min\": 1,\n \"avg\": 1.5,\n \"last\": 2,\n \"last-5-avg\": 1.5,\n \"last-10-avg\": 1.5\n },\n \"time_this_iter_s\": {\n \"max\": 73.56608295440674,\n \"min\": 72.80412983894348,\n \"avg\": 73.18510639667511,\n \"last\": 72.80412983894348,\n \"last-5-avg\": 73.18510639667511,\n \"last-10-avg\": 73.18510639667511\n },\n \"time_total_s\": {\n \"max\": 146.37021279335022,\n \"min\": 73.56608295440674,\n \"avg\": 109.96814787387848,\n \"last\": 146.37021279335022,\n \"last-5-avg\": 109.96814787387848,\n \"last-10-avg\": 109.96814787387848\n },\n \"time_since_restore\": {\n \"max\": 73.56608295440674,\n \"min\": 72.80412983894348,\n \"avg\": 73.18510639667511,\n \"last\": 72.80412983894348,\n \"last-5-avg\": 73.18510639667511,\n \"last-10-avg\": 73.18510639667511\n },\n \"iterations_since_restore\": {\n \"max\": 1,\n \"min\": 1,\n \"avg\": 1.0,\n \"last\": 1,\n \"last-5-avg\": 1.0,\n \"last-10-avg\": 1.0\n }\n },\n \"_n_steps\": [\n 5,\n 10\n ],\n \"metric_n_steps\": {\n \"loss\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005959d000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294288c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b0074946243087bbd5e5fa8a59e4094869452946807680d4308e8f3f98c3d8d9e409486945294652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005959d000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294288c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b0074946243087bbd5e5fa8a59e4094869452946807680d4308e8f3f98c3d8d9e409486945294652e\"\n }\n },\n \"should_checkpoint\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059524000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294288888652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059524000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294288888652e\"\n }\n },\n \"done\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059524000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294288989652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059524000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294288989652e\"\n }\n },\n \"training_iteration\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059526000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294284b014b02652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059526000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294284b014b02652e\"\n }\n },\n \"time_this_iter_s\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059534000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b058694529428474052643ab40000004740523376dd000000652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059534000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a8694529428474052643ab40000004740523376dd000000652e\"\n }\n },\n \"time_total_s\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059534000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b058694529428474052643ab40000004740624bd8c8800000652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059534000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a8694529428474052643ab40000004740624bd8c8800000652e\"\n }\n },\n \"time_since_restore\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059534000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b058694529428474052643ab40000004740523376dd000000652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059534000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a8694529428474052643ab40000004740523376dd000000652e\"\n }\n },\n \"iterations_since_restore\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059526000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294284b014b01652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059526000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294284b014b01652e\"\n }\n }\n },\n \"checkpoint_manager\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"800595a5090000000000008c267261792e747261696e2e5f696e7465726e616c2e636865636b706f696e745f6d616e61676572948c125f436865636b706f696e744d616e616765729493942981947d94288c125f636865636b706f696e745f636f6e666967948c097261792e747261696e948c10436865636b706f696e74436f6e6669679493942981947d94288c0b6e756d5f746f5f6b656570944e8c1a636865636b706f696e745f73636f72655f617474726962757465944e8c16636865636b706f696e745f73636f72655f6f72646572948c036d6178948c14636865636b706f696e745f6672657175656e6379944b008c11636865636b706f696e745f61745f656e6494898c1a5f636865636b706f696e745f6b6565705f616c6c5f72616e6b73948c0a44455052454341544544948c1f5f636865636b706f696e745f75706c6f61645f66726f6d5f776f726b65727394681275628c135f636865636b706f696e745f726573756c7473945d94288c1b7261792e747261696e2e5f696e7465726e616c2e73657373696f6e948c0f5f547261696e696e67526573756c749493942981947d94288c0a636865636b706f696e749468068c0a436865636b706f696e749493942981947d94288c0470617468948cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f31646534363061385f31305f62617463685f73697a653d313032342c6465636f6465725f696e745f66663d313032342c6465636f6465725f696e745f6e756d5f6c617965723d382c6465636f6465725f72745f66663d3531322c6465636f6465725f72745f6e756d5f6c617965723d342c64726f705f5f323032342d30362d32355f31362d32302d33342f636865636b706f696e745f303030303030948c0a66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c055f75756964948c0475756964948c04555549449493942981947d948c03696e74948a1055810ffc8fb1d7ab1141f27ccac18639736275628c076d657472696373947d94288c046c6f7373948c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b0074946243087bbd5e5fa8a59e4094869452948c0974696d657374616d70944acbd27a668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303030948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b018c08747269616c5f6964948c083164653436306138948c0464617465948c13323032342d30362d32355f31362d32332d3037948c1074696d655f746869735f697465725f7394474052643ab40000008c0c74696d655f746f74616c5f7394474052643ab40000008c03706964944ac60e01008c08686f73746e616d65948c06723369356e36948c076e6f64655f6970948c0c31302e3135392e382e313539948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b028c146465636f6465725f72745f6e756d5f6c61796572944b048c156465636f6465725f696e745f6e756d5f6c61796572944b088c0d656d62656464696e675f64696d944b108c0a656e636f6465725f6666944d00088c0d6465636f6465725f72745f6666944d00028c0e6465636f6465725f696e745f6666944d00048c066e5f68656164944b018c0964726f705f7261746594473fa041a3beb29a908c026c7294473f6c27387aac57a48c0a62617463685f73697a65944d0004758c1274696d655f73696e63655f726573746f726594474052643ab40000008c18697465726174696f6e735f73696e63655f726573746f7265944b0175756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f31646534363061385f31305f62617463685f73697a653d313032342c6465636f6465725f696e745f66663d313032342c6465636f6465725f696e745f6e756d5f6c617965723d382c6465636f6465725f72745f66663d3531322c6465636f6465725f72745f6e756d5f6c617965723d342c64726f705f5f323032342d30362d32355f31362d32302d33342f636865636b706f696e745f30303030303194682268257d946827897385945294682a682d2981947d9468308a11b8b96603598218a15144e463ba24cf8e007362756268317d94288c046c6f7373946836683c4308e8f3f98c3d8d9e4094869452948c0974696d657374616d70944a13dc7a668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303031948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b028c08747269616c5f6964948c083164653436306138948c0464617465948c13323032342d30362d32355f31372d30322d3433948c1074696d655f746869735f697465725f73944740523376dd0000008c0c74696d655f746f74616c5f73944740624bd8c88000008c03706964944ae4fb01008c08686f73746e616d65948c06723369356e36948c076e6f64655f6970948c0c31302e3135392e382e313539948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b028c146465636f6465725f72745f6e756d5f6c61796572944b048c156465636f6465725f696e745f6e756d5f6c61796572944b088c0d656d62656464696e675f64696d944b108c0a656e636f6465725f6666944d00088c0d6465636f6465725f72745f6666944d00028c0e6465636f6465725f696e745f6666944d00048c066e5f68656164944b018c0964726f705f7261746594473fa041a3beb29a908c026c7294473f6c27387aac57a48c0a62617463685f73697a65944d0004758c1274696d655f73696e63655f726573746f7265944740523376dd0000008c18697465726174696f6e735f73696e63655f726573746f7265944b01757562658c195f6c61746573745f636865636b706f696e745f726573756c7494686275622e\"\n }\n}"
+ ],
+ [
+ "{\n \"stub\": false,\n \"trainable_name\": \"train_model\",\n \"trial_id\": \"1ef7c88b\",\n \"storage\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059572040000000000008c1b7261792e747261696e2e5f696e7465726e616c2e73746f72616765948c0e53746f72616765436f6e746578749493942981947d94288c12637573746f6d5f66735f70726f766964656494898c1273746f726167655f6c6f63616c5f70617468948c2b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c7473948c136578706572696d656e745f6469725f6e616d65948c0f746573745f6578706572696d656e74948c0e747269616c5f6469725f6e616d65948c96747261696e5f6d6f64656c5f31656637633838625f31315f62617463685f73697a653d313032342c6465636f6465725f696e745f66663d313032342c6465636f6465725f696e745f6e756d5f6c617965723d342c6465636f6465725f72745f66663d3531322c6465636f6465725f72745f6e756d5f6c617965723d342c64726f705f5f323032342d30362d32355f31362d32312d3534948c1863757272656e745f636865636b706f696e745f696e646578944b008c0b73796e635f636f6e666967948c097261792e747261696e948c0a53796e63436f6e6669679493942981947d94288c0b73796e635f706572696f64944d2c018c0c73796e635f74696d656f7574944d08078c0e73796e635f61727469666163747394898c1c73796e635f6172746966616374735f6f6e5f636865636b706f696e7494888c0a75706c6f61645f646972948c0a44455052454341544544948c0673796e6365729468188c1273796e635f6f6e5f636865636b706f696e7494681875628c1273746f726167655f66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c0f73746f726167655f66735f70617468948c452f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f7465737494681968008c115f46696c6573797374656d53796e6365729493942981947d9428681b682268134d2c0168144d08078c116c6173745f73796e635f75705f74696d65944741d99eb463df15148c136c6173745f73796e635f646f776e5f74696d659447fff00000000000008c0d5f73796e635f70726f63657373944e8c0c5f63757272656e745f636d649468008c125f75706c6f61645f746f5f66735f706174689493947d94288c0a6c6f63616c5f70617468948c3b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c74732f746573745f6578706572696d656e74948c0266739468228c0766735f70617468948c552f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e74948c076578636c756465945d948c0e2a2f636865636b706f696e745f2a9461758694756275622e\"\n },\n \"config\": {\n \"encoder_num_layer\": 8,\n \"decoder_rt_num_layer\": 4,\n \"decoder_int_num_layer\": 4,\n \"embedding_dim\": 1024,\n \"encoder_ff\": 1024,\n \"decoder_rt_ff\": 512,\n \"decoder_int_ff\": 1024,\n \"n_head\": 2,\n \"drop_rate\": 0.14711494497891564,\n \"lr\": 0.00018307057253269556,\n \"batch_size\": 1024\n },\n \"_Trial__unresolved_config\": {\n \"encoder_num_layer\": 8,\n \"decoder_rt_num_layer\": 4,\n \"decoder_int_num_layer\": 4,\n \"embedding_dim\": 1024,\n \"encoder_ff\": 1024,\n \"decoder_rt_ff\": 512,\n \"decoder_int_ff\": 1024,\n \"n_head\": 2,\n \"drop_rate\": 0.14711494497891564,\n \"lr\": 0.00018307057253269556,\n \"batch_size\": 1024\n },\n \"evaluated_params\": {\n \"batch_size\": 1024,\n \"decoder_int_ff\": 1024,\n \"decoder_int_num_layer\": 4,\n \"decoder_rt_ff\": 512,\n \"decoder_rt_num_layer\": 4,\n \"drop_rate\": 0.14711494497891564,\n \"embedding_dim\": 1024,\n \"encoder_ff\": 1024,\n \"encoder_num_layer\": 8,\n \"lr\": 0.00018307057253269556,\n \"n_head\": 2\n },\n \"experiment_tag\": \"11_batch_size=1024,decoder_int_ff=1024,decoder_int_num_layer=4,decoder_rt_ff=512,decoder_rt_num_layer=4,drop_rate=0.1471,embedding_dim=1024,encoder_ff=1024,encoder_num_layer=8,lr=0.0002,n_head=2\",\n \"stopping_criterion\": {},\n \"_setup_default_resource\": true,\n \"_default_placement_group_factory\": \"80054e2e\",\n \"placement_group_factory\": \"800595ba000000000000008c237261792e74756e652e657865637574696f6e2e706c6163656d656e745f67726f757073948c15506c6163656d656e7447726f7570466163746f72799493942981947d94288c085f62756e646c6573945d947d94288c03435055944740540000000000008c034750559447401000000000000075618c155f686561645f62756e646c655f69735f656d70747994898c095f7374726174656779948c045041434b948c055f6172677394298c075f6b7761726773947d9475622e\",\n \"log_to_file\": [\n null,\n null\n ],\n \"max_failures\": 0,\n \"_default_result_or_future\": null,\n \"export_formats\": [],\n \"status\": \"ERROR\",\n \"relative_logdir\": \"train_model_1ef7c88b_11_batch_size=1024,decoder_int_ff=1024,decoder_int_num_layer=4,decoder_rt_ff=512,decoder_rt_num_layer=4,drop__2024-06-25_16-21-54\",\n \"trial_name_creator\": null,\n \"trial_dirname_creator\": null,\n \"custom_trial_name\": null,\n \"custom_dirname\": null,\n \"restore_path\": null,\n \"_restore_checkpoint_result\": null,\n \"_state_json\": null,\n \"results\": \"80054e2e\",\n \"extra_arg\": \"80054e2e\",\n \"_resources\": \"80054e2e\"\n}",
+ "{\n \"start_time\": 1719325388.1127121,\n \"num_failures\": 1,\n \"num_failures_after_restore\": 0,\n \"error_filename\": \"error.txt\",\n \"pickled_error_filename\": \"error.pkl\",\n \"last_result\": {\n \"loss\": 1963.8436942513533,\n \"timestamp\": 1719326402,\n \"checkpoint_dir_name\": \"checkpoint_000000\",\n \"should_checkpoint\": true,\n \"done\": false,\n \"training_iteration\": 1,\n \"trial_id\": \"1ef7c88b\",\n \"date\": \"2024-06-25_16-40-02\",\n \"time_this_iter_s\": 1014.7334928512573,\n \"time_total_s\": 1014.7334928512573,\n \"pid\": 69318,\n \"hostname\": \"r3i5n6\",\n \"node_ip\": \"10.159.8.159\",\n \"config\": {\n \"encoder_num_layer\": 8,\n \"decoder_rt_num_layer\": 4,\n \"decoder_int_num_layer\": 4,\n \"embedding_dim\": 1024,\n \"encoder_ff\": 1024,\n \"decoder_rt_ff\": 512,\n \"decoder_int_ff\": 1024,\n \"n_head\": 2,\n \"drop_rate\": 0.14711494497891564,\n \"lr\": 0.00018307057253269556,\n \"batch_size\": 1024\n },\n \"time_since_restore\": 1014.7334928512573,\n \"iterations_since_restore\": 1\n },\n \"last_result_time\": 1719326402.8500564,\n \"metric_analysis\": {\n \"loss\": {\n \"max\": 1963.8436942513533,\n \"min\": 1963.8436942513533,\n \"avg\": 1963.8436942513533,\n \"last\": 1963.8436942513533,\n \"last-5-avg\": 1963.8436942513533,\n \"last-10-avg\": 1963.8436942513533\n },\n \"should_checkpoint\": {\n \"max\": true,\n \"min\": true,\n \"avg\": true,\n \"last\": true,\n \"last-5-avg\": true,\n \"last-10-avg\": true\n },\n \"done\": {\n \"max\": false,\n \"min\": false,\n \"avg\": false,\n \"last\": false,\n \"last-5-avg\": false,\n \"last-10-avg\": false\n },\n \"training_iteration\": {\n \"max\": 1,\n \"min\": 1,\n \"avg\": 1,\n \"last\": 1,\n \"last-5-avg\": 1,\n \"last-10-avg\": 1\n },\n \"time_this_iter_s\": {\n \"max\": 1014.7334928512573,\n \"min\": 1014.7334928512573,\n \"avg\": 1014.7334928512573,\n \"last\": 1014.7334928512573,\n \"last-5-avg\": 1014.7334928512573,\n \"last-10-avg\": 1014.7334928512573\n },\n \"time_total_s\": {\n \"max\": 1014.7334928512573,\n \"min\": 1014.7334928512573,\n \"avg\": 1014.7334928512573,\n \"last\": 1014.7334928512573,\n \"last-5-avg\": 1014.7334928512573,\n \"last-10-avg\": 1014.7334928512573\n },\n \"time_since_restore\": {\n \"max\": 1014.7334928512573,\n \"min\": 1014.7334928512573,\n \"avg\": 1014.7334928512573,\n \"last\": 1014.7334928512573,\n \"last-5-avg\": 1014.7334928512573,\n \"last-10-avg\": 1014.7334928512573\n },\n \"iterations_since_restore\": {\n \"max\": 1,\n \"min\": 1,\n \"avg\": 1,\n \"last\": 1,\n \"last-5-avg\": 1,\n \"last-10-avg\": 1\n }\n },\n \"_n_steps\": [\n 5,\n 10\n ],\n \"metric_n_steps\": {\n \"loss\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059589000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452948c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b0074946243088bc562f15faf9e409486945294612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059589000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452948c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b0074946243088bc562f15faf9e409486945294612e\"\n }\n },\n \"should_checkpoint\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059522000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b058694529488612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059522000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a8694529488612e\"\n }\n },\n \"done\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059522000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b058694529489612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059522000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a8694529489612e\"\n }\n },\n \"training_iteration\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059523000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452944b01612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059523000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452944b01612e\"\n }\n },\n \"time_this_iter_s\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b058694529447408fb5de31800000612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a8694529447408fb5de31800000612e\"\n }\n },\n \"time_total_s\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b058694529447408fb5de31800000612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a8694529447408fb5de31800000612e\"\n }\n },\n \"time_since_restore\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b058694529447408fb5de31800000612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a8694529447408fb5de31800000612e\"\n }\n },\n \"iterations_since_restore\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059523000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452944b01612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059523000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452944b01612e\"\n }\n }\n },\n \"checkpoint_manager\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059519060000000000008c267261792e747261696e2e5f696e7465726e616c2e636865636b706f696e745f6d616e61676572948c125f436865636b706f696e744d616e616765729493942981947d94288c125f636865636b706f696e745f636f6e666967948c097261792e747261696e948c10436865636b706f696e74436f6e6669679493942981947d94288c0b6e756d5f746f5f6b656570944e8c1a636865636b706f696e745f73636f72655f617474726962757465944e8c16636865636b706f696e745f73636f72655f6f72646572948c036d6178948c14636865636b706f696e745f6672657175656e6379944b008c11636865636b706f696e745f61745f656e6494898c1a5f636865636b706f696e745f6b6565705f616c6c5f72616e6b73948c0a44455052454341544544948c1f5f636865636b706f696e745f75706c6f61645f66726f6d5f776f726b65727394681275628c135f636865636b706f696e745f726573756c7473945d948c1b7261792e747261696e2e5f696e7465726e616c2e73657373696f6e948c0f5f547261696e696e67526573756c749493942981947d94288c0a636865636b706f696e749468068c0a436865636b706f696e749493942981947d94288c0470617468948cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f31656637633838625f31315f62617463685f73697a653d313032342c6465636f6465725f696e745f66663d313032342c6465636f6465725f696e745f6e756d5f6c617965723d342c6465636f6465725f72745f66663d3531322c6465636f6465725f72745f6e756d5f6c617965723d342c64726f705f5f323032342d30362d32355f31362d32312d35342f636865636b706f696e745f303030303030948c0a66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c055f75756964948c0475756964948c04555549449493942981947d948c03696e74948a117d30e4131f2c1ba2eb43a2955dff97f100736275628c076d657472696373947d94288c046c6f7373948c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b0074946243088bc562f15faf9e4094869452948c0974696d657374616d70944ac2d67a668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303030948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b018c08747269616c5f6964948c083165663763383862948c0464617465948c13323032342d30362d32355f31362d34302d3032948c1074696d655f746869735f697465725f739447408fb5de318000008c0c74696d655f746f74616c5f739447408fb5de318000008c03706964944ac60e01008c08686f73746e616d65948c06723369356e36948c076e6f64655f6970948c0c31302e3135392e382e313539948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b088c146465636f6465725f72745f6e756d5f6c61796572944b048c156465636f6465725f696e745f6e756d5f6c61796572944b048c0d656d62656464696e675f64696d944d00048c0a656e636f6465725f6666944d00048c0d6465636f6465725f72745f6666944d00028c0e6465636f6465725f696e745f6666944d00048c066e5f68656164944b028c0964726f705f7261746594473fc2d4a99ab7f8dc8c026c7294473f27fed43e6818098c0a62617463685f73697a65944d0004758c1274696d655f73696e63655f726573746f72659447408fb5de318000008c18697465726174696f6e735f73696e63655f726573746f7265944b01757562618c195f6c61746573745f636865636b706f696e745f726573756c7494681975622e\"\n }\n}"
+ ],
+ [
+ "{\n \"stub\": false,\n \"trainable_name\": \"train_model\",\n \"trial_id\": \"651349c4\",\n \"storage\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059572040000000000008c1b7261792e747261696e2e5f696e7465726e616c2e73746f72616765948c0e53746f72616765436f6e746578749493942981947d94288c12637573746f6d5f66735f70726f766964656494898c1273746f726167655f6c6f63616c5f70617468948c2b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c7473948c136578706572696d656e745f6469725f6e616d65948c0f746573745f6578706572696d656e74948c0e747269616c5f6469725f6e616d65948c96747261696e5f6d6f64656c5f36353133343963345f31325f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d323034382c6465636f6465725f696e745f6e756d5f6c617965723d382c6465636f6465725f72745f66663d323034382c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f323032342d30362d32355f31362d32332d3038948c1863757272656e745f636865636b706f696e745f696e646578944b008c0b73796e635f636f6e666967948c097261792e747261696e948c0a53796e63436f6e6669679493942981947d94288c0b73796e635f706572696f64944d2c018c0c73796e635f74696d656f7574944d08078c0e73796e635f61727469666163747394898c1c73796e635f6172746966616374735f6f6e5f636865636b706f696e7494888c0a75706c6f61645f646972948c0a44455052454341544544948c0673796e6365729468188c1273796e635f6f6e5f636865636b706f696e7494681875628c1273746f726167655f66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c0f73746f726167655f66735f70617468948c452f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f7465737494681968008c115f46696c6573797374656d53796e6365729493942981947d9428681b682268134d2c0168144d08078c116c6173745f73796e635f75705f74696d65944741d99eb4af484b348c136c6173745f73796e635f646f776e5f74696d659447fff00000000000008c0d5f73796e635f70726f63657373944e8c0c5f63757272656e745f636d649468008c125f75706c6f61645f746f5f66735f706174689493947d94288c0a6c6f63616c5f70617468948c3b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c74732f746573745f6578706572696d656e74948c0266739468228c0766735f70617468948c552f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e74948c076578636c756465945d948c0e2a2f636865636b706f696e745f2a9461758694756275622e\"\n },\n \"config\": {\n \"encoder_num_layer\": 8,\n \"decoder_rt_num_layer\": 1,\n \"decoder_int_num_layer\": 8,\n \"embedding_dim\": 16,\n \"encoder_ff\": 2048,\n \"decoder_rt_ff\": 2048,\n \"decoder_int_ff\": 2048,\n \"n_head\": 4,\n \"drop_rate\": 0.6683907000004579,\n \"lr\": 0.0001175178193869245,\n \"batch_size\": 2048\n },\n \"_Trial__unresolved_config\": {\n \"encoder_num_layer\": 8,\n \"decoder_rt_num_layer\": 1,\n \"decoder_int_num_layer\": 8,\n \"embedding_dim\": 16,\n \"encoder_ff\": 2048,\n \"decoder_rt_ff\": 2048,\n \"decoder_int_ff\": 2048,\n \"n_head\": 4,\n \"drop_rate\": 0.6683907000004579,\n \"lr\": 0.0001175178193869245,\n \"batch_size\": 2048\n },\n \"evaluated_params\": {\n \"batch_size\": 2048,\n \"decoder_int_ff\": 2048,\n \"decoder_int_num_layer\": 8,\n \"decoder_rt_ff\": 2048,\n \"decoder_rt_num_layer\": 1,\n \"drop_rate\": 0.6683907000004579,\n \"embedding_dim\": 16,\n \"encoder_ff\": 2048,\n \"encoder_num_layer\": 8,\n \"lr\": 0.0001175178193869245,\n \"n_head\": 4\n },\n \"experiment_tag\": \"12_batch_size=2048,decoder_int_ff=2048,decoder_int_num_layer=8,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop_rate=0.6684,embedding_dim=16,encoder_ff=2048,encoder_num_layer=8,lr=0.0001,n_head=4\",\n \"stopping_criterion\": {},\n \"_setup_default_resource\": true,\n \"_default_placement_group_factory\": \"80054e2e\",\n \"placement_group_factory\": \"800595ba000000000000008c237261792e74756e652e657865637574696f6e2e706c6163656d656e745f67726f757073948c15506c6163656d656e7447726f7570466163746f72799493942981947d94288c085f62756e646c6573945d947d94288c03435055944740540000000000008c034750559447401000000000000075618c155f686561645f62756e646c655f69735f656d70747994898c095f7374726174656779948c045041434b948c055f6172677394298c075f6b7761726773947d9475622e\",\n \"log_to_file\": [\n null,\n null\n ],\n \"max_failures\": 0,\n \"_default_result_or_future\": null,\n \"export_formats\": [],\n \"status\": \"PENDING\",\n \"relative_logdir\": \"train_model_651349c4_12_batch_size=2048,decoder_int_ff=2048,decoder_int_num_layer=8,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop_2024-06-25_16-23-08\",\n \"trial_name_creator\": null,\n \"trial_dirname_creator\": null,\n \"custom_trial_name\": null,\n \"custom_dirname\": null,\n \"restore_path\": null,\n \"_restore_checkpoint_result\": null,\n \"_state_json\": null,\n \"results\": \"80054e2e\",\n \"extra_arg\": \"80054e2e\",\n \"_resources\": \"80054e2e\"\n}",
+ "{\n \"start_time\": 1719326403.0031497,\n \"num_failures\": 0,\n \"num_failures_after_restore\": 0,\n \"error_filename\": null,\n \"pickled_error_filename\": null,\n \"last_result\": {\n \"loss\": 1981.618032380352,\n \"timestamp\": 1719326489,\n \"checkpoint_dir_name\": \"checkpoint_000000\",\n \"should_checkpoint\": true,\n \"done\": false,\n \"training_iteration\": 1,\n \"trial_id\": \"651349c4\",\n \"date\": \"2024-06-25_16-41-29\",\n \"time_this_iter_s\": 86.11733651161194,\n \"time_total_s\": 86.11733651161194,\n \"pid\": 69318,\n \"hostname\": \"r3i5n6\",\n \"node_ip\": \"10.159.8.159\",\n \"config\": {\n \"encoder_num_layer\": 8,\n \"decoder_rt_num_layer\": 1,\n \"decoder_int_num_layer\": 8,\n \"embedding_dim\": 16,\n \"encoder_ff\": 2048,\n \"decoder_rt_ff\": 2048,\n \"decoder_int_ff\": 2048,\n \"n_head\": 4,\n \"drop_rate\": 0.6683907000004579,\n \"lr\": 0.0001175178193869245,\n \"batch_size\": 2048\n },\n \"time_since_restore\": 86.11733651161194,\n \"iterations_since_restore\": 1,\n \"experiment_tag\": \"12_batch_size=2048,decoder_int_ff=2048,decoder_int_num_layer=8,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop_rate=0.6684,embedding_dim=16,encoder_ff=2048,encoder_num_layer=8,lr=0.0001,n_head=4\"\n },\n \"last_result_time\": 1719326489.1258004,\n \"metric_analysis\": {\n \"loss\": {\n \"max\": 1981.618032380352,\n \"min\": 1981.618032380352,\n \"avg\": 1981.618032380352,\n \"last\": 1981.618032380352,\n \"last-5-avg\": 1981.618032380352,\n \"last-10-avg\": 1981.618032380352\n },\n \"should_checkpoint\": {\n \"max\": true,\n \"min\": true,\n \"avg\": true,\n \"last\": true,\n \"last-5-avg\": true,\n \"last-10-avg\": true\n },\n \"done\": {\n \"max\": false,\n \"min\": false,\n \"avg\": false,\n \"last\": false,\n \"last-5-avg\": false,\n \"last-10-avg\": false\n },\n \"training_iteration\": {\n \"max\": 1,\n \"min\": 1,\n \"avg\": 1,\n \"last\": 1,\n \"last-5-avg\": 1,\n \"last-10-avg\": 1\n },\n \"time_this_iter_s\": {\n \"max\": 86.11733651161194,\n \"min\": 86.11733651161194,\n \"avg\": 86.11733651161194,\n \"last\": 86.11733651161194,\n \"last-5-avg\": 86.11733651161194,\n \"last-10-avg\": 86.11733651161194\n },\n \"time_total_s\": {\n \"max\": 86.11733651161194,\n \"min\": 86.11733651161194,\n \"avg\": 86.11733651161194,\n \"last\": 86.11733651161194,\n \"last-5-avg\": 86.11733651161194,\n \"last-10-avg\": 86.11733651161194\n },\n \"time_since_restore\": {\n \"max\": 86.11733651161194,\n \"min\": 86.11733651161194,\n \"avg\": 86.11733651161194,\n \"last\": 86.11733651161194,\n \"last-5-avg\": 86.11733651161194,\n \"last-10-avg\": 86.11733651161194\n },\n \"iterations_since_restore\": {\n \"max\": 1,\n \"min\": 1,\n \"avg\": 1,\n \"last\": 1,\n \"last-5-avg\": 1,\n \"last-10-avg\": 1\n }\n },\n \"_n_steps\": [\n 5,\n 10\n ],\n \"metric_n_steps\": {\n \"loss\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059589000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452948c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b007494624308ecf57add78f69e409486945294612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059589000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452948c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b007494624308ecf57add78f69e409486945294612e\"\n }\n },\n \"should_checkpoint\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059522000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b058694529488612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059522000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a8694529488612e\"\n }\n },\n \"done\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059522000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b058694529489612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059522000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a8694529489612e\"\n }\n },\n \"training_iteration\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059523000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452944b01612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059523000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452944b01612e\"\n }\n },\n \"time_this_iter_s\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294474055878271000000612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294474055878271000000612e\"\n }\n },\n \"time_total_s\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294474055878271000000612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294474055878271000000612e\"\n }\n },\n \"time_since_restore\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294474055878271000000612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294474055878271000000612e\"\n }\n },\n \"iterations_since_restore\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059523000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452944b01612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059523000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452944b01612e\"\n }\n }\n },\n \"checkpoint_manager\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059517060000000000008c267261792e747261696e2e5f696e7465726e616c2e636865636b706f696e745f6d616e61676572948c125f436865636b706f696e744d616e616765729493942981947d94288c125f636865636b706f696e745f636f6e666967948c097261792e747261696e948c10436865636b706f696e74436f6e6669679493942981947d94288c0b6e756d5f746f5f6b656570944e8c1a636865636b706f696e745f73636f72655f617474726962757465944e8c16636865636b706f696e745f73636f72655f6f72646572948c036d6178948c14636865636b706f696e745f6672657175656e6379944b008c11636865636b706f696e745f61745f656e6494898c1a5f636865636b706f696e745f6b6565705f616c6c5f72616e6b73948c0a44455052454341544544948c1f5f636865636b706f696e745f75706c6f61645f66726f6d5f776f726b65727394681275628c135f636865636b706f696e745f726573756c7473945d948c1b7261792e747261696e2e5f696e7465726e616c2e73657373696f6e948c0f5f547261696e696e67526573756c749493942981947d94288c0a636865636b706f696e749468068c0a436865636b706f696e749493942981947d94288c0470617468948cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f36353133343963345f31325f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d323034382c6465636f6465725f696e745f6e756d5f6c617965723d382c6465636f6465725f72745f66663d323034382c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f323032342d30362d32355f31362d32332d30382f636865636b706f696e745f303030303030948c0a66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c055f75756964948c0475756964948c04555549449493942981947d948c03696e74948a103cd44f9d38a2e7bec446dcaeaf332b13736275628c076d657472696373947d94288c046c6f7373948c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b007494624308ecf57add78f69e4094869452948c0974696d657374616d70944a19d77a668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303030948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b018c08747269616c5f6964948c083635313334396334948c0464617465948c13323032342d30362d32355f31362d34312d3239948c1074696d655f746869735f697465725f73944740558782710000008c0c74696d655f746f74616c5f73944740558782710000008c03706964944ac60e01008c08686f73746e616d65948c06723369356e36948c076e6f64655f6970948c0c31302e3135392e382e313539948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b088c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b088c0d656d62656464696e675f64696d944b108c0a656e636f6465725f6666944d00088c0d6465636f6465725f72745f6666944d00088c0e6465636f6465725f696e745f6666944d00088c066e5f68656164944b048c0964726f705f7261746594473fe56374e4ae7afe8c026c7294473f1ece7cc38c162c8c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f7265944740558782710000008c18697465726174696f6e735f73696e63655f726573746f7265944b01757562618c195f6c61746573745f636865636b706f696e745f726573756c7494681975622e\"\n }\n}"
+ ],
+ [
+ "{\n \"stub\": false,\n \"trainable_name\": \"train_model\",\n \"trial_id\": \"e566e32a\",\n \"storage\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059572040000000000008c1b7261792e747261696e2e5f696e7465726e616c2e73746f72616765948c0e53746f72616765436f6e746578749493942981947d94288c12637573746f6d5f66735f70726f766964656494898c1273746f726167655f6c6f63616c5f70617468948c2b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c7473948c136578706572696d656e745f6469725f6e616d65948c0f746573745f6578706572696d656e74948c0e747269616c5f6469725f6e616d65948c96747261696e5f6d6f64656c5f65353636653332615f31335f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d323034382c6465636f6465725f72745f6e756d5f6c617965723d322c64726f705f5f323032342d30362d32355f31362d34302d3033948c1863757272656e745f636865636b706f696e745f696e646578944b008c0b73796e635f636f6e666967948c097261792e747261696e948c0a53796e63436f6e6669679493942981947d94288c0b73796e635f706572696f64944d2c018c0c73796e635f74696d656f7574944d08078c0e73796e635f61727469666163747394898c1c73796e635f6172746966616374735f6f6e5f636865636b706f696e7494888c0a75706c6f61645f646972948c0a44455052454341544544948c0673796e6365729468188c1273796e635f6f6e5f636865636b706f696e7494681875628c1273746f726167655f66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c0f73746f726167655f66735f70617468948c452f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f7465737494681968008c115f46696c6573797374656d53796e6365729493942981947d9428681b682268134d2c0168144d08078c116c6173745f73796e635f75705f74696d65944741d99eb5915a973d8c136c6173745f73796e635f646f776e5f74696d659447fff00000000000008c0d5f73796e635f70726f63657373944e8c0c5f63757272656e745f636d649468008c125f75706c6f61645f746f5f66735f706174689493947d94288c0a6c6f63616c5f70617468948c3b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c74732f746573745f6578706572696d656e74948c0266739468228c0766735f70617468948c552f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e74948c076578636c756465945d948c0e2a2f636865636b706f696e745f2a9461758694756275622e\"\n },\n \"config\": {\n \"encoder_num_layer\": 8,\n \"decoder_rt_num_layer\": 2,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 16,\n \"encoder_ff\": 1024,\n \"decoder_rt_ff\": 2048,\n \"decoder_int_ff\": 512,\n \"n_head\": 4,\n \"drop_rate\": 0.7731110479994988,\n \"lr\": 0.03425650698356203,\n \"batch_size\": 2048\n },\n \"_Trial__unresolved_config\": {\n \"encoder_num_layer\": 8,\n \"decoder_rt_num_layer\": 2,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 16,\n \"encoder_ff\": 1024,\n \"decoder_rt_ff\": 2048,\n \"decoder_int_ff\": 512,\n \"n_head\": 4,\n \"drop_rate\": 0.7731110479994988,\n \"lr\": 0.03425650698356203,\n \"batch_size\": 2048\n },\n \"evaluated_params\": {\n \"batch_size\": 2048,\n \"decoder_int_ff\": 512,\n \"decoder_int_num_layer\": 1,\n \"decoder_rt_ff\": 2048,\n \"decoder_rt_num_layer\": 2,\n \"drop_rate\": 0.7731110479994988,\n \"embedding_dim\": 16,\n \"encoder_ff\": 1024,\n \"encoder_num_layer\": 8,\n \"lr\": 0.03425650698356203,\n \"n_head\": 4\n },\n \"experiment_tag\": \"13_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=2,drop_rate=0.7731,embedding_dim=16,encoder_ff=1024,encoder_num_layer=8,lr=0.0343,n_head=4\",\n \"stopping_criterion\": {},\n \"_setup_default_resource\": true,\n \"_default_placement_group_factory\": \"80054e2e\",\n \"placement_group_factory\": \"800595ba000000000000008c237261792e74756e652e657865637574696f6e2e706c6163656d656e745f67726f757073948c15506c6163656d656e7447726f7570466163746f72799493942981947d94288c085f62756e646c6573945d947d94288c03435055944740540000000000008c034750559447401000000000000075618c155f686561645f62756e646c655f69735f656d70747994898c095f7374726174656779948c045041434b948c055f6172677394298c075f6b7761726773947d9475622e\",\n \"log_to_file\": [\n null,\n null\n ],\n \"max_failures\": 0,\n \"_default_result_or_future\": null,\n \"export_formats\": [],\n \"status\": \"PAUSED\",\n \"relative_logdir\": \"train_model_e566e32a_13_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=2,drop__2024-06-25_16-40-03\",\n \"trial_name_creator\": null,\n \"trial_dirname_creator\": null,\n \"custom_trial_name\": null,\n \"custom_dirname\": null,\n \"restore_path\": null,\n \"_restore_checkpoint_result\": null,\n \"_state_json\": null,\n \"results\": \"80054e2e\",\n \"extra_arg\": \"80054e2e\",\n \"_resources\": \"80054e2e\"\n}",
+ "{\n \"start_time\": 1719326489.2697742,\n \"num_failures\": 0,\n \"num_failures_after_restore\": 0,\n \"error_filename\": null,\n \"pickled_error_filename\": null,\n \"last_result\": {\n \"loss\": 2129.4940118264026,\n \"timestamp\": 1719326571,\n \"checkpoint_dir_name\": \"checkpoint_000000\",\n \"should_checkpoint\": true,\n \"done\": false,\n \"training_iteration\": 1,\n \"trial_id\": \"e566e32a\",\n \"date\": \"2024-06-25_16-42-51\",\n \"time_this_iter_s\": 82.61908912658691,\n \"time_total_s\": 82.61908912658691,\n \"pid\": 69318,\n \"hostname\": \"r3i5n6\",\n \"node_ip\": \"10.159.8.159\",\n \"config\": {\n \"encoder_num_layer\": 8,\n \"decoder_rt_num_layer\": 2,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 16,\n \"encoder_ff\": 1024,\n \"decoder_rt_ff\": 2048,\n \"decoder_int_ff\": 512,\n \"n_head\": 4,\n \"drop_rate\": 0.7731110479994988,\n \"lr\": 0.03425650698356203,\n \"batch_size\": 2048\n },\n \"time_since_restore\": 82.61908912658691,\n \"iterations_since_restore\": 1,\n \"experiment_tag\": \"13_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=2,drop_rate=0.7731,embedding_dim=16,encoder_ff=1024,encoder_num_layer=8,lr=0.0343,n_head=4\"\n },\n \"last_result_time\": 1719326571.8931763,\n \"metric_analysis\": {\n \"loss\": {\n \"max\": 2129.4940118264026,\n \"min\": 2129.4940118264026,\n \"avg\": 2129.4940118264026,\n \"last\": 2129.4940118264026,\n \"last-5-avg\": 2129.4940118264026,\n \"last-10-avg\": 2129.4940118264026\n },\n \"should_checkpoint\": {\n \"max\": true,\n \"min\": true,\n \"avg\": true,\n \"last\": true,\n \"last-5-avg\": true,\n \"last-10-avg\": true\n },\n \"done\": {\n \"max\": false,\n \"min\": false,\n \"avg\": false,\n \"last\": false,\n \"last-5-avg\": false,\n \"last-10-avg\": false\n },\n \"training_iteration\": {\n \"max\": 1,\n \"min\": 1,\n \"avg\": 1,\n \"last\": 1,\n \"last-5-avg\": 1,\n \"last-10-avg\": 1\n },\n \"time_this_iter_s\": {\n \"max\": 82.61908912658691,\n \"min\": 82.61908912658691,\n \"avg\": 82.61908912658691,\n \"last\": 82.61908912658691,\n \"last-5-avg\": 82.61908912658691,\n \"last-10-avg\": 82.61908912658691\n },\n \"time_total_s\": {\n \"max\": 82.61908912658691,\n \"min\": 82.61908912658691,\n \"avg\": 82.61908912658691,\n \"last\": 82.61908912658691,\n \"last-5-avg\": 82.61908912658691,\n \"last-10-avg\": 82.61908912658691\n },\n \"time_since_restore\": {\n \"max\": 82.61908912658691,\n \"min\": 82.61908912658691,\n \"avg\": 82.61908912658691,\n \"last\": 82.61908912658691,\n \"last-5-avg\": 82.61908912658691,\n \"last-10-avg\": 82.61908912658691\n },\n \"iterations_since_restore\": {\n \"max\": 1,\n \"min\": 1,\n \"avg\": 1,\n \"last\": 1,\n \"last-5-avg\": 1,\n \"last-10-avg\": 1\n }\n },\n \"_n_steps\": [\n 5,\n 10\n ],\n \"metric_n_steps\": {\n \"loss\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059589000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452948c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b007494624308793c1eeffca2a0409486945294612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059589000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452948c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b007494624308793c1eeffca2a0409486945294612e\"\n }\n },\n \"should_checkpoint\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059522000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b058694529488612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059522000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a8694529488612e\"\n }\n },\n \"done\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059522000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b058694529489612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059522000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a8694529489612e\"\n }\n },\n \"training_iteration\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059523000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452944b01612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059523000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452944b01612e\"\n }\n },\n \"time_this_iter_s\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294474054a79f28000000612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294474054a79f28000000612e\"\n }\n },\n \"time_total_s\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294474054a79f28000000612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294474054a79f28000000612e\"\n }\n },\n \"time_since_restore\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294474054a79f28000000612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294474054a79f28000000612e\"\n }\n },\n \"iterations_since_restore\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059523000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452944b01612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059523000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452944b01612e\"\n }\n }\n },\n \"checkpoint_manager\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059517060000000000008c267261792e747261696e2e5f696e7465726e616c2e636865636b706f696e745f6d616e61676572948c125f436865636b706f696e744d616e616765729493942981947d94288c125f636865636b706f696e745f636f6e666967948c097261792e747261696e948c10436865636b706f696e74436f6e6669679493942981947d94288c0b6e756d5f746f5f6b656570944e8c1a636865636b706f696e745f73636f72655f617474726962757465944e8c16636865636b706f696e745f73636f72655f6f72646572948c036d6178948c14636865636b706f696e745f6672657175656e6379944b008c11636865636b706f696e745f61745f656e6494898c1a5f636865636b706f696e745f6b6565705f616c6c5f72616e6b73948c0a44455052454341544544948c1f5f636865636b706f696e745f75706c6f61645f66726f6d5f776f726b65727394681275628c135f636865636b706f696e745f726573756c7473945d948c1b7261792e747261696e2e5f696e7465726e616c2e73657373696f6e948c0f5f547261696e696e67526573756c749493942981947d94288c0a636865636b706f696e749468068c0a436865636b706f696e749493942981947d94288c0470617468948cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f65353636653332615f31335f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d323034382c6465636f6465725f72745f6e756d5f6c617965723d322c64726f705f5f323032342d30362d32355f31362d34302d30332f636865636b706f696e745f303030303030948c0a66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c055f75756964948c0475756964948c04555549449493942981947d948c03696e74948a108eef8c58375f918166442f09d4e1302f736275628c076d657472696373947d94288c046c6f7373948c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b007494624308793c1eeffca2a04094869452948c0974696d657374616d70944a6bd77a668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303030948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b018c08747269616c5f6964948c086535363665333261948c0464617465948c13323032342d30362d32355f31362d34322d3531948c1074696d655f746869735f697465725f7394474054a79f280000008c0c74696d655f746f74616c5f7394474054a79f280000008c03706964944ac60e01008c08686f73746e616d65948c06723369356e36948c076e6f64655f6970948c0c31302e3135392e382e313539948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b088c146465636f6465725f72745f6e756d5f6c61796572944b028c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b108c0a656e636f6465725f6666944d00048c0d6465636f6465725f72745f6666944d00088c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b048c0964726f705f7261746594473fe8bd53616ab1398c026c7294473fa18a11a256d5188c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f726594474054a79f280000008c18697465726174696f6e735f73696e63655f726573746f7265944b01757562618c195f6c61746573745f636865636b706f696e745f726573756c7494681975622e\"\n }\n}"
+ ],
+ [
+ "{\n \"stub\": false,\n \"trainable_name\": \"train_model\",\n \"trial_id\": \"8c6e7040\",\n \"storage\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059575040000000000008c1b7261792e747261696e2e5f696e7465726e616c2e73746f72616765948c0e53746f72616765436f6e746578749493942981947d94288c12637573746f6d5f66735f70726f766964656494898c1273746f726167655f6c6f63616c5f70617468948c2b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c7473948c136578706572696d656e745f6469725f6e616d65948c0f746573745f6578706572696d656e74948c0e747269616c5f6469725f6e616d65948c96747261696e5f6d6f64656c5f38633665373034305f31345f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d323034382c6465636f6465725f696e745f6e756d5f6c617965723d382c6465636f6465725f72745f66663d313032342c6465636f6465725f72745f6e756d5f6c617965723d382c64726f705f323032342d30362d32355f31362d34312d3239948c1863757272656e745f636865636b706f696e745f696e646578944affffffff8c0b73796e635f636f6e666967948c097261792e747261696e948c0a53796e63436f6e6669679493942981947d94288c0b73796e635f706572696f64944d2c018c0c73796e635f74696d656f7574944d08078c0e73796e635f61727469666163747394898c1c73796e635f6172746966616374735f6f6e5f636865636b706f696e7494888c0a75706c6f61645f646972948c0a44455052454341544544948c0673796e6365729468188c1273796e635f6f6e5f636865636b706f696e7494681875628c1273746f726167655f66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c0f73746f726167655f66735f70617468948c452f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f7465737494681968008c115f46696c6573797374656d53796e6365729493942981947d9428681b682268134d2c0168144d08078c116c6173745f73796e635f75705f74696d65944741d99eb5915a973d8c136c6173745f73796e635f646f776e5f74696d659447fff00000000000008c0d5f73796e635f70726f63657373944e8c0c5f63757272656e745f636d649468008c125f75706c6f61645f746f5f66735f706174689493947d94288c0a6c6f63616c5f70617468948c3b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c74732f746573745f6578706572696d656e74948c0266739468228c0766735f70617468948c552f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e74948c076578636c756465945d948c0e2a2f636865636b706f696e745f2a9461758694756275622e\"\n },\n \"config\": {\n \"encoder_num_layer\": 8,\n \"decoder_rt_num_layer\": 8,\n \"decoder_int_num_layer\": 8,\n \"embedding_dim\": 256,\n \"encoder_ff\": 2048,\n \"decoder_rt_ff\": 1024,\n \"decoder_int_ff\": 2048,\n \"n_head\": 1,\n \"drop_rate\": 0.6537122299874158,\n \"lr\": 0.010515037912087563,\n \"batch_size\": 2048\n },\n \"_Trial__unresolved_config\": {\n \"encoder_num_layer\": 8,\n \"decoder_rt_num_layer\": 8,\n \"decoder_int_num_layer\": 8,\n \"embedding_dim\": 256,\n \"encoder_ff\": 2048,\n \"decoder_rt_ff\": 1024,\n \"decoder_int_ff\": 2048,\n \"n_head\": 1,\n \"drop_rate\": 0.6537122299874158,\n \"lr\": 0.010515037912087563,\n \"batch_size\": 2048\n },\n \"evaluated_params\": {\n \"batch_size\": 2048,\n \"decoder_int_ff\": 2048,\n \"decoder_int_num_layer\": 8,\n \"decoder_rt_ff\": 1024,\n \"decoder_rt_num_layer\": 8,\n \"drop_rate\": 0.6537122299874158,\n \"embedding_dim\": 256,\n \"encoder_ff\": 2048,\n \"encoder_num_layer\": 8,\n \"lr\": 0.010515037912087563,\n \"n_head\": 1\n },\n \"experiment_tag\": \"14_batch_size=2048,decoder_int_ff=2048,decoder_int_num_layer=8,decoder_rt_ff=1024,decoder_rt_num_layer=8,drop_rate=0.6537,embedding_dim=256,encoder_ff=2048,encoder_num_layer=8,lr=0.0105,n_head=1\",\n \"stopping_criterion\": {},\n \"_setup_default_resource\": true,\n \"_default_placement_group_factory\": \"80054e2e\",\n \"placement_group_factory\": \"800595ba000000000000008c237261792e74756e652e657865637574696f6e2e706c6163656d656e745f67726f757073948c15506c6163656d656e7447726f7570466163746f72799493942981947d94288c085f62756e646c6573945d947d94288c03435055944740540000000000008c034750559447401000000000000075618c155f686561645f62756e646c655f69735f656d70747994898c095f7374726174656779948c045041434b948c055f6172677394298c075f6b7761726773947d9475622e\",\n \"log_to_file\": [\n null,\n null\n ],\n \"max_failures\": 0,\n \"_default_result_or_future\": null,\n \"export_formats\": [],\n \"status\": \"ERROR\",\n \"relative_logdir\": \"train_model_8c6e7040_14_batch_size=2048,decoder_int_ff=2048,decoder_int_num_layer=8,decoder_rt_ff=1024,decoder_rt_num_layer=8,drop_2024-06-25_16-41-29\",\n \"trial_name_creator\": null,\n \"trial_dirname_creator\": null,\n \"custom_trial_name\": null,\n \"custom_dirname\": null,\n \"restore_path\": null,\n \"_restore_checkpoint_result\": null,\n \"_state_json\": null,\n \"results\": \"80054e2e\",\n \"extra_arg\": \"80054e2e\",\n \"_resources\": \"80054e2e\"\n}",
+ "{\n \"start_time\": 1719326572.0475543,\n \"num_failures\": 1,\n \"num_failures_after_restore\": 0,\n \"error_filename\": \"error.txt\",\n \"pickled_error_filename\": \"error.pkl\",\n \"last_result\": {\n \"trial_id\": \"8c6e7040\"\n },\n \"last_result_time\": -Infinity,\n \"metric_analysis\": {},\n \"_n_steps\": [\n 5,\n 10\n ],\n \"metric_n_steps\": {},\n \"checkpoint_manager\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059584010000000000008c267261792e747261696e2e5f696e7465726e616c2e636865636b706f696e745f6d616e61676572948c125f436865636b706f696e744d616e616765729493942981947d94288c125f636865636b706f696e745f636f6e666967948c097261792e747261696e948c10436865636b706f696e74436f6e6669679493942981947d94288c0b6e756d5f746f5f6b656570944e8c1a636865636b706f696e745f73636f72655f617474726962757465944e8c16636865636b706f696e745f73636f72655f6f72646572948c036d6178948c14636865636b706f696e745f6672657175656e6379944b008c11636865636b706f696e745f61745f656e6494898c1a5f636865636b706f696e745f6b6565705f616c6c5f72616e6b73948c0a44455052454341544544948c1f5f636865636b706f696e745f75706c6f61645f66726f6d5f776f726b65727394681275628c135f636865636b706f696e745f726573756c7473945d948c195f6c61746573745f636865636b706f696e745f726573756c74944e75622e\"\n }\n}"
+ ],
+ [
+ "{\n \"stub\": false,\n \"trainable_name\": \"train_model\",\n \"trial_id\": \"25b42194\",\n \"storage\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059572040000000000008c1b7261792e747261696e2e5f696e7465726e616c2e73746f72616765948c0e53746f72616765436f6e746578749493942981947d94288c12637573746f6d5f66735f70726f766964656494898c1273746f726167655f6c6f63616c5f70617468948c2b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c7473948c136578706572696d656e745f6469725f6e616d65948c0f746573745f6578706572696d656e74948c0e747269616c5f6469725f6e616d65948c96747261696e5f6d6f64656c5f32356234323139345f31355f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d323034382c6465636f6465725f696e745f6e756d5f6c617965723d382c6465636f6465725f72745f66663d313032342c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f323032342d30362d32355f31362d34322d3532948c1863757272656e745f636865636b706f696e745f696e646578944b008c0b73796e635f636f6e666967948c097261792e747261696e948c0a53796e63436f6e6669679493942981947d94288c0b73796e635f706572696f64944d2c018c0c73796e635f74696d656f7574944d08078c0e73796e635f61727469666163747394898c1c73796e635f6172746966616374735f6f6e5f636865636b706f696e7494888c0a75706c6f61645f646972948c0a44455052454341544544948c0673796e6365729468188c1273796e635f6f6e5f636865636b706f696e7494681875628c1273746f726167655f66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c0f73746f726167655f66735f70617468948c452f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f7465737494681968008c115f46696c6573797374656d53796e6365729493942981947d9428681b682268134d2c0168144d08078c116c6173745f73796e635f75705f74696d65944741d99eb5915a973d8c136c6173745f73796e635f646f776e5f74696d659447fff00000000000008c0d5f73796e635f70726f63657373944e8c0c5f63757272656e745f636d649468008c125f75706c6f61645f746f5f66735f706174689493947d94288c0a6c6f63616c5f70617468948c3b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c74732f746573745f6578706572696d656e74948c0266739468228c0766735f70617468948c552f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e74948c076578636c756465945d948c0e2a2f636865636b706f696e745f2a9461758694756275622e\"\n },\n \"config\": {\n \"encoder_num_layer\": 1,\n \"decoder_rt_num_layer\": 1,\n \"decoder_int_num_layer\": 8,\n \"embedding_dim\": 1024,\n \"encoder_ff\": 512,\n \"decoder_rt_ff\": 1024,\n \"decoder_int_ff\": 2048,\n \"n_head\": 16,\n \"drop_rate\": 0.3256301932631026,\n \"lr\": 0.0018178629337167497,\n \"batch_size\": 2048\n },\n \"_Trial__unresolved_config\": {\n \"encoder_num_layer\": 1,\n \"decoder_rt_num_layer\": 1,\n \"decoder_int_num_layer\": 8,\n \"embedding_dim\": 1024,\n \"encoder_ff\": 512,\n \"decoder_rt_ff\": 1024,\n \"decoder_int_ff\": 2048,\n \"n_head\": 16,\n \"drop_rate\": 0.3256301932631026,\n \"lr\": 0.0018178629337167497,\n \"batch_size\": 2048\n },\n \"evaluated_params\": {\n \"batch_size\": 2048,\n \"decoder_int_ff\": 2048,\n \"decoder_int_num_layer\": 8,\n \"decoder_rt_ff\": 1024,\n \"decoder_rt_num_layer\": 1,\n \"drop_rate\": 0.3256301932631026,\n \"embedding_dim\": 1024,\n \"encoder_ff\": 512,\n \"encoder_num_layer\": 1,\n \"lr\": 0.0018178629337167497,\n \"n_head\": 16\n },\n \"experiment_tag\": \"15_batch_size=2048,decoder_int_ff=2048,decoder_int_num_layer=8,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop_rate=0.3256,embedding_dim=1024,encoder_ff=512,encoder_num_layer=1,lr=0.0018,n_head=16\",\n \"stopping_criterion\": {},\n \"_setup_default_resource\": true,\n \"_default_placement_group_factory\": \"80054e2e\",\n \"placement_group_factory\": \"800595ba000000000000008c237261792e74756e652e657865637574696f6e2e706c6163656d656e745f67726f757073948c15506c6163656d656e7447726f7570466163746f72799493942981947d94288c085f62756e646c6573945d947d94288c03435055944740540000000000008c034750559447401000000000000075618c155f686561645f62756e646c655f69735f656d70747994898c095f7374726174656779948c045041434b948c055f6172677394298c075f6b7761726773947d9475622e\",\n \"log_to_file\": [\n null,\n null\n ],\n \"max_failures\": 0,\n \"_default_result_or_future\": null,\n \"export_formats\": [],\n \"status\": \"PAUSED\",\n \"relative_logdir\": \"train_model_25b42194_15_batch_size=2048,decoder_int_ff=2048,decoder_int_num_layer=8,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop_2024-06-25_16-42-52\",\n \"trial_name_creator\": null,\n \"trial_dirname_creator\": null,\n \"custom_trial_name\": null,\n \"custom_dirname\": null,\n \"restore_path\": null,\n \"_restore_checkpoint_result\": null,\n \"_state_json\": null,\n \"results\": \"80054e2e\",\n \"extra_arg\": \"80054e2e\",\n \"_resources\": \"80054e2e\"\n}",
+ "{\n \"start_time\": 1719326598.1773813,\n \"num_failures\": 0,\n \"num_failures_after_restore\": 0,\n \"error_filename\": null,\n \"pickled_error_filename\": null,\n \"last_result\": {\n \"loss\": 5061.415592704232,\n \"timestamp\": 1719326792,\n \"checkpoint_dir_name\": \"checkpoint_000000\",\n \"should_checkpoint\": true,\n \"done\": false,\n \"training_iteration\": 1,\n \"trial_id\": \"25b42194\",\n \"date\": \"2024-06-25_16-46-32\",\n \"time_this_iter_s\": 194.36064219474792,\n \"time_total_s\": 194.36064219474792,\n \"pid\": 122466,\n \"hostname\": \"r3i5n6\",\n \"node_ip\": \"10.159.8.159\",\n \"config\": {\n \"encoder_num_layer\": 1,\n \"decoder_rt_num_layer\": 1,\n \"decoder_int_num_layer\": 8,\n \"embedding_dim\": 1024,\n \"encoder_ff\": 512,\n \"decoder_rt_ff\": 1024,\n \"decoder_int_ff\": 2048,\n \"n_head\": 16,\n \"drop_rate\": 0.3256301932631026,\n \"lr\": 0.0018178629337167497,\n \"batch_size\": 2048\n },\n \"time_since_restore\": 194.36064219474792,\n \"iterations_since_restore\": 1,\n \"experiment_tag\": \"15_batch_size=2048,decoder_int_ff=2048,decoder_int_num_layer=8,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop_rate=0.3256,embedding_dim=1024,encoder_ff=512,encoder_num_layer=1,lr=0.0018,n_head=16\"\n },\n \"last_result_time\": 1719326792.5431535,\n \"metric_analysis\": {\n \"loss\": {\n \"max\": 5061.415592704232,\n \"min\": 5061.415592704232,\n \"avg\": 5061.415592704232,\n \"last\": 5061.415592704232,\n \"last-5-avg\": 5061.415592704232,\n \"last-10-avg\": 5061.415592704232\n },\n \"should_checkpoint\": {\n \"max\": true,\n \"min\": true,\n \"avg\": true,\n \"last\": true,\n \"last-5-avg\": true,\n \"last-10-avg\": true\n },\n \"done\": {\n \"max\": false,\n \"min\": false,\n \"avg\": false,\n \"last\": false,\n \"last-5-avg\": false,\n \"last-10-avg\": false\n },\n \"training_iteration\": {\n \"max\": 1,\n \"min\": 1,\n \"avg\": 1,\n \"last\": 1,\n \"last-5-avg\": 1,\n \"last-10-avg\": 1\n },\n \"time_this_iter_s\": {\n \"max\": 194.36064219474792,\n \"min\": 194.36064219474792,\n \"avg\": 194.36064219474792,\n \"last\": 194.36064219474792,\n \"last-5-avg\": 194.36064219474792,\n \"last-10-avg\": 194.36064219474792\n },\n \"time_total_s\": {\n \"max\": 194.36064219474792,\n \"min\": 194.36064219474792,\n \"avg\": 194.36064219474792,\n \"last\": 194.36064219474792,\n \"last-5-avg\": 194.36064219474792,\n \"last-10-avg\": 194.36064219474792\n },\n \"time_since_restore\": {\n \"max\": 194.36064219474792,\n \"min\": 194.36064219474792,\n \"avg\": 194.36064219474792,\n \"last\": 194.36064219474792,\n \"last-5-avg\": 194.36064219474792,\n \"last-10-avg\": 194.36064219474792\n },\n \"iterations_since_restore\": {\n \"max\": 1,\n \"min\": 1,\n \"avg\": 1,\n \"last\": 1,\n \"last-5-avg\": 1,\n \"last-10-avg\": 1\n }\n },\n \"_n_steps\": [\n 5,\n 10\n ],\n \"metric_n_steps\": {\n \"loss\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059589000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452948c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b007494624308229148646ac5b3409486945294612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059589000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452948c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b007494624308229148646ac5b3409486945294612e\"\n }\n },\n \"should_checkpoint\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059522000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b058694529488612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059522000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a8694529488612e\"\n }\n },\n \"done\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059522000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b058694529489612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059522000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a8694529489612e\"\n }\n },\n \"training_iteration\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059523000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452944b01612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059523000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452944b01612e\"\n }\n },\n \"time_this_iter_s\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452944740684b8a61800000612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452944740684b8a61800000612e\"\n }\n },\n \"time_total_s\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452944740684b8a61800000612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452944740684b8a61800000612e\"\n }\n },\n \"time_since_restore\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452944740684b8a61800000612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452944740684b8a61800000612e\"\n }\n },\n \"iterations_since_restore\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059523000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452944b01612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059523000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452944b01612e\"\n }\n }\n },\n \"checkpoint_manager\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059519060000000000008c267261792e747261696e2e5f696e7465726e616c2e636865636b706f696e745f6d616e61676572948c125f436865636b706f696e744d616e616765729493942981947d94288c125f636865636b706f696e745f636f6e666967948c097261792e747261696e948c10436865636b706f696e74436f6e6669679493942981947d94288c0b6e756d5f746f5f6b656570944e8c1a636865636b706f696e745f73636f72655f617474726962757465944e8c16636865636b706f696e745f73636f72655f6f72646572948c036d6178948c14636865636b706f696e745f6672657175656e6379944b008c11636865636b706f696e745f61745f656e6494898c1a5f636865636b706f696e745f6b6565705f616c6c5f72616e6b73948c0a44455052454341544544948c1f5f636865636b706f696e745f75706c6f61645f66726f6d5f776f726b65727394681275628c135f636865636b706f696e745f726573756c7473945d948c1b7261792e747261696e2e5f696e7465726e616c2e73657373696f6e948c0f5f547261696e696e67526573756c749493942981947d94288c0a636865636b706f696e749468068c0a436865636b706f696e749493942981947d94288c0470617468948cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f32356234323139345f31355f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d323034382c6465636f6465725f696e745f6e756d5f6c617965723d382c6465636f6465725f72745f66663d313032342c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f323032342d30362d32355f31362d34322d35322f636865636b706f696e745f303030303030948c0a66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c055f75756964948c0475756964948c04555549449493942981947d948c03696e74948a1134c6fe3ca07c39bd6c4247e744cd77b300736275628c076d657472696373947d94288c046c6f7373948c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b007494624308229148646ac5b34094869452948c0974696d657374616d70944a48d87a668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303030948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b018c08747269616c5f6964948c083235623432313934948c0464617465948c13323032342d30362d32355f31362d34362d3332948c1074696d655f746869735f697465725f73944740684b8a618000008c0c74696d655f746f74616c5f73944740684b8a618000008c03706964944a62de01008c08686f73746e616d65948c06723369356e36948c076e6f64655f6970948c0c31302e3135392e382e313539948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b018c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b088c0d656d62656464696e675f64696d944d00048c0a656e636f6465725f6666944d00028c0d6465636f6465725f72745f6666944d00048c0e6465636f6465725f696e745f6666944d00088c066e5f68656164944b108c0964726f705f7261746594473fd4d72005a9ee8e8c026c7294473f5dc8ab7654c5908c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f7265944740684b8a618000008c18697465726174696f6e735f73696e63655f726573746f7265944b01757562618c195f6c61746573745f636865636b706f696e745f726573756c7494681975622e\"\n }\n}"
+ ],
+ [
+ "{\n \"stub\": false,\n \"trainable_name\": \"train_model\",\n \"trial_id\": \"b53cfe05\",\n \"storage\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059572040000000000008c1b7261792e747261696e2e5f696e7465726e616c2e73746f72616765948c0e53746f72616765436f6e746578749493942981947d94288c12637573746f6d5f66735f70726f766964656494898c1273746f726167655f6c6f63616c5f70617468948c2b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c7473948c136578706572696d656e745f6469725f6e616d65948c0f746573745f6578706572696d656e74948c0e747269616c5f6469725f6e616d65948c96747261696e5f6d6f64656c5f62353363666530355f31365f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d382c6465636f6465725f72745f66663d3531322c6465636f6465725f72745f6e756d5f6c617965723d322c64726f705f725f323032342d30362d32355f31362d34332d3138948c1863757272656e745f636865636b706f696e745f696e646578944b008c0b73796e635f636f6e666967948c097261792e747261696e948c0a53796e63436f6e6669679493942981947d94288c0b73796e635f706572696f64944d2c018c0c73796e635f74696d656f7574944d08078c0e73796e635f61727469666163747394898c1c73796e635f6172746966616374735f6f6e5f636865636b706f696e7494888c0a75706c6f61645f646972948c0a44455052454341544544948c0673796e6365729468188c1273796e635f6f6e5f636865636b706f696e7494681875628c1273746f726167655f66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c0f73746f726167655f66735f70617468948c452f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f7465737494681968008c115f46696c6573797374656d53796e6365729493942981947d9428681b682268134d2c0168144d08078c116c6173745f73796e635f75705f74696d65944741d99eb5dcbfed3f8c136c6173745f73796e635f646f776e5f74696d659447fff00000000000008c0d5f73796e635f70726f63657373944e8c0c5f63757272656e745f636d649468008c125f75706c6f61645f746f5f66735f706174689493947d94288c0a6c6f63616c5f70617468948c3b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c74732f746573745f6578706572696d656e74948c0266739468228c0766735f70617468948c552f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e74948c076578636c756465945d948c0e2a2f636865636b706f696e745f2a9461758694756275622e\"\n },\n \"config\": {\n \"encoder_num_layer\": 2,\n \"decoder_rt_num_layer\": 2,\n \"decoder_int_num_layer\": 8,\n \"embedding_dim\": 64,\n \"encoder_ff\": 512,\n \"decoder_rt_ff\": 512,\n \"decoder_int_ff\": 512,\n \"n_head\": 4,\n \"drop_rate\": 0.9936594412219328,\n \"lr\": 0.000836614555813242,\n \"batch_size\": 2048\n },\n \"_Trial__unresolved_config\": {\n \"encoder_num_layer\": 2,\n \"decoder_rt_num_layer\": 2,\n \"decoder_int_num_layer\": 8,\n \"embedding_dim\": 64,\n \"encoder_ff\": 512,\n \"decoder_rt_ff\": 512,\n \"decoder_int_ff\": 512,\n \"n_head\": 4,\n \"drop_rate\": 0.9936594412219328,\n \"lr\": 0.000836614555813242,\n \"batch_size\": 2048\n },\n \"evaluated_params\": {\n \"batch_size\": 2048,\n \"decoder_int_ff\": 512,\n \"decoder_int_num_layer\": 8,\n \"decoder_rt_ff\": 512,\n \"decoder_rt_num_layer\": 2,\n \"drop_rate\": 0.9936594412219328,\n \"embedding_dim\": 64,\n \"encoder_ff\": 512,\n \"encoder_num_layer\": 2,\n \"lr\": 0.000836614555813242,\n \"n_head\": 4\n },\n \"experiment_tag\": \"16_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=8,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_rate=0.9937,embedding_dim=64,encoder_ff=512,encoder_num_layer=2,lr=0.0008,n_head=4\",\n \"stopping_criterion\": {},\n \"_setup_default_resource\": true,\n \"_default_placement_group_factory\": \"80054e2e\",\n \"placement_group_factory\": \"800595ba000000000000008c237261792e74756e652e657865637574696f6e2e706c6163656d656e745f67726f757073948c15506c6163656d656e7447726f7570466163746f72799493942981947d94288c085f62756e646c6573945d947d94288c03435055944740540000000000008c034750559447401000000000000075618c155f686561645f62756e646c655f69735f656d70747994898c095f7374726174656779948c045041434b948c055f6172677394298c075f6b7761726773947d9475622e\",\n \"log_to_file\": [\n null,\n null\n ],\n \"max_failures\": 0,\n \"_default_result_or_future\": null,\n \"export_formats\": [],\n \"status\": \"PAUSED\",\n \"relative_logdir\": \"train_model_b53cfe05_16_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=8,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_16-43-18\",\n \"trial_name_creator\": null,\n \"trial_dirname_creator\": null,\n \"custom_trial_name\": null,\n \"custom_dirname\": null,\n \"restore_path\": null,\n \"_restore_checkpoint_result\": null,\n \"_state_json\": null,\n \"results\": \"80054e2e\",\n \"extra_arg\": \"80054e2e\",\n \"_resources\": \"80054e2e\"\n}",
+ "{\n \"start_time\": 1719326792.6860473,\n \"num_failures\": 0,\n \"num_failures_after_restore\": 0,\n \"error_filename\": null,\n \"pickled_error_filename\": null,\n \"last_result\": {\n \"loss\": 3022.543074480192,\n \"timestamp\": 1719326860,\n \"checkpoint_dir_name\": \"checkpoint_000000\",\n \"should_checkpoint\": true,\n \"done\": false,\n \"training_iteration\": 1,\n \"trial_id\": \"b53cfe05\",\n \"date\": \"2024-06-25_16-47-40\",\n \"time_this_iter_s\": 67.51874232292175,\n \"time_total_s\": 67.51874232292175,\n \"pid\": 122466,\n \"hostname\": \"r3i5n6\",\n \"node_ip\": \"10.159.8.159\",\n \"config\": {\n \"encoder_num_layer\": 2,\n \"decoder_rt_num_layer\": 2,\n \"decoder_int_num_layer\": 8,\n \"embedding_dim\": 64,\n \"encoder_ff\": 512,\n \"decoder_rt_ff\": 512,\n \"decoder_int_ff\": 512,\n \"n_head\": 4,\n \"drop_rate\": 0.9936594412219328,\n \"lr\": 0.000836614555813242,\n \"batch_size\": 2048\n },\n \"time_since_restore\": 67.51874232292175,\n \"iterations_since_restore\": 1,\n \"experiment_tag\": \"16_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=8,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_rate=0.9937,embedding_dim=64,encoder_ff=512,encoder_num_layer=2,lr=0.0008,n_head=4\"\n },\n \"last_result_time\": 1719326860.2093158,\n \"metric_analysis\": {\n \"loss\": {\n \"max\": 3022.543074480192,\n \"min\": 3022.543074480192,\n \"avg\": 3022.543074480192,\n \"last\": 3022.543074480192,\n \"last-5-avg\": 3022.543074480192,\n \"last-10-avg\": 3022.543074480192\n },\n \"should_checkpoint\": {\n \"max\": true,\n \"min\": true,\n \"avg\": true,\n \"last\": true,\n \"last-5-avg\": true,\n \"last-10-avg\": true\n },\n \"done\": {\n \"max\": false,\n \"min\": false,\n \"avg\": false,\n \"last\": false,\n \"last-5-avg\": false,\n \"last-10-avg\": false\n },\n \"training_iteration\": {\n \"max\": 1,\n \"min\": 1,\n \"avg\": 1,\n \"last\": 1,\n \"last-5-avg\": 1,\n \"last-10-avg\": 1\n },\n \"time_this_iter_s\": {\n \"max\": 67.51874232292175,\n \"min\": 67.51874232292175,\n \"avg\": 67.51874232292175,\n \"last\": 67.51874232292175,\n \"last-5-avg\": 67.51874232292175,\n \"last-10-avg\": 67.51874232292175\n },\n \"time_total_s\": {\n \"max\": 67.51874232292175,\n \"min\": 67.51874232292175,\n \"avg\": 67.51874232292175,\n \"last\": 67.51874232292175,\n \"last-5-avg\": 67.51874232292175,\n \"last-10-avg\": 67.51874232292175\n },\n \"time_since_restore\": {\n \"max\": 67.51874232292175,\n \"min\": 67.51874232292175,\n \"avg\": 67.51874232292175,\n \"last\": 67.51874232292175,\n \"last-5-avg\": 67.51874232292175,\n \"last-10-avg\": 67.51874232292175\n },\n \"iterations_since_restore\": {\n \"max\": 1,\n \"min\": 1,\n \"avg\": 1,\n \"last\": 1,\n \"last-5-avg\": 1,\n \"last-10-avg\": 1\n }\n },\n \"_n_steps\": [\n 5,\n 10\n ],\n \"metric_n_steps\": {\n \"loss\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059589000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452948c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b0074946243086fb7db0d169da7409486945294612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059589000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452948c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b0074946243086fb7db0d169da7409486945294612e\"\n }\n },\n \"should_checkpoint\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059522000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b058694529488612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059522000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a8694529488612e\"\n }\n },\n \"done\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059522000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b058694529489612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059522000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a8694529489612e\"\n }\n },\n \"training_iteration\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059523000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452944b01612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059523000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452944b01612e\"\n }\n },\n \"time_this_iter_s\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294474050e13313000000612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294474050e13313000000612e\"\n }\n },\n \"time_total_s\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294474050e13313000000612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294474050e13313000000612e\"\n }\n },\n \"time_since_restore\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294474050e13313000000612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294474050e13313000000612e\"\n }\n },\n \"iterations_since_restore\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059523000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452944b01612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059523000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452944b01612e\"\n }\n }\n },\n \"checkpoint_manager\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059517060000000000008c267261792e747261696e2e5f696e7465726e616c2e636865636b706f696e745f6d616e61676572948c125f436865636b706f696e744d616e616765729493942981947d94288c125f636865636b706f696e745f636f6e666967948c097261792e747261696e948c10436865636b706f696e74436f6e6669679493942981947d94288c0b6e756d5f746f5f6b656570944e8c1a636865636b706f696e745f73636f72655f617474726962757465944e8c16636865636b706f696e745f73636f72655f6f72646572948c036d6178948c14636865636b706f696e745f6672657175656e6379944b008c11636865636b706f696e745f61745f656e6494898c1a5f636865636b706f696e745f6b6565705f616c6c5f72616e6b73948c0a44455052454341544544948c1f5f636865636b706f696e745f75706c6f61645f66726f6d5f776f726b65727394681275628c135f636865636b706f696e745f726573756c7473945d948c1b7261792e747261696e2e5f696e7465726e616c2e73657373696f6e948c0f5f547261696e696e67526573756c749493942981947d94288c0a636865636b706f696e749468068c0a436865636b706f696e749493942981947d94288c0470617468948cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f62353363666530355f31365f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d382c6465636f6465725f72745f66663d3531322c6465636f6465725f72745f6e756d5f6c617965723d322c64726f705f725f323032342d30362d32355f31362d34332d31382f636865636b706f696e745f303030303030948c0a66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c055f75756964948c0475756964948c04555549449493942981947d948c03696e74948a100ded1d5120b71bb6d94d1874c6c85758736275628c076d657472696373947d94288c046c6f7373948c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b0074946243086fb7db0d169da74094869452948c0974696d657374616d70944a8cd87a668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303030948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b018c08747269616c5f6964948c086235336366653035948c0464617465948c13323032342d30362d32355f31362d34372d3430948c1074696d655f746869735f697465725f7394474050e133130000008c0c74696d655f746f74616c5f7394474050e133130000008c03706964944a62de01008c08686f73746e616d65948c06723369356e36948c076e6f64655f6970948c0c31302e3135392e382e313539948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b028c146465636f6465725f72745f6e756d5f6c61796572944b028c156465636f6465725f696e745f6e756d5f6c61796572944b088c0d656d62656464696e675f64696d944b408c0a656e636f6465725f6666944d00028c0d6465636f6465725f72745f6666944d00028c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b048c0964726f705f7261746594473fefcc0ee26d1d608c026c7294473f4b6a08140aa9ff8c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f726594474050e133130000008c18697465726174696f6e735f73696e63655f726573746f7265944b01757562618c195f6c61746573745f636865636b706f696e745f726573756c7494681975622e\"\n }\n}"
+ ],
+ [
+ "{\n \"stub\": false,\n \"trainable_name\": \"train_model\",\n \"trial_id\": \"45b8cfdd\",\n \"storage\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059575040000000000008c1b7261792e747261696e2e5f696e7465726e616c2e73746f72616765948c0e53746f72616765436f6e746578749493942981947d94288c12637573746f6d5f66735f70726f766964656494898c1273746f726167655f6c6f63616c5f70617468948c2b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c7473948c136578706572696d656e745f6469725f6e616d65948c0f746573745f6578706572696d656e74948c0e747269616c5f6469725f6e616d65948c96747261696e5f6d6f64656c5f34356238636664645f31375f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d323034382c6465636f6465725f696e745f6e756d5f6c617965723d322c6465636f6465725f72745f66663d313032342c6465636f6465725f72745f6e756d5f6c617965723d342c64726f705f323032342d30362d32355f31362d34362d3332948c1863757272656e745f636865636b706f696e745f696e646578944affffffff8c0b73796e635f636f6e666967948c097261792e747261696e948c0a53796e63436f6e6669679493942981947d94288c0b73796e635f706572696f64944d2c018c0c73796e635f74696d656f7574944d08078c0e73796e635f61727469666163747394898c1c73796e635f6172746966616374735f6f6e5f636865636b706f696e7494888c0a75706c6f61645f646972948c0a44455052454341544544948c0673796e6365729468188c1273796e635f6f6e5f636865636b706f696e7494681875628c1273746f726167655f66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c0f73746f726167655f66735f70617468948c452f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f7465737494681968008c115f46696c6573797374656d53796e6365729493942981947d9428681b682268134d2c0168144d08078c116c6173745f73796e635f75705f74696d65944741d99eb5dcbfed3f8c136c6173745f73796e635f646f776e5f74696d659447fff00000000000008c0d5f73796e635f70726f63657373944e8c0c5f63757272656e745f636d649468008c125f75706c6f61645f746f5f66735f706174689493947d94288c0a6c6f63616c5f70617468948c3b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c74732f746573745f6578706572696d656e74948c0266739468228c0766735f70617468948c552f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e74948c076578636c756465945d948c0e2a2f636865636b706f696e745f2a9461758694756275622e\"\n },\n \"config\": {\n \"encoder_num_layer\": 2,\n \"decoder_rt_num_layer\": 4,\n \"decoder_int_num_layer\": 2,\n \"embedding_dim\": 1024,\n \"encoder_ff\": 2048,\n \"decoder_rt_ff\": 1024,\n \"decoder_int_ff\": 2048,\n \"n_head\": 4,\n \"drop_rate\": 0.18931069188451255,\n \"lr\": 0.0002168775970949337,\n \"batch_size\": 2048\n },\n \"_Trial__unresolved_config\": {\n \"encoder_num_layer\": 2,\n \"decoder_rt_num_layer\": 4,\n \"decoder_int_num_layer\": 2,\n \"embedding_dim\": 1024,\n \"encoder_ff\": 2048,\n \"decoder_rt_ff\": 1024,\n \"decoder_int_ff\": 2048,\n \"n_head\": 4,\n \"drop_rate\": 0.18931069188451255,\n \"lr\": 0.0002168775970949337,\n \"batch_size\": 2048\n },\n \"evaluated_params\": {\n \"batch_size\": 2048,\n \"decoder_int_ff\": 2048,\n \"decoder_int_num_layer\": 2,\n \"decoder_rt_ff\": 1024,\n \"decoder_rt_num_layer\": 4,\n \"drop_rate\": 0.18931069188451255,\n \"embedding_dim\": 1024,\n \"encoder_ff\": 2048,\n \"encoder_num_layer\": 2,\n \"lr\": 0.0002168775970949337,\n \"n_head\": 4\n },\n \"experiment_tag\": \"17_batch_size=2048,decoder_int_ff=2048,decoder_int_num_layer=2,decoder_rt_ff=1024,decoder_rt_num_layer=4,drop_rate=0.1893,embedding_dim=1024,encoder_ff=2048,encoder_num_layer=2,lr=0.0002,n_head=4\",\n \"stopping_criterion\": {},\n \"_setup_default_resource\": true,\n \"_default_placement_group_factory\": \"80054e2e\",\n \"placement_group_factory\": \"800595ba000000000000008c237261792e74756e652e657865637574696f6e2e706c6163656d656e745f67726f757073948c15506c6163656d656e7447726f7570466163746f72799493942981947d94288c085f62756e646c6573945d947d94288c03435055944740540000000000008c034750559447401000000000000075618c155f686561645f62756e646c655f69735f656d70747994898c095f7374726174656779948c045041434b948c055f6172677394298c075f6b7761726773947d9475622e\",\n \"log_to_file\": [\n null,\n null\n ],\n \"max_failures\": 0,\n \"_default_result_or_future\": null,\n \"export_formats\": [],\n \"status\": \"ERROR\",\n \"relative_logdir\": \"train_model_45b8cfdd_17_batch_size=2048,decoder_int_ff=2048,decoder_int_num_layer=2,decoder_rt_ff=1024,decoder_rt_num_layer=4,drop_2024-06-25_16-46-32\",\n \"trial_name_creator\": null,\n \"trial_dirname_creator\": null,\n \"custom_trial_name\": null,\n \"custom_dirname\": null,\n \"restore_path\": null,\n \"_restore_checkpoint_result\": null,\n \"_state_json\": null,\n \"results\": \"80054e2e\",\n \"extra_arg\": \"80054e2e\",\n \"_resources\": \"80054e2e\"\n}",
+ "{\n \"start_time\": 1719326860.3577213,\n \"num_failures\": 1,\n \"num_failures_after_restore\": 0,\n \"error_filename\": \"error.txt\",\n \"pickled_error_filename\": \"error.pkl\",\n \"last_result\": {\n \"trial_id\": \"45b8cfdd\"\n },\n \"last_result_time\": -Infinity,\n \"metric_analysis\": {},\n \"_n_steps\": [\n 5,\n 10\n ],\n \"metric_n_steps\": {},\n \"checkpoint_manager\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059584010000000000008c267261792e747261696e2e5f696e7465726e616c2e636865636b706f696e745f6d616e61676572948c125f436865636b706f696e744d616e616765729493942981947d94288c125f636865636b706f696e745f636f6e666967948c097261792e747261696e948c10436865636b706f696e74436f6e6669679493942981947d94288c0b6e756d5f746f5f6b656570944e8c1a636865636b706f696e745f73636f72655f617474726962757465944e8c16636865636b706f696e745f73636f72655f6f72646572948c036d6178948c14636865636b706f696e745f6672657175656e6379944b008c11636865636b706f696e745f61745f656e6494898c1a5f636865636b706f696e745f6b6565705f616c6c5f72616e6b73948c0a44455052454341544544948c1f5f636865636b706f696e745f75706c6f61645f66726f6d5f776f726b65727394681275628c135f636865636b706f696e745f726573756c7473945d948c195f6c61746573745f636865636b706f696e745f726573756c74944e75622e\"\n }\n}"
+ ],
+ [
+ "{\n \"stub\": false,\n \"trainable_name\": \"train_model\",\n \"trial_id\": \"8e58579c\",\n \"storage\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059572040000000000008c1b7261792e747261696e2e5f696e7465726e616c2e73746f72616765948c0e53746f72616765436f6e746578749493942981947d94288c12637573746f6d5f66735f70726f766964656494898c1273746f726167655f6c6f63616c5f70617468948c2b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c7473948c136578706572696d656e745f6469725f6e616d65948c0f746573745f6578706572696d656e74948c0e747269616c5f6469725f6e616d65948c96747261696e5f6d6f64656c5f38653538353739635f31385f62617463685f73697a653d313032342c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d382c6465636f6465725f72745f66663d323034382c6465636f6465725f72745f6e756d5f6c617965723d382c64726f705f5f323032342d30362d32355f31362d34372d3430948c1863757272656e745f636865636b706f696e745f696e646578944b018c0b73796e635f636f6e666967948c097261792e747261696e948c0a53796e63436f6e6669679493942981947d94288c0b73796e635f706572696f64944d2c018c0c73796e635f74696d656f7574944d08078c0e73796e635f61727469666163747394898c1c73796e635f6172746966616374735f6f6e5f636865636b706f696e7494888c0a75706c6f61645f646972948c0a44455052454341544544948c0673796e6365729468188c1273796e635f6f6e5f636865636b706f696e7494681875628c1273746f726167655f66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c0f73746f726167655f66735f70617468948c452f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f7465737494681968008c115f46696c6573797374656d53796e6365729493942981947d9428681b682268134d2c0168144d08078c116c6173745f73796e635f75705f74696d65944741d99eb5dcbfed3f8c136c6173745f73796e635f646f776e5f74696d659447fff00000000000008c0d5f73796e635f70726f63657373944e8c0c5f63757272656e745f636d649468008c125f75706c6f61645f746f5f66735f706174689493947d94288c0a6c6f63616c5f70617468948c3b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c74732f746573745f6578706572696d656e74948c0266739468228c0766735f70617468948c552f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e74948c076578636c756465945d948c0e2a2f636865636b706f696e745f2a9461758694756275622e\"\n },\n \"config\": {\n \"encoder_num_layer\": 8,\n \"decoder_rt_num_layer\": 8,\n \"decoder_int_num_layer\": 8,\n \"embedding_dim\": 64,\n \"encoder_ff\": 1024,\n \"decoder_rt_ff\": 2048,\n \"decoder_int_ff\": 512,\n \"n_head\": 8,\n \"drop_rate\": 0.8999111579906955,\n \"lr\": 0.000663247583015271,\n \"batch_size\": 1024\n },\n \"_Trial__unresolved_config\": {\n \"encoder_num_layer\": 8,\n \"decoder_rt_num_layer\": 8,\n \"decoder_int_num_layer\": 8,\n \"embedding_dim\": 64,\n \"encoder_ff\": 1024,\n \"decoder_rt_ff\": 2048,\n \"decoder_int_ff\": 512,\n \"n_head\": 8,\n \"drop_rate\": 0.8999111579906955,\n \"lr\": 0.000663247583015271,\n \"batch_size\": 1024\n },\n \"evaluated_params\": {\n \"batch_size\": 1024,\n \"decoder_int_ff\": 512,\n \"decoder_int_num_layer\": 8,\n \"decoder_rt_ff\": 2048,\n \"decoder_rt_num_layer\": 8,\n \"drop_rate\": 0.8999111579906955,\n \"embedding_dim\": 64,\n \"encoder_ff\": 1024,\n \"encoder_num_layer\": 8,\n \"lr\": 0.000663247583015271,\n \"n_head\": 8\n },\n \"experiment_tag\": \"18_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=8,decoder_rt_ff=2048,decoder_rt_num_layer=8,drop_rate=0.8999,embedding_dim=64,encoder_ff=1024,encoder_num_layer=8,lr=0.0007,n_head=8\",\n \"stopping_criterion\": {},\n \"_setup_default_resource\": true,\n \"_default_placement_group_factory\": \"80054e2e\",\n \"placement_group_factory\": \"800595ba000000000000008c237261792e74756e652e657865637574696f6e2e706c6163656d656e745f67726f757073948c15506c6163656d656e7447726f7570466163746f72799493942981947d94288c085f62756e646c6573945d947d94288c03435055944740540000000000008c034750559447401000000000000075618c155f686561645f62756e646c655f69735f656d70747994898c095f7374726174656779948c045041434b948c055f6172677394298c075f6b7761726773947d9475622e\",\n \"log_to_file\": [\n null,\n null\n ],\n \"max_failures\": 0,\n \"_default_result_or_future\": null,\n \"export_formats\": [],\n \"status\": \"PAUSED\",\n \"relative_logdir\": \"train_model_8e58579c_18_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=8,decoder_rt_ff=2048,decoder_rt_num_layer=8,drop__2024-06-25_16-47-40\",\n \"trial_name_creator\": null,\n \"trial_dirname_creator\": null,\n \"custom_trial_name\": null,\n \"custom_dirname\": null,\n \"restore_path\": null,\n \"_restore_checkpoint_result\": null,\n \"_state_json\": null,\n \"results\": \"80054e2e\",\n \"extra_arg\": \"80054e2e\",\n \"_resources\": \"80054e2e\"\n}",
+ "{\n \"start_time\": 1719326883.2966387,\n \"num_failures\": 0,\n \"num_failures_after_restore\": 0,\n \"error_filename\": null,\n \"pickled_error_filename\": null,\n \"last_result\": {\n \"loss\": 2056.04917078694,\n \"timestamp\": 1719327182,\n \"checkpoint_dir_name\": \"checkpoint_000001\",\n \"should_checkpoint\": true,\n \"done\": false,\n \"training_iteration\": 2,\n \"trial_id\": \"8e58579c\",\n \"date\": \"2024-06-25_16-53-02\",\n \"time_this_iter_s\": 142.2777497768402,\n \"time_total_s\": 298.91499376296997,\n \"pid\": 130020,\n \"hostname\": \"r3i5n6\",\n \"node_ip\": \"10.159.8.159\",\n \"config\": {\n \"encoder_num_layer\": 8,\n \"decoder_rt_num_layer\": 8,\n \"decoder_int_num_layer\": 8,\n \"embedding_dim\": 64,\n \"encoder_ff\": 1024,\n \"decoder_rt_ff\": 2048,\n \"decoder_int_ff\": 512,\n \"n_head\": 8,\n \"drop_rate\": 0.8999111579906955,\n \"lr\": 0.000663247583015271,\n \"batch_size\": 1024\n },\n \"time_since_restore\": 298.91499376296997,\n \"iterations_since_restore\": 2,\n \"experiment_tag\": \"18_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=8,decoder_rt_ff=2048,decoder_rt_num_layer=8,drop_rate=0.8999,embedding_dim=64,encoder_ff=1024,encoder_num_layer=8,lr=0.0007,n_head=8\"\n },\n \"last_result_time\": 1719327182.2214158,\n \"metric_analysis\": {\n \"loss\": {\n \"max\": 2062.3793243648497,\n \"min\": 2056.04917078694,\n \"avg\": 2059.2142475758947,\n \"last\": 2056.04917078694,\n \"last-5-avg\": 2059.2142475758947,\n \"last-10-avg\": 2059.2142475758947\n },\n \"should_checkpoint\": {\n \"max\": true,\n \"min\": true,\n \"avg\": 1.0,\n \"last\": true,\n \"last-5-avg\": 1.0,\n \"last-10-avg\": 1.0\n },\n \"done\": {\n \"max\": false,\n \"min\": false,\n \"avg\": 0.0,\n \"last\": false,\n \"last-5-avg\": 0.0,\n \"last-10-avg\": 0.0\n },\n \"training_iteration\": {\n \"max\": 2,\n \"min\": 1,\n \"avg\": 1.5,\n \"last\": 2,\n \"last-5-avg\": 1.5,\n \"last-10-avg\": 1.5\n },\n \"time_this_iter_s\": {\n \"max\": 156.63724398612976,\n \"min\": 142.2777497768402,\n \"avg\": 149.45749688148499,\n \"last\": 142.2777497768402,\n \"last-5-avg\": 149.45749688148499,\n \"last-10-avg\": 149.45749688148499\n },\n \"time_total_s\": {\n \"max\": 298.91499376296997,\n \"min\": 156.63724398612976,\n \"avg\": 227.77611887454987,\n \"last\": 298.91499376296997,\n \"last-5-avg\": 227.77611887454987,\n \"last-10-avg\": 227.77611887454987\n },\n \"time_since_restore\": {\n \"max\": 298.91499376296997,\n \"min\": 156.63724398612976,\n \"avg\": 227.77611887454987,\n \"last\": 298.91499376296997,\n \"last-5-avg\": 227.77611887454987,\n \"last-10-avg\": 227.77611887454987\n },\n \"iterations_since_restore\": {\n \"max\": 2,\n \"min\": 1,\n \"avg\": 1.5,\n \"last\": 2,\n \"last-5-avg\": 1.5,\n \"last-10-avg\": 1.5\n }\n },\n \"_n_steps\": [\n 5,\n 10\n ],\n \"metric_n_steps\": {\n \"loss\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005959d000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294288c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b007494624308369bcd36c21ca04094869452946807680d4308a7d3e92c1910a0409486945294652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005959d000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294288c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b007494624308369bcd36c21ca04094869452946807680d4308a7d3e92c1910a0409486945294652e\"\n }\n },\n \"should_checkpoint\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059524000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294288888652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059524000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294288888652e\"\n }\n },\n \"done\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059524000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294288989652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059524000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294288989652e\"\n }\n },\n \"training_iteration\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059526000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294284b014b02652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059526000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294284b014b02652e\"\n }\n },\n \"time_this_iter_s\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059534000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452942847406394644d800000474061c8e353800000652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059534000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452942847406394644d800000474061c8e353800000652e\"\n }\n },\n \"time_total_s\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059534000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452942847406394644d800000474072aea3d0800000652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059534000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452942847406394644d800000474072aea3d0800000652e\"\n }\n },\n \"time_since_restore\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059534000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452942847406394644d800000474072aea3d0800000652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059534000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452942847406394644d800000474072aea3d0800000652e\"\n }\n },\n \"iterations_since_restore\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059526000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294284b014b02652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059526000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294284b014b02652e\"\n }\n }\n },\n \"checkpoint_manager\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"800595a4090000000000008c267261792e747261696e2e5f696e7465726e616c2e636865636b706f696e745f6d616e61676572948c125f436865636b706f696e744d616e616765729493942981947d94288c125f636865636b706f696e745f636f6e666967948c097261792e747261696e948c10436865636b706f696e74436f6e6669679493942981947d94288c0b6e756d5f746f5f6b656570944e8c1a636865636b706f696e745f73636f72655f617474726962757465944e8c16636865636b706f696e745f73636f72655f6f72646572948c036d6178948c14636865636b706f696e745f6672657175656e6379944b008c11636865636b706f696e745f61745f656e6494898c1a5f636865636b706f696e745f6b6565705f616c6c5f72616e6b73948c0a44455052454341544544948c1f5f636865636b706f696e745f75706c6f61645f66726f6d5f776f726b65727394681275628c135f636865636b706f696e745f726573756c7473945d94288c1b7261792e747261696e2e5f696e7465726e616c2e73657373696f6e948c0f5f547261696e696e67526573756c749493942981947d94288c0a636865636b706f696e749468068c0a436865636b706f696e749493942981947d94288c0470617468948cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f38653538353739635f31385f62617463685f73697a653d313032342c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d382c6465636f6465725f72745f66663d323034382c6465636f6465725f72745f6e756d5f6c617965723d382c64726f705f5f323032342d30362d32355f31362d34372d34302f636865636b706f696e745f303030303030948c0a66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c055f75756964948c0475756964948c04555549449493942981947d948c03696e74948a102fd40e2ad2aee0ab774f9b6b0d8e1422736275628c076d657472696373947d94288c046c6f7373948c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b007494624308369bcd36c21ca04094869452948c0974696d657374616d70944a3fd97a668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303030948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b018c08747269616c5f6964948c083865353835373963948c0464617465948c13323032342d30362d32355f31362d35302d3339948c1074696d655f746869735f697465725f739447406394644d8000008c0c74696d655f746f74616c5f739447406394644d8000008c03706964944ae4fb01008c08686f73746e616d65948c06723369356e36948c076e6f64655f6970948c0c31302e3135392e382e313539948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b088c146465636f6465725f72745f6e756d5f6c61796572944b088c156465636f6465725f696e745f6e756d5f6c61796572944b088c0d656d62656464696e675f64696d944b408c0a656e636f6465725f6666944d00048c0d6465636f6465725f72745f6666944d00088c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b088c0964726f705f7261746594473feccc127c1c04508c026c7294473f45bbb956cf4fa78c0a62617463685f73697a65944d0004758c1274696d655f73696e63655f726573746f72659447406394644d8000008c18697465726174696f6e735f73696e63655f726573746f7265944b0175756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f38653538353739635f31385f62617463685f73697a653d313032342c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d382c6465636f6465725f72745f66663d323034382c6465636f6465725f72745f6e756d5f6c617965723d382c64726f705f5f323032342d30362d32355f31362d34372d34302f636865636b706f696e745f30303030303194682268257d946827897385945294682a682d2981947d9468308a109082755a26183b9e5c416e75fbaec42e7362756268317d94288c046c6f7373946836683c4308a7d3e92c1910a04094869452948c0974696d657374616d70944aced97a668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303031948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b028c08747269616c5f6964948c083865353835373963948c0464617465948c13323032342d30362d32355f31362d35332d3032948c1074696d655f746869735f697465725f7394474061c8e3538000008c0c74696d655f746f74616c5f7394474072aea3d08000008c03706964944ae4fb01008c08686f73746e616d65948c06723369356e36948c076e6f64655f6970948c0c31302e3135392e382e313539948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b088c146465636f6465725f72745f6e756d5f6c61796572944b088c156465636f6465725f696e745f6e756d5f6c61796572944b088c0d656d62656464696e675f64696d944b408c0a656e636f6465725f6666944d00048c0d6465636f6465725f72745f6666944d00088c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b088c0964726f705f7261746594473feccc127c1c04508c026c7294473f45bbb956cf4fa78c0a62617463685f73697a65944d0004758c1274696d655f73696e63655f726573746f726594474072aea3d08000008c18697465726174696f6e735f73696e63655f726573746f7265944b02757562658c195f6c61746573745f636865636b706f696e745f726573756c7494686275622e\"\n }\n}"
+ ],
+ [
+ "{\n \"stub\": false,\n \"trainable_name\": \"train_model\",\n \"trial_id\": \"375ed3d7\",\n \"storage\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059572040000000000008c1b7261792e747261696e2e5f696e7465726e616c2e73746f72616765948c0e53746f72616765436f6e746578749493942981947d94288c12637573746f6d5f66735f70726f766964656494898c1273746f726167655f6c6f63616c5f70617468948c2b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c7473948c136578706572696d656e745f6469725f6e616d65948c0f746573745f6578706572696d656e74948c0e747269616c5f6469725f6e616d65948c96747261696e5f6d6f64656c5f33373565643364375f31395f62617463685f73697a653d313032342c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d322c6465636f6465725f72745f66663d313032342c6465636f6465725f72745f6e756d5f6c617965723d342c64726f705f5f323032342d30362d32355f31362d35362d3132948c1863757272656e745f636865636b706f696e745f696e646578944b008c0b73796e635f636f6e666967948c097261792e747261696e948c0a53796e63436f6e6669679493942981947d94288c0b73796e635f706572696f64944d2c018c0c73796e635f74696d656f7574944d08078c0e73796e635f61727469666163747394898c1c73796e635f6172746966616374735f6f6e5f636865636b706f696e7494888c0a75706c6f61645f646972948c0a44455052454341544544948c0673796e6365729468188c1273796e635f6f6e5f636865636b706f696e7494681875628c1273746f726167655f66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c0f73746f726167655f66735f70617468948c452f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f7465737494681968008c115f46696c6573797374656d53796e6365729493942981947d9428681b682268134d2c0168144d08078c116c6173745f73796e635f75705f74696d65944741d99eb673a554b88c136c6173745f73796e635f646f776e5f74696d659447fff00000000000008c0d5f73796e635f70726f63657373944e8c0c5f63757272656e745f636d649468008c125f75706c6f61645f746f5f66735f706174689493947d94288c0a6c6f63616c5f70617468948c3b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c74732f746573745f6578706572696d656e74948c0266739468228c0766735f70617468948c552f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e74948c076578636c756465945d948c0e2a2f636865636b706f696e745f2a9461758694756275622e\"\n },\n \"config\": {\n \"encoder_num_layer\": 1,\n \"decoder_rt_num_layer\": 4,\n \"decoder_int_num_layer\": 2,\n \"embedding_dim\": 16,\n \"encoder_ff\": 1024,\n \"decoder_rt_ff\": 1024,\n \"decoder_int_ff\": 512,\n \"n_head\": 1,\n \"drop_rate\": 0.29850788173794873,\n \"lr\": 0.017371208825505195,\n \"batch_size\": 1024\n },\n \"_Trial__unresolved_config\": {\n \"encoder_num_layer\": 1,\n \"decoder_rt_num_layer\": 4,\n \"decoder_int_num_layer\": 2,\n \"embedding_dim\": 16,\n \"encoder_ff\": 1024,\n \"decoder_rt_ff\": 1024,\n \"decoder_int_ff\": 512,\n \"n_head\": 1,\n \"drop_rate\": 0.29850788173794873,\n \"lr\": 0.017371208825505195,\n \"batch_size\": 1024\n },\n \"evaluated_params\": {\n \"batch_size\": 1024,\n \"decoder_int_ff\": 512,\n \"decoder_int_num_layer\": 2,\n \"decoder_rt_ff\": 1024,\n \"decoder_rt_num_layer\": 4,\n \"drop_rate\": 0.29850788173794873,\n \"embedding_dim\": 16,\n \"encoder_ff\": 1024,\n \"encoder_num_layer\": 1,\n \"lr\": 0.017371208825505195,\n \"n_head\": 1\n },\n \"experiment_tag\": \"19_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=2,decoder_rt_ff=1024,decoder_rt_num_layer=4,drop_rate=0.2985,embedding_dim=16,encoder_ff=1024,encoder_num_layer=1,lr=0.0174,n_head=1\",\n \"stopping_criterion\": {},\n \"_setup_default_resource\": true,\n \"_default_placement_group_factory\": \"80054e2e\",\n \"placement_group_factory\": \"800595ba000000000000008c237261792e74756e652e657865637574696f6e2e706c6163656d656e745f67726f757073948c15506c6163656d656e7447726f7570466163746f72799493942981947d94288c085f62756e646c6573945d947d94288c03435055944740540000000000008c034750559447401000000000000075618c155f686561645f62756e646c655f69735f656d70747994898c095f7374726174656779948c045041434b948c055f6172677394298c075f6b7761726773947d9475622e\",\n \"log_to_file\": [\n null,\n null\n ],\n \"max_failures\": 0,\n \"_default_result_or_future\": null,\n \"export_formats\": [],\n \"status\": \"RUNNING\",\n \"relative_logdir\": \"train_model_375ed3d7_19_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=2,decoder_rt_ff=1024,decoder_rt_num_layer=4,drop__2024-06-25_16-56-12\",\n \"trial_name_creator\": null,\n \"trial_dirname_creator\": null,\n \"custom_trial_name\": null,\n \"custom_dirname\": null,\n \"restore_path\": null,\n \"_restore_checkpoint_result\": null,\n \"_state_json\": null,\n \"results\": \"80054e2e\",\n \"extra_arg\": \"80054e2e\",\n \"_resources\": \"80054e2e\"\n}",
+ "{\n \"start_time\": 1719327790.388722,\n \"num_failures\": 0,\n \"num_failures_after_restore\": 0,\n \"error_filename\": null,\n \"pickled_error_filename\": null,\n \"last_result\": {\n \"loss\": 1971.010832058163,\n \"timestamp\": 1719327860,\n \"checkpoint_dir_name\": \"checkpoint_000000\",\n \"should_checkpoint\": true,\n \"done\": false,\n \"training_iteration\": 1,\n \"trial_id\": \"375ed3d7\",\n \"date\": \"2024-06-25_17-04-20\",\n \"time_this_iter_s\": 70.38656640052795,\n \"time_total_s\": 70.38656640052795,\n \"pid\": 168065,\n \"hostname\": \"r3i5n6\",\n \"node_ip\": \"10.159.8.159\",\n \"config\": {\n \"encoder_num_layer\": 1,\n \"decoder_rt_num_layer\": 4,\n \"decoder_int_num_layer\": 2,\n \"embedding_dim\": 16,\n \"encoder_ff\": 1024,\n \"decoder_rt_ff\": 1024,\n \"decoder_int_ff\": 512,\n \"n_head\": 1,\n \"drop_rate\": 0.29850788173794873,\n \"lr\": 0.017371208825505195,\n \"batch_size\": 1024\n },\n \"time_since_restore\": 70.38656640052795,\n \"iterations_since_restore\": 1,\n \"experiment_tag\": \"19_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=2,decoder_rt_ff=1024,decoder_rt_num_layer=4,drop_rate=0.2985,embedding_dim=16,encoder_ff=1024,encoder_num_layer=1,lr=0.0174,n_head=1\"\n },\n \"last_result_time\": 1719327860.7799456,\n \"metric_analysis\": {\n \"loss\": {\n \"max\": 1971.010832058163,\n \"min\": 1971.010832058163,\n \"avg\": 1971.010832058163,\n \"last\": 1971.010832058163,\n \"last-5-avg\": 1971.010832058163,\n \"last-10-avg\": 1971.010832058163\n },\n \"should_checkpoint\": {\n \"max\": true,\n \"min\": true,\n \"avg\": true,\n \"last\": true,\n \"last-5-avg\": true,\n \"last-10-avg\": true\n },\n \"done\": {\n \"max\": false,\n \"min\": false,\n \"avg\": false,\n \"last\": false,\n \"last-5-avg\": false,\n \"last-10-avg\": false\n },\n \"training_iteration\": {\n \"max\": 1,\n \"min\": 1,\n \"avg\": 1,\n \"last\": 1,\n \"last-5-avg\": 1,\n \"last-10-avg\": 1\n },\n \"time_this_iter_s\": {\n \"max\": 70.38656640052795,\n \"min\": 70.38656640052795,\n \"avg\": 70.38656640052795,\n \"last\": 70.38656640052795,\n \"last-5-avg\": 70.38656640052795,\n \"last-10-avg\": 70.38656640052795\n },\n \"time_total_s\": {\n \"max\": 70.38656640052795,\n \"min\": 70.38656640052795,\n \"avg\": 70.38656640052795,\n \"last\": 70.38656640052795,\n \"last-5-avg\": 70.38656640052795,\n \"last-10-avg\": 70.38656640052795\n },\n \"time_since_restore\": {\n \"max\": 70.38656640052795,\n \"min\": 70.38656640052795,\n \"avg\": 70.38656640052795,\n \"last\": 70.38656640052795,\n \"last-5-avg\": 70.38656640052795,\n \"last-10-avg\": 70.38656640052795\n },\n \"iterations_since_restore\": {\n \"max\": 1,\n \"min\": 1,\n \"avg\": 1,\n \"last\": 1,\n \"last-5-avg\": 1,\n \"last-10-avg\": 1\n }\n },\n \"_n_steps\": [\n 5,\n 10\n ],\n \"metric_n_steps\": {\n \"loss\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059589000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452948c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b0074946243083c1e8f170bcc9e409486945294612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059589000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452948c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b0074946243083c1e8f170bcc9e409486945294612e\"\n }\n },\n \"should_checkpoint\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059522000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b058694529488612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059522000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a8694529488612e\"\n }\n },\n \"done\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059522000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b058694529489612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059522000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a8694529489612e\"\n }\n },\n \"training_iteration\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059523000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452944b01612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059523000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452944b01612e\"\n }\n },\n \"time_this_iter_s\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b058694529447405198bd81000000612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a8694529447405198bd81000000612e\"\n }\n },\n \"time_total_s\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b058694529447405198bd81000000612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a8694529447405198bd81000000612e\"\n }\n },\n \"time_since_restore\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b058694529447405198bd81000000612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a8694529447405198bd81000000612e\"\n }\n },\n \"iterations_since_restore\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059523000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452944b01612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059523000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452944b01612e\"\n }\n }\n },\n \"checkpoint_manager\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059517060000000000008c267261792e747261696e2e5f696e7465726e616c2e636865636b706f696e745f6d616e61676572948c125f436865636b706f696e744d616e616765729493942981947d94288c125f636865636b706f696e745f636f6e666967948c097261792e747261696e948c10436865636b706f696e74436f6e6669679493942981947d94288c0b6e756d5f746f5f6b656570944e8c1a636865636b706f696e745f73636f72655f617474726962757465944e8c16636865636b706f696e745f73636f72655f6f72646572948c036d6178948c14636865636b706f696e745f6672657175656e6379944b008c11636865636b706f696e745f61745f656e6494898c1a5f636865636b706f696e745f6b6565705f616c6c5f72616e6b73948c0a44455052454341544544948c1f5f636865636b706f696e745f75706c6f61645f66726f6d5f776f726b65727394681275628c135f636865636b706f696e745f726573756c7473945d948c1b7261792e747261696e2e5f696e7465726e616c2e73657373696f6e948c0f5f547261696e696e67526573756c749493942981947d94288c0a636865636b706f696e749468068c0a436865636b706f696e749493942981947d94288c0470617468948cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f33373565643364375f31395f62617463685f73697a653d313032342c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d322c6465636f6465725f72745f66663d313032342c6465636f6465725f72745f6e756d5f6c617965723d342c64726f705f5f323032342d30362d32355f31362d35362d31322f636865636b706f696e745f303030303030948c0a66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c055f75756964948c0475756964948c04555549449493942981947d948c03696e74948a10396822c682368391904e67012945ac5a736275628c076d657472696373947d94288c046c6f7373948c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b0074946243083c1e8f170bcc9e4094869452948c0974696d657374616d70944a74dc7a668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303030948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b018c08747269616c5f6964948c083337356564336437948c0464617465948c13323032342d30362d32355f31372d30342d3230948c1074696d655f746869735f697465725f739447405198bd810000008c0c74696d655f746f74616c5f739447405198bd810000008c03706964944a819002008c08686f73746e616d65948c06723369356e36948c076e6f64655f6970948c0c31302e3135392e382e313539948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b018c146465636f6465725f72745f6e756d5f6c61796572944b048c156465636f6465725f696e745f6e756d5f6c61796572944b028c0d656d62656464696e675f64696d944b108c0a656e636f6465725f6666944d00048c0d6465636f6465725f72745f6666944d00048c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b018c0964726f705f7261746594473fd31ac0cd6a6a188c026c7294473f91c9c21730ac118c0a62617463685f73697a65944d0004758c1274696d655f73696e63655f726573746f72659447405198bd810000008c18697465726174696f6e735f73696e63655f726573746f7265944b01757562618c195f6c61746573745f636865636b706f696e745f726573756c7494681975622e\"\n }\n}"
+ ]
+ ],
+ "runner_data": {
+ "_earliest_stopping_actor": 35513.70259362,
+ "_actor_cleanup_timeout": 600,
+ "_actor_force_cleanup_timeout": 10,
+ "_reuse_actors": true,
+ "_buffer_length": 1,
+ "_buffer_min_time_s": 0.0,
+ "_buffer_max_time_s": 100.0,
+ "_max_pending_trials": 1,
+ "_metric": "loss",
+ "_total_time": 3296.2440366744995,
+ "_iteration": 34237,
+ "_has_errored": true,
+ "_fail_fast": false,
+ "_print_trial_errors": true,
+ "_cached_trial_decisions": {},
+ "_queued_trial_decisions": {},
+ "_should_stop_experiment": false,
+ "_stopper": {
+ "_type": "CLOUDPICKLE_FALLBACK",
+ "value": "8005952c000000000000008c157261792e74756e652e73746f707065722e6e6f6f70948c0b4e6f6f7053746f707065729493942981942e"
+ },
+ "_start_time": 1719324468.4160578,
+ "_last_checkpoint_time": -Infinity,
+ "_session_str": "2024-06-25_16-07-48",
+ "_checkpoint_period": "auto",
+ "_trial_checkpoint_config": {
+ "_type": "CLOUDPICKLE_FALLBACK",
+ "value": "800595f2000000000000008c097261792e747261696e948c10436865636b706f696e74436f6e6669679493942981947d94288c0b6e756d5f746f5f6b656570944e8c1a636865636b706f696e745f73636f72655f617474726962757465944e8c16636865636b706f696e745f73636f72655f6f72646572948c036d6178948c14636865636b706f696e745f6672657175656e6379944b008c11636865636b706f696e745f61745f656e6494898c1a5f636865636b706f696e745f6b6565705f616c6c5f72616e6b73948c0a44455052454341544544948c1f5f636865636b706f696e745f75706c6f61645f66726f6d5f776f726b65727394680c75622e"
+ },
+ "_resumed": false
+ },
+ "stats": {
+ "start_time": 1719324468.4160578,
+ "timestamp": -Infinity
+ }
+}
\ No newline at end of file
diff --git a/ray_results/test_experiment/experiment_state-2024-06-25_17-10-00.json b/ray_results/test_experiment/experiment_state-2024-06-25_17-10-00.json
new file mode 100644
index 0000000000000000000000000000000000000000..97e43a7772f0ed7f59f3a262876e78364ee38735
--- /dev/null
+++ b/ray_results/test_experiment/experiment_state-2024-06-25_17-10-00.json
@@ -0,0 +1,168 @@
+{
+ "trial_data": [
+ [
+ "{\n \"stub\": false,\n \"trainable_name\": \"train_model\",\n \"trial_id\": \"5763f130\",\n \"storage\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059582030000000000008c1b7261792e747261696e2e5f696e7465726e616c2e73746f72616765948c0e53746f72616765436f6e746578749493942981947d94288c12637573746f6d5f66735f70726f766964656494898c1273746f726167655f6c6f63616c5f70617468948c2b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c7473948c136578706572696d656e745f6469725f6e616d65948c0f746573745f6578706572696d656e74948c0e747269616c5f6469725f6e616d65948c96747261696e5f6d6f64656c5f35373633663133305f315f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d323034382c6465636f6465725f72745f6e756d5f6c617965723d382c64726f705f725f323032342d30362d32355f31372d31302d3030948c1863757272656e745f636865636b706f696e745f696e646578944b008c0b73796e635f636f6e666967948c097261792e747261696e948c0a53796e63436f6e6669679493942981947d94288c0b73796e635f706572696f64944d2c018c0c73796e635f74696d656f7574944d08078c0e73796e635f61727469666163747394898c1c73796e635f6172746966616374735f6f6e5f636865636b706f696e7494888c0a75706c6f61645f646972948c0a44455052454341544544948c0673796e6365729468188c1273796e635f6f6e5f636865636b706f696e7494681875628c1273746f726167655f66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c0f73746f726167655f66735f70617468948c452f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f7465737494681968008c115f46696c6573797374656d53796e6365729493942981947d9428681b682268134d2c0168144d08078c116c6173745f73796e635f75705f74696d659447fff00000000000008c136c6173745f73796e635f646f776e5f74696d659447fff00000000000008c0d5f73796e635f70726f63657373944e8c0c5f63757272656e745f636d64944e756275622e\"\n },\n \"config\": {\n \"encoder_num_layer\": 4,\n \"decoder_rt_num_layer\": 8,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 256,\n \"encoder_ff\": 512,\n \"decoder_rt_ff\": 2048,\n \"decoder_int_ff\": 512,\n \"n_head\": 2,\n \"drop_rate\": 0.7224904221599877,\n \"lr\": 0.04561252173503944,\n \"batch_size\": 2048\n },\n \"_Trial__unresolved_config\": {\n \"encoder_num_layer\": 4,\n \"decoder_rt_num_layer\": 8,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 256,\n \"encoder_ff\": 512,\n \"decoder_rt_ff\": 2048,\n \"decoder_int_ff\": 512,\n \"n_head\": 2,\n \"drop_rate\": 0.7224904221599877,\n \"lr\": 0.04561252173503944,\n \"batch_size\": 2048\n },\n \"evaluated_params\": {\n \"batch_size\": 2048,\n \"decoder_int_ff\": 512,\n \"decoder_int_num_layer\": 1,\n \"decoder_rt_ff\": 2048,\n \"decoder_rt_num_layer\": 8,\n \"drop_rate\": 0.7224904221599877,\n \"embedding_dim\": 256,\n \"encoder_ff\": 512,\n \"encoder_num_layer\": 4,\n \"lr\": 0.04561252173503944,\n \"n_head\": 2\n },\n \"experiment_tag\": \"1_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=8,drop_rate=0.7225,embedding_dim=256,encoder_ff=512,encoder_num_layer=4,lr=0.0456,n_head=2\",\n \"stopping_criterion\": {},\n \"_setup_default_resource\": true,\n \"_default_placement_group_factory\": \"80054e2e\",\n \"placement_group_factory\": \"800595ba000000000000008c237261792e74756e652e657865637574696f6e2e706c6163656d656e745f67726f757073948c15506c6163656d656e7447726f7570466163746f72799493942981947d94288c085f62756e646c6573945d947d94288c03435055944740540000000000008c034750559447401000000000000075618c155f686561645f62756e646c655f69735f656d70747994898c095f7374726174656779948c045041434b948c055f6172677394298c075f6b7761726773947d9475622e\",\n \"log_to_file\": [\n null,\n null\n ],\n \"max_failures\": 0,\n \"_default_result_or_future\": null,\n \"export_formats\": [],\n \"status\": \"PAUSED\",\n \"relative_logdir\": \"train_model_5763f130_1_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=8,drop_r_2024-06-25_17-10-00\",\n \"trial_name_creator\": null,\n \"trial_dirname_creator\": null,\n \"custom_trial_name\": null,\n \"custom_dirname\": null,\n \"restore_path\": null,\n \"_restore_checkpoint_result\": null,\n \"_state_json\": null,\n \"results\": \"80054e2e\",\n \"extra_arg\": \"80054e2e\",\n \"_resources\": \"80054e2e\"\n}",
+ "{\n \"start_time\": 1719328213.2508516,\n \"num_failures\": 0,\n \"num_failures_after_restore\": 0,\n \"error_filename\": null,\n \"pickled_error_filename\": null,\n \"last_result\": {\n \"loss\": 4538.754975086122,\n \"timestamp\": 1719328455,\n \"checkpoint_dir_name\": \"checkpoint_000000\",\n \"should_checkpoint\": true,\n \"done\": false,\n \"training_iteration\": 1,\n \"trial_id\": \"5763f130\",\n \"date\": \"2024-06-25_17-14-16\",\n \"time_this_iter_s\": 242.80822491645813,\n \"time_total_s\": 242.80822491645813,\n \"pid\": 2195253,\n \"hostname\": \"r8i6n2\",\n \"node_ip\": \"10.159.28.60\",\n \"config\": {\n \"encoder_num_layer\": 4,\n \"decoder_rt_num_layer\": 8,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 256,\n \"encoder_ff\": 512,\n \"decoder_rt_ff\": 2048,\n \"decoder_int_ff\": 512,\n \"n_head\": 2,\n \"drop_rate\": 0.7224904221599877,\n \"lr\": 0.04561252173503944,\n \"batch_size\": 2048\n },\n \"time_since_restore\": 242.80822491645813,\n \"iterations_since_restore\": 1,\n \"experiment_tag\": \"1_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=8,drop_rate=0.7225,embedding_dim=256,encoder_ff=512,encoder_num_layer=4,lr=0.0456,n_head=2\"\n },\n \"last_result_time\": 1719328456.063987,\n \"metric_analysis\": {\n \"loss\": {\n \"max\": 4538.754975086122,\n \"min\": 4538.754975086122,\n \"avg\": 4538.754975086122,\n \"last\": 4538.754975086122,\n \"last-5-avg\": 4538.754975086122,\n \"last-10-avg\": 4538.754975086122\n },\n \"should_checkpoint\": {\n \"max\": true,\n \"min\": true,\n \"avg\": true,\n \"last\": true,\n \"last-5-avg\": true,\n \"last-10-avg\": true\n },\n \"done\": {\n \"max\": false,\n \"min\": false,\n \"avg\": false,\n \"last\": false,\n \"last-5-avg\": false,\n \"last-10-avg\": false\n },\n \"training_iteration\": {\n \"max\": 1,\n \"min\": 1,\n \"avg\": 1,\n \"last\": 1,\n \"last-5-avg\": 1,\n \"last-10-avg\": 1\n },\n \"time_this_iter_s\": {\n \"max\": 242.80822491645813,\n \"min\": 242.80822491645813,\n \"avg\": 242.80822491645813,\n \"last\": 242.80822491645813,\n \"last-5-avg\": 242.80822491645813,\n \"last-10-avg\": 242.80822491645813\n },\n \"time_total_s\": {\n \"max\": 242.80822491645813,\n \"min\": 242.80822491645813,\n \"avg\": 242.80822491645813,\n \"last\": 242.80822491645813,\n \"last-5-avg\": 242.80822491645813,\n \"last-10-avg\": 242.80822491645813\n },\n \"time_since_restore\": {\n \"max\": 242.80822491645813,\n \"min\": 242.80822491645813,\n \"avg\": 242.80822491645813,\n \"last\": 242.80822491645813,\n \"last-5-avg\": 242.80822491645813,\n \"last-10-avg\": 242.80822491645813\n },\n \"iterations_since_restore\": {\n \"max\": 1,\n \"min\": 1,\n \"avg\": 1,\n \"last\": 1,\n \"last-5-avg\": 1,\n \"last-10-avg\": 1\n }\n },\n \"_n_steps\": [\n 5,\n 10\n ],\n \"metric_n_steps\": {\n \"loss\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059589000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452948c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b00749462430830180c46c1bab1409486945294612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059589000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452948c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b00749462430830180c46c1bab1409486945294612e\"\n }\n },\n \"should_checkpoint\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059522000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b058694529488612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059522000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a8694529488612e\"\n }\n },\n \"done\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059522000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b058694529489612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059522000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a8694529489612e\"\n }\n },\n \"training_iteration\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059523000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452944b01612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059523000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452944b01612e\"\n }\n },\n \"time_this_iter_s\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b058694529447406e59dcfa800000612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a8694529447406e59dcfa800000612e\"\n }\n },\n \"time_total_s\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b058694529447406e59dcfa800000612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a8694529447406e59dcfa800000612e\"\n }\n },\n \"time_since_restore\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b058694529447406e59dcfa800000612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a8694529447406e59dcfa800000612e\"\n }\n },\n \"iterations_since_restore\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059523000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452944b01612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059523000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452944b01612e\"\n }\n }\n },\n \"checkpoint_manager\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059518060000000000008c267261792e747261696e2e5f696e7465726e616c2e636865636b706f696e745f6d616e61676572948c125f436865636b706f696e744d616e616765729493942981947d94288c125f636865636b706f696e745f636f6e666967948c097261792e747261696e948c10436865636b706f696e74436f6e6669679493942981947d94288c0b6e756d5f746f5f6b656570944e8c1a636865636b706f696e745f73636f72655f617474726962757465944e8c16636865636b706f696e745f73636f72655f6f72646572948c036d6178948c14636865636b706f696e745f6672657175656e6379944b008c11636865636b706f696e745f61745f656e6494898c1a5f636865636b706f696e745f6b6565705f616c6c5f72616e6b73948c0a44455052454341544544948c1f5f636865636b706f696e745f75706c6f61645f66726f6d5f776f726b65727394681275628c135f636865636b706f696e745f726573756c7473945d948c1b7261792e747261696e2e5f696e7465726e616c2e73657373696f6e948c0f5f547261696e696e67526573756c749493942981947d94288c0a636865636b706f696e749468068c0a436865636b706f696e749493942981947d94288c0470617468948cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f35373633663133305f315f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d323034382c6465636f6465725f72745f6e756d5f6c617965723d382c64726f705f725f323032342d30362d32355f31372d31302d30302f636865636b706f696e745f303030303030948c0a66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c055f75756964948c0475756964948c04555549449493942981947d948c03696e74948a10b0dd19201197f187aa4e48898e873e6f736275628c076d657472696373947d94288c046c6f7373948c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b00749462430830180c46c1bab14094869452948c0974696d657374616d70944ac7de7a668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303030948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b018c08747269616c5f6964948c083537363366313330948c0464617465948c13323032342d30362d32355f31372d31342d3136948c1074696d655f746869735f697465725f739447406e59dcfa8000008c0c74696d655f746f74616c5f739447406e59dcfa8000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b048c146465636f6465725f72745f6e756d5f6c61796572944b088c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944d00018c0a656e636f6465725f6666944d00028c0d6465636f6465725f72745f6666944d00088c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b028c0964726f705f7261746594473fe71ea43bdb36528c026c7294473fa75a864247b9098c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f72659447406e59dcfa8000008c18697465726174696f6e735f73696e63655f726573746f7265944b01757562618c195f6c61746573745f636865636b706f696e745f726573756c7494681975622e\"\n }\n}"
+ ],
+ [
+ "{\n \"stub\": false,\n \"trainable_name\": \"train_model\",\n \"trial_id\": \"a00d3259\",\n \"storage\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059572040000000000008c1b7261792e747261696e2e5f696e7465726e616c2e73746f72616765948c0e53746f72616765436f6e746578749493942981947d94288c12637573746f6d5f66735f70726f766964656494898c1273746f726167655f6c6f63616c5f70617468948c2b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c7473948c136578706572696d656e745f6469725f6e616d65948c0f746573745f6578706572696d656e74948c0e747269616c5f6469725f6e616d65948c96747261696e5f6d6f64656c5f61303064333235395f325f62617463685f73697a653d313032342c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d3531322c6465636f6465725f72745f6e756d5f6c617965723d342c64726f705f72615f323032342d30362d32355f31372d31302d3133948c1863757272656e745f636865636b706f696e745f696e646578944b078c0b73796e635f636f6e666967948c097261792e747261696e948c0a53796e63436f6e6669679493942981947d94288c0b73796e635f706572696f64944d2c018c0c73796e635f74696d656f7574944d08078c0e73796e635f61727469666163747394898c1c73796e635f6172746966616374735f6f6e5f636865636b706f696e7494888c0a75706c6f61645f646972948c0a44455052454341544544948c0673796e6365729468188c1273796e635f6f6e5f636865636b706f696e7494681875628c1273746f726167655f66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c0f73746f726167655f66735f70617468948c452f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f7465737494681968008c115f46696c6573797374656d53796e6365729493942981947d9428681b682268134d2c0168144d08078c116c6173745f73796e635f75705f74696d65944741d99eb7722637668c136c6173745f73796e635f646f776e5f74696d659447fff00000000000008c0d5f73796e635f70726f63657373944e8c0c5f63757272656e745f636d649468008c125f75706c6f61645f746f5f66735f706174689493947d94288c0a6c6f63616c5f70617468948c3b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c74732f746573745f6578706572696d656e74948c0266739468228c0766735f70617468948c552f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e74948c076578636c756465945d948c0e2a2f636865636b706f696e745f2a9461758694756275622e\"\n },\n \"config\": {\n \"encoder_num_layer\": 1,\n \"decoder_rt_num_layer\": 4,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 16,\n \"encoder_ff\": 1024,\n \"decoder_rt_ff\": 512,\n \"decoder_int_ff\": 512,\n \"n_head\": 4,\n \"drop_rate\": 0.22544213936627444,\n \"lr\": 0.0013157536168827595,\n \"batch_size\": 1024\n },\n \"_Trial__unresolved_config\": {\n \"encoder_num_layer\": 1,\n \"decoder_rt_num_layer\": 4,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 16,\n \"encoder_ff\": 1024,\n \"decoder_rt_ff\": 512,\n \"decoder_int_ff\": 512,\n \"n_head\": 4,\n \"drop_rate\": 0.22544213936627444,\n \"lr\": 0.0013157536168827595,\n \"batch_size\": 1024\n },\n \"evaluated_params\": {\n \"batch_size\": 1024,\n \"decoder_int_ff\": 512,\n \"decoder_int_num_layer\": 1,\n \"decoder_rt_ff\": 512,\n \"decoder_rt_num_layer\": 4,\n \"drop_rate\": 0.22544213936627444,\n \"embedding_dim\": 16,\n \"encoder_ff\": 1024,\n \"encoder_num_layer\": 1,\n \"lr\": 0.0013157536168827595,\n \"n_head\": 4\n },\n \"experiment_tag\": \"2_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=4,drop_rate=0.2254,embedding_dim=16,encoder_ff=1024,encoder_num_layer=1,lr=0.0013,n_head=4\",\n \"stopping_criterion\": {},\n \"_setup_default_resource\": true,\n \"_default_placement_group_factory\": \"80054e2e\",\n \"placement_group_factory\": \"800595ba000000000000008c237261792e74756e652e657865637574696f6e2e706c6163656d656e745f67726f757073948c15506c6163656d656e7447726f7570466163746f72799493942981947d94288c085f62756e646c6573945d947d94288c03435055944740540000000000008c034750559447401000000000000075618c155f686561645f62756e646c655f69735f656d70747994898c095f7374726174656779948c045041434b948c055f6172677394298c075f6b7761726773947d9475622e\",\n \"log_to_file\": [\n null,\n null\n ],\n \"max_failures\": 0,\n \"_default_result_or_future\": null,\n \"export_formats\": [],\n \"status\": \"PAUSED\",\n \"relative_logdir\": \"train_model_a00d3259_2_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=4,drop_ra_2024-06-25_17-10-13\",\n \"trial_name_creator\": null,\n \"trial_dirname_creator\": null,\n \"custom_trial_name\": null,\n \"custom_dirname\": null,\n \"restore_path\": null,\n \"_restore_checkpoint_result\": null,\n \"_state_json\": null,\n \"results\": \"80054e2e\",\n \"extra_arg\": \"80054e2e\",\n \"_resources\": \"80054e2e\"\n}",
+ "{\n \"start_time\": 1719328456.2145321,\n \"num_failures\": 0,\n \"num_failures_after_restore\": 0,\n \"error_filename\": null,\n \"pickled_error_filename\": null,\n \"last_result\": {\n \"loss\": 119.19234625748761,\n \"timestamp\": 1719332071,\n \"checkpoint_dir_name\": \"checkpoint_000007\",\n \"should_checkpoint\": true,\n \"done\": false,\n \"training_iteration\": 8,\n \"trial_id\": \"a00d3259\",\n \"date\": \"2024-06-25_18-14-31\",\n \"time_this_iter_s\": 58.22098970413208,\n \"time_total_s\": 537.5048625469208,\n \"pid\": 2195253,\n \"hostname\": \"r8i6n2\",\n \"node_ip\": \"10.159.28.60\",\n \"config\": {\n \"encoder_num_layer\": 1,\n \"decoder_rt_num_layer\": 4,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 16,\n \"encoder_ff\": 1024,\n \"decoder_rt_ff\": 512,\n \"decoder_int_ff\": 512,\n \"n_head\": 4,\n \"drop_rate\": 0.22544213936627444,\n \"lr\": 0.0013157536168827595,\n \"batch_size\": 1024\n },\n \"time_since_restore\": 246.92507433891296,\n \"iterations_since_restore\": 4,\n \"experiment_tag\": \"2_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=4,drop_rate=0.2254,embedding_dim=16,encoder_ff=1024,encoder_num_layer=1,lr=0.0013,n_head=4\"\n },\n \"last_result_time\": 1719332071.4645963,\n \"metric_analysis\": {\n \"loss\": {\n \"max\": 253.8278117743064,\n \"min\": 119.19234625748761,\n \"avg\": 155.9507788898438,\n \"last\": 119.19234625748761,\n \"last-5-avg\": 133.0517245735709,\n \"last-10-avg\": 155.9507788898438\n },\n \"should_checkpoint\": {\n \"max\": true,\n \"min\": true,\n \"avg\": 1.0,\n \"last\": true,\n \"last-5-avg\": 1.0,\n \"last-10-avg\": 1.0\n },\n \"done\": {\n \"max\": false,\n \"min\": false,\n \"avg\": 0.0,\n \"last\": false,\n \"last-5-avg\": 0.0,\n \"last-10-avg\": 0.0\n },\n \"training_iteration\": {\n \"max\": 8,\n \"min\": 1,\n \"avg\": 4.5,\n \"last\": 8,\n \"last-5-avg\": 6.0,\n \"last-10-avg\": 4.5\n },\n \"time_this_iter_s\": {\n \"max\": 88.16569352149963,\n \"min\": 58.202213287353516,\n \"avg\": 67.1881078183651,\n \"last\": 58.22098970413208,\n \"last-5-avg\": 61.9065589427948,\n \"last-10-avg\": 67.1881078183651\n },\n \"time_total_s\": {\n \"max\": 537.5048625469208,\n \"min\": 70.23847699165344,\n \"avg\": 316.16187810897827,\n \"last\": 537.5048625469208,\n \"last-5-avg\": 418.25562114715575,\n \"last-10-avg\": 316.16187810897827\n },\n \"time_since_restore\": {\n \"max\": 246.92507433891296,\n \"min\": 69.5678973197937,\n \"avg\": 127.14058080315588,\n \"last\": 246.92507433891296,\n \"last-5-avg\": 157.83051571846008,\n \"last-10-avg\": 127.1405808031559\n },\n \"iterations_since_restore\": {\n \"max\": 4,\n \"min\": 1,\n \"avg\": 1.875,\n \"last\": 4,\n \"last-5-avg\": 2.4,\n \"last-10-avg\": 1.875\n }\n },\n \"_n_steps\": [\n 5,\n 10\n ],\n \"metric_n_steps\": {\n \"loss\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"800595d6000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294288c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b0074946243080c068339a894614094869452946807680d4308bbdd6e5f00c0624094869452946807680d4308000000b8f85f604094869452946807680d430873b95c3efb1a5f4094869452946807680d4308b55aad664fcc5d409486945294652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005950f010000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294288c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b007494624308793c1e6f7dba6f4094869452946807680d430883c160c03b16654094869452946807680d43085fafd76b66fa634094869452946807680d43080c068339a894614094869452946807680d4308bbdd6e5f00c0624094869452946807680d4308000000b8f85f604094869452946807680d430873b95c3efb1a5f4094869452946807680d4308b55aad664fcc5d409486945294652e\"\n }\n },\n \"should_checkpoint\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059527000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294288888888888652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294288888888888888888652e\"\n }\n },\n \"done\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059527000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294288989898989652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294288989898989898989652e\"\n }\n },\n \"training_iteration\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952c000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294284b044b054b064b074b08652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059532000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294284b014b024b034b044b054b064b074b08652e\"\n }\n },\n \"time_this_iter_s\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005954f000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452942847404f4dc9c800000047405211878400000047404d19e22000000047404d1d2e4a00000047404d1c4964000000652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005956a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294284740518f433500000047405164586e0000004740560a9ab900000047404f4dc9c800000047405211878400000047404d19e22000000047404d1d2e4a00000047404d1c4964000000652e\"\n }\n },\n \"time_total_s\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005954f000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294284740722946d0000000474076ada8b100000047407a50e4f500000047407df48abe400000474080cc09f5600000652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005956a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294284740518f433500000047406179cdd180000047406c7f1b2e0000004740722946d0000000474076ada8b100000047407a50e4f500000047407df48abe400000474080cc09f5600000652e\"\n }\n },\n \"time_since_restore\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005954f000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b058694529428474062d8bfce8000004740521187840000004740604f3c4a0000004740679687dc80000047406edd9a35800000652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005956a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294284740518f433500000047405164586e0000004740560a9ab9000000474062d8bfce8000004740521187840000004740604f3c4a0000004740679687dc80000047406edd9a35800000652e\"\n }\n },\n \"iterations_since_restore\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952c000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294284b024b014b024b034b04652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059532000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294284b014b014b014b024b014b024b034b04652e\"\n }\n }\n },\n \"checkpoint_manager\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"800595f41e0000000000008c267261792e747261696e2e5f696e7465726e616c2e636865636b706f696e745f6d616e61676572948c125f436865636b706f696e744d616e616765729493942981947d94288c125f636865636b706f696e745f636f6e666967948c097261792e747261696e948c10436865636b706f696e74436f6e6669679493942981947d94288c0b6e756d5f746f5f6b656570944e8c1a636865636b706f696e745f73636f72655f617474726962757465944e8c16636865636b706f696e745f73636f72655f6f72646572948c036d6178948c14636865636b706f696e745f6672657175656e6379944b008c11636865636b706f696e745f61745f656e6494898c1a5f636865636b706f696e745f6b6565705f616c6c5f72616e6b73948c0a44455052454341544544948c1f5f636865636b706f696e745f75706c6f61645f66726f6d5f776f726b65727394681275628c135f636865636b706f696e745f726573756c7473945d94288c1b7261792e747261696e2e5f696e7465726e616c2e73657373696f6e948c0f5f547261696e696e67526573756c749493942981947d94288c0a636865636b706f696e749468068c0a436865636b706f696e749493942981947d94288c0470617468948cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f61303064333235395f325f62617463685f73697a653d313032342c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d3531322c6465636f6465725f72745f6e756d5f6c617965723d342c64726f705f72615f323032342d30362d32355f31372d31302d31332f636865636b706f696e745f303030303030948c0a66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c055f75756964948c0475756964948c04555549449493942981947d948c03696e74948a110e9e5dc1217ec9bebd476cec37a884c400736275628c076d657472696373947d94288c046c6f7373948c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b007494624308793c1e6f7dba6f4094869452948c0974696d657374616d70944a0edf7a668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303030948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b018c08747269616c5f6964948c086130306433323539948c0464617465948c13323032342d30362d32355f31372d31352d3236948c1074696d655f746869735f697465725f73944740518f43350000008c0c74696d655f746f74616c5f73944740518f43350000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b018c146465636f6465725f72745f6e756d5f6c61796572944b048c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b108c0a656e636f6465725f6666944d00048c0d6465636f6465725f72745f6666944d00028c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b048c0964726f705f7261746594473fccdb49bbdbf5488c026c7294473f558eabb04342508c0a62617463685f73697a65944d0004758c1274696d655f73696e63655f726573746f7265944740518f43350000008c18697465726174696f6e735f73696e63655f726573746f7265944b0175756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f61303064333235395f325f62617463685f73697a653d313032342c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d3531322c6465636f6465725f72745f6e756d5f6c617965723d342c64726f705f72615f323032342d30362d32355f31372d31302d31332f636865636b706f696e745f30303030303194682268257d946827897385945294682a682d2981947d9468308a11cc42bb71e00b7b97494800e3d3acd9b2007362756268317d94288c046c6f7373946836683c430883c160c03b16654094869452948c0974696d657374616d70944a1be77a668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303031948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b028c08747269616c5f6964948c086130306433323539948c0464617465948c13323032342d30362d32355f31372d34392d3437948c1074696d655f746869735f697465725f739447405164586e0000008c0c74696d655f746f74616c5f739447406179cdd18000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b018c146465636f6465725f72745f6e756d5f6c61796572944b048c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b108c0a656e636f6465725f6666944d00048c0d6465636f6465725f72745f6666944d00028c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b048c0964726f705f7261746594473fccdb49bbdbf5488c026c7294473f558eabb04342508c0a62617463685f73697a65944d0004758c1274696d655f73696e63655f726573746f72659447405164586e0000008c18697465726174696f6e735f73696e63655f726573746f7265944b0175756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f61303064333235395f325f62617463685f73697a653d313032342c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d3531322c6465636f6465725f72745f6e756d5f6c617965723d342c64726f705f72615f323032342d30362d32355f31372d31302d31332f636865636b706f696e745f30303030303294682268257d946827897385945294682a682d2981947d9468308a10141deb64e2f047a7ca4b06bf191963437362756268317d94288c046c6f7373946836683c43085fafd76b66fa634094869452948c0974696d657374616d70944a93e97a668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303032948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b038c08747269616c5f6964948c086130306433323539948c0464617465948c13323032342d30362d32355f31382d30302d3139948c1074696d655f746869735f697465725f73944740560a9ab90000008c0c74696d655f746f74616c5f739447406c7f1b2e0000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b018c146465636f6465725f72745f6e756d5f6c61796572944b048c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b108c0a656e636f6465725f6666944d00048c0d6465636f6465725f72745f6666944d00028c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b048c0964726f705f7261746594473fccdb49bbdbf5488c026c7294473f558eabb04342508c0a62617463685f73697a65944d0004758c1274696d655f73696e63655f726573746f7265944740560a9ab90000008c18697465726174696f6e735f73696e63655f726573746f7265944b0175756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f61303064333235395f325f62617463685f73697a653d313032342c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d3531322c6465636f6465725f72745f6e756d5f6c617965723d342c64726f705f72615f323032342d30362d32355f31372d31302d31332f636865636b706f696e745f30303030303394682268257d946827897385945294682a682d2981947d9468308a119f5735580a8fcb92b74321bfa361c4b9007362756268317d94288c046c6f7373946836683c43080c068339a894614094869452948c0974696d657374616d70944ad2e97a668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303033948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b048c08747269616c5f6964948c086130306433323539948c0464617465948c13323032342d30362d32355f31382d30312d3232948c1074696d655f746869735f697465725f739447404f4dc9c80000008c0c74696d655f746f74616c5f73944740722946d00000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b018c146465636f6465725f72745f6e756d5f6c61796572944b048c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b108c0a656e636f6465725f6666944d00048c0d6465636f6465725f72745f6666944d00028c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b048c0964726f705f7261746594473fccdb49bbdbf5488c026c7294473f558eabb04342508c0a62617463685f73697a65944d0004758c1274696d655f73696e63655f726573746f726594474062d8bfce8000008c18697465726174696f6e735f73696e63655f726573746f7265944b0275756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f61303064333235395f325f62617463685f73697a653d313032342c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d3531322c6465636f6465725f72745f6e756d5f6c617965723d342c64726f705f72615f323032342d30362d32355f31372d31302d31332f636865636b706f696e745f30303030303494682268257d946827897385945294682a682d2981947d9468308a1178391f217b7e0e8b8e484fe617cc4aa7007362756268317d94288c046c6f7373946836683c4308bbdd6e5f00c0624094869452948c0974696d657374616d70944a38ec7a668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303034948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b058c08747269616c5f6964948c086130306433323539948c0464617465948c13323032342d30362d32355f31382d31312d3336948c1074696d655f746869735f697465725f73944740521187840000008c0c74696d655f746f74616c5f7394474076ada8b10000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b018c146465636f6465725f72745f6e756d5f6c61796572944b048c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b108c0a656e636f6465725f6666944d00048c0d6465636f6465725f72745f6666944d00028c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b048c0964726f705f7261746594473fccdb49bbdbf5488c026c7294473f558eabb04342508c0a62617463685f73697a65944d0004758c1274696d655f73696e63655f726573746f7265944740521187840000008c18697465726174696f6e735f73696e63655f726573746f7265944b0175756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f61303064333235395f325f62617463685f73697a653d313032342c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d3531322c6465636f6465725f72745f6e756d5f6c617965723d342c64726f705f72615f323032342d30362d32355f31372d31302d31332f636865636b706f696e745f30303030303594682268257d946827897385945294682a682d2981947d9468308a11b01bc90fcfb3849519437ed644c87283007362756268317d94288c046c6f7373946836683c4308000000b8f85f604094869452948c0974696d657374616d70944a72ec7a668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303035948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b068c08747269616c5f6964948c086130306433323539948c0464617465948c13323032342d30362d32355f31382d31322d3334948c1074696d655f746869735f697465725f739447404d19e2200000008c0c74696d655f746f74616c5f739447407a50e4f50000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b018c146465636f6465725f72745f6e756d5f6c61796572944b048c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b108c0a656e636f6465725f6666944d00048c0d6465636f6465725f72745f6666944d00028c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b048c0964726f705f7261746594473fccdb49bbdbf5488c026c7294473f558eabb04342508c0a62617463685f73697a65944d0004758c1274696d655f73696e63655f726573746f7265944740604f3c4a0000008c18697465726174696f6e735f73696e63655f726573746f7265944b0275756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f61303064333235395f325f62617463685f73697a653d313032342c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d3531322c6465636f6465725f72745f6e756d5f6c617965723d342c64726f705f72615f323032342d30362d32355f31372d31302d31332f636865636b706f696e745f30303030303694682268257d946827897385945294682a682d2981947d9468308a100e2bc27ee9b7d8baec4b607719848f1a7362756268317d94288c046c6f7373946836683c430873b95c3efb1a5f4094869452948c0974696d657374616d70944aadec7a668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303036948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b078c08747269616c5f6964948c086130306433323539948c0464617465948c13323032342d30362d32355f31382d31332d3333948c1074696d655f746869735f697465725f739447404d1d2e4a0000008c0c74696d655f746f74616c5f739447407df48abe4000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b018c146465636f6465725f72745f6e756d5f6c61796572944b048c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b108c0a656e636f6465725f6666944d00048c0d6465636f6465725f72745f6666944d00028c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b048c0964726f705f7261746594473fccdb49bbdbf5488c026c7294473f558eabb04342508c0a62617463685f73697a65944d0004758c1274696d655f73696e63655f726573746f7265944740679687dc8000008c18697465726174696f6e735f73696e63655f726573746f7265944b0375756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f61303064333235395f325f62617463685f73697a653d313032342c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d3531322c6465636f6465725f72745f6e756d5f6c617965723d342c64726f705f72615f323032342d30362d32355f31372d31302d31332f636865636b706f696e745f30303030303794682268257d946827897385945294682a682d2981947d9468308a10ede47fd5253ab29d7647405ee5896a327362756268317d94288c046c6f7373946836683c4308b55aad664fcc5d4094869452948c0974696d657374616d70944ae7ec7a668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303037948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b088c08747269616c5f6964948c086130306433323539948c0464617465948c13323032342d30362d32355f31382d31342d3331948c1074696d655f746869735f697465725f739447404d1c49640000008c0c74696d655f746f74616c5f7394474080cc09f56000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b018c146465636f6465725f72745f6e756d5f6c61796572944b048c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b108c0a656e636f6465725f6666944d00048c0d6465636f6465725f72745f6666944d00028c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b048c0964726f705f7261746594473fccdb49bbdbf5488c026c7294473f558eabb04342508c0a62617463685f73697a65944d0004758c1274696d655f73696e63655f726573746f72659447406edd9a358000008c18697465726174696f6e735f73696e63655f726573746f7265944b04757562658c195f6c61746573745f636865636b706f696e745f726573756c74946a7c01000075622e\"\n }\n}"
+ ],
+ [
+ "{\n \"stub\": false,\n \"trainable_name\": \"train_model\",\n \"trial_id\": \"7ddcd025\",\n \"storage\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059572040000000000008c1b7261792e747261696e2e5f696e7465726e616c2e73746f72616765948c0e53746f72616765436f6e746578749493942981947d94288c12637573746f6d5f66735f70726f766964656494898c1273746f726167655f6c6f63616c5f70617468948c2b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c7473948c136578706572696d656e745f6469725f6e616d65948c0f746573745f6578706572696d656e74948c0e747269616c5f6469725f6e616d65948c96747261696e5f6d6f64656c5f37646463643032355f335f62617463685f73697a653d313032342c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d3531322c6465636f6465725f72745f6e756d5f6c617965723d382c64726f705f72615f323032342d30362d32355f31372d31342d3136948c1863757272656e745f636865636b706f696e745f696e646578944b008c0b73796e635f636f6e666967948c097261792e747261696e948c0a53796e63436f6e6669679493942981947d94288c0b73796e635f706572696f64944d2c018c0c73796e635f74696d656f7574944d08078c0e73796e635f61727469666163747394898c1c73796e635f6172746966616374735f6f6e5f636865636b706f696e7494888c0a75706c6f61645f646972948c0a44455052454341544544948c0673796e6365729468188c1273796e635f6f6e5f636865636b706f696e7494681875628c1273746f726167655f66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c0f73746f726167655f66735f70617468948c452f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f7465737494681968008c115f46696c6573797374656d53796e6365729493942981947d9428681b682268134d2c0168144d08078c116c6173745f73796e635f75705f74696d65944741d99eb7722637668c136c6173745f73796e635f646f776e5f74696d659447fff00000000000008c0d5f73796e635f70726f63657373944e8c0c5f63757272656e745f636d649468008c125f75706c6f61645f746f5f66735f706174689493947d94288c0a6c6f63616c5f70617468948c3b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c74732f746573745f6578706572696d656e74948c0266739468228c0766735f70617468948c552f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e74948c076578636c756465945d948c0e2a2f636865636b706f696e745f2a9461758694756275622e\"\n },\n \"config\": {\n \"encoder_num_layer\": 2,\n \"decoder_rt_num_layer\": 8,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 64,\n \"encoder_ff\": 2048,\n \"decoder_rt_ff\": 512,\n \"decoder_int_ff\": 512,\n \"n_head\": 1,\n \"drop_rate\": 0.474834603771515,\n \"lr\": 0.015369204694575145,\n \"batch_size\": 1024\n },\n \"_Trial__unresolved_config\": {\n \"encoder_num_layer\": 2,\n \"decoder_rt_num_layer\": 8,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 64,\n \"encoder_ff\": 2048,\n \"decoder_rt_ff\": 512,\n \"decoder_int_ff\": 512,\n \"n_head\": 1,\n \"drop_rate\": 0.474834603771515,\n \"lr\": 0.015369204694575145,\n \"batch_size\": 1024\n },\n \"evaluated_params\": {\n \"batch_size\": 1024,\n \"decoder_int_ff\": 512,\n \"decoder_int_num_layer\": 1,\n \"decoder_rt_ff\": 512,\n \"decoder_rt_num_layer\": 8,\n \"drop_rate\": 0.474834603771515,\n \"embedding_dim\": 64,\n \"encoder_ff\": 2048,\n \"encoder_num_layer\": 2,\n \"lr\": 0.015369204694575145,\n \"n_head\": 1\n },\n \"experiment_tag\": \"3_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=8,drop_rate=0.4748,embedding_dim=64,encoder_ff=2048,encoder_num_layer=2,lr=0.0154,n_head=1\",\n \"stopping_criterion\": {},\n \"_setup_default_resource\": true,\n \"_default_placement_group_factory\": \"80054e2e\",\n \"placement_group_factory\": \"800595ba000000000000008c237261792e74756e652e657865637574696f6e2e706c6163656d656e745f67726f757073948c15506c6163656d656e7447726f7570466163746f72799493942981947d94288c085f62756e646c6573945d947d94288c03435055944740540000000000008c034750559447401000000000000075618c155f686561645f62756e646c655f69735f656d70747994898c095f7374726174656779948c045041434b948c055f6172677394298c075f6b7761726773947d9475622e\",\n \"log_to_file\": [\n null,\n null\n ],\n \"max_failures\": 0,\n \"_default_result_or_future\": null,\n \"export_formats\": [],\n \"status\": \"PAUSED\",\n \"relative_logdir\": \"train_model_7ddcd025_3_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=8,drop_ra_2024-06-25_17-14-16\",\n \"trial_name_creator\": null,\n \"trial_dirname_creator\": null,\n \"custom_trial_name\": null,\n \"custom_dirname\": null,\n \"restore_path\": null,\n \"_restore_checkpoint_result\": null,\n \"_state_json\": null,\n \"results\": \"80054e2e\",\n \"extra_arg\": \"80054e2e\",\n \"_resources\": \"80054e2e\"\n}",
+ "{\n \"start_time\": 1719328526.6072588,\n \"num_failures\": 0,\n \"num_failures_after_restore\": 0,\n \"error_filename\": null,\n \"pickled_error_filename\": null,\n \"last_result\": {\n \"loss\": 2079.0407700425994,\n \"timestamp\": 1719328617,\n \"checkpoint_dir_name\": \"checkpoint_000000\",\n \"should_checkpoint\": true,\n \"done\": false,\n \"training_iteration\": 1,\n \"trial_id\": \"7ddcd025\",\n \"date\": \"2024-06-25_17-16-57\",\n \"time_this_iter_s\": 90.59219408035278,\n \"time_total_s\": 90.59219408035278,\n \"pid\": 2195253,\n \"hostname\": \"r8i6n2\",\n \"node_ip\": \"10.159.28.60\",\n \"config\": {\n \"encoder_num_layer\": 2,\n \"decoder_rt_num_layer\": 8,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 64,\n \"encoder_ff\": 2048,\n \"decoder_rt_ff\": 512,\n \"decoder_int_ff\": 512,\n \"n_head\": 1,\n \"drop_rate\": 0.474834603771515,\n \"lr\": 0.015369204694575145,\n \"batch_size\": 1024\n },\n \"time_since_restore\": 90.59219408035278,\n \"iterations_since_restore\": 1,\n \"experiment_tag\": \"3_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=8,drop_rate=0.4748,embedding_dim=64,encoder_ff=2048,encoder_num_layer=2,lr=0.0154,n_head=1\"\n },\n \"last_result_time\": 1719328617.2057276,\n \"metric_analysis\": {\n \"loss\": {\n \"max\": 2079.0407700425994,\n \"min\": 2079.0407700425994,\n \"avg\": 2079.0407700425994,\n \"last\": 2079.0407700425994,\n \"last-5-avg\": 2079.0407700425994,\n \"last-10-avg\": 2079.0407700425994\n },\n \"should_checkpoint\": {\n \"max\": true,\n \"min\": true,\n \"avg\": true,\n \"last\": true,\n \"last-5-avg\": true,\n \"last-10-avg\": true\n },\n \"done\": {\n \"max\": false,\n \"min\": false,\n \"avg\": false,\n \"last\": false,\n \"last-5-avg\": false,\n \"last-10-avg\": false\n },\n \"training_iteration\": {\n \"max\": 1,\n \"min\": 1,\n \"avg\": 1,\n \"last\": 1,\n \"last-5-avg\": 1,\n \"last-10-avg\": 1\n },\n \"time_this_iter_s\": {\n \"max\": 90.59219408035278,\n \"min\": 90.59219408035278,\n \"avg\": 90.59219408035278,\n \"last\": 90.59219408035278,\n \"last-5-avg\": 90.59219408035278,\n \"last-10-avg\": 90.59219408035278\n },\n \"time_total_s\": {\n \"max\": 90.59219408035278,\n \"min\": 90.59219408035278,\n \"avg\": 90.59219408035278,\n \"last\": 90.59219408035278,\n \"last-5-avg\": 90.59219408035278,\n \"last-10-avg\": 90.59219408035278\n },\n \"time_since_restore\": {\n \"max\": 90.59219408035278,\n \"min\": 90.59219408035278,\n \"avg\": 90.59219408035278,\n \"last\": 90.59219408035278,\n \"last-5-avg\": 90.59219408035278,\n \"last-10-avg\": 90.59219408035278\n },\n \"iterations_since_restore\": {\n \"max\": 1,\n \"min\": 1,\n \"avg\": 1,\n \"last\": 1,\n \"last-5-avg\": 1,\n \"last-10-avg\": 1\n }\n },\n \"_n_steps\": [\n 5,\n 10\n ],\n \"metric_n_steps\": {\n \"loss\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059589000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452948c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b0074946243083e9fcfdf143ea0409486945294612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059589000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452948c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b0074946243083e9fcfdf143ea0409486945294612e\"\n }\n },\n \"should_checkpoint\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059522000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b058694529488612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059522000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a8694529488612e\"\n }\n },\n \"done\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059522000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b058694529489612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059522000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a8694529489612e\"\n }\n },\n \"training_iteration\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059523000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452944b01612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059523000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452944b01612e\"\n }\n },\n \"time_this_iter_s\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294474056a5e682000000612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294474056a5e682000000612e\"\n }\n },\n \"time_total_s\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294474056a5e682000000612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294474056a5e682000000612e\"\n }\n },\n \"time_since_restore\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294474056a5e682000000612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294474056a5e682000000612e\"\n }\n },\n \"iterations_since_restore\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059523000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452944b01612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059523000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452944b01612e\"\n }\n }\n },\n \"checkpoint_manager\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059517060000000000008c267261792e747261696e2e5f696e7465726e616c2e636865636b706f696e745f6d616e61676572948c125f436865636b706f696e744d616e616765729493942981947d94288c125f636865636b706f696e745f636f6e666967948c097261792e747261696e948c10436865636b706f696e74436f6e6669679493942981947d94288c0b6e756d5f746f5f6b656570944e8c1a636865636b706f696e745f73636f72655f617474726962757465944e8c16636865636b706f696e745f73636f72655f6f72646572948c036d6178948c14636865636b706f696e745f6672657175656e6379944b008c11636865636b706f696e745f61745f656e6494898c1a5f636865636b706f696e745f6b6565705f616c6c5f72616e6b73948c0a44455052454341544544948c1f5f636865636b706f696e745f75706c6f61645f66726f6d5f776f726b65727394681275628c135f636865636b706f696e745f726573756c7473945d948c1b7261792e747261696e2e5f696e7465726e616c2e73657373696f6e948c0f5f547261696e696e67526573756c749493942981947d94288c0a636865636b706f696e749468068c0a436865636b706f696e749493942981947d94288c0470617468948cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f37646463643032355f335f62617463685f73697a653d313032342c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d3531322c6465636f6465725f72745f6e756d5f6c617965723d382c64726f705f72615f323032342d30362d32355f31372d31342d31362f636865636b706f696e745f303030303030948c0a66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c055f75756964948c0475756964948c04555549449493942981947d948c03696e74948a105ac6f07c076b1b94404e03bf2719e871736275628c076d657472696373947d94288c046c6f7373948c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b0074946243083e9fcfdf143ea04094869452948c0974696d657374616d70944a69df7a668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303030948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b018c08747269616c5f6964948c083764646364303235948c0464617465948c13323032342d30362d32355f31372d31362d3537948c1074696d655f746869735f697465725f7394474056a5e6820000008c0c74696d655f746f74616c5f7394474056a5e6820000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b028c146465636f6465725f72745f6e756d5f6c61796572944b088c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b408c0a656e636f6465725f6666944d00088c0d6465636f6465725f72745f6666944d00028c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b018c0964726f705f7261746594473fde63b0ad8d4c308c026c7294473f8f79e3bc3ad6c08c0a62617463685f73697a65944d0004758c1274696d655f73696e63655f726573746f726594474056a5e6820000008c18697465726174696f6e735f73696e63655f726573746f7265944b01757562618c195f6c61746573745f636865636b706f696e745f726573756c7494681975622e\"\n }\n}"
+ ],
+ [
+ "{\n \"stub\": false,\n \"trainable_name\": \"train_model\",\n \"trial_id\": \"7dfe67c7\",\n \"storage\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059572040000000000008c1b7261792e747261696e2e5f696e7465726e616c2e73746f72616765948c0e53746f72616765436f6e746578749493942981947d94288c12637573746f6d5f66735f70726f766964656494898c1273746f726167655f6c6f63616c5f70617468948c2b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c7473948c136578706572696d656e745f6469725f6e616d65948c0f746573745f6578706572696d656e74948c0e747269616c5f6469725f6e616d65948c96747261696e5f6d6f64656c5f37646665363763375f345f62617463685f73697a653d313032342c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d323034382c6465636f6465725f72745f6e756d5f6c617965723d322c64726f705f725f323032342d30362d32355f31372d31352d3236948c1863757272656e745f636865636b706f696e745f696e646578944b008c0b73796e635f636f6e666967948c097261792e747261696e948c0a53796e63436f6e6669679493942981947d94288c0b73796e635f706572696f64944d2c018c0c73796e635f74696d656f7574944d08078c0e73796e635f61727469666163747394898c1c73796e635f6172746966616374735f6f6e5f636865636b706f696e7494888c0a75706c6f61645f646972948c0a44455052454341544544948c0673796e6365729468188c1273796e635f6f6e5f636865636b706f696e7494681875628c1273746f726167655f66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c0f73746f726167655f66735f70617468948c452f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f7465737494681968008c115f46696c6573797374656d53796e6365729493942981947d9428681b682268134d2c0168144d08078c116c6173745f73796e635f75705f74696d65944741d99eb7bda3c2988c136c6173745f73796e635f646f776e5f74696d659447fff00000000000008c0d5f73796e635f70726f63657373944e8c0c5f63757272656e745f636d649468008c125f75706c6f61645f746f5f66735f706174689493947d94288c0a6c6f63616c5f70617468948c3b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c74732f746573745f6578706572696d656e74948c0266739468228c0766735f70617468948c552f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e74948c076578636c756465945d948c0e2a2f636865636b706f696e745f2a9461758694756275622e\"\n },\n \"config\": {\n \"encoder_num_layer\": 2,\n \"decoder_rt_num_layer\": 2,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 256,\n \"encoder_ff\": 2048,\n \"decoder_rt_ff\": 2048,\n \"decoder_int_ff\": 512,\n \"n_head\": 2,\n \"drop_rate\": 0.6066107852445892,\n \"lr\": 0.0011313709304158431,\n \"batch_size\": 1024\n },\n \"_Trial__unresolved_config\": {\n \"encoder_num_layer\": 2,\n \"decoder_rt_num_layer\": 2,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 256,\n \"encoder_ff\": 2048,\n \"decoder_rt_ff\": 2048,\n \"decoder_int_ff\": 512,\n \"n_head\": 2,\n \"drop_rate\": 0.6066107852445892,\n \"lr\": 0.0011313709304158431,\n \"batch_size\": 1024\n },\n \"evaluated_params\": {\n \"batch_size\": 1024,\n \"decoder_int_ff\": 512,\n \"decoder_int_num_layer\": 1,\n \"decoder_rt_ff\": 2048,\n \"decoder_rt_num_layer\": 2,\n \"drop_rate\": 0.6066107852445892,\n \"embedding_dim\": 256,\n \"encoder_ff\": 2048,\n \"encoder_num_layer\": 2,\n \"lr\": 0.0011313709304158431,\n \"n_head\": 2\n },\n \"experiment_tag\": \"4_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=2,drop_rate=0.6066,embedding_dim=256,encoder_ff=2048,encoder_num_layer=2,lr=0.0011,n_head=2\",\n \"stopping_criterion\": {},\n \"_setup_default_resource\": true,\n \"_default_placement_group_factory\": \"80054e2e\",\n \"placement_group_factory\": \"800595ba000000000000008c237261792e74756e652e657865637574696f6e2e706c6163656d656e745f67726f757073948c15506c6163656d656e7447726f7570466163746f72799493942981947d94288c085f62756e646c6573945d947d94288c03435055944740540000000000008c034750559447401000000000000075618c155f686561645f62756e646c655f69735f656d70747994898c095f7374726174656779948c045041434b948c055f6172677394298c075f6b7761726773947d9475622e\",\n \"log_to_file\": [\n null,\n null\n ],\n \"max_failures\": 0,\n \"_default_result_or_future\": null,\n \"export_formats\": [],\n \"status\": \"PAUSED\",\n \"relative_logdir\": \"train_model_7dfe67c7_4_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=2,drop_r_2024-06-25_17-15-26\",\n \"trial_name_creator\": null,\n \"trial_dirname_creator\": null,\n \"custom_trial_name\": null,\n \"custom_dirname\": null,\n \"restore_path\": null,\n \"_restore_checkpoint_result\": null,\n \"_state_json\": null,\n \"results\": \"80054e2e\",\n \"extra_arg\": \"80054e2e\",\n \"_resources\": \"80054e2e\"\n}",
+ "{\n \"start_time\": 1719328617.3522246,\n \"num_failures\": 0,\n \"num_failures_after_restore\": 0,\n \"error_filename\": null,\n \"pickled_error_filename\": null,\n \"last_result\": {\n \"loss\": 1979.0773276982345,\n \"timestamp\": 1719328732,\n \"checkpoint_dir_name\": \"checkpoint_000000\",\n \"should_checkpoint\": true,\n \"done\": false,\n \"training_iteration\": 1,\n \"trial_id\": \"7dfe67c7\",\n \"date\": \"2024-06-25_17-18-52\",\n \"time_this_iter_s\": 115.28918504714966,\n \"time_total_s\": 115.28918504714966,\n \"pid\": 2195253,\n \"hostname\": \"r8i6n2\",\n \"node_ip\": \"10.159.28.60\",\n \"config\": {\n \"encoder_num_layer\": 2,\n \"decoder_rt_num_layer\": 2,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 256,\n \"encoder_ff\": 2048,\n \"decoder_rt_ff\": 2048,\n \"decoder_int_ff\": 512,\n \"n_head\": 2,\n \"drop_rate\": 0.6066107852445892,\n \"lr\": 0.0011313709304158431,\n \"batch_size\": 1024\n },\n \"time_since_restore\": 115.28918504714966,\n \"iterations_since_restore\": 1,\n \"experiment_tag\": \"4_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=2,drop_rate=0.6066,embedding_dim=256,encoder_ff=2048,encoder_num_layer=2,lr=0.0011,n_head=2\"\n },\n \"last_result_time\": 1719328732.6464097,\n \"metric_analysis\": {\n \"loss\": {\n \"max\": 1979.0773276982345,\n \"min\": 1979.0773276982345,\n \"avg\": 1979.0773276982345,\n \"last\": 1979.0773276982345,\n \"last-5-avg\": 1979.0773276982345,\n \"last-10-avg\": 1979.0773276982345\n },\n \"should_checkpoint\": {\n \"max\": true,\n \"min\": true,\n \"avg\": true,\n \"last\": true,\n \"last-5-avg\": true,\n \"last-10-avg\": true\n },\n \"done\": {\n \"max\": false,\n \"min\": false,\n \"avg\": false,\n \"last\": false,\n \"last-5-avg\": false,\n \"last-10-avg\": false\n },\n \"training_iteration\": {\n \"max\": 1,\n \"min\": 1,\n \"avg\": 1,\n \"last\": 1,\n \"last-5-avg\": 1,\n \"last-10-avg\": 1\n },\n \"time_this_iter_s\": {\n \"max\": 115.28918504714966,\n \"min\": 115.28918504714966,\n \"avg\": 115.28918504714966,\n \"last\": 115.28918504714966,\n \"last-5-avg\": 115.28918504714966,\n \"last-10-avg\": 115.28918504714966\n },\n \"time_total_s\": {\n \"max\": 115.28918504714966,\n \"min\": 115.28918504714966,\n \"avg\": 115.28918504714966,\n \"last\": 115.28918504714966,\n \"last-5-avg\": 115.28918504714966,\n \"last-10-avg\": 115.28918504714966\n },\n \"time_since_restore\": {\n \"max\": 115.28918504714966,\n \"min\": 115.28918504714966,\n \"avg\": 115.28918504714966,\n \"last\": 115.28918504714966,\n \"last-5-avg\": 115.28918504714966,\n \"last-10-avg\": 115.28918504714966\n },\n \"iterations_since_restore\": {\n \"max\": 1,\n \"min\": 1,\n \"avg\": 1,\n \"last\": 1,\n \"last-5-avg\": 1,\n \"last-10-avg\": 1\n }\n },\n \"_n_steps\": [\n 5,\n 10\n ],\n \"metric_n_steps\": {\n \"loss\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059589000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452948c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b007494624308f8fbfd2e4fec9e409486945294612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059589000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452948c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b007494624308f8fbfd2e4fec9e409486945294612e\"\n }\n },\n \"should_checkpoint\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059522000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b058694529488612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059522000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a8694529488612e\"\n }\n },\n \"done\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059522000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b058694529489612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059522000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a8694529489612e\"\n }\n },\n \"training_iteration\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059523000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452944b01612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059523000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452944b01612e\"\n }\n },\n \"time_this_iter_s\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b058694529447405cd28202000000612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a8694529447405cd28202000000612e\"\n }\n },\n \"time_total_s\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b058694529447405cd28202000000612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a8694529447405cd28202000000612e\"\n }\n },\n \"time_since_restore\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b058694529447405cd28202000000612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a8694529447405cd28202000000612e\"\n }\n },\n \"iterations_since_restore\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059523000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452944b01612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059523000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452944b01612e\"\n }\n }\n },\n \"checkpoint_manager\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059518060000000000008c267261792e747261696e2e5f696e7465726e616c2e636865636b706f696e745f6d616e61676572948c125f436865636b706f696e744d616e616765729493942981947d94288c125f636865636b706f696e745f636f6e666967948c097261792e747261696e948c10436865636b706f696e74436f6e6669679493942981947d94288c0b6e756d5f746f5f6b656570944e8c1a636865636b706f696e745f73636f72655f617474726962757465944e8c16636865636b706f696e745f73636f72655f6f72646572948c036d6178948c14636865636b706f696e745f6672657175656e6379944b008c11636865636b706f696e745f61745f656e6494898c1a5f636865636b706f696e745f6b6565705f616c6c5f72616e6b73948c0a44455052454341544544948c1f5f636865636b706f696e745f75706c6f61645f66726f6d5f776f726b65727394681275628c135f636865636b706f696e745f726573756c7473945d948c1b7261792e747261696e2e5f696e7465726e616c2e73657373696f6e948c0f5f547261696e696e67526573756c749493942981947d94288c0a636865636b706f696e749468068c0a436865636b706f696e749493942981947d94288c0470617468948cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f37646665363763375f345f62617463685f73697a653d313032342c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d323034382c6465636f6465725f72745f6e756d5f6c617965723d322c64726f705f725f323032342d30362d32355f31372d31352d32362f636865636b706f696e745f303030303030948c0a66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c055f75756964948c0475756964948c04555549449493942981947d948c03696e74948a10d00a7865e6fde98160434a4fa02a8c2c736275628c076d657472696373947d94288c046c6f7373948c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b007494624308f8fbfd2e4fec9e4094869452948c0974696d657374616d70944adcdf7a668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303030948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b018c08747269616c5f6964948c083764666536376337948c0464617465948c13323032342d30362d32355f31372d31382d3532948c1074696d655f746869735f697465725f739447405cd282020000008c0c74696d655f746f74616c5f739447405cd282020000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b028c146465636f6465725f72745f6e756d5f6c61796572944b028c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944d00018c0a656e636f6465725f6666944d00088c0d6465636f6465725f72745f6666944d00088c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b028c0964726f705f7261746594473fe3695b0580d8308c026c7294473f52895049547a698c0a62617463685f73697a65944d0004758c1274696d655f73696e63655f726573746f72659447405cd282020000008c18697465726174696f6e735f73696e63655f726573746f7265944b01757562618c195f6c61746573745f636865636b706f696e745f726573756c7494681975622e\"\n }\n}"
+ ],
+ [
+ "{\n \"stub\": false,\n \"trainable_name\": \"train_model\",\n \"trial_id\": \"f1fbbaf2\",\n \"storage\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059572040000000000008c1b7261792e747261696e2e5f696e7465726e616c2e73746f72616765948c0e53746f72616765436f6e746578749493942981947d94288c12637573746f6d5f66735f70726f766964656494898c1273746f726167655f6c6f63616c5f70617468948c2b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c7473948c136578706572696d656e745f6469725f6e616d65948c0f746573745f6578706572696d656e74948c0e747269616c5f6469725f6e616d65948c96747261696e5f6d6f64656c5f66316662626166325f355f62617463685f73697a653d313032342c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d313032342c6465636f6465725f72745f6e756d5f6c617965723d322c64726f705f725f323032342d30362d32355f31372d31362d3537948c1863757272656e745f636865636b706f696e745f696e646578944b008c0b73796e635f636f6e666967948c097261792e747261696e948c0a53796e63436f6e6669679493942981947d94288c0b73796e635f706572696f64944d2c018c0c73796e635f74696d656f7574944d08078c0e73796e635f61727469666163747394898c1c73796e635f6172746966616374735f6f6e5f636865636b706f696e7494888c0a75706c6f61645f646972948c0a44455052454341544544948c0673796e6365729468188c1273796e635f6f6e5f636865636b706f696e7494681875628c1273746f726167655f66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c0f73746f726167655f66735f70617468948c452f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f7465737494681968008c115f46696c6573797374656d53796e6365729493942981947d9428681b682268134d2c0168144d08078c116c6173745f73796e635f75705f74696d65944741d99eb7bda3c2988c136c6173745f73796e635f646f776e5f74696d659447fff00000000000008c0d5f73796e635f70726f63657373944e8c0c5f63757272656e745f636d649468008c125f75706c6f61645f746f5f66735f706174689493947d94288c0a6c6f63616c5f70617468948c3b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c74732f746573745f6578706572696d656e74948c0266739468228c0766735f70617468948c552f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e74948c076578636c756465945d948c0e2a2f636865636b706f696e745f2a9461758694756275622e\"\n },\n \"config\": {\n \"encoder_num_layer\": 4,\n \"decoder_rt_num_layer\": 2,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 1024,\n \"encoder_ff\": 2048,\n \"decoder_rt_ff\": 1024,\n \"decoder_int_ff\": 512,\n \"n_head\": 1,\n \"drop_rate\": 0.24540187796715263,\n \"lr\": 0.0006485525268785047,\n \"batch_size\": 1024\n },\n \"_Trial__unresolved_config\": {\n \"encoder_num_layer\": 4,\n \"decoder_rt_num_layer\": 2,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 1024,\n \"encoder_ff\": 2048,\n \"decoder_rt_ff\": 1024,\n \"decoder_int_ff\": 512,\n \"n_head\": 1,\n \"drop_rate\": 0.24540187796715263,\n \"lr\": 0.0006485525268785047,\n \"batch_size\": 1024\n },\n \"evaluated_params\": {\n \"batch_size\": 1024,\n \"decoder_int_ff\": 512,\n \"decoder_int_num_layer\": 1,\n \"decoder_rt_ff\": 1024,\n \"decoder_rt_num_layer\": 2,\n \"drop_rate\": 0.24540187796715263,\n \"embedding_dim\": 1024,\n \"encoder_ff\": 2048,\n \"encoder_num_layer\": 4,\n \"lr\": 0.0006485525268785047,\n \"n_head\": 1\n },\n \"experiment_tag\": \"5_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop_rate=0.2454,embedding_dim=1024,encoder_ff=2048,encoder_num_layer=4,lr=0.0006,n_head=1\",\n \"stopping_criterion\": {},\n \"_setup_default_resource\": true,\n \"_default_placement_group_factory\": \"80054e2e\",\n \"placement_group_factory\": \"800595ba000000000000008c237261792e74756e652e657865637574696f6e2e706c6163656d656e745f67726f757073948c15506c6163656d656e7447726f7570466163746f72799493942981947d94288c085f62756e646c6573945d947d94288c03435055944740540000000000008c034750559447401000000000000075618c155f686561645f62756e646c655f69735f656d70747994898c095f7374726174656779948c045041434b948c055f6172677394298c075f6b7761726773947d9475622e\",\n \"log_to_file\": [\n null,\n null\n ],\n \"max_failures\": 0,\n \"_default_result_or_future\": null,\n \"export_formats\": [],\n \"status\": \"PAUSED\",\n \"relative_logdir\": \"train_model_f1fbbaf2_5_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop_r_2024-06-25_17-16-57\",\n \"trial_name_creator\": null,\n \"trial_dirname_creator\": null,\n \"custom_trial_name\": null,\n \"custom_dirname\": null,\n \"restore_path\": null,\n \"_restore_checkpoint_result\": null,\n \"_state_json\": null,\n \"results\": \"80054e2e\",\n \"extra_arg\": \"80054e2e\",\n \"_resources\": \"80054e2e\"\n}",
+ "{\n \"start_time\": 1719328732.7937407,\n \"num_failures\": 0,\n \"num_failures_after_restore\": 0,\n \"error_filename\": null,\n \"pickled_error_filename\": null,\n \"last_result\": {\n \"loss\": 1986.5954556202325,\n \"timestamp\": 1719329376,\n \"checkpoint_dir_name\": \"checkpoint_000000\",\n \"should_checkpoint\": true,\n \"done\": false,\n \"training_iteration\": 1,\n \"trial_id\": \"f1fbbaf2\",\n \"date\": \"2024-06-25_17-29-36\",\n \"time_this_iter_s\": 643.849014043808,\n \"time_total_s\": 643.849014043808,\n \"pid\": 2195253,\n \"hostname\": \"r8i6n2\",\n \"node_ip\": \"10.159.28.60\",\n \"config\": {\n \"encoder_num_layer\": 4,\n \"decoder_rt_num_layer\": 2,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 1024,\n \"encoder_ff\": 2048,\n \"decoder_rt_ff\": 1024,\n \"decoder_int_ff\": 512,\n \"n_head\": 1,\n \"drop_rate\": 0.24540187796715263,\n \"lr\": 0.0006485525268785047,\n \"batch_size\": 1024\n },\n \"time_since_restore\": 643.849014043808,\n \"iterations_since_restore\": 1,\n \"experiment_tag\": \"5_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop_rate=0.2454,embedding_dim=1024,encoder_ff=2048,encoder_num_layer=4,lr=0.0006,n_head=1\"\n },\n \"last_result_time\": 1719329376.6484032,\n \"metric_analysis\": {\n \"loss\": {\n \"max\": 1986.5954556202325,\n \"min\": 1986.5954556202325,\n \"avg\": 1986.5954556202325,\n \"last\": 1986.5954556202325,\n \"last-5-avg\": 1986.5954556202325,\n \"last-10-avg\": 1986.5954556202325\n },\n \"should_checkpoint\": {\n \"max\": true,\n \"min\": true,\n \"avg\": true,\n \"last\": true,\n \"last-5-avg\": true,\n \"last-10-avg\": true\n },\n \"done\": {\n \"max\": false,\n \"min\": false,\n \"avg\": false,\n \"last\": false,\n \"last-5-avg\": false,\n \"last-10-avg\": false\n },\n \"training_iteration\": {\n \"max\": 1,\n \"min\": 1,\n \"avg\": 1,\n \"last\": 1,\n \"last-5-avg\": 1,\n \"last-10-avg\": 1\n },\n \"time_this_iter_s\": {\n \"max\": 643.849014043808,\n \"min\": 643.849014043808,\n \"avg\": 643.849014043808,\n \"last\": 643.849014043808,\n \"last-5-avg\": 643.849014043808,\n \"last-10-avg\": 643.849014043808\n },\n \"time_total_s\": {\n \"max\": 643.849014043808,\n \"min\": 643.849014043808,\n \"avg\": 643.849014043808,\n \"last\": 643.849014043808,\n \"last-5-avg\": 643.849014043808,\n \"last-10-avg\": 643.849014043808\n },\n \"time_since_restore\": {\n \"max\": 643.849014043808,\n \"min\": 643.849014043808,\n \"avg\": 643.849014043808,\n \"last\": 643.849014043808,\n \"last-5-avg\": 643.849014043808,\n \"last-10-avg\": 643.849014043808\n },\n \"iterations_since_restore\": {\n \"max\": 1,\n \"min\": 1,\n \"avg\": 1,\n \"last\": 1,\n \"last-5-avg\": 1,\n \"last-10-avg\": 1\n }\n },\n \"_n_steps\": [\n 5,\n 10\n ],\n \"metric_n_steps\": {\n \"loss\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059589000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452948c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b007494624308793c1ebf610a9f409486945294612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059589000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452948c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b007494624308793c1ebf610a9f409486945294612e\"\n }\n },\n \"should_checkpoint\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059522000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b058694529488612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059522000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a8694529488612e\"\n }\n },\n \"done\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059522000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b058694529489612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059522000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a8694529489612e\"\n }\n },\n \"training_iteration\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059523000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452944b01612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059523000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452944b01612e\"\n }\n },\n \"time_this_iter_s\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452944740841ecac7e00000612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452944740841ecac7e00000612e\"\n }\n },\n \"time_total_s\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452944740841ecac7e00000612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452944740841ecac7e00000612e\"\n }\n },\n \"time_since_restore\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452944740841ecac7e00000612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452944740841ecac7e00000612e\"\n }\n },\n \"iterations_since_restore\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059523000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452944b01612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059523000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452944b01612e\"\n }\n }\n },\n \"checkpoint_manager\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059518060000000000008c267261792e747261696e2e5f696e7465726e616c2e636865636b706f696e745f6d616e61676572948c125f436865636b706f696e744d616e616765729493942981947d94288c125f636865636b706f696e745f636f6e666967948c097261792e747261696e948c10436865636b706f696e74436f6e6669679493942981947d94288c0b6e756d5f746f5f6b656570944e8c1a636865636b706f696e745f73636f72655f617474726962757465944e8c16636865636b706f696e745f73636f72655f6f72646572948c036d6178948c14636865636b706f696e745f6672657175656e6379944b008c11636865636b706f696e745f61745f656e6494898c1a5f636865636b706f696e745f6b6565705f616c6c5f72616e6b73948c0a44455052454341544544948c1f5f636865636b706f696e745f75706c6f61645f66726f6d5f776f726b65727394681275628c135f636865636b706f696e745f726573756c7473945d948c1b7261792e747261696e2e5f696e7465726e616c2e73657373696f6e948c0f5f547261696e696e67526573756c749493942981947d94288c0a636865636b706f696e749468068c0a436865636b706f696e749493942981947d94288c0470617468948cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f66316662626166325f355f62617463685f73697a653d313032342c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d313032342c6465636f6465725f72745f6e756d5f6c617965723d322c64726f705f725f323032342d30362d32355f31372d31362d35372f636865636b706f696e745f303030303030948c0a66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c055f75756964948c0475756964948c04555549449493942981947d948c03696e74948a107ceec938c58a36a286442505bc700a43736275628c076d657472696373947d94288c046c6f7373948c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b007494624308793c1ebf610a9f4094869452948c0974696d657374616d70944a60e27a668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303030948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b018c08747269616c5f6964948c086631666262616632948c0464617465948c13323032342d30362d32355f31372d32392d3336948c1074696d655f746869735f697465725f73944740841ecac7e000008c0c74696d655f746f74616c5f73944740841ecac7e000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b048c146465636f6465725f72745f6e756d5f6c61796572944b028c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944d00048c0a656e636f6465725f6666944d00088c0d6465636f6465725f72745f6666944d00048c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b018c0964726f705f7261746594473fcf6954281f79c48c026c7294473f454073f24361d28c0a62617463685f73697a65944d0004758c1274696d655f73696e63655f726573746f7265944740841ecac7e000008c18697465726174696f6e735f73696e63655f726573746f7265944b01757562618c195f6c61746573745f636865636b706f696e745f726573756c7494681975622e\"\n }\n}"
+ ],
+ [
+ "{\n \"stub\": false,\n \"trainable_name\": \"train_model\",\n \"trial_id\": \"5b0a0ae2\",\n \"storage\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059572040000000000008c1b7261792e747261696e2e5f696e7465726e616c2e73746f72616765948c0e53746f72616765436f6e746578749493942981947d94288c12637573746f6d5f66735f70726f766964656494898c1273746f726167655f6c6f63616c5f70617468948c2b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c7473948c136578706572696d656e745f6469725f6e616d65948c0f746573745f6578706572696d656e74948c0e747269616c5f6469725f6e616d65948c96747261696e5f6d6f64656c5f35623061306165325f365f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d323034382c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f725f323032342d30362d32355f31372d31382d3532948c1863757272656e745f636865636b706f696e745f696e646578944b018c0b73796e635f636f6e666967948c097261792e747261696e948c0a53796e63436f6e6669679493942981947d94288c0b73796e635f706572696f64944d2c018c0c73796e635f74696d656f7574944d08078c0e73796e635f61727469666163747394898c1c73796e635f6172746966616374735f6f6e5f636865636b706f696e7494888c0a75706c6f61645f646972948c0a44455052454341544544948c0673796e6365729468188c1273796e635f6f6e5f636865636b706f696e7494681875628c1273746f726167655f66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c0f73746f726167655f66735f70617468948c452f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f7465737494681968008c115f46696c6573797374656d53796e6365729493942981947d9428681b682268134d2c0168144d08078c116c6173745f73796e635f75705f74696d65944741d99eb7bda3c2988c136c6173745f73796e635f646f776e5f74696d659447fff00000000000008c0d5f73796e635f70726f63657373944e8c0c5f63757272656e745f636d649468008c125f75706c6f61645f746f5f66735f706174689493947d94288c0a6c6f63616c5f70617468948c3b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c74732f746573745f6578706572696d656e74948c0266739468228c0766735f70617468948c552f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e74948c076578636c756465945d948c0e2a2f636865636b706f696e745f2a9461758694756275622e\"\n },\n \"config\": {\n \"encoder_num_layer\": 8,\n \"decoder_rt_num_layer\": 1,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 16,\n \"encoder_ff\": 512,\n \"decoder_rt_ff\": 2048,\n \"decoder_int_ff\": 512,\n \"n_head\": 16,\n \"drop_rate\": 0.0063231604639778505,\n \"lr\": 0.0021079084039053323,\n \"batch_size\": 2048\n },\n \"_Trial__unresolved_config\": {\n \"encoder_num_layer\": 8,\n \"decoder_rt_num_layer\": 1,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 16,\n \"encoder_ff\": 512,\n \"decoder_rt_ff\": 2048,\n \"decoder_int_ff\": 512,\n \"n_head\": 16,\n \"drop_rate\": 0.0063231604639778505,\n \"lr\": 0.0021079084039053323,\n \"batch_size\": 2048\n },\n \"evaluated_params\": {\n \"batch_size\": 2048,\n \"decoder_int_ff\": 512,\n \"decoder_int_num_layer\": 1,\n \"decoder_rt_ff\": 2048,\n \"decoder_rt_num_layer\": 1,\n \"drop_rate\": 0.0063231604639778505,\n \"embedding_dim\": 16,\n \"encoder_ff\": 512,\n \"encoder_num_layer\": 8,\n \"lr\": 0.0021079084039053323,\n \"n_head\": 16\n },\n \"experiment_tag\": \"6_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop_rate=0.0063,embedding_dim=16,encoder_ff=512,encoder_num_layer=8,lr=0.0021,n_head=16\",\n \"stopping_criterion\": {},\n \"_setup_default_resource\": true,\n \"_default_placement_group_factory\": \"80054e2e\",\n \"placement_group_factory\": \"800595ba000000000000008c237261792e74756e652e657865637574696f6e2e706c6163656d656e745f67726f757073948c15506c6163656d656e7447726f7570466163746f72799493942981947d94288c085f62756e646c6573945d947d94288c03435055944740540000000000008c034750559447401000000000000075618c155f686561645f62756e646c655f69735f656d70747994898c095f7374726174656779948c045041434b948c055f6172677394298c075f6b7761726773947d9475622e\",\n \"log_to_file\": [\n null,\n null\n ],\n \"max_failures\": 0,\n \"_default_result_or_future\": null,\n \"export_formats\": [],\n \"status\": \"PAUSED\",\n \"relative_logdir\": \"train_model_5b0a0ae2_6_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop_r_2024-06-25_17-18-52\",\n \"trial_name_creator\": null,\n \"trial_dirname_creator\": null,\n \"custom_trial_name\": null,\n \"custom_dirname\": null,\n \"restore_path\": null,\n \"_restore_checkpoint_result\": null,\n \"_state_json\": null,\n \"results\": \"80054e2e\",\n \"extra_arg\": \"80054e2e\",\n \"_resources\": \"80054e2e\"\n}",
+ "{\n \"start_time\": 1719329376.7954574,\n \"num_failures\": 0,\n \"num_failures_after_restore\": 0,\n \"error_filename\": null,\n \"pickled_error_filename\": null,\n \"last_result\": {\n \"loss\": 1963.9745468980684,\n \"timestamp\": 1719330671,\n \"checkpoint_dir_name\": \"checkpoint_000001\",\n \"should_checkpoint\": true,\n \"done\": false,\n \"training_iteration\": 2,\n \"trial_id\": \"5b0a0ae2\",\n \"date\": \"2024-06-25_17-51-11\",\n \"time_this_iter_s\": 84.48489356040955,\n \"time_total_s\": 168.87538933753967,\n \"pid\": 2195253,\n \"hostname\": \"r8i6n2\",\n \"node_ip\": \"10.159.28.60\",\n \"config\": {\n \"encoder_num_layer\": 8,\n \"decoder_rt_num_layer\": 1,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 16,\n \"encoder_ff\": 512,\n \"decoder_rt_ff\": 2048,\n \"decoder_int_ff\": 512,\n \"n_head\": 16,\n \"drop_rate\": 0.0063231604639778505,\n \"lr\": 0.0021079084039053323,\n \"batch_size\": 2048\n },\n \"time_since_restore\": 84.48489356040955,\n \"iterations_since_restore\": 1,\n \"experiment_tag\": \"6_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop_rate=0.0063,embedding_dim=16,encoder_ff=512,encoder_num_layer=8,lr=0.0021,n_head=16\"\n },\n \"last_result_time\": 1719330671.9681833,\n \"metric_analysis\": {\n \"loss\": {\n \"max\": 1963.9745468980684,\n \"min\": 1952.9751620555487,\n \"avg\": 1958.4748544768086,\n \"last\": 1963.9745468980684,\n \"last-5-avg\": 1958.4748544768086,\n \"last-10-avg\": 1958.4748544768086\n },\n \"should_checkpoint\": {\n \"max\": true,\n \"min\": true,\n \"avg\": 1.0,\n \"last\": true,\n \"last-5-avg\": 1.0,\n \"last-10-avg\": 1.0\n },\n \"done\": {\n \"max\": false,\n \"min\": false,\n \"avg\": 0.0,\n \"last\": false,\n \"last-5-avg\": 0.0,\n \"last-10-avg\": 0.0\n },\n \"training_iteration\": {\n \"max\": 2,\n \"min\": 1,\n \"avg\": 1.5,\n \"last\": 2,\n \"last-5-avg\": 1.5,\n \"last-10-avg\": 1.5\n },\n \"time_this_iter_s\": {\n \"max\": 84.48489356040955,\n \"min\": 84.39049577713013,\n \"avg\": 84.43769466876984,\n \"last\": 84.48489356040955,\n \"last-5-avg\": 84.43769466876984,\n \"last-10-avg\": 84.43769466876984\n },\n \"time_total_s\": {\n \"max\": 168.87538933753967,\n \"min\": 84.39049577713013,\n \"avg\": 126.6329425573349,\n \"last\": 168.87538933753967,\n \"last-5-avg\": 126.6329425573349,\n \"last-10-avg\": 126.6329425573349\n },\n \"time_since_restore\": {\n \"max\": 84.48489356040955,\n \"min\": 84.39049577713013,\n \"avg\": 84.43769466876984,\n \"last\": 84.48489356040955,\n \"last-5-avg\": 84.43769466876984,\n \"last-10-avg\": 84.43769466876984\n },\n \"iterations_since_restore\": {\n \"max\": 1,\n \"min\": 1,\n \"avg\": 1.0,\n \"last\": 1,\n \"last-5-avg\": 1.0,\n \"last-10-avg\": 1.0\n }\n },\n \"_n_steps\": [\n 5,\n 10\n ],\n \"metric_n_steps\": {\n \"loss\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005959d000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294288c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b00749462430887c3e190e6839e4094869452946807680d43087d3e9fefe5af9e409486945294652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005959d000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294288c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b00749462430887c3e190e6839e4094869452946807680d43087d3e9fefe5af9e409486945294652e\"\n }\n },\n \"should_checkpoint\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059524000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294288888652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059524000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294288888652e\"\n }\n },\n \"done\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059524000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294288989652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059524000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294288989652e\"\n }\n },\n \"training_iteration\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059526000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294284b014b02652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059526000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294284b014b02652e\"\n }\n },\n \"time_this_iter_s\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059534000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452942847405518fde20000004740551f087f000000652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059534000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452942847405518fde20000004740551f087f000000652e\"\n }\n },\n \"time_total_s\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059534000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452942847405518fde20000004740651c0330800000652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059534000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452942847405518fde20000004740651c0330800000652e\"\n }\n },\n \"time_since_restore\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059534000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452942847405518fde20000004740551f087f000000652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059534000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452942847405518fde20000004740551f087f000000652e\"\n }\n },\n \"iterations_since_restore\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059526000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294284b014b01652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059526000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294284b014b01652e\"\n }\n }\n },\n \"checkpoint_manager\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"800595a5090000000000008c267261792e747261696e2e5f696e7465726e616c2e636865636b706f696e745f6d616e61676572948c125f436865636b706f696e744d616e616765729493942981947d94288c125f636865636b706f696e745f636f6e666967948c097261792e747261696e948c10436865636b706f696e74436f6e6669679493942981947d94288c0b6e756d5f746f5f6b656570944e8c1a636865636b706f696e745f73636f72655f617474726962757465944e8c16636865636b706f696e745f73636f72655f6f72646572948c036d6178948c14636865636b706f696e745f6672657175656e6379944b008c11636865636b706f696e745f61745f656e6494898c1a5f636865636b706f696e745f6b6565705f616c6c5f72616e6b73948c0a44455052454341544544948c1f5f636865636b706f696e745f75706c6f61645f66726f6d5f776f726b65727394681275628c135f636865636b706f696e745f726573756c7473945d94288c1b7261792e747261696e2e5f696e7465726e616c2e73657373696f6e948c0f5f547261696e696e67526573756c749493942981947d94288c0a636865636b706f696e749468068c0a436865636b706f696e749493942981947d94288c0470617468948cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f35623061306165325f365f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d323034382c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f725f323032342d30362d32355f31372d31382d35322f636865636b706f696e745f303030303030948c0a66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c055f75756964948c0475756964948c04555549449493942981947d948c03696e74948a1070db152d94d48fa82e42a314639aa52d736275628c076d657472696373947d94288c046c6f7373948c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b00749462430887c3e190e6839e4094869452948c0974696d657374616d70944ab5e27a668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303030948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b018c08747269616c5f6964948c083562306130616532948c0464617465948c13323032342d30362d32355f31372d33312d3031948c1074696d655f746869735f697465725f739447405518fde20000008c0c74696d655f746f74616c5f739447405518fde20000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b088c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b108c0a656e636f6465725f6666944d00028c0d6465636f6465725f72745f6666944d00088c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b108c0964726f705f7261746594473f79e6507666fe808c026c7294473f61449ab50c2e628c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f72659447405518fde20000008c18697465726174696f6e735f73696e63655f726573746f7265944b0175756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f35623061306165325f365f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d323034382c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f725f323032342d30362d32355f31372d31382d35322f636865636b706f696e745f30303030303194682268257d946827897385945294682a682d2981947d9468308a1137570e916e59b7a6e64247e3a7f2e0a5007362756268317d94288c046c6f7373946836683c43087d3e9fefe5af9e4094869452948c0974696d657374616d70944a6fe77a668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303031948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b028c08747269616c5f6964948c083562306130616532948c0464617465948c13323032342d30362d32355f31372d35312d3131948c1074696d655f746869735f697465725f73944740551f087f0000008c0c74696d655f746f74616c5f73944740651c03308000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b088c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b108c0a656e636f6465725f6666944d00028c0d6465636f6465725f72745f6666944d00088c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b108c0964726f705f7261746594473f79e6507666fe808c026c7294473f61449ab50c2e628c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f7265944740551f087f0000008c18697465726174696f6e735f73696e63655f726573746f7265944b01757562658c195f6c61746573745f636865636b706f696e745f726573756c7494686275622e\"\n }\n}"
+ ],
+ [
+ "{\n \"stub\": false,\n \"trainable_name\": \"train_model\",\n \"trial_id\": \"3f37f45c\",\n \"storage\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059572040000000000008c1b7261792e747261696e2e5f696e7465726e616c2e73746f72616765948c0e53746f72616765436f6e746578749493942981947d94288c12637573746f6d5f66735f70726f766964656494898c1273746f726167655f6c6f63616c5f70617468948c2b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c7473948c136578706572696d656e745f6469725f6e616d65948c0f746573745f6578706572696d656e74948c0e747269616c5f6469725f6e616d65948c96747261696e5f6d6f64656c5f33663337663435635f375f62617463685f73697a653d313032342c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d323034382c6465636f6465725f72745f6e756d5f6c617965723d322c64726f705f725f323032342d30362d32355f31372d32392d3336948c1863757272656e745f636865636b706f696e745f696e646578944b008c0b73796e635f636f6e666967948c097261792e747261696e948c0a53796e63436f6e6669679493942981947d94288c0b73796e635f706572696f64944d2c018c0c73796e635f74696d656f7574944d08078c0e73796e635f61727469666163747394898c1c73796e635f6172746966616374735f6f6e5f636865636b706f696e7494888c0a75706c6f61645f646972948c0a44455052454341544544948c0673796e6365729468188c1273796e635f6f6e5f636865636b706f696e7494681875628c1273746f726167655f66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c0f73746f726167655f66735f70617468948c452f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f7465737494681968008c115f46696c6573797374656d53796e6365729493942981947d9428681b682268134d2c0168144d08078c116c6173745f73796e635f75705f74696d65944741d99eb8547484f48c136c6173745f73796e635f646f776e5f74696d659447fff00000000000008c0d5f73796e635f70726f63657373944e8c0c5f63757272656e745f636d649468008c125f75706c6f61645f746f5f66735f706174689493947d94288c0a6c6f63616c5f70617468948c3b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c74732f746573745f6578706572696d656e74948c0266739468228c0766735f70617468948c552f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e74948c076578636c756465945d948c0e2a2f636865636b706f696e745f2a9461758694756275622e\"\n },\n \"config\": {\n \"encoder_num_layer\": 1,\n \"decoder_rt_num_layer\": 2,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 256,\n \"encoder_ff\": 512,\n \"decoder_rt_ff\": 2048,\n \"decoder_int_ff\": 512,\n \"n_head\": 4,\n \"drop_rate\": 0.6691316467787182,\n \"lr\": 0.004731694425031267,\n \"batch_size\": 1024\n },\n \"_Trial__unresolved_config\": {\n \"encoder_num_layer\": 1,\n \"decoder_rt_num_layer\": 2,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 256,\n \"encoder_ff\": 512,\n \"decoder_rt_ff\": 2048,\n \"decoder_int_ff\": 512,\n \"n_head\": 4,\n \"drop_rate\": 0.6691316467787182,\n \"lr\": 0.004731694425031267,\n \"batch_size\": 1024\n },\n \"evaluated_params\": {\n \"batch_size\": 1024,\n \"decoder_int_ff\": 512,\n \"decoder_int_num_layer\": 1,\n \"decoder_rt_ff\": 2048,\n \"decoder_rt_num_layer\": 2,\n \"drop_rate\": 0.6691316467787182,\n \"embedding_dim\": 256,\n \"encoder_ff\": 512,\n \"encoder_num_layer\": 1,\n \"lr\": 0.004731694425031267,\n \"n_head\": 4\n },\n \"experiment_tag\": \"7_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=2,drop_rate=0.6691,embedding_dim=256,encoder_ff=512,encoder_num_layer=1,lr=0.0047,n_head=4\",\n \"stopping_criterion\": {},\n \"_setup_default_resource\": true,\n \"_default_placement_group_factory\": \"80054e2e\",\n \"placement_group_factory\": \"800595ba000000000000008c237261792e74756e652e657865637574696f6e2e706c6163656d656e745f67726f757073948c15506c6163656d656e7447726f7570466163746f72799493942981947d94288c085f62756e646c6573945d947d94288c03435055944740540000000000008c034750559447401000000000000075618c155f686561645f62756e646c655f69735f656d70747994898c095f7374726174656779948c045041434b948c055f6172677394298c075f6b7761726773947d9475622e\",\n \"log_to_file\": [\n null,\n null\n ],\n \"max_failures\": 0,\n \"_default_result_or_future\": null,\n \"export_formats\": [],\n \"status\": \"PAUSED\",\n \"relative_logdir\": \"train_model_3f37f45c_7_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=2,drop_r_2024-06-25_17-29-36\",\n \"trial_name_creator\": null,\n \"trial_dirname_creator\": null,\n \"custom_trial_name\": null,\n \"custom_dirname\": null,\n \"restore_path\": null,\n \"_restore_checkpoint_result\": null,\n \"_state_json\": null,\n \"results\": \"80054e2e\",\n \"extra_arg\": \"80054e2e\",\n \"_resources\": \"80054e2e\"\n}",
+ "{\n \"start_time\": 1719329461.3491297,\n \"num_failures\": 0,\n \"num_failures_after_restore\": 0,\n \"error_filename\": null,\n \"pickled_error_filename\": null,\n \"last_result\": {\n \"loss\": 2052.4382574126475,\n \"timestamp\": 1719329544,\n \"checkpoint_dir_name\": \"checkpoint_000000\",\n \"should_checkpoint\": true,\n \"done\": false,\n \"training_iteration\": 1,\n \"trial_id\": \"3f37f45c\",\n \"date\": \"2024-06-25_17-32-25\",\n \"time_this_iter_s\": 83.68853616714478,\n \"time_total_s\": 83.68853616714478,\n \"pid\": 2195253,\n \"hostname\": \"r8i6n2\",\n \"node_ip\": \"10.159.28.60\",\n \"config\": {\n \"encoder_num_layer\": 1,\n \"decoder_rt_num_layer\": 2,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 256,\n \"encoder_ff\": 512,\n \"decoder_rt_ff\": 2048,\n \"decoder_int_ff\": 512,\n \"n_head\": 4,\n \"drop_rate\": 0.6691316467787182,\n \"lr\": 0.004731694425031267,\n \"batch_size\": 1024\n },\n \"time_since_restore\": 83.68853616714478,\n \"iterations_since_restore\": 1,\n \"experiment_tag\": \"7_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=2,drop_rate=0.6691,embedding_dim=256,encoder_ff=512,encoder_num_layer=1,lr=0.0047,n_head=4\"\n },\n \"last_result_time\": 1719329545.0414808,\n \"metric_analysis\": {\n \"loss\": {\n \"max\": 2052.4382574126475,\n \"min\": 2052.4382574126475,\n \"avg\": 2052.4382574126475,\n \"last\": 2052.4382574126475,\n \"last-5-avg\": 2052.4382574126475,\n \"last-10-avg\": 2052.4382574126475\n },\n \"should_checkpoint\": {\n \"max\": true,\n \"min\": true,\n \"avg\": true,\n \"last\": true,\n \"last-5-avg\": true,\n \"last-10-avg\": true\n },\n \"done\": {\n \"max\": false,\n \"min\": false,\n \"avg\": false,\n \"last\": false,\n \"last-5-avg\": false,\n \"last-10-avg\": false\n },\n \"training_iteration\": {\n \"max\": 1,\n \"min\": 1,\n \"avg\": 1,\n \"last\": 1,\n \"last-5-avg\": 1,\n \"last-10-avg\": 1\n },\n \"time_this_iter_s\": {\n \"max\": 83.68853616714478,\n \"min\": 83.68853616714478,\n \"avg\": 83.68853616714478,\n \"last\": 83.68853616714478,\n \"last-5-avg\": 83.68853616714478,\n \"last-10-avg\": 83.68853616714478\n },\n \"time_total_s\": {\n \"max\": 83.68853616714478,\n \"min\": 83.68853616714478,\n \"avg\": 83.68853616714478,\n \"last\": 83.68853616714478,\n \"last-5-avg\": 83.68853616714478,\n \"last-10-avg\": 83.68853616714478\n },\n \"time_since_restore\": {\n \"max\": 83.68853616714478,\n \"min\": 83.68853616714478,\n \"avg\": 83.68853616714478,\n \"last\": 83.68853616714478,\n \"last-5-avg\": 83.68853616714478,\n \"last-10-avg\": 83.68853616714478\n },\n \"iterations_since_restore\": {\n \"max\": 1,\n \"min\": 1,\n \"avg\": 1,\n \"last\": 1,\n \"last-5-avg\": 1,\n \"last-10-avg\": 1\n }\n },\n \"_n_steps\": [\n 5,\n 10\n ],\n \"metric_n_steps\": {\n \"loss\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059589000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452948c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b0074946243081a8d4663e008a0409486945294612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059589000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452948c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b0074946243081a8d4663e008a0409486945294612e\"\n }\n },\n \"should_checkpoint\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059522000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b058694529488612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059522000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a8694529488612e\"\n }\n },\n \"done\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059522000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b058694529489612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059522000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a8694529489612e\"\n }\n },\n \"training_iteration\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059523000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452944b01612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059523000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452944b01612e\"\n }\n },\n \"time_this_iter_s\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294474054ec10fa000000612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294474054ec10fa000000612e\"\n }\n },\n \"time_total_s\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294474054ec10fa000000612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294474054ec10fa000000612e\"\n }\n },\n \"time_since_restore\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294474054ec10fa000000612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294474054ec10fa000000612e\"\n }\n },\n \"iterations_since_restore\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059523000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452944b01612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059523000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452944b01612e\"\n }\n }\n },\n \"checkpoint_manager\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059519060000000000008c267261792e747261696e2e5f696e7465726e616c2e636865636b706f696e745f6d616e61676572948c125f436865636b706f696e744d616e616765729493942981947d94288c125f636865636b706f696e745f636f6e666967948c097261792e747261696e948c10436865636b706f696e74436f6e6669679493942981947d94288c0b6e756d5f746f5f6b656570944e8c1a636865636b706f696e745f73636f72655f617474726962757465944e8c16636865636b706f696e745f73636f72655f6f72646572948c036d6178948c14636865636b706f696e745f6672657175656e6379944b008c11636865636b706f696e745f61745f656e6494898c1a5f636865636b706f696e745f6b6565705f616c6c5f72616e6b73948c0a44455052454341544544948c1f5f636865636b706f696e745f75706c6f61645f66726f6d5f776f726b65727394681275628c135f636865636b706f696e745f726573756c7473945d948c1b7261792e747261696e2e5f696e7465726e616c2e73657373696f6e948c0f5f547261696e696e67526573756c749493942981947d94288c0a636865636b706f696e749468068c0a436865636b706f696e749493942981947d94288c0470617468948cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f33663337663435635f375f62617463685f73697a653d313032342c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d323034382c6465636f6465725f72745f6e756d5f6c617965723d322c64726f705f725f323032342d30362d32355f31372d32392d33362f636865636b706f696e745f303030303030948c0a66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c055f75756964948c0475756964948c04555549449493942981947d948c03696e74948a1138d348b093235c974043ee23a2e6638700736275628c076d657472696373947d94288c046c6f7373948c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b0074946243081a8d4663e008a04094869452948c0974696d657374616d70944a08e37a668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303030948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b018c08747269616c5f6964948c083366333766343563948c0464617465948c13323032342d30362d32355f31372d33322d3235948c1074696d655f746869735f697465725f7394474054ec10fa0000008c0c74696d655f746f74616c5f7394474054ec10fa0000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b018c146465636f6465725f72745f6e756d5f6c61796572944b028c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944d00018c0a656e636f6465725f6666944d00028c0d6465636f6465725f72745f6666944d00088c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b048c0964726f705f7261746594473fe56986c57443538c026c7294473f73618a8cf679de8c0a62617463685f73697a65944d0004758c1274696d655f73696e63655f726573746f726594474054ec10fa0000008c18697465726174696f6e735f73696e63655f726573746f7265944b01757562618c195f6c61746573745f636865636b706f696e745f726573756c7494681975622e\"\n }\n}"
+ ],
+ [
+ "{\n \"stub\": false,\n \"trainable_name\": \"train_model\",\n \"trial_id\": \"040fcf09\",\n \"storage\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059572040000000000008c1b7261792e747261696e2e5f696e7465726e616c2e73746f72616765948c0e53746f72616765436f6e746578749493942981947d94288c12637573746f6d5f66735f70726f766964656494898c1273746f726167655f6c6f63616c5f70617468948c2b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c7473948c136578706572696d656e745f6469725f6e616d65948c0f746573745f6578706572696d656e74948c0e747269616c5f6469725f6e616d65948c96747261696e5f6d6f64656c5f30343066636630395f385f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d3531322c6465636f6465725f72745f6e756d5f6c617965723d342c64726f705f72615f323032342d30362d32355f31372d33312d3031948c1863757272656e745f636865636b706f696e745f696e646578944b008c0b73796e635f636f6e666967948c097261792e747261696e948c0a53796e63436f6e6669679493942981947d94288c0b73796e635f706572696f64944d2c018c0c73796e635f74696d656f7574944d08078c0e73796e635f61727469666163747394898c1c73796e635f6172746966616374735f6f6e5f636865636b706f696e7494888c0a75706c6f61645f646972948c0a44455052454341544544948c0673796e6365729468188c1273796e635f6f6e5f636865636b706f696e7494681875628c1273746f726167655f66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c0f73746f726167655f66735f70617468948c452f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f7465737494681968008c115f46696c6573797374656d53796e6365729493942981947d9428681b682268134d2c0168144d08078c116c6173745f73796e635f75705f74696d65944741d99eb89fd0cfec8c136c6173745f73796e635f646f776e5f74696d659447fff00000000000008c0d5f73796e635f70726f63657373944e8c0c5f63757272656e745f636d649468008c125f75706c6f61645f746f5f66735f706174689493947d94288c0a6c6f63616c5f70617468948c3b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c74732f746573745f6578706572696d656e74948c0266739468228c0766735f70617468948c552f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e74948c076578636c756465945d948c0e2a2f636865636b706f696e745f2a9461758694756275622e\"\n },\n \"config\": {\n \"encoder_num_layer\": 1,\n \"decoder_rt_num_layer\": 4,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 16,\n \"encoder_ff\": 1024,\n \"decoder_rt_ff\": 512,\n \"decoder_int_ff\": 512,\n \"n_head\": 1,\n \"drop_rate\": 0.18759871111111892,\n \"lr\": 0.02750689103642202,\n \"batch_size\": 2048\n },\n \"_Trial__unresolved_config\": {\n \"encoder_num_layer\": 1,\n \"decoder_rt_num_layer\": 4,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 16,\n \"encoder_ff\": 1024,\n \"decoder_rt_ff\": 512,\n \"decoder_int_ff\": 512,\n \"n_head\": 1,\n \"drop_rate\": 0.18759871111111892,\n \"lr\": 0.02750689103642202,\n \"batch_size\": 2048\n },\n \"evaluated_params\": {\n \"batch_size\": 2048,\n \"decoder_int_ff\": 512,\n \"decoder_int_num_layer\": 1,\n \"decoder_rt_ff\": 512,\n \"decoder_rt_num_layer\": 4,\n \"drop_rate\": 0.18759871111111892,\n \"embedding_dim\": 16,\n \"encoder_ff\": 1024,\n \"encoder_num_layer\": 1,\n \"lr\": 0.02750689103642202,\n \"n_head\": 1\n },\n \"experiment_tag\": \"8_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=4,drop_rate=0.1876,embedding_dim=16,encoder_ff=1024,encoder_num_layer=1,lr=0.0275,n_head=1\",\n \"stopping_criterion\": {},\n \"_setup_default_resource\": true,\n \"_default_placement_group_factory\": \"80054e2e\",\n \"placement_group_factory\": \"800595ba000000000000008c237261792e74756e652e657865637574696f6e2e706c6163656d656e745f67726f757073948c15506c6163656d656e7447726f7570466163746f72799493942981947d94288c085f62756e646c6573945d947d94288c03435055944740540000000000008c034750559447401000000000000075618c155f686561645f62756e646c655f69735f656d70747994898c095f7374726174656779948c045041434b948c055f6172677394298c075f6b7761726773947d9475622e\",\n \"log_to_file\": [\n null,\n null\n ],\n \"max_failures\": 0,\n \"_default_result_or_future\": null,\n \"export_formats\": [],\n \"status\": \"PAUSED\",\n \"relative_logdir\": \"train_model_040fcf09_8_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=4,drop_ra_2024-06-25_17-31-01\",\n \"trial_name_creator\": null,\n \"trial_dirname_creator\": null,\n \"custom_trial_name\": null,\n \"custom_dirname\": null,\n \"restore_path\": null,\n \"_restore_checkpoint_result\": null,\n \"_state_json\": null,\n \"results\": \"80054e2e\",\n \"extra_arg\": \"80054e2e\",\n \"_resources\": \"80054e2e\"\n}",
+ "{\n \"start_time\": 1719329545.1895237,\n \"num_failures\": 0,\n \"num_failures_after_restore\": 0,\n \"error_filename\": null,\n \"pickled_error_filename\": null,\n \"last_result\": {\n \"loss\": 2019.4176592488927,\n \"timestamp\": 1719329610,\n \"checkpoint_dir_name\": \"checkpoint_000000\",\n \"should_checkpoint\": true,\n \"done\": false,\n \"training_iteration\": 1,\n \"trial_id\": \"040fcf09\",\n \"date\": \"2024-06-25_17-33-30\",\n \"time_this_iter_s\": 65.59718012809753,\n \"time_total_s\": 65.59718012809753,\n \"pid\": 2195253,\n \"hostname\": \"r8i6n2\",\n \"node_ip\": \"10.159.28.60\",\n \"config\": {\n \"encoder_num_layer\": 1,\n \"decoder_rt_num_layer\": 4,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 16,\n \"encoder_ff\": 1024,\n \"decoder_rt_ff\": 512,\n \"decoder_int_ff\": 512,\n \"n_head\": 1,\n \"drop_rate\": 0.18759871111111892,\n \"lr\": 0.02750689103642202,\n \"batch_size\": 2048\n },\n \"time_since_restore\": 65.59718012809753,\n \"iterations_since_restore\": 1,\n \"experiment_tag\": \"8_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=4,drop_rate=0.1876,embedding_dim=16,encoder_ff=1024,encoder_num_layer=1,lr=0.0275,n_head=1\"\n },\n \"last_result_time\": 1719329610.791616,\n \"metric_analysis\": {\n \"loss\": {\n \"max\": 2019.4176592488927,\n \"min\": 2019.4176592488927,\n \"avg\": 2019.4176592488927,\n \"last\": 2019.4176592488927,\n \"last-5-avg\": 2019.4176592488927,\n \"last-10-avg\": 2019.4176592488927\n },\n \"should_checkpoint\": {\n \"max\": true,\n \"min\": true,\n \"avg\": true,\n \"last\": true,\n \"last-5-avg\": true,\n \"last-10-avg\": true\n },\n \"done\": {\n \"max\": false,\n \"min\": false,\n \"avg\": false,\n \"last\": false,\n \"last-5-avg\": false,\n \"last-10-avg\": false\n },\n \"training_iteration\": {\n \"max\": 1,\n \"min\": 1,\n \"avg\": 1,\n \"last\": 1,\n \"last-5-avg\": 1,\n \"last-10-avg\": 1\n },\n \"time_this_iter_s\": {\n \"max\": 65.59718012809753,\n \"min\": 65.59718012809753,\n \"avg\": 65.59718012809753,\n \"last\": 65.59718012809753,\n \"last-5-avg\": 65.59718012809753,\n \"last-10-avg\": 65.59718012809753\n },\n \"time_total_s\": {\n \"max\": 65.59718012809753,\n \"min\": 65.59718012809753,\n \"avg\": 65.59718012809753,\n \"last\": 65.59718012809753,\n \"last-5-avg\": 65.59718012809753,\n \"last-10-avg\": 65.59718012809753\n },\n \"time_since_restore\": {\n \"max\": 65.59718012809753,\n \"min\": 65.59718012809753,\n \"avg\": 65.59718012809753,\n \"last\": 65.59718012809753,\n \"last-5-avg\": 65.59718012809753,\n \"last-10-avg\": 65.59718012809753\n },\n \"iterations_since_restore\": {\n \"max\": 1,\n \"min\": 1,\n \"avg\": 1,\n \"last\": 1,\n \"last-5-avg\": 1,\n \"last-10-avg\": 1\n }\n },\n \"_n_steps\": [\n 5,\n 10\n ],\n \"metric_n_steps\": {\n \"loss\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059589000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452948c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b00749462430877bbddaeab8d9f409486945294612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059589000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452948c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b00749462430877bbddaeab8d9f409486945294612e\"\n }\n },\n \"should_checkpoint\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059522000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b058694529488612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059522000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a8694529488612e\"\n }\n },\n \"done\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059522000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b058694529489612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059522000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a8694529489612e\"\n }\n },\n \"training_iteration\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059523000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452944b01612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059523000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452944b01612e\"\n }\n },\n \"time_this_iter_s\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294474050663833000000612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294474050663833000000612e\"\n }\n },\n \"time_total_s\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294474050663833000000612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294474050663833000000612e\"\n }\n },\n \"time_since_restore\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294474050663833000000612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294474050663833000000612e\"\n }\n },\n \"iterations_since_restore\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059523000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452944b01612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059523000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452944b01612e\"\n }\n }\n },\n \"checkpoint_manager\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059518060000000000008c267261792e747261696e2e5f696e7465726e616c2e636865636b706f696e745f6d616e61676572948c125f436865636b706f696e744d616e616765729493942981947d94288c125f636865636b706f696e745f636f6e666967948c097261792e747261696e948c10436865636b706f696e74436f6e6669679493942981947d94288c0b6e756d5f746f5f6b656570944e8c1a636865636b706f696e745f73636f72655f617474726962757465944e8c16636865636b706f696e745f73636f72655f6f72646572948c036d6178948c14636865636b706f696e745f6672657175656e6379944b008c11636865636b706f696e745f61745f656e6494898c1a5f636865636b706f696e745f6b6565705f616c6c5f72616e6b73948c0a44455052454341544544948c1f5f636865636b706f696e745f75706c6f61645f66726f6d5f776f726b65727394681275628c135f636865636b706f696e745f726573756c7473945d948c1b7261792e747261696e2e5f696e7465726e616c2e73657373696f6e948c0f5f547261696e696e67526573756c749493942981947d94288c0a636865636b706f696e749468068c0a436865636b706f696e749493942981947d94288c0470617468948cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f30343066636630395f385f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d3531322c6465636f6465725f72745f6e756d5f6c617965723d342c64726f705f72615f323032342d30362d32355f31372d33312d30312f636865636b706f696e745f303030303030948c0a66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c055f75756964948c0475756964948c04555549449493942981947d948c03696e74948a11fc7754c809abffae8d4f2132ea00c58200736275628c076d657472696373947d94288c046c6f7373948c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b00749462430877bbddaeab8d9f4094869452948c0974696d657374616d70944a4ae37a668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303030948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b018c08747269616c5f6964948c083034306663663039948c0464617465948c13323032342d30362d32355f31372d33332d3330948c1074696d655f746869735f697465725f73944740506638330000008c0c74696d655f746f74616c5f73944740506638330000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b018c146465636f6465725f72745f6e756d5f6c61796572944b048c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b108c0a656e636f6465725f6666944d00048c0d6465636f6465725f72745f6666944d00028c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b018c0964726f705f7261746594473fc8033c0c7f3ba48c026c7294473f9c2ac435aa0dc38c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f7265944740506638330000008c18697465726174696f6e735f73696e63655f726573746f7265944b01757562618c195f6c61746573745f636865636b706f696e745f726573756c7494681975622e\"\n }\n}"
+ ],
+ [
+ "{\n \"stub\": false,\n \"trainable_name\": \"train_model\",\n \"trial_id\": \"934c5104\",\n \"storage\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059572040000000000008c1b7261792e747261696e2e5f696e7465726e616c2e73746f72616765948c0e53746f72616765436f6e746578749493942981947d94288c12637573746f6d5f66735f70726f766964656494898c1273746f726167655f6c6f63616c5f70617468948c2b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c7473948c136578706572696d656e745f6469725f6e616d65948c0f746573745f6578706572696d656e74948c0e747269616c5f6469725f6e616d65948c96747261696e5f6d6f64656c5f39333463353130345f395f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d313032342c6465636f6465725f72745f6e756d5f6c617965723d322c64726f705f725f323032342d30362d32355f31372d33322d3235948c1863757272656e745f636865636b706f696e745f696e646578944b0d8c0b73796e635f636f6e666967948c097261792e747261696e948c0a53796e63436f6e6669679493942981947d94288c0b73796e635f706572696f64944d2c018c0c73796e635f74696d656f7574944d08078c0e73796e635f61727469666163747394898c1c73796e635f6172746966616374735f6f6e5f636865636b706f696e7494888c0a75706c6f61645f646972948c0a44455052454341544544948c0673796e6365729468188c1273796e635f6f6e5f636865636b706f696e7494681875628c1273746f726167655f66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c0f73746f726167655f66735f70617468948c452f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f7465737494681968008c115f46696c6573797374656d53796e6365729493942981947d9428681b682268134d2c0168144d08078c116c6173745f73796e635f75705f74696d65944741d99eb89fd0cfec8c136c6173745f73796e635f646f776e5f74696d659447fff00000000000008c0d5f73796e635f70726f63657373944e8c0c5f63757272656e745f636d649468008c125f75706c6f61645f746f5f66735f706174689493947d94288c0a6c6f63616c5f70617468948c3b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c74732f746573745f6578706572696d656e74948c0266739468228c0766735f70617468948c552f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e74948c076578636c756465945d948c0e2a2f636865636b706f696e745f2a9461758694756275622e\"\n },\n \"config\": {\n \"encoder_num_layer\": 1,\n \"decoder_rt_num_layer\": 2,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 256,\n \"encoder_ff\": 2048,\n \"decoder_rt_ff\": 1024,\n \"decoder_int_ff\": 512,\n \"n_head\": 8,\n \"drop_rate\": 0.2811887966312956,\n \"lr\": 0.0005389075253520159,\n \"batch_size\": 2048\n },\n \"_Trial__unresolved_config\": {\n \"encoder_num_layer\": 1,\n \"decoder_rt_num_layer\": 2,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 256,\n \"encoder_ff\": 2048,\n \"decoder_rt_ff\": 1024,\n \"decoder_int_ff\": 512,\n \"n_head\": 8,\n \"drop_rate\": 0.2811887966312956,\n \"lr\": 0.0005389075253520159,\n \"batch_size\": 2048\n },\n \"evaluated_params\": {\n \"batch_size\": 2048,\n \"decoder_int_ff\": 512,\n \"decoder_int_num_layer\": 1,\n \"decoder_rt_ff\": 1024,\n \"decoder_rt_num_layer\": 2,\n \"drop_rate\": 0.2811887966312956,\n \"embedding_dim\": 256,\n \"encoder_ff\": 2048,\n \"encoder_num_layer\": 1,\n \"lr\": 0.0005389075253520159,\n \"n_head\": 8\n },\n \"experiment_tag\": \"9_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop_rate=0.2812,embedding_dim=256,encoder_ff=2048,encoder_num_layer=1,lr=0.0005,n_head=8\",\n \"stopping_criterion\": {},\n \"_setup_default_resource\": true,\n \"_default_placement_group_factory\": \"80054e2e\",\n \"placement_group_factory\": \"800595ba000000000000008c237261792e74756e652e657865637574696f6e2e706c6163656d656e745f67726f757073948c15506c6163656d656e7447726f7570466163746f72799493942981947d94288c085f62756e646c6573945d947d94288c03435055944740540000000000008c034750559447401000000000000075618c155f686561645f62756e646c655f69735f656d70747994898c095f7374726174656779948c045041434b948c055f6172677394298c075f6b7761726773947d9475622e\",\n \"log_to_file\": [\n null,\n null\n ],\n \"max_failures\": 0,\n \"_default_result_or_future\": null,\n \"export_formats\": [],\n \"status\": \"TERMINATED\",\n \"relative_logdir\": \"train_model_934c5104_9_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop_r_2024-06-25_17-32-25\",\n \"trial_name_creator\": null,\n \"trial_dirname_creator\": null,\n \"custom_trial_name\": null,\n \"custom_dirname\": null,\n \"restore_path\": null,\n \"_restore_checkpoint_result\": null,\n \"_state_json\": null,\n \"results\": \"80054e2e\",\n \"extra_arg\": \"80054e2e\",\n \"_resources\": \"80054e2e\"\n}",
+ "{\n \"start_time\": 1719329610.941942,\n \"num_failures\": 0,\n \"num_failures_after_restore\": 0,\n \"error_filename\": null,\n \"pickled_error_filename\": null,\n \"last_result\": {\n \"loss\": 98.24798722154512,\n \"timestamp\": 1719332767,\n \"checkpoint_dir_name\": \"checkpoint_000013\",\n \"should_checkpoint\": true,\n \"done\": true,\n \"training_iteration\": 14,\n \"trial_id\": \"934c5104\",\n \"date\": \"2024-06-25_18-26-08\",\n \"time_this_iter_s\": 68.14825201034546,\n \"time_total_s\": 1037.901340007782,\n \"pid\": 2195253,\n \"hostname\": \"r8i6n2\",\n \"node_ip\": \"10.159.28.60\",\n \"config\": {\n \"encoder_num_layer\": 1,\n \"decoder_rt_num_layer\": 2,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 256,\n \"encoder_ff\": 2048,\n \"decoder_rt_ff\": 1024,\n \"decoder_int_ff\": 512,\n \"n_head\": 8,\n \"drop_rate\": 0.2811887966312956,\n \"lr\": 0.0005389075253520159,\n \"batch_size\": 2048\n },\n \"time_since_restore\": 696.4155676364899,\n \"iterations_since_restore\": 10,\n \"experiment_tag\": \"9_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop_rate=0.2812,embedding_dim=256,encoder_ff=2048,encoder_num_layer=1,lr=0.0005,n_head=8\"\n },\n \"last_result_time\": 1719332768.0763395,\n \"metric_analysis\": {\n \"loss\": {\n \"max\": 234.06637176753966,\n \"min\": 98.24798722154512,\n \"avg\": 132.7836964438817,\n \"last\": 98.24798722154512,\n \"last-5-avg\": 102.28564795546644,\n \"last-10-avg\": 110.75848803783026\n },\n \"should_checkpoint\": {\n \"max\": true,\n \"min\": true,\n \"avg\": 1.0,\n \"last\": true,\n \"last-5-avg\": 1.0,\n \"last-10-avg\": 1.0\n },\n \"done\": {\n \"max\": false,\n \"min\": false,\n \"avg\": 0.0,\n \"last\": false,\n \"last-5-avg\": 0.0,\n \"last-10-avg\": 0.0\n },\n \"training_iteration\": {\n \"max\": 14,\n \"min\": 1,\n \"avg\": 7.5,\n \"last\": 14,\n \"last-5-avg\": 12.0,\n \"last-10-avg\": 9.5\n },\n \"time_this_iter_s\": {\n \"max\": 95.2866895198822,\n \"min\": 67.7198805809021,\n \"avg\": 74.13581000055585,\n \"last\": 68.14825201034546,\n \"last-5-avg\": 68.35834956169128,\n \"last-10-avg\": 69.64155676364899\n },\n \"time_total_s\": {\n \"max\": 1037.901340007782,\n \"min\": 82.2380793094635,\n \"avg\": 582.5065506185805,\n \"last\": 1037.901340007782,\n \"last-5-avg\": 901.4340446949005,\n \"last-10-avg\": 730.7140613794327\n },\n \"time_since_restore\": {\n \"max\": 696.4155676364899,\n \"min\": 82.23219752311707,\n \"avg\": 309.21823942661285,\n \"last\": 696.4155676364899,\n \"last-5-avg\": 559.9482723236084,\n \"last-10-avg\": 389.22828900814056\n },\n \"iterations_since_restore\": {\n \"max\": 10,\n \"min\": 1,\n \"avg\": 4.285714285714286,\n \"last\": 10,\n \"last-5-avg\": 8.0,\n \"last-10-avg\": 5.5\n }\n },\n \"_n_steps\": [\n 5,\n 10\n ],\n \"metric_n_steps\": {\n \"loss\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"800595d6000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294288c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b00749462430832994c86a42a5b4094869452946807680d430863b1586c5982594094869452946807680d4308653299ac3316594094869452946807680d4308168bc5a25788594094869452946807680d43082e97cb05df8f58409486945294652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059535010000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294288c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b00749462430853a9547a09bf614094869452946807680d43084fa7d389576d5d4094869452946807680d430847a3d12863595c4094869452946807680d43089bcd661342ee5b4094869452946807680d43085d2e97abf6d65b4094869452946807680d430832994c86a42a5b4094869452946807680d430863b1586c5982594094869452946807680d4308653299ac3316594094869452946807680d4308168bc5a25788594094869452946807680d43082e97cb05df8f58409486945294652e\"\n }\n },\n \"should_checkpoint\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059527000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294288888888888652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952c000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452942888888888888888888888652e\"\n }\n },\n \"done\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059527000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294288989898989652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952c000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452942889898989898989898989652e\"\n }\n },\n \"training_iteration\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952c000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294284b0a4b0b4b0c4b0d4b0e652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059536000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294284b054b064b074b084b094b0a4b0b4b0c4b0d4b0e652e\"\n }\n },\n \"time_this_iter_s\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005954f000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294284740513f6b2000000047405103f21400000047405105d2960000004740511fff3f000000474051097cf6000000652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005957c000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452942847405499f0d5000000474051054b19000000474050ee128600000047405126f212000000474050f3ac240000004740513f6b2000000047405103f21400000047405105d2960000004740511fff3f000000474051097cf6000000652e\"\n }\n },\n \"time_total_s\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005954f000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b058694529428474087e8cdd5e0000047408a094c1860000047408c2a066b20000047408e4e0653000000474090379af8e00000652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005957c000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452942847407a7e41ee80000047407ebf94b4c000004740817d8cab200000474083a26aed600000474085c0e071e00000474087e8cdd5e0000047408a094c1860000047408c2a066b20000047408e4e0653000000474090379af8e00000652e\"\n }\n },\n \"time_since_restore\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005954f000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452942847407a79d5f280000047407ebad2778000004740817e238e800000474083a22376600000474085c35315200000652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005957c000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452942847405499f0d5000000474062cf9df700000047406b46a73a000000474071ed102180000047407629fb2a80000047407a79d5f280000047407ebad2778000004740817e238e800000474083a22376600000474085c35315200000652e\"\n }\n },\n \"iterations_since_restore\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952c000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294284b064b074b084b094b0a652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059536000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294284b014b024b034b044b054b064b074b084b094b0a652e\"\n }\n }\n },\n \"checkpoint_manager\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005954d340000000000008c267261792e747261696e2e5f696e7465726e616c2e636865636b706f696e745f6d616e61676572948c125f436865636b706f696e744d616e616765729493942981947d94288c125f636865636b706f696e745f636f6e666967948c097261792e747261696e948c10436865636b706f696e74436f6e6669679493942981947d94288c0b6e756d5f746f5f6b656570944e8c1a636865636b706f696e745f73636f72655f617474726962757465944e8c16636865636b706f696e745f73636f72655f6f72646572948c036d6178948c14636865636b706f696e745f6672657175656e6379944b008c11636865636b706f696e745f61745f656e6494898c1a5f636865636b706f696e745f6b6565705f616c6c5f72616e6b73948c0a44455052454341544544948c1f5f636865636b706f696e745f75706c6f61645f66726f6d5f776f726b65727394681275628c135f636865636b706f696e745f726573756c7473945d94288c1b7261792e747261696e2e5f696e7465726e616c2e73657373696f6e948c0f5f547261696e696e67526573756c749493942981947d94288c0a636865636b706f696e749468068c0a436865636b706f696e749493942981947d94288c0470617468948cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f39333463353130345f395f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d313032342c6465636f6465725f72745f6e756d5f6c617965723d322c64726f705f725f323032342d30362d32355f31372d33322d32352f636865636b706f696e745f303030303030948c0a66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c055f75756964948c0475756964948c04555549449493942981947d948c03696e74948a113ca5f21f44513f9f654790e892756aaa00736275628c076d657472696373947d94288c046c6f7373948c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b007494624308bd5eafb71f426d4094869452948c0974696d657374616d70944a9de37a668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303030948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b018c08747269616c5f6964948c083933346335313034948c0464617465948c13323032342d30362d32355f31372d33342d3533948c1074696d655f746869735f697465725f73944740548f3cb10000008c0c74696d655f746f74616c5f73944740548f3cb10000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b018c146465636f6465725f72745f6e756d5f6c61796572944b028c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944d00018c0a656e636f6465725f6666944d00088c0d6465636f6465725f72745f6666944d00048c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b088c0964726f705f7261746594473fd1feff4b621cd48c026c7294473f41a8af1935d3f68c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f7265944740548f3cb10000008c18697465726174696f6e735f73696e63655f726573746f7265944b0175756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f39333463353130345f395f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d313032342c6465636f6465725f72745f6e756d5f6c617965723d322c64726f705f725f323032342d30362d32355f31372d33322d32352f636865636b706f696e745f30303030303194682268257d946827897385945294682a682d2981947d9468308a117ca28ae951829cbc5b4c9022326fceab007362756268317d94288c046c6f7373946836683c43084fa7d3690fe9674094869452948c0974696d657374616d70944ac2e77a668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303031948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b028c08747269616c5f6964948c083933346335313034948c0464617465948c13323032342d30362d32355f31372d35322d3334948c1074696d655f746869735f697465725f73944740548edc530000008c0c74696d655f746f74616c5f73944740648f0c820000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b018c146465636f6465725f72745f6e756d5f6c61796572944b028c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944d00018c0a656e636f6465725f6666944d00088c0d6465636f6465725f72745f6666944d00048c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b088c0964726f705f7261746594473fd1feff4b621cd48c026c7294473f41a8af1935d3f68c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f7265944740548edc530000008c18697465726174696f6e735f73696e63655f726573746f7265944b0175756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f39333463353130345f395f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d313032342c6465636f6465725f72745f6e756d5f6c617965723d322c64726f705f725f323032342d30362d32355f31372d33322d32352f636865636b706f696e745f30303030303294682268257d946827897385945294682a682d2981947d9468308a11f8ffe8fb191278bdc64dd54f425d0895007362756268317d94288c046c6f7373946836683c4308592c166bdec1644094869452948c0974696d657374616d70944a31ea7a668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303032948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b038c08747269616c5f6964948c083933346335313034948c0464617465948c13323032342d30362d32355f31382d30322d3537948c1074696d655f746869735f697465725f7394474057d2591f0000008c0c74696d655f746f74616c5f73944740703c1c88c000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b018c146465636f6465725f72745f6e756d5f6c61796572944b028c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944d00018c0a656e636f6465725f6666944d00088c0d6465636f6465725f72745f6666944d00048c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b088c0964726f705f7261746594473fd1feff4b621cd48c026c7294473f41a8af1935d3f68c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f726594474057d2591f0000008c18697465726174696f6e735f73696e63655f726573746f7265944b0175756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f39333463353130345f395f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d313032342c6465636f6465725f72745f6e756d5f6c617965723d322c64726f705f725f323032342d30362d32355f31372d33322d32352f636865636b706f696e745f30303030303394682268257d946827897385945294682a682d2981947d9468308a102377a72e3654ecaa034308385248700f7362756268317d94288c046c6f7373946836683c4308028140b053ff634094869452948c0974696d657374616d70944a83ea7a668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303033948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b048c08747269616c5f6964948c083933346335313034948c0464617465948c13323032342d30362d32355f31382d30342d3139948c1074696d655f746869735f697465725f73944740546ea4c20000008c0c74696d655f746f74616c5f739447407557c5b94000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b018c146465636f6465725f72745f6e756d5f6c61796572944b028c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944d00018c0a656e636f6465725f6666944d00088c0d6465636f6465725f72745f6666944d00048c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b088c0964726f705f7261746594473fd1feff4b621cd48c026c7294473f41a8af1935d3f68c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f726594474066207ef08000008c18697465726174696f6e735f73696e63655f726573746f7265944b0275756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f39333463353130345f395f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d313032342c6465636f6465725f72745f6e756d5f6c617965723d322c64726f705f725f323032342d30362d32355f31372d33322d32352f636865636b706f696e745f30303030303494682268257d946827897385945294682a682d2981947d9468308a10e4fddd941927c4a8ec4f33d7e5fd991c7362756268317d94288c046c6f7373946836683c430853a9547a09bf614094869452948c0974696d657374616d70944a39ed7a668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303034948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b058c08747269616c5f6964948c083933346335313034948c0464617465948c13323032342d30362d32355f31382d31352d3534948c1074696d655f746869735f697465725f739447405499f0d50000008c0c74696d655f746f74616c5f739447407a7e41ee8000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b018c146465636f6465725f72745f6e756d5f6c61796572944b028c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944d00018c0a656e636f6465725f6666944d00088c0d6465636f6465725f72745f6666944d00048c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b088c0964726f705f7261746594473fd1feff4b621cd48c026c7294473f41a8af1935d3f68c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f72659447405499f0d50000008c18697465726174696f6e735f73696e63655f726573746f7265944b0175756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f39333463353130345f395f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d313032342c6465636f6465725f72745f6e756d5f6c617965723d322c64726f705f725f323032342d30362d32355f31372d33322d32352f636865636b706f696e745f30303030303594682268257d946827897385945294682a682d2981947d9468308a1121aa1fe7b3527aa95a439d1c8db5f0a5007362756268317d94288c046c6f7373946836683c43084fa7d389576d5d4094869452948c0974696d657374616d70944a7eed7a668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303035948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b068c08747269616c5f6964948c083933346335313034948c0464617465948c13323032342d30362d32355f31382d31372d3032948c1074696d655f746869735f697465725f7394474051054b190000008c0c74696d655f746f74616c5f739447407ebf94b4c000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b018c146465636f6465725f72745f6e756d5f6c61796572944b028c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944d00018c0a656e636f6465725f6666944d00088c0d6465636f6465725f72745f6666944d00048c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b088c0964726f705f7261746594473fd1feff4b621cd48c026c7294473f41a8af1935d3f68c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f726594474062cf9df70000008c18697465726174696f6e735f73696e63655f726573746f7265944b0275756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f39333463353130345f395f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d313032342c6465636f6465725f72745f6e756d5f6c617965723d322c64726f705f725f323032342d30362d32355f31372d33322d32352f636865636b706f696e745f30303030303694682268257d946827897385945294682a682d2981947d9468308a11c74c9ef6372bbdbe514cde6a5c8f09ff007362756268317d94288c046c6f7373946836683c430847a3d12863595c4094869452948c0974696d657374616d70944ac1ed7a668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303036948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b078c08747269616c5f6964948c083933346335313034948c0464617465948c13323032342d30362d32355f31382d31382d3039948c1074696d655f746869735f697465725f7394474050ee12860000008c0c74696d655f746f74616c5f73944740817d8cab2000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b018c146465636f6465725f72745f6e756d5f6c61796572944b028c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944d00018c0a656e636f6465725f6666944d00088c0d6465636f6465725f72745f6666944d00048c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b088c0964726f705f7261746594473fd1feff4b621cd48c026c7294473f41a8af1935d3f68c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f72659447406b46a73a0000008c18697465726174696f6e735f73696e63655f726573746f7265944b0375756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f39333463353130345f395f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d313032342c6465636f6465725f72745f6e756d5f6c617965723d322c64726f705f725f323032342d30362d32355f31372d33322d32352f636865636b706f696e745f30303030303794682268257d946827897385945294682a682d2981947d9468308a1053bcef571437bd9cc04adacd19b3473d7362756268317d94288c046c6f7373946836683c43089bcd661342ee5b4094869452948c0974696d657374616d70944a06ee7a668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303037948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b088c08747269616c5f6964948c083933346335313034948c0464617465948c13323032342d30362d32355f31382d31392d3138948c1074696d655f746869735f697465725f739447405126f2120000008c0c74696d655f746f74616c5f7394474083a26aed6000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b018c146465636f6465725f72745f6e756d5f6c61796572944b028c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944d00018c0a656e636f6465725f6666944d00088c0d6465636f6465725f72745f6666944d00048c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b088c0964726f705f7261746594473fd1feff4b621cd48c026c7294473f41a8af1935d3f68c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f726594474071ed10218000008c18697465726174696f6e735f73696e63655f726573746f7265944b0475756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f39333463353130345f395f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d313032342c6465636f6465725f72745f6e756d5f6c617965723d322c64726f705f725f323032342d30362d32355f31372d33322d32352f636865636b706f696e745f30303030303894682268257d946827897385945294682a682d2981947d9468308a1195212ae085dbfb82a54a82fe466abbc1007362756268317d94288c046c6f7373946836683c43085d2e97abf6d65b4094869452948c0974696d657374616d70944a4aee7a668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303038948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b098c08747269616c5f6964948c083933346335313034948c0464617465948c13323032342d30362d32355f31382d32302d3236948c1074696d655f746869735f697465725f7394474050f3ac240000008c0c74696d655f746f74616c5f7394474085c0e071e000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b018c146465636f6465725f72745f6e756d5f6c61796572944b028c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944d00018c0a656e636f6465725f6666944d00088c0d6465636f6465725f72745f6666944d00048c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b088c0964726f705f7261746594473fd1feff4b621cd48c026c7294473f41a8af1935d3f68c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f72659447407629fb2a8000008c18697465726174696f6e735f73696e63655f726573746f7265944b0575756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f39333463353130345f395f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d313032342c6465636f6465725f72745f6e756d5f6c617965723d322c64726f705f725f323032342d30362d32355f31372d33322d32352f636865636b706f696e745f30303030303994682268257d946827897385945294682a682d2981947d9468308a10ee3248a264cfe3a4c54fdc972eeda71b7362756268317d94288c046c6f7373946836683c430832994c86a42a5b4094869452948c0974696d657374616d70944a8eee7a668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303039948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b0a8c08747269616c5f6964948c083933346335313034948c0464617465948c13323032342d30362d32355f31382d32312d3335948c1074696d655f746869735f697465725f73944740513f6b200000008c0c74696d655f746f74616c5f7394474087e8cdd5e000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b018c146465636f6465725f72745f6e756d5f6c61796572944b028c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944d00018c0a656e636f6465725f6666944d00088c0d6465636f6465725f72745f6666944d00048c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b088c0964726f705f7261746594473fd1feff4b621cd48c026c7294473f41a8af1935d3f68c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f72659447407a79d5f28000008c18697465726174696f6e735f73696e63655f726573746f7265944b0675756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f39333463353130345f395f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d313032342c6465636f6465725f72745f6e756d5f6c617965723d322c64726f705f725f323032342d30362d32355f31372d33322d32352f636865636b706f696e745f30303030313094682268257d946827897385945294682a682d2981947d9468308a11ddb581b325cb91b0374ddf2264a944d4007362756268317d94288c046c6f7373946836683c430863b1586c5982594094869452948c0974696d657374616d70944ad3ee7a668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303130948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b0b8c08747269616c5f6964948c083933346335313034948c0464617465948c13323032342d30362d32355f31382d32322d3433948c1074696d655f746869735f697465725f739447405103f2140000008c0c74696d655f746f74616c5f739447408a094c186000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b018c146465636f6465725f72745f6e756d5f6c61796572944b028c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944d00018c0a656e636f6465725f6666944d00088c0d6465636f6465725f72745f6666944d00048c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b088c0964726f705f7261746594473fd1feff4b621cd48c026c7294473f41a8af1935d3f68c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f72659447407ebad2778000008c18697465726174696f6e735f73696e63655f726573746f7265944b0775756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f39333463353130345f395f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d313032342c6465636f6465725f72745f6e756d5f6c617965723d322c64726f705f725f323032342d30362d32355f31372d33322d32352f636865636b706f696e745f30303030313194682268257d946827897385945294682a682d2981947d9468308a10e7b8684994fe77a9a34e7f60393270667362756268317d94288c046c6f7373946836683c4308653299ac3316594094869452948c0974696d657374616d70944a17ef7a668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303131948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b0c8c08747269616c5f6964948c083933346335313034948c0464617465948c13323032342d30362d32355f31382d32332d3531948c1074696d655f746869735f697465725f739447405105d2960000008c0c74696d655f746f74616c5f739447408c2a066b2000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b018c146465636f6465725f72745f6e756d5f6c61796572944b028c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944d00018c0a656e636f6465725f6666944d00088c0d6465636f6465725f72745f6666944d00048c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b088c0964726f705f7261746594473fd1feff4b621cd48c026c7294473f41a8af1935d3f68c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f7265944740817e238e8000008c18697465726174696f6e735f73696e63655f726573746f7265944b0875756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f39333463353130345f395f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d313032342c6465636f6465725f72745f6e756d5f6c617965723d322c64726f705f725f323032342d30362d32355f31372d33322d32352f636865636b706f696e745f30303030313294682268257d946827897385945294682a682d2981947d9468308a1155973d114de961a7f24898e54966f6d9007362756268317d94288c046c6f7373946836683c4308168bc5a25788594094869452948c0974696d657374616d70944a5bef7a668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303132948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b0d8c08747269616c5f6964948c083933346335313034948c0464617465948c13323032342d30362d32355f31382d32342d3539948c1074696d655f746869735f697465725f73944740511fff3f0000008c0c74696d655f746f74616c5f739447408e4e06530000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b018c146465636f6465725f72745f6e756d5f6c61796572944b028c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944d00018c0a656e636f6465725f6666944d00088c0d6465636f6465725f72745f6666944d00048c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b088c0964726f705f7261746594473fd1feff4b621cd48c026c7294473f41a8af1935d3f68c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f726594474083a223766000008c18697465726174696f6e735f73696e63655f726573746f7265944b0975756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f39333463353130345f395f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d313032342c6465636f6465725f72745f6e756d5f6c617965723d322c64726f705f725f323032342d30362d32355f31372d33322d32352f636865636b706f696e745f30303030313394682268257d946827897385945294682a682d2981947d9468308a10cb0a04e2aef0c18bb140650baea389707362756268317d94288c046c6f7373946836683c43082e97cb05df8f584094869452948c0974696d657374616d70944a9fef7a668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303133948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b0e8c08747269616c5f6964948c083933346335313034948c0464617465948c13323032342d30362d32355f31382d32362d3038948c1074696d655f746869735f697465725f7394474051097cf60000008c0c74696d655f746f74616c5f7394474090379af8e000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b018c146465636f6465725f72745f6e756d5f6c61796572944b028c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944d00018c0a656e636f6465725f6666944d00088c0d6465636f6465725f72745f6666944d00048c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b088c0964726f705f7261746594473fd1feff4b621cd48c026c7294473f41a8af1935d3f68c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f726594474085c353152000008c18697465726174696f6e735f73696e63655f726573746f7265944b0a757562658c195f6c61746573745f636865636b706f696e745f726573756c74946a9602000075622e\"\n }\n}"
+ ],
+ [
+ "{\n \"stub\": false,\n \"trainable_name\": \"train_model\",\n \"trial_id\": \"481f1fff\",\n \"storage\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059572040000000000008c1b7261792e747261696e2e5f696e7465726e616c2e73746f72616765948c0e53746f72616765436f6e746578749493942981947d94288c12637573746f6d5f66735f70726f766964656494898c1273746f726167655f6c6f63616c5f70617468948c2b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c7473948c136578706572696d656e745f6469725f6e616d65948c0f746573745f6578706572696d656e74948c0e747269616c5f6469725f6e616d65948c96747261696e5f6d6f64656c5f34383166316666665f31305f62617463685f73697a653d313032342c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d313032342c6465636f6465725f72745f6e756d5f6c617965723d382c64726f705f5f323032342d30362d32355f31372d33332d3330948c1863757272656e745f636865636b706f696e745f696e646578944b018c0b73796e635f636f6e666967948c097261792e747261696e948c0a53796e63436f6e6669679493942981947d94288c0b73796e635f706572696f64944d2c018c0c73796e635f74696d656f7574944d08078c0e73796e635f61727469666163747394898c1c73796e635f6172746966616374735f6f6e5f636865636b706f696e7494888c0a75706c6f61645f646972948c0a44455052454341544544948c0673796e6365729468188c1273796e635f6f6e5f636865636b706f696e7494681875628c1273746f726167655f66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c0f73746f726167655f66735f70617468948c452f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f7465737494681968008c115f46696c6573797374656d53796e6365729493942981947d9428681b682268134d2c0168144d08078c116c6173745f73796e635f75705f74696d65944741d99eb89fd0cfec8c136c6173745f73796e635f646f776e5f74696d659447fff00000000000008c0d5f73796e635f70726f63657373944e8c0c5f63757272656e745f636d649468008c125f75706c6f61645f746f5f66735f706174689493947d94288c0a6c6f63616c5f70617468948c3b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c74732f746573745f6578706572696d656e74948c0266739468228c0766735f70617468948c552f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e74948c076578636c756465945d948c0e2a2f636865636b706f696e745f2a9461758694756275622e\"\n },\n \"config\": {\n \"encoder_num_layer\": 2,\n \"decoder_rt_num_layer\": 8,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 64,\n \"encoder_ff\": 1024,\n \"decoder_rt_ff\": 1024,\n \"decoder_int_ff\": 512,\n \"n_head\": 16,\n \"drop_rate\": 0.15995138350968297,\n \"lr\": 0.004656630423530019,\n \"batch_size\": 1024\n },\n \"_Trial__unresolved_config\": {\n \"encoder_num_layer\": 2,\n \"decoder_rt_num_layer\": 8,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 64,\n \"encoder_ff\": 1024,\n \"decoder_rt_ff\": 1024,\n \"decoder_int_ff\": 512,\n \"n_head\": 16,\n \"drop_rate\": 0.15995138350968297,\n \"lr\": 0.004656630423530019,\n \"batch_size\": 1024\n },\n \"evaluated_params\": {\n \"batch_size\": 1024,\n \"decoder_int_ff\": 512,\n \"decoder_int_num_layer\": 1,\n \"decoder_rt_ff\": 1024,\n \"decoder_rt_num_layer\": 8,\n \"drop_rate\": 0.15995138350968297,\n \"embedding_dim\": 64,\n \"encoder_ff\": 1024,\n \"encoder_num_layer\": 2,\n \"lr\": 0.004656630423530019,\n \"n_head\": 16\n },\n \"experiment_tag\": \"10_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=8,drop_rate=0.1600,embedding_dim=64,encoder_ff=1024,encoder_num_layer=2,lr=0.0047,n_head=16\",\n \"stopping_criterion\": {},\n \"_setup_default_resource\": true,\n \"_default_placement_group_factory\": \"80054e2e\",\n \"placement_group_factory\": \"800595ba000000000000008c237261792e74756e652e657865637574696f6e2e706c6163656d656e745f67726f757073948c15506c6163656d656e7447726f7570466163746f72799493942981947d94288c085f62756e646c6573945d947d94288c03435055944740540000000000008c034750559447401000000000000075618c155f686561645f62756e646c655f69735f656d70747994898c095f7374726174656779948c045041434b948c055f6172677394298c075f6b7761726773947d9475622e\",\n \"log_to_file\": [\n null,\n null\n ],\n \"max_failures\": 0,\n \"_default_result_or_future\": null,\n \"export_formats\": [],\n \"status\": \"PAUSED\",\n \"relative_logdir\": \"train_model_481f1fff_10_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=8,drop__2024-06-25_17-33-30\",\n \"trial_name_creator\": null,\n \"trial_dirname_creator\": null,\n \"custom_trial_name\": null,\n \"custom_dirname\": null,\n \"restore_path\": null,\n \"_restore_checkpoint_result\": null,\n \"_state_json\": null,\n \"results\": \"80054e2e\",\n \"extra_arg\": \"80054e2e\",\n \"_resources\": \"80054e2e\"\n}",
+ "{\n \"start_time\": 1719329693.3274555,\n \"num_failures\": 0,\n \"num_failures_after_restore\": 0,\n \"error_filename\": null,\n \"pickled_error_filename\": null,\n \"last_result\": {\n \"loss\": 1959.9800554410679,\n \"timestamp\": 1719330869,\n \"checkpoint_dir_name\": \"checkpoint_000001\",\n \"should_checkpoint\": true,\n \"done\": false,\n \"training_iteration\": 2,\n \"trial_id\": \"481f1fff\",\n \"date\": \"2024-06-25_17-54-29\",\n \"time_this_iter_s\": 114.88935971260071,\n \"time_total_s\": 228.74907898902893,\n \"pid\": 2195253,\n \"hostname\": \"r8i6n2\",\n \"node_ip\": \"10.159.28.60\",\n \"config\": {\n \"encoder_num_layer\": 2,\n \"decoder_rt_num_layer\": 8,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 64,\n \"encoder_ff\": 1024,\n \"decoder_rt_ff\": 1024,\n \"decoder_int_ff\": 512,\n \"n_head\": 16,\n \"drop_rate\": 0.15995138350968297,\n \"lr\": 0.004656630423530019,\n \"batch_size\": 1024\n },\n \"time_since_restore\": 114.88935971260071,\n \"iterations_since_restore\": 1,\n \"experiment_tag\": \"10_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=8,drop_rate=0.1600,embedding_dim=64,encoder_ff=1024,encoder_num_layer=2,lr=0.0047,n_head=16\"\n },\n \"last_result_time\": 1719330869.3889194,\n \"metric_analysis\": {\n \"loss\": {\n \"max\": 1961.035730076587,\n \"min\": 1959.9800554410679,\n \"avg\": 1960.5078927588274,\n \"last\": 1959.9800554410679,\n \"last-5-avg\": 1960.5078927588274,\n \"last-10-avg\": 1960.5078927588274\n },\n \"should_checkpoint\": {\n \"max\": true,\n \"min\": true,\n \"avg\": 1.0,\n \"last\": true,\n \"last-5-avg\": 1.0,\n \"last-10-avg\": 1.0\n },\n \"done\": {\n \"max\": false,\n \"min\": false,\n \"avg\": 0.0,\n \"last\": false,\n \"last-5-avg\": 0.0,\n \"last-10-avg\": 0.0\n },\n \"training_iteration\": {\n \"max\": 2,\n \"min\": 1,\n \"avg\": 1.5,\n \"last\": 2,\n \"last-5-avg\": 1.5,\n \"last-10-avg\": 1.5\n },\n \"time_this_iter_s\": {\n \"max\": 114.88935971260071,\n \"min\": 113.85971927642822,\n \"avg\": 114.37453949451447,\n \"last\": 114.88935971260071,\n \"last-5-avg\": 114.37453949451447,\n \"last-10-avg\": 114.37453949451447\n },\n \"time_total_s\": {\n \"max\": 228.74907898902893,\n \"min\": 113.85971927642822,\n \"avg\": 171.30439913272858,\n \"last\": 228.74907898902893,\n \"last-5-avg\": 171.30439913272858,\n \"last-10-avg\": 171.30439913272858\n },\n \"time_since_restore\": {\n \"max\": 114.88935971260071,\n \"min\": 113.85971927642822,\n \"avg\": 114.37453949451447,\n \"last\": 114.88935971260071,\n \"last-5-avg\": 114.37453949451447,\n \"last-10-avg\": 114.37453949451447\n },\n \"iterations_since_restore\": {\n \"max\": 1,\n \"min\": 1,\n \"avg\": 1.0,\n \"last\": 1,\n \"last-5-avg\": 1.0,\n \"last-10-avg\": 1.0\n }\n },\n \"_n_steps\": [\n 5,\n 10\n ],\n \"metric_n_steps\": {\n \"loss\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005959d000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294288c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b007494624308b3d96c9624a49e4094869452946807680d43089d4ea793eb9f9e409486945294652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005959d000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294288c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b007494624308b3d96c9624a49e4094869452946807680d43089d4ea793eb9f9e409486945294652e\"\n }\n },\n \"should_checkpoint\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059524000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294288888652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059524000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294288888652e\"\n }\n },\n \"done\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059524000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294288989652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059524000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294288989652e\"\n }\n },\n \"training_iteration\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059526000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294284b014b02652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059526000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294284b014b02652e\"\n }\n },\n \"time_this_iter_s\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059534000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452942847405c7705a400000047405cb8eb45000000652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059534000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452942847405c7705a400000047405cb8eb45000000652e\"\n }\n },\n \"time_total_s\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059534000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452942847405c7705a400000047406c97f874800000652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059534000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452942847405c7705a400000047406c97f874800000652e\"\n }\n },\n \"time_since_restore\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059534000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452942847405c7705a400000047405cb8eb45000000652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059534000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452942847405c7705a400000047405cb8eb45000000652e\"\n }\n },\n \"iterations_since_restore\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059526000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294284b014b01652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059526000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294284b014b01652e\"\n }\n }\n },\n \"checkpoint_manager\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"800595a5090000000000008c267261792e747261696e2e5f696e7465726e616c2e636865636b706f696e745f6d616e61676572948c125f436865636b706f696e744d616e616765729493942981947d94288c125f636865636b706f696e745f636f6e666967948c097261792e747261696e948c10436865636b706f696e74436f6e6669679493942981947d94288c0b6e756d5f746f5f6b656570944e8c1a636865636b706f696e745f73636f72655f617474726962757465944e8c16636865636b706f696e745f73636f72655f6f72646572948c036d6178948c14636865636b706f696e745f6672657175656e6379944b008c11636865636b706f696e745f61745f656e6494898c1a5f636865636b706f696e745f6b6565705f616c6c5f72616e6b73948c0a44455052454341544544948c1f5f636865636b706f696e745f75706c6f61645f66726f6d5f776f726b65727394681275628c135f636865636b706f696e745f726573756c7473945d94288c1b7261792e747261696e2e5f696e7465726e616c2e73657373696f6e948c0f5f547261696e696e67526573756c749493942981947d94288c0a636865636b706f696e749468068c0a436865636b706f696e749493942981947d94288c0470617468948cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f34383166316666665f31305f62617463685f73697a653d313032342c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d313032342c6465636f6465725f72745f6e756d5f6c617965723d382c64726f705f5f323032342d30362d32355f31372d33332d33302f636865636b706f696e745f303030303030948c0a66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c055f75756964948c0475756964948c04555549449493942981947d948c03696e74948a101a1ee3d5d45707b6114cb3034c146e7e736275628c076d657472696373947d94288c046c6f7373948c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b007494624308b3d96c9624a49e4094869452948c0974696d657374616d70944a0fe47a668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303030948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b018c08747269616c5f6964948c083438316631666666948c0464617465948c13323032342d30362d32355f31372d33362d3437948c1074696d655f746869735f697465725f739447405c7705a40000008c0c74696d655f746f74616c5f739447405c7705a40000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b028c146465636f6465725f72745f6e756d5f6c61796572944b088c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b408c0a656e636f6465725f6666944d00048c0d6465636f6465725f72745f6666944d00048c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b108c0964726f705f7261746594473fc47949748fe09c8c026c7294473f7312d4b60ed3e98c0a62617463685f73697a65944d0004758c1274696d655f73696e63655f726573746f72659447405c7705a40000008c18697465726174696f6e735f73696e63655f726573746f7265944b0175756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f34383166316666665f31305f62617463685f73697a653d313032342c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d313032342c6465636f6465725f72745f6e756d5f6c617965723d382c64726f705f5f323032342d30362d32355f31372d33332d33302f636865636b706f696e745f30303030303194682268257d946827897385945294682a682d2981947d9468308a1103a55102964a82888840273d3f2dbac6007362756268317d94288c046c6f7373946836683c43089d4ea793eb9f9e4094869452948c0974696d657374616d70944a35e87a668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303031948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b028c08747269616c5f6964948c083438316631666666948c0464617465948c13323032342d30362d32355f31372d35342d3239948c1074696d655f746869735f697465725f739447405cb8eb450000008c0c74696d655f746f74616c5f739447406c97f8748000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b028c146465636f6465725f72745f6e756d5f6c61796572944b088c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b408c0a656e636f6465725f6666944d00048c0d6465636f6465725f72745f6666944d00048c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b108c0964726f705f7261746594473fc47949748fe09c8c026c7294473f7312d4b60ed3e98c0a62617463685f73697a65944d0004758c1274696d655f73696e63655f726573746f72659447405cb8eb450000008c18697465726174696f6e735f73696e63655f726573746f7265944b01757562658c195f6c61746573745f636865636b706f696e745f726573756c7494686275622e\"\n }\n}"
+ ],
+ [
+ "{\n \"stub\": false,\n \"trainable_name\": \"train_model\",\n \"trial_id\": \"2465ee2c\",\n \"storage\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059572040000000000008c1b7261792e747261696e2e5f696e7465726e616c2e73746f72616765948c0e53746f72616765436f6e746578749493942981947d94288c12637573746f6d5f66735f70726f766964656494898c1273746f726167655f6c6f63616c5f70617468948c2b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c7473948c136578706572696d656e745f6469725f6e616d65948c0f746573745f6578706572696d656e74948c0e747269616c5f6469725f6e616d65948c96747261696e5f6d6f64656c5f32343635656532635f31315f62617463685f73697a653d313032342c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d313032342c6465636f6465725f72745f6e756d5f6c617965723d322c64726f705f5f323032342d30362d32355f31372d33342d3533948c1863757272656e745f636865636b706f696e745f696e646578944b008c0b73796e635f636f6e666967948c097261792e747261696e948c0a53796e63436f6e6669679493942981947d94288c0b73796e635f706572696f64944d2c018c0c73796e635f74696d656f7574944d08078c0e73796e635f61727469666163747394898c1c73796e635f6172746966616374735f6f6e5f636865636b706f696e7494888c0a75706c6f61645f646972948c0a44455052454341544544948c0673796e6365729468188c1273796e635f6f6e5f636865636b706f696e7494681875628c1273746f726167655f66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c0f73746f726167655f66735f70617468948c452f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f7465737494681968008c115f46696c6573797374656d53796e6365729493942981947d9428681b682268134d2c0168144d08078c116c6173745f73796e635f75705f74696d65944741d99eb89fd0cfec8c136c6173745f73796e635f646f776e5f74696d659447fff00000000000008c0d5f73796e635f70726f63657373944e8c0c5f63757272656e745f636d649468008c125f75706c6f61645f746f5f66735f706174689493947d94288c0a6c6f63616c5f70617468948c3b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c74732f746573745f6578706572696d656e74948c0266739468228c0766735f70617468948c552f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e74948c076578636c756465945d948c0e2a2f636865636b706f696e745f2a9461758694756275622e\"\n },\n \"config\": {\n \"encoder_num_layer\": 4,\n \"decoder_rt_num_layer\": 2,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 64,\n \"encoder_ff\": 2048,\n \"decoder_rt_ff\": 1024,\n \"decoder_int_ff\": 512,\n \"n_head\": 8,\n \"drop_rate\": 0.3089996310219745,\n \"lr\": 0.062282868368169514,\n \"batch_size\": 1024\n },\n \"_Trial__unresolved_config\": {\n \"encoder_num_layer\": 4,\n \"decoder_rt_num_layer\": 2,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 64,\n \"encoder_ff\": 2048,\n \"decoder_rt_ff\": 1024,\n \"decoder_int_ff\": 512,\n \"n_head\": 8,\n \"drop_rate\": 0.3089996310219745,\n \"lr\": 0.062282868368169514,\n \"batch_size\": 1024\n },\n \"evaluated_params\": {\n \"batch_size\": 1024,\n \"decoder_int_ff\": 512,\n \"decoder_int_num_layer\": 1,\n \"decoder_rt_ff\": 1024,\n \"decoder_rt_num_layer\": 2,\n \"drop_rate\": 0.3089996310219745,\n \"embedding_dim\": 64,\n \"encoder_ff\": 2048,\n \"encoder_num_layer\": 4,\n \"lr\": 0.062282868368169514,\n \"n_head\": 8\n },\n \"experiment_tag\": \"11_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop_rate=0.3090,embedding_dim=64,encoder_ff=2048,encoder_num_layer=4,lr=0.0623,n_head=8\",\n \"stopping_criterion\": {},\n \"_setup_default_resource\": true,\n \"_default_placement_group_factory\": \"80054e2e\",\n \"placement_group_factory\": \"800595ba000000000000008c237261792e74756e652e657865637574696f6e2e706c6163656d656e745f67726f757073948c15506c6163656d656e7447726f7570466163746f72799493942981947d94288c085f62756e646c6573945d947d94288c03435055944740540000000000008c034750559447401000000000000075618c155f686561645f62756e646c655f69735f656d70747994898c095f7374726174656779948c045041434b948c055f6172677394298c075f6b7761726773947d9475622e\",\n \"log_to_file\": [\n null,\n null\n ],\n \"max_failures\": 0,\n \"_default_result_or_future\": null,\n \"export_formats\": [],\n \"status\": \"PAUSED\",\n \"relative_logdir\": \"train_model_2465ee2c_11_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop__2024-06-25_17-34-53\",\n \"trial_name_creator\": null,\n \"trial_dirname_creator\": null,\n \"custom_trial_name\": null,\n \"custom_dirname\": null,\n \"restore_path\": null,\n \"_restore_checkpoint_result\": null,\n \"_state_json\": null,\n \"results\": \"80054e2e\",\n \"extra_arg\": \"80054e2e\",\n \"_resources\": \"80054e2e\"\n}",
+ "{\n \"start_time\": 1719329807.375336,\n \"num_failures\": 0,\n \"num_failures_after_restore\": 0,\n \"error_filename\": null,\n \"pickled_error_filename\": null,\n \"last_result\": {\n \"loss\": 2699.211756428396,\n \"timestamp\": 1719329892,\n \"checkpoint_dir_name\": \"checkpoint_000000\",\n \"should_checkpoint\": true,\n \"done\": false,\n \"training_iteration\": 1,\n \"trial_id\": \"2465ee2c\",\n \"date\": \"2024-06-25_17-38-12\",\n \"time_this_iter_s\": 85.10484099388123,\n \"time_total_s\": 85.10484099388123,\n \"pid\": 2195253,\n \"hostname\": \"r8i6n2\",\n \"node_ip\": \"10.159.28.60\",\n \"config\": {\n \"encoder_num_layer\": 4,\n \"decoder_rt_num_layer\": 2,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 64,\n \"encoder_ff\": 2048,\n \"decoder_rt_ff\": 1024,\n \"decoder_int_ff\": 512,\n \"n_head\": 8,\n \"drop_rate\": 0.3089996310219745,\n \"lr\": 0.062282868368169514,\n \"batch_size\": 1024\n },\n \"time_since_restore\": 85.10484099388123,\n \"iterations_since_restore\": 1,\n \"experiment_tag\": \"11_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop_rate=0.3090,embedding_dim=64,encoder_ff=2048,encoder_num_layer=4,lr=0.0623,n_head=8\"\n },\n \"last_result_time\": 1719329892.485186,\n \"metric_analysis\": {\n \"loss\": {\n \"max\": 2699.211756428396,\n \"min\": 2699.211756428396,\n \"avg\": 2699.211756428396,\n \"last\": 2699.211756428396,\n \"last-5-avg\": 2699.211756428396,\n \"last-10-avg\": 2699.211756428396\n },\n \"should_checkpoint\": {\n \"max\": true,\n \"min\": true,\n \"avg\": true,\n \"last\": true,\n \"last-5-avg\": true,\n \"last-10-avg\": true\n },\n \"done\": {\n \"max\": false,\n \"min\": false,\n \"avg\": false,\n \"last\": false,\n \"last-5-avg\": false,\n \"last-10-avg\": false\n },\n \"training_iteration\": {\n \"max\": 1,\n \"min\": 1,\n \"avg\": 1,\n \"last\": 1,\n \"last-5-avg\": 1,\n \"last-10-avg\": 1\n },\n \"time_this_iter_s\": {\n \"max\": 85.10484099388123,\n \"min\": 85.10484099388123,\n \"avg\": 85.10484099388123,\n \"last\": 85.10484099388123,\n \"last-5-avg\": 85.10484099388123,\n \"last-10-avg\": 85.10484099388123\n },\n \"time_total_s\": {\n \"max\": 85.10484099388123,\n \"min\": 85.10484099388123,\n \"avg\": 85.10484099388123,\n \"last\": 85.10484099388123,\n \"last-5-avg\": 85.10484099388123,\n \"last-10-avg\": 85.10484099388123\n },\n \"time_since_restore\": {\n \"max\": 85.10484099388123,\n \"min\": 85.10484099388123,\n \"avg\": 85.10484099388123,\n \"last\": 85.10484099388123,\n \"last-5-avg\": 85.10484099388123,\n \"last-10-avg\": 85.10484099388123\n },\n \"iterations_since_restore\": {\n \"max\": 1,\n \"min\": 1,\n \"avg\": 1,\n \"last\": 1,\n \"last-5-avg\": 1,\n \"last-10-avg\": 1\n }\n },\n \"_n_steps\": [\n 5,\n 10\n ],\n \"metric_n_steps\": {\n \"loss\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059589000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452948c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b0074946243085bad566b6c16a5409486945294612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059589000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452948c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b0074946243085bad566b6c16a5409486945294612e\"\n }\n },\n \"should_checkpoint\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059522000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b058694529488612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059522000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a8694529488612e\"\n }\n },\n \"done\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059522000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b058694529489612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059522000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a8694529489612e\"\n }\n },\n \"training_iteration\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059523000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452944b01612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059523000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452944b01612e\"\n }\n },\n \"time_this_iter_s\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b058694529447405546b5b7000000612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a8694529447405546b5b7000000612e\"\n }\n },\n \"time_total_s\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b058694529447405546b5b7000000612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a8694529447405546b5b7000000612e\"\n }\n },\n \"time_since_restore\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b058694529447405546b5b7000000612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a8694529447405546b5b7000000612e\"\n }\n },\n \"iterations_since_restore\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059523000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452944b01612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059523000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452944b01612e\"\n }\n }\n },\n \"checkpoint_manager\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059518060000000000008c267261792e747261696e2e5f696e7465726e616c2e636865636b706f696e745f6d616e61676572948c125f436865636b706f696e744d616e616765729493942981947d94288c125f636865636b706f696e745f636f6e666967948c097261792e747261696e948c10436865636b706f696e74436f6e6669679493942981947d94288c0b6e756d5f746f5f6b656570944e8c1a636865636b706f696e745f73636f72655f617474726962757465944e8c16636865636b706f696e745f73636f72655f6f72646572948c036d6178948c14636865636b706f696e745f6672657175656e6379944b008c11636865636b706f696e745f61745f656e6494898c1a5f636865636b706f696e745f6b6565705f616c6c5f72616e6b73948c0a44455052454341544544948c1f5f636865636b706f696e745f75706c6f61645f66726f6d5f776f726b65727394681275628c135f636865636b706f696e745f726573756c7473945d948c1b7261792e747261696e2e5f696e7465726e616c2e73657373696f6e948c0f5f547261696e696e67526573756c749493942981947d94288c0a636865636b706f696e749468068c0a436865636b706f696e749493942981947d94288c0470617468948cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f32343635656532635f31315f62617463685f73697a653d313032342c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d313032342c6465636f6465725f72745f6e756d5f6c617965723d322c64726f705f5f323032342d30362d32355f31372d33342d35332f636865636b706f696e745f303030303030948c0a66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c055f75756964948c0475756964948c04555549449493942981947d948c03696e74948a11d69238a42f7cd0b08f42f95515b8d7e000736275628c076d657472696373947d94288c046c6f7373948c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b0074946243085bad566b6c16a54094869452948c0974696d657374616d70944a64e47a668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303030948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b018c08747269616c5f6964948c083234363565653263948c0464617465948c13323032342d30362d32355f31372d33382d3132948c1074696d655f746869735f697465725f739447405546b5b70000008c0c74696d655f746f74616c5f739447405546b5b70000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b048c146465636f6465725f72745f6e756d5f6c61796572944b028c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b408c0a656e636f6465725f6666944d00088c0d6465636f6465725f72745f6666944d00048c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b088c0964726f705f7261746594473fd3c6a6636dc9e48c026c7294473fafe38a457c16bc8c0a62617463685f73697a65944d0004758c1274696d655f73696e63655f726573746f72659447405546b5b70000008c18697465726174696f6e735f73696e63655f726573746f7265944b01757562618c195f6c61746573745f636865636b706f696e745f726573756c7494681975622e\"\n }\n}"
+ ],
+ [
+ "{\n \"stub\": false,\n \"trainable_name\": \"train_model\",\n \"trial_id\": \"ad12346e\",\n \"storage\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059572040000000000008c1b7261792e747261696e2e5f696e7465726e616c2e73746f72616765948c0e53746f72616765436f6e746578749493942981947d94288c12637573746f6d5f66735f70726f766964656494898c1273746f726167655f6c6f63616c5f70617468948c2b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c7473948c136578706572696d656e745f6469725f6e616d65948c0f746573745f6578706572696d656e74948c0e747269616c5f6469725f6e616d65948c96747261696e5f6d6f64656c5f61643132333436655f31325f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d313032342c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f5f323032342d30362d32355f31372d33362d3437948c1863757272656e745f636865636b706f696e745f696e646578944b008c0b73796e635f636f6e666967948c097261792e747261696e948c0a53796e63436f6e6669679493942981947d94288c0b73796e635f706572696f64944d2c018c0c73796e635f74696d656f7574944d08078c0e73796e635f61727469666163747394898c1c73796e635f6172746966616374735f6f6e5f636865636b706f696e7494888c0a75706c6f61645f646972948c0a44455052454341544544948c0673796e6365729468188c1273796e635f6f6e5f636865636b706f696e7494681875628c1273746f726167655f66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c0f73746f726167655f66735f70617468948c452f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f7465737494681968008c115f46696c6573797374656d53796e6365729493942981947d9428681b682268134d2c0168144d08078c116c6173745f73796e635f75705f74696d65944741d99eb8eb5008998c136c6173745f73796e635f646f776e5f74696d659447fff00000000000008c0d5f73796e635f70726f63657373944e8c0c5f63757272656e745f636d649468008c125f75706c6f61645f746f5f66735f706174689493947d94288c0a6c6f63616c5f70617468948c3b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c74732f746573745f6578706572696d656e74948c0266739468228c0766735f70617468948c552f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e74948c076578636c756465945d948c0e2a2f636865636b706f696e745f2a9461758694756275622e\"\n },\n \"config\": {\n \"encoder_num_layer\": 1,\n \"decoder_rt_num_layer\": 1,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 1024,\n \"encoder_ff\": 1024,\n \"decoder_rt_ff\": 1024,\n \"decoder_int_ff\": 512,\n \"n_head\": 4,\n \"drop_rate\": 0.9358999268282086,\n \"lr\": 0.00020838214141938324,\n \"batch_size\": 2048\n },\n \"_Trial__unresolved_config\": {\n \"encoder_num_layer\": 1,\n \"decoder_rt_num_layer\": 1,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 1024,\n \"encoder_ff\": 1024,\n \"decoder_rt_ff\": 1024,\n \"decoder_int_ff\": 512,\n \"n_head\": 4,\n \"drop_rate\": 0.9358999268282086,\n \"lr\": 0.00020838214141938324,\n \"batch_size\": 2048\n },\n \"evaluated_params\": {\n \"batch_size\": 2048,\n \"decoder_int_ff\": 512,\n \"decoder_int_num_layer\": 1,\n \"decoder_rt_ff\": 1024,\n \"decoder_rt_num_layer\": 1,\n \"drop_rate\": 0.9358999268282086,\n \"embedding_dim\": 1024,\n \"encoder_ff\": 1024,\n \"encoder_num_layer\": 1,\n \"lr\": 0.00020838214141938324,\n \"n_head\": 4\n },\n \"experiment_tag\": \"12_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop_rate=0.9359,embedding_dim=1024,encoder_ff=1024,encoder_num_layer=1,lr=0.0002,n_head=4\",\n \"stopping_criterion\": {},\n \"_setup_default_resource\": true,\n \"_default_placement_group_factory\": \"80054e2e\",\n \"placement_group_factory\": \"800595ba000000000000008c237261792e74756e652e657865637574696f6e2e706c6163656d656e745f67726f757073948c15506c6163656d656e7447726f7570466163746f72799493942981947d94288c085f62756e646c6573945d947d94288c03435055944740540000000000008c034750559447401000000000000075618c155f686561645f62756e646c655f69735f656d70747994898c095f7374726174656779948c045041434b948c055f6172677394298c075f6b7761726773947d9475622e\",\n \"log_to_file\": [\n null,\n null\n ],\n \"max_failures\": 0,\n \"_default_result_or_future\": null,\n \"export_formats\": [],\n \"status\": \"PAUSED\",\n \"relative_logdir\": \"train_model_ad12346e_12_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_17-36-47\",\n \"trial_name_creator\": null,\n \"trial_dirname_creator\": null,\n \"custom_trial_name\": null,\n \"custom_dirname\": null,\n \"restore_path\": null,\n \"_restore_checkpoint_result\": null,\n \"_state_json\": null,\n \"results\": \"80054e2e\",\n \"extra_arg\": \"80054e2e\",\n \"_resources\": \"80054e2e\"\n}",
+ "{\n \"start_time\": 1719329892.630865,\n \"num_failures\": 0,\n \"num_failures_after_restore\": 0,\n \"error_filename\": null,\n \"pickled_error_filename\": null,\n \"last_result\": {\n \"loss\": 2126.6908641424707,\n \"timestamp\": 1719330099,\n \"checkpoint_dir_name\": \"checkpoint_000000\",\n \"should_checkpoint\": true,\n \"done\": false,\n \"training_iteration\": 1,\n \"trial_id\": \"ad12346e\",\n \"date\": \"2024-06-25_17-41-39\",\n \"time_this_iter_s\": 206.83131527900696,\n \"time_total_s\": 206.83131527900696,\n \"pid\": 2195253,\n \"hostname\": \"r8i6n2\",\n \"node_ip\": \"10.159.28.60\",\n \"config\": {\n \"encoder_num_layer\": 1,\n \"decoder_rt_num_layer\": 1,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 1024,\n \"encoder_ff\": 1024,\n \"decoder_rt_ff\": 1024,\n \"decoder_int_ff\": 512,\n \"n_head\": 4,\n \"drop_rate\": 0.9358999268282086,\n \"lr\": 0.00020838214141938324,\n \"batch_size\": 2048\n },\n \"time_since_restore\": 206.83131527900696,\n \"iterations_since_restore\": 1,\n \"experiment_tag\": \"12_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop_rate=0.9359,embedding_dim=1024,encoder_ff=1024,encoder_num_layer=1,lr=0.0002,n_head=4\"\n },\n \"last_result_time\": 1719330099.4666245,\n \"metric_analysis\": {\n \"loss\": {\n \"max\": 2126.6908641424707,\n \"min\": 2126.6908641424707,\n \"avg\": 2126.6908641424707,\n \"last\": 2126.6908641424707,\n \"last-5-avg\": 2126.6908641424707,\n \"last-10-avg\": 2126.6908641424707\n },\n \"should_checkpoint\": {\n \"max\": true,\n \"min\": true,\n \"avg\": true,\n \"last\": true,\n \"last-5-avg\": true,\n \"last-10-avg\": true\n },\n \"done\": {\n \"max\": false,\n \"min\": false,\n \"avg\": false,\n \"last\": false,\n \"last-5-avg\": false,\n \"last-10-avg\": false\n },\n \"training_iteration\": {\n \"max\": 1,\n \"min\": 1,\n \"avg\": 1,\n \"last\": 1,\n \"last-5-avg\": 1,\n \"last-10-avg\": 1\n },\n \"time_this_iter_s\": {\n \"max\": 206.83131527900696,\n \"min\": 206.83131527900696,\n \"avg\": 206.83131527900696,\n \"last\": 206.83131527900696,\n \"last-5-avg\": 206.83131527900696,\n \"last-10-avg\": 206.83131527900696\n },\n \"time_total_s\": {\n \"max\": 206.83131527900696,\n \"min\": 206.83131527900696,\n \"avg\": 206.83131527900696,\n \"last\": 206.83131527900696,\n \"last-5-avg\": 206.83131527900696,\n \"last-10-avg\": 206.83131527900696\n },\n \"time_since_restore\": {\n \"max\": 206.83131527900696,\n \"min\": 206.83131527900696,\n \"avg\": 206.83131527900696,\n \"last\": 206.83131527900696,\n \"last-5-avg\": 206.83131527900696,\n \"last-10-avg\": 206.83131527900696\n },\n \"iterations_since_restore\": {\n \"max\": 1,\n \"min\": 1,\n \"avg\": 1,\n \"last\": 1,\n \"last-5-avg\": 1,\n \"last-10-avg\": 1\n }\n },\n \"_n_steps\": [\n 5,\n 10\n ],\n \"metric_n_steps\": {\n \"loss\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059589000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452948c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b007494624308c8e3f1b8619da0409486945294612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059589000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452948c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b007494624308c8e3f1b8619da0409486945294612e\"\n }\n },\n \"should_checkpoint\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059522000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b058694529488612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059522000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a8694529488612e\"\n }\n },\n \"done\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059522000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b058694529489612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059522000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a8694529489612e\"\n }\n },\n \"training_iteration\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059523000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452944b01612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059523000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452944b01612e\"\n }\n },\n \"time_this_iter_s\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294474069da9a22800000612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294474069da9a22800000612e\"\n }\n },\n \"time_total_s\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294474069da9a22800000612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294474069da9a22800000612e\"\n }\n },\n \"time_since_restore\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294474069da9a22800000612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294474069da9a22800000612e\"\n }\n },\n \"iterations_since_restore\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059523000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452944b01612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059523000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452944b01612e\"\n }\n }\n },\n \"checkpoint_manager\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059519060000000000008c267261792e747261696e2e5f696e7465726e616c2e636865636b706f696e745f6d616e61676572948c125f436865636b706f696e744d616e616765729493942981947d94288c125f636865636b706f696e745f636f6e666967948c097261792e747261696e948c10436865636b706f696e74436f6e6669679493942981947d94288c0b6e756d5f746f5f6b656570944e8c1a636865636b706f696e745f73636f72655f617474726962757465944e8c16636865636b706f696e745f73636f72655f6f72646572948c036d6178948c14636865636b706f696e745f6672657175656e6379944b008c11636865636b706f696e745f61745f656e6494898c1a5f636865636b706f696e745f6b6565705f616c6c5f72616e6b73948c0a44455052454341544544948c1f5f636865636b706f696e745f75706c6f61645f66726f6d5f776f726b65727394681275628c135f636865636b706f696e745f726573756c7473945d948c1b7261792e747261696e2e5f696e7465726e616c2e73657373696f6e948c0f5f547261696e696e67526573756c749493942981947d94288c0a636865636b706f696e749468068c0a436865636b706f696e749493942981947d94288c0470617468948cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f61643132333436655f31325f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d313032342c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f5f323032342d30362d32355f31372d33362d34372f636865636b706f696e745f303030303030948c0a66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c055f75756964948c0475756964948c04555549449493942981947d948c03696e74948a11d6a8427ced244b852d454a69b0fff7f400736275628c076d657472696373947d94288c046c6f7373948c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b007494624308c8e3f1b8619da04094869452948c0974696d657374616d70944a33e57a668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303030948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b018c08747269616c5f6964948c086164313233343665948c0464617465948c13323032342d30362d32355f31372d34312d3339948c1074696d655f746869735f697465725f7394474069da9a228000008c0c74696d655f746f74616c5f7394474069da9a228000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b018c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944d00048c0a656e636f6465725f6666944d00048c0d6465636f6465725f72745f6666944d00048c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b048c0964726f705f7261746594473fedf2e46741ca558c026c7294473f2b5024f705e0038c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f726594474069da9a228000008c18697465726174696f6e735f73696e63655f726573746f7265944b01757562618c195f6c61746573745f636865636b706f696e745f726573756c7494681975622e\"\n }\n}"
+ ],
+ [
+ "{\n \"stub\": false,\n \"trainable_name\": \"train_model\",\n \"trial_id\": \"25a9f43a\",\n \"storage\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059572040000000000008c1b7261792e747261696e2e5f696e7465726e616c2e73746f72616765948c0e53746f72616765436f6e746578749493942981947d94288c12637573746f6d5f66735f70726f766964656494898c1273746f726167655f6c6f63616c5f70617468948c2b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c7473948c136578706572696d656e745f6469725f6e616d65948c0f746573745f6578706572696d656e74948c0e747269616c5f6469725f6e616d65948c96747261696e5f6d6f64656c5f32356139663433615f31335f62617463685f73697a653d313032342c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d313032342c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f5f323032342d30362d32355f31372d33382d3132948c1863757272656e745f636865636b706f696e745f696e646578944b038c0b73796e635f636f6e666967948c097261792e747261696e948c0a53796e63436f6e6669679493942981947d94288c0b73796e635f706572696f64944d2c018c0c73796e635f74696d656f7574944d08078c0e73796e635f61727469666163747394898c1c73796e635f6172746966616374735f6f6e5f636865636b706f696e7494888c0a75706c6f61645f646972948c0a44455052454341544544948c0673796e6365729468188c1273796e635f6f6e5f636865636b706f696e7494681875628c1273746f726167655f66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c0f73746f726167655f66735f70617468948c452f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f7465737494681968008c115f46696c6573797374656d53796e6365729493942981947d9428681b682268134d2c0168144d08078c116c6173745f73796e635f75705f74696d65944741d99eb8eb5008998c136c6173745f73796e635f646f776e5f74696d659447fff00000000000008c0d5f73796e635f70726f63657373944e8c0c5f63757272656e745f636d649468008c125f75706c6f61645f746f5f66735f706174689493947d94288c0a6c6f63616c5f70617468948c3b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c74732f746573745f6578706572696d656e74948c0266739468228c0766735f70617468948c552f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e74948c076578636c756465945d948c0e2a2f636865636b706f696e745f2a9461758694756275622e\"\n },\n \"config\": {\n \"encoder_num_layer\": 4,\n \"decoder_rt_num_layer\": 1,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 64,\n \"encoder_ff\": 512,\n \"decoder_rt_ff\": 1024,\n \"decoder_int_ff\": 512,\n \"n_head\": 16,\n \"drop_rate\": 0.7474706763430264,\n \"lr\": 0.00045083297498032556,\n \"batch_size\": 1024\n },\n \"_Trial__unresolved_config\": {\n \"encoder_num_layer\": 4,\n \"decoder_rt_num_layer\": 1,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 64,\n \"encoder_ff\": 512,\n \"decoder_rt_ff\": 1024,\n \"decoder_int_ff\": 512,\n \"n_head\": 16,\n \"drop_rate\": 0.7474706763430264,\n \"lr\": 0.00045083297498032556,\n \"batch_size\": 1024\n },\n \"evaluated_params\": {\n \"batch_size\": 1024,\n \"decoder_int_ff\": 512,\n \"decoder_int_num_layer\": 1,\n \"decoder_rt_ff\": 1024,\n \"decoder_rt_num_layer\": 1,\n \"drop_rate\": 0.7474706763430264,\n \"embedding_dim\": 64,\n \"encoder_ff\": 512,\n \"encoder_num_layer\": 4,\n \"lr\": 0.00045083297498032556,\n \"n_head\": 16\n },\n \"experiment_tag\": \"13_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop_rate=0.7475,embedding_dim=64,encoder_ff=512,encoder_num_layer=4,lr=0.0005,n_head=16\",\n \"stopping_criterion\": {},\n \"_setup_default_resource\": true,\n \"_default_placement_group_factory\": \"80054e2e\",\n \"placement_group_factory\": \"800595ba000000000000008c237261792e74756e652e657865637574696f6e2e706c6163656d656e745f67726f757073948c15506c6163656d656e7447726f7570466163746f72799493942981947d94288c085f62756e646c6573945d947d94288c03435055944740540000000000008c034750559447401000000000000075618c155f686561645f62756e646c655f69735f656d70747994898c095f7374726174656779948c045041434b948c055f6172677394298c075f6b7761726773947d9475622e\",\n \"log_to_file\": [\n null,\n null\n ],\n \"max_failures\": 0,\n \"_default_result_or_future\": null,\n \"export_formats\": [],\n \"status\": \"PAUSED\",\n \"relative_logdir\": \"train_model_25a9f43a_13_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_17-38-12\",\n \"trial_name_creator\": null,\n \"trial_dirname_creator\": null,\n \"custom_trial_name\": null,\n \"custom_dirname\": null,\n \"restore_path\": null,\n \"_restore_checkpoint_result\": null,\n \"_state_json\": null,\n \"results\": \"80054e2e\",\n \"extra_arg\": \"80054e2e\",\n \"_resources\": \"80054e2e\"\n}",
+ "{\n \"start_time\": 1719330099.6124241,\n \"num_failures\": 0,\n \"num_failures_after_restore\": 0,\n \"error_filename\": null,\n \"pickled_error_filename\": null,\n \"last_result\": {\n \"loss\": 381.2108863169753,\n \"timestamp\": 1719331634,\n \"checkpoint_dir_name\": \"checkpoint_000003\",\n \"should_checkpoint\": true,\n \"done\": false,\n \"training_iteration\": 4,\n \"trial_id\": \"25a9f43a\",\n \"date\": \"2024-06-25_18-07-14\",\n \"time_this_iter_s\": 80.07035231590271,\n \"time_total_s\": 332.0298240184784,\n \"pid\": 2195253,\n \"hostname\": \"r8i6n2\",\n \"node_ip\": \"10.159.28.60\",\n \"config\": {\n \"encoder_num_layer\": 4,\n \"decoder_rt_num_layer\": 1,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 64,\n \"encoder_ff\": 512,\n \"decoder_rt_ff\": 1024,\n \"decoder_int_ff\": 512,\n \"n_head\": 16,\n \"drop_rate\": 0.7474706763430264,\n \"lr\": 0.00045083297498032556,\n \"batch_size\": 1024\n },\n \"time_since_restore\": 174.64248299598694,\n \"iterations_since_restore\": 2,\n \"experiment_tag\": \"13_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop_rate=0.7475,embedding_dim=64,encoder_ff=512,encoder_num_layer=4,lr=0.0005,n_head=16\"\n },\n \"last_result_time\": 1719331634.3944979,\n \"metric_analysis\": {\n \"loss\": {\n \"max\": 1284.8956039308564,\n \"min\": 381.2108863169753,\n \"avg\": 652.1886275584304,\n \"last\": 381.2108863169753,\n \"last-5-avg\": 652.1886275584304,\n \"last-10-avg\": 652.1886275584304\n },\n \"should_checkpoint\": {\n \"max\": true,\n \"min\": true,\n \"avg\": 1.0,\n \"last\": true,\n \"last-5-avg\": 1.0,\n \"last-10-avg\": 1.0\n },\n \"done\": {\n \"max\": false,\n \"min\": false,\n \"avg\": 0.0,\n \"last\": false,\n \"last-5-avg\": 0.0,\n \"last-10-avg\": 0.0\n },\n \"training_iteration\": {\n \"max\": 4,\n \"min\": 1,\n \"avg\": 2.5,\n \"last\": 4,\n \"last-5-avg\": 2.5,\n \"last-10-avg\": 2.5\n },\n \"time_this_iter_s\": {\n \"max\": 94.57213068008423,\n \"min\": 77.99471831321716,\n \"avg\": 83.0074560046196,\n \"last\": 80.07035231590271,\n \"last-5-avg\": 83.0074560046196,\n \"last-10-avg\": 83.0074560046196\n },\n \"time_total_s\": {\n \"max\": 332.0298240184784,\n \"min\": 79.39262270927429,\n \"avg\": 205.19231486320496,\n \"last\": 332.0298240184784,\n \"last-5-avg\": 205.19231486320496,\n \"last-10-avg\": 205.19231486320496\n },\n \"time_since_restore\": {\n \"max\": 174.64248299598694,\n \"min\": 77.99471831321716,\n \"avg\": 106.65048867464066,\n \"last\": 174.64248299598694,\n \"last-5-avg\": 106.65048867464066,\n \"last-10-avg\": 106.65048867464066\n },\n \"iterations_since_restore\": {\n \"max\": 2,\n \"min\": 1,\n \"avg\": 1.25,\n \"last\": 2,\n \"last-5-avg\": 1.25,\n \"last-10-avg\": 1.25\n }\n },\n \"_n_steps\": [\n 5,\n 10\n ],\n \"metric_n_steps\": {\n \"loss\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"800595c3000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294288c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b007494624308ca6432199513944094869452946807680d43087fbfdf1fe42d804094869452946807680d4308d4e9740a968e7a4094869452946807680d430853a954ca5fd377409486945294652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"800595c3000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294288c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b007494624308ca6432199513944094869452946807680d43087fbfdf1fe42d804094869452946807680d4308d4e9740a968e7a4094869452946807680d430853a954ca5fd377409486945294652e\"\n }\n },\n \"should_checkpoint\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059526000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452942888888888652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059526000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452942888888888652e\"\n }\n },\n \"done\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059526000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452942889898989652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059526000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452942889898989652e\"\n }\n },\n \"training_iteration\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294284b014b024b034b04652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294284b014b024b034b04652e\"\n }\n },\n \"time_this_iter_s\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059546000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b058694529428474053d920bb0000004740537fa977000000474057a49dca0000004740540480a7000000652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059546000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a8694529428474053d920bb0000004740537fa977000000474057a49dca0000004740540480a7000000652e\"\n }\n },\n \"time_total_s\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059546000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b058694529428474053d920bb000000474063ac651900000047406f7eb3fe000000474074c07a28c00000652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059546000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a8694529428474053d920bb000000474063ac651900000047406f7eb3fe000000474074c07a28c00000652e\"\n }\n },\n \"time_since_restore\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059546000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b058694529428474053d920bb0000004740537fa977000000474057a49dca000000474065d48f38800000652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059546000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a8694529428474053d920bb0000004740537fa977000000474057a49dca000000474065d48f38800000652e\"\n }\n },\n \"iterations_since_restore\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294284b014b014b014b02652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294284b014b014b014b02652e\"\n }\n }\n },\n \"checkpoint_manager\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"800595be100000000000008c267261792e747261696e2e5f696e7465726e616c2e636865636b706f696e745f6d616e61676572948c125f436865636b706f696e744d616e616765729493942981947d94288c125f636865636b706f696e745f636f6e666967948c097261792e747261696e948c10436865636b706f696e74436f6e6669679493942981947d94288c0b6e756d5f746f5f6b656570944e8c1a636865636b706f696e745f73636f72655f617474726962757465944e8c16636865636b706f696e745f73636f72655f6f72646572948c036d6178948c14636865636b706f696e745f6672657175656e6379944b008c11636865636b706f696e745f61745f656e6494898c1a5f636865636b706f696e745f6b6565705f616c6c5f72616e6b73948c0a44455052454341544544948c1f5f636865636b706f696e745f75706c6f61645f66726f6d5f776f726b65727394681275628c135f636865636b706f696e745f726573756c7473945d94288c1b7261792e747261696e2e5f696e7465726e616c2e73657373696f6e948c0f5f547261696e696e67526573756c749493942981947d94288c0a636865636b706f696e749468068c0a436865636b706f696e749493942981947d94288c0470617468948cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f32356139663433615f31335f62617463685f73697a653d313032342c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d313032342c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f5f323032342d30362d32355f31372d33382d31322f636865636b706f696e745f303030303030948c0a66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c055f75756964948c0475756964948c04555549449493942981947d948c03696e74948a10a129e095eddc25b4d044f32a0b7ba13f736275628c076d657472696373947d94288c046c6f7373948c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b007494624308ca6432199513944094869452948c0974696d657374616d70944a82e57a668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303030948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b018c08747269616c5f6964948c083235613966343361948c0464617465948c13323032342d30362d32355f31372d34322d3539948c1074696d655f746869735f697465725f7394474053d920bb0000008c0c74696d655f746f74616c5f7394474053d920bb0000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b048c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b408c0a656e636f6465725f6666944d00028c0d6465636f6465725f72745f6666944d00048c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b108c0964726f705f7261746594473fe7eb479fb397f58c026c7294473f3d8bb8e22cfb9f8c0a62617463685f73697a65944d0004758c1274696d655f73696e63655f726573746f726594474053d920bb0000008c18697465726174696f6e735f73696e63655f726573746f7265944b0175756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f32356139663433615f31335f62617463685f73697a653d313032342c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d313032342c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f5f323032342d30362d32355f31372d33382d31322f636865636b706f696e745f30303030303194682268257d946827897385945294682a682d2981947d9468308a11e92ab01417c513b2a345f8709b6dc5a0007362756268317d94288c046c6f7373946836683c43087fbfdf1fe42d804094869452948c0974696d657374616d70944a83e87a668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303031948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b028c08747269616c5f6964948c083235613966343361948c0464617465948c13323032342d30362d32355f31372d35352d3437948c1074696d655f746869735f697465725f73944740537fa9770000008c0c74696d655f746f74616c5f7394474063ac65190000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b048c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b408c0a656e636f6465725f6666944d00028c0d6465636f6465725f72745f6666944d00048c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b108c0964726f705f7261746594473fe7eb479fb397f58c026c7294473f3d8bb8e22cfb9f8c0a62617463685f73697a65944d0004758c1274696d655f73696e63655f726573746f7265944740537fa9770000008c18697465726174696f6e735f73696e63655f726573746f7265944b0175756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f32356139663433615f31335f62617463685f73697a653d313032342c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d313032342c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f5f323032342d30362d32355f31372d33382d31322f636865636b706f696e745f30303030303294682268257d946827897385945294682a682d2981947d9468308a1092c9f175bddcd68a0848129376eec62e7362756268317d94288c046c6f7373946836683c4308d4e9740a968e7a4094869452948c0974696d657374616d70944ae2ea7a668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303032948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b038c08747269616c5f6964948c083235613966343361948c0464617465948c13323032342d30362d32355f31382d30352d3534948c1074696d655f746869735f697465725f7394474057a49dca0000008c0c74696d655f746f74616c5f739447406f7eb3fe0000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b048c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b408c0a656e636f6465725f6666944d00028c0d6465636f6465725f72745f6666944d00048c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b108c0964726f705f7261746594473fe7eb479fb397f58c026c7294473f3d8bb8e22cfb9f8c0a62617463685f73697a65944d0004758c1274696d655f73696e63655f726573746f726594474057a49dca0000008c18697465726174696f6e735f73696e63655f726573746f7265944b0175756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f32356139663433615f31335f62617463685f73697a653d313032342c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d313032342c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f5f323032342d30362d32355f31372d33382d31322f636865636b706f696e745f30303030303394682268257d946827897385945294682a682d2981947d9468308a11fbd0d64e460991bc9d4e4220c9ebd9cb007362756268317d94288c046c6f7373946836683c430853a954ca5fd3774094869452948c0974696d657374616d70944a32eb7a668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303033948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b048c08747269616c5f6964948c083235613966343361948c0464617465948c13323032342d30362d32355f31382d30372d3134948c1074696d655f746869735f697465725f73944740540480a70000008c0c74696d655f746f74616c5f7394474074c07a28c000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b048c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b408c0a656e636f6465725f6666944d00028c0d6465636f6465725f72745f6666944d00048c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b108c0964726f705f7261746594473fe7eb479fb397f58c026c7294473f3d8bb8e22cfb9f8c0a62617463685f73697a65944d0004758c1274696d655f73696e63655f726573746f726594474065d48f388000008c18697465726174696f6e735f73696e63655f726573746f7265944b02757562658c195f6c61746573745f636865636b706f696e745f726573756c749468c075622e\"\n }\n}"
+ ],
+ [
+ "{\n \"stub\": false,\n \"trainable_name\": \"train_model\",\n \"trial_id\": \"110f0a27\",\n \"storage\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059572040000000000008c1b7261792e747261696e2e5f696e7465726e616c2e73746f72616765948c0e53746f72616765436f6e746578749493942981947d94288c12637573746f6d5f66735f70726f766964656494898c1273746f726167655f6c6f63616c5f70617468948c2b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c7473948c136578706572696d656e745f6469725f6e616d65948c0f746573745f6578706572696d656e74948c0e747269616c5f6469725f6e616d65948c96747261696e5f6d6f64656c5f31313066306132375f31345f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d313032342c6465636f6465725f72745f6e756d5f6c617965723d322c64726f705f5f323032342d30362d32355f31372d34312d3339948c1863757272656e745f636865636b706f696e745f696e646578944b018c0b73796e635f636f6e666967948c097261792e747261696e948c0a53796e63436f6e6669679493942981947d94288c0b73796e635f706572696f64944d2c018c0c73796e635f74696d656f7574944d08078c0e73796e635f61727469666163747394898c1c73796e635f6172746966616374735f6f6e5f636865636b706f696e7494888c0a75706c6f61645f646972948c0a44455052454341544544948c0673796e6365729468188c1273796e635f6f6e5f636865636b706f696e7494681875628c1273746f726167655f66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c0f73746f726167655f66735f70617468948c452f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f7465737494681968008c115f46696c6573797374656d53796e6365729493942981947d9428681b682268134d2c0168144d08078c116c6173745f73796e635f75705f74696d65944741d99eb936cf4b1e8c136c6173745f73796e635f646f776e5f74696d659447fff00000000000008c0d5f73796e635f70726f63657373944e8c0c5f63757272656e745f636d649468008c125f75706c6f61645f746f5f66735f706174689493947d94288c0a6c6f63616c5f70617468948c3b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c74732f746573745f6578706572696d656e74948c0266739468228c0766735f70617468948c552f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e74948c076578636c756465945d948c0e2a2f636865636b706f696e745f2a9461758694756275622e\"\n },\n \"config\": {\n \"encoder_num_layer\": 2,\n \"decoder_rt_num_layer\": 2,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 256,\n \"encoder_ff\": 2048,\n \"decoder_rt_ff\": 1024,\n \"decoder_int_ff\": 512,\n \"n_head\": 16,\n \"drop_rate\": 0.025073968761468723,\n \"lr\": 0.0010259835678295768,\n \"batch_size\": 2048\n },\n \"_Trial__unresolved_config\": {\n \"encoder_num_layer\": 2,\n \"decoder_rt_num_layer\": 2,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 256,\n \"encoder_ff\": 2048,\n \"decoder_rt_ff\": 1024,\n \"decoder_int_ff\": 512,\n \"n_head\": 16,\n \"drop_rate\": 0.025073968761468723,\n \"lr\": 0.0010259835678295768,\n \"batch_size\": 2048\n },\n \"evaluated_params\": {\n \"batch_size\": 2048,\n \"decoder_int_ff\": 512,\n \"decoder_int_num_layer\": 1,\n \"decoder_rt_ff\": 1024,\n \"decoder_rt_num_layer\": 2,\n \"drop_rate\": 0.025073968761468723,\n \"embedding_dim\": 256,\n \"encoder_ff\": 2048,\n \"encoder_num_layer\": 2,\n \"lr\": 0.0010259835678295768,\n \"n_head\": 16\n },\n \"experiment_tag\": \"14_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop_rate=0.0251,embedding_dim=256,encoder_ff=2048,encoder_num_layer=2,lr=0.0010,n_head=16\",\n \"stopping_criterion\": {},\n \"_setup_default_resource\": true,\n \"_default_placement_group_factory\": \"80054e2e\",\n \"placement_group_factory\": \"800595ba000000000000008c237261792e74756e652e657865637574696f6e2e706c6163656d656e745f67726f757073948c15506c6163656d656e7447726f7570466163746f72799493942981947d94288c085f62756e646c6573945d947d94288c03435055944740540000000000008c034750559447401000000000000075618c155f686561645f62756e646c655f69735f656d70747994898c095f7374726174656779948c045041434b948c055f6172677394298c075f6b7761726773947d9475622e\",\n \"log_to_file\": [\n null,\n null\n ],\n \"max_failures\": 0,\n \"_default_result_or_future\": null,\n \"export_formats\": [],\n \"status\": \"PAUSED\",\n \"relative_logdir\": \"train_model_110f0a27_14_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop__2024-06-25_17-41-39\",\n \"trial_name_creator\": null,\n \"trial_dirname_creator\": null,\n \"custom_trial_name\": null,\n \"custom_dirname\": null,\n \"restore_path\": null,\n \"_restore_checkpoint_result\": null,\n \"_state_json\": null,\n \"results\": \"80054e2e\",\n \"extra_arg\": \"80054e2e\",\n \"_resources\": \"80054e2e\"\n}",
+ "{\n \"start_time\": 1719330179.1827745,\n \"num_failures\": 0,\n \"num_failures_after_restore\": 0,\n \"error_filename\": null,\n \"pickled_error_filename\": null,\n \"last_result\": {\n \"loss\": 1952.48307740219,\n \"timestamp\": 1719331057,\n \"checkpoint_dir_name\": \"checkpoint_000001\",\n \"should_checkpoint\": true,\n \"done\": false,\n \"training_iteration\": 2,\n \"trial_id\": \"110f0a27\",\n \"date\": \"2024-06-25_17-57-37\",\n \"time_this_iter_s\": 110.16742134094238,\n \"time_total_s\": 220.83409094810486,\n \"pid\": 2195253,\n \"hostname\": \"r8i6n2\",\n \"node_ip\": \"10.159.28.60\",\n \"config\": {\n \"encoder_num_layer\": 2,\n \"decoder_rt_num_layer\": 2,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 256,\n \"encoder_ff\": 2048,\n \"decoder_rt_ff\": 1024,\n \"decoder_int_ff\": 512,\n \"n_head\": 16,\n \"drop_rate\": 0.025073968761468723,\n \"lr\": 0.0010259835678295768,\n \"batch_size\": 2048\n },\n \"time_since_restore\": 110.16742134094238,\n \"iterations_since_restore\": 1,\n \"experiment_tag\": \"14_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop_rate=0.0251,embedding_dim=256,encoder_ff=2048,encoder_num_layer=2,lr=0.0010,n_head=16\"\n },\n \"last_result_time\": 1719331057.906955,\n \"metric_analysis\": {\n \"loss\": {\n \"max\": 1956.1862581508367,\n \"min\": 1952.48307740219,\n \"avg\": 1954.3346677765135,\n \"last\": 1952.48307740219,\n \"last-5-avg\": 1954.3346677765135,\n \"last-10-avg\": 1954.3346677765135\n },\n \"should_checkpoint\": {\n \"max\": true,\n \"min\": true,\n \"avg\": 1.0,\n \"last\": true,\n \"last-5-avg\": 1.0,\n \"last-10-avg\": 1.0\n },\n \"done\": {\n \"max\": false,\n \"min\": false,\n \"avg\": 0.0,\n \"last\": false,\n \"last-5-avg\": 0.0,\n \"last-10-avg\": 0.0\n },\n \"training_iteration\": {\n \"max\": 2,\n \"min\": 1,\n \"avg\": 1.5,\n \"last\": 2,\n \"last-5-avg\": 1.5,\n \"last-10-avg\": 1.5\n },\n \"time_this_iter_s\": {\n \"max\": 110.66666960716248,\n \"min\": 110.16742134094238,\n \"avg\": 110.41704547405243,\n \"last\": 110.16742134094238,\n \"last-5-avg\": 110.41704547405243,\n \"last-10-avg\": 110.41704547405243\n },\n \"time_total_s\": {\n \"max\": 220.83409094810486,\n \"min\": 110.66666960716248,\n \"avg\": 165.75038027763367,\n \"last\": 220.83409094810486,\n \"last-5-avg\": 165.75038027763367,\n \"last-10-avg\": 165.75038027763367\n },\n \"time_since_restore\": {\n \"max\": 110.66666960716248,\n \"min\": 110.16742134094238,\n \"avg\": 110.41704547405243,\n \"last\": 110.16742134094238,\n \"last-5-avg\": 110.41704547405243,\n \"last-10-avg\": 110.41704547405243\n },\n \"iterations_since_restore\": {\n \"max\": 1,\n \"min\": 1,\n \"avg\": 1.0,\n \"last\": 1,\n \"last-5-avg\": 1.0,\n \"last-10-avg\": 1.0\n }\n },\n \"_n_steps\": [\n 5,\n 10\n ],\n \"metric_n_steps\": {\n \"loss\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005959d000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294288c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b007494624308d4e974babe909e4094869452946807680d43085fafd7abee819e409486945294652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005959d000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294288c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b007494624308d4e974babe909e4094869452946807680d43085fafd7abee819e409486945294652e\"\n }\n },\n \"should_checkpoint\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059524000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294288888652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059524000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294288888652e\"\n }\n },\n \"done\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059524000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294288989652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059524000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294288989652e\"\n }\n },\n \"training_iteration\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059526000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294284b014b02652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059526000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294284b014b02652e\"\n }\n },\n \"time_this_iter_s\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059534000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452942847405baaaab700000047405b8ab708000000652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059534000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452942847405baaaab700000047405b8ab708000000652e\"\n }\n },\n \"time_total_s\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059534000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452942847405baaaab700000047406b9ab0df800000652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059534000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452942847405baaaab700000047406b9ab0df800000652e\"\n }\n },\n \"time_since_restore\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059534000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452942847405baaaab700000047405b8ab708000000652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059534000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452942847405baaaab700000047405b8ab708000000652e\"\n }\n },\n \"iterations_since_restore\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059526000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294284b014b01652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059526000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294284b014b01652e\"\n }\n }\n },\n \"checkpoint_manager\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"800595a8090000000000008c267261792e747261696e2e5f696e7465726e616c2e636865636b706f696e745f6d616e61676572948c125f436865636b706f696e744d616e616765729493942981947d94288c125f636865636b706f696e745f636f6e666967948c097261792e747261696e948c10436865636b706f696e74436f6e6669679493942981947d94288c0b6e756d5f746f5f6b656570944e8c1a636865636b706f696e745f73636f72655f617474726962757465944e8c16636865636b706f696e745f73636f72655f6f72646572948c036d6178948c14636865636b706f696e745f6672657175656e6379944b008c11636865636b706f696e745f61745f656e6494898c1a5f636865636b706f696e745f6b6565705f616c6c5f72616e6b73948c0a44455052454341544544948c1f5f636865636b706f696e745f75706c6f61645f66726f6d5f776f726b65727394681275628c135f636865636b706f696e745f726573756c7473945d94288c1b7261792e747261696e2e5f696e7465726e616c2e73657373696f6e948c0f5f547261696e696e67526573756c749493942981947d94288c0a636865636b706f696e749468068c0a436865636b706f696e749493942981947d94288c0470617468948cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f31313066306132375f31345f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d313032342c6465636f6465725f72745f6e756d5f6c617965723d322c64726f705f5f323032342d30362d32355f31372d34312d33392f636865636b706f696e745f303030303030948c0a66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c055f75756964948c0475756964948c04555549449493942981947d948c03696e74948a11ca83eb928c757b9c114894f700e3c3b900736275628c076d657472696373947d94288c046c6f7373948c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b007494624308d4e974babe909e4094869452948c0974696d657374616d70944af1e57a668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303030948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b018c08747269616c5f6964948c083131306630613237948c0464617465948c13323032342d30362d32355f31372d34342d3439948c1074696d655f746869735f697465725f739447405baaaab70000008c0c74696d655f746f74616c5f739447405baaaab70000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b028c146465636f6465725f72745f6e756d5f6c61796572944b028c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944d00018c0a656e636f6465725f6666944d00088c0d6465636f6465725f72745f6666944d00048c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b108c0964726f705f7261746594473f99acfd8f3ee8808c026c7294473f50cf4977af16238c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f72659447405baaaab70000008c18697465726174696f6e735f73696e63655f726573746f7265944b0175756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f31313066306132375f31345f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d313032342c6465636f6465725f72745f6e756d5f6c617965723d322c64726f705f5f323032342d30362d32355f31372d34312d33392f636865636b706f696e745f30303030303194682268257d946827897385945294682a682d2981947d9468308a11aca660fb96d059b81742d1fa316ebf82007362756268317d94288c046c6f7373946836683c43085fafd7abee819e4094869452948c0974696d657374616d70944af1e87a668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303031948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b028c08747269616c5f6964948c083131306630613237948c0464617465948c13323032342d30362d32355f31372d35372d3337948c1074696d655f746869735f697465725f739447405b8ab7080000008c0c74696d655f746f74616c5f739447406b9ab0df8000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b028c146465636f6465725f72745f6e756d5f6c61796572944b028c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944d00018c0a656e636f6465725f6666944d00088c0d6465636f6465725f72745f6666944d00048c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b108c0964726f705f7261746594473f99acfd8f3ee8808c026c7294473f50cf4977af16238c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f72659447405b8ab7080000008c18697465726174696f6e735f73696e63655f726573746f7265944b01757562658c195f6c61746573745f636865636b706f696e745f726573756c7494686275622e\"\n }\n}"
+ ],
+ [
+ "{\n \"stub\": false,\n \"trainable_name\": \"train_model\",\n \"trial_id\": \"3510e84b\",\n \"storage\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059572040000000000008c1b7261792e747261696e2e5f696e7465726e616c2e73746f72616765948c0e53746f72616765436f6e746578749493942981947d94288c12637573746f6d5f66735f70726f766964656494898c1273746f726167655f6c6f63616c5f70617468948c2b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c7473948c136578706572696d656e745f6469725f6e616d65948c0f746573745f6578706572696d656e74948c0e747269616c5f6469725f6e616d65948c96747261696e5f6d6f64656c5f33353130653834625f31355f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d323034382c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f5f323032342d30362d32355f31372d34322d3539948c1863757272656e745f636865636b706f696e745f696e646578944b018c0b73796e635f636f6e666967948c097261792e747261696e948c0a53796e63436f6e6669679493942981947d94288c0b73796e635f706572696f64944d2c018c0c73796e635f74696d656f7574944d08078c0e73796e635f61727469666163747394898c1c73796e635f6172746966616374735f6f6e5f636865636b706f696e7494888c0a75706c6f61645f646972948c0a44455052454341544544948c0673796e6365729468188c1273796e635f6f6e5f636865636b706f696e7494681875628c1273746f726167655f66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c0f73746f726167655f66735f70617468948c452f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f7465737494681968008c115f46696c6573797374656d53796e6365729493942981947d9428681b682268134d2c0168144d08078c116c6173745f73796e635f75705f74696d65944741d99eb936cf4b1e8c136c6173745f73796e635f646f776e5f74696d659447fff00000000000008c0d5f73796e635f70726f63657373944e8c0c5f63757272656e745f636d649468008c125f75706c6f61645f746f5f66735f706174689493947d94288c0a6c6f63616c5f70617468948c3b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c74732f746573745f6578706572696d656e74948c0266739468228c0766735f70617468948c552f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e74948c076578636c756465945d948c0e2a2f636865636b706f696e745f2a9461758694756275622e\"\n },\n \"config\": {\n \"encoder_num_layer\": 4,\n \"decoder_rt_num_layer\": 1,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 16,\n \"encoder_ff\": 2048,\n \"decoder_rt_ff\": 2048,\n \"decoder_int_ff\": 512,\n \"n_head\": 4,\n \"drop_rate\": 0.3408951993486181,\n \"lr\": 0.009004430142657121,\n \"batch_size\": 2048\n },\n \"_Trial__unresolved_config\": {\n \"encoder_num_layer\": 4,\n \"decoder_rt_num_layer\": 1,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 16,\n \"encoder_ff\": 2048,\n \"decoder_rt_ff\": 2048,\n \"decoder_int_ff\": 512,\n \"n_head\": 4,\n \"drop_rate\": 0.3408951993486181,\n \"lr\": 0.009004430142657121,\n \"batch_size\": 2048\n },\n \"evaluated_params\": {\n \"batch_size\": 2048,\n \"decoder_int_ff\": 512,\n \"decoder_int_num_layer\": 1,\n \"decoder_rt_ff\": 2048,\n \"decoder_rt_num_layer\": 1,\n \"drop_rate\": 0.3408951993486181,\n \"embedding_dim\": 16,\n \"encoder_ff\": 2048,\n \"encoder_num_layer\": 4,\n \"lr\": 0.009004430142657121,\n \"n_head\": 4\n },\n \"experiment_tag\": \"15_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop_rate=0.3409,embedding_dim=16,encoder_ff=2048,encoder_num_layer=4,lr=0.0090,n_head=4\",\n \"stopping_criterion\": {},\n \"_setup_default_resource\": true,\n \"_default_placement_group_factory\": \"80054e2e\",\n \"placement_group_factory\": \"800595ba000000000000008c237261792e74756e652e657865637574696f6e2e706c6163656d656e745f67726f757073948c15506c6163656d656e7447726f7570466163746f72799493942981947d94288c085f62756e646c6573945d947d94288c03435055944740540000000000008c034750559447401000000000000075618c155f686561645f62756e646c655f69735f656d70747994898c095f7374726174656779948c045041434b948c055f6172677394298c075f6b7761726773947d9475622e\",\n \"log_to_file\": [\n null,\n null\n ],\n \"max_failures\": 0,\n \"_default_result_or_future\": null,\n \"export_formats\": [],\n \"status\": \"TERMINATED\",\n \"relative_logdir\": \"train_model_3510e84b_15_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop__2024-06-25_17-42-59\",\n \"trial_name_creator\": null,\n \"trial_dirname_creator\": null,\n \"custom_trial_name\": null,\n \"custom_dirname\": null,\n \"restore_path\": null,\n \"_restore_checkpoint_result\": null,\n \"_state_json\": null,\n \"results\": \"80054e2e\",\n \"extra_arg\": \"80054e2e\",\n \"_resources\": \"80054e2e\"\n}",
+ "{\n \"start_time\": 1719330290.0214992,\n \"num_failures\": 0,\n \"num_failures_after_restore\": 0,\n \"error_filename\": null,\n \"pickled_error_filename\": null,\n \"last_result\": {\n \"loss\": 1964.9452173428272,\n \"timestamp\": 1719331131,\n \"checkpoint_dir_name\": \"checkpoint_000001\",\n \"should_checkpoint\": true,\n \"done\": true,\n \"training_iteration\": 2,\n \"trial_id\": \"3510e84b\",\n \"date\": \"2024-06-25_17-58-51\",\n \"time_this_iter_s\": 73.41431927680969,\n \"time_total_s\": 146.1404869556427,\n \"pid\": 2195253,\n \"hostname\": \"r8i6n2\",\n \"node_ip\": \"10.159.28.60\",\n \"config\": {\n \"encoder_num_layer\": 4,\n \"decoder_rt_num_layer\": 1,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 16,\n \"encoder_ff\": 2048,\n \"decoder_rt_ff\": 2048,\n \"decoder_int_ff\": 512,\n \"n_head\": 4,\n \"drop_rate\": 0.3408951993486181,\n \"lr\": 0.009004430142657121,\n \"batch_size\": 2048\n },\n \"time_since_restore\": 73.41431927680969,\n \"iterations_since_restore\": 1,\n \"experiment_tag\": \"15_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop_rate=0.3409,embedding_dim=16,encoder_ff=2048,encoder_num_layer=4,lr=0.0090,n_head=4\"\n },\n \"last_result_time\": 1719331131.4748592,\n \"metric_analysis\": {\n \"loss\": {\n \"max\": 1964.9452173428272,\n \"min\": 1964.014622485544,\n \"avg\": 1964.4799199141855,\n \"last\": 1964.9452173428272,\n \"last-5-avg\": 1964.4799199141855,\n \"last-10-avg\": 1964.4799199141855\n },\n \"should_checkpoint\": {\n \"max\": true,\n \"min\": true,\n \"avg\": 1.0,\n \"last\": true,\n \"last-5-avg\": 1.0,\n \"last-10-avg\": 1.0\n },\n \"done\": {\n \"max\": true,\n \"min\": false,\n \"avg\": 0.5,\n \"last\": true,\n \"last-5-avg\": 0.5,\n \"last-10-avg\": 0.5\n },\n \"training_iteration\": {\n \"max\": 2,\n \"min\": 1,\n \"avg\": 1.5,\n \"last\": 2,\n \"last-5-avg\": 1.5,\n \"last-10-avg\": 1.5\n },\n \"time_this_iter_s\": {\n \"max\": 73.41431927680969,\n \"min\": 72.72616767883301,\n \"avg\": 73.07024347782135,\n \"last\": 73.41431927680969,\n \"last-5-avg\": 73.07024347782135,\n \"last-10-avg\": 73.07024347782135\n },\n \"time_total_s\": {\n \"max\": 146.1404869556427,\n \"min\": 72.72616767883301,\n \"avg\": 109.43332731723785,\n \"last\": 146.1404869556427,\n \"last-5-avg\": 109.43332731723785,\n \"last-10-avg\": 109.43332731723785\n },\n \"time_since_restore\": {\n \"max\": 73.41431927680969,\n \"min\": 72.72616767883301,\n \"avg\": 73.07024347782135,\n \"last\": 73.41431927680969,\n \"last-5-avg\": 73.07024347782135,\n \"last-10-avg\": 73.07024347782135\n },\n \"iterations_since_restore\": {\n \"max\": 1,\n \"min\": 1,\n \"avg\": 1.0,\n \"last\": 1,\n \"last-5-avg\": 1.0,\n \"last-10-avg\": 1.0\n }\n },\n \"_n_steps\": [\n 5,\n 10\n ],\n \"metric_n_steps\": {\n \"loss\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005959d000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294288c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b007494624308ca6432f90eb09e4094869452946807680d4308381c0ee7c7b39e409486945294652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005959d000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294288c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b007494624308ca6432f90eb09e4094869452946807680d4308381c0ee7c7b39e409486945294652e\"\n }\n },\n \"should_checkpoint\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059524000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294288888652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059524000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294288888652e\"\n }\n },\n \"done\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059524000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294288988652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059524000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294288988652e\"\n }\n },\n \"training_iteration\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059526000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294284b014b02652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059526000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294284b014b02652e\"\n }\n },\n \"time_this_iter_s\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059534000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294284740522e79880000004740525a8435000000652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059534000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294284740522e79880000004740525a8435000000652e\"\n }\n },\n \"time_total_s\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059534000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294284740522e7988000000474062447ede800000652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059534000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294284740522e7988000000474062447ede800000652e\"\n }\n },\n \"time_since_restore\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059534000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294284740522e79880000004740525a8435000000652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059534000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294284740522e79880000004740525a8435000000652e\"\n }\n },\n \"iterations_since_restore\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059526000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294284b014b01652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059526000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294284b014b01652e\"\n }\n }\n },\n \"checkpoint_manager\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"800595a4090000000000008c267261792e747261696e2e5f696e7465726e616c2e636865636b706f696e745f6d616e61676572948c125f436865636b706f696e744d616e616765729493942981947d94288c125f636865636b706f696e745f636f6e666967948c097261792e747261696e948c10436865636b706f696e74436f6e6669679493942981947d94288c0b6e756d5f746f5f6b656570944e8c1a636865636b706f696e745f73636f72655f617474726962757465944e8c16636865636b706f696e745f73636f72655f6f72646572948c036d6178948c14636865636b706f696e745f6672657175656e6379944b008c11636865636b706f696e745f61745f656e6494898c1a5f636865636b706f696e745f6b6565705f616c6c5f72616e6b73948c0a44455052454341544544948c1f5f636865636b706f696e745f75706c6f61645f66726f6d5f776f726b65727394681275628c135f636865636b706f696e745f726573756c7473945d94288c1b7261792e747261696e2e5f696e7465726e616c2e73657373696f6e948c0f5f547261696e696e67526573756c749493942981947d94288c0a636865636b706f696e749468068c0a436865636b706f696e749493942981947d94288c0470617468948cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f33353130653834625f31355f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d323034382c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f5f323032342d30362d32355f31372d34322d35392f636865636b706f696e745f303030303030948c0a66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c055f75756964948c0475756964948c04555549449493942981947d948c03696e74948a1063fb3088b10820947b4c52d91eb57649736275628c076d657472696373947d94288c046c6f7373948c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b007494624308ca6432f90eb09e4094869452948c0974696d657374616d70944a3ae67a668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303030948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b018c08747269616c5f6964948c083335313065383462948c0464617465948c13323032342d30362d32355f31372d34362d3032948c1074696d655f746869735f697465725f73944740522e79880000008c0c74696d655f746f74616c5f73944740522e79880000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b048c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b108c0a656e636f6465725f6666944d00088c0d6465636f6465725f72745f6666944d00088c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b048c0964726f705f7261746594473fd5d13a192434ae8c026c7294473f8270ea27dac9238c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f7265944740522e79880000008c18697465726174696f6e735f73696e63655f726573746f7265944b0175756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f33353130653834625f31355f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d323034382c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f5f323032342d30362d32355f31372d34322d35392f636865636b706f696e745f30303030303194682268257d946827897385945294682a682d2981947d9468308a1032d8d537008cf58e134bb9d8c30d880d7362756268317d94288c046c6f7373946836683c4308381c0ee7c7b39e4094869452948c0974696d657374616d70944a3be97a668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303031948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b028c08747269616c5f6964948c083335313065383462948c0464617465948c13323032342d30362d32355f31372d35382d3531948c1074696d655f746869735f697465725f73944740525a84350000008c0c74696d655f746f74616c5f7394474062447ede8000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b048c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b108c0a656e636f6465725f6666944d00088c0d6465636f6465725f72745f6666944d00088c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b048c0964726f705f7261746594473fd5d13a192434ae8c026c7294473f8270ea27dac9238c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f7265944740525a84350000008c18697465726174696f6e735f73696e63655f726573746f7265944b01757562658c195f6c61746573745f636865636b706f696e745f726573756c7494686275622e\"\n }\n}"
+ ],
+ [
+ "{\n \"stub\": false,\n \"trainable_name\": \"train_model\",\n \"trial_id\": \"a01c9b2d\",\n \"storage\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059572040000000000008c1b7261792e747261696e2e5f696e7465726e616c2e73746f72616765948c0e53746f72616765436f6e746578749493942981947d94288c12637573746f6d5f66735f70726f766964656494898c1273746f726167655f6c6f63616c5f70617468948c2b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c7473948c136578706572696d656e745f6469725f6e616d65948c0f746573745f6578706572696d656e74948c0e747269616c5f6469725f6e616d65948c96747261696e5f6d6f64656c5f61303163396232645f31365f62617463685f73697a653d313032342c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d313032342c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f5f323032342d30362d32355f31372d34342d3530948c1863757272656e745f636865636b706f696e745f696e646578944b038c0b73796e635f636f6e666967948c097261792e747261696e948c0a53796e63436f6e6669679493942981947d94288c0b73796e635f706572696f64944d2c018c0c73796e635f74696d656f7574944d08078c0e73796e635f61727469666163747394898c1c73796e635f6172746966616374735f6f6e5f636865636b706f696e7494888c0a75706c6f61645f646972948c0a44455052454341544544948c0673796e6365729468188c1273796e635f6f6e5f636865636b706f696e7494681875628c1273746f726167655f66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c0f73746f726167655f66735f70617468948c452f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f7465737494681968008c115f46696c6573797374656d53796e6365729493942981947d9428681b682268134d2c0168144d08078c116c6173745f73796e635f75705f74696d65944741d99eb936cf4b1e8c136c6173745f73796e635f646f776e5f74696d659447fff00000000000008c0d5f73796e635f70726f63657373944e8c0c5f63757272656e745f636d649468008c125f75706c6f61645f746f5f66735f706174689493947d94288c0a6c6f63616c5f70617468948c3b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c74732f746573745f6578706572696d656e74948c0266739468228c0766735f70617468948c552f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e74948c076578636c756465945d948c0e2a2f636865636b706f696e745f2a9461758694756275622e\"\n },\n \"config\": {\n \"encoder_num_layer\": 8,\n \"decoder_rt_num_layer\": 1,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 16,\n \"encoder_ff\": 1024,\n \"decoder_rt_ff\": 1024,\n \"decoder_int_ff\": 512,\n \"n_head\": 4,\n \"drop_rate\": 0.23338023981015343,\n \"lr\": 0.00010269515886648154,\n \"batch_size\": 1024\n },\n \"_Trial__unresolved_config\": {\n \"encoder_num_layer\": 8,\n \"decoder_rt_num_layer\": 1,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 16,\n \"encoder_ff\": 1024,\n \"decoder_rt_ff\": 1024,\n \"decoder_int_ff\": 512,\n \"n_head\": 4,\n \"drop_rate\": 0.23338023981015343,\n \"lr\": 0.00010269515886648154,\n \"batch_size\": 1024\n },\n \"evaluated_params\": {\n \"batch_size\": 1024,\n \"decoder_int_ff\": 512,\n \"decoder_int_num_layer\": 1,\n \"decoder_rt_ff\": 1024,\n \"decoder_rt_num_layer\": 1,\n \"drop_rate\": 0.23338023981015343,\n \"embedding_dim\": 16,\n \"encoder_ff\": 1024,\n \"encoder_num_layer\": 8,\n \"lr\": 0.00010269515886648154,\n \"n_head\": 4\n },\n \"experiment_tag\": \"16_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop_rate=0.2334,embedding_dim=16,encoder_ff=1024,encoder_num_layer=8,lr=0.0001,n_head=4\",\n \"stopping_criterion\": {},\n \"_setup_default_resource\": true,\n \"_default_placement_group_factory\": \"80054e2e\",\n \"placement_group_factory\": \"800595ba000000000000008c237261792e74756e652e657865637574696f6e2e706c6163656d656e745f67726f757073948c15506c6163656d656e7447726f7570466163746f72799493942981947d94288c085f62756e646c6573945d947d94288c03435055944740540000000000008c034750559447401000000000000075618c155f686561645f62756e646c655f69735f656d70747994898c095f7374726174656779948c045041434b948c055f6172677394298c075f6b7761726773947d9475622e\",\n \"log_to_file\": [\n null,\n null\n ],\n \"max_failures\": 0,\n \"_default_result_or_future\": null,\n \"export_formats\": [],\n \"status\": \"TERMINATED\",\n \"relative_logdir\": \"train_model_a01c9b2d_16_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_17-44-50\",\n \"trial_name_creator\": null,\n \"trial_dirname_creator\": null,\n \"custom_trial_name\": null,\n \"custom_dirname\": null,\n \"restore_path\": null,\n \"_restore_checkpoint_result\": null,\n \"_state_json\": null,\n \"results\": \"80054e2e\",\n \"extra_arg\": \"80054e2e\",\n \"_resources\": \"80054e2e\"\n}",
+ "{\n \"start_time\": 1719330362.8999166,\n \"num_failures\": 0,\n \"num_failures_after_restore\": 0,\n \"error_filename\": null,\n \"pickled_error_filename\": null,\n \"last_result\": {\n \"loss\": 290.2177711246521,\n \"timestamp\": 1719331824,\n \"checkpoint_dir_name\": \"checkpoint_000003\",\n \"should_checkpoint\": true,\n \"done\": true,\n \"training_iteration\": 4,\n \"trial_id\": \"a01c9b2d\",\n \"date\": \"2024-06-25_18-10-24\",\n \"time_this_iter_s\": 87.97965884208679,\n \"time_total_s\": 344.4578182697296,\n \"pid\": 2195253,\n \"hostname\": \"r8i6n2\",\n \"node_ip\": \"10.159.28.60\",\n \"config\": {\n \"encoder_num_layer\": 8,\n \"decoder_rt_num_layer\": 1,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 16,\n \"encoder_ff\": 1024,\n \"decoder_rt_ff\": 1024,\n \"decoder_int_ff\": 512,\n \"n_head\": 4,\n \"drop_rate\": 0.23338023981015343,\n \"lr\": 0.00010269515886648154,\n \"batch_size\": 1024\n },\n \"time_since_restore\": 189.79181909561157,\n \"iterations_since_restore\": 2,\n \"experiment_tag\": \"16_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop_rate=0.2334,embedding_dim=16,encoder_ff=1024,encoder_num_layer=8,lr=0.0001,n_head=4\"\n },\n \"last_result_time\": 1719331824.3503153,\n \"metric_analysis\": {\n \"loss\": {\n \"max\": 983.9757466954509,\n \"min\": 290.2177711246521,\n \"avg\": 534.5706713143296,\n \"last\": 290.2177711246521,\n \"last-5-avg\": 534.5706713143296,\n \"last-10-avg\": 534.5706713143296\n },\n \"should_checkpoint\": {\n \"max\": true,\n \"min\": true,\n \"avg\": 1.0,\n \"last\": true,\n \"last-5-avg\": 1.0,\n \"last-10-avg\": 1.0\n },\n \"done\": {\n \"max\": true,\n \"min\": false,\n \"avg\": 0.25,\n \"last\": true,\n \"last-5-avg\": 0.25,\n \"last-10-avg\": 0.25\n },\n \"training_iteration\": {\n \"max\": 4,\n \"min\": 1,\n \"avg\": 2.5,\n \"last\": 4,\n \"last-5-avg\": 2.5,\n \"last-10-avg\": 2.5\n },\n \"time_this_iter_s\": {\n \"max\": 101.81216025352478,\n \"min\": 70.83729195594788,\n \"avg\": 86.1144545674324,\n \"last\": 87.97965884208679,\n \"last-5-avg\": 86.1144545674324,\n \"last-10-avg\": 86.1144545674324\n },\n \"time_total_s\": {\n \"max\": 344.4578182697296,\n \"min\": 83.82870721817017,\n \"avg\": 209.85767102241516,\n \"last\": 344.4578182697296,\n \"last-5-avg\": 209.85767102241516,\n \"last-10-avg\": 209.85767102241516\n },\n \"time_since_restore\": {\n \"max\": 189.79181909561157,\n \"min\": 83.82870721817017,\n \"avg\": 132.52467143535614,\n \"last\": 189.79181909561157,\n \"last-5-avg\": 132.52467143535614,\n \"last-10-avg\": 132.52467143535614\n },\n \"iterations_since_restore\": {\n \"max\": 2,\n \"min\": 1,\n \"avg\": 1.5,\n \"last\": 2,\n \"last-5-avg\": 1.5,\n \"last-10-avg\": 1.5\n }\n },\n \"_n_steps\": [\n 5,\n 10\n ],\n \"metric_n_steps\": {\n \"loss\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"800595c3000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294288c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b00749462430822914854cebf8e4094869452946807680d4308552a95bad1567f4094869452946807680d4308028140809baa764094869452946807680d43084d2693fd7b2372409486945294652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"800595c3000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294288c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b00749462430822914854cebf8e4094869452946807680d4308552a95bad1567f4094869452946807680d4308028140809baa764094869452946807680d43084d2693fd7b2372409486945294652e\"\n }\n },\n \"should_checkpoint\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059526000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452942888888888652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059526000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452942888888888652e\"\n }\n },\n \"done\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059526000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452942889898988652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059526000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452942889898988652e\"\n }\n },\n \"training_iteration\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294284b014b024b034b04652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294284b014b024b034b04652e\"\n }\n },\n \"time_this_iter_s\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059546000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b058694529428474054f5098a000000474051b5963100000047405973fa6f000000474055feb2bb000000652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059546000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a8694529428474054f5098a000000474051b5963100000047405973fa6f000000474055feb2bb000000652e\"\n }\n },\n \"time_total_s\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059546000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b058694529428474054f5098a000000474063554fdd80000047407007a68a800000474075875339400000652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059546000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a8694529428474054f5098a000000474063554fdd80000047407007a68a800000474075875339400000652e\"\n }\n },\n \"time_since_restore\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059546000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b058694529428474054f5098a000000474063554fdd80000047405973fa6f000000474067b95695000000652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059546000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a8694529428474054f5098a000000474063554fdd80000047405973fa6f000000474067b95695000000652e\"\n }\n },\n \"iterations_since_restore\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294284b014b024b014b02652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294284b014b024b014b02652e\"\n }\n }\n },\n \"checkpoint_manager\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"800595be100000000000008c267261792e747261696e2e5f696e7465726e616c2e636865636b706f696e745f6d616e61676572948c125f436865636b706f696e744d616e616765729493942981947d94288c125f636865636b706f696e745f636f6e666967948c097261792e747261696e948c10436865636b706f696e74436f6e6669679493942981947d94288c0b6e756d5f746f5f6b656570944e8c1a636865636b706f696e745f73636f72655f617474726962757465944e8c16636865636b706f696e745f73636f72655f6f72646572948c036d6178948c14636865636b706f696e745f6672657175656e6379944b008c11636865636b706f696e745f61745f656e6494898c1a5f636865636b706f696e745f6b6565705f616c6c5f72616e6b73948c0a44455052454341544544948c1f5f636865636b706f696e745f75706c6f61645f66726f6d5f776f726b65727394681275628c135f636865636b706f696e745f726573756c7473945d94288c1b7261792e747261696e2e5f696e7465726e616c2e73657373696f6e948c0f5f547261696e696e67526573756c749493942981947d94288c0a636865636b706f696e749468068c0a436865636b706f696e749493942981947d94288c0470617468948cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f61303163396232645f31365f62617463685f73697a653d313032342c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d313032342c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f5f323032342d30362d32355f31372d34342d35302f636865636b706f696e745f303030303030948c0a66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c055f75756964948c0475756964948c04555549449493942981947d948c03696e74948a10c8ae3bd2cfa83498264691db42891e69736275628c076d657472696373947d94288c046c6f7373948c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b00749462430822914854cebf8e4094869452948c0974696d657374616d70944a8ee67a668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303030948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b018c08747269616c5f6964948c086130316339623264948c0464617465948c13323032342d30362d32355f31372d34372d3236948c1074696d655f746869735f697465725f7394474054f5098a0000008c0c74696d655f746f74616c5f7394474054f5098a0000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b088c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b108c0a656e636f6465725f6666944d00048c0d6465636f6465725f72745f6666944d00048c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b048c0964726f705f7261746594473fcddf6758c235208c026c7294473f1aebc16528f5c58c0a62617463685f73697a65944d0004758c1274696d655f73696e63655f726573746f726594474054f5098a0000008c18697465726174696f6e735f73696e63655f726573746f7265944b0175756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f61303163396232645f31365f62617463685f73697a653d313032342c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d313032342c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f5f323032342d30362d32355f31372d34342d35302f636865636b706f696e745f30303030303194682268257d946827897385945294682a682d2981947d9468308a10bb086198d988a581a94a4fbe433a685d7362756268317d94288c046c6f7373946836683c4308552a95bad1567f4094869452948c0974696d657374616d70944ad5e67a668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303031948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b028c08747269616c5f6964948c086130316339623264948c0464617465948c13323032342d30362d32355f31372d34382d3337948c1074696d655f746869735f697465725f7394474051b596310000008c0c74696d655f746f74616c5f7394474063554fdd8000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b088c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b108c0a656e636f6465725f6666944d00048c0d6465636f6465725f72745f6666944d00048c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b048c0964726f705f7261746594473fcddf6758c235208c026c7294473f1aebc16528f5c58c0a62617463685f73697a65944d0004758c1274696d655f73696e63655f726573746f726594474063554fdd8000008c18697465726174696f6e735f73696e63655f726573746f7265944b0275756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f61303163396232645f31365f62617463685f73697a653d313032342c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d313032342c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f5f323032342d30362d32355f31372d34342d35302f636865636b706f696e745f30303030303294682268257d946827897385945294682a682d2981947d9468308a119f83ae4eac3d058fa948287dc2caf5ed007362756268317d94288c046c6f7373946836683c4308028140809baa764094869452948c0974696d657374616d70944a98eb7a668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303032948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b038c08747269616c5f6964948c086130316339623264948c0464617465948c13323032342d30362d32355f31382d30382d3536948c1074696d655f746869735f697465725f739447405973fa6f0000008c0c74696d655f746f74616c5f739447407007a68a8000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b088c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b108c0a656e636f6465725f6666944d00048c0d6465636f6465725f72745f6666944d00048c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b048c0964726f705f7261746594473fcddf6758c235208c026c7294473f1aebc16528f5c58c0a62617463685f73697a65944d0004758c1274696d655f73696e63655f726573746f72659447405973fa6f0000008c18697465726174696f6e735f73696e63655f726573746f7265944b0175756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f61303163396232645f31365f62617463685f73697a653d313032342c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d313032342c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f5f323032342d30362d32355f31372d34342d35302f636865636b706f696e745f30303030303394682268257d946827897385945294682a682d2981947d9468308a115a8bbc5bf2511f88d34992ba4b8dfef7007362756268317d94288c046c6f7373946836683c43084d2693fd7b23724094869452948c0974696d657374616d70944af0eb7a668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303033948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b048c08747269616c5f6964948c086130316339623264948c0464617465948c13323032342d30362d32355f31382d31302d3234948c1074696d655f746869735f697465725f7394474055feb2bb0000008c0c74696d655f746f74616c5f73944740758753394000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b088c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b108c0a656e636f6465725f6666944d00048c0d6465636f6465725f72745f6666944d00048c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b048c0964726f705f7261746594473fcddf6758c235208c026c7294473f1aebc16528f5c58c0a62617463685f73697a65944d0004758c1274696d655f73696e63655f726573746f726594474067b956950000008c18697465726174696f6e735f73696e63655f726573746f7265944b02757562658c195f6c61746573745f636865636b706f696e745f726573756c749468c075622e\"\n }\n}"
+ ],
+ [
+ "{\n \"stub\": false,\n \"trainable_name\": \"train_model\",\n \"trial_id\": \"bea15841\",\n \"storage\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059572040000000000008c1b7261792e747261696e2e5f696e7465726e616c2e73746f72616765948c0e53746f72616765436f6e746578749493942981947d94288c12637573746f6d5f66735f70726f766964656494898c1273746f726167655f6c6f63616c5f70617468948c2b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c7473948c136578706572696d656e745f6469725f6e616d65948c0f746573745f6578706572696d656e74948c0e747269616c5f6469725f6e616d65948c96747261696e5f6d6f64656c5f62656131353834315f31375f62617463685f73697a653d313032342c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d313032342c6465636f6465725f72745f6e756d5f6c617965723d342c64726f705f5f323032342d30362d32355f31372d34362d3032948c1863757272656e745f636865636b706f696e745f696e646578944b018c0b73796e635f636f6e666967948c097261792e747261696e948c0a53796e63436f6e6669679493942981947d94288c0b73796e635f706572696f64944d2c018c0c73796e635f74696d656f7574944d08078c0e73796e635f61727469666163747394898c1c73796e635f6172746966616374735f6f6e5f636865636b706f696e7494888c0a75706c6f61645f646972948c0a44455052454341544544948c0673796e6365729468188c1273796e635f6f6e5f636865636b706f696e7494681875628c1273746f726167655f66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c0f73746f726167655f66735f70617468948c452f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f7465737494681968008c115f46696c6573797374656d53796e6365729493942981947d9428681b682268134d2c0168144d08078c116c6173745f73796e635f75705f74696d65944741d99eb9823a4e538c136c6173745f73796e635f646f776e5f74696d659447fff00000000000008c0d5f73796e635f70726f63657373944e8c0c5f63757272656e745f636d649468008c125f75706c6f61645f746f5f66735f706174689493947d94288c0a6c6f63616c5f70617468948c3b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c74732f746573745f6578706572696d656e74948c0266739468228c0766735f70617468948c552f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e74948c076578636c756465945d948c0e2a2f636865636b706f696e745f2a9461758694756275622e\"\n },\n \"config\": {\n \"encoder_num_layer\": 4,\n \"decoder_rt_num_layer\": 4,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 64,\n \"encoder_ff\": 1024,\n \"decoder_rt_ff\": 1024,\n \"decoder_int_ff\": 512,\n \"n_head\": 4,\n \"drop_rate\": 0.8101220088903347,\n \"lr\": 0.0005780102991375737,\n \"batch_size\": 1024\n },\n \"_Trial__unresolved_config\": {\n \"encoder_num_layer\": 4,\n \"decoder_rt_num_layer\": 4,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 64,\n \"encoder_ff\": 1024,\n \"decoder_rt_ff\": 1024,\n \"decoder_int_ff\": 512,\n \"n_head\": 4,\n \"drop_rate\": 0.8101220088903347,\n \"lr\": 0.0005780102991375737,\n \"batch_size\": 1024\n },\n \"evaluated_params\": {\n \"batch_size\": 1024,\n \"decoder_int_ff\": 512,\n \"decoder_int_num_layer\": 1,\n \"decoder_rt_ff\": 1024,\n \"decoder_rt_num_layer\": 4,\n \"drop_rate\": 0.8101220088903347,\n \"embedding_dim\": 64,\n \"encoder_ff\": 1024,\n \"encoder_num_layer\": 4,\n \"lr\": 0.0005780102991375737,\n \"n_head\": 4\n },\n \"experiment_tag\": \"17_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=4,drop_rate=0.8101,embedding_dim=64,encoder_ff=1024,encoder_num_layer=4,lr=0.0006,n_head=4\",\n \"stopping_criterion\": {},\n \"_setup_default_resource\": true,\n \"_default_placement_group_factory\": \"80054e2e\",\n \"placement_group_factory\": \"800595ba000000000000008c237261792e74756e652e657865637574696f6e2e706c6163656d656e745f67726f757073948c15506c6163656d656e7447726f7570466163746f72799493942981947d94288c085f62756e646c6573945d947d94288c03435055944740540000000000008c034750559447401000000000000075618c155f686561645f62756e646c655f69735f656d70747994898c095f7374726174656779948c045041434b948c055f6172677394298c075f6b7761726773947d9475622e\",\n \"log_to_file\": [\n null,\n null\n ],\n \"max_failures\": 0,\n \"_default_result_or_future\": null,\n \"export_formats\": [],\n \"status\": \"PAUSED\",\n \"relative_logdir\": \"train_model_bea15841_17_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=4,drop__2024-06-25_17-46-02\",\n \"trial_name_creator\": null,\n \"trial_dirname_creator\": null,\n \"custom_trial_name\": null,\n \"custom_dirname\": null,\n \"restore_path\": null,\n \"_restore_checkpoint_result\": null,\n \"_state_json\": null,\n \"results\": \"80054e2e\",\n \"extra_arg\": \"80054e2e\",\n \"_resources\": \"80054e2e\"\n}",
+ "{\n \"start_time\": 1719332768.2796977,\n \"num_failures\": 0,\n \"num_failures_after_restore\": 0,\n \"error_filename\": null,\n \"pickled_error_filename\": null,\n \"last_result\": {\n \"loss\": 2008.3096395177165,\n \"timestamp\": 1719332932,\n \"checkpoint_dir_name\": \"checkpoint_000001\",\n \"should_checkpoint\": true,\n \"done\": false,\n \"training_iteration\": 2,\n \"trial_id\": \"bea15841\",\n \"date\": \"2024-06-25_18-28-52\",\n \"time_this_iter_s\": 75.03496551513672,\n \"time_total_s\": 164.1780436038971,\n \"pid\": 2195253,\n \"hostname\": \"r8i6n2\",\n \"node_ip\": \"10.159.28.60\",\n \"config\": {\n \"encoder_num_layer\": 4,\n \"decoder_rt_num_layer\": 4,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 64,\n \"encoder_ff\": 1024,\n \"decoder_rt_ff\": 1024,\n \"decoder_int_ff\": 512,\n \"n_head\": 4,\n \"drop_rate\": 0.8101220088903347,\n \"lr\": 0.0005780102991375737,\n \"batch_size\": 1024\n },\n \"time_since_restore\": 164.1780436038971,\n \"iterations_since_restore\": 2,\n \"experiment_tag\": \"17_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=4,drop_rate=0.8101,embedding_dim=64,encoder_ff=1024,encoder_num_layer=4,lr=0.0006,n_head=4\"\n },\n \"last_result_time\": 1719332932.4666936,\n \"metric_analysis\": {\n \"loss\": {\n \"max\": 2012.1837220680056,\n \"min\": 2008.3096395177165,\n \"avg\": 2010.246680792861,\n \"last\": 2008.3096395177165,\n \"last-5-avg\": 2010.246680792861,\n \"last-10-avg\": 2010.246680792861\n },\n \"should_checkpoint\": {\n \"max\": true,\n \"min\": true,\n \"avg\": 1.0,\n \"last\": true,\n \"last-5-avg\": 1.0,\n \"last-10-avg\": 1.0\n },\n \"done\": {\n \"max\": false,\n \"min\": false,\n \"avg\": 0.0,\n \"last\": false,\n \"last-5-avg\": 0.0,\n \"last-10-avg\": 0.0\n },\n \"training_iteration\": {\n \"max\": 2,\n \"min\": 1,\n \"avg\": 1.5,\n \"last\": 2,\n \"last-5-avg\": 1.5,\n \"last-10-avg\": 1.5\n },\n \"time_this_iter_s\": {\n \"max\": 89.14307808876038,\n \"min\": 75.03496551513672,\n \"avg\": 82.08902180194855,\n \"last\": 75.03496551513672,\n \"last-5-avg\": 82.08902180194855,\n \"last-10-avg\": 82.08902180194855\n },\n \"time_total_s\": {\n \"max\": 164.1780436038971,\n \"min\": 89.14307808876038,\n \"avg\": 126.66056084632874,\n \"last\": 164.1780436038971,\n \"last-5-avg\": 126.66056084632874,\n \"last-10-avg\": 126.66056084632874\n },\n \"time_since_restore\": {\n \"max\": 164.1780436038971,\n \"min\": 89.14307808876038,\n \"avg\": 126.66056084632874,\n \"last\": 164.1780436038971,\n \"last-5-avg\": 126.66056084632874,\n \"last-10-avg\": 126.66056084632874\n },\n \"iterations_since_restore\": {\n \"max\": 2,\n \"min\": 1,\n \"avg\": 1.5,\n \"last\": 2,\n \"last-5-avg\": 1.5,\n \"last-10-avg\": 1.5\n }\n },\n \"_n_steps\": [\n 5,\n 10\n ],\n \"metric_n_steps\": {\n \"loss\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005959d000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294288c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b0074946243088d46a321bc709f4094869452946807680d4308914824123d619f409486945294652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005959d000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294288c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b0074946243088d46a321bc709f4094869452946807680d4308914824123d619f409486945294652e\"\n }\n },\n \"should_checkpoint\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059524000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294288888652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059524000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294288888652e\"\n }\n },\n \"done\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059524000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294288989652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059524000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294288989652e\"\n }\n },\n \"training_iteration\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059526000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294284b014b02652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059526000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294284b014b02652e\"\n }\n },\n \"time_this_iter_s\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059534000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b058694529428474056492831000000474052c23ce0000000652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059534000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a8694529428474056492831000000474052c23ce0000000652e\"\n }\n },\n \"time_total_s\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059534000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452942847405649283100000047406485b288800000652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059534000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452942847405649283100000047406485b288800000652e\"\n }\n },\n \"time_since_restore\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059534000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452942847405649283100000047406485b288800000652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059534000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452942847405649283100000047406485b288800000652e\"\n }\n },\n \"iterations_since_restore\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059526000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294284b014b02652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059526000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294284b014b02652e\"\n }\n }\n },\n \"checkpoint_manager\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"800595a4090000000000008c267261792e747261696e2e5f696e7465726e616c2e636865636b706f696e745f6d616e61676572948c125f436865636b706f696e744d616e616765729493942981947d94288c125f636865636b706f696e745f636f6e666967948c097261792e747261696e948c10436865636b706f696e74436f6e6669679493942981947d94288c0b6e756d5f746f5f6b656570944e8c1a636865636b706f696e745f73636f72655f617474726962757465944e8c16636865636b706f696e745f73636f72655f6f72646572948c036d6178948c14636865636b706f696e745f6672657175656e6379944b008c11636865636b706f696e745f61745f656e6494898c1a5f636865636b706f696e745f6b6565705f616c6c5f72616e6b73948c0a44455052454341544544948c1f5f636865636b706f696e745f75706c6f61645f66726f6d5f776f726b65727394681275628c135f636865636b706f696e745f726573756c7473945d94288c1b7261792e747261696e2e5f696e7465726e616c2e73657373696f6e948c0f5f547261696e696e67526573756c749493942981947d94288c0a636865636b706f696e749468068c0a436865636b706f696e749493942981947d94288c0470617468948cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f62656131353834315f31375f62617463685f73697a653d313032342c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d313032342c6465636f6465725f72745f6e756d5f6c617965723d342c64726f705f5f323032342d30362d32355f31372d34362d30322f636865636b706f696e745f303030303030948c0a66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c055f75756964948c0475756964948c04555549449493942981947d948c03696e74948a105352ddbd558402981243df8184916b6d736275628c076d657472696373947d94288c046c6f7373948c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b0074946243088d46a321bc709f4094869452948c0974696d657374616d70944af9ef7a668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303030948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b018c08747269616c5f6964948c086265613135383431948c0464617465948c13323032342d30362d32355f31382d32372d3337948c1074696d655f746869735f697465725f73944740564928310000008c0c74696d655f746f74616c5f73944740564928310000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b048c146465636f6465725f72745f6e756d5f6c61796572944b048c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b408c0a656e636f6465725f6666944d00048c0d6465636f6465725f72745f6666944d00048c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b048c0964726f705f7261746594473fe9ec84fdbe859a8c026c7294473f42f0b3aa7022498c0a62617463685f73697a65944d0004758c1274696d655f73696e63655f726573746f7265944740564928310000008c18697465726174696f6e735f73696e63655f726573746f7265944b0175756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f62656131353834315f31375f62617463685f73697a653d313032342c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d313032342c6465636f6465725f72745f6e756d5f6c617965723d342c64726f705f5f323032342d30362d32355f31372d34362d30322f636865636b706f696e745f30303030303194682268257d946827897385945294682a682d2981947d9468308a10be83920c3952d4a97344b9f324a4f52e7362756268317d94288c046c6f7373946836683c4308914824123d619f4094869452948c0974696d657374616d70944a44f07a668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303031948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b028c08747269616c5f6964948c086265613135383431948c0464617465948c13323032342d30362d32355f31382d32382d3532948c1074696d655f746869735f697465725f7394474052c23ce00000008c0c74696d655f746f74616c5f739447406485b2888000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b048c146465636f6465725f72745f6e756d5f6c61796572944b048c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b408c0a656e636f6465725f6666944d00048c0d6465636f6465725f72745f6666944d00048c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b048c0964726f705f7261746594473fe9ec84fdbe859a8c026c7294473f42f0b3aa7022498c0a62617463685f73697a65944d0004758c1274696d655f73696e63655f726573746f72659447406485b2888000008c18697465726174696f6e735f73696e63655f726573746f7265944b02757562658c195f6c61746573745f636865636b706f696e745f726573756c7494686275622e\"\n }\n}"
+ ],
+ [
+ "{\n \"stub\": false,\n \"trainable_name\": \"train_model\",\n \"trial_id\": \"c2c99680\",\n \"storage\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059572040000000000008c1b7261792e747261696e2e5f696e7465726e616c2e73746f72616765948c0e53746f72616765436f6e746578749493942981947d94288c12637573746f6d5f66735f70726f766964656494898c1273746f726167655f6c6f63616c5f70617468948c2b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c7473948c136578706572696d656e745f6469725f6e616d65948c0f746573745f6578706572696d656e74948c0e747269616c5f6469725f6e616d65948c96747261696e5f6d6f64656c5f63326339393638305f31385f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d313032342c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f5f323032342d30362d32355f31382d32362d3038948c1863757272656e745f636865636b706f696e745f696e646578944b038c0b73796e635f636f6e666967948c097261792e747261696e948c0a53796e63436f6e6669679493942981947d94288c0b73796e635f706572696f64944d2c018c0c73796e635f74696d656f7574944d08078c0e73796e635f61727469666163747394898c1c73796e635f6172746966616374735f6f6e5f636865636b706f696e7494888c0a75706c6f61645f646972948c0a44455052454341544544948c0673796e6365729468188c1273796e635f6f6e5f636865636b706f696e7494681875628c1273746f726167655f66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c0f73746f726167655f66735f70617468948c452f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f7465737494681968008c115f46696c6573797374656d53796e6365729493942981947d9428681b682268134d2c0168144d08078c116c6173745f73796e635f75705f74696d65944741d99ebbde5539d88c136c6173745f73796e635f646f776e5f74696d659447fff00000000000008c0d5f73796e635f70726f63657373944e8c0c5f63757272656e745f636d649468008c125f75706c6f61645f746f5f66735f706174689493947d94288c0a6c6f63616c5f70617468948c3b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c74732f746573745f6578706572696d656e74948c0266739468228c0766735f70617468948c552f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e74948c076578636c756465945d948c0e2a2f636865636b706f696e745f2a9461758694756275622e\"\n },\n \"config\": {\n \"encoder_num_layer\": 4,\n \"decoder_rt_num_layer\": 1,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 1024,\n \"encoder_ff\": 1024,\n \"decoder_rt_ff\": 1024,\n \"decoder_int_ff\": 512,\n \"n_head\": 1,\n \"drop_rate\": 0.5887703154000944,\n \"lr\": 0.0008721653331207079,\n \"batch_size\": 2048\n },\n \"_Trial__unresolved_config\": {\n \"encoder_num_layer\": 4,\n \"decoder_rt_num_layer\": 1,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 1024,\n \"encoder_ff\": 1024,\n \"decoder_rt_ff\": 1024,\n \"decoder_int_ff\": 512,\n \"n_head\": 1,\n \"drop_rate\": 0.5887703154000944,\n \"lr\": 0.0008721653331207079,\n \"batch_size\": 2048\n },\n \"evaluated_params\": {\n \"batch_size\": 2048,\n \"decoder_int_ff\": 512,\n \"decoder_int_num_layer\": 1,\n \"decoder_rt_ff\": 1024,\n \"decoder_rt_num_layer\": 1,\n \"drop_rate\": 0.5887703154000944,\n \"embedding_dim\": 1024,\n \"encoder_ff\": 1024,\n \"encoder_num_layer\": 4,\n \"lr\": 0.0008721653331207079,\n \"n_head\": 1\n },\n \"experiment_tag\": \"18_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop_rate=0.5888,embedding_dim=1024,encoder_ff=1024,encoder_num_layer=4,lr=0.0009,n_head=1\",\n \"stopping_criterion\": {},\n \"_setup_default_resource\": true,\n \"_default_placement_group_factory\": \"80054e2e\",\n \"placement_group_factory\": \"800595ba000000000000008c237261792e74756e652e657865637574696f6e2e706c6163656d656e745f67726f757073948c15506c6163656d656e7447726f7570466163746f72799493942981947d94288c085f62756e646c6573945d947d94288c03435055944740540000000000008c034750559447401000000000000075618c155f686561645f62756e646c655f69735f656d70747994898c095f7374726174656779948c045041434b948c055f6172677394298c075f6b7761726773947d9475622e\",\n \"log_to_file\": [\n null,\n null\n ],\n \"max_failures\": 0,\n \"_default_result_or_future\": null,\n \"export_formats\": [],\n \"status\": \"PAUSED\",\n \"relative_logdir\": \"train_model_c2c99680_18_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_18-26-08\",\n \"trial_name_creator\": null,\n \"trial_dirname_creator\": null,\n \"custom_trial_name\": null,\n \"custom_dirname\": null,\n \"restore_path\": null,\n \"_restore_checkpoint_result\": null,\n \"_state_json\": null,\n \"results\": \"80054e2e\",\n \"extra_arg\": \"80054e2e\",\n \"_resources\": \"80054e2e\"\n}",
+ "{\n \"start_time\": 1719332932.6360667,\n \"num_failures\": 0,\n \"num_failures_after_restore\": 0,\n \"error_filename\": null,\n \"pickled_error_filename\": null,\n \"last_result\": {\n \"loss\": 2023.2190220149482,\n \"timestamp\": 1719338188,\n \"checkpoint_dir_name\": \"checkpoint_000003\",\n \"should_checkpoint\": true,\n \"done\": false,\n \"training_iteration\": 4,\n \"trial_id\": \"c2c99680\",\n \"date\": \"2024-06-25_19-56-29\",\n \"time_this_iter_s\": 439.1456036567688,\n \"time_total_s\": 1790.0054636001587,\n \"pid\": 2195253,\n \"hostname\": \"r8i6n2\",\n \"node_ip\": \"10.159.28.60\",\n \"config\": {\n \"encoder_num_layer\": 4,\n \"decoder_rt_num_layer\": 1,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 1024,\n \"encoder_ff\": 1024,\n \"decoder_rt_ff\": 1024,\n \"decoder_int_ff\": 512,\n \"n_head\": 1,\n \"drop_rate\": 0.5887703154000944,\n \"lr\": 0.0008721653331207079,\n \"batch_size\": 2048\n },\n \"time_since_restore\": 893.1749258041382,\n \"iterations_since_restore\": 2,\n \"experiment_tag\": \"18_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop_rate=0.5888,embedding_dim=1024,encoder_ff=1024,encoder_num_layer=4,lr=0.0009,n_head=1\"\n },\n \"last_result_time\": 1719338189.219069,\n \"metric_analysis\": {\n \"loss\": {\n \"max\": 2023.2190220149482,\n \"min\": 1982.5428274560163,\n \"avg\": 2001.3316611943283,\n \"last\": 2023.2190220149482,\n \"last-5-avg\": 2001.3316611943283,\n \"last-10-avg\": 2001.3316611943283\n },\n \"should_checkpoint\": {\n \"max\": true,\n \"min\": true,\n \"avg\": 1.0,\n \"last\": true,\n \"last-5-avg\": 1.0,\n \"last-10-avg\": 1.0\n },\n \"done\": {\n \"max\": false,\n \"min\": false,\n \"avg\": 0.0,\n \"last\": false,\n \"last-5-avg\": 0.0,\n \"last-10-avg\": 0.0\n },\n \"training_iteration\": {\n \"max\": 4,\n \"min\": 1,\n \"avg\": 2.5,\n \"last\": 4,\n \"last-5-avg\": 2.5,\n \"last-10-avg\": 2.5\n },\n \"time_this_iter_s\": {\n \"max\": 456.8247263431549,\n \"min\": 439.1456036567688,\n \"avg\": 447.5013659000397,\n \"last\": 439.1456036567688,\n \"last-5-avg\": 447.5013659000397,\n \"last-10-avg\": 447.5013659000397\n },\n \"time_total_s\": {\n \"max\": 1790.0054636001587,\n \"min\": 456.8247263431549,\n \"avg\": 1123.630146920681,\n \"last\": 1790.0054636001587,\n \"last-5-avg\": 1123.630146920681,\n \"last-10-avg\": 1123.630146920681\n },\n \"time_since_restore\": {\n \"max\": 896.8305377960205,\n \"min\": 454.0293221473694,\n \"avg\": 675.2148780226707,\n \"last\": 893.1749258041382,\n \"last-5-avg\": 675.2148780226707,\n \"last-10-avg\": 675.2148780226707\n },\n \"iterations_since_restore\": {\n \"max\": 2,\n \"min\": 1,\n \"avg\": 1.5,\n \"last\": 2,\n \"last-5-avg\": 1.5,\n \"last-10-avg\": 1.5\n }\n },\n \"_n_steps\": [\n 5,\n 10\n ],\n \"metric_n_steps\": {\n \"loss\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"800595c3000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294288c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b00749462430824128924c40c9f4094869452946807680d4308d8ebf5da2bfa9e4094869452946807680d4308a9542a357e719f4094869452946807680d43083a9d4e47e09c9f409486945294652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"800595c3000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294288c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b00749462430824128924c40c9f4094869452946807680d4308d8ebf5da2bfa9e4094869452946807680d4308a9542a357e719f4094869452946807680d43083a9d4e47e09c9f409486945294652e\"\n }\n },\n \"should_checkpoint\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059526000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452942888888888652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059526000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452942888888888652e\"\n }\n },\n \"done\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059526000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452942889898989652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059526000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452942889898989652e\"\n }\n },\n \"training_iteration\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294284b014b024b034b04652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294284b014b024b034b04652e\"\n }\n },\n \"time_this_iter_s\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059546000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452942847407c8d321440000047407b8017cdc0000047407c60781a80000047407b725464800000652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059546000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452942847407c8d321440000047407b8017cdc0000047407c60781a80000047407b725464800000652e\"\n }\n },\n \"time_total_s\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059546000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452942847407c8d321440000047408c06a4f10000004740951b707f20000047409bf80598400000652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059546000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452942847407c8d321440000047408c06a4f10000004740951b707f20000047409bf80598400000652e\"\n }\n },\n \"time_since_restore\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059546000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452942847407c8d321440000047408c06a4f100000047407c60781a80000047408be9663f800000652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059546000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452942847407c8d321440000047408c06a4f100000047407c60781a80000047408be9663f800000652e\"\n }\n },\n \"iterations_since_restore\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294284b014b024b014b02652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294284b014b024b014b02652e\"\n }\n }\n },\n \"checkpoint_manager\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"800595c1100000000000008c267261792e747261696e2e5f696e7465726e616c2e636865636b706f696e745f6d616e61676572948c125f436865636b706f696e744d616e616765729493942981947d94288c125f636865636b706f696e745f636f6e666967948c097261792e747261696e948c10436865636b706f696e74436f6e6669679493942981947d94288c0b6e756d5f746f5f6b656570944e8c1a636865636b706f696e745f73636f72655f617474726962757465944e8c16636865636b706f696e745f73636f72655f6f72646572948c036d6178948c14636865636b706f696e745f6672657175656e6379944b008c11636865636b706f696e745f61745f656e6494898c1a5f636865636b706f696e745f6b6565705f616c6c5f72616e6b73948c0a44455052454341544544948c1f5f636865636b706f696e745f75706c6f61645f66726f6d5f776f726b65727394681275628c135f636865636b706f696e745f726573756c7473945d94288c1b7261792e747261696e2e5f696e7465726e616c2e73657373696f6e948c0f5f547261696e696e67526573756c749493942981947d94288c0a636865636b706f696e749468068c0a436865636b706f696e749493942981947d94288c0470617468948cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f63326339393638305f31385f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d313032342c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f5f323032342d30362d32355f31382d32362d30382f636865636b706f696e745f303030303030948c0a66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c055f75756964948c0475756964948c04555549449493942981947d948c03696e74948a101c8edb50b935b7a61d419595d7acd718736275628c076d657472696373947d94288c046c6f7373948c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b00749462430824128924c40c9f4094869452948c0974696d657374616d70944a0df27a668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303030948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b018c08747269616c5f6964948c086332633939363830948c0464617465948c13323032342d30362d32355f31382d33362d3239948c1074696d655f746869735f697465725f739447407c8d32144000008c0c74696d655f746f74616c5f739447407c8d32144000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b048c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944d00048c0a656e636f6465725f6666944d00048c0d6465636f6465725f72745f6666944d00048c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b018c0964726f705f7261746594473fe2d734d82ff7e58c026c7294473f4c9440ca8dff148c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f72659447407c8d32144000008c18697465726174696f6e735f73696e63655f726573746f7265944b0175756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f63326339393638305f31385f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d313032342c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f5f323032342d30362d32355f31382d32362d30382f636865636b706f696e745f30303030303194682268257d946827897385945294682a682d2981947d9468308a10e1b8940c1af27eabef4bf5010b4ec7667362756268317d94288c046c6f7373946836683c4308d8ebf5da2bfa9e4094869452948c0974696d657374616d70944ac5f37a668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303031948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b028c08747269616c5f6964948c086332633939363830948c0464617465948c13323032342d30362d32355f31382d34332d3439948c1074696d655f746869735f697465725f739447407b8017cdc000008c0c74696d655f746f74616c5f739447408c06a4f10000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b048c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944d00048c0a656e636f6465725f6666944d00048c0d6465636f6465725f72745f6666944d00048c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b018c0964726f705f7261746594473fe2d734d82ff7e58c026c7294473f4c9440ca8dff148c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f72659447408c06a4f10000008c18697465726174696f6e735f73696e63655f726573746f7265944b0275756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f63326339393638305f31385f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d313032342c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f5f323032342d30362d32355f31382d32362d30382f636865636b706f696e745f30303030303294682268257d946827897385945294682a682d2981947d9468308a1135863e593a8482982843ef83852573c9007362756268317d94288c046c6f7373946836683c4308a9542a357e719f4094869452948c0974696d657374616d70944a15037b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303032948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b038c08747269616c5f6964948c086332633939363830948c0464617465948c13323032342d30362d32355f31392d34392d3130948c1074696d655f746869735f697465725f739447407c60781a8000008c0c74696d655f746f74616c5f73944740951b707f2000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b048c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944d00048c0a656e636f6465725f6666944d00048c0d6465636f6465725f72745f6666944d00048c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b018c0964726f705f7261746594473fe2d734d82ff7e58c026c7294473f4c9440ca8dff148c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f72659447407c60781a8000008c18697465726174696f6e735f73696e63655f726573746f7265944b0175756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f63326339393638305f31385f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d313032342c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f5f323032342d30362d32355f31382d32362d30382f636865636b706f696e745f30303030303394682268257d946827897385945294682a682d2981947d9468308a1021b63c16e40a7298204def415d0d94067362756268317d94288c046c6f7373946836683c43083a9d4e47e09c9f4094869452948c0974696d657374616d70944acc047b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303033948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b048c08747269616c5f6964948c086332633939363830948c0464617465948c13323032342d30362d32355f31392d35362d3239948c1074696d655f746869735f697465725f739447407b7254648000008c0c74696d655f746f74616c5f739447409bf805984000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b048c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944d00048c0a656e636f6465725f6666944d00048c0d6465636f6465725f72745f6666944d00048c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b018c0964726f705f7261746594473fe2d734d82ff7e58c026c7294473f4c9440ca8dff148c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f72659447408be9663f8000008c18697465726174696f6e735f73696e63655f726573746f7265944b02757562658c195f6c61746573745f636865636b706f696e745f726573756c749468c075622e\"\n }\n}"
+ ],
+ [
+ "{\n \"stub\": false,\n \"trainable_name\": \"train_model\",\n \"trial_id\": \"43bba37c\",\n \"storage\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059572040000000000008c1b7261792e747261696e2e5f696e7465726e616c2e73746f72616765948c0e53746f72616765436f6e746578749493942981947d94288c12637573746f6d5f66735f70726f766964656494898c1273746f726167655f6c6f63616c5f70617468948c2b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c7473948c136578706572696d656e745f6469725f6e616d65948c0f746573745f6578706572696d656e74948c0e747269616c5f6469725f6e616d65948c96747261696e5f6d6f64656c5f34336262613337635f31395f62617463685f73697a653d313032342c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d313032342c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f5f323032342d30362d32355f31382d32382d3532948c1863757272656e745f636865636b706f696e745f696e646578944b078c0b73796e635f636f6e666967948c097261792e747261696e948c0a53796e63436f6e6669679493942981947d94288c0b73796e635f706572696f64944d2c018c0c73796e635f74696d656f7574944d08078c0e73796e635f61727469666163747394898c1c73796e635f6172746966616374735f6f6e5f636865636b706f696e7494888c0a75706c6f61645f646972948c0a44455052454341544544948c0673796e6365729468188c1273796e635f6f6e5f636865636b706f696e7494681875628c1273746f726167655f66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c0f73746f726167655f66735f70617468948c452f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f7465737494681968008c115f46696c6573797374656d53796e6365729493942981947d9428681b682268134d2c0168144d08078c116c6173745f73796e635f75705f74696d65944741d99ebbde5539d88c136c6173745f73796e635f646f776e5f74696d659447fff00000000000008c0d5f73796e635f70726f63657373944e8c0c5f63757272656e745f636d649468008c125f75706c6f61645f746f5f66735f706174689493947d94288c0a6c6f63616c5f70617468948c3b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c74732f746573745f6578706572696d656e74948c0266739468228c0766735f70617468948c552f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e74948c076578636c756465945d948c0e2a2f636865636b706f696e745f2a9461758694756275622e\"\n },\n \"config\": {\n \"encoder_num_layer\": 8,\n \"decoder_rt_num_layer\": 1,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 256,\n \"encoder_ff\": 1024,\n \"decoder_rt_ff\": 1024,\n \"decoder_int_ff\": 512,\n \"n_head\": 1,\n \"drop_rate\": 0.4036073673456261,\n \"lr\": 0.0007661776725543005,\n \"batch_size\": 1024\n },\n \"_Trial__unresolved_config\": {\n \"encoder_num_layer\": 8,\n \"decoder_rt_num_layer\": 1,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 256,\n \"encoder_ff\": 1024,\n \"decoder_rt_ff\": 1024,\n \"decoder_int_ff\": 512,\n \"n_head\": 1,\n \"drop_rate\": 0.4036073673456261,\n \"lr\": 0.0007661776725543005,\n \"batch_size\": 1024\n },\n \"evaluated_params\": {\n \"batch_size\": 1024,\n \"decoder_int_ff\": 512,\n \"decoder_int_num_layer\": 1,\n \"decoder_rt_ff\": 1024,\n \"decoder_rt_num_layer\": 1,\n \"drop_rate\": 0.4036073673456261,\n \"embedding_dim\": 256,\n \"encoder_ff\": 1024,\n \"encoder_num_layer\": 8,\n \"lr\": 0.0007661776725543005,\n \"n_head\": 1\n },\n \"experiment_tag\": \"19_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop_rate=0.4036,embedding_dim=256,encoder_ff=1024,encoder_num_layer=8,lr=0.0008,n_head=1\",\n \"stopping_criterion\": {},\n \"_setup_default_resource\": true,\n \"_default_placement_group_factory\": \"80054e2e\",\n \"placement_group_factory\": \"800595ba000000000000008c237261792e74756e652e657865637574696f6e2e706c6163656d656e745f67726f757073948c15506c6163656d656e7447726f7570466163746f72799493942981947d94288c085f62756e646c6573945d947d94288c03435055944740540000000000008c034750559447401000000000000075618c155f686561645f62756e646c655f69735f656d70747994898c095f7374726174656779948c045041434b948c055f6172677394298c075f6b7761726773947d9475622e\",\n \"log_to_file\": [\n null,\n null\n ],\n \"max_failures\": 0,\n \"_default_result_or_future\": null,\n \"export_formats\": [],\n \"status\": \"PAUSED\",\n \"relative_logdir\": \"train_model_43bba37c_19_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_18-28-52\",\n \"trial_name_creator\": null,\n \"trial_dirname_creator\": null,\n \"custom_trial_name\": null,\n \"custom_dirname\": null,\n \"restore_path\": null,\n \"_restore_checkpoint_result\": null,\n \"_state_json\": null,\n \"results\": \"80054e2e\",\n \"extra_arg\": \"80054e2e\",\n \"_resources\": \"80054e2e\"\n}",
+ "{\n \"start_time\": 1719333829.6209524,\n \"num_failures\": 0,\n \"num_failures_after_restore\": 0,\n \"error_filename\": null,\n \"pickled_error_filename\": null,\n \"last_result\": {\n \"loss\": 1962.8120477631337,\n \"timestamp\": 1719340670,\n \"checkpoint_dir_name\": \"checkpoint_000007\",\n \"should_checkpoint\": true,\n \"done\": false,\n \"training_iteration\": 8,\n \"trial_id\": \"43bba37c\",\n \"date\": \"2024-06-25_20-37-50\",\n \"time_this_iter_s\": 147.67359066009521,\n \"time_total_s\": 1221.8980810642242,\n \"pid\": 2195253,\n \"hostname\": \"r8i6n2\",\n \"node_ip\": \"10.159.28.60\",\n \"config\": {\n \"encoder_num_layer\": 8,\n \"decoder_rt_num_layer\": 1,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 256,\n \"encoder_ff\": 1024,\n \"decoder_rt_ff\": 1024,\n \"decoder_int_ff\": 512,\n \"n_head\": 1,\n \"drop_rate\": 0.4036073673456261,\n \"lr\": 0.0007661776725543005,\n \"batch_size\": 1024\n },\n \"time_since_restore\": 603.0734503269196,\n \"iterations_since_restore\": 4,\n \"experiment_tag\": \"19_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop_rate=0.4036,embedding_dim=256,encoder_ff=1024,encoder_num_layer=8,lr=0.0008,n_head=1\"\n },\n \"last_result_time\": 1719340670.7245317,\n \"metric_analysis\": {\n \"loss\": {\n \"max\": 1972.7257733682948,\n \"min\": 1961.9788803941622,\n \"avg\": 1966.0090920155444,\n \"last\": 1962.8120477631337,\n \"last-5-avg\": 1966.325843534695,\n \"last-10-avg\": 1966.0090920155444\n },\n \"should_checkpoint\": {\n \"max\": true,\n \"min\": true,\n \"avg\": 1.0,\n \"last\": true,\n \"last-5-avg\": 1.0,\n \"last-10-avg\": 1.0\n },\n \"done\": {\n \"max\": false,\n \"min\": false,\n \"avg\": 0.0,\n \"last\": false,\n \"last-5-avg\": 0.0,\n \"last-10-avg\": 0.0\n },\n \"training_iteration\": {\n \"max\": 8,\n \"min\": 1,\n \"avg\": 4.5,\n \"last\": 8,\n \"last-5-avg\": 6.0,\n \"last-10-avg\": 4.5\n },\n \"time_this_iter_s\": {\n \"max\": 161.8538646697998,\n \"min\": 147.0123507976532,\n \"avg\": 152.737260133028,\n \"last\": 147.67359066009521,\n \"last-5-avg\": 150.12170810699462,\n \"last-10-avg\": 152.73726013302803\n },\n \"time_total_s\": {\n \"max\": 1221.8980810642242,\n \"min\": 161.8538646697998,\n \"avg\": 695.531242519617,\n \"last\": 1221.8980810642242,\n \"last-5-avg\": 924.2763486385345,\n \"last-10-avg\": 695.5312425196171\n },\n \"time_since_restore\": {\n \"max\": 603.0734503269196,\n \"min\": 160.8864643573761,\n \"avg\": 308.6877292096615,\n \"last\": 603.0734503269196,\n \"last-5-avg\": 367.2716856956482,\n \"last-10-avg\": 308.6877292096615\n },\n \"iterations_since_restore\": {\n \"max\": 4,\n \"min\": 1,\n \"avg\": 2.0,\n \"last\": 4,\n \"last-5-avg\": 2.4,\n \"last-10-avg\": 2.0\n }\n },\n \"_n_steps\": [\n 5,\n 10\n ],\n \"metric_n_steps\": {\n \"loss\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"800595d6000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294288c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b00749462430889442231e7d29e4094869452946807680d4308fe7ebf1f56b69e4094869452946807680d4308381c0e7758b49e4094869452946807680d430802814000afb59e4094869452946807680d4308cce572893fab9e409486945294652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005950f010000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294288c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b0074946243087d3e9f5feaa79e4094869452946807680d430863b158bc38bb9e4094869452946807680d4308ee76bb0da3be9e4094869452946807680d430889442231e7d29e4094869452946807680d4308fe7ebf1f56b69e4094869452946807680d4308381c0e7758b49e4094869452946807680d430802814000afb59e4094869452946807680d4308cce572893fab9e409486945294652e\"\n }\n },\n \"should_checkpoint\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059527000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294288888888888652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294288888888888888888652e\"\n }\n },\n \"done\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059527000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294288989898989652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294288989898989898989652e\"\n }\n },\n \"training_iteration\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952c000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294284b044b054b064b074b08652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059532000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294284b014b024b034b044b054b064b074b08652e\"\n }\n },\n \"time_this_iter_s\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005954f000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b058694529428474062711f758000004740641c5dea80000047406260652d80000047406270088e800000474062758e0e000000652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005956a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294284740643b52dc0000004740627bdea280000047406432126c000000474062711f758000004740641c5dea80000047406260652d80000047406270088e800000474062758e0e000000652e\"\n }\n },\n \"time_total_s\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005954f000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294284740835698d80000004740885db052a0000047408cf5c99e000000474090c8e5e0d000004740931797a2900000652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005956a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294284740643b52dc0000004740735b98bf40000047407d74a1f54000004740835698d80000004740885db052a0000047408cf5c99e000000474090c8e5e0d000004740931797a2900000652e\"\n }\n },\n \"time_since_restore\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005954f000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294284740735198f0c000004740641c5dea8000004740733e618c00000047407c7665d3400000474082d8966d200000652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005956a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294284740643b52dc0000004740735b98bf40000047406432126c0000004740735198f0c000004740641c5dea8000004740733e618c00000047407c7665d3400000474082d8966d200000652e\"\n }\n },\n \"iterations_since_restore\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952c000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294284b024b014b024b034b04652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059532000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294284b014b024b014b024b014b024b034b04652e\"\n }\n }\n },\n \"checkpoint_manager\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"800595fb1e0000000000008c267261792e747261696e2e5f696e7465726e616c2e636865636b706f696e745f6d616e61676572948c125f436865636b706f696e744d616e616765729493942981947d94288c125f636865636b706f696e745f636f6e666967948c097261792e747261696e948c10436865636b706f696e74436f6e6669679493942981947d94288c0b6e756d5f746f5f6b656570944e8c1a636865636b706f696e745f73636f72655f617474726962757465944e8c16636865636b706f696e745f73636f72655f6f72646572948c036d6178948c14636865636b706f696e745f6672657175656e6379944b008c11636865636b706f696e745f61745f656e6494898c1a5f636865636b706f696e745f6b6565705f616c6c5f72616e6b73948c0a44455052454341544544948c1f5f636865636b706f696e745f75706c6f61645f66726f6d5f776f726b65727394681275628c135f636865636b706f696e745f726573756c7473945d94288c1b7261792e747261696e2e5f696e7465726e616c2e73657373696f6e948c0f5f547261696e696e67526573756c749493942981947d94288c0a636865636b706f696e749468068c0a436865636b706f696e749493942981947d94288c0470617468948cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f34336262613337635f31395f62617463685f73697a653d313032342c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d313032342c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f5f323032342d30362d32355f31382d32382d35322f636865636b706f696e745f303030303030948c0a66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c055f75756964948c0475756964948c04555549449493942981947d948c03696e74948a10c55bb4c2568adfa6d54834ade0808d70736275628c076d657472696373947d94288c046c6f7373948c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b0074946243087d3e9f5feaa79e4094869452948c0974696d657374616d70944a67f47a668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303030948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b018c08747269616c5f6964948c083433626261333763948c0464617465948c13323032342d30362d32355f31382d34362d3331948c1074696d655f746869735f697465725f73944740643b52dc0000008c0c74696d655f746f74616c5f73944740643b52dc0000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b088c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944d00018c0a656e636f6465725f6666944d00048c0d6465636f6465725f72745f6666944d00048c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b018c0964726f705f7261746594473fd9d4b3fecb24d28c026c7294473f491b2a05f539228c0a62617463685f73697a65944d0004758c1274696d655f73696e63655f726573746f7265944740643b52dc0000008c18697465726174696f6e735f73696e63655f726573746f7265944b0175756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f34336262613337635f31395f62617463685f73697a653d313032342c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d313032342c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f5f323032342d30362d32355f31382d32382d35322f636865636b706f696e745f30303030303194682268257d946827897385945294682a682d2981947d9468308a118c9b38b9a32c22987742418adc523bfb007362756268317d94288c046c6f7373946836683c430863b158bc38bb9e4094869452948c0974696d657374616d70944afbf47a668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303031948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b028c08747269616c5f6964948c083433626261333763948c0464617465948c13323032342d30362d32355f31382d34382d3539948c1074696d655f746869735f697465725f73944740627bdea28000008c0c74696d655f746f74616c5f73944740735b98bf4000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b088c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944d00018c0a656e636f6465725f6666944d00048c0d6465636f6465725f72745f6666944d00048c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b018c0964726f705f7261746594473fd9d4b3fecb24d28c026c7294473f491b2a05f539228c0a62617463685f73697a65944d0004758c1274696d655f73696e63655f726573746f7265944740735b98bf4000008c18697465726174696f6e735f73696e63655f726573746f7265944b0275756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f34336262613337635f31395f62617463685f73697a653d313032342c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d313032342c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f5f323032342d30362d32355f31382d32382d35322f636865636b706f696e745f30303030303294682268257d946827897385945294682a682d2981947d9468308a11556bfc02404457ad8f4a07c98b89dc83007362756268317d94288c046c6f7373946836683c4308ee76bb0da3be9e4094869452948c0974696d657374616d70944a6e057b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303032948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b038c08747269616c5f6964948c083433626261333763948c0464617465948c13323032342d30362d32355f31392d35392d3130948c1074696d655f746869735f697465725f739447406432126c0000008c0c74696d655f746f74616c5f739447407d74a1f54000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b088c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944d00018c0a656e636f6465725f6666944d00048c0d6465636f6465725f72745f6666944d00048c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b018c0964726f705f7261746594473fd9d4b3fecb24d28c026c7294473f491b2a05f539228c0a62617463685f73697a65944d0004758c1274696d655f73696e63655f726573746f72659447406432126c0000008c18697465726174696f6e735f73696e63655f726573746f7265944b0175756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f34336262613337635f31395f62617463685f73697a653d313032342c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d313032342c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f5f323032342d30362d32355f31382d32382d35322f636865636b706f696e745f30303030303394682268257d946827897385945294682a682d2981947d9468308a115a68d41d442743a97541c3eb24d52f92007362756268317d94288c046c6f7373946836683c430889442231e7d29e4094869452948c0974696d657374616d70944a02067b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303033948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b048c08747269616c5f6964948c083433626261333763948c0464617465948c13323032342d30362d32355f32302d30312d3338948c1074696d655f746869735f697465725f7394474062711f758000008c0c74696d655f746f74616c5f73944740835698d80000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b088c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944d00018c0a656e636f6465725f6666944d00048c0d6465636f6465725f72745f6666944d00048c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b018c0964726f705f7261746594473fd9d4b3fecb24d28c026c7294473f491b2a05f539228c0a62617463685f73697a65944d0004758c1274696d655f73696e63655f726573746f7265944740735198f0c000008c18697465726174696f6e735f73696e63655f726573746f7265944b0275756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f34336262613337635f31395f62617463685f73697a653d313032342c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d313032342c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f5f323032342d30362d32355f31382d32382d35322f636865636b706f696e745f30303030303494682268257d946827897385945294682a682d2981947d9468308a1024dd1969cdf9cbb17b4e4994e75700317362756268317d94288c046c6f7373946836683c4308fe7ebf1f56b69e4094869452948c0974696d657374616d70944ac40c7b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303034948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b058c08747269616c5f6964948c083433626261333763948c0464617465948c13323032342d30362d32355f32302d33302d3238948c1074696d655f746869735f697465725f73944740641c5dea8000008c0c74696d655f746f74616c5f73944740885db052a000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b088c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944d00018c0a656e636f6465725f6666944d00048c0d6465636f6465725f72745f6666944d00048c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b018c0964726f705f7261746594473fd9d4b3fecb24d28c026c7294473f491b2a05f539228c0a62617463685f73697a65944d0004758c1274696d655f73696e63655f726573746f7265944740641c5dea8000008c18697465726174696f6e735f73696e63655f726573746f7265944b0175756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f34336262613337635f31395f62617463685f73697a653d313032342c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d313032342c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f5f323032342d30362d32355f31382d32382d35322f636865636b706f696e745f30303030303594682268257d946827897385945294682a682d2981947d9468308a101be42d7b0b232c97b2421babd3c09e087362756268317d94288c046c6f7373946836683c4308381c0e7758b49e4094869452948c0974696d657374616d70944a570d7b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303035948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b068c08747269616c5f6964948c083433626261333763948c0464617465948c13323032342d30362d32355f32302d33322d3535948c1074696d655f746869735f697465725f739447406260652d8000008c0c74696d655f746f74616c5f739447408cf5c99e0000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b088c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944d00018c0a656e636f6465725f6666944d00048c0d6465636f6465725f72745f6666944d00048c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b018c0964726f705f7261746594473fd9d4b3fecb24d28c026c7294473f491b2a05f539228c0a62617463685f73697a65944d0004758c1274696d655f73696e63655f726573746f7265944740733e618c0000008c18697465726174696f6e735f73696e63655f726573746f7265944b0275756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f34336262613337635f31395f62617463685f73697a653d313032342c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d313032342c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f5f323032342d30362d32355f31382d32382d35322f636865636b706f696e745f30303030303694682268257d946827897385945294682a682d2981947d9468308a1055798b272905eb9bf84a5c121809c9137362756268317d94288c046c6f7373946836683c430802814000afb59e4094869452948c0974696d657374616d70944aea0d7b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303036948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b078c08747269616c5f6964948c083433626261333763948c0464617465948c13323032342d30362d32355f32302d33352d3233948c1074696d655f746869735f697465725f739447406270088e8000008c0c74696d655f746f74616c5f7394474090c8e5e0d000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b088c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944d00018c0a656e636f6465725f6666944d00048c0d6465636f6465725f72745f6666944d00048c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b018c0964726f705f7261746594473fd9d4b3fecb24d28c026c7294473f491b2a05f539228c0a62617463685f73697a65944d0004758c1274696d655f73696e63655f726573746f72659447407c7665d34000008c18697465726174696f6e735f73696e63655f726573746f7265944b0375756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f34336262613337635f31395f62617463685f73697a653d313032342c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d313032342c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f5f323032342d30362d32355f31382d32382d35322f636865636b706f696e745f30303030303794682268257d946827897385945294682a682d2981947d9468308a11fbb129ac075b9a9a344ecce4395bca9d007362756268317d94288c046c6f7373946836683c4308cce572893fab9e4094869452948c0974696d657374616d70944a7e0e7b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303037948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b088c08747269616c5f6964948c083433626261333763948c0464617465948c13323032342d30362d32355f32302d33372d3530948c1074696d655f746869735f697465725f7394474062758e0e0000008c0c74696d655f746f74616c5f73944740931797a29000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b088c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944d00018c0a656e636f6465725f6666944d00048c0d6465636f6465725f72745f6666944d00048c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b018c0964726f705f7261746594473fd9d4b3fecb24d28c026c7294473f491b2a05f539228c0a62617463685f73697a65944d0004758c1274696d655f73696e63655f726573746f726594474082d8966d2000008c18697465726174696f6e735f73696e63655f726573746f7265944b04757562658c195f6c61746573745f636865636b706f696e745f726573756c74946a7c01000075622e\"\n }\n}"
+ ],
+ [
+ "{\n \"stub\": false,\n \"trainable_name\": \"train_model\",\n \"trial_id\": \"6c8b476a\",\n \"storage\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059572040000000000008c1b7261792e747261696e2e5f696e7465726e616c2e73746f72616765948c0e53746f72616765436f6e746578749493942981947d94288c12637573746f6d5f66735f70726f766964656494898c1273746f726167655f6c6f63616c5f70617468948c2b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c7473948c136578706572696d656e745f6469725f6e616d65948c0f746573745f6578706572696d656e74948c0e747269616c5f6469725f6e616d65948c96747261696e5f6d6f64656c5f36633862343736615f32305f62617463685f73697a653d313032342c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d3531322c6465636f6465725f72745f6e756d5f6c617965723d342c64726f705f725f323032342d30362d32355f31382d34332d3439948c1863757272656e745f636865636b706f696e745f696e646578944b038c0b73796e635f636f6e666967948c097261792e747261696e948c0a53796e63436f6e6669679493942981947d94288c0b73796e635f706572696f64944d2c018c0c73796e635f74696d656f7574944d08078c0e73796e635f61727469666163747394898c1c73796e635f6172746966616374735f6f6e5f636865636b706f696e7494888c0a75706c6f61645f646972948c0a44455052454341544544948c0673796e6365729468188c1273796e635f6f6e5f636865636b706f696e7494681875628c1273746f726167655f66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c0f73746f726167655f66735f70617468948c452f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f7465737494681968008c115f46696c6573797374656d53796e6365729493942981947d9428681b682268134d2c0168144d08078c116c6173745f73796e635f75705f74696d65944741d99ebcc08782328c136c6173745f73796e635f646f776e5f74696d659447fff00000000000008c0d5f73796e635f70726f63657373944e8c0c5f63757272656e745f636d649468008c125f75706c6f61645f746f5f66735f706174689493947d94288c0a6c6f63616c5f70617468948c3b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c74732f746573745f6578706572696d656e74948c0266739468228c0766735f70617468948c552f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e74948c076578636c756465945d948c0e2a2f636865636b706f696e745f2a9461758694756275622e\"\n },\n \"config\": {\n \"encoder_num_layer\": 8,\n \"decoder_rt_num_layer\": 4,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 64,\n \"encoder_ff\": 1024,\n \"decoder_rt_ff\": 512,\n \"decoder_int_ff\": 512,\n \"n_head\": 16,\n \"drop_rate\": 0.5088357248512263,\n \"lr\": 0.010524289583233754,\n \"batch_size\": 1024\n },\n \"_Trial__unresolved_config\": {\n \"encoder_num_layer\": 8,\n \"decoder_rt_num_layer\": 4,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 64,\n \"encoder_ff\": 1024,\n \"decoder_rt_ff\": 512,\n \"decoder_int_ff\": 512,\n \"n_head\": 16,\n \"drop_rate\": 0.5088357248512263,\n \"lr\": 0.010524289583233754,\n \"batch_size\": 1024\n },\n \"evaluated_params\": {\n \"batch_size\": 1024,\n \"decoder_int_ff\": 512,\n \"decoder_int_num_layer\": 1,\n \"decoder_rt_ff\": 512,\n \"decoder_rt_num_layer\": 4,\n \"drop_rate\": 0.5088357248512263,\n \"embedding_dim\": 64,\n \"encoder_ff\": 1024,\n \"encoder_num_layer\": 8,\n \"lr\": 0.010524289583233754,\n \"n_head\": 16\n },\n \"experiment_tag\": \"20_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=4,drop_rate=0.5088,embedding_dim=64,encoder_ff=1024,encoder_num_layer=8,lr=0.0105,n_head=16\",\n \"stopping_criterion\": {},\n \"_setup_default_resource\": true,\n \"_default_placement_group_factory\": \"80054e2e\",\n \"placement_group_factory\": \"800595ba000000000000008c237261792e74756e652e657865637574696f6e2e706c6163656d656e745f67726f757073948c15506c6163656d656e7447726f7570466163746f72799493942981947d94288c085f62756e646c6573945d947d94288c03435055944740540000000000008c034750559447401000000000000075618c155f686561645f62756e646c655f69735f656d70747994898c095f7374726174656779948c045041434b948c055f6172677394298c075f6b7761726773947d9475622e\",\n \"log_to_file\": [\n null,\n null\n ],\n \"max_failures\": 0,\n \"_default_result_or_future\": null,\n \"export_formats\": [],\n \"status\": \"PAUSED\",\n \"relative_logdir\": \"train_model_6c8b476a_20_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=4,drop_r_2024-06-25_18-43-49\",\n \"trial_name_creator\": null,\n \"trial_dirname_creator\": null,\n \"custom_trial_name\": null,\n \"custom_dirname\": null,\n \"restore_path\": null,\n \"_restore_checkpoint_result\": null,\n \"_state_json\": null,\n \"results\": \"80054e2e\",\n \"extra_arg\": \"80054e2e\",\n \"_resources\": \"80054e2e\"\n}",
+ "{\n \"start_time\": 1719334139.5021868,\n \"num_failures\": 0,\n \"num_failures_after_restore\": 0,\n \"error_filename\": null,\n \"pickled_error_filename\": null,\n \"last_result\": {\n \"loss\": 1991.5076505405696,\n \"timestamp\": 1719338738,\n \"checkpoint_dir_name\": \"checkpoint_000003\",\n \"should_checkpoint\": true,\n \"done\": false,\n \"training_iteration\": 4,\n \"trial_id\": \"6c8b476a\",\n \"date\": \"2024-06-25_20-05-38\",\n \"time_this_iter_s\": 113.23560690879822,\n \"time_total_s\": 480.9753620624542,\n \"pid\": 2195253,\n \"hostname\": \"r8i6n2\",\n \"node_ip\": \"10.159.28.60\",\n \"config\": {\n \"encoder_num_layer\": 8,\n \"decoder_rt_num_layer\": 4,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 64,\n \"encoder_ff\": 1024,\n \"decoder_rt_ff\": 512,\n \"decoder_int_ff\": 512,\n \"n_head\": 16,\n \"drop_rate\": 0.5088357248512263,\n \"lr\": 0.010524289583233754,\n \"batch_size\": 1024\n },\n \"time_since_restore\": 239.98935389518738,\n \"iterations_since_restore\": 2,\n \"experiment_tag\": \"20_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=4,drop_rate=0.5088,embedding_dim=64,encoder_ff=1024,encoder_num_layer=8,lr=0.0105,n_head=16\"\n },\n \"last_result_time\": 1719338738.6413836,\n \"metric_analysis\": {\n \"loss\": {\n \"max\": 2006.8072533795214,\n \"min\": 1991.5076505405696,\n \"avg\": 1997.4594356506825,\n \"last\": 1991.5076505405696,\n \"last-5-avg\": 1997.4594356506827,\n \"last-10-avg\": 1997.4594356506827\n },\n \"should_checkpoint\": {\n \"max\": true,\n \"min\": true,\n \"avg\": 1.0,\n \"last\": true,\n \"last-5-avg\": 1.0,\n \"last-10-avg\": 1.0\n },\n \"done\": {\n \"max\": false,\n \"min\": false,\n \"avg\": 0.0,\n \"last\": false,\n \"last-5-avg\": 0.0,\n \"last-10-avg\": 0.0\n },\n \"training_iteration\": {\n \"max\": 4,\n \"min\": 1,\n \"avg\": 2.5,\n \"last\": 4,\n \"last-5-avg\": 2.5,\n \"last-10-avg\": 2.5\n },\n \"time_this_iter_s\": {\n \"max\": 127.51717782020569,\n \"min\": 113.23560690879822,\n \"avg\": 120.24384051561356,\n \"last\": 113.23560690879822,\n \"last-5-avg\": 120.24384051561356,\n \"last-10-avg\": 120.24384051561356\n },\n \"time_total_s\": {\n \"max\": 480.9753620624542,\n \"min\": 127.51717782020569,\n \"avg\": 304.3045758008957,\n \"last\": 480.9753620624542,\n \"last-5-avg\": 304.3045758008957,\n \"last-10-avg\": 304.3045758008957\n },\n \"time_since_restore\": {\n \"max\": 240.98600816726685,\n \"min\": 126.75374698638916,\n \"avg\": 183.81157171726227,\n \"last\": 239.98935389518738,\n \"last-5-avg\": 183.81157171726227,\n \"last-10-avg\": 183.81157171726227\n },\n \"iterations_since_restore\": {\n \"max\": 2,\n \"min\": 1,\n \"avg\": 1.5,\n \"last\": 2,\n \"last-5-avg\": 1.5,\n \"last-10-avg\": 1.5\n }\n },\n \"_n_steps\": [\n 5,\n 10\n ],\n \"metric_n_steps\": {\n \"loss\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"800595c3000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294288c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b0074946243088542a1a03a5b9f4094869452946807680d430828140a75fc2c9f4094869452946807680d4308f0f7fbed1a319f4094869452946807680d43082c168bd5071e9f409486945294652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"800595c3000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294288c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b0074946243088542a1a03a5b9f4094869452946807680d430828140a75fc2c9f4094869452946807680d4308f0f7fbed1a319f4094869452946807680d43082c168bd5071e9f409486945294652e\"\n }\n },\n \"should_checkpoint\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059526000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452942888888888652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059526000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452942888888888652e\"\n }\n },\n \"done\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059526000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452942889898989652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059526000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452942889898989652e\"\n }\n },\n \"training_iteration\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294284b014b024b034b04652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294284b014b024b034b04652e\"\n }\n },\n \"time_this_iter_s\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059546000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452942847405fe1197100000047405c5e015100000047405fb03d6400000047405c4f142f000000652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059546000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452942847405fe1197100000047405c5e015100000047405fb03d6400000047405c4f142f000000652e\"\n }\n },\n \"time_total_s\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059546000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452942847405fe1197100000047406e1f8d61000000474076fbd60980000047407e0f9b15400000652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059546000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452942847405fe1197100000047406e1f8d61000000474076fbd60980000047407e0f9b15400000652e\"\n }\n },\n \"time_since_restore\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059546000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452942847405fe1197100000047406e1f8d6100000047405fb03d6400000047406dffa8c9800000652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059546000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452942847405fe1197100000047406e1f8d6100000047405fb03d6400000047406dffa8c9800000652e\"\n }\n },\n \"iterations_since_restore\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294284b014b024b014b02652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294284b014b024b014b02652e\"\n }\n }\n },\n \"checkpoint_manager\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"800595be100000000000008c267261792e747261696e2e5f696e7465726e616c2e636865636b706f696e745f6d616e61676572948c125f436865636b706f696e744d616e616765729493942981947d94288c125f636865636b706f696e745f636f6e666967948c097261792e747261696e948c10436865636b706f696e74436f6e6669679493942981947d94288c0b6e756d5f746f5f6b656570944e8c1a636865636b706f696e745f73636f72655f617474726962757465944e8c16636865636b706f696e745f73636f72655f6f72646572948c036d6178948c14636865636b706f696e745f6672657175656e6379944b008c11636865636b706f696e745f61745f656e6494898c1a5f636865636b706f696e745f6b6565705f616c6c5f72616e6b73948c0a44455052454341544544948c1f5f636865636b706f696e745f75706c6f61645f66726f6d5f776f726b65727394681275628c135f636865636b706f696e745f726573756c7473945d94288c1b7261792e747261696e2e5f696e7465726e616c2e73657373696f6e948c0f5f547261696e696e67526573756c749493942981947d94288c0a636865636b706f696e749468068c0a436865636b706f696e749493942981947d94288c0470617468948cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f36633862343736615f32305f62617463685f73697a653d313032342c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d3531322c6465636f6465725f72745f6e756d5f6c617965723d342c64726f705f725f323032342d30362d32355f31382d34332d34392f636865636b706f696e745f303030303030948c0a66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c055f75756964948c0475756964948c04555549449493942981947d948c03696e74948a11803e749eadb9749b79460273979bdcab00736275628c076d657472696373947d94288c046c6f7373948c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b0074946243088542a1a03a5b9f4094869452948c0974696d657374616d70944a7bf57a668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303030948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b018c08747269616c5f6964948c083663386234373661948c0464617465948c13323032342d30362d32355f31382d35312d3037948c1074696d655f746869735f697465725f739447405fe119710000008c0c74696d655f746f74616c5f739447405fe119710000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b088c146465636f6465725f72745f6e756d5f6c61796572944b048c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b408c0a656e636f6465725f6666944d00048c0d6465636f6465725f72745f6666944d00028c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b108c0964726f705f7261746594473fe04861dba8b8168c026c7294473f858dc23c96c75d8c0a62617463685f73697a65944d0004758c1274696d655f73696e63655f726573746f72659447405fe119710000008c18697465726174696f6e735f73696e63655f726573746f7265944b0175756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f36633862343736615f32305f62617463685f73697a653d313032342c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d3531322c6465636f6465725f72745f6e756d5f6c617965723d342c64726f705f725f323032342d30362d32355f31382d34332d34392f636865636b706f696e745f30303030303194682268257d946827897385945294682a682d2981947d9468308a11fccc6197e14f14be294ffea124b8adb9007362756268317d94288c046c6f7373946836683c430828140a75fc2c9f4094869452948c0974696d657374616d70944aecf57a668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303031948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b028c08747269616c5f6964948c083663386234373661948c0464617465948c13323032342d30362d32355f31382d35332d3030948c1074696d655f746869735f697465725f739447405c5e01510000008c0c74696d655f746f74616c5f739447406e1f8d610000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b088c146465636f6465725f72745f6e756d5f6c61796572944b048c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b408c0a656e636f6465725f6666944d00048c0d6465636f6465725f72745f6666944d00028c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b108c0964726f705f7261746594473fe04861dba8b8168c026c7294473f858dc23c96c75d8c0a62617463685f73697a65944d0004758c1274696d655f73696e63655f726573746f72659447406e1f8d610000008c18697465726174696f6e735f73696e63655f726573746f7265944b0275756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f36633862343736615f32305f62617463685f73697a653d313032342c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d3531322c6465636f6465725f72745f6e756d5f6c617965723d342c64726f705f725f323032342d30362d32355f31382d34332d34392f636865636b706f696e745f30303030303294682268257d946827897385945294682a682d2981947d9468308a105ca15e0bab76d6a61c457fae0a0690687362756268317d94288c046c6f7373946836683c4308f0f7fbed1a319f4094869452948c0974696d657374616d70944a81067b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303032948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b038c08747269616c5f6964948c083663386234373661948c0464617465948c13323032342d30362d32355f32302d30332d3435948c1074696d655f746869735f697465725f739447405fb03d640000008c0c74696d655f746f74616c5f7394474076fbd6098000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b088c146465636f6465725f72745f6e756d5f6c61796572944b048c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b408c0a656e636f6465725f6666944d00048c0d6465636f6465725f72745f6666944d00028c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b108c0964726f705f7261746594473fe04861dba8b8168c026c7294473f858dc23c96c75d8c0a62617463685f73697a65944d0004758c1274696d655f73696e63655f726573746f72659447405fb03d640000008c18697465726174696f6e735f73696e63655f726573746f7265944b0175756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f36633862343736615f32305f62617463685f73697a653d313032342c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d3531322c6465636f6465725f72745f6e756d5f6c617965723d342c64726f705f725f323032342d30362d32355f31382d34332d34392f636865636b706f696e745f30303030303394682268257d946827897385945294682a682d2981947d9468308a10085a31f812052289c447ecf15e9e87147362756268317d94288c046c6f7373946836683c43082c168bd5071e9f4094869452948c0974696d657374616d70944af2067b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303033948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b048c08747269616c5f6964948c083663386234373661948c0464617465948c13323032342d30362d32355f32302d30352d3338948c1074696d655f746869735f697465725f739447405c4f142f0000008c0c74696d655f746f74616c5f739447407e0f9b154000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b088c146465636f6465725f72745f6e756d5f6c61796572944b048c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b408c0a656e636f6465725f6666944d00048c0d6465636f6465725f72745f6666944d00028c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b108c0964726f705f7261746594473fe04861dba8b8168c026c7294473f858dc23c96c75d8c0a62617463685f73697a65944d0004758c1274696d655f73696e63655f726573746f72659447406dffa8c98000008c18697465726174696f6e735f73696e63655f726573746f7265944b02757562658c195f6c61746573745f636865636b706f696e745f726573756c749468c075622e\"\n }\n}"
+ ],
+ [
+ "{\n \"stub\": false,\n \"trainable_name\": \"train_model\",\n \"trial_id\": \"6743cdee\",\n \"storage\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059572040000000000008c1b7261792e747261696e2e5f696e7465726e616c2e73746f72616765948c0e53746f72616765436f6e746578749493942981947d94288c12637573746f6d5f66735f70726f766964656494898c1273746f726167655f6c6f63616c5f70617468948c2b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c7473948c136578706572696d656e745f6469725f6e616d65948c0f746573745f6578706572696d656e74948c0e747269616c5f6469725f6e616d65948c96747261696e5f6d6f64656c5f36373433636465655f32315f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d323034382c6465636f6465725f72745f6e756d5f6c617965723d342c64726f705f5f323032342d30362d32355f31382d34382d3539948c1863757272656e745f636865636b706f696e745f696e646578944b018c0b73796e635f636f6e666967948c097261792e747261696e948c0a53796e63436f6e6669679493942981947d94288c0b73796e635f706572696f64944d2c018c0c73796e635f74696d656f7574944d08078c0e73796e635f61727469666163747394898c1c73796e635f6172746966616374735f6f6e5f636865636b706f696e7494888c0a75706c6f61645f646972948c0a44455052454341544544948c0673796e6365729468188c1273796e635f6f6e5f636865636b706f696e7494681875628c1273746f726167655f66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c0f73746f726167655f66735f70617468948c452f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f7465737494681968008c115f46696c6573797374656d53796e6365729493942981947d9428681b682268134d2c0168144d08078c116c6173745f73796e635f75705f74696d65944741d99ebd0beabbc18c136c6173745f73796e635f646f776e5f74696d659447fff00000000000008c0d5f73796e635f70726f63657373944e8c0c5f63757272656e745f636d649468008c125f75706c6f61645f746f5f66735f706174689493947d94288c0a6c6f63616c5f70617468948c3b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c74732f746573745f6578706572696d656e74948c0266739468228c0766735f70617468948c552f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e74948c076578636c756465945d948c0e2a2f636865636b706f696e745f2a9461758694756275622e\"\n },\n \"config\": {\n \"encoder_num_layer\": 2,\n \"decoder_rt_num_layer\": 4,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 256,\n \"encoder_ff\": 512,\n \"decoder_rt_ff\": 2048,\n \"decoder_int_ff\": 512,\n \"n_head\": 8,\n \"drop_rate\": 0.4436137502039823,\n \"lr\": 0.07179226263764402,\n \"batch_size\": 2048\n },\n \"_Trial__unresolved_config\": {\n \"encoder_num_layer\": 2,\n \"decoder_rt_num_layer\": 4,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 256,\n \"encoder_ff\": 512,\n \"decoder_rt_ff\": 2048,\n \"decoder_int_ff\": 512,\n \"n_head\": 8,\n \"drop_rate\": 0.4436137502039823,\n \"lr\": 0.07179226263764402,\n \"batch_size\": 2048\n },\n \"evaluated_params\": {\n \"batch_size\": 2048,\n \"decoder_int_ff\": 512,\n \"decoder_int_num_layer\": 1,\n \"decoder_rt_ff\": 2048,\n \"decoder_rt_num_layer\": 4,\n \"drop_rate\": 0.4436137502039823,\n \"embedding_dim\": 256,\n \"encoder_ff\": 512,\n \"encoder_num_layer\": 2,\n \"lr\": 0.07179226263764402,\n \"n_head\": 8\n },\n \"experiment_tag\": \"21_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=4,drop_rate=0.4436,embedding_dim=256,encoder_ff=512,encoder_num_layer=2,lr=0.0718,n_head=8\",\n \"stopping_criterion\": {},\n \"_setup_default_resource\": true,\n \"_default_placement_group_factory\": \"80054e2e\",\n \"placement_group_factory\": \"800595ba000000000000008c237261792e74756e652e657865637574696f6e2e706c6163656d656e745f67726f757073948c15506c6163656d656e7447726f7570466163746f72799493942981947d94288c085f62756e646c6573945d947d94288c03435055944740540000000000008c034750559447401000000000000075618c155f686561645f62756e646c655f69735f656d70747994898c095f7374726174656779948c045041434b948c055f6172677394298c075f6b7761726773947d9475622e\",\n \"log_to_file\": [\n null,\n null\n ],\n \"max_failures\": 0,\n \"_default_result_or_future\": null,\n \"export_formats\": [],\n \"status\": \"PAUSED\",\n \"relative_logdir\": \"train_model_6743cdee_21_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=4,drop__2024-06-25_18-48-59\",\n \"trial_name_creator\": null,\n \"trial_dirname_creator\": null,\n \"custom_trial_name\": null,\n \"custom_dirname\": null,\n \"restore_path\": null,\n \"_restore_checkpoint_result\": null,\n \"_state_json\": null,\n \"results\": \"80054e2e\",\n \"extra_arg\": \"80054e2e\",\n \"_resources\": \"80054e2e\"\n}",
+ "{\n \"start_time\": 1719334380.648466,\n \"num_failures\": 0,\n \"num_failures_after_restore\": 0,\n \"error_filename\": null,\n \"pickled_error_filename\": null,\n \"last_result\": {\n \"loss\": 2680.254179226132,\n \"timestamp\": 1719334648,\n \"checkpoint_dir_name\": \"checkpoint_000001\",\n \"should_checkpoint\": true,\n \"done\": false,\n \"training_iteration\": 2,\n \"trial_id\": \"6743cdee\",\n \"date\": \"2024-06-25_18-57-29\",\n \"time_this_iter_s\": 127.1407482624054,\n \"time_total_s\": 268.34629464149475,\n \"pid\": 2195253,\n \"hostname\": \"r8i6n2\",\n \"node_ip\": \"10.159.28.60\",\n \"config\": {\n \"encoder_num_layer\": 2,\n \"decoder_rt_num_layer\": 4,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 256,\n \"encoder_ff\": 512,\n \"decoder_rt_ff\": 2048,\n \"decoder_int_ff\": 512,\n \"n_head\": 8,\n \"drop_rate\": 0.4436137502039823,\n \"lr\": 0.07179226263764402,\n \"batch_size\": 2048\n },\n \"time_since_restore\": 268.34629464149475,\n \"iterations_since_restore\": 2,\n \"experiment_tag\": \"21_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=4,drop_rate=0.4436,embedding_dim=256,encoder_ff=512,encoder_num_layer=2,lr=0.0718,n_head=8\"\n },\n \"last_result_time\": 1719334649.0042667,\n \"metric_analysis\": {\n \"loss\": {\n \"max\": 3811.964426596334,\n \"min\": 2680.254179226132,\n \"avg\": 3246.1093029112326,\n \"last\": 2680.254179226132,\n \"last-5-avg\": 3246.1093029112326,\n \"last-10-avg\": 3246.1093029112326\n },\n \"should_checkpoint\": {\n \"max\": true,\n \"min\": true,\n \"avg\": 1.0,\n \"last\": true,\n \"last-5-avg\": 1.0,\n \"last-10-avg\": 1.0\n },\n \"done\": {\n \"max\": false,\n \"min\": false,\n \"avg\": 0.0,\n \"last\": false,\n \"last-5-avg\": 0.0,\n \"last-10-avg\": 0.0\n },\n \"training_iteration\": {\n \"max\": 2,\n \"min\": 1,\n \"avg\": 1.5,\n \"last\": 2,\n \"last-5-avg\": 1.5,\n \"last-10-avg\": 1.5\n },\n \"time_this_iter_s\": {\n \"max\": 141.20554637908936,\n \"min\": 127.1407482624054,\n \"avg\": 134.17314732074738,\n \"last\": 127.1407482624054,\n \"last-5-avg\": 134.17314732074738,\n \"last-10-avg\": 134.17314732074738\n },\n \"time_total_s\": {\n \"max\": 268.34629464149475,\n \"min\": 141.20554637908936,\n \"avg\": 204.77592051029205,\n \"last\": 268.34629464149475,\n \"last-5-avg\": 204.77592051029205,\n \"last-10-avg\": 204.77592051029205\n },\n \"time_since_restore\": {\n \"max\": 268.34629464149475,\n \"min\": 141.20554637908936,\n \"avg\": 204.77592051029205,\n \"last\": 268.34629464149475,\n \"last-5-avg\": 204.77592051029205,\n \"last-10-avg\": 204.77592051029205\n },\n \"iterations_since_restore\": {\n \"max\": 2,\n \"min\": 1,\n \"avg\": 1.5,\n \"last\": 2,\n \"last-5-avg\": 1.5,\n \"last-10-avg\": 1.5\n }\n },\n \"_n_steps\": [\n 5,\n 10\n ],\n \"metric_n_steps\": {\n \"loss\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005959d000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294288c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b0074946243084ba552c9edc7ad4094869452946807680d43081e8fc72382f0a4409486945294652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005959d000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294288c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b0074946243084ba552c9edc7ad4094869452946807680d43081e8fc72382f0a4409486945294652e\"\n }\n },\n \"should_checkpoint\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059524000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294288888652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059524000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294288888652e\"\n }\n },\n \"done\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059524000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294288989652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059524000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294288989652e\"\n }\n },\n \"training_iteration\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059526000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294284b014b02652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059526000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294284b014b02652e\"\n }\n },\n \"time_this_iter_s\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059534000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b058694529428474061a693d600000047405fc90205000000652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059534000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a8694529428474061a693d600000047405fc90205000000652e\"\n }\n },\n \"time_total_s\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059534000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b058694529428474061a693d6000000474070c58a6c400000652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059534000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a8694529428474061a693d6000000474070c58a6c400000652e\"\n }\n },\n \"time_since_restore\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059534000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b058694529428474061a693d6000000474070c58a6c400000652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059534000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a8694529428474061a693d6000000474070c58a6c400000652e\"\n }\n },\n \"iterations_since_restore\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059526000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294284b014b02652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059526000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294284b014b02652e\"\n }\n }\n },\n \"checkpoint_manager\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"800595a6090000000000008c267261792e747261696e2e5f696e7465726e616c2e636865636b706f696e745f6d616e61676572948c125f436865636b706f696e744d616e616765729493942981947d94288c125f636865636b706f696e745f636f6e666967948c097261792e747261696e948c10436865636b706f696e74436f6e6669679493942981947d94288c0b6e756d5f746f5f6b656570944e8c1a636865636b706f696e745f73636f72655f617474726962757465944e8c16636865636b706f696e745f73636f72655f6f72646572948c036d6178948c14636865636b706f696e745f6672657175656e6379944b008c11636865636b706f696e745f61745f656e6494898c1a5f636865636b706f696e745f6b6565705f616c6c5f72616e6b73948c0a44455052454341544544948c1f5f636865636b706f696e745f75706c6f61645f66726f6d5f776f726b65727394681275628c135f636865636b706f696e745f726573756c7473945d94288c1b7261792e747261696e2e5f696e7465726e616c2e73657373696f6e948c0f5f547261696e696e67526573756c749493942981947d94288c0a636865636b706f696e749468068c0a436865636b706f696e749493942981947d94288c0470617468948cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f36373433636465655f32315f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d323034382c6465636f6465725f72745f6e756d5f6c617965723d342c64726f705f5f323032342d30362d32355f31382d34382d35392f636865636b706f696e745f303030303030948c0a66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c055f75756964948c0475756964948c04555549449493942981947d948c03696e74948a1030324741ea99cb8426455c335ab8d75d736275628c076d657472696373947d94288c046c6f7373948c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b0074946243084ba552c9edc7ad4094869452948c0974696d657374616d70944a79f67a668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303030948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b018c08747269616c5f6964948c083637343363646565948c0464617465948c13323032342d30362d32355f31382d35352d3231948c1074696d655f746869735f697465725f7394474061a693d60000008c0c74696d655f746f74616c5f7394474061a693d60000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b028c146465636f6465725f72745f6e756d5f6c61796572944b048c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944d00018c0a656e636f6465725f6666944d00028c0d6465636f6465725f72745f6666944d00088c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b088c0964726f705f7261746594473fdc642aed4ba62c8c026c7294473fb260fa4c2270268c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f726594474061a693d60000008c18697465726174696f6e735f73696e63655f726573746f7265944b0175756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f36373433636465655f32315f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d323034382c6465636f6465725f72745f6e756d5f6c617965723d342c64726f705f5f323032342d30362d32355f31382d34382d35392f636865636b706f696e745f30303030303194682268257d946827897385945294682a682d2981947d9468308a10abdc9a77d96eacb24f45169ee215a2007362756268317d94288c046c6f7373946836683c43081e8fc72382f0a44094869452948c0974696d657374616d70944af8f67a668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303031948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b028c08747269616c5f6964948c083637343363646565948c0464617465948c13323032342d30362d32355f31382d35372d3239948c1074696d655f746869735f697465725f739447405fc902050000008c0c74696d655f746f74616c5f7394474070c58a6c4000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b028c146465636f6465725f72745f6e756d5f6c61796572944b048c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944d00018c0a656e636f6465725f6666944d00028c0d6465636f6465725f72745f6666944d00088c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b088c0964726f705f7261746594473fdc642aed4ba62c8c026c7294473fb260fa4c2270268c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f726594474070c58a6c4000008c18697465726174696f6e735f73696e63655f726573746f7265944b02757562658c195f6c61746573745f636865636b706f696e745f726573756c7494686275622e\"\n }\n}"
+ ],
+ [
+ "{\n \"stub\": false,\n \"trainable_name\": \"train_model\",\n \"trial_id\": \"0f9cc97d\",\n \"storage\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059572040000000000008c1b7261792e747261696e2e5f696e7465726e616c2e73746f72616765948c0e53746f72616765436f6e746578749493942981947d94288c12637573746f6d5f66735f70726f766964656494898c1273746f726167655f6c6f63616c5f70617468948c2b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c7473948c136578706572696d656e745f6469725f6e616d65948c0f746573745f6578706572696d656e74948c0e747269616c5f6469725f6e616d65948c96747261696e5f6d6f64656c5f30663963633937645f32325f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d3531322c6465636f6465725f72745f6e756d5f6c617965723d322c64726f705f725f323032342d30362d32355f31382d35332d3030948c1863757272656e745f636865636b706f696e745f696e646578944b0d8c0b73796e635f636f6e666967948c097261792e747261696e948c0a53796e63436f6e6669679493942981947d94288c0b73796e635f706572696f64944d2c018c0c73796e635f74696d656f7574944d08078c0e73796e635f61727469666163747394898c1c73796e635f6172746966616374735f6f6e5f636865636b706f696e7494888c0a75706c6f61645f646972948c0a44455052454341544544948c0673796e6365729468188c1273796e635f6f6e5f636865636b706f696e7494681875628c1273746f726167655f66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c0f73746f726167655f66735f70617468948c452f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f7465737494681968008c115f46696c6573797374656d53796e6365729493942981947d9428681b682268134d2c0168144d08078c116c6173745f73796e635f75705f74696d65944741d99ebd577452378c136c6173745f73796e635f646f776e5f74696d659447fff00000000000008c0d5f73796e635f70726f63657373944e8c0c5f63757272656e745f636d649468008c125f75706c6f61645f746f5f66735f706174689493947d94288c0a6c6f63616c5f70617468948c3b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c74732f746573745f6578706572696d656e74948c0266739468228c0766735f70617468948c552f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e74948c076578636c756465945d948c0e2a2f636865636b706f696e745f2a9461758694756275622e\"\n },\n \"config\": {\n \"encoder_num_layer\": 4,\n \"decoder_rt_num_layer\": 2,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 1024,\n \"encoder_ff\": 1024,\n \"decoder_rt_ff\": 512,\n \"decoder_int_ff\": 512,\n \"n_head\": 1,\n \"drop_rate\": 0.005508735010543275,\n \"lr\": 0.0011880094129501908,\n \"batch_size\": 2048\n },\n \"_Trial__unresolved_config\": {\n \"encoder_num_layer\": 4,\n \"decoder_rt_num_layer\": 2,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 1024,\n \"encoder_ff\": 1024,\n \"decoder_rt_ff\": 512,\n \"decoder_int_ff\": 512,\n \"n_head\": 1,\n \"drop_rate\": 0.005508735010543275,\n \"lr\": 0.0011880094129501908,\n \"batch_size\": 2048\n },\n \"evaluated_params\": {\n \"batch_size\": 2048,\n \"decoder_int_ff\": 512,\n \"decoder_int_num_layer\": 1,\n \"decoder_rt_ff\": 512,\n \"decoder_rt_num_layer\": 2,\n \"drop_rate\": 0.005508735010543275,\n \"embedding_dim\": 1024,\n \"encoder_ff\": 1024,\n \"encoder_num_layer\": 4,\n \"lr\": 0.0011880094129501908,\n \"n_head\": 1\n },\n \"experiment_tag\": \"22_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_rate=0.0055,embedding_dim=1024,encoder_ff=1024,encoder_num_layer=4,lr=0.0012,n_head=1\",\n \"stopping_criterion\": {},\n \"_setup_default_resource\": true,\n \"_default_placement_group_factory\": \"80054e2e\",\n \"placement_group_factory\": \"800595ba000000000000008c237261792e74756e652e657865637574696f6e2e706c6163656d656e745f67726f757073948c15506c6163656d656e7447726f7570466163746f72799493942981947d94288c085f62756e646c6573945d947d94288c03435055944740540000000000008c034750559447401000000000000075618c155f686561645f62756e646c655f69735f656d70747994898c095f7374726174656779948c045041434b948c055f6172677394298c075f6b7761726773947d9475622e\",\n \"log_to_file\": [\n null,\n null\n ],\n \"max_failures\": 0,\n \"_default_result_or_future\": null,\n \"export_formats\": [],\n \"status\": \"TERMINATED\",\n \"relative_logdir\": \"train_model_0f9cc97d_22_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_18-53-00\",\n \"trial_name_creator\": null,\n \"trial_dirname_creator\": null,\n \"custom_trial_name\": null,\n \"custom_dirname\": null,\n \"restore_path\": null,\n \"_restore_checkpoint_result\": null,\n \"_state_json\": null,\n \"results\": \"80054e2e\",\n \"extra_arg\": \"80054e2e\",\n \"_resources\": \"80054e2e\"\n}",
+ "{\n \"start_time\": 1719334649.1537328,\n \"num_failures\": 0,\n \"num_failures_after_restore\": 0,\n \"error_filename\": null,\n \"pickled_error_filename\": null,\n \"last_result\": {\n \"loss\": 1956.4829832062007,\n \"timestamp\": 1719345747,\n \"checkpoint_dir_name\": \"checkpoint_000013\",\n \"should_checkpoint\": true,\n \"done\": true,\n \"training_iteration\": 14,\n \"trial_id\": \"0f9cc97d\",\n \"date\": \"2024-06-25_22-02-27\",\n \"time_this_iter_s\": 506.25228786468506,\n \"time_total_s\": 7130.941356182098,\n \"pid\": 2195253,\n \"hostname\": \"r8i6n2\",\n \"node_ip\": \"10.159.28.60\",\n \"config\": {\n \"encoder_num_layer\": 4,\n \"decoder_rt_num_layer\": 2,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 1024,\n \"encoder_ff\": 1024,\n \"decoder_rt_ff\": 512,\n \"decoder_int_ff\": 512,\n \"n_head\": 1,\n \"drop_rate\": 0.005508735010543275,\n \"lr\": 0.0011880094129501908,\n \"batch_size\": 2048\n },\n \"time_since_restore\": 5076.989426612854,\n \"iterations_since_restore\": 10,\n \"experiment_tag\": \"22_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_rate=0.0055,embedding_dim=1024,encoder_ff=1024,encoder_num_layer=4,lr=0.0012,n_head=1\"\n },\n \"last_result_time\": 1719345747.9423904,\n \"metric_analysis\": {\n \"loss\": {\n \"max\": 1970.2912184347317,\n \"min\": 1951.9356997031866,\n \"avg\": 1956.8511505641873,\n \"last\": 1956.4829832062007,\n \"last-5-avg\": 1954.9646307517226,\n \"last-10-avg\": 1954.5448684331939\n },\n \"should_checkpoint\": {\n \"max\": true,\n \"min\": true,\n \"avg\": 1.0,\n \"last\": true,\n \"last-5-avg\": 1.0,\n \"last-10-avg\": 1.0\n },\n \"done\": {\n \"max\": false,\n \"min\": false,\n \"avg\": 0.0,\n \"last\": false,\n \"last-5-avg\": 0.0,\n \"last-10-avg\": 0.0\n },\n \"training_iteration\": {\n \"max\": 14,\n \"min\": 1,\n \"avg\": 7.5,\n \"last\": 14,\n \"last-5-avg\": 12.0,\n \"last-10-avg\": 9.5\n },\n \"time_this_iter_s\": {\n \"max\": 522.4360599517822,\n \"min\": 505.6786599159241,\n \"avg\": 509.352954013007,\n \"last\": 506.25228786468506,\n \"last-5-avg\": 506.09470949172976,\n \"last-10-avg\": 507.6989426612854\n },\n \"time_total_s\": {\n \"max\": 7130.941356182098,\n \"min\": 519.6532871723175,\n \"avg\": 3834.4753139189315,\n \"last\": 7130.941356182098,\n \"last-5-avg\": 6118.492047595978,\n \"last-10-avg\": 4853.56500992775\n },\n \"time_since_restore\": {\n \"max\": 5076.989426612854,\n \"min\": 519.6532871723175,\n \"avg\": 2220.8551683255605,\n \"last\": 5076.989426612854,\n \"last-5-avg\": 4064.5401180267336,\n \"last-10-avg\": 2799.613080358505\n },\n \"iterations_since_restore\": {\n \"max\": 10,\n \"min\": 1,\n \"avg\": 4.357142857142857,\n \"last\": 10,\n \"last-5-avg\": 8.0,\n \"last-10-avg\": 5.5\n }\n },\n \"_n_steps\": [\n 5,\n 10\n ],\n \"metric_n_steps\": {\n \"loss\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"800595d6000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294288c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b0074946243089fcfe773fa859e4094869452946807680d43088fc7e3911a849e4094869452946807680d4308f8fbfd3e21919e4094869452946807680d430887c3e110268e9e4094869452946807680d4308994c2693ee919e409486945294652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059535010000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294288c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b00749462430841201028be7f9e4094869452946807680d43083a9d4e27ca839e4094869452946807680d4308bfdfefd79d8b9e4094869452946807680d43089bcd6613ce8b9e4094869452946807680d430883c160508c8f9e4094869452946807680d43089fcfe773fa859e4094869452946807680d43088fc7e3911a849e4094869452946807680d4308f8fbfd3e21919e4094869452946807680d430887c3e110268e9e4094869452946807680d4308994c2693ee919e409486945294652e\"\n }\n },\n \"should_checkpoint\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059527000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294288888888888652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952c000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452942888888888888888888888652e\"\n }\n },\n \"done\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059527000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294288989898989652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952c000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452942889898989898989898989652e\"\n }\n },\n \"training_iteration\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952c000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294284b0a4b0b4b0c4b0d4b0e652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059536000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294284b054b064b074b084b094b0a4b0b4b0c4b0d4b0e652e\"\n }\n },\n \"time_this_iter_s\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005954f000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452942847407f9d4d0480000047407f9bdb4600000047407fa734e200000047407fa32d1b00000047407fa4095f000000652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005957c000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a8694529428474080537d0d00000047407fa48b5a40000047407fa336da80000047407f9ea8f140000047407f9adbca80000047407f9d4d0480000047407f9bdb4600000047407fa734e200000047407fa32d1b00000047407fa4095f000000652e\"\n }\n },\n \"time_total_s\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005954f000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294284740b3f24c929800004740b5ec0a46f800004740b7e67d951800004740b9e0b066c800004740bbdaf0fcb80000652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005957c000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294284740a420c6a69000004740a8155811d800004740ac09beed2800004740affd940b5000004740b1f877c25000004740b3f24c929800004740b5ec0a46f800004740b7e67d951800004740b9e0b066c800004740bbdaf0fcb80000652e\"\n }\n },\n \"time_since_restore\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005954f000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294284740a7d8b1c1e000004740abcc2d2aa000004740afc113c6e000004740b1dabcb52000004740b3d4fd4b100000652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005957c000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a8694529428474080537d0d00000047409012e15d100000474097fbaf13b0000047409fe359500000004740a3e508215000004740a7d8b1c1e000004740abcc2d2aa000004740afc113c6e000004740b1dabcb52000004740b3d4fd4b100000652e\"\n }\n },\n \"iterations_since_restore\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952c000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294284b064b074b084b094b0a652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059536000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294284b014b024b034b044b054b064b074b084b094b0a652e\"\n }\n }\n },\n \"checkpoint_manager\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005954a340000000000008c267261792e747261696e2e5f696e7465726e616c2e636865636b706f696e745f6d616e61676572948c125f436865636b706f696e744d616e616765729493942981947d94288c125f636865636b706f696e745f636f6e666967948c097261792e747261696e948c10436865636b706f696e74436f6e6669679493942981947d94288c0b6e756d5f746f5f6b656570944e8c1a636865636b706f696e745f73636f72655f617474726962757465944e8c16636865636b706f696e745f73636f72655f6f72646572948c036d6178948c14636865636b706f696e745f6672657175656e6379944b008c11636865636b706f696e745f61745f656e6494898c1a5f636865636b706f696e745f6b6565705f616c6c5f72616e6b73948c0a44455052454341544544948c1f5f636865636b706f696e745f75706c6f61645f66726f6d5f776f726b65727394681275628c135f636865636b706f696e745f726573756c7473945d94288c1b7261792e747261696e2e5f696e7465726e616c2e73657373696f6e948c0f5f547261696e696e67526573756c749493942981947d94288c0a636865636b706f696e749468068c0a436865636b706f696e749493942981947d94288c0470617468948cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f30663963633937645f32325f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d3531322c6465636f6465725f72745f6e756d5f6c617965723d322c64726f705f725f323032342d30362d32355f31382d35332d30302f636865636b706f696e745f303030303030948c0a66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c055f75756964948c0475756964948c04555549449493942981947d948c03696e74948a11bc8b62e8c3edbcb2c8485d28b207aeb100736275628c076d657472696373947d94288c046c6f7373948c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b007494624308d268349a08909e4094869452948c0974696d657374616d70944a00f97a668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303030948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b018c08747269616c5f6964948c083066396363393764948c0464617465948c13323032342d30362d32355f31392d30362d3038948c1074696d655f746869735f697465725f73944740803d39eea000008c0c74696d655f746f74616c5f73944740803d39eea000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b048c146465636f6465725f72745f6e756d5f6c61796572944b028c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944d00048c0a656e636f6465725f6666944d00048c0d6465636f6465725f72745f6666944d00028c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b018c0964726f705f7261746594473f769053cb66dd808c026c7294473f5376df64dc8c7c8c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f7265944740803d39eea000008c18697465726174696f6e735f73696e63655f726573746f7265944b0175756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f30663963633937645f32325f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d3531322c6465636f6465725f72745f6e756d5f6c617965723d322c64726f705f725f323032342d30362d32355f31382d35332d30302f636865636b706f696e745f30303030303194682268257d946827897385945294682a682d2981947d9468308a105e13c0c82c9e3191cf421fa8c4e1fc307362756268317d94288c046c6f7373946836683c4308a9542a352ac99e4094869452948c0974696d657374616d70944afafa7a668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303031948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b028c08747269616c5f6964948c083066396363393764948c0464617465948c13323032342d30362d32355f31392d31342d3334948c1074696d655f746869735f697465725f739447407f9ed96ec000008c0c74696d655f746f74616c5f73944740900653530000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b048c146465636f6465725f72745f6e756d5f6c61796572944b028c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944d00048c0a656e636f6465725f6666944d00048c0d6465636f6465725f72745f6666944d00028c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b018c0964726f705f7261746594473f769053cb66dd808c026c7294473f5376df64dc8c7c8c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f7265944740900653530000008c18697465726174696f6e735f73696e63655f726573746f7265944b0275756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f30663963633937645f32325f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d3531322c6465636f6465725f72745f6e756d5f6c617965723d322c64726f705f725f323032342d30362d32355f31382d35332d30302f636865636b706f696e745f30303030303294682268257d946827897385945294682a682d2981947d9468308a1047a51823437053b9eb45d1bfde1b56417362756268317d94288c046c6f7373946836683c4308d4e974fab0899e4094869452948c0974696d657374616d70944afc087b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303032948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b038c08747269616c5f6964948c083066396363393764948c0464617465948c13323032342d30362d32355f32302d31342d3231948c1074696d655f746869735f697465725f739447408051e4044000008c0c74696d655f746f74616c5f73944740982f45552000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b048c146465636f6465725f72745f6e756d5f6c61796572944b028c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944d00048c0a656e636f6465725f6666944d00048c0d6465636f6465725f72745f6666944d00028c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b018c0964726f705f7261746594473f769053cb66dd808c026c7294473f5376df64dc8c7c8c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f72659447408051e4044000008c18697465726174696f6e735f73696e63655f726573746f7265944b0175756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f30663963633937645f32325f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d3531322c6465636f6465725f72745f6e756d5f6c617965723d322c64726f705f725f323032342d30362d32355f31382d35332d30302f636865636b706f696e745f30303030303394682268257d946827897385945294682a682d2981947d9468308a11fe5eef75c2c2fb9c0b4a0a747000a89f007362756268317d94288c046c6f7373946836683c4308d4e974dafac69e4094869452948c0974696d657374616d70944af60a7b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303033948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b048c08747269616c5f6964948c083066396363393764948c0464617465948c13323032342d30362d32355f32302d32322d3437948c1074696d655f746869735f697465725f739447407fa225c60000008c0c74696d655f746f74616c5f73944740a00be7635000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b048c146465636f6465725f72745f6e756d5f6c61796572944b028c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944d00048c0a656e636f6465725f6666944d00048c0d6465636f6465725f72745f6666944d00028c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b018c0964726f705f7261746594473f769053cb66dd808c026c7294473f5376df64dc8c7c8c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f726594474090117b73a000008c18697465726174696f6e735f73696e63655f726573746f7265944b0275756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f30663963633937645f32325f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d3531322c6465636f6465725f72745f6e756d5f6c617965723d322c64726f705f725f323032342d30362d32355f31382d35332d30302f636865636b706f696e745f30303030303494682268257d946827897385945294682a682d2981947d9468308a10085c6792f9ded8bf46472e09e7339a767362756268317d94288c046c6f7373946836683c430841201028be7f9e4094869452948c0974696d657374616d70944a88107b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303034948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b058c08747269616c5f6964948c083066396363393764948c0464617465948c13323032342d30362d32355f32302d34362d3333948c1074696d655f746869735f697465725f7394474080537d0d0000008c0c74696d655f746f74616c5f73944740a420c6a69000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b048c146465636f6465725f72745f6e756d5f6c61796572944b028c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944d00048c0a656e636f6465725f6666944d00048c0d6465636f6465725f72745f6666944d00028c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b018c0964726f705f7261746594473f769053cb66dd808c026c7294473f5376df64dc8c7c8c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f726594474080537d0d0000008c18697465726174696f6e735f73696e63655f726573746f7265944b0175756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f30663963633937645f32325f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d3531322c6465636f6465725f72745f6e756d5f6c617965723d322c64726f705f725f323032342d30362d32355f31382d35332d30302f636865636b706f696e745f30303030303594682268257d946827897385945294682a682d2981947d9468308a1068d58297224d9a964f43f4e1c1d90f797362756268317d94288c046c6f7373946836683c43083a9d4e27ca839e4094869452948c0974696d657374616d70944a83127b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303035948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b068c08747269616c5f6964948c083066396363393764948c0464617465948c13323032342d30362d32355f32302d35342d3539948c1074696d655f746869735f697465725f739447407fa48b5a4000008c0c74696d655f746f74616c5f73944740a8155811d800008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b048c146465636f6465725f72745f6e756d5f6c61796572944b028c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944d00048c0a656e636f6465725f6666944d00048c0d6465636f6465725f72745f6666944d00028c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b018c0964726f705f7261746594473f769053cb66dd808c026c7294473f5376df64dc8c7c8c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f72659447409012e15d1000008c18697465726174696f6e735f73696e63655f726573746f7265944b0275756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f30663963633937645f32325f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d3531322c6465636f6465725f72745f6e756d5f6c617965723d322c64726f705f725f323032342d30362d32355f31382d35332d30302f636865636b706f696e745f30303030303694682268257d946827897385945294682a682d2981947d9468308a10d78dd9d3647e568eb845a94a734b6b4b7362756268317d94288c046c6f7373946836683c4308bfdfefd79d8b9e4094869452948c0974696d657374616d70944a7d147b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303036948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b078c08747269616c5f6964948c083066396363393764948c0464617465948c13323032342d30362d32355f32312d30332d3235948c1074696d655f746869735f697465725f739447407fa336da8000008c0c74696d655f746f74616c5f73944740ac09beed2800008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b048c146465636f6465725f72745f6e756d5f6c61796572944b028c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944d00048c0a656e636f6465725f6666944d00048c0d6465636f6465725f72745f6666944d00028c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b018c0964726f705f7261746594473f769053cb66dd808c026c7294473f5376df64dc8c7c8c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f726594474097fbaf13b000008c18697465726174696f6e735f73696e63655f726573746f7265944b0375756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f30663963633937645f32325f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d3531322c6465636f6465725f72745f6e756d5f6c617965723d322c64726f705f725f323032342d30362d32355f31382d35332d30302f636865636b706f696e745f30303030303794682268257d946827897385945294682a682d2981947d9468308a11e00c723a18786c94ee4853bb68c682c6007362756268317d94288c046c6f7373946836683c43089bcd6613ce8b9e4094869452948c0974696d657374616d70944a77167b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303037948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b088c08747269616c5f6964948c083066396363393764948c0464617465948c13323032342d30362d32355f32312d31312d3531948c1074696d655f746869735f697465725f739447407f9ea8f14000008c0c74696d655f746f74616c5f73944740affd940b5000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b048c146465636f6465725f72745f6e756d5f6c61796572944b028c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944d00048c0a656e636f6465725f6666944d00048c0d6465636f6465725f72745f6666944d00028c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b018c0964726f705f7261746594473f769053cb66dd808c026c7294473f5376df64dc8c7c8c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f72659447409fe359500000008c18697465726174696f6e735f73696e63655f726573746f7265944b0475756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f30663963633937645f32325f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d3531322c6465636f6465725f72745f6e756d5f6c617965723d322c64726f705f725f323032342d30362d32355f31382d35332d30302f636865636b706f696e745f30303030303894682268257d946827897385945294682a682d2981947d9468308a114c9987b028f3149ab84de430cd82a0aa007362756268317d94288c046c6f7373946836683c430883c160508c8f9e4094869452948c0974696d657374616d70944a71187b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303038948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b098c08747269616c5f6964948c083066396363393764948c0464617465948c13323032342d30362d32355f32312d32302d3137948c1074696d655f746869735f697465725f739447407f9adbca8000008c0c74696d655f746f74616c5f73944740b1f877c25000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b048c146465636f6465725f72745f6e756d5f6c61796572944b028c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944d00048c0a656e636f6465725f6666944d00048c0d6465636f6465725f72745f6666944d00028c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b018c0964726f705f7261746594473f769053cb66dd808c026c7294473f5376df64dc8c7c8c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f7265944740a3e508215000008c18697465726174696f6e735f73696e63655f726573746f7265944b0575756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f30663963633937645f32325f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d3531322c6465636f6465725f72745f6e756d5f6c617965723d322c64726f705f725f323032342d30362d32355f31382d35332d30302f636865636b706f696e745f30303030303994682268257d946827897385945294682a682d2981947d9468308a100e8a22064078bc9d22409ef1e05347527362756268317d94288c046c6f7373946836683c43089fcfe773fa859e4094869452948c0974696d657374616d70944a6a1a7b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303039948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b0a8c08747269616c5f6964948c083066396363393764948c0464617465948c13323032342d30362d32355f32312d32382d3433948c1074696d655f746869735f697465725f739447407f9d4d048000008c0c74696d655f746f74616c5f73944740b3f24c929800008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b048c146465636f6465725f72745f6e756d5f6c61796572944b028c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944d00048c0a656e636f6465725f6666944d00048c0d6465636f6465725f72745f6666944d00028c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b018c0964726f705f7261746594473f769053cb66dd808c026c7294473f5376df64dc8c7c8c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f7265944740a7d8b1c1e000008c18697465726174696f6e735f73696e63655f726573746f7265944b0675756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f30663963633937645f32325f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d3531322c6465636f6465725f72745f6e756d5f6c617965723d322c64726f705f725f323032342d30362d32355f31382d35332d30302f636865636b706f696e745f30303030313094682268257d946827897385945294682a682d2981947d9468308a11f07707441e5ecb8e5043ef37cdf3579f007362756268317d94288c046c6f7373946836683c43088fc7e3911a849e4094869452948c0974696d657374616d70944a641c7b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303130948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b0b8c08747269616c5f6964948c083066396363393764948c0464617465948c13323032342d30362d32355f32312d33372d3039948c1074696d655f746869735f697465725f739447407f9bdb460000008c0c74696d655f746f74616c5f73944740b5ec0a46f800008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b048c146465636f6465725f72745f6e756d5f6c61796572944b028c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944d00048c0a656e636f6465725f6666944d00048c0d6465636f6465725f72745f6666944d00028c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b018c0964726f705f7261746594473f769053cb66dd808c026c7294473f5376df64dc8c7c8c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f7265944740abcc2d2aa000008c18697465726174696f6e735f73696e63655f726573746f7265944b0775756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f30663963633937645f32325f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d3531322c6465636f6465725f72745f6e756d5f6c617965723d322c64726f705f725f323032342d30362d32355f31382d35332d30302f636865636b706f696e745f30303030313194682268257d946827897385945294682a682d2981947d9468308a1095be09bc9c1548877342e6ddc1583d0b7362756268317d94288c046c6f7373946836683c4308f8fbfd3e21919e4094869452948c0974696d657374616d70944a5f1e7b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303131948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b0c8c08747269616c5f6964948c083066396363393764948c0464617465948c13323032342d30362d32355f32312d34352d3335948c1074696d655f746869735f697465725f739447407fa734e20000008c0c74696d655f746f74616c5f73944740b7e67d951800008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b048c146465636f6465725f72745f6e756d5f6c61796572944b028c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944d00048c0a656e636f6465725f6666944d00048c0d6465636f6465725f72745f6666944d00028c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b018c0964726f705f7261746594473f769053cb66dd808c026c7294473f5376df64dc8c7c8c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f7265944740afc113c6e000008c18697465726174696f6e735f73696e63655f726573746f7265944b0875756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f30663963633937645f32325f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d3531322c6465636f6465725f72745f6e756d5f6c617965723d322c64726f705f725f323032342d30362d32355f31382d35332d30302f636865636b706f696e745f30303030313294682268257d946827897385945294682a682d2981947d9468308a10c47a732ff00386986d4175a68c1701057362756268317d94288c046c6f7373946836683c430887c3e110268e9e4094869452948c0974696d657374616d70944a59207b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303132948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b0d8c08747269616c5f6964948c083066396363393764948c0464617465948c13323032342d30362d32355f32312d35342d3031948c1074696d655f746869735f697465725f739447407fa32d1b0000008c0c74696d655f746f74616c5f73944740b9e0b066c800008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b048c146465636f6465725f72745f6e756d5f6c61796572944b028c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944d00048c0a656e636f6465725f6666944d00048c0d6465636f6465725f72745f6666944d00028c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b018c0964726f705f7261746594473f769053cb66dd808c026c7294473f5376df64dc8c7c8c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f7265944740b1dabcb52000008c18697465726174696f6e735f73696e63655f726573746f7265944b0975756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f30663963633937645f32325f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d3531322c6465636f6465725f72745f6e756d5f6c617965723d322c64726f705f725f323032342d30362d32355f31382d35332d30302f636865636b706f696e745f30303030313394682268257d946827897385945294682a682d2981947d9468308a10638046ba80165599f44a08949e6b85357362756268317d94288c046c6f7373946836683c4308994c2693ee919e4094869452948c0974696d657374616d70944a53227b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303133948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b0e8c08747269616c5f6964948c083066396363393764948c0464617465948c13323032342d30362d32355f32322d30322d3237948c1074696d655f746869735f697465725f739447407fa4095f0000008c0c74696d655f746f74616c5f73944740bbdaf0fcb800008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b048c146465636f6465725f72745f6e756d5f6c61796572944b028c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944d00048c0a656e636f6465725f6666944d00048c0d6465636f6465725f72745f6666944d00028c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b018c0964726f705f7261746594473f769053cb66dd808c026c7294473f5376df64dc8c7c8c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f7265944740b3d4fd4b1000008c18697465726174696f6e735f73696e63655f726573746f7265944b0a757562658c195f6c61746573745f636865636b706f696e745f726573756c74946a9602000075622e\"\n }\n}"
+ ],
+ [
+ "{\n \"stub\": false,\n \"trainable_name\": \"train_model\",\n \"trial_id\": \"5eb1e282\",\n \"storage\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059572040000000000008c1b7261792e747261696e2e5f696e7465726e616c2e73746f72616765948c0e53746f72616765436f6e746578749493942981947d94288c12637573746f6d5f66735f70726f766964656494898c1273746f726167655f6c6f63616c5f70617468948c2b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c7473948c136578706572696d656e745f6469725f6e616d65948c0f746573745f6578706572696d656e74948c0e747269616c5f6469725f6e616d65948c96747261696e5f6d6f64656c5f35656231653238325f32335f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d3531322c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f725f323032342d30362d32355f31382d35372d3239948c1863757272656e745f636865636b706f696e745f696e646578944b118c0b73796e635f636f6e666967948c097261792e747261696e948c0a53796e63436f6e6669679493942981947d94288c0b73796e635f706572696f64944d2c018c0c73796e635f74696d656f7574944d08078c0e73796e635f61727469666163747394898c1c73796e635f6172746966616374735f6f6e5f636865636b706f696e7494888c0a75706c6f61645f646972948c0a44455052454341544544948c0673796e6365729468188c1273796e635f6f6e5f636865636b706f696e7494681875628c1273746f726167655f66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c0f73746f726167655f66735f70617468948c452f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f7465737494681968008c115f46696c6573797374656d53796e6365729493942981947d9428681b682268134d2c0168144d08078c116c6173745f73796e635f75705f74696d65944741d99ebda2f220858c136c6173745f73796e635f646f776e5f74696d659447fff00000000000008c0d5f73796e635f70726f63657373944e8c0c5f63757272656e745f636d649468008c125f75706c6f61645f746f5f66735f706174689493947d94288c0a6c6f63616c5f70617468948c3b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c74732f746573745f6578706572696d656e74948c0266739468228c0766735f70617468948c552f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e74948c076578636c756465945d948c0e2a2f636865636b706f696e745f2a9461758694756275622e\"\n },\n \"config\": {\n \"encoder_num_layer\": 1,\n \"decoder_rt_num_layer\": 1,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 16,\n \"encoder_ff\": 1024,\n \"decoder_rt_ff\": 512,\n \"decoder_int_ff\": 512,\n \"n_head\": 4,\n \"drop_rate\": 0.2842529108982538,\n \"lr\": 0.00015364900402465778,\n \"batch_size\": 2048\n },\n \"_Trial__unresolved_config\": {\n \"encoder_num_layer\": 1,\n \"decoder_rt_num_layer\": 1,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 16,\n \"encoder_ff\": 1024,\n \"decoder_rt_ff\": 512,\n \"decoder_int_ff\": 512,\n \"n_head\": 4,\n \"drop_rate\": 0.2842529108982538,\n \"lr\": 0.00015364900402465778,\n \"batch_size\": 2048\n },\n \"evaluated_params\": {\n \"batch_size\": 2048,\n \"decoder_int_ff\": 512,\n \"decoder_int_num_layer\": 1,\n \"decoder_rt_ff\": 512,\n \"decoder_rt_num_layer\": 1,\n \"drop_rate\": 0.2842529108982538,\n \"embedding_dim\": 16,\n \"encoder_ff\": 1024,\n \"encoder_num_layer\": 1,\n \"lr\": 0.00015364900402465778,\n \"n_head\": 4\n },\n \"experiment_tag\": \"23_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=1,drop_rate=0.2843,embedding_dim=16,encoder_ff=1024,encoder_num_layer=1,lr=0.0002,n_head=4\",\n \"stopping_criterion\": {},\n \"_setup_default_resource\": true,\n \"_default_placement_group_factory\": \"80054e2e\",\n \"placement_group_factory\": \"800595ba000000000000008c237261792e74756e652e657865637574696f6e2e706c6163656d656e745f67726f757073948c15506c6163656d656e7447726f7570466163746f72799493942981947d94288c085f62756e646c6573945d947d94288c03435055944740540000000000008c034750559447401000000000000075618c155f686561645f62756e646c655f69735f656d70747994898c095f7374726174656779948c045041434b948c055f6172677394298c075f6b7761726773947d9475622e\",\n \"log_to_file\": [\n null,\n null\n ],\n \"max_failures\": 0,\n \"_default_result_or_future\": null,\n \"export_formats\": [],\n \"status\": \"TERMINATED\",\n \"relative_logdir\": \"train_model_5eb1e282_23_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=1,drop_r_2024-06-25_18-57-29\",\n \"trial_name_creator\": null,\n \"trial_dirname_creator\": null,\n \"custom_trial_name\": null,\n \"custom_dirname\": null,\n \"restore_path\": null,\n \"_restore_checkpoint_result\": null,\n \"_state_json\": null,\n \"results\": \"80054e2e\",\n \"extra_arg\": \"80054e2e\",\n \"_resources\": \"80054e2e\"\n}",
+ "{\n \"start_time\": 1719335674.8936594,\n \"num_failures\": 0,\n \"num_failures_after_restore\": 0,\n \"error_filename\": null,\n \"pickled_error_filename\": null,\n \"last_result\": {\n \"loss\": 183.96574522003414,\n \"timestamp\": 1719346234,\n \"checkpoint_dir_name\": \"checkpoint_000017\",\n \"should_checkpoint\": true,\n \"done\": true,\n \"training_iteration\": 18,\n \"trial_id\": \"5eb1e282\",\n \"date\": \"2024-06-25_22-10-34\",\n \"time_this_iter_s\": 47.40534067153931,\n \"time_total_s\": 894.9221906661987,\n \"pid\": 2195253,\n \"hostname\": \"r8i6n2\",\n \"node_ip\": \"10.159.28.60\",\n \"config\": {\n \"encoder_num_layer\": 1,\n \"decoder_rt_num_layer\": 1,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 16,\n \"encoder_ff\": 1024,\n \"decoder_rt_ff\": 512,\n \"decoder_int_ff\": 512,\n \"n_head\": 4,\n \"drop_rate\": 0.2842529108982538,\n \"lr\": 0.00015364900402465778,\n \"batch_size\": 2048\n },\n \"time_since_restore\": 486.2922348976135,\n \"iterations_since_restore\": 10,\n \"experiment_tag\": \"23_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=1,drop_rate=0.2843,embedding_dim=16,encoder_ff=1024,encoder_num_layer=1,lr=0.0002,n_head=4\"\n },\n \"last_result_time\": 1719346234.4968047,\n \"metric_analysis\": {\n \"loss\": {\n \"max\": 1030.0586158572219,\n \"min\": 183.96574522003414,\n \"avg\": 322.085730137996,\n \"last\": 183.96574522003414,\n \"last-5-avg\": 195.48832729069267,\n \"last-10-avg\": 213.65420443317083\n },\n \"should_checkpoint\": {\n \"max\": true,\n \"min\": true,\n \"avg\": 1.0,\n \"last\": true,\n \"last-5-avg\": 1.0,\n \"last-10-avg\": 1.0\n },\n \"done\": {\n \"max\": false,\n \"min\": false,\n \"avg\": 0.0,\n \"last\": false,\n \"last-5-avg\": 0.0,\n \"last-10-avg\": 0.0\n },\n \"training_iteration\": {\n \"max\": 18,\n \"min\": 1,\n \"avg\": 9.5,\n \"last\": 18,\n \"last-5-avg\": 16.0,\n \"last-10-avg\": 13.5\n },\n \"time_this_iter_s\": {\n \"max\": 61.87126088142395,\n \"min\": 46.741618633270264,\n \"avg\": 49.71789948145548,\n \"last\": 47.40534067153931,\n \"last-5-avg\": 47.24765267372131,\n \"last-10-avg\": 48.62922348976135\n },\n \"time_total_s\": {\n \"max\": 894.9221906661987,\n \"min\": 60.84843850135803,\n \"avg\": 485.0399955908457,\n \"last\": 894.9221906661987,\n \"last-5-avg\": 800.4915899276733,\n \"last-10-avg\": 682.4233734607697\n },\n \"time_since_restore\": {\n \"max\": 486.2922348976135,\n \"min\": 60.84843850135803,\n \"avg\": 221.83922640482584,\n \"last\": 486.2922348976135,\n \"last-5-avg\": 391.8616341590881,\n \"last-10-avg\": 273.79341769218445\n },\n \"iterations_since_restore\": {\n \"max\": 10,\n \"min\": 1,\n \"avg\": 4.388888888888888,\n \"last\": 10,\n \"last-5-avg\": 8.0,\n \"last-10-avg\": 5.5\n }\n },\n \"_n_steps\": [\n 5,\n 10\n ],\n \"metric_n_steps\": {\n \"loss\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"800595d6000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294288c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b007494624308180c06c30e90694094869452946807680d4308e4f178bcd9c1684094869452946807680d4308bfdfef77d742684094869452946807680d430847a3d1887a9a684094869452946807680d4308140a8562e7fe66409486945294652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059535010000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294288c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b007494624308369bcd96da0e704094869452946807680d4308241289a404126e4094869452946807680d430845229168ebcd6c4094869452946807680d43085128142a096d6b4094869452946807680d43082e97cb2588786a4094869452946807680d4308180c06c30e90694094869452946807680d4308e4f178bcd9c1684094869452946807680d4308bfdfef77d742684094869452946807680d430847a3d1887a9a684094869452946807680d4308140a8562e7fe66409486945294652e\"\n }\n },\n \"should_checkpoint\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059527000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294288888888888652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952c000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452942888888888888888888888652e\"\n }\n },\n \"done\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059527000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294288989898989652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952c000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452942889898989898989898989652e\"\n }\n },\n \"training_iteration\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952c000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294284b0e4b0f4b104b114b12652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059536000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294284b094b0a4b0b4b0c4b0d4b0e4b0f4b104b114b12652e\"\n }\n },\n \"time_this_iter_s\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005954f000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b058694529428474047e31db20000004740475eed5c00000047404794970400000047404793fb24000000474047b3e234000000652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005957c000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452942847404eb3e57200000047404792ce3800000047404797ce0e000000474047919be200000047404796caf0000000474047e31db20000004740475eed5c00000047404794970400000047404793fb24000000474047b3e234000000652e\"\n }\n },\n \"time_total_s\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005954f000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452942847408613aa8a00000047408789995fc0000047408902e2d000000047408a7c228240000047408bf760a5800000652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005957c000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452942847407d6090fac00000474080297560e00000474081a2f241c000004740831c0bffe000004740849578aee0000047408613aa8a00000047408789995fc0000047408902e2d000000047408a7c228240000047408bf760a5800000652e\"\n }\n },\n \"time_since_restore\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005954f000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294284740729d40c7800000474075891e730000004740787bb15380000047407b6e30b800000047407e64acfe800000652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005957c000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452942847404eb3e57200000047405b2359d500000047406377a06e0000004740695c076680000047406f41ba228000004740729d40c7800000474075891e730000004740787bb15380000047407b6e30b800000047407e64acfe800000652e\"\n }\n },\n \"iterations_since_restore\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952c000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294284b064b074b084b094b0a652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059536000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294284b014b024b034b044b054b064b074b084b094b0a652e\"\n }\n }\n },\n \"checkpoint_manager\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059572420000000000008c267261792e747261696e2e5f696e7465726e616c2e636865636b706f696e745f6d616e61676572948c125f436865636b706f696e744d616e616765729493942981947d94288c125f636865636b706f696e745f636f6e666967948c097261792e747261696e948c10436865636b706f696e74436f6e6669679493942981947d94288c0b6e756d5f746f5f6b656570944e8c1a636865636b706f696e745f73636f72655f617474726962757465944e8c16636865636b706f696e745f73636f72655f6f72646572948c036d6178948c14636865636b706f696e745f6672657175656e6379944b008c11636865636b706f696e745f61745f656e6494898c1a5f636865636b706f696e745f6b6565705f616c6c5f72616e6b73948c0a44455052454341544544948c1f5f636865636b706f696e745f75706c6f61645f66726f6d5f776f726b65727394681275628c135f636865636b706f696e745f726573756c7473945d94288c1b7261792e747261696e2e5f696e7465726e616c2e73657373696f6e948c0f5f547261696e696e67526573756c749493942981947d94288c0a636865636b706f696e749468068c0a436865636b706f696e749493942981947d94288c0470617468948cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f35656231653238325f32335f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d3531322c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f725f323032342d30362d32355f31382d35372d32392f636865636b706f696e745f303030303030948c0a66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c055f75756964948c0475756964948c04555549449493942981947d948c03696e74948a1195d20293345c8a99de4f658c432f75ee00736275628c076d657472696373947d94288c046c6f7373948c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b0074946243082e97cb053c18904094869452948c0974696d657374616d70944a37fb7a668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303030948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b018c08747269616c5f6964948c083565623165323832948c0464617465948c13323032342d30362d32355f31392d31352d3335948c1074696d655f746869735f697465725f739447404e6c99a20000008c0c74696d655f746f74616c5f739447404e6c99a20000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b018c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b108c0a656e636f6465725f6666944d00048c0d6465636f6465725f72745f6666944d00028c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b048c0964726f705f7261746594473fd231331f06738a8c026c7294473f24239ae50ae97e8c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f72659447404e6c99a20000008c18697465726174696f6e735f73696e63655f726573746f7265944b0175756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f35656231653238325f32335f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d3531322c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f725f323032342d30362d32355f31382d35372d32392f636865636b706f696e745f30303030303194682268257d946827897385945294682a682d2981947d9468308a1178af4dae568327b5074207261e5053d2007362756268317d94288c046c6f7373946836683c4308201008043b90814094869452948c0974696d657374616d70944a67fb7a668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303031948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b028c08747269616c5f6964948c083565623165323832948c0464617465948c13323032342d30362d32355f31392d31362d3233948c1074696d655f746869735f697465725f7394474047da1acc0000008c0c74696d655f746f74616c5f739447405b235a370000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b018c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b108c0a656e636f6465725f6666944d00048c0d6465636f6465725f72745f6666944d00028c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b048c0964726f705f7261746594473fd231331f06738a8c026c7294473f24239ae50ae97e8c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f72659447405b235a370000008c18697465726174696f6e735f73696e63655f726573746f7265944b0275756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f35656231653238325f32335f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d3531322c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f725f323032342d30362d32355f31382d35372d32392f636865636b706f696e745f30303030303294682268257d946827897385945294682a682d2981947d9468308a1188f94f4311288da1e14082c0ef818cb8007362756268317d94288c046c6f7373946836683c43084d2693a981ce7b4094869452948c0974696d657374616d70944a350b7b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303032948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b038c08747269616c5f6964948c083565623165323832948c0464617465948c13323032342d30362d32355f32302d32332d3439948c1074696d655f746869735f697465725f739447404eef857a0000008c0c74696d655f746f74616c5f73944740654d8e7a0000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b018c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b108c0a656e636f6465725f6666944d00048c0d6465636f6465725f72745f6666944d00028c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b048c0964726f705f7261746594473fd231331f06738a8c026c7294473f24239ae50ae97e8c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f72659447404eef857a0000008c18697465726174696f6e735f73696e63655f726573746f7265944b0175756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f35656231653238325f32335f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d3531322c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f725f323032342d30362d32355f31382d35372d32392f636865636b706f696e745f30303030303394682268257d946827897385945294682a682d2981947d9468308a102651890a9afad186f34145110bb9e22f7362756268317d94288c046c6f7373946836683c43085bad568b4f1c784094869452948c0974696d657374616d70944a640b7b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303033948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b048c08747269616c5f6964948c083565623165323832948c0464617465948c13323032342d30362d32355f32302d32342d3336948c1074696d655f746869735f697465725f7394474047b3f4240000008c0c74696d655f746f74616c5f739447406b3a8b830000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b018c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b108c0a656e636f6465725f6666944d00048c0d6465636f6465725f72745f6666944d00028c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b048c0964726f705f7261746594473fd231331f06738a8c026c7294473f24239ae50ae97e8c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f72659447405b51bccf0000008c18697465726174696f6e735f73696e63655f726573746f7265944b0275756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f35656231653238325f32335f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d3531322c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f725f323032342d30362d32355f31382d35372d32392f636865636b706f696e745f30303030303494682268257d946827897385945294682a682d2981947d9468308a11d26c790b9ccacf9ac248298dcb6f6bee007362756268317d94288c046c6f7373946836683c4308140a85a2cef1754094869452948c0974696d657374616d70944a940b7b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303034948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b058c08747269616c5f6964948c083565623165323832948c0464617465948c13323032342d30362d32355f32302d32352d3234948c1074696d655f746869735f697465725f7394474047fd1bc00000008c0c74696d655f746f74616c5f73944740709ce9398000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b018c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b108c0a656e636f6465725f6666944d00048c0d6465636f6465725f72745f6666944d00028c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b048c0964726f705f7261746594473fd231331f06738a8c026c7294473f24239ae50ae97e8c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f726594474063a825578000008c18697465726174696f6e735f73696e63655f726573746f7265944b0375756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f35656231653238325f32335f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d3531322c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f725f323032342d30362d32355f31382d35372d32392f636865636b706f696e745f30303030303594682268257d946827897385945294682a682d2981947d9468308a1055e673ebcf0b81bede4bfa3c05941c5e7362756268317d94288c046c6f7373946836683c4308f67abd3e7007744094869452948c0974696d657374616d70944ac40b7b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303035948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b068c08747269616c5f6964948c083565623165323832948c0464617465948c13323032342d30362d32355f32302d32362d3132948c1074696d655f746869735f697465725f7394474047b75a200000008c0c74696d655f746f74616c5f739447407393d47d8000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b018c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b108c0a656e636f6465725f6666944d00048c0d6465636f6465725f72745f6666944d00028c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b048c0964726f705f7261746594473fd231331f06738a8c026c7294473f24239ae50ae97e8c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f72659447406995fbdf8000008c18697465726174696f6e735f73696e63655f726573746f7265944b0475756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f35656231653238325f32335f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d3531322c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f725f323032342d30362d32355f31382d35372d32392f636865636b706f696e745f30303030303694682268257d946827897385945294682a682d2981947d9468308a108c30c8a2a1ec26aa4d45bb6fa954dc1d7362756268317d94288c046c6f7373946836683c43086fb7dbfd5d57724094869452948c0974696d657374616d70944af30b7b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303036948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b078c08747269616c5f6964948c083565623165323832948c0464617465948c13323032342d30362d32355f32302d32362d3539948c1074696d655f746869735f697465725f7394474047f5514c0000008c0c74696d655f746f74616c5f7394474076927ea70000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b018c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b108c0a656e636f6465725f6666944d00048c0d6465636f6465725f72745f6666944d00028c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b048c0964726f705f7261746594473fd231331f06738a8c026c7294473f24239ae50ae97e8c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f72659447406f9350328000008c18697465726174696f6e735f73696e63655f726573746f7265944b0575756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f35656231653238325f32335f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d3531322c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f725f323032342d30362d32355f31382d35372d32392f636865636b706f696e745f30303030303794682268257d946827897385945294682a682d2981947d9468308a1047ecf3292ecafbaf8043f4325d7111457362756268317d94288c046c6f7373946836683c4308e270384c3013714094869452948c0974696d657374616d70944a230c7b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303037948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b088c08747269616c5f6964948c083565623165323832948c0464617465948c13323032342d30362d32355f32302d32372d3437948c1074696d655f746869735f697465725f7394474047bcad2c0000008c0c74696d655f746f74616c5f73944740798a144c8000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b018c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b108c0a656e636f6465725f6666944d00048c0d6465636f6465725f72745f6666944d00028c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b048c0964726f705f7261746594473fd231331f06738a8c026c7294473f24239ae50ae97e8c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f726594474072c13dbec000008c18697465726174696f6e735f73696e63655f726573746f7265944b0675756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f35656231653238325f32335f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d3531322c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f725f323032342d30362d32355f31382d35372d32392f636865636b706f696e745f30303030303894682268257d946827897385945294682a682d2981947d9468308a1076123843a8ac16adcf4931a9534560767362756268317d94288c046c6f7373946836683c4308369bcd96da0e704094869452948c0974696d657374616d70944a91227b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303038948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b098c08747269616c5f6964948c083565623165323832948c0464617465948c13323032342d30362d32355f32322d30332d3239948c1074696d655f746869735f697465725f739447404eb3e5720000008c0c74696d655f746f74616c5f739447407d6090fac000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b018c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b108c0a656e636f6465725f6666944d00048c0d6465636f6465725f72745f6666944d00028c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b048c0964726f705f7261746594473fd231331f06738a8c026c7294473f24239ae50ae97e8c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f72659447404eb3e5720000008c18697465726174696f6e735f73696e63655f726573746f7265944b0175756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f35656231653238325f32335f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d3531322c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f725f323032342d30362d32355f31382d35372d32392f636865636b706f696e745f30303030303994682268257d946827897385945294682a682d2981947d9468308a11c8b346056ef247af62461e359ba9e3ed007362756268317d94288c046c6f7373946836683c4308241289a404126e4094869452948c0974696d657374616d70944ac0227b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303039948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b0a8c08747269616c5f6964948c083565623165323832948c0464617465948c13323032342d30362d32355f32322d30342d3136948c1074696d655f746869735f697465725f739447404792ce380000008c0c74696d655f746f74616c5f7394474080297560e000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b018c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b108c0a656e636f6465725f6666944d00048c0d6465636f6465725f72745f6666944d00028c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b048c0964726f705f7261746594473fd231331f06738a8c026c7294473f24239ae50ae97e8c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f72659447405b2359d50000008c18697465726174696f6e735f73696e63655f726573746f7265944b0275756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f35656231653238325f32335f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d3531322c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f725f323032342d30362d32355f31382d35372d32392f636865636b706f696e745f30303030313094682268257d946827897385945294682a682d2981947d9468308a114bae079ae08a7393eb42144a99fde4e9007362756268317d94288c046c6f7373946836683c430845229168ebcd6c4094869452948c0974696d657374616d70944aef227b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303130948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b0b8c08747269616c5f6964948c083565623165323832948c0464617465948c13323032342d30362d32355f32322d30352d3033948c1074696d655f746869735f697465725f739447404797ce0e0000008c0c74696d655f746f74616c5f7394474081a2f241c000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b018c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b108c0a656e636f6465725f6666944d00048c0d6465636f6465725f72745f6666944d00028c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b048c0964726f705f7261746594473fd231331f06738a8c026c7294473f24239ae50ae97e8c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f72659447406377a06e0000008c18697465726174696f6e735f73696e63655f726573746f7265944b0375756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f35656231653238325f32335f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d3531322c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f725f323032342d30362d32355f31382d35372d32392f636865636b706f696e745f30303030313194682268257d946827897385945294682a682d2981947d9468308a11f00e26ed3e9b8eaf074cdf1ffa7501dd007362756268317d94288c046c6f7373946836683c43085128142a096d6b4094869452948c0974696d657374616d70944a1f237b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303131948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b0c8c08747269616c5f6964948c083565623165323832948c0464617465948c13323032342d30362d32355f32322d30352d3531948c1074696d655f746869735f697465725f7394474047919be20000008c0c74696d655f746f74616c5f73944740831c0bffe000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b018c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b108c0a656e636f6465725f6666944d00048c0d6465636f6465725f72745f6666944d00028c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b048c0964726f705f7261746594473fd231331f06738a8c026c7294473f24239ae50ae97e8c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f7265944740695c07668000008c18697465726174696f6e735f73696e63655f726573746f7265944b0475756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f35656231653238325f32335f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d3531322c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f725f323032342d30362d32355f31382d35372d32392f636865636b706f696e745f30303030313294682268257d946827897385945294682a682d2981947d9468308a11fe873f03ff0f78ad294ca6bd93705bf8007362756268317d94288c046c6f7373946836683c43082e97cb2588786a4094869452948c0974696d657374616d70944a4e237b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303132948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b0d8c08747269616c5f6964948c083565623165323832948c0464617465948c13323032342d30362d32355f32322d30362d3338948c1074696d655f746869735f697465725f739447404796caf00000008c0c74696d655f746f74616c5f73944740849578aee000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b018c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b108c0a656e636f6465725f6666944d00048c0d6465636f6465725f72745f6666944d00028c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b048c0964726f705f7261746594473fd231331f06738a8c026c7294473f24239ae50ae97e8c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f72659447406f41ba228000008c18697465726174696f6e735f73696e63655f726573746f7265944b0575756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f35656231653238325f32335f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d3531322c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f725f323032342d30362d32355f31382d35372d32392f636865636b706f696e745f30303030313394682268257d946827897385945294682a682d2981947d9468308a1124e339c8fc2638a8374fbe2521a165f2007362756268317d94288c046c6f7373946836683c4308180c06c30e90694094869452948c0974696d657374616d70944a7e237b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303133948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b0e8c08747269616c5f6964948c083565623165323832948c0464617465948c13323032342d30362d32355f32322d30372d3236948c1074696d655f746869735f697465725f7394474047e31db20000008c0c74696d655f746f74616c5f739447408613aa8a0000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b018c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b108c0a656e636f6465725f6666944d00048c0d6465636f6465725f72745f6666944d00028c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b048c0964726f705f7261746594473fd231331f06738a8c026c7294473f24239ae50ae97e8c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f7265944740729d40c78000008c18697465726174696f6e735f73696e63655f726573746f7265944b0675756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f35656231653238325f32335f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d3531322c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f725f323032342d30362d32355f31382d35372d32392f636865636b706f696e745f30303030313494682268257d946827897385945294682a682d2981947d9468308a113d2d0972ce2ffa8ef048bd410c626484007362756268317d94288c046c6f7373946836683c4308e4f178bcd9c1684094869452948c0974696d657374616d70944aac237b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303134948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b0f8c08747269616c5f6964948c083565623165323832948c0464617465948c13323032342d30362d32355f32322d30382d3132948c1074696d655f746869735f697465725f73944740475eed5c0000008c0c74696d655f746f74616c5f739447408789995fc000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b018c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b108c0a656e636f6465725f6666944d00048c0d6465636f6465725f72745f6666944d00028c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b048c0964726f705f7261746594473fd231331f06738a8c026c7294473f24239ae50ae97e8c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f726594474075891e730000008c18697465726174696f6e735f73696e63655f726573746f7265944b0775756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f35656231653238325f32335f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d3531322c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f725f323032342d30362d32355f31382d35372d32392f636865636b706f696e745f30303030313594682268257d946827897385945294682a682d2981947d9468308a10a143eb21091fa9a2394734ed33e86c357362756268317d94288c046c6f7373946836683c4308bfdfef77d742684094869452948c0974696d657374616d70944adb237b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303135948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b108c08747269616c5f6964948c083565623165323832948c0464617465948c13323032342d30362d32355f32322d30382d3539948c1074696d655f746869735f697465725f73944740479497040000008c0c74696d655f746f74616c5f739447408902e2d00000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b018c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b108c0a656e636f6465725f6666944d00048c0d6465636f6465725f72745f6666944d00028c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b048c0964726f705f7261746594473fd231331f06738a8c026c7294473f24239ae50ae97e8c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f7265944740787bb1538000008c18697465726174696f6e735f73696e63655f726573746f7265944b0875756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f35656231653238325f32335f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d3531322c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f725f323032342d30362d32355f31382d35372d32392f636865636b706f696e745f30303030313694682268257d946827897385945294682a682d2981947d9468308a11b5a2db847af6a690174b20e00c776398007362756268317d94288c046c6f7373946836683c430847a3d1887a9a684094869452948c0974696d657374616d70944a0b247b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303136948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b118c08747269616c5f6964948c083565623165323832948c0464617465948c13323032342d30362d32355f32322d30392d3437948c1074696d655f746869735f697465725f739447404793fb240000008c0c74696d655f746f74616c5f739447408a7c22824000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b018c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b108c0a656e636f6465725f6666944d00048c0d6465636f6465725f72745f6666944d00028c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b048c0964726f705f7261746594473fd231331f06738a8c026c7294473f24239ae50ae97e8c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f72659447407b6e30b80000008c18697465726174696f6e735f73696e63655f726573746f7265944b0975756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f35656231653238325f32335f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d3531322c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f725f323032342d30362d32355f31382d35372d32392f636865636b706f696e745f30303030313794682268257d946827897385945294682a682d2981947d9468308a10f8f9a9758b204aba2e454a28a0bd42257362756268317d94288c046c6f7373946836683c4308140a8562e7fe664094869452948c0974696d657374616d70944a3a247b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303137948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b128c08747269616c5f6964948c083565623165323832948c0464617465948c13323032342d30362d32355f32322d31302d3334948c1074696d655f746869735f697465725f7394474047b3e2340000008c0c74696d655f746f74616c5f739447408bf760a58000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b018c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b108c0a656e636f6465725f6666944d00048c0d6465636f6465725f72745f6666944d00028c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b048c0964726f705f7261746594473fd231331f06738a8c026c7294473f24239ae50ae97e8c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f72659447407e64acfe8000008c18697465726174696f6e735f73696e63655f726573746f7265944b0a757562658c195f6c61746573745f636865636b706f696e745f726573756c74946a5203000075622e\"\n }\n}"
+ ],
+ [
+ "{\n \"stub\": false,\n \"trainable_name\": \"train_model\",\n \"trial_id\": \"6be79717\",\n \"storage\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059572040000000000008c1b7261792e747261696e2e5f696e7465726e616c2e73746f72616765948c0e53746f72616765436f6e746578749493942981947d94288c12637573746f6d5f66735f70726f766964656494898c1273746f726167655f6c6f63616c5f70617468948c2b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c7473948c136578706572696d656e745f6469725f6e616d65948c0f746573745f6578706572696d656e74948c0e747269616c5f6469725f6e616d65948c96747261696e5f6d6f64656c5f36626537393731375f32345f62617463685f73697a653d313032342c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d313032342c6465636f6465725f72745f6e756d5f6c617965723d322c64726f705f5f323032342d30362d32355f31392d31342d3334948c1863757272656e745f636865636b706f696e745f696e646578944b018c0b73796e635f636f6e666967948c097261792e747261696e948c0a53796e63436f6e6669679493942981947d94288c0b73796e635f706572696f64944d2c018c0c73796e635f74696d656f7574944d08078c0e73796e635f61727469666163747394898c1c73796e635f6172746966616374735f6f6e5f636865636b706f696e7494888c0a75706c6f61645f646972948c0a44455052454341544544948c0673796e6365729468188c1273796e635f6f6e5f636865636b706f696e7494681875628c1273746f726167655f66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c0f73746f726167655f66735f70617468948c452f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f7465737494681968008c115f46696c6573797374656d53796e6365729493942981947d9428681b682268134d2c0168144d08078c116c6173745f73796e635f75705f74696d65944741d99ebe858154548c136c6173745f73796e635f646f776e5f74696d659447fff00000000000008c0d5f73796e635f70726f63657373944e8c0c5f63757272656e745f636d649468008c125f75706c6f61645f746f5f66735f706174689493947d94288c0a6c6f63616c5f70617468948c3b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c74732f746573745f6578706572696d656e74948c0266739468228c0766735f70617468948c552f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e74948c076578636c756465945d948c0e2a2f636865636b706f696e745f2a9461758694756275622e\"\n },\n \"config\": {\n \"encoder_num_layer\": 1,\n \"decoder_rt_num_layer\": 2,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 64,\n \"encoder_ff\": 2048,\n \"decoder_rt_ff\": 1024,\n \"decoder_int_ff\": 512,\n \"n_head\": 8,\n \"drop_rate\": 0.7446531774853204,\n \"lr\": 0.006000257122673396,\n \"batch_size\": 1024\n },\n \"_Trial__unresolved_config\": {\n \"encoder_num_layer\": 1,\n \"decoder_rt_num_layer\": 2,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 64,\n \"encoder_ff\": 2048,\n \"decoder_rt_ff\": 1024,\n \"decoder_int_ff\": 512,\n \"n_head\": 8,\n \"drop_rate\": 0.7446531774853204,\n \"lr\": 0.006000257122673396,\n \"batch_size\": 1024\n },\n \"evaluated_params\": {\n \"batch_size\": 1024,\n \"decoder_int_ff\": 512,\n \"decoder_int_num_layer\": 1,\n \"decoder_rt_ff\": 1024,\n \"decoder_rt_num_layer\": 2,\n \"drop_rate\": 0.7446531774853204,\n \"embedding_dim\": 64,\n \"encoder_ff\": 2048,\n \"encoder_num_layer\": 1,\n \"lr\": 0.006000257122673396,\n \"n_head\": 8\n },\n \"experiment_tag\": \"24_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop_rate=0.7447,embedding_dim=64,encoder_ff=2048,encoder_num_layer=1,lr=0.0060,n_head=8\",\n \"stopping_criterion\": {},\n \"_setup_default_resource\": true,\n \"_default_placement_group_factory\": \"80054e2e\",\n \"placement_group_factory\": \"800595ba000000000000008c237261792e74756e652e657865637574696f6e2e706c6163656d656e745f67726f757073948c15506c6163656d656e7447726f7570466163746f72799493942981947d94288c085f62756e646c6573945d947d94288c03435055944740540000000000008c034750559447401000000000000075618c155f686561645f62756e646c655f69735f656d70747994898c095f7374726174656779948c045041434b948c055f6172677394298c075f6b7761726773947d9475622e\",\n \"log_to_file\": [\n null,\n null\n ],\n \"max_failures\": 0,\n \"_default_result_or_future\": null,\n \"export_formats\": [],\n \"status\": \"PAUSED\",\n \"relative_logdir\": \"train_model_6be79717_24_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop__2024-06-25_19-14-34\",\n \"trial_name_creator\": null,\n \"trial_dirname_creator\": null,\n \"custom_trial_name\": null,\n \"custom_dirname\": null,\n \"restore_path\": null,\n \"_restore_checkpoint_result\": null,\n \"_state_json\": null,\n \"results\": \"80054e2e\",\n \"extra_arg\": \"80054e2e\",\n \"_resources\": \"80054e2e\"\n}",
+ "{\n \"start_time\": 1719335783.6048226,\n \"num_failures\": 0,\n \"num_failures_after_restore\": 0,\n \"error_filename\": null,\n \"pickled_error_filename\": null,\n \"last_result\": {\n \"loss\": 2003.3902054433747,\n \"timestamp\": 1719335912,\n \"checkpoint_dir_name\": \"checkpoint_000001\",\n \"should_checkpoint\": true,\n \"done\": false,\n \"training_iteration\": 2,\n \"trial_id\": \"6be79717\",\n \"date\": \"2024-06-25_19-18-32\",\n \"time_this_iter_s\": 57.25079274177551,\n \"time_total_s\": 128.72498559951782,\n \"pid\": 2195253,\n \"hostname\": \"r8i6n2\",\n \"node_ip\": \"10.159.28.60\",\n \"config\": {\n \"encoder_num_layer\": 1,\n \"decoder_rt_num_layer\": 2,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 64,\n \"encoder_ff\": 2048,\n \"decoder_rt_ff\": 1024,\n \"decoder_int_ff\": 512,\n \"n_head\": 8,\n \"drop_rate\": 0.7446531774853204,\n \"lr\": 0.006000257122673396,\n \"batch_size\": 1024\n },\n \"time_since_restore\": 128.72498559951782,\n \"iterations_since_restore\": 2,\n \"experiment_tag\": \"24_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop_rate=0.7447,embedding_dim=64,encoder_ff=2048,encoder_num_layer=1,lr=0.0060,n_head=8\"\n },\n \"last_result_time\": 1719335912.3383458,\n \"metric_analysis\": {\n \"loss\": {\n \"max\": 2020.6805025836613,\n \"min\": 2003.3902054433747,\n \"avg\": 2012.035354013518,\n \"last\": 2003.3902054433747,\n \"last-5-avg\": 2012.035354013518,\n \"last-10-avg\": 2012.035354013518\n },\n \"should_checkpoint\": {\n \"max\": true,\n \"min\": true,\n \"avg\": 1.0,\n \"last\": true,\n \"last-5-avg\": 1.0,\n \"last-10-avg\": 1.0\n },\n \"done\": {\n \"max\": false,\n \"min\": false,\n \"avg\": 0.0,\n \"last\": false,\n \"last-5-avg\": 0.0,\n \"last-10-avg\": 0.0\n },\n \"training_iteration\": {\n \"max\": 2,\n \"min\": 1,\n \"avg\": 1.5,\n \"last\": 2,\n \"last-5-avg\": 1.5,\n \"last-10-avg\": 1.5\n },\n \"time_this_iter_s\": {\n \"max\": 71.47419285774231,\n \"min\": 57.25079274177551,\n \"avg\": 64.36249279975891,\n \"last\": 57.25079274177551,\n \"last-5-avg\": 64.36249279975891,\n \"last-10-avg\": 64.36249279975891\n },\n \"time_total_s\": {\n \"max\": 128.72498559951782,\n \"min\": 71.47419285774231,\n \"avg\": 100.09958922863007,\n \"last\": 128.72498559951782,\n \"last-5-avg\": 100.09958922863007,\n \"last-10-avg\": 100.09958922863007\n },\n \"time_since_restore\": {\n \"max\": 128.72498559951782,\n \"min\": 71.47419285774231,\n \"avg\": 100.09958922863007,\n \"last\": 128.72498559951782,\n \"last-5-avg\": 100.09958922863007,\n \"last-10-avg\": 100.09958922863007\n },\n \"iterations_since_restore\": {\n \"max\": 2,\n \"min\": 1,\n \"avg\": 1.5,\n \"last\": 2,\n \"last-5-avg\": 1.5,\n \"last-10-avg\": 1.5\n }\n },\n \"_n_steps\": [\n 5,\n 10\n ],\n \"metric_n_steps\": {\n \"loss\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005959d000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294288c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b007494624308ad56abd5b8929f4094869452946807680d4308100804928f4d9f409486945294652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005959d000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294288c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b007494624308ad56abd5b8929f4094869452946807680d4308100804928f4d9f409486945294652e\"\n }\n },\n \"should_checkpoint\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059524000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294288888652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059524000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294288888652e\"\n }\n },\n \"done\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059524000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294288989652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059524000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294288989652e\"\n }\n },\n \"training_iteration\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059526000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294284b014b02652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059526000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294284b014b02652e\"\n }\n },\n \"time_this_iter_s\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059534000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b058694529428474051de592d00000047404ca019fa000000652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059534000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a8694529428474051de592d00000047404ca019fa000000652e\"\n }\n },\n \"time_total_s\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059534000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b058694529428474051de592d000000474060173315000000652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059534000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a8694529428474051de592d000000474060173315000000652e\"\n }\n },\n \"time_since_restore\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059534000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b058694529428474051de592d000000474060173315000000652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059534000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a8694529428474051de592d000000474060173315000000652e\"\n }\n },\n \"iterations_since_restore\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059526000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294284b014b02652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059526000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294284b014b02652e\"\n }\n }\n },\n \"checkpoint_manager\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"800595a6090000000000008c267261792e747261696e2e5f696e7465726e616c2e636865636b706f696e745f6d616e61676572948c125f436865636b706f696e744d616e616765729493942981947d94288c125f636865636b706f696e745f636f6e666967948c097261792e747261696e948c10436865636b706f696e74436f6e6669679493942981947d94288c0b6e756d5f746f5f6b656570944e8c1a636865636b706f696e745f73636f72655f617474726962757465944e8c16636865636b706f696e745f73636f72655f6f72646572948c036d6178948c14636865636b706f696e745f6672657175656e6379944b008c11636865636b706f696e745f61745f656e6494898c1a5f636865636b706f696e745f6b6565705f616c6c5f72616e6b73948c0a44455052454341544544948c1f5f636865636b706f696e745f75706c6f61645f66726f6d5f776f726b65727394681275628c135f636865636b706f696e745f726573756c7473945d94288c1b7261792e747261696e2e5f696e7465726e616c2e73657373696f6e948c0f5f547261696e696e67526573756c749493942981947d94288c0a636865636b706f696e749468068c0a436865636b706f696e749493942981947d94288c0470617468948cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f36626537393731375f32345f62617463685f73697a653d313032342c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d313032342c6465636f6465725f72745f6e756d5f6c617965723d322c64726f705f5f323032342d30362d32355f31392d31342d33342f636865636b706f696e745f303030303030948c0a66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c055f75756964948c0475756964948c04555549449493942981947d948c03696e74948a117534ee679e09c7a75841a69709c52f9900736275628c076d657472696373947d94288c046c6f7373948c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b007494624308ad56abd5b8929f4094869452948c0974696d657374616d70944aaffb7a668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303030948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b018c08747269616c5f6964948c083662653739373137948c0464617465948c13323032342d30362d32355f31392d31372d3335948c1074696d655f746869735f697465725f7394474051de592d0000008c0c74696d655f746f74616c5f7394474051de592d0000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b018c146465636f6465725f72745f6e756d5f6c61796572944b028c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b408c0a656e636f6465725f6666944d00088c0d6465636f6465725f72745f6666944d00048c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b088c0964726f705f7261746594473fe7d432e6852e928c026c7294473f7893b9c1c067128c0a62617463685f73697a65944d0004758c1274696d655f73696e63655f726573746f726594474051de592d0000008c18697465726174696f6e735f73696e63655f726573746f7265944b0175756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f36626537393731375f32345f62617463685f73697a653d313032342c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d313032342c6465636f6465725f72745f6e756d5f6c617965723d322c64726f705f5f323032342d30362d32355f31392d31342d33342f636865636b706f696e745f30303030303194682268257d946827897385945294682a682d2981947d9468308a11ef0301a808ba10b9e348ae9d26f051ed007362756268317d94288c046c6f7373946836683c4308100804928f4d9f4094869452948c0974696d657374616d70944ae8fb7a668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303031948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b028c08747269616c5f6964948c083662653739373137948c0464617465948c13323032342d30362d32355f31392d31382d3332948c1074696d655f746869735f697465725f739447404ca019fa0000008c0c74696d655f746f74616c5f73944740601733150000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b018c146465636f6465725f72745f6e756d5f6c61796572944b028c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b408c0a656e636f6465725f6666944d00088c0d6465636f6465725f72745f6666944d00048c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b088c0964726f705f7261746594473fe7d432e6852e928c026c7294473f7893b9c1c067128c0a62617463685f73697a65944d0004758c1274696d655f73696e63655f726573746f7265944740601733150000008c18697465726174696f6e735f73696e63655f726573746f7265944b02757562658c195f6c61746573745f636865636b706f696e745f726573756c7494686275622e\"\n }\n}"
+ ],
+ [
+ "{\n \"stub\": false,\n \"trainable_name\": \"train_model\",\n \"trial_id\": \"82396cf3\",\n \"storage\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059572040000000000008c1b7261792e747261696e2e5f696e7465726e616c2e73746f72616765948c0e53746f72616765436f6e746578749493942981947d94288c12637573746f6d5f66735f70726f766964656494898c1273746f726167655f6c6f63616c5f70617468948c2b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c7473948c136578706572696d656e745f6469725f6e616d65948c0f746573745f6578706572696d656e74948c0e747269616c5f6469725f6e616d65948c96747261696e5f6d6f64656c5f38323339366366335f32355f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d3531322c6465636f6465725f72745f6e756d5f6c617965723d322c64726f705f725f323032342d30362d32355f31392d31362d3233948c1863757272656e745f636865636b706f696e745f696e646578944b018c0b73796e635f636f6e666967948c097261792e747261696e948c0a53796e63436f6e6669679493942981947d94288c0b73796e635f706572696f64944d2c018c0c73796e635f74696d656f7574944d08078c0e73796e635f61727469666163747394898c1c73796e635f6172746966616374735f6f6e5f636865636b706f696e7494888c0a75706c6f61645f646972948c0a44455052454341544544948c0673796e6365729468188c1273796e635f6f6e5f636865636b706f696e7494681875628c1273746f726167655f66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c0f73746f726167655f66735f70617468948c452f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f7465737494681968008c115f46696c6573797374656d53796e6365729493942981947d9428681b682268134d2c0168144d08078c116c6173745f73796e635f75705f74696d65944741d99ebed10938d48c136c6173745f73796e635f646f776e5f74696d659447fff00000000000008c0d5f73796e635f70726f63657373944e8c0c5f63757272656e745f636d649468008c125f75706c6f61645f746f5f66735f706174689493947d94288c0a6c6f63616c5f70617468948c3b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c74732f746573745f6578706572696d656e74948c0266739468228c0766735f70617468948c552f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e74948c076578636c756465945d948c0e2a2f636865636b706f696e745f2a9461758694756275622e\"\n },\n \"config\": {\n \"encoder_num_layer\": 4,\n \"decoder_rt_num_layer\": 2,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 1024,\n \"encoder_ff\": 2048,\n \"decoder_rt_ff\": 512,\n \"decoder_int_ff\": 512,\n \"n_head\": 4,\n \"drop_rate\": 0.9507029678825294,\n \"lr\": 0.00025487621898754583,\n \"batch_size\": 2048\n },\n \"_Trial__unresolved_config\": {\n \"encoder_num_layer\": 4,\n \"decoder_rt_num_layer\": 2,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 1024,\n \"encoder_ff\": 2048,\n \"decoder_rt_ff\": 512,\n \"decoder_int_ff\": 512,\n \"n_head\": 4,\n \"drop_rate\": 0.9507029678825294,\n \"lr\": 0.00025487621898754583,\n \"batch_size\": 2048\n },\n \"evaluated_params\": {\n \"batch_size\": 2048,\n \"decoder_int_ff\": 512,\n \"decoder_int_num_layer\": 1,\n \"decoder_rt_ff\": 512,\n \"decoder_rt_num_layer\": 2,\n \"drop_rate\": 0.9507029678825294,\n \"embedding_dim\": 1024,\n \"encoder_ff\": 2048,\n \"encoder_num_layer\": 4,\n \"lr\": 0.00025487621898754583,\n \"n_head\": 4\n },\n \"experiment_tag\": \"25_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_rate=0.9507,embedding_dim=1024,encoder_ff=2048,encoder_num_layer=4,lr=0.0003,n_head=4\",\n \"stopping_criterion\": {},\n \"_setup_default_resource\": true,\n \"_default_placement_group_factory\": \"80054e2e\",\n \"placement_group_factory\": \"800595ba000000000000008c237261792e74756e652e657865637574696f6e2e706c6163656d656e745f67726f757073948c15506c6163656d656e7447726f7570466163746f72799493942981947d94288c085f62756e646c6573945d947d94288c03435055944740540000000000008c034750559447401000000000000075618c155f686561645f62756e646c655f69735f656d70747994898c095f7374726174656779948c045041434b948c055f6172677394298c075f6b7761726773947d9475622e\",\n \"log_to_file\": [\n null,\n null\n ],\n \"max_failures\": 0,\n \"_default_result_or_future\": null,\n \"export_formats\": [],\n \"status\": \"PAUSED\",\n \"relative_logdir\": \"train_model_82396cf3_25_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_19-16-23\",\n \"trial_name_creator\": null,\n \"trial_dirname_creator\": null,\n \"custom_trial_name\": null,\n \"custom_dirname\": null,\n \"restore_path\": null,\n \"_restore_checkpoint_result\": null,\n \"_state_json\": null,\n \"results\": \"80054e2e\",\n \"extra_arg\": \"80054e2e\",\n \"_resources\": \"80054e2e\"\n}",
+ "{\n \"start_time\": 1719335912.4933662,\n \"num_failures\": 0,\n \"num_failures_after_restore\": 0,\n \"error_filename\": null,\n \"pickled_error_filename\": null,\n \"last_result\": {\n \"loss\": 2187.193873608206,\n \"timestamp\": 1719337142,\n \"checkpoint_dir_name\": \"checkpoint_000001\",\n \"should_checkpoint\": true,\n \"done\": false,\n \"training_iteration\": 2,\n \"trial_id\": \"82396cf3\",\n \"date\": \"2024-06-25_19-39-03\",\n \"time_this_iter_s\": 608.1205008029938,\n \"time_total_s\": 1230.5634942054749,\n \"pid\": 2195253,\n \"hostname\": \"r8i6n2\",\n \"node_ip\": \"10.159.28.60\",\n \"config\": {\n \"encoder_num_layer\": 4,\n \"decoder_rt_num_layer\": 2,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 1024,\n \"encoder_ff\": 2048,\n \"decoder_rt_ff\": 512,\n \"decoder_int_ff\": 512,\n \"n_head\": 4,\n \"drop_rate\": 0.9507029678825294,\n \"lr\": 0.00025487621898754583,\n \"batch_size\": 2048\n },\n \"time_since_restore\": 1230.5634942054749,\n \"iterations_since_restore\": 2,\n \"experiment_tag\": \"25_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_rate=0.9507,embedding_dim=1024,encoder_ff=2048,encoder_num_layer=4,lr=0.0003,n_head=4\"\n },\n \"last_result_time\": 1719337143.0662968,\n \"metric_analysis\": {\n \"loss\": {\n \"max\": 2187.193873608206,\n \"min\": 2178.1833553764764,\n \"avg\": 2182.688614492341,\n \"last\": 2187.193873608206,\n \"last-5-avg\": 2182.688614492341,\n \"last-10-avg\": 2182.688614492341\n },\n \"should_checkpoint\": {\n \"max\": true,\n \"min\": true,\n \"avg\": 1.0,\n \"last\": true,\n \"last-5-avg\": 1.0,\n \"last-10-avg\": 1.0\n },\n \"done\": {\n \"max\": false,\n \"min\": false,\n \"avg\": 0.0,\n \"last\": false,\n \"last-5-avg\": 0.0,\n \"last-10-avg\": 0.0\n },\n \"training_iteration\": {\n \"max\": 2,\n \"min\": 1,\n \"avg\": 1.5,\n \"last\": 2,\n \"last-5-avg\": 1.5,\n \"last-10-avg\": 1.5\n },\n \"time_this_iter_s\": {\n \"max\": 622.4429934024811,\n \"min\": 608.1205008029938,\n \"avg\": 615.2817471027374,\n \"last\": 608.1205008029938,\n \"last-5-avg\": 615.2817471027374,\n \"last-10-avg\": 615.2817471027374\n },\n \"time_total_s\": {\n \"max\": 1230.5634942054749,\n \"min\": 622.4429934024811,\n \"avg\": 926.503243803978,\n \"last\": 1230.5634942054749,\n \"last-5-avg\": 926.503243803978,\n \"last-10-avg\": 926.503243803978\n },\n \"time_since_restore\": {\n \"max\": 1230.5634942054749,\n \"min\": 622.4429934024811,\n \"avg\": 926.503243803978,\n \"last\": 1230.5634942054749,\n \"last-5-avg\": 926.503243803978,\n \"last-10-avg\": 926.503243803978\n },\n \"iterations_since_restore\": {\n \"max\": 2,\n \"min\": 1,\n \"avg\": 1.5,\n \"last\": 2,\n \"last-5-avg\": 1.5,\n \"last-10-avg\": 1.5\n }\n },\n \"_n_steps\": [\n 5,\n 10\n ],\n \"metric_n_steps\": {\n \"loss\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005959d000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294288c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b0074946243080683c1e05d04a14094869452946807680d43089bcd66436316a1409486945294652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005959d000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294288c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b0074946243080683c1e05d04a14094869452946807680d43089bcd66436316a1409486945294652e\"\n }\n },\n \"should_checkpoint\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059524000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294288888652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059524000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294288888652e\"\n }\n },\n \"done\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059524000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294288989652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059524000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294288989652e\"\n }\n },\n \"training_iteration\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059526000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294284b014b02652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059526000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294284b014b02652e\"\n }\n },\n \"time_this_iter_s\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059534000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b058694529428474083738b4020000047408300f6c9200000652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059534000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a8694529428474083738b4020000047408300f6c9200000652e\"\n }\n },\n \"time_total_s\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059534000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b058694529428474083738b402000004740933a4104a00000652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059534000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a8694529428474083738b402000004740933a4104a00000652e\"\n }\n },\n \"time_since_restore\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059534000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b058694529428474083738b402000004740933a4104a00000652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059534000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a8694529428474083738b402000004740933a4104a00000652e\"\n }\n },\n \"iterations_since_restore\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059526000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294284b014b02652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059526000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294284b014b02652e\"\n }\n }\n },\n \"checkpoint_manager\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"800595a7090000000000008c267261792e747261696e2e5f696e7465726e616c2e636865636b706f696e745f6d616e61676572948c125f436865636b706f696e744d616e616765729493942981947d94288c125f636865636b706f696e745f636f6e666967948c097261792e747261696e948c10436865636b706f696e74436f6e6669679493942981947d94288c0b6e756d5f746f5f6b656570944e8c1a636865636b706f696e745f73636f72655f617474726962757465944e8c16636865636b706f696e745f73636f72655f6f72646572948c036d6178948c14636865636b706f696e745f6672657175656e6379944b008c11636865636b706f696e745f61745f656e6494898c1a5f636865636b706f696e745f6b6565705f616c6c5f72616e6b73948c0a44455052454341544544948c1f5f636865636b706f696e745f75706c6f61645f66726f6d5f776f726b65727394681275628c135f636865636b706f696e745f726573756c7473945d94288c1b7261792e747261696e2e5f696e7465726e616c2e73657373696f6e948c0f5f547261696e696e67526573756c749493942981947d94288c0a636865636b706f696e749468068c0a436865636b706f696e749493942981947d94288c0470617468948cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f38323339366366335f32355f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d3531322c6465636f6465725f72745f6e756d5f6c617965723d322c64726f705f725f323032342d30362d32355f31392d31362d32332f636865636b706f696e745f303030303030948c0a66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c055f75756964948c0475756964948c04555549449493942981947d948c03696e74948a10b0521178b5d8959c1040a702210f1421736275628c076d657472696373947d94288c046c6f7373948c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b0074946243080683c1e05d04a14094869452948c0974696d657374616d70944a56fe7a668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303030948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b018c08747269616c5f6964948c083832333936636633948c0464617465948c13323032342d30362d32355f31392d32382d3534948c1074696d655f746869735f697465725f7394474083738b402000008c0c74696d655f746f74616c5f7394474083738b402000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b048c146465636f6465725f72745f6e756d5f6c61796572944b028c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944d00048c0a656e636f6465725f6666944d00088c0d6465636f6465725f72745f6666944d00028c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b048c0964726f705f7261746594473fee6c28a1687fd08c026c7294473f30b41d066b9e998c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f726594474083738b402000008c18697465726174696f6e735f73696e63655f726573746f7265944b0175756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f38323339366366335f32355f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d3531322c6465636f6465725f72745f6e756d5f6c617965723d322c64726f705f725f323032342d30362d32355f31392d31362d32332f636865636b706f696e745f30303030303194682268257d946827897385945294682a682d2981947d9468308a1109d9a6b3fe1045b69f4feea938dd89bf007362756268317d94288c046c6f7373946836683c43089bcd66436316a14094869452948c0974696d657374616d70944ab6007b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303031948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b028c08747269616c5f6964948c083832333936636633948c0464617465948c13323032342d30362d32355f31392d33392d3033948c1074696d655f746869735f697465725f739447408300f6c92000008c0c74696d655f746f74616c5f73944740933a4104a000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b048c146465636f6465725f72745f6e756d5f6c61796572944b028c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944d00048c0a656e636f6465725f6666944d00088c0d6465636f6465725f72745f6666944d00028c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b048c0964726f705f7261746594473fee6c28a1687fd08c026c7294473f30b41d066b9e998c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f7265944740933a4104a000008c18697465726174696f6e735f73696e63655f726573746f7265944b02757562658c195f6c61746573745f636865636b706f696e745f726573756c7494686275622e\"\n }\n}"
+ ],
+ [
+ "{\n \"stub\": false,\n \"trainable_name\": \"train_model\",\n \"trial_id\": \"e31d7ca7\",\n \"storage\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059572040000000000008c1b7261792e747261696e2e5f696e7465726e616c2e73746f72616765948c0e53746f72616765436f6e746578749493942981947d94288c12637573746f6d5f66735f70726f766964656494898c1273746f726167655f6c6f63616c5f70617468948c2b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c7473948c136578706572696d656e745f6469725f6e616d65948c0f746573745f6578706572696d656e74948c0e747269616c5f6469725f6e616d65948c96747261696e5f6d6f64656c5f65333164376361375f32365f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d323034382c6465636f6465725f72745f6e756d5f6c617965723d322c64726f705f5f323032342d30362d32355f31392d31382d3332948c1863757272656e745f636865636b706f696e745f696e646578944b018c0b73796e635f636f6e666967948c097261792e747261696e948c0a53796e63436f6e6669679493942981947d94288c0b73796e635f706572696f64944d2c018c0c73796e635f74696d656f7574944d08078c0e73796e635f61727469666163747394898c1c73796e635f6172746966616374735f6f6e5f636865636b706f696e7494888c0a75706c6f61645f646972948c0a44455052454341544544948c0673796e6365729468188c1273796e635f6f6e5f636865636b706f696e7494681875628c1273746f726167655f66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c0f73746f726167655f66735f70617468948c452f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f7465737494681968008c115f46696c6573797374656d53796e6365729493942981947d9428681b682268134d2c0168144d08078c116c6173745f73796e635f75705f74696d65944741d99ebed10938d48c136c6173745f73796e635f646f776e5f74696d659447fff00000000000008c0d5f73796e635f70726f63657373944e8c0c5f63757272656e745f636d649468008c125f75706c6f61645f746f5f66735f706174689493947d94288c0a6c6f63616c5f70617468948c3b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c74732f746573745f6578706572696d656e74948c0266739468228c0766735f70617468948c552f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e74948c076578636c756465945d948c0e2a2f636865636b706f696e745f2a9461758694756275622e\"\n },\n \"config\": {\n \"encoder_num_layer\": 4,\n \"decoder_rt_num_layer\": 2,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 64,\n \"encoder_ff\": 2048,\n \"decoder_rt_ff\": 2048,\n \"decoder_int_ff\": 512,\n \"n_head\": 4,\n \"drop_rate\": 0.7500609138892806,\n \"lr\": 0.0713230632683364,\n \"batch_size\": 2048\n },\n \"_Trial__unresolved_config\": {\n \"encoder_num_layer\": 4,\n \"decoder_rt_num_layer\": 2,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 64,\n \"encoder_ff\": 2048,\n \"decoder_rt_ff\": 2048,\n \"decoder_int_ff\": 512,\n \"n_head\": 4,\n \"drop_rate\": 0.7500609138892806,\n \"lr\": 0.0713230632683364,\n \"batch_size\": 2048\n },\n \"evaluated_params\": {\n \"batch_size\": 2048,\n \"decoder_int_ff\": 512,\n \"decoder_int_num_layer\": 1,\n \"decoder_rt_ff\": 2048,\n \"decoder_rt_num_layer\": 2,\n \"drop_rate\": 0.7500609138892806,\n \"embedding_dim\": 64,\n \"encoder_ff\": 2048,\n \"encoder_num_layer\": 4,\n \"lr\": 0.0713230632683364,\n \"n_head\": 4\n },\n \"experiment_tag\": \"26_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=2,drop_rate=0.7501,embedding_dim=64,encoder_ff=2048,encoder_num_layer=4,lr=0.0713,n_head=4\",\n \"stopping_criterion\": {},\n \"_setup_default_resource\": true,\n \"_default_placement_group_factory\": \"80054e2e\",\n \"placement_group_factory\": \"800595ba000000000000008c237261792e74756e652e657865637574696f6e2e706c6163656d656e745f67726f757073948c15506c6163656d656e7447726f7570466163746f72799493942981947d94288c085f62756e646c6573945d947d94288c03435055944740540000000000008c034750559447401000000000000075618c155f686561645f62756e646c655f69735f656d70747994898c095f7374726174656779948c045041434b948c055f6172677394298c075f6b7761726773947d9475622e\",\n \"log_to_file\": [\n null,\n null\n ],\n \"max_failures\": 0,\n \"_default_result_or_future\": null,\n \"export_formats\": [],\n \"status\": \"TERMINATED\",\n \"relative_logdir\": \"train_model_e31d7ca7_26_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=2,drop__2024-06-25_19-18-32\",\n \"trial_name_creator\": null,\n \"trial_dirname_creator\": null,\n \"custom_trial_name\": null,\n \"custom_dirname\": null,\n \"restore_path\": null,\n \"_restore_checkpoint_result\": null,\n \"_state_json\": null,\n \"results\": \"80054e2e\",\n \"extra_arg\": \"80054e2e\",\n \"_resources\": \"80054e2e\"\n}",
+ "{\n \"start_time\": 1719337143.2143147,\n \"num_failures\": 0,\n \"num_failures_after_restore\": 0,\n \"error_filename\": null,\n \"pickled_error_filename\": null,\n \"last_result\": {\n \"loss\": 2698.886155496432,\n \"timestamp\": 1719337295,\n \"checkpoint_dir_name\": \"checkpoint_000001\",\n \"should_checkpoint\": true,\n \"done\": true,\n \"training_iteration\": 2,\n \"trial_id\": \"e31d7ca7\",\n \"date\": \"2024-06-25_19-41-35\",\n \"time_this_iter_s\": 69.29531860351562,\n \"time_total_s\": 152.63765025138855,\n \"pid\": 2195253,\n \"hostname\": \"r8i6n2\",\n \"node_ip\": \"10.159.28.60\",\n \"config\": {\n \"encoder_num_layer\": 4,\n \"decoder_rt_num_layer\": 2,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 64,\n \"encoder_ff\": 2048,\n \"decoder_rt_ff\": 2048,\n \"decoder_int_ff\": 512,\n \"n_head\": 4,\n \"drop_rate\": 0.7500609138892806,\n \"lr\": 0.0713230632683364,\n \"batch_size\": 2048\n },\n \"time_since_restore\": 152.63765025138855,\n \"iterations_since_restore\": 2,\n \"experiment_tag\": \"26_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=2,drop_rate=0.7501,embedding_dim=64,encoder_ff=2048,encoder_num_layer=4,lr=0.0713,n_head=4\"\n },\n \"last_result_time\": 1719337295.8805537,\n \"metric_analysis\": {\n \"loss\": {\n \"max\": 3842.66859736405,\n \"min\": 2698.886155496432,\n \"avg\": 3270.777376430241,\n \"last\": 2698.886155496432,\n \"last-5-avg\": 3270.777376430241,\n \"last-10-avg\": 3270.777376430241\n },\n \"should_checkpoint\": {\n \"max\": true,\n \"min\": true,\n \"avg\": 1.0,\n \"last\": true,\n \"last-5-avg\": 1.0,\n \"last-10-avg\": 1.0\n },\n \"done\": {\n \"max\": true,\n \"min\": false,\n \"avg\": 0.5,\n \"last\": true,\n \"last-5-avg\": 0.5,\n \"last-10-avg\": 0.5\n },\n \"training_iteration\": {\n \"max\": 2,\n \"min\": 1,\n \"avg\": 1.5,\n \"last\": 2,\n \"last-5-avg\": 1.5,\n \"last-10-avg\": 1.5\n },\n \"time_this_iter_s\": {\n \"max\": 83.34233164787292,\n \"min\": 69.29531860351562,\n \"avg\": 76.31882512569427,\n \"last\": 69.29531860351562,\n \"last-5-avg\": 76.31882512569427,\n \"last-10-avg\": 76.31882512569427\n },\n \"time_total_s\": {\n \"max\": 152.63765025138855,\n \"min\": 83.34233164787292,\n \"avg\": 117.98999094963074,\n \"last\": 152.63765025138855,\n \"last-5-avg\": 117.98999094963074,\n \"last-10-avg\": 117.98999094963074\n },\n \"time_since_restore\": {\n \"max\": 152.63765025138855,\n \"min\": 83.34233164787292,\n \"avg\": 117.98999094963074,\n \"last\": 152.63765025138855,\n \"last-5-avg\": 117.98999094963074,\n \"last-10-avg\": 117.98999094963074\n },\n \"iterations_since_restore\": {\n \"max\": 2,\n \"min\": 1,\n \"avg\": 1.5,\n \"last\": 2,\n \"last-5-avg\": 1.5,\n \"last-10-avg\": 1.5\n }\n },\n \"_n_steps\": [\n 5,\n 10\n ],\n \"metric_n_steps\": {\n \"loss\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005959d000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294288c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b00749462430893c964525605ae4094869452946807680d4308b1582cb6c515a5409486945294652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005959d000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294288c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b00749462430893c964525605ae4094869452946807680d4308b1582cb6c515a5409486945294652e\"\n }\n },\n \"should_checkpoint\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059524000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294288888652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059524000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294288888652e\"\n }\n },\n \"done\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059524000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294288988652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059524000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294288988652e\"\n }\n },\n \"training_iteration\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059526000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294284b014b02652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059526000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294284b014b02652e\"\n }\n },\n \"time_this_iter_s\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059534000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b058694529428474054d5e8c300000047405152e680000000652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059534000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a8694529428474054d5e8c300000047405152e680000000652e\"\n }\n },\n \"time_total_s\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059534000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b058694529428474054d5e8c30000004740631467a1800000652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059534000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a8694529428474054d5e8c30000004740631467a1800000652e\"\n }\n },\n \"time_since_restore\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059534000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b058694529428474054d5e8c30000004740631467a1800000652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059534000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a8694529428474054d5e8c30000004740631467a1800000652e\"\n }\n },\n \"iterations_since_restore\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059526000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294284b014b02652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059526000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294284b014b02652e\"\n }\n }\n },\n \"checkpoint_manager\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"800595a5090000000000008c267261792e747261696e2e5f696e7465726e616c2e636865636b706f696e745f6d616e61676572948c125f436865636b706f696e744d616e616765729493942981947d94288c125f636865636b706f696e745f636f6e666967948c097261792e747261696e948c10436865636b706f696e74436f6e6669679493942981947d94288c0b6e756d5f746f5f6b656570944e8c1a636865636b706f696e745f73636f72655f617474726962757465944e8c16636865636b706f696e745f73636f72655f6f72646572948c036d6178948c14636865636b706f696e745f6672657175656e6379944b008c11636865636b706f696e745f61745f656e6494898c1a5f636865636b706f696e745f6b6565705f616c6c5f72616e6b73948c0a44455052454341544544948c1f5f636865636b706f696e745f75706c6f61645f66726f6d5f776f726b65727394681275628c135f636865636b706f696e745f726573756c7473945d94288c1b7261792e747261696e2e5f696e7465726e616c2e73657373696f6e948c0f5f547261696e696e67526573756c749493942981947d94288c0a636865636b706f696e749468068c0a436865636b706f696e749493942981947d94288c0470617468948cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f65333164376361375f32365f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d323034382c6465636f6465725f72745f6e756d5f6c617965723d322c64726f705f5f323032342d30362d32355f31392d31382d33322f636865636b706f696e745f303030303030948c0a66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c055f75756964948c0475756964948c04555549449493942981947d948c03696e74948a1069f0f04dc7e8f8a8d14373899995645a736275628c076d657472696373947d94288c046c6f7373948c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b00749462430893c964525605ae4094869452948c0974696d657374616d70944a0a017b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303030948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b018c08747269616c5f6964948c086533316437636137948c0464617465948c13323032342d30362d32355f31392d34302d3236948c1074696d655f746869735f697465725f7394474054d5e8c30000008c0c74696d655f746f74616c5f7394474054d5e8c30000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b048c146465636f6465725f72745f6e756d5f6c61796572944b028c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b408c0a656e636f6465725f6666944d00088c0d6465636f6465725f72745f6666944d00088c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b048c0964726f705f7261746594473fe8007fbee531d48c026c7294473fb2423a703023a28c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f726594474054d5e8c30000008c18697465726174696f6e735f73696e63655f726573746f7265944b0175756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f65333164376361375f32365f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d323034382c6465636f6465725f72745f6e756d5f6c617965723d322c64726f705f5f323032342d30362d32355f31392d31382d33322f636865636b706f696e745f30303030303194682268257d946827897385945294682a682d2981947d9468308a1116b3c5101b040fad064e3d871bb62198007362756268317d94288c046c6f7373946836683c4308b1582cb6c515a54094869452948c0974696d657374616d70944a4f017b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303031948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b028c08747269616c5f6964948c086533316437636137948c0464617465948c13323032342d30362d32355f31392d34312d3335948c1074696d655f746869735f697465725f739447405152e6800000008c0c74696d655f746f74616c5f73944740631467a18000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b048c146465636f6465725f72745f6e756d5f6c61796572944b028c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b408c0a656e636f6465725f6666944d00088c0d6465636f6465725f72745f6666944d00088c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b048c0964726f705f7261746594473fe8007fbee531d48c026c7294473fb2423a703023a28c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f7265944740631467a18000008c18697465726174696f6e735f73696e63655f726573746f7265944b02757562658c195f6c61746573745f636865636b706f696e745f726573756c7494686275622e\"\n }\n}"
+ ],
+ [
+ "{\n \"stub\": false,\n \"trainable_name\": \"train_model\",\n \"trial_id\": \"18c11cbc\",\n \"storage\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059572040000000000008c1b7261792e747261696e2e5f696e7465726e616c2e73746f72616765948c0e53746f72616765436f6e746578749493942981947d94288c12637573746f6d5f66735f70726f766964656494898c1273746f726167655f6c6f63616c5f70617468948c2b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c7473948c136578706572696d656e745f6469725f6e616d65948c0f746573745f6578706572696d656e74948c0e747269616c5f6469725f6e616d65948c96747261696e5f6d6f64656c5f31386331316362635f32375f62617463685f73697a653d313032342c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d323034382c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f5f323032342d30362d32355f31392d33392d3033948c1863757272656e745f636865636b706f696e745f696e646578944b138c0b73796e635f636f6e666967948c097261792e747261696e948c0a53796e63436f6e6669679493942981947d94288c0b73796e635f706572696f64944d2c018c0c73796e635f74696d656f7574944d08078c0e73796e635f61727469666163747394898c1c73796e635f6172746966616374735f6f6e5f636865636b706f696e7494888c0a75706c6f61645f646972948c0a44455052454341544544948c0673796e6365729468188c1273796e635f6f6e5f636865636b706f696e7494681875628c1273746f726167655f66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c0f73746f726167655f66735f70617468948c452f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f7465737494681968008c115f46696c6573797374656d53796e6365729493942981947d9428681b682268134d2c0168144d08078c116c6173745f73796e635f75705f74696d65944741d99ebfff24893c8c136c6173745f73796e635f646f776e5f74696d659447fff00000000000008c0d5f73796e635f70726f63657373944e8c0c5f63757272656e745f636d649468008c125f75706c6f61645f746f5f66735f706174689493947d94288c0a6c6f63616c5f70617468948c3b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c74732f746573745f6578706572696d656e74948c0266739468228c0766735f70617468948c552f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e74948c076578636c756465945d948c0e2a2f636865636b706f696e745f2a9461758694756275622e\"\n },\n \"config\": {\n \"encoder_num_layer\": 4,\n \"decoder_rt_num_layer\": 1,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 16,\n \"encoder_ff\": 512,\n \"decoder_rt_ff\": 2048,\n \"decoder_int_ff\": 512,\n \"n_head\": 8,\n \"drop_rate\": 0.3688667410196246,\n \"lr\": 0.0018036063294034547,\n \"batch_size\": 1024\n },\n \"_Trial__unresolved_config\": {\n \"encoder_num_layer\": 4,\n \"decoder_rt_num_layer\": 1,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 16,\n \"encoder_ff\": 512,\n \"decoder_rt_ff\": 2048,\n \"decoder_int_ff\": 512,\n \"n_head\": 8,\n \"drop_rate\": 0.3688667410196246,\n \"lr\": 0.0018036063294034547,\n \"batch_size\": 1024\n },\n \"evaluated_params\": {\n \"batch_size\": 1024,\n \"decoder_int_ff\": 512,\n \"decoder_int_num_layer\": 1,\n \"decoder_rt_ff\": 2048,\n \"decoder_rt_num_layer\": 1,\n \"drop_rate\": 0.3688667410196246,\n \"embedding_dim\": 16,\n \"encoder_ff\": 512,\n \"encoder_num_layer\": 4,\n \"lr\": 0.0018036063294034547,\n \"n_head\": 8\n },\n \"experiment_tag\": \"27_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop_rate=0.3689,embedding_dim=16,encoder_ff=512,encoder_num_layer=4,lr=0.0018,n_head=8\",\n \"stopping_criterion\": {},\n \"_setup_default_resource\": true,\n \"_default_placement_group_factory\": \"80054e2e\",\n \"placement_group_factory\": \"800595ba000000000000008c237261792e74756e652e657865637574696f6e2e706c6163656d656e745f67726f757073948c15506c6163656d656e7447726f7570466163746f72799493942981947d94288c085f62756e646c6573945d947d94288c03435055944740540000000000008c034750559447401000000000000075618c155f686561645f62756e646c655f69735f656d70747994898c095f7374726174656779948c045041434b948c055f6172677394298c075f6b7761726773947d9475622e\",\n \"log_to_file\": [\n null,\n null\n ],\n \"max_failures\": 0,\n \"_default_result_or_future\": null,\n \"export_formats\": [],\n \"status\": \"TERMINATED\",\n \"relative_logdir\": \"train_model_18c11cbc_27_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop__2024-06-25_19-39-03\",\n \"trial_name_creator\": null,\n \"trial_dirname_creator\": null,\n \"custom_trial_name\": null,\n \"custom_dirname\": null,\n \"restore_path\": null,\n \"_restore_checkpoint_result\": null,\n \"_state_json\": null,\n \"results\": \"80054e2e\",\n \"extra_arg\": \"80054e2e\",\n \"_resources\": \"80054e2e\"\n}",
+ "{\n \"start_time\": 1719346234.668183,\n \"num_failures\": 0,\n \"num_failures_after_restore\": 0,\n \"error_filename\": null,\n \"pickled_error_filename\": null,\n \"last_result\": {\n \"loss\": 118.33789474006713,\n \"timestamp\": 1719357871,\n \"checkpoint_dir_name\": \"checkpoint_000019\",\n \"should_checkpoint\": true,\n \"done\": true,\n \"training_iteration\": 20,\n \"trial_id\": \"18c11cbc\",\n \"date\": \"2024-06-26_01-24-31\",\n \"time_this_iter_s\": 59.315247774124146,\n \"time_total_s\": 1226.7515268325806,\n \"pid\": 2850562,\n \"hostname\": \"r8i6n2\",\n \"node_ip\": \"10.159.28.60\",\n \"config\": {\n \"encoder_num_layer\": 4,\n \"decoder_rt_num_layer\": 1,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 16,\n \"encoder_ff\": 512,\n \"decoder_rt_ff\": 2048,\n \"decoder_int_ff\": 512,\n \"n_head\": 8,\n \"drop_rate\": 0.3688667410196246,\n \"lr\": 0.0018036063294034547,\n \"batch_size\": 1024\n },\n \"time_since_restore\": 606.2512559890747,\n \"iterations_since_restore\": 10,\n \"experiment_tag\": \"27_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop_rate=0.3689,embedding_dim=16,encoder_ff=512,encoder_num_layer=4,lr=0.0018,n_head=8\"\n },\n \"last_result_time\": 1719357871.9733982,\n \"metric_analysis\": {\n \"loss\": {\n \"max\": 266.66766681821326,\n \"min\": 118.33789474006713,\n \"avg\": 143.6234760126729,\n \"last\": 118.33789474006713,\n \"last-5-avg\": 120.82310969134953,\n \"last-10-avg\": 123.46259958913008\n },\n \"should_checkpoint\": {\n \"max\": true,\n \"min\": true,\n \"avg\": 1.0,\n \"last\": true,\n \"last-5-avg\": 1.0,\n \"last-10-avg\": 1.0\n },\n \"done\": {\n \"max\": false,\n \"min\": false,\n \"avg\": 0.0,\n \"last\": false,\n \"last-5-avg\": 0.0,\n \"last-10-avg\": 0.0\n },\n \"training_iteration\": {\n \"max\": 20,\n \"min\": 1,\n \"avg\": 10.5,\n \"last\": 20,\n \"last-5-avg\": 18.0,\n \"last-10-avg\": 15.5\n },\n \"time_this_iter_s\": {\n \"max\": 74.13704776763916,\n \"min\": 58.23622250556946,\n \"avg\": 61.33757634162903,\n \"last\": 59.315247774124146,\n \"last-5-avg\": 59.27494025230408,\n \"last-10-avg\": 60.62512559890747\n },\n \"time_total_s\": {\n \"max\": 1226.7515268325806,\n \"min\": 74.13704776763916,\n \"avg\": 654.1219100713729,\n \"last\": 1226.7515268325806,\n \"last-5-avg\": 1108.3144157409668,\n \"last-10-avg\": 960.0894476175308\n },\n \"time_since_restore\": {\n \"max\": 606.2512559890747,\n \"min\": 72.75415802001953,\n \"avg\": 265.71221474409106,\n \"last\": 606.2512559890747,\n \"last-5-avg\": 487.81414489746095,\n \"last-10-avg\": 339.58917677402496\n },\n \"iterations_since_restore\": {\n \"max\": 10,\n \"min\": 1,\n \"avg\": 4.25,\n \"last\": 10,\n \"last-5-avg\": 8.0,\n \"last-10-avg\": 5.5\n }\n },\n \"_n_steps\": [\n 5,\n 10\n ],\n \"metric_n_steps\": {\n \"loss\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"800595d6000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294288c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b007494624308d8ebf5c27a5c5e4094869452946807680d4308613018cc31075f4094869452946807680d4308241289b4d7615e4094869452946807680d4308040281d040ac5d4094869452946807680d43080a854211a0955d409486945294652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059535010000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294288c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b00749462430897cbe50acf00604094869452946807680d4308e2703894b34f604094869452946807680d43089fcfe793d4635f4094869452946807680d430843a150183a005f4094869452946807680d43080a854241979b5e4094869452946807680d4308d8ebf5c27a5c5e4094869452946807680d4308613018cc31075f4094869452946807680d4308241289b4d7615e4094869452946807680d4308040281d040ac5d4094869452946807680d43080a854211a0955d409486945294652e\"\n }\n },\n \"should_checkpoint\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059527000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294288888888888652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952c000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452942888888888888888888888652e\"\n }\n },\n \"done\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059527000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294288989898989652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952c000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452942889898989898989898989652e\"\n }\n },\n \"training_iteration\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952c000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294284b104b114b124b134b14652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059536000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294284b0b4b0c4b0d4b0e4b0f4b104b114b124b134b14652e\"\n }\n },\n \"time_this_iter_s\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005954f000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452942847404dcea81600000047404d92bc7200000047404d8f0b3e00000047404d972c6600000047404da85a0a000000652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005957c000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452942847405230442000000047404dac917e00000047404da1a58400000047404dadc47e00000047404d93af3200000047404dcea81600000047404d92bc7200000047404d8f0b3e00000047404d972c6600000047404da85a0a000000652e\"\n }\n },\n \"time_total_s\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005954f000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452942847408eefee3e800000474090648d02d0000047409151055cc000004740923dbebff000004740932b0190400000652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005957c000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a8694529428474085aa091200000047408784d229e000004740895eec8220000047408b39c8ca00000047408d1303bd20000047408eefee3e800000474090648d02d0000047409151055cc000004740923dbebff000004740932b0190400000652e\"\n }\n },\n \"time_since_restore\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005954f000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452942847407717db6100000047407aca32ef40000047407e7c1457000000474081177cf1e00000474082f20292800000652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005957c000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452942847405230442000000047406083466f800000474067ebafd080000047406f5720f00000004740735e065e40000047407717db6100000047407aca32ef40000047407e7c1457000000474081177cf1e00000474082f20292800000652e\"\n }\n },\n \"iterations_since_restore\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952c000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294284b064b074b084b094b0a652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059536000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294284b014b024b034b044b054b064b074b084b094b0a652e\"\n }\n }\n },\n \"checkpoint_manager\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059589490000000000008c267261792e747261696e2e5f696e7465726e616c2e636865636b706f696e745f6d616e61676572948c125f436865636b706f696e744d616e616765729493942981947d94288c125f636865636b706f696e745f636f6e666967948c097261792e747261696e948c10436865636b706f696e74436f6e6669679493942981947d94288c0b6e756d5f746f5f6b656570944e8c1a636865636b706f696e745f73636f72655f617474726962757465944e8c16636865636b706f696e745f73636f72655f6f72646572948c036d6178948c14636865636b706f696e745f6672657175656e6379944b008c11636865636b706f696e745f61745f656e6494898c1a5f636865636b706f696e745f6b6565705f616c6c5f72616e6b73948c0a44455052454341544544948c1f5f636865636b706f696e745f75706c6f61645f66726f6d5f776f726b65727394681275628c135f636865636b706f696e745f726573756c7473945d94288c1b7261792e747261696e2e5f696e7465726e616c2e73657373696f6e948c0f5f547261696e696e67526573756c749493942981947d94288c0a636865636b706f696e749468068c0a436865636b706f696e749493942981947d94288c0470617468948cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f31386331316362635f32375f62617463685f73697a653d313032342c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d323034382c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f5f323032342d30362d32355f31392d33392d30332f636865636b706f696e745f303030303030948c0a66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c055f75756964948c0475756964948c04555549449493942981947d948c03696e74948a1086df3ed6b22978aaa74314407424f10b736275628c076d657472696373947d94288c046c6f7373948c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b0074946243089bcd66c3aeaa704094869452948c0974696d657374616d70944a84247b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303030948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b018c08747269616c5f6964948c083138633131636263948c0464617465948c13323032342d30362d32355f32322d31312d3438948c1074696d655f746869735f697465725f739447405288c5640000008c0c74696d655f746f74616c5f739447405288c5640000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b048c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b108c0a656e636f6465725f6666944d00028c0d6465636f6465725f72745f6666944d00088c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b088c0964726f705f7261746594473fd79b833f50ba9a8c026c7294473f5d8cdf8cc701de8c0a62617463685f73697a65944d0004758c1274696d655f73696e63655f726573746f72659447405288c5640000008c18697465726174696f6e735f73696e63655f726573746f7265944b0175756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f31386331316362635f32375f62617463685f73697a653d313032342c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d323034382c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f5f323032342d30362d32355f31392d33392d30332f636865636b706f696e745f30303030303194682268257d946827897385945294682a682d2981947d9468308a11a000894b3dda7f875b4e024ae8b9f2eb007362756268317d94288c046c6f7373946836683c430853a9548a88fb674094869452948c0974696d657374616d70944abf247b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303031948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b028c08747269616c5f6964948c083138633131636263948c0464617465948c13323032342d30362d32355f32322d31322d3437948c1074696d655f746869735f697465725f739447404d912d960000008c0c74696d655f746f74616c5f7394474060a8ae178000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b048c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b108c0a656e636f6465725f6666944d00028c0d6465636f6465725f72745f6666944d00088c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b088c0964726f705f7261746594473fd79b833f50ba9a8c026c7294473f5d8cdf8cc701de8c0a62617463685f73697a65944d0004758c1274696d655f73696e63655f726573746f726594474060a8ae178000008c18697465726174696f6e735f73696e63655f726573746f7265944b0275756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f31386331316362635f32375f62617463685f73697a653d313032342c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d323034382c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f5f323032342d30362d32355f31392d33392d30332f636865636b706f696e745f30303030303294682268257d946827897385945294682a682d2981947d9468308a11ef4b086337798fab724707bc8d770398007362756268317d94288c046c6f7373946836683c4308180c06031189654094869452948c0974696d657374616d70944afb247b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303032948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b038c08747269616c5f6964948c083138633131636263948c0464617465948c13323032342d30362d32355f32322d31332d3437948c1074696d655f746869735f697465725f739447404dc3c7900000008c0c74696d655f746f74616c5f7394474068199ffb8000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b048c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b108c0a656e636f6465725f6666944d00028c0d6465636f6465725f72745f6666944d00088c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b088c0964726f705f7261746594473fd79b833f50ba9a8c026c7294473f5d8cdf8cc701de8c0a62617463685f73697a65944d0004758c1274696d655f73696e63655f726573746f726594474068199ffb8000008c18697465726174696f6e735f73696e63655f726573746f7265944b0375756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f31386331316362635f32375f62617463685f73697a653d313032342c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d323034382c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f5f323032342d30362d32355f31392d33392d30332f636865636b706f696e745f30303030303394682268257d946827897385945294682a682d2981947d9468308a10c0cac13ebaaa329b3049d64165106b4b7362756268317d94288c046c6f7373946836683c430828140af519bc644094869452948c0974696d657374616d70944a37257b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303033948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b048c08747269616c5f6964948c083138633131636263948c0464617465948c13323032342d30362d32355f32322d31342d3437948c1074696d655f746869735f697465725f739447404dd443c80000008c0c74696d655f746f74616c5f739447406f8eb0ed8000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b048c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b108c0a656e636f6465725f6666944d00028c0d6465636f6465725f72745f6666944d00088c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b088c0964726f705f7261746594473fd79b833f50ba9a8c026c7294473f5d8cdf8cc701de8c0a62617463685f73697a65944d0004758c1274696d655f73696e63655f726573746f72659447406f8eb0ed8000008c18697465726174696f6e735f73696e63655f726573746f7265944b0475756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f31386331316362635f32375f62617463685f73697a653d313032342c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d323034382c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f5f323032342d30362d32355f31392d33392d30332f636865636b706f696e745f30303030303494682268257d946827897385945294682a682d2981947d9468308a116d61ff6fc4f453a2604803a0673145a3007362756268317d94288c046c6f7373946836683c4308dced763babae624094869452948c0974696d657374616d70944a73257b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303034948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b058c08747269616c5f6964948c083138633131636263948c0464617465948c13323032342d30362d32355f32322d31352d3437948c1074696d655f746869735f697465725f739447404e16ee200000008c0c74696d655f746f74616c5f73944740738a363ac000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b048c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b108c0a656e636f6465725f6666944d00028c0d6465636f6465725f72745f6666944d00088c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b088c0964726f705f7261746594473fd79b833f50ba9a8c026c7294473f5d8cdf8cc701de8c0a62617463685f73697a65944d0004758c1274696d655f73696e63655f726573746f7265944740738a363ac000008c18697465726174696f6e735f73696e63655f726573746f7265944b0575756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f31386331316362635f32375f62617463685f73697a653d313032342c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d323034382c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f5f323032342d30362d32355f31392d33392d30332f636865636b706f696e745f30303030303594682268257d946827897385945294682a682d2981947d9468308a10f26bae67b6730cb51f4cc81fe59f67167362756268317d94288c046c6f7373946836683c4308369bcd3ec5d9614094869452948c0974696d657374616d70944a5e457b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303035948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b068c08747269616c5f6964948c083138633131636263948c0464617465948c13323032342d30362d32365f30302d33312d3538948c1074696d655f746869735f697465725f73944740525eacfc0000008c0c74696d655f746f74616c5f739447407821e179c000008c03706964944a027f2b008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b048c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b108c0a656e636f6465725f6666944d00028c0d6465636f6465725f72745f6666944d00088c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b088c0964726f705f7261746594473fd79b833f50ba9a8c026c7294473f5d8cdf8cc701de8c0a62617463685f73697a65944d0004758c1274696d655f73696e63655f726573746f7265944740525eacfc0000008c18697465726174696f6e735f73696e63655f726573746f7265944b0175756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f31386331316362635f32375f62617463685f73697a653d313032342c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d323034382c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f5f323032342d30362d32355f31392d33392d30332f636865636b706f696e745f30303030303694682268257d946827897385945294682a682d2981947d9468308a10c04ca19083252abc3d4f3c39852a7e727362756268317d94288c046c6f7373946836683c4308c26030d0cd73624094869452948c0974696d657374616d70944a98457b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303036948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b078c08747269616c5f6964948c083138633131636263948c0464617465948c13323032342d30362d32365f30302d33322d3536948c1074696d655f746869735f697465725f739447404d1e3c8a0000008c0c74696d655f746f74616c5f739447407bc5a90b0000008c03706964944a027f2b008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b048c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b108c0a656e636f6465725f6666944d00028c0d6465636f6465725f72745f6666944d00088c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b088c0964726f705f7261746594473fd79b833f50ba9a8c026c7294473f5d8cdf8cc701de8c0a62617463685f73697a65944d0004758c1274696d655f73696e63655f726573746f72659447406076e5a08000008c18697465726174696f6e735f73696e63655f726573746f7265944b0275756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f31386331316362635f32375f62617463685f73697a653d313032342c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d323034382c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f5f323032342d30362d32355f31392d33392d30332f636865636b706f696e745f30303030303794682268257d946827897385945294682a682d2981947d9468308a10aa1b24d1c873da9cde48140f766316467362756268317d94288c046c6f7373946836683c4308b1582c7eef1d614094869452948c0974696d657374616d70944ad3457b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303037948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b088c08747269616c5f6964948c083138633131636263948c0464617465948c13323032342d30362d32365f30302d33332d3535948c1074696d655f746869735f697465725f739447404d8a674c0000008c0c74696d655f746f74616c5f739447407f76f5f48000008c03706964944a027f2b008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b048c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b108c0a656e636f6465725f6666944d00028c0d6465636f6465725f72745f6666944d00088c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b088c0964726f705f7261746594473fd79b833f50ba9a8c026c7294473f5d8cdf8cc701de8c0a62617463685f73697a65944d0004758c1274696d655f73696e63655f726573746f726594474067d97f738000008c18697465726174696f6e735f73696e63655f726573746f7265944b0375756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f31386331316362635f32375f62617463685f73697a653d313032342c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d323034382c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f5f323032342d30362d32355f31392d33392d30332f636865636b706f696e745f30303030303894682268257d946827897385945294682a682d2981947d9468308a109112222aaa942e97504068bbe7cbc45d7362756268317d94288c046c6f7373946836683c4308e27038f485ac604094869452948c0974696d657374616d70944a0d467b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303038948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b098c08747269616c5f6964948c083138633131636263948c0464617465948c13323032342d30362d32365f30302d33342d3533948c1074696d655f746869735f697465725f739447404d32a3dc0000008c0c74696d655f746f74616c5f73944740818ea5380000008c03706964944a027f2b008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b048c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b108c0a656e636f6465725f6666944d00028c0d6465636f6465725f72745f6666944d00088c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b088c0964726f705f7261746594473fd79b833f50ba9a8c026c7294473f5d8cdf8cc701de8c0a62617463685f73697a65944d0004758c1274696d655f73696e63655f726573746f72659447406f26286a8000008c18697465726174696f6e735f73696e63655f726573746f7265944b0475756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f31386331316362635f32375f62617463685f73697a653d313032342c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d323034382c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f5f323032342d30362d32355f31392d33392d30332f636865636b706f696e745f30303030303994682268257d946827897385945294682a682d2981947d9468308a119785aa279b54399f9a40e1bc294f0ff3007362756268317d94288c046c6f7373946836683c43080683c160395e604094869452948c0974696d657374616d70944a48467b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303039948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b0a8c08747269616c5f6964948c083138633131636263948c0464617465948c13323032342d30362d32365f30302d33352d3532948c1074696d655f746869735f697465725f739447404d55b5600000008c0c74696d655f746f74616c5f739447408364008e0000008c03706964944a027f2b008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b048c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b108c0a656e636f6465725f6666944d00028c0d6465636f6465725f72745f6666944d00088c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b088c0964726f705f7261746594473fd79b833f50ba9a8c026c7294473f5d8cdf8cc701de8c0a62617463685f73697a65944d0004758c1274696d655f73696e63655f726573746f7265944740733dcae14000008c18697465726174696f6e735f73696e63655f726573746f7265944b0575756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f31386331316362635f32375f62617463685f73697a653d313032342c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d323034382c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f5f323032342d30362d32355f31392d33392d30332f636865636b706f696e745f30303030313094682268257d946827897385945294682a682d2981947d9468308a11df99e3d8c62135b4a14e96ee86bb889b007362756268317d94288c046c6f7373946836683c430897cbe50acf00604094869452948c0974696d657374616d70944a9a4f7b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303130948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b0b8c08747269616c5f6964948c083138633131636263948c0464617465948c13323032342d30362d32365f30312d31352d3338948c1074696d655f746869735f697465725f73944740523044200000008c0c74696d655f746f74616c5f7394474085aa09120000008c03706964944a027f2b008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b048c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b108c0a656e636f6465725f6666944d00028c0d6465636f6465725f72745f6666944d00088c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b088c0964726f705f7261746594473fd79b833f50ba9a8c026c7294473f5d8cdf8cc701de8c0a62617463685f73697a65944d0004758c1274696d655f73696e63655f726573746f7265944740523044200000008c18697465726174696f6e735f73696e63655f726573746f7265944b0175756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f31386331316362635f32375f62617463685f73697a653d313032342c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d323034382c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f5f323032342d30362d32355f31392d33392d30332f636865636b706f696e745f30303030313194682268257d946827897385945294682a682d2981947d9468308a11c09cc703c01eecb8294d5a9db423d3a1007362756268317d94288c046c6f7373946836683c4308e2703894b34f604094869452948c0974696d657374616d70944ad54f7b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303131948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b0c8c08747269616c5f6964948c083138633131636263948c0464617465948c13323032342d30362d32365f30312d31362d3337948c1074696d655f746869735f697465725f739447404dac917e0000008c0c74696d655f746f74616c5f739447408784d229e000008c03706964944a027f2b008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b048c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b108c0a656e636f6465725f6666944d00028c0d6465636f6465725f72745f6666944d00088c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b088c0964726f705f7261746594473fd79b833f50ba9a8c026c7294473f5d8cdf8cc701de8c0a62617463685f73697a65944d0004758c1274696d655f73696e63655f726573746f72659447406083466f8000008c18697465726174696f6e735f73696e63655f726573746f7265944b0275756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f31386331316362635f32375f62617463685f73697a653d313032342c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d323034382c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f5f323032342d30362d32355f31392d33392d30332f636865636b706f696e745f30303030313294682268257d946827897385945294682a682d2981947d9468308a11508686547d02c2817449c27b146b35b3007362756268317d94288c046c6f7373946836683c43089fcfe793d4635f4094869452948c0974696d657374616d70944a11507b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303132948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b0d8c08747269616c5f6964948c083138633131636263948c0464617465948c13323032342d30362d32365f30312d31372d3337948c1074696d655f746869735f697465725f739447404da1a5840000008c0c74696d655f746f74616c5f73944740895eec822000008c03706964944a027f2b008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b048c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b108c0a656e636f6465725f6666944d00028c0d6465636f6465725f72745f6666944d00088c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b088c0964726f705f7261746594473fd79b833f50ba9a8c026c7294473f5d8cdf8cc701de8c0a62617463685f73697a65944d0004758c1274696d655f73696e63655f726573746f726594474067ebafd08000008c18697465726174696f6e735f73696e63655f726573746f7265944b0375756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f31386331316362635f32375f62617463685f73697a653d313032342c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d323034382c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f5f323032342d30362d32355f31392d33392d30332f636865636b706f696e745f30303030313394682268257d946827897385945294682a682d2981947d9468308a116cbb66f88fca9b852f40838bb78628f4007362756268317d94288c046c6f7373946836683c430843a150183a005f4094869452948c0974696d657374616d70944a4c507b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303133948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b0e8c08747269616c5f6964948c083138633131636263948c0464617465948c13323032342d30362d32365f30312d31382d3336948c1074696d655f746869735f697465725f739447404dadc47e0000008c0c74696d655f746f74616c5f739447408b39c8ca0000008c03706964944a027f2b008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b048c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b108c0a656e636f6465725f6666944d00028c0d6465636f6465725f72745f6666944d00088c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b088c0964726f705f7261746594473fd79b833f50ba9a8c026c7294473f5d8cdf8cc701de8c0a62617463685f73697a65944d0004758c1274696d655f73696e63655f726573746f72659447406f5720f00000008c18697465726174696f6e735f73696e63655f726573746f7265944b0475756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f31386331316362635f32375f62617463685f73697a653d313032342c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d323034382c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f5f323032342d30362d32355f31392d33392d30332f636865636b706f696e745f30303030313494682268257d946827897385945294682a682d2981947d9468308a11613624b5cce4d2b99b4d25d68d3f1d8b007362756268317d94288c046c6f7373946836683c43080a854241979b5e4094869452948c0974696d657374616d70944a87507b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303134948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b0f8c08747269616c5f6964948c083138633131636263948c0464617465948c13323032342d30362d32365f30312d31392d3335948c1074696d655f746869735f697465725f739447404d93af320000008c0c74696d655f746f74616c5f739447408d1303bd2000008c03706964944a027f2b008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b048c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b108c0a656e636f6465725f6666944d00028c0d6465636f6465725f72745f6666944d00088c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b088c0964726f705f7261746594473fd79b833f50ba9a8c026c7294473f5d8cdf8cc701de8c0a62617463685f73697a65944d0004758c1274696d655f73696e63655f726573746f7265944740735e065e4000008c18697465726174696f6e735f73696e63655f726573746f7265944b0575756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f31386331316362635f32375f62617463685f73697a653d313032342c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d323034382c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f5f323032342d30362d32355f31392d33392d30332f636865636b706f696e745f30303030313594682268257d946827897385945294682a682d2981947d9468308a10938eba92221ca199b44bc0aef16e165b7362756268317d94288c046c6f7373946836683c4308d8ebf5c27a5c5e4094869452948c0974696d657374616d70944ac3507b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303135948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b108c08747269616c5f6964948c083138633131636263948c0464617465948c13323032342d30362d32365f30312d32302d3335948c1074696d655f746869735f697465725f739447404dcea8160000008c0c74696d655f746f74616c5f739447408eefee3e8000008c03706964944a027f2b008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b048c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b108c0a656e636f6465725f6666944d00028c0d6465636f6465725f72745f6666944d00088c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b088c0964726f705f7261746594473fd79b833f50ba9a8c026c7294473f5d8cdf8cc701de8c0a62617463685f73697a65944d0004758c1274696d655f73696e63655f726573746f72659447407717db610000008c18697465726174696f6e735f73696e63655f726573746f7265944b0675756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f31386331316362635f32375f62617463685f73697a653d313032342c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d323034382c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f5f323032342d30362d32355f31392d33392d30332f636865636b706f696e745f30303030313694682268257d946827897385945294682a682d2981947d9468308a10d774e2adcfce7da9b04371b86194f5047362756268317d94288c046c6f7373946836683c4308613018cc31075f4094869452948c0974696d657374616d70944afe507b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303136948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b118c08747269616c5f6964948c083138633131636263948c0464617465948c13323032342d30362d32365f30312d32312d3334948c1074696d655f746869735f697465725f739447404d92bc720000008c0c74696d655f746f74616c5f7394474090648d02d000008c03706964944a027f2b008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b048c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b108c0a656e636f6465725f6666944d00028c0d6465636f6465725f72745f6666944d00088c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b088c0964726f705f7261746594473fd79b833f50ba9a8c026c7294473f5d8cdf8cc701de8c0a62617463685f73697a65944d0004758c1274696d655f73696e63655f726573746f72659447407aca32ef4000008c18697465726174696f6e735f73696e63655f726573746f7265944b0775756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f31386331316362635f32375f62617463685f73697a653d313032342c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d323034382c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f5f323032342d30362d32355f31392d33392d30332f636865636b706f696e745f30303030313794682268257d946827897385945294682a682d2981947d9468308a103a52109f48a6a791e44e1b0991e471477362756268317d94288c046c6f7373946836683c4308241289b4d7615e4094869452948c0974696d657374616d70944a39517b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303137948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b128c08747269616c5f6964948c083138633131636263948c0464617465948c13323032342d30362d32365f30312d32322d3333948c1074696d655f746869735f697465725f739447404d8f0b3e0000008c0c74696d655f746f74616c5f739447409151055cc000008c03706964944a027f2b008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b048c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b108c0a656e636f6465725f6666944d00028c0d6465636f6465725f72745f6666944d00088c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b088c0964726f705f7261746594473fd79b833f50ba9a8c026c7294473f5d8cdf8cc701de8c0a62617463685f73697a65944d0004758c1274696d655f73696e63655f726573746f72659447407e7c14570000008c18697465726174696f6e735f73696e63655f726573746f7265944b0875756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f31386331316362635f32375f62617463685f73697a653d313032342c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d323034382c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f5f323032342d30362d32355f31392d33392d30332f636865636b706f696e745f30303030313894682268257d946827897385945294682a682d2981947d9468308a1015dc357caba8f7a13b4ebd0ba57182407362756268317d94288c046c6f7373946836683c4308040281d040ac5d4094869452948c0974696d657374616d70944a74517b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303138948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b138c08747269616c5f6964948c083138633131636263948c0464617465948c13323032342d30362d32365f30312d32332d3332948c1074696d655f746869735f697465725f739447404d972c660000008c0c74696d655f746f74616c5f73944740923dbebff000008c03706964944a027f2b008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b048c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b108c0a656e636f6465725f6666944d00028c0d6465636f6465725f72745f6666944d00088c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b088c0964726f705f7261746594473fd79b833f50ba9a8c026c7294473f5d8cdf8cc701de8c0a62617463685f73697a65944d0004758c1274696d655f73696e63655f726573746f726594474081177cf1e000008c18697465726174696f6e735f73696e63655f726573746f7265944b0975756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f31386331316362635f32375f62617463685f73697a653d313032342c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d323034382c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f5f323032342d30362d32355f31392d33392d30332f636865636b706f696e745f30303030313994682268257d946827897385945294682a682d2981947d9468308a1147920dc8f570e8b19e4e6779415f42a8007362756268317d94288c046c6f7373946836683c43080a854211a0955d4094869452948c0974696d657374616d70944aaf517b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303139948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b148c08747269616c5f6964948c083138633131636263948c0464617465948c13323032342d30362d32365f30312d32342d3331948c1074696d655f746869735f697465725f739447404da85a0a0000008c0c74696d655f746f74616c5f73944740932b01904000008c03706964944a027f2b008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b048c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b108c0a656e636f6465725f6666944d00028c0d6465636f6465725f72745f6666944d00088c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b088c0964726f705f7261746594473fd79b833f50ba9a8c026c7294473f5d8cdf8cc701de8c0a62617463685f73697a65944d0004758c1274696d655f73696e63655f726573746f726594474082f202928000008c18697465726174696f6e735f73696e63655f726573746f7265944b0a757562658c195f6c61746573745f636865636b706f696e745f726573756c74946ab003000075622e\"\n }\n}"
+ ],
+ [
+ "{\n \"stub\": false,\n \"trainable_name\": \"train_model\",\n \"trial_id\": \"3ab2f301\",\n \"storage\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059572040000000000008c1b7261792e747261696e2e5f696e7465726e616c2e73746f72616765948c0e53746f72616765436f6e746578749493942981947d94288c12637573746f6d5f66735f70726f766964656494898c1273746f726167655f6c6f63616c5f70617468948c2b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c7473948c136578706572696d656e745f6469725f6e616d65948c0f746573745f6578706572696d656e74948c0e747269616c5f6469725f6e616d65948c96747261696e5f6d6f64656c5f33616232663330315f32385f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d323034382c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f5f323032342d30362d32355f32322d31302d3334948c1863757272656e745f636865636b706f696e745f696e646578944b138c0b73796e635f636f6e666967948c097261792e747261696e948c0a53796e63436f6e6669679493942981947d94288c0b73796e635f706572696f64944d2c018c0c73796e635f74696d656f7574944d08078c0e73796e635f61727469666163747394898c1c73796e635f6172746966616374735f6f6e5f636865636b706f696e7494888c0a75706c6f61645f646972948c0a44455052454341544544948c0673796e6365729468188c1273796e635f6f6e5f636865636b706f696e7494681875628c1273746f726167655f66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c0f73746f726167655f66735f70617468948c452f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f7465737494681968008c115f46696c6573797374656d53796e6365729493942981947d9428681b682268134d2c0168144d08078c116c6173745f73796e635f75705f74696d65944741d99ec8d7e1c03a8c136c6173745f73796e635f646f776e5f74696d659447fff00000000000008c0d5f73796e635f70726f63657373944e8c0c5f63757272656e745f636d649468008c125f75706c6f61645f746f5f66735f706174689493947d94288c0a6c6f63616c5f70617468948c3b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c74732f746573745f6578706572696d656e74948c0266739468228c0766735f70617468948c552f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e74948c076578636c756465945d948c0e2a2f636865636b706f696e745f2a9461758694756275622e\"\n },\n \"config\": {\n \"encoder_num_layer\": 2,\n \"decoder_rt_num_layer\": 1,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 16,\n \"encoder_ff\": 2048,\n \"decoder_rt_ff\": 2048,\n \"decoder_int_ff\": 512,\n \"n_head\": 1,\n \"drop_rate\": 0.034481700426631456,\n \"lr\": 0.0010186128360476897,\n \"batch_size\": 2048\n },\n \"_Trial__unresolved_config\": {\n \"encoder_num_layer\": 2,\n \"decoder_rt_num_layer\": 1,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 16,\n \"encoder_ff\": 2048,\n \"decoder_rt_ff\": 2048,\n \"decoder_int_ff\": 512,\n \"n_head\": 1,\n \"drop_rate\": 0.034481700426631456,\n \"lr\": 0.0010186128360476897,\n \"batch_size\": 2048\n },\n \"evaluated_params\": {\n \"batch_size\": 2048,\n \"decoder_int_ff\": 512,\n \"decoder_int_num_layer\": 1,\n \"decoder_rt_ff\": 2048,\n \"decoder_rt_num_layer\": 1,\n \"drop_rate\": 0.034481700426631456,\n \"embedding_dim\": 16,\n \"encoder_ff\": 2048,\n \"encoder_num_layer\": 2,\n \"lr\": 0.0010186128360476897,\n \"n_head\": 1\n },\n \"experiment_tag\": \"28_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop_rate=0.0345,embedding_dim=16,encoder_ff=2048,encoder_num_layer=2,lr=0.0010,n_head=1\",\n \"stopping_criterion\": {},\n \"_setup_default_resource\": true,\n \"_default_placement_group_factory\": \"80054e2e\",\n \"placement_group_factory\": \"800595ba000000000000008c237261792e74756e652e657865637574696f6e2e706c6163656d656e745f67726f757073948c15506c6163656d656e7447726f7570466163746f72799493942981947d94288c085f62756e646c6573945d947d94288c03435055944740540000000000008c034750559447401000000000000075618c155f686561645f62756e646c655f69735f656d70747994898c095f7374726174656779948c045041434b948c055f6172677394298c075f6b7761726773947d9475622e\",\n \"log_to_file\": [\n null,\n null\n ],\n \"max_failures\": 0,\n \"_default_result_or_future\": null,\n \"export_formats\": [],\n \"status\": \"TERMINATED\",\n \"relative_logdir\": \"train_model_3ab2f301_28_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop__2024-06-25_22-10-34\",\n \"trial_name_creator\": null,\n \"trial_dirname_creator\": null,\n \"custom_trial_name\": null,\n \"custom_dirname\": null,\n \"restore_path\": null,\n \"_restore_checkpoint_result\": null,\n \"_state_json\": null,\n \"results\": \"80054e2e\",\n \"extra_arg\": \"80054e2e\",\n \"_resources\": \"80054e2e\"\n}",
+ "{\n \"start_time\": 1719346547.487432,\n \"num_failures\": 0,\n \"num_failures_after_restore\": 0,\n \"error_filename\": null,\n \"pickled_error_filename\": null,\n \"last_result\": {\n \"loss\": 84.81797225832001,\n \"timestamp\": 1719358398,\n \"checkpoint_dir_name\": \"checkpoint_000019\",\n \"should_checkpoint\": true,\n \"done\": true,\n \"training_iteration\": 20,\n \"trial_id\": \"3ab2f301\",\n \"date\": \"2024-06-26_01-33-18\",\n \"time_this_iter_s\": 51.372178077697754,\n \"time_total_s\": 1072.982276916504,\n \"pid\": 2850562,\n \"hostname\": \"r8i6n2\",\n \"node_ip\": \"10.159.28.60\",\n \"config\": {\n \"encoder_num_layer\": 2,\n \"decoder_rt_num_layer\": 1,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 16,\n \"encoder_ff\": 2048,\n \"decoder_rt_ff\": 2048,\n \"decoder_int_ff\": 512,\n \"n_head\": 1,\n \"drop_rate\": 0.034481700426631456,\n \"lr\": 0.0010186128360476897,\n \"batch_size\": 2048\n },\n \"time_since_restore\": 525.9760255813599,\n \"iterations_since_restore\": 10,\n \"experiment_tag\": \"28_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop_rate=0.0345,embedding_dim=16,encoder_ff=2048,encoder_num_layer=2,lr=0.0010,n_head=1\"\n },\n \"last_result_time\": 1719358398.1699922,\n \"metric_analysis\": {\n \"loss\": {\n \"max\": 217.46355960875985,\n \"min\": 84.81797225832001,\n \"avg\": 111.01648353816957,\n \"last\": 84.81797225832001,\n \"last-5-avg\": 88.37300739438516,\n \"last-10-avg\": 90.88470410925197\n },\n \"should_checkpoint\": {\n \"max\": true,\n \"min\": true,\n \"avg\": 1.0,\n \"last\": true,\n \"last-5-avg\": 1.0,\n \"last-10-avg\": 1.0\n },\n \"done\": {\n \"max\": false,\n \"min\": false,\n \"avg\": 0.0,\n \"last\": false,\n \"last-5-avg\": 0.0,\n \"last-10-avg\": 0.0\n },\n \"training_iteration\": {\n \"max\": 20,\n \"min\": 1,\n \"avg\": 10.5,\n \"last\": 20,\n \"last-5-avg\": 18.0,\n \"last-10-avg\": 15.5\n },\n \"time_this_iter_s\": {\n \"max\": 65.41611504554749,\n \"min\": 50.784724712371826,\n \"avg\": 53.6491138458252,\n \"last\": 51.372178077697754,\n \"last-5-avg\": 51.40114240646362,\n \"last-10-avg\": 52.59760255813599\n },\n \"time_total_s\": {\n \"max\": 1072.982276916504,\n \"min\": 65.41611504554749,\n \"avg\": 574.7836294054985,\n \"last\": 1072.982276916504,\n \"last-5-avg\": 970.4215782165527,\n \"last-10-avg\": 842.11748483181\n },\n \"time_since_restore\": {\n \"max\": 525.9760255813599,\n \"min\": 64.502512216568,\n \"avg\": 232.20214306116105,\n \"last\": 525.9760255813599,\n \"last-5-avg\": 423.4153268814087,\n \"last-10-avg\": 295.11123349666593\n },\n \"iterations_since_restore\": {\n \"max\": 10,\n \"min\": 1,\n \"avg\": 4.25,\n \"last\": 10,\n \"last-5-avg\": 8.0,\n \"last-10-avg\": 5.5\n }\n },\n \"_n_steps\": [\n 5,\n 10\n ],\n \"metric_n_steps\": {\n \"loss\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"800595d6000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294288c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b007494624308abd56af55780574094869452946807680d4308f67abd5e1ade554094869452946807680d4308cce572191604564094869452946807680d430871381cae7ae0554094869452946807680d430843a150a8593455409486945294652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059535010000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294288c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b007494624308d0e7f3b9ce9f584094869452946807680d4308f2783c7ebf5f574094869452946807680d4308fcfd7e1f86b2574094869452946807680d43083e9fcfa7c9dd564094869452946807680d430843a15028fb2e564094869452946807680d4308abd56af55780574094869452946807680d4308f67abd5e1ade554094869452946807680d4308cce572191604564094869452946807680d430871381cae7ae0554094869452946807680d430843a150a8593455409486945294652e\"\n }\n },\n \"should_checkpoint\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059527000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294288888888888652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952c000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452942888888888888888888888652e\"\n }\n },\n \"done\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059527000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294288989898989652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952c000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452942889898989898989898989652e\"\n }\n },\n \"training_iteration\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952c000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294284b104b114b124b134b14652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059536000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294284b0b4b0c4b0d4b0e4b0f4b104b114b124b134b14652e\"\n }\n },\n \"time_this_iter_s\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005954f000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b058694529428474049ccd45e000000474049ebdd980000004740498cc3e20000004740498ba1cc000000474049afa388000000652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005957c000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294284740502029290000004740496e97f2000000474049d276600000004740499660bc0000004740496471dc000000474049ccd45e000000474049ebdd980000004740498cc3e20000004740498ba1cc000000474049afa388000000652e\"\n }\n },\n \"time_total_s\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005954f000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452942847408b1c9d4720000047408cbb5b20a0000047408e54275ec0000047408fece17b800000474090c3edda000000652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005957c000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294284740831c11f2a00000474084b2fb71c000004740865022d7c00000474087e988e38000004740897fd00140000047408b1c9d4720000047408cbb5b20a0000047408e54275ec0000047408fece17b800000474090c3edda000000652e\"\n }\n },\n \"time_since_restore\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005954f000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294284740740920f3400000474077469ca640000047407a78352280000047407da9a95c0000004740806fcee6800000652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005957c000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452942847405020292900000047405cd77522000000474064e0582900000047406b45f058000000474070cf86678000004740740920f3400000474077469ca640000047407a78352280000047407da9a95c0000004740806fcee6800000652e\"\n }\n },\n \"iterations_since_restore\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952c000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294284b064b074b084b094b0a652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059536000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294284b014b024b034b044b054b064b074b084b094b0a652e\"\n }\n }\n },\n \"checkpoint_manager\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059587490000000000008c267261792e747261696e2e5f696e7465726e616c2e636865636b706f696e745f6d616e61676572948c125f436865636b706f696e744d616e616765729493942981947d94288c125f636865636b706f696e745f636f6e666967948c097261792e747261696e948c10436865636b706f696e74436f6e6669679493942981947d94288c0b6e756d5f746f5f6b656570944e8c1a636865636b706f696e745f73636f72655f617474726962757465944e8c16636865636b706f696e745f73636f72655f6f72646572948c036d6178948c14636865636b706f696e745f6672657175656e6379944b008c11636865636b706f696e745f61745f656e6494898c1a5f636865636b706f696e745f6b6565705f616c6c5f72616e6b73948c0a44455052454341544544948c1f5f636865636b706f696e745f75706c6f61645f66726f6d5f776f726b65727394681275628c135f636865636b706f696e745f726573756c7473945d94288c1b7261792e747261696e2e5f696e7465726e616c2e73657373696f6e948c0f5f547261696e696e67526573756c749493942981947d94288c0a636865636b706f696e749468068c0a436865636b706f696e749493942981947d94288c0470617468948cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f33616232663330315f32385f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d323034382c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f5f323032342d30362d32355f32322d31302d33342f636865636b706f696e745f303030303030948c0a66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c055f75756964948c0475756964948c04555549449493942981947d948c03696e74948a10e8ed94ab740a45ae324a3111f342bd41736275628c076d657472696373947d94288c046c6f7373948c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b007494624308d8ebf57ad52e6b4094869452948c0974696d657374616d70944ab4257b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303030948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b018c08747269616c5f6964948c083361623266333031948c0464617465948c13323032342d30362d32355f32322d31362d3532948c1074696d655f746869735f697465725f73944740505aa1a10000008c0c74696d655f746f74616c5f73944740505aa1a10000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b028c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b108c0a656e636f6465725f6666944d00088c0d6465636f6465725f72745f6666944d00088c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b018c0964726f705f7261746594473fa1a795df4923d08c026c7294473f50b05f345becb88c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f7265944740505aa1a10000008c18697465726174696f6e735f73696e63655f726573746f7265944b0175756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f33616232663330315f32385f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d323034382c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f5f323032342d30362d32355f32322d31302d33342f636865636b706f696e745f30303030303194682268257d946827897385945294682a682d2981947d9468308a102fede7cdb9c96a8d2144477b65b563337362756268317d94288c046c6f7373946836683c43086130184c0a7b644094869452948c0974696d657374616d70944ae8257b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303031948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b028c08747269616c5f6964948c083361623266333031948c0464617465948c13323032342d30362d32355f32322d31372d3435948c1074696d655f746869735f697465725f739447404a336ca00000008c0c74696d655f746f74616c5f739447405d7457f10000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b028c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b108c0a656e636f6465725f6666944d00088c0d6465636f6465725f72745f6666944d00088c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b018c0964726f705f7261746594473fa1a795df4923d08c026c7294473f50b05f345becb88c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f72659447405d7457f10000008c18697465726174696f6e735f73696e63655f726573746f7265944b0275756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f33616232663330315f32385f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d323034382c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f5f323032342d30362d32355f32322d31302d33342f636865636b706f696e745f30303030303294682268257d946827897385945294682a682d2981947d9468308a1140a5e00664cf47862e44f66032ca9e9c007362756268317d94288c046c6f7373946836683c430893c964529e70614094869452948c0974696d657374616d70944a1d267b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303032948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b038c08747269616c5f6964948c083361623266333031948c0464617465948c13323032342d30362d32355f32322d31382d3337948c1074696d655f746869735f697465725f739447404a1a97f60000008c0c74696d655f746f74616c5f739447406540d1f60000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b028c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b108c0a656e636f6465725f6666944d00088c0d6465636f6465725f72745f6666944d00088c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b018c0964726f705f7261746594473fa1a795df4923d08c026c7294473f50b05f345becb88c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f72659447406540d1f60000008c18697465726174696f6e735f73696e63655f726573746f7265944b0375756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f33616232663330315f32385f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d323034382c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f5f323032342d30362d32355f32322d31302d33342f636865636b706f696e745f30303030303394682268257d946827897385945294682a682d2981947d9468308a1058db2733e21f6db671481e20128b04797362756268317d94288c046c6f7373946836683c43087fbfdf4ffd745f4094869452948c0974696d657374616d70944a52267b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303033948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b048c08747269616c5f6964948c083361623266333031948c0464617465948c13323032342d30362d32355f32322d31392d3330948c1074696d655f746869735f697465725f739447404a8bc79c0000008c0c74696d655f746f74616c5f739447406be3c3dd0000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b028c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b108c0a656e636f6465725f6666944d00088c0d6465636f6465725f72745f6666944d00088c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b018c0964726f705f7261746594473fa1a795df4923d08c026c7294473f50b05f345becb88c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f72659447406be3c3dd0000008c18697465726174696f6e735f73696e63655f726573746f7265944b0475756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f33616232663330315f32385f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d323034382c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f5f323032342d30362d32355f32322d31302d33342f636865636b706f696e745f30303030303494682268257d946827897385945294682a682d2981947d9468308a110132d6caf4940790b5401c1c6f6270bb007362756268317d94288c046c6f7373946836683c43088bc562f16a5d5d4094869452948c0974696d657374616d70944a87267b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303034948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b058c08747269616c5f6964948c083361623266333031948c0464617465948c13323032342d30362d32355f32322d32302d3233948c1074696d655f746869735f697465725f739447404a990f700000008c0c74696d655f746f74616c5f73944740714503dc8000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b028c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b108c0a656e636f6465725f6666944d00088c0d6465636f6465725f72745f6666944d00088c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b018c0964726f705f7261746594473fa1a795df4923d08c026c7294473f50b05f345becb88c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f7265944740714503dc8000008c18697465726174696f6e735f73696e63655f726573746f7265944b0575756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f33616232663330315f32385f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d323034382c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f5f323032342d30362d32355f32322d31302d33342f636865636b706f696e745f30303030303594682268257d946827897385945294682a682d2981947d9468308a113c614213f41e10ab864c5182885dc6f9007362756268317d94288c046c6f7373946836683c4308894422f1949e5c4094869452948c0974696d657374616d70944a89467b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303035948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b068c08747269616c5f6964948c083361623266333031948c0464617465948c13323032342d30362d32365f30302d33362d3537948c1074696d655f746869735f697465725f73944740505018840000008c0c74696d655f746f74616c5f73944740755909fd8000008c03706964944a027f2b008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b028c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b108c0a656e636f6465725f6666944d00088c0d6465636f6465725f72745f6666944d00088c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b018c0964726f705f7261746594473fa1a795df4923d08c026c7294473f50b05f345becb88c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f7265944740505018840000008c18697465726174696f6e735f73696e63655f726573746f7265944b0175756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f33616232663330315f32385f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d323034382c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f5f323032342d30362d32355f32322d31302d33342f636865636b706f696e745f30303030303694682268257d946827897385945294682a682d2981947d9468308a11b0b7fbae3a25b0b0e1426175363775cf007362756268317d94288c046c6f7373946836683c430893c964d2f24f5b4094869452948c0974696d657374616d70944abd467b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303036948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b078c08747269616c5f6964948c083361623266333031948c0464617465948c13323032342d30362d32365f30302d33372d3439948c1074696d655f746869735f697465725f7394474049bf61580000008c0c74696d655f746f74616c5f739447407890f6288000008c03706964944a027f2b008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b028c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b108c0a656e636f6465725f6666944d00088c0d6465636f6465725f72745f6666944d00088c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b018c0964726f705f7261746594473fa1a795df4923d08c026c7294473f50b05f345becb88c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f72659447405d2fc9300000008c18697465726174696f6e735f73696e63655f726573746f7265944b0275756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f33616232663330315f32385f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d323034382c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f5f323032342d30362d32355f32322d31302d33342f636865636b706f696e745f30303030303794682268257d946827897385945294682a682d2981947d9468308a110ee569d210e17ba781414ebdd6974df0007362756268317d94288c046c6f7373946836683c430843a15068e24c5a4094869452948c0974696d657374616d70944af0467b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303037948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b088c08747269616c5f6964948c083361623266333031948c0464617465948c13323032342d30362d32365f30302d33382d3430948c1074696d655f746869735f697465725f7394474049bf9ac40000008c0c74696d655f746f74616c5f739447407bc8e9810000008c03706964944a027f2b008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b028c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b108c0a656e636f6465725f6666944d00088c0d6465636f6465725f72745f6666944d00088c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b018c0964726f705f7261746594473fa1a795df4923d08c026c7294473f50b05f345becb88c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f72659447406507cb490000008c18697465726174696f6e735f73696e63655f726573746f7265944b0375756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f33616232663330315f32385f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d323034382c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f5f323032342d30362d32355f32322d31302d33342f636865636b706f696e745f30303030303894682268257d946827897385945294682a682d2981947d9468308a10da33af97d2baa297f64dc751c8d7d27a7362756268317d94288c046c6f7373946836683c4308100804a23aa55d4094869452948c0974696d657374616d70944a23477b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303038948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b098c08747269616c5f6964948c083361623266333031948c0464617465948c13323032342d30362d32365f30302d33392d3331948c1074696d655f746869735f697465725f739447404987e6840000008c0c74696d655f746f74616c5f739447407ef9e6518000008c03706964944a027f2b008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b028c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b108c0a656e636f6465725f6666944d00088c0d6465636f6465725f72745f6666944d00088c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b018c0964726f705f7261746594473fa1a795df4923d08c026c7294473f50b05f345becb88c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f72659447406b69c4ea0000008c18697465726174696f6e735f73696e63655f726573746f7265944b0475756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f33616232663330315f32385f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d323034382c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f5f323032342d30362d32355f32322d31302d33342f636865636b706f696e745f30303030303994682268257d946827897385945294682a682d2981947d9468308a1018334bbd4a62dc974f4049e30037562e7362756268317d94288c046c6f7373946836683c43082c168b25daf6584094869452948c0974696d657374616d70944a57477b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303039948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b0a8c08747269616c5f6964948c083361623266333031948c0464617465948c13323032342d30362d32365f30302d34302d3233948c1074696d655f746869735f697465725f7394474049b19a4c0000008c0c74696d655f746f74616c5f7394474081180ccd8000008c03706964944a027f2b008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b028c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b108c0a656e636f6465725f6666944d00088c0d6465636f6465725f72745f6666944d00088c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b018c0964726f705f7261746594473fa1a795df4923d08c026c7294473f50b05f345becb88c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f726594474070eb15be8000008c18697465726174696f6e735f73696e63655f726573746f7265944b0575756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f33616232663330315f32385f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d323034382c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f5f323032342d30362d32355f32322d31302d33342f636865636b706f696e745f30303030313094682268257d946827897385945294682a682d2981947d9468308a10d94fee9fcf871d9a2d4bc8368f829f327362756268317d94288c046c6f7373946836683c4308d0e7f3b9ce9f584094869452948c0974696d657374616d70944af0517b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303130948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b0b8c08747269616c5f6964948c083361623266333031948c0464617465948c13323032342d30362d32365f30312d32352d3336948c1074696d655f746869735f697465725f73944740502029290000008c0c74696d655f746f74616c5f73944740831c11f2a000008c03706964944a027f2b008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b028c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b108c0a656e636f6465725f6666944d00088c0d6465636f6465725f72745f6666944d00088c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b018c0964726f705f7261746594473fa1a795df4923d08c026c7294473f50b05f345becb88c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f7265944740502029290000008c18697465726174696f6e735f73696e63655f726573746f7265944b0175756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f33616232663330315f32385f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d323034382c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f5f323032342d30362d32355f32322d31302d33342f636865636b706f696e745f30303030313194682268257d946827897385945294682a682d2981947d9468308a101d55dd00b02e91ba9f4d9dc2d447b2387362756268317d94288c046c6f7373946836683c4308f2783c7ebf5f574094869452948c0974696d657374616d70944a23527b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303131948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b0c8c08747269616c5f6964948c083361623266333031948c0464617465948c13323032342d30362d32365f30312d32362d3237948c1074696d655f746869735f697465725f73944740496e97f20000008c0c74696d655f746f74616c5f7394474084b2fb71c000008c03706964944a027f2b008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b028c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b108c0a656e636f6465725f6666944d00088c0d6465636f6465725f72745f6666944d00088c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b018c0964726f705f7261746594473fa1a795df4923d08c026c7294473f50b05f345becb88c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f72659447405cd775220000008c18697465726174696f6e735f73696e63655f726573746f7265944b0275756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f33616232663330315f32385f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d323034382c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f5f323032342d30362d32355f32322d31302d33342f636865636b706f696e745f30303030313294682268257d946827897385945294682a682d2981947d9468308a108b607ff288319d88e9491b307a617c547362756268317d94288c046c6f7373946836683c4308fcfd7e1f86b2574094869452948c0974696d657374616d70944a57527b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303132948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b0d8c08747269616c5f6964948c083361623266333031948c0464617465948c13323032342d30362d32365f30312d32372d3139948c1074696d655f746869735f697465725f7394474049d276600000008c0c74696d655f746f74616c5f73944740865022d7c000008c03706964944a027f2b008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b028c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b108c0a656e636f6465725f6666944d00088c0d6465636f6465725f72745f6666944d00088c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b018c0964726f705f7261746594473fa1a795df4923d08c026c7294473f50b05f345becb88c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f726594474064e058290000008c18697465726174696f6e735f73696e63655f726573746f7265944b0375756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f33616232663330315f32385f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d323034382c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f5f323032342d30362d32355f32322d31302d33342f636865636b706f696e745f30303030313394682268257d946827897385945294682a682d2981947d9468308a10f8312aa627f59fa458496df894d64f087362756268317d94288c046c6f7373946836683c43083e9fcfa7c9dd564094869452948c0974696d657374616d70944a8a527b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303133948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b0e8c08747269616c5f6964948c083361623266333031948c0464617465948c13323032342d30362d32365f30312d32382d3130948c1074696d655f746869735f697465725f73944740499660bc0000008c0c74696d655f746f74616c5f7394474087e988e38000008c03706964944a027f2b008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b028c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b108c0a656e636f6465725f6666944d00088c0d6465636f6465725f72745f6666944d00088c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b018c0964726f705f7261746594473fa1a795df4923d08c026c7294473f50b05f345becb88c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f72659447406b45f0580000008c18697465726174696f6e735f73696e63655f726573746f7265944b0475756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f33616232663330315f32385f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d323034382c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f5f323032342d30362d32355f32322d31302d33342f636865636b706f696e745f30303030313494682268257d946827897385945294682a682d2981947d9468308a101c4d5ed7924e63920a4809ae3bcd0d197362756268317d94288c046c6f7373946836683c430843a15028fb2e564094869452948c0974696d657374616d70944abd527b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303134948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b0f8c08747269616c5f6964948c083361623266333031948c0464617465948c13323032342d30362d32365f30312d32392d3031948c1074696d655f746869735f697465725f73944740496471dc0000008c0c74696d655f746f74616c5f73944740897fd0014000008c03706964944a027f2b008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b028c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b108c0a656e636f6465725f6666944d00088c0d6465636f6465725f72745f6666944d00088c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b018c0964726f705f7261746594473fa1a795df4923d08c026c7294473f50b05f345becb88c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f726594474070cf86678000008c18697465726174696f6e735f73696e63655f726573746f7265944b0575756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f33616232663330315f32385f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d323034382c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f5f323032342d30362d32355f32322d31302d33342f636865636b706f696e745f30303030313594682268257d946827897385945294682a682d2981947d9468308a109380808904b901aafa42edbf9fe1513c7362756268317d94288c046c6f7373946836683c4308abd56af55780574094869452948c0974696d657374616d70944af0527b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303135948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b108c08747269616c5f6964948c083361623266333031948c0464617465948c13323032342d30362d32365f30312d32392d3532948c1074696d655f746869735f697465725f7394474049ccd45e0000008c0c74696d655f746f74616c5f739447408b1c9d472000008c03706964944a027f2b008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b028c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b108c0a656e636f6465725f6666944d00088c0d6465636f6465725f72745f6666944d00088c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b018c0964726f705f7261746594473fa1a795df4923d08c026c7294473f50b05f345becb88c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f7265944740740920f34000008c18697465726174696f6e735f73696e63655f726573746f7265944b0675756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f33616232663330315f32385f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d323034382c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f5f323032342d30362d32355f32322d31302d33342f636865636b706f696e745f30303030313694682268257d946827897385945294682a682d2981947d9468308a111c21b0db9c483bba1245d7f6ed0624d4007362756268317d94288c046c6f7373946836683c4308f67abd5e1ade554094869452948c0974696d657374616d70944a24537b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303136948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b118c08747269616c5f6964948c083361623266333031948c0464617465948c13323032342d30362d32365f30312d33302d3434948c1074696d655f746869735f697465725f7394474049ebdd980000008c0c74696d655f746f74616c5f739447408cbb5b20a000008c03706964944a027f2b008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b028c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b108c0a656e636f6465725f6666944d00088c0d6465636f6465725f72745f6666944d00088c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b018c0964726f705f7261746594473fa1a795df4923d08c026c7294473f50b05f345becb88c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f726594474077469ca64000008c18697465726174696f6e735f73696e63655f726573746f7265944b0775756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f33616232663330315f32385f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d323034382c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f5f323032342d30362d32355f32322d31302d33342f636865636b706f696e745f30303030313794682268257d946827897385945294682a682d2981947d9468308a10696a9c34166e88864a42e14741b4af2e7362756268317d94288c046c6f7373946836683c4308cce572191604564094869452948c0974696d657374616d70944a57537b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303137948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b128c08747269616c5f6964948c083361623266333031948c0464617465948c13323032342d30362d32365f30312d33312d3335948c1074696d655f746869735f697465725f73944740498cc3e20000008c0c74696d655f746f74616c5f739447408e54275ec000008c03706964944a027f2b008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b028c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b108c0a656e636f6465725f6666944d00088c0d6465636f6465725f72745f6666944d00088c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b018c0964726f705f7261746594473fa1a795df4923d08c026c7294473f50b05f345becb88c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f72659447407a7835228000008c18697465726174696f6e735f73696e63655f726573746f7265944b0875756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f33616232663330315f32385f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d323034382c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f5f323032342d30362d32355f32322d31302d33342f636865636b706f696e745f30303030313894682268257d946827897385945294682a682d2981947d9468308a11ab454fccca6de7be48460b4ecd666cfa007362756268317d94288c046c6f7373946836683c430871381cae7ae0554094869452948c0974696d657374616d70944a8a537b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303138948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b138c08747269616c5f6964948c083361623266333031948c0464617465948c13323032342d30362d32365f30312d33322d3236948c1074696d655f746869735f697465725f73944740498ba1cc0000008c0c74696d655f746f74616c5f739447408fece17b8000008c03706964944a027f2b008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b028c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b108c0a656e636f6465725f6666944d00088c0d6465636f6465725f72745f6666944d00088c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b018c0964726f705f7261746594473fa1a795df4923d08c026c7294473f50b05f345becb88c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f72659447407da9a95c0000008c18697465726174696f6e735f73696e63655f726573746f7265944b0975756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f33616232663330315f32385f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d323034382c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f5f323032342d30362d32355f32322d31302d33342f636865636b706f696e745f30303030313994682268257d946827897385945294682a682d2981947d9468308a11c2289f66b1a5fb8e3745534de55c89d4007362756268317d94288c046c6f7373946836683c430843a150a85934554094869452948c0974696d657374616d70944abe537b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303139948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b148c08747269616c5f6964948c083361623266333031948c0464617465948c13323032342d30362d32365f30312d33332d3138948c1074696d655f746869735f697465725f7394474049afa3880000008c0c74696d655f746f74616c5f7394474090c3edda0000008c03706964944a027f2b008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b028c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b108c0a656e636f6465725f6666944d00088c0d6465636f6465725f72745f6666944d00088c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b018c0964726f705f7261746594473fa1a795df4923d08c026c7294473f50b05f345becb88c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f7265944740806fcee68000008c18697465726174696f6e735f73696e63655f726573746f7265944b0a757562658c195f6c61746573745f636865636b706f696e745f726573756c74946ab003000075622e\"\n }\n}"
+ ],
+ [
+ "{\n \"stub\": false,\n \"trainable_name\": \"train_model\",\n \"trial_id\": \"40101b9a\",\n \"storage\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059572040000000000008c1b7261792e747261696e2e5f696e7465726e616c2e73746f72616765948c0e53746f72616765436f6e746578749493942981947d94288c12637573746f6d5f66735f70726f766964656494898c1273746f726167655f6c6f63616c5f70617468948c2b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c7473948c136578706572696d656e745f6469725f6e616d65948c0f746573745f6578706572696d656e74948c0e747269616c5f6469725f6e616d65948c96747261696e5f6d6f64656c5f34303130316239615f32395f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d3531322c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f725f323032342d30362d32355f32322d31352d3437948c1863757272656e745f636865636b706f696e745f696e646578944b048c0b73796e635f636f6e666967948c097261792e747261696e948c0a53796e63436f6e6669679493942981947d94288c0b73796e635f706572696f64944d2c018c0c73796e635f74696d656f7574944d08078c0e73796e635f61727469666163747394898c1c73796e635f6172746966616374735f6f6e5f636865636b706f696e7494888c0a75706c6f61645f646972948c0a44455052454341544544948c0673796e6365729468188c1273796e635f6f6e5f636865636b706f696e7494681875628c1273746f726167655f66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c0f73746f726167655f66735f70617468948c452f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f7465737494681968008c115f46696c6573797374656d53796e6365729493942981947d9428681b682268134d2c0168144d08078c116c6173745f73796e635f75705f74696d65944741d99ec923518d8b8c136c6173745f73796e635f646f776e5f74696d659447fff00000000000008c0d5f73796e635f70726f63657373944e8c0c5f63757272656e745f636d649468008c125f75706c6f61645f746f5f66735f706174689493947d94288c0a6c6f63616c5f70617468948c3b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c74732f746573745f6578706572696d656e74948c0266739468228c0766735f70617468948c552f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e74948c076578636c756465945d948c0e2a2f636865636b706f696e745f2a9461758694756275622e\"\n },\n \"config\": {\n \"encoder_num_layer\": 2,\n \"decoder_rt_num_layer\": 1,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 1024,\n \"encoder_ff\": 2048,\n \"decoder_rt_ff\": 512,\n \"decoder_int_ff\": 512,\n \"n_head\": 1,\n \"drop_rate\": 0.8983976265900149,\n \"lr\": 0.039339947723305395,\n \"batch_size\": 2048\n },\n \"_Trial__unresolved_config\": {\n \"encoder_num_layer\": 2,\n \"decoder_rt_num_layer\": 1,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 1024,\n \"encoder_ff\": 2048,\n \"decoder_rt_ff\": 512,\n \"decoder_int_ff\": 512,\n \"n_head\": 1,\n \"drop_rate\": 0.8983976265900149,\n \"lr\": 0.039339947723305395,\n \"batch_size\": 2048\n },\n \"evaluated_params\": {\n \"batch_size\": 2048,\n \"decoder_int_ff\": 512,\n \"decoder_int_num_layer\": 1,\n \"decoder_rt_ff\": 512,\n \"decoder_rt_num_layer\": 1,\n \"drop_rate\": 0.8983976265900149,\n \"embedding_dim\": 1024,\n \"encoder_ff\": 2048,\n \"encoder_num_layer\": 2,\n \"lr\": 0.039339947723305395,\n \"n_head\": 1\n },\n \"experiment_tag\": \"29_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=1,drop_rate=0.8984,embedding_dim=1024,encoder_ff=2048,encoder_num_layer=2,lr=0.0393,n_head=1\",\n \"stopping_criterion\": {},\n \"_setup_default_resource\": true,\n \"_default_placement_group_factory\": \"80054e2e\",\n \"placement_group_factory\": \"800595ba000000000000008c237261792e74756e652e657865637574696f6e2e706c6163656d656e745f67726f757073948c15506c6163656d656e7447726f7570466163746f72799493942981947d94288c085f62756e646c6573945d947d94288c03435055944740540000000000008c034750559447401000000000000075618c155f686561645f62756e646c655f69735f656d70747994898c095f7374726174656779948c045041434b948c055f6172677394298c075f6b7761726773947d9475622e\",\n \"log_to_file\": [\n null,\n null\n ],\n \"max_failures\": 0,\n \"_default_result_or_future\": null,\n \"export_formats\": [],\n \"status\": \"PAUSED\",\n \"relative_logdir\": \"train_model_40101b9a_29_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=1,drop_r_2024-06-25_22-15-47\",\n \"trial_name_creator\": null,\n \"trial_dirname_creator\": null,\n \"custom_trial_name\": null,\n \"custom_dirname\": null,\n \"restore_path\": null,\n \"_restore_checkpoint_result\": null,\n \"_state_json\": null,\n \"results\": \"80054e2e\",\n \"extra_arg\": \"80054e2e\",\n \"_resources\": \"80054e2e\"\n}",
+ "{\n \"start_time\": 1719346823.9898968,\n \"num_failures\": 0,\n \"num_failures_after_restore\": 0,\n \"error_filename\": null,\n \"pickled_error_filename\": null,\n \"last_result\": {\n \"loss\": 2031.5720868448573,\n \"timestamp\": 1719348338,\n \"checkpoint_dir_name\": \"checkpoint_000004\",\n \"should_checkpoint\": true,\n \"done\": false,\n \"training_iteration\": 5,\n \"trial_id\": \"40101b9a\",\n \"date\": \"2024-06-25_22-45-38\",\n \"time_this_iter_s\": 300.0205228328705,\n \"time_total_s\": 1514.9287722110748,\n \"pid\": 2195253,\n \"hostname\": \"r8i6n2\",\n \"node_ip\": \"10.159.28.60\",\n \"config\": {\n \"encoder_num_layer\": 2,\n \"decoder_rt_num_layer\": 1,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 1024,\n \"encoder_ff\": 2048,\n \"decoder_rt_ff\": 512,\n \"decoder_int_ff\": 512,\n \"n_head\": 1,\n \"drop_rate\": 0.8983976265900149,\n \"lr\": 0.039339947723305395,\n \"batch_size\": 2048\n },\n \"time_since_restore\": 1514.9287722110748,\n \"iterations_since_restore\": 5,\n \"experiment_tag\": \"29_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=1,drop_rate=0.8984,embedding_dim=1024,encoder_ff=2048,encoder_num_layer=2,lr=0.0393,n_head=1\"\n },\n \"last_result_time\": 1719348338.9425833,\n \"metric_analysis\": {\n \"loss\": {\n \"max\": 4106.712271622785,\n \"min\": 2031.5720868448573,\n \"avg\": 2792.4098417507375,\n \"last\": 2031.5720868448573,\n \"last-5-avg\": 2792.409841750738,\n \"last-10-avg\": 2792.409841750738\n },\n \"should_checkpoint\": {\n \"max\": true,\n \"min\": true,\n \"avg\": 1.0,\n \"last\": true,\n \"last-5-avg\": 1.0,\n \"last-10-avg\": 1.0\n },\n \"done\": {\n \"max\": false,\n \"min\": false,\n \"avg\": 0.0,\n \"last\": false,\n \"last-5-avg\": 0.0,\n \"last-10-avg\": 0.0\n },\n \"training_iteration\": {\n \"max\": 5,\n \"min\": 1,\n \"avg\": 3.0,\n \"last\": 5,\n \"last-5-avg\": 3.0,\n \"last-10-avg\": 3.0\n },\n \"time_this_iter_s\": {\n \"max\": 313.700572013855,\n \"min\": 300.0205228328705,\n \"avg\": 302.98575444221495,\n \"last\": 300.0205228328705,\n \"last-5-avg\": 302.98575444221495,\n \"last-10-avg\": 302.98575444221495\n },\n \"time_total_s\": {\n \"max\": 1514.9287722110748,\n \"min\": 313.700572013855,\n \"avg\": 914.4032258033753,\n \"last\": 1514.9287722110748,\n \"last-5-avg\": 914.4032258033752,\n \"last-10-avg\": 914.4032258033752\n },\n \"time_since_restore\": {\n \"max\": 1514.9287722110748,\n \"min\": 313.700572013855,\n \"avg\": 914.4032258033753,\n \"last\": 1514.9287722110748,\n \"last-5-avg\": 914.4032258033752,\n \"last-10-avg\": 914.4032258033752\n },\n \"iterations_since_restore\": {\n \"max\": 5,\n \"min\": 1,\n \"avg\": 3.0,\n \"last\": 5,\n \"last-5-avg\": 3.0,\n \"last-10-avg\": 3.0\n }\n },\n \"_n_steps\": [\n 5,\n 10\n ],\n \"metric_n_steps\": {\n \"loss\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"800595d6000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294288c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b007494624308bbdd6e57b60ab04094869452946807680d43083e9fcfa72d74a84094869452946807680d4308080402e157a1a34094869452946807680d43088d46a311020aa14094869452946807680d4308894422d149be9f409486945294652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"800595d6000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294288c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b007494624308bbdd6e57b60ab04094869452946807680d43083e9fcfa72d74a84094869452946807680d4308080402e157a1a34094869452946807680d43088d46a311020aa14094869452946807680d4308894422d149be9f409486945294652e\"\n }\n },\n \"should_checkpoint\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059527000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294288888888888652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059527000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294288888888888652e\"\n }\n },\n \"done\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059527000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294288989898989652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059527000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294288989898989652e\"\n }\n },\n \"training_iteration\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952c000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294284b014b024b034b044b05652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952c000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294284b014b024b034b044b05652e\"\n }\n },\n \"time_this_iter_s\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005954f000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294284740739b358b000000474072c7fbec800000474072c14525c00000474072ca1193400000474072c0540fc00000652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005954f000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294284740739b358b000000474072c7fbec800000474072c14525c00000474072ca1193400000474072c0540fc00000652e\"\n }\n },\n \"time_total_s\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005954f000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294284740739b358b0000004740833198bbc0000047408c923b4ea00000474092fba20c200000474097abb710100000652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005954f000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294284740739b358b0000004740833198bbc0000047408c923b4ea00000474092fba20c200000474097abb710100000652e\"\n }\n },\n \"time_since_restore\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005954f000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294284740739b358b0000004740833198bbc0000047408c923b4ea00000474092fba20c200000474097abb710100000652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005954f000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294284740739b358b0000004740833198bbc0000047408c923b4ea00000474092fba20c200000474097abb710100000652e\"\n }\n },\n \"iterations_since_restore\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952c000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294284b014b024b034b044b05652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952c000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294284b014b024b034b044b05652e\"\n }\n }\n },\n \"checkpoint_manager\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005954e140000000000008c267261792e747261696e2e5f696e7465726e616c2e636865636b706f696e745f6d616e61676572948c125f436865636b706f696e744d616e616765729493942981947d94288c125f636865636b706f696e745f636f6e666967948c097261792e747261696e948c10436865636b706f696e74436f6e6669679493942981947d94288c0b6e756d5f746f5f6b656570944e8c1a636865636b706f696e745f73636f72655f617474726962757465944e8c16636865636b706f696e745f73636f72655f6f72646572948c036d6178948c14636865636b706f696e745f6672657175656e6379944b008c11636865636b706f696e745f61745f656e6494898c1a5f636865636b706f696e745f6b6565705f616c6c5f72616e6b73948c0a44455052454341544544948c1f5f636865636b706f696e745f75706c6f61645f66726f6d5f776f726b65727394681275628c135f636865636b706f696e745f726573756c7473945d94288c1b7261792e747261696e2e5f696e7465726e616c2e73657373696f6e948c0f5f547261696e696e67526573756c749493942981947d94288c0a636865636b706f696e749468068c0a436865636b706f696e749493942981947d94288c0470617468948cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f34303130316239615f32395f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d3531322c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f725f323032342d30362d32355f32322d31352d34372f636865636b706f696e745f303030303030948c0a66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c055f75756964948c0475756964948c04555549449493942981947d948c03696e74948a10466204a4ea70b689744ef652d6f7e478736275628c076d657472696373947d94288c046c6f7373948c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b007494624308bbdd6e57b60ab04094869452948c0974696d657374616d70944ac1277b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303030948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b018c08747269616c5f6964948c083430313031623961948c0464617465948c13323032342d30362d32355f32322d32352d3337948c1074696d655f746869735f697465725f73944740739b358b0000008c0c74696d655f746f74616c5f73944740739b358b0000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b028c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944d00048c0a656e636f6465725f6666944d00088c0d6465636f6465725f72745f6666944d00028c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b018c0964726f705f7261746594473fecbfac612042a28c026c7294473fa4245d99cbbf9f8c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f7265944740739b358b0000008c18697465726174696f6e735f73696e63655f726573746f7265944b0175756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f34303130316239615f32395f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d3531322c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f725f323032342d30362d32355f32322d31352d34372f636865636b706f696e745f30303030303194682268257d946827897385945294682a682d2981947d9468308a109d9c39eefbe09e9c564bf1f9a07b40327362756268317d94288c046c6f7373946836683c43083e9fcfa72d74a84094869452948c0974696d657374616d70944aed287b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303031948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b028c08747269616c5f6964948c083430313031623961948c0464617465948c13323032342d30362d32355f32322d33302d3338948c1074696d655f746869735f697465725f7394474072c7fbec8000008c0c74696d655f746f74616c5f73944740833198bbc000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b028c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944d00048c0a656e636f6465725f6666944d00088c0d6465636f6465725f72745f6666944d00028c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b018c0964726f705f7261746594473fecbfac612042a28c026c7294473fa4245d99cbbf9f8c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f7265944740833198bbc000008c18697465726174696f6e735f73696e63655f726573746f7265944b0275756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f34303130316239615f32395f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d3531322c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f725f323032342d30362d32355f32322d31352d34372f636865636b706f696e745f30303030303294682268257d946827897385945294682a682d2981947d9468308a10c5592afb394617b1fe42039a7b0872297362756268317d94288c046c6f7373946836683c4308080402e157a1a34094869452948c0974696d657374616d70944a192a7b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303032948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b038c08747269616c5f6964948c083430313031623961948c0464617465948c13323032342d30362d32355f32322d33352d3338948c1074696d655f746869735f697465725f7394474072c14525c000008c0c74696d655f746f74616c5f739447408c923b4ea000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b028c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944d00048c0a656e636f6465725f6666944d00088c0d6465636f6465725f72745f6666944d00028c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b018c0964726f705f7261746594473fecbfac612042a28c026c7294473fa4245d99cbbf9f8c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f72659447408c923b4ea000008c18697465726174696f6e735f73696e63655f726573746f7265944b0375756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f34303130316239615f32395f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d3531322c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f725f323032342d30362d32355f32322d31352d34372f636865636b706f696e745f30303030303394682268257d946827897385945294682a682d2981947d9468308a103e648eca43fcfebd824735f6a5de520d7362756268317d94288c046c6f7373946836683c43088d46a311020aa14094869452948c0974696d657374616d70944a462b7b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303033948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b048c08747269616c5f6964948c083430313031623961948c0464617465948c13323032342d30362d32355f32322d34302d3338948c1074696d655f746869735f697465725f7394474072ca11934000008c0c74696d655f746f74616c5f7394474092fba20c2000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b028c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944d00048c0a656e636f6465725f6666944d00088c0d6465636f6465725f72745f6666944d00028c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b018c0964726f705f7261746594473fecbfac612042a28c026c7294473fa4245d99cbbf9f8c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f726594474092fba20c2000008c18697465726174696f6e735f73696e63655f726573746f7265944b0475756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f34303130316239615f32395f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d3531322c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f725f323032342d30362d32355f32322d31352d34372f636865636b706f696e745f30303030303494682268257d946827897385945294682a682d2981947d9468308a11e83d97333973ffa71d4c11b616163ec9007362756268317d94288c046c6f7373946836683c4308894422d149be9f4094869452948c0974696d657374616d70944a722c7b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303034948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b058c08747269616c5f6964948c083430313031623961948c0464617465948c13323032342d30362d32355f32322d34352d3338948c1074696d655f746869735f697465725f7394474072c0540fc000008c0c74696d655f746f74616c5f7394474097abb7101000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b028c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944d00048c0a656e636f6465725f6666944d00088c0d6465636f6465725f72745f6666944d00028c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b018c0964726f705f7261746594473fecbfac612042a28c026c7294473fa4245d99cbbf9f8c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f726594474097abb7101000008c18697465726174696f6e735f73696e63655f726573746f7265944b05757562658c195f6c61746573745f636865636b706f696e745f726573756c749468ef75622e\"\n }\n}"
+ ],
+ [
+ "{\n \"stub\": false,\n \"trainable_name\": \"train_model\",\n \"trial_id\": \"de1e9134\",\n \"storage\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059572040000000000008c1b7261792e747261696e2e5f696e7465726e616c2e73746f72616765948c0e53746f72616765436f6e746578749493942981947d94288c12637573746f6d5f66735f70726f766964656494898c1273746f726167655f6c6f63616c5f70617468948c2b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c7473948c136578706572696d656e745f6469725f6e616d65948c0f746573745f6578706572696d656e74948c0e747269616c5f6469725f6e616d65948c96747261696e5f6d6f64656c5f64653165393133345f33305f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d3531322c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f725f323032342d30362d32355f32322d32302d3233948c1863757272656e745f636865636b706f696e745f696e646578944b098c0b73796e635f636f6e666967948c097261792e747261696e948c0a53796e63436f6e6669679493942981947d94288c0b73796e635f706572696f64944d2c018c0c73796e635f74696d656f7574944d08078c0e73796e635f61727469666163747394898c1c73796e635f6172746966616374735f6f6e5f636865636b706f696e7494888c0a75706c6f61645f646972948c0a44455052454341544544948c0673796e6365729468188c1273796e635f6f6e5f636865636b706f696e7494681875628c1273746f726167655f66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c0f73746f726167655f66735f70617468948c452f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f7465737494681968008c115f46696c6573797374656d53796e6365729493942981947d9428681b682268134d2c0168144d08078c116c6173745f73796e635f75705f74696d65944741d99ec96ee4bc5f8c136c6173745f73796e635f646f776e5f74696d659447fff00000000000008c0d5f73796e635f70726f63657373944e8c0c5f63757272656e745f636d649468008c125f75706c6f61645f746f5f66735f706174689493947d94288c0a6c6f63616c5f70617468948c3b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c74732f746573745f6578706572696d656e74948c0266739468228c0766735f70617468948c552f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e74948c076578636c756465945d948c0e2a2f636865636b706f696e745f2a9461758694756275622e\"\n },\n \"config\": {\n \"encoder_num_layer\": 4,\n \"decoder_rt_num_layer\": 1,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 16,\n \"encoder_ff\": 1024,\n \"decoder_rt_ff\": 512,\n \"decoder_int_ff\": 512,\n \"n_head\": 1,\n \"drop_rate\": 0.12892236942851165,\n \"lr\": 0.005626677279957184,\n \"batch_size\": 2048\n },\n \"_Trial__unresolved_config\": {\n \"encoder_num_layer\": 4,\n \"decoder_rt_num_layer\": 1,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 16,\n \"encoder_ff\": 1024,\n \"decoder_rt_ff\": 512,\n \"decoder_int_ff\": 512,\n \"n_head\": 1,\n \"drop_rate\": 0.12892236942851165,\n \"lr\": 0.005626677279957184,\n \"batch_size\": 2048\n },\n \"evaluated_params\": {\n \"batch_size\": 2048,\n \"decoder_int_ff\": 512,\n \"decoder_int_num_layer\": 1,\n \"decoder_rt_ff\": 512,\n \"decoder_rt_num_layer\": 1,\n \"drop_rate\": 0.12892236942851165,\n \"embedding_dim\": 16,\n \"encoder_ff\": 1024,\n \"encoder_num_layer\": 4,\n \"lr\": 0.005626677279957184,\n \"n_head\": 1\n },\n \"experiment_tag\": \"30_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=1,drop_rate=0.1289,embedding_dim=16,encoder_ff=1024,encoder_num_layer=4,lr=0.0056,n_head=1\",\n \"stopping_criterion\": {},\n \"_setup_default_resource\": true,\n \"_default_placement_group_factory\": \"80054e2e\",\n \"placement_group_factory\": \"800595ba000000000000008c237261792e74756e652e657865637574696f6e2e706c6163656d656e745f67726f757073948c15506c6163656d656e7447726f7570466163746f72799493942981947d94288c085f62756e646c6573945d947d94288c03435055944740540000000000008c034750559447401000000000000075618c155f686561645f62756e646c655f69735f656d70747994898c095f7374726174656779948c045041434b948c055f6172677394298c075f6b7761726773947d9475622e\",\n \"log_to_file\": [\n null,\n null\n ],\n \"max_failures\": 0,\n \"_default_result_or_future\": null,\n \"export_formats\": [],\n \"status\": \"PAUSED\",\n \"relative_logdir\": \"train_model_de1e9134_30_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=1,drop_r_2024-06-25_22-20-23\",\n \"trial_name_creator\": null,\n \"trial_dirname_creator\": null,\n \"custom_trial_name\": null,\n \"custom_dirname\": null,\n \"restore_path\": null,\n \"_restore_checkpoint_result\": null,\n \"_state_json\": null,\n \"results\": \"80054e2e\",\n \"extra_arg\": \"80054e2e\",\n \"_resources\": \"80054e2e\"\n}",
+ "{\n \"start_time\": 1719348339.0969658,\n \"num_failures\": 0,\n \"num_failures_after_restore\": 0,\n \"error_filename\": null,\n \"pickled_error_filename\": null,\n \"last_result\": {\n \"loss\": 1957.9578991987573,\n \"timestamp\": 1719355501,\n \"checkpoint_dir_name\": \"checkpoint_000009\",\n \"should_checkpoint\": true,\n \"done\": false,\n \"training_iteration\": 10,\n \"trial_id\": \"de1e9134\",\n \"date\": \"2024-06-26_00-45-01\",\n \"time_this_iter_s\": 52.88777208328247,\n \"time_total_s\": 560.1831469535828,\n \"pid\": 2850562,\n \"hostname\": \"r8i6n2\",\n \"node_ip\": \"10.159.28.60\",\n \"config\": {\n \"encoder_num_layer\": 4,\n \"decoder_rt_num_layer\": 1,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 16,\n \"encoder_ff\": 1024,\n \"decoder_rt_ff\": 512,\n \"decoder_int_ff\": 512,\n \"n_head\": 1,\n \"drop_rate\": 0.12892236942851165,\n \"lr\": 0.005626677279957184,\n \"batch_size\": 2048\n },\n \"time_since_restore\": 278.3368046283722,\n \"iterations_since_restore\": 5,\n \"experiment_tag\": \"30_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=1,drop_rate=0.1289,embedding_dim=16,encoder_ff=1024,encoder_num_layer=4,lr=0.0056,n_head=1\"\n },\n \"last_result_time\": 1719355501.903357,\n \"metric_analysis\": {\n \"loss\": {\n \"max\": 1966.5979321096827,\n \"min\": 1954.8947446327509,\n \"avg\": 1957.8368239996003,\n \"last\": 1957.9578991987573,\n \"last-5-avg\": 1957.8378904327633,\n \"last-10-avg\": 1957.8368239996005\n },\n \"should_checkpoint\": {\n \"max\": true,\n \"min\": true,\n \"avg\": 1.0,\n \"last\": true,\n \"last-5-avg\": 1.0,\n \"last-10-avg\": 1.0\n },\n \"done\": {\n \"max\": false,\n \"min\": false,\n \"avg\": 0.0,\n \"last\": false,\n \"last-5-avg\": 0.0,\n \"last-10-avg\": 0.0\n },\n \"training_iteration\": {\n \"max\": 10,\n \"min\": 1,\n \"avg\": 5.5,\n \"last\": 10,\n \"last-5-avg\": 8.0,\n \"last-10-avg\": 5.5\n },\n \"time_this_iter_s\": {\n \"max\": 67.86075568199158,\n \"min\": 52.848015785217285,\n \"avg\": 56.01831469535828,\n \"last\": 52.88777208328247,\n \"last-5-avg\": 55.66736092567444,\n \"last-10-avg\": 56.01831469535828\n },\n \"time_total_s\": {\n \"max\": 560.1831469535828,\n \"min\": 67.86075568199158,\n \"avg\": 314.47027125358585,\n \"last\": 560.1831469535828,\n \"last-5-avg\": 454.1982753276825,\n \"last-10-avg\": 314.4702712535858\n },\n \"time_since_restore\": {\n \"max\": 281.84634232521057,\n \"min\": 66.45859718322754,\n \"avg\": 173.5471000909805,\n \"last\": 278.3368046283722,\n \"last-5-avg\": 172.35193300247192,\n \"last-10-avg\": 173.54710009098054\n },\n \"iterations_since_restore\": {\n \"max\": 5,\n \"min\": 1,\n \"avg\": 3.0,\n \"last\": 5,\n \"last-5-avg\": 3.0,\n \"last-10-avg\": 3.0\n }\n },\n \"_n_steps\": [\n 5,\n 10\n ],\n \"metric_n_steps\": {\n \"loss\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"800595d6000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294288c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b0074946243086532998c198c9e4094869452946807680d430847a3d14879a99e4094869452946807680d430845229188ed949e4094869452946807680d4308f2783cbe6c929e4094869452946807680d43081c0e87e3d4979e409486945294652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059535010000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294288c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b007494624308a9542af5cb929e4094869452946807680d430843a1504864ba9e4094869452946807680d4308bfdfef37948b9e4094869452946807680d430893c96472978f9e4094869452946807680d430863b1582c5b8c9e4094869452946807680d43086532998c198c9e4094869452946807680d430847a3d14879a99e4094869452946807680d430845229188ed949e4094869452946807680d4308f2783cbe6c929e4094869452946807680d43081c0e87e3d4979e409486945294652e\"\n }\n },\n \"should_checkpoint\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059527000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294288888888888652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952c000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452942888888888888888888888652e\"\n }\n },\n \"done\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059527000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294288989898989652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952c000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452942889898989898989898989652e\"\n }\n },\n \"training_iteration\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952c000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294284b064b074b084b094b0a652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059536000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294284b014b024b034b044b054b064b074b084b094b0a652e\"\n }\n },\n \"time_this_iter_s\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005954f000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294284740509d59a800000047404a6c8bc800000047404a7374e400000047404a9ec5ea00000047404a71a284000000652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005957c000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a8694529428474050f7169f00000047404ab327b400000047404ab51a8a00000047404ab256f400000047404ae38e820000004740509d59a800000047404a6c8bc800000047404a7374e400000047404a9ec5ea00000047404a71a284000000652e\"\n }\n },\n \"time_total_s\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005954f000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b058694529428474075c4e10840000047407912728140000047407c60e11dc0000047407fb4b9db000000474081817715c00000652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005957c000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a8694529428474050f7169f00000047405e50aa79000000474065d59bdf00000047406c82319c0000004740719d8a9e400000474075c4e10840000047407912728140000047407c60e11dc0000047407fb4b9db000000474081817715c00000652e\"\n }\n },\n \"time_since_restore\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005954f000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294284740509d59a800000047405dd39f8c00000047406586acff00000047406c2e5e7980000047407165638d400000652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005957c000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a8694529428474050f7169f00000047405e50aa79000000474065d59bdf00000047406c82319c0000004740719d8a9e4000004740509d59a800000047405dd39f8c00000047406586acff00000047406c2e5e7980000047407165638d400000652e\"\n }\n },\n \"iterations_since_restore\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952c000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294284b014b024b034b044b05652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059536000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294284b014b024b034b044b054b014b024b034b044b05652e\"\n }\n }\n },\n \"checkpoint_manager\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005950b260000000000008c267261792e747261696e2e5f696e7465726e616c2e636865636b706f696e745f6d616e61676572948c125f436865636b706f696e744d616e616765729493942981947d94288c125f636865636b706f696e745f636f6e666967948c097261792e747261696e948c10436865636b706f696e74436f6e6669679493942981947d94288c0b6e756d5f746f5f6b656570944e8c1a636865636b706f696e745f73636f72655f617474726962757465944e8c16636865636b706f696e745f73636f72655f6f72646572948c036d6178948c14636865636b706f696e745f6672657175656e6379944b008c11636865636b706f696e745f61745f656e6494898c1a5f636865636b706f696e745f6b6565705f616c6c5f72616e6b73948c0a44455052454341544544948c1f5f636865636b706f696e745f75706c6f61645f66726f6d5f776f726b65727394681275628c135f636865636b706f696e745f726573756c7473945d94288c1b7261792e747261696e2e5f696e7465726e616c2e73657373696f6e948c0f5f547261696e696e67526573756c749493942981947d94288c0a636865636b706f696e749468068c0a436865636b706f696e749493942981947d94288c0470617468948cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f64653165393133345f33305f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d3531322c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f725f323032342d30362d32355f32322d32302d32332f636865636b706f696e745f303030303030948c0a66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c055f75756964948c0475756964948c04555549449493942981947d948c03696e74948a117e99e52d3a1ad38e0d44aea7c54e25dc00736275628c076d657472696373947d94288c046c6f7373948c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b007494624308a9542af5cb929e4094869452948c0974696d657374616d70944ab62c7b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303030948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b018c08747269616c5f6964948c086465316539313334948c0464617465948c13323032342d30362d32355f32322d34362d3436948c1074696d655f746869735f697465725f7394474050f7169f0000008c0c74696d655f746f74616c5f7394474050f7169f0000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b048c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b108c0a656e636f6465725f6666944d00048c0d6465636f6465725f72745f6666944d00028c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b018c0964726f705f7261746594473fc0808738358a748c026c7294473f770bffae70e5368c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f726594474050f7169f0000008c18697465726174696f6e735f73696e63655f726573746f7265944b0175756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f64653165393133345f33305f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d3531322c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f725f323032342d30362d32355f32322d32302d32332f636865636b706f696e745f30303030303194682268257d946827897385945294682a682d2981947d9468308a11e0766b93104e69aa2a4ce8efe19787dc007362756268317d94288c046c6f7373946836683c430843a1504864ba9e4094869452948c0974696d657374616d70944aec2c7b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303031948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b028c08747269616c5f6964948c086465316539313334948c0464617465948c13323032342d30362d32355f32322d34372d3430948c1074696d655f746869735f697465725f739447404ab327b40000008c0c74696d655f746f74616c5f739447405e50aa790000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b048c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b108c0a656e636f6465725f6666944d00048c0d6465636f6465725f72745f6666944d00028c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b018c0964726f705f7261746594473fc0808738358a748c026c7294473f770bffae70e5368c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f72659447405e50aa790000008c18697465726174696f6e735f73696e63655f726573746f7265944b0275756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f64653165393133345f33305f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d3531322c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f725f323032342d30362d32355f32322d32302d32332f636865636b706f696e745f30303030303294682268257d946827897385945294682a682d2981947d9468308a1172f8bcad354c3998c04ca697a29c22d1007362756268317d94288c046c6f7373946836683c4308bfdfef37948b9e4094869452948c0974696d657374616d70944a212d7b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303032948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b038c08747269616c5f6964948c086465316539313334948c0464617465948c13323032342d30362d32355f32322d34382d3333948c1074696d655f746869735f697465725f739447404ab51a8a0000008c0c74696d655f746f74616c5f7394474065d59bdf0000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b048c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b108c0a656e636f6465725f6666944d00048c0d6465636f6465725f72745f6666944d00028c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b018c0964726f705f7261746594473fc0808738358a748c026c7294473f770bffae70e5368c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f726594474065d59bdf0000008c18697465726174696f6e735f73696e63655f726573746f7265944b0375756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f64653165393133345f33305f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d3531322c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f725f323032342d30362d32355f32322d32302d32332f636865636b706f696e745f30303030303394682268257d946827897385945294682a682d2981947d9468308a1029599fd908f38f99194dfe050a975e4b7362756268317d94288c046c6f7373946836683c430893c96472978f9e4094869452948c0974696d657374616d70944a572d7b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303033948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b048c08747269616c5f6964948c086465316539313334948c0464617465948c13323032342d30362d32355f32322d34392d3237948c1074696d655f746869735f697465725f739447404ab256f40000008c0c74696d655f746f74616c5f739447406c82319c0000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b048c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b108c0a656e636f6465725f6666944d00048c0d6465636f6465725f72745f6666944d00028c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b018c0964726f705f7261746594473fc0808738358a748c026c7294473f770bffae70e5368c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f72659447406c82319c0000008c18697465726174696f6e735f73696e63655f726573746f7265944b0475756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f64653165393133345f33305f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d3531322c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f725f323032342d30362d32355f32322d32302d32332f636865636b706f696e745f30303030303494682268257d946827897385945294682a682d2981947d9468308a108717ed70f9eab9802a4cfc2b28770c597362756268317d94288c046c6f7373946836683c430863b1582c5b8c9e4094869452948c0974696d657374616d70944a8c2d7b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303034948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b058c08747269616c5f6964948c086465316539313334948c0464617465948c13323032342d30362d32355f32322d35302d3230948c1074696d655f746869735f697465725f739447404ae38e820000008c0c74696d655f746f74616c5f73944740719d8a9e4000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b048c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b108c0a656e636f6465725f6666944d00048c0d6465636f6465725f72745f6666944d00028c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b018c0964726f705f7261746594473fc0808738358a748c026c7294473f770bffae70e5368c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f7265944740719d8a9e4000008c18697465726174696f6e735f73696e63655f726573746f7265944b0575756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f64653165393133345f33305f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d3531322c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f725f323032342d30362d32355f32322d32302d32332f636865636b706f696e745f30303030303594682268257d946827897385945294682a682d2981947d9468308a11a1d04c24c760538f2248b1b0ce71ab9a007362756268317d94288c046c6f7373946836683c43086532998c198c9e4094869452948c0974696d657374616d70944a99477b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303035948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b068c08747269616c5f6964948c086465316539313334948c0464617465948c13323032342d30362d32365f30302d34312d3330948c1074696d655f746869735f697465725f73944740509d59a80000008c0c74696d655f746f74616c5f7394474075c4e1084000008c03706964944a027f2b008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b048c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b108c0a656e636f6465725f6666944d00048c0d6465636f6465725f72745f6666944d00028c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b018c0964726f705f7261746594473fc0808738358a748c026c7294473f770bffae70e5368c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f7265944740509d59a80000008c18697465726174696f6e735f73696e63655f726573746f7265944b0175756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f64653165393133345f33305f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d3531322c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f725f323032342d30362d32355f32322d32302d32332f636865636b706f696e745f30303030303694682268257d946827897385945294682a682d2981947d9468308a100f0006703543e68bd642176c71a376337362756268317d94288c046c6f7373946836683c430847a3d14879a99e4094869452948c0974696d657374616d70944ace477b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303036948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b078c08747269616c5f6964948c086465316539313334948c0464617465948c13323032342d30362d32365f30302d34322d3232948c1074696d655f746869735f697465725f739447404a6c8bc80000008c0c74696d655f746f74616c5f73944740791272814000008c03706964944a027f2b008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b048c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b108c0a656e636f6465725f6666944d00048c0d6465636f6465725f72745f6666944d00028c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b018c0964726f705f7261746594473fc0808738358a748c026c7294473f770bffae70e5368c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f72659447405dd39f8c0000008c18697465726174696f6e735f73696e63655f726573746f7265944b0275756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f64653165393133345f33305f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d3531322c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f725f323032342d30362d32355f32322d32302d32332f636865636b706f696e745f30303030303794682268257d946827897385945294682a682d2981947d9468308a10846da0a32c220f9d3b4fdaea935a21477362756268317d94288c046c6f7373946836683c430845229188ed949e4094869452948c0974696d657374616d70944a03487b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303037948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b088c08747269616c5f6964948c086465316539313334948c0464617465948c13323032342d30362d32365f30302d34332d3135948c1074696d655f746869735f697465725f739447404a7374e40000008c0c74696d655f746f74616c5f739447407c60e11dc000008c03706964944a027f2b008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b048c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b108c0a656e636f6465725f6666944d00048c0d6465636f6465725f72745f6666944d00028c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b018c0964726f705f7261746594473fc0808738358a748c026c7294473f770bffae70e5368c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f72659447406586acff0000008c18697465726174696f6e735f73696e63655f726573746f7265944b0375756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f64653165393133345f33305f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d3531322c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f725f323032342d30362d32355f32322d32302d32332f636865636b706f696e745f30303030303894682268257d946827897385945294682a682d2981947d9468308a10f3fa414a50a891a01a4c61ba0c7c707b7362756268317d94288c046c6f7373946836683c4308f2783cbe6c929e4094869452948c0974696d657374616d70944a38487b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303038948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b098c08747269616c5f6964948c086465316539313334948c0464617465948c13323032342d30362d32365f30302d34342d3039948c1074696d655f746869735f697465725f739447404a9ec5ea0000008c0c74696d655f746f74616c5f739447407fb4b9db0000008c03706964944a027f2b008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b048c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b108c0a656e636f6465725f6666944d00048c0d6465636f6465725f72745f6666944d00028c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b018c0964726f705f7261746594473fc0808738358a748c026c7294473f770bffae70e5368c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f72659447406c2e5e798000008c18697465726174696f6e735f73696e63655f726573746f7265944b0475756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f64653165393133345f33305f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d3531322c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f725f323032342d30362d32355f32322d32302d32332f636865636b706f696e745f30303030303994682268257d946827897385945294682a682d2981947d9468308a1058e700d0973d4caa6949f5ee7039cd737362756268317d94288c046c6f7373946836683c43081c0e87e3d4979e4094869452948c0974696d657374616d70944a6d487b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303039948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b0a8c08747269616c5f6964948c086465316539313334948c0464617465948c13323032342d30362d32365f30302d34352d3031948c1074696d655f746869735f697465725f739447404a71a2840000008c0c74696d655f746f74616c5f7394474081817715c000008c03706964944a027f2b008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b048c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b108c0a656e636f6465725f6666944d00048c0d6465636f6465725f72745f6666944d00028c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b018c0964726f705f7261746594473fc0808738358a748c026c7294473f770bffae70e5368c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f72659447407165638d4000008c18697465726174696f6e735f73696e63655f726573746f7265944b05757562658c195f6c61746573745f636865636b706f696e745f726573756c74946ada01000075622e\"\n }\n}"
+ ],
+ [
+ "{\n \"stub\": false,\n \"trainable_name\": \"train_model\",\n \"trial_id\": \"0f80a6d7\",\n \"storage\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059572040000000000008c1b7261792e747261696e2e5f696e7465726e616c2e73746f72616765948c0e53746f72616765436f6e746578749493942981947d94288c12637573746f6d5f66735f70726f766964656494898c1273746f726167655f6c6f63616c5f70617468948c2b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c7473948c136578706572696d656e745f6469725f6e616d65948c0f746573745f6578706572696d656e74948c0e747269616c5f6469725f6e616d65948c96747261696e5f6d6f64656c5f30663830613664375f33315f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d3531322c6465636f6465725f72745f6e756d5f6c617965723d322c64726f705f725f323032342d30362d32355f32322d34352d3339948c1863757272656e745f636865636b706f696e745f696e646578944b098c0b73796e635f636f6e666967948c097261792e747261696e948c0a53796e63436f6e6669679493942981947d94288c0b73796e635f706572696f64944d2c018c0c73796e635f74696d656f7574944d08078c0e73796e635f61727469666163747394898c1c73796e635f6172746966616374735f6f6e5f636865636b706f696e7494888c0a75706c6f61645f646972948c0a44455052454341544544948c0673796e6365729468188c1273796e635f6f6e5f636865636b706f696e7494681875628c1273746f726167655f66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c0f73746f726167655f66735f70617468948c452f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f7465737494681968008c115f46696c6573797374656d53796e6365729493942981947d9428681b682268134d2c0168144d08078c116c6173745f73796e635f75705f74696d65944741d99ecafb428f1a8c136c6173745f73796e635f646f776e5f74696d659447fff00000000000008c0d5f73796e635f70726f63657373944e8c0c5f63757272656e745f636d649468008c125f75706c6f61645f746f5f66735f706174689493947d94288c0a6c6f63616c5f70617468948c3b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c74732f746573745f6578706572696d656e74948c0266739468228c0766735f70617468948c552f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e74948c076578636c756465945d948c0e2a2f636865636b706f696e745f2a9461758694756275622e\"\n },\n \"config\": {\n \"encoder_num_layer\": 2,\n \"decoder_rt_num_layer\": 2,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 1024,\n \"encoder_ff\": 1024,\n \"decoder_rt_ff\": 512,\n \"decoder_int_ff\": 512,\n \"n_head\": 16,\n \"drop_rate\": 0.023920599186226577,\n \"lr\": 0.0024687399238039302,\n \"batch_size\": 2048\n },\n \"_Trial__unresolved_config\": {\n \"encoder_num_layer\": 2,\n \"decoder_rt_num_layer\": 2,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 1024,\n \"encoder_ff\": 1024,\n \"decoder_rt_ff\": 512,\n \"decoder_int_ff\": 512,\n \"n_head\": 16,\n \"drop_rate\": 0.023920599186226577,\n \"lr\": 0.0024687399238039302,\n \"batch_size\": 2048\n },\n \"evaluated_params\": {\n \"batch_size\": 2048,\n \"decoder_int_ff\": 512,\n \"decoder_int_num_layer\": 1,\n \"decoder_rt_ff\": 512,\n \"decoder_rt_num_layer\": 2,\n \"drop_rate\": 0.023920599186226577,\n \"embedding_dim\": 1024,\n \"encoder_ff\": 1024,\n \"encoder_num_layer\": 2,\n \"lr\": 0.0024687399238039302,\n \"n_head\": 16\n },\n \"experiment_tag\": \"31_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_rate=0.0239,embedding_dim=1024,encoder_ff=1024,encoder_num_layer=2,lr=0.0025,n_head=16\",\n \"stopping_criterion\": {},\n \"_setup_default_resource\": true,\n \"_default_placement_group_factory\": \"80054e2e\",\n \"placement_group_factory\": \"800595ba000000000000008c237261792e74756e652e657865637574696f6e2e706c6163656d656e745f67726f757073948c15506c6163656d656e7447726f7570466163746f72799493942981947d94288c085f62756e646c6573945d947d94288c03435055944740540000000000008c034750559447401000000000000075618c155f686561645f62756e646c655f69735f656d70747994898c095f7374726174656779948c045041434b948c055f6172677394298c075f6b7761726773947d9475622e\",\n \"log_to_file\": [\n null,\n null\n ],\n \"max_failures\": 0,\n \"_default_result_or_future\": null,\n \"export_formats\": [],\n \"status\": \"TERMINATED\",\n \"relative_logdir\": \"train_model_0f80a6d7_31_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_22-45-39\",\n \"trial_name_creator\": null,\n \"trial_dirname_creator\": null,\n \"custom_trial_name\": null,\n \"custom_dirname\": null,\n \"restore_path\": null,\n \"_restore_checkpoint_result\": null,\n \"_state_json\": null,\n \"results\": \"80054e2e\",\n \"extra_arg\": \"80054e2e\",\n \"_resources\": \"80054e2e\"\n}",
+ "{\n \"start_time\": 1719348621.1283247,\n \"num_failures\": 0,\n \"num_failures_after_restore\": 0,\n \"error_filename\": null,\n \"pickled_error_filename\": null,\n \"last_result\": {\n \"loss\": 1959.91755063515,\n \"timestamp\": 1719357265,\n \"checkpoint_dir_name\": \"checkpoint_000009\",\n \"should_checkpoint\": true,\n \"done\": true,\n \"training_iteration\": 10,\n \"trial_id\": \"0f80a6d7\",\n \"date\": \"2024-06-26_01-14-25\",\n \"time_this_iter_s\": 349.87603521347046,\n \"time_total_s\": 3527.171615600586,\n \"pid\": 2850562,\n \"hostname\": \"r8i6n2\",\n \"node_ip\": \"10.159.28.60\",\n \"config\": {\n \"encoder_num_layer\": 2,\n \"decoder_rt_num_layer\": 2,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 1024,\n \"encoder_ff\": 1024,\n \"decoder_rt_ff\": 512,\n \"decoder_int_ff\": 512,\n \"n_head\": 16,\n \"drop_rate\": 0.023920599186226577,\n \"lr\": 0.0024687399238039302,\n \"batch_size\": 2048\n },\n \"time_since_restore\": 1763.417979001999,\n \"iterations_since_restore\": 5,\n \"experiment_tag\": \"31_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_rate=0.0239,embedding_dim=1024,encoder_ff=1024,encoder_num_layer=2,lr=0.0025,n_head=16\"\n },\n \"last_result_time\": 1719357265.5030398,\n \"metric_analysis\": {\n \"loss\": {\n \"max\": 1986.2198524775467,\n \"min\": 1955.6729476808564,\n \"avg\": 1965.4819975124567,\n \"last\": 1959.91755063515,\n \"last-5-avg\": 1962.4524979238436,\n \"last-10-avg\": 1965.4819975124567\n },\n \"should_checkpoint\": {\n \"max\": true,\n \"min\": true,\n \"avg\": 1.0,\n \"last\": true,\n \"last-5-avg\": 1.0,\n \"last-10-avg\": 1.0\n },\n \"done\": {\n \"max\": true,\n \"min\": false,\n \"avg\": 0.1,\n \"last\": true,\n \"last-5-avg\": 0.2,\n \"last-10-avg\": 0.1\n },\n \"training_iteration\": {\n \"max\": 10,\n \"min\": 1,\n \"avg\": 5.5,\n \"last\": 10,\n \"last-5-avg\": 8.0,\n \"last-10-avg\": 5.5\n },\n \"time_this_iter_s\": {\n \"max\": 365.2865264415741,\n \"min\": 349.18421268463135,\n \"avg\": 352.71716156005857,\n \"last\": 349.87603521347046,\n \"last-5-avg\": 352.68359580039976,\n \"last-10-avg\": 352.71716156005857\n },\n \"time_total_s\": {\n \"max\": 3527.171615600586,\n \"min\": 363.18635082244873,\n \"avg\": 1945.676479148865,\n \"last\": 3527.171615600586,\n \"last-5-avg\": 2827.933716440201,\n \"last-10-avg\": 1945.6764791488647\n },\n \"time_since_restore\": {\n \"max\": 1763.753636598587,\n \"min\": 363.18635082244873,\n \"avg\": 1063.7996608495712,\n \"last\": 1763.417979001999,\n \"last-5-avg\": 1064.1800798416139,\n \"last-10-avg\": 1063.7996608495712\n },\n \"iterations_since_restore\": {\n \"max\": 5,\n \"min\": 1,\n \"avg\": 3.0,\n \"last\": 5,\n \"last-5-avg\": 3.0,\n \"last-10-avg\": 3.0\n }\n },\n \"_n_steps\": [\n 5,\n 10\n ],\n \"metric_n_steps\": {\n \"loss\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"800595d6000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294288c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b00749462430867b3d96c98cf9e4094869452946807680d4308ca643219b18e9e4094869452946807680d4308d0e7f339fd959e4094869452946807680d4308bd5eaf771abd9e4094869452946807680d430893c96492ab9f9e409486945294652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059535010000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294288c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b00749462430808040221e1089f4094869452946807680d4308fa7c3e7f32ee9e4094869452946807680d430808040241bdb39e4094869452946807680d4308d4e9745ab38e9e4094869452946807680d43080e87c3a1b6909e4094869452946807680d430867b3d96c98cf9e4094869452946807680d4308ca643219b18e9e4094869452946807680d4308d0e7f339fd959e4094869452946807680d4308bd5eaf771abd9e4094869452946807680d430893c96492ab9f9e409486945294652e\"\n }\n },\n \"should_checkpoint\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059527000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294288888888888652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952c000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452942888888888888888888888652e\"\n }\n },\n \"done\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059527000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294288989898988652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952c000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452942889898989898989898988652e\"\n }\n },\n \"training_iteration\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952c000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294284b064b074b084b094b0a652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059536000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294284b014b024b034b044b054b064b074b084b094b0a652e\"\n }\n },\n \"time_this_iter_s\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005954f000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b058694529428474076d4959cc00000474075d2f289000000474075db6648c00000474075d5bd5ec00000474075de043d800000652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005957c000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a8694529428474076b2fb4b000000474075e4b866c00000474075d9fe77c00000474075e767ddc00000474075e2f4de000000474076d4959cc00000474075d2f289000000474075db6648c00000474075d5bd5ec00000474075de043d800000652e\"\n }\n },\n \"time_total_s\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005954f000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294284740a0a214904000004740a35c72e16000004740a617dfaa7800004740a8d297565000004740ab8e57de000000652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005957c000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a8694529428474076b2fb4b0000004740864bd9d8e000004740909c6c8a600000474096164681d0000047409b8f03b95000004740a0a214904000004740a35c72e16000004740a617dfaa7800004740a8d297565000004740ab8e57de000000652e\"\n }\n },\n \"time_since_restore\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005954f000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b058694529428474076d4959cc0000047408653c412e00000474090a0bb9ba00000474096162af350000047409b8dac02b00000652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005957c000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a8694529428474076b2fb4b0000004740864bd9d8e000004740909c6c8a600000474096164681d0000047409b8f03b9500000474076d4959cc0000047408653c412e00000474090a0bb9ba00000474096162af350000047409b8dac02b00000652e\"\n }\n },\n \"iterations_since_restore\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952c000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294284b014b024b034b044b05652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059536000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294284b014b024b034b044b054b014b024b034b044b05652e\"\n }\n }\n },\n \"checkpoint_manager\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059514260000000000008c267261792e747261696e2e5f696e7465726e616c2e636865636b706f696e745f6d616e61676572948c125f436865636b706f696e744d616e616765729493942981947d94288c125f636865636b706f696e745f636f6e666967948c097261792e747261696e948c10436865636b706f696e74436f6e6669679493942981947d94288c0b6e756d5f746f5f6b656570944e8c1a636865636b706f696e745f73636f72655f617474726962757465944e8c16636865636b706f696e745f73636f72655f6f72646572948c036d6178948c14636865636b706f696e745f6672657175656e6379944b008c11636865636b706f696e745f61745f656e6494898c1a5f636865636b706f696e745f6b6565705f616c6c5f72616e6b73948c0a44455052454341544544948c1f5f636865636b706f696e745f75706c6f61645f66726f6d5f776f726b65727394681275628c135f636865636b706f696e745f726573756c7473945d94288c1b7261792e747261696e2e5f696e7465726e616c2e73657373696f6e948c0f5f547261696e696e67526573756c749493942981947d94288c0a636865636b706f696e749468068c0a436865636b706f696e749493942981947d94288c0470617468948cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f30663830613664375f33315f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d3531322c6465636f6465725f72745f6e756d5f6c617965723d322c64726f705f725f323032342d30362d32355f32322d34352d33392f636865636b706f696e745f303030303030948c0a66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c055f75756964948c0475756964948c04555549449493942981947d948c03696e74948a1161e27b7800fdc1924e4b3679cc099dd800736275628c076d657472696373947d94288c046c6f7373948c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b00749462430808040221e1089f4094869452948c0974696d657374616d70944af72e7b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303030948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b018c08747269616c5f6964948c083066383061366437948c0464617465948c13323032342d30362d32355f32322d35362d3234948c1074696d655f746869735f697465725f7394474076b2fb4b0000008c0c74696d655f746f74616c5f7394474076b2fb4b0000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b028c146465636f6465725f72745f6e756d5f6c61796572944b028c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944d00048c0a656e636f6465725f6666944d00048c0d6465636f6465725f72745f6666944d00028c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b108c0964726f705f7261746594473f987ea43cd280408c026c7294473f643952a785ac188c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f726594474076b2fb4b0000008c18697465726174696f6e735f73696e63655f726573746f7265944b0175756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f30663830613664375f33315f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d3531322c6465636f6465725f72745f6e756d5f6c617965723d322c64726f705f725f323032342d30362d32355f32322d34352d33392f636865636b706f696e745f30303030303194682268257d946827897385945294682a682d2981947d9468308a101bd69a0e219bcbaedd476198d47ebf7f7362756268317d94288c046c6f7373946836683c4308fa7c3e7f32ee9e4094869452948c0974696d657374616d70944a56307b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303031948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b028c08747269616c5f6964948c083066383061366437948c0464617465948c13323032342d30362d32355f32332d30322d3134948c1074696d655f746869735f697465725f7394474075e4b866c000008c0c74696d655f746f74616c5f73944740864bd9d8e000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b028c146465636f6465725f72745f6e756d5f6c61796572944b028c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944d00048c0a656e636f6465725f6666944d00048c0d6465636f6465725f72745f6666944d00028c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b108c0964726f705f7261746594473f987ea43cd280408c026c7294473f643952a785ac188c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f7265944740864bd9d8e000008c18697465726174696f6e735f73696e63655f726573746f7265944b0275756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f30663830613664375f33315f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d3531322c6465636f6465725f72745f6e756d5f6c617965723d322c64726f705f725f323032342d30362d32355f32322d34352d33392f636865636b706f696e745f30303030303294682268257d946827897385945294682a682d2981947d9468308a103c1ee8c2549795b4c943114c6667993c7362756268317d94288c046c6f7373946836683c430808040241bdb39e4094869452948c0974696d657374616d70944ab3317b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303032948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b038c08747269616c5f6964948c083066383061366437948c0464617465948c13323032342d30362d32355f32332d30382d3034948c1074696d655f746869735f697465725f7394474075d9fe77c000008c0c74696d655f746f74616c5f73944740909c6c8a6000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b028c146465636f6465725f72745f6e756d5f6c61796572944b028c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944d00048c0a656e636f6465725f6666944d00048c0d6465636f6465725f72745f6666944d00028c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b108c0964726f705f7261746594473f987ea43cd280408c026c7294473f643952a785ac188c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f7265944740909c6c8a6000008c18697465726174696f6e735f73696e63655f726573746f7265944b0375756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f30663830613664375f33315f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d3531322c6465636f6465725f72745f6e756d5f6c617965723d322c64726f705f725f323032342d30362d32355f32322d34352d33392f636865636b706f696e745f30303030303394682268257d946827897385945294682a682d2981947d9468308a10ec8e63351823f2892247c258a97ee73d7362756268317d94288c046c6f7373946836683c4308d4e9745ab38e9e4094869452948c0974696d657374616d70944a12337b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303033948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b048c08747269616c5f6964948c083066383061366437948c0464617465948c13323032342d30362d32355f32332d31332d3534948c1074696d655f746869735f697465725f7394474075e767ddc000008c0c74696d655f746f74616c5f7394474096164681d000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b028c146465636f6465725f72745f6e756d5f6c61796572944b028c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944d00048c0a656e636f6465725f6666944d00048c0d6465636f6465725f72745f6666944d00028c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b108c0964726f705f7261746594473f987ea43cd280408c026c7294473f643952a785ac188c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f726594474096164681d000008c18697465726174696f6e735f73696e63655f726573746f7265944b0475756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f30663830613664375f33315f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d3531322c6465636f6465725f72745f6e756d5f6c617965723d322c64726f705f725f323032342d30362d32355f32322d34352d33392f636865636b706f696e745f30303030303494682268257d946827897385945294682a682d2981947d9468308a1067d6c95952ced697984797b362b8927d7362756268317d94288c046c6f7373946836683c43080e87c3a1b6909e4094869452948c0974696d657374616d70944a70347b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303034948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b058c08747269616c5f6964948c083066383061366437948c0464617465948c13323032342d30362d32355f32332d31392d3434948c1074696d655f746869735f697465725f7394474075e2f4de0000008c0c74696d655f746f74616c5f739447409b8f03b95000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b028c146465636f6465725f72745f6e756d5f6c61796572944b028c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944d00048c0a656e636f6465725f6666944d00048c0d6465636f6465725f72745f6666944d00028c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b108c0964726f705f7261746594473f987ea43cd280408c026c7294473f643952a785ac188c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f72659447409b8f03b95000008c18697465726174696f6e735f73696e63655f726573746f7265944b0575756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f30663830613664375f33315f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d3531322c6465636f6465725f72745f6e756d5f6c617965723d322c64726f705f725f323032342d30362d32355f32322d34352d33392f636865636b706f696e745f30303030303594682268257d946827897385945294682a682d2981947d9468308a1188174b41c08d35aff24e950105dea5c6007362756268317d94288c046c6f7373946836683c430867b3d96c98cf9e4094869452948c0974696d657374616d70944ada497b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303035948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b068c08747269616c5f6964948c083066383061366437948c0464617465948c13323032342d30362d32365f30302d35312d3037948c1074696d655f746869735f697465725f7394474076d4959cc000008c0c74696d655f746f74616c5f73944740a0a214904000008c03706964944a027f2b008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b028c146465636f6465725f72745f6e756d5f6c61796572944b028c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944d00048c0a656e636f6465725f6666944d00048c0d6465636f6465725f72745f6666944d00028c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b108c0964726f705f7261746594473f987ea43cd280408c026c7294473f643952a785ac188c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f726594474076d4959cc000008c18697465726174696f6e735f73696e63655f726573746f7265944b0175756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f30663830613664375f33315f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d3531322c6465636f6465725f72745f6e756d5f6c617965723d322c64726f705f725f323032342d30362d32355f32322d34352d33392f636865636b706f696e745f30303030303694682268257d946827897385945294682a682d2981947d9468308a1057cce3d9822a14a9614bea04201a966b7362756268317d94288c046c6f7373946836683c4308ca643219b18e9e4094869452948c0974696d657374616d70944a384b7b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303036948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b078c08747269616c5f6964948c083066383061366437948c0464617465948c13323032342d30362d32365f30302d35362d3536948c1074696d655f746869735f697465725f7394474075d2f2890000008c0c74696d655f746f74616c5f73944740a35c72e16000008c03706964944a027f2b008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b028c146465636f6465725f72745f6e756d5f6c61796572944b028c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944d00048c0a656e636f6465725f6666944d00048c0d6465636f6465725f72745f6666944d00028c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b108c0964726f705f7261746594473f987ea43cd280408c026c7294473f643952a785ac188c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f72659447408653c412e000008c18697465726174696f6e735f73696e63655f726573746f7265944b0275756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f30663830613664375f33315f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d3531322c6465636f6465725f72745f6e756d5f6c617965723d322c64726f705f725f323032342d30362d32355f32322d34352d33392f636865636b706f696e745f30303030303794682268257d946827897385945294682a682d2981947d9468308a102a5c119bf60755beb2446b96b09e55537362756268317d94288c046c6f7373946836683c4308d0e7f339fd959e4094869452948c0974696d657374616d70944a954c7b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303037948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b088c08747269616c5f6964948c083066383061366437948c0464617465948c13323032342d30362d32365f30312d30322d3436948c1074696d655f746869735f697465725f7394474075db6648c000008c0c74696d655f746f74616c5f73944740a617dfaa7800008c03706964944a027f2b008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b028c146465636f6465725f72745f6e756d5f6c61796572944b028c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944d00048c0a656e636f6465725f6666944d00048c0d6465636f6465725f72745f6666944d00028c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b108c0964726f705f7261746594473f987ea43cd280408c026c7294473f643952a785ac188c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f726594474090a0bb9ba000008c18697465726174696f6e735f73696e63655f726573746f7265944b0375756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f30663830613664375f33315f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d3531322c6465636f6465725f72745f6e756d5f6c617965723d322c64726f705f725f323032342d30362d32355f32322d34352d33392f636865636b706f696e745f30303030303894682268257d946827897385945294682a682d2981947d9468308a11a9087b4978719e871e443c6bfcdaf6e0007362756268317d94288c046c6f7373946836683c4308bd5eaf771abd9e4094869452948c0974696d657374616d70944af34d7b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303038948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b098c08747269616c5f6964948c083066383061366437948c0464617465948c13323032342d30362d32365f30312d30382d3335948c1074696d655f746869735f697465725f7394474075d5bd5ec000008c0c74696d655f746f74616c5f73944740a8d297565000008c03706964944a027f2b008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b028c146465636f6465725f72745f6e756d5f6c61796572944b028c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944d00048c0a656e636f6465725f6666944d00048c0d6465636f6465725f72745f6666944d00028c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b108c0964726f705f7261746594473f987ea43cd280408c026c7294473f643952a785ac188c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f726594474096162af35000008c18697465726174696f6e735f73696e63655f726573746f7265944b0475756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f30663830613664375f33315f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d3531322c6465636f6465725f72745f6e756d5f6c617965723d322c64726f705f725f323032342d30362d32355f32322d34352d33392f636865636b706f696e745f30303030303994682268257d946827897385945294682a682d2981947d9468308a10bcc31758fd28c490b547d6435831d4687362756268317d94288c046c6f7373946836683c430893c96492ab9f9e4094869452948c0974696d657374616d70944a514f7b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303039948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b0a8c08747269616c5f6964948c083066383061366437948c0464617465948c13323032342d30362d32365f30312d31342d3235948c1074696d655f746869735f697465725f7394474075de043d8000008c0c74696d655f746f74616c5f73944740ab8e57de0000008c03706964944a027f2b008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b028c146465636f6465725f72745f6e756d5f6c61796572944b028c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944d00048c0a656e636f6465725f6666944d00048c0d6465636f6465725f72745f6666944d00028c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b108c0964726f705f7261746594473f987ea43cd280408c026c7294473f643952a785ac188c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f72659447409b8dac02b000008c18697465726174696f6e735f73696e63655f726573746f7265944b05757562658c195f6c61746573745f636865636b706f696e745f726573756c74946ada01000075622e\"\n }\n}"
+ ],
+ [
+ "{\n \"stub\": false,\n \"trainable_name\": \"train_model\",\n \"trial_id\": \"fe083630\",\n \"storage\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059572040000000000008c1b7261792e747261696e2e5f696e7465726e616c2e73746f72616765948c0e53746f72616765436f6e746578749493942981947d94288c12637573746f6d5f66735f70726f766964656494898c1273746f726167655f6c6f63616c5f70617468948c2b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c7473948c136578706572696d656e745f6469725f6e616d65948c0f746573745f6578706572696d656e74948c0e747269616c5f6469725f6e616d65948c96747261696e5f6d6f64656c5f66653038333633305f33325f62617463685f73697a653d313032342c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d3531322c6465636f6465725f72745f6e756d5f6c617965723d322c64726f705f725f323032342d30362d32355f32322d35302d3231948c1863757272656e745f636865636b706f696e745f696e646578944b048c0b73796e635f636f6e666967948c097261792e747261696e948c0a53796e63436f6e6669679493942981947d94288c0b73796e635f706572696f64944d2c018c0c73796e635f74696d656f7574944d08078c0e73796e635f61727469666163747394898c1c73796e635f6172746966616374735f6f6e5f636865636b706f696e7494888c0a75706c6f61645f646972948c0a44455052454341544544948c0673796e6365729468188c1273796e635f6f6e5f636865636b706f696e7494681875628c1273746f726167655f66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c0f73746f726167655f66735f70617468948c452f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f7465737494681968008c115f46696c6573797374656d53796e6365729493942981947d9428681b682268134d2c0168144d08078c116c6173745f73796e635f75705f74696d65944741d99ecb46c1710d8c136c6173745f73796e635f646f776e5f74696d659447fff00000000000008c0d5f73796e635f70726f63657373944e8c0c5f63757272656e745f636d649468008c125f75706c6f61645f746f5f66735f706174689493947d94288c0a6c6f63616c5f70617468948c3b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c74732f746573745f6578706572696d656e74948c0266739468228c0766735f70617468948c552f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e74948c076578636c756465945d948c0e2a2f636865636b706f696e745f2a9461758694756275622e\"\n },\n \"config\": {\n \"encoder_num_layer\": 8,\n \"decoder_rt_num_layer\": 2,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 1024,\n \"encoder_ff\": 1024,\n \"decoder_rt_ff\": 512,\n \"decoder_int_ff\": 512,\n \"n_head\": 2,\n \"drop_rate\": 0.7814344084863447,\n \"lr\": 0.030816743385966763,\n \"batch_size\": 1024\n },\n \"_Trial__unresolved_config\": {\n \"encoder_num_layer\": 8,\n \"decoder_rt_num_layer\": 2,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 1024,\n \"encoder_ff\": 1024,\n \"decoder_rt_ff\": 512,\n \"decoder_int_ff\": 512,\n \"n_head\": 2,\n \"drop_rate\": 0.7814344084863447,\n \"lr\": 0.030816743385966763,\n \"batch_size\": 1024\n },\n \"evaluated_params\": {\n \"batch_size\": 1024,\n \"decoder_int_ff\": 512,\n \"decoder_int_num_layer\": 1,\n \"decoder_rt_ff\": 512,\n \"decoder_rt_num_layer\": 2,\n \"drop_rate\": 0.7814344084863447,\n \"embedding_dim\": 1024,\n \"encoder_ff\": 1024,\n \"encoder_num_layer\": 8,\n \"lr\": 0.030816743385966763,\n \"n_head\": 2\n },\n \"experiment_tag\": \"32_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_rate=0.7814,embedding_dim=1024,encoder_ff=1024,encoder_num_layer=8,lr=0.0308,n_head=2\",\n \"stopping_criterion\": {},\n \"_setup_default_resource\": true,\n \"_default_placement_group_factory\": \"80054e2e\",\n \"placement_group_factory\": \"800595ba000000000000008c237261792e74756e652e657865637574696f6e2e706c6163656d656e745f67726f757073948c15506c6163656d656e7447726f7570466163746f72799493942981947d94288c085f62756e646c6573945d947d94288c03435055944740540000000000008c034750559447401000000000000075618c155f686561645f62756e646c655f69735f656d70747994898c095f7374726174656779948c045041434b948c055f6172677394298c075f6b7761726773947d9475622e\",\n \"log_to_file\": [\n null,\n null\n ],\n \"max_failures\": 0,\n \"_default_result_or_future\": null,\n \"export_formats\": [],\n \"status\": \"TERMINATED\",\n \"relative_logdir\": \"train_model_fe083630_32_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_22-50-21\",\n \"trial_name_creator\": null,\n \"trial_dirname_creator\": null,\n \"custom_trial_name\": null,\n \"custom_dirname\": null,\n \"restore_path\": null,\n \"_restore_checkpoint_result\": null,\n \"_state_json\": null,\n \"results\": \"80054e2e\",\n \"extra_arg\": \"80054e2e\",\n \"_resources\": \"80054e2e\"\n}",
+ "{\n \"start_time\": 1719350385.0636432,\n \"num_failures\": 0,\n \"num_failures_after_restore\": 0,\n \"error_filename\": null,\n \"pickled_error_filename\": null,\n \"last_result\": {\n \"loss\": 1961.6992687315453,\n \"timestamp\": 1719354625,\n \"checkpoint_dir_name\": \"checkpoint_000004\",\n \"should_checkpoint\": true,\n \"done\": false,\n \"training_iteration\": 5,\n \"trial_id\": \"fe083630\",\n \"date\": \"2024-06-26_00-30-25\",\n \"time_this_iter_s\": 845.8089480400085,\n \"time_total_s\": 4240.718232393265,\n \"pid\": 2195253,\n \"hostname\": \"r8i6n2\",\n \"node_ip\": \"10.159.28.60\",\n \"config\": {\n \"encoder_num_layer\": 8,\n \"decoder_rt_num_layer\": 2,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 1024,\n \"encoder_ff\": 1024,\n \"decoder_rt_ff\": 512,\n \"decoder_int_ff\": 512,\n \"n_head\": 2,\n \"drop_rate\": 0.7814344084863447,\n \"lr\": 0.030816743385966763,\n \"batch_size\": 1024\n },\n \"time_since_restore\": 4240.718232393265,\n \"iterations_since_restore\": 5,\n \"experiment_tag\": \"32_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_rate=0.7814,embedding_dim=1024,encoder_ff=1024,encoder_num_layer=8,lr=0.0308,n_head=2\"\n },\n \"last_result_time\": 1719354625.8083603,\n \"metric_analysis\": {\n \"loss\": {\n \"max\": 4034.1917479507565,\n \"min\": 1961.6992687315453,\n \"avg\": 2624.951602004645,\n \"last\": 1961.6992687315453,\n \"last-5-avg\": 2624.9516020046444,\n \"last-10-avg\": 2624.9516020046444\n },\n \"should_checkpoint\": {\n \"max\": true,\n \"min\": true,\n \"avg\": 1.0,\n \"last\": true,\n \"last-5-avg\": 1.0,\n \"last-10-avg\": 1.0\n },\n \"done\": {\n \"max\": false,\n \"min\": false,\n \"avg\": 0.0,\n \"last\": false,\n \"last-5-avg\": 0.0,\n \"last-10-avg\": 0.0\n },\n \"training_iteration\": {\n \"max\": 5,\n \"min\": 1,\n \"avg\": 3.0,\n \"last\": 5,\n \"last-5-avg\": 3.0,\n \"last-10-avg\": 3.0\n },\n \"time_this_iter_s\": {\n \"max\": 858.9566829204559,\n \"min\": 845.1621363162994,\n \"avg\": 848.143646478653,\n \"last\": 845.8089480400085,\n \"last-5-avg\": 848.143646478653,\n \"last-10-avg\": 848.143646478653\n },\n \"time_total_s\": {\n \"max\": 4240.718232393265,\n \"min\": 858.9566829204559,\n \"avg\": 2549.6743980884553,\n \"last\": 4240.718232393265,\n \"last-5-avg\": 2549.6743980884553,\n \"last-10-avg\": 2549.6743980884553\n },\n \"time_since_restore\": {\n \"max\": 4240.718232393265,\n \"min\": 858.9566829204559,\n \"avg\": 2549.6743980884553,\n \"last\": 4240.718232393265,\n \"last-5-avg\": 2549.6743980884553,\n \"last-10-avg\": 2549.6743980884553\n },\n \"iterations_since_restore\": {\n \"max\": 5,\n \"min\": 1,\n \"avg\": 3.0,\n \"last\": 5,\n \"last-5-avg\": 3.0,\n \"last-10-avg\": 3.0\n }\n },\n \"_n_steps\": [\n 5,\n 10\n ],\n \"metric_n_steps\": {\n \"loss\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"800595d6000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294288c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b0074946243082693c92c6284af4094869452946807680d4308ecf57a5da882a64094869452946807680d4308ee76bb157c7ea14094869452946807680d4308b55aade62e619f4094869452946807680d430869341a0dcca69e409486945294652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"800595d6000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294288c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b0074946243082693c92c6284af4094869452946807680d4308ecf57a5da882a64094869452946807680d4308ee76bb157c7ea14094869452946807680d4308b55aade62e619f4094869452946807680d430869341a0dcca69e409486945294652e\"\n }\n },\n \"should_checkpoint\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059527000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294288888888888652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059527000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294288888888888652e\"\n }\n },\n \"done\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059527000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294288989898989652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059527000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294288989898989652e\"\n }\n },\n \"training_iteration\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952c000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294284b014b024b034b044b05652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952c000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294284b014b024b034b044b05652e\"\n }\n },\n \"time_this_iter_s\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005954f000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452942847408ad7a74960000047408a694c0e20000047408a6c66b640000047408a69ec2920000047408a6e78b9c00000652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005954f000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452942847408ad7a74960000047408a694c0e20000047408a6c66b640000047408a69ec2920000047408a6e78b9c00000652e\"\n }\n },\n \"time_total_s\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005954f000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452942847408ad7a74960000047409aa079abc000004740a3eb56837000004740aa85d18db800004740b090b7de140000652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005954f000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452942847408ad7a74960000047409aa079abc000004740a3eb56837000004740aa85d18db800004740b090b7de140000652e\"\n }\n },\n \"time_since_restore\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005954f000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452942847408ad7a74960000047409aa079abc000004740a3eb56837000004740aa85d18db800004740b090b7de140000652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005954f000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452942847408ad7a74960000047409aa079abc000004740a3eb56837000004740aa85d18db800004740b090b7de140000652e\"\n }\n },\n \"iterations_since_restore\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952c000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294284b014b024b034b044b05652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952c000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294284b014b024b034b044b05652e\"\n }\n }\n },\n \"checkpoint_manager\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005954f140000000000008c267261792e747261696e2e5f696e7465726e616c2e636865636b706f696e745f6d616e61676572948c125f436865636b706f696e744d616e616765729493942981947d94288c125f636865636b706f696e745f636f6e666967948c097261792e747261696e948c10436865636b706f696e74436f6e6669679493942981947d94288c0b6e756d5f746f5f6b656570944e8c1a636865636b706f696e745f73636f72655f617474726962757465944e8c16636865636b706f696e745f73636f72655f6f72646572948c036d6178948c14636865636b706f696e745f6672657175656e6379944b008c11636865636b706f696e745f61745f656e6494898c1a5f636865636b706f696e745f6b6565705f616c6c5f72616e6b73948c0a44455052454341544544948c1f5f636865636b706f696e745f75706c6f61645f66726f6d5f776f726b65727394681275628c135f636865636b706f696e745f726573756c7473945d94288c1b7261792e747261696e2e5f696e7465726e616c2e73657373696f6e948c0f5f547261696e696e67526573756c749493942981947d94288c0a636865636b706f696e749468068c0a436865636b706f696e749493942981947d94288c0470617468948cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f66653038333633305f33325f62617463685f73697a653d313032342c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d3531322c6465636f6465725f72745f6e756d5f6c617965723d322c64726f705f725f323032342d30362d32355f32322d35302d32312f636865636b706f696e745f303030303030948c0a66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c055f75756964948c0475756964948c04555549449493942981947d948c03696e74948a112ec45172fa7983bffa44a44d0e218ae900736275628c076d657472696373947d94288c046c6f7373948c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b0074946243082693c92c6284af4094869452948c0974696d657374616d70944acb377b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303030948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b018c08747269616c5f6964948c086665303833363330948c0464617465948c13323032342d30362d32355f32332d33342d3034948c1074696d655f746869735f697465725f739447408ad7a7496000008c0c74696d655f746f74616c5f739447408ad7a7496000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b088c146465636f6465725f72745f6e756d5f6c61796572944b028c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944d00048c0a656e636f6465725f6666944d00048c0d6465636f6465725f72745f6666944d00028c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b028c0964726f705f7261746594473fe90182bb8d5fe48c026c7294473f9f8e6ca40c3d078c0a62617463685f73697a65944d0004758c1274696d655f73696e63655f726573746f72659447408ad7a7496000008c18697465726174696f6e735f73696e63655f726573746f7265944b0175756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f66653038333633305f33325f62617463685f73697a653d313032342c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d3531322c6465636f6465725f72745f6e756d5f6c617965723d322c64726f705f725f323032342d30362d32355f32322d35302d32312f636865636b706f696e745f30303030303194682268257d946827897385945294682a682d2981947d9468308a10955cd877237e09ba474c31d05669167a7362756268317d94288c046c6f7373946836683c4308ecf57a5da882a64094869452948c0974696d657374616d70944a183b7b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303031948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b028c08747269616c5f6964948c086665303833363330948c0464617465948c13323032342d30362d32355f32332d34382d3039948c1074696d655f746869735f697465725f739447408a694c0e2000008c0c74696d655f746f74616c5f739447409aa079abc000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b088c146465636f6465725f72745f6e756d5f6c61796572944b028c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944d00048c0a656e636f6465725f6666944d00048c0d6465636f6465725f72745f6666944d00028c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b028c0964726f705f7261746594473fe90182bb8d5fe48c026c7294473f9f8e6ca40c3d078c0a62617463685f73697a65944d0004758c1274696d655f73696e63655f726573746f72659447409aa079abc000008c18697465726174696f6e735f73696e63655f726573746f7265944b0275756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f66653038333633305f33325f62617463685f73697a653d313032342c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d3531322c6465636f6465725f72745f6e756d5f6c617965723d322c64726f705f725f323032342d30362d32355f32322d35302d32312f636865636b706f696e745f30303030303294682268257d946827897385945294682a682d2981947d9468308a1034ac3c87313821b8c74393a0b5d49d377362756268317d94288c046c6f7373946836683c4308ee76bb157c7ea14094869452948c0974696d657374616d70944a663e7b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303032948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b038c08747269616c5f6964948c086665303833363330948c0464617465948c13323032342d30362d32365f30302d30322d3134948c1074696d655f746869735f697465725f739447408a6c66b64000008c0c74696d655f746f74616c5f73944740a3eb56837000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b088c146465636f6465725f72745f6e756d5f6c61796572944b028c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944d00048c0a656e636f6465725f6666944d00048c0d6465636f6465725f72745f6666944d00028c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b028c0964726f705f7261746594473fe90182bb8d5fe48c026c7294473f9f8e6ca40c3d078c0a62617463685f73697a65944d0004758c1274696d655f73696e63655f726573746f7265944740a3eb56837000008c18697465726174696f6e735f73696e63655f726573746f7265944b0375756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f66653038333633305f33325f62617463685f73697a653d313032342c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d3531322c6465636f6465725f72745f6e756d5f6c617965723d322c64726f705f725f323032342d30362d32355f32322d35302d32312f636865636b706f696e745f30303030303394682268257d946827897385945294682a682d2981947d9468308a11cc60ea54f385009b774823ed0f83bb96007362756268317d94288c046c6f7373946836683c4308b55aade62e619f4094869452948c0974696d657374616d70944ab3417b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303033948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b048c08747269616c5f6964948c086665303833363330948c0464617465948c13323032342d30362d32365f30302d31362d3139948c1074696d655f746869735f697465725f739447408a69ec292000008c0c74696d655f746f74616c5f73944740aa85d18db800008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b088c146465636f6465725f72745f6e756d5f6c61796572944b028c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944d00048c0a656e636f6465725f6666944d00048c0d6465636f6465725f72745f6666944d00028c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b028c0964726f705f7261746594473fe90182bb8d5fe48c026c7294473f9f8e6ca40c3d078c0a62617463685f73697a65944d0004758c1274696d655f73696e63655f726573746f7265944740aa85d18db800008c18697465726174696f6e735f73696e63655f726573746f7265944b0475756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f66653038333633305f33325f62617463685f73697a653d313032342c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d3531322c6465636f6465725f72745f6e756d5f6c617965723d322c64726f705f725f323032342d30362d32355f32322d35302d32312f636865636b706f696e745f30303030303494682268257d946827897385945294682a682d2981947d9468308a105c998f80b0fad298484f48365ecb77397362756268317d94288c046c6f7373946836683c430869341a0dcca69e4094869452948c0974696d657374616d70944a01457b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303034948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b058c08747269616c5f6964948c086665303833363330948c0464617465948c13323032342d30362d32365f30302d33302d3235948c1074696d655f746869735f697465725f739447408a6e78b9c000008c0c74696d655f746f74616c5f73944740b090b7de1400008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b088c146465636f6465725f72745f6e756d5f6c61796572944b028c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944d00048c0a656e636f6465725f6666944d00048c0d6465636f6465725f72745f6666944d00028c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b028c0964726f705f7261746594473fe90182bb8d5fe48c026c7294473f9f8e6ca40c3d078c0a62617463685f73697a65944d0004758c1274696d655f73696e63655f726573746f7265944740b090b7de1400008c18697465726174696f6e735f73696e63655f726573746f7265944b05757562658c195f6c61746573745f636865636b706f696e745f726573756c749468ef75622e\"\n }\n}"
+ ]
+ ],
+ "runner_data": {
+ "_earliest_stopping_actor": 401634.092423264,
+ "_actor_cleanup_timeout": 600,
+ "_actor_force_cleanup_timeout": 10,
+ "_reuse_actors": true,
+ "_buffer_length": 1,
+ "_buffer_min_time_s": 0.0,
+ "_buffer_max_time_s": 100.0,
+ "_max_pending_trials": 1,
+ "_metric": "loss",
+ "_total_time": 30888.675180912018,
+ "_iteration": 300268,
+ "_has_errored": false,
+ "_fail_fast": false,
+ "_print_trial_errors": true,
+ "_cached_trial_decisions": {},
+ "_queued_trial_decisions": {},
+ "_should_stop_experiment": false,
+ "_stopper": {
+ "_type": "CLOUDPICKLE_FALLBACK",
+ "value": "8005952c000000000000008c157261792e74756e652e73746f707065722e6e6f6f70948c0b4e6f6f7053746f707065729493942981942e"
+ },
+ "_start_time": 1719328200.4031954,
+ "_last_checkpoint_time": -Infinity,
+ "_session_str": "2024-06-25_17-10-00",
+ "_checkpoint_period": "auto",
+ "_trial_checkpoint_config": {
+ "_type": "CLOUDPICKLE_FALLBACK",
+ "value": "800595f2000000000000008c097261792e747261696e948c10436865636b706f696e74436f6e6669679493942981947d94288c0b6e756d5f746f5f6b656570944e8c1a636865636b706f696e745f73636f72655f617474726962757465944e8c16636865636b706f696e745f73636f72655f6f72646572948c036d6178948c14636865636b706f696e745f6672657175656e6379944b008c11636865636b706f696e745f61745f656e6494898c1a5f636865636b706f696e745f6b6565705f616c6c5f72616e6b73948c0a44455052454341544544948c1f5f636865636b706f696e745f75706c6f61645f66726f6d5f776f726b65727394680c75622e"
+ },
+ "_resumed": false
+ },
+ "stats": {
+ "start_time": 1719328200.4031954,
+ "timestamp": -Infinity
+ }
+}
\ No newline at end of file
diff --git a/ray_results/test_experiment/search_gen_state-2024-06-25_15-19-46.json b/ray_results/test_experiment/search_gen_state-2024-06-25_15-19-46.json
new file mode 100644
index 0000000000000000000000000000000000000000..bd56b6ad2aec0c95c881d75662ca024e9d960928
Binary files /dev/null and b/ray_results/test_experiment/search_gen_state-2024-06-25_15-19-46.json differ
diff --git a/ray_results/test_experiment/search_gen_state-2024-06-25_16-07-48.json b/ray_results/test_experiment/search_gen_state-2024-06-25_16-07-48.json
new file mode 100644
index 0000000000000000000000000000000000000000..c109608d0177f83f265c982cca5e36e6eed43cbc
Binary files /dev/null and b/ray_results/test_experiment/search_gen_state-2024-06-25_16-07-48.json differ
diff --git a/ray_results/test_experiment/search_gen_state-2024-06-25_17-10-00.json b/ray_results/test_experiment/search_gen_state-2024-06-25_17-10-00.json
new file mode 100644
index 0000000000000000000000000000000000000000..2a60067bb11a8389929614a20cceda74b2a2a3ce
Binary files /dev/null and b/ray_results/test_experiment/search_gen_state-2024-06-25_17-10-00.json differ
diff --git a/ray_results/test_experiment/searcher-state-2024-06-25_15-19-46.pkl b/ray_results/test_experiment/searcher-state-2024-06-25_15-19-46.pkl
new file mode 100644
index 0000000000000000000000000000000000000000..1413e7a92ee7ac42e62bf31f6637bfdebb340a7d
Binary files /dev/null and b/ray_results/test_experiment/searcher-state-2024-06-25_15-19-46.pkl differ
diff --git a/ray_results/test_experiment/searcher-state-2024-06-25_16-07-48.pkl b/ray_results/test_experiment/searcher-state-2024-06-25_16-07-48.pkl
new file mode 100644
index 0000000000000000000000000000000000000000..304ac7bdae36124fb18b38216267623b0f8202b8
Binary files /dev/null and b/ray_results/test_experiment/searcher-state-2024-06-25_16-07-48.pkl differ
diff --git a/ray_results/test_experiment/searcher-state-2024-06-25_17-10-00.pkl b/ray_results/test_experiment/searcher-state-2024-06-25_17-10-00.pkl
new file mode 100644
index 0000000000000000000000000000000000000000..862455da2971212f5b405296f4c3e232559a06d0
Binary files /dev/null and b/ray_results/test_experiment/searcher-state-2024-06-25_17-10-00.pkl differ
diff --git a/ray_results/test_experiment/train_model_040fcf09_8_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=4,drop_ra_2024-06-25_17-31-01/events.out.tfevents.1719329545.r8i6n2 b/ray_results/test_experiment/train_model_040fcf09_8_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=4,drop_ra_2024-06-25_17-31-01/events.out.tfevents.1719329545.r8i6n2
new file mode 100644
index 0000000000000000000000000000000000000000..a4b8f2b87bd1c314e64ca0fd1a3070aa0e558e5d
Binary files /dev/null and b/ray_results/test_experiment/train_model_040fcf09_8_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=4,drop_ra_2024-06-25_17-31-01/events.out.tfevents.1719329545.r8i6n2 differ
diff --git a/ray_results/test_experiment/train_model_040fcf09_8_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=4,drop_ra_2024-06-25_17-31-01/params.json b/ray_results/test_experiment/train_model_040fcf09_8_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=4,drop_ra_2024-06-25_17-31-01/params.json
new file mode 100644
index 0000000000000000000000000000000000000000..b034105813c1160d84c755fb8e8a11adf89722af
--- /dev/null
+++ b/ray_results/test_experiment/train_model_040fcf09_8_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=4,drop_ra_2024-06-25_17-31-01/params.json
@@ -0,0 +1,13 @@
+{
+ "batch_size": 2048,
+ "decoder_int_ff": 512,
+ "decoder_int_num_layer": 1,
+ "decoder_rt_ff": 512,
+ "decoder_rt_num_layer": 4,
+ "drop_rate": 0.18759871111111892,
+ "embedding_dim": 16,
+ "encoder_ff": 1024,
+ "encoder_num_layer": 1,
+ "lr": 0.02750689103642202,
+ "n_head": 1
+}
\ No newline at end of file
diff --git a/ray_results/test_experiment/train_model_040fcf09_8_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=4,drop_ra_2024-06-25_17-31-01/params.pkl b/ray_results/test_experiment/train_model_040fcf09_8_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=4,drop_ra_2024-06-25_17-31-01/params.pkl
new file mode 100644
index 0000000000000000000000000000000000000000..cb40d5843f9214339d4eed4e4022ca573d6a2b4f
Binary files /dev/null and b/ray_results/test_experiment/train_model_040fcf09_8_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=4,drop_ra_2024-06-25_17-31-01/params.pkl differ
diff --git a/ray_results/test_experiment/train_model_040fcf09_8_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=4,drop_ra_2024-06-25_17-31-01/progress.csv b/ray_results/test_experiment/train_model_040fcf09_8_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=4,drop_ra_2024-06-25_17-31-01/progress.csv
new file mode 100644
index 0000000000000000000000000000000000000000..53f618e48b1bd08b2d7679df866d1bd237cdda35
--- /dev/null
+++ b/ray_results/test_experiment/train_model_040fcf09_8_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=4,drop_ra_2024-06-25_17-31-01/progress.csv
@@ -0,0 +1,2 @@
+loss,timestamp,checkpoint_dir_name,should_checkpoint,done,training_iteration,trial_id,date,time_this_iter_s,time_total_s,pid,hostname,node_ip,time_since_restore,iterations_since_restore
+2019.4176592488927,1719329610,checkpoint_000000,True,False,1,040fcf09,2024-06-25_17-33-30,65.59718012809753,65.59718012809753,2195253,r8i6n2,10.159.28.60,65.59718012809753,1
diff --git a/ray_results/test_experiment/train_model_040fcf09_8_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=4,drop_ra_2024-06-25_17-31-01/result.json b/ray_results/test_experiment/train_model_040fcf09_8_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=4,drop_ra_2024-06-25_17-31-01/result.json
new file mode 100644
index 0000000000000000000000000000000000000000..ecbddee35871bcbd1ad4419902fbbd91c91463c8
--- /dev/null
+++ b/ray_results/test_experiment/train_model_040fcf09_8_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=4,drop_ra_2024-06-25_17-31-01/result.json
@@ -0,0 +1 @@
+{"loss": 2019.4176592488927, "timestamp": 1719329610, "checkpoint_dir_name": "checkpoint_000000", "should_checkpoint": true, "done": false, "training_iteration": 1, "trial_id": "040fcf09", "date": "2024-06-25_17-33-30", "time_this_iter_s": 65.59718012809753, "time_total_s": 65.59718012809753, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 1, "decoder_rt_num_layer": 4, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 1024, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 1, "drop_rate": 0.18759871111111892, "lr": 0.02750689103642202, "batch_size": 2048}, "time_since_restore": 65.59718012809753, "iterations_since_restore": 1}
diff --git a/ray_results/test_experiment/train_model_0f80a6d7_31_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_22-45-39/events.out.tfevents.1719348621.r8i6n2 b/ray_results/test_experiment/train_model_0f80a6d7_31_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_22-45-39/events.out.tfevents.1719348621.r8i6n2
new file mode 100644
index 0000000000000000000000000000000000000000..3e4fe552af2e0836befa83eef18b9fff71dcdd87
Binary files /dev/null and b/ray_results/test_experiment/train_model_0f80a6d7_31_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_22-45-39/events.out.tfevents.1719348621.r8i6n2 differ
diff --git a/ray_results/test_experiment/train_model_0f80a6d7_31_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_22-45-39/events.out.tfevents.1719355502.r8i6n2 b/ray_results/test_experiment/train_model_0f80a6d7_31_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_22-45-39/events.out.tfevents.1719355502.r8i6n2
new file mode 100644
index 0000000000000000000000000000000000000000..fcfab8bfd030b3cca70231ad4044c1ae29c1a234
Binary files /dev/null and b/ray_results/test_experiment/train_model_0f80a6d7_31_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_22-45-39/events.out.tfevents.1719355502.r8i6n2 differ
diff --git a/ray_results/test_experiment/train_model_0f80a6d7_31_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_22-45-39/params.json b/ray_results/test_experiment/train_model_0f80a6d7_31_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_22-45-39/params.json
new file mode 100644
index 0000000000000000000000000000000000000000..4db5d85861f98c4ada0ca68997102edfbf869b40
--- /dev/null
+++ b/ray_results/test_experiment/train_model_0f80a6d7_31_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_22-45-39/params.json
@@ -0,0 +1,13 @@
+{
+ "batch_size": 2048,
+ "decoder_int_ff": 512,
+ "decoder_int_num_layer": 1,
+ "decoder_rt_ff": 512,
+ "decoder_rt_num_layer": 2,
+ "drop_rate": 0.023920599186226577,
+ "embedding_dim": 1024,
+ "encoder_ff": 1024,
+ "encoder_num_layer": 2,
+ "lr": 0.0024687399238039302,
+ "n_head": 16
+}
\ No newline at end of file
diff --git a/ray_results/test_experiment/train_model_0f80a6d7_31_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_22-45-39/params.pkl b/ray_results/test_experiment/train_model_0f80a6d7_31_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_22-45-39/params.pkl
new file mode 100644
index 0000000000000000000000000000000000000000..611a751f78351d0edeaa1fbe25feacf85cea253e
Binary files /dev/null and b/ray_results/test_experiment/train_model_0f80a6d7_31_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_22-45-39/params.pkl differ
diff --git a/ray_results/test_experiment/train_model_0f80a6d7_31_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_22-45-39/progress.csv b/ray_results/test_experiment/train_model_0f80a6d7_31_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_22-45-39/progress.csv
new file mode 100644
index 0000000000000000000000000000000000000000..946ee52dcdbeb3f12442b247aab4d57dcadf3ec3
--- /dev/null
+++ b/ray_results/test_experiment/train_model_0f80a6d7_31_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_22-45-39/progress.csv
@@ -0,0 +1,11 @@
+loss,timestamp,checkpoint_dir_name,should_checkpoint,done,training_iteration,trial_id,date,time_this_iter_s,time_total_s,pid,hostname,node_ip,time_since_restore,iterations_since_restore
+1986.2198524775467,1719348983,checkpoint_000000,True,False,1,0f80a6d7,2024-06-25_22-56-24,363.18635082244873,363.18635082244873,2195253,r8i6n2,10.159.28.60,363.18635082244873,1
+1979.5493135226993,1719349334,checkpoint_000001,True,False,2,0f80a6d7,2024-06-25_23-02-14,350.295019865036,713.4813706874847,2195253,r8i6n2,10.159.28.60,713.4813706874847,2
+1964.9348182978592,1719349683,checkpoint_000002,True,False,3,0f80a6d7,2024-06-25_23-08-04,349.6246259212494,1063.1059966087341,2195253,r8i6n2,10.159.28.60,1063.1059966087341,3
+1955.6751497523992,1719350034,checkpoint_000003,True,False,4,0f80a6d7,2024-06-25_23-13-54,350.46285796165466,1413.5688545703888,2195253,r8i6n2,10.159.28.60,1413.5688545703888,4
+1956.1783514548474,1719350384,checkpoint_000004,True,False,5,0f80a6d7,2024-06-25_23-19-44,350.18478202819824,1763.753636598587,2195253,r8i6n2,10.159.28.60,1763.753636598587,5
+1971.8988527312993,1719355866,checkpoint_000005,True,False,6,0f80a6d7,2024-06-26_00-51-07,365.2865264415741,2129.040163040161,2850562,r8i6n2,10.159.28.60,365.2865264415741,1
+1955.6729476808564,1719356216,checkpoint_000006,True,False,7,0f80a6d7,2024-06-26_00-56-56,349.18421268463135,2478.2243757247925,2850562,r8i6n2,10.159.28.60,714.4707391262054,2
+1957.4972913847196,1719356565,checkpoint_000007,True,False,8,0f80a6d7,2024-06-26_01-02-46,349.7124717235565,2827.936847448349,2850562,r8i6n2,10.159.28.60,1064.183210849762,3
+1967.2758471871923,1719356915,checkpoint_000008,True,False,9,0f80a6d7,2024-06-26_01-08-35,349.3587329387665,3177.2955803871155,2850562,r8i6n2,10.159.28.60,1413.5419437885284,4
+1959.91755063515,1719357265,checkpoint_000009,True,True,10,0f80a6d7,2024-06-26_01-14-25,349.87603521347046,3527.171615600586,2850562,r8i6n2,10.159.28.60,1763.417979001999,5
diff --git a/ray_results/test_experiment/train_model_0f80a6d7_31_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_22-45-39/result.json b/ray_results/test_experiment/train_model_0f80a6d7_31_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_22-45-39/result.json
new file mode 100644
index 0000000000000000000000000000000000000000..5dc123f57c5e2efe0b8940579ad73ad6f88d1357
--- /dev/null
+++ b/ray_results/test_experiment/train_model_0f80a6d7_31_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_22-45-39/result.json
@@ -0,0 +1,10 @@
+{"loss": 1986.2198524775467, "timestamp": 1719348983, "checkpoint_dir_name": "checkpoint_000000", "should_checkpoint": true, "done": false, "training_iteration": 1, "trial_id": "0f80a6d7", "date": "2024-06-25_22-56-24", "time_this_iter_s": 363.18635082244873, "time_total_s": 363.18635082244873, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 2, "decoder_rt_num_layer": 2, "decoder_int_num_layer": 1, "embedding_dim": 1024, "encoder_ff": 1024, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 16, "drop_rate": 0.023920599186226577, "lr": 0.0024687399238039302, "batch_size": 2048}, "time_since_restore": 363.18635082244873, "iterations_since_restore": 1}
+{"loss": 1979.5493135226993, "timestamp": 1719349334, "checkpoint_dir_name": "checkpoint_000001", "should_checkpoint": true, "done": false, "training_iteration": 2, "trial_id": "0f80a6d7", "date": "2024-06-25_23-02-14", "time_this_iter_s": 350.295019865036, "time_total_s": 713.4813706874847, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 2, "decoder_rt_num_layer": 2, "decoder_int_num_layer": 1, "embedding_dim": 1024, "encoder_ff": 1024, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 16, "drop_rate": 0.023920599186226577, "lr": 0.0024687399238039302, "batch_size": 2048}, "time_since_restore": 713.4813706874847, "iterations_since_restore": 2}
+{"loss": 1964.9348182978592, "timestamp": 1719349683, "checkpoint_dir_name": "checkpoint_000002", "should_checkpoint": true, "done": false, "training_iteration": 3, "trial_id": "0f80a6d7", "date": "2024-06-25_23-08-04", "time_this_iter_s": 349.6246259212494, "time_total_s": 1063.1059966087341, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 2, "decoder_rt_num_layer": 2, "decoder_int_num_layer": 1, "embedding_dim": 1024, "encoder_ff": 1024, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 16, "drop_rate": 0.023920599186226577, "lr": 0.0024687399238039302, "batch_size": 2048}, "time_since_restore": 1063.1059966087341, "iterations_since_restore": 3}
+{"loss": 1955.6751497523992, "timestamp": 1719350034, "checkpoint_dir_name": "checkpoint_000003", "should_checkpoint": true, "done": false, "training_iteration": 4, "trial_id": "0f80a6d7", "date": "2024-06-25_23-13-54", "time_this_iter_s": 350.46285796165466, "time_total_s": 1413.5688545703888, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 2, "decoder_rt_num_layer": 2, "decoder_int_num_layer": 1, "embedding_dim": 1024, "encoder_ff": 1024, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 16, "drop_rate": 0.023920599186226577, "lr": 0.0024687399238039302, "batch_size": 2048}, "time_since_restore": 1413.5688545703888, "iterations_since_restore": 4}
+{"loss": 1956.1783514548474, "timestamp": 1719350384, "checkpoint_dir_name": "checkpoint_000004", "should_checkpoint": true, "done": false, "training_iteration": 5, "trial_id": "0f80a6d7", "date": "2024-06-25_23-19-44", "time_this_iter_s": 350.18478202819824, "time_total_s": 1763.753636598587, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 2, "decoder_rt_num_layer": 2, "decoder_int_num_layer": 1, "embedding_dim": 1024, "encoder_ff": 1024, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 16, "drop_rate": 0.023920599186226577, "lr": 0.0024687399238039302, "batch_size": 2048}, "time_since_restore": 1763.753636598587, "iterations_since_restore": 5}
+{"loss": 1971.8988527312993, "timestamp": 1719355866, "checkpoint_dir_name": "checkpoint_000005", "should_checkpoint": true, "done": false, "training_iteration": 6, "trial_id": "0f80a6d7", "date": "2024-06-26_00-51-07", "time_this_iter_s": 365.2865264415741, "time_total_s": 2129.040163040161, "pid": 2850562, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 2, "decoder_rt_num_layer": 2, "decoder_int_num_layer": 1, "embedding_dim": 1024, "encoder_ff": 1024, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 16, "drop_rate": 0.023920599186226577, "lr": 0.0024687399238039302, "batch_size": 2048}, "time_since_restore": 365.2865264415741, "iterations_since_restore": 1}
+{"loss": 1955.6729476808564, "timestamp": 1719356216, "checkpoint_dir_name": "checkpoint_000006", "should_checkpoint": true, "done": false, "training_iteration": 7, "trial_id": "0f80a6d7", "date": "2024-06-26_00-56-56", "time_this_iter_s": 349.18421268463135, "time_total_s": 2478.2243757247925, "pid": 2850562, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 2, "decoder_rt_num_layer": 2, "decoder_int_num_layer": 1, "embedding_dim": 1024, "encoder_ff": 1024, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 16, "drop_rate": 0.023920599186226577, "lr": 0.0024687399238039302, "batch_size": 2048}, "time_since_restore": 714.4707391262054, "iterations_since_restore": 2}
+{"loss": 1957.4972913847196, "timestamp": 1719356565, "checkpoint_dir_name": "checkpoint_000007", "should_checkpoint": true, "done": false, "training_iteration": 8, "trial_id": "0f80a6d7", "date": "2024-06-26_01-02-46", "time_this_iter_s": 349.7124717235565, "time_total_s": 2827.936847448349, "pid": 2850562, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 2, "decoder_rt_num_layer": 2, "decoder_int_num_layer": 1, "embedding_dim": 1024, "encoder_ff": 1024, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 16, "drop_rate": 0.023920599186226577, "lr": 0.0024687399238039302, "batch_size": 2048}, "time_since_restore": 1064.183210849762, "iterations_since_restore": 3}
+{"loss": 1967.2758471871923, "timestamp": 1719356915, "checkpoint_dir_name": "checkpoint_000008", "should_checkpoint": true, "done": false, "training_iteration": 9, "trial_id": "0f80a6d7", "date": "2024-06-26_01-08-35", "time_this_iter_s": 349.3587329387665, "time_total_s": 3177.2955803871155, "pid": 2850562, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 2, "decoder_rt_num_layer": 2, "decoder_int_num_layer": 1, "embedding_dim": 1024, "encoder_ff": 1024, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 16, "drop_rate": 0.023920599186226577, "lr": 0.0024687399238039302, "batch_size": 2048}, "time_since_restore": 1413.5419437885284, "iterations_since_restore": 4}
+{"loss": 1959.91755063515, "timestamp": 1719357265, "checkpoint_dir_name": "checkpoint_000009", "should_checkpoint": true, "done": true, "training_iteration": 10, "trial_id": "0f80a6d7", "date": "2024-06-26_01-14-25", "time_this_iter_s": 349.87603521347046, "time_total_s": 3527.171615600586, "pid": 2850562, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 2, "decoder_rt_num_layer": 2, "decoder_int_num_layer": 1, "embedding_dim": 1024, "encoder_ff": 1024, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 16, "drop_rate": 0.023920599186226577, "lr": 0.0024687399238039302, "batch_size": 2048}, "time_since_restore": 1763.417979001999, "iterations_since_restore": 5}
diff --git a/ray_results/test_experiment/train_model_0f9cc97d_22_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_18-53-00/events.out.tfevents.1719334649.r8i6n2 b/ray_results/test_experiment/train_model_0f9cc97d_22_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_18-53-00/events.out.tfevents.1719334649.r8i6n2
new file mode 100644
index 0000000000000000000000000000000000000000..dc0dea9a1381339a860c4465b489b19537d51c90
Binary files /dev/null and b/ray_results/test_experiment/train_model_0f9cc97d_22_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_18-53-00/events.out.tfevents.1719334649.r8i6n2 differ
diff --git a/ray_results/test_experiment/train_model_0f9cc97d_22_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_18-53-00/events.out.tfevents.1719338738.r8i6n2 b/ray_results/test_experiment/train_model_0f9cc97d_22_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_18-53-00/events.out.tfevents.1719338738.r8i6n2
new file mode 100644
index 0000000000000000000000000000000000000000..dce79638b4556a1330da3fa9372c75e4df06ad7f
Binary files /dev/null and b/ray_results/test_experiment/train_model_0f9cc97d_22_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_18-53-00/events.out.tfevents.1719338738.r8i6n2 differ
diff --git a/ray_results/test_experiment/train_model_0f9cc97d_22_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_18-53-00/events.out.tfevents.1719340670.r8i6n2 b/ray_results/test_experiment/train_model_0f9cc97d_22_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_18-53-00/events.out.tfevents.1719340670.r8i6n2
new file mode 100644
index 0000000000000000000000000000000000000000..f808e7d784a188e9088b8ff8d17f1f6ebef71267
Binary files /dev/null and b/ray_results/test_experiment/train_model_0f9cc97d_22_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_18-53-00/events.out.tfevents.1719340670.r8i6n2 differ
diff --git a/ray_results/test_experiment/train_model_0f9cc97d_22_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_18-53-00/params.json b/ray_results/test_experiment/train_model_0f9cc97d_22_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_18-53-00/params.json
new file mode 100644
index 0000000000000000000000000000000000000000..3eb44acf7d5e673ecf6a6120c8425ce117b0d245
--- /dev/null
+++ b/ray_results/test_experiment/train_model_0f9cc97d_22_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_18-53-00/params.json
@@ -0,0 +1,13 @@
+{
+ "batch_size": 2048,
+ "decoder_int_ff": 512,
+ "decoder_int_num_layer": 1,
+ "decoder_rt_ff": 512,
+ "decoder_rt_num_layer": 2,
+ "drop_rate": 0.005508735010543275,
+ "embedding_dim": 1024,
+ "encoder_ff": 1024,
+ "encoder_num_layer": 4,
+ "lr": 0.0011880094129501908,
+ "n_head": 1
+}
\ No newline at end of file
diff --git a/ray_results/test_experiment/train_model_0f9cc97d_22_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_18-53-00/params.pkl b/ray_results/test_experiment/train_model_0f9cc97d_22_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_18-53-00/params.pkl
new file mode 100644
index 0000000000000000000000000000000000000000..0b14b147032aaa52de8463ec6daaf91430fbcb3f
Binary files /dev/null and b/ray_results/test_experiment/train_model_0f9cc97d_22_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_18-53-00/params.pkl differ
diff --git a/ray_results/test_experiment/train_model_0f9cc97d_22_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_18-53-00/progress.csv b/ray_results/test_experiment/train_model_0f9cc97d_22_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_18-53-00/progress.csv
new file mode 100644
index 0000000000000000000000000000000000000000..56220b4b4e7ad1948c8e27282d5467e86953c013
--- /dev/null
+++ b/ray_results/test_experiment/train_model_0f9cc97d_22_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_18-53-00/progress.csv
@@ -0,0 +1,15 @@
+loss,timestamp,checkpoint_dir_name,should_checkpoint,done,training_iteration,trial_id,date,time_this_iter_s,time_total_s,pid,hostname,node_ip,time_since_restore,iterations_since_restore
+1956.0084007443406,1719335168,checkpoint_000000,True,False,1,0f9cc97d,2024-06-25_19-06-08,519.6532871723175,519.6532871723175,2195253,r8i6n2,10.159.28.60,519.6532871723175,1
+1970.2912184347317,1719335674,checkpoint_000001,True,False,2,0f9cc97d,2024-06-25_19-14-34,505.92808413505554,1025.581371307373,2195253,r8i6n2,10.159.28.60,1025.581371307373,2
+1954.4228304164617,1719339260,checkpoint_000002,True,False,3,0f9cc97d,2024-06-25_20-14-21,522.2363362312317,1547.8177075386047,2195253,r8i6n2,10.159.28.60,522.2363362312317,1
+1969.7449739711492,1719339766,checkpoint_000003,True,False,4,0f9cc97d,2024-06-25_20-22-47,506.13422203063965,2053.9519295692444,2195253,r8i6n2,10.159.28.60,1028.3705582618713,2
+1951.9356997031866,1719341192,checkpoint_000004,True,False,5,0f9cc97d,2024-06-25_20-46-33,522.4360599517822,2576.3879895210266,2195253,r8i6n2,10.159.28.60,522.4360599517822,1
+1952.9474155696357,1719341699,checkpoint_000005,True,False,6,0f9cc97d,2024-06-25_20-54-59,506.28402161598206,3082.6720111370087,2195253,r8i6n2,10.159.28.60,1028.7200815677643,2
+1954.9041440468134,1719342205,checkpoint_000006,True,False,7,0f9cc97d,2024-06-25_21-03-25,506.20089197158813,3588.872903108597,2195253,r8i6n2,10.159.28.60,1534.9209735393524,3
+1954.9512458861343,1719342711,checkpoint_000007,True,False,8,0f9cc97d,2024-06-25_21-11-51,505.91624569892883,4094.7891488075256,2195253,r8i6n2,10.159.28.60,2040.8372192382812,4
+1955.8870253675566,1719343217,checkpoint_000008,True,False,9,0f9cc97d,2024-06-25_21-20-17,505.6786599159241,4600.46780872345,2195253,r8i6n2,10.159.28.60,2546.5158791542053,5
+1953.494582769439,1719343722,checkpoint_000009,True,False,10,0f9cc97d,2024-06-25_21-28-43,505.8313031196594,5106.299111843109,2195253,r8i6n2,10.159.28.60,3052.3471822738647,6
+1953.025947150283,1719344228,checkpoint_000010,True,False,11,0f9cc97d,2024-06-25_21-37-09,505.74103355407715,5612.040145397186,2195253,r8i6n2,10.159.28.60,3558.088215827942,7
+1956.2824668583908,1719344735,checkpoint_000011,True,False,12,0f9cc97d,2024-06-25_21-45-35,506.4504108428955,6118.490556240082,2195253,r8i6n2,10.159.28.60,4064.5386266708374,8
+1955.5371737742987,1719345241,checkpoint_000012,True,False,13,0f9cc97d,2024-06-25_21-54-01,506.19851207733154,6624.689068317413,2195253,r8i6n2,10.159.28.60,4570.737138748169,9
+1956.4829832062007,1719345747,checkpoint_000013,True,False,14,0f9cc97d,2024-06-25_22-02-27,506.25228786468506,7130.941356182098,2195253,r8i6n2,10.159.28.60,5076.989426612854,10
diff --git a/ray_results/test_experiment/train_model_0f9cc97d_22_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_18-53-00/result.json b/ray_results/test_experiment/train_model_0f9cc97d_22_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_18-53-00/result.json
new file mode 100644
index 0000000000000000000000000000000000000000..a0b4919885bb848434a2cfdac180bf70715777e6
--- /dev/null
+++ b/ray_results/test_experiment/train_model_0f9cc97d_22_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_18-53-00/result.json
@@ -0,0 +1,14 @@
+{"loss": 1956.0084007443406, "timestamp": 1719335168, "checkpoint_dir_name": "checkpoint_000000", "should_checkpoint": true, "done": false, "training_iteration": 1, "trial_id": "0f9cc97d", "date": "2024-06-25_19-06-08", "time_this_iter_s": 519.6532871723175, "time_total_s": 519.6532871723175, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 4, "decoder_rt_num_layer": 2, "decoder_int_num_layer": 1, "embedding_dim": 1024, "encoder_ff": 1024, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 1, "drop_rate": 0.005508735010543275, "lr": 0.0011880094129501908, "batch_size": 2048}, "time_since_restore": 519.6532871723175, "iterations_since_restore": 1}
+{"loss": 1970.2912184347317, "timestamp": 1719335674, "checkpoint_dir_name": "checkpoint_000001", "should_checkpoint": true, "done": false, "training_iteration": 2, "trial_id": "0f9cc97d", "date": "2024-06-25_19-14-34", "time_this_iter_s": 505.92808413505554, "time_total_s": 1025.581371307373, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 4, "decoder_rt_num_layer": 2, "decoder_int_num_layer": 1, "embedding_dim": 1024, "encoder_ff": 1024, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 1, "drop_rate": 0.005508735010543275, "lr": 0.0011880094129501908, "batch_size": 2048}, "time_since_restore": 1025.581371307373, "iterations_since_restore": 2}
+{"loss": 1954.4228304164617, "timestamp": 1719339260, "checkpoint_dir_name": "checkpoint_000002", "should_checkpoint": true, "done": false, "training_iteration": 3, "trial_id": "0f9cc97d", "date": "2024-06-25_20-14-21", "time_this_iter_s": 522.2363362312317, "time_total_s": 1547.8177075386047, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 4, "decoder_rt_num_layer": 2, "decoder_int_num_layer": 1, "embedding_dim": 1024, "encoder_ff": 1024, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 1, "drop_rate": 0.005508735010543275, "lr": 0.0011880094129501908, "batch_size": 2048}, "time_since_restore": 522.2363362312317, "iterations_since_restore": 1}
+{"loss": 1969.7449739711492, "timestamp": 1719339766, "checkpoint_dir_name": "checkpoint_000003", "should_checkpoint": true, "done": false, "training_iteration": 4, "trial_id": "0f9cc97d", "date": "2024-06-25_20-22-47", "time_this_iter_s": 506.13422203063965, "time_total_s": 2053.9519295692444, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 4, "decoder_rt_num_layer": 2, "decoder_int_num_layer": 1, "embedding_dim": 1024, "encoder_ff": 1024, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 1, "drop_rate": 0.005508735010543275, "lr": 0.0011880094129501908, "batch_size": 2048}, "time_since_restore": 1028.3705582618713, "iterations_since_restore": 2}
+{"loss": 1951.9356997031866, "timestamp": 1719341192, "checkpoint_dir_name": "checkpoint_000004", "should_checkpoint": true, "done": false, "training_iteration": 5, "trial_id": "0f9cc97d", "date": "2024-06-25_20-46-33", "time_this_iter_s": 522.4360599517822, "time_total_s": 2576.3879895210266, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 4, "decoder_rt_num_layer": 2, "decoder_int_num_layer": 1, "embedding_dim": 1024, "encoder_ff": 1024, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 1, "drop_rate": 0.005508735010543275, "lr": 0.0011880094129501908, "batch_size": 2048}, "time_since_restore": 522.4360599517822, "iterations_since_restore": 1}
+{"loss": 1952.9474155696357, "timestamp": 1719341699, "checkpoint_dir_name": "checkpoint_000005", "should_checkpoint": true, "done": false, "training_iteration": 6, "trial_id": "0f9cc97d", "date": "2024-06-25_20-54-59", "time_this_iter_s": 506.28402161598206, "time_total_s": 3082.6720111370087, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 4, "decoder_rt_num_layer": 2, "decoder_int_num_layer": 1, "embedding_dim": 1024, "encoder_ff": 1024, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 1, "drop_rate": 0.005508735010543275, "lr": 0.0011880094129501908, "batch_size": 2048}, "time_since_restore": 1028.7200815677643, "iterations_since_restore": 2}
+{"loss": 1954.9041440468134, "timestamp": 1719342205, "checkpoint_dir_name": "checkpoint_000006", "should_checkpoint": true, "done": false, "training_iteration": 7, "trial_id": "0f9cc97d", "date": "2024-06-25_21-03-25", "time_this_iter_s": 506.20089197158813, "time_total_s": 3588.872903108597, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 4, "decoder_rt_num_layer": 2, "decoder_int_num_layer": 1, "embedding_dim": 1024, "encoder_ff": 1024, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 1, "drop_rate": 0.005508735010543275, "lr": 0.0011880094129501908, "batch_size": 2048}, "time_since_restore": 1534.9209735393524, "iterations_since_restore": 3}
+{"loss": 1954.9512458861343, "timestamp": 1719342711, "checkpoint_dir_name": "checkpoint_000007", "should_checkpoint": true, "done": false, "training_iteration": 8, "trial_id": "0f9cc97d", "date": "2024-06-25_21-11-51", "time_this_iter_s": 505.91624569892883, "time_total_s": 4094.7891488075256, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 4, "decoder_rt_num_layer": 2, "decoder_int_num_layer": 1, "embedding_dim": 1024, "encoder_ff": 1024, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 1, "drop_rate": 0.005508735010543275, "lr": 0.0011880094129501908, "batch_size": 2048}, "time_since_restore": 2040.8372192382812, "iterations_since_restore": 4}
+{"loss": 1955.8870253675566, "timestamp": 1719343217, "checkpoint_dir_name": "checkpoint_000008", "should_checkpoint": true, "done": false, "training_iteration": 9, "trial_id": "0f9cc97d", "date": "2024-06-25_21-20-17", "time_this_iter_s": 505.6786599159241, "time_total_s": 4600.46780872345, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 4, "decoder_rt_num_layer": 2, "decoder_int_num_layer": 1, "embedding_dim": 1024, "encoder_ff": 1024, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 1, "drop_rate": 0.005508735010543275, "lr": 0.0011880094129501908, "batch_size": 2048}, "time_since_restore": 2546.5158791542053, "iterations_since_restore": 5}
+{"loss": 1953.494582769439, "timestamp": 1719343722, "checkpoint_dir_name": "checkpoint_000009", "should_checkpoint": true, "done": false, "training_iteration": 10, "trial_id": "0f9cc97d", "date": "2024-06-25_21-28-43", "time_this_iter_s": 505.8313031196594, "time_total_s": 5106.299111843109, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 4, "decoder_rt_num_layer": 2, "decoder_int_num_layer": 1, "embedding_dim": 1024, "encoder_ff": 1024, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 1, "drop_rate": 0.005508735010543275, "lr": 0.0011880094129501908, "batch_size": 2048}, "time_since_restore": 3052.3471822738647, "iterations_since_restore": 6}
+{"loss": 1953.025947150283, "timestamp": 1719344228, "checkpoint_dir_name": "checkpoint_000010", "should_checkpoint": true, "done": false, "training_iteration": 11, "trial_id": "0f9cc97d", "date": "2024-06-25_21-37-09", "time_this_iter_s": 505.74103355407715, "time_total_s": 5612.040145397186, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 4, "decoder_rt_num_layer": 2, "decoder_int_num_layer": 1, "embedding_dim": 1024, "encoder_ff": 1024, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 1, "drop_rate": 0.005508735010543275, "lr": 0.0011880094129501908, "batch_size": 2048}, "time_since_restore": 3558.088215827942, "iterations_since_restore": 7}
+{"loss": 1956.2824668583908, "timestamp": 1719344735, "checkpoint_dir_name": "checkpoint_000011", "should_checkpoint": true, "done": false, "training_iteration": 12, "trial_id": "0f9cc97d", "date": "2024-06-25_21-45-35", "time_this_iter_s": 506.4504108428955, "time_total_s": 6118.490556240082, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 4, "decoder_rt_num_layer": 2, "decoder_int_num_layer": 1, "embedding_dim": 1024, "encoder_ff": 1024, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 1, "drop_rate": 0.005508735010543275, "lr": 0.0011880094129501908, "batch_size": 2048}, "time_since_restore": 4064.5386266708374, "iterations_since_restore": 8}
+{"loss": 1955.5371737742987, "timestamp": 1719345241, "checkpoint_dir_name": "checkpoint_000012", "should_checkpoint": true, "done": false, "training_iteration": 13, "trial_id": "0f9cc97d", "date": "2024-06-25_21-54-01", "time_this_iter_s": 506.19851207733154, "time_total_s": 6624.689068317413, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 4, "decoder_rt_num_layer": 2, "decoder_int_num_layer": 1, "embedding_dim": 1024, "encoder_ff": 1024, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 1, "drop_rate": 0.005508735010543275, "lr": 0.0011880094129501908, "batch_size": 2048}, "time_since_restore": 4570.737138748169, "iterations_since_restore": 9}
+{"loss": 1956.4829832062007, "timestamp": 1719345747, "checkpoint_dir_name": "checkpoint_000013", "should_checkpoint": true, "done": false, "training_iteration": 14, "trial_id": "0f9cc97d", "date": "2024-06-25_22-02-27", "time_this_iter_s": 506.25228786468506, "time_total_s": 7130.941356182098, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 4, "decoder_rt_num_layer": 2, "decoder_int_num_layer": 1, "embedding_dim": 1024, "encoder_ff": 1024, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 1, "drop_rate": 0.005508735010543275, "lr": 0.0011880094129501908, "batch_size": 2048}, "time_since_restore": 5076.989426612854, "iterations_since_restore": 10}
diff --git a/ray_results/test_experiment/train_model_110f0a27_14_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop__2024-06-25_17-41-39/events.out.tfevents.1719330179.r8i6n2 b/ray_results/test_experiment/train_model_110f0a27_14_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop__2024-06-25_17-41-39/events.out.tfevents.1719330179.r8i6n2
new file mode 100644
index 0000000000000000000000000000000000000000..b82acd39abc29482c82924ea3f47329be3374488
Binary files /dev/null and b/ray_results/test_experiment/train_model_110f0a27_14_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop__2024-06-25_17-41-39/events.out.tfevents.1719330179.r8i6n2 differ
diff --git a/ray_results/test_experiment/train_model_110f0a27_14_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop__2024-06-25_17-41-39/events.out.tfevents.1719330947.r8i6n2 b/ray_results/test_experiment/train_model_110f0a27_14_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop__2024-06-25_17-41-39/events.out.tfevents.1719330947.r8i6n2
new file mode 100644
index 0000000000000000000000000000000000000000..79a594ff1a06e34d9edd77d26a0c5e4c29d15082
Binary files /dev/null and b/ray_results/test_experiment/train_model_110f0a27_14_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop__2024-06-25_17-41-39/events.out.tfevents.1719330947.r8i6n2 differ
diff --git a/ray_results/test_experiment/train_model_110f0a27_14_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop__2024-06-25_17-41-39/params.json b/ray_results/test_experiment/train_model_110f0a27_14_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop__2024-06-25_17-41-39/params.json
new file mode 100644
index 0000000000000000000000000000000000000000..b2dd97ebd74972769f79172bfc00f0a850dc87bb
--- /dev/null
+++ b/ray_results/test_experiment/train_model_110f0a27_14_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop__2024-06-25_17-41-39/params.json
@@ -0,0 +1,13 @@
+{
+ "batch_size": 2048,
+ "decoder_int_ff": 512,
+ "decoder_int_num_layer": 1,
+ "decoder_rt_ff": 1024,
+ "decoder_rt_num_layer": 2,
+ "drop_rate": 0.025073968761468723,
+ "embedding_dim": 256,
+ "encoder_ff": 2048,
+ "encoder_num_layer": 2,
+ "lr": 0.0010259835678295768,
+ "n_head": 16
+}
\ No newline at end of file
diff --git a/ray_results/test_experiment/train_model_110f0a27_14_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop__2024-06-25_17-41-39/params.pkl b/ray_results/test_experiment/train_model_110f0a27_14_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop__2024-06-25_17-41-39/params.pkl
new file mode 100644
index 0000000000000000000000000000000000000000..b400c94d2014abf046ea4082ed13be2b65b4c668
Binary files /dev/null and b/ray_results/test_experiment/train_model_110f0a27_14_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop__2024-06-25_17-41-39/params.pkl differ
diff --git a/ray_results/test_experiment/train_model_110f0a27_14_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop__2024-06-25_17-41-39/progress.csv b/ray_results/test_experiment/train_model_110f0a27_14_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop__2024-06-25_17-41-39/progress.csv
new file mode 100644
index 0000000000000000000000000000000000000000..a5487a498902487aa3da0294d9e5d591a647309b
--- /dev/null
+++ b/ray_results/test_experiment/train_model_110f0a27_14_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop__2024-06-25_17-41-39/progress.csv
@@ -0,0 +1,3 @@
+loss,timestamp,checkpoint_dir_name,should_checkpoint,done,training_iteration,trial_id,date,time_this_iter_s,time_total_s,pid,hostname,node_ip,time_since_restore,iterations_since_restore
+1956.1862581508367,1719330289,checkpoint_000000,True,False,1,110f0a27,2024-06-25_17-44-49,110.66666960716248,110.66666960716248,2195253,r8i6n2,10.159.28.60,110.66666960716248,1
+1952.48307740219,1719331057,checkpoint_000001,True,False,2,110f0a27,2024-06-25_17-57-37,110.16742134094238,220.83409094810486,2195253,r8i6n2,10.159.28.60,110.16742134094238,1
diff --git a/ray_results/test_experiment/train_model_110f0a27_14_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop__2024-06-25_17-41-39/result.json b/ray_results/test_experiment/train_model_110f0a27_14_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop__2024-06-25_17-41-39/result.json
new file mode 100644
index 0000000000000000000000000000000000000000..32a0ab934e6ac9f0385bc329edf55d0dc7166fef
--- /dev/null
+++ b/ray_results/test_experiment/train_model_110f0a27_14_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop__2024-06-25_17-41-39/result.json
@@ -0,0 +1,2 @@
+{"loss": 1956.1862581508367, "timestamp": 1719330289, "checkpoint_dir_name": "checkpoint_000000", "should_checkpoint": true, "done": false, "training_iteration": 1, "trial_id": "110f0a27", "date": "2024-06-25_17-44-49", "time_this_iter_s": 110.66666960716248, "time_total_s": 110.66666960716248, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 2, "decoder_rt_num_layer": 2, "decoder_int_num_layer": 1, "embedding_dim": 256, "encoder_ff": 2048, "decoder_rt_ff": 1024, "decoder_int_ff": 512, "n_head": 16, "drop_rate": 0.025073968761468723, "lr": 0.0010259835678295768, "batch_size": 2048}, "time_since_restore": 110.66666960716248, "iterations_since_restore": 1}
+{"loss": 1952.48307740219, "timestamp": 1719331057, "checkpoint_dir_name": "checkpoint_000001", "should_checkpoint": true, "done": false, "training_iteration": 2, "trial_id": "110f0a27", "date": "2024-06-25_17-57-37", "time_this_iter_s": 110.16742134094238, "time_total_s": 220.83409094810486, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 2, "decoder_rt_num_layer": 2, "decoder_int_num_layer": 1, "embedding_dim": 256, "encoder_ff": 2048, "decoder_rt_ff": 1024, "decoder_int_ff": 512, "n_head": 16, "drop_rate": 0.025073968761468723, "lr": 0.0010259835678295768, "batch_size": 2048}, "time_since_restore": 110.16742134094238, "iterations_since_restore": 1}
diff --git a/ray_results/test_experiment/train_model_18c11cbc_27_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop__2024-06-25_19-39-03/events.out.tfevents.1719346234.r8i6n2 b/ray_results/test_experiment/train_model_18c11cbc_27_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop__2024-06-25_19-39-03/events.out.tfevents.1719346234.r8i6n2
new file mode 100644
index 0000000000000000000000000000000000000000..5f06f7f639bc2052375f3401b6e605202feaf7d7
Binary files /dev/null and b/ray_results/test_experiment/train_model_18c11cbc_27_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop__2024-06-25_19-39-03/events.out.tfevents.1719346234.r8i6n2 differ
diff --git a/ray_results/test_experiment/train_model_18c11cbc_27_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop__2024-06-25_19-39-03/events.out.tfevents.1719354644.r8i6n2 b/ray_results/test_experiment/train_model_18c11cbc_27_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop__2024-06-25_19-39-03/events.out.tfevents.1719354644.r8i6n2
new file mode 100644
index 0000000000000000000000000000000000000000..aada72158deb33f4f2ccf3c71d50f792587e0963
Binary files /dev/null and b/ray_results/test_experiment/train_model_18c11cbc_27_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop__2024-06-25_19-39-03/events.out.tfevents.1719354644.r8i6n2 differ
diff --git a/ray_results/test_experiment/train_model_18c11cbc_27_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop__2024-06-25_19-39-03/events.out.tfevents.1719357265.r8i6n2 b/ray_results/test_experiment/train_model_18c11cbc_27_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop__2024-06-25_19-39-03/events.out.tfevents.1719357265.r8i6n2
new file mode 100644
index 0000000000000000000000000000000000000000..cff8c2b4224ff135612f5376adb41d5505a365a6
Binary files /dev/null and b/ray_results/test_experiment/train_model_18c11cbc_27_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop__2024-06-25_19-39-03/events.out.tfevents.1719357265.r8i6n2 differ
diff --git a/ray_results/test_experiment/train_model_18c11cbc_27_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop__2024-06-25_19-39-03/params.json b/ray_results/test_experiment/train_model_18c11cbc_27_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop__2024-06-25_19-39-03/params.json
new file mode 100644
index 0000000000000000000000000000000000000000..99a97a2e7eb732b5d2f76ada7f6823456190de9d
--- /dev/null
+++ b/ray_results/test_experiment/train_model_18c11cbc_27_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop__2024-06-25_19-39-03/params.json
@@ -0,0 +1,13 @@
+{
+ "batch_size": 1024,
+ "decoder_int_ff": 512,
+ "decoder_int_num_layer": 1,
+ "decoder_rt_ff": 2048,
+ "decoder_rt_num_layer": 1,
+ "drop_rate": 0.3688667410196246,
+ "embedding_dim": 16,
+ "encoder_ff": 512,
+ "encoder_num_layer": 4,
+ "lr": 0.0018036063294034547,
+ "n_head": 8
+}
\ No newline at end of file
diff --git a/ray_results/test_experiment/train_model_18c11cbc_27_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop__2024-06-25_19-39-03/params.pkl b/ray_results/test_experiment/train_model_18c11cbc_27_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop__2024-06-25_19-39-03/params.pkl
new file mode 100644
index 0000000000000000000000000000000000000000..ed17526d37e22fc1e652a7c64e27ec1665e71efa
Binary files /dev/null and b/ray_results/test_experiment/train_model_18c11cbc_27_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop__2024-06-25_19-39-03/params.pkl differ
diff --git a/ray_results/test_experiment/train_model_18c11cbc_27_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop__2024-06-25_19-39-03/progress.csv b/ray_results/test_experiment/train_model_18c11cbc_27_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop__2024-06-25_19-39-03/progress.csv
new file mode 100644
index 0000000000000000000000000000000000000000..0ed9c15574718811a791b10bacc440be13b4414f
--- /dev/null
+++ b/ray_results/test_experiment/train_model_18c11cbc_27_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop__2024-06-25_19-39-03/progress.csv
@@ -0,0 +1,21 @@
+loss,timestamp,checkpoint_dir_name,should_checkpoint,done,training_iteration,trial_id,date,time_this_iter_s,time_total_s,pid,hostname,node_ip,time_since_restore,iterations_since_restore
+266.66766681821326,1719346308,checkpoint_000000,True,False,1,18c11cbc,2024-06-25_22-11-48,74.13704776763916,74.13704776763916,2195253,r8i6n2,10.159.28.60,74.13704776763916,1
+191.86041752372202,1719346367,checkpoint_000001,True,False,2,18c11cbc,2024-06-25_22-12-47,59.13420367240906,133.27125144004822,2195253,r8i6n2,10.159.28.60,133.27125144004822,2
+172.28332663708784,1719346427,checkpoint_000002,True,False,3,18c11cbc,2024-06-25_22-13-47,59.52952766418457,192.8007791042328,2195253,r8i6n2,10.159.28.60,192.8007791042328,3
+165.87816860168937,1719346487,checkpoint_000003,True,False,4,18c11cbc,2024-06-25_22-14-47,59.658318519592285,252.45909762382507,2195253,r8i6n2,10.159.28.60,252.45909762382507,4
+149.45840237835262,1719346547,checkpoint_000004,True,False,5,18c11cbc,2024-06-25_22-15-47,60.179141998291016,312.6382396221161,2195253,r8i6n2,10.159.28.60,312.6382396221161,5
+142.80532779843787,1719354718,checkpoint_000005,True,False,6,18c11cbc,2024-06-26_00-31-58,73.47930812835693,386.117547750473,2850562,r8i6n2,10.159.28.60,73.47930812835693,1
+147.61887368630238,1719354776,checkpoint_000006,True,False,7,18c11cbc,2024-06-26_00-32-56,58.23622250556946,444.3537702560425,2850562,r8i6n2,10.159.28.60,131.7155306339264,2
+136.93548496877114,1719354835,checkpoint_000007,True,False,8,18c11cbc,2024-06-26_00-33-55,59.08127737045288,503.43504762649536,2850562,r8i6n2,10.159.28.60,190.79680800437927,3
+133.39135180495856,1719354893,checkpoint_000008,True,False,9,18c11cbc,2024-06-26_00-34-53,58.395625591278076,561.8306732177734,2850562,r8i6n2,10.159.28.60,249.19243359565735,4
+130.94450414462352,1719354952,checkpoint_000009,True,False,10,18c11cbc,2024-06-26_00-35-52,58.66959762573242,620.5002708435059,2850562,r8i6n2,10.159.28.60,307.86203122138977,5
+128.02527375108613,1719357338,checkpoint_000010,True,False,11,18c11cbc,2024-06-26_01-15-38,72.75415802001953,693.2544288635254,2850562,r8i6n2,10.159.28.60,72.75415802001953,1
+130.49067126296637,1719357397,checkpoint_000011,True,False,12,18c11cbc,2024-06-26_01-16-37,59.34819006919861,752.602618932724,2850562,r8i6n2,10.159.28.60,132.10234808921814,2
+125.55984971654696,1719357457,checkpoint_000012,True,False,13,18c11cbc,2024-06-26_01-17-37,59.262863636016846,811.8654825687408,2850562,r8i6n2,10.159.28.60,191.36521172523499,3
+124.00354583620087,1719357516,checkpoint_000013,True,False,14,18c11cbc,2024-06-26_01-18-36,59.35755896568298,871.2230415344238,2850562,r8i6n2,10.159.28.60,250.72277069091797,4
+122.43110686775267,1719357575,checkpoint_000014,True,False,15,18c11cbc,2024-06-26_01-19-35,59.15378403663635,930.3768255710602,2850562,r8i6n2,10.159.28.60,309.8765547275543,5
+121.44499277129887,1719357635,checkpoint_000015,True,False,16,18c11cbc,2024-06-26_01-20-35,59.61450457572937,989.9913301467896,2850562,r8i6n2,10.159.28.60,369.4910593032837,6
+124.11241438257413,1719357694,checkpoint_000016,True,False,17,18c11cbc,2024-06-26_01-21-34,59.14637589454651,1049.137706041336,2850562,r8i6n2,10.159.28.60,428.6374351978302,7
+121.52879060159518,1719357753,checkpoint_000017,True,False,18,18c11cbc,2024-06-26_01-22-33,59.11753058433533,1108.2552366256714,2850562,r8i6n2,10.159.28.60,487.7549657821655,8
+118.6914559612124,1719357812,checkpoint_000018,True,False,19,18c11cbc,2024-06-26_01-23-32,59.181042432785034,1167.4362790584564,2850562,r8i6n2,10.159.28.60,546.9360082149506,9
+118.33789474006713,1719357871,checkpoint_000019,True,False,20,18c11cbc,2024-06-26_01-24-31,59.315247774124146,1226.7515268325806,2850562,r8i6n2,10.159.28.60,606.2512559890747,10
diff --git a/ray_results/test_experiment/train_model_18c11cbc_27_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop__2024-06-25_19-39-03/result.json b/ray_results/test_experiment/train_model_18c11cbc_27_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop__2024-06-25_19-39-03/result.json
new file mode 100644
index 0000000000000000000000000000000000000000..db58b6736becb24329bb220a9c2676988e3f04cc
--- /dev/null
+++ b/ray_results/test_experiment/train_model_18c11cbc_27_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop__2024-06-25_19-39-03/result.json
@@ -0,0 +1,20 @@
+{"loss": 266.66766681821326, "timestamp": 1719346308, "checkpoint_dir_name": "checkpoint_000000", "should_checkpoint": true, "done": false, "training_iteration": 1, "trial_id": "18c11cbc", "date": "2024-06-25_22-11-48", "time_this_iter_s": 74.13704776763916, "time_total_s": 74.13704776763916, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 4, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 512, "decoder_rt_ff": 2048, "decoder_int_ff": 512, "n_head": 8, "drop_rate": 0.3688667410196246, "lr": 0.0018036063294034547, "batch_size": 1024}, "time_since_restore": 74.13704776763916, "iterations_since_restore": 1}
+{"loss": 191.86041752372202, "timestamp": 1719346367, "checkpoint_dir_name": "checkpoint_000001", "should_checkpoint": true, "done": false, "training_iteration": 2, "trial_id": "18c11cbc", "date": "2024-06-25_22-12-47", "time_this_iter_s": 59.13420367240906, "time_total_s": 133.27125144004822, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 4, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 512, "decoder_rt_ff": 2048, "decoder_int_ff": 512, "n_head": 8, "drop_rate": 0.3688667410196246, "lr": 0.0018036063294034547, "batch_size": 1024}, "time_since_restore": 133.27125144004822, "iterations_since_restore": 2}
+{"loss": 172.28332663708784, "timestamp": 1719346427, "checkpoint_dir_name": "checkpoint_000002", "should_checkpoint": true, "done": false, "training_iteration": 3, "trial_id": "18c11cbc", "date": "2024-06-25_22-13-47", "time_this_iter_s": 59.52952766418457, "time_total_s": 192.8007791042328, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 4, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 512, "decoder_rt_ff": 2048, "decoder_int_ff": 512, "n_head": 8, "drop_rate": 0.3688667410196246, "lr": 0.0018036063294034547, "batch_size": 1024}, "time_since_restore": 192.8007791042328, "iterations_since_restore": 3}
+{"loss": 165.87816860168937, "timestamp": 1719346487, "checkpoint_dir_name": "checkpoint_000003", "should_checkpoint": true, "done": false, "training_iteration": 4, "trial_id": "18c11cbc", "date": "2024-06-25_22-14-47", "time_this_iter_s": 59.658318519592285, "time_total_s": 252.45909762382507, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 4, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 512, "decoder_rt_ff": 2048, "decoder_int_ff": 512, "n_head": 8, "drop_rate": 0.3688667410196246, "lr": 0.0018036063294034547, "batch_size": 1024}, "time_since_restore": 252.45909762382507, "iterations_since_restore": 4}
+{"loss": 149.45840237835262, "timestamp": 1719346547, "checkpoint_dir_name": "checkpoint_000004", "should_checkpoint": true, "done": false, "training_iteration": 5, "trial_id": "18c11cbc", "date": "2024-06-25_22-15-47", "time_this_iter_s": 60.179141998291016, "time_total_s": 312.6382396221161, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 4, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 512, "decoder_rt_ff": 2048, "decoder_int_ff": 512, "n_head": 8, "drop_rate": 0.3688667410196246, "lr": 0.0018036063294034547, "batch_size": 1024}, "time_since_restore": 312.6382396221161, "iterations_since_restore": 5}
+{"loss": 142.80532779843787, "timestamp": 1719354718, "checkpoint_dir_name": "checkpoint_000005", "should_checkpoint": true, "done": false, "training_iteration": 6, "trial_id": "18c11cbc", "date": "2024-06-26_00-31-58", "time_this_iter_s": 73.47930812835693, "time_total_s": 386.117547750473, "pid": 2850562, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 4, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 512, "decoder_rt_ff": 2048, "decoder_int_ff": 512, "n_head": 8, "drop_rate": 0.3688667410196246, "lr": 0.0018036063294034547, "batch_size": 1024}, "time_since_restore": 73.47930812835693, "iterations_since_restore": 1}
+{"loss": 147.61887368630238, "timestamp": 1719354776, "checkpoint_dir_name": "checkpoint_000006", "should_checkpoint": true, "done": false, "training_iteration": 7, "trial_id": "18c11cbc", "date": "2024-06-26_00-32-56", "time_this_iter_s": 58.23622250556946, "time_total_s": 444.3537702560425, "pid": 2850562, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 4, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 512, "decoder_rt_ff": 2048, "decoder_int_ff": 512, "n_head": 8, "drop_rate": 0.3688667410196246, "lr": 0.0018036063294034547, "batch_size": 1024}, "time_since_restore": 131.7155306339264, "iterations_since_restore": 2}
+{"loss": 136.93548496877114, "timestamp": 1719354835, "checkpoint_dir_name": "checkpoint_000007", "should_checkpoint": true, "done": false, "training_iteration": 8, "trial_id": "18c11cbc", "date": "2024-06-26_00-33-55", "time_this_iter_s": 59.08127737045288, "time_total_s": 503.43504762649536, "pid": 2850562, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 4, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 512, "decoder_rt_ff": 2048, "decoder_int_ff": 512, "n_head": 8, "drop_rate": 0.3688667410196246, "lr": 0.0018036063294034547, "batch_size": 1024}, "time_since_restore": 190.79680800437927, "iterations_since_restore": 3}
+{"loss": 133.39135180495856, "timestamp": 1719354893, "checkpoint_dir_name": "checkpoint_000008", "should_checkpoint": true, "done": false, "training_iteration": 9, "trial_id": "18c11cbc", "date": "2024-06-26_00-34-53", "time_this_iter_s": 58.395625591278076, "time_total_s": 561.8306732177734, "pid": 2850562, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 4, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 512, "decoder_rt_ff": 2048, "decoder_int_ff": 512, "n_head": 8, "drop_rate": 0.3688667410196246, "lr": 0.0018036063294034547, "batch_size": 1024}, "time_since_restore": 249.19243359565735, "iterations_since_restore": 4}
+{"loss": 130.94450414462352, "timestamp": 1719354952, "checkpoint_dir_name": "checkpoint_000009", "should_checkpoint": true, "done": false, "training_iteration": 10, "trial_id": "18c11cbc", "date": "2024-06-26_00-35-52", "time_this_iter_s": 58.66959762573242, "time_total_s": 620.5002708435059, "pid": 2850562, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 4, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 512, "decoder_rt_ff": 2048, "decoder_int_ff": 512, "n_head": 8, "drop_rate": 0.3688667410196246, "lr": 0.0018036063294034547, "batch_size": 1024}, "time_since_restore": 307.86203122138977, "iterations_since_restore": 5}
+{"loss": 128.02527375108613, "timestamp": 1719357338, "checkpoint_dir_name": "checkpoint_000010", "should_checkpoint": true, "done": false, "training_iteration": 11, "trial_id": "18c11cbc", "date": "2024-06-26_01-15-38", "time_this_iter_s": 72.75415802001953, "time_total_s": 693.2544288635254, "pid": 2850562, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 4, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 512, "decoder_rt_ff": 2048, "decoder_int_ff": 512, "n_head": 8, "drop_rate": 0.3688667410196246, "lr": 0.0018036063294034547, "batch_size": 1024}, "time_since_restore": 72.75415802001953, "iterations_since_restore": 1}
+{"loss": 130.49067126296637, "timestamp": 1719357397, "checkpoint_dir_name": "checkpoint_000011", "should_checkpoint": true, "done": false, "training_iteration": 12, "trial_id": "18c11cbc", "date": "2024-06-26_01-16-37", "time_this_iter_s": 59.34819006919861, "time_total_s": 752.602618932724, "pid": 2850562, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 4, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 512, "decoder_rt_ff": 2048, "decoder_int_ff": 512, "n_head": 8, "drop_rate": 0.3688667410196246, "lr": 0.0018036063294034547, "batch_size": 1024}, "time_since_restore": 132.10234808921814, "iterations_since_restore": 2}
+{"loss": 125.55984971654696, "timestamp": 1719357457, "checkpoint_dir_name": "checkpoint_000012", "should_checkpoint": true, "done": false, "training_iteration": 13, "trial_id": "18c11cbc", "date": "2024-06-26_01-17-37", "time_this_iter_s": 59.262863636016846, "time_total_s": 811.8654825687408, "pid": 2850562, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 4, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 512, "decoder_rt_ff": 2048, "decoder_int_ff": 512, "n_head": 8, "drop_rate": 0.3688667410196246, "lr": 0.0018036063294034547, "batch_size": 1024}, "time_since_restore": 191.36521172523499, "iterations_since_restore": 3}
+{"loss": 124.00354583620087, "timestamp": 1719357516, "checkpoint_dir_name": "checkpoint_000013", "should_checkpoint": true, "done": false, "training_iteration": 14, "trial_id": "18c11cbc", "date": "2024-06-26_01-18-36", "time_this_iter_s": 59.35755896568298, "time_total_s": 871.2230415344238, "pid": 2850562, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 4, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 512, "decoder_rt_ff": 2048, "decoder_int_ff": 512, "n_head": 8, "drop_rate": 0.3688667410196246, "lr": 0.0018036063294034547, "batch_size": 1024}, "time_since_restore": 250.72277069091797, "iterations_since_restore": 4}
+{"loss": 122.43110686775267, "timestamp": 1719357575, "checkpoint_dir_name": "checkpoint_000014", "should_checkpoint": true, "done": false, "training_iteration": 15, "trial_id": "18c11cbc", "date": "2024-06-26_01-19-35", "time_this_iter_s": 59.15378403663635, "time_total_s": 930.3768255710602, "pid": 2850562, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 4, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 512, "decoder_rt_ff": 2048, "decoder_int_ff": 512, "n_head": 8, "drop_rate": 0.3688667410196246, "lr": 0.0018036063294034547, "batch_size": 1024}, "time_since_restore": 309.8765547275543, "iterations_since_restore": 5}
+{"loss": 121.44499277129887, "timestamp": 1719357635, "checkpoint_dir_name": "checkpoint_000015", "should_checkpoint": true, "done": false, "training_iteration": 16, "trial_id": "18c11cbc", "date": "2024-06-26_01-20-35", "time_this_iter_s": 59.61450457572937, "time_total_s": 989.9913301467896, "pid": 2850562, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 4, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 512, "decoder_rt_ff": 2048, "decoder_int_ff": 512, "n_head": 8, "drop_rate": 0.3688667410196246, "lr": 0.0018036063294034547, "batch_size": 1024}, "time_since_restore": 369.4910593032837, "iterations_since_restore": 6}
+{"loss": 124.11241438257413, "timestamp": 1719357694, "checkpoint_dir_name": "checkpoint_000016", "should_checkpoint": true, "done": false, "training_iteration": 17, "trial_id": "18c11cbc", "date": "2024-06-26_01-21-34", "time_this_iter_s": 59.14637589454651, "time_total_s": 1049.137706041336, "pid": 2850562, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 4, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 512, "decoder_rt_ff": 2048, "decoder_int_ff": 512, "n_head": 8, "drop_rate": 0.3688667410196246, "lr": 0.0018036063294034547, "batch_size": 1024}, "time_since_restore": 428.6374351978302, "iterations_since_restore": 7}
+{"loss": 121.52879060159518, "timestamp": 1719357753, "checkpoint_dir_name": "checkpoint_000017", "should_checkpoint": true, "done": false, "training_iteration": 18, "trial_id": "18c11cbc", "date": "2024-06-26_01-22-33", "time_this_iter_s": 59.11753058433533, "time_total_s": 1108.2552366256714, "pid": 2850562, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 4, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 512, "decoder_rt_ff": 2048, "decoder_int_ff": 512, "n_head": 8, "drop_rate": 0.3688667410196246, "lr": 0.0018036063294034547, "batch_size": 1024}, "time_since_restore": 487.7549657821655, "iterations_since_restore": 8}
+{"loss": 118.6914559612124, "timestamp": 1719357812, "checkpoint_dir_name": "checkpoint_000018", "should_checkpoint": true, "done": false, "training_iteration": 19, "trial_id": "18c11cbc", "date": "2024-06-26_01-23-32", "time_this_iter_s": 59.181042432785034, "time_total_s": 1167.4362790584564, "pid": 2850562, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 4, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 512, "decoder_rt_ff": 2048, "decoder_int_ff": 512, "n_head": 8, "drop_rate": 0.3688667410196246, "lr": 0.0018036063294034547, "batch_size": 1024}, "time_since_restore": 546.9360082149506, "iterations_since_restore": 9}
+{"loss": 118.33789474006713, "timestamp": 1719357871, "checkpoint_dir_name": "checkpoint_000019", "should_checkpoint": true, "done": false, "training_iteration": 20, "trial_id": "18c11cbc", "date": "2024-06-26_01-24-31", "time_this_iter_s": 59.315247774124146, "time_total_s": 1226.7515268325806, "pid": 2850562, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 4, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 512, "decoder_rt_ff": 2048, "decoder_int_ff": 512, "n_head": 8, "drop_rate": 0.3688667410196246, "lr": 0.0018036063294034547, "batch_size": 1024}, "time_since_restore": 606.2512559890747, "iterations_since_restore": 10}
diff --git a/ray_results/test_experiment/train_model_1920f493_9_batch_size=1024,decoder_int_ff=1024,decoder_int_num_layer=2,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop__2024-06-25_16-19-16/events.out.tfevents.1719325234.r3i5n6 b/ray_results/test_experiment/train_model_1920f493_9_batch_size=1024,decoder_int_ff=1024,decoder_int_num_layer=2,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop__2024-06-25_16-19-16/events.out.tfevents.1719325234.r3i5n6
new file mode 100644
index 0000000000000000000000000000000000000000..3d56d70c86b59b1df86c8bd559ee0c2dc402b4f1
Binary files /dev/null and b/ray_results/test_experiment/train_model_1920f493_9_batch_size=1024,decoder_int_ff=1024,decoder_int_num_layer=2,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop__2024-06-25_16-19-16/events.out.tfevents.1719325234.r3i5n6 differ
diff --git a/ray_results/test_experiment/train_model_1920f493_9_batch_size=1024,decoder_int_ff=1024,decoder_int_num_layer=2,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop__2024-06-25_16-19-16/params.json b/ray_results/test_experiment/train_model_1920f493_9_batch_size=1024,decoder_int_ff=1024,decoder_int_num_layer=2,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop__2024-06-25_16-19-16/params.json
new file mode 100644
index 0000000000000000000000000000000000000000..3a113351c84e232ce3e4ed0cd5df30addfd04668
--- /dev/null
+++ b/ray_results/test_experiment/train_model_1920f493_9_batch_size=1024,decoder_int_ff=1024,decoder_int_num_layer=2,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop__2024-06-25_16-19-16/params.json
@@ -0,0 +1,13 @@
+{
+ "batch_size": 1024,
+ "decoder_int_ff": 1024,
+ "decoder_int_num_layer": 2,
+ "decoder_rt_ff": 2048,
+ "decoder_rt_num_layer": 1,
+ "drop_rate": 0.922082156004742,
+ "embedding_dim": 64,
+ "encoder_ff": 2048,
+ "encoder_num_layer": 4,
+ "lr": 0.0016525285062566051,
+ "n_head": 1
+}
\ No newline at end of file
diff --git a/ray_results/test_experiment/train_model_1920f493_9_batch_size=1024,decoder_int_ff=1024,decoder_int_num_layer=2,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop__2024-06-25_16-19-16/params.pkl b/ray_results/test_experiment/train_model_1920f493_9_batch_size=1024,decoder_int_ff=1024,decoder_int_num_layer=2,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop__2024-06-25_16-19-16/params.pkl
new file mode 100644
index 0000000000000000000000000000000000000000..0d692d3c5ce4a7a704acc89c2aa87714b70d3b2b
Binary files /dev/null and b/ray_results/test_experiment/train_model_1920f493_9_batch_size=1024,decoder_int_ff=1024,decoder_int_num_layer=2,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop__2024-06-25_16-19-16/params.pkl differ
diff --git a/ray_results/test_experiment/train_model_1920f493_9_batch_size=1024,decoder_int_ff=1024,decoder_int_num_layer=2,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop__2024-06-25_16-19-16/progress.csv b/ray_results/test_experiment/train_model_1920f493_9_batch_size=1024,decoder_int_ff=1024,decoder_int_num_layer=2,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop__2024-06-25_16-19-16/progress.csv
new file mode 100644
index 0000000000000000000000000000000000000000..71eb8bc0d058441e953da5b6234f909d93319adb
--- /dev/null
+++ b/ray_results/test_experiment/train_model_1920f493_9_batch_size=1024,decoder_int_ff=1024,decoder_int_num_layer=2,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop__2024-06-25_16-19-16/progress.csv
@@ -0,0 +1,2 @@
+loss,timestamp,checkpoint_dir_name,should_checkpoint,done,training_iteration,trial_id,date,time_this_iter_s,time_total_s,pid,hostname,node_ip,time_since_restore,iterations_since_restore
+2102.196185254675,1719325314,checkpoint_000000,True,False,1,1920f493,2024-06-25_16-21-54,79.38143539428711,79.38143539428711,69318,r3i5n6,10.159.8.159,79.38143539428711,1
diff --git a/ray_results/test_experiment/train_model_1920f493_9_batch_size=1024,decoder_int_ff=1024,decoder_int_num_layer=2,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop__2024-06-25_16-19-16/result.json b/ray_results/test_experiment/train_model_1920f493_9_batch_size=1024,decoder_int_ff=1024,decoder_int_num_layer=2,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop__2024-06-25_16-19-16/result.json
new file mode 100644
index 0000000000000000000000000000000000000000..fd14144be4fab88a3c1b3071b7c4f659da8671f0
--- /dev/null
+++ b/ray_results/test_experiment/train_model_1920f493_9_batch_size=1024,decoder_int_ff=1024,decoder_int_num_layer=2,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop__2024-06-25_16-19-16/result.json
@@ -0,0 +1 @@
+{"loss": 2102.196185254675, "timestamp": 1719325314, "checkpoint_dir_name": "checkpoint_000000", "should_checkpoint": true, "done": false, "training_iteration": 1, "trial_id": "1920f493", "date": "2024-06-25_16-21-54", "time_this_iter_s": 79.38143539428711, "time_total_s": 79.38143539428711, "pid": 69318, "hostname": "r3i5n6", "node_ip": "10.159.8.159", "config": {"encoder_num_layer": 4, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 2, "embedding_dim": 64, "encoder_ff": 2048, "decoder_rt_ff": 2048, "decoder_int_ff": 1024, "n_head": 1, "drop_rate": 0.922082156004742, "lr": 0.0016525285062566051, "batch_size": 1024}, "time_since_restore": 79.38143539428711, "iterations_since_restore": 1}
diff --git a/ray_results/test_experiment/train_model_1de460a8_10_batch_size=1024,decoder_int_ff=1024,decoder_int_num_layer=8,decoder_rt_ff=512,decoder_rt_num_layer=4,drop__2024-06-25_16-20-34/events.out.tfevents.1719325314.r3i5n6 b/ray_results/test_experiment/train_model_1de460a8_10_batch_size=1024,decoder_int_ff=1024,decoder_int_num_layer=8,decoder_rt_ff=512,decoder_rt_num_layer=4,drop__2024-06-25_16-20-34/events.out.tfevents.1719325314.r3i5n6
new file mode 100644
index 0000000000000000000000000000000000000000..f44e779c8b68dd1ae1feb09c2f227703e7c7ffe6
Binary files /dev/null and b/ray_results/test_experiment/train_model_1de460a8_10_batch_size=1024,decoder_int_ff=1024,decoder_int_num_layer=8,decoder_rt_ff=512,decoder_rt_num_layer=4,drop__2024-06-25_16-20-34/events.out.tfevents.1719325314.r3i5n6 differ
diff --git a/ray_results/test_experiment/train_model_1de460a8_10_batch_size=1024,decoder_int_ff=1024,decoder_int_num_layer=8,decoder_rt_ff=512,decoder_rt_num_layer=4,drop__2024-06-25_16-20-34/events.out.tfevents.1719327690.r3i5n6 b/ray_results/test_experiment/train_model_1de460a8_10_batch_size=1024,decoder_int_ff=1024,decoder_int_num_layer=8,decoder_rt_ff=512,decoder_rt_num_layer=4,drop__2024-06-25_16-20-34/events.out.tfevents.1719327690.r3i5n6
new file mode 100644
index 0000000000000000000000000000000000000000..984a613d6411f46d57a8e1857294676fecce33b3
Binary files /dev/null and b/ray_results/test_experiment/train_model_1de460a8_10_batch_size=1024,decoder_int_ff=1024,decoder_int_num_layer=8,decoder_rt_ff=512,decoder_rt_num_layer=4,drop__2024-06-25_16-20-34/events.out.tfevents.1719327690.r3i5n6 differ
diff --git a/ray_results/test_experiment/train_model_1de460a8_10_batch_size=1024,decoder_int_ff=1024,decoder_int_num_layer=8,decoder_rt_ff=512,decoder_rt_num_layer=4,drop__2024-06-25_16-20-34/params.json b/ray_results/test_experiment/train_model_1de460a8_10_batch_size=1024,decoder_int_ff=1024,decoder_int_num_layer=8,decoder_rt_ff=512,decoder_rt_num_layer=4,drop__2024-06-25_16-20-34/params.json
new file mode 100644
index 0000000000000000000000000000000000000000..cb9dc533b4a755b51779113fd702b420f2affd42
--- /dev/null
+++ b/ray_results/test_experiment/train_model_1de460a8_10_batch_size=1024,decoder_int_ff=1024,decoder_int_num_layer=8,decoder_rt_ff=512,decoder_rt_num_layer=4,drop__2024-06-25_16-20-34/params.json
@@ -0,0 +1,13 @@
+{
+ "batch_size": 1024,
+ "decoder_int_ff": 1024,
+ "decoder_int_num_layer": 8,
+ "decoder_rt_ff": 512,
+ "decoder_rt_num_layer": 4,
+ "drop_rate": 0.03175079062321118,
+ "embedding_dim": 16,
+ "encoder_ff": 2048,
+ "encoder_num_layer": 2,
+ "lr": 0.003436670599863372,
+ "n_head": 1
+}
\ No newline at end of file
diff --git a/ray_results/test_experiment/train_model_1de460a8_10_batch_size=1024,decoder_int_ff=1024,decoder_int_num_layer=8,decoder_rt_ff=512,decoder_rt_num_layer=4,drop__2024-06-25_16-20-34/params.pkl b/ray_results/test_experiment/train_model_1de460a8_10_batch_size=1024,decoder_int_ff=1024,decoder_int_num_layer=8,decoder_rt_ff=512,decoder_rt_num_layer=4,drop__2024-06-25_16-20-34/params.pkl
new file mode 100644
index 0000000000000000000000000000000000000000..7407cc3806eeb80607eabed513e14637907ac226
Binary files /dev/null and b/ray_results/test_experiment/train_model_1de460a8_10_batch_size=1024,decoder_int_ff=1024,decoder_int_num_layer=8,decoder_rt_ff=512,decoder_rt_num_layer=4,drop__2024-06-25_16-20-34/params.pkl differ
diff --git a/ray_results/test_experiment/train_model_1de460a8_10_batch_size=1024,decoder_int_ff=1024,decoder_int_num_layer=8,decoder_rt_ff=512,decoder_rt_num_layer=4,drop__2024-06-25_16-20-34/progress.csv b/ray_results/test_experiment/train_model_1de460a8_10_batch_size=1024,decoder_int_ff=1024,decoder_int_num_layer=8,decoder_rt_ff=512,decoder_rt_num_layer=4,drop__2024-06-25_16-20-34/progress.csv
new file mode 100644
index 0000000000000000000000000000000000000000..f2c7e56c8575af06bca376007b063468f475849c
--- /dev/null
+++ b/ray_results/test_experiment/train_model_1de460a8_10_batch_size=1024,decoder_int_ff=1024,decoder_int_num_layer=8,decoder_rt_ff=512,decoder_rt_num_layer=4,drop__2024-06-25_16-20-34/progress.csv
@@ -0,0 +1,3 @@
+loss,timestamp,checkpoint_dir_name,should_checkpoint,done,training_iteration,trial_id,date,time_this_iter_s,time_total_s,pid,hostname,node_ip,time_since_restore,iterations_since_restore
+1961.4144263079786,1719325387,checkpoint_000000,True,False,1,1de460a8,2024-06-25_16-23-07,73.56608295440674,73.56608295440674,69318,r3i5n6,10.159.8.159,73.56608295440674,1
+1955.3101080947035,1719327763,checkpoint_000001,True,False,2,1de460a8,2024-06-25_17-02-43,72.80412983894348,146.37021279335022,130020,r3i5n6,10.159.8.159,72.80412983894348,1
diff --git a/ray_results/test_experiment/train_model_1de460a8_10_batch_size=1024,decoder_int_ff=1024,decoder_int_num_layer=8,decoder_rt_ff=512,decoder_rt_num_layer=4,drop__2024-06-25_16-20-34/result.json b/ray_results/test_experiment/train_model_1de460a8_10_batch_size=1024,decoder_int_ff=1024,decoder_int_num_layer=8,decoder_rt_ff=512,decoder_rt_num_layer=4,drop__2024-06-25_16-20-34/result.json
new file mode 100644
index 0000000000000000000000000000000000000000..58cd15de84b9e5e5967d6338c351cd63e6c0a693
--- /dev/null
+++ b/ray_results/test_experiment/train_model_1de460a8_10_batch_size=1024,decoder_int_ff=1024,decoder_int_num_layer=8,decoder_rt_ff=512,decoder_rt_num_layer=4,drop__2024-06-25_16-20-34/result.json
@@ -0,0 +1,2 @@
+{"loss": 1961.4144263079786, "timestamp": 1719325387, "checkpoint_dir_name": "checkpoint_000000", "should_checkpoint": true, "done": false, "training_iteration": 1, "trial_id": "1de460a8", "date": "2024-06-25_16-23-07", "time_this_iter_s": 73.56608295440674, "time_total_s": 73.56608295440674, "pid": 69318, "hostname": "r3i5n6", "node_ip": "10.159.8.159", "config": {"encoder_num_layer": 2, "decoder_rt_num_layer": 4, "decoder_int_num_layer": 8, "embedding_dim": 16, "encoder_ff": 2048, "decoder_rt_ff": 512, "decoder_int_ff": 1024, "n_head": 1, "drop_rate": 0.03175079062321118, "lr": 0.003436670599863372, "batch_size": 1024}, "time_since_restore": 73.56608295440674, "iterations_since_restore": 1}
+{"loss": 1955.3101080947035, "timestamp": 1719327763, "checkpoint_dir_name": "checkpoint_000001", "should_checkpoint": true, "done": false, "training_iteration": 2, "trial_id": "1de460a8", "date": "2024-06-25_17-02-43", "time_this_iter_s": 72.80412983894348, "time_total_s": 146.37021279335022, "pid": 130020, "hostname": "r3i5n6", "node_ip": "10.159.8.159", "config": {"encoder_num_layer": 2, "decoder_rt_num_layer": 4, "decoder_int_num_layer": 8, "embedding_dim": 16, "encoder_ff": 2048, "decoder_rt_ff": 512, "decoder_int_ff": 1024, "n_head": 1, "drop_rate": 0.03175079062321118, "lr": 0.003436670599863372, "batch_size": 1024}, "time_since_restore": 72.80412983894348, "iterations_since_restore": 1}
diff --git a/ray_results/test_experiment/train_model_1ef7c88b_11_batch_size=1024,decoder_int_ff=1024,decoder_int_num_layer=4,decoder_rt_ff=512,decoder_rt_num_layer=4,drop__2024-06-25_16-21-54/error.pkl b/ray_results/test_experiment/train_model_1ef7c88b_11_batch_size=1024,decoder_int_ff=1024,decoder_int_num_layer=4,decoder_rt_ff=512,decoder_rt_num_layer=4,drop__2024-06-25_16-21-54/error.pkl
new file mode 100644
index 0000000000000000000000000000000000000000..dc96d91fc2cdcb1d28ff67aa443125d6624678a0
Binary files /dev/null and b/ray_results/test_experiment/train_model_1ef7c88b_11_batch_size=1024,decoder_int_ff=1024,decoder_int_num_layer=4,decoder_rt_ff=512,decoder_rt_num_layer=4,drop__2024-06-25_16-21-54/error.pkl differ
diff --git a/ray_results/test_experiment/train_model_1ef7c88b_11_batch_size=1024,decoder_int_ff=1024,decoder_int_num_layer=4,decoder_rt_ff=512,decoder_rt_num_layer=4,drop__2024-06-25_16-21-54/error.txt b/ray_results/test_experiment/train_model_1ef7c88b_11_batch_size=1024,decoder_int_ff=1024,decoder_int_num_layer=4,decoder_rt_ff=512,decoder_rt_num_layer=4,drop__2024-06-25_16-21-54/error.txt
new file mode 100644
index 0000000000000000000000000000000000000000..da55c0d242bd939db4b4e294ffe5bde84ba71c78
--- /dev/null
+++ b/ray_results/test_experiment/train_model_1ef7c88b_11_batch_size=1024,decoder_int_ff=1024,decoder_int_num_layer=4,decoder_rt_ff=512,decoder_rt_num_layer=4,drop__2024-06-25_16-21-54/error.txt
@@ -0,0 +1,26 @@
+Failure # 1 (occurred at 2024-06-25_17-03-02)
+[36mray::ImplicitFunc.train()[39m (pid=130020, ip=10.159.8.159, actor_id=595d5a321c4a0818d73ea9fa01000000, repr=train_model)
+ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+ File "/gpfslocalsup/pub/anaconda-py3/2023.09/envs/pytorch-gpu-2.2.0+py3.11.7/lib/python3.11/site-packages/ray/tune/trainable/trainable.py", line 342, in train
+ raise skipped from exception_cause(skipped)
+ File "/gpfslocalsup/pub/anaconda-py3/2023.09/envs/pytorch-gpu-2.2.0+py3.11.7/lib/python3.11/site-packages/ray/air/_internal/util.py", line 88, in run
+ self._ret = self._target(*self._args, **self._kwargs)
+ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+ File "/gpfslocalsup/pub/anaconda-py3/2023.09/envs/pytorch-gpu-2.2.0+py3.11.7/lib/python3.11/site-packages/ray/tune/trainable/function_trainable.py", line 249, in <lambda>
+ training_func=lambda: self._trainable_func(self.config),
+ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+ File "/gpfslocalsup/pub/anaconda-py3/2023.09/envs/pytorch-gpu-2.2.0+py3.11.7/lib/python3.11/site-packages/ray/tune/trainable/function_trainable.py", line 332, in _trainable_func
+ output = fn()
+ ^^^^
+ File "/gpfslocalsup/pub/anaconda-py3/2023.09/envs/pytorch-gpu-2.2.0+py3.11.7/lib/python3.11/site-packages/ray/tune/trainable/util.py", line 138, in inner
+ return trainable(config, **fn_kwargs)
+ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+ File "/gpfswork/rech/ute/ucg81ws/these/LC-MS-RT-prediction/main_ray_tune.py", line 98, in train_model
+ loss.backward()
+ File "/gpfslocalsup/pub/anaconda-py3/2023.09/envs/pytorch-gpu-2.2.0+py3.11.7/lib/python3.11/site-packages/torch/_tensor.py", line 522, in backward
+ torch.autograd.backward(
+ File "/gpfslocalsup/pub/anaconda-py3/2023.09/envs/pytorch-gpu-2.2.0+py3.11.7/lib/python3.11/site-packages/torch/autograd/__init__.py", line 266, in backward
+ Variable._execution_engine.run_backward( # Calls into the C++ engine to run the backward pass
+torch.cuda.OutOfMemoryError: CUDA out of memory. Tried to allocate 258.00 MiB. GPU 0 has a total capacity of 15.77 GiB of which 216.69 MiB is free. Including non-PyTorch memory, this process has 15.55 GiB memory in use. Of the allocated memory 15.02 GiB is allocated by PyTorch, and 161.15 MiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables)
diff --git a/ray_results/test_experiment/train_model_1ef7c88b_11_batch_size=1024,decoder_int_ff=1024,decoder_int_num_layer=4,decoder_rt_ff=512,decoder_rt_num_layer=4,drop__2024-06-25_16-21-54/events.out.tfevents.1719325388.r3i5n6 b/ray_results/test_experiment/train_model_1ef7c88b_11_batch_size=1024,decoder_int_ff=1024,decoder_int_num_layer=4,decoder_rt_ff=512,decoder_rt_num_layer=4,drop__2024-06-25_16-21-54/events.out.tfevents.1719325388.r3i5n6
new file mode 100644
index 0000000000000000000000000000000000000000..81eb448e22e971b303b3b631227ff8873a5741e2
Binary files /dev/null and b/ray_results/test_experiment/train_model_1ef7c88b_11_batch_size=1024,decoder_int_ff=1024,decoder_int_num_layer=4,decoder_rt_ff=512,decoder_rt_num_layer=4,drop__2024-06-25_16-21-54/events.out.tfevents.1719325388.r3i5n6 differ
diff --git a/ray_results/test_experiment/train_model_1ef7c88b_11_batch_size=1024,decoder_int_ff=1024,decoder_int_num_layer=4,decoder_rt_ff=512,decoder_rt_num_layer=4,drop__2024-06-25_16-21-54/events.out.tfevents.1719327763.r3i5n6 b/ray_results/test_experiment/train_model_1ef7c88b_11_batch_size=1024,decoder_int_ff=1024,decoder_int_num_layer=4,decoder_rt_ff=512,decoder_rt_num_layer=4,drop__2024-06-25_16-21-54/events.out.tfevents.1719327763.r3i5n6
new file mode 100644
index 0000000000000000000000000000000000000000..34cccb21c50303885980210d17dac2401e234568
Binary files /dev/null and b/ray_results/test_experiment/train_model_1ef7c88b_11_batch_size=1024,decoder_int_ff=1024,decoder_int_num_layer=4,decoder_rt_ff=512,decoder_rt_num_layer=4,drop__2024-06-25_16-21-54/events.out.tfevents.1719327763.r3i5n6 differ
diff --git a/ray_results/test_experiment/train_model_1ef7c88b_11_batch_size=1024,decoder_int_ff=1024,decoder_int_num_layer=4,decoder_rt_ff=512,decoder_rt_num_layer=4,drop__2024-06-25_16-21-54/params.json b/ray_results/test_experiment/train_model_1ef7c88b_11_batch_size=1024,decoder_int_ff=1024,decoder_int_num_layer=4,decoder_rt_ff=512,decoder_rt_num_layer=4,drop__2024-06-25_16-21-54/params.json
new file mode 100644
index 0000000000000000000000000000000000000000..17c6a866d7f3ff71880ba72e4290e8f6785f242e
--- /dev/null
+++ b/ray_results/test_experiment/train_model_1ef7c88b_11_batch_size=1024,decoder_int_ff=1024,decoder_int_num_layer=4,decoder_rt_ff=512,decoder_rt_num_layer=4,drop__2024-06-25_16-21-54/params.json
@@ -0,0 +1,13 @@
+{
+ "batch_size": 1024,
+ "decoder_int_ff": 1024,
+ "decoder_int_num_layer": 4,
+ "decoder_rt_ff": 512,
+ "decoder_rt_num_layer": 4,
+ "drop_rate": 0.14711494497891564,
+ "embedding_dim": 1024,
+ "encoder_ff": 1024,
+ "encoder_num_layer": 8,
+ "lr": 0.00018307057253269556,
+ "n_head": 2
+}
\ No newline at end of file
diff --git a/ray_results/test_experiment/train_model_1ef7c88b_11_batch_size=1024,decoder_int_ff=1024,decoder_int_num_layer=4,decoder_rt_ff=512,decoder_rt_num_layer=4,drop__2024-06-25_16-21-54/params.pkl b/ray_results/test_experiment/train_model_1ef7c88b_11_batch_size=1024,decoder_int_ff=1024,decoder_int_num_layer=4,decoder_rt_ff=512,decoder_rt_num_layer=4,drop__2024-06-25_16-21-54/params.pkl
new file mode 100644
index 0000000000000000000000000000000000000000..761dbb8411c02b9bd6cb33d071a631c4d2c0b735
Binary files /dev/null and b/ray_results/test_experiment/train_model_1ef7c88b_11_batch_size=1024,decoder_int_ff=1024,decoder_int_num_layer=4,decoder_rt_ff=512,decoder_rt_num_layer=4,drop__2024-06-25_16-21-54/params.pkl differ
diff --git a/ray_results/test_experiment/train_model_1ef7c88b_11_batch_size=1024,decoder_int_ff=1024,decoder_int_num_layer=4,decoder_rt_ff=512,decoder_rt_num_layer=4,drop__2024-06-25_16-21-54/progress.csv b/ray_results/test_experiment/train_model_1ef7c88b_11_batch_size=1024,decoder_int_ff=1024,decoder_int_num_layer=4,decoder_rt_ff=512,decoder_rt_num_layer=4,drop__2024-06-25_16-21-54/progress.csv
new file mode 100644
index 0000000000000000000000000000000000000000..aef8460922bcec0e15f91cc9f30d86fb38cdacbc
--- /dev/null
+++ b/ray_results/test_experiment/train_model_1ef7c88b_11_batch_size=1024,decoder_int_ff=1024,decoder_int_num_layer=4,decoder_rt_ff=512,decoder_rt_num_layer=4,drop__2024-06-25_16-21-54/progress.csv
@@ -0,0 +1,2 @@
+loss,timestamp,checkpoint_dir_name,should_checkpoint,done,training_iteration,trial_id,date,time_this_iter_s,time_total_s,pid,hostname,node_ip,time_since_restore,iterations_since_restore
+1963.8436942513533,1719326402,checkpoint_000000,True,False,1,1ef7c88b,2024-06-25_16-40-02,1014.7334928512573,1014.7334928512573,69318,r3i5n6,10.159.8.159,1014.7334928512573,1
diff --git a/ray_results/test_experiment/train_model_1ef7c88b_11_batch_size=1024,decoder_int_ff=1024,decoder_int_num_layer=4,decoder_rt_ff=512,decoder_rt_num_layer=4,drop__2024-06-25_16-21-54/result.json b/ray_results/test_experiment/train_model_1ef7c88b_11_batch_size=1024,decoder_int_ff=1024,decoder_int_num_layer=4,decoder_rt_ff=512,decoder_rt_num_layer=4,drop__2024-06-25_16-21-54/result.json
new file mode 100644
index 0000000000000000000000000000000000000000..9a00620ed155016b3f73b332a7d87fb2a78b9841
--- /dev/null
+++ b/ray_results/test_experiment/train_model_1ef7c88b_11_batch_size=1024,decoder_int_ff=1024,decoder_int_num_layer=4,decoder_rt_ff=512,decoder_rt_num_layer=4,drop__2024-06-25_16-21-54/result.json
@@ -0,0 +1 @@
+{"loss": 1963.8436942513533, "timestamp": 1719326402, "checkpoint_dir_name": "checkpoint_000000", "should_checkpoint": true, "done": false, "training_iteration": 1, "trial_id": "1ef7c88b", "date": "2024-06-25_16-40-02", "time_this_iter_s": 1014.7334928512573, "time_total_s": 1014.7334928512573, "pid": 69318, "hostname": "r3i5n6", "node_ip": "10.159.8.159", "config": {"encoder_num_layer": 8, "decoder_rt_num_layer": 4, "decoder_int_num_layer": 4, "embedding_dim": 1024, "encoder_ff": 1024, "decoder_rt_ff": 512, "decoder_int_ff": 1024, "n_head": 2, "drop_rate": 0.14711494497891564, "lr": 0.00018307057253269556, "batch_size": 1024}, "time_since_restore": 1014.7334928512573, "iterations_since_restore": 1}
diff --git a/ray_results/test_experiment/train_model_2465ee2c_11_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop__2024-06-25_17-34-53/events.out.tfevents.1719329807.r8i6n2 b/ray_results/test_experiment/train_model_2465ee2c_11_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop__2024-06-25_17-34-53/events.out.tfevents.1719329807.r8i6n2
new file mode 100644
index 0000000000000000000000000000000000000000..a58bd4ff7d62f770216de0cf32d7ec8ad74425f3
Binary files /dev/null and b/ray_results/test_experiment/train_model_2465ee2c_11_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop__2024-06-25_17-34-53/events.out.tfevents.1719329807.r8i6n2 differ
diff --git a/ray_results/test_experiment/train_model_2465ee2c_11_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop__2024-06-25_17-34-53/params.json b/ray_results/test_experiment/train_model_2465ee2c_11_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop__2024-06-25_17-34-53/params.json
new file mode 100644
index 0000000000000000000000000000000000000000..534a5f9bdae7aeffd7d8dee12154fc867a212de7
--- /dev/null
+++ b/ray_results/test_experiment/train_model_2465ee2c_11_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop__2024-06-25_17-34-53/params.json
@@ -0,0 +1,13 @@
+{
+ "batch_size": 1024,
+ "decoder_int_ff": 512,
+ "decoder_int_num_layer": 1,
+ "decoder_rt_ff": 1024,
+ "decoder_rt_num_layer": 2,
+ "drop_rate": 0.3089996310219745,
+ "embedding_dim": 64,
+ "encoder_ff": 2048,
+ "encoder_num_layer": 4,
+ "lr": 0.062282868368169514,
+ "n_head": 8
+}
\ No newline at end of file
diff --git a/ray_results/test_experiment/train_model_2465ee2c_11_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop__2024-06-25_17-34-53/params.pkl b/ray_results/test_experiment/train_model_2465ee2c_11_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop__2024-06-25_17-34-53/params.pkl
new file mode 100644
index 0000000000000000000000000000000000000000..bcc16a0a09e990a2b6020c319aed895424b217d0
Binary files /dev/null and b/ray_results/test_experiment/train_model_2465ee2c_11_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop__2024-06-25_17-34-53/params.pkl differ
diff --git a/ray_results/test_experiment/train_model_2465ee2c_11_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop__2024-06-25_17-34-53/progress.csv b/ray_results/test_experiment/train_model_2465ee2c_11_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop__2024-06-25_17-34-53/progress.csv
new file mode 100644
index 0000000000000000000000000000000000000000..69e0cb3eee538f9987373eb9b36398b86e14f93f
--- /dev/null
+++ b/ray_results/test_experiment/train_model_2465ee2c_11_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop__2024-06-25_17-34-53/progress.csv
@@ -0,0 +1,2 @@
+loss,timestamp,checkpoint_dir_name,should_checkpoint,done,training_iteration,trial_id,date,time_this_iter_s,time_total_s,pid,hostname,node_ip,time_since_restore,iterations_since_restore
+2699.211756428396,1719329892,checkpoint_000000,True,False,1,2465ee2c,2024-06-25_17-38-12,85.10484099388123,85.10484099388123,2195253,r8i6n2,10.159.28.60,85.10484099388123,1
diff --git a/ray_results/test_experiment/train_model_2465ee2c_11_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop__2024-06-25_17-34-53/result.json b/ray_results/test_experiment/train_model_2465ee2c_11_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop__2024-06-25_17-34-53/result.json
new file mode 100644
index 0000000000000000000000000000000000000000..ea44c5a7918eae88c8fc50aafa838f09ea5d91b9
--- /dev/null
+++ b/ray_results/test_experiment/train_model_2465ee2c_11_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop__2024-06-25_17-34-53/result.json
@@ -0,0 +1 @@
+{"loss": 2699.211756428396, "timestamp": 1719329892, "checkpoint_dir_name": "checkpoint_000000", "should_checkpoint": true, "done": false, "training_iteration": 1, "trial_id": "2465ee2c", "date": "2024-06-25_17-38-12", "time_this_iter_s": 85.10484099388123, "time_total_s": 85.10484099388123, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 4, "decoder_rt_num_layer": 2, "decoder_int_num_layer": 1, "embedding_dim": 64, "encoder_ff": 2048, "decoder_rt_ff": 1024, "decoder_int_ff": 512, "n_head": 8, "drop_rate": 0.3089996310219745, "lr": 0.062282868368169514, "batch_size": 1024}, "time_since_restore": 85.10484099388123, "iterations_since_restore": 1}
diff --git a/ray_results/test_experiment/train_model_25a9f43a_13_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_17-38-12/events.out.tfevents.1719330099.r8i6n2 b/ray_results/test_experiment/train_model_25a9f43a_13_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_17-38-12/events.out.tfevents.1719330099.r8i6n2
new file mode 100644
index 0000000000000000000000000000000000000000..581d2e5eb098e8e17386809ef437026c18d7184c
Binary files /dev/null and b/ray_results/test_experiment/train_model_25a9f43a_13_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_17-38-12/events.out.tfevents.1719330099.r8i6n2 differ
diff --git a/ray_results/test_experiment/train_model_25a9f43a_13_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_17-38-12/events.out.tfevents.1719330869.r8i6n2 b/ray_results/test_experiment/train_model_25a9f43a_13_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_17-38-12/events.out.tfevents.1719330869.r8i6n2
new file mode 100644
index 0000000000000000000000000000000000000000..f004f6238575b423ce99b9281e20c0ba39f98197
Binary files /dev/null and b/ray_results/test_experiment/train_model_25a9f43a_13_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_17-38-12/events.out.tfevents.1719330869.r8i6n2 differ
diff --git a/ray_results/test_experiment/train_model_25a9f43a_13_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_17-38-12/events.out.tfevents.1719331459.r8i6n2 b/ray_results/test_experiment/train_model_25a9f43a_13_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_17-38-12/events.out.tfevents.1719331459.r8i6n2
new file mode 100644
index 0000000000000000000000000000000000000000..0a564b32bc39785eb520b097cf4fdc0c50d185c7
Binary files /dev/null and b/ray_results/test_experiment/train_model_25a9f43a_13_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_17-38-12/events.out.tfevents.1719331459.r8i6n2 differ
diff --git a/ray_results/test_experiment/train_model_25a9f43a_13_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_17-38-12/params.json b/ray_results/test_experiment/train_model_25a9f43a_13_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_17-38-12/params.json
new file mode 100644
index 0000000000000000000000000000000000000000..e292b13fc433cde73b229257b81c3c75f8fdce64
--- /dev/null
+++ b/ray_results/test_experiment/train_model_25a9f43a_13_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_17-38-12/params.json
@@ -0,0 +1,13 @@
+{
+ "batch_size": 1024,
+ "decoder_int_ff": 512,
+ "decoder_int_num_layer": 1,
+ "decoder_rt_ff": 1024,
+ "decoder_rt_num_layer": 1,
+ "drop_rate": 0.7474706763430264,
+ "embedding_dim": 64,
+ "encoder_ff": 512,
+ "encoder_num_layer": 4,
+ "lr": 0.00045083297498032556,
+ "n_head": 16
+}
\ No newline at end of file
diff --git a/ray_results/test_experiment/train_model_25a9f43a_13_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_17-38-12/params.pkl b/ray_results/test_experiment/train_model_25a9f43a_13_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_17-38-12/params.pkl
new file mode 100644
index 0000000000000000000000000000000000000000..7004f92dc9779858835e886d54e70af12fbc782f
Binary files /dev/null and b/ray_results/test_experiment/train_model_25a9f43a_13_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_17-38-12/params.pkl differ
diff --git a/ray_results/test_experiment/train_model_25a9f43a_13_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_17-38-12/progress.csv b/ray_results/test_experiment/train_model_25a9f43a_13_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_17-38-12/progress.csv
new file mode 100644
index 0000000000000000000000000000000000000000..47d8699f6d39cbd327c8fec36c002b334369addb
--- /dev/null
+++ b/ray_results/test_experiment/train_model_25a9f43a_13_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_17-38-12/progress.csv
@@ -0,0 +1,5 @@
+loss,timestamp,checkpoint_dir_name,should_checkpoint,done,training_iteration,trial_id,date,time_this_iter_s,time_total_s,pid,hostname,node_ip,time_since_restore,iterations_since_restore
+1284.8956039308564,1719330178,checkpoint_000000,True,False,1,25a9f43a,2024-06-25_17-42-59,79.39262270927429,79.39262270927429,2195253,r8i6n2,10.159.28.60,79.39262270927429,1
+517.7363889198604,1719330947,checkpoint_000001,True,False,2,25a9f43a,2024-06-25_17-55-47,77.99471831321716,157.38734102249146,2195253,r8i6n2,10.159.28.60,77.99471831321716,1
+424.9116310660295,1719331554,checkpoint_000002,True,False,3,25a9f43a,2024-06-25_18-05-54,94.57213068008423,251.95947170257568,2195253,r8i6n2,10.159.28.60,94.57213068008423,1
+381.2108863169753,1719331634,checkpoint_000003,True,False,4,25a9f43a,2024-06-25_18-07-14,80.07035231590271,332.0298240184784,2195253,r8i6n2,10.159.28.60,174.64248299598694,2
diff --git a/ray_results/test_experiment/train_model_25a9f43a_13_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_17-38-12/result.json b/ray_results/test_experiment/train_model_25a9f43a_13_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_17-38-12/result.json
new file mode 100644
index 0000000000000000000000000000000000000000..e7cb0d2fbcc809acdcff6775e6ece41d2ac84d9d
--- /dev/null
+++ b/ray_results/test_experiment/train_model_25a9f43a_13_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_17-38-12/result.json
@@ -0,0 +1,4 @@
+{"loss": 1284.8956039308564, "timestamp": 1719330178, "checkpoint_dir_name": "checkpoint_000000", "should_checkpoint": true, "done": false, "training_iteration": 1, "trial_id": "25a9f43a", "date": "2024-06-25_17-42-59", "time_this_iter_s": 79.39262270927429, "time_total_s": 79.39262270927429, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 4, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 64, "encoder_ff": 512, "decoder_rt_ff": 1024, "decoder_int_ff": 512, "n_head": 16, "drop_rate": 0.7474706763430264, "lr": 0.00045083297498032556, "batch_size": 1024}, "time_since_restore": 79.39262270927429, "iterations_since_restore": 1}
+{"loss": 517.7363889198604, "timestamp": 1719330947, "checkpoint_dir_name": "checkpoint_000001", "should_checkpoint": true, "done": false, "training_iteration": 2, "trial_id": "25a9f43a", "date": "2024-06-25_17-55-47", "time_this_iter_s": 77.99471831321716, "time_total_s": 157.38734102249146, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 4, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 64, "encoder_ff": 512, "decoder_rt_ff": 1024, "decoder_int_ff": 512, "n_head": 16, "drop_rate": 0.7474706763430264, "lr": 0.00045083297498032556, "batch_size": 1024}, "time_since_restore": 77.99471831321716, "iterations_since_restore": 1}
+{"loss": 424.9116310660295, "timestamp": 1719331554, "checkpoint_dir_name": "checkpoint_000002", "should_checkpoint": true, "done": false, "training_iteration": 3, "trial_id": "25a9f43a", "date": "2024-06-25_18-05-54", "time_this_iter_s": 94.57213068008423, "time_total_s": 251.95947170257568, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 4, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 64, "encoder_ff": 512, "decoder_rt_ff": 1024, "decoder_int_ff": 512, "n_head": 16, "drop_rate": 0.7474706763430264, "lr": 0.00045083297498032556, "batch_size": 1024}, "time_since_restore": 94.57213068008423, "iterations_since_restore": 1}
+{"loss": 381.2108863169753, "timestamp": 1719331634, "checkpoint_dir_name": "checkpoint_000003", "should_checkpoint": true, "done": false, "training_iteration": 4, "trial_id": "25a9f43a", "date": "2024-06-25_18-07-14", "time_this_iter_s": 80.07035231590271, "time_total_s": 332.0298240184784, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 4, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 64, "encoder_ff": 512, "decoder_rt_ff": 1024, "decoder_int_ff": 512, "n_head": 16, "drop_rate": 0.7474706763430264, "lr": 0.00045083297498032556, "batch_size": 1024}, "time_since_restore": 174.64248299598694, "iterations_since_restore": 2}
diff --git a/ray_results/test_experiment/train_model_25b42194_15_batch_size=2048,decoder_int_ff=2048,decoder_int_num_layer=8,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop_2024-06-25_16-42-52/events.out.tfevents.1719326598.r3i5n6 b/ray_results/test_experiment/train_model_25b42194_15_batch_size=2048,decoder_int_ff=2048,decoder_int_num_layer=8,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop_2024-06-25_16-42-52/events.out.tfevents.1719326598.r3i5n6
new file mode 100644
index 0000000000000000000000000000000000000000..1de1084d20c575948087454049071e447b8ab3cf
Binary files /dev/null and b/ray_results/test_experiment/train_model_25b42194_15_batch_size=2048,decoder_int_ff=2048,decoder_int_num_layer=8,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop_2024-06-25_16-42-52/events.out.tfevents.1719326598.r3i5n6 differ
diff --git a/ray_results/test_experiment/train_model_25b42194_15_batch_size=2048,decoder_int_ff=2048,decoder_int_num_layer=8,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop_2024-06-25_16-42-52/params.json b/ray_results/test_experiment/train_model_25b42194_15_batch_size=2048,decoder_int_ff=2048,decoder_int_num_layer=8,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop_2024-06-25_16-42-52/params.json
new file mode 100644
index 0000000000000000000000000000000000000000..11cf8d5a68cb1fdd5393c46c876fe910a608c140
--- /dev/null
+++ b/ray_results/test_experiment/train_model_25b42194_15_batch_size=2048,decoder_int_ff=2048,decoder_int_num_layer=8,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop_2024-06-25_16-42-52/params.json
@@ -0,0 +1,13 @@
+{
+ "batch_size": 2048,
+ "decoder_int_ff": 2048,
+ "decoder_int_num_layer": 8,
+ "decoder_rt_ff": 1024,
+ "decoder_rt_num_layer": 1,
+ "drop_rate": 0.3256301932631026,
+ "embedding_dim": 1024,
+ "encoder_ff": 512,
+ "encoder_num_layer": 1,
+ "lr": 0.0018178629337167497,
+ "n_head": 16
+}
\ No newline at end of file
diff --git a/ray_results/test_experiment/train_model_25b42194_15_batch_size=2048,decoder_int_ff=2048,decoder_int_num_layer=8,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop_2024-06-25_16-42-52/params.pkl b/ray_results/test_experiment/train_model_25b42194_15_batch_size=2048,decoder_int_ff=2048,decoder_int_num_layer=8,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop_2024-06-25_16-42-52/params.pkl
new file mode 100644
index 0000000000000000000000000000000000000000..488fd5b39c866d502307fe72e6f5acd8cb2e9bd7
Binary files /dev/null and b/ray_results/test_experiment/train_model_25b42194_15_batch_size=2048,decoder_int_ff=2048,decoder_int_num_layer=8,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop_2024-06-25_16-42-52/params.pkl differ
diff --git a/ray_results/test_experiment/train_model_25b42194_15_batch_size=2048,decoder_int_ff=2048,decoder_int_num_layer=8,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop_2024-06-25_16-42-52/progress.csv b/ray_results/test_experiment/train_model_25b42194_15_batch_size=2048,decoder_int_ff=2048,decoder_int_num_layer=8,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop_2024-06-25_16-42-52/progress.csv
new file mode 100644
index 0000000000000000000000000000000000000000..ef0228f56ebfa86cbf6fc14049bfc47242425393
--- /dev/null
+++ b/ray_results/test_experiment/train_model_25b42194_15_batch_size=2048,decoder_int_ff=2048,decoder_int_num_layer=8,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop_2024-06-25_16-42-52/progress.csv
@@ -0,0 +1,2 @@
+loss,timestamp,checkpoint_dir_name,should_checkpoint,done,training_iteration,trial_id,date,time_this_iter_s,time_total_s,pid,hostname,node_ip,time_since_restore,iterations_since_restore
+5061.415592704232,1719326792,checkpoint_000000,True,False,1,25b42194,2024-06-25_16-46-32,194.36064219474792,194.36064219474792,122466,r3i5n6,10.159.8.159,194.36064219474792,1
diff --git a/ray_results/test_experiment/train_model_25b42194_15_batch_size=2048,decoder_int_ff=2048,decoder_int_num_layer=8,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop_2024-06-25_16-42-52/result.json b/ray_results/test_experiment/train_model_25b42194_15_batch_size=2048,decoder_int_ff=2048,decoder_int_num_layer=8,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop_2024-06-25_16-42-52/result.json
new file mode 100644
index 0000000000000000000000000000000000000000..2bed45591f0fdb56b11171b3835888d428b80432
--- /dev/null
+++ b/ray_results/test_experiment/train_model_25b42194_15_batch_size=2048,decoder_int_ff=2048,decoder_int_num_layer=8,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop_2024-06-25_16-42-52/result.json
@@ -0,0 +1 @@
+{"loss": 5061.415592704232, "timestamp": 1719326792, "checkpoint_dir_name": "checkpoint_000000", "should_checkpoint": true, "done": false, "training_iteration": 1, "trial_id": "25b42194", "date": "2024-06-25_16-46-32", "time_this_iter_s": 194.36064219474792, "time_total_s": 194.36064219474792, "pid": 122466, "hostname": "r3i5n6", "node_ip": "10.159.8.159", "config": {"encoder_num_layer": 1, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 8, "embedding_dim": 1024, "encoder_ff": 512, "decoder_rt_ff": 1024, "decoder_int_ff": 2048, "n_head": 16, "drop_rate": 0.3256301932631026, "lr": 0.0018178629337167497, "batch_size": 2048}, "time_since_restore": 194.36064219474792, "iterations_since_restore": 1}
diff --git a/ray_results/test_experiment/train_model_3510e84b_15_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop__2024-06-25_17-42-59/events.out.tfevents.1719330290.r8i6n2 b/ray_results/test_experiment/train_model_3510e84b_15_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop__2024-06-25_17-42-59/events.out.tfevents.1719330290.r8i6n2
new file mode 100644
index 0000000000000000000000000000000000000000..42f4d6e380450ceb2064f42298987f704a563cff
Binary files /dev/null and b/ray_results/test_experiment/train_model_3510e84b_15_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop__2024-06-25_17-42-59/events.out.tfevents.1719330290.r8i6n2 differ
diff --git a/ray_results/test_experiment/train_model_3510e84b_15_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop__2024-06-25_17-42-59/events.out.tfevents.1719331058.r8i6n2 b/ray_results/test_experiment/train_model_3510e84b_15_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop__2024-06-25_17-42-59/events.out.tfevents.1719331058.r8i6n2
new file mode 100644
index 0000000000000000000000000000000000000000..0e0d499863dcfe68eed6184f569508c2a3993d90
Binary files /dev/null and b/ray_results/test_experiment/train_model_3510e84b_15_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop__2024-06-25_17-42-59/events.out.tfevents.1719331058.r8i6n2 differ
diff --git a/ray_results/test_experiment/train_model_3510e84b_15_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop__2024-06-25_17-42-59/params.json b/ray_results/test_experiment/train_model_3510e84b_15_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop__2024-06-25_17-42-59/params.json
new file mode 100644
index 0000000000000000000000000000000000000000..b2801fd53e5af95dfaf6900909bb125aed4776e3
--- /dev/null
+++ b/ray_results/test_experiment/train_model_3510e84b_15_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop__2024-06-25_17-42-59/params.json
@@ -0,0 +1,13 @@
+{
+ "batch_size": 2048,
+ "decoder_int_ff": 512,
+ "decoder_int_num_layer": 1,
+ "decoder_rt_ff": 2048,
+ "decoder_rt_num_layer": 1,
+ "drop_rate": 0.3408951993486181,
+ "embedding_dim": 16,
+ "encoder_ff": 2048,
+ "encoder_num_layer": 4,
+ "lr": 0.009004430142657121,
+ "n_head": 4
+}
\ No newline at end of file
diff --git a/ray_results/test_experiment/train_model_3510e84b_15_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop__2024-06-25_17-42-59/params.pkl b/ray_results/test_experiment/train_model_3510e84b_15_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop__2024-06-25_17-42-59/params.pkl
new file mode 100644
index 0000000000000000000000000000000000000000..c983efd0bd3463d016ccf3fdf090ad866722f9f5
Binary files /dev/null and b/ray_results/test_experiment/train_model_3510e84b_15_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop__2024-06-25_17-42-59/params.pkl differ
diff --git a/ray_results/test_experiment/train_model_3510e84b_15_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop__2024-06-25_17-42-59/progress.csv b/ray_results/test_experiment/train_model_3510e84b_15_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop__2024-06-25_17-42-59/progress.csv
new file mode 100644
index 0000000000000000000000000000000000000000..7c7780aa1bdf4ebf523be9cd8420343cba158f0c
--- /dev/null
+++ b/ray_results/test_experiment/train_model_3510e84b_15_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop__2024-06-25_17-42-59/progress.csv
@@ -0,0 +1,3 @@
+loss,timestamp,checkpoint_dir_name,should_checkpoint,done,training_iteration,trial_id,date,time_this_iter_s,time_total_s,pid,hostname,node_ip,time_since_restore,iterations_since_restore
+1964.014622485544,1719330362,checkpoint_000000,True,False,1,3510e84b,2024-06-25_17-46-02,72.72616767883301,72.72616767883301,2195253,r8i6n2,10.159.28.60,72.72616767883301,1
+1964.9452173428272,1719331131,checkpoint_000001,True,True,2,3510e84b,2024-06-25_17-58-51,73.41431927680969,146.1404869556427,2195253,r8i6n2,10.159.28.60,73.41431927680969,1
diff --git a/ray_results/test_experiment/train_model_3510e84b_15_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop__2024-06-25_17-42-59/result.json b/ray_results/test_experiment/train_model_3510e84b_15_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop__2024-06-25_17-42-59/result.json
new file mode 100644
index 0000000000000000000000000000000000000000..19328aa43a4d6d3f8f09052eb9c121f9eedd428d
--- /dev/null
+++ b/ray_results/test_experiment/train_model_3510e84b_15_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop__2024-06-25_17-42-59/result.json
@@ -0,0 +1,2 @@
+{"loss": 1964.014622485544, "timestamp": 1719330362, "checkpoint_dir_name": "checkpoint_000000", "should_checkpoint": true, "done": false, "training_iteration": 1, "trial_id": "3510e84b", "date": "2024-06-25_17-46-02", "time_this_iter_s": 72.72616767883301, "time_total_s": 72.72616767883301, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 4, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 2048, "decoder_rt_ff": 2048, "decoder_int_ff": 512, "n_head": 4, "drop_rate": 0.3408951993486181, "lr": 0.009004430142657121, "batch_size": 2048}, "time_since_restore": 72.72616767883301, "iterations_since_restore": 1}
+{"loss": 1964.9452173428272, "timestamp": 1719331131, "checkpoint_dir_name": "checkpoint_000001", "should_checkpoint": true, "done": true, "training_iteration": 2, "trial_id": "3510e84b", "date": "2024-06-25_17-58-51", "time_this_iter_s": 73.41431927680969, "time_total_s": 146.1404869556427, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 4, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 2048, "decoder_rt_ff": 2048, "decoder_int_ff": 512, "n_head": 4, "drop_rate": 0.3408951993486181, "lr": 0.009004430142657121, "batch_size": 2048}, "time_since_restore": 73.41431927680969, "iterations_since_restore": 1}
diff --git a/ray_results/test_experiment/train_model_375ed3d7_19_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=2,decoder_rt_ff=1024,decoder_rt_num_layer=4,drop__2024-06-25_16-56-12/events.out.tfevents.1719327790.r3i5n6 b/ray_results/test_experiment/train_model_375ed3d7_19_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=2,decoder_rt_ff=1024,decoder_rt_num_layer=4,drop__2024-06-25_16-56-12/events.out.tfevents.1719327790.r3i5n6
new file mode 100644
index 0000000000000000000000000000000000000000..08e5459ca585e45531b5d6b1109a0c8472d3c574
Binary files /dev/null and b/ray_results/test_experiment/train_model_375ed3d7_19_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=2,decoder_rt_ff=1024,decoder_rt_num_layer=4,drop__2024-06-25_16-56-12/events.out.tfevents.1719327790.r3i5n6 differ
diff --git a/ray_results/test_experiment/train_model_375ed3d7_19_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=2,decoder_rt_ff=1024,decoder_rt_num_layer=4,drop__2024-06-25_16-56-12/params.json b/ray_results/test_experiment/train_model_375ed3d7_19_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=2,decoder_rt_ff=1024,decoder_rt_num_layer=4,drop__2024-06-25_16-56-12/params.json
new file mode 100644
index 0000000000000000000000000000000000000000..970fbc421d7181351bd824e4e908ba093c1d1033
--- /dev/null
+++ b/ray_results/test_experiment/train_model_375ed3d7_19_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=2,decoder_rt_ff=1024,decoder_rt_num_layer=4,drop__2024-06-25_16-56-12/params.json
@@ -0,0 +1,13 @@
+{
+ "batch_size": 1024,
+ "decoder_int_ff": 512,
+ "decoder_int_num_layer": 2,
+ "decoder_rt_ff": 1024,
+ "decoder_rt_num_layer": 4,
+ "drop_rate": 0.29850788173794873,
+ "embedding_dim": 16,
+ "encoder_ff": 1024,
+ "encoder_num_layer": 1,
+ "lr": 0.017371208825505195,
+ "n_head": 1
+}
\ No newline at end of file
diff --git a/ray_results/test_experiment/train_model_375ed3d7_19_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=2,decoder_rt_ff=1024,decoder_rt_num_layer=4,drop__2024-06-25_16-56-12/params.pkl b/ray_results/test_experiment/train_model_375ed3d7_19_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=2,decoder_rt_ff=1024,decoder_rt_num_layer=4,drop__2024-06-25_16-56-12/params.pkl
new file mode 100644
index 0000000000000000000000000000000000000000..38d5b145945a4bf9e65a628967eafd993a61f6b3
Binary files /dev/null and b/ray_results/test_experiment/train_model_375ed3d7_19_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=2,decoder_rt_ff=1024,decoder_rt_num_layer=4,drop__2024-06-25_16-56-12/params.pkl differ
diff --git a/ray_results/test_experiment/train_model_375ed3d7_19_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=2,decoder_rt_ff=1024,decoder_rt_num_layer=4,drop__2024-06-25_16-56-12/progress.csv b/ray_results/test_experiment/train_model_375ed3d7_19_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=2,decoder_rt_ff=1024,decoder_rt_num_layer=4,drop__2024-06-25_16-56-12/progress.csv
new file mode 100644
index 0000000000000000000000000000000000000000..0f8fff3c200893434bced60b5f54c6d7d4fb3e49
--- /dev/null
+++ b/ray_results/test_experiment/train_model_375ed3d7_19_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=2,decoder_rt_ff=1024,decoder_rt_num_layer=4,drop__2024-06-25_16-56-12/progress.csv
@@ -0,0 +1,2 @@
+loss,timestamp,checkpoint_dir_name,should_checkpoint,done,training_iteration,trial_id,date,time_this_iter_s,time_total_s,pid,hostname,node_ip,time_since_restore,iterations_since_restore
+1971.010832058163,1719327860,checkpoint_000000,True,False,1,375ed3d7,2024-06-25_17-04-20,70.38656640052795,70.38656640052795,168065,r3i5n6,10.159.8.159,70.38656640052795,1
diff --git a/ray_results/test_experiment/train_model_375ed3d7_19_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=2,decoder_rt_ff=1024,decoder_rt_num_layer=4,drop__2024-06-25_16-56-12/result.json b/ray_results/test_experiment/train_model_375ed3d7_19_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=2,decoder_rt_ff=1024,decoder_rt_num_layer=4,drop__2024-06-25_16-56-12/result.json
new file mode 100644
index 0000000000000000000000000000000000000000..08a067b4fbff8cc21a36cb2d9d3548b1dd6c5b0d
--- /dev/null
+++ b/ray_results/test_experiment/train_model_375ed3d7_19_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=2,decoder_rt_ff=1024,decoder_rt_num_layer=4,drop__2024-06-25_16-56-12/result.json
@@ -0,0 +1 @@
+{"loss": 1971.010832058163, "timestamp": 1719327860, "checkpoint_dir_name": "checkpoint_000000", "should_checkpoint": true, "done": false, "training_iteration": 1, "trial_id": "375ed3d7", "date": "2024-06-25_17-04-20", "time_this_iter_s": 70.38656640052795, "time_total_s": 70.38656640052795, "pid": 168065, "hostname": "r3i5n6", "node_ip": "10.159.8.159", "config": {"encoder_num_layer": 1, "decoder_rt_num_layer": 4, "decoder_int_num_layer": 2, "embedding_dim": 16, "encoder_ff": 1024, "decoder_rt_ff": 1024, "decoder_int_ff": 512, "n_head": 1, "drop_rate": 0.29850788173794873, "lr": 0.017371208825505195, "batch_size": 1024}, "time_since_restore": 70.38656640052795, "iterations_since_restore": 1}
diff --git a/ray_results/test_experiment/train_model_3ab2f301_28_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop__2024-06-25_22-10-34/events.out.tfevents.1719346547.r8i6n2 b/ray_results/test_experiment/train_model_3ab2f301_28_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop__2024-06-25_22-10-34/events.out.tfevents.1719346547.r8i6n2
new file mode 100644
index 0000000000000000000000000000000000000000..33636372c728ca899def9f9d20e3bcc1bdaf621c
Binary files /dev/null and b/ray_results/test_experiment/train_model_3ab2f301_28_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop__2024-06-25_22-10-34/events.out.tfevents.1719346547.r8i6n2 differ
diff --git a/ray_results/test_experiment/train_model_3ab2f301_28_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop__2024-06-25_22-10-34/events.out.tfevents.1719354952.r8i6n2 b/ray_results/test_experiment/train_model_3ab2f301_28_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop__2024-06-25_22-10-34/events.out.tfevents.1719354952.r8i6n2
new file mode 100644
index 0000000000000000000000000000000000000000..6bc61b20030cbda1b55aa35ee326d6af18723a77
Binary files /dev/null and b/ray_results/test_experiment/train_model_3ab2f301_28_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop__2024-06-25_22-10-34/events.out.tfevents.1719354952.r8i6n2 differ
diff --git a/ray_results/test_experiment/train_model_3ab2f301_28_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop__2024-06-25_22-10-34/events.out.tfevents.1719357872.r8i6n2 b/ray_results/test_experiment/train_model_3ab2f301_28_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop__2024-06-25_22-10-34/events.out.tfevents.1719357872.r8i6n2
new file mode 100644
index 0000000000000000000000000000000000000000..72a5f6bff7d168e3aef88a728e9af5d4168b3f80
Binary files /dev/null and b/ray_results/test_experiment/train_model_3ab2f301_28_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop__2024-06-25_22-10-34/events.out.tfevents.1719357872.r8i6n2 differ
diff --git a/ray_results/test_experiment/train_model_3ab2f301_28_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop__2024-06-25_22-10-34/params.json b/ray_results/test_experiment/train_model_3ab2f301_28_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop__2024-06-25_22-10-34/params.json
new file mode 100644
index 0000000000000000000000000000000000000000..1e2acc471ea898afa2bd246d5b6c44a29dfa2a54
--- /dev/null
+++ b/ray_results/test_experiment/train_model_3ab2f301_28_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop__2024-06-25_22-10-34/params.json
@@ -0,0 +1,13 @@
+{
+ "batch_size": 2048,
+ "decoder_int_ff": 512,
+ "decoder_int_num_layer": 1,
+ "decoder_rt_ff": 2048,
+ "decoder_rt_num_layer": 1,
+ "drop_rate": 0.034481700426631456,
+ "embedding_dim": 16,
+ "encoder_ff": 2048,
+ "encoder_num_layer": 2,
+ "lr": 0.0010186128360476897,
+ "n_head": 1
+}
\ No newline at end of file
diff --git a/ray_results/test_experiment/train_model_3ab2f301_28_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop__2024-06-25_22-10-34/params.pkl b/ray_results/test_experiment/train_model_3ab2f301_28_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop__2024-06-25_22-10-34/params.pkl
new file mode 100644
index 0000000000000000000000000000000000000000..5741d83de357f483edaa8b4c62494ccccb584429
Binary files /dev/null and b/ray_results/test_experiment/train_model_3ab2f301_28_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop__2024-06-25_22-10-34/params.pkl differ
diff --git a/ray_results/test_experiment/train_model_3ab2f301_28_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop__2024-06-25_22-10-34/progress.csv b/ray_results/test_experiment/train_model_3ab2f301_28_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop__2024-06-25_22-10-34/progress.csv
new file mode 100644
index 0000000000000000000000000000000000000000..65321cfcd3e870505a7b73f1b8d6e85df132d285
--- /dev/null
+++ b/ray_results/test_experiment/train_model_3ab2f301_28_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop__2024-06-25_22-10-34/progress.csv
@@ -0,0 +1,21 @@
+loss,timestamp,checkpoint_dir_name,should_checkpoint,done,training_iteration,trial_id,date,time_this_iter_s,time_total_s,pid,hostname,node_ip,time_since_restore,iterations_since_restore
+217.46355960875985,1719346612,checkpoint_000000,True,False,1,3ab2f301,2024-06-25_22-16-52,65.41611504554749,65.41611504554749,2195253,r8i6n2,10.159.28.60,65.41611504554749,1
+163.8450069878045,1719346664,checkpoint_000001,True,False,2,3ab2f301,2024-06-25_22-17-45,52.40175247192383,117.81786751747131,2195253,r8i6n2,10.159.28.60,117.81786751747131,2
+139.51932639775313,1719346717,checkpoint_000002,True,False,3,3ab2f301,2024-06-25_22-18-37,52.207762479782104,170.02562999725342,2195253,r8i6n2,10.159.28.60,170.02562999725342,3
+125.82796093798059,1719346770,checkpoint_000003,True,False,4,3ab2f301,2024-06-25_22-19-30,53.092029094696045,223.11765909194946,2195253,r8i6n2,10.159.28.60,223.11765909194946,4
+117.45965227742833,1719346823,checkpoint_000004,True,False,5,3ab2f301,2024-06-25_22-20-23,53.195783615112305,276.31344270706177,2195253,r8i6n2,10.159.28.60,276.31344270706177,5
+114.47784069391686,1719355017,checkpoint_000005,True,False,6,3ab2f301,2024-06-26_00-36-57,65.25149631500244,341.5649390220642,2850562,r8i6n2,10.159.28.60,65.25149631500244,1
+109.24919566957969,1719355069,checkpoint_000006,True,False,7,3ab2f301,2024-06-26_00-37-49,51.495158195495605,393.0600972175598,2850562,r8i6n2,10.159.28.60,116.74665451049805,2
+105.2013188159372,1719355120,checkpoint_000007,True,False,8,3ab2f301,2024-06-26_00-38-40,51.496910572052,444.5570077896118,2850562,r8i6n2,10.159.28.60,168.24356508255005,3
+118.58170366662694,1719355171,checkpoint_000008,True,False,9,3ab2f301,2024-06-26_00-39-31,51.06172227859497,495.6187300682068,2850562,r8i6n2,10.159.28.60,219.30528736114502,4
+99.85706461508443,1719355223,checkpoint_000009,True,False,10,3ab2f301,2024-06-26_00-40-23,51.387521266937256,547.006251335144,2850562,r8i6n2,10.159.28.60,270.6928086280823,5
+98.49699257677935,1719357936,checkpoint_000010,True,False,11,3ab2f301,2024-06-26_01-25-36,64.502512216568,611.508763551712,2850562,r8i6n2,10.159.28.60,64.502512216568,1
+93.49606281190407,1719357987,checkpoint_000011,True,False,12,3ab2f301,2024-06-26_01-26-27,50.864012002944946,662.372775554657,2850562,r8i6n2,10.159.28.60,115.36652421951294,2
+94.78943622018409,1719358039,checkpoint_000012,True,False,13,3ab2f301,2024-06-26_01-27-19,51.64423751831055,714.0170130729675,2850562,r8i6n2,10.159.28.60,167.0107617378235,3
+91.46543307567205,1719358090,checkpoint_000013,True,False,14,3ab2f301,2024-06-26_01-28-10,51.174827098846436,765.191840171814,2850562,r8i6n2,10.159.28.60,218.18558883666992,4
+88.73407943605439,1719358141,checkpoint_000014,True,False,15,3ab2f301,2024-06-26_01-29-01,50.784724712371826,815.9765648841858,2850562,r8i6n2,10.159.28.60,268.97031354904175,5
+94.00536857064314,1719358192,checkpoint_000015,True,False,16,3ab2f301,2024-06-26_01-29-52,51.60023093223572,867.5767958164215,2850562,r8i6n2,10.159.28.60,320.57054448127747,6
+87.47035950187623,1719358244,checkpoint_000016,True,False,17,3ab2f301,2024-06-26_01-30-44,51.84270000457764,919.4194958209991,2850562,r8i6n2,10.159.28.60,372.4132444858551,7
+88.06384884090875,1719358295,checkpoint_000017,True,False,18,3ab2f301,2024-06-26_01-31-35,51.09972786903381,970.519223690033,2850562,r8i6n2,10.159.28.60,423.5129723548889,8
+87.50748780017763,1719358346,checkpoint_000018,True,False,19,3ab2f301,2024-06-26_01-32-26,51.09087514877319,1021.6100988388062,2850562,r8i6n2,10.159.28.60,474.6038475036621,9
+84.81797225832001,1719358398,checkpoint_000019,True,False,20,3ab2f301,2024-06-26_01-33-18,51.372178077697754,1072.982276916504,2850562,r8i6n2,10.159.28.60,525.9760255813599,10
diff --git a/ray_results/test_experiment/train_model_3ab2f301_28_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop__2024-06-25_22-10-34/result.json b/ray_results/test_experiment/train_model_3ab2f301_28_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop__2024-06-25_22-10-34/result.json
new file mode 100644
index 0000000000000000000000000000000000000000..044b6351ac2ab0a11841aa9f90a4249e007c874c
--- /dev/null
+++ b/ray_results/test_experiment/train_model_3ab2f301_28_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop__2024-06-25_22-10-34/result.json
@@ -0,0 +1,20 @@
+{"loss": 217.46355960875985, "timestamp": 1719346612, "checkpoint_dir_name": "checkpoint_000000", "should_checkpoint": true, "done": false, "training_iteration": 1, "trial_id": "3ab2f301", "date": "2024-06-25_22-16-52", "time_this_iter_s": 65.41611504554749, "time_total_s": 65.41611504554749, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 2, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 2048, "decoder_rt_ff": 2048, "decoder_int_ff": 512, "n_head": 1, "drop_rate": 0.034481700426631456, "lr": 0.0010186128360476897, "batch_size": 2048}, "time_since_restore": 65.41611504554749, "iterations_since_restore": 1}
+{"loss": 163.8450069878045, "timestamp": 1719346664, "checkpoint_dir_name": "checkpoint_000001", "should_checkpoint": true, "done": false, "training_iteration": 2, "trial_id": "3ab2f301", "date": "2024-06-25_22-17-45", "time_this_iter_s": 52.40175247192383, "time_total_s": 117.81786751747131, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 2, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 2048, "decoder_rt_ff": 2048, "decoder_int_ff": 512, "n_head": 1, "drop_rate": 0.034481700426631456, "lr": 0.0010186128360476897, "batch_size": 2048}, "time_since_restore": 117.81786751747131, "iterations_since_restore": 2}
+{"loss": 139.51932639775313, "timestamp": 1719346717, "checkpoint_dir_name": "checkpoint_000002", "should_checkpoint": true, "done": false, "training_iteration": 3, "trial_id": "3ab2f301", "date": "2024-06-25_22-18-37", "time_this_iter_s": 52.207762479782104, "time_total_s": 170.02562999725342, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 2, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 2048, "decoder_rt_ff": 2048, "decoder_int_ff": 512, "n_head": 1, "drop_rate": 0.034481700426631456, "lr": 0.0010186128360476897, "batch_size": 2048}, "time_since_restore": 170.02562999725342, "iterations_since_restore": 3}
+{"loss": 125.82796093798059, "timestamp": 1719346770, "checkpoint_dir_name": "checkpoint_000003", "should_checkpoint": true, "done": false, "training_iteration": 4, "trial_id": "3ab2f301", "date": "2024-06-25_22-19-30", "time_this_iter_s": 53.092029094696045, "time_total_s": 223.11765909194946, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 2, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 2048, "decoder_rt_ff": 2048, "decoder_int_ff": 512, "n_head": 1, "drop_rate": 0.034481700426631456, "lr": 0.0010186128360476897, "batch_size": 2048}, "time_since_restore": 223.11765909194946, "iterations_since_restore": 4}
+{"loss": 117.45965227742833, "timestamp": 1719346823, "checkpoint_dir_name": "checkpoint_000004", "should_checkpoint": true, "done": false, "training_iteration": 5, "trial_id": "3ab2f301", "date": "2024-06-25_22-20-23", "time_this_iter_s": 53.195783615112305, "time_total_s": 276.31344270706177, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 2, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 2048, "decoder_rt_ff": 2048, "decoder_int_ff": 512, "n_head": 1, "drop_rate": 0.034481700426631456, "lr": 0.0010186128360476897, "batch_size": 2048}, "time_since_restore": 276.31344270706177, "iterations_since_restore": 5}
+{"loss": 114.47784069391686, "timestamp": 1719355017, "checkpoint_dir_name": "checkpoint_000005", "should_checkpoint": true, "done": false, "training_iteration": 6, "trial_id": "3ab2f301", "date": "2024-06-26_00-36-57", "time_this_iter_s": 65.25149631500244, "time_total_s": 341.5649390220642, "pid": 2850562, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 2, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 2048, "decoder_rt_ff": 2048, "decoder_int_ff": 512, "n_head": 1, "drop_rate": 0.034481700426631456, "lr": 0.0010186128360476897, "batch_size": 2048}, "time_since_restore": 65.25149631500244, "iterations_since_restore": 1}
+{"loss": 109.24919566957969, "timestamp": 1719355069, "checkpoint_dir_name": "checkpoint_000006", "should_checkpoint": true, "done": false, "training_iteration": 7, "trial_id": "3ab2f301", "date": "2024-06-26_00-37-49", "time_this_iter_s": 51.495158195495605, "time_total_s": 393.0600972175598, "pid": 2850562, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 2, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 2048, "decoder_rt_ff": 2048, "decoder_int_ff": 512, "n_head": 1, "drop_rate": 0.034481700426631456, "lr": 0.0010186128360476897, "batch_size": 2048}, "time_since_restore": 116.74665451049805, "iterations_since_restore": 2}
+{"loss": 105.2013188159372, "timestamp": 1719355120, "checkpoint_dir_name": "checkpoint_000007", "should_checkpoint": true, "done": false, "training_iteration": 8, "trial_id": "3ab2f301", "date": "2024-06-26_00-38-40", "time_this_iter_s": 51.496910572052, "time_total_s": 444.5570077896118, "pid": 2850562, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 2, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 2048, "decoder_rt_ff": 2048, "decoder_int_ff": 512, "n_head": 1, "drop_rate": 0.034481700426631456, "lr": 0.0010186128360476897, "batch_size": 2048}, "time_since_restore": 168.24356508255005, "iterations_since_restore": 3}
+{"loss": 118.58170366662694, "timestamp": 1719355171, "checkpoint_dir_name": "checkpoint_000008", "should_checkpoint": true, "done": false, "training_iteration": 9, "trial_id": "3ab2f301", "date": "2024-06-26_00-39-31", "time_this_iter_s": 51.06172227859497, "time_total_s": 495.6187300682068, "pid": 2850562, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 2, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 2048, "decoder_rt_ff": 2048, "decoder_int_ff": 512, "n_head": 1, "drop_rate": 0.034481700426631456, "lr": 0.0010186128360476897, "batch_size": 2048}, "time_since_restore": 219.30528736114502, "iterations_since_restore": 4}
+{"loss": 99.85706461508443, "timestamp": 1719355223, "checkpoint_dir_name": "checkpoint_000009", "should_checkpoint": true, "done": false, "training_iteration": 10, "trial_id": "3ab2f301", "date": "2024-06-26_00-40-23", "time_this_iter_s": 51.387521266937256, "time_total_s": 547.006251335144, "pid": 2850562, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 2, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 2048, "decoder_rt_ff": 2048, "decoder_int_ff": 512, "n_head": 1, "drop_rate": 0.034481700426631456, "lr": 0.0010186128360476897, "batch_size": 2048}, "time_since_restore": 270.6928086280823, "iterations_since_restore": 5}
+{"loss": 98.49699257677935, "timestamp": 1719357936, "checkpoint_dir_name": "checkpoint_000010", "should_checkpoint": true, "done": false, "training_iteration": 11, "trial_id": "3ab2f301", "date": "2024-06-26_01-25-36", "time_this_iter_s": 64.502512216568, "time_total_s": 611.508763551712, "pid": 2850562, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 2, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 2048, "decoder_rt_ff": 2048, "decoder_int_ff": 512, "n_head": 1, "drop_rate": 0.034481700426631456, "lr": 0.0010186128360476897, "batch_size": 2048}, "time_since_restore": 64.502512216568, "iterations_since_restore": 1}
+{"loss": 93.49606281190407, "timestamp": 1719357987, "checkpoint_dir_name": "checkpoint_000011", "should_checkpoint": true, "done": false, "training_iteration": 12, "trial_id": "3ab2f301", "date": "2024-06-26_01-26-27", "time_this_iter_s": 50.864012002944946, "time_total_s": 662.372775554657, "pid": 2850562, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 2, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 2048, "decoder_rt_ff": 2048, "decoder_int_ff": 512, "n_head": 1, "drop_rate": 0.034481700426631456, "lr": 0.0010186128360476897, "batch_size": 2048}, "time_since_restore": 115.36652421951294, "iterations_since_restore": 2}
+{"loss": 94.78943622018409, "timestamp": 1719358039, "checkpoint_dir_name": "checkpoint_000012", "should_checkpoint": true, "done": false, "training_iteration": 13, "trial_id": "3ab2f301", "date": "2024-06-26_01-27-19", "time_this_iter_s": 51.64423751831055, "time_total_s": 714.0170130729675, "pid": 2850562, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 2, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 2048, "decoder_rt_ff": 2048, "decoder_int_ff": 512, "n_head": 1, "drop_rate": 0.034481700426631456, "lr": 0.0010186128360476897, "batch_size": 2048}, "time_since_restore": 167.0107617378235, "iterations_since_restore": 3}
+{"loss": 91.46543307567205, "timestamp": 1719358090, "checkpoint_dir_name": "checkpoint_000013", "should_checkpoint": true, "done": false, "training_iteration": 14, "trial_id": "3ab2f301", "date": "2024-06-26_01-28-10", "time_this_iter_s": 51.174827098846436, "time_total_s": 765.191840171814, "pid": 2850562, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 2, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 2048, "decoder_rt_ff": 2048, "decoder_int_ff": 512, "n_head": 1, "drop_rate": 0.034481700426631456, "lr": 0.0010186128360476897, "batch_size": 2048}, "time_since_restore": 218.18558883666992, "iterations_since_restore": 4}
+{"loss": 88.73407943605439, "timestamp": 1719358141, "checkpoint_dir_name": "checkpoint_000014", "should_checkpoint": true, "done": false, "training_iteration": 15, "trial_id": "3ab2f301", "date": "2024-06-26_01-29-01", "time_this_iter_s": 50.784724712371826, "time_total_s": 815.9765648841858, "pid": 2850562, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 2, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 2048, "decoder_rt_ff": 2048, "decoder_int_ff": 512, "n_head": 1, "drop_rate": 0.034481700426631456, "lr": 0.0010186128360476897, "batch_size": 2048}, "time_since_restore": 268.97031354904175, "iterations_since_restore": 5}
+{"loss": 94.00536857064314, "timestamp": 1719358192, "checkpoint_dir_name": "checkpoint_000015", "should_checkpoint": true, "done": false, "training_iteration": 16, "trial_id": "3ab2f301", "date": "2024-06-26_01-29-52", "time_this_iter_s": 51.60023093223572, "time_total_s": 867.5767958164215, "pid": 2850562, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 2, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 2048, "decoder_rt_ff": 2048, "decoder_int_ff": 512, "n_head": 1, "drop_rate": 0.034481700426631456, "lr": 0.0010186128360476897, "batch_size": 2048}, "time_since_restore": 320.57054448127747, "iterations_since_restore": 6}
+{"loss": 87.47035950187623, "timestamp": 1719358244, "checkpoint_dir_name": "checkpoint_000016", "should_checkpoint": true, "done": false, "training_iteration": 17, "trial_id": "3ab2f301", "date": "2024-06-26_01-30-44", "time_this_iter_s": 51.84270000457764, "time_total_s": 919.4194958209991, "pid": 2850562, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 2, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 2048, "decoder_rt_ff": 2048, "decoder_int_ff": 512, "n_head": 1, "drop_rate": 0.034481700426631456, "lr": 0.0010186128360476897, "batch_size": 2048}, "time_since_restore": 372.4132444858551, "iterations_since_restore": 7}
+{"loss": 88.06384884090875, "timestamp": 1719358295, "checkpoint_dir_name": "checkpoint_000017", "should_checkpoint": true, "done": false, "training_iteration": 18, "trial_id": "3ab2f301", "date": "2024-06-26_01-31-35", "time_this_iter_s": 51.09972786903381, "time_total_s": 970.519223690033, "pid": 2850562, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 2, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 2048, "decoder_rt_ff": 2048, "decoder_int_ff": 512, "n_head": 1, "drop_rate": 0.034481700426631456, "lr": 0.0010186128360476897, "batch_size": 2048}, "time_since_restore": 423.5129723548889, "iterations_since_restore": 8}
+{"loss": 87.50748780017763, "timestamp": 1719358346, "checkpoint_dir_name": "checkpoint_000018", "should_checkpoint": true, "done": false, "training_iteration": 19, "trial_id": "3ab2f301", "date": "2024-06-26_01-32-26", "time_this_iter_s": 51.09087514877319, "time_total_s": 1021.6100988388062, "pid": 2850562, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 2, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 2048, "decoder_rt_ff": 2048, "decoder_int_ff": 512, "n_head": 1, "drop_rate": 0.034481700426631456, "lr": 0.0010186128360476897, "batch_size": 2048}, "time_since_restore": 474.6038475036621, "iterations_since_restore": 9}
+{"loss": 84.81797225832001, "timestamp": 1719358398, "checkpoint_dir_name": "checkpoint_000019", "should_checkpoint": true, "done": false, "training_iteration": 20, "trial_id": "3ab2f301", "date": "2024-06-26_01-33-18", "time_this_iter_s": 51.372178077697754, "time_total_s": 1072.982276916504, "pid": 2850562, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 2, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 2048, "decoder_rt_ff": 2048, "decoder_int_ff": 512, "n_head": 1, "drop_rate": 0.034481700426631456, "lr": 0.0010186128360476897, "batch_size": 2048}, "time_since_restore": 525.9760255813599, "iterations_since_restore": 10}
diff --git a/ray_results/test_experiment/train_model_3f37f45c_7_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=2,drop_r_2024-06-25_17-29-36/events.out.tfevents.1719329461.r8i6n2 b/ray_results/test_experiment/train_model_3f37f45c_7_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=2,drop_r_2024-06-25_17-29-36/events.out.tfevents.1719329461.r8i6n2
new file mode 100644
index 0000000000000000000000000000000000000000..c59d8753d6c3273090a67d066fdd18c09c512b16
Binary files /dev/null and b/ray_results/test_experiment/train_model_3f37f45c_7_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=2,drop_r_2024-06-25_17-29-36/events.out.tfevents.1719329461.r8i6n2 differ
diff --git a/ray_results/test_experiment/train_model_3f37f45c_7_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=2,drop_r_2024-06-25_17-29-36/params.json b/ray_results/test_experiment/train_model_3f37f45c_7_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=2,drop_r_2024-06-25_17-29-36/params.json
new file mode 100644
index 0000000000000000000000000000000000000000..9cba325d8b83587de89311fe38329581935a1d13
--- /dev/null
+++ b/ray_results/test_experiment/train_model_3f37f45c_7_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=2,drop_r_2024-06-25_17-29-36/params.json
@@ -0,0 +1,13 @@
+{
+ "batch_size": 1024,
+ "decoder_int_ff": 512,
+ "decoder_int_num_layer": 1,
+ "decoder_rt_ff": 2048,
+ "decoder_rt_num_layer": 2,
+ "drop_rate": 0.6691316467787182,
+ "embedding_dim": 256,
+ "encoder_ff": 512,
+ "encoder_num_layer": 1,
+ "lr": 0.004731694425031267,
+ "n_head": 4
+}
\ No newline at end of file
diff --git a/ray_results/test_experiment/train_model_3f37f45c_7_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=2,drop_r_2024-06-25_17-29-36/params.pkl b/ray_results/test_experiment/train_model_3f37f45c_7_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=2,drop_r_2024-06-25_17-29-36/params.pkl
new file mode 100644
index 0000000000000000000000000000000000000000..3d20730fcb1235a43c5ba169b8895406c66a4a50
Binary files /dev/null and b/ray_results/test_experiment/train_model_3f37f45c_7_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=2,drop_r_2024-06-25_17-29-36/params.pkl differ
diff --git a/ray_results/test_experiment/train_model_3f37f45c_7_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=2,drop_r_2024-06-25_17-29-36/progress.csv b/ray_results/test_experiment/train_model_3f37f45c_7_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=2,drop_r_2024-06-25_17-29-36/progress.csv
new file mode 100644
index 0000000000000000000000000000000000000000..3eef0014fc61fc03687c1f2965647f106a8df246
--- /dev/null
+++ b/ray_results/test_experiment/train_model_3f37f45c_7_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=2,drop_r_2024-06-25_17-29-36/progress.csv
@@ -0,0 +1,2 @@
+loss,timestamp,checkpoint_dir_name,should_checkpoint,done,training_iteration,trial_id,date,time_this_iter_s,time_total_s,pid,hostname,node_ip,time_since_restore,iterations_since_restore
+2052.4382574126475,1719329544,checkpoint_000000,True,False,1,3f37f45c,2024-06-25_17-32-25,83.68853616714478,83.68853616714478,2195253,r8i6n2,10.159.28.60,83.68853616714478,1
diff --git a/ray_results/test_experiment/train_model_3f37f45c_7_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=2,drop_r_2024-06-25_17-29-36/result.json b/ray_results/test_experiment/train_model_3f37f45c_7_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=2,drop_r_2024-06-25_17-29-36/result.json
new file mode 100644
index 0000000000000000000000000000000000000000..e888aa04ce0325c88489aafc063f634ba211d370
--- /dev/null
+++ b/ray_results/test_experiment/train_model_3f37f45c_7_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=2,drop_r_2024-06-25_17-29-36/result.json
@@ -0,0 +1 @@
+{"loss": 2052.4382574126475, "timestamp": 1719329544, "checkpoint_dir_name": "checkpoint_000000", "should_checkpoint": true, "done": false, "training_iteration": 1, "trial_id": "3f37f45c", "date": "2024-06-25_17-32-25", "time_this_iter_s": 83.68853616714478, "time_total_s": 83.68853616714478, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 1, "decoder_rt_num_layer": 2, "decoder_int_num_layer": 1, "embedding_dim": 256, "encoder_ff": 512, "decoder_rt_ff": 2048, "decoder_int_ff": 512, "n_head": 4, "drop_rate": 0.6691316467787182, "lr": 0.004731694425031267, "batch_size": 1024}, "time_since_restore": 83.68853616714478, "iterations_since_restore": 1}
diff --git a/ray_results/test_experiment/train_model_40101b9a_29_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=1,drop_r_2024-06-25_22-15-47/events.out.tfevents.1719346823.r8i6n2 b/ray_results/test_experiment/train_model_40101b9a_29_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=1,drop_r_2024-06-25_22-15-47/events.out.tfevents.1719346823.r8i6n2
new file mode 100644
index 0000000000000000000000000000000000000000..910e4dab8aa5c8154370eb77fea90f04beacf996
Binary files /dev/null and b/ray_results/test_experiment/train_model_40101b9a_29_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=1,drop_r_2024-06-25_22-15-47/events.out.tfevents.1719346823.r8i6n2 differ
diff --git a/ray_results/test_experiment/train_model_40101b9a_29_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=1,drop_r_2024-06-25_22-15-47/params.json b/ray_results/test_experiment/train_model_40101b9a_29_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=1,drop_r_2024-06-25_22-15-47/params.json
new file mode 100644
index 0000000000000000000000000000000000000000..0027cad7cdd85fe48b5005aa2f12d28816fcadd5
--- /dev/null
+++ b/ray_results/test_experiment/train_model_40101b9a_29_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=1,drop_r_2024-06-25_22-15-47/params.json
@@ -0,0 +1,13 @@
+{
+ "batch_size": 2048,
+ "decoder_int_ff": 512,
+ "decoder_int_num_layer": 1,
+ "decoder_rt_ff": 512,
+ "decoder_rt_num_layer": 1,
+ "drop_rate": 0.8983976265900149,
+ "embedding_dim": 1024,
+ "encoder_ff": 2048,
+ "encoder_num_layer": 2,
+ "lr": 0.039339947723305395,
+ "n_head": 1
+}
\ No newline at end of file
diff --git a/ray_results/test_experiment/train_model_40101b9a_29_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=1,drop_r_2024-06-25_22-15-47/params.pkl b/ray_results/test_experiment/train_model_40101b9a_29_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=1,drop_r_2024-06-25_22-15-47/params.pkl
new file mode 100644
index 0000000000000000000000000000000000000000..3c52c6e8e2388ddf5b799b58a8d80fcf752792dd
Binary files /dev/null and b/ray_results/test_experiment/train_model_40101b9a_29_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=1,drop_r_2024-06-25_22-15-47/params.pkl differ
diff --git a/ray_results/test_experiment/train_model_40101b9a_29_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=1,drop_r_2024-06-25_22-15-47/progress.csv b/ray_results/test_experiment/train_model_40101b9a_29_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=1,drop_r_2024-06-25_22-15-47/progress.csv
new file mode 100644
index 0000000000000000000000000000000000000000..2534081b8c975cb7f1c52f5b87025695ca84649e
--- /dev/null
+++ b/ray_results/test_experiment/train_model_40101b9a_29_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=1,drop_r_2024-06-25_22-15-47/progress.csv
@@ -0,0 +1,6 @@
+loss,timestamp,checkpoint_dir_name,should_checkpoint,done,training_iteration,trial_id,date,time_this_iter_s,time_total_s,pid,hostname,node_ip,time_since_restore,iterations_since_restore
+4106.712271622785,1719347137,checkpoint_000000,True,False,1,40101b9a,2024-06-25_22-25-37,313.700572013855,313.700572013855,2195253,r8i6n2,10.159.28.60,313.700572013855,1
+3130.0891709215057,1719347437,checkpoint_000001,True,False,2,40101b9a,2024-06-25_22-30-38,300.49900484085083,614.1995768547058,2195253,r8i6n2,10.159.28.60,614.1995768547058,2
+2512.6716385488435,1719347737,checkpoint_000002,True,False,3,40101b9a,2024-06-25_22-35-38,300.07938170433044,914.2789585590363,2195253,r8i6n2,10.159.28.60,914.2789585590363,3
+2181.0040408156988,1719348038,checkpoint_000003,True,False,4,40101b9a,2024-06-25_22-40-38,300.6292908191681,1214.9082493782043,2195253,r8i6n2,10.159.28.60,1214.9082493782043,4
+2031.5720868448573,1719348338,checkpoint_000004,True,False,5,40101b9a,2024-06-25_22-45-38,300.0205228328705,1514.9287722110748,2195253,r8i6n2,10.159.28.60,1514.9287722110748,5
diff --git a/ray_results/test_experiment/train_model_40101b9a_29_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=1,drop_r_2024-06-25_22-15-47/result.json b/ray_results/test_experiment/train_model_40101b9a_29_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=1,drop_r_2024-06-25_22-15-47/result.json
new file mode 100644
index 0000000000000000000000000000000000000000..cb94596134c436371b18116e323bdac2213d07ba
--- /dev/null
+++ b/ray_results/test_experiment/train_model_40101b9a_29_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=1,drop_r_2024-06-25_22-15-47/result.json
@@ -0,0 +1,5 @@
+{"loss": 4106.712271622785, "timestamp": 1719347137, "checkpoint_dir_name": "checkpoint_000000", "should_checkpoint": true, "done": false, "training_iteration": 1, "trial_id": "40101b9a", "date": "2024-06-25_22-25-37", "time_this_iter_s": 313.700572013855, "time_total_s": 313.700572013855, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 2, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 1024, "encoder_ff": 2048, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 1, "drop_rate": 0.8983976265900149, "lr": 0.039339947723305395, "batch_size": 2048}, "time_since_restore": 313.700572013855, "iterations_since_restore": 1}
+{"loss": 3130.0891709215057, "timestamp": 1719347437, "checkpoint_dir_name": "checkpoint_000001", "should_checkpoint": true, "done": false, "training_iteration": 2, "trial_id": "40101b9a", "date": "2024-06-25_22-30-38", "time_this_iter_s": 300.49900484085083, "time_total_s": 614.1995768547058, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 2, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 1024, "encoder_ff": 2048, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 1, "drop_rate": 0.8983976265900149, "lr": 0.039339947723305395, "batch_size": 2048}, "time_since_restore": 614.1995768547058, "iterations_since_restore": 2}
+{"loss": 2512.6716385488435, "timestamp": 1719347737, "checkpoint_dir_name": "checkpoint_000002", "should_checkpoint": true, "done": false, "training_iteration": 3, "trial_id": "40101b9a", "date": "2024-06-25_22-35-38", "time_this_iter_s": 300.07938170433044, "time_total_s": 914.2789585590363, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 2, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 1024, "encoder_ff": 2048, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 1, "drop_rate": 0.8983976265900149, "lr": 0.039339947723305395, "batch_size": 2048}, "time_since_restore": 914.2789585590363, "iterations_since_restore": 3}
+{"loss": 2181.0040408156988, "timestamp": 1719348038, "checkpoint_dir_name": "checkpoint_000003", "should_checkpoint": true, "done": false, "training_iteration": 4, "trial_id": "40101b9a", "date": "2024-06-25_22-40-38", "time_this_iter_s": 300.6292908191681, "time_total_s": 1214.9082493782043, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 2, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 1024, "encoder_ff": 2048, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 1, "drop_rate": 0.8983976265900149, "lr": 0.039339947723305395, "batch_size": 2048}, "time_since_restore": 1214.9082493782043, "iterations_since_restore": 4}
+{"loss": 2031.5720868448573, "timestamp": 1719348338, "checkpoint_dir_name": "checkpoint_000004", "should_checkpoint": true, "done": false, "training_iteration": 5, "trial_id": "40101b9a", "date": "2024-06-25_22-45-38", "time_this_iter_s": 300.0205228328705, "time_total_s": 1514.9287722110748, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 2, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 1024, "encoder_ff": 2048, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 1, "drop_rate": 0.8983976265900149, "lr": 0.039339947723305395, "batch_size": 2048}, "time_since_restore": 1514.9287722110748, "iterations_since_restore": 5}
diff --git a/ray_results/test_experiment/train_model_43bba37c_19_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_18-28-52/events.out.tfevents.1719333829.r8i6n2 b/ray_results/test_experiment/train_model_43bba37c_19_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_18-28-52/events.out.tfevents.1719333829.r8i6n2
new file mode 100644
index 0000000000000000000000000000000000000000..de4459e3351b53546d3fd1186a9b0a9a936caeae
Binary files /dev/null and b/ray_results/test_experiment/train_model_43bba37c_19_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_18-28-52/events.out.tfevents.1719333829.r8i6n2 differ
diff --git a/ray_results/test_experiment/train_model_43bba37c_19_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_18-28-52/events.out.tfevents.1719338189.r8i6n2 b/ray_results/test_experiment/train_model_43bba37c_19_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_18-28-52/events.out.tfevents.1719338189.r8i6n2
new file mode 100644
index 0000000000000000000000000000000000000000..d71c6d523313f49eca5605e6b3e008b1e341c65c
Binary files /dev/null and b/ray_results/test_experiment/train_model_43bba37c_19_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_18-28-52/events.out.tfevents.1719338189.r8i6n2 differ
diff --git a/ray_results/test_experiment/train_model_43bba37c_19_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_18-28-52/events.out.tfevents.1719340067.r8i6n2 b/ray_results/test_experiment/train_model_43bba37c_19_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_18-28-52/events.out.tfevents.1719340067.r8i6n2
new file mode 100644
index 0000000000000000000000000000000000000000..cbac79e5035f208089df4716762053af9d5a158e
Binary files /dev/null and b/ray_results/test_experiment/train_model_43bba37c_19_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_18-28-52/events.out.tfevents.1719340067.r8i6n2 differ
diff --git a/ray_results/test_experiment/train_model_43bba37c_19_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_18-28-52/params.json b/ray_results/test_experiment/train_model_43bba37c_19_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_18-28-52/params.json
new file mode 100644
index 0000000000000000000000000000000000000000..950914f36fd2a2f73d4ae77d50ef9031a6eda320
--- /dev/null
+++ b/ray_results/test_experiment/train_model_43bba37c_19_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_18-28-52/params.json
@@ -0,0 +1,13 @@
+{
+ "batch_size": 1024,
+ "decoder_int_ff": 512,
+ "decoder_int_num_layer": 1,
+ "decoder_rt_ff": 1024,
+ "decoder_rt_num_layer": 1,
+ "drop_rate": 0.4036073673456261,
+ "embedding_dim": 256,
+ "encoder_ff": 1024,
+ "encoder_num_layer": 8,
+ "lr": 0.0007661776725543005,
+ "n_head": 1
+}
\ No newline at end of file
diff --git a/ray_results/test_experiment/train_model_43bba37c_19_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_18-28-52/params.pkl b/ray_results/test_experiment/train_model_43bba37c_19_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_18-28-52/params.pkl
new file mode 100644
index 0000000000000000000000000000000000000000..841f248f50619e300739577d668357c35b3abf6d
Binary files /dev/null and b/ray_results/test_experiment/train_model_43bba37c_19_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_18-28-52/params.pkl differ
diff --git a/ray_results/test_experiment/train_model_43bba37c_19_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_18-28-52/progress.csv b/ray_results/test_experiment/train_model_43bba37c_19_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_18-28-52/progress.csv
new file mode 100644
index 0000000000000000000000000000000000000000..ac45833cd19f28ffdb899d36fb285b16e597e1a7
--- /dev/null
+++ b/ray_results/test_experiment/train_model_43bba37c_19_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_18-28-52/progress.csv
@@ -0,0 +1,9 @@
+loss,timestamp,checkpoint_dir_name,should_checkpoint,done,training_iteration,trial_id,date,time_this_iter_s,time_total_s,pid,hostname,node_ip,time_since_restore,iterations_since_restore
+1961.9788803941622,1719333991,checkpoint_000000,True,False,1,43bba37c,2024-06-25_18-46-31,161.8538646697998,161.8538646697998,2195253,r8i6n2,10.159.28.60,161.8538646697998,1
+1966.8054059847134,1719334139,checkpoint_000001,True,False,2,43bba37c,2024-06-25_18-48-59,147.8709270954132,309.724791765213,2195253,r8i6n2,10.159.28.60,309.724791765213,2
+1967.6592320720042,1719338350,checkpoint_000002,True,False,3,43bba37c,2024-06-25_19-59-10,161.5647487640381,471.2895405292511,2195253,r8i6n2,10.159.28.60,161.5647487640381,1
+1972.7257733682948,1719338498,checkpoint_000003,True,False,4,43bba37c,2024-06-25_20-01-38,147.5350902080536,618.8246307373047,2195253,r8i6n2,10.159.28.60,309.0998389720917,2
+1965.584105484129,1719340228,checkpoint_000004,True,False,5,43bba37c,2024-06-25_20-30-28,160.8864643573761,779.7110950946808,2195253,r8i6n2,10.159.28.60,160.8864643573761,1
+1965.0863916592334,1719340375,checkpoint_000005,True,False,6,43bba37c,2024-06-25_20-32-55,147.0123507976532,926.723445892334,2195253,r8i6n2,10.159.28.60,307.8988151550293,2
+1965.4208993986836,1719340522,checkpoint_000006,True,False,7,43bba37c,2024-06-25_20-35-23,147.50104451179504,1074.224490404129,2195253,r8i6n2,10.159.28.60,455.39985966682434,3
+1962.8120477631337,1719340670,checkpoint_000007,True,False,8,43bba37c,2024-06-25_20-37-50,147.67359066009521,1221.8980810642242,2195253,r8i6n2,10.159.28.60,603.0734503269196,4
diff --git a/ray_results/test_experiment/train_model_43bba37c_19_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_18-28-52/result.json b/ray_results/test_experiment/train_model_43bba37c_19_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_18-28-52/result.json
new file mode 100644
index 0000000000000000000000000000000000000000..8a62215f061891f789eb444a7e4ce30af80e0541
--- /dev/null
+++ b/ray_results/test_experiment/train_model_43bba37c_19_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_18-28-52/result.json
@@ -0,0 +1,8 @@
+{"loss": 1961.9788803941622, "timestamp": 1719333991, "checkpoint_dir_name": "checkpoint_000000", "should_checkpoint": true, "done": false, "training_iteration": 1, "trial_id": "43bba37c", "date": "2024-06-25_18-46-31", "time_this_iter_s": 161.8538646697998, "time_total_s": 161.8538646697998, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 8, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 256, "encoder_ff": 1024, "decoder_rt_ff": 1024, "decoder_int_ff": 512, "n_head": 1, "drop_rate": 0.4036073673456261, "lr": 0.0007661776725543005, "batch_size": 1024}, "time_since_restore": 161.8538646697998, "iterations_since_restore": 1}
+{"loss": 1966.8054059847134, "timestamp": 1719334139, "checkpoint_dir_name": "checkpoint_000001", "should_checkpoint": true, "done": false, "training_iteration": 2, "trial_id": "43bba37c", "date": "2024-06-25_18-48-59", "time_this_iter_s": 147.8709270954132, "time_total_s": 309.724791765213, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 8, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 256, "encoder_ff": 1024, "decoder_rt_ff": 1024, "decoder_int_ff": 512, "n_head": 1, "drop_rate": 0.4036073673456261, "lr": 0.0007661776725543005, "batch_size": 1024}, "time_since_restore": 309.724791765213, "iterations_since_restore": 2}
+{"loss": 1967.6592320720042, "timestamp": 1719338350, "checkpoint_dir_name": "checkpoint_000002", "should_checkpoint": true, "done": false, "training_iteration": 3, "trial_id": "43bba37c", "date": "2024-06-25_19-59-10", "time_this_iter_s": 161.5647487640381, "time_total_s": 471.2895405292511, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 8, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 256, "encoder_ff": 1024, "decoder_rt_ff": 1024, "decoder_int_ff": 512, "n_head": 1, "drop_rate": 0.4036073673456261, "lr": 0.0007661776725543005, "batch_size": 1024}, "time_since_restore": 161.5647487640381, "iterations_since_restore": 1}
+{"loss": 1972.7257733682948, "timestamp": 1719338498, "checkpoint_dir_name": "checkpoint_000003", "should_checkpoint": true, "done": false, "training_iteration": 4, "trial_id": "43bba37c", "date": "2024-06-25_20-01-38", "time_this_iter_s": 147.5350902080536, "time_total_s": 618.8246307373047, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 8, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 256, "encoder_ff": 1024, "decoder_rt_ff": 1024, "decoder_int_ff": 512, "n_head": 1, "drop_rate": 0.4036073673456261, "lr": 0.0007661776725543005, "batch_size": 1024}, "time_since_restore": 309.0998389720917, "iterations_since_restore": 2}
+{"loss": 1965.584105484129, "timestamp": 1719340228, "checkpoint_dir_name": "checkpoint_000004", "should_checkpoint": true, "done": false, "training_iteration": 5, "trial_id": "43bba37c", "date": "2024-06-25_20-30-28", "time_this_iter_s": 160.8864643573761, "time_total_s": 779.7110950946808, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 8, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 256, "encoder_ff": 1024, "decoder_rt_ff": 1024, "decoder_int_ff": 512, "n_head": 1, "drop_rate": 0.4036073673456261, "lr": 0.0007661776725543005, "batch_size": 1024}, "time_since_restore": 160.8864643573761, "iterations_since_restore": 1}
+{"loss": 1965.0863916592334, "timestamp": 1719340375, "checkpoint_dir_name": "checkpoint_000005", "should_checkpoint": true, "done": false, "training_iteration": 6, "trial_id": "43bba37c", "date": "2024-06-25_20-32-55", "time_this_iter_s": 147.0123507976532, "time_total_s": 926.723445892334, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 8, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 256, "encoder_ff": 1024, "decoder_rt_ff": 1024, "decoder_int_ff": 512, "n_head": 1, "drop_rate": 0.4036073673456261, "lr": 0.0007661776725543005, "batch_size": 1024}, "time_since_restore": 307.8988151550293, "iterations_since_restore": 2}
+{"loss": 1965.4208993986836, "timestamp": 1719340522, "checkpoint_dir_name": "checkpoint_000006", "should_checkpoint": true, "done": false, "training_iteration": 7, "trial_id": "43bba37c", "date": "2024-06-25_20-35-23", "time_this_iter_s": 147.50104451179504, "time_total_s": 1074.224490404129, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 8, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 256, "encoder_ff": 1024, "decoder_rt_ff": 1024, "decoder_int_ff": 512, "n_head": 1, "drop_rate": 0.4036073673456261, "lr": 0.0007661776725543005, "batch_size": 1024}, "time_since_restore": 455.39985966682434, "iterations_since_restore": 3}
+{"loss": 1962.8120477631337, "timestamp": 1719340670, "checkpoint_dir_name": "checkpoint_000007", "should_checkpoint": true, "done": false, "training_iteration": 8, "trial_id": "43bba37c", "date": "2024-06-25_20-37-50", "time_this_iter_s": 147.67359066009521, "time_total_s": 1221.8980810642242, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 8, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 256, "encoder_ff": 1024, "decoder_rt_ff": 1024, "decoder_int_ff": 512, "n_head": 1, "drop_rate": 0.4036073673456261, "lr": 0.0007661776725543005, "batch_size": 1024}, "time_since_restore": 603.0734503269196, "iterations_since_restore": 4}
diff --git a/ray_results/test_experiment/train_model_45b8cfdd_17_batch_size=2048,decoder_int_ff=2048,decoder_int_num_layer=2,decoder_rt_ff=1024,decoder_rt_num_layer=4,drop_2024-06-25_16-46-32/error.pkl b/ray_results/test_experiment/train_model_45b8cfdd_17_batch_size=2048,decoder_int_ff=2048,decoder_int_num_layer=2,decoder_rt_ff=1024,decoder_rt_num_layer=4,drop_2024-06-25_16-46-32/error.pkl
new file mode 100644
index 0000000000000000000000000000000000000000..16db52c26d343e291b1302cac14cba752cf0ff4b
Binary files /dev/null and b/ray_results/test_experiment/train_model_45b8cfdd_17_batch_size=2048,decoder_int_ff=2048,decoder_int_num_layer=2,decoder_rt_ff=1024,decoder_rt_num_layer=4,drop_2024-06-25_16-46-32/error.pkl differ
diff --git a/ray_results/test_experiment/train_model_45b8cfdd_17_batch_size=2048,decoder_int_ff=2048,decoder_int_num_layer=2,decoder_rt_ff=1024,decoder_rt_num_layer=4,drop_2024-06-25_16-46-32/error.txt b/ray_results/test_experiment/train_model_45b8cfdd_17_batch_size=2048,decoder_int_ff=2048,decoder_int_num_layer=2,decoder_rt_ff=1024,decoder_rt_num_layer=4,drop_2024-06-25_16-46-32/error.txt
new file mode 100644
index 0000000000000000000000000000000000000000..3129628b81d0e235d5f8f24ee728ab64780db02f
--- /dev/null
+++ b/ray_results/test_experiment/train_model_45b8cfdd_17_batch_size=2048,decoder_int_ff=2048,decoder_int_num_layer=2,decoder_rt_ff=1024,decoder_rt_num_layer=4,drop_2024-06-25_16-46-32/error.txt
@@ -0,0 +1,68 @@
+Failure # 1 (occurred at 2024-06-25_16-47-55)
+[36mray::ImplicitFunc.train()[39m (pid=122466, ip=10.159.8.159, actor_id=3b1fd592afc06be8c677c69e01000000, repr=train_model)
+ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+ File "/gpfslocalsup/pub/anaconda-py3/2023.09/envs/pytorch-gpu-2.2.0+py3.11.7/lib/python3.11/site-packages/ray/tune/trainable/trainable.py", line 342, in train
+ raise skipped from exception_cause(skipped)
+ File "/gpfslocalsup/pub/anaconda-py3/2023.09/envs/pytorch-gpu-2.2.0+py3.11.7/lib/python3.11/site-packages/ray/air/_internal/util.py", line 88, in run
+ self._ret = self._target(*self._args, **self._kwargs)
+ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+ File "/gpfslocalsup/pub/anaconda-py3/2023.09/envs/pytorch-gpu-2.2.0+py3.11.7/lib/python3.11/site-packages/ray/tune/trainable/function_trainable.py", line 249, in <lambda>
+ training_func=lambda: self._trainable_func(self.config),
+ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+ File "/gpfslocalsup/pub/anaconda-py3/2023.09/envs/pytorch-gpu-2.2.0+py3.11.7/lib/python3.11/site-packages/ray/tune/trainable/function_trainable.py", line 332, in _trainable_func
+ output = fn()
+ ^^^^
+ File "/gpfslocalsup/pub/anaconda-py3/2023.09/envs/pytorch-gpu-2.2.0+py3.11.7/lib/python3.11/site-packages/ray/tune/trainable/util.py", line 138, in inner
+ return trainable(config, **fn_kwargs)
+ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+ File "/gpfswork/rech/ute/ucg81ws/these/LC-MS-RT-prediction/main_ray_tune.py", line 66, in train_model
+ pred_rt = net.module.forward_rt(seq)
+ ^^^^^^^^^^^^^^^^^^^^^^^^^^
+ File "/gpfsdswork/projects/rech/ute/ucg81ws/these/LC-MS-RT-prediction/model_custom.py", line 111, in forward_rt
+ out_rt = self.decoder_RT(enc)
+ ^^^^^^^^^^^^^^^^^^^^
+ File "/gpfslocalsup/pub/anaconda-py3/2023.09/envs/pytorch-gpu-2.2.0+py3.11.7/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1511, in _wrapped_call_impl
+ return self._call_impl(*args, **kwargs)
+ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+ File "/gpfslocalsup/pub/anaconda-py3/2023.09/envs/pytorch-gpu-2.2.0+py3.11.7/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1520, in _call_impl
+ return forward_call(*args, **kwargs)
+ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+ File "/gpfslocalsup/pub/anaconda-py3/2023.09/envs/pytorch-gpu-2.2.0+py3.11.7/lib/python3.11/site-packages/torch/nn/modules/container.py", line 217, in forward
+ input = module(input)
+ ^^^^^^^^^^^^^
+ File "/gpfslocalsup/pub/anaconda-py3/2023.09/envs/pytorch-gpu-2.2.0+py3.11.7/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1511, in _wrapped_call_impl
+ return self._call_impl(*args, **kwargs)
+ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+ File "/gpfslocalsup/pub/anaconda-py3/2023.09/envs/pytorch-gpu-2.2.0+py3.11.7/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1520, in _call_impl
+ return forward_call(*args, **kwargs)
+ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+ File "/gpfslocalsup/pub/anaconda-py3/2023.09/envs/pytorch-gpu-2.2.0+py3.11.7/lib/python3.11/site-packages/torch/nn/modules/transformer.py", line 391, in forward
+ output = mod(output, src_mask=mask, is_causal=is_causal, src_key_padding_mask=src_key_padding_mask_for_layers)
+ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+ File "/gpfslocalsup/pub/anaconda-py3/2023.09/envs/pytorch-gpu-2.2.0+py3.11.7/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1511, in _wrapped_call_impl
+ return self._call_impl(*args, **kwargs)
+ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+ File "/gpfslocalsup/pub/anaconda-py3/2023.09/envs/pytorch-gpu-2.2.0+py3.11.7/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1520, in _call_impl
+ return forward_call(*args, **kwargs)
+ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+ File "/gpfslocalsup/pub/anaconda-py3/2023.09/envs/pytorch-gpu-2.2.0+py3.11.7/lib/python3.11/site-packages/torch/nn/modules/transformer.py", line 715, in forward
+ x = self.norm2(x + self._ff_block(x))
+ ^^^^^^^^^^^^^^^^^
+ File "/gpfslocalsup/pub/anaconda-py3/2023.09/envs/pytorch-gpu-2.2.0+py3.11.7/lib/python3.11/site-packages/torch/nn/modules/transformer.py", line 731, in _ff_block
+ return self.dropout2(x)
+ ^^^^^^^^^^^^^^^^
+ File "/gpfslocalsup/pub/anaconda-py3/2023.09/envs/pytorch-gpu-2.2.0+py3.11.7/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1511, in _wrapped_call_impl
+ return self._call_impl(*args, **kwargs)
+ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+ File "/gpfslocalsup/pub/anaconda-py3/2023.09/envs/pytorch-gpu-2.2.0+py3.11.7/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1520, in _call_impl
+ return forward_call(*args, **kwargs)
+ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+ File "/gpfslocalsup/pub/anaconda-py3/2023.09/envs/pytorch-gpu-2.2.0+py3.11.7/lib/python3.11/site-packages/torch/nn/modules/dropout.py", line 59, in forward
+ return F.dropout(input, self.p, self.training, self.inplace)
+ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+ File "/gpfslocalsup/pub/anaconda-py3/2023.09/envs/pytorch-gpu-2.2.0+py3.11.7/lib/python3.11/site-packages/torch/nn/functional.py", line 1268, in dropout
+ return _VF.dropout_(input, p, training) if inplace else _VF.dropout(input, p, training)
+ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+torch.cuda.OutOfMemoryError: CUDA out of memory. Tried to allocate 200.00 MiB. GPU 0 has a total capacity of 15.77 GiB of which 68.69 MiB is free. Including non-PyTorch memory, this process has 15.70 GiB memory in use. Of the allocated memory 15.24 GiB is allocated by PyTorch, and 83.18 MiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables)
diff --git a/ray_results/test_experiment/train_model_45b8cfdd_17_batch_size=2048,decoder_int_ff=2048,decoder_int_num_layer=2,decoder_rt_ff=1024,decoder_rt_num_layer=4,drop_2024-06-25_16-46-32/events.out.tfevents.1719326860.r3i5n6 b/ray_results/test_experiment/train_model_45b8cfdd_17_batch_size=2048,decoder_int_ff=2048,decoder_int_num_layer=2,decoder_rt_ff=1024,decoder_rt_num_layer=4,drop_2024-06-25_16-46-32/events.out.tfevents.1719326860.r3i5n6
new file mode 100644
index 0000000000000000000000000000000000000000..f667b75df449fba850966736612c1eb6f560a35b
Binary files /dev/null and b/ray_results/test_experiment/train_model_45b8cfdd_17_batch_size=2048,decoder_int_ff=2048,decoder_int_num_layer=2,decoder_rt_ff=1024,decoder_rt_num_layer=4,drop_2024-06-25_16-46-32/events.out.tfevents.1719326860.r3i5n6 differ
diff --git a/ray_results/test_experiment/train_model_45b8cfdd_17_batch_size=2048,decoder_int_ff=2048,decoder_int_num_layer=2,decoder_rt_ff=1024,decoder_rt_num_layer=4,drop_2024-06-25_16-46-32/params.json b/ray_results/test_experiment/train_model_45b8cfdd_17_batch_size=2048,decoder_int_ff=2048,decoder_int_num_layer=2,decoder_rt_ff=1024,decoder_rt_num_layer=4,drop_2024-06-25_16-46-32/params.json
new file mode 100644
index 0000000000000000000000000000000000000000..dab584b9bc0498d2bc172260c8b423a2673ef432
--- /dev/null
+++ b/ray_results/test_experiment/train_model_45b8cfdd_17_batch_size=2048,decoder_int_ff=2048,decoder_int_num_layer=2,decoder_rt_ff=1024,decoder_rt_num_layer=4,drop_2024-06-25_16-46-32/params.json
@@ -0,0 +1,13 @@
+{
+ "batch_size": 2048,
+ "decoder_int_ff": 2048,
+ "decoder_int_num_layer": 2,
+ "decoder_rt_ff": 1024,
+ "decoder_rt_num_layer": 4,
+ "drop_rate": 0.18931069188451255,
+ "embedding_dim": 1024,
+ "encoder_ff": 2048,
+ "encoder_num_layer": 2,
+ "lr": 0.0002168775970949337,
+ "n_head": 4
+}
\ No newline at end of file
diff --git a/ray_results/test_experiment/train_model_45b8cfdd_17_batch_size=2048,decoder_int_ff=2048,decoder_int_num_layer=2,decoder_rt_ff=1024,decoder_rt_num_layer=4,drop_2024-06-25_16-46-32/params.pkl b/ray_results/test_experiment/train_model_45b8cfdd_17_batch_size=2048,decoder_int_ff=2048,decoder_int_num_layer=2,decoder_rt_ff=1024,decoder_rt_num_layer=4,drop_2024-06-25_16-46-32/params.pkl
new file mode 100644
index 0000000000000000000000000000000000000000..9bbaaa64ed5a70faf93a5c97e5680d371d33f75c
Binary files /dev/null and b/ray_results/test_experiment/train_model_45b8cfdd_17_batch_size=2048,decoder_int_ff=2048,decoder_int_num_layer=2,decoder_rt_ff=1024,decoder_rt_num_layer=4,drop_2024-06-25_16-46-32/params.pkl differ
diff --git a/ray_results/test_experiment/train_model_45b8cfdd_17_batch_size=2048,decoder_int_ff=2048,decoder_int_num_layer=2,decoder_rt_ff=1024,decoder_rt_num_layer=4,drop_2024-06-25_16-46-32/result.json b/ray_results/test_experiment/train_model_45b8cfdd_17_batch_size=2048,decoder_int_ff=2048,decoder_int_num_layer=2,decoder_rt_ff=1024,decoder_rt_num_layer=4,drop_2024-06-25_16-46-32/result.json
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/ray_results/test_experiment/train_model_481f1fff_10_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=8,drop__2024-06-25_17-33-30/events.out.tfevents.1719329693.r8i6n2 b/ray_results/test_experiment/train_model_481f1fff_10_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=8,drop__2024-06-25_17-33-30/events.out.tfevents.1719329693.r8i6n2
new file mode 100644
index 0000000000000000000000000000000000000000..0020c2e157c4fe99b9dd259b94491ab4dc45a174
Binary files /dev/null and b/ray_results/test_experiment/train_model_481f1fff_10_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=8,drop__2024-06-25_17-33-30/events.out.tfevents.1719329693.r8i6n2 differ
diff --git a/ray_results/test_experiment/train_model_481f1fff_10_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=8,drop__2024-06-25_17-33-30/events.out.tfevents.1719330754.r8i6n2 b/ray_results/test_experiment/train_model_481f1fff_10_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=8,drop__2024-06-25_17-33-30/events.out.tfevents.1719330754.r8i6n2
new file mode 100644
index 0000000000000000000000000000000000000000..ead04b117dd3c7918b653da129dc0043476292d2
Binary files /dev/null and b/ray_results/test_experiment/train_model_481f1fff_10_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=8,drop__2024-06-25_17-33-30/events.out.tfevents.1719330754.r8i6n2 differ
diff --git a/ray_results/test_experiment/train_model_481f1fff_10_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=8,drop__2024-06-25_17-33-30/params.json b/ray_results/test_experiment/train_model_481f1fff_10_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=8,drop__2024-06-25_17-33-30/params.json
new file mode 100644
index 0000000000000000000000000000000000000000..4e01c9e388fa2eaf57720ae2b27166f33f2ad5f8
--- /dev/null
+++ b/ray_results/test_experiment/train_model_481f1fff_10_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=8,drop__2024-06-25_17-33-30/params.json
@@ -0,0 +1,13 @@
+{
+ "batch_size": 1024,
+ "decoder_int_ff": 512,
+ "decoder_int_num_layer": 1,
+ "decoder_rt_ff": 1024,
+ "decoder_rt_num_layer": 8,
+ "drop_rate": 0.15995138350968297,
+ "embedding_dim": 64,
+ "encoder_ff": 1024,
+ "encoder_num_layer": 2,
+ "lr": 0.004656630423530019,
+ "n_head": 16
+}
\ No newline at end of file
diff --git a/ray_results/test_experiment/train_model_481f1fff_10_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=8,drop__2024-06-25_17-33-30/params.pkl b/ray_results/test_experiment/train_model_481f1fff_10_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=8,drop__2024-06-25_17-33-30/params.pkl
new file mode 100644
index 0000000000000000000000000000000000000000..079cac3087e864cbb4289e74c79f3ee3b44eee0c
Binary files /dev/null and b/ray_results/test_experiment/train_model_481f1fff_10_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=8,drop__2024-06-25_17-33-30/params.pkl differ
diff --git a/ray_results/test_experiment/train_model_481f1fff_10_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=8,drop__2024-06-25_17-33-30/progress.csv b/ray_results/test_experiment/train_model_481f1fff_10_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=8,drop__2024-06-25_17-33-30/progress.csv
new file mode 100644
index 0000000000000000000000000000000000000000..78438d975d13f945bbc054034f5933a2b8e850ed
--- /dev/null
+++ b/ray_results/test_experiment/train_model_481f1fff_10_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=8,drop__2024-06-25_17-33-30/progress.csv
@@ -0,0 +1,3 @@
+loss,timestamp,checkpoint_dir_name,should_checkpoint,done,training_iteration,trial_id,date,time_this_iter_s,time_total_s,pid,hostname,node_ip,time_since_restore,iterations_since_restore
+1961.035730076587,1719329807,checkpoint_000000,True,False,1,481f1fff,2024-06-25_17-36-47,113.85971927642822,113.85971927642822,2195253,r8i6n2,10.159.28.60,113.85971927642822,1
+1959.9800554410679,1719330869,checkpoint_000001,True,False,2,481f1fff,2024-06-25_17-54-29,114.88935971260071,228.74907898902893,2195253,r8i6n2,10.159.28.60,114.88935971260071,1
diff --git a/ray_results/test_experiment/train_model_481f1fff_10_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=8,drop__2024-06-25_17-33-30/result.json b/ray_results/test_experiment/train_model_481f1fff_10_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=8,drop__2024-06-25_17-33-30/result.json
new file mode 100644
index 0000000000000000000000000000000000000000..e6d1117d03a84c620b31ad73b3a706bbd512fdb0
--- /dev/null
+++ b/ray_results/test_experiment/train_model_481f1fff_10_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=8,drop__2024-06-25_17-33-30/result.json
@@ -0,0 +1,2 @@
+{"loss": 1961.035730076587, "timestamp": 1719329807, "checkpoint_dir_name": "checkpoint_000000", "should_checkpoint": true, "done": false, "training_iteration": 1, "trial_id": "481f1fff", "date": "2024-06-25_17-36-47", "time_this_iter_s": 113.85971927642822, "time_total_s": 113.85971927642822, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 2, "decoder_rt_num_layer": 8, "decoder_int_num_layer": 1, "embedding_dim": 64, "encoder_ff": 1024, "decoder_rt_ff": 1024, "decoder_int_ff": 512, "n_head": 16, "drop_rate": 0.15995138350968297, "lr": 0.004656630423530019, "batch_size": 1024}, "time_since_restore": 113.85971927642822, "iterations_since_restore": 1}
+{"loss": 1959.9800554410679, "timestamp": 1719330869, "checkpoint_dir_name": "checkpoint_000001", "should_checkpoint": true, "done": false, "training_iteration": 2, "trial_id": "481f1fff", "date": "2024-06-25_17-54-29", "time_this_iter_s": 114.88935971260071, "time_total_s": 228.74907898902893, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 2, "decoder_rt_num_layer": 8, "decoder_int_num_layer": 1, "embedding_dim": 64, "encoder_ff": 1024, "decoder_rt_ff": 1024, "decoder_int_ff": 512, "n_head": 16, "drop_rate": 0.15995138350968297, "lr": 0.004656630423530019, "batch_size": 1024}, "time_since_restore": 114.88935971260071, "iterations_since_restore": 1}
diff --git a/ray_results/test_experiment/train_model_52d8dfa1_2_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=4,drop_r_2024-06-25_16-08-04/events.out.tfevents.1719324574.r3i5n6 b/ray_results/test_experiment/train_model_52d8dfa1_2_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=4,drop_r_2024-06-25_16-08-04/events.out.tfevents.1719324574.r3i5n6
new file mode 100644
index 0000000000000000000000000000000000000000..0c8fa347ef81f7c40770382a058151a3d74cbe79
Binary files /dev/null and b/ray_results/test_experiment/train_model_52d8dfa1_2_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=4,drop_r_2024-06-25_16-08-04/events.out.tfevents.1719324574.r3i5n6 differ
diff --git a/ray_results/test_experiment/train_model_52d8dfa1_2_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=4,drop_r_2024-06-25_16-08-04/params.json b/ray_results/test_experiment/train_model_52d8dfa1_2_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=4,drop_r_2024-06-25_16-08-04/params.json
new file mode 100644
index 0000000000000000000000000000000000000000..cef4024b58873cca841cf82a602ae6076d231d8a
--- /dev/null
+++ b/ray_results/test_experiment/train_model_52d8dfa1_2_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=4,drop_r_2024-06-25_16-08-04/params.json
@@ -0,0 +1,13 @@
+{
+ "batch_size": 2048,
+ "decoder_int_ff": 512,
+ "decoder_int_num_layer": 1,
+ "decoder_rt_ff": 1024,
+ "decoder_rt_num_layer": 4,
+ "drop_rate": 0.982125481793317,
+ "embedding_dim": 16,
+ "encoder_ff": 1024,
+ "encoder_num_layer": 8,
+ "lr": 0.002789281371289888,
+ "n_head": 8
+}
\ No newline at end of file
diff --git a/ray_results/test_experiment/train_model_52d8dfa1_2_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=4,drop_r_2024-06-25_16-08-04/params.pkl b/ray_results/test_experiment/train_model_52d8dfa1_2_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=4,drop_r_2024-06-25_16-08-04/params.pkl
new file mode 100644
index 0000000000000000000000000000000000000000..7c3abfbca8fa2251d2a0adc0e85159d217bcd515
Binary files /dev/null and b/ray_results/test_experiment/train_model_52d8dfa1_2_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=4,drop_r_2024-06-25_16-08-04/params.pkl differ
diff --git a/ray_results/test_experiment/train_model_52d8dfa1_2_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=4,drop_r_2024-06-25_16-08-04/progress.csv b/ray_results/test_experiment/train_model_52d8dfa1_2_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=4,drop_r_2024-06-25_16-08-04/progress.csv
new file mode 100644
index 0000000000000000000000000000000000000000..1e9afeb8ed9762bd3f24a905c756d1be91d33540
--- /dev/null
+++ b/ray_results/test_experiment/train_model_52d8dfa1_2_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=4,drop_r_2024-06-25_16-08-04/progress.csv
@@ -0,0 +1,2 @@
+loss,timestamp,checkpoint_dir_name,should_checkpoint,done,training_iteration,trial_id,date,time_this_iter_s,time_total_s,pid,hostname,node_ip,time_since_restore,iterations_since_restore
+nan,1719324663,checkpoint_000000,True,False,1,52d8dfa1,2024-06-25_16-11-03,89.4755928516388,89.4755928516388,69318,r3i5n6,10.159.8.159,89.4755928516388,1
diff --git a/ray_results/test_experiment/train_model_52d8dfa1_2_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=4,drop_r_2024-06-25_16-08-04/result.json b/ray_results/test_experiment/train_model_52d8dfa1_2_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=4,drop_r_2024-06-25_16-08-04/result.json
new file mode 100644
index 0000000000000000000000000000000000000000..624a4df7276510f941e4e44bae7a84706f6403e7
--- /dev/null
+++ b/ray_results/test_experiment/train_model_52d8dfa1_2_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=4,drop_r_2024-06-25_16-08-04/result.json
@@ -0,0 +1 @@
+{"loss": NaN, "timestamp": 1719324663, "checkpoint_dir_name": "checkpoint_000000", "should_checkpoint": true, "done": false, "training_iteration": 1, "trial_id": "52d8dfa1", "date": "2024-06-25_16-11-03", "time_this_iter_s": 89.4755928516388, "time_total_s": 89.4755928516388, "pid": 69318, "hostname": "r3i5n6", "node_ip": "10.159.8.159", "config": {"encoder_num_layer": 8, "decoder_rt_num_layer": 4, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 1024, "decoder_rt_ff": 1024, "decoder_int_ff": 512, "n_head": 8, "drop_rate": 0.982125481793317, "lr": 0.002789281371289888, "batch_size": 2048}, "time_since_restore": 89.4755928516388, "iterations_since_restore": 1}
diff --git a/ray_results/test_experiment/train_model_5763f130_1_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=8,drop_r_2024-06-25_17-10-00/events.out.tfevents.1719328213.r8i6n2 b/ray_results/test_experiment/train_model_5763f130_1_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=8,drop_r_2024-06-25_17-10-00/events.out.tfevents.1719328213.r8i6n2
new file mode 100644
index 0000000000000000000000000000000000000000..a15214dc49bed0fae69d1e250b80c268488a66a4
Binary files /dev/null and b/ray_results/test_experiment/train_model_5763f130_1_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=8,drop_r_2024-06-25_17-10-00/events.out.tfevents.1719328213.r8i6n2 differ
diff --git a/ray_results/test_experiment/train_model_5763f130_1_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=8,drop_r_2024-06-25_17-10-00/params.json b/ray_results/test_experiment/train_model_5763f130_1_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=8,drop_r_2024-06-25_17-10-00/params.json
new file mode 100644
index 0000000000000000000000000000000000000000..554644b8ace65820295e2d4fd52fd4a45f11a91b
--- /dev/null
+++ b/ray_results/test_experiment/train_model_5763f130_1_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=8,drop_r_2024-06-25_17-10-00/params.json
@@ -0,0 +1,13 @@
+{
+ "batch_size": 2048,
+ "decoder_int_ff": 512,
+ "decoder_int_num_layer": 1,
+ "decoder_rt_ff": 2048,
+ "decoder_rt_num_layer": 8,
+ "drop_rate": 0.7224904221599877,
+ "embedding_dim": 256,
+ "encoder_ff": 512,
+ "encoder_num_layer": 4,
+ "lr": 0.04561252173503944,
+ "n_head": 2
+}
\ No newline at end of file
diff --git a/ray_results/test_experiment/train_model_5763f130_1_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=8,drop_r_2024-06-25_17-10-00/params.pkl b/ray_results/test_experiment/train_model_5763f130_1_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=8,drop_r_2024-06-25_17-10-00/params.pkl
new file mode 100644
index 0000000000000000000000000000000000000000..4395f1ff6fd0ff4d452b715ee4e214e7f9c36808
Binary files /dev/null and b/ray_results/test_experiment/train_model_5763f130_1_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=8,drop_r_2024-06-25_17-10-00/params.pkl differ
diff --git a/ray_results/test_experiment/train_model_5763f130_1_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=8,drop_r_2024-06-25_17-10-00/progress.csv b/ray_results/test_experiment/train_model_5763f130_1_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=8,drop_r_2024-06-25_17-10-00/progress.csv
new file mode 100644
index 0000000000000000000000000000000000000000..c3d04e5ceebb5f965800d469b50ada9cdef5a5d8
--- /dev/null
+++ b/ray_results/test_experiment/train_model_5763f130_1_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=8,drop_r_2024-06-25_17-10-00/progress.csv
@@ -0,0 +1,2 @@
+loss,timestamp,checkpoint_dir_name,should_checkpoint,done,training_iteration,trial_id,date,time_this_iter_s,time_total_s,pid,hostname,node_ip,time_since_restore,iterations_since_restore
+4538.754975086122,1719328455,checkpoint_000000,True,False,1,5763f130,2024-06-25_17-14-16,242.80822491645813,242.80822491645813,2195253,r8i6n2,10.159.28.60,242.80822491645813,1
diff --git a/ray_results/test_experiment/train_model_5763f130_1_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=8,drop_r_2024-06-25_17-10-00/result.json b/ray_results/test_experiment/train_model_5763f130_1_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=8,drop_r_2024-06-25_17-10-00/result.json
new file mode 100644
index 0000000000000000000000000000000000000000..ab246236189d451cde9d8c042cf56878c1053d04
--- /dev/null
+++ b/ray_results/test_experiment/train_model_5763f130_1_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=8,drop_r_2024-06-25_17-10-00/result.json
@@ -0,0 +1 @@
+{"loss": 4538.754975086122, "timestamp": 1719328455, "checkpoint_dir_name": "checkpoint_000000", "should_checkpoint": true, "done": false, "training_iteration": 1, "trial_id": "5763f130", "date": "2024-06-25_17-14-16", "time_this_iter_s": 242.80822491645813, "time_total_s": 242.80822491645813, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 4, "decoder_rt_num_layer": 8, "decoder_int_num_layer": 1, "embedding_dim": 256, "encoder_ff": 512, "decoder_rt_ff": 2048, "decoder_int_ff": 512, "n_head": 2, "drop_rate": 0.7224904221599877, "lr": 0.04561252173503944, "batch_size": 2048}, "time_since_restore": 242.80822491645813, "iterations_since_restore": 1}
diff --git a/ray_results/test_experiment/train_model_5b0a0ae2_6_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop_r_2024-06-25_17-18-52/events.out.tfevents.1719329376.r8i6n2 b/ray_results/test_experiment/train_model_5b0a0ae2_6_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop_r_2024-06-25_17-18-52/events.out.tfevents.1719329376.r8i6n2
new file mode 100644
index 0000000000000000000000000000000000000000..f6d895802d3803de895243eb6a9f615d9abfdf13
Binary files /dev/null and b/ray_results/test_experiment/train_model_5b0a0ae2_6_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop_r_2024-06-25_17-18-52/events.out.tfevents.1719329376.r8i6n2 differ
diff --git a/ray_results/test_experiment/train_model_5b0a0ae2_6_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop_r_2024-06-25_17-18-52/events.out.tfevents.1719330587.r8i6n2 b/ray_results/test_experiment/train_model_5b0a0ae2_6_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop_r_2024-06-25_17-18-52/events.out.tfevents.1719330587.r8i6n2
new file mode 100644
index 0000000000000000000000000000000000000000..4f8b36a8361df3d3b03014589393ace09171acf0
Binary files /dev/null and b/ray_results/test_experiment/train_model_5b0a0ae2_6_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop_r_2024-06-25_17-18-52/events.out.tfevents.1719330587.r8i6n2 differ
diff --git a/ray_results/test_experiment/train_model_5b0a0ae2_6_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop_r_2024-06-25_17-18-52/params.json b/ray_results/test_experiment/train_model_5b0a0ae2_6_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop_r_2024-06-25_17-18-52/params.json
new file mode 100644
index 0000000000000000000000000000000000000000..e8426151f69bd4ed948bad9132a30a9d7b558fcd
--- /dev/null
+++ b/ray_results/test_experiment/train_model_5b0a0ae2_6_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop_r_2024-06-25_17-18-52/params.json
@@ -0,0 +1,13 @@
+{
+ "batch_size": 2048,
+ "decoder_int_ff": 512,
+ "decoder_int_num_layer": 1,
+ "decoder_rt_ff": 2048,
+ "decoder_rt_num_layer": 1,
+ "drop_rate": 0.0063231604639778505,
+ "embedding_dim": 16,
+ "encoder_ff": 512,
+ "encoder_num_layer": 8,
+ "lr": 0.0021079084039053323,
+ "n_head": 16
+}
\ No newline at end of file
diff --git a/ray_results/test_experiment/train_model_5b0a0ae2_6_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop_r_2024-06-25_17-18-52/params.pkl b/ray_results/test_experiment/train_model_5b0a0ae2_6_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop_r_2024-06-25_17-18-52/params.pkl
new file mode 100644
index 0000000000000000000000000000000000000000..736546c8843456938dfbfec2824c7e47ab0926d8
Binary files /dev/null and b/ray_results/test_experiment/train_model_5b0a0ae2_6_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop_r_2024-06-25_17-18-52/params.pkl differ
diff --git a/ray_results/test_experiment/train_model_5b0a0ae2_6_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop_r_2024-06-25_17-18-52/progress.csv b/ray_results/test_experiment/train_model_5b0a0ae2_6_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop_r_2024-06-25_17-18-52/progress.csv
new file mode 100644
index 0000000000000000000000000000000000000000..942260564f74608d0d71012e7151f44e6bbd5724
--- /dev/null
+++ b/ray_results/test_experiment/train_model_5b0a0ae2_6_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop_r_2024-06-25_17-18-52/progress.csv
@@ -0,0 +1,3 @@
+loss,timestamp,checkpoint_dir_name,should_checkpoint,done,training_iteration,trial_id,date,time_this_iter_s,time_total_s,pid,hostname,node_ip,time_since_restore,iterations_since_restore
+1952.9751620555487,1719329461,checkpoint_000000,True,False,1,5b0a0ae2,2024-06-25_17-31-01,84.39049577713013,84.39049577713013,2195253,r8i6n2,10.159.28.60,84.39049577713013,1
+1963.9745468980684,1719330671,checkpoint_000001,True,False,2,5b0a0ae2,2024-06-25_17-51-11,84.48489356040955,168.87538933753967,2195253,r8i6n2,10.159.28.60,84.48489356040955,1
diff --git a/ray_results/test_experiment/train_model_5b0a0ae2_6_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop_r_2024-06-25_17-18-52/result.json b/ray_results/test_experiment/train_model_5b0a0ae2_6_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop_r_2024-06-25_17-18-52/result.json
new file mode 100644
index 0000000000000000000000000000000000000000..dd7065b0fe374873636812f1c5cb0b07b13f6a8f
--- /dev/null
+++ b/ray_results/test_experiment/train_model_5b0a0ae2_6_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop_r_2024-06-25_17-18-52/result.json
@@ -0,0 +1,2 @@
+{"loss": 1952.9751620555487, "timestamp": 1719329461, "checkpoint_dir_name": "checkpoint_000000", "should_checkpoint": true, "done": false, "training_iteration": 1, "trial_id": "5b0a0ae2", "date": "2024-06-25_17-31-01", "time_this_iter_s": 84.39049577713013, "time_total_s": 84.39049577713013, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 8, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 512, "decoder_rt_ff": 2048, "decoder_int_ff": 512, "n_head": 16, "drop_rate": 0.0063231604639778505, "lr": 0.0021079084039053323, "batch_size": 2048}, "time_since_restore": 84.39049577713013, "iterations_since_restore": 1}
+{"loss": 1963.9745468980684, "timestamp": 1719330671, "checkpoint_dir_name": "checkpoint_000001", "should_checkpoint": true, "done": false, "training_iteration": 2, "trial_id": "5b0a0ae2", "date": "2024-06-25_17-51-11", "time_this_iter_s": 84.48489356040955, "time_total_s": 168.87538933753967, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 8, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 512, "decoder_rt_ff": 2048, "decoder_int_ff": 512, "n_head": 16, "drop_rate": 0.0063231604639778505, "lr": 0.0021079084039053323, "batch_size": 2048}, "time_since_restore": 84.48489356040955, "iterations_since_restore": 1}
diff --git a/ray_results/test_experiment/train_model_5eb1e282_23_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=1,drop_r_2024-06-25_18-57-29/events.out.tfevents.1719335674.r8i6n2 b/ray_results/test_experiment/train_model_5eb1e282_23_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=1,drop_r_2024-06-25_18-57-29/events.out.tfevents.1719335674.r8i6n2
new file mode 100644
index 0000000000000000000000000000000000000000..bb8893a139aa6e56fab04eaf274dd18721a436a5
Binary files /dev/null and b/ray_results/test_experiment/train_model_5eb1e282_23_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=1,drop_r_2024-06-25_18-57-29/events.out.tfevents.1719335674.r8i6n2 differ
diff --git a/ray_results/test_experiment/train_model_5eb1e282_23_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=1,drop_r_2024-06-25_18-57-29/events.out.tfevents.1719339767.r8i6n2 b/ray_results/test_experiment/train_model_5eb1e282_23_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=1,drop_r_2024-06-25_18-57-29/events.out.tfevents.1719339767.r8i6n2
new file mode 100644
index 0000000000000000000000000000000000000000..948ba6c57f934e4a1d8025f5603eda83973ad968
Binary files /dev/null and b/ray_results/test_experiment/train_model_5eb1e282_23_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=1,drop_r_2024-06-25_18-57-29/events.out.tfevents.1719339767.r8i6n2 differ
diff --git a/ray_results/test_experiment/train_model_5eb1e282_23_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=1,drop_r_2024-06-25_18-57-29/events.out.tfevents.1719345748.r8i6n2 b/ray_results/test_experiment/train_model_5eb1e282_23_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=1,drop_r_2024-06-25_18-57-29/events.out.tfevents.1719345748.r8i6n2
new file mode 100644
index 0000000000000000000000000000000000000000..cc958918d7198f3e194bf3add12949bb2505abe6
Binary files /dev/null and b/ray_results/test_experiment/train_model_5eb1e282_23_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=1,drop_r_2024-06-25_18-57-29/events.out.tfevents.1719345748.r8i6n2 differ
diff --git a/ray_results/test_experiment/train_model_5eb1e282_23_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=1,drop_r_2024-06-25_18-57-29/params.json b/ray_results/test_experiment/train_model_5eb1e282_23_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=1,drop_r_2024-06-25_18-57-29/params.json
new file mode 100644
index 0000000000000000000000000000000000000000..a4aefb0c2eb135aff9a15f009919801619086eb3
--- /dev/null
+++ b/ray_results/test_experiment/train_model_5eb1e282_23_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=1,drop_r_2024-06-25_18-57-29/params.json
@@ -0,0 +1,13 @@
+{
+ "batch_size": 2048,
+ "decoder_int_ff": 512,
+ "decoder_int_num_layer": 1,
+ "decoder_rt_ff": 512,
+ "decoder_rt_num_layer": 1,
+ "drop_rate": 0.2842529108982538,
+ "embedding_dim": 16,
+ "encoder_ff": 1024,
+ "encoder_num_layer": 1,
+ "lr": 0.00015364900402465778,
+ "n_head": 4
+}
\ No newline at end of file
diff --git a/ray_results/test_experiment/train_model_5eb1e282_23_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=1,drop_r_2024-06-25_18-57-29/params.pkl b/ray_results/test_experiment/train_model_5eb1e282_23_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=1,drop_r_2024-06-25_18-57-29/params.pkl
new file mode 100644
index 0000000000000000000000000000000000000000..18b52d2f95739444bc3641862802d17e21716ccb
Binary files /dev/null and b/ray_results/test_experiment/train_model_5eb1e282_23_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=1,drop_r_2024-06-25_18-57-29/params.pkl differ
diff --git a/ray_results/test_experiment/train_model_5eb1e282_23_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=1,drop_r_2024-06-25_18-57-29/progress.csv b/ray_results/test_experiment/train_model_5eb1e282_23_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=1,drop_r_2024-06-25_18-57-29/progress.csv
new file mode 100644
index 0000000000000000000000000000000000000000..0363c5e332fd9be01f40571378308f8f3932fd60
--- /dev/null
+++ b/ray_results/test_experiment/train_model_5eb1e282_23_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=1,drop_r_2024-06-25_18-57-29/progress.csv
@@ -0,0 +1,19 @@
+loss,timestamp,checkpoint_dir_name,should_checkpoint,done,training_iteration,trial_id,date,time_this_iter_s,time_total_s,pid,hostname,node_ip,time_since_restore,iterations_since_restore
+1030.0586158572219,1719335735,checkpoint_000000,True,False,1,5eb1e282,2024-06-25_19-15-35,60.84843850135803,60.84843850135803,2195253,r8i6n2,10.159.28.60,60.84843850135803,1
+562.0288162832185,1719335783,checkpoint_000001,True,False,2,5eb1e282,2024-06-25_19-16-23,47.70394277572632,108.55238127708435,2195253,r8i6n2,10.159.28.60,108.55238127708435,2
+444.9066558597595,1719339829,checkpoint_000002,True,False,3,5eb1e282,2024-06-25_20-23-49,61.87126088142395,170.4236421585083,2195253,r8i6n2,10.159.28.60,61.87126088142395,1
+385.7694199930026,1719339876,checkpoint_000003,True,False,4,5eb1e282,2024-06-25_20-24-36,47.405888080596924,217.82953023910522,2195253,r8i6n2,10.159.28.60,109.27714896202087,2
+351.1129479595995,1719339924,checkpoint_000004,True,False,5,5eb1e282,2024-06-25_20-25-24,47.97740936279297,265.8069396018982,2195253,r8i6n2,10.159.28.60,157.25455832481384,3
+320.4649035836768,1719339972,checkpoint_000005,True,False,6,5eb1e282,2024-06-25_20-26-12,47.432437896728516,313.2393774986267,2195253,r8i6n2,10.159.28.60,204.68699622154236,4
+293.46044717623494,1719340019,checkpoint_000006,True,False,7,5eb1e282,2024-06-25_20-26-59,47.91654348373413,361.15592098236084,2195253,r8i6n2,10.159.28.60,252.6035397052765,5
+273.19929143950696,1719340067,checkpoint_000007,True,False,8,5eb1e282,2024-06-25_20-27-47,47.474034786224365,408.6299557685852,2195253,r8i6n2,10.159.28.60,300.07757449150085,6
+256.9283664733406,1719345809,checkpoint_000008,True,False,9,5eb1e282,2024-06-25_22-03-29,61.405439615249634,470.03539538383484,2195253,r8i6n2,10.159.28.60,61.405439615249634,1
+240.56306673785832,1719345856,checkpoint_000009,True,False,10,5eb1e282,2024-06-25_22-04-16,47.146918296813965,517.1823136806488,2195253,r8i6n2,10.159.28.60,108.5523579120636,2
+230.43498638483484,1719345903,checkpoint_000010,True,False,11,5eb1e282,2024-06-25_22-05-03,47.185975790023804,564.3682894706726,2195253,r8i6n2,10.159.28.60,155.7383337020874,3
+219.4073686975194,1719345951,checkpoint_000011,True,False,12,5eb1e282,2024-06-25_22-05-51,47.13756966590881,611.5058591365814,2195253,r8i6n2,10.159.28.60,202.87590336799622,4
+211.7666195846918,1719345998,checkpoint_000012,True,False,13,5eb1e282,2024-06-25_22-06-38,47.17806816101074,658.6839272975922,2195253,r8i6n2,10.159.28.60,250.05397152900696,5
+204.5018019788847,1719346046,checkpoint_000013,True,False,14,5eb1e282,2024-06-25_22-07-26,47.774343729019165,706.4582710266113,2195253,r8i6n2,10.159.28.60,297.8283152580261,6
+198.05782912847565,1719346092,checkpoint_000014,True,False,15,5eb1e282,2024-06-25_22-08-12,46.741618633270264,753.1998896598816,2195253,r8i6n2,10.159.28.60,344.5699338912964,7
+194.08880230760948,1719346139,checkpoint_000015,True,False,16,5eb1e282,2024-06-25_22-08-59,47.16085863113403,800.3607482910156,2195253,r8i6n2,10.159.28.60,391.7307925224304,8
+196.82745781845935,1719346187,checkpoint_000016,True,False,17,5eb1e282,2024-06-25_22-09-47,47.1561017036438,847.5168499946594,2195253,r8i6n2,10.159.28.60,438.8868942260742,9
+183.96574522003414,1719346234,checkpoint_000017,True,False,18,5eb1e282,2024-06-25_22-10-34,47.40534067153931,894.9221906661987,2195253,r8i6n2,10.159.28.60,486.2922348976135,10
diff --git a/ray_results/test_experiment/train_model_5eb1e282_23_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=1,drop_r_2024-06-25_18-57-29/result.json b/ray_results/test_experiment/train_model_5eb1e282_23_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=1,drop_r_2024-06-25_18-57-29/result.json
new file mode 100644
index 0000000000000000000000000000000000000000..db1051e1977364347a4053570795bb1eec9ca59a
--- /dev/null
+++ b/ray_results/test_experiment/train_model_5eb1e282_23_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=1,drop_r_2024-06-25_18-57-29/result.json
@@ -0,0 +1,18 @@
+{"loss": 1030.0586158572219, "timestamp": 1719335735, "checkpoint_dir_name": "checkpoint_000000", "should_checkpoint": true, "done": false, "training_iteration": 1, "trial_id": "5eb1e282", "date": "2024-06-25_19-15-35", "time_this_iter_s": 60.84843850135803, "time_total_s": 60.84843850135803, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 1, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 1024, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 4, "drop_rate": 0.2842529108982538, "lr": 0.00015364900402465778, "batch_size": 2048}, "time_since_restore": 60.84843850135803, "iterations_since_restore": 1}
+{"loss": 562.0288162832185, "timestamp": 1719335783, "checkpoint_dir_name": "checkpoint_000001", "should_checkpoint": true, "done": false, "training_iteration": 2, "trial_id": "5eb1e282", "date": "2024-06-25_19-16-23", "time_this_iter_s": 47.70394277572632, "time_total_s": 108.55238127708435, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 1, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 1024, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 4, "drop_rate": 0.2842529108982538, "lr": 0.00015364900402465778, "batch_size": 2048}, "time_since_restore": 108.55238127708435, "iterations_since_restore": 2}
+{"loss": 444.9066558597595, "timestamp": 1719339829, "checkpoint_dir_name": "checkpoint_000002", "should_checkpoint": true, "done": false, "training_iteration": 3, "trial_id": "5eb1e282", "date": "2024-06-25_20-23-49", "time_this_iter_s": 61.87126088142395, "time_total_s": 170.4236421585083, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 1, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 1024, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 4, "drop_rate": 0.2842529108982538, "lr": 0.00015364900402465778, "batch_size": 2048}, "time_since_restore": 61.87126088142395, "iterations_since_restore": 1}
+{"loss": 385.7694199930026, "timestamp": 1719339876, "checkpoint_dir_name": "checkpoint_000003", "should_checkpoint": true, "done": false, "training_iteration": 4, "trial_id": "5eb1e282", "date": "2024-06-25_20-24-36", "time_this_iter_s": 47.405888080596924, "time_total_s": 217.82953023910522, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 1, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 1024, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 4, "drop_rate": 0.2842529108982538, "lr": 0.00015364900402465778, "batch_size": 2048}, "time_since_restore": 109.27714896202087, "iterations_since_restore": 2}
+{"loss": 351.1129479595995, "timestamp": 1719339924, "checkpoint_dir_name": "checkpoint_000004", "should_checkpoint": true, "done": false, "training_iteration": 5, "trial_id": "5eb1e282", "date": "2024-06-25_20-25-24", "time_this_iter_s": 47.97740936279297, "time_total_s": 265.8069396018982, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 1, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 1024, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 4, "drop_rate": 0.2842529108982538, "lr": 0.00015364900402465778, "batch_size": 2048}, "time_since_restore": 157.25455832481384, "iterations_since_restore": 3}
+{"loss": 320.4649035836768, "timestamp": 1719339972, "checkpoint_dir_name": "checkpoint_000005", "should_checkpoint": true, "done": false, "training_iteration": 6, "trial_id": "5eb1e282", "date": "2024-06-25_20-26-12", "time_this_iter_s": 47.432437896728516, "time_total_s": 313.2393774986267, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 1, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 1024, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 4, "drop_rate": 0.2842529108982538, "lr": 0.00015364900402465778, "batch_size": 2048}, "time_since_restore": 204.68699622154236, "iterations_since_restore": 4}
+{"loss": 293.46044717623494, "timestamp": 1719340019, "checkpoint_dir_name": "checkpoint_000006", "should_checkpoint": true, "done": false, "training_iteration": 7, "trial_id": "5eb1e282", "date": "2024-06-25_20-26-59", "time_this_iter_s": 47.91654348373413, "time_total_s": 361.15592098236084, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 1, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 1024, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 4, "drop_rate": 0.2842529108982538, "lr": 0.00015364900402465778, "batch_size": 2048}, "time_since_restore": 252.6035397052765, "iterations_since_restore": 5}
+{"loss": 273.19929143950696, "timestamp": 1719340067, "checkpoint_dir_name": "checkpoint_000007", "should_checkpoint": true, "done": false, "training_iteration": 8, "trial_id": "5eb1e282", "date": "2024-06-25_20-27-47", "time_this_iter_s": 47.474034786224365, "time_total_s": 408.6299557685852, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 1, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 1024, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 4, "drop_rate": 0.2842529108982538, "lr": 0.00015364900402465778, "batch_size": 2048}, "time_since_restore": 300.07757449150085, "iterations_since_restore": 6}
+{"loss": 256.9283664733406, "timestamp": 1719345809, "checkpoint_dir_name": "checkpoint_000008", "should_checkpoint": true, "done": false, "training_iteration": 9, "trial_id": "5eb1e282", "date": "2024-06-25_22-03-29", "time_this_iter_s": 61.405439615249634, "time_total_s": 470.03539538383484, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 1, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 1024, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 4, "drop_rate": 0.2842529108982538, "lr": 0.00015364900402465778, "batch_size": 2048}, "time_since_restore": 61.405439615249634, "iterations_since_restore": 1}
+{"loss": 240.56306673785832, "timestamp": 1719345856, "checkpoint_dir_name": "checkpoint_000009", "should_checkpoint": true, "done": false, "training_iteration": 10, "trial_id": "5eb1e282", "date": "2024-06-25_22-04-16", "time_this_iter_s": 47.146918296813965, "time_total_s": 517.1823136806488, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 1, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 1024, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 4, "drop_rate": 0.2842529108982538, "lr": 0.00015364900402465778, "batch_size": 2048}, "time_since_restore": 108.5523579120636, "iterations_since_restore": 2}
+{"loss": 230.43498638483484, "timestamp": 1719345903, "checkpoint_dir_name": "checkpoint_000010", "should_checkpoint": true, "done": false, "training_iteration": 11, "trial_id": "5eb1e282", "date": "2024-06-25_22-05-03", "time_this_iter_s": 47.185975790023804, "time_total_s": 564.3682894706726, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 1, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 1024, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 4, "drop_rate": 0.2842529108982538, "lr": 0.00015364900402465778, "batch_size": 2048}, "time_since_restore": 155.7383337020874, "iterations_since_restore": 3}
+{"loss": 219.4073686975194, "timestamp": 1719345951, "checkpoint_dir_name": "checkpoint_000011", "should_checkpoint": true, "done": false, "training_iteration": 12, "trial_id": "5eb1e282", "date": "2024-06-25_22-05-51", "time_this_iter_s": 47.13756966590881, "time_total_s": 611.5058591365814, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 1, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 1024, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 4, "drop_rate": 0.2842529108982538, "lr": 0.00015364900402465778, "batch_size": 2048}, "time_since_restore": 202.87590336799622, "iterations_since_restore": 4}
+{"loss": 211.7666195846918, "timestamp": 1719345998, "checkpoint_dir_name": "checkpoint_000012", "should_checkpoint": true, "done": false, "training_iteration": 13, "trial_id": "5eb1e282", "date": "2024-06-25_22-06-38", "time_this_iter_s": 47.17806816101074, "time_total_s": 658.6839272975922, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 1, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 1024, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 4, "drop_rate": 0.2842529108982538, "lr": 0.00015364900402465778, "batch_size": 2048}, "time_since_restore": 250.05397152900696, "iterations_since_restore": 5}
+{"loss": 204.5018019788847, "timestamp": 1719346046, "checkpoint_dir_name": "checkpoint_000013", "should_checkpoint": true, "done": false, "training_iteration": 14, "trial_id": "5eb1e282", "date": "2024-06-25_22-07-26", "time_this_iter_s": 47.774343729019165, "time_total_s": 706.4582710266113, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 1, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 1024, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 4, "drop_rate": 0.2842529108982538, "lr": 0.00015364900402465778, "batch_size": 2048}, "time_since_restore": 297.8283152580261, "iterations_since_restore": 6}
+{"loss": 198.05782912847565, "timestamp": 1719346092, "checkpoint_dir_name": "checkpoint_000014", "should_checkpoint": true, "done": false, "training_iteration": 15, "trial_id": "5eb1e282", "date": "2024-06-25_22-08-12", "time_this_iter_s": 46.741618633270264, "time_total_s": 753.1998896598816, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 1, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 1024, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 4, "drop_rate": 0.2842529108982538, "lr": 0.00015364900402465778, "batch_size": 2048}, "time_since_restore": 344.5699338912964, "iterations_since_restore": 7}
+{"loss": 194.08880230760948, "timestamp": 1719346139, "checkpoint_dir_name": "checkpoint_000015", "should_checkpoint": true, "done": false, "training_iteration": 16, "trial_id": "5eb1e282", "date": "2024-06-25_22-08-59", "time_this_iter_s": 47.16085863113403, "time_total_s": 800.3607482910156, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 1, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 1024, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 4, "drop_rate": 0.2842529108982538, "lr": 0.00015364900402465778, "batch_size": 2048}, "time_since_restore": 391.7307925224304, "iterations_since_restore": 8}
+{"loss": 196.82745781845935, "timestamp": 1719346187, "checkpoint_dir_name": "checkpoint_000016", "should_checkpoint": true, "done": false, "training_iteration": 17, "trial_id": "5eb1e282", "date": "2024-06-25_22-09-47", "time_this_iter_s": 47.1561017036438, "time_total_s": 847.5168499946594, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 1, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 1024, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 4, "drop_rate": 0.2842529108982538, "lr": 0.00015364900402465778, "batch_size": 2048}, "time_since_restore": 438.8868942260742, "iterations_since_restore": 9}
+{"loss": 183.96574522003414, "timestamp": 1719346234, "checkpoint_dir_name": "checkpoint_000017", "should_checkpoint": true, "done": false, "training_iteration": 18, "trial_id": "5eb1e282", "date": "2024-06-25_22-10-34", "time_this_iter_s": 47.40534067153931, "time_total_s": 894.9221906661987, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 1, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 1024, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 4, "drop_rate": 0.2842529108982538, "lr": 0.00015364900402465778, "batch_size": 2048}, "time_since_restore": 486.2922348976135, "iterations_since_restore": 10}
diff --git a/ray_results/test_experiment/train_model_651349c4_12_batch_size=2048,decoder_int_ff=2048,decoder_int_num_layer=8,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop_2024-06-25_16-23-08/events.out.tfevents.1719326403.r3i5n6 b/ray_results/test_experiment/train_model_651349c4_12_batch_size=2048,decoder_int_ff=2048,decoder_int_num_layer=8,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop_2024-06-25_16-23-08/events.out.tfevents.1719326403.r3i5n6
new file mode 100644
index 0000000000000000000000000000000000000000..0cf7bb0efd26ac5cf2723e720f5a8dda9f21725f
Binary files /dev/null and b/ray_results/test_experiment/train_model_651349c4_12_batch_size=2048,decoder_int_ff=2048,decoder_int_num_layer=8,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop_2024-06-25_16-23-08/events.out.tfevents.1719326403.r3i5n6 differ
diff --git a/ray_results/test_experiment/train_model_651349c4_12_batch_size=2048,decoder_int_ff=2048,decoder_int_num_layer=8,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop_2024-06-25_16-23-08/params.json b/ray_results/test_experiment/train_model_651349c4_12_batch_size=2048,decoder_int_ff=2048,decoder_int_num_layer=8,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop_2024-06-25_16-23-08/params.json
new file mode 100644
index 0000000000000000000000000000000000000000..7745c6e9c6ba7521606df9b806a9bdfe914a8be8
--- /dev/null
+++ b/ray_results/test_experiment/train_model_651349c4_12_batch_size=2048,decoder_int_ff=2048,decoder_int_num_layer=8,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop_2024-06-25_16-23-08/params.json
@@ -0,0 +1,13 @@
+{
+ "batch_size": 2048,
+ "decoder_int_ff": 2048,
+ "decoder_int_num_layer": 8,
+ "decoder_rt_ff": 2048,
+ "decoder_rt_num_layer": 1,
+ "drop_rate": 0.6683907000004579,
+ "embedding_dim": 16,
+ "encoder_ff": 2048,
+ "encoder_num_layer": 8,
+ "lr": 0.0001175178193869245,
+ "n_head": 4
+}
\ No newline at end of file
diff --git a/ray_results/test_experiment/train_model_651349c4_12_batch_size=2048,decoder_int_ff=2048,decoder_int_num_layer=8,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop_2024-06-25_16-23-08/params.pkl b/ray_results/test_experiment/train_model_651349c4_12_batch_size=2048,decoder_int_ff=2048,decoder_int_num_layer=8,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop_2024-06-25_16-23-08/params.pkl
new file mode 100644
index 0000000000000000000000000000000000000000..74a714b2d04cf1b94b7f8ea4dc828e85b345fe6f
Binary files /dev/null and b/ray_results/test_experiment/train_model_651349c4_12_batch_size=2048,decoder_int_ff=2048,decoder_int_num_layer=8,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop_2024-06-25_16-23-08/params.pkl differ
diff --git a/ray_results/test_experiment/train_model_651349c4_12_batch_size=2048,decoder_int_ff=2048,decoder_int_num_layer=8,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop_2024-06-25_16-23-08/progress.csv b/ray_results/test_experiment/train_model_651349c4_12_batch_size=2048,decoder_int_ff=2048,decoder_int_num_layer=8,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop_2024-06-25_16-23-08/progress.csv
new file mode 100644
index 0000000000000000000000000000000000000000..6bff8a1f098245eda5a92ab2da4989c58936765b
--- /dev/null
+++ b/ray_results/test_experiment/train_model_651349c4_12_batch_size=2048,decoder_int_ff=2048,decoder_int_num_layer=8,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop_2024-06-25_16-23-08/progress.csv
@@ -0,0 +1,2 @@
+loss,timestamp,checkpoint_dir_name,should_checkpoint,done,training_iteration,trial_id,date,time_this_iter_s,time_total_s,pid,hostname,node_ip,time_since_restore,iterations_since_restore
+1981.618032380352,1719326489,checkpoint_000000,True,False,1,651349c4,2024-06-25_16-41-29,86.11733651161194,86.11733651161194,69318,r3i5n6,10.159.8.159,86.11733651161194,1
diff --git a/ray_results/test_experiment/train_model_651349c4_12_batch_size=2048,decoder_int_ff=2048,decoder_int_num_layer=8,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop_2024-06-25_16-23-08/result.json b/ray_results/test_experiment/train_model_651349c4_12_batch_size=2048,decoder_int_ff=2048,decoder_int_num_layer=8,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop_2024-06-25_16-23-08/result.json
new file mode 100644
index 0000000000000000000000000000000000000000..717abb4e0d15e5cc740002841bd78a6ce1881350
--- /dev/null
+++ b/ray_results/test_experiment/train_model_651349c4_12_batch_size=2048,decoder_int_ff=2048,decoder_int_num_layer=8,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop_2024-06-25_16-23-08/result.json
@@ -0,0 +1 @@
+{"loss": 1981.618032380352, "timestamp": 1719326489, "checkpoint_dir_name": "checkpoint_000000", "should_checkpoint": true, "done": false, "training_iteration": 1, "trial_id": "651349c4", "date": "2024-06-25_16-41-29", "time_this_iter_s": 86.11733651161194, "time_total_s": 86.11733651161194, "pid": 69318, "hostname": "r3i5n6", "node_ip": "10.159.8.159", "config": {"encoder_num_layer": 8, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 8, "embedding_dim": 16, "encoder_ff": 2048, "decoder_rt_ff": 2048, "decoder_int_ff": 2048, "n_head": 4, "drop_rate": 0.6683907000004579, "lr": 0.0001175178193869245, "batch_size": 2048}, "time_since_restore": 86.11733651161194, "iterations_since_restore": 1}
diff --git a/ray_results/test_experiment/train_model_6743cdee_21_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=4,drop__2024-06-25_18-48-59/events.out.tfevents.1719334380.r8i6n2 b/ray_results/test_experiment/train_model_6743cdee_21_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=4,drop__2024-06-25_18-48-59/events.out.tfevents.1719334380.r8i6n2
new file mode 100644
index 0000000000000000000000000000000000000000..1561fef9a4f2ae211498717f45c884da72657154
Binary files /dev/null and b/ray_results/test_experiment/train_model_6743cdee_21_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=4,drop__2024-06-25_18-48-59/events.out.tfevents.1719334380.r8i6n2 differ
diff --git a/ray_results/test_experiment/train_model_6743cdee_21_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=4,drop__2024-06-25_18-48-59/params.json b/ray_results/test_experiment/train_model_6743cdee_21_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=4,drop__2024-06-25_18-48-59/params.json
new file mode 100644
index 0000000000000000000000000000000000000000..75a5d4a692897b6ec7ff68d740a7e9cd8a89d1b2
--- /dev/null
+++ b/ray_results/test_experiment/train_model_6743cdee_21_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=4,drop__2024-06-25_18-48-59/params.json
@@ -0,0 +1,13 @@
+{
+ "batch_size": 2048,
+ "decoder_int_ff": 512,
+ "decoder_int_num_layer": 1,
+ "decoder_rt_ff": 2048,
+ "decoder_rt_num_layer": 4,
+ "drop_rate": 0.4436137502039823,
+ "embedding_dim": 256,
+ "encoder_ff": 512,
+ "encoder_num_layer": 2,
+ "lr": 0.07179226263764402,
+ "n_head": 8
+}
\ No newline at end of file
diff --git a/ray_results/test_experiment/train_model_6743cdee_21_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=4,drop__2024-06-25_18-48-59/params.pkl b/ray_results/test_experiment/train_model_6743cdee_21_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=4,drop__2024-06-25_18-48-59/params.pkl
new file mode 100644
index 0000000000000000000000000000000000000000..a1a7d11f1f3c5afd5ac4eb6ac8174ddc4a8fdef4
Binary files /dev/null and b/ray_results/test_experiment/train_model_6743cdee_21_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=4,drop__2024-06-25_18-48-59/params.pkl differ
diff --git a/ray_results/test_experiment/train_model_6743cdee_21_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=4,drop__2024-06-25_18-48-59/progress.csv b/ray_results/test_experiment/train_model_6743cdee_21_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=4,drop__2024-06-25_18-48-59/progress.csv
new file mode 100644
index 0000000000000000000000000000000000000000..31ceed3ce98cf1e92d50227b0096963e4d7cc54c
--- /dev/null
+++ b/ray_results/test_experiment/train_model_6743cdee_21_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=4,drop__2024-06-25_18-48-59/progress.csv
@@ -0,0 +1,3 @@
+loss,timestamp,checkpoint_dir_name,should_checkpoint,done,training_iteration,trial_id,date,time_this_iter_s,time_total_s,pid,hostname,node_ip,time_since_restore,iterations_since_restore
+3811.964426596334,1719334521,checkpoint_000000,True,False,1,6743cdee,2024-06-25_18-55-21,141.20554637908936,141.20554637908936,2195253,r8i6n2,10.159.28.60,141.20554637908936,1
+2680.254179226132,1719334648,checkpoint_000001,True,False,2,6743cdee,2024-06-25_18-57-29,127.1407482624054,268.34629464149475,2195253,r8i6n2,10.159.28.60,268.34629464149475,2
diff --git a/ray_results/test_experiment/train_model_6743cdee_21_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=4,drop__2024-06-25_18-48-59/result.json b/ray_results/test_experiment/train_model_6743cdee_21_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=4,drop__2024-06-25_18-48-59/result.json
new file mode 100644
index 0000000000000000000000000000000000000000..c0df350d50219031464da4afb0d32252b77e8f62
--- /dev/null
+++ b/ray_results/test_experiment/train_model_6743cdee_21_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=4,drop__2024-06-25_18-48-59/result.json
@@ -0,0 +1,2 @@
+{"loss": 3811.964426596334, "timestamp": 1719334521, "checkpoint_dir_name": "checkpoint_000000", "should_checkpoint": true, "done": false, "training_iteration": 1, "trial_id": "6743cdee", "date": "2024-06-25_18-55-21", "time_this_iter_s": 141.20554637908936, "time_total_s": 141.20554637908936, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 2, "decoder_rt_num_layer": 4, "decoder_int_num_layer": 1, "embedding_dim": 256, "encoder_ff": 512, "decoder_rt_ff": 2048, "decoder_int_ff": 512, "n_head": 8, "drop_rate": 0.4436137502039823, "lr": 0.07179226263764402, "batch_size": 2048}, "time_since_restore": 141.20554637908936, "iterations_since_restore": 1}
+{"loss": 2680.254179226132, "timestamp": 1719334648, "checkpoint_dir_name": "checkpoint_000001", "should_checkpoint": true, "done": false, "training_iteration": 2, "trial_id": "6743cdee", "date": "2024-06-25_18-57-29", "time_this_iter_s": 127.1407482624054, "time_total_s": 268.34629464149475, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 2, "decoder_rt_num_layer": 4, "decoder_int_num_layer": 1, "embedding_dim": 256, "encoder_ff": 512, "decoder_rt_ff": 2048, "decoder_int_ff": 512, "n_head": 8, "drop_rate": 0.4436137502039823, "lr": 0.07179226263764402, "batch_size": 2048}, "time_since_restore": 268.34629464149475, "iterations_since_restore": 2}
diff --git a/ray_results/test_experiment/train_model_6a62b85a_3_batch_size=2048,decoder_int_ff=1024,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop__2024-06-25_16-09-34/events.out.tfevents.1719324663.r3i5n6 b/ray_results/test_experiment/train_model_6a62b85a_3_batch_size=2048,decoder_int_ff=1024,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop__2024-06-25_16-09-34/events.out.tfevents.1719324663.r3i5n6
new file mode 100644
index 0000000000000000000000000000000000000000..7c047983b9f9067296085771161c520497f33864
Binary files /dev/null and b/ray_results/test_experiment/train_model_6a62b85a_3_batch_size=2048,decoder_int_ff=1024,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop__2024-06-25_16-09-34/events.out.tfevents.1719324663.r3i5n6 differ
diff --git a/ray_results/test_experiment/train_model_6a62b85a_3_batch_size=2048,decoder_int_ff=1024,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop__2024-06-25_16-09-34/events.out.tfevents.1719327182.r3i5n6 b/ray_results/test_experiment/train_model_6a62b85a_3_batch_size=2048,decoder_int_ff=1024,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop__2024-06-25_16-09-34/events.out.tfevents.1719327182.r3i5n6
new file mode 100644
index 0000000000000000000000000000000000000000..8c1b7f32f44d3b52e1917d5eda8319a27b1190a7
Binary files /dev/null and b/ray_results/test_experiment/train_model_6a62b85a_3_batch_size=2048,decoder_int_ff=1024,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop__2024-06-25_16-09-34/events.out.tfevents.1719327182.r3i5n6 differ
diff --git a/ray_results/test_experiment/train_model_6a62b85a_3_batch_size=2048,decoder_int_ff=1024,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop__2024-06-25_16-09-34/params.json b/ray_results/test_experiment/train_model_6a62b85a_3_batch_size=2048,decoder_int_ff=1024,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop__2024-06-25_16-09-34/params.json
new file mode 100644
index 0000000000000000000000000000000000000000..8c61cf16f82dbc7b3cfcffae8616594cc3fa7367
--- /dev/null
+++ b/ray_results/test_experiment/train_model_6a62b85a_3_batch_size=2048,decoder_int_ff=1024,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop__2024-06-25_16-09-34/params.json
@@ -0,0 +1,13 @@
+{
+ "batch_size": 2048,
+ "decoder_int_ff": 1024,
+ "decoder_int_num_layer": 1,
+ "decoder_rt_ff": 1024,
+ "decoder_rt_num_layer": 2,
+ "drop_rate": 0.06860896590384413,
+ "embedding_dim": 16,
+ "encoder_ff": 512,
+ "encoder_num_layer": 1,
+ "lr": 0.00016561766448063182,
+ "n_head": 4
+}
\ No newline at end of file
diff --git a/ray_results/test_experiment/train_model_6a62b85a_3_batch_size=2048,decoder_int_ff=1024,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop__2024-06-25_16-09-34/params.pkl b/ray_results/test_experiment/train_model_6a62b85a_3_batch_size=2048,decoder_int_ff=1024,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop__2024-06-25_16-09-34/params.pkl
new file mode 100644
index 0000000000000000000000000000000000000000..56b64ee286f614698ef2640771033fd2c68c1269
Binary files /dev/null and b/ray_results/test_experiment/train_model_6a62b85a_3_batch_size=2048,decoder_int_ff=1024,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop__2024-06-25_16-09-34/params.pkl differ
diff --git a/ray_results/test_experiment/train_model_6a62b85a_3_batch_size=2048,decoder_int_ff=1024,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop__2024-06-25_16-09-34/progress.csv b/ray_results/test_experiment/train_model_6a62b85a_3_batch_size=2048,decoder_int_ff=1024,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop__2024-06-25_16-09-34/progress.csv
new file mode 100644
index 0000000000000000000000000000000000000000..f1fef35aaec7bbdebda45fbe6120b056c06f3fd2
--- /dev/null
+++ b/ray_results/test_experiment/train_model_6a62b85a_3_batch_size=2048,decoder_int_ff=1024,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop__2024-06-25_16-09-34/progress.csv
@@ -0,0 +1,3 @@
+loss,timestamp,checkpoint_dir_name,should_checkpoint,done,training_iteration,trial_id,date,time_this_iter_s,time_total_s,pid,hostname,node_ip,time_since_restore,iterations_since_restore
+510.42659056656004,1719324728,checkpoint_000000,True,False,1,6a62b85a,2024-06-25_16-12-08,64.32230615615845,64.32230615615845,69318,r3i5n6,10.159.8.159,64.32230615615845,1
+310.75900604968933,1719327245,checkpoint_000001,True,False,2,6a62b85a,2024-06-25_16-54-05,63.18879699707031,127.51110315322876,130020,r3i5n6,10.159.8.159,63.18879699707031,1
diff --git a/ray_results/test_experiment/train_model_6a62b85a_3_batch_size=2048,decoder_int_ff=1024,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop__2024-06-25_16-09-34/result.json b/ray_results/test_experiment/train_model_6a62b85a_3_batch_size=2048,decoder_int_ff=1024,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop__2024-06-25_16-09-34/result.json
new file mode 100644
index 0000000000000000000000000000000000000000..c222fa4d6ae5fb24bb0157e1143ae6511d728f96
--- /dev/null
+++ b/ray_results/test_experiment/train_model_6a62b85a_3_batch_size=2048,decoder_int_ff=1024,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop__2024-06-25_16-09-34/result.json
@@ -0,0 +1,2 @@
+{"loss": 510.42659056656004, "timestamp": 1719324728, "checkpoint_dir_name": "checkpoint_000000", "should_checkpoint": true, "done": false, "training_iteration": 1, "trial_id": "6a62b85a", "date": "2024-06-25_16-12-08", "time_this_iter_s": 64.32230615615845, "time_total_s": 64.32230615615845, "pid": 69318, "hostname": "r3i5n6", "node_ip": "10.159.8.159", "config": {"encoder_num_layer": 1, "decoder_rt_num_layer": 2, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 512, "decoder_rt_ff": 1024, "decoder_int_ff": 1024, "n_head": 4, "drop_rate": 0.06860896590384413, "lr": 0.00016561766448063182, "batch_size": 2048}, "time_since_restore": 64.32230615615845, "iterations_since_restore": 1}
+{"loss": 310.75900604968933, "timestamp": 1719327245, "checkpoint_dir_name": "checkpoint_000001", "should_checkpoint": true, "done": false, "training_iteration": 2, "trial_id": "6a62b85a", "date": "2024-06-25_16-54-05", "time_this_iter_s": 63.18879699707031, "time_total_s": 127.51110315322876, "pid": 130020, "hostname": "r3i5n6", "node_ip": "10.159.8.159", "config": {"encoder_num_layer": 1, "decoder_rt_num_layer": 2, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 512, "decoder_rt_ff": 1024, "decoder_int_ff": 1024, "n_head": 4, "drop_rate": 0.06860896590384413, "lr": 0.00016561766448063182, "batch_size": 2048}, "time_since_restore": 63.18879699707031, "iterations_since_restore": 1}
diff --git a/ray_results/test_experiment/train_model_6be79717_24_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop__2024-06-25_19-14-34/events.out.tfevents.1719335783.r8i6n2 b/ray_results/test_experiment/train_model_6be79717_24_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop__2024-06-25_19-14-34/events.out.tfevents.1719335783.r8i6n2
new file mode 100644
index 0000000000000000000000000000000000000000..c4aacef1d7de666f23944a20c1ba73cc59c14404
Binary files /dev/null and b/ray_results/test_experiment/train_model_6be79717_24_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop__2024-06-25_19-14-34/events.out.tfevents.1719335783.r8i6n2 differ
diff --git a/ray_results/test_experiment/train_model_6be79717_24_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop__2024-06-25_19-14-34/params.json b/ray_results/test_experiment/train_model_6be79717_24_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop__2024-06-25_19-14-34/params.json
new file mode 100644
index 0000000000000000000000000000000000000000..a573c350fed5501f3c0065983d42a68af7f0bdab
--- /dev/null
+++ b/ray_results/test_experiment/train_model_6be79717_24_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop__2024-06-25_19-14-34/params.json
@@ -0,0 +1,13 @@
+{
+ "batch_size": 1024,
+ "decoder_int_ff": 512,
+ "decoder_int_num_layer": 1,
+ "decoder_rt_ff": 1024,
+ "decoder_rt_num_layer": 2,
+ "drop_rate": 0.7446531774853204,
+ "embedding_dim": 64,
+ "encoder_ff": 2048,
+ "encoder_num_layer": 1,
+ "lr": 0.006000257122673396,
+ "n_head": 8
+}
\ No newline at end of file
diff --git a/ray_results/test_experiment/train_model_6be79717_24_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop__2024-06-25_19-14-34/params.pkl b/ray_results/test_experiment/train_model_6be79717_24_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop__2024-06-25_19-14-34/params.pkl
new file mode 100644
index 0000000000000000000000000000000000000000..aad1abecc4a67ea997619c3837e518a0aae66cd4
Binary files /dev/null and b/ray_results/test_experiment/train_model_6be79717_24_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop__2024-06-25_19-14-34/params.pkl differ
diff --git a/ray_results/test_experiment/train_model_6be79717_24_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop__2024-06-25_19-14-34/progress.csv b/ray_results/test_experiment/train_model_6be79717_24_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop__2024-06-25_19-14-34/progress.csv
new file mode 100644
index 0000000000000000000000000000000000000000..555eff0f4cab5e30bff5850400f11683011eb2f4
--- /dev/null
+++ b/ray_results/test_experiment/train_model_6be79717_24_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop__2024-06-25_19-14-34/progress.csv
@@ -0,0 +1,3 @@
+loss,timestamp,checkpoint_dir_name,should_checkpoint,done,training_iteration,trial_id,date,time_this_iter_s,time_total_s,pid,hostname,node_ip,time_since_restore,iterations_since_restore
+2020.6805025836613,1719335855,checkpoint_000000,True,False,1,6be79717,2024-06-25_19-17-35,71.47419285774231,71.47419285774231,2195253,r8i6n2,10.159.28.60,71.47419285774231,1
+2003.3902054433747,1719335912,checkpoint_000001,True,False,2,6be79717,2024-06-25_19-18-32,57.25079274177551,128.72498559951782,2195253,r8i6n2,10.159.28.60,128.72498559951782,2
diff --git a/ray_results/test_experiment/train_model_6be79717_24_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop__2024-06-25_19-14-34/result.json b/ray_results/test_experiment/train_model_6be79717_24_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop__2024-06-25_19-14-34/result.json
new file mode 100644
index 0000000000000000000000000000000000000000..a6ff692e4aa862f70436d11d62946c69249995f1
--- /dev/null
+++ b/ray_results/test_experiment/train_model_6be79717_24_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop__2024-06-25_19-14-34/result.json
@@ -0,0 +1,2 @@
+{"loss": 2020.6805025836613, "timestamp": 1719335855, "checkpoint_dir_name": "checkpoint_000000", "should_checkpoint": true, "done": false, "training_iteration": 1, "trial_id": "6be79717", "date": "2024-06-25_19-17-35", "time_this_iter_s": 71.47419285774231, "time_total_s": 71.47419285774231, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 1, "decoder_rt_num_layer": 2, "decoder_int_num_layer": 1, "embedding_dim": 64, "encoder_ff": 2048, "decoder_rt_ff": 1024, "decoder_int_ff": 512, "n_head": 8, "drop_rate": 0.7446531774853204, "lr": 0.006000257122673396, "batch_size": 1024}, "time_since_restore": 71.47419285774231, "iterations_since_restore": 1}
+{"loss": 2003.3902054433747, "timestamp": 1719335912, "checkpoint_dir_name": "checkpoint_000001", "should_checkpoint": true, "done": false, "training_iteration": 2, "trial_id": "6be79717", "date": "2024-06-25_19-18-32", "time_this_iter_s": 57.25079274177551, "time_total_s": 128.72498559951782, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 1, "decoder_rt_num_layer": 2, "decoder_int_num_layer": 1, "embedding_dim": 64, "encoder_ff": 2048, "decoder_rt_ff": 1024, "decoder_int_ff": 512, "n_head": 8, "drop_rate": 0.7446531774853204, "lr": 0.006000257122673396, "batch_size": 1024}, "time_since_restore": 128.72498559951782, "iterations_since_restore": 2}
diff --git a/ray_results/test_experiment/train_model_6c8b476a_20_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=4,drop_r_2024-06-25_18-43-49/events.out.tfevents.1719334139.r8i6n2 b/ray_results/test_experiment/train_model_6c8b476a_20_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=4,drop_r_2024-06-25_18-43-49/events.out.tfevents.1719334139.r8i6n2
new file mode 100644
index 0000000000000000000000000000000000000000..e229310bfa517cb8926df9f2af4925df493548a9
Binary files /dev/null and b/ray_results/test_experiment/train_model_6c8b476a_20_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=4,drop_r_2024-06-25_18-43-49/events.out.tfevents.1719334139.r8i6n2 differ
diff --git a/ray_results/test_experiment/train_model_6c8b476a_20_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=4,drop_r_2024-06-25_18-43-49/events.out.tfevents.1719338498.r8i6n2 b/ray_results/test_experiment/train_model_6c8b476a_20_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=4,drop_r_2024-06-25_18-43-49/events.out.tfevents.1719338498.r8i6n2
new file mode 100644
index 0000000000000000000000000000000000000000..b9c37540f6834b97b3aa22e56e87c3716256be97
Binary files /dev/null and b/ray_results/test_experiment/train_model_6c8b476a_20_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=4,drop_r_2024-06-25_18-43-49/events.out.tfevents.1719338498.r8i6n2 differ
diff --git a/ray_results/test_experiment/train_model_6c8b476a_20_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=4,drop_r_2024-06-25_18-43-49/params.json b/ray_results/test_experiment/train_model_6c8b476a_20_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=4,drop_r_2024-06-25_18-43-49/params.json
new file mode 100644
index 0000000000000000000000000000000000000000..cdc8e6a645304f1cbdc25c7c6cd3849c9e53eaac
--- /dev/null
+++ b/ray_results/test_experiment/train_model_6c8b476a_20_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=4,drop_r_2024-06-25_18-43-49/params.json
@@ -0,0 +1,13 @@
+{
+ "batch_size": 1024,
+ "decoder_int_ff": 512,
+ "decoder_int_num_layer": 1,
+ "decoder_rt_ff": 512,
+ "decoder_rt_num_layer": 4,
+ "drop_rate": 0.5088357248512263,
+ "embedding_dim": 64,
+ "encoder_ff": 1024,
+ "encoder_num_layer": 8,
+ "lr": 0.010524289583233754,
+ "n_head": 16
+}
\ No newline at end of file
diff --git a/ray_results/test_experiment/train_model_6c8b476a_20_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=4,drop_r_2024-06-25_18-43-49/params.pkl b/ray_results/test_experiment/train_model_6c8b476a_20_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=4,drop_r_2024-06-25_18-43-49/params.pkl
new file mode 100644
index 0000000000000000000000000000000000000000..25e06ac30468867cc36b1fb368e21f3aff3e8c36
Binary files /dev/null and b/ray_results/test_experiment/train_model_6c8b476a_20_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=4,drop_r_2024-06-25_18-43-49/params.pkl differ
diff --git a/ray_results/test_experiment/train_model_6c8b476a_20_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=4,drop_r_2024-06-25_18-43-49/progress.csv b/ray_results/test_experiment/train_model_6c8b476a_20_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=4,drop_r_2024-06-25_18-43-49/progress.csv
new file mode 100644
index 0000000000000000000000000000000000000000..4ba5ff39b35c970c2e6e4e5ede2bff78e2dc5187
--- /dev/null
+++ b/ray_results/test_experiment/train_model_6c8b476a_20_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=4,drop_r_2024-06-25_18-43-49/progress.csv
@@ -0,0 +1,5 @@
+loss,timestamp,checkpoint_dir_name,should_checkpoint,done,training_iteration,trial_id,date,time_this_iter_s,time_total_s,pid,hostname,node_ip,time_since_restore,iterations_since_restore
+2006.8072533795214,1719334267,checkpoint_000000,True,False,1,6c8b476a,2024-06-25_18-51-07,127.51717782020569,127.51717782020569,2195253,r8i6n2,10.159.28.60,127.51717782020569,1
+1995.246540219765,1719334380,checkpoint_000001,True,False,2,6c8b476a,2024-06-25_18-53-00,113.46883034706116,240.98600816726685,2195253,r8i6n2,10.159.28.60,240.98600816726685,2
+1996.2762984628753,1719338625,checkpoint_000002,True,False,3,6c8b476a,2024-06-25_20-03-45,126.75374698638916,367.739755153656,2195253,r8i6n2,10.159.28.60,126.75374698638916,1
+1991.5076505405696,1719338738,checkpoint_000003,True,False,4,6c8b476a,2024-06-25_20-05-38,113.23560690879822,480.9753620624542,2195253,r8i6n2,10.159.28.60,239.98935389518738,2
diff --git a/ray_results/test_experiment/train_model_6c8b476a_20_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=4,drop_r_2024-06-25_18-43-49/result.json b/ray_results/test_experiment/train_model_6c8b476a_20_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=4,drop_r_2024-06-25_18-43-49/result.json
new file mode 100644
index 0000000000000000000000000000000000000000..9a54df5811c4202c5183bcd155109ce9f1e90d97
--- /dev/null
+++ b/ray_results/test_experiment/train_model_6c8b476a_20_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=4,drop_r_2024-06-25_18-43-49/result.json
@@ -0,0 +1,4 @@
+{"loss": 2006.8072533795214, "timestamp": 1719334267, "checkpoint_dir_name": "checkpoint_000000", "should_checkpoint": true, "done": false, "training_iteration": 1, "trial_id": "6c8b476a", "date": "2024-06-25_18-51-07", "time_this_iter_s": 127.51717782020569, "time_total_s": 127.51717782020569, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 8, "decoder_rt_num_layer": 4, "decoder_int_num_layer": 1, "embedding_dim": 64, "encoder_ff": 1024, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 16, "drop_rate": 0.5088357248512263, "lr": 0.010524289583233754, "batch_size": 1024}, "time_since_restore": 127.51717782020569, "iterations_since_restore": 1}
+{"loss": 1995.246540219765, "timestamp": 1719334380, "checkpoint_dir_name": "checkpoint_000001", "should_checkpoint": true, "done": false, "training_iteration": 2, "trial_id": "6c8b476a", "date": "2024-06-25_18-53-00", "time_this_iter_s": 113.46883034706116, "time_total_s": 240.98600816726685, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 8, "decoder_rt_num_layer": 4, "decoder_int_num_layer": 1, "embedding_dim": 64, "encoder_ff": 1024, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 16, "drop_rate": 0.5088357248512263, "lr": 0.010524289583233754, "batch_size": 1024}, "time_since_restore": 240.98600816726685, "iterations_since_restore": 2}
+{"loss": 1996.2762984628753, "timestamp": 1719338625, "checkpoint_dir_name": "checkpoint_000002", "should_checkpoint": true, "done": false, "training_iteration": 3, "trial_id": "6c8b476a", "date": "2024-06-25_20-03-45", "time_this_iter_s": 126.75374698638916, "time_total_s": 367.739755153656, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 8, "decoder_rt_num_layer": 4, "decoder_int_num_layer": 1, "embedding_dim": 64, "encoder_ff": 1024, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 16, "drop_rate": 0.5088357248512263, "lr": 0.010524289583233754, "batch_size": 1024}, "time_since_restore": 126.75374698638916, "iterations_since_restore": 1}
+{"loss": 1991.5076505405696, "timestamp": 1719338738, "checkpoint_dir_name": "checkpoint_000003", "should_checkpoint": true, "done": false, "training_iteration": 4, "trial_id": "6c8b476a", "date": "2024-06-25_20-05-38", "time_this_iter_s": 113.23560690879822, "time_total_s": 480.9753620624542, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 8, "decoder_rt_num_layer": 4, "decoder_int_num_layer": 1, "embedding_dim": 64, "encoder_ff": 1024, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 16, "drop_rate": 0.5088357248512263, "lr": 0.010524289583233754, "batch_size": 1024}, "time_since_restore": 239.98935389518738, "iterations_since_restore": 2}
diff --git a/ray_results/test_experiment/train_model_7ddcd025_3_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=8,drop_ra_2024-06-25_17-14-16/events.out.tfevents.1719328526.r8i6n2 b/ray_results/test_experiment/train_model_7ddcd025_3_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=8,drop_ra_2024-06-25_17-14-16/events.out.tfevents.1719328526.r8i6n2
new file mode 100644
index 0000000000000000000000000000000000000000..9bac5693357821843ba482583e0c8d0988a565f3
Binary files /dev/null and b/ray_results/test_experiment/train_model_7ddcd025_3_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=8,drop_ra_2024-06-25_17-14-16/events.out.tfevents.1719328526.r8i6n2 differ
diff --git a/ray_results/test_experiment/train_model_7ddcd025_3_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=8,drop_ra_2024-06-25_17-14-16/params.json b/ray_results/test_experiment/train_model_7ddcd025_3_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=8,drop_ra_2024-06-25_17-14-16/params.json
new file mode 100644
index 0000000000000000000000000000000000000000..ae356ec5704e9b16fe2fc50959a5c3a5f393c87b
--- /dev/null
+++ b/ray_results/test_experiment/train_model_7ddcd025_3_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=8,drop_ra_2024-06-25_17-14-16/params.json
@@ -0,0 +1,13 @@
+{
+ "batch_size": 1024,
+ "decoder_int_ff": 512,
+ "decoder_int_num_layer": 1,
+ "decoder_rt_ff": 512,
+ "decoder_rt_num_layer": 8,
+ "drop_rate": 0.474834603771515,
+ "embedding_dim": 64,
+ "encoder_ff": 2048,
+ "encoder_num_layer": 2,
+ "lr": 0.015369204694575145,
+ "n_head": 1
+}
\ No newline at end of file
diff --git a/ray_results/test_experiment/train_model_7ddcd025_3_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=8,drop_ra_2024-06-25_17-14-16/params.pkl b/ray_results/test_experiment/train_model_7ddcd025_3_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=8,drop_ra_2024-06-25_17-14-16/params.pkl
new file mode 100644
index 0000000000000000000000000000000000000000..f7324d1996709089de50c6898ab7840b47452eee
Binary files /dev/null and b/ray_results/test_experiment/train_model_7ddcd025_3_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=8,drop_ra_2024-06-25_17-14-16/params.pkl differ
diff --git a/ray_results/test_experiment/train_model_7ddcd025_3_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=8,drop_ra_2024-06-25_17-14-16/progress.csv b/ray_results/test_experiment/train_model_7ddcd025_3_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=8,drop_ra_2024-06-25_17-14-16/progress.csv
new file mode 100644
index 0000000000000000000000000000000000000000..d5f9bcf7eaa42ca457dd3dd8c963060707c98ba7
--- /dev/null
+++ b/ray_results/test_experiment/train_model_7ddcd025_3_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=8,drop_ra_2024-06-25_17-14-16/progress.csv
@@ -0,0 +1,2 @@
+loss,timestamp,checkpoint_dir_name,should_checkpoint,done,training_iteration,trial_id,date,time_this_iter_s,time_total_s,pid,hostname,node_ip,time_since_restore,iterations_since_restore
+2079.0407700425994,1719328617,checkpoint_000000,True,False,1,7ddcd025,2024-06-25_17-16-57,90.59219408035278,90.59219408035278,2195253,r8i6n2,10.159.28.60,90.59219408035278,1
diff --git a/ray_results/test_experiment/train_model_7ddcd025_3_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=8,drop_ra_2024-06-25_17-14-16/result.json b/ray_results/test_experiment/train_model_7ddcd025_3_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=8,drop_ra_2024-06-25_17-14-16/result.json
new file mode 100644
index 0000000000000000000000000000000000000000..5c795bae9dce5d374e94b6df1ee4d1629abb226c
--- /dev/null
+++ b/ray_results/test_experiment/train_model_7ddcd025_3_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=8,drop_ra_2024-06-25_17-14-16/result.json
@@ -0,0 +1 @@
+{"loss": 2079.0407700425994, "timestamp": 1719328617, "checkpoint_dir_name": "checkpoint_000000", "should_checkpoint": true, "done": false, "training_iteration": 1, "trial_id": "7ddcd025", "date": "2024-06-25_17-16-57", "time_this_iter_s": 90.59219408035278, "time_total_s": 90.59219408035278, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 2, "decoder_rt_num_layer": 8, "decoder_int_num_layer": 1, "embedding_dim": 64, "encoder_ff": 2048, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 1, "drop_rate": 0.474834603771515, "lr": 0.015369204694575145, "batch_size": 1024}, "time_since_restore": 90.59219408035278, "iterations_since_restore": 1}
diff --git a/ray_results/test_experiment/train_model_7dfe67c7_4_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=2,drop_r_2024-06-25_17-15-26/events.out.tfevents.1719328617.r8i6n2 b/ray_results/test_experiment/train_model_7dfe67c7_4_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=2,drop_r_2024-06-25_17-15-26/events.out.tfevents.1719328617.r8i6n2
new file mode 100644
index 0000000000000000000000000000000000000000..7367e226129739c3987e7bb20d4b3e14b1e03ed1
Binary files /dev/null and b/ray_results/test_experiment/train_model_7dfe67c7_4_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=2,drop_r_2024-06-25_17-15-26/events.out.tfevents.1719328617.r8i6n2 differ
diff --git a/ray_results/test_experiment/train_model_7dfe67c7_4_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=2,drop_r_2024-06-25_17-15-26/params.json b/ray_results/test_experiment/train_model_7dfe67c7_4_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=2,drop_r_2024-06-25_17-15-26/params.json
new file mode 100644
index 0000000000000000000000000000000000000000..adf6e7ca5d1b3f49b2d6c1d55cc8efa19a47a77d
--- /dev/null
+++ b/ray_results/test_experiment/train_model_7dfe67c7_4_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=2,drop_r_2024-06-25_17-15-26/params.json
@@ -0,0 +1,13 @@
+{
+ "batch_size": 1024,
+ "decoder_int_ff": 512,
+ "decoder_int_num_layer": 1,
+ "decoder_rt_ff": 2048,
+ "decoder_rt_num_layer": 2,
+ "drop_rate": 0.6066107852445892,
+ "embedding_dim": 256,
+ "encoder_ff": 2048,
+ "encoder_num_layer": 2,
+ "lr": 0.0011313709304158431,
+ "n_head": 2
+}
\ No newline at end of file
diff --git a/ray_results/test_experiment/train_model_7dfe67c7_4_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=2,drop_r_2024-06-25_17-15-26/params.pkl b/ray_results/test_experiment/train_model_7dfe67c7_4_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=2,drop_r_2024-06-25_17-15-26/params.pkl
new file mode 100644
index 0000000000000000000000000000000000000000..04ed4ecc4f6a23dc67e03a1661d603cfd8caa48c
Binary files /dev/null and b/ray_results/test_experiment/train_model_7dfe67c7_4_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=2,drop_r_2024-06-25_17-15-26/params.pkl differ
diff --git a/ray_results/test_experiment/train_model_7dfe67c7_4_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=2,drop_r_2024-06-25_17-15-26/progress.csv b/ray_results/test_experiment/train_model_7dfe67c7_4_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=2,drop_r_2024-06-25_17-15-26/progress.csv
new file mode 100644
index 0000000000000000000000000000000000000000..dc8c1f95e1ac345fae8765bf79bb3df464776932
--- /dev/null
+++ b/ray_results/test_experiment/train_model_7dfe67c7_4_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=2,drop_r_2024-06-25_17-15-26/progress.csv
@@ -0,0 +1,2 @@
+loss,timestamp,checkpoint_dir_name,should_checkpoint,done,training_iteration,trial_id,date,time_this_iter_s,time_total_s,pid,hostname,node_ip,time_since_restore,iterations_since_restore
+1979.0773276982345,1719328732,checkpoint_000000,True,False,1,7dfe67c7,2024-06-25_17-18-52,115.28918504714966,115.28918504714966,2195253,r8i6n2,10.159.28.60,115.28918504714966,1
diff --git a/ray_results/test_experiment/train_model_7dfe67c7_4_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=2,drop_r_2024-06-25_17-15-26/result.json b/ray_results/test_experiment/train_model_7dfe67c7_4_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=2,drop_r_2024-06-25_17-15-26/result.json
new file mode 100644
index 0000000000000000000000000000000000000000..a11df1a6f9ad4dc4eb14bd319e717eed60db5336
--- /dev/null
+++ b/ray_results/test_experiment/train_model_7dfe67c7_4_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=2,drop_r_2024-06-25_17-15-26/result.json
@@ -0,0 +1 @@
+{"loss": 1979.0773276982345, "timestamp": 1719328732, "checkpoint_dir_name": "checkpoint_000000", "should_checkpoint": true, "done": false, "training_iteration": 1, "trial_id": "7dfe67c7", "date": "2024-06-25_17-18-52", "time_this_iter_s": 115.28918504714966, "time_total_s": 115.28918504714966, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 2, "decoder_rt_num_layer": 2, "decoder_int_num_layer": 1, "embedding_dim": 256, "encoder_ff": 2048, "decoder_rt_ff": 2048, "decoder_int_ff": 512, "n_head": 2, "drop_rate": 0.6066107852445892, "lr": 0.0011313709304158431, "batch_size": 1024}, "time_since_restore": 115.28918504714966, "iterations_since_restore": 1}
diff --git a/ray_results/test_experiment/train_model_82396cf3_25_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_19-16-23/events.out.tfevents.1719335912.r8i6n2 b/ray_results/test_experiment/train_model_82396cf3_25_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_19-16-23/events.out.tfevents.1719335912.r8i6n2
new file mode 100644
index 0000000000000000000000000000000000000000..026fe5133f8eec3c18e3c3ede008ba87963143ff
Binary files /dev/null and b/ray_results/test_experiment/train_model_82396cf3_25_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_19-16-23/events.out.tfevents.1719335912.r8i6n2 differ
diff --git a/ray_results/test_experiment/train_model_82396cf3_25_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_19-16-23/params.json b/ray_results/test_experiment/train_model_82396cf3_25_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_19-16-23/params.json
new file mode 100644
index 0000000000000000000000000000000000000000..3afc3a3232142476d14869fc210d51b0c9adbf33
--- /dev/null
+++ b/ray_results/test_experiment/train_model_82396cf3_25_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_19-16-23/params.json
@@ -0,0 +1,13 @@
+{
+ "batch_size": 2048,
+ "decoder_int_ff": 512,
+ "decoder_int_num_layer": 1,
+ "decoder_rt_ff": 512,
+ "decoder_rt_num_layer": 2,
+ "drop_rate": 0.9507029678825294,
+ "embedding_dim": 1024,
+ "encoder_ff": 2048,
+ "encoder_num_layer": 4,
+ "lr": 0.00025487621898754583,
+ "n_head": 4
+}
\ No newline at end of file
diff --git a/ray_results/test_experiment/train_model_82396cf3_25_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_19-16-23/params.pkl b/ray_results/test_experiment/train_model_82396cf3_25_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_19-16-23/params.pkl
new file mode 100644
index 0000000000000000000000000000000000000000..5822f94cb1e2ebddcad8543851807b8549ac0c89
Binary files /dev/null and b/ray_results/test_experiment/train_model_82396cf3_25_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_19-16-23/params.pkl differ
diff --git a/ray_results/test_experiment/train_model_82396cf3_25_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_19-16-23/progress.csv b/ray_results/test_experiment/train_model_82396cf3_25_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_19-16-23/progress.csv
new file mode 100644
index 0000000000000000000000000000000000000000..1f3b6b453f7aa3f386923870ca7d7203035b36dc
--- /dev/null
+++ b/ray_results/test_experiment/train_model_82396cf3_25_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_19-16-23/progress.csv
@@ -0,0 +1,3 @@
+loss,timestamp,checkpoint_dir_name,should_checkpoint,done,training_iteration,trial_id,date,time_this_iter_s,time_total_s,pid,hostname,node_ip,time_since_restore,iterations_since_restore
+2178.1833553764764,1719336534,checkpoint_000000,True,False,1,82396cf3,2024-06-25_19-28-54,622.4429934024811,622.4429934024811,2195253,r8i6n2,10.159.28.60,622.4429934024811,1
+2187.193873608206,1719337142,checkpoint_000001,True,False,2,82396cf3,2024-06-25_19-39-03,608.1205008029938,1230.5634942054749,2195253,r8i6n2,10.159.28.60,1230.5634942054749,2
diff --git a/ray_results/test_experiment/train_model_82396cf3_25_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_19-16-23/result.json b/ray_results/test_experiment/train_model_82396cf3_25_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_19-16-23/result.json
new file mode 100644
index 0000000000000000000000000000000000000000..6aabf4ae683de46d9cdfadfe1bbfe5bcb96d079e
--- /dev/null
+++ b/ray_results/test_experiment/train_model_82396cf3_25_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_19-16-23/result.json
@@ -0,0 +1,2 @@
+{"loss": 2178.1833553764764, "timestamp": 1719336534, "checkpoint_dir_name": "checkpoint_000000", "should_checkpoint": true, "done": false, "training_iteration": 1, "trial_id": "82396cf3", "date": "2024-06-25_19-28-54", "time_this_iter_s": 622.4429934024811, "time_total_s": 622.4429934024811, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 4, "decoder_rt_num_layer": 2, "decoder_int_num_layer": 1, "embedding_dim": 1024, "encoder_ff": 2048, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 4, "drop_rate": 0.9507029678825294, "lr": 0.00025487621898754583, "batch_size": 2048}, "time_since_restore": 622.4429934024811, "iterations_since_restore": 1}
+{"loss": 2187.193873608206, "timestamp": 1719337142, "checkpoint_dir_name": "checkpoint_000001", "should_checkpoint": true, "done": false, "training_iteration": 2, "trial_id": "82396cf3", "date": "2024-06-25_19-39-03", "time_this_iter_s": 608.1205008029938, "time_total_s": 1230.5634942054749, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 4, "decoder_rt_num_layer": 2, "decoder_int_num_layer": 1, "embedding_dim": 1024, "encoder_ff": 2048, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 4, "drop_rate": 0.9507029678825294, "lr": 0.00025487621898754583, "batch_size": 2048}, "time_since_restore": 1230.5634942054749, "iterations_since_restore": 2}
diff --git a/ray_results/test_experiment/train_model_86ae258f_7_batch_size=2048,decoder_int_ff=1024,decoder_int_num_layer=4,decoder_rt_ff=1024,decoder_rt_num_layer=4,drop__2024-06-25_16-15-18/events.out.tfevents.1719325014.r3i5n6 b/ray_results/test_experiment/train_model_86ae258f_7_batch_size=2048,decoder_int_ff=1024,decoder_int_num_layer=4,decoder_rt_ff=1024,decoder_rt_num_layer=4,drop__2024-06-25_16-15-18/events.out.tfevents.1719325014.r3i5n6
new file mode 100644
index 0000000000000000000000000000000000000000..10b61213c97a5e881e015339b840ca4732ffdcd8
Binary files /dev/null and b/ray_results/test_experiment/train_model_86ae258f_7_batch_size=2048,decoder_int_ff=1024,decoder_int_num_layer=4,decoder_rt_ff=1024,decoder_rt_num_layer=4,drop__2024-06-25_16-15-18/events.out.tfevents.1719325014.r3i5n6 differ
diff --git a/ray_results/test_experiment/train_model_86ae258f_7_batch_size=2048,decoder_int_ff=1024,decoder_int_num_layer=4,decoder_rt_ff=1024,decoder_rt_num_layer=4,drop__2024-06-25_16-15-18/events.out.tfevents.1719327469.r3i5n6 b/ray_results/test_experiment/train_model_86ae258f_7_batch_size=2048,decoder_int_ff=1024,decoder_int_num_layer=4,decoder_rt_ff=1024,decoder_rt_num_layer=4,drop__2024-06-25_16-15-18/events.out.tfevents.1719327469.r3i5n6
new file mode 100644
index 0000000000000000000000000000000000000000..dd67d6d6f8550c83efa236fe944e13de4dfd5c06
Binary files /dev/null and b/ray_results/test_experiment/train_model_86ae258f_7_batch_size=2048,decoder_int_ff=1024,decoder_int_num_layer=4,decoder_rt_ff=1024,decoder_rt_num_layer=4,drop__2024-06-25_16-15-18/events.out.tfevents.1719327469.r3i5n6 differ
diff --git a/ray_results/test_experiment/train_model_86ae258f_7_batch_size=2048,decoder_int_ff=1024,decoder_int_num_layer=4,decoder_rt_ff=1024,decoder_rt_num_layer=4,drop__2024-06-25_16-15-18/params.json b/ray_results/test_experiment/train_model_86ae258f_7_batch_size=2048,decoder_int_ff=1024,decoder_int_num_layer=4,decoder_rt_ff=1024,decoder_rt_num_layer=4,drop__2024-06-25_16-15-18/params.json
new file mode 100644
index 0000000000000000000000000000000000000000..ee925cb745fa96d8c7a10dd76b86825dedca8f2c
--- /dev/null
+++ b/ray_results/test_experiment/train_model_86ae258f_7_batch_size=2048,decoder_int_ff=1024,decoder_int_num_layer=4,decoder_rt_ff=1024,decoder_rt_num_layer=4,drop__2024-06-25_16-15-18/params.json
@@ -0,0 +1,13 @@
+{
+ "batch_size": 2048,
+ "decoder_int_ff": 1024,
+ "decoder_int_num_layer": 4,
+ "decoder_rt_ff": 1024,
+ "decoder_rt_num_layer": 4,
+ "drop_rate": 0.7568171644612052,
+ "embedding_dim": 256,
+ "encoder_ff": 2048,
+ "encoder_num_layer": 2,
+ "lr": 0.0002629821336450532,
+ "n_head": 16
+}
\ No newline at end of file
diff --git a/ray_results/test_experiment/train_model_86ae258f_7_batch_size=2048,decoder_int_ff=1024,decoder_int_num_layer=4,decoder_rt_ff=1024,decoder_rt_num_layer=4,drop__2024-06-25_16-15-18/params.pkl b/ray_results/test_experiment/train_model_86ae258f_7_batch_size=2048,decoder_int_ff=1024,decoder_int_num_layer=4,decoder_rt_ff=1024,decoder_rt_num_layer=4,drop__2024-06-25_16-15-18/params.pkl
new file mode 100644
index 0000000000000000000000000000000000000000..289c7c0772501541dc96ebcea495ecc9910ae5c3
Binary files /dev/null and b/ray_results/test_experiment/train_model_86ae258f_7_batch_size=2048,decoder_int_ff=1024,decoder_int_num_layer=4,decoder_rt_ff=1024,decoder_rt_num_layer=4,drop__2024-06-25_16-15-18/params.pkl differ
diff --git a/ray_results/test_experiment/train_model_86ae258f_7_batch_size=2048,decoder_int_ff=1024,decoder_int_num_layer=4,decoder_rt_ff=1024,decoder_rt_num_layer=4,drop__2024-06-25_16-15-18/progress.csv b/ray_results/test_experiment/train_model_86ae258f_7_batch_size=2048,decoder_int_ff=1024,decoder_int_num_layer=4,decoder_rt_ff=1024,decoder_rt_num_layer=4,drop__2024-06-25_16-15-18/progress.csv
new file mode 100644
index 0000000000000000000000000000000000000000..68b7d899baa71da98ba7962e013b2f45a387a4fb
--- /dev/null
+++ b/ray_results/test_experiment/train_model_86ae258f_7_batch_size=2048,decoder_int_ff=1024,decoder_int_num_layer=4,decoder_rt_ff=1024,decoder_rt_num_layer=4,drop__2024-06-25_16-15-18/progress.csv
@@ -0,0 +1,3 @@
+loss,timestamp,checkpoint_dir_name,should_checkpoint,done,training_iteration,trial_id,date,time_this_iter_s,time_total_s,pid,hostname,node_ip,time_since_restore,iterations_since_restore
+1993.6307767132137,1719325156,checkpoint_000000,True,False,1,86ae258f,2024-06-25_16-19-16,141.57161211967468,141.57161211967468,69318,r3i5n6,10.159.8.159,141.57161211967468,1
+1995.102591927596,1719327610,checkpoint_000001,True,False,2,86ae258f,2024-06-25_17-00-10,141.5520534515381,283.12366557121277,130020,r3i5n6,10.159.8.159,141.5520534515381,1
diff --git a/ray_results/test_experiment/train_model_86ae258f_7_batch_size=2048,decoder_int_ff=1024,decoder_int_num_layer=4,decoder_rt_ff=1024,decoder_rt_num_layer=4,drop__2024-06-25_16-15-18/result.json b/ray_results/test_experiment/train_model_86ae258f_7_batch_size=2048,decoder_int_ff=1024,decoder_int_num_layer=4,decoder_rt_ff=1024,decoder_rt_num_layer=4,drop__2024-06-25_16-15-18/result.json
new file mode 100644
index 0000000000000000000000000000000000000000..1a450f4490a88ed6039da312734fc88a8e7bac4d
--- /dev/null
+++ b/ray_results/test_experiment/train_model_86ae258f_7_batch_size=2048,decoder_int_ff=1024,decoder_int_num_layer=4,decoder_rt_ff=1024,decoder_rt_num_layer=4,drop__2024-06-25_16-15-18/result.json
@@ -0,0 +1,2 @@
+{"loss": 1993.6307767132137, "timestamp": 1719325156, "checkpoint_dir_name": "checkpoint_000000", "should_checkpoint": true, "done": false, "training_iteration": 1, "trial_id": "86ae258f", "date": "2024-06-25_16-19-16", "time_this_iter_s": 141.57161211967468, "time_total_s": 141.57161211967468, "pid": 69318, "hostname": "r3i5n6", "node_ip": "10.159.8.159", "config": {"encoder_num_layer": 2, "decoder_rt_num_layer": 4, "decoder_int_num_layer": 4, "embedding_dim": 256, "encoder_ff": 2048, "decoder_rt_ff": 1024, "decoder_int_ff": 1024, "n_head": 16, "drop_rate": 0.7568171644612052, "lr": 0.0002629821336450532, "batch_size": 2048}, "time_since_restore": 141.57161211967468, "iterations_since_restore": 1}
+{"loss": 1995.102591927596, "timestamp": 1719327610, "checkpoint_dir_name": "checkpoint_000001", "should_checkpoint": true, "done": false, "training_iteration": 2, "trial_id": "86ae258f", "date": "2024-06-25_17-00-10", "time_this_iter_s": 141.5520534515381, "time_total_s": 283.12366557121277, "pid": 130020, "hostname": "r3i5n6", "node_ip": "10.159.8.159", "config": {"encoder_num_layer": 2, "decoder_rt_num_layer": 4, "decoder_int_num_layer": 4, "embedding_dim": 256, "encoder_ff": 2048, "decoder_rt_ff": 1024, "decoder_int_ff": 1024, "n_head": 16, "drop_rate": 0.7568171644612052, "lr": 0.0002629821336450532, "batch_size": 2048}, "time_since_restore": 141.5520534515381, "iterations_since_restore": 1}
diff --git a/ray_results/test_experiment/train_model_8c6e7040_14_batch_size=2048,decoder_int_ff=2048,decoder_int_num_layer=8,decoder_rt_ff=1024,decoder_rt_num_layer=8,drop_2024-06-25_16-41-29/error.pkl b/ray_results/test_experiment/train_model_8c6e7040_14_batch_size=2048,decoder_int_ff=2048,decoder_int_num_layer=8,decoder_rt_ff=1024,decoder_rt_num_layer=8,drop_2024-06-25_16-41-29/error.pkl
new file mode 100644
index 0000000000000000000000000000000000000000..2fc8daba03d604d911ffe4f4d7076265d28d6b64
Binary files /dev/null and b/ray_results/test_experiment/train_model_8c6e7040_14_batch_size=2048,decoder_int_ff=2048,decoder_int_num_layer=8,decoder_rt_ff=1024,decoder_rt_num_layer=8,drop_2024-06-25_16-41-29/error.pkl differ
diff --git a/ray_results/test_experiment/train_model_8c6e7040_14_batch_size=2048,decoder_int_ff=2048,decoder_int_num_layer=8,decoder_rt_ff=1024,decoder_rt_num_layer=8,drop_2024-06-25_16-41-29/error.txt b/ray_results/test_experiment/train_model_8c6e7040_14_batch_size=2048,decoder_int_ff=2048,decoder_int_num_layer=8,decoder_rt_ff=1024,decoder_rt_num_layer=8,drop_2024-06-25_16-41-29/error.txt
new file mode 100644
index 0000000000000000000000000000000000000000..1f1f25f7dfca38496adaa0a5967477fb6a6fe8c8
--- /dev/null
+++ b/ray_results/test_experiment/train_model_8c6e7040_14_batch_size=2048,decoder_int_ff=2048,decoder_int_num_layer=8,decoder_rt_ff=1024,decoder_rt_num_layer=8,drop_2024-06-25_16-41-29/error.txt
@@ -0,0 +1,59 @@
+Failure # 1 (occurred at 2024-06-25_16-43-06)
+[36mray::ImplicitFunc.train()[39m (pid=69318, ip=10.159.8.159, actor_id=eae11dc975075dcb8b04fdbf01000000, repr=train_model)
+ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+ File "/gpfslocalsup/pub/anaconda-py3/2023.09/envs/pytorch-gpu-2.2.0+py3.11.7/lib/python3.11/site-packages/ray/tune/trainable/trainable.py", line 342, in train
+ raise skipped from exception_cause(skipped)
+ File "/gpfslocalsup/pub/anaconda-py3/2023.09/envs/pytorch-gpu-2.2.0+py3.11.7/lib/python3.11/site-packages/ray/air/_internal/util.py", line 88, in run
+ self._ret = self._target(*self._args, **self._kwargs)
+ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+ File "/gpfslocalsup/pub/anaconda-py3/2023.09/envs/pytorch-gpu-2.2.0+py3.11.7/lib/python3.11/site-packages/ray/tune/trainable/function_trainable.py", line 249, in <lambda>
+ training_func=lambda: self._trainable_func(self.config),
+ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+ File "/gpfslocalsup/pub/anaconda-py3/2023.09/envs/pytorch-gpu-2.2.0+py3.11.7/lib/python3.11/site-packages/ray/tune/trainable/function_trainable.py", line 332, in _trainable_func
+ output = fn()
+ ^^^^
+ File "/gpfslocalsup/pub/anaconda-py3/2023.09/envs/pytorch-gpu-2.2.0+py3.11.7/lib/python3.11/site-packages/ray/tune/trainable/util.py", line 138, in inner
+ return trainable(config, **fn_kwargs)
+ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+ File "/gpfswork/rech/ute/ucg81ws/these/LC-MS-RT-prediction/main_ray_tune.py", line 66, in train_model
+ pred_rt = net.module.forward_rt(seq)
+ ^^^^^^^^^^^^^^^^^^^^^^^^^^
+ File "/gpfsdswork/projects/rech/ute/ucg81ws/these/LC-MS-RT-prediction/model_custom.py", line 111, in forward_rt
+ out_rt = self.decoder_RT(enc)
+ ^^^^^^^^^^^^^^^^^^^^
+ File "/gpfslocalsup/pub/anaconda-py3/2023.09/envs/pytorch-gpu-2.2.0+py3.11.7/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1511, in _wrapped_call_impl
+ return self._call_impl(*args, **kwargs)
+ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+ File "/gpfslocalsup/pub/anaconda-py3/2023.09/envs/pytorch-gpu-2.2.0+py3.11.7/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1520, in _call_impl
+ return forward_call(*args, **kwargs)
+ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+ File "/gpfslocalsup/pub/anaconda-py3/2023.09/envs/pytorch-gpu-2.2.0+py3.11.7/lib/python3.11/site-packages/torch/nn/modules/container.py", line 217, in forward
+ input = module(input)
+ ^^^^^^^^^^^^^
+ File "/gpfslocalsup/pub/anaconda-py3/2023.09/envs/pytorch-gpu-2.2.0+py3.11.7/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1511, in _wrapped_call_impl
+ return self._call_impl(*args, **kwargs)
+ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+ File "/gpfslocalsup/pub/anaconda-py3/2023.09/envs/pytorch-gpu-2.2.0+py3.11.7/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1520, in _call_impl
+ return forward_call(*args, **kwargs)
+ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+ File "/gpfslocalsup/pub/anaconda-py3/2023.09/envs/pytorch-gpu-2.2.0+py3.11.7/lib/python3.11/site-packages/torch/nn/modules/transformer.py", line 391, in forward
+ output = mod(output, src_mask=mask, is_causal=is_causal, src_key_padding_mask=src_key_padding_mask_for_layers)
+ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+ File "/gpfslocalsup/pub/anaconda-py3/2023.09/envs/pytorch-gpu-2.2.0+py3.11.7/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1511, in _wrapped_call_impl
+ return self._call_impl(*args, **kwargs)
+ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+ File "/gpfslocalsup/pub/anaconda-py3/2023.09/envs/pytorch-gpu-2.2.0+py3.11.7/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1520, in _call_impl
+ return forward_call(*args, **kwargs)
+ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+ File "/gpfslocalsup/pub/anaconda-py3/2023.09/envs/pytorch-gpu-2.2.0+py3.11.7/lib/python3.11/site-packages/torch/nn/modules/transformer.py", line 715, in forward
+ x = self.norm2(x + self._ff_block(x))
+ ^^^^^^^^^^^^^^^^^
+ File "/gpfslocalsup/pub/anaconda-py3/2023.09/envs/pytorch-gpu-2.2.0+py3.11.7/lib/python3.11/site-packages/torch/nn/modules/transformer.py", line 730, in _ff_block
+ x = self.linear2(self.dropout(self.activation(self.linear1(x))))
+ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+ File "/gpfslocalsup/pub/anaconda-py3/2023.09/envs/pytorch-gpu-2.2.0+py3.11.7/lib/python3.11/site-packages/torch/nn/functional.py", line 1473, in relu
+ result = torch.relu(input)
+ ^^^^^^^^^^^^^^^^^
+torch.cuda.OutOfMemoryError: CUDA out of memory. Tried to allocate 200.00 MiB. GPU 0 has a total capacity of 15.77 GiB of which 164.69 MiB is free. Including non-PyTorch memory, this process has 15.60 GiB memory in use. Of the allocated memory 15.18 GiB is allocated by PyTorch, and 51.15 MiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables)
diff --git a/ray_results/test_experiment/train_model_8c6e7040_14_batch_size=2048,decoder_int_ff=2048,decoder_int_num_layer=8,decoder_rt_ff=1024,decoder_rt_num_layer=8,drop_2024-06-25_16-41-29/events.out.tfevents.1719326572.r3i5n6 b/ray_results/test_experiment/train_model_8c6e7040_14_batch_size=2048,decoder_int_ff=2048,decoder_int_num_layer=8,decoder_rt_ff=1024,decoder_rt_num_layer=8,drop_2024-06-25_16-41-29/events.out.tfevents.1719326572.r3i5n6
new file mode 100644
index 0000000000000000000000000000000000000000..bb8b5d2f12702c6f37d63b73939f9db314e074bf
Binary files /dev/null and b/ray_results/test_experiment/train_model_8c6e7040_14_batch_size=2048,decoder_int_ff=2048,decoder_int_num_layer=8,decoder_rt_ff=1024,decoder_rt_num_layer=8,drop_2024-06-25_16-41-29/events.out.tfevents.1719326572.r3i5n6 differ
diff --git a/ray_results/test_experiment/train_model_8c6e7040_14_batch_size=2048,decoder_int_ff=2048,decoder_int_num_layer=8,decoder_rt_ff=1024,decoder_rt_num_layer=8,drop_2024-06-25_16-41-29/params.json b/ray_results/test_experiment/train_model_8c6e7040_14_batch_size=2048,decoder_int_ff=2048,decoder_int_num_layer=8,decoder_rt_ff=1024,decoder_rt_num_layer=8,drop_2024-06-25_16-41-29/params.json
new file mode 100644
index 0000000000000000000000000000000000000000..8af95a6ed61c792e34156c22ee75bb705ca4613c
--- /dev/null
+++ b/ray_results/test_experiment/train_model_8c6e7040_14_batch_size=2048,decoder_int_ff=2048,decoder_int_num_layer=8,decoder_rt_ff=1024,decoder_rt_num_layer=8,drop_2024-06-25_16-41-29/params.json
@@ -0,0 +1,13 @@
+{
+ "batch_size": 2048,
+ "decoder_int_ff": 2048,
+ "decoder_int_num_layer": 8,
+ "decoder_rt_ff": 1024,
+ "decoder_rt_num_layer": 8,
+ "drop_rate": 0.6537122299874158,
+ "embedding_dim": 256,
+ "encoder_ff": 2048,
+ "encoder_num_layer": 8,
+ "lr": 0.010515037912087563,
+ "n_head": 1
+}
\ No newline at end of file
diff --git a/ray_results/test_experiment/train_model_8c6e7040_14_batch_size=2048,decoder_int_ff=2048,decoder_int_num_layer=8,decoder_rt_ff=1024,decoder_rt_num_layer=8,drop_2024-06-25_16-41-29/params.pkl b/ray_results/test_experiment/train_model_8c6e7040_14_batch_size=2048,decoder_int_ff=2048,decoder_int_num_layer=8,decoder_rt_ff=1024,decoder_rt_num_layer=8,drop_2024-06-25_16-41-29/params.pkl
new file mode 100644
index 0000000000000000000000000000000000000000..b03d70fc1f346840504450aed94126874aed0347
Binary files /dev/null and b/ray_results/test_experiment/train_model_8c6e7040_14_batch_size=2048,decoder_int_ff=2048,decoder_int_num_layer=8,decoder_rt_ff=1024,decoder_rt_num_layer=8,drop_2024-06-25_16-41-29/params.pkl differ
diff --git a/ray_results/test_experiment/train_model_8c6e7040_14_batch_size=2048,decoder_int_ff=2048,decoder_int_num_layer=8,decoder_rt_ff=1024,decoder_rt_num_layer=8,drop_2024-06-25_16-41-29/result.json b/ray_results/test_experiment/train_model_8c6e7040_14_batch_size=2048,decoder_int_ff=2048,decoder_int_num_layer=8,decoder_rt_ff=1024,decoder_rt_num_layer=8,drop_2024-06-25_16-41-29/result.json
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/ray_results/test_experiment/train_model_8e58579c_18_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=8,decoder_rt_ff=2048,decoder_rt_num_layer=8,drop__2024-06-25_16-47-40/events.out.tfevents.1719326883.r3i5n6 b/ray_results/test_experiment/train_model_8e58579c_18_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=8,decoder_rt_ff=2048,decoder_rt_num_layer=8,drop__2024-06-25_16-47-40/events.out.tfevents.1719326883.r3i5n6
new file mode 100644
index 0000000000000000000000000000000000000000..7864aece5d8ff7dc3e8e074eb73881f1d778e557
Binary files /dev/null and b/ray_results/test_experiment/train_model_8e58579c_18_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=8,decoder_rt_ff=2048,decoder_rt_num_layer=8,drop__2024-06-25_16-47-40/events.out.tfevents.1719326883.r3i5n6 differ
diff --git a/ray_results/test_experiment/train_model_8e58579c_18_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=8,decoder_rt_ff=2048,decoder_rt_num_layer=8,drop__2024-06-25_16-47-40/params.json b/ray_results/test_experiment/train_model_8e58579c_18_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=8,decoder_rt_ff=2048,decoder_rt_num_layer=8,drop__2024-06-25_16-47-40/params.json
new file mode 100644
index 0000000000000000000000000000000000000000..6c356800fa7c651961ed91e66b850ae07bdd8ae8
--- /dev/null
+++ b/ray_results/test_experiment/train_model_8e58579c_18_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=8,decoder_rt_ff=2048,decoder_rt_num_layer=8,drop__2024-06-25_16-47-40/params.json
@@ -0,0 +1,13 @@
+{
+ "batch_size": 1024,
+ "decoder_int_ff": 512,
+ "decoder_int_num_layer": 8,
+ "decoder_rt_ff": 2048,
+ "decoder_rt_num_layer": 8,
+ "drop_rate": 0.8999111579906955,
+ "embedding_dim": 64,
+ "encoder_ff": 1024,
+ "encoder_num_layer": 8,
+ "lr": 0.000663247583015271,
+ "n_head": 8
+}
\ No newline at end of file
diff --git a/ray_results/test_experiment/train_model_8e58579c_18_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=8,decoder_rt_ff=2048,decoder_rt_num_layer=8,drop__2024-06-25_16-47-40/params.pkl b/ray_results/test_experiment/train_model_8e58579c_18_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=8,decoder_rt_ff=2048,decoder_rt_num_layer=8,drop__2024-06-25_16-47-40/params.pkl
new file mode 100644
index 0000000000000000000000000000000000000000..6bbd9fa4c9b60d515a5378a4c5267e0d057fe47c
Binary files /dev/null and b/ray_results/test_experiment/train_model_8e58579c_18_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=8,decoder_rt_ff=2048,decoder_rt_num_layer=8,drop__2024-06-25_16-47-40/params.pkl differ
diff --git a/ray_results/test_experiment/train_model_8e58579c_18_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=8,decoder_rt_ff=2048,decoder_rt_num_layer=8,drop__2024-06-25_16-47-40/progress.csv b/ray_results/test_experiment/train_model_8e58579c_18_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=8,decoder_rt_ff=2048,decoder_rt_num_layer=8,drop__2024-06-25_16-47-40/progress.csv
new file mode 100644
index 0000000000000000000000000000000000000000..1b028f4d87c51f9bbd83798848a93662d3297571
--- /dev/null
+++ b/ray_results/test_experiment/train_model_8e58579c_18_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=8,decoder_rt_ff=2048,decoder_rt_num_layer=8,drop__2024-06-25_16-47-40/progress.csv
@@ -0,0 +1,3 @@
+loss,timestamp,checkpoint_dir_name,should_checkpoint,done,training_iteration,trial_id,date,time_this_iter_s,time_total_s,pid,hostname,node_ip,time_since_restore,iterations_since_restore
+2062.3793243648497,1719327039,checkpoint_000000,True,False,1,8e58579c,2024-06-25_16-50-39,156.63724398612976,156.63724398612976,130020,r3i5n6,10.159.8.159,156.63724398612976,1
+2056.04917078694,1719327182,checkpoint_000001,True,False,2,8e58579c,2024-06-25_16-53-02,142.2777497768402,298.91499376296997,130020,r3i5n6,10.159.8.159,298.91499376296997,2
diff --git a/ray_results/test_experiment/train_model_8e58579c_18_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=8,decoder_rt_ff=2048,decoder_rt_num_layer=8,drop__2024-06-25_16-47-40/result.json b/ray_results/test_experiment/train_model_8e58579c_18_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=8,decoder_rt_ff=2048,decoder_rt_num_layer=8,drop__2024-06-25_16-47-40/result.json
new file mode 100644
index 0000000000000000000000000000000000000000..f6ba9861f4501cdbab887168892ec296ca506187
--- /dev/null
+++ b/ray_results/test_experiment/train_model_8e58579c_18_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=8,decoder_rt_ff=2048,decoder_rt_num_layer=8,drop__2024-06-25_16-47-40/result.json
@@ -0,0 +1,2 @@
+{"loss": 2062.3793243648497, "timestamp": 1719327039, "checkpoint_dir_name": "checkpoint_000000", "should_checkpoint": true, "done": false, "training_iteration": 1, "trial_id": "8e58579c", "date": "2024-06-25_16-50-39", "time_this_iter_s": 156.63724398612976, "time_total_s": 156.63724398612976, "pid": 130020, "hostname": "r3i5n6", "node_ip": "10.159.8.159", "config": {"encoder_num_layer": 8, "decoder_rt_num_layer": 8, "decoder_int_num_layer": 8, "embedding_dim": 64, "encoder_ff": 1024, "decoder_rt_ff": 2048, "decoder_int_ff": 512, "n_head": 8, "drop_rate": 0.8999111579906955, "lr": 0.000663247583015271, "batch_size": 1024}, "time_since_restore": 156.63724398612976, "iterations_since_restore": 1}
+{"loss": 2056.04917078694, "timestamp": 1719327182, "checkpoint_dir_name": "checkpoint_000001", "should_checkpoint": true, "done": false, "training_iteration": 2, "trial_id": "8e58579c", "date": "2024-06-25_16-53-02", "time_this_iter_s": 142.2777497768402, "time_total_s": 298.91499376296997, "pid": 130020, "hostname": "r3i5n6", "node_ip": "10.159.8.159", "config": {"encoder_num_layer": 8, "decoder_rt_num_layer": 8, "decoder_int_num_layer": 8, "embedding_dim": 64, "encoder_ff": 1024, "decoder_rt_ff": 2048, "decoder_int_ff": 512, "n_head": 8, "drop_rate": 0.8999111579906955, "lr": 0.000663247583015271, "batch_size": 1024}, "time_since_restore": 298.91499376296997, "iterations_since_restore": 2}
diff --git a/ray_results/test_experiment/train_model_934c5104_9_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop_r_2024-06-25_17-32-25/events.out.tfevents.1719329610.r8i6n2 b/ray_results/test_experiment/train_model_934c5104_9_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop_r_2024-06-25_17-32-25/events.out.tfevents.1719329610.r8i6n2
new file mode 100644
index 0000000000000000000000000000000000000000..6286bd7b76e0ed6b0f2508218e6d19bd2fb5afbb
Binary files /dev/null and b/ray_results/test_experiment/train_model_934c5104_9_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop_r_2024-06-25_17-32-25/events.out.tfevents.1719329610.r8i6n2 differ
diff --git a/ray_results/test_experiment/train_model_934c5104_9_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop_r_2024-06-25_17-32-25/events.out.tfevents.1719330672.r8i6n2 b/ray_results/test_experiment/train_model_934c5104_9_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop_r_2024-06-25_17-32-25/events.out.tfevents.1719330672.r8i6n2
new file mode 100644
index 0000000000000000000000000000000000000000..0e757bb36b88857565d42e05f02c84edb9ff24d3
Binary files /dev/null and b/ray_results/test_experiment/train_model_934c5104_9_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop_r_2024-06-25_17-32-25/events.out.tfevents.1719330672.r8i6n2 differ
diff --git a/ray_results/test_experiment/train_model_934c5104_9_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop_r_2024-06-25_17-32-25/events.out.tfevents.1719331282.r8i6n2 b/ray_results/test_experiment/train_model_934c5104_9_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop_r_2024-06-25_17-32-25/events.out.tfevents.1719331282.r8i6n2
new file mode 100644
index 0000000000000000000000000000000000000000..09c5868db57d6f4925f92cad890b331bfe74ebf1
Binary files /dev/null and b/ray_results/test_experiment/train_model_934c5104_9_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop_r_2024-06-25_17-32-25/events.out.tfevents.1719331282.r8i6n2 differ
diff --git a/ray_results/test_experiment/train_model_934c5104_9_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop_r_2024-06-25_17-32-25/events.out.tfevents.1719332071.r8i6n2 b/ray_results/test_experiment/train_model_934c5104_9_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop_r_2024-06-25_17-32-25/events.out.tfevents.1719332071.r8i6n2
new file mode 100644
index 0000000000000000000000000000000000000000..49e30115265afe258f1d2759701312484da2e83f
Binary files /dev/null and b/ray_results/test_experiment/train_model_934c5104_9_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop_r_2024-06-25_17-32-25/events.out.tfevents.1719332071.r8i6n2 differ
diff --git a/ray_results/test_experiment/train_model_934c5104_9_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop_r_2024-06-25_17-32-25/params.json b/ray_results/test_experiment/train_model_934c5104_9_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop_r_2024-06-25_17-32-25/params.json
new file mode 100644
index 0000000000000000000000000000000000000000..74d001d17fd9b747da8ffb1f8200ba1a778811c7
--- /dev/null
+++ b/ray_results/test_experiment/train_model_934c5104_9_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop_r_2024-06-25_17-32-25/params.json
@@ -0,0 +1,13 @@
+{
+ "batch_size": 2048,
+ "decoder_int_ff": 512,
+ "decoder_int_num_layer": 1,
+ "decoder_rt_ff": 1024,
+ "decoder_rt_num_layer": 2,
+ "drop_rate": 0.2811887966312956,
+ "embedding_dim": 256,
+ "encoder_ff": 2048,
+ "encoder_num_layer": 1,
+ "lr": 0.0005389075253520159,
+ "n_head": 8
+}
\ No newline at end of file
diff --git a/ray_results/test_experiment/train_model_934c5104_9_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop_r_2024-06-25_17-32-25/params.pkl b/ray_results/test_experiment/train_model_934c5104_9_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop_r_2024-06-25_17-32-25/params.pkl
new file mode 100644
index 0000000000000000000000000000000000000000..bf09f8838ddc89a7bb42c709c4600a089ff8a403
Binary files /dev/null and b/ray_results/test_experiment/train_model_934c5104_9_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop_r_2024-06-25_17-32-25/params.pkl differ
diff --git a/ray_results/test_experiment/train_model_934c5104_9_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop_r_2024-06-25_17-32-25/progress.csv b/ray_results/test_experiment/train_model_934c5104_9_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop_r_2024-06-25_17-32-25/progress.csv
new file mode 100644
index 0000000000000000000000000000000000000000..1935394caedcf6778e39ffd2821f27653f5edb11
--- /dev/null
+++ b/ray_results/test_experiment/train_model_934c5104_9_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop_r_2024-06-25_17-32-25/progress.csv
@@ -0,0 +1,15 @@
+loss,timestamp,checkpoint_dir_name,should_checkpoint,done,training_iteration,trial_id,date,time_this_iter_s,time_total_s,pid,hostname,node_ip,time_since_restore,iterations_since_restore
+234.06637176753966,1719329693,checkpoint_000000,True,False,1,934c5104,2024-06-25_17-34-53,82.2380793094635,82.2380793094635,2195253,r8i6n2,10.159.28.60,82.2380793094635,1
+191.28313151682457,1719330754,checkpoint_000001,True,False,2,934c5104,2024-06-25_17-52-34,82.23219752311707,164.47027683258057,2195253,r8i6n2,10.159.28.60,82.23219752311707,1
+166.0584006722518,1719331377,checkpoint_000002,True,False,3,934c5104,2024-06-25_18-02-57,95.2866895198822,259.75696635246277,2195253,r8i6n2,10.159.28.60,95.2866895198822,1
+159.9789658794253,1719331459,checkpoint_000003,True,False,4,934c5104,2024-06-25_18-04-19,81.72880601882935,341.4857723712921,2195253,r8i6n2,10.159.28.60,177.01549553871155,2
+141.96990696464,1719332153,checkpoint_000004,True,False,5,934c5104,2024-06-25_18-15-54,82.40532422065735,423.89109659194946,2195253,r8i6n2,10.159.28.60,82.40532422065735,1
+117.70846791905682,1719332222,checkpoint_000005,True,False,6,934c5104,2024-06-25_18-17-02,68.08270859718323,491.9738051891327,2195253,r8i6n2,10.159.28.60,150.48803281784058,2
+113.39667721245233,1719332289,checkpoint_000006,True,False,7,934c5104,2024-06-25_18-18-09,67.7198805809021,559.6936857700348,2195253,r8i6n2,10.159.28.60,218.20791339874268,3
+111.7227829460084,1719332358,checkpoint_000007,True,False,8,934c5104,2024-06-25_18-19-18,68.60852479934692,628.3022105693817,2195253,r8i6n2,10.159.28.60,286.8164381980896,4
+111.3588055588129,1719332426,checkpoint_000008,True,False,9,934c5104,2024-06-25_18-20-26,67.80738162994385,696.1095921993256,2195253,r8i6n2,10.159.28.60,354.62381982803345,5
+108.66629178505244,1719332494,checkpoint_000009,True,False,10,934c5104,2024-06-25_18-21-35,68.99091339111328,765.1005055904388,2195253,r8i6n2,10.159.28.60,423.61473321914673,6
+102.03670796071451,1719332563,checkpoint_000010,True,False,11,934c5104,2024-06-25_18-22-43,68.06165027618408,833.1621558666229,2195253,r8i6n2,10.159.28.60,491.6763834953308,7
+100.34690394364004,1719332631,checkpoint_000011,True,False,12,934c5104,2024-06-25_18-23-51,68.09097814559937,901.2531340122223,2195253,r8i6n2,10.159.28.60,559.7673616409302,8
+102.1303488663801,1719332699,checkpoint_000012,True,False,13,934c5104,2024-06-25_18-24-59,68.49995398521423,969.7530879974365,2195253,r8i6n2,10.159.28.60,628.2673156261444,9
+98.24798722154512,1719332767,checkpoint_000013,True,False,14,934c5104,2024-06-25_18-26-08,68.14825201034546,1037.901340007782,2195253,r8i6n2,10.159.28.60,696.4155676364899,10
diff --git a/ray_results/test_experiment/train_model_934c5104_9_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop_r_2024-06-25_17-32-25/result.json b/ray_results/test_experiment/train_model_934c5104_9_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop_r_2024-06-25_17-32-25/result.json
new file mode 100644
index 0000000000000000000000000000000000000000..dba173744518e8ab5acc1851bdc0c388610b36b0
--- /dev/null
+++ b/ray_results/test_experiment/train_model_934c5104_9_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop_r_2024-06-25_17-32-25/result.json
@@ -0,0 +1,14 @@
+{"loss": 234.06637176753966, "timestamp": 1719329693, "checkpoint_dir_name": "checkpoint_000000", "should_checkpoint": true, "done": false, "training_iteration": 1, "trial_id": "934c5104", "date": "2024-06-25_17-34-53", "time_this_iter_s": 82.2380793094635, "time_total_s": 82.2380793094635, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 1, "decoder_rt_num_layer": 2, "decoder_int_num_layer": 1, "embedding_dim": 256, "encoder_ff": 2048, "decoder_rt_ff": 1024, "decoder_int_ff": 512, "n_head": 8, "drop_rate": 0.2811887966312956, "lr": 0.0005389075253520159, "batch_size": 2048}, "time_since_restore": 82.2380793094635, "iterations_since_restore": 1}
+{"loss": 191.28313151682457, "timestamp": 1719330754, "checkpoint_dir_name": "checkpoint_000001", "should_checkpoint": true, "done": false, "training_iteration": 2, "trial_id": "934c5104", "date": "2024-06-25_17-52-34", "time_this_iter_s": 82.23219752311707, "time_total_s": 164.47027683258057, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 1, "decoder_rt_num_layer": 2, "decoder_int_num_layer": 1, "embedding_dim": 256, "encoder_ff": 2048, "decoder_rt_ff": 1024, "decoder_int_ff": 512, "n_head": 8, "drop_rate": 0.2811887966312956, "lr": 0.0005389075253520159, "batch_size": 2048}, "time_since_restore": 82.23219752311707, "iterations_since_restore": 1}
+{"loss": 166.0584006722518, "timestamp": 1719331377, "checkpoint_dir_name": "checkpoint_000002", "should_checkpoint": true, "done": false, "training_iteration": 3, "trial_id": "934c5104", "date": "2024-06-25_18-02-57", "time_this_iter_s": 95.2866895198822, "time_total_s": 259.75696635246277, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 1, "decoder_rt_num_layer": 2, "decoder_int_num_layer": 1, "embedding_dim": 256, "encoder_ff": 2048, "decoder_rt_ff": 1024, "decoder_int_ff": 512, "n_head": 8, "drop_rate": 0.2811887966312956, "lr": 0.0005389075253520159, "batch_size": 2048}, "time_since_restore": 95.2866895198822, "iterations_since_restore": 1}
+{"loss": 159.9789658794253, "timestamp": 1719331459, "checkpoint_dir_name": "checkpoint_000003", "should_checkpoint": true, "done": false, "training_iteration": 4, "trial_id": "934c5104", "date": "2024-06-25_18-04-19", "time_this_iter_s": 81.72880601882935, "time_total_s": 341.4857723712921, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 1, "decoder_rt_num_layer": 2, "decoder_int_num_layer": 1, "embedding_dim": 256, "encoder_ff": 2048, "decoder_rt_ff": 1024, "decoder_int_ff": 512, "n_head": 8, "drop_rate": 0.2811887966312956, "lr": 0.0005389075253520159, "batch_size": 2048}, "time_since_restore": 177.01549553871155, "iterations_since_restore": 2}
+{"loss": 141.96990696464, "timestamp": 1719332153, "checkpoint_dir_name": "checkpoint_000004", "should_checkpoint": true, "done": false, "training_iteration": 5, "trial_id": "934c5104", "date": "2024-06-25_18-15-54", "time_this_iter_s": 82.40532422065735, "time_total_s": 423.89109659194946, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 1, "decoder_rt_num_layer": 2, "decoder_int_num_layer": 1, "embedding_dim": 256, "encoder_ff": 2048, "decoder_rt_ff": 1024, "decoder_int_ff": 512, "n_head": 8, "drop_rate": 0.2811887966312956, "lr": 0.0005389075253520159, "batch_size": 2048}, "time_since_restore": 82.40532422065735, "iterations_since_restore": 1}
+{"loss": 117.70846791905682, "timestamp": 1719332222, "checkpoint_dir_name": "checkpoint_000005", "should_checkpoint": true, "done": false, "training_iteration": 6, "trial_id": "934c5104", "date": "2024-06-25_18-17-02", "time_this_iter_s": 68.08270859718323, "time_total_s": 491.9738051891327, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 1, "decoder_rt_num_layer": 2, "decoder_int_num_layer": 1, "embedding_dim": 256, "encoder_ff": 2048, "decoder_rt_ff": 1024, "decoder_int_ff": 512, "n_head": 8, "drop_rate": 0.2811887966312956, "lr": 0.0005389075253520159, "batch_size": 2048}, "time_since_restore": 150.48803281784058, "iterations_since_restore": 2}
+{"loss": 113.39667721245233, "timestamp": 1719332289, "checkpoint_dir_name": "checkpoint_000006", "should_checkpoint": true, "done": false, "training_iteration": 7, "trial_id": "934c5104", "date": "2024-06-25_18-18-09", "time_this_iter_s": 67.7198805809021, "time_total_s": 559.6936857700348, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 1, "decoder_rt_num_layer": 2, "decoder_int_num_layer": 1, "embedding_dim": 256, "encoder_ff": 2048, "decoder_rt_ff": 1024, "decoder_int_ff": 512, "n_head": 8, "drop_rate": 0.2811887966312956, "lr": 0.0005389075253520159, "batch_size": 2048}, "time_since_restore": 218.20791339874268, "iterations_since_restore": 3}
+{"loss": 111.7227829460084, "timestamp": 1719332358, "checkpoint_dir_name": "checkpoint_000007", "should_checkpoint": true, "done": false, "training_iteration": 8, "trial_id": "934c5104", "date": "2024-06-25_18-19-18", "time_this_iter_s": 68.60852479934692, "time_total_s": 628.3022105693817, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 1, "decoder_rt_num_layer": 2, "decoder_int_num_layer": 1, "embedding_dim": 256, "encoder_ff": 2048, "decoder_rt_ff": 1024, "decoder_int_ff": 512, "n_head": 8, "drop_rate": 0.2811887966312956, "lr": 0.0005389075253520159, "batch_size": 2048}, "time_since_restore": 286.8164381980896, "iterations_since_restore": 4}
+{"loss": 111.3588055588129, "timestamp": 1719332426, "checkpoint_dir_name": "checkpoint_000008", "should_checkpoint": true, "done": false, "training_iteration": 9, "trial_id": "934c5104", "date": "2024-06-25_18-20-26", "time_this_iter_s": 67.80738162994385, "time_total_s": 696.1095921993256, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 1, "decoder_rt_num_layer": 2, "decoder_int_num_layer": 1, "embedding_dim": 256, "encoder_ff": 2048, "decoder_rt_ff": 1024, "decoder_int_ff": 512, "n_head": 8, "drop_rate": 0.2811887966312956, "lr": 0.0005389075253520159, "batch_size": 2048}, "time_since_restore": 354.62381982803345, "iterations_since_restore": 5}
+{"loss": 108.66629178505244, "timestamp": 1719332494, "checkpoint_dir_name": "checkpoint_000009", "should_checkpoint": true, "done": false, "training_iteration": 10, "trial_id": "934c5104", "date": "2024-06-25_18-21-35", "time_this_iter_s": 68.99091339111328, "time_total_s": 765.1005055904388, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 1, "decoder_rt_num_layer": 2, "decoder_int_num_layer": 1, "embedding_dim": 256, "encoder_ff": 2048, "decoder_rt_ff": 1024, "decoder_int_ff": 512, "n_head": 8, "drop_rate": 0.2811887966312956, "lr": 0.0005389075253520159, "batch_size": 2048}, "time_since_restore": 423.61473321914673, "iterations_since_restore": 6}
+{"loss": 102.03670796071451, "timestamp": 1719332563, "checkpoint_dir_name": "checkpoint_000010", "should_checkpoint": true, "done": false, "training_iteration": 11, "trial_id": "934c5104", "date": "2024-06-25_18-22-43", "time_this_iter_s": 68.06165027618408, "time_total_s": 833.1621558666229, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 1, "decoder_rt_num_layer": 2, "decoder_int_num_layer": 1, "embedding_dim": 256, "encoder_ff": 2048, "decoder_rt_ff": 1024, "decoder_int_ff": 512, "n_head": 8, "drop_rate": 0.2811887966312956, "lr": 0.0005389075253520159, "batch_size": 2048}, "time_since_restore": 491.6763834953308, "iterations_since_restore": 7}
+{"loss": 100.34690394364004, "timestamp": 1719332631, "checkpoint_dir_name": "checkpoint_000011", "should_checkpoint": true, "done": false, "training_iteration": 12, "trial_id": "934c5104", "date": "2024-06-25_18-23-51", "time_this_iter_s": 68.09097814559937, "time_total_s": 901.2531340122223, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 1, "decoder_rt_num_layer": 2, "decoder_int_num_layer": 1, "embedding_dim": 256, "encoder_ff": 2048, "decoder_rt_ff": 1024, "decoder_int_ff": 512, "n_head": 8, "drop_rate": 0.2811887966312956, "lr": 0.0005389075253520159, "batch_size": 2048}, "time_since_restore": 559.7673616409302, "iterations_since_restore": 8}
+{"loss": 102.1303488663801, "timestamp": 1719332699, "checkpoint_dir_name": "checkpoint_000012", "should_checkpoint": true, "done": false, "training_iteration": 13, "trial_id": "934c5104", "date": "2024-06-25_18-24-59", "time_this_iter_s": 68.49995398521423, "time_total_s": 969.7530879974365, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 1, "decoder_rt_num_layer": 2, "decoder_int_num_layer": 1, "embedding_dim": 256, "encoder_ff": 2048, "decoder_rt_ff": 1024, "decoder_int_ff": 512, "n_head": 8, "drop_rate": 0.2811887966312956, "lr": 0.0005389075253520159, "batch_size": 2048}, "time_since_restore": 628.2673156261444, "iterations_since_restore": 9}
+{"loss": 98.24798722154512, "timestamp": 1719332767, "checkpoint_dir_name": "checkpoint_000013", "should_checkpoint": true, "done": false, "training_iteration": 14, "trial_id": "934c5104", "date": "2024-06-25_18-26-08", "time_this_iter_s": 68.14825201034546, "time_total_s": 1037.901340007782, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 1, "decoder_rt_num_layer": 2, "decoder_int_num_layer": 1, "embedding_dim": 256, "encoder_ff": 2048, "decoder_rt_ff": 1024, "decoder_int_ff": 512, "n_head": 8, "drop_rate": 0.2811887966312956, "lr": 0.0005389075253520159, "batch_size": 2048}, "time_since_restore": 696.4155676364899, "iterations_since_restore": 10}
diff --git a/ray_results/test_experiment/train_model_975aa6a0_4_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=2,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop_r_2024-06-25_16-11-03/events.out.tfevents.1719324728.r3i5n6 b/ray_results/test_experiment/train_model_975aa6a0_4_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=2,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop_r_2024-06-25_16-11-03/events.out.tfevents.1719324728.r3i5n6
new file mode 100644
index 0000000000000000000000000000000000000000..751ead7bdc6e653c4adc03dec7f4ad59e03acae9
Binary files /dev/null and b/ray_results/test_experiment/train_model_975aa6a0_4_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=2,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop_r_2024-06-25_16-11-03/events.out.tfevents.1719324728.r3i5n6 differ
diff --git a/ray_results/test_experiment/train_model_975aa6a0_4_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=2,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop_r_2024-06-25_16-11-03/params.json b/ray_results/test_experiment/train_model_975aa6a0_4_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=2,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop_r_2024-06-25_16-11-03/params.json
new file mode 100644
index 0000000000000000000000000000000000000000..a21fd66094fca7a83a07b23a898bb7095743ad4b
--- /dev/null
+++ b/ray_results/test_experiment/train_model_975aa6a0_4_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=2,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop_r_2024-06-25_16-11-03/params.json
@@ -0,0 +1,13 @@
+{
+ "batch_size": 2048,
+ "decoder_int_ff": 512,
+ "decoder_int_num_layer": 2,
+ "decoder_rt_ff": 1024,
+ "decoder_rt_num_layer": 1,
+ "drop_rate": 0.8156275326399022,
+ "embedding_dim": 64,
+ "encoder_ff": 1024,
+ "encoder_num_layer": 1,
+ "lr": 0.00010404955423531405,
+ "n_head": 16
+}
\ No newline at end of file
diff --git a/ray_results/test_experiment/train_model_975aa6a0_4_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=2,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop_r_2024-06-25_16-11-03/params.pkl b/ray_results/test_experiment/train_model_975aa6a0_4_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=2,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop_r_2024-06-25_16-11-03/params.pkl
new file mode 100644
index 0000000000000000000000000000000000000000..e196681f5f7a5e368943bf3fb54a03fefb5f87e9
Binary files /dev/null and b/ray_results/test_experiment/train_model_975aa6a0_4_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=2,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop_r_2024-06-25_16-11-03/params.pkl differ
diff --git a/ray_results/test_experiment/train_model_975aa6a0_4_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=2,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop_r_2024-06-25_16-11-03/progress.csv b/ray_results/test_experiment/train_model_975aa6a0_4_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=2,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop_r_2024-06-25_16-11-03/progress.csv
new file mode 100644
index 0000000000000000000000000000000000000000..cc55625fa767504c743ae1ea347027ed5389f3ee
--- /dev/null
+++ b/ray_results/test_experiment/train_model_975aa6a0_4_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=2,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop_r_2024-06-25_16-11-03/progress.csv
@@ -0,0 +1,2 @@
+loss,timestamp,checkpoint_dir_name,should_checkpoint,done,training_iteration,trial_id,date,time_this_iter_s,time_total_s,pid,hostname,node_ip,time_since_restore,iterations_since_restore
+2002.7163720318651,1719324792,checkpoint_000000,True,False,1,975aa6a0,2024-06-25_16-13-12,64.14145565032959,64.14145565032959,69318,r3i5n6,10.159.8.159,64.14145565032959,1
diff --git a/ray_results/test_experiment/train_model_975aa6a0_4_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=2,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop_r_2024-06-25_16-11-03/result.json b/ray_results/test_experiment/train_model_975aa6a0_4_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=2,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop_r_2024-06-25_16-11-03/result.json
new file mode 100644
index 0000000000000000000000000000000000000000..28d61e95cbac8d1d2b16ba0f37adfa97780e5daa
--- /dev/null
+++ b/ray_results/test_experiment/train_model_975aa6a0_4_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=2,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop_r_2024-06-25_16-11-03/result.json
@@ -0,0 +1 @@
+{"loss": 2002.7163720318651, "timestamp": 1719324792, "checkpoint_dir_name": "checkpoint_000000", "should_checkpoint": true, "done": false, "training_iteration": 1, "trial_id": "975aa6a0", "date": "2024-06-25_16-13-12", "time_this_iter_s": 64.14145565032959, "time_total_s": 64.14145565032959, "pid": 69318, "hostname": "r3i5n6", "node_ip": "10.159.8.159", "config": {"encoder_num_layer": 1, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 2, "embedding_dim": 64, "encoder_ff": 1024, "decoder_rt_ff": 1024, "decoder_int_ff": 512, "n_head": 16, "drop_rate": 0.8156275326399022, "lr": 0.00010404955423531405, "batch_size": 2048}, "time_since_restore": 64.14145565032959, "iterations_since_restore": 1}
diff --git a/ray_results/test_experiment/train_model_a00d3259_2_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=4,drop_ra_2024-06-25_17-10-13/events.out.tfevents.1719328456.r8i6n2 b/ray_results/test_experiment/train_model_a00d3259_2_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=4,drop_ra_2024-06-25_17-10-13/events.out.tfevents.1719328456.r8i6n2
new file mode 100644
index 0000000000000000000000000000000000000000..4cc40b099c6d264f4885bfa7e1e600d9fbd88405
Binary files /dev/null and b/ray_results/test_experiment/train_model_a00d3259_2_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=4,drop_ra_2024-06-25_17-10-13/events.out.tfevents.1719328456.r8i6n2 differ
diff --git a/ray_results/test_experiment/train_model_a00d3259_2_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=4,drop_ra_2024-06-25_17-10-13/events.out.tfevents.1719330517.r8i6n2 b/ray_results/test_experiment/train_model_a00d3259_2_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=4,drop_ra_2024-06-25_17-10-13/events.out.tfevents.1719330517.r8i6n2
new file mode 100644
index 0000000000000000000000000000000000000000..058684f608701ed90d6d293a6424317a3ea4cda4
Binary files /dev/null and b/ray_results/test_experiment/train_model_a00d3259_2_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=4,drop_ra_2024-06-25_17-10-13/events.out.tfevents.1719330517.r8i6n2 differ
diff --git a/ray_results/test_experiment/train_model_a00d3259_2_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=4,drop_ra_2024-06-25_17-10-13/events.out.tfevents.1719331131.r8i6n2 b/ray_results/test_experiment/train_model_a00d3259_2_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=4,drop_ra_2024-06-25_17-10-13/events.out.tfevents.1719331131.r8i6n2
new file mode 100644
index 0000000000000000000000000000000000000000..7cf1b5ad836eef601320c037e2736229dd488702
Binary files /dev/null and b/ray_results/test_experiment/train_model_a00d3259_2_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=4,drop_ra_2024-06-25_17-10-13/events.out.tfevents.1719331131.r8i6n2 differ
diff --git a/ray_results/test_experiment/train_model_a00d3259_2_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=4,drop_ra_2024-06-25_17-10-13/events.out.tfevents.1719331824.r8i6n2 b/ray_results/test_experiment/train_model_a00d3259_2_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=4,drop_ra_2024-06-25_17-10-13/events.out.tfevents.1719331824.r8i6n2
new file mode 100644
index 0000000000000000000000000000000000000000..2992aec175590f0f1e210addbbd192505204f9b3
Binary files /dev/null and b/ray_results/test_experiment/train_model_a00d3259_2_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=4,drop_ra_2024-06-25_17-10-13/events.out.tfevents.1719331824.r8i6n2 differ
diff --git a/ray_results/test_experiment/train_model_a00d3259_2_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=4,drop_ra_2024-06-25_17-10-13/params.json b/ray_results/test_experiment/train_model_a00d3259_2_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=4,drop_ra_2024-06-25_17-10-13/params.json
new file mode 100644
index 0000000000000000000000000000000000000000..9155b49fa735e9c3630e8248d0b8144b123b7b02
--- /dev/null
+++ b/ray_results/test_experiment/train_model_a00d3259_2_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=4,drop_ra_2024-06-25_17-10-13/params.json
@@ -0,0 +1,13 @@
+{
+ "batch_size": 1024,
+ "decoder_int_ff": 512,
+ "decoder_int_num_layer": 1,
+ "decoder_rt_ff": 512,
+ "decoder_rt_num_layer": 4,
+ "drop_rate": 0.22544213936627444,
+ "embedding_dim": 16,
+ "encoder_ff": 1024,
+ "encoder_num_layer": 1,
+ "lr": 0.0013157536168827595,
+ "n_head": 4
+}
\ No newline at end of file
diff --git a/ray_results/test_experiment/train_model_a00d3259_2_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=4,drop_ra_2024-06-25_17-10-13/params.pkl b/ray_results/test_experiment/train_model_a00d3259_2_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=4,drop_ra_2024-06-25_17-10-13/params.pkl
new file mode 100644
index 0000000000000000000000000000000000000000..cf049c9f581d612bc599d1f8f0bbb72e6fd079c8
Binary files /dev/null and b/ray_results/test_experiment/train_model_a00d3259_2_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=4,drop_ra_2024-06-25_17-10-13/params.pkl differ
diff --git a/ray_results/test_experiment/train_model_a00d3259_2_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=4,drop_ra_2024-06-25_17-10-13/progress.csv b/ray_results/test_experiment/train_model_a00d3259_2_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=4,drop_ra_2024-06-25_17-10-13/progress.csv
new file mode 100644
index 0000000000000000000000000000000000000000..3ec1538b65af1e46553b471671341a8d3853e2ba
--- /dev/null
+++ b/ray_results/test_experiment/train_model_a00d3259_2_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=4,drop_ra_2024-06-25_17-10-13/progress.csv
@@ -0,0 +1,9 @@
+loss,timestamp,checkpoint_dir_name,should_checkpoint,done,training_iteration,trial_id,date,time_this_iter_s,time_total_s,pid,hostname,node_ip,time_since_restore,iterations_since_restore
+253.8278117743064,1719328526,checkpoint_000000,True,False,1,a00d3259,2024-06-25_17-15-26,70.23847699165344,70.23847699165344,2195253,r8i6n2,10.159.28.60,70.23847699165344,1
+168.6947938813938,1719330587,checkpoint_000001,True,False,2,a00d3259,2024-06-25_17-49-47,69.5678973197937,139.80637431144714,2195253,r8i6n2,10.159.28.60,69.5678973197937,1
+159.82500259519563,1719331219,checkpoint_000002,True,False,3,a00d3259,2024-06-25_18-00-19,88.16569352149963,227.97206783294678,2195253,r8i6n2,10.159.28.60,88.16569352149963,1
+140.64553523626853,1719331282,checkpoint_000003,True,False,4,a00d3259,2024-06-25_18-01-22,62.607720375061035,290.5797882080078,2195253,r8i6n2,10.159.28.60,150.77341389656067,2
+150.0000455060343,1719331896,checkpoint_000004,True,False,5,a00d3259,2024-06-25_18-11-36,72.27389621734619,362.853684425354,2195253,r8i6n2,10.159.28.60,72.27389621734619,1
+130.9991111755371,1719331954,checkpoint_000005,True,False,6,a00d3259,2024-06-25_18-12-34,58.202213287353516,421.0558977127075,2195253,r8i6n2,10.159.28.60,130.4761095046997,2
+124.42158469252699,1719332013,checkpoint_000006,True,False,7,a00d3259,2024-06-25_18-13-33,58.22797513008118,479.2838728427887,2195253,r8i6n2,10.159.28.60,188.70408463478088,3
+119.19234625748761,1719332071,checkpoint_000007,True,False,8,a00d3259,2024-06-25_18-14-31,58.22098970413208,537.5048625469208,2195253,r8i6n2,10.159.28.60,246.92507433891296,4
diff --git a/ray_results/test_experiment/train_model_a00d3259_2_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=4,drop_ra_2024-06-25_17-10-13/result.json b/ray_results/test_experiment/train_model_a00d3259_2_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=4,drop_ra_2024-06-25_17-10-13/result.json
new file mode 100644
index 0000000000000000000000000000000000000000..58069336ccdaa2ca6824e28da2f0a04216663257
--- /dev/null
+++ b/ray_results/test_experiment/train_model_a00d3259_2_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=4,drop_ra_2024-06-25_17-10-13/result.json
@@ -0,0 +1,8 @@
+{"loss": 253.8278117743064, "timestamp": 1719328526, "checkpoint_dir_name": "checkpoint_000000", "should_checkpoint": true, "done": false, "training_iteration": 1, "trial_id": "a00d3259", "date": "2024-06-25_17-15-26", "time_this_iter_s": 70.23847699165344, "time_total_s": 70.23847699165344, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 1, "decoder_rt_num_layer": 4, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 1024, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 4, "drop_rate": 0.22544213936627444, "lr": 0.0013157536168827595, "batch_size": 1024}, "time_since_restore": 70.23847699165344, "iterations_since_restore": 1}
+{"loss": 168.6947938813938, "timestamp": 1719330587, "checkpoint_dir_name": "checkpoint_000001", "should_checkpoint": true, "done": false, "training_iteration": 2, "trial_id": "a00d3259", "date": "2024-06-25_17-49-47", "time_this_iter_s": 69.5678973197937, "time_total_s": 139.80637431144714, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 1, "decoder_rt_num_layer": 4, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 1024, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 4, "drop_rate": 0.22544213936627444, "lr": 0.0013157536168827595, "batch_size": 1024}, "time_since_restore": 69.5678973197937, "iterations_since_restore": 1}
+{"loss": 159.82500259519563, "timestamp": 1719331219, "checkpoint_dir_name": "checkpoint_000002", "should_checkpoint": true, "done": false, "training_iteration": 3, "trial_id": "a00d3259", "date": "2024-06-25_18-00-19", "time_this_iter_s": 88.16569352149963, "time_total_s": 227.97206783294678, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 1, "decoder_rt_num_layer": 4, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 1024, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 4, "drop_rate": 0.22544213936627444, "lr": 0.0013157536168827595, "batch_size": 1024}, "time_since_restore": 88.16569352149963, "iterations_since_restore": 1}
+{"loss": 140.64553523626853, "timestamp": 1719331282, "checkpoint_dir_name": "checkpoint_000003", "should_checkpoint": true, "done": false, "training_iteration": 4, "trial_id": "a00d3259", "date": "2024-06-25_18-01-22", "time_this_iter_s": 62.607720375061035, "time_total_s": 290.5797882080078, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 1, "decoder_rt_num_layer": 4, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 1024, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 4, "drop_rate": 0.22544213936627444, "lr": 0.0013157536168827595, "batch_size": 1024}, "time_since_restore": 150.77341389656067, "iterations_since_restore": 2}
+{"loss": 150.0000455060343, "timestamp": 1719331896, "checkpoint_dir_name": "checkpoint_000004", "should_checkpoint": true, "done": false, "training_iteration": 5, "trial_id": "a00d3259", "date": "2024-06-25_18-11-36", "time_this_iter_s": 72.27389621734619, "time_total_s": 362.853684425354, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 1, "decoder_rt_num_layer": 4, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 1024, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 4, "drop_rate": 0.22544213936627444, "lr": 0.0013157536168827595, "batch_size": 1024}, "time_since_restore": 72.27389621734619, "iterations_since_restore": 1}
+{"loss": 130.9991111755371, "timestamp": 1719331954, "checkpoint_dir_name": "checkpoint_000005", "should_checkpoint": true, "done": false, "training_iteration": 6, "trial_id": "a00d3259", "date": "2024-06-25_18-12-34", "time_this_iter_s": 58.202213287353516, "time_total_s": 421.0558977127075, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 1, "decoder_rt_num_layer": 4, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 1024, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 4, "drop_rate": 0.22544213936627444, "lr": 0.0013157536168827595, "batch_size": 1024}, "time_since_restore": 130.4761095046997, "iterations_since_restore": 2}
+{"loss": 124.42158469252699, "timestamp": 1719332013, "checkpoint_dir_name": "checkpoint_000006", "should_checkpoint": true, "done": false, "training_iteration": 7, "trial_id": "a00d3259", "date": "2024-06-25_18-13-33", "time_this_iter_s": 58.22797513008118, "time_total_s": 479.2838728427887, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 1, "decoder_rt_num_layer": 4, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 1024, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 4, "drop_rate": 0.22544213936627444, "lr": 0.0013157536168827595, "batch_size": 1024}, "time_since_restore": 188.70408463478088, "iterations_since_restore": 3}
+{"loss": 119.19234625748761, "timestamp": 1719332071, "checkpoint_dir_name": "checkpoint_000007", "should_checkpoint": true, "done": false, "training_iteration": 8, "trial_id": "a00d3259", "date": "2024-06-25_18-14-31", "time_this_iter_s": 58.22098970413208, "time_total_s": 537.5048625469208, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 1, "decoder_rt_num_layer": 4, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 1024, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 4, "drop_rate": 0.22544213936627444, "lr": 0.0013157536168827595, "batch_size": 1024}, "time_since_restore": 246.92507433891296, "iterations_since_restore": 4}
diff --git a/ray_results/test_experiment/train_model_a01c9b2d_16_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_17-44-50/events.out.tfevents.1719330362.r8i6n2 b/ray_results/test_experiment/train_model_a01c9b2d_16_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_17-44-50/events.out.tfevents.1719330362.r8i6n2
new file mode 100644
index 0000000000000000000000000000000000000000..12da38bddd4ec933a17d9c29821179e8a2d15e84
Binary files /dev/null and b/ray_results/test_experiment/train_model_a01c9b2d_16_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_17-44-50/events.out.tfevents.1719330362.r8i6n2 differ
diff --git a/ray_results/test_experiment/train_model_a01c9b2d_16_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_17-44-50/events.out.tfevents.1719331634.r8i6n2 b/ray_results/test_experiment/train_model_a01c9b2d_16_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_17-44-50/events.out.tfevents.1719331634.r8i6n2
new file mode 100644
index 0000000000000000000000000000000000000000..edaa9f5f5aab1cc489cdb86002b338f3871994da
Binary files /dev/null and b/ray_results/test_experiment/train_model_a01c9b2d_16_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_17-44-50/events.out.tfevents.1719331634.r8i6n2 differ
diff --git a/ray_results/test_experiment/train_model_a01c9b2d_16_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_17-44-50/params.json b/ray_results/test_experiment/train_model_a01c9b2d_16_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_17-44-50/params.json
new file mode 100644
index 0000000000000000000000000000000000000000..6fc28c7d054c07b5433663602fe7ab861a67c4a9
--- /dev/null
+++ b/ray_results/test_experiment/train_model_a01c9b2d_16_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_17-44-50/params.json
@@ -0,0 +1,13 @@
+{
+ "batch_size": 1024,
+ "decoder_int_ff": 512,
+ "decoder_int_num_layer": 1,
+ "decoder_rt_ff": 1024,
+ "decoder_rt_num_layer": 1,
+ "drop_rate": 0.23338023981015343,
+ "embedding_dim": 16,
+ "encoder_ff": 1024,
+ "encoder_num_layer": 8,
+ "lr": 0.00010269515886648154,
+ "n_head": 4
+}
\ No newline at end of file
diff --git a/ray_results/test_experiment/train_model_a01c9b2d_16_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_17-44-50/params.pkl b/ray_results/test_experiment/train_model_a01c9b2d_16_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_17-44-50/params.pkl
new file mode 100644
index 0000000000000000000000000000000000000000..450eb78874348bec9322c08bf9cba50d42dd5617
Binary files /dev/null and b/ray_results/test_experiment/train_model_a01c9b2d_16_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_17-44-50/params.pkl differ
diff --git a/ray_results/test_experiment/train_model_a01c9b2d_16_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_17-44-50/progress.csv b/ray_results/test_experiment/train_model_a01c9b2d_16_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_17-44-50/progress.csv
new file mode 100644
index 0000000000000000000000000000000000000000..962131dadafabe3d6da973d1f034a29a753c4d13
--- /dev/null
+++ b/ray_results/test_experiment/train_model_a01c9b2d_16_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_17-44-50/progress.csv
@@ -0,0 +1,5 @@
+loss,timestamp,checkpoint_dir_name,should_checkpoint,done,training_iteration,trial_id,date,time_this_iter_s,time_total_s,pid,hostname,node_ip,time_since_restore,iterations_since_restore
+983.9757466954509,1719330446,checkpoint_000000,True,False,1,a01c9b2d,2024-06-25_17-47-26,83.82870721817017,83.82870721817017,2195253,r8i6n2,10.159.28.60,83.82870721817017,1
+501.4262033297321,1719330517,checkpoint_000001,True,False,2,a01c9b2d,2024-06-25_17-48-37,70.83729195594788,154.66599917411804,2195253,r8i6n2,10.159.28.60,154.66599917411804,2
+362.6629641074834,1719331736,checkpoint_000002,True,False,3,a01c9b2d,2024-06-25_18-08-56,101.81216025352478,256.4781594276428,2195253,r8i6n2,10.159.28.60,101.81216025352478,1
+290.2177711246521,1719331824,checkpoint_000003,True,True,4,a01c9b2d,2024-06-25_18-10-24,87.97965884208679,344.4578182697296,2195253,r8i6n2,10.159.28.60,189.79181909561157,2
diff --git a/ray_results/test_experiment/train_model_a01c9b2d_16_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_17-44-50/result.json b/ray_results/test_experiment/train_model_a01c9b2d_16_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_17-44-50/result.json
new file mode 100644
index 0000000000000000000000000000000000000000..9899007282dd273b59b34659af32c6cd233e8bd1
--- /dev/null
+++ b/ray_results/test_experiment/train_model_a01c9b2d_16_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_17-44-50/result.json
@@ -0,0 +1,4 @@
+{"loss": 983.9757466954509, "timestamp": 1719330446, "checkpoint_dir_name": "checkpoint_000000", "should_checkpoint": true, "done": false, "training_iteration": 1, "trial_id": "a01c9b2d", "date": "2024-06-25_17-47-26", "time_this_iter_s": 83.82870721817017, "time_total_s": 83.82870721817017, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 8, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 1024, "decoder_rt_ff": 1024, "decoder_int_ff": 512, "n_head": 4, "drop_rate": 0.23338023981015343, "lr": 0.00010269515886648154, "batch_size": 1024}, "time_since_restore": 83.82870721817017, "iterations_since_restore": 1}
+{"loss": 501.4262033297321, "timestamp": 1719330517, "checkpoint_dir_name": "checkpoint_000001", "should_checkpoint": true, "done": false, "training_iteration": 2, "trial_id": "a01c9b2d", "date": "2024-06-25_17-48-37", "time_this_iter_s": 70.83729195594788, "time_total_s": 154.66599917411804, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 8, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 1024, "decoder_rt_ff": 1024, "decoder_int_ff": 512, "n_head": 4, "drop_rate": 0.23338023981015343, "lr": 0.00010269515886648154, "batch_size": 1024}, "time_since_restore": 154.66599917411804, "iterations_since_restore": 2}
+{"loss": 362.6629641074834, "timestamp": 1719331736, "checkpoint_dir_name": "checkpoint_000002", "should_checkpoint": true, "done": false, "training_iteration": 3, "trial_id": "a01c9b2d", "date": "2024-06-25_18-08-56", "time_this_iter_s": 101.81216025352478, "time_total_s": 256.4781594276428, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 8, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 1024, "decoder_rt_ff": 1024, "decoder_int_ff": 512, "n_head": 4, "drop_rate": 0.23338023981015343, "lr": 0.00010269515886648154, "batch_size": 1024}, "time_since_restore": 101.81216025352478, "iterations_since_restore": 1}
+{"loss": 290.2177711246521, "timestamp": 1719331824, "checkpoint_dir_name": "checkpoint_000003", "should_checkpoint": true, "done": true, "training_iteration": 4, "trial_id": "a01c9b2d", "date": "2024-06-25_18-10-24", "time_this_iter_s": 87.97965884208679, "time_total_s": 344.4578182697296, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 8, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 1024, "decoder_rt_ff": 1024, "decoder_int_ff": 512, "n_head": 4, "drop_rate": 0.23338023981015343, "lr": 0.00010269515886648154, "batch_size": 1024}, "time_since_restore": 189.79181909561157, "iterations_since_restore": 2}
diff --git a/ray_results/test_experiment/train_model_a0e8b2af_1_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=8,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop_r_2024-06-25_16-07-48/events.out.tfevents.1719324484.r3i5n6 b/ray_results/test_experiment/train_model_a0e8b2af_1_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=8,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop_r_2024-06-25_16-07-48/events.out.tfevents.1719324484.r3i5n6
new file mode 100644
index 0000000000000000000000000000000000000000..c7b5f480763345b2895ae6385b6313c3e4b6ad63
Binary files /dev/null and b/ray_results/test_experiment/train_model_a0e8b2af_1_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=8,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop_r_2024-06-25_16-07-48/events.out.tfevents.1719324484.r3i5n6 differ
diff --git a/ray_results/test_experiment/train_model_a0e8b2af_1_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=8,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop_r_2024-06-25_16-07-48/params.json b/ray_results/test_experiment/train_model_a0e8b2af_1_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=8,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop_r_2024-06-25_16-07-48/params.json
new file mode 100644
index 0000000000000000000000000000000000000000..2acdbf55bfeb48321d70a874c947f96ec7996f37
--- /dev/null
+++ b/ray_results/test_experiment/train_model_a0e8b2af_1_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=8,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop_r_2024-06-25_16-07-48/params.json
@@ -0,0 +1,13 @@
+{
+ "batch_size": 1024,
+ "decoder_int_ff": 512,
+ "decoder_int_num_layer": 8,
+ "decoder_rt_ff": 1024,
+ "decoder_rt_num_layer": 1,
+ "drop_rate": 0.4534009171604666,
+ "embedding_dim": 256,
+ "encoder_ff": 2048,
+ "encoder_num_layer": 2,
+ "lr": 0.019472817375775914,
+ "n_head": 8
+}
\ No newline at end of file
diff --git a/ray_results/test_experiment/train_model_a0e8b2af_1_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=8,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop_r_2024-06-25_16-07-48/params.pkl b/ray_results/test_experiment/train_model_a0e8b2af_1_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=8,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop_r_2024-06-25_16-07-48/params.pkl
new file mode 100644
index 0000000000000000000000000000000000000000..b954d8aa1d31bdc54af695f028f87b2d395e5d26
Binary files /dev/null and b/ray_results/test_experiment/train_model_a0e8b2af_1_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=8,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop_r_2024-06-25_16-07-48/params.pkl differ
diff --git a/ray_results/test_experiment/train_model_a0e8b2af_1_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=8,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop_r_2024-06-25_16-07-48/progress.csv b/ray_results/test_experiment/train_model_a0e8b2af_1_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=8,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop_r_2024-06-25_16-07-48/progress.csv
new file mode 100644
index 0000000000000000000000000000000000000000..2ab7a964b5cde6a77cb9add891113b35cf121142
--- /dev/null
+++ b/ray_results/test_experiment/train_model_a0e8b2af_1_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=8,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop_r_2024-06-25_16-07-48/progress.csv
@@ -0,0 +1,2 @@
+loss,timestamp,checkpoint_dir_name,should_checkpoint,done,training_iteration,trial_id,date,time_this_iter_s,time_total_s,pid,hostname,node_ip,time_since_restore,iterations_since_restore
+2773.484795037217,1719324574,checkpoint_000000,True,False,1,a0e8b2af,2024-06-25_16-09-34,89.29933595657349,89.29933595657349,69318,r3i5n6,10.159.8.159,89.29933595657349,1
diff --git a/ray_results/test_experiment/train_model_a0e8b2af_1_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=8,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop_r_2024-06-25_16-07-48/result.json b/ray_results/test_experiment/train_model_a0e8b2af_1_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=8,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop_r_2024-06-25_16-07-48/result.json
new file mode 100644
index 0000000000000000000000000000000000000000..d494e654bf5095745a0f0a2e426f142a70b492ba
--- /dev/null
+++ b/ray_results/test_experiment/train_model_a0e8b2af_1_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=8,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop_r_2024-06-25_16-07-48/result.json
@@ -0,0 +1 @@
+{"loss": 2773.484795037217, "timestamp": 1719324574, "checkpoint_dir_name": "checkpoint_000000", "should_checkpoint": true, "done": false, "training_iteration": 1, "trial_id": "a0e8b2af", "date": "2024-06-25_16-09-34", "time_this_iter_s": 89.29933595657349, "time_total_s": 89.29933595657349, "pid": 69318, "hostname": "r3i5n6", "node_ip": "10.159.8.159", "config": {"encoder_num_layer": 2, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 8, "embedding_dim": 256, "encoder_ff": 2048, "decoder_rt_ff": 1024, "decoder_int_ff": 512, "n_head": 8, "drop_rate": 0.4534009171604666, "lr": 0.019472817375775914, "batch_size": 1024}, "time_since_restore": 89.29933595657349, "iterations_since_restore": 1}
diff --git a/ray_results/test_experiment/train_model_ad12346e_12_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_17-36-47/events.out.tfevents.1719329892.r8i6n2 b/ray_results/test_experiment/train_model_ad12346e_12_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_17-36-47/events.out.tfevents.1719329892.r8i6n2
new file mode 100644
index 0000000000000000000000000000000000000000..f0a469aba51f5e59528842fed0cf9c272fd182d1
Binary files /dev/null and b/ray_results/test_experiment/train_model_ad12346e_12_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_17-36-47/events.out.tfevents.1719329892.r8i6n2 differ
diff --git a/ray_results/test_experiment/train_model_ad12346e_12_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_17-36-47/params.json b/ray_results/test_experiment/train_model_ad12346e_12_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_17-36-47/params.json
new file mode 100644
index 0000000000000000000000000000000000000000..d8b9b667c5492071e962c2aa2f698a2eb3471137
--- /dev/null
+++ b/ray_results/test_experiment/train_model_ad12346e_12_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_17-36-47/params.json
@@ -0,0 +1,13 @@
+{
+ "batch_size": 2048,
+ "decoder_int_ff": 512,
+ "decoder_int_num_layer": 1,
+ "decoder_rt_ff": 1024,
+ "decoder_rt_num_layer": 1,
+ "drop_rate": 0.9358999268282086,
+ "embedding_dim": 1024,
+ "encoder_ff": 1024,
+ "encoder_num_layer": 1,
+ "lr": 0.00020838214141938324,
+ "n_head": 4
+}
\ No newline at end of file
diff --git a/ray_results/test_experiment/train_model_ad12346e_12_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_17-36-47/params.pkl b/ray_results/test_experiment/train_model_ad12346e_12_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_17-36-47/params.pkl
new file mode 100644
index 0000000000000000000000000000000000000000..8c10fae9a1db4cee70588af8af9e5116360d6216
Binary files /dev/null and b/ray_results/test_experiment/train_model_ad12346e_12_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_17-36-47/params.pkl differ
diff --git a/ray_results/test_experiment/train_model_ad12346e_12_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_17-36-47/progress.csv b/ray_results/test_experiment/train_model_ad12346e_12_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_17-36-47/progress.csv
new file mode 100644
index 0000000000000000000000000000000000000000..9c22dec332e0d40348badc43ccc82d7b472f75c2
--- /dev/null
+++ b/ray_results/test_experiment/train_model_ad12346e_12_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_17-36-47/progress.csv
@@ -0,0 +1,2 @@
+loss,timestamp,checkpoint_dir_name,should_checkpoint,done,training_iteration,trial_id,date,time_this_iter_s,time_total_s,pid,hostname,node_ip,time_since_restore,iterations_since_restore
+2126.6908641424707,1719330099,checkpoint_000000,True,False,1,ad12346e,2024-06-25_17-41-39,206.83131527900696,206.83131527900696,2195253,r8i6n2,10.159.28.60,206.83131527900696,1
diff --git a/ray_results/test_experiment/train_model_ad12346e_12_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_17-36-47/result.json b/ray_results/test_experiment/train_model_ad12346e_12_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_17-36-47/result.json
new file mode 100644
index 0000000000000000000000000000000000000000..e6771dcd324c57470b082be984e547ac0b228c07
--- /dev/null
+++ b/ray_results/test_experiment/train_model_ad12346e_12_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_17-36-47/result.json
@@ -0,0 +1 @@
+{"loss": 2126.6908641424707, "timestamp": 1719330099, "checkpoint_dir_name": "checkpoint_000000", "should_checkpoint": true, "done": false, "training_iteration": 1, "trial_id": "ad12346e", "date": "2024-06-25_17-41-39", "time_this_iter_s": 206.83131527900696, "time_total_s": 206.83131527900696, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 1, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 1024, "encoder_ff": 1024, "decoder_rt_ff": 1024, "decoder_int_ff": 512, "n_head": 4, "drop_rate": 0.9358999268282086, "lr": 0.00020838214141938324, "batch_size": 2048}, "time_since_restore": 206.83131527900696, "iterations_since_restore": 1}
diff --git a/ray_results/test_experiment/train_model_af6d9c38_5_batch_size=2048,decoder_int_ff=1024,decoder_int_num_layer=8,decoder_rt_ff=1024,decoder_rt_num_layer=8,drop__2024-06-25_16-12-08/events.out.tfevents.1719324792.r3i5n6 b/ray_results/test_experiment/train_model_af6d9c38_5_batch_size=2048,decoder_int_ff=1024,decoder_int_num_layer=8,decoder_rt_ff=1024,decoder_rt_num_layer=8,drop__2024-06-25_16-12-08/events.out.tfevents.1719324792.r3i5n6
new file mode 100644
index 0000000000000000000000000000000000000000..f0f7171be1e2e83b5650da58c10d891a20709dc8
Binary files /dev/null and b/ray_results/test_experiment/train_model_af6d9c38_5_batch_size=2048,decoder_int_ff=1024,decoder_int_num_layer=8,decoder_rt_ff=1024,decoder_rt_num_layer=8,drop__2024-06-25_16-12-08/events.out.tfevents.1719324792.r3i5n6 differ
diff --git a/ray_results/test_experiment/train_model_af6d9c38_5_batch_size=2048,decoder_int_ff=1024,decoder_int_num_layer=8,decoder_rt_ff=1024,decoder_rt_num_layer=8,drop__2024-06-25_16-12-08/events.out.tfevents.1719327245.r3i5n6 b/ray_results/test_experiment/train_model_af6d9c38_5_batch_size=2048,decoder_int_ff=1024,decoder_int_num_layer=8,decoder_rt_ff=1024,decoder_rt_num_layer=8,drop__2024-06-25_16-12-08/events.out.tfevents.1719327245.r3i5n6
new file mode 100644
index 0000000000000000000000000000000000000000..1840bfa396cc6a9655fd8c325054ad02eb51df9d
Binary files /dev/null and b/ray_results/test_experiment/train_model_af6d9c38_5_batch_size=2048,decoder_int_ff=1024,decoder_int_num_layer=8,decoder_rt_ff=1024,decoder_rt_num_layer=8,drop__2024-06-25_16-12-08/events.out.tfevents.1719327245.r3i5n6 differ
diff --git a/ray_results/test_experiment/train_model_af6d9c38_5_batch_size=2048,decoder_int_ff=1024,decoder_int_num_layer=8,decoder_rt_ff=1024,decoder_rt_num_layer=8,drop__2024-06-25_16-12-08/params.json b/ray_results/test_experiment/train_model_af6d9c38_5_batch_size=2048,decoder_int_ff=1024,decoder_int_num_layer=8,decoder_rt_ff=1024,decoder_rt_num_layer=8,drop__2024-06-25_16-12-08/params.json
new file mode 100644
index 0000000000000000000000000000000000000000..a2c32c93fc9f21c68ff4ad1052aff65e4b1fc380
--- /dev/null
+++ b/ray_results/test_experiment/train_model_af6d9c38_5_batch_size=2048,decoder_int_ff=1024,decoder_int_num_layer=8,decoder_rt_ff=1024,decoder_rt_num_layer=8,drop__2024-06-25_16-12-08/params.json
@@ -0,0 +1,13 @@
+{
+ "batch_size": 2048,
+ "decoder_int_ff": 1024,
+ "decoder_int_num_layer": 8,
+ "decoder_rt_ff": 1024,
+ "decoder_rt_num_layer": 8,
+ "drop_rate": 0.5912116123671598,
+ "embedding_dim": 16,
+ "encoder_ff": 512,
+ "encoder_num_layer": 8,
+ "lr": 0.009202977404873457,
+ "n_head": 16
+}
\ No newline at end of file
diff --git a/ray_results/test_experiment/train_model_af6d9c38_5_batch_size=2048,decoder_int_ff=1024,decoder_int_num_layer=8,decoder_rt_ff=1024,decoder_rt_num_layer=8,drop__2024-06-25_16-12-08/params.pkl b/ray_results/test_experiment/train_model_af6d9c38_5_batch_size=2048,decoder_int_ff=1024,decoder_int_num_layer=8,decoder_rt_ff=1024,decoder_rt_num_layer=8,drop__2024-06-25_16-12-08/params.pkl
new file mode 100644
index 0000000000000000000000000000000000000000..4b516bb5606b09c8d54c523cf3bed0451adcc9d3
Binary files /dev/null and b/ray_results/test_experiment/train_model_af6d9c38_5_batch_size=2048,decoder_int_ff=1024,decoder_int_num_layer=8,decoder_rt_ff=1024,decoder_rt_num_layer=8,drop__2024-06-25_16-12-08/params.pkl differ
diff --git a/ray_results/test_experiment/train_model_af6d9c38_5_batch_size=2048,decoder_int_ff=1024,decoder_int_num_layer=8,decoder_rt_ff=1024,decoder_rt_num_layer=8,drop__2024-06-25_16-12-08/progress.csv b/ray_results/test_experiment/train_model_af6d9c38_5_batch_size=2048,decoder_int_ff=1024,decoder_int_num_layer=8,decoder_rt_ff=1024,decoder_rt_num_layer=8,drop__2024-06-25_16-12-08/progress.csv
new file mode 100644
index 0000000000000000000000000000000000000000..7becd21f153d613bc6f46b641ea9315fb8b6d20b
--- /dev/null
+++ b/ray_results/test_experiment/train_model_af6d9c38_5_batch_size=2048,decoder_int_ff=1024,decoder_int_num_layer=8,decoder_rt_ff=1024,decoder_rt_num_layer=8,drop__2024-06-25_16-12-08/progress.csv
@@ -0,0 +1,3 @@
+loss,timestamp,checkpoint_dir_name,should_checkpoint,done,training_iteration,trial_id,date,time_this_iter_s,time_total_s,pid,hostname,node_ip,time_since_restore,iterations_since_restore
+1987.8132929764395,1719324918,checkpoint_000000,True,False,1,af6d9c38,2024-06-25_16-15-18,125.93394875526428,125.93394875526428,69318,r3i5n6,10.159.8.159,125.93394875526428,1
+1982.2825418307086,1719327372,checkpoint_000001,True,False,2,af6d9c38,2024-06-25_16-56-12,126.36091089248657,252.29485964775085,130020,r3i5n6,10.159.8.159,126.36091089248657,1
diff --git a/ray_results/test_experiment/train_model_af6d9c38_5_batch_size=2048,decoder_int_ff=1024,decoder_int_num_layer=8,decoder_rt_ff=1024,decoder_rt_num_layer=8,drop__2024-06-25_16-12-08/result.json b/ray_results/test_experiment/train_model_af6d9c38_5_batch_size=2048,decoder_int_ff=1024,decoder_int_num_layer=8,decoder_rt_ff=1024,decoder_rt_num_layer=8,drop__2024-06-25_16-12-08/result.json
new file mode 100644
index 0000000000000000000000000000000000000000..e9b68a0743b1db1c75e47798f014e42e48243658
--- /dev/null
+++ b/ray_results/test_experiment/train_model_af6d9c38_5_batch_size=2048,decoder_int_ff=1024,decoder_int_num_layer=8,decoder_rt_ff=1024,decoder_rt_num_layer=8,drop__2024-06-25_16-12-08/result.json
@@ -0,0 +1,2 @@
+{"loss": 1987.8132929764395, "timestamp": 1719324918, "checkpoint_dir_name": "checkpoint_000000", "should_checkpoint": true, "done": false, "training_iteration": 1, "trial_id": "af6d9c38", "date": "2024-06-25_16-15-18", "time_this_iter_s": 125.93394875526428, "time_total_s": 125.93394875526428, "pid": 69318, "hostname": "r3i5n6", "node_ip": "10.159.8.159", "config": {"encoder_num_layer": 8, "decoder_rt_num_layer": 8, "decoder_int_num_layer": 8, "embedding_dim": 16, "encoder_ff": 512, "decoder_rt_ff": 1024, "decoder_int_ff": 1024, "n_head": 16, "drop_rate": 0.5912116123671598, "lr": 0.009202977404873457, "batch_size": 2048}, "time_since_restore": 125.93394875526428, "iterations_since_restore": 1}
+{"loss": 1982.2825418307086, "timestamp": 1719327372, "checkpoint_dir_name": "checkpoint_000001", "should_checkpoint": true, "done": false, "training_iteration": 2, "trial_id": "af6d9c38", "date": "2024-06-25_16-56-12", "time_this_iter_s": 126.36091089248657, "time_total_s": 252.29485964775085, "pid": 130020, "hostname": "r3i5n6", "node_ip": "10.159.8.159", "config": {"encoder_num_layer": 8, "decoder_rt_num_layer": 8, "decoder_int_num_layer": 8, "embedding_dim": 16, "encoder_ff": 512, "decoder_rt_ff": 1024, "decoder_int_ff": 1024, "n_head": 16, "drop_rate": 0.5912116123671598, "lr": 0.009202977404873457, "batch_size": 2048}, "time_since_restore": 126.36091089248657, "iterations_since_restore": 1}
diff --git a/ray_results/test_experiment/train_model_b53cfe05_16_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=8,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_16-43-18/events.out.tfevents.1719326792.r3i5n6 b/ray_results/test_experiment/train_model_b53cfe05_16_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=8,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_16-43-18/events.out.tfevents.1719326792.r3i5n6
new file mode 100644
index 0000000000000000000000000000000000000000..84159e01221465ea023e494cc53832129b11d176
Binary files /dev/null and b/ray_results/test_experiment/train_model_b53cfe05_16_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=8,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_16-43-18/events.out.tfevents.1719326792.r3i5n6 differ
diff --git a/ray_results/test_experiment/train_model_b53cfe05_16_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=8,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_16-43-18/params.json b/ray_results/test_experiment/train_model_b53cfe05_16_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=8,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_16-43-18/params.json
new file mode 100644
index 0000000000000000000000000000000000000000..bbdf643cadd5c33ca5cb0e8c4f82620d2e5fdd5b
--- /dev/null
+++ b/ray_results/test_experiment/train_model_b53cfe05_16_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=8,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_16-43-18/params.json
@@ -0,0 +1,13 @@
+{
+ "batch_size": 2048,
+ "decoder_int_ff": 512,
+ "decoder_int_num_layer": 8,
+ "decoder_rt_ff": 512,
+ "decoder_rt_num_layer": 2,
+ "drop_rate": 0.9936594412219328,
+ "embedding_dim": 64,
+ "encoder_ff": 512,
+ "encoder_num_layer": 2,
+ "lr": 0.000836614555813242,
+ "n_head": 4
+}
\ No newline at end of file
diff --git a/ray_results/test_experiment/train_model_b53cfe05_16_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=8,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_16-43-18/params.pkl b/ray_results/test_experiment/train_model_b53cfe05_16_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=8,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_16-43-18/params.pkl
new file mode 100644
index 0000000000000000000000000000000000000000..523a2a3cdde8d8143f3e74b4e9e1f7a93eaaaf31
Binary files /dev/null and b/ray_results/test_experiment/train_model_b53cfe05_16_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=8,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_16-43-18/params.pkl differ
diff --git a/ray_results/test_experiment/train_model_b53cfe05_16_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=8,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_16-43-18/progress.csv b/ray_results/test_experiment/train_model_b53cfe05_16_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=8,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_16-43-18/progress.csv
new file mode 100644
index 0000000000000000000000000000000000000000..81cd2257717d5e1c24862e510ffb938d1e867565
--- /dev/null
+++ b/ray_results/test_experiment/train_model_b53cfe05_16_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=8,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_16-43-18/progress.csv
@@ -0,0 +1,2 @@
+loss,timestamp,checkpoint_dir_name,should_checkpoint,done,training_iteration,trial_id,date,time_this_iter_s,time_total_s,pid,hostname,node_ip,time_since_restore,iterations_since_restore
+3022.543074480192,1719326860,checkpoint_000000,True,False,1,b53cfe05,2024-06-25_16-47-40,67.51874232292175,67.51874232292175,122466,r3i5n6,10.159.8.159,67.51874232292175,1
diff --git a/ray_results/test_experiment/train_model_b53cfe05_16_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=8,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_16-43-18/result.json b/ray_results/test_experiment/train_model_b53cfe05_16_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=8,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_16-43-18/result.json
new file mode 100644
index 0000000000000000000000000000000000000000..5643fc7983c246ec0818ceb24833e39f5daa4b66
--- /dev/null
+++ b/ray_results/test_experiment/train_model_b53cfe05_16_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=8,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_16-43-18/result.json
@@ -0,0 +1 @@
+{"loss": 3022.543074480192, "timestamp": 1719326860, "checkpoint_dir_name": "checkpoint_000000", "should_checkpoint": true, "done": false, "training_iteration": 1, "trial_id": "b53cfe05", "date": "2024-06-25_16-47-40", "time_this_iter_s": 67.51874232292175, "time_total_s": 67.51874232292175, "pid": 122466, "hostname": "r3i5n6", "node_ip": "10.159.8.159", "config": {"encoder_num_layer": 2, "decoder_rt_num_layer": 2, "decoder_int_num_layer": 8, "embedding_dim": 64, "encoder_ff": 512, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 4, "drop_rate": 0.9936594412219328, "lr": 0.000836614555813242, "batch_size": 2048}, "time_since_restore": 67.51874232292175, "iterations_since_restore": 1}
diff --git a/ray_results/test_experiment/train_model_b577349d_6_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=2,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop_r_2024-06-25_16-13-12/events.out.tfevents.1719324918.r3i5n6 b/ray_results/test_experiment/train_model_b577349d_6_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=2,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop_r_2024-06-25_16-13-12/events.out.tfevents.1719324918.r3i5n6
new file mode 100644
index 0000000000000000000000000000000000000000..5b02452d78cd4c9ff5c103f3cc7a608047903a81
Binary files /dev/null and b/ray_results/test_experiment/train_model_b577349d_6_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=2,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop_r_2024-06-25_16-13-12/events.out.tfevents.1719324918.r3i5n6 differ
diff --git a/ray_results/test_experiment/train_model_b577349d_6_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=2,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop_r_2024-06-25_16-13-12/events.out.tfevents.1719327372.r3i5n6 b/ray_results/test_experiment/train_model_b577349d_6_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=2,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop_r_2024-06-25_16-13-12/events.out.tfevents.1719327372.r3i5n6
new file mode 100644
index 0000000000000000000000000000000000000000..06891b4c4f58089104309f635f7954f62a6f1cdf
Binary files /dev/null and b/ray_results/test_experiment/train_model_b577349d_6_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=2,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop_r_2024-06-25_16-13-12/events.out.tfevents.1719327372.r3i5n6 differ
diff --git a/ray_results/test_experiment/train_model_b577349d_6_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=2,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop_r_2024-06-25_16-13-12/params.json b/ray_results/test_experiment/train_model_b577349d_6_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=2,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop_r_2024-06-25_16-13-12/params.json
new file mode 100644
index 0000000000000000000000000000000000000000..02e2b5410abf0385d0c5378fef9b9746124f754c
--- /dev/null
+++ b/ray_results/test_experiment/train_model_b577349d_6_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=2,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop_r_2024-06-25_16-13-12/params.json
@@ -0,0 +1,13 @@
+{
+ "batch_size": 2048,
+ "decoder_int_ff": 512,
+ "decoder_int_num_layer": 2,
+ "decoder_rt_ff": 1024,
+ "decoder_rt_num_layer": 1,
+ "drop_rate": 0.2945022566291742,
+ "embedding_dim": 64,
+ "encoder_ff": 2048,
+ "encoder_num_layer": 8,
+ "lr": 0.000496318687335349,
+ "n_head": 2
+}
\ No newline at end of file
diff --git a/ray_results/test_experiment/train_model_b577349d_6_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=2,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop_r_2024-06-25_16-13-12/params.pkl b/ray_results/test_experiment/train_model_b577349d_6_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=2,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop_r_2024-06-25_16-13-12/params.pkl
new file mode 100644
index 0000000000000000000000000000000000000000..e91d5dbb24c552bd6ce68238e51e96d05f3fa139
Binary files /dev/null and b/ray_results/test_experiment/train_model_b577349d_6_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=2,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop_r_2024-06-25_16-13-12/params.pkl differ
diff --git a/ray_results/test_experiment/train_model_b577349d_6_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=2,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop_r_2024-06-25_16-13-12/progress.csv b/ray_results/test_experiment/train_model_b577349d_6_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=2,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop_r_2024-06-25_16-13-12/progress.csv
new file mode 100644
index 0000000000000000000000000000000000000000..82a172d3e1e8d1d6e99432f9ead767568891cda3
--- /dev/null
+++ b/ray_results/test_experiment/train_model_b577349d_6_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=2,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop_r_2024-06-25_16-13-12/progress.csv
@@ -0,0 +1,3 @@
+loss,timestamp,checkpoint_dir_name,should_checkpoint,done,training_iteration,trial_id,date,time_this_iter_s,time_total_s,pid,hostname,node_ip,time_since_restore,iterations_since_restore
+1958.5306944359006,1719325014,checkpoint_000000,True,False,1,b577349d,2024-06-25_16-16-54,95.59347629547119,95.59347629547119,69318,r3i5n6,10.159.8.159,95.59347629547119,1
+1958.5695377860482,1719327468,checkpoint_000001,True,False,2,b577349d,2024-06-25_16-57-48,96.6448974609375,192.2383737564087,130020,r3i5n6,10.159.8.159,96.6448974609375,1
diff --git a/ray_results/test_experiment/train_model_b577349d_6_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=2,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop_r_2024-06-25_16-13-12/result.json b/ray_results/test_experiment/train_model_b577349d_6_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=2,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop_r_2024-06-25_16-13-12/result.json
new file mode 100644
index 0000000000000000000000000000000000000000..8befa381317c77ec5a9b21d22b2b54ceb1be0644
--- /dev/null
+++ b/ray_results/test_experiment/train_model_b577349d_6_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=2,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop_r_2024-06-25_16-13-12/result.json
@@ -0,0 +1,2 @@
+{"loss": 1958.5306944359006, "timestamp": 1719325014, "checkpoint_dir_name": "checkpoint_000000", "should_checkpoint": true, "done": false, "training_iteration": 1, "trial_id": "b577349d", "date": "2024-06-25_16-16-54", "time_this_iter_s": 95.59347629547119, "time_total_s": 95.59347629547119, "pid": 69318, "hostname": "r3i5n6", "node_ip": "10.159.8.159", "config": {"encoder_num_layer": 8, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 2, "embedding_dim": 64, "encoder_ff": 2048, "decoder_rt_ff": 1024, "decoder_int_ff": 512, "n_head": 2, "drop_rate": 0.2945022566291742, "lr": 0.000496318687335349, "batch_size": 2048}, "time_since_restore": 95.59347629547119, "iterations_since_restore": 1}
+{"loss": 1958.5695377860482, "timestamp": 1719327468, "checkpoint_dir_name": "checkpoint_000001", "should_checkpoint": true, "done": false, "training_iteration": 2, "trial_id": "b577349d", "date": "2024-06-25_16-57-48", "time_this_iter_s": 96.6448974609375, "time_total_s": 192.2383737564087, "pid": 130020, "hostname": "r3i5n6", "node_ip": "10.159.8.159", "config": {"encoder_num_layer": 8, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 2, "embedding_dim": 64, "encoder_ff": 2048, "decoder_rt_ff": 1024, "decoder_int_ff": 512, "n_head": 2, "drop_rate": 0.2945022566291742, "lr": 0.000496318687335349, "batch_size": 2048}, "time_since_restore": 96.6448974609375, "iterations_since_restore": 1}
diff --git a/ray_results/test_experiment/train_model_bea15841_17_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=4,drop__2024-06-25_17-46-02/events.out.tfevents.1719332768.r8i6n2 b/ray_results/test_experiment/train_model_bea15841_17_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=4,drop__2024-06-25_17-46-02/events.out.tfevents.1719332768.r8i6n2
new file mode 100644
index 0000000000000000000000000000000000000000..2e003b6b19e54d5f43ca7c25feeb64658d56a1f6
Binary files /dev/null and b/ray_results/test_experiment/train_model_bea15841_17_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=4,drop__2024-06-25_17-46-02/events.out.tfevents.1719332768.r8i6n2 differ
diff --git a/ray_results/test_experiment/train_model_bea15841_17_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=4,drop__2024-06-25_17-46-02/params.json b/ray_results/test_experiment/train_model_bea15841_17_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=4,drop__2024-06-25_17-46-02/params.json
new file mode 100644
index 0000000000000000000000000000000000000000..7fa69d655453b43365521eb78094906a7fdd4cad
--- /dev/null
+++ b/ray_results/test_experiment/train_model_bea15841_17_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=4,drop__2024-06-25_17-46-02/params.json
@@ -0,0 +1,13 @@
+{
+ "batch_size": 1024,
+ "decoder_int_ff": 512,
+ "decoder_int_num_layer": 1,
+ "decoder_rt_ff": 1024,
+ "decoder_rt_num_layer": 4,
+ "drop_rate": 0.8101220088903347,
+ "embedding_dim": 64,
+ "encoder_ff": 1024,
+ "encoder_num_layer": 4,
+ "lr": 0.0005780102991375737,
+ "n_head": 4
+}
\ No newline at end of file
diff --git a/ray_results/test_experiment/train_model_bea15841_17_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=4,drop__2024-06-25_17-46-02/params.pkl b/ray_results/test_experiment/train_model_bea15841_17_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=4,drop__2024-06-25_17-46-02/params.pkl
new file mode 100644
index 0000000000000000000000000000000000000000..70deec0b1ae58c8b4addc77a860c6182b5ffdb33
Binary files /dev/null and b/ray_results/test_experiment/train_model_bea15841_17_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=4,drop__2024-06-25_17-46-02/params.pkl differ
diff --git a/ray_results/test_experiment/train_model_bea15841_17_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=4,drop__2024-06-25_17-46-02/progress.csv b/ray_results/test_experiment/train_model_bea15841_17_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=4,drop__2024-06-25_17-46-02/progress.csv
new file mode 100644
index 0000000000000000000000000000000000000000..2fd8fe1a77236df53b207293e63bb6686fb1e6ed
--- /dev/null
+++ b/ray_results/test_experiment/train_model_bea15841_17_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=4,drop__2024-06-25_17-46-02/progress.csv
@@ -0,0 +1,3 @@
+loss,timestamp,checkpoint_dir_name,should_checkpoint,done,training_iteration,trial_id,date,time_this_iter_s,time_total_s,pid,hostname,node_ip,time_since_restore,iterations_since_restore
+2012.1837220680056,1719332857,checkpoint_000000,True,False,1,bea15841,2024-06-25_18-27-37,89.14307808876038,89.14307808876038,2195253,r8i6n2,10.159.28.60,89.14307808876038,1
+2008.3096395177165,1719332932,checkpoint_000001,True,False,2,bea15841,2024-06-25_18-28-52,75.03496551513672,164.1780436038971,2195253,r8i6n2,10.159.28.60,164.1780436038971,2
diff --git a/ray_results/test_experiment/train_model_bea15841_17_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=4,drop__2024-06-25_17-46-02/result.json b/ray_results/test_experiment/train_model_bea15841_17_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=4,drop__2024-06-25_17-46-02/result.json
new file mode 100644
index 0000000000000000000000000000000000000000..bbf7ae8deab991780f54329ecb38475be30ad979
--- /dev/null
+++ b/ray_results/test_experiment/train_model_bea15841_17_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=4,drop__2024-06-25_17-46-02/result.json
@@ -0,0 +1,2 @@
+{"loss": 2012.1837220680056, "timestamp": 1719332857, "checkpoint_dir_name": "checkpoint_000000", "should_checkpoint": true, "done": false, "training_iteration": 1, "trial_id": "bea15841", "date": "2024-06-25_18-27-37", "time_this_iter_s": 89.14307808876038, "time_total_s": 89.14307808876038, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 4, "decoder_rt_num_layer": 4, "decoder_int_num_layer": 1, "embedding_dim": 64, "encoder_ff": 1024, "decoder_rt_ff": 1024, "decoder_int_ff": 512, "n_head": 4, "drop_rate": 0.8101220088903347, "lr": 0.0005780102991375737, "batch_size": 1024}, "time_since_restore": 89.14307808876038, "iterations_since_restore": 1}
+{"loss": 2008.3096395177165, "timestamp": 1719332932, "checkpoint_dir_name": "checkpoint_000001", "should_checkpoint": true, "done": false, "training_iteration": 2, "trial_id": "bea15841", "date": "2024-06-25_18-28-52", "time_this_iter_s": 75.03496551513672, "time_total_s": 164.1780436038971, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 4, "decoder_rt_num_layer": 4, "decoder_int_num_layer": 1, "embedding_dim": 64, "encoder_ff": 1024, "decoder_rt_ff": 1024, "decoder_int_ff": 512, "n_head": 4, "drop_rate": 0.8101220088903347, "lr": 0.0005780102991375737, "batch_size": 1024}, "time_since_restore": 164.1780436038971, "iterations_since_restore": 2}
diff --git a/ray_results/test_experiment/train_model_c2c99680_18_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_18-26-08/events.out.tfevents.1719332932.r8i6n2 b/ray_results/test_experiment/train_model_c2c99680_18_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_18-26-08/events.out.tfevents.1719332932.r8i6n2
new file mode 100644
index 0000000000000000000000000000000000000000..80186fdfd44a0eb3cc6b9b6fd09db2190d925fe2
Binary files /dev/null and b/ray_results/test_experiment/train_model_c2c99680_18_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_18-26-08/events.out.tfevents.1719332932.r8i6n2 differ
diff --git a/ray_results/test_experiment/train_model_c2c99680_18_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_18-26-08/events.out.tfevents.1719337296.r8i6n2 b/ray_results/test_experiment/train_model_c2c99680_18_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_18-26-08/events.out.tfevents.1719337296.r8i6n2
new file mode 100644
index 0000000000000000000000000000000000000000..8cc354ae2695e10c3f3312f2dd0ea2ed51a45b18
Binary files /dev/null and b/ray_results/test_experiment/train_model_c2c99680_18_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_18-26-08/events.out.tfevents.1719337296.r8i6n2 differ
diff --git a/ray_results/test_experiment/train_model_c2c99680_18_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_18-26-08/params.json b/ray_results/test_experiment/train_model_c2c99680_18_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_18-26-08/params.json
new file mode 100644
index 0000000000000000000000000000000000000000..073d45844bf3552fd66e4ff0fad38953785da549
--- /dev/null
+++ b/ray_results/test_experiment/train_model_c2c99680_18_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_18-26-08/params.json
@@ -0,0 +1,13 @@
+{
+ "batch_size": 2048,
+ "decoder_int_ff": 512,
+ "decoder_int_num_layer": 1,
+ "decoder_rt_ff": 1024,
+ "decoder_rt_num_layer": 1,
+ "drop_rate": 0.5887703154000944,
+ "embedding_dim": 1024,
+ "encoder_ff": 1024,
+ "encoder_num_layer": 4,
+ "lr": 0.0008721653331207079,
+ "n_head": 1
+}
\ No newline at end of file
diff --git a/ray_results/test_experiment/train_model_c2c99680_18_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_18-26-08/params.pkl b/ray_results/test_experiment/train_model_c2c99680_18_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_18-26-08/params.pkl
new file mode 100644
index 0000000000000000000000000000000000000000..82eaeb8610755e83829654c9acc2d2d1ef3e9c33
Binary files /dev/null and b/ray_results/test_experiment/train_model_c2c99680_18_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_18-26-08/params.pkl differ
diff --git a/ray_results/test_experiment/train_model_c2c99680_18_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_18-26-08/progress.csv b/ray_results/test_experiment/train_model_c2c99680_18_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_18-26-08/progress.csv
new file mode 100644
index 0000000000000000000000000000000000000000..74c0983943a91dc79efb983783d582bde062e0bf
--- /dev/null
+++ b/ray_results/test_experiment/train_model_c2c99680_18_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_18-26-08/progress.csv
@@ -0,0 +1,5 @@
+loss,timestamp,checkpoint_dir_name,should_checkpoint,done,training_iteration,trial_id,date,time_this_iter_s,time_total_s,pid,hostname,node_ip,time_since_restore,iterations_since_restore
+1987.1915456216166,1719333389,checkpoint_000000,True,False,1,c2c99680,2024-06-25_18-36-29,456.8247263431549,456.8247263431549,2195253,r8i6n2,10.159.28.60,456.8247263431549,1
+1982.5428274560163,1719333829,checkpoint_000001,True,False,2,c2c99680,2024-06-25_18-43-49,440.0058114528656,896.8305377960205,2195253,r8i6n2,10.159.28.60,896.8305377960205,2
+2012.3732496847317,1719337749,checkpoint_000002,True,False,3,c2c99680,2024-06-25_19-49-10,454.0293221473694,1350.85985994339,2195253,r8i6n2,10.159.28.60,454.0293221473694,1
+2023.2190220149482,1719338188,checkpoint_000003,True,False,4,c2c99680,2024-06-25_19-56-29,439.1456036567688,1790.0054636001587,2195253,r8i6n2,10.159.28.60,893.1749258041382,2
diff --git a/ray_results/test_experiment/train_model_c2c99680_18_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_18-26-08/result.json b/ray_results/test_experiment/train_model_c2c99680_18_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_18-26-08/result.json
new file mode 100644
index 0000000000000000000000000000000000000000..98eb2842bf7fce6c1291764957048b9c9eea1cc6
--- /dev/null
+++ b/ray_results/test_experiment/train_model_c2c99680_18_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_18-26-08/result.json
@@ -0,0 +1,4 @@
+{"loss": 1987.1915456216166, "timestamp": 1719333389, "checkpoint_dir_name": "checkpoint_000000", "should_checkpoint": true, "done": false, "training_iteration": 1, "trial_id": "c2c99680", "date": "2024-06-25_18-36-29", "time_this_iter_s": 456.8247263431549, "time_total_s": 456.8247263431549, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 4, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 1024, "encoder_ff": 1024, "decoder_rt_ff": 1024, "decoder_int_ff": 512, "n_head": 1, "drop_rate": 0.5887703154000944, "lr": 0.0008721653331207079, "batch_size": 2048}, "time_since_restore": 456.8247263431549, "iterations_since_restore": 1}
+{"loss": 1982.5428274560163, "timestamp": 1719333829, "checkpoint_dir_name": "checkpoint_000001", "should_checkpoint": true, "done": false, "training_iteration": 2, "trial_id": "c2c99680", "date": "2024-06-25_18-43-49", "time_this_iter_s": 440.0058114528656, "time_total_s": 896.8305377960205, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 4, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 1024, "encoder_ff": 1024, "decoder_rt_ff": 1024, "decoder_int_ff": 512, "n_head": 1, "drop_rate": 0.5887703154000944, "lr": 0.0008721653331207079, "batch_size": 2048}, "time_since_restore": 896.8305377960205, "iterations_since_restore": 2}
+{"loss": 2012.3732496847317, "timestamp": 1719337749, "checkpoint_dir_name": "checkpoint_000002", "should_checkpoint": true, "done": false, "training_iteration": 3, "trial_id": "c2c99680", "date": "2024-06-25_19-49-10", "time_this_iter_s": 454.0293221473694, "time_total_s": 1350.85985994339, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 4, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 1024, "encoder_ff": 1024, "decoder_rt_ff": 1024, "decoder_int_ff": 512, "n_head": 1, "drop_rate": 0.5887703154000944, "lr": 0.0008721653331207079, "batch_size": 2048}, "time_since_restore": 454.0293221473694, "iterations_since_restore": 1}
+{"loss": 2023.2190220149482, "timestamp": 1719338188, "checkpoint_dir_name": "checkpoint_000003", "should_checkpoint": true, "done": false, "training_iteration": 4, "trial_id": "c2c99680", "date": "2024-06-25_19-56-29", "time_this_iter_s": 439.1456036567688, "time_total_s": 1790.0054636001587, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 4, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 1024, "encoder_ff": 1024, "decoder_rt_ff": 1024, "decoder_int_ff": 512, "n_head": 1, "drop_rate": 0.5887703154000944, "lr": 0.0008721653331207079, "batch_size": 2048}, "time_since_restore": 893.1749258041382, "iterations_since_restore": 2}
diff --git a/ray_results/test_experiment/train_model_c4172973_8_batch_size=2048,decoder_int_ff=2048,decoder_int_num_layer=8,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_16-16-54/events.out.tfevents.1719325156.r3i5n6 b/ray_results/test_experiment/train_model_c4172973_8_batch_size=2048,decoder_int_ff=2048,decoder_int_num_layer=8,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_16-16-54/events.out.tfevents.1719325156.r3i5n6
new file mode 100644
index 0000000000000000000000000000000000000000..857d34b750d8b2e30b5df5659eea38d004cdbc52
Binary files /dev/null and b/ray_results/test_experiment/train_model_c4172973_8_batch_size=2048,decoder_int_ff=2048,decoder_int_num_layer=8,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_16-16-54/events.out.tfevents.1719325156.r3i5n6 differ
diff --git a/ray_results/test_experiment/train_model_c4172973_8_batch_size=2048,decoder_int_ff=2048,decoder_int_num_layer=8,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_16-16-54/events.out.tfevents.1719327610.r3i5n6 b/ray_results/test_experiment/train_model_c4172973_8_batch_size=2048,decoder_int_ff=2048,decoder_int_num_layer=8,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_16-16-54/events.out.tfevents.1719327610.r3i5n6
new file mode 100644
index 0000000000000000000000000000000000000000..be46ae5ad11034bb8f06f1cfcc1d9f6026229e6e
Binary files /dev/null and b/ray_results/test_experiment/train_model_c4172973_8_batch_size=2048,decoder_int_ff=2048,decoder_int_num_layer=8,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_16-16-54/events.out.tfevents.1719327610.r3i5n6 differ
diff --git a/ray_results/test_experiment/train_model_c4172973_8_batch_size=2048,decoder_int_ff=2048,decoder_int_num_layer=8,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_16-16-54/params.json b/ray_results/test_experiment/train_model_c4172973_8_batch_size=2048,decoder_int_ff=2048,decoder_int_num_layer=8,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_16-16-54/params.json
new file mode 100644
index 0000000000000000000000000000000000000000..f7623f2f2affd489b9e21d7e3c765fc94f49afe8
--- /dev/null
+++ b/ray_results/test_experiment/train_model_c4172973_8_batch_size=2048,decoder_int_ff=2048,decoder_int_num_layer=8,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_16-16-54/params.json
@@ -0,0 +1,13 @@
+{
+ "batch_size": 2048,
+ "decoder_int_ff": 2048,
+ "decoder_int_num_layer": 8,
+ "decoder_rt_ff": 512,
+ "decoder_rt_num_layer": 2,
+ "drop_rate": 0.4176788223006639,
+ "embedding_dim": 256,
+ "encoder_ff": 1024,
+ "encoder_num_layer": 2,
+ "lr": 0.00038150746043137153,
+ "n_head": 2
+}
\ No newline at end of file
diff --git a/ray_results/test_experiment/train_model_c4172973_8_batch_size=2048,decoder_int_ff=2048,decoder_int_num_layer=8,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_16-16-54/params.pkl b/ray_results/test_experiment/train_model_c4172973_8_batch_size=2048,decoder_int_ff=2048,decoder_int_num_layer=8,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_16-16-54/params.pkl
new file mode 100644
index 0000000000000000000000000000000000000000..22dbfe027d19fb71871939982c85229f0b51a0ef
Binary files /dev/null and b/ray_results/test_experiment/train_model_c4172973_8_batch_size=2048,decoder_int_ff=2048,decoder_int_num_layer=8,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_16-16-54/params.pkl differ
diff --git a/ray_results/test_experiment/train_model_c4172973_8_batch_size=2048,decoder_int_ff=2048,decoder_int_num_layer=8,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_16-16-54/progress.csv b/ray_results/test_experiment/train_model_c4172973_8_batch_size=2048,decoder_int_ff=2048,decoder_int_num_layer=8,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_16-16-54/progress.csv
new file mode 100644
index 0000000000000000000000000000000000000000..7ed4de839614f90d77c551e97719cd578c842279
--- /dev/null
+++ b/ray_results/test_experiment/train_model_c4172973_8_batch_size=2048,decoder_int_ff=2048,decoder_int_num_layer=8,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_16-16-54/progress.csv
@@ -0,0 +1,3 @@
+loss,timestamp,checkpoint_dir_name,should_checkpoint,done,training_iteration,trial_id,date,time_this_iter_s,time_total_s,pid,hostname,node_ip,time_since_restore,iterations_since_restore
+357.444681242695,1719325234,checkpoint_000000,True,False,1,c4172973,2024-06-25_16-20-34,78.41421294212341,78.41421294212341,69318,r3i5n6,10.159.8.159,78.41421294212341,1
+261.25574487400803,1719327690,checkpoint_000001,True,False,2,c4172973,2024-06-25_17-01-30,79.34292578697205,157.75713872909546,130020,r3i5n6,10.159.8.159,79.34292578697205,1
diff --git a/ray_results/test_experiment/train_model_c4172973_8_batch_size=2048,decoder_int_ff=2048,decoder_int_num_layer=8,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_16-16-54/result.json b/ray_results/test_experiment/train_model_c4172973_8_batch_size=2048,decoder_int_ff=2048,decoder_int_num_layer=8,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_16-16-54/result.json
new file mode 100644
index 0000000000000000000000000000000000000000..cee08e9750b7d4d1d50c49476006ee2e3615a18f
--- /dev/null
+++ b/ray_results/test_experiment/train_model_c4172973_8_batch_size=2048,decoder_int_ff=2048,decoder_int_num_layer=8,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_16-16-54/result.json
@@ -0,0 +1,2 @@
+{"loss": 357.444681242695, "timestamp": 1719325234, "checkpoint_dir_name": "checkpoint_000000", "should_checkpoint": true, "done": false, "training_iteration": 1, "trial_id": "c4172973", "date": "2024-06-25_16-20-34", "time_this_iter_s": 78.41421294212341, "time_total_s": 78.41421294212341, "pid": 69318, "hostname": "r3i5n6", "node_ip": "10.159.8.159", "config": {"encoder_num_layer": 2, "decoder_rt_num_layer": 2, "decoder_int_num_layer": 8, "embedding_dim": 256, "encoder_ff": 1024, "decoder_rt_ff": 512, "decoder_int_ff": 2048, "n_head": 2, "drop_rate": 0.4176788223006639, "lr": 0.00038150746043137153, "batch_size": 2048}, "time_since_restore": 78.41421294212341, "iterations_since_restore": 1}
+{"loss": 261.25574487400803, "timestamp": 1719327690, "checkpoint_dir_name": "checkpoint_000001", "should_checkpoint": true, "done": false, "training_iteration": 2, "trial_id": "c4172973", "date": "2024-06-25_17-01-30", "time_this_iter_s": 79.34292578697205, "time_total_s": 157.75713872909546, "pid": 130020, "hostname": "r3i5n6", "node_ip": "10.159.8.159", "config": {"encoder_num_layer": 2, "decoder_rt_num_layer": 2, "decoder_int_num_layer": 8, "embedding_dim": 256, "encoder_ff": 1024, "decoder_rt_ff": 512, "decoder_int_ff": 2048, "n_head": 2, "drop_rate": 0.4176788223006639, "lr": 0.00038150746043137153, "batch_size": 2048}, "time_since_restore": 79.34292578697205, "iterations_since_restore": 1}
diff --git a/ray_results/test_experiment/train_model_dd950274_1_batch_size=2048,decoder_int_ff=1024,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_15-19-46/events.out.tfevents.1719321603.r7i5n7 b/ray_results/test_experiment/train_model_dd950274_1_batch_size=2048,decoder_int_ff=1024,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_15-19-46/events.out.tfevents.1719321603.r7i5n7
new file mode 100644
index 0000000000000000000000000000000000000000..fa935586fae83d743c2e340fe35b13aabe51b2ae
Binary files /dev/null and b/ray_results/test_experiment/train_model_dd950274_1_batch_size=2048,decoder_int_ff=1024,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_15-19-46/events.out.tfevents.1719321603.r7i5n7 differ
diff --git a/ray_results/test_experiment/train_model_dd950274_1_batch_size=2048,decoder_int_ff=1024,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_15-19-46/params.json b/ray_results/test_experiment/train_model_dd950274_1_batch_size=2048,decoder_int_ff=1024,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_15-19-46/params.json
new file mode 100644
index 0000000000000000000000000000000000000000..6e93c3a3a8e54d866a9f5c51584d6108e62235e3
--- /dev/null
+++ b/ray_results/test_experiment/train_model_dd950274_1_batch_size=2048,decoder_int_ff=1024,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_15-19-46/params.json
@@ -0,0 +1,13 @@
+{
+ "batch_size": 2048,
+ "decoder_int_ff": 1024,
+ "decoder_int_num_layer": 1,
+ "decoder_rt_ff": 1024,
+ "decoder_rt_num_layer": 1,
+ "drop_rate": 0.7776595209025089,
+ "embedding_dim": 16,
+ "encoder_ff": 1024,
+ "encoder_num_layer": 8,
+ "lr": 0.0006106789085082022,
+ "n_head": 16
+}
\ No newline at end of file
diff --git a/ray_results/test_experiment/train_model_dd950274_1_batch_size=2048,decoder_int_ff=1024,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_15-19-46/params.pkl b/ray_results/test_experiment/train_model_dd950274_1_batch_size=2048,decoder_int_ff=1024,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_15-19-46/params.pkl
new file mode 100644
index 0000000000000000000000000000000000000000..f685246d35e73936eedd2db61601ee2b0a9426ae
Binary files /dev/null and b/ray_results/test_experiment/train_model_dd950274_1_batch_size=2048,decoder_int_ff=1024,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_15-19-46/params.pkl differ
diff --git a/ray_results/test_experiment/train_model_dd950274_1_batch_size=2048,decoder_int_ff=1024,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_15-19-46/progress.csv b/ray_results/test_experiment/train_model_dd950274_1_batch_size=2048,decoder_int_ff=1024,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_15-19-46/progress.csv
new file mode 100644
index 0000000000000000000000000000000000000000..28c92556a14f9b3e3bc0ebf1fad165d4aa69ce37
--- /dev/null
+++ b/ray_results/test_experiment/train_model_dd950274_1_batch_size=2048,decoder_int_ff=1024,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_15-19-46/progress.csv
@@ -0,0 +1,2 @@
+loss,timestamp,checkpoint_dir_name,should_checkpoint,done,training_iteration,trial_id,date,time_this_iter_s,time_total_s,pid,hostname,node_ip,time_since_restore,iterations_since_restore
+2001.7704808993603,1719321692,checkpoint_000000,True,False,1,dd950274,2024-06-25_15-21-32,89.34920597076416,89.34920597076416,355319,r7i5n7,10.159.24.56,89.34920597076416,1
diff --git a/ray_results/test_experiment/train_model_dd950274_1_batch_size=2048,decoder_int_ff=1024,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_15-19-46/result.json b/ray_results/test_experiment/train_model_dd950274_1_batch_size=2048,decoder_int_ff=1024,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_15-19-46/result.json
new file mode 100644
index 0000000000000000000000000000000000000000..c20e0ec205e155ef6a5ac9d19650e8454ec0cea8
--- /dev/null
+++ b/ray_results/test_experiment/train_model_dd950274_1_batch_size=2048,decoder_int_ff=1024,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_15-19-46/result.json
@@ -0,0 +1 @@
+{"loss": 2001.7704808993603, "timestamp": 1719321692, "checkpoint_dir_name": "checkpoint_000000", "should_checkpoint": true, "done": false, "training_iteration": 1, "trial_id": "dd950274", "date": "2024-06-25_15-21-32", "time_this_iter_s": 89.34920597076416, "time_total_s": 89.34920597076416, "pid": 355319, "hostname": "r7i5n7", "node_ip": "10.159.24.56", "config": {"encoder_num_layer": 8, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 1024, "decoder_rt_ff": 1024, "decoder_int_ff": 1024, "n_head": 16, "drop_rate": 0.7776595209025089, "lr": 0.0006106789085082022, "batch_size": 2048}, "time_since_restore": 89.34920597076416, "iterations_since_restore": 1}
diff --git a/ray_results/test_experiment/train_model_de1e9134_30_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=1,drop_r_2024-06-25_22-20-23/events.out.tfevents.1719348339.r8i6n2 b/ray_results/test_experiment/train_model_de1e9134_30_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=1,drop_r_2024-06-25_22-20-23/events.out.tfevents.1719348339.r8i6n2
new file mode 100644
index 0000000000000000000000000000000000000000..7870273a07987a5f6341b5403eed9658bab51da4
Binary files /dev/null and b/ray_results/test_experiment/train_model_de1e9134_30_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=1,drop_r_2024-06-25_22-20-23/events.out.tfevents.1719348339.r8i6n2 differ
diff --git a/ray_results/test_experiment/train_model_de1e9134_30_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=1,drop_r_2024-06-25_22-20-23/events.out.tfevents.1719355223.r8i6n2 b/ray_results/test_experiment/train_model_de1e9134_30_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=1,drop_r_2024-06-25_22-20-23/events.out.tfevents.1719355223.r8i6n2
new file mode 100644
index 0000000000000000000000000000000000000000..56e3055f9bf9fe7ae68cd9d7b51dcf121c27b383
Binary files /dev/null and b/ray_results/test_experiment/train_model_de1e9134_30_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=1,drop_r_2024-06-25_22-20-23/events.out.tfevents.1719355223.r8i6n2 differ
diff --git a/ray_results/test_experiment/train_model_de1e9134_30_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=1,drop_r_2024-06-25_22-20-23/params.json b/ray_results/test_experiment/train_model_de1e9134_30_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=1,drop_r_2024-06-25_22-20-23/params.json
new file mode 100644
index 0000000000000000000000000000000000000000..4f6c97338393cce93234669d30656ff23fac07b8
--- /dev/null
+++ b/ray_results/test_experiment/train_model_de1e9134_30_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=1,drop_r_2024-06-25_22-20-23/params.json
@@ -0,0 +1,13 @@
+{
+ "batch_size": 2048,
+ "decoder_int_ff": 512,
+ "decoder_int_num_layer": 1,
+ "decoder_rt_ff": 512,
+ "decoder_rt_num_layer": 1,
+ "drop_rate": 0.12892236942851165,
+ "embedding_dim": 16,
+ "encoder_ff": 1024,
+ "encoder_num_layer": 4,
+ "lr": 0.005626677279957184,
+ "n_head": 1
+}
\ No newline at end of file
diff --git a/ray_results/test_experiment/train_model_de1e9134_30_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=1,drop_r_2024-06-25_22-20-23/params.pkl b/ray_results/test_experiment/train_model_de1e9134_30_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=1,drop_r_2024-06-25_22-20-23/params.pkl
new file mode 100644
index 0000000000000000000000000000000000000000..f5d49d3c3306a54c36545122768fe638be6711c8
Binary files /dev/null and b/ray_results/test_experiment/train_model_de1e9134_30_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=1,drop_r_2024-06-25_22-20-23/params.pkl differ
diff --git a/ray_results/test_experiment/train_model_de1e9134_30_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=1,drop_r_2024-06-25_22-20-23/progress.csv b/ray_results/test_experiment/train_model_de1e9134_30_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=1,drop_r_2024-06-25_22-20-23/progress.csv
new file mode 100644
index 0000000000000000000000000000000000000000..771cf5990f19e892a514dd70b3205b12736bf64c
--- /dev/null
+++ b/ray_results/test_experiment/train_model_de1e9134_30_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=1,drop_r_2024-06-25_22-20-23/progress.csv
@@ -0,0 +1,11 @@
+loss,timestamp,checkpoint_dir_name,should_checkpoint,done,training_iteration,trial_id,date,time_this_iter_s,time_total_s,pid,hostname,node_ip,time_since_restore,iterations_since_restore
+1956.6991774191067,1719348406,checkpoint_000000,True,False,1,de1e9134,2024-06-25_22-46-46,67.86075568199158,67.86075568199158,2195253,r8i6n2,10.159.28.60,67.86075568199158,1
+1966.5979321096827,1719348460,checkpoint_000001,True,False,2,de1e9134,2024-06-25_22-47-40,53.399649143218994,121.26040482521057,2195253,r8i6n2,10.159.28.60,121.26040482521057,2
+1954.8947446327509,1719348513,checkpoint_000002,True,False,3,de1e9134,2024-06-25_22-48-33,53.41487240791321,174.67527723312378,2195253,r8i6n2,10.159.28.60,174.67527723312378,3
+1955.8978973148376,1719348567,checkpoint_000003,True,False,4,de1e9134,2024-06-25_22-49-27,53.3932785987854,228.06855583190918,2195253,r8i6n2,10.159.28.60,228.06855583190918,4
+1955.0890363558071,1719348620,checkpoint_000004,True,False,5,de1e9134,2024-06-25_22-50-20,53.77778649330139,281.84634232521057,2195253,r8i6n2,10.159.28.60,281.84634232521057,5
+1955.0249504029282,1719355289,checkpoint_000005,True,False,6,de1e9134,2024-06-26_00-41-30,66.45859718322754,348.3049395084381,2850562,r8i6n2,10.159.28.60,66.45859718322754,1
+1962.3684418445498,1719355342,checkpoint_000006,True,False,7,de1e9134,2024-06-26_00-42-22,52.848015785217285,401.1529552936554,2850562,r8i6n2,10.159.28.60,119.30661296844482,2
+1957.2319662739912,1719355395,checkpoint_000007,True,False,8,de1e9134,2024-06-26_00-43-15,52.90200471878052,454.0549600124359,2850562,r8i6n2,10.159.28.60,172.20861768722534,3
+1956.60619444359,1719355448,checkpoint_000008,True,False,9,de1e9134,2024-06-26_00-44-09,53.24041485786438,507.2953748703003,2850562,r8i6n2,10.159.28.60,225.44903254508972,4
+1957.9578991987573,1719355501,checkpoint_000009,True,False,10,de1e9134,2024-06-26_00-45-01,52.88777208328247,560.1831469535828,2850562,r8i6n2,10.159.28.60,278.3368046283722,5
diff --git a/ray_results/test_experiment/train_model_de1e9134_30_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=1,drop_r_2024-06-25_22-20-23/result.json b/ray_results/test_experiment/train_model_de1e9134_30_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=1,drop_r_2024-06-25_22-20-23/result.json
new file mode 100644
index 0000000000000000000000000000000000000000..2ecc83bd7d109a69079af88556b08db814acc21d
--- /dev/null
+++ b/ray_results/test_experiment/train_model_de1e9134_30_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=1,drop_r_2024-06-25_22-20-23/result.json
@@ -0,0 +1,10 @@
+{"loss": 1956.6991774191067, "timestamp": 1719348406, "checkpoint_dir_name": "checkpoint_000000", "should_checkpoint": true, "done": false, "training_iteration": 1, "trial_id": "de1e9134", "date": "2024-06-25_22-46-46", "time_this_iter_s": 67.86075568199158, "time_total_s": 67.86075568199158, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 4, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 1024, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 1, "drop_rate": 0.12892236942851165, "lr": 0.005626677279957184, "batch_size": 2048}, "time_since_restore": 67.86075568199158, "iterations_since_restore": 1}
+{"loss": 1966.5979321096827, "timestamp": 1719348460, "checkpoint_dir_name": "checkpoint_000001", "should_checkpoint": true, "done": false, "training_iteration": 2, "trial_id": "de1e9134", "date": "2024-06-25_22-47-40", "time_this_iter_s": 53.399649143218994, "time_total_s": 121.26040482521057, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 4, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 1024, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 1, "drop_rate": 0.12892236942851165, "lr": 0.005626677279957184, "batch_size": 2048}, "time_since_restore": 121.26040482521057, "iterations_since_restore": 2}
+{"loss": 1954.8947446327509, "timestamp": 1719348513, "checkpoint_dir_name": "checkpoint_000002", "should_checkpoint": true, "done": false, "training_iteration": 3, "trial_id": "de1e9134", "date": "2024-06-25_22-48-33", "time_this_iter_s": 53.41487240791321, "time_total_s": 174.67527723312378, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 4, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 1024, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 1, "drop_rate": 0.12892236942851165, "lr": 0.005626677279957184, "batch_size": 2048}, "time_since_restore": 174.67527723312378, "iterations_since_restore": 3}
+{"loss": 1955.8978973148376, "timestamp": 1719348567, "checkpoint_dir_name": "checkpoint_000003", "should_checkpoint": true, "done": false, "training_iteration": 4, "trial_id": "de1e9134", "date": "2024-06-25_22-49-27", "time_this_iter_s": 53.3932785987854, "time_total_s": 228.06855583190918, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 4, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 1024, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 1, "drop_rate": 0.12892236942851165, "lr": 0.005626677279957184, "batch_size": 2048}, "time_since_restore": 228.06855583190918, "iterations_since_restore": 4}
+{"loss": 1955.0890363558071, "timestamp": 1719348620, "checkpoint_dir_name": "checkpoint_000004", "should_checkpoint": true, "done": false, "training_iteration": 5, "trial_id": "de1e9134", "date": "2024-06-25_22-50-20", "time_this_iter_s": 53.77778649330139, "time_total_s": 281.84634232521057, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 4, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 1024, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 1, "drop_rate": 0.12892236942851165, "lr": 0.005626677279957184, "batch_size": 2048}, "time_since_restore": 281.84634232521057, "iterations_since_restore": 5}
+{"loss": 1955.0249504029282, "timestamp": 1719355289, "checkpoint_dir_name": "checkpoint_000005", "should_checkpoint": true, "done": false, "training_iteration": 6, "trial_id": "de1e9134", "date": "2024-06-26_00-41-30", "time_this_iter_s": 66.45859718322754, "time_total_s": 348.3049395084381, "pid": 2850562, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 4, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 1024, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 1, "drop_rate": 0.12892236942851165, "lr": 0.005626677279957184, "batch_size": 2048}, "time_since_restore": 66.45859718322754, "iterations_since_restore": 1}
+{"loss": 1962.3684418445498, "timestamp": 1719355342, "checkpoint_dir_name": "checkpoint_000006", "should_checkpoint": true, "done": false, "training_iteration": 7, "trial_id": "de1e9134", "date": "2024-06-26_00-42-22", "time_this_iter_s": 52.848015785217285, "time_total_s": 401.1529552936554, "pid": 2850562, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 4, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 1024, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 1, "drop_rate": 0.12892236942851165, "lr": 0.005626677279957184, "batch_size": 2048}, "time_since_restore": 119.30661296844482, "iterations_since_restore": 2}
+{"loss": 1957.2319662739912, "timestamp": 1719355395, "checkpoint_dir_name": "checkpoint_000007", "should_checkpoint": true, "done": false, "training_iteration": 8, "trial_id": "de1e9134", "date": "2024-06-26_00-43-15", "time_this_iter_s": 52.90200471878052, "time_total_s": 454.0549600124359, "pid": 2850562, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 4, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 1024, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 1, "drop_rate": 0.12892236942851165, "lr": 0.005626677279957184, "batch_size": 2048}, "time_since_restore": 172.20861768722534, "iterations_since_restore": 3}
+{"loss": 1956.60619444359, "timestamp": 1719355448, "checkpoint_dir_name": "checkpoint_000008", "should_checkpoint": true, "done": false, "training_iteration": 9, "trial_id": "de1e9134", "date": "2024-06-26_00-44-09", "time_this_iter_s": 53.24041485786438, "time_total_s": 507.2953748703003, "pid": 2850562, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 4, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 1024, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 1, "drop_rate": 0.12892236942851165, "lr": 0.005626677279957184, "batch_size": 2048}, "time_since_restore": 225.44903254508972, "iterations_since_restore": 4}
+{"loss": 1957.9578991987573, "timestamp": 1719355501, "checkpoint_dir_name": "checkpoint_000009", "should_checkpoint": true, "done": false, "training_iteration": 10, "trial_id": "de1e9134", "date": "2024-06-26_00-45-01", "time_this_iter_s": 52.88777208328247, "time_total_s": 560.1831469535828, "pid": 2850562, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 4, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 1024, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 1, "drop_rate": 0.12892236942851165, "lr": 0.005626677279957184, "batch_size": 2048}, "time_since_restore": 278.3368046283722, "iterations_since_restore": 5}
diff --git a/ray_results/test_experiment/train_model_e31d7ca7_26_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=2,drop__2024-06-25_19-18-32/events.out.tfevents.1719337143.r8i6n2 b/ray_results/test_experiment/train_model_e31d7ca7_26_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=2,drop__2024-06-25_19-18-32/events.out.tfevents.1719337143.r8i6n2
new file mode 100644
index 0000000000000000000000000000000000000000..80ebb7b865552e87926cf6142fbd51dbf047ce91
Binary files /dev/null and b/ray_results/test_experiment/train_model_e31d7ca7_26_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=2,drop__2024-06-25_19-18-32/events.out.tfevents.1719337143.r8i6n2 differ
diff --git a/ray_results/test_experiment/train_model_e31d7ca7_26_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=2,drop__2024-06-25_19-18-32/params.json b/ray_results/test_experiment/train_model_e31d7ca7_26_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=2,drop__2024-06-25_19-18-32/params.json
new file mode 100644
index 0000000000000000000000000000000000000000..d0fef20f5b1e14cfccd1aadd95e46674fa753005
--- /dev/null
+++ b/ray_results/test_experiment/train_model_e31d7ca7_26_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=2,drop__2024-06-25_19-18-32/params.json
@@ -0,0 +1,13 @@
+{
+ "batch_size": 2048,
+ "decoder_int_ff": 512,
+ "decoder_int_num_layer": 1,
+ "decoder_rt_ff": 2048,
+ "decoder_rt_num_layer": 2,
+ "drop_rate": 0.7500609138892806,
+ "embedding_dim": 64,
+ "encoder_ff": 2048,
+ "encoder_num_layer": 4,
+ "lr": 0.0713230632683364,
+ "n_head": 4
+}
\ No newline at end of file
diff --git a/ray_results/test_experiment/train_model_e31d7ca7_26_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=2,drop__2024-06-25_19-18-32/params.pkl b/ray_results/test_experiment/train_model_e31d7ca7_26_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=2,drop__2024-06-25_19-18-32/params.pkl
new file mode 100644
index 0000000000000000000000000000000000000000..97d8b946d49e4185cba644765668d2b857017c28
Binary files /dev/null and b/ray_results/test_experiment/train_model_e31d7ca7_26_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=2,drop__2024-06-25_19-18-32/params.pkl differ
diff --git a/ray_results/test_experiment/train_model_e31d7ca7_26_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=2,drop__2024-06-25_19-18-32/progress.csv b/ray_results/test_experiment/train_model_e31d7ca7_26_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=2,drop__2024-06-25_19-18-32/progress.csv
new file mode 100644
index 0000000000000000000000000000000000000000..2fa4e7c417d121cda2e5b7fbb81e7573dd73d376
--- /dev/null
+++ b/ray_results/test_experiment/train_model_e31d7ca7_26_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=2,drop__2024-06-25_19-18-32/progress.csv
@@ -0,0 +1,3 @@
+loss,timestamp,checkpoint_dir_name,should_checkpoint,done,training_iteration,trial_id,date,time_this_iter_s,time_total_s,pid,hostname,node_ip,time_since_restore,iterations_since_restore
+3842.66859736405,1719337226,checkpoint_000000,True,False,1,e31d7ca7,2024-06-25_19-40-26,83.34233164787292,83.34233164787292,2195253,r8i6n2,10.159.28.60,83.34233164787292,1
+2698.886155496432,1719337295,checkpoint_000001,True,True,2,e31d7ca7,2024-06-25_19-41-35,69.29531860351562,152.63765025138855,2195253,r8i6n2,10.159.28.60,152.63765025138855,2
diff --git a/ray_results/test_experiment/train_model_e31d7ca7_26_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=2,drop__2024-06-25_19-18-32/result.json b/ray_results/test_experiment/train_model_e31d7ca7_26_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=2,drop__2024-06-25_19-18-32/result.json
new file mode 100644
index 0000000000000000000000000000000000000000..da281f72a30edaf10244a822f7d8720886e79a3b
--- /dev/null
+++ b/ray_results/test_experiment/train_model_e31d7ca7_26_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=2,drop__2024-06-25_19-18-32/result.json
@@ -0,0 +1,2 @@
+{"loss": 3842.66859736405, "timestamp": 1719337226, "checkpoint_dir_name": "checkpoint_000000", "should_checkpoint": true, "done": false, "training_iteration": 1, "trial_id": "e31d7ca7", "date": "2024-06-25_19-40-26", "time_this_iter_s": 83.34233164787292, "time_total_s": 83.34233164787292, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 4, "decoder_rt_num_layer": 2, "decoder_int_num_layer": 1, "embedding_dim": 64, "encoder_ff": 2048, "decoder_rt_ff": 2048, "decoder_int_ff": 512, "n_head": 4, "drop_rate": 0.7500609138892806, "lr": 0.0713230632683364, "batch_size": 2048}, "time_since_restore": 83.34233164787292, "iterations_since_restore": 1}
+{"loss": 2698.886155496432, "timestamp": 1719337295, "checkpoint_dir_name": "checkpoint_000001", "should_checkpoint": true, "done": true, "training_iteration": 2, "trial_id": "e31d7ca7", "date": "2024-06-25_19-41-35", "time_this_iter_s": 69.29531860351562, "time_total_s": 152.63765025138855, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 4, "decoder_rt_num_layer": 2, "decoder_int_num_layer": 1, "embedding_dim": 64, "encoder_ff": 2048, "decoder_rt_ff": 2048, "decoder_int_ff": 512, "n_head": 4, "drop_rate": 0.7500609138892806, "lr": 0.0713230632683364, "batch_size": 2048}, "time_since_restore": 152.63765025138855, "iterations_since_restore": 2}
diff --git a/ray_results/test_experiment/train_model_e566e32a_13_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=2,drop__2024-06-25_16-40-03/events.out.tfevents.1719326489.r3i5n6 b/ray_results/test_experiment/train_model_e566e32a_13_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=2,drop__2024-06-25_16-40-03/events.out.tfevents.1719326489.r3i5n6
new file mode 100644
index 0000000000000000000000000000000000000000..3b93ff1cc2833af8ac754fc78faee86a4f4d3a97
Binary files /dev/null and b/ray_results/test_experiment/train_model_e566e32a_13_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=2,drop__2024-06-25_16-40-03/events.out.tfevents.1719326489.r3i5n6 differ
diff --git a/ray_results/test_experiment/train_model_e566e32a_13_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=2,drop__2024-06-25_16-40-03/params.json b/ray_results/test_experiment/train_model_e566e32a_13_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=2,drop__2024-06-25_16-40-03/params.json
new file mode 100644
index 0000000000000000000000000000000000000000..f5b422f7a35efedf51f4d814c54d27669631719e
--- /dev/null
+++ b/ray_results/test_experiment/train_model_e566e32a_13_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=2,drop__2024-06-25_16-40-03/params.json
@@ -0,0 +1,13 @@
+{
+ "batch_size": 2048,
+ "decoder_int_ff": 512,
+ "decoder_int_num_layer": 1,
+ "decoder_rt_ff": 2048,
+ "decoder_rt_num_layer": 2,
+ "drop_rate": 0.7731110479994988,
+ "embedding_dim": 16,
+ "encoder_ff": 1024,
+ "encoder_num_layer": 8,
+ "lr": 0.03425650698356203,
+ "n_head": 4
+}
\ No newline at end of file
diff --git a/ray_results/test_experiment/train_model_e566e32a_13_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=2,drop__2024-06-25_16-40-03/params.pkl b/ray_results/test_experiment/train_model_e566e32a_13_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=2,drop__2024-06-25_16-40-03/params.pkl
new file mode 100644
index 0000000000000000000000000000000000000000..0828fd20418e00661a7e5a866774b997b9408aaf
Binary files /dev/null and b/ray_results/test_experiment/train_model_e566e32a_13_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=2,drop__2024-06-25_16-40-03/params.pkl differ
diff --git a/ray_results/test_experiment/train_model_e566e32a_13_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=2,drop__2024-06-25_16-40-03/progress.csv b/ray_results/test_experiment/train_model_e566e32a_13_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=2,drop__2024-06-25_16-40-03/progress.csv
new file mode 100644
index 0000000000000000000000000000000000000000..151739c566c6620e35d4ed9909a44a32e70e677f
--- /dev/null
+++ b/ray_results/test_experiment/train_model_e566e32a_13_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=2,drop__2024-06-25_16-40-03/progress.csv
@@ -0,0 +1,2 @@
+loss,timestamp,checkpoint_dir_name,should_checkpoint,done,training_iteration,trial_id,date,time_this_iter_s,time_total_s,pid,hostname,node_ip,time_since_restore,iterations_since_restore
+2129.4940118264026,1719326571,checkpoint_000000,True,False,1,e566e32a,2024-06-25_16-42-51,82.61908912658691,82.61908912658691,69318,r3i5n6,10.159.8.159,82.61908912658691,1
diff --git a/ray_results/test_experiment/train_model_e566e32a_13_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=2,drop__2024-06-25_16-40-03/result.json b/ray_results/test_experiment/train_model_e566e32a_13_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=2,drop__2024-06-25_16-40-03/result.json
new file mode 100644
index 0000000000000000000000000000000000000000..fd337bc43bd4b25655d1005b31302c9d6a6950a9
--- /dev/null
+++ b/ray_results/test_experiment/train_model_e566e32a_13_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=2,drop__2024-06-25_16-40-03/result.json
@@ -0,0 +1 @@
+{"loss": 2129.4940118264026, "timestamp": 1719326571, "checkpoint_dir_name": "checkpoint_000000", "should_checkpoint": true, "done": false, "training_iteration": 1, "trial_id": "e566e32a", "date": "2024-06-25_16-42-51", "time_this_iter_s": 82.61908912658691, "time_total_s": 82.61908912658691, "pid": 69318, "hostname": "r3i5n6", "node_ip": "10.159.8.159", "config": {"encoder_num_layer": 8, "decoder_rt_num_layer": 2, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 1024, "decoder_rt_ff": 2048, "decoder_int_ff": 512, "n_head": 4, "drop_rate": 0.7731110479994988, "lr": 0.03425650698356203, "batch_size": 2048}, "time_since_restore": 82.61908912658691, "iterations_since_restore": 1}
diff --git a/ray_results/test_experiment/train_model_f1fbbaf2_5_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop_r_2024-06-25_17-16-57/events.out.tfevents.1719328732.r8i6n2 b/ray_results/test_experiment/train_model_f1fbbaf2_5_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop_r_2024-06-25_17-16-57/events.out.tfevents.1719328732.r8i6n2
new file mode 100644
index 0000000000000000000000000000000000000000..66fb04c000a2f00097bcaaea4d9f438d49a1a197
Binary files /dev/null and b/ray_results/test_experiment/train_model_f1fbbaf2_5_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop_r_2024-06-25_17-16-57/events.out.tfevents.1719328732.r8i6n2 differ
diff --git a/ray_results/test_experiment/train_model_f1fbbaf2_5_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop_r_2024-06-25_17-16-57/params.json b/ray_results/test_experiment/train_model_f1fbbaf2_5_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop_r_2024-06-25_17-16-57/params.json
new file mode 100644
index 0000000000000000000000000000000000000000..65f2092fb6630a7ebb6dab5e101f3d965ad58e07
--- /dev/null
+++ b/ray_results/test_experiment/train_model_f1fbbaf2_5_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop_r_2024-06-25_17-16-57/params.json
@@ -0,0 +1,13 @@
+{
+ "batch_size": 1024,
+ "decoder_int_ff": 512,
+ "decoder_int_num_layer": 1,
+ "decoder_rt_ff": 1024,
+ "decoder_rt_num_layer": 2,
+ "drop_rate": 0.24540187796715263,
+ "embedding_dim": 1024,
+ "encoder_ff": 2048,
+ "encoder_num_layer": 4,
+ "lr": 0.0006485525268785047,
+ "n_head": 1
+}
\ No newline at end of file
diff --git a/ray_results/test_experiment/train_model_f1fbbaf2_5_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop_r_2024-06-25_17-16-57/params.pkl b/ray_results/test_experiment/train_model_f1fbbaf2_5_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop_r_2024-06-25_17-16-57/params.pkl
new file mode 100644
index 0000000000000000000000000000000000000000..d2806b583545ebac95768737b473342db3d6b524
Binary files /dev/null and b/ray_results/test_experiment/train_model_f1fbbaf2_5_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop_r_2024-06-25_17-16-57/params.pkl differ
diff --git a/ray_results/test_experiment/train_model_f1fbbaf2_5_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop_r_2024-06-25_17-16-57/progress.csv b/ray_results/test_experiment/train_model_f1fbbaf2_5_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop_r_2024-06-25_17-16-57/progress.csv
new file mode 100644
index 0000000000000000000000000000000000000000..3d8d7bbef1a044de4bc1a7500c380027246f2f69
--- /dev/null
+++ b/ray_results/test_experiment/train_model_f1fbbaf2_5_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop_r_2024-06-25_17-16-57/progress.csv
@@ -0,0 +1,2 @@
+loss,timestamp,checkpoint_dir_name,should_checkpoint,done,training_iteration,trial_id,date,time_this_iter_s,time_total_s,pid,hostname,node_ip,time_since_restore,iterations_since_restore
+1986.5954556202325,1719329376,checkpoint_000000,True,False,1,f1fbbaf2,2024-06-25_17-29-36,643.849014043808,643.849014043808,2195253,r8i6n2,10.159.28.60,643.849014043808,1
diff --git a/ray_results/test_experiment/train_model_f1fbbaf2_5_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop_r_2024-06-25_17-16-57/result.json b/ray_results/test_experiment/train_model_f1fbbaf2_5_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop_r_2024-06-25_17-16-57/result.json
new file mode 100644
index 0000000000000000000000000000000000000000..a73f3b5496457e703a0257d21e440da6d17f0644
--- /dev/null
+++ b/ray_results/test_experiment/train_model_f1fbbaf2_5_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop_r_2024-06-25_17-16-57/result.json
@@ -0,0 +1 @@
+{"loss": 1986.5954556202325, "timestamp": 1719329376, "checkpoint_dir_name": "checkpoint_000000", "should_checkpoint": true, "done": false, "training_iteration": 1, "trial_id": "f1fbbaf2", "date": "2024-06-25_17-29-36", "time_this_iter_s": 643.849014043808, "time_total_s": 643.849014043808, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 4, "decoder_rt_num_layer": 2, "decoder_int_num_layer": 1, "embedding_dim": 1024, "encoder_ff": 2048, "decoder_rt_ff": 1024, "decoder_int_ff": 512, "n_head": 1, "drop_rate": 0.24540187796715263, "lr": 0.0006485525268785047, "batch_size": 1024}, "time_since_restore": 643.849014043808, "iterations_since_restore": 1}
diff --git a/ray_results/test_experiment/train_model_f74eb513_2_batch_size=1024,decoder_int_ff=1024,decoder_int_num_layer=8,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop__2024-06-25_15-20-03/events.out.tfevents.1719321692.r7i5n7 b/ray_results/test_experiment/train_model_f74eb513_2_batch_size=1024,decoder_int_ff=1024,decoder_int_num_layer=8,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop__2024-06-25_15-20-03/events.out.tfevents.1719321692.r7i5n7
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/ray_results/test_experiment/train_model_f74eb513_2_batch_size=1024,decoder_int_ff=1024,decoder_int_num_layer=8,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop__2024-06-25_15-20-03/params.json b/ray_results/test_experiment/train_model_f74eb513_2_batch_size=1024,decoder_int_ff=1024,decoder_int_num_layer=8,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop__2024-06-25_15-20-03/params.json
new file mode 100644
index 0000000000000000000000000000000000000000..bb0521fd1bdf4e706a94f9e857d97cb4e7348187
--- /dev/null
+++ b/ray_results/test_experiment/train_model_f74eb513_2_batch_size=1024,decoder_int_ff=1024,decoder_int_num_layer=8,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop__2024-06-25_15-20-03/params.json
@@ -0,0 +1,13 @@
+{
+ "batch_size": 1024,
+ "decoder_int_ff": 1024,
+ "decoder_int_num_layer": 8,
+ "decoder_rt_ff": 1024,
+ "decoder_rt_num_layer": 2,
+ "drop_rate": 0.5887719140440579,
+ "embedding_dim": 1024,
+ "encoder_ff": 2048,
+ "encoder_num_layer": 8,
+ "lr": 0.009580025887992769,
+ "n_head": 8
+}
\ No newline at end of file
diff --git a/ray_results/test_experiment/train_model_f74eb513_2_batch_size=1024,decoder_int_ff=1024,decoder_int_num_layer=8,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop__2024-06-25_15-20-03/params.pkl b/ray_results/test_experiment/train_model_f74eb513_2_batch_size=1024,decoder_int_ff=1024,decoder_int_num_layer=8,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop__2024-06-25_15-20-03/params.pkl
new file mode 100644
index 0000000000000000000000000000000000000000..ce79ddfb021ca60b7aa0c2634f11fac094cd75c0
Binary files /dev/null and b/ray_results/test_experiment/train_model_f74eb513_2_batch_size=1024,decoder_int_ff=1024,decoder_int_num_layer=8,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop__2024-06-25_15-20-03/params.pkl differ
diff --git a/ray_results/test_experiment/train_model_f74eb513_2_batch_size=1024,decoder_int_ff=1024,decoder_int_num_layer=8,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop__2024-06-25_15-20-03/result.json b/ray_results/test_experiment/train_model_f74eb513_2_batch_size=1024,decoder_int_ff=1024,decoder_int_num_layer=8,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop__2024-06-25_15-20-03/result.json
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/ray_results/test_experiment/train_model_fe083630_32_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_22-50-21/events.out.tfevents.1719350385.r8i6n2 b/ray_results/test_experiment/train_model_fe083630_32_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_22-50-21/events.out.tfevents.1719350385.r8i6n2
new file mode 100644
index 0000000000000000000000000000000000000000..d70d04e52ed2d24c2a20a14d61d62a92e59ca943
Binary files /dev/null and b/ray_results/test_experiment/train_model_fe083630_32_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_22-50-21/events.out.tfevents.1719350385.r8i6n2 differ
diff --git a/ray_results/test_experiment/train_model_fe083630_32_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_22-50-21/params.json b/ray_results/test_experiment/train_model_fe083630_32_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_22-50-21/params.json
new file mode 100644
index 0000000000000000000000000000000000000000..f528e7d1bec8d6ae7b61e1ac15c235a55dbf4a9d
--- /dev/null
+++ b/ray_results/test_experiment/train_model_fe083630_32_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_22-50-21/params.json
@@ -0,0 +1,13 @@
+{
+ "batch_size": 1024,
+ "decoder_int_ff": 512,
+ "decoder_int_num_layer": 1,
+ "decoder_rt_ff": 512,
+ "decoder_rt_num_layer": 2,
+ "drop_rate": 0.7814344084863447,
+ "embedding_dim": 1024,
+ "encoder_ff": 1024,
+ "encoder_num_layer": 8,
+ "lr": 0.030816743385966763,
+ "n_head": 2
+}
\ No newline at end of file
diff --git a/ray_results/test_experiment/train_model_fe083630_32_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_22-50-21/params.pkl b/ray_results/test_experiment/train_model_fe083630_32_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_22-50-21/params.pkl
new file mode 100644
index 0000000000000000000000000000000000000000..eabd75552a221282c9ed756addafb311b81036a0
Binary files /dev/null and b/ray_results/test_experiment/train_model_fe083630_32_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_22-50-21/params.pkl differ
diff --git a/ray_results/test_experiment/train_model_fe083630_32_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_22-50-21/progress.csv b/ray_results/test_experiment/train_model_fe083630_32_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_22-50-21/progress.csv
new file mode 100644
index 0000000000000000000000000000000000000000..4242a15f99e40b0388fef8d9fc0821f84a05fd8a
--- /dev/null
+++ b/ray_results/test_experiment/train_model_fe083630_32_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_22-50-21/progress.csv
@@ -0,0 +1,6 @@
+loss,timestamp,checkpoint_dir_name,should_checkpoint,done,training_iteration,trial_id,date,time_this_iter_s,time_total_s,pid,hostname,node_ip,time_since_restore,iterations_since_restore
+4034.1917479507565,1719351243,checkpoint_000000,True,False,1,fe083630,2024-06-25_23-34-04,858.9566829204559,858.9566829204559,2195253,r8i6n2,10.159.28.60,858.9566829204559,1
+2881.328838198204,1719352088,checkpoint_000001,True,False,2,fe083630,2024-06-25_23-48-09,845.1621363162994,1704.1188192367554,2195253,r8i6n2,10.159.28.60,1704.1188192367554,2
+2239.2423533041647,1719352934,checkpoint_000002,True,False,3,fe083630,2024-06-26_00-02-14,845.5501523017883,2549.6689715385437,2195253,r8i6n2,10.159.28.60,2549.6689715385437,3
+2008.2958018385518,1719353779,checkpoint_000003,True,False,4,fe083630,2024-06-26_00-16-19,845.2403128147125,3394.909284353256,2195253,r8i6n2,10.159.28.60,3394.909284353256,4
+1961.6992687315453,1719354625,checkpoint_000004,True,False,5,fe083630,2024-06-26_00-30-25,845.8089480400085,4240.718232393265,2195253,r8i6n2,10.159.28.60,4240.718232393265,5
diff --git a/ray_results/test_experiment/train_model_fe083630_32_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_22-50-21/result.json b/ray_results/test_experiment/train_model_fe083630_32_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_22-50-21/result.json
new file mode 100644
index 0000000000000000000000000000000000000000..68c099d8769d7749d24db768c6ff23a79bbb91dc
--- /dev/null
+++ b/ray_results/test_experiment/train_model_fe083630_32_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_22-50-21/result.json
@@ -0,0 +1,5 @@
+{"loss": 4034.1917479507565, "timestamp": 1719351243, "checkpoint_dir_name": "checkpoint_000000", "should_checkpoint": true, "done": false, "training_iteration": 1, "trial_id": "fe083630", "date": "2024-06-25_23-34-04", "time_this_iter_s": 858.9566829204559, "time_total_s": 858.9566829204559, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 8, "decoder_rt_num_layer": 2, "decoder_int_num_layer": 1, "embedding_dim": 1024, "encoder_ff": 1024, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 2, "drop_rate": 0.7814344084863447, "lr": 0.030816743385966763, "batch_size": 1024}, "time_since_restore": 858.9566829204559, "iterations_since_restore": 1}
+{"loss": 2881.328838198204, "timestamp": 1719352088, "checkpoint_dir_name": "checkpoint_000001", "should_checkpoint": true, "done": false, "training_iteration": 2, "trial_id": "fe083630", "date": "2024-06-25_23-48-09", "time_this_iter_s": 845.1621363162994, "time_total_s": 1704.1188192367554, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 8, "decoder_rt_num_layer": 2, "decoder_int_num_layer": 1, "embedding_dim": 1024, "encoder_ff": 1024, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 2, "drop_rate": 0.7814344084863447, "lr": 0.030816743385966763, "batch_size": 1024}, "time_since_restore": 1704.1188192367554, "iterations_since_restore": 2}
+{"loss": 2239.2423533041647, "timestamp": 1719352934, "checkpoint_dir_name": "checkpoint_000002", "should_checkpoint": true, "done": false, "training_iteration": 3, "trial_id": "fe083630", "date": "2024-06-26_00-02-14", "time_this_iter_s": 845.5501523017883, "time_total_s": 2549.6689715385437, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 8, "decoder_rt_num_layer": 2, "decoder_int_num_layer": 1, "embedding_dim": 1024, "encoder_ff": 1024, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 2, "drop_rate": 0.7814344084863447, "lr": 0.030816743385966763, "batch_size": 1024}, "time_since_restore": 2549.6689715385437, "iterations_since_restore": 3}
+{"loss": 2008.2958018385518, "timestamp": 1719353779, "checkpoint_dir_name": "checkpoint_000003", "should_checkpoint": true, "done": false, "training_iteration": 4, "trial_id": "fe083630", "date": "2024-06-26_00-16-19", "time_this_iter_s": 845.2403128147125, "time_total_s": 3394.909284353256, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 8, "decoder_rt_num_layer": 2, "decoder_int_num_layer": 1, "embedding_dim": 1024, "encoder_ff": 1024, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 2, "drop_rate": 0.7814344084863447, "lr": 0.030816743385966763, "batch_size": 1024}, "time_since_restore": 3394.909284353256, "iterations_since_restore": 4}
+{"loss": 1961.6992687315453, "timestamp": 1719354625, "checkpoint_dir_name": "checkpoint_000004", "should_checkpoint": true, "done": false, "training_iteration": 5, "trial_id": "fe083630", "date": "2024-06-26_00-30-25", "time_this_iter_s": 845.8089480400085, "time_total_s": 4240.718232393265, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 8, "decoder_rt_num_layer": 2, "decoder_int_num_layer": 1, "embedding_dim": 1024, "encoder_ff": 1024, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 2, "drop_rate": 0.7814344084863447, "lr": 0.030816743385966763, "batch_size": 1024}, "time_since_restore": 4240.718232393265, "iterations_since_restore": 5}
diff --git a/ray_results/test_experiment/tuner.pkl b/ray_results/test_experiment/tuner.pkl
index cf4c3d302f96c6d70b0fcc0431588892ca366842..064732323e8c0aae443330fb73506baaa5ff28c9 100644
Binary files a/ray_results/test_experiment/tuner.pkl and b/ray_results/test_experiment/tuner.pkl differ