From a790c5c2599a515945ebfe278c0e4fa985cc2879 Mon Sep 17 00:00:00 2001
From: lschneider <leo.schneider@ecl19.ec-lyon.fr>
Date: Wed, 26 Jun 2024 15:43:28 +0200
Subject: [PATCH] res
---
.../experiment_state-2024-06-25_15-19-46.json | 52 ++++++
.../experiment_state-2024-06-25_16-07-48.json | 116 ++++++++++++
.../experiment_state-2024-06-25_17-10-00.json | 168 ++++++++++++++++++
.../search_gen_state-2024-06-25_15-19-46.json | Bin 0 -> 2463 bytes
.../search_gen_state-2024-06-25_16-07-48.json | Bin 0 -> 2463 bytes
.../search_gen_state-2024-06-25_17-10-00.json | Bin 0 -> 2449 bytes
.../searcher-state-2024-06-25_15-19-46.pkl | Bin 0 -> 6726 bytes
.../searcher-state-2024-06-25_16-07-48.pkl | Bin 0 -> 10106 bytes
.../searcher-state-2024-06-25_17-10-00.pkl | Bin 0 -> 13726 bytes
.../events.out.tfevents.1719329545.r8i6n2 | Bin 0 -> 1311 bytes
.../params.json | 13 ++
.../params.pkl | Bin 0 -> 224 bytes
.../progress.csv | 2 +
.../result.json | 1 +
.../events.out.tfevents.1719348621.r8i6n2 | Bin 0 -> 1865 bytes
.../events.out.tfevents.1719355502.r8i6n2 | Bin 0 -> 2771 bytes
.../params.json | 13 ++
.../params.pkl | Bin 0 -> 225 bytes
.../progress.csv | 11 ++
.../result.json | 10 ++
.../events.out.tfevents.1719334649.r8i6n2 | Bin 0 -> 770 bytes
.../events.out.tfevents.1719338738.r8i6n2 | Bin 0 -> 770 bytes
.../events.out.tfevents.1719340670.r8i6n2 | Bin 0 -> 4596 bytes
.../params.json | 13 ++
.../params.pkl | Bin 0 -> 225 bytes
.../progress.csv | 15 ++
.../result.json | 14 ++
.../events.out.tfevents.1719330179.r8i6n2 | Bin 0 -> 405 bytes
.../events.out.tfevents.1719330947.r8i6n2 | Bin 0 -> 1311 bytes
.../params.json | 13 ++
.../params.pkl | Bin 0 -> 225 bytes
.../progress.csv | 3 +
.../result.json | 2 +
.../events.out.tfevents.1719346234.r8i6n2 | Bin 0 -> 1865 bytes
.../events.out.tfevents.1719354644.r8i6n2 | Bin 0 -> 1865 bytes
.../events.out.tfevents.1719357265.r8i6n2 | Bin 0 -> 4596 bytes
.../params.json | 13 ++
.../params.pkl | Bin 0 -> 224 bytes
.../progress.csv | 21 +++
.../result.json | 20 +++
.../events.out.tfevents.1719325234.r3i5n6 | Bin 0 -> 1311 bytes
.../params.json | 13 ++
.../params.pkl | Bin 0 -> 224 bytes
.../progress.csv | 2 +
.../result.json | 1 +
.../events.out.tfevents.1719325314.r3i5n6 | Bin 0 -> 405 bytes
.../events.out.tfevents.1719327690.r3i5n6 | Bin 0 -> 405 bytes
.../params.json | 13 ++
.../params.pkl | Bin 0 -> 224 bytes
.../progress.csv | 3 +
.../result.json | 2 +
.../error.pkl | Bin 0 -> 4759 bytes
.../error.txt | 26 +++
.../events.out.tfevents.1719325388.r3i5n6 | Bin 0 -> 405 bytes
.../events.out.tfevents.1719327763.r3i5n6 | Bin 0 -> 40 bytes
.../params.json | 13 ++
.../params.pkl | Bin 0 -> 225 bytes
.../progress.csv | 2 +
.../result.json | 1 +
.../events.out.tfevents.1719329807.r8i6n2 | Bin 0 -> 1311 bytes
.../params.json | 13 ++
.../params.pkl | Bin 0 -> 224 bytes
.../progress.csv | 2 +
.../result.json | 1 +
.../events.out.tfevents.1719330099.r8i6n2 | Bin 0 -> 405 bytes
.../events.out.tfevents.1719330869.r8i6n2 | Bin 0 -> 405 bytes
.../events.out.tfevents.1719331459.r8i6n2 | Bin 0 -> 1676 bytes
.../params.json | 13 ++
.../params.pkl | Bin 0 -> 224 bytes
.../progress.csv | 5 +
.../result.json | 4 +
.../events.out.tfevents.1719326598.r3i5n6 | Bin 0 -> 1311 bytes
.../params.json | 13 ++
.../params.pkl | Bin 0 -> 225 bytes
.../progress.csv | 2 +
.../result.json | 1 +
.../events.out.tfevents.1719330290.r8i6n2 | Bin 0 -> 405 bytes
.../events.out.tfevents.1719331058.r8i6n2 | Bin 0 -> 1311 bytes
.../params.json | 13 ++
.../params.pkl | Bin 0 -> 224 bytes
.../progress.csv | 3 +
.../result.json | 2 +
.../events.out.tfevents.1719327790.r3i5n6 | Bin 0 -> 405 bytes
.../params.json | 13 ++
.../params.pkl | Bin 0 -> 224 bytes
.../progress.csv | 2 +
.../result.json | 1 +
.../events.out.tfevents.1719346547.r8i6n2 | Bin 0 -> 1865 bytes
.../events.out.tfevents.1719354952.r8i6n2 | Bin 0 -> 1865 bytes
.../events.out.tfevents.1719357872.r8i6n2 | Bin 0 -> 4596 bytes
.../params.json | 13 ++
.../params.pkl | Bin 0 -> 224 bytes
.../progress.csv | 21 +++
.../result.json | 20 +++
.../events.out.tfevents.1719329461.r8i6n2 | Bin 0 -> 1311 bytes
.../params.json | 13 ++
.../params.pkl | Bin 0 -> 225 bytes
.../progress.csv | 2 +
.../result.json | 1 +
.../events.out.tfevents.1719346823.r8i6n2 | Bin 0 -> 2771 bytes
.../params.json | 13 ++
.../params.pkl | Bin 0 -> 225 bytes
.../progress.csv | 6 +
.../result.json | 5 +
.../events.out.tfevents.1719333829.r8i6n2 | Bin 0 -> 770 bytes
.../events.out.tfevents.1719338189.r8i6n2 | Bin 0 -> 770 bytes
.../events.out.tfevents.1719340067.r8i6n2 | Bin 0 -> 2406 bytes
.../params.json | 13 ++
.../params.pkl | Bin 0 -> 225 bytes
.../progress.csv | 9 +
.../result.json | 8 +
.../error.pkl | Bin 0 -> 8271 bytes
.../error.txt | 68 +++++++
.../events.out.tfevents.1719326860.r3i5n6 | Bin 0 -> 40 bytes
.../params.json | 13 ++
.../params.pkl | Bin 0 -> 225 bytes
.../result.json | 0
.../events.out.tfevents.1719329693.r8i6n2 | Bin 0 -> 405 bytes
.../events.out.tfevents.1719330754.r8i6n2 | Bin 0 -> 1311 bytes
.../params.json | 13 ++
.../params.pkl | Bin 0 -> 224 bytes
.../progress.csv | 3 +
.../result.json | 2 +
.../events.out.tfevents.1719324574.r3i5n6 | Bin 0 -> 1241 bytes
.../params.json | 13 ++
.../params.pkl | Bin 0 -> 224 bytes
.../progress.csv | 2 +
.../result.json | 1 +
.../events.out.tfevents.1719328213.r8i6n2 | Bin 0 -> 1311 bytes
.../params.json | 13 ++
.../params.pkl | Bin 0 -> 225 bytes
.../progress.csv | 2 +
.../result.json | 1 +
.../events.out.tfevents.1719329376.r8i6n2 | Bin 0 -> 405 bytes
.../events.out.tfevents.1719330587.r8i6n2 | Bin 0 -> 1311 bytes
.../params.json | 13 ++
.../params.pkl | Bin 0 -> 224 bytes
.../progress.csv | 3 +
.../result.json | 2 +
.../events.out.tfevents.1719335674.r8i6n2 | Bin 0 -> 770 bytes
.../events.out.tfevents.1719339767.r8i6n2 | Bin 0 -> 2230 bytes
.../events.out.tfevents.1719345748.r8i6n2 | Bin 0 -> 4596 bytes
.../params.json | 13 ++
.../params.pkl | Bin 0 -> 224 bytes
.../progress.csv | 19 ++
.../result.json | 18 ++
.../events.out.tfevents.1719326403.r3i5n6 | Bin 0 -> 405 bytes
.../params.json | 13 ++
.../params.pkl | Bin 0 -> 224 bytes
.../progress.csv | 2 +
.../result.json | 1 +
.../events.out.tfevents.1719334380.r8i6n2 | Bin 0 -> 1676 bytes
.../params.json | 13 ++
.../params.pkl | Bin 0 -> 225 bytes
.../progress.csv | 3 +
.../result.json | 2 +
.../events.out.tfevents.1719324663.r3i5n6 | Bin 0 -> 405 bytes
.../events.out.tfevents.1719327182.r3i5n6 | Bin 0 -> 405 bytes
.../params.json | 13 ++
.../params.pkl | Bin 0 -> 224 bytes
.../progress.csv | 3 +
.../result.json | 2 +
.../events.out.tfevents.1719335783.r8i6n2 | Bin 0 -> 1676 bytes
.../params.json | 13 ++
.../params.pkl | Bin 0 -> 224 bytes
.../progress.csv | 3 +
.../result.json | 2 +
.../events.out.tfevents.1719334139.r8i6n2 | Bin 0 -> 770 bytes
.../events.out.tfevents.1719338498.r8i6n2 | Bin 0 -> 1676 bytes
.../params.json | 13 ++
.../params.pkl | Bin 0 -> 224 bytes
.../progress.csv | 5 +
.../result.json | 4 +
.../events.out.tfevents.1719328526.r8i6n2 | Bin 0 -> 1311 bytes
.../params.json | 13 ++
.../params.pkl | Bin 0 -> 224 bytes
.../progress.csv | 2 +
.../result.json | 1 +
.../events.out.tfevents.1719328617.r8i6n2 | Bin 0 -> 1311 bytes
.../params.json | 13 ++
.../params.pkl | Bin 0 -> 225 bytes
.../progress.csv | 2 +
.../result.json | 1 +
.../events.out.tfevents.1719335912.r8i6n2 | Bin 0 -> 1676 bytes
.../params.json | 13 ++
.../params.pkl | Bin 0 -> 225 bytes
.../progress.csv | 3 +
.../result.json | 2 +
.../events.out.tfevents.1719325014.r3i5n6 | Bin 0 -> 405 bytes
.../events.out.tfevents.1719327469.r3i5n6 | Bin 0 -> 405 bytes
.../params.json | 13 ++
.../params.pkl | Bin 0 -> 225 bytes
.../progress.csv | 3 +
.../result.json | 2 +
.../error.pkl | Bin 0 -> 7437 bytes
.../error.txt | 59 ++++++
.../events.out.tfevents.1719326572.r3i5n6 | Bin 0 -> 40 bytes
.../params.json | 13 ++
.../params.pkl | Bin 0 -> 225 bytes
.../result.json | 0
.../events.out.tfevents.1719326883.r3i5n6 | Bin 0 -> 770 bytes
.../params.json | 13 ++
.../params.pkl | Bin 0 -> 224 bytes
.../progress.csv | 3 +
.../result.json | 2 +
.../events.out.tfevents.1719329610.r8i6n2 | Bin 0 -> 405 bytes
.../events.out.tfevents.1719330672.r8i6n2 | Bin 0 -> 405 bytes
.../events.out.tfevents.1719331282.r8i6n2 | Bin 0 -> 770 bytes
.../events.out.tfevents.1719332071.r8i6n2 | Bin 0 -> 4596 bytes
.../params.json | 13 ++
.../params.pkl | Bin 0 -> 225 bytes
.../progress.csv | 15 ++
.../result.json | 14 ++
.../events.out.tfevents.1719324728.r3i5n6 | Bin 0 -> 1311 bytes
.../params.json | 13 ++
.../params.pkl | Bin 0 -> 224 bytes
.../progress.csv | 2 +
.../result.json | 1 +
.../events.out.tfevents.1719328456.r8i6n2 | Bin 0 -> 405 bytes
.../events.out.tfevents.1719330517.r8i6n2 | Bin 0 -> 405 bytes
.../events.out.tfevents.1719331131.r8i6n2 | Bin 0 -> 770 bytes
.../events.out.tfevents.1719331824.r8i6n2 | Bin 0 -> 2406 bytes
.../params.json | 13 ++
.../params.pkl | Bin 0 -> 224 bytes
.../progress.csv | 9 +
.../result.json | 8 +
.../events.out.tfevents.1719330362.r8i6n2 | Bin 0 -> 770 bytes
.../events.out.tfevents.1719331634.r8i6n2 | Bin 0 -> 1676 bytes
.../params.json | 13 ++
.../params.pkl | Bin 0 -> 224 bytes
.../progress.csv | 5 +
.../result.json | 4 +
.../events.out.tfevents.1719324484.r3i5n6 | Bin 0 -> 1311 bytes
.../params.json | 13 ++
.../params.pkl | Bin 0 -> 225 bytes
.../progress.csv | 2 +
.../result.json | 1 +
.../events.out.tfevents.1719329892.r8i6n2 | Bin 0 -> 1311 bytes
.../params.json | 13 ++
.../params.pkl | Bin 0 -> 225 bytes
.../progress.csv | 2 +
.../result.json | 1 +
.../events.out.tfevents.1719324792.r3i5n6 | Bin 0 -> 405 bytes
.../events.out.tfevents.1719327245.r3i5n6 | Bin 0 -> 405 bytes
.../params.json | 13 ++
.../params.pkl | Bin 0 -> 224 bytes
.../progress.csv | 3 +
.../result.json | 2 +
.../events.out.tfevents.1719326792.r3i5n6 | Bin 0 -> 1311 bytes
.../params.json | 13 ++
.../params.pkl | Bin 0 -> 224 bytes
.../progress.csv | 2 +
.../result.json | 1 +
.../events.out.tfevents.1719324918.r3i5n6 | Bin 0 -> 405 bytes
.../events.out.tfevents.1719327372.r3i5n6 | Bin 0 -> 405 bytes
.../params.json | 13 ++
.../params.pkl | Bin 0 -> 224 bytes
.../progress.csv | 3 +
.../result.json | 2 +
.../events.out.tfevents.1719332768.r8i6n2 | Bin 0 -> 1676 bytes
.../params.json | 13 ++
.../params.pkl | Bin 0 -> 224 bytes
.../progress.csv | 3 +
.../result.json | 2 +
.../events.out.tfevents.1719332932.r8i6n2 | Bin 0 -> 770 bytes
.../events.out.tfevents.1719337296.r8i6n2 | Bin 0 -> 1676 bytes
.../params.json | 13 ++
.../params.pkl | Bin 0 -> 225 bytes
.../progress.csv | 5 +
.../result.json | 4 +
.../events.out.tfevents.1719325156.r3i5n6 | Bin 0 -> 405 bytes
.../events.out.tfevents.1719327610.r3i5n6 | Bin 0 -> 405 bytes
.../params.json | 13 ++
.../params.pkl | Bin 0 -> 225 bytes
.../progress.csv | 3 +
.../result.json | 2 +
.../events.out.tfevents.1719321603.r7i5n7 | Bin 0 -> 405 bytes
.../params.json | 13 ++
.../params.pkl | Bin 0 -> 224 bytes
.../progress.csv | 2 +
.../result.json | 1 +
.../events.out.tfevents.1719348339.r8i6n2 | Bin 0 -> 1865 bytes
.../events.out.tfevents.1719355223.r8i6n2 | Bin 0 -> 2771 bytes
.../params.json | 13 ++
.../params.pkl | Bin 0 -> 224 bytes
.../progress.csv | 11 ++
.../result.json | 10 ++
.../events.out.tfevents.1719337143.r8i6n2 | Bin 0 -> 1676 bytes
.../params.json | 13 ++
.../params.pkl | Bin 0 -> 224 bytes
.../progress.csv | 3 +
.../result.json | 2 +
.../events.out.tfevents.1719326489.r3i5n6 | Bin 0 -> 1311 bytes
.../params.json | 13 ++
.../params.pkl | Bin 0 -> 224 bytes
.../progress.csv | 2 +
.../result.json | 1 +
.../events.out.tfevents.1719328732.r8i6n2 | Bin 0 -> 1311 bytes
.../params.json | 13 ++
.../params.pkl | Bin 0 -> 225 bytes
.../progress.csv | 2 +
.../result.json | 1 +
.../events.out.tfevents.1719321692.r7i5n7 | 0
.../params.json | 13 ++
.../params.pkl | Bin 0 -> 225 bytes
.../result.json | 0
.../events.out.tfevents.1719350385.r8i6n2 | Bin 0 -> 2771 bytes
.../params.json | 13 ++
.../params.pkl | Bin 0 -> 225 bytes
.../progress.csv | 6 +
.../result.json | 5 +
ray_results/test_experiment/tuner.pkl | Bin 3586 -> 3586 bytes
312 files changed, 1626 insertions(+)
create mode 100644 ray_results/test_experiment/experiment_state-2024-06-25_15-19-46.json
create mode 100644 ray_results/test_experiment/experiment_state-2024-06-25_16-07-48.json
create mode 100644 ray_results/test_experiment/experiment_state-2024-06-25_17-10-00.json
create mode 100644 ray_results/test_experiment/search_gen_state-2024-06-25_15-19-46.json
create mode 100644 ray_results/test_experiment/search_gen_state-2024-06-25_16-07-48.json
create mode 100644 ray_results/test_experiment/search_gen_state-2024-06-25_17-10-00.json
create mode 100644 ray_results/test_experiment/searcher-state-2024-06-25_15-19-46.pkl
create mode 100644 ray_results/test_experiment/searcher-state-2024-06-25_16-07-48.pkl
create mode 100644 ray_results/test_experiment/searcher-state-2024-06-25_17-10-00.pkl
create mode 100644 ray_results/test_experiment/train_model_040fcf09_8_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=4,drop_ra_2024-06-25_17-31-01/events.out.tfevents.1719329545.r8i6n2
create mode 100644 ray_results/test_experiment/train_model_040fcf09_8_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=4,drop_ra_2024-06-25_17-31-01/params.json
create mode 100644 ray_results/test_experiment/train_model_040fcf09_8_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=4,drop_ra_2024-06-25_17-31-01/params.pkl
create mode 100644 ray_results/test_experiment/train_model_040fcf09_8_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=4,drop_ra_2024-06-25_17-31-01/progress.csv
create mode 100644 ray_results/test_experiment/train_model_040fcf09_8_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=4,drop_ra_2024-06-25_17-31-01/result.json
create mode 100644 ray_results/test_experiment/train_model_0f80a6d7_31_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_22-45-39/events.out.tfevents.1719348621.r8i6n2
create mode 100644 ray_results/test_experiment/train_model_0f80a6d7_31_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_22-45-39/events.out.tfevents.1719355502.r8i6n2
create mode 100644 ray_results/test_experiment/train_model_0f80a6d7_31_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_22-45-39/params.json
create mode 100644 ray_results/test_experiment/train_model_0f80a6d7_31_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_22-45-39/params.pkl
create mode 100644 ray_results/test_experiment/train_model_0f80a6d7_31_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_22-45-39/progress.csv
create mode 100644 ray_results/test_experiment/train_model_0f80a6d7_31_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_22-45-39/result.json
create mode 100644 ray_results/test_experiment/train_model_0f9cc97d_22_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_18-53-00/events.out.tfevents.1719334649.r8i6n2
create mode 100644 ray_results/test_experiment/train_model_0f9cc97d_22_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_18-53-00/events.out.tfevents.1719338738.r8i6n2
create mode 100644 ray_results/test_experiment/train_model_0f9cc97d_22_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_18-53-00/events.out.tfevents.1719340670.r8i6n2
create mode 100644 ray_results/test_experiment/train_model_0f9cc97d_22_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_18-53-00/params.json
create mode 100644 ray_results/test_experiment/train_model_0f9cc97d_22_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_18-53-00/params.pkl
create mode 100644 ray_results/test_experiment/train_model_0f9cc97d_22_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_18-53-00/progress.csv
create mode 100644 ray_results/test_experiment/train_model_0f9cc97d_22_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_18-53-00/result.json
create mode 100644 ray_results/test_experiment/train_model_110f0a27_14_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop__2024-06-25_17-41-39/events.out.tfevents.1719330179.r8i6n2
create mode 100644 ray_results/test_experiment/train_model_110f0a27_14_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop__2024-06-25_17-41-39/events.out.tfevents.1719330947.r8i6n2
create mode 100644 ray_results/test_experiment/train_model_110f0a27_14_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop__2024-06-25_17-41-39/params.json
create mode 100644 ray_results/test_experiment/train_model_110f0a27_14_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop__2024-06-25_17-41-39/params.pkl
create mode 100644 ray_results/test_experiment/train_model_110f0a27_14_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop__2024-06-25_17-41-39/progress.csv
create mode 100644 ray_results/test_experiment/train_model_110f0a27_14_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop__2024-06-25_17-41-39/result.json
create mode 100644 ray_results/test_experiment/train_model_18c11cbc_27_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop__2024-06-25_19-39-03/events.out.tfevents.1719346234.r8i6n2
create mode 100644 ray_results/test_experiment/train_model_18c11cbc_27_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop__2024-06-25_19-39-03/events.out.tfevents.1719354644.r8i6n2
create mode 100644 ray_results/test_experiment/train_model_18c11cbc_27_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop__2024-06-25_19-39-03/events.out.tfevents.1719357265.r8i6n2
create mode 100644 ray_results/test_experiment/train_model_18c11cbc_27_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop__2024-06-25_19-39-03/params.json
create mode 100644 ray_results/test_experiment/train_model_18c11cbc_27_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop__2024-06-25_19-39-03/params.pkl
create mode 100644 ray_results/test_experiment/train_model_18c11cbc_27_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop__2024-06-25_19-39-03/progress.csv
create mode 100644 ray_results/test_experiment/train_model_18c11cbc_27_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop__2024-06-25_19-39-03/result.json
create mode 100644 ray_results/test_experiment/train_model_1920f493_9_batch_size=1024,decoder_int_ff=1024,decoder_int_num_layer=2,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop__2024-06-25_16-19-16/events.out.tfevents.1719325234.r3i5n6
create mode 100644 ray_results/test_experiment/train_model_1920f493_9_batch_size=1024,decoder_int_ff=1024,decoder_int_num_layer=2,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop__2024-06-25_16-19-16/params.json
create mode 100644 ray_results/test_experiment/train_model_1920f493_9_batch_size=1024,decoder_int_ff=1024,decoder_int_num_layer=2,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop__2024-06-25_16-19-16/params.pkl
create mode 100644 ray_results/test_experiment/train_model_1920f493_9_batch_size=1024,decoder_int_ff=1024,decoder_int_num_layer=2,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop__2024-06-25_16-19-16/progress.csv
create mode 100644 ray_results/test_experiment/train_model_1920f493_9_batch_size=1024,decoder_int_ff=1024,decoder_int_num_layer=2,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop__2024-06-25_16-19-16/result.json
create mode 100644 ray_results/test_experiment/train_model_1de460a8_10_batch_size=1024,decoder_int_ff=1024,decoder_int_num_layer=8,decoder_rt_ff=512,decoder_rt_num_layer=4,drop__2024-06-25_16-20-34/events.out.tfevents.1719325314.r3i5n6
create mode 100644 ray_results/test_experiment/train_model_1de460a8_10_batch_size=1024,decoder_int_ff=1024,decoder_int_num_layer=8,decoder_rt_ff=512,decoder_rt_num_layer=4,drop__2024-06-25_16-20-34/events.out.tfevents.1719327690.r3i5n6
create mode 100644 ray_results/test_experiment/train_model_1de460a8_10_batch_size=1024,decoder_int_ff=1024,decoder_int_num_layer=8,decoder_rt_ff=512,decoder_rt_num_layer=4,drop__2024-06-25_16-20-34/params.json
create mode 100644 ray_results/test_experiment/train_model_1de460a8_10_batch_size=1024,decoder_int_ff=1024,decoder_int_num_layer=8,decoder_rt_ff=512,decoder_rt_num_layer=4,drop__2024-06-25_16-20-34/params.pkl
create mode 100644 ray_results/test_experiment/train_model_1de460a8_10_batch_size=1024,decoder_int_ff=1024,decoder_int_num_layer=8,decoder_rt_ff=512,decoder_rt_num_layer=4,drop__2024-06-25_16-20-34/progress.csv
create mode 100644 ray_results/test_experiment/train_model_1de460a8_10_batch_size=1024,decoder_int_ff=1024,decoder_int_num_layer=8,decoder_rt_ff=512,decoder_rt_num_layer=4,drop__2024-06-25_16-20-34/result.json
create mode 100644 ray_results/test_experiment/train_model_1ef7c88b_11_batch_size=1024,decoder_int_ff=1024,decoder_int_num_layer=4,decoder_rt_ff=512,decoder_rt_num_layer=4,drop__2024-06-25_16-21-54/error.pkl
create mode 100644 ray_results/test_experiment/train_model_1ef7c88b_11_batch_size=1024,decoder_int_ff=1024,decoder_int_num_layer=4,decoder_rt_ff=512,decoder_rt_num_layer=4,drop__2024-06-25_16-21-54/error.txt
create mode 100644 ray_results/test_experiment/train_model_1ef7c88b_11_batch_size=1024,decoder_int_ff=1024,decoder_int_num_layer=4,decoder_rt_ff=512,decoder_rt_num_layer=4,drop__2024-06-25_16-21-54/events.out.tfevents.1719325388.r3i5n6
create mode 100644 ray_results/test_experiment/train_model_1ef7c88b_11_batch_size=1024,decoder_int_ff=1024,decoder_int_num_layer=4,decoder_rt_ff=512,decoder_rt_num_layer=4,drop__2024-06-25_16-21-54/events.out.tfevents.1719327763.r3i5n6
create mode 100644 ray_results/test_experiment/train_model_1ef7c88b_11_batch_size=1024,decoder_int_ff=1024,decoder_int_num_layer=4,decoder_rt_ff=512,decoder_rt_num_layer=4,drop__2024-06-25_16-21-54/params.json
create mode 100644 ray_results/test_experiment/train_model_1ef7c88b_11_batch_size=1024,decoder_int_ff=1024,decoder_int_num_layer=4,decoder_rt_ff=512,decoder_rt_num_layer=4,drop__2024-06-25_16-21-54/params.pkl
create mode 100644 ray_results/test_experiment/train_model_1ef7c88b_11_batch_size=1024,decoder_int_ff=1024,decoder_int_num_layer=4,decoder_rt_ff=512,decoder_rt_num_layer=4,drop__2024-06-25_16-21-54/progress.csv
create mode 100644 ray_results/test_experiment/train_model_1ef7c88b_11_batch_size=1024,decoder_int_ff=1024,decoder_int_num_layer=4,decoder_rt_ff=512,decoder_rt_num_layer=4,drop__2024-06-25_16-21-54/result.json
create mode 100644 ray_results/test_experiment/train_model_2465ee2c_11_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop__2024-06-25_17-34-53/events.out.tfevents.1719329807.r8i6n2
create mode 100644 ray_results/test_experiment/train_model_2465ee2c_11_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop__2024-06-25_17-34-53/params.json
create mode 100644 ray_results/test_experiment/train_model_2465ee2c_11_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop__2024-06-25_17-34-53/params.pkl
create mode 100644 ray_results/test_experiment/train_model_2465ee2c_11_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop__2024-06-25_17-34-53/progress.csv
create mode 100644 ray_results/test_experiment/train_model_2465ee2c_11_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop__2024-06-25_17-34-53/result.json
create mode 100644 ray_results/test_experiment/train_model_25a9f43a_13_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_17-38-12/events.out.tfevents.1719330099.r8i6n2
create mode 100644 ray_results/test_experiment/train_model_25a9f43a_13_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_17-38-12/events.out.tfevents.1719330869.r8i6n2
create mode 100644 ray_results/test_experiment/train_model_25a9f43a_13_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_17-38-12/events.out.tfevents.1719331459.r8i6n2
create mode 100644 ray_results/test_experiment/train_model_25a9f43a_13_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_17-38-12/params.json
create mode 100644 ray_results/test_experiment/train_model_25a9f43a_13_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_17-38-12/params.pkl
create mode 100644 ray_results/test_experiment/train_model_25a9f43a_13_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_17-38-12/progress.csv
create mode 100644 ray_results/test_experiment/train_model_25a9f43a_13_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_17-38-12/result.json
create mode 100644 ray_results/test_experiment/train_model_25b42194_15_batch_size=2048,decoder_int_ff=2048,decoder_int_num_layer=8,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop_2024-06-25_16-42-52/events.out.tfevents.1719326598.r3i5n6
create mode 100644 ray_results/test_experiment/train_model_25b42194_15_batch_size=2048,decoder_int_ff=2048,decoder_int_num_layer=8,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop_2024-06-25_16-42-52/params.json
create mode 100644 ray_results/test_experiment/train_model_25b42194_15_batch_size=2048,decoder_int_ff=2048,decoder_int_num_layer=8,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop_2024-06-25_16-42-52/params.pkl
create mode 100644 ray_results/test_experiment/train_model_25b42194_15_batch_size=2048,decoder_int_ff=2048,decoder_int_num_layer=8,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop_2024-06-25_16-42-52/progress.csv
create mode 100644 ray_results/test_experiment/train_model_25b42194_15_batch_size=2048,decoder_int_ff=2048,decoder_int_num_layer=8,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop_2024-06-25_16-42-52/result.json
create mode 100644 ray_results/test_experiment/train_model_3510e84b_15_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop__2024-06-25_17-42-59/events.out.tfevents.1719330290.r8i6n2
create mode 100644 ray_results/test_experiment/train_model_3510e84b_15_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop__2024-06-25_17-42-59/events.out.tfevents.1719331058.r8i6n2
create mode 100644 ray_results/test_experiment/train_model_3510e84b_15_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop__2024-06-25_17-42-59/params.json
create mode 100644 ray_results/test_experiment/train_model_3510e84b_15_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop__2024-06-25_17-42-59/params.pkl
create mode 100644 ray_results/test_experiment/train_model_3510e84b_15_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop__2024-06-25_17-42-59/progress.csv
create mode 100644 ray_results/test_experiment/train_model_3510e84b_15_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop__2024-06-25_17-42-59/result.json
create mode 100644 ray_results/test_experiment/train_model_375ed3d7_19_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=2,decoder_rt_ff=1024,decoder_rt_num_layer=4,drop__2024-06-25_16-56-12/events.out.tfevents.1719327790.r3i5n6
create mode 100644 ray_results/test_experiment/train_model_375ed3d7_19_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=2,decoder_rt_ff=1024,decoder_rt_num_layer=4,drop__2024-06-25_16-56-12/params.json
create mode 100644 ray_results/test_experiment/train_model_375ed3d7_19_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=2,decoder_rt_ff=1024,decoder_rt_num_layer=4,drop__2024-06-25_16-56-12/params.pkl
create mode 100644 ray_results/test_experiment/train_model_375ed3d7_19_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=2,decoder_rt_ff=1024,decoder_rt_num_layer=4,drop__2024-06-25_16-56-12/progress.csv
create mode 100644 ray_results/test_experiment/train_model_375ed3d7_19_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=2,decoder_rt_ff=1024,decoder_rt_num_layer=4,drop__2024-06-25_16-56-12/result.json
create mode 100644 ray_results/test_experiment/train_model_3ab2f301_28_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop__2024-06-25_22-10-34/events.out.tfevents.1719346547.r8i6n2
create mode 100644 ray_results/test_experiment/train_model_3ab2f301_28_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop__2024-06-25_22-10-34/events.out.tfevents.1719354952.r8i6n2
create mode 100644 ray_results/test_experiment/train_model_3ab2f301_28_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop__2024-06-25_22-10-34/events.out.tfevents.1719357872.r8i6n2
create mode 100644 ray_results/test_experiment/train_model_3ab2f301_28_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop__2024-06-25_22-10-34/params.json
create mode 100644 ray_results/test_experiment/train_model_3ab2f301_28_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop__2024-06-25_22-10-34/params.pkl
create mode 100644 ray_results/test_experiment/train_model_3ab2f301_28_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop__2024-06-25_22-10-34/progress.csv
create mode 100644 ray_results/test_experiment/train_model_3ab2f301_28_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop__2024-06-25_22-10-34/result.json
create mode 100644 ray_results/test_experiment/train_model_3f37f45c_7_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=2,drop_r_2024-06-25_17-29-36/events.out.tfevents.1719329461.r8i6n2
create mode 100644 ray_results/test_experiment/train_model_3f37f45c_7_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=2,drop_r_2024-06-25_17-29-36/params.json
create mode 100644 ray_results/test_experiment/train_model_3f37f45c_7_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=2,drop_r_2024-06-25_17-29-36/params.pkl
create mode 100644 ray_results/test_experiment/train_model_3f37f45c_7_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=2,drop_r_2024-06-25_17-29-36/progress.csv
create mode 100644 ray_results/test_experiment/train_model_3f37f45c_7_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=2,drop_r_2024-06-25_17-29-36/result.json
create mode 100644 ray_results/test_experiment/train_model_40101b9a_29_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=1,drop_r_2024-06-25_22-15-47/events.out.tfevents.1719346823.r8i6n2
create mode 100644 ray_results/test_experiment/train_model_40101b9a_29_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=1,drop_r_2024-06-25_22-15-47/params.json
create mode 100644 ray_results/test_experiment/train_model_40101b9a_29_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=1,drop_r_2024-06-25_22-15-47/params.pkl
create mode 100644 ray_results/test_experiment/train_model_40101b9a_29_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=1,drop_r_2024-06-25_22-15-47/progress.csv
create mode 100644 ray_results/test_experiment/train_model_40101b9a_29_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=1,drop_r_2024-06-25_22-15-47/result.json
create mode 100644 ray_results/test_experiment/train_model_43bba37c_19_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_18-28-52/events.out.tfevents.1719333829.r8i6n2
create mode 100644 ray_results/test_experiment/train_model_43bba37c_19_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_18-28-52/events.out.tfevents.1719338189.r8i6n2
create mode 100644 ray_results/test_experiment/train_model_43bba37c_19_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_18-28-52/events.out.tfevents.1719340067.r8i6n2
create mode 100644 ray_results/test_experiment/train_model_43bba37c_19_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_18-28-52/params.json
create mode 100644 ray_results/test_experiment/train_model_43bba37c_19_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_18-28-52/params.pkl
create mode 100644 ray_results/test_experiment/train_model_43bba37c_19_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_18-28-52/progress.csv
create mode 100644 ray_results/test_experiment/train_model_43bba37c_19_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_18-28-52/result.json
create mode 100644 ray_results/test_experiment/train_model_45b8cfdd_17_batch_size=2048,decoder_int_ff=2048,decoder_int_num_layer=2,decoder_rt_ff=1024,decoder_rt_num_layer=4,drop_2024-06-25_16-46-32/error.pkl
create mode 100644 ray_results/test_experiment/train_model_45b8cfdd_17_batch_size=2048,decoder_int_ff=2048,decoder_int_num_layer=2,decoder_rt_ff=1024,decoder_rt_num_layer=4,drop_2024-06-25_16-46-32/error.txt
create mode 100644 ray_results/test_experiment/train_model_45b8cfdd_17_batch_size=2048,decoder_int_ff=2048,decoder_int_num_layer=2,decoder_rt_ff=1024,decoder_rt_num_layer=4,drop_2024-06-25_16-46-32/events.out.tfevents.1719326860.r3i5n6
create mode 100644 ray_results/test_experiment/train_model_45b8cfdd_17_batch_size=2048,decoder_int_ff=2048,decoder_int_num_layer=2,decoder_rt_ff=1024,decoder_rt_num_layer=4,drop_2024-06-25_16-46-32/params.json
create mode 100644 ray_results/test_experiment/train_model_45b8cfdd_17_batch_size=2048,decoder_int_ff=2048,decoder_int_num_layer=2,decoder_rt_ff=1024,decoder_rt_num_layer=4,drop_2024-06-25_16-46-32/params.pkl
create mode 100644 ray_results/test_experiment/train_model_45b8cfdd_17_batch_size=2048,decoder_int_ff=2048,decoder_int_num_layer=2,decoder_rt_ff=1024,decoder_rt_num_layer=4,drop_2024-06-25_16-46-32/result.json
create mode 100644 ray_results/test_experiment/train_model_481f1fff_10_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=8,drop__2024-06-25_17-33-30/events.out.tfevents.1719329693.r8i6n2
create mode 100644 ray_results/test_experiment/train_model_481f1fff_10_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=8,drop__2024-06-25_17-33-30/events.out.tfevents.1719330754.r8i6n2
create mode 100644 ray_results/test_experiment/train_model_481f1fff_10_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=8,drop__2024-06-25_17-33-30/params.json
create mode 100644 ray_results/test_experiment/train_model_481f1fff_10_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=8,drop__2024-06-25_17-33-30/params.pkl
create mode 100644 ray_results/test_experiment/train_model_481f1fff_10_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=8,drop__2024-06-25_17-33-30/progress.csv
create mode 100644 ray_results/test_experiment/train_model_481f1fff_10_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=8,drop__2024-06-25_17-33-30/result.json
create mode 100644 ray_results/test_experiment/train_model_52d8dfa1_2_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=4,drop_r_2024-06-25_16-08-04/events.out.tfevents.1719324574.r3i5n6
create mode 100644 ray_results/test_experiment/train_model_52d8dfa1_2_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=4,drop_r_2024-06-25_16-08-04/params.json
create mode 100644 ray_results/test_experiment/train_model_52d8dfa1_2_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=4,drop_r_2024-06-25_16-08-04/params.pkl
create mode 100644 ray_results/test_experiment/train_model_52d8dfa1_2_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=4,drop_r_2024-06-25_16-08-04/progress.csv
create mode 100644 ray_results/test_experiment/train_model_52d8dfa1_2_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=4,drop_r_2024-06-25_16-08-04/result.json
create mode 100644 ray_results/test_experiment/train_model_5763f130_1_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=8,drop_r_2024-06-25_17-10-00/events.out.tfevents.1719328213.r8i6n2
create mode 100644 ray_results/test_experiment/train_model_5763f130_1_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=8,drop_r_2024-06-25_17-10-00/params.json
create mode 100644 ray_results/test_experiment/train_model_5763f130_1_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=8,drop_r_2024-06-25_17-10-00/params.pkl
create mode 100644 ray_results/test_experiment/train_model_5763f130_1_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=8,drop_r_2024-06-25_17-10-00/progress.csv
create mode 100644 ray_results/test_experiment/train_model_5763f130_1_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=8,drop_r_2024-06-25_17-10-00/result.json
create mode 100644 ray_results/test_experiment/train_model_5b0a0ae2_6_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop_r_2024-06-25_17-18-52/events.out.tfevents.1719329376.r8i6n2
create mode 100644 ray_results/test_experiment/train_model_5b0a0ae2_6_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop_r_2024-06-25_17-18-52/events.out.tfevents.1719330587.r8i6n2
create mode 100644 ray_results/test_experiment/train_model_5b0a0ae2_6_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop_r_2024-06-25_17-18-52/params.json
create mode 100644 ray_results/test_experiment/train_model_5b0a0ae2_6_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop_r_2024-06-25_17-18-52/params.pkl
create mode 100644 ray_results/test_experiment/train_model_5b0a0ae2_6_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop_r_2024-06-25_17-18-52/progress.csv
create mode 100644 ray_results/test_experiment/train_model_5b0a0ae2_6_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop_r_2024-06-25_17-18-52/result.json
create mode 100644 ray_results/test_experiment/train_model_5eb1e282_23_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=1,drop_r_2024-06-25_18-57-29/events.out.tfevents.1719335674.r8i6n2
create mode 100644 ray_results/test_experiment/train_model_5eb1e282_23_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=1,drop_r_2024-06-25_18-57-29/events.out.tfevents.1719339767.r8i6n2
create mode 100644 ray_results/test_experiment/train_model_5eb1e282_23_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=1,drop_r_2024-06-25_18-57-29/events.out.tfevents.1719345748.r8i6n2
create mode 100644 ray_results/test_experiment/train_model_5eb1e282_23_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=1,drop_r_2024-06-25_18-57-29/params.json
create mode 100644 ray_results/test_experiment/train_model_5eb1e282_23_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=1,drop_r_2024-06-25_18-57-29/params.pkl
create mode 100644 ray_results/test_experiment/train_model_5eb1e282_23_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=1,drop_r_2024-06-25_18-57-29/progress.csv
create mode 100644 ray_results/test_experiment/train_model_5eb1e282_23_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=1,drop_r_2024-06-25_18-57-29/result.json
create mode 100644 ray_results/test_experiment/train_model_651349c4_12_batch_size=2048,decoder_int_ff=2048,decoder_int_num_layer=8,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop_2024-06-25_16-23-08/events.out.tfevents.1719326403.r3i5n6
create mode 100644 ray_results/test_experiment/train_model_651349c4_12_batch_size=2048,decoder_int_ff=2048,decoder_int_num_layer=8,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop_2024-06-25_16-23-08/params.json
create mode 100644 ray_results/test_experiment/train_model_651349c4_12_batch_size=2048,decoder_int_ff=2048,decoder_int_num_layer=8,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop_2024-06-25_16-23-08/params.pkl
create mode 100644 ray_results/test_experiment/train_model_651349c4_12_batch_size=2048,decoder_int_ff=2048,decoder_int_num_layer=8,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop_2024-06-25_16-23-08/progress.csv
create mode 100644 ray_results/test_experiment/train_model_651349c4_12_batch_size=2048,decoder_int_ff=2048,decoder_int_num_layer=8,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop_2024-06-25_16-23-08/result.json
create mode 100644 ray_results/test_experiment/train_model_6743cdee_21_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=4,drop__2024-06-25_18-48-59/events.out.tfevents.1719334380.r8i6n2
create mode 100644 ray_results/test_experiment/train_model_6743cdee_21_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=4,drop__2024-06-25_18-48-59/params.json
create mode 100644 ray_results/test_experiment/train_model_6743cdee_21_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=4,drop__2024-06-25_18-48-59/params.pkl
create mode 100644 ray_results/test_experiment/train_model_6743cdee_21_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=4,drop__2024-06-25_18-48-59/progress.csv
create mode 100644 ray_results/test_experiment/train_model_6743cdee_21_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=4,drop__2024-06-25_18-48-59/result.json
create mode 100644 ray_results/test_experiment/train_model_6a62b85a_3_batch_size=2048,decoder_int_ff=1024,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop__2024-06-25_16-09-34/events.out.tfevents.1719324663.r3i5n6
create mode 100644 ray_results/test_experiment/train_model_6a62b85a_3_batch_size=2048,decoder_int_ff=1024,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop__2024-06-25_16-09-34/events.out.tfevents.1719327182.r3i5n6
create mode 100644 ray_results/test_experiment/train_model_6a62b85a_3_batch_size=2048,decoder_int_ff=1024,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop__2024-06-25_16-09-34/params.json
create mode 100644 ray_results/test_experiment/train_model_6a62b85a_3_batch_size=2048,decoder_int_ff=1024,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop__2024-06-25_16-09-34/params.pkl
create mode 100644 ray_results/test_experiment/train_model_6a62b85a_3_batch_size=2048,decoder_int_ff=1024,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop__2024-06-25_16-09-34/progress.csv
create mode 100644 ray_results/test_experiment/train_model_6a62b85a_3_batch_size=2048,decoder_int_ff=1024,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop__2024-06-25_16-09-34/result.json
create mode 100644 ray_results/test_experiment/train_model_6be79717_24_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop__2024-06-25_19-14-34/events.out.tfevents.1719335783.r8i6n2
create mode 100644 ray_results/test_experiment/train_model_6be79717_24_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop__2024-06-25_19-14-34/params.json
create mode 100644 ray_results/test_experiment/train_model_6be79717_24_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop__2024-06-25_19-14-34/params.pkl
create mode 100644 ray_results/test_experiment/train_model_6be79717_24_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop__2024-06-25_19-14-34/progress.csv
create mode 100644 ray_results/test_experiment/train_model_6be79717_24_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop__2024-06-25_19-14-34/result.json
create mode 100644 ray_results/test_experiment/train_model_6c8b476a_20_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=4,drop_r_2024-06-25_18-43-49/events.out.tfevents.1719334139.r8i6n2
create mode 100644 ray_results/test_experiment/train_model_6c8b476a_20_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=4,drop_r_2024-06-25_18-43-49/events.out.tfevents.1719338498.r8i6n2
create mode 100644 ray_results/test_experiment/train_model_6c8b476a_20_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=4,drop_r_2024-06-25_18-43-49/params.json
create mode 100644 ray_results/test_experiment/train_model_6c8b476a_20_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=4,drop_r_2024-06-25_18-43-49/params.pkl
create mode 100644 ray_results/test_experiment/train_model_6c8b476a_20_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=4,drop_r_2024-06-25_18-43-49/progress.csv
create mode 100644 ray_results/test_experiment/train_model_6c8b476a_20_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=4,drop_r_2024-06-25_18-43-49/result.json
create mode 100644 ray_results/test_experiment/train_model_7ddcd025_3_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=8,drop_ra_2024-06-25_17-14-16/events.out.tfevents.1719328526.r8i6n2
create mode 100644 ray_results/test_experiment/train_model_7ddcd025_3_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=8,drop_ra_2024-06-25_17-14-16/params.json
create mode 100644 ray_results/test_experiment/train_model_7ddcd025_3_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=8,drop_ra_2024-06-25_17-14-16/params.pkl
create mode 100644 ray_results/test_experiment/train_model_7ddcd025_3_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=8,drop_ra_2024-06-25_17-14-16/progress.csv
create mode 100644 ray_results/test_experiment/train_model_7ddcd025_3_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=8,drop_ra_2024-06-25_17-14-16/result.json
create mode 100644 ray_results/test_experiment/train_model_7dfe67c7_4_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=2,drop_r_2024-06-25_17-15-26/events.out.tfevents.1719328617.r8i6n2
create mode 100644 ray_results/test_experiment/train_model_7dfe67c7_4_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=2,drop_r_2024-06-25_17-15-26/params.json
create mode 100644 ray_results/test_experiment/train_model_7dfe67c7_4_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=2,drop_r_2024-06-25_17-15-26/params.pkl
create mode 100644 ray_results/test_experiment/train_model_7dfe67c7_4_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=2,drop_r_2024-06-25_17-15-26/progress.csv
create mode 100644 ray_results/test_experiment/train_model_7dfe67c7_4_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=2,drop_r_2024-06-25_17-15-26/result.json
create mode 100644 ray_results/test_experiment/train_model_82396cf3_25_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_19-16-23/events.out.tfevents.1719335912.r8i6n2
create mode 100644 ray_results/test_experiment/train_model_82396cf3_25_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_19-16-23/params.json
create mode 100644 ray_results/test_experiment/train_model_82396cf3_25_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_19-16-23/params.pkl
create mode 100644 ray_results/test_experiment/train_model_82396cf3_25_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_19-16-23/progress.csv
create mode 100644 ray_results/test_experiment/train_model_82396cf3_25_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_19-16-23/result.json
create mode 100644 ray_results/test_experiment/train_model_86ae258f_7_batch_size=2048,decoder_int_ff=1024,decoder_int_num_layer=4,decoder_rt_ff=1024,decoder_rt_num_layer=4,drop__2024-06-25_16-15-18/events.out.tfevents.1719325014.r3i5n6
create mode 100644 ray_results/test_experiment/train_model_86ae258f_7_batch_size=2048,decoder_int_ff=1024,decoder_int_num_layer=4,decoder_rt_ff=1024,decoder_rt_num_layer=4,drop__2024-06-25_16-15-18/events.out.tfevents.1719327469.r3i5n6
create mode 100644 ray_results/test_experiment/train_model_86ae258f_7_batch_size=2048,decoder_int_ff=1024,decoder_int_num_layer=4,decoder_rt_ff=1024,decoder_rt_num_layer=4,drop__2024-06-25_16-15-18/params.json
create mode 100644 ray_results/test_experiment/train_model_86ae258f_7_batch_size=2048,decoder_int_ff=1024,decoder_int_num_layer=4,decoder_rt_ff=1024,decoder_rt_num_layer=4,drop__2024-06-25_16-15-18/params.pkl
create mode 100644 ray_results/test_experiment/train_model_86ae258f_7_batch_size=2048,decoder_int_ff=1024,decoder_int_num_layer=4,decoder_rt_ff=1024,decoder_rt_num_layer=4,drop__2024-06-25_16-15-18/progress.csv
create mode 100644 ray_results/test_experiment/train_model_86ae258f_7_batch_size=2048,decoder_int_ff=1024,decoder_int_num_layer=4,decoder_rt_ff=1024,decoder_rt_num_layer=4,drop__2024-06-25_16-15-18/result.json
create mode 100644 ray_results/test_experiment/train_model_8c6e7040_14_batch_size=2048,decoder_int_ff=2048,decoder_int_num_layer=8,decoder_rt_ff=1024,decoder_rt_num_layer=8,drop_2024-06-25_16-41-29/error.pkl
create mode 100644 ray_results/test_experiment/train_model_8c6e7040_14_batch_size=2048,decoder_int_ff=2048,decoder_int_num_layer=8,decoder_rt_ff=1024,decoder_rt_num_layer=8,drop_2024-06-25_16-41-29/error.txt
create mode 100644 ray_results/test_experiment/train_model_8c6e7040_14_batch_size=2048,decoder_int_ff=2048,decoder_int_num_layer=8,decoder_rt_ff=1024,decoder_rt_num_layer=8,drop_2024-06-25_16-41-29/events.out.tfevents.1719326572.r3i5n6
create mode 100644 ray_results/test_experiment/train_model_8c6e7040_14_batch_size=2048,decoder_int_ff=2048,decoder_int_num_layer=8,decoder_rt_ff=1024,decoder_rt_num_layer=8,drop_2024-06-25_16-41-29/params.json
create mode 100644 ray_results/test_experiment/train_model_8c6e7040_14_batch_size=2048,decoder_int_ff=2048,decoder_int_num_layer=8,decoder_rt_ff=1024,decoder_rt_num_layer=8,drop_2024-06-25_16-41-29/params.pkl
create mode 100644 ray_results/test_experiment/train_model_8c6e7040_14_batch_size=2048,decoder_int_ff=2048,decoder_int_num_layer=8,decoder_rt_ff=1024,decoder_rt_num_layer=8,drop_2024-06-25_16-41-29/result.json
create mode 100644 ray_results/test_experiment/train_model_8e58579c_18_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=8,decoder_rt_ff=2048,decoder_rt_num_layer=8,drop__2024-06-25_16-47-40/events.out.tfevents.1719326883.r3i5n6
create mode 100644 ray_results/test_experiment/train_model_8e58579c_18_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=8,decoder_rt_ff=2048,decoder_rt_num_layer=8,drop__2024-06-25_16-47-40/params.json
create mode 100644 ray_results/test_experiment/train_model_8e58579c_18_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=8,decoder_rt_ff=2048,decoder_rt_num_layer=8,drop__2024-06-25_16-47-40/params.pkl
create mode 100644 ray_results/test_experiment/train_model_8e58579c_18_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=8,decoder_rt_ff=2048,decoder_rt_num_layer=8,drop__2024-06-25_16-47-40/progress.csv
create mode 100644 ray_results/test_experiment/train_model_8e58579c_18_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=8,decoder_rt_ff=2048,decoder_rt_num_layer=8,drop__2024-06-25_16-47-40/result.json
create mode 100644 ray_results/test_experiment/train_model_934c5104_9_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop_r_2024-06-25_17-32-25/events.out.tfevents.1719329610.r8i6n2
create mode 100644 ray_results/test_experiment/train_model_934c5104_9_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop_r_2024-06-25_17-32-25/events.out.tfevents.1719330672.r8i6n2
create mode 100644 ray_results/test_experiment/train_model_934c5104_9_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop_r_2024-06-25_17-32-25/events.out.tfevents.1719331282.r8i6n2
create mode 100644 ray_results/test_experiment/train_model_934c5104_9_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop_r_2024-06-25_17-32-25/events.out.tfevents.1719332071.r8i6n2
create mode 100644 ray_results/test_experiment/train_model_934c5104_9_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop_r_2024-06-25_17-32-25/params.json
create mode 100644 ray_results/test_experiment/train_model_934c5104_9_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop_r_2024-06-25_17-32-25/params.pkl
create mode 100644 ray_results/test_experiment/train_model_934c5104_9_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop_r_2024-06-25_17-32-25/progress.csv
create mode 100644 ray_results/test_experiment/train_model_934c5104_9_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop_r_2024-06-25_17-32-25/result.json
create mode 100644 ray_results/test_experiment/train_model_975aa6a0_4_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=2,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop_r_2024-06-25_16-11-03/events.out.tfevents.1719324728.r3i5n6
create mode 100644 ray_results/test_experiment/train_model_975aa6a0_4_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=2,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop_r_2024-06-25_16-11-03/params.json
create mode 100644 ray_results/test_experiment/train_model_975aa6a0_4_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=2,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop_r_2024-06-25_16-11-03/params.pkl
create mode 100644 ray_results/test_experiment/train_model_975aa6a0_4_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=2,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop_r_2024-06-25_16-11-03/progress.csv
create mode 100644 ray_results/test_experiment/train_model_975aa6a0_4_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=2,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop_r_2024-06-25_16-11-03/result.json
create mode 100644 ray_results/test_experiment/train_model_a00d3259_2_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=4,drop_ra_2024-06-25_17-10-13/events.out.tfevents.1719328456.r8i6n2
create mode 100644 ray_results/test_experiment/train_model_a00d3259_2_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=4,drop_ra_2024-06-25_17-10-13/events.out.tfevents.1719330517.r8i6n2
create mode 100644 ray_results/test_experiment/train_model_a00d3259_2_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=4,drop_ra_2024-06-25_17-10-13/events.out.tfevents.1719331131.r8i6n2
create mode 100644 ray_results/test_experiment/train_model_a00d3259_2_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=4,drop_ra_2024-06-25_17-10-13/events.out.tfevents.1719331824.r8i6n2
create mode 100644 ray_results/test_experiment/train_model_a00d3259_2_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=4,drop_ra_2024-06-25_17-10-13/params.json
create mode 100644 ray_results/test_experiment/train_model_a00d3259_2_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=4,drop_ra_2024-06-25_17-10-13/params.pkl
create mode 100644 ray_results/test_experiment/train_model_a00d3259_2_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=4,drop_ra_2024-06-25_17-10-13/progress.csv
create mode 100644 ray_results/test_experiment/train_model_a00d3259_2_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=4,drop_ra_2024-06-25_17-10-13/result.json
create mode 100644 ray_results/test_experiment/train_model_a01c9b2d_16_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_17-44-50/events.out.tfevents.1719330362.r8i6n2
create mode 100644 ray_results/test_experiment/train_model_a01c9b2d_16_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_17-44-50/events.out.tfevents.1719331634.r8i6n2
create mode 100644 ray_results/test_experiment/train_model_a01c9b2d_16_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_17-44-50/params.json
create mode 100644 ray_results/test_experiment/train_model_a01c9b2d_16_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_17-44-50/params.pkl
create mode 100644 ray_results/test_experiment/train_model_a01c9b2d_16_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_17-44-50/progress.csv
create mode 100644 ray_results/test_experiment/train_model_a01c9b2d_16_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_17-44-50/result.json
create mode 100644 ray_results/test_experiment/train_model_a0e8b2af_1_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=8,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop_r_2024-06-25_16-07-48/events.out.tfevents.1719324484.r3i5n6
create mode 100644 ray_results/test_experiment/train_model_a0e8b2af_1_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=8,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop_r_2024-06-25_16-07-48/params.json
create mode 100644 ray_results/test_experiment/train_model_a0e8b2af_1_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=8,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop_r_2024-06-25_16-07-48/params.pkl
create mode 100644 ray_results/test_experiment/train_model_a0e8b2af_1_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=8,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop_r_2024-06-25_16-07-48/progress.csv
create mode 100644 ray_results/test_experiment/train_model_a0e8b2af_1_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=8,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop_r_2024-06-25_16-07-48/result.json
create mode 100644 ray_results/test_experiment/train_model_ad12346e_12_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_17-36-47/events.out.tfevents.1719329892.r8i6n2
create mode 100644 ray_results/test_experiment/train_model_ad12346e_12_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_17-36-47/params.json
create mode 100644 ray_results/test_experiment/train_model_ad12346e_12_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_17-36-47/params.pkl
create mode 100644 ray_results/test_experiment/train_model_ad12346e_12_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_17-36-47/progress.csv
create mode 100644 ray_results/test_experiment/train_model_ad12346e_12_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_17-36-47/result.json
create mode 100644 ray_results/test_experiment/train_model_af6d9c38_5_batch_size=2048,decoder_int_ff=1024,decoder_int_num_layer=8,decoder_rt_ff=1024,decoder_rt_num_layer=8,drop__2024-06-25_16-12-08/events.out.tfevents.1719324792.r3i5n6
create mode 100644 ray_results/test_experiment/train_model_af6d9c38_5_batch_size=2048,decoder_int_ff=1024,decoder_int_num_layer=8,decoder_rt_ff=1024,decoder_rt_num_layer=8,drop__2024-06-25_16-12-08/events.out.tfevents.1719327245.r3i5n6
create mode 100644 ray_results/test_experiment/train_model_af6d9c38_5_batch_size=2048,decoder_int_ff=1024,decoder_int_num_layer=8,decoder_rt_ff=1024,decoder_rt_num_layer=8,drop__2024-06-25_16-12-08/params.json
create mode 100644 ray_results/test_experiment/train_model_af6d9c38_5_batch_size=2048,decoder_int_ff=1024,decoder_int_num_layer=8,decoder_rt_ff=1024,decoder_rt_num_layer=8,drop__2024-06-25_16-12-08/params.pkl
create mode 100644 ray_results/test_experiment/train_model_af6d9c38_5_batch_size=2048,decoder_int_ff=1024,decoder_int_num_layer=8,decoder_rt_ff=1024,decoder_rt_num_layer=8,drop__2024-06-25_16-12-08/progress.csv
create mode 100644 ray_results/test_experiment/train_model_af6d9c38_5_batch_size=2048,decoder_int_ff=1024,decoder_int_num_layer=8,decoder_rt_ff=1024,decoder_rt_num_layer=8,drop__2024-06-25_16-12-08/result.json
create mode 100644 ray_results/test_experiment/train_model_b53cfe05_16_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=8,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_16-43-18/events.out.tfevents.1719326792.r3i5n6
create mode 100644 ray_results/test_experiment/train_model_b53cfe05_16_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=8,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_16-43-18/params.json
create mode 100644 ray_results/test_experiment/train_model_b53cfe05_16_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=8,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_16-43-18/params.pkl
create mode 100644 ray_results/test_experiment/train_model_b53cfe05_16_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=8,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_16-43-18/progress.csv
create mode 100644 ray_results/test_experiment/train_model_b53cfe05_16_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=8,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_16-43-18/result.json
create mode 100644 ray_results/test_experiment/train_model_b577349d_6_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=2,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop_r_2024-06-25_16-13-12/events.out.tfevents.1719324918.r3i5n6
create mode 100644 ray_results/test_experiment/train_model_b577349d_6_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=2,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop_r_2024-06-25_16-13-12/events.out.tfevents.1719327372.r3i5n6
create mode 100644 ray_results/test_experiment/train_model_b577349d_6_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=2,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop_r_2024-06-25_16-13-12/params.json
create mode 100644 ray_results/test_experiment/train_model_b577349d_6_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=2,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop_r_2024-06-25_16-13-12/params.pkl
create mode 100644 ray_results/test_experiment/train_model_b577349d_6_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=2,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop_r_2024-06-25_16-13-12/progress.csv
create mode 100644 ray_results/test_experiment/train_model_b577349d_6_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=2,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop_r_2024-06-25_16-13-12/result.json
create mode 100644 ray_results/test_experiment/train_model_bea15841_17_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=4,drop__2024-06-25_17-46-02/events.out.tfevents.1719332768.r8i6n2
create mode 100644 ray_results/test_experiment/train_model_bea15841_17_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=4,drop__2024-06-25_17-46-02/params.json
create mode 100644 ray_results/test_experiment/train_model_bea15841_17_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=4,drop__2024-06-25_17-46-02/params.pkl
create mode 100644 ray_results/test_experiment/train_model_bea15841_17_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=4,drop__2024-06-25_17-46-02/progress.csv
create mode 100644 ray_results/test_experiment/train_model_bea15841_17_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=4,drop__2024-06-25_17-46-02/result.json
create mode 100644 ray_results/test_experiment/train_model_c2c99680_18_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_18-26-08/events.out.tfevents.1719332932.r8i6n2
create mode 100644 ray_results/test_experiment/train_model_c2c99680_18_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_18-26-08/events.out.tfevents.1719337296.r8i6n2
create mode 100644 ray_results/test_experiment/train_model_c2c99680_18_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_18-26-08/params.json
create mode 100644 ray_results/test_experiment/train_model_c2c99680_18_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_18-26-08/params.pkl
create mode 100644 ray_results/test_experiment/train_model_c2c99680_18_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_18-26-08/progress.csv
create mode 100644 ray_results/test_experiment/train_model_c2c99680_18_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_18-26-08/result.json
create mode 100644 ray_results/test_experiment/train_model_c4172973_8_batch_size=2048,decoder_int_ff=2048,decoder_int_num_layer=8,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_16-16-54/events.out.tfevents.1719325156.r3i5n6
create mode 100644 ray_results/test_experiment/train_model_c4172973_8_batch_size=2048,decoder_int_ff=2048,decoder_int_num_layer=8,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_16-16-54/events.out.tfevents.1719327610.r3i5n6
create mode 100644 ray_results/test_experiment/train_model_c4172973_8_batch_size=2048,decoder_int_ff=2048,decoder_int_num_layer=8,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_16-16-54/params.json
create mode 100644 ray_results/test_experiment/train_model_c4172973_8_batch_size=2048,decoder_int_ff=2048,decoder_int_num_layer=8,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_16-16-54/params.pkl
create mode 100644 ray_results/test_experiment/train_model_c4172973_8_batch_size=2048,decoder_int_ff=2048,decoder_int_num_layer=8,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_16-16-54/progress.csv
create mode 100644 ray_results/test_experiment/train_model_c4172973_8_batch_size=2048,decoder_int_ff=2048,decoder_int_num_layer=8,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_16-16-54/result.json
create mode 100644 ray_results/test_experiment/train_model_dd950274_1_batch_size=2048,decoder_int_ff=1024,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_15-19-46/events.out.tfevents.1719321603.r7i5n7
create mode 100644 ray_results/test_experiment/train_model_dd950274_1_batch_size=2048,decoder_int_ff=1024,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_15-19-46/params.json
create mode 100644 ray_results/test_experiment/train_model_dd950274_1_batch_size=2048,decoder_int_ff=1024,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_15-19-46/params.pkl
create mode 100644 ray_results/test_experiment/train_model_dd950274_1_batch_size=2048,decoder_int_ff=1024,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_15-19-46/progress.csv
create mode 100644 ray_results/test_experiment/train_model_dd950274_1_batch_size=2048,decoder_int_ff=1024,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_15-19-46/result.json
create mode 100644 ray_results/test_experiment/train_model_de1e9134_30_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=1,drop_r_2024-06-25_22-20-23/events.out.tfevents.1719348339.r8i6n2
create mode 100644 ray_results/test_experiment/train_model_de1e9134_30_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=1,drop_r_2024-06-25_22-20-23/events.out.tfevents.1719355223.r8i6n2
create mode 100644 ray_results/test_experiment/train_model_de1e9134_30_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=1,drop_r_2024-06-25_22-20-23/params.json
create mode 100644 ray_results/test_experiment/train_model_de1e9134_30_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=1,drop_r_2024-06-25_22-20-23/params.pkl
create mode 100644 ray_results/test_experiment/train_model_de1e9134_30_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=1,drop_r_2024-06-25_22-20-23/progress.csv
create mode 100644 ray_results/test_experiment/train_model_de1e9134_30_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=1,drop_r_2024-06-25_22-20-23/result.json
create mode 100644 ray_results/test_experiment/train_model_e31d7ca7_26_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=2,drop__2024-06-25_19-18-32/events.out.tfevents.1719337143.r8i6n2
create mode 100644 ray_results/test_experiment/train_model_e31d7ca7_26_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=2,drop__2024-06-25_19-18-32/params.json
create mode 100644 ray_results/test_experiment/train_model_e31d7ca7_26_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=2,drop__2024-06-25_19-18-32/params.pkl
create mode 100644 ray_results/test_experiment/train_model_e31d7ca7_26_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=2,drop__2024-06-25_19-18-32/progress.csv
create mode 100644 ray_results/test_experiment/train_model_e31d7ca7_26_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=2,drop__2024-06-25_19-18-32/result.json
create mode 100644 ray_results/test_experiment/train_model_e566e32a_13_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=2,drop__2024-06-25_16-40-03/events.out.tfevents.1719326489.r3i5n6
create mode 100644 ray_results/test_experiment/train_model_e566e32a_13_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=2,drop__2024-06-25_16-40-03/params.json
create mode 100644 ray_results/test_experiment/train_model_e566e32a_13_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=2,drop__2024-06-25_16-40-03/params.pkl
create mode 100644 ray_results/test_experiment/train_model_e566e32a_13_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=2,drop__2024-06-25_16-40-03/progress.csv
create mode 100644 ray_results/test_experiment/train_model_e566e32a_13_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=2,drop__2024-06-25_16-40-03/result.json
create mode 100644 ray_results/test_experiment/train_model_f1fbbaf2_5_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop_r_2024-06-25_17-16-57/events.out.tfevents.1719328732.r8i6n2
create mode 100644 ray_results/test_experiment/train_model_f1fbbaf2_5_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop_r_2024-06-25_17-16-57/params.json
create mode 100644 ray_results/test_experiment/train_model_f1fbbaf2_5_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop_r_2024-06-25_17-16-57/params.pkl
create mode 100644 ray_results/test_experiment/train_model_f1fbbaf2_5_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop_r_2024-06-25_17-16-57/progress.csv
create mode 100644 ray_results/test_experiment/train_model_f1fbbaf2_5_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop_r_2024-06-25_17-16-57/result.json
create mode 100644 ray_results/test_experiment/train_model_f74eb513_2_batch_size=1024,decoder_int_ff=1024,decoder_int_num_layer=8,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop__2024-06-25_15-20-03/events.out.tfevents.1719321692.r7i5n7
create mode 100644 ray_results/test_experiment/train_model_f74eb513_2_batch_size=1024,decoder_int_ff=1024,decoder_int_num_layer=8,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop__2024-06-25_15-20-03/params.json
create mode 100644 ray_results/test_experiment/train_model_f74eb513_2_batch_size=1024,decoder_int_ff=1024,decoder_int_num_layer=8,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop__2024-06-25_15-20-03/params.pkl
create mode 100644 ray_results/test_experiment/train_model_f74eb513_2_batch_size=1024,decoder_int_ff=1024,decoder_int_num_layer=8,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop__2024-06-25_15-20-03/result.json
create mode 100644 ray_results/test_experiment/train_model_fe083630_32_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_22-50-21/events.out.tfevents.1719350385.r8i6n2
create mode 100644 ray_results/test_experiment/train_model_fe083630_32_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_22-50-21/params.json
create mode 100644 ray_results/test_experiment/train_model_fe083630_32_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_22-50-21/params.pkl
create mode 100644 ray_results/test_experiment/train_model_fe083630_32_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_22-50-21/progress.csv
create mode 100644 ray_results/test_experiment/train_model_fe083630_32_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_22-50-21/result.json
diff --git a/ray_results/test_experiment/experiment_state-2024-06-25_15-19-46.json b/ray_results/test_experiment/experiment_state-2024-06-25_15-19-46.json
new file mode 100644
index 0000000..86a72d4
--- /dev/null
+++ b/ray_results/test_experiment/experiment_state-2024-06-25_15-19-46.json
@@ -0,0 +1,52 @@
+{
+ "trial_data": [
+ [
+ "{\n \"stub\": false,\n \"trainable_name\": \"train_model\",\n \"trial_id\": \"dd950274\",\n \"storage\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059582030000000000008c1b7261792e747261696e2e5f696e7465726e616c2e73746f72616765948c0e53746f72616765436f6e746578749493942981947d94288c12637573746f6d5f66735f70726f766964656494898c1273746f726167655f6c6f63616c5f70617468948c2b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c7473948c136578706572696d656e745f6469725f6e616d65948c0f746573745f6578706572696d656e74948c0e747269616c5f6469725f6e616d65948c96747261696e5f6d6f64656c5f64643935303237345f315f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d313032342c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d313032342c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f5f323032342d30362d32355f31352d31392d3436948c1863757272656e745f636865636b706f696e745f696e646578944b008c0b73796e635f636f6e666967948c097261792e747261696e948c0a53796e63436f6e6669679493942981947d94288c0b73796e635f706572696f64944d2c018c0c73796e635f74696d656f7574944d08078c0e73796e635f61727469666163747394898c1c73796e635f6172746966616374735f6f6e5f636865636b706f696e7494888c0a75706c6f61645f646972948c0a44455052454341544544948c0673796e6365729468188c1273796e635f6f6e5f636865636b706f696e7494681875628c1273746f726167655f66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c0f73746f726167655f66735f70617468948c452f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f7465737494681968008c115f46696c6573797374656d53796e6365729493942981947d9428681b682268134d2c0168144d08078c116c6173745f73796e635f75705f74696d659447fff00000000000008c136c6173745f73796e635f646f776e5f74696d659447fff00000000000008c0d5f73796e635f70726f63657373944e8c0c5f63757272656e745f636d64944e756275622e\"\n },\n \"config\": {\n \"encoder_num_layer\": 8,\n \"decoder_rt_num_layer\": 1,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 16,\n \"encoder_ff\": 1024,\n \"decoder_rt_ff\": 1024,\n \"decoder_int_ff\": 1024,\n \"n_head\": 16,\n \"drop_rate\": 0.7776595209025089,\n \"lr\": 0.0006106789085082022,\n \"batch_size\": 2048\n },\n \"_Trial__unresolved_config\": {\n \"encoder_num_layer\": 8,\n \"decoder_rt_num_layer\": 1,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 16,\n \"encoder_ff\": 1024,\n \"decoder_rt_ff\": 1024,\n \"decoder_int_ff\": 1024,\n \"n_head\": 16,\n \"drop_rate\": 0.7776595209025089,\n \"lr\": 0.0006106789085082022,\n \"batch_size\": 2048\n },\n \"evaluated_params\": {\n \"batch_size\": 2048,\n \"decoder_int_ff\": 1024,\n \"decoder_int_num_layer\": 1,\n \"decoder_rt_ff\": 1024,\n \"decoder_rt_num_layer\": 1,\n \"drop_rate\": 0.7776595209025089,\n \"embedding_dim\": 16,\n \"encoder_ff\": 1024,\n \"encoder_num_layer\": 8,\n \"lr\": 0.0006106789085082022,\n \"n_head\": 16\n },\n \"experiment_tag\": \"1_batch_size=2048,decoder_int_ff=1024,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop_rate=0.7777,embedding_dim=16,encoder_ff=1024,encoder_num_layer=8,lr=0.0006,n_head=16\",\n \"stopping_criterion\": {},\n \"_setup_default_resource\": true,\n \"_default_placement_group_factory\": \"80054e2e\",\n \"placement_group_factory\": \"800595ba000000000000008c237261792e74756e652e657865637574696f6e2e706c6163656d656e745f67726f757073948c15506c6163656d656e7447726f7570466163746f72799493942981947d94288c085f62756e646c6573945d947d94288c03435055944740540000000000008c0347505594473ff000000000000075618c155f686561645f62756e646c655f69735f656d70747994898c095f7374726174656779948c045041434b948c055f6172677394298c075f6b7761726773947d9475622e\",\n \"log_to_file\": [\n null,\n null\n ],\n \"max_failures\": 0,\n \"_default_result_or_future\": null,\n \"export_formats\": [],\n \"status\": \"PAUSED\",\n \"relative_logdir\": \"train_model_dd950274_1_batch_size=2048,decoder_int_ff=1024,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_15-19-46\",\n \"trial_name_creator\": null,\n \"trial_dirname_creator\": null,\n \"custom_trial_name\": null,\n \"custom_dirname\": null,\n \"restore_path\": null,\n \"_restore_checkpoint_result\": null,\n \"_state_json\": null,\n \"results\": \"80054e2e\",\n \"extra_arg\": \"80054e2e\",\n \"_resources\": \"80054e2e\"\n}",
+ "{\n \"start_time\": 1719321603.1018882,\n \"num_failures\": 0,\n \"num_failures_after_restore\": 0,\n \"error_filename\": null,\n \"pickled_error_filename\": null,\n \"last_result\": {\n \"loss\": 2001.7704808993603,\n \"timestamp\": 1719321692,\n \"checkpoint_dir_name\": \"checkpoint_000000\",\n \"should_checkpoint\": true,\n \"done\": false,\n \"training_iteration\": 1,\n \"trial_id\": \"dd950274\",\n \"date\": \"2024-06-25_15-21-32\",\n \"time_this_iter_s\": 89.34920597076416,\n \"time_total_s\": 89.34920597076416,\n \"pid\": 355319,\n \"hostname\": \"r7i5n7\",\n \"node_ip\": \"10.159.24.56\",\n \"config\": {\n \"encoder_num_layer\": 8,\n \"decoder_rt_num_layer\": 1,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 16,\n \"encoder_ff\": 1024,\n \"decoder_rt_ff\": 1024,\n \"decoder_int_ff\": 1024,\n \"n_head\": 16,\n \"drop_rate\": 0.7776595209025089,\n \"lr\": 0.0006106789085082022,\n \"batch_size\": 2048\n },\n \"time_since_restore\": 89.34920597076416,\n \"iterations_since_restore\": 1,\n \"experiment_tag\": \"1_batch_size=2048,decoder_int_ff=1024,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop_rate=0.7777,embedding_dim=16,encoder_ff=1024,encoder_num_layer=8,lr=0.0006,n_head=16\"\n },\n \"last_result_time\": 1719321692.456364,\n \"metric_analysis\": {\n \"loss\": {\n \"max\": 2001.7704808993603,\n \"min\": 2001.7704808993603,\n \"avg\": 2001.7704808993603,\n \"last\": 2001.7704808993603,\n \"last-5-avg\": 2001.7704808993603,\n \"last-10-avg\": 2001.7704808993603\n },\n \"should_checkpoint\": {\n \"max\": true,\n \"min\": true,\n \"avg\": true,\n \"last\": true,\n \"last-5-avg\": true,\n \"last-10-avg\": true\n },\n \"done\": {\n \"max\": false,\n \"min\": false,\n \"avg\": false,\n \"last\": false,\n \"last-5-avg\": false,\n \"last-10-avg\": false\n },\n \"training_iteration\": {\n \"max\": 1,\n \"min\": 1,\n \"avg\": 1,\n \"last\": 1,\n \"last-5-avg\": 1,\n \"last-10-avg\": 1\n },\n \"time_this_iter_s\": {\n \"max\": 89.34920597076416,\n \"min\": 89.34920597076416,\n \"avg\": 89.34920597076416,\n \"last\": 89.34920597076416,\n \"last-5-avg\": 89.34920597076416,\n \"last-10-avg\": 89.34920597076416\n },\n \"time_total_s\": {\n \"max\": 89.34920597076416,\n \"min\": 89.34920597076416,\n \"avg\": 89.34920597076416,\n \"last\": 89.34920597076416,\n \"last-5-avg\": 89.34920597076416,\n \"last-10-avg\": 89.34920597076416\n },\n \"time_since_restore\": {\n \"max\": 89.34920597076416,\n \"min\": 89.34920597076416,\n \"avg\": 89.34920597076416,\n \"last\": 89.34920597076416,\n \"last-5-avg\": 89.34920597076416,\n \"last-10-avg\": 89.34920597076416\n },\n \"iterations_since_restore\": {\n \"max\": 1,\n \"min\": 1,\n \"avg\": 1,\n \"last\": 1,\n \"last-5-avg\": 1,\n \"last-10-avg\": 1\n }\n },\n \"_n_steps\": [\n 5,\n 10\n ],\n \"metric_n_steps\": {\n \"loss\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059589000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452948c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b007494624308c8e3f1f814479f409486945294612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059589000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452948c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b007494624308c8e3f1f814479f409486945294612e\"\n }\n },\n \"should_checkpoint\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059522000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b058694529488612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059522000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a8694529488612e\"\n }\n },\n \"done\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059522000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b058694529489612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059522000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a8694529489612e\"\n }\n },\n \"training_iteration\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059523000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452944b01612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059523000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452944b01612e\"\n }\n },\n \"time_this_iter_s\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294474056565964000000612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294474056565964000000612e\"\n }\n },\n \"time_total_s\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294474056565964000000612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294474056565964000000612e\"\n }\n },\n \"time_since_restore\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294474056565964000000612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294474056565964000000612e\"\n }\n },\n \"iterations_since_restore\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059523000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452944b01612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059523000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452944b01612e\"\n }\n }\n },\n \"checkpoint_manager\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059517060000000000008c267261792e747261696e2e5f696e7465726e616c2e636865636b706f696e745f6d616e61676572948c125f436865636b706f696e744d616e616765729493942981947d94288c125f636865636b706f696e745f636f6e666967948c097261792e747261696e948c10436865636b706f696e74436f6e6669679493942981947d94288c0b6e756d5f746f5f6b656570944e8c1a636865636b706f696e745f73636f72655f617474726962757465944e8c16636865636b706f696e745f73636f72655f6f72646572948c036d6178948c14636865636b706f696e745f6672657175656e6379944b008c11636865636b706f696e745f61745f656e6494898c1a5f636865636b706f696e745f6b6565705f616c6c5f72616e6b73948c0a44455052454341544544948c1f5f636865636b706f696e745f75706c6f61645f66726f6d5f776f726b65727394681275628c135f636865636b706f696e745f726573756c7473945d948c1b7261792e747261696e2e5f696e7465726e616c2e73657373696f6e948c0f5f547261696e696e67526573756c749493942981947d94288c0a636865636b706f696e749468068c0a436865636b706f696e749493942981947d94288c0470617468948cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f64643935303237345f315f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d313032342c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d313032342c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f5f323032342d30362d32355f31352d31392d34362f636865636b706f696e745f303030303030948c0a66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c055f75756964948c0475756964948c04555549449493942981947d948c03696e74948a1050959366a0fa3fa36746f99b4aeaad34736275628c076d657472696373947d94288c046c6f7373948c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b007494624308c8e3f1f814479f4094869452948c0974696d657374616d70944a5cc47a668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303030948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b018c08747269616c5f6964948c086464393530323734948c0464617465948c13323032342d30362d32355f31352d32312d3332948c1074696d655f746869735f697465725f73944740565659640000008c0c74696d655f746f74616c5f73944740565659640000008c03706964944af76b05008c08686f73746e616d65948c06723769356e37948c076e6f64655f6970948c0c31302e3135392e32342e3536948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b088c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b108c0a656e636f6465725f6666944d00048c0d6465636f6465725f72745f6666944d00048c0e6465636f6465725f696e745f6666944d00048c066e5f68656164944b108c0964726f705f7261746594473fe8e296383660b38c026c7294473f4402bef85f047c8c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f7265944740565659640000008c18697465726174696f6e735f73696e63655f726573746f7265944b01757562618c195f6c61746573745f636865636b706f696e745f726573756c7494681975622e\"\n }\n}"
+ ],
+ [
+ "{\n \"stub\": false,\n \"trainable_name\": \"train_model\",\n \"trial_id\": \"f74eb513\",\n \"storage\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059575040000000000008c1b7261792e747261696e2e5f696e7465726e616c2e73746f72616765948c0e53746f72616765436f6e746578749493942981947d94288c12637573746f6d5f66735f70726f766964656494898c1273746f726167655f6c6f63616c5f70617468948c2b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c7473948c136578706572696d656e745f6469725f6e616d65948c0f746573745f6578706572696d656e74948c0e747269616c5f6469725f6e616d65948c96747261696e5f6d6f64656c5f66373465623531335f325f62617463685f73697a653d313032342c6465636f6465725f696e745f66663d313032342c6465636f6465725f696e745f6e756d5f6c617965723d382c6465636f6465725f72745f66663d313032342c6465636f6465725f72745f6e756d5f6c617965723d322c64726f705f5f323032342d30362d32355f31352d32302d3033948c1863757272656e745f636865636b706f696e745f696e646578944affffffff8c0b73796e635f636f6e666967948c097261792e747261696e948c0a53796e63436f6e6669679493942981947d94288c0b73796e635f706572696f64944d2c018c0c73796e635f74696d656f7574944d08078c0e73796e635f61727469666163747394898c1c73796e635f6172746966616374735f6f6e5f636865636b706f696e7494888c0a75706c6f61645f646972948c0a44455052454341544544948c0673796e6365729468188c1273796e635f6f6e5f636865636b706f696e7494681875628c1273746f726167655f66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c0f73746f726167655f66735f70617468948c452f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f7465737494681968008c115f46696c6573797374656d53796e6365729493942981947d9428681b682268134d2c0168144d08078c116c6173745f73796e635f75705f74696d65944741d99eb0fc98aa368c136c6173745f73796e635f646f776e5f74696d659447fff00000000000008c0d5f73796e635f70726f63657373944e8c0c5f63757272656e745f636d649468008c125f75706c6f61645f746f5f66735f706174689493947d94288c0a6c6f63616c5f70617468948c3b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c74732f746573745f6578706572696d656e74948c0266739468228c0766735f70617468948c552f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e74948c076578636c756465945d948c0e2a2f636865636b706f696e745f2a9461758694756275622e\"\n },\n \"config\": {\n \"encoder_num_layer\": 8,\n \"decoder_rt_num_layer\": 2,\n \"decoder_int_num_layer\": 8,\n \"embedding_dim\": 1024,\n \"encoder_ff\": 2048,\n \"decoder_rt_ff\": 1024,\n \"decoder_int_ff\": 1024,\n \"n_head\": 8,\n \"drop_rate\": 0.5887719140440579,\n \"lr\": 0.009580025887992769,\n \"batch_size\": 1024\n },\n \"_Trial__unresolved_config\": {\n \"encoder_num_layer\": 8,\n \"decoder_rt_num_layer\": 2,\n \"decoder_int_num_layer\": 8,\n \"embedding_dim\": 1024,\n \"encoder_ff\": 2048,\n \"decoder_rt_ff\": 1024,\n \"decoder_int_ff\": 1024,\n \"n_head\": 8,\n \"drop_rate\": 0.5887719140440579,\n \"lr\": 0.009580025887992769,\n \"batch_size\": 1024\n },\n \"evaluated_params\": {\n \"batch_size\": 1024,\n \"decoder_int_ff\": 1024,\n \"decoder_int_num_layer\": 8,\n \"decoder_rt_ff\": 1024,\n \"decoder_rt_num_layer\": 2,\n \"drop_rate\": 0.5887719140440579,\n \"embedding_dim\": 1024,\n \"encoder_ff\": 2048,\n \"encoder_num_layer\": 8,\n \"lr\": 0.009580025887992769,\n \"n_head\": 8\n },\n \"experiment_tag\": \"2_batch_size=1024,decoder_int_ff=1024,decoder_int_num_layer=8,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop_rate=0.5888,embedding_dim=1024,encoder_ff=2048,encoder_num_layer=8,lr=0.0096,n_head=8\",\n \"stopping_criterion\": {},\n \"_setup_default_resource\": true,\n \"_default_placement_group_factory\": \"80054e2e\",\n \"placement_group_factory\": \"800595ba000000000000008c237261792e74756e652e657865637574696f6e2e706c6163656d656e745f67726f757073948c15506c6163656d656e7447726f7570466163746f72799493942981947d94288c085f62756e646c6573945d947d94288c03435055944740540000000000008c0347505594473ff000000000000075618c155f686561645f62756e646c655f69735f656d70747994898c095f7374726174656779948c045041434b948c055f6172677394298c075f6b7761726773947d9475622e\",\n \"log_to_file\": [\n null,\n null\n ],\n \"max_failures\": 0,\n \"_default_result_or_future\": null,\n \"export_formats\": [],\n \"status\": \"RUNNING\",\n \"relative_logdir\": \"train_model_f74eb513_2_batch_size=1024,decoder_int_ff=1024,decoder_int_num_layer=8,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop__2024-06-25_15-20-03\",\n \"trial_name_creator\": null,\n \"trial_dirname_creator\": null,\n \"custom_trial_name\": null,\n \"custom_dirname\": null,\n \"restore_path\": null,\n \"_restore_checkpoint_result\": null,\n \"_state_json\": null,\n \"results\": \"80054e2e\",\n \"extra_arg\": \"80054e2e\",\n \"_resources\": \"80054e2e\"\n}",
+ "{\n \"start_time\": 1719321692.6103284,\n \"num_failures\": 0,\n \"num_failures_after_restore\": 0,\n \"error_filename\": null,\n \"pickled_error_filename\": null,\n \"last_result\": {\n \"trial_id\": \"f74eb513\"\n },\n \"last_result_time\": -Infinity,\n \"metric_analysis\": {},\n \"_n_steps\": [\n 5,\n 10\n ],\n \"metric_n_steps\": {},\n \"checkpoint_manager\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059584010000000000008c267261792e747261696e2e5f696e7465726e616c2e636865636b706f696e745f6d616e61676572948c125f436865636b706f696e744d616e616765729493942981947d94288c125f636865636b706f696e745f636f6e666967948c097261792e747261696e948c10436865636b706f696e74436f6e6669679493942981947d94288c0b6e756d5f746f5f6b656570944e8c1a636865636b706f696e745f73636f72655f617474726962757465944e8c16636865636b706f696e745f73636f72655f6f72646572948c036d6178948c14636865636b706f696e745f6672657175656e6379944b008c11636865636b706f696e745f61745f656e6494898c1a5f636865636b706f696e745f6b6565705f616c6c5f72616e6b73948c0a44455052454341544544948c1f5f636865636b706f696e745f75706c6f61645f66726f6d5f776f726b65727394681275628c135f636865636b706f696e745f726573756c7473945d948c195f6c61746573745f636865636b706f696e745f726573756c74944e75622e\"\n }\n}"
+ ],
+ [
+ "{\n \"stub\": false,\n \"trainable_name\": \"train_model\",\n \"trial_id\": \"53fe56f2\",\n \"storage\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059575040000000000008c1b7261792e747261696e2e5f696e7465726e616c2e73746f72616765948c0e53746f72616765436f6e746578749493942981947d94288c12637573746f6d5f66735f70726f766964656494898c1273746f726167655f6c6f63616c5f70617468948c2b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c7473948c136578706572696d656e745f6469725f6e616d65948c0f746573745f6578706572696d656e74948c0e747269616c5f6469725f6e616d65948c96747261696e5f6d6f64656c5f35336665353666325f335f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d323034382c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f725f323032342d30362d32355f31352d32312d3332948c1863757272656e745f636865636b706f696e745f696e646578944affffffff8c0b73796e635f636f6e666967948c097261792e747261696e948c0a53796e63436f6e6669679493942981947d94288c0b73796e635f706572696f64944d2c018c0c73796e635f74696d656f7574944d08078c0e73796e635f61727469666163747394898c1c73796e635f6172746966616374735f6f6e5f636865636b706f696e7494888c0a75706c6f61645f646972948c0a44455052454341544544948c0673796e6365729468188c1273796e635f6f6e5f636865636b706f696e7494681875628c1273746f726167655f66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c0f73746f726167655f66735f70617468948c452f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f7465737494681968008c115f46696c6573797374656d53796e6365729493942981947d9428681b682268134d2c0168144d08078c116c6173745f73796e635f75705f74696d65944741d99eb0fc98aa368c136c6173745f73796e635f646f776e5f74696d659447fff00000000000008c0d5f73796e635f70726f63657373944e8c0c5f63757272656e745f636d649468008c125f75706c6f61645f746f5f66735f706174689493947d94288c0a6c6f63616c5f70617468948c3b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c74732f746573745f6578706572696d656e74948c0266739468228c0766735f70617468948c552f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e74948c076578636c756465945d948c0e2a2f636865636b706f696e745f2a9461758694756275622e\"\n },\n \"config\": {\n \"encoder_num_layer\": 1,\n \"decoder_rt_num_layer\": 1,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 16,\n \"encoder_ff\": 2048,\n \"decoder_rt_ff\": 2048,\n \"decoder_int_ff\": 512,\n \"n_head\": 1,\n \"drop_rate\": 0.21039295796266755,\n \"lr\": 0.00010129411733818616,\n \"batch_size\": 2048\n },\n \"_Trial__unresolved_config\": {\n \"encoder_num_layer\": 1,\n \"decoder_rt_num_layer\": 1,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 16,\n \"encoder_ff\": 2048,\n \"decoder_rt_ff\": 2048,\n \"decoder_int_ff\": 512,\n \"n_head\": 1,\n \"drop_rate\": 0.21039295796266755,\n \"lr\": 0.00010129411733818616,\n \"batch_size\": 2048\n },\n \"evaluated_params\": {\n \"batch_size\": 2048,\n \"decoder_int_ff\": 512,\n \"decoder_int_num_layer\": 1,\n \"decoder_rt_ff\": 2048,\n \"decoder_rt_num_layer\": 1,\n \"drop_rate\": 0.21039295796266755,\n \"embedding_dim\": 16,\n \"encoder_ff\": 2048,\n \"encoder_num_layer\": 1,\n \"lr\": 0.00010129411733818616,\n \"n_head\": 1\n },\n \"experiment_tag\": \"3_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop_rate=0.2104,embedding_dim=16,encoder_ff=2048,encoder_num_layer=1,lr=0.0001,n_head=1\",\n \"stopping_criterion\": {},\n \"_setup_default_resource\": true,\n \"_default_placement_group_factory\": \"80054e2e\",\n \"placement_group_factory\": \"800595ba000000000000008c237261792e74756e652e657865637574696f6e2e706c6163656d656e745f67726f757073948c15506c6163656d656e7447726f7570466163746f72799493942981947d94288c085f62756e646c6573945d947d94288c03435055944740540000000000008c0347505594473ff000000000000075618c155f686561645f62756e646c655f69735f656d70747994898c095f7374726174656779948c045041434b948c055f6172677394298c075f6b7761726773947d9475622e\",\n \"log_to_file\": [\n null,\n null\n ],\n \"max_failures\": 0,\n \"_default_result_or_future\": null,\n \"export_formats\": [],\n \"status\": \"PENDING\",\n \"relative_logdir\": \"train_model_53fe56f2_3_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop_r_2024-06-25_15-21-32\",\n \"trial_name_creator\": null,\n \"trial_dirname_creator\": null,\n \"custom_trial_name\": null,\n \"custom_dirname\": null,\n \"restore_path\": null,\n \"_restore_checkpoint_result\": null,\n \"_state_json\": null,\n \"results\": \"80054e2e\",\n \"extra_arg\": \"80054e2e\",\n \"_resources\": \"80054e2e\"\n}",
+ "{\n \"start_time\": null,\n \"num_failures\": 0,\n \"num_failures_after_restore\": 0,\n \"error_filename\": null,\n \"pickled_error_filename\": null,\n \"last_result\": {},\n \"last_result_time\": -Infinity,\n \"metric_analysis\": {},\n \"_n_steps\": [\n 5,\n 10\n ],\n \"metric_n_steps\": {},\n \"checkpoint_manager\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059584010000000000008c267261792e747261696e2e5f696e7465726e616c2e636865636b706f696e745f6d616e61676572948c125f436865636b706f696e744d616e616765729493942981947d94288c125f636865636b706f696e745f636f6e666967948c097261792e747261696e948c10436865636b706f696e74436f6e6669679493942981947d94288c0b6e756d5f746f5f6b656570944e8c1a636865636b706f696e745f73636f72655f617474726962757465944e8c16636865636b706f696e745f73636f72655f6f72646572948c036d6178948c14636865636b706f696e745f6672657175656e6379944b008c11636865636b706f696e745f61745f656e6494898c1a5f636865636b706f696e745f6b6565705f616c6c5f72616e6b73948c0a44455052454341544544948c1f5f636865636b706f696e745f75706c6f61645f66726f6d5f776f726b65727394681275628c135f636865636b706f696e745f726573756c7473945d948c195f6c61746573745f636865636b706f696e745f726573756c74944e75622e\"\n }\n}"
+ ]
+ ],
+ "runner_data": {
+ "_earliest_stopping_actor": Infinity,
+ "_actor_cleanup_timeout": 600,
+ "_actor_force_cleanup_timeout": 10,
+ "_reuse_actors": true,
+ "_buffer_length": 1,
+ "_buffer_min_time_s": 0.0,
+ "_buffer_max_time_s": 100.0,
+ "_max_pending_trials": 1,
+ "_metric": "loss",
+ "_total_time": 89.34920597076416,
+ "_iteration": 4001,
+ "_has_errored": false,
+ "_fail_fast": false,
+ "_print_trial_errors": true,
+ "_cached_trial_decisions": {},
+ "_queued_trial_decisions": {},
+ "_should_stop_experiment": false,
+ "_stopper": {
+ "_type": "CLOUDPICKLE_FALLBACK",
+ "value": "8005952c000000000000008c157261792e74756e652e73746f707065722e6e6f6f70948c0b4e6f6f7053746f707065729493942981942e"
+ },
+ "_start_time": 1719321586.2168348,
+ "_last_checkpoint_time": -Infinity,
+ "_session_str": "2024-06-25_15-19-46",
+ "_checkpoint_period": "auto",
+ "_trial_checkpoint_config": {
+ "_type": "CLOUDPICKLE_FALLBACK",
+ "value": "800595f2000000000000008c097261792e747261696e948c10436865636b706f696e74436f6e6669679493942981947d94288c0b6e756d5f746f5f6b656570944e8c1a636865636b706f696e745f73636f72655f617474726962757465944e8c16636865636b706f696e745f73636f72655f6f72646572948c036d6178948c14636865636b706f696e745f6672657175656e6379944b008c11636865636b706f696e745f61745f656e6494898c1a5f636865636b706f696e745f6b6565705f616c6c5f72616e6b73948c0a44455052454341544544948c1f5f636865636b706f696e745f75706c6f61645f66726f6d5f776f726b65727394680c75622e"
+ },
+ "_resumed": false
+ },
+ "stats": {
+ "start_time": 1719321586.2168348,
+ "timestamp": -Infinity
+ }
+}
\ No newline at end of file
diff --git a/ray_results/test_experiment/experiment_state-2024-06-25_16-07-48.json b/ray_results/test_experiment/experiment_state-2024-06-25_16-07-48.json
new file mode 100644
index 0000000..f1a806f
--- /dev/null
+++ b/ray_results/test_experiment/experiment_state-2024-06-25_16-07-48.json
@@ -0,0 +1,116 @@
+{
+ "trial_data": [
+ [
+ "{\n \"stub\": false,\n \"trainable_name\": \"train_model\",\n \"trial_id\": \"a0e8b2af\",\n \"storage\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059582030000000000008c1b7261792e747261696e2e5f696e7465726e616c2e73746f72616765948c0e53746f72616765436f6e746578749493942981947d94288c12637573746f6d5f66735f70726f766964656494898c1273746f726167655f6c6f63616c5f70617468948c2b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c7473948c136578706572696d656e745f6469725f6e616d65948c0f746573745f6578706572696d656e74948c0e747269616c5f6469725f6e616d65948c96747261696e5f6d6f64656c5f61306538623261665f315f62617463685f73697a653d313032342c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d382c6465636f6465725f72745f66663d313032342c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f725f323032342d30362d32355f31362d30372d3438948c1863757272656e745f636865636b706f696e745f696e646578944b008c0b73796e635f636f6e666967948c097261792e747261696e948c0a53796e63436f6e6669679493942981947d94288c0b73796e635f706572696f64944d2c018c0c73796e635f74696d656f7574944d08078c0e73796e635f61727469666163747394898c1c73796e635f6172746966616374735f6f6e5f636865636b706f696e7494888c0a75706c6f61645f646972948c0a44455052454341544544948c0673796e6365729468188c1273796e635f6f6e5f636865636b706f696e7494681875628c1273746f726167655f66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c0f73746f726167655f66735f70617468948c452f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f7465737494681968008c115f46696c6573797374656d53796e6365729493942981947d9428681b682268134d2c0168144d08078c116c6173745f73796e635f75705f74696d659447fff00000000000008c136c6173745f73796e635f646f776e5f74696d659447fff00000000000008c0d5f73796e635f70726f63657373944e8c0c5f63757272656e745f636d64944e756275622e\"\n },\n \"config\": {\n \"encoder_num_layer\": 2,\n \"decoder_rt_num_layer\": 1,\n \"decoder_int_num_layer\": 8,\n \"embedding_dim\": 256,\n \"encoder_ff\": 2048,\n \"decoder_rt_ff\": 1024,\n \"decoder_int_ff\": 512,\n \"n_head\": 8,\n \"drop_rate\": 0.4534009171604666,\n \"lr\": 0.019472817375775914,\n \"batch_size\": 1024\n },\n \"_Trial__unresolved_config\": {\n \"encoder_num_layer\": 2,\n \"decoder_rt_num_layer\": 1,\n \"decoder_int_num_layer\": 8,\n \"embedding_dim\": 256,\n \"encoder_ff\": 2048,\n \"decoder_rt_ff\": 1024,\n \"decoder_int_ff\": 512,\n \"n_head\": 8,\n \"drop_rate\": 0.4534009171604666,\n \"lr\": 0.019472817375775914,\n \"batch_size\": 1024\n },\n \"evaluated_params\": {\n \"batch_size\": 1024,\n \"decoder_int_ff\": 512,\n \"decoder_int_num_layer\": 8,\n \"decoder_rt_ff\": 1024,\n \"decoder_rt_num_layer\": 1,\n \"drop_rate\": 0.4534009171604666,\n \"embedding_dim\": 256,\n \"encoder_ff\": 2048,\n \"encoder_num_layer\": 2,\n \"lr\": 0.019472817375775914,\n \"n_head\": 8\n },\n \"experiment_tag\": \"1_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=8,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop_rate=0.4534,embedding_dim=256,encoder_ff=2048,encoder_num_layer=2,lr=0.0195,n_head=8\",\n \"stopping_criterion\": {},\n \"_setup_default_resource\": true,\n \"_default_placement_group_factory\": \"80054e2e\",\n \"placement_group_factory\": \"800595ba000000000000008c237261792e74756e652e657865637574696f6e2e706c6163656d656e745f67726f757073948c15506c6163656d656e7447726f7570466163746f72799493942981947d94288c085f62756e646c6573945d947d94288c03435055944740540000000000008c034750559447401000000000000075618c155f686561645f62756e646c655f69735f656d70747994898c095f7374726174656779948c045041434b948c055f6172677394298c075f6b7761726773947d9475622e\",\n \"log_to_file\": [\n null,\n null\n ],\n \"max_failures\": 0,\n \"_default_result_or_future\": null,\n \"export_formats\": [],\n \"status\": \"PAUSED\",\n \"relative_logdir\": \"train_model_a0e8b2af_1_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=8,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop_r_2024-06-25_16-07-48\",\n \"trial_name_creator\": null,\n \"trial_dirname_creator\": null,\n \"custom_trial_name\": null,\n \"custom_dirname\": null,\n \"restore_path\": null,\n \"_restore_checkpoint_result\": null,\n \"_state_json\": null,\n \"results\": \"80054e2e\",\n \"extra_arg\": \"80054e2e\",\n \"_resources\": \"80054e2e\"\n}",
+ "{\n \"start_time\": 1719324484.8937905,\n \"num_failures\": 0,\n \"num_failures_after_restore\": 0,\n \"error_filename\": null,\n \"pickled_error_filename\": null,\n \"last_result\": {\n \"loss\": 2773.484795037217,\n \"timestamp\": 1719324574,\n \"checkpoint_dir_name\": \"checkpoint_000000\",\n \"should_checkpoint\": true,\n \"done\": false,\n \"training_iteration\": 1,\n \"trial_id\": \"a0e8b2af\",\n \"date\": \"2024-06-25_16-09-34\",\n \"time_this_iter_s\": 89.29933595657349,\n \"time_total_s\": 89.29933595657349,\n \"pid\": 69318,\n \"hostname\": \"r3i5n6\",\n \"node_ip\": \"10.159.8.159\",\n \"config\": {\n \"encoder_num_layer\": 2,\n \"decoder_rt_num_layer\": 1,\n \"decoder_int_num_layer\": 8,\n \"embedding_dim\": 256,\n \"encoder_ff\": 2048,\n \"decoder_rt_ff\": 1024,\n \"decoder_int_ff\": 512,\n \"n_head\": 8,\n \"drop_rate\": 0.4534009171604666,\n \"lr\": 0.019472817375775914,\n \"batch_size\": 1024\n },\n \"time_since_restore\": 89.29933595657349,\n \"iterations_since_restore\": 1,\n \"experiment_tag\": \"1_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=8,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop_rate=0.4534,embedding_dim=256,encoder_ff=2048,encoder_num_layer=2,lr=0.0195,n_head=8\"\n },\n \"last_result_time\": 1719324574.1991315,\n \"metric_analysis\": {\n \"loss\": {\n \"max\": 2773.484795037217,\n \"min\": 2773.484795037217,\n \"avg\": 2773.484795037217,\n \"last\": 2773.484795037217,\n \"last-5-avg\": 2773.484795037217,\n \"last-10-avg\": 2773.484795037217\n },\n \"should_checkpoint\": {\n \"max\": true,\n \"min\": true,\n \"avg\": true,\n \"last\": true,\n \"last-5-avg\": true,\n \"last-10-avg\": true\n },\n \"done\": {\n \"max\": false,\n \"min\": false,\n \"avg\": false,\n \"last\": false,\n \"last-5-avg\": false,\n \"last-10-avg\": false\n },\n \"training_iteration\": {\n \"max\": 1,\n \"min\": 1,\n \"avg\": 1,\n \"last\": 1,\n \"last-5-avg\": 1,\n \"last-10-avg\": 1\n },\n \"time_this_iter_s\": {\n \"max\": 89.29933595657349,\n \"min\": 89.29933595657349,\n \"avg\": 89.29933595657349,\n \"last\": 89.29933595657349,\n \"last-5-avg\": 89.29933595657349,\n \"last-10-avg\": 89.29933595657349\n },\n \"time_total_s\": {\n \"max\": 89.29933595657349,\n \"min\": 89.29933595657349,\n \"avg\": 89.29933595657349,\n \"last\": 89.29933595657349,\n \"last-5-avg\": 89.29933595657349,\n \"last-10-avg\": 89.29933595657349\n },\n \"time_since_restore\": {\n \"max\": 89.29933595657349,\n \"min\": 89.29933595657349,\n \"avg\": 89.29933595657349,\n \"last\": 89.29933595657349,\n \"last-5-avg\": 89.29933595657349,\n \"last-10-avg\": 89.29933595657349\n },\n \"iterations_since_restore\": {\n \"max\": 1,\n \"min\": 1,\n \"avg\": 1,\n \"last\": 1,\n \"last-5-avg\": 1,\n \"last-10-avg\": 1\n }\n },\n \"_n_steps\": [\n 5,\n 10\n ],\n \"metric_n_steps\": {\n \"loss\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059589000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452948c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b007494624308381c0e37f8aaa5409486945294612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059589000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452948c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b007494624308381c0e37f8aaa5409486945294612e\"\n }\n },\n \"should_checkpoint\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059522000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b058694529488612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059522000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a8694529488612e\"\n }\n },\n \"done\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059522000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b058694529489612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059522000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a8694529489612e\"\n }\n },\n \"training_iteration\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059523000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452944b01612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059523000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452944b01612e\"\n }\n },\n \"time_this_iter_s\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294474056532852000000612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294474056532852000000612e\"\n }\n },\n \"time_total_s\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294474056532852000000612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294474056532852000000612e\"\n }\n },\n \"time_since_restore\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294474056532852000000612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294474056532852000000612e\"\n }\n },\n \"iterations_since_restore\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059523000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452944b01612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059523000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452944b01612e\"\n }\n }\n },\n \"checkpoint_manager\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059519060000000000008c267261792e747261696e2e5f696e7465726e616c2e636865636b706f696e745f6d616e61676572948c125f436865636b706f696e744d616e616765729493942981947d94288c125f636865636b706f696e745f636f6e666967948c097261792e747261696e948c10436865636b706f696e74436f6e6669679493942981947d94288c0b6e756d5f746f5f6b656570944e8c1a636865636b706f696e745f73636f72655f617474726962757465944e8c16636865636b706f696e745f73636f72655f6f72646572948c036d6178948c14636865636b706f696e745f6672657175656e6379944b008c11636865636b706f696e745f61745f656e6494898c1a5f636865636b706f696e745f6b6565705f616c6c5f72616e6b73948c0a44455052454341544544948c1f5f636865636b706f696e745f75706c6f61645f66726f6d5f776f726b65727394681275628c135f636865636b706f696e745f726573756c7473945d948c1b7261792e747261696e2e5f696e7465726e616c2e73657373696f6e948c0f5f547261696e696e67526573756c749493942981947d94288c0a636865636b706f696e749468068c0a436865636b706f696e749493942981947d94288c0470617468948cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f61306538623261665f315f62617463685f73697a653d313032342c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d382c6465636f6465725f72745f66663d313032342c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f725f323032342d30362d32355f31362d30372d34382f636865636b706f696e745f303030303030948c0a66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c055f75756964948c0475756964948c04555549449493942981947d948c03696e74948a117618d482644e63b29e4dd84919f39dde00736275628c076d657472696373947d94288c046c6f7373948c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b007494624308381c0e37f8aaa54094869452948c0974696d657374616d70944a9ecf7a668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303030948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b018c08747269616c5f6964948c086130653862326166948c0464617465948c13323032342d30362d32355f31362d30392d3334948c1074696d655f746869735f697465725f73944740565328520000008c0c74696d655f746f74616c5f73944740565328520000008c03706964944ac60e01008c08686f73746e616d65948c06723369356e36948c076e6f64655f6970948c0c31302e3135392e382e313539948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b028c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b088c0d656d62656464696e675f64696d944d00018c0a656e636f6465725f6666944d00088c0d6465636f6465725f72745f6666944d00048c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b088c0964726f705f7261746594473fdd048547cb8f1a8c026c7294473f93f0aea728e5888c0a62617463685f73697a65944d0004758c1274696d655f73696e63655f726573746f7265944740565328520000008c18697465726174696f6e735f73696e63655f726573746f7265944b01757562618c195f6c61746573745f636865636b706f696e745f726573756c7494681975622e\"\n }\n}"
+ ],
+ [
+ "{\n \"stub\": false,\n \"trainable_name\": \"train_model\",\n \"trial_id\": \"52d8dfa1\",\n \"storage\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059572040000000000008c1b7261792e747261696e2e5f696e7465726e616c2e73746f72616765948c0e53746f72616765436f6e746578749493942981947d94288c12637573746f6d5f66735f70726f766964656494898c1273746f726167655f6c6f63616c5f70617468948c2b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c7473948c136578706572696d656e745f6469725f6e616d65948c0f746573745f6578706572696d656e74948c0e747269616c5f6469725f6e616d65948c96747261696e5f6d6f64656c5f35326438646661315f325f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d313032342c6465636f6465725f72745f6e756d5f6c617965723d342c64726f705f725f323032342d30362d32355f31362d30382d3034948c1863757272656e745f636865636b706f696e745f696e646578944b008c0b73796e635f636f6e666967948c097261792e747261696e948c0a53796e63436f6e6669679493942981947d94288c0b73796e635f706572696f64944d2c018c0c73796e635f74696d656f7574944d08078c0e73796e635f61727469666163747394898c1c73796e635f6172746966616374735f6f6e5f636865636b706f696e7494888c0a75706c6f61645f646972948c0a44455052454341544544948c0673796e6365729468188c1273796e635f6f6e5f636865636b706f696e7494681875628c1273746f726167655f66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c0f73746f726167655f66735f70617468948c452f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f7465737494681968008c115f46696c6573797374656d53796e6365729493942981947d9428681b682268134d2c0168144d08078c116c6173745f73796e635f75705f74696d65944741d99eb3cd22d0378c136c6173745f73796e635f646f776e5f74696d659447fff00000000000008c0d5f73796e635f70726f63657373944e8c0c5f63757272656e745f636d649468008c125f75706c6f61645f746f5f66735f706174689493947d94288c0a6c6f63616c5f70617468948c3b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c74732f746573745f6578706572696d656e74948c0266739468228c0766735f70617468948c552f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e74948c076578636c756465945d948c0e2a2f636865636b706f696e745f2a9461758694756275622e\"\n },\n \"config\": {\n \"encoder_num_layer\": 8,\n \"decoder_rt_num_layer\": 4,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 16,\n \"encoder_ff\": 1024,\n \"decoder_rt_ff\": 1024,\n \"decoder_int_ff\": 512,\n \"n_head\": 8,\n \"drop_rate\": 0.982125481793317,\n \"lr\": 0.002789281371289888,\n \"batch_size\": 2048\n },\n \"_Trial__unresolved_config\": {\n \"encoder_num_layer\": 8,\n \"decoder_rt_num_layer\": 4,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 16,\n \"encoder_ff\": 1024,\n \"decoder_rt_ff\": 1024,\n \"decoder_int_ff\": 512,\n \"n_head\": 8,\n \"drop_rate\": 0.982125481793317,\n \"lr\": 0.002789281371289888,\n \"batch_size\": 2048\n },\n \"evaluated_params\": {\n \"batch_size\": 2048,\n \"decoder_int_ff\": 512,\n \"decoder_int_num_layer\": 1,\n \"decoder_rt_ff\": 1024,\n \"decoder_rt_num_layer\": 4,\n \"drop_rate\": 0.982125481793317,\n \"embedding_dim\": 16,\n \"encoder_ff\": 1024,\n \"encoder_num_layer\": 8,\n \"lr\": 0.002789281371289888,\n \"n_head\": 8\n },\n \"experiment_tag\": \"2_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=4,drop_rate=0.9821,embedding_dim=16,encoder_ff=1024,encoder_num_layer=8,lr=0.0028,n_head=8\",\n \"stopping_criterion\": {},\n \"_setup_default_resource\": true,\n \"_default_placement_group_factory\": \"80054e2e\",\n \"placement_group_factory\": \"800595ba000000000000008c237261792e74756e652e657865637574696f6e2e706c6163656d656e745f67726f757073948c15506c6163656d656e7447726f7570466163746f72799493942981947d94288c085f62756e646c6573945d947d94288c03435055944740540000000000008c034750559447401000000000000075618c155f686561645f62756e646c655f69735f656d70747994898c095f7374726174656779948c045041434b948c055f6172677394298c075f6b7761726773947d9475622e\",\n \"log_to_file\": [\n null,\n null\n ],\n \"max_failures\": 0,\n \"_default_result_or_future\": null,\n \"export_formats\": [],\n \"status\": \"PAUSED\",\n \"relative_logdir\": \"train_model_52d8dfa1_2_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=4,drop_r_2024-06-25_16-08-04\",\n \"trial_name_creator\": null,\n \"trial_dirname_creator\": null,\n \"custom_trial_name\": null,\n \"custom_dirname\": null,\n \"restore_path\": null,\n \"_restore_checkpoint_result\": null,\n \"_state_json\": null,\n \"results\": \"80054e2e\",\n \"extra_arg\": \"80054e2e\",\n \"_resources\": \"80054e2e\"\n}",
+ "{\n \"start_time\": 1719324574.3448324,\n \"num_failures\": 0,\n \"num_failures_after_restore\": 0,\n \"error_filename\": null,\n \"pickled_error_filename\": null,\n \"last_result\": {\n \"loss\": NaN,\n \"timestamp\": 1719324663,\n \"checkpoint_dir_name\": \"checkpoint_000000\",\n \"should_checkpoint\": true,\n \"done\": false,\n \"training_iteration\": 1,\n \"trial_id\": \"52d8dfa1\",\n \"date\": \"2024-06-25_16-11-03\",\n \"time_this_iter_s\": 89.4755928516388,\n \"time_total_s\": 89.4755928516388,\n \"pid\": 69318,\n \"hostname\": \"r3i5n6\",\n \"node_ip\": \"10.159.8.159\",\n \"config\": {\n \"encoder_num_layer\": 8,\n \"decoder_rt_num_layer\": 4,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 16,\n \"encoder_ff\": 1024,\n \"decoder_rt_ff\": 1024,\n \"decoder_int_ff\": 512,\n \"n_head\": 8,\n \"drop_rate\": 0.982125481793317,\n \"lr\": 0.002789281371289888,\n \"batch_size\": 2048\n },\n \"time_since_restore\": 89.4755928516388,\n \"iterations_since_restore\": 1,\n \"experiment_tag\": \"2_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=4,drop_rate=0.9821,embedding_dim=16,encoder_ff=1024,encoder_num_layer=8,lr=0.0028,n_head=8\"\n },\n \"last_result_time\": 1719324663.8246083,\n \"metric_analysis\": {\n \"loss\": {\n \"max\": NaN,\n \"min\": NaN,\n \"avg\": NaN,\n \"last\": NaN,\n \"last-5-avg\": NaN,\n \"last-10-avg\": NaN\n },\n \"should_checkpoint\": {\n \"max\": true,\n \"min\": true,\n \"avg\": true,\n \"last\": true,\n \"last-5-avg\": true,\n \"last-10-avg\": true\n },\n \"done\": {\n \"max\": false,\n \"min\": false,\n \"avg\": false,\n \"last\": false,\n \"last-5-avg\": false,\n \"last-10-avg\": false\n },\n \"training_iteration\": {\n \"max\": 1,\n \"min\": 1,\n \"avg\": 1,\n \"last\": 1,\n \"last-5-avg\": 1,\n \"last-10-avg\": 1\n },\n \"time_this_iter_s\": {\n \"max\": 89.4755928516388,\n \"min\": 89.4755928516388,\n \"avg\": 89.4755928516388,\n \"last\": 89.4755928516388,\n \"last-5-avg\": 89.4755928516388,\n \"last-10-avg\": 89.4755928516388\n },\n \"time_total_s\": {\n \"max\": 89.4755928516388,\n \"min\": 89.4755928516388,\n \"avg\": 89.4755928516388,\n \"last\": 89.4755928516388,\n \"last-5-avg\": 89.4755928516388,\n \"last-10-avg\": 89.4755928516388\n },\n \"time_since_restore\": {\n \"max\": 89.4755928516388,\n \"min\": 89.4755928516388,\n \"avg\": 89.4755928516388,\n \"last\": 89.4755928516388,\n \"last-5-avg\": 89.4755928516388,\n \"last-10-avg\": 89.4755928516388\n },\n \"iterations_since_restore\": {\n \"max\": 1,\n \"min\": 1,\n \"avg\": 1,\n \"last\": 1,\n \"last-5-avg\": 1,\n \"last-10-avg\": 1\n }\n },\n \"_n_steps\": [\n 5,\n 10\n ],\n \"metric_n_steps\": {\n \"loss\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059589000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452948c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b007494624308000000e0ffffff7f9486945294612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059589000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452948c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b007494624308000000e0ffffff7f9486945294612e\"\n }\n },\n \"should_checkpoint\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059522000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b058694529488612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059522000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a8694529488612e\"\n }\n },\n \"done\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059522000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b058694529489612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059522000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a8694529489612e\"\n }\n },\n \"training_iteration\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059523000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452944b01612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059523000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452944b01612e\"\n }\n },\n \"time_this_iter_s\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452944740565e701d000000612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452944740565e701d000000612e\"\n }\n },\n \"time_total_s\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452944740565e701d000000612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452944740565e701d000000612e\"\n }\n },\n \"time_since_restore\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452944740565e701d000000612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452944740565e701d000000612e\"\n }\n },\n \"iterations_since_restore\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059523000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452944b01612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059523000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452944b01612e\"\n }\n }\n },\n \"checkpoint_manager\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059518060000000000008c267261792e747261696e2e5f696e7465726e616c2e636865636b706f696e745f6d616e61676572948c125f436865636b706f696e744d616e616765729493942981947d94288c125f636865636b706f696e745f636f6e666967948c097261792e747261696e948c10436865636b706f696e74436f6e6669679493942981947d94288c0b6e756d5f746f5f6b656570944e8c1a636865636b706f696e745f73636f72655f617474726962757465944e8c16636865636b706f696e745f73636f72655f6f72646572948c036d6178948c14636865636b706f696e745f6672657175656e6379944b008c11636865636b706f696e745f61745f656e6494898c1a5f636865636b706f696e745f6b6565705f616c6c5f72616e6b73948c0a44455052454341544544948c1f5f636865636b706f696e745f75706c6f61645f66726f6d5f776f726b65727394681275628c135f636865636b706f696e745f726573756c7473945d948c1b7261792e747261696e2e5f696e7465726e616c2e73657373696f6e948c0f5f547261696e696e67526573756c749493942981947d94288c0a636865636b706f696e749468068c0a436865636b706f696e749493942981947d94288c0470617468948cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f35326438646661315f325f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d313032342c6465636f6465725f72745f6e756d5f6c617965723d342c64726f705f725f323032342d30362d32355f31362d30382d30342f636865636b706f696e745f303030303030948c0a66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c055f75756964948c0475756964948c04555549449493942981947d948c03696e74948a11c21e3bc53b8b4899854129ba1f17e9de00736275628c076d657472696373947d94288c046c6f7373948c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b007494624308000000e0ffffff7f94869452948c0974696d657374616d70944af7cf7a668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303030948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b018c08747269616c5f6964948c083532643864666131948c0464617465948c13323032342d30362d32355f31362d31312d3033948c1074696d655f746869735f697465725f73944740565e701d0000008c0c74696d655f746f74616c5f73944740565e701d0000008c03706964944ac60e01008c08686f73746e616d65948c06723369356e36948c076e6f64655f6970948c0c31302e3135392e382e313539948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b088c146465636f6465725f72745f6e756d5f6c61796572944b048c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b108c0a656e636f6465725f6666944d00048c0d6465636f6465725f72745f6666944d00048c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b088c0964726f705f7261746594473fef6d926b1bdb778c026c7294473f66d98c089be9368c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f7265944740565e701d0000008c18697465726174696f6e735f73696e63655f726573746f7265944b01757562618c195f6c61746573745f636865636b706f696e745f726573756c7494681975622e\"\n }\n}"
+ ],
+ [
+ "{\n \"stub\": false,\n \"trainable_name\": \"train_model\",\n \"trial_id\": \"6a62b85a\",\n \"storage\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059572040000000000008c1b7261792e747261696e2e5f696e7465726e616c2e73746f72616765948c0e53746f72616765436f6e746578749493942981947d94288c12637573746f6d5f66735f70726f766964656494898c1273746f726167655f6c6f63616c5f70617468948c2b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c7473948c136578706572696d656e745f6469725f6e616d65948c0f746573745f6578706572696d656e74948c0e747269616c5f6469725f6e616d65948c96747261696e5f6d6f64656c5f36613632623835615f335f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d313032342c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d313032342c6465636f6465725f72745f6e756d5f6c617965723d322c64726f705f5f323032342d30362d32355f31362d30392d3334948c1863757272656e745f636865636b706f696e745f696e646578944b018c0b73796e635f636f6e666967948c097261792e747261696e948c0a53796e63436f6e6669679493942981947d94288c0b73796e635f706572696f64944d2c018c0c73796e635f74696d656f7574944d08078c0e73796e635f61727469666163747394898c1c73796e635f6172746966616374735f6f6e5f636865636b706f696e7494888c0a75706c6f61645f646972948c0a44455052454341544544948c0673796e6365729468188c1273796e635f6f6e5f636865636b706f696e7494681875628c1273746f726167655f66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c0f73746f726167655f66735f70617468948c452f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f7465737494681968008c115f46696c6573797374656d53796e6365729493942981947d9428681b682268134d2c0168144d08078c116c6173745f73796e635f75705f74696d65944741d99eb3cd22d0378c136c6173745f73796e635f646f776e5f74696d659447fff00000000000008c0d5f73796e635f70726f63657373944e8c0c5f63757272656e745f636d649468008c125f75706c6f61645f746f5f66735f706174689493947d94288c0a6c6f63616c5f70617468948c3b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c74732f746573745f6578706572696d656e74948c0266739468228c0766735f70617468948c552f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e74948c076578636c756465945d948c0e2a2f636865636b706f696e745f2a9461758694756275622e\"\n },\n \"config\": {\n \"encoder_num_layer\": 1,\n \"decoder_rt_num_layer\": 2,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 16,\n \"encoder_ff\": 512,\n \"decoder_rt_ff\": 1024,\n \"decoder_int_ff\": 1024,\n \"n_head\": 4,\n \"drop_rate\": 0.06860896590384413,\n \"lr\": 0.00016561766448063182,\n \"batch_size\": 2048\n },\n \"_Trial__unresolved_config\": {\n \"encoder_num_layer\": 1,\n \"decoder_rt_num_layer\": 2,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 16,\n \"encoder_ff\": 512,\n \"decoder_rt_ff\": 1024,\n \"decoder_int_ff\": 1024,\n \"n_head\": 4,\n \"drop_rate\": 0.06860896590384413,\n \"lr\": 0.00016561766448063182,\n \"batch_size\": 2048\n },\n \"evaluated_params\": {\n \"batch_size\": 2048,\n \"decoder_int_ff\": 1024,\n \"decoder_int_num_layer\": 1,\n \"decoder_rt_ff\": 1024,\n \"decoder_rt_num_layer\": 2,\n \"drop_rate\": 0.06860896590384413,\n \"embedding_dim\": 16,\n \"encoder_ff\": 512,\n \"encoder_num_layer\": 1,\n \"lr\": 0.00016561766448063182,\n \"n_head\": 4\n },\n \"experiment_tag\": \"3_batch_size=2048,decoder_int_ff=1024,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop_rate=0.0686,embedding_dim=16,encoder_ff=512,encoder_num_layer=1,lr=0.0002,n_head=4\",\n \"stopping_criterion\": {},\n \"_setup_default_resource\": true,\n \"_default_placement_group_factory\": \"80054e2e\",\n \"placement_group_factory\": \"800595ba000000000000008c237261792e74756e652e657865637574696f6e2e706c6163656d656e745f67726f757073948c15506c6163656d656e7447726f7570466163746f72799493942981947d94288c085f62756e646c6573945d947d94288c03435055944740540000000000008c034750559447401000000000000075618c155f686561645f62756e646c655f69735f656d70747994898c095f7374726174656779948c045041434b948c055f6172677394298c075f6b7761726773947d9475622e\",\n \"log_to_file\": [\n null,\n null\n ],\n \"max_failures\": 0,\n \"_default_result_or_future\": null,\n \"export_formats\": [],\n \"status\": \"PAUSED\",\n \"relative_logdir\": \"train_model_6a62b85a_3_batch_size=2048,decoder_int_ff=1024,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop__2024-06-25_16-09-34\",\n \"trial_name_creator\": null,\n \"trial_dirname_creator\": null,\n \"custom_trial_name\": null,\n \"custom_dirname\": null,\n \"restore_path\": null,\n \"_restore_checkpoint_result\": null,\n \"_state_json\": null,\n \"results\": \"80054e2e\",\n \"extra_arg\": \"80054e2e\",\n \"_resources\": \"80054e2e\"\n}",
+ "{\n \"start_time\": 1719324663.967162,\n \"num_failures\": 0,\n \"num_failures_after_restore\": 0,\n \"error_filename\": null,\n \"pickled_error_filename\": null,\n \"last_result\": {\n \"loss\": 310.75900604968933,\n \"timestamp\": 1719327245,\n \"checkpoint_dir_name\": \"checkpoint_000001\",\n \"should_checkpoint\": true,\n \"done\": false,\n \"training_iteration\": 2,\n \"trial_id\": \"6a62b85a\",\n \"date\": \"2024-06-25_16-54-05\",\n \"time_this_iter_s\": 63.18879699707031,\n \"time_total_s\": 127.51110315322876,\n \"pid\": 130020,\n \"hostname\": \"r3i5n6\",\n \"node_ip\": \"10.159.8.159\",\n \"config\": {\n \"encoder_num_layer\": 1,\n \"decoder_rt_num_layer\": 2,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 16,\n \"encoder_ff\": 512,\n \"decoder_rt_ff\": 1024,\n \"decoder_int_ff\": 1024,\n \"n_head\": 4,\n \"drop_rate\": 0.06860896590384413,\n \"lr\": 0.00016561766448063182,\n \"batch_size\": 2048\n },\n \"time_since_restore\": 63.18879699707031,\n \"iterations_since_restore\": 1,\n \"experiment_tag\": \"3_batch_size=2048,decoder_int_ff=1024,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop_rate=0.0686,embedding_dim=16,encoder_ff=512,encoder_num_layer=1,lr=0.0002,n_head=4\"\n },\n \"last_result_time\": 1719327245.5672662,\n \"metric_analysis\": {\n \"loss\": {\n \"max\": 510.42659056656004,\n \"min\": 310.75900604968933,\n \"avg\": 410.59279830812466,\n \"last\": 310.75900604968933,\n \"last-5-avg\": 410.59279830812466,\n \"last-10-avg\": 410.59279830812466\n },\n \"should_checkpoint\": {\n \"max\": true,\n \"min\": true,\n \"avg\": 1.0,\n \"last\": true,\n \"last-5-avg\": 1.0,\n \"last-10-avg\": 1.0\n },\n \"done\": {\n \"max\": false,\n \"min\": false,\n \"avg\": 0.0,\n \"last\": false,\n \"last-5-avg\": 0.0,\n \"last-10-avg\": 0.0\n },\n \"training_iteration\": {\n \"max\": 2,\n \"min\": 1,\n \"avg\": 1.5,\n \"last\": 2,\n \"last-5-avg\": 1.5,\n \"last-10-avg\": 1.5\n },\n \"time_this_iter_s\": {\n \"max\": 64.32230615615845,\n \"min\": 63.18879699707031,\n \"avg\": 63.75555157661438,\n \"last\": 63.18879699707031,\n \"last-5-avg\": 63.75555157661438,\n \"last-10-avg\": 63.75555157661438\n },\n \"time_total_s\": {\n \"max\": 127.51110315322876,\n \"min\": 64.32230615615845,\n \"avg\": 95.9167046546936,\n \"last\": 127.51110315322876,\n \"last-5-avg\": 95.9167046546936,\n \"last-10-avg\": 95.9167046546936\n },\n \"time_since_restore\": {\n \"max\": 64.32230615615845,\n \"min\": 63.18879699707031,\n \"avg\": 63.75555157661438,\n \"last\": 63.18879699707031,\n \"last-5-avg\": 63.75555157661438,\n \"last-10-avg\": 63.75555157661438\n },\n \"iterations_since_restore\": {\n \"max\": 1,\n \"min\": 1,\n \"avg\": 1.0,\n \"last\": 1,\n \"last-5-avg\": 1.0,\n \"last-10-avg\": 1.0\n }\n },\n \"_n_steps\": [\n 5,\n 10\n ],\n \"metric_n_steps\": {\n \"loss\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005959d000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294288c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b0074946243088542a150d3e67f4094869452946807680d43081c0e87e3246c73409486945294652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005959d000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294288c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b0074946243088542a150d3e67f4094869452946807680d43081c0e87e3246c73409486945294652e\"\n }\n },\n \"should_checkpoint\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059524000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294288888652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059524000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294288888652e\"\n }\n },\n \"done\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059524000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294288989652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059524000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294288989652e\"\n }\n },\n \"training_iteration\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059526000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294284b014b02652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059526000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294284b014b02652e\"\n }\n },\n \"time_this_iter_s\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059534000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452942847405014a0aa00000047404f982a80000000652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059534000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452942847405014a0aa00000047404f982a80000000652e\"\n }\n },\n \"time_total_s\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059534000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452942847405014a0aa00000047405fe0b5ea000000652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059534000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452942847405014a0aa00000047405fe0b5ea000000652e\"\n }\n },\n \"time_since_restore\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059534000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452942847405014a0aa00000047404f982a80000000652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059534000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452942847405014a0aa00000047404f982a80000000652e\"\n }\n },\n \"iterations_since_restore\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059526000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294284b014b01652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059526000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294284b014b01652e\"\n }\n }\n },\n \"checkpoint_manager\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"800595a5090000000000008c267261792e747261696e2e5f696e7465726e616c2e636865636b706f696e745f6d616e61676572948c125f436865636b706f696e744d616e616765729493942981947d94288c125f636865636b706f696e745f636f6e666967948c097261792e747261696e948c10436865636b706f696e74436f6e6669679493942981947d94288c0b6e756d5f746f5f6b656570944e8c1a636865636b706f696e745f73636f72655f617474726962757465944e8c16636865636b706f696e745f73636f72655f6f72646572948c036d6178948c14636865636b706f696e745f6672657175656e6379944b008c11636865636b706f696e745f61745f656e6494898c1a5f636865636b706f696e745f6b6565705f616c6c5f72616e6b73948c0a44455052454341544544948c1f5f636865636b706f696e745f75706c6f61645f66726f6d5f776f726b65727394681275628c135f636865636b706f696e745f726573756c7473945d94288c1b7261792e747261696e2e5f696e7465726e616c2e73657373696f6e948c0f5f547261696e696e67526573756c749493942981947d94288c0a636865636b706f696e749468068c0a436865636b706f696e749493942981947d94288c0470617468948cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f36613632623835615f335f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d313032342c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d313032342c6465636f6465725f72745f6e756d5f6c617965723d322c64726f705f5f323032342d30362d32355f31362d30392d33342f636865636b706f696e745f303030303030948c0a66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c055f75756964948c0475756964948c04555549449493942981947d948c03696e74948a111770d9a5a2b89096d44582c2ceb22a9d00736275628c076d657472696373947d94288c046c6f7373948c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b0074946243088542a150d3e67f4094869452948c0974696d657374616d70944a38d07a668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303030948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b018c08747269616c5f6964948c083661363262383561948c0464617465948c13323032342d30362d32355f31362d31322d3038948c1074696d655f746869735f697465725f739447405014a0aa0000008c0c74696d655f746f74616c5f739447405014a0aa0000008c03706964944ac60e01008c08686f73746e616d65948c06723369356e36948c076e6f64655f6970948c0c31302e3135392e382e313539948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b018c146465636f6465725f72745f6e756d5f6c61796572944b028c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b108c0a656e636f6465725f6666944d00028c0d6465636f6465725f72745f6666944d00048c0e6465636f6465725f696e745f6666944d00048c066e5f68656164944b048c0964726f705f7261746594473fb1905b70c4f6b88c026c7294473f25b534e7b91e408c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f72659447405014a0aa0000008c18697465726174696f6e735f73696e63655f726573746f7265944b0175756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f36613632623835615f335f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d313032342c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d313032342c6465636f6465725f72745f6e756d5f6c617965723d322c64726f705f5f323032342d30362d32355f31362d30392d33342f636865636b706f696e745f30303030303194682268257d946827897385945294682a682d2981947d9468308a103744df4a18a121ad344e9c027800e32b7362756268317d94288c046c6f7373946836683c43081c0e87e3246c734094869452948c0974696d657374616d70944a0dda7a668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303031948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b028c08747269616c5f6964948c083661363262383561948c0464617465948c13323032342d30362d32355f31362d35342d3035948c1074696d655f746869735f697465725f739447404f982a800000008c0c74696d655f746f74616c5f739447405fe0b5ea0000008c03706964944ae4fb01008c08686f73746e616d65948c06723369356e36948c076e6f64655f6970948c0c31302e3135392e382e313539948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b018c146465636f6465725f72745f6e756d5f6c61796572944b028c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b108c0a656e636f6465725f6666944d00028c0d6465636f6465725f72745f6666944d00048c0e6465636f6465725f696e745f6666944d00048c066e5f68656164944b048c0964726f705f7261746594473fb1905b70c4f6b88c026c7294473f25b534e7b91e408c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f72659447404f982a800000008c18697465726174696f6e735f73696e63655f726573746f7265944b01757562658c195f6c61746573745f636865636b706f696e745f726573756c7494686275622e\"\n }\n}"
+ ],
+ [
+ "{\n \"stub\": false,\n \"trainable_name\": \"train_model\",\n \"trial_id\": \"975aa6a0\",\n \"storage\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059572040000000000008c1b7261792e747261696e2e5f696e7465726e616c2e73746f72616765948c0e53746f72616765436f6e746578749493942981947d94288c12637573746f6d5f66735f70726f766964656494898c1273746f726167655f6c6f63616c5f70617468948c2b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c7473948c136578706572696d656e745f6469725f6e616d65948c0f746573745f6578706572696d656e74948c0e747269616c5f6469725f6e616d65948c96747261696e5f6d6f64656c5f39373561613661305f345f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d322c6465636f6465725f72745f66663d313032342c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f725f323032342d30362d32355f31362d31312d3033948c1863757272656e745f636865636b706f696e745f696e646578944b008c0b73796e635f636f6e666967948c097261792e747261696e948c0a53796e63436f6e6669679493942981947d94288c0b73796e635f706572696f64944d2c018c0c73796e635f74696d656f7574944d08078c0e73796e635f61727469666163747394898c1c73796e635f6172746966616374735f6f6e5f636865636b706f696e7494888c0a75706c6f61645f646972948c0a44455052454341544544948c0673796e6365729468188c1273796e635f6f6e5f636865636b706f696e7494681875628c1273746f726167655f66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c0f73746f726167655f66735f70617468948c452f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f7465737494681968008c115f46696c6573797374656d53796e6365729493942981947d9428681b682268134d2c0168144d08078c116c6173745f73796e635f75705f74696d65944741d99eb3cd22d0378c136c6173745f73796e635f646f776e5f74696d659447fff00000000000008c0d5f73796e635f70726f63657373944e8c0c5f63757272656e745f636d649468008c125f75706c6f61645f746f5f66735f706174689493947d94288c0a6c6f63616c5f70617468948c3b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c74732f746573745f6578706572696d656e74948c0266739468228c0766735f70617468948c552f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e74948c076578636c756465945d948c0e2a2f636865636b706f696e745f2a9461758694756275622e\"\n },\n \"config\": {\n \"encoder_num_layer\": 1,\n \"decoder_rt_num_layer\": 1,\n \"decoder_int_num_layer\": 2,\n \"embedding_dim\": 64,\n \"encoder_ff\": 1024,\n \"decoder_rt_ff\": 1024,\n \"decoder_int_ff\": 512,\n \"n_head\": 16,\n \"drop_rate\": 0.8156275326399022,\n \"lr\": 0.00010404955423531405,\n \"batch_size\": 2048\n },\n \"_Trial__unresolved_config\": {\n \"encoder_num_layer\": 1,\n \"decoder_rt_num_layer\": 1,\n \"decoder_int_num_layer\": 2,\n \"embedding_dim\": 64,\n \"encoder_ff\": 1024,\n \"decoder_rt_ff\": 1024,\n \"decoder_int_ff\": 512,\n \"n_head\": 16,\n \"drop_rate\": 0.8156275326399022,\n \"lr\": 0.00010404955423531405,\n \"batch_size\": 2048\n },\n \"evaluated_params\": {\n \"batch_size\": 2048,\n \"decoder_int_ff\": 512,\n \"decoder_int_num_layer\": 2,\n \"decoder_rt_ff\": 1024,\n \"decoder_rt_num_layer\": 1,\n \"drop_rate\": 0.8156275326399022,\n \"embedding_dim\": 64,\n \"encoder_ff\": 1024,\n \"encoder_num_layer\": 1,\n \"lr\": 0.00010404955423531405,\n \"n_head\": 16\n },\n \"experiment_tag\": \"4_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=2,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop_rate=0.8156,embedding_dim=64,encoder_ff=1024,encoder_num_layer=1,lr=0.0001,n_head=16\",\n \"stopping_criterion\": {},\n \"_setup_default_resource\": true,\n \"_default_placement_group_factory\": \"80054e2e\",\n \"placement_group_factory\": \"800595ba000000000000008c237261792e74756e652e657865637574696f6e2e706c6163656d656e745f67726f757073948c15506c6163656d656e7447726f7570466163746f72799493942981947d94288c085f62756e646c6573945d947d94288c03435055944740540000000000008c034750559447401000000000000075618c155f686561645f62756e646c655f69735f656d70747994898c095f7374726174656779948c045041434b948c055f6172677394298c075f6b7761726773947d9475622e\",\n \"log_to_file\": [\n null,\n null\n ],\n \"max_failures\": 0,\n \"_default_result_or_future\": null,\n \"export_formats\": [],\n \"status\": \"PAUSED\",\n \"relative_logdir\": \"train_model_975aa6a0_4_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=2,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop_r_2024-06-25_16-11-03\",\n \"trial_name_creator\": null,\n \"trial_dirname_creator\": null,\n \"custom_trial_name\": null,\n \"custom_dirname\": null,\n \"restore_path\": null,\n \"_restore_checkpoint_result\": null,\n \"_state_json\": null,\n \"results\": \"80054e2e\",\n \"extra_arg\": \"80054e2e\",\n \"_resources\": \"80054e2e\"\n}",
+ "{\n \"start_time\": 1719324728.438826,\n \"num_failures\": 0,\n \"num_failures_after_restore\": 0,\n \"error_filename\": null,\n \"pickled_error_filename\": null,\n \"last_result\": {\n \"loss\": 2002.7163720318651,\n \"timestamp\": 1719324792,\n \"checkpoint_dir_name\": \"checkpoint_000000\",\n \"should_checkpoint\": true,\n \"done\": false,\n \"training_iteration\": 1,\n \"trial_id\": \"975aa6a0\",\n \"date\": \"2024-06-25_16-13-12\",\n \"time_this_iter_s\": 64.14145565032959,\n \"time_total_s\": 64.14145565032959,\n \"pid\": 69318,\n \"hostname\": \"r3i5n6\",\n \"node_ip\": \"10.159.8.159\",\n \"config\": {\n \"encoder_num_layer\": 1,\n \"decoder_rt_num_layer\": 1,\n \"decoder_int_num_layer\": 2,\n \"embedding_dim\": 64,\n \"encoder_ff\": 1024,\n \"decoder_rt_ff\": 1024,\n \"decoder_int_ff\": 512,\n \"n_head\": 16,\n \"drop_rate\": 0.8156275326399022,\n \"lr\": 0.00010404955423531405,\n \"batch_size\": 2048\n },\n \"time_since_restore\": 64.14145565032959,\n \"iterations_since_restore\": 1,\n \"experiment_tag\": \"4_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=2,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop_rate=0.8156,embedding_dim=64,encoder_ff=1024,encoder_num_layer=1,lr=0.0001,n_head=16\"\n },\n \"last_result_time\": 1719324792.5855737,\n \"metric_analysis\": {\n \"loss\": {\n \"max\": 2002.7163720318651,\n \"min\": 2002.7163720318651,\n \"avg\": 2002.7163720318651,\n \"last\": 2002.7163720318651,\n \"last-5-avg\": 2002.7163720318651,\n \"last-10-avg\": 2002.7163720318651\n },\n \"should_checkpoint\": {\n \"max\": true,\n \"min\": true,\n \"avg\": true,\n \"last\": true,\n \"last-5-avg\": true,\n \"last-10-avg\": true\n },\n \"done\": {\n \"max\": false,\n \"min\": false,\n \"avg\": false,\n \"last\": false,\n \"last-5-avg\": false,\n \"last-10-avg\": false\n },\n \"training_iteration\": {\n \"max\": 1,\n \"min\": 1,\n \"avg\": 1,\n \"last\": 1,\n \"last-5-avg\": 1,\n \"last-10-avg\": 1\n },\n \"time_this_iter_s\": {\n \"max\": 64.14145565032959,\n \"min\": 64.14145565032959,\n \"avg\": 64.14145565032959,\n \"last\": 64.14145565032959,\n \"last-5-avg\": 64.14145565032959,\n \"last-10-avg\": 64.14145565032959\n },\n \"time_total_s\": {\n \"max\": 64.14145565032959,\n \"min\": 64.14145565032959,\n \"avg\": 64.14145565032959,\n \"last\": 64.14145565032959,\n \"last-5-avg\": 64.14145565032959,\n \"last-10-avg\": 64.14145565032959\n },\n \"time_since_restore\": {\n \"max\": 64.14145565032959,\n \"min\": 64.14145565032959,\n \"avg\": 64.14145565032959,\n \"last\": 64.14145565032959,\n \"last-5-avg\": 64.14145565032959,\n \"last-10-avg\": 64.14145565032959\n },\n \"iterations_since_restore\": {\n \"max\": 1,\n \"min\": 1,\n \"avg\": 1,\n \"last\": 1,\n \"last-5-avg\": 1,\n \"last-10-avg\": 1\n }\n },\n \"_n_steps\": [\n 5,\n 10\n ],\n \"metric_n_steps\": {\n \"loss\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059589000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452948c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b0074946243088542a190dd4a9f409486945294612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059589000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452948c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b0074946243088542a190dd4a9f409486945294612e\"\n }\n },\n \"should_checkpoint\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059522000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b058694529488612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059522000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a8694529488612e\"\n }\n },\n \"done\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059522000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b058694529489612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059522000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a8694529489612e\"\n }\n },\n \"training_iteration\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059523000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452944b01612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059523000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452944b01612e\"\n }\n },\n \"time_this_iter_s\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294474050090d9c000000612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294474050090d9c000000612e\"\n }\n },\n \"time_total_s\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294474050090d9c000000612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294474050090d9c000000612e\"\n }\n },\n \"time_since_restore\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294474050090d9c000000612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294474050090d9c000000612e\"\n }\n },\n \"iterations_since_restore\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059523000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452944b01612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059523000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452944b01612e\"\n }\n }\n },\n \"checkpoint_manager\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059517060000000000008c267261792e747261696e2e5f696e7465726e616c2e636865636b706f696e745f6d616e61676572948c125f436865636b706f696e744d616e616765729493942981947d94288c125f636865636b706f696e745f636f6e666967948c097261792e747261696e948c10436865636b706f696e74436f6e6669679493942981947d94288c0b6e756d5f746f5f6b656570944e8c1a636865636b706f696e745f73636f72655f617474726962757465944e8c16636865636b706f696e745f73636f72655f6f72646572948c036d6178948c14636865636b706f696e745f6672657175656e6379944b008c11636865636b706f696e745f61745f656e6494898c1a5f636865636b706f696e745f6b6565705f616c6c5f72616e6b73948c0a44455052454341544544948c1f5f636865636b706f696e745f75706c6f61645f66726f6d5f776f726b65727394681275628c135f636865636b706f696e745f726573756c7473945d948c1b7261792e747261696e2e5f696e7465726e616c2e73657373696f6e948c0f5f547261696e696e67526573756c749493942981947d94288c0a636865636b706f696e749468068c0a436865636b706f696e749493942981947d94288c0470617468948cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f39373561613661305f345f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d322c6465636f6465725f72745f66663d313032342c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f725f323032342d30362d32355f31362d31312d30332f636865636b706f696e745f303030303030948c0a66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c055f75756964948c0475756964948c04555549449493942981947d948c03696e74948a104d5f716ab4722ebf3b45fadaa52f804f736275628c076d657472696373947d94288c046c6f7373948c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b0074946243088542a190dd4a9f4094869452948c0974696d657374616d70944a78d07a668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303030948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b018c08747269616c5f6964948c083937356161366130948c0464617465948c13323032342d30362d32355f31362d31332d3132948c1074696d655f746869735f697465725f7394474050090d9c0000008c0c74696d655f746f74616c5f7394474050090d9c0000008c03706964944ac60e01008c08686f73746e616d65948c06723369356e36948c076e6f64655f6970948c0c31302e3135392e382e313539948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b018c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b028c0d656d62656464696e675f64696d944b408c0a656e636f6465725f6666944d00048c0d6465636f6465725f72745f6666944d00048c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b108c0964726f705f7261746594473fea199ee94cfa498c026c7294473f1b46a5bafc8e808c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f726594474050090d9c0000008c18697465726174696f6e735f73696e63655f726573746f7265944b01757562618c195f6c61746573745f636865636b706f696e745f726573756c7494681975622e\"\n }\n}"
+ ],
+ [
+ "{\n \"stub\": false,\n \"trainable_name\": \"train_model\",\n \"trial_id\": \"af6d9c38\",\n \"storage\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059572040000000000008c1b7261792e747261696e2e5f696e7465726e616c2e73746f72616765948c0e53746f72616765436f6e746578749493942981947d94288c12637573746f6d5f66735f70726f766964656494898c1273746f726167655f6c6f63616c5f70617468948c2b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c7473948c136578706572696d656e745f6469725f6e616d65948c0f746573745f6578706572696d656e74948c0e747269616c5f6469725f6e616d65948c96747261696e5f6d6f64656c5f61663664396333385f355f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d313032342c6465636f6465725f696e745f6e756d5f6c617965723d382c6465636f6465725f72745f66663d313032342c6465636f6465725f72745f6e756d5f6c617965723d382c64726f705f5f323032342d30362d32355f31362d31322d3038948c1863757272656e745f636865636b706f696e745f696e646578944b018c0b73796e635f636f6e666967948c097261792e747261696e948c0a53796e63436f6e6669679493942981947d94288c0b73796e635f706572696f64944d2c018c0c73796e635f74696d656f7574944d08078c0e73796e635f61727469666163747394898c1c73796e635f6172746966616374735f6f6e5f636865636b706f696e7494888c0a75706c6f61645f646972948c0a44455052454341544544948c0673796e6365729468188c1273796e635f6f6e5f636865636b706f696e7494681875628c1273746f726167655f66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c0f73746f726167655f66735f70617468948c452f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f7465737494681968008c115f46696c6573797374656d53796e6365729493942981947d9428681b682268134d2c0168144d08078c116c6173745f73796e635f75705f74696d65944741d99eb3cd22d0378c136c6173745f73796e635f646f776e5f74696d659447fff00000000000008c0d5f73796e635f70726f63657373944e8c0c5f63757272656e745f636d649468008c125f75706c6f61645f746f5f66735f706174689493947d94288c0a6c6f63616c5f70617468948c3b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c74732f746573745f6578706572696d656e74948c0266739468228c0766735f70617468948c552f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e74948c076578636c756465945d948c0e2a2f636865636b706f696e745f2a9461758694756275622e\"\n },\n \"config\": {\n \"encoder_num_layer\": 8,\n \"decoder_rt_num_layer\": 8,\n \"decoder_int_num_layer\": 8,\n \"embedding_dim\": 16,\n \"encoder_ff\": 512,\n \"decoder_rt_ff\": 1024,\n \"decoder_int_ff\": 1024,\n \"n_head\": 16,\n \"drop_rate\": 0.5912116123671598,\n \"lr\": 0.009202977404873457,\n \"batch_size\": 2048\n },\n \"_Trial__unresolved_config\": {\n \"encoder_num_layer\": 8,\n \"decoder_rt_num_layer\": 8,\n \"decoder_int_num_layer\": 8,\n \"embedding_dim\": 16,\n \"encoder_ff\": 512,\n \"decoder_rt_ff\": 1024,\n \"decoder_int_ff\": 1024,\n \"n_head\": 16,\n \"drop_rate\": 0.5912116123671598,\n \"lr\": 0.009202977404873457,\n \"batch_size\": 2048\n },\n \"evaluated_params\": {\n \"batch_size\": 2048,\n \"decoder_int_ff\": 1024,\n \"decoder_int_num_layer\": 8,\n \"decoder_rt_ff\": 1024,\n \"decoder_rt_num_layer\": 8,\n \"drop_rate\": 0.5912116123671598,\n \"embedding_dim\": 16,\n \"encoder_ff\": 512,\n \"encoder_num_layer\": 8,\n \"lr\": 0.009202977404873457,\n \"n_head\": 16\n },\n \"experiment_tag\": \"5_batch_size=2048,decoder_int_ff=1024,decoder_int_num_layer=8,decoder_rt_ff=1024,decoder_rt_num_layer=8,drop_rate=0.5912,embedding_dim=16,encoder_ff=512,encoder_num_layer=8,lr=0.0092,n_head=16\",\n \"stopping_criterion\": {},\n \"_setup_default_resource\": true,\n \"_default_placement_group_factory\": \"80054e2e\",\n \"placement_group_factory\": \"800595ba000000000000008c237261792e74756e652e657865637574696f6e2e706c6163656d656e745f67726f757073948c15506c6163656d656e7447726f7570466163746f72799493942981947d94288c085f62756e646c6573945d947d94288c03435055944740540000000000008c034750559447401000000000000075618c155f686561645f62756e646c655f69735f656d70747994898c095f7374726174656779948c045041434b948c055f6172677394298c075f6b7761726773947d9475622e\",\n \"log_to_file\": [\n null,\n null\n ],\n \"max_failures\": 0,\n \"_default_result_or_future\": null,\n \"export_formats\": [],\n \"status\": \"PAUSED\",\n \"relative_logdir\": \"train_model_af6d9c38_5_batch_size=2048,decoder_int_ff=1024,decoder_int_num_layer=8,decoder_rt_ff=1024,decoder_rt_num_layer=8,drop__2024-06-25_16-12-08\",\n \"trial_name_creator\": null,\n \"trial_dirname_creator\": null,\n \"custom_trial_name\": null,\n \"custom_dirname\": null,\n \"restore_path\": null,\n \"_restore_checkpoint_result\": null,\n \"_state_json\": null,\n \"results\": \"80054e2e\",\n \"extra_arg\": \"80054e2e\",\n \"_resources\": \"80054e2e\"\n}",
+ "{\n \"start_time\": 1719324792.730206,\n \"num_failures\": 0,\n \"num_failures_after_restore\": 0,\n \"error_filename\": null,\n \"pickled_error_filename\": null,\n \"last_result\": {\n \"loss\": 1982.2825418307086,\n \"timestamp\": 1719327372,\n \"checkpoint_dir_name\": \"checkpoint_000001\",\n \"should_checkpoint\": true,\n \"done\": false,\n \"training_iteration\": 2,\n \"trial_id\": \"af6d9c38\",\n \"date\": \"2024-06-25_16-56-12\",\n \"time_this_iter_s\": 126.36091089248657,\n \"time_total_s\": 252.29485964775085,\n \"pid\": 130020,\n \"hostname\": \"r3i5n6\",\n \"node_ip\": \"10.159.8.159\",\n \"config\": {\n \"encoder_num_layer\": 8,\n \"decoder_rt_num_layer\": 8,\n \"decoder_int_num_layer\": 8,\n \"embedding_dim\": 16,\n \"encoder_ff\": 512,\n \"decoder_rt_ff\": 1024,\n \"decoder_int_ff\": 1024,\n \"n_head\": 16,\n \"drop_rate\": 0.5912116123671598,\n \"lr\": 0.009202977404873457,\n \"batch_size\": 2048\n },\n \"time_since_restore\": 126.36091089248657,\n \"iterations_since_restore\": 1,\n \"experiment_tag\": \"5_batch_size=2048,decoder_int_ff=1024,decoder_int_num_layer=8,decoder_rt_ff=1024,decoder_rt_num_layer=8,drop_rate=0.5912,embedding_dim=16,encoder_ff=512,encoder_num_layer=8,lr=0.0092,n_head=16\"\n },\n \"last_result_time\": 1719327372.0857606,\n \"metric_analysis\": {\n \"loss\": {\n \"max\": 1987.8132929764395,\n \"min\": 1982.2825418307086,\n \"avg\": 1985.047917403574,\n \"last\": 1982.2825418307086,\n \"last-5-avg\": 1985.047917403574,\n \"last-10-avg\": 1985.047917403574\n },\n \"should_checkpoint\": {\n \"max\": true,\n \"min\": true,\n \"avg\": 1.0,\n \"last\": true,\n \"last-5-avg\": 1.0,\n \"last-10-avg\": 1.0\n },\n \"done\": {\n \"max\": false,\n \"min\": false,\n \"avg\": 0.0,\n \"last\": false,\n \"last-5-avg\": 0.0,\n \"last-10-avg\": 0.0\n },\n \"training_iteration\": {\n \"max\": 2,\n \"min\": 1,\n \"avg\": 1.5,\n \"last\": 2,\n \"last-5-avg\": 1.5,\n \"last-10-avg\": 1.5\n },\n \"time_this_iter_s\": {\n \"max\": 126.36091089248657,\n \"min\": 125.93394875526428,\n \"avg\": 126.14742982387543,\n \"last\": 126.36091089248657,\n \"last-5-avg\": 126.14742982387543,\n \"last-10-avg\": 126.14742982387543\n },\n \"time_total_s\": {\n \"max\": 252.29485964775085,\n \"min\": 125.93394875526428,\n \"avg\": 189.11440420150757,\n \"last\": 252.29485964775085,\n \"last-5-avg\": 189.11440420150757,\n \"last-10-avg\": 189.11440420150757\n },\n \"time_since_restore\": {\n \"max\": 126.36091089248657,\n \"min\": 125.93394875526428,\n \"avg\": 126.14742982387543,\n \"last\": 126.36091089248657,\n \"last-5-avg\": 126.14742982387543,\n \"last-10-avg\": 126.14742982387543\n },\n \"iterations_since_restore\": {\n \"max\": 1,\n \"min\": 1,\n \"avg\": 1.0,\n \"last\": 1,\n \"last-5-avg\": 1.0,\n \"last-10-avg\": 1.0\n }\n },\n \"_n_steps\": [\n 5,\n 10\n ],\n \"metric_n_steps\": {\n \"loss\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005959d000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294288c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b0074946243087fbfdfcf400f9f4094869452946807680d4308954aa55221f99e409486945294652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005959d000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294288c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b0074946243087fbfdfcf400f9f4094869452946807680d4308954aa55221f99e409486945294652e\"\n }\n },\n \"should_checkpoint\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059524000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294288888652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059524000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294288888652e\"\n }\n },\n \"done\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059524000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294288989652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059524000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294288989652e\"\n }\n },\n \"training_iteration\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059526000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294284b014b02652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059526000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294284b014b02652e\"\n }\n },\n \"time_this_iter_s\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059534000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452942847405f7bc5d100000047405f97192a000000652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059534000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452942847405f7bc5d100000047405f97192a000000652e\"\n }\n },\n \"time_total_s\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059534000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452942847405f7bc5d100000047406f896f7d800000652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059534000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452942847405f7bc5d100000047406f896f7d800000652e\"\n }\n },\n \"time_since_restore\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059534000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452942847405f7bc5d100000047405f97192a000000652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059534000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452942847405f7bc5d100000047405f97192a000000652e\"\n }\n },\n \"iterations_since_restore\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059526000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294284b014b01652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059526000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294284b014b01652e\"\n }\n }\n },\n \"checkpoint_manager\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"800595a5090000000000008c267261792e747261696e2e5f696e7465726e616c2e636865636b706f696e745f6d616e61676572948c125f436865636b706f696e744d616e616765729493942981947d94288c125f636865636b706f696e745f636f6e666967948c097261792e747261696e948c10436865636b706f696e74436f6e6669679493942981947d94288c0b6e756d5f746f5f6b656570944e8c1a636865636b706f696e745f73636f72655f617474726962757465944e8c16636865636b706f696e745f73636f72655f6f72646572948c036d6178948c14636865636b706f696e745f6672657175656e6379944b008c11636865636b706f696e745f61745f656e6494898c1a5f636865636b706f696e745f6b6565705f616c6c5f72616e6b73948c0a44455052454341544544948c1f5f636865636b706f696e745f75706c6f61645f66726f6d5f776f726b65727394681275628c135f636865636b706f696e745f726573756c7473945d94288c1b7261792e747261696e2e5f696e7465726e616c2e73657373696f6e948c0f5f547261696e696e67526573756c749493942981947d94288c0a636865636b706f696e749468068c0a436865636b706f696e749493942981947d94288c0470617468948cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f61663664396333385f355f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d313032342c6465636f6465725f696e745f6e756d5f6c617965723d382c6465636f6465725f72745f66663d313032342c6465636f6465725f72745f6e756d5f6c617965723d382c64726f705f5f323032342d30362d32355f31362d31322d30382f636865636b706f696e745f303030303030948c0a66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c055f75756964948c0475756964948c04555549449493942981947d948c03696e74948a115d4407c8c02494bf934f054d85e6a9e100736275628c076d657472696373947d94288c046c6f7373948c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b0074946243087fbfdfcf400f9f4094869452948c0974696d657374616d70944af6d07a668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303030948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b018c08747269616c5f6964948c086166366439633338948c0464617465948c13323032342d30362d32355f31362d31352d3138948c1074696d655f746869735f697465725f739447405f7bc5d10000008c0c74696d655f746f74616c5f739447405f7bc5d10000008c03706964944ac60e01008c08686f73746e616d65948c06723369356e36948c076e6f64655f6970948c0c31302e3135392e382e313539948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b088c146465636f6465725f72745f6e756d5f6c61796572944b088c156465636f6465725f696e745f6e756d5f6c61796572944b088c0d656d62656464696e675f64696d944b108c0a656e636f6465725f6666944d00028c0d6465636f6465725f72745f6666944d00048c0e6465636f6465725f696e745f6666944d00048c066e5f68656164944b108c0964726f705f7261746594473fe2eb349d843c768c026c7294473f82d902b7d68b998c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f72659447405f7bc5d10000008c18697465726174696f6e735f73696e63655f726573746f7265944b0175756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f61663664396333385f355f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d313032342c6465636f6465725f696e745f6e756d5f6c617965723d382c6465636f6465725f72745f66663d313032342c6465636f6465725f72745f6e756d5f6c617965723d382c64726f705f5f323032342d30362d32355f31362d31322d30382f636865636b706f696e745f30303030303194682268257d946827897385945294682a682d2981947d9468308a10d273959b2f684f801f41e4e3e4159f7b7362756268317d94288c046c6f7373946836683c4308954aa55221f99e4094869452948c0974696d657374616d70944a8cda7a668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303031948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b028c08747269616c5f6964948c086166366439633338948c0464617465948c13323032342d30362d32355f31362d35362d3132948c1074696d655f746869735f697465725f739447405f97192a0000008c0c74696d655f746f74616c5f739447406f896f7d8000008c03706964944ae4fb01008c08686f73746e616d65948c06723369356e36948c076e6f64655f6970948c0c31302e3135392e382e313539948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b088c146465636f6465725f72745f6e756d5f6c61796572944b088c156465636f6465725f696e745f6e756d5f6c61796572944b088c0d656d62656464696e675f64696d944b108c0a656e636f6465725f6666944d00028c0d6465636f6465725f72745f6666944d00048c0e6465636f6465725f696e745f6666944d00048c066e5f68656164944b108c0964726f705f7261746594473fe2eb349d843c768c026c7294473f82d902b7d68b998c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f72659447405f97192a0000008c18697465726174696f6e735f73696e63655f726573746f7265944b01757562658c195f6c61746573745f636865636b706f696e745f726573756c7494686275622e\"\n }\n}"
+ ],
+ [
+ "{\n \"stub\": false,\n \"trainable_name\": \"train_model\",\n \"trial_id\": \"b577349d\",\n \"storage\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059572040000000000008c1b7261792e747261696e2e5f696e7465726e616c2e73746f72616765948c0e53746f72616765436f6e746578749493942981947d94288c12637573746f6d5f66735f70726f766964656494898c1273746f726167655f6c6f63616c5f70617468948c2b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c7473948c136578706572696d656e745f6469725f6e616d65948c0f746573745f6578706572696d656e74948c0e747269616c5f6469725f6e616d65948c96747261696e5f6d6f64656c5f62353737333439645f365f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d322c6465636f6465725f72745f66663d313032342c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f725f323032342d30362d32355f31362d31332d3132948c1863757272656e745f636865636b706f696e745f696e646578944b018c0b73796e635f636f6e666967948c097261792e747261696e948c0a53796e63436f6e6669679493942981947d94288c0b73796e635f706572696f64944d2c018c0c73796e635f74696d656f7574944d08078c0e73796e635f61727469666163747394898c1c73796e635f6172746966616374735f6f6e5f636865636b706f696e7494888c0a75706c6f61645f646972948c0a44455052454341544544948c0673796e6365729468188c1273796e635f6f6e5f636865636b706f696e7494681875628c1273746f726167655f66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c0f73746f726167655f66735f70617468948c452f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f7465737494681968008c115f46696c6573797374656d53796e6365729493942981947d9428681b682268134d2c0168144d08078c116c6173745f73796e635f75705f74696d65944741d99eb418846a298c136c6173745f73796e635f646f776e5f74696d659447fff00000000000008c0d5f73796e635f70726f63657373944e8c0c5f63757272656e745f636d649468008c125f75706c6f61645f746f5f66735f706174689493947d94288c0a6c6f63616c5f70617468948c3b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c74732f746573745f6578706572696d656e74948c0266739468228c0766735f70617468948c552f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e74948c076578636c756465945d948c0e2a2f636865636b706f696e745f2a9461758694756275622e\"\n },\n \"config\": {\n \"encoder_num_layer\": 8,\n \"decoder_rt_num_layer\": 1,\n \"decoder_int_num_layer\": 2,\n \"embedding_dim\": 64,\n \"encoder_ff\": 2048,\n \"decoder_rt_ff\": 1024,\n \"decoder_int_ff\": 512,\n \"n_head\": 2,\n \"drop_rate\": 0.2945022566291742,\n \"lr\": 0.000496318687335349,\n \"batch_size\": 2048\n },\n \"_Trial__unresolved_config\": {\n \"encoder_num_layer\": 8,\n \"decoder_rt_num_layer\": 1,\n \"decoder_int_num_layer\": 2,\n \"embedding_dim\": 64,\n \"encoder_ff\": 2048,\n \"decoder_rt_ff\": 1024,\n \"decoder_int_ff\": 512,\n \"n_head\": 2,\n \"drop_rate\": 0.2945022566291742,\n \"lr\": 0.000496318687335349,\n \"batch_size\": 2048\n },\n \"evaluated_params\": {\n \"batch_size\": 2048,\n \"decoder_int_ff\": 512,\n \"decoder_int_num_layer\": 2,\n \"decoder_rt_ff\": 1024,\n \"decoder_rt_num_layer\": 1,\n \"drop_rate\": 0.2945022566291742,\n \"embedding_dim\": 64,\n \"encoder_ff\": 2048,\n \"encoder_num_layer\": 8,\n \"lr\": 0.000496318687335349,\n \"n_head\": 2\n },\n \"experiment_tag\": \"6_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=2,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop_rate=0.2945,embedding_dim=64,encoder_ff=2048,encoder_num_layer=8,lr=0.0005,n_head=2\",\n \"stopping_criterion\": {},\n \"_setup_default_resource\": true,\n \"_default_placement_group_factory\": \"80054e2e\",\n \"placement_group_factory\": \"800595ba000000000000008c237261792e74756e652e657865637574696f6e2e706c6163656d656e745f67726f757073948c15506c6163656d656e7447726f7570466163746f72799493942981947d94288c085f62756e646c6573945d947d94288c03435055944740540000000000008c034750559447401000000000000075618c155f686561645f62756e646c655f69735f656d70747994898c095f7374726174656779948c045041434b948c055f6172677394298c075f6b7761726773947d9475622e\",\n \"log_to_file\": [\n null,\n null\n ],\n \"max_failures\": 0,\n \"_default_result_or_future\": null,\n \"export_formats\": [],\n \"status\": \"PAUSED\",\n \"relative_logdir\": \"train_model_b577349d_6_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=2,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop_r_2024-06-25_16-13-12\",\n \"trial_name_creator\": null,\n \"trial_dirname_creator\": null,\n \"custom_trial_name\": null,\n \"custom_dirname\": null,\n \"restore_path\": null,\n \"_restore_checkpoint_result\": null,\n \"_state_json\": null,\n \"results\": \"80054e2e\",\n \"extra_arg\": \"80054e2e\",\n \"_resources\": \"80054e2e\"\n}",
+ "{\n \"start_time\": 1719324918.8126864,\n \"num_failures\": 0,\n \"num_failures_after_restore\": 0,\n \"error_filename\": null,\n \"pickled_error_filename\": null,\n \"last_result\": {\n \"loss\": 1958.5695377860482,\n \"timestamp\": 1719327468,\n \"checkpoint_dir_name\": \"checkpoint_000001\",\n \"should_checkpoint\": true,\n \"done\": false,\n \"training_iteration\": 2,\n \"trial_id\": \"b577349d\",\n \"date\": \"2024-06-25_16-57-48\",\n \"time_this_iter_s\": 96.6448974609375,\n \"time_total_s\": 192.2383737564087,\n \"pid\": 130020,\n \"hostname\": \"r3i5n6\",\n \"node_ip\": \"10.159.8.159\",\n \"config\": {\n \"encoder_num_layer\": 8,\n \"decoder_rt_num_layer\": 1,\n \"decoder_int_num_layer\": 2,\n \"embedding_dim\": 64,\n \"encoder_ff\": 2048,\n \"decoder_rt_ff\": 1024,\n \"decoder_int_ff\": 512,\n \"n_head\": 2,\n \"drop_rate\": 0.2945022566291742,\n \"lr\": 0.000496318687335349,\n \"batch_size\": 2048\n },\n \"time_since_restore\": 96.6448974609375,\n \"iterations_since_restore\": 1,\n \"experiment_tag\": \"6_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=2,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop_rate=0.2945,embedding_dim=64,encoder_ff=2048,encoder_num_layer=8,lr=0.0005,n_head=2\"\n },\n \"last_result_time\": 1719327468.8863273,\n \"metric_analysis\": {\n \"loss\": {\n \"max\": 1958.5695377860482,\n \"min\": 1958.5306944359006,\n \"avg\": 1958.5501161109744,\n \"last\": 1958.5695377860482,\n \"last-5-avg\": 1958.5501161109744,\n \"last-10-avg\": 1958.5501161109744\n },\n \"should_checkpoint\": {\n \"max\": true,\n \"min\": true,\n \"avg\": 1.0,\n \"last\": true,\n \"last-5-avg\": 1.0,\n \"last-10-avg\": 1.0\n },\n \"done\": {\n \"max\": false,\n \"min\": false,\n \"avg\": 0.0,\n \"last\": false,\n \"last-5-avg\": 0.0,\n \"last-10-avg\": 0.0\n },\n \"training_iteration\": {\n \"max\": 2,\n \"min\": 1,\n \"avg\": 1.5,\n \"last\": 2,\n \"last-5-avg\": 1.5,\n \"last-10-avg\": 1.5\n },\n \"time_this_iter_s\": {\n \"max\": 96.6448974609375,\n \"min\": 95.59347629547119,\n \"avg\": 96.11918687820435,\n \"last\": 96.6448974609375,\n \"last-5-avg\": 96.11918687820435,\n \"last-10-avg\": 96.11918687820435\n },\n \"time_total_s\": {\n \"max\": 192.2383737564087,\n \"min\": 95.59347629547119,\n \"avg\": 143.91592502593994,\n \"last\": 192.2383737564087,\n \"last-5-avg\": 143.91592502593994,\n \"last-10-avg\": 143.91592502593994\n },\n \"time_since_restore\": {\n \"max\": 96.6448974609375,\n \"min\": 95.59347629547119,\n \"avg\": 96.11918687820435,\n \"last\": 96.6448974609375,\n \"last-5-avg\": 96.11918687820435,\n \"last-10-avg\": 96.11918687820435\n },\n \"iterations_since_restore\": {\n \"max\": 1,\n \"min\": 1,\n \"avg\": 1.0,\n \"last\": 1,\n \"last-5-avg\": 1.0,\n \"last-10-avg\": 1.0\n }\n },\n \"_n_steps\": [\n 5,\n 10\n ],\n \"metric_n_steps\": {\n \"loss\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005959d000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294288c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b00749462430873b95c6e1f9a9e4094869452946807680d4308a7d3e934479a9e409486945294652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005959d000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294288c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b00749462430873b95c6e1f9a9e4094869452946807680d4308a7d3e934479a9e409486945294652e\"\n }\n },\n \"should_checkpoint\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059524000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294288888652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059524000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294288888652e\"\n }\n },\n \"done\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059524000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294288989652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059524000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294288989652e\"\n }\n },\n \"training_iteration\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059526000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294284b014b02652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059526000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294284b014b02652e\"\n }\n },\n \"time_this_iter_s\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059534000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b058694529428474057e5fb84000000474058294600000000652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059534000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a8694529428474057e5fb84000000474058294600000000652e\"\n }\n },\n \"time_total_s\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059534000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b058694529428474057e5fb8400000047406807a0c2000000652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059534000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a8694529428474057e5fb8400000047406807a0c2000000652e\"\n }\n },\n \"time_since_restore\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059534000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b058694529428474057e5fb84000000474058294600000000652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059534000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a8694529428474057e5fb84000000474058294600000000652e\"\n }\n },\n \"iterations_since_restore\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059526000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294284b014b01652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059526000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294284b014b01652e\"\n }\n }\n },\n \"checkpoint_manager\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"800595a6090000000000008c267261792e747261696e2e5f696e7465726e616c2e636865636b706f696e745f6d616e61676572948c125f436865636b706f696e744d616e616765729493942981947d94288c125f636865636b706f696e745f636f6e666967948c097261792e747261696e948c10436865636b706f696e74436f6e6669679493942981947d94288c0b6e756d5f746f5f6b656570944e8c1a636865636b706f696e745f73636f72655f617474726962757465944e8c16636865636b706f696e745f73636f72655f6f72646572948c036d6178948c14636865636b706f696e745f6672657175656e6379944b008c11636865636b706f696e745f61745f656e6494898c1a5f636865636b706f696e745f6b6565705f616c6c5f72616e6b73948c0a44455052454341544544948c1f5f636865636b706f696e745f75706c6f61645f66726f6d5f776f726b65727394681275628c135f636865636b706f696e745f726573756c7473945d94288c1b7261792e747261696e2e5f696e7465726e616c2e73657373696f6e948c0f5f547261696e696e67526573756c749493942981947d94288c0a636865636b706f696e749468068c0a436865636b706f696e749493942981947d94288c0470617468948cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f62353737333439645f365f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d322c6465636f6465725f72745f66663d313032342c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f725f323032342d30362d32355f31362d31332d31322f636865636b706f696e745f303030303030948c0a66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c055f75756964948c0475756964948c04555549449493942981947d948c03696e74948a11539cca0b2b6fcbbd0f44024aa04e0da500736275628c076d657472696373947d94288c046c6f7373948c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b00749462430873b95c6e1f9a9e4094869452948c0974696d657374616d70944a56d17a668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303030948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b018c08747269616c5f6964948c086235373733343964948c0464617465948c13323032342d30362d32355f31362d31362d3534948c1074696d655f746869735f697465725f7394474057e5fb840000008c0c74696d655f746f74616c5f7394474057e5fb840000008c03706964944ac60e01008c08686f73746e616d65948c06723369356e36948c076e6f64655f6970948c0c31302e3135392e382e313539948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b088c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b028c0d656d62656464696e675f64696d944b408c0a656e636f6465725f6666944d00088c0d6465636f6465725f72745f6666944d00048c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b028c0964726f705f7261746594473fd2d91ffe34831c8c026c7294473f40436c43e763ef8c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f726594474057e5fb840000008c18697465726174696f6e735f73696e63655f726573746f7265944b0175756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f62353737333439645f365f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d322c6465636f6465725f72745f66663d313032342c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f725f323032342d30362d32355f31362d31332d31322f636865636b706f696e745f30303030303194682268257d946827897385945294682a682d2981947d9468308a11a771b6bee3cbf999f74d2cbec6d3de8c007362756268317d94288c046c6f7373946836683c4308a7d3e934479a9e4094869452948c0974696d657374616d70944aecda7a668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303031948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b028c08747269616c5f6964948c086235373733343964948c0464617465948c13323032342d30362d32355f31362d35372d3438948c1074696d655f746869735f697465725f73944740582946000000008c0c74696d655f746f74616c5f739447406807a0c20000008c03706964944ae4fb01008c08686f73746e616d65948c06723369356e36948c076e6f64655f6970948c0c31302e3135392e382e313539948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b088c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b028c0d656d62656464696e675f64696d944b408c0a656e636f6465725f6666944d00088c0d6465636f6465725f72745f6666944d00048c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b028c0964726f705f7261746594473fd2d91ffe34831c8c026c7294473f40436c43e763ef8c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f7265944740582946000000008c18697465726174696f6e735f73696e63655f726573746f7265944b01757562658c195f6c61746573745f636865636b706f696e745f726573756c7494686275622e\"\n }\n}"
+ ],
+ [
+ "{\n \"stub\": false,\n \"trainable_name\": \"train_model\",\n \"trial_id\": \"86ae258f\",\n \"storage\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059572040000000000008c1b7261792e747261696e2e5f696e7465726e616c2e73746f72616765948c0e53746f72616765436f6e746578749493942981947d94288c12637573746f6d5f66735f70726f766964656494898c1273746f726167655f6c6f63616c5f70617468948c2b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c7473948c136578706572696d656e745f6469725f6e616d65948c0f746573745f6578706572696d656e74948c0e747269616c5f6469725f6e616d65948c96747261696e5f6d6f64656c5f38366165323538665f375f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d313032342c6465636f6465725f696e745f6e756d5f6c617965723d342c6465636f6465725f72745f66663d313032342c6465636f6465725f72745f6e756d5f6c617965723d342c64726f705f5f323032342d30362d32355f31362d31352d3138948c1863757272656e745f636865636b706f696e745f696e646578944b018c0b73796e635f636f6e666967948c097261792e747261696e948c0a53796e63436f6e6669679493942981947d94288c0b73796e635f706572696f64944d2c018c0c73796e635f74696d656f7574944d08078c0e73796e635f61727469666163747394898c1c73796e635f6172746966616374735f6f6e5f636865636b706f696e7494888c0a75706c6f61645f646972948c0a44455052454341544544948c0673796e6365729468188c1273796e635f6f6e5f636865636b706f696e7494681875628c1273746f726167655f66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c0f73746f726167655f66735f70617468948c452f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f7465737494681968008c115f46696c6573797374656d53796e6365729493942981947d9428681b682268134d2c0168144d08078c116c6173745f73796e635f75705f74696d65944741d99eb418846a298c136c6173745f73796e635f646f776e5f74696d659447fff00000000000008c0d5f73796e635f70726f63657373944e8c0c5f63757272656e745f636d649468008c125f75706c6f61645f746f5f66735f706174689493947d94288c0a6c6f63616c5f70617468948c3b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c74732f746573745f6578706572696d656e74948c0266739468228c0766735f70617468948c552f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e74948c076578636c756465945d948c0e2a2f636865636b706f696e745f2a9461758694756275622e\"\n },\n \"config\": {\n \"encoder_num_layer\": 2,\n \"decoder_rt_num_layer\": 4,\n \"decoder_int_num_layer\": 4,\n \"embedding_dim\": 256,\n \"encoder_ff\": 2048,\n \"decoder_rt_ff\": 1024,\n \"decoder_int_ff\": 1024,\n \"n_head\": 16,\n \"drop_rate\": 0.7568171644612052,\n \"lr\": 0.0002629821336450532,\n \"batch_size\": 2048\n },\n \"_Trial__unresolved_config\": {\n \"encoder_num_layer\": 2,\n \"decoder_rt_num_layer\": 4,\n \"decoder_int_num_layer\": 4,\n \"embedding_dim\": 256,\n \"encoder_ff\": 2048,\n \"decoder_rt_ff\": 1024,\n \"decoder_int_ff\": 1024,\n \"n_head\": 16,\n \"drop_rate\": 0.7568171644612052,\n \"lr\": 0.0002629821336450532,\n \"batch_size\": 2048\n },\n \"evaluated_params\": {\n \"batch_size\": 2048,\n \"decoder_int_ff\": 1024,\n \"decoder_int_num_layer\": 4,\n \"decoder_rt_ff\": 1024,\n \"decoder_rt_num_layer\": 4,\n \"drop_rate\": 0.7568171644612052,\n \"embedding_dim\": 256,\n \"encoder_ff\": 2048,\n \"encoder_num_layer\": 2,\n \"lr\": 0.0002629821336450532,\n \"n_head\": 16\n },\n \"experiment_tag\": \"7_batch_size=2048,decoder_int_ff=1024,decoder_int_num_layer=4,decoder_rt_ff=1024,decoder_rt_num_layer=4,drop_rate=0.7568,embedding_dim=256,encoder_ff=2048,encoder_num_layer=2,lr=0.0003,n_head=16\",\n \"stopping_criterion\": {},\n \"_setup_default_resource\": true,\n \"_default_placement_group_factory\": \"80054e2e\",\n \"placement_group_factory\": \"800595ba000000000000008c237261792e74756e652e657865637574696f6e2e706c6163656d656e745f67726f757073948c15506c6163656d656e7447726f7570466163746f72799493942981947d94288c085f62756e646c6573945d947d94288c03435055944740540000000000008c034750559447401000000000000075618c155f686561645f62756e646c655f69735f656d70747994898c095f7374726174656779948c045041434b948c055f6172677394298c075f6b7761726773947d9475622e\",\n \"log_to_file\": [\n null,\n null\n ],\n \"max_failures\": 0,\n \"_default_result_or_future\": null,\n \"export_formats\": [],\n \"status\": \"PAUSED\",\n \"relative_logdir\": \"train_model_86ae258f_7_batch_size=2048,decoder_int_ff=1024,decoder_int_num_layer=4,decoder_rt_ff=1024,decoder_rt_num_layer=4,drop__2024-06-25_16-15-18\",\n \"trial_name_creator\": null,\n \"trial_dirname_creator\": null,\n \"custom_trial_name\": null,\n \"custom_dirname\": null,\n \"restore_path\": null,\n \"_restore_checkpoint_result\": null,\n \"_state_json\": null,\n \"results\": \"80054e2e\",\n \"extra_arg\": \"80054e2e\",\n \"_resources\": \"80054e2e\"\n}",
+ "{\n \"start_time\": 1719325014.5539198,\n \"num_failures\": 0,\n \"num_failures_after_restore\": 0,\n \"error_filename\": null,\n \"pickled_error_filename\": null,\n \"last_result\": {\n \"loss\": 1995.102591927596,\n \"timestamp\": 1719327610,\n \"checkpoint_dir_name\": \"checkpoint_000001\",\n \"should_checkpoint\": true,\n \"done\": false,\n \"training_iteration\": 2,\n \"trial_id\": \"86ae258f\",\n \"date\": \"2024-06-25_17-00-10\",\n \"time_this_iter_s\": 141.5520534515381,\n \"time_total_s\": 283.12366557121277,\n \"pid\": 130020,\n \"hostname\": \"r3i5n6\",\n \"node_ip\": \"10.159.8.159\",\n \"config\": {\n \"encoder_num_layer\": 2,\n \"decoder_rt_num_layer\": 4,\n \"decoder_int_num_layer\": 4,\n \"embedding_dim\": 256,\n \"encoder_ff\": 2048,\n \"decoder_rt_ff\": 1024,\n \"decoder_int_ff\": 1024,\n \"n_head\": 16,\n \"drop_rate\": 0.7568171644612052,\n \"lr\": 0.0002629821336450532,\n \"batch_size\": 2048\n },\n \"time_since_restore\": 141.5520534515381,\n \"iterations_since_restore\": 1,\n \"experiment_tag\": \"7_batch_size=2048,decoder_int_ff=1024,decoder_int_num_layer=4,decoder_rt_ff=1024,decoder_rt_num_layer=4,drop_rate=0.7568,embedding_dim=256,encoder_ff=2048,encoder_num_layer=2,lr=0.0003,n_head=16\"\n },\n \"last_result_time\": 1719327610.6072223,\n \"metric_analysis\": {\n \"loss\": {\n \"max\": 1995.102591927596,\n \"min\": 1993.6307767132137,\n \"avg\": 1994.3666843204048,\n \"last\": 1995.102591927596,\n \"last-5-avg\": 1994.3666843204048,\n \"last-10-avg\": 1994.3666843204048\n },\n \"should_checkpoint\": {\n \"max\": true,\n \"min\": true,\n \"avg\": 1.0,\n \"last\": true,\n \"last-5-avg\": 1.0,\n \"last-10-avg\": 1.0\n },\n \"done\": {\n \"max\": false,\n \"min\": false,\n \"avg\": 0.0,\n \"last\": false,\n \"last-5-avg\": 0.0,\n \"last-10-avg\": 0.0\n },\n \"training_iteration\": {\n \"max\": 2,\n \"min\": 1,\n \"avg\": 1.5,\n \"last\": 2,\n \"last-5-avg\": 1.5,\n \"last-10-avg\": 1.5\n },\n \"time_this_iter_s\": {\n \"max\": 141.57161211967468,\n \"min\": 141.5520534515381,\n \"avg\": 141.56183278560638,\n \"last\": 141.5520534515381,\n \"last-5-avg\": 141.56183278560638,\n \"last-10-avg\": 141.56183278560638\n },\n \"time_total_s\": {\n \"max\": 283.12366557121277,\n \"min\": 141.57161211967468,\n \"avg\": 212.34763884544373,\n \"last\": 283.12366557121277,\n \"last-5-avg\": 212.34763884544373,\n \"last-10-avg\": 212.34763884544373\n },\n \"time_since_restore\": {\n \"max\": 141.57161211967468,\n \"min\": 141.5520534515381,\n \"avg\": 141.56183278560638,\n \"last\": 141.5520534515381,\n \"last-5-avg\": 141.56183278560638,\n \"last-10-avg\": 141.56183278560638\n },\n \"iterations_since_restore\": {\n \"max\": 1,\n \"min\": 1,\n \"avg\": 1.0,\n \"last\": 1,\n \"last-5-avg\": 1.0,\n \"last-10-avg\": 1.0\n }\n },\n \"_n_steps\": [\n 5,\n 10\n ],\n \"metric_n_steps\": {\n \"loss\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005959d000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294288c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b00749462430853a954ea85269f4094869452946807680d43086fb7db0d692c9f409486945294652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005959d000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294288c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b00749462430853a954ea85269f4094869452946807680d43086fb7db0d692c9f409486945294652e\"\n }\n },\n \"should_checkpoint\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059524000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294288888652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059524000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294288888652e\"\n }\n },\n \"done\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059524000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294288989652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059524000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294288989652e\"\n }\n },\n \"training_iteration\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059526000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294284b014b02652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059526000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294284b014b02652e\"\n }\n },\n \"time_this_iter_s\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059534000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b058694529428474061b24aa5800000474061b1aa6c000000652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059534000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a8694529428474061b24aa5800000474061b1aa6c000000652e\"\n }\n },\n \"time_total_s\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059534000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b058694529428474061b24aa5800000474071b1fa88c00000652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059534000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a8694529428474061b24aa5800000474071b1fa88c00000652e\"\n }\n },\n \"time_since_restore\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059534000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b058694529428474061b24aa5800000474061b1aa6c000000652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059534000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a8694529428474061b24aa5800000474061b1aa6c000000652e\"\n }\n },\n \"iterations_since_restore\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059526000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294284b014b01652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059526000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294284b014b01652e\"\n }\n }\n },\n \"checkpoint_manager\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"800595a7090000000000008c267261792e747261696e2e5f696e7465726e616c2e636865636b706f696e745f6d616e61676572948c125f436865636b706f696e744d616e616765729493942981947d94288c125f636865636b706f696e745f636f6e666967948c097261792e747261696e948c10436865636b706f696e74436f6e6669679493942981947d94288c0b6e756d5f746f5f6b656570944e8c1a636865636b706f696e745f73636f72655f617474726962757465944e8c16636865636b706f696e745f73636f72655f6f72646572948c036d6178948c14636865636b706f696e745f6672657175656e6379944b008c11636865636b706f696e745f61745f656e6494898c1a5f636865636b706f696e745f6b6565705f616c6c5f72616e6b73948c0a44455052454341544544948c1f5f636865636b706f696e745f75706c6f61645f66726f6d5f776f726b65727394681275628c135f636865636b706f696e745f726573756c7473945d94288c1b7261792e747261696e2e5f696e7465726e616c2e73657373696f6e948c0f5f547261696e696e67526573756c749493942981947d94288c0a636865636b706f696e749468068c0a436865636b706f696e749493942981947d94288c0470617468948cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f38366165323538665f375f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d313032342c6465636f6465725f696e745f6e756d5f6c617965723d342c6465636f6465725f72745f66663d313032342c6465636f6465725f72745f6e756d5f6c617965723d342c64726f705f5f323032342d30362d32355f31362d31352d31382f636865636b706f696e745f303030303030948c0a66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c055f75756964948c0475756964948c04555549449493942981947d948c03696e74948a11f44260e4487af3908249f6c6bea7609700736275628c076d657472696373947d94288c046c6f7373948c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b00749462430853a954ea85269f4094869452948c0974696d657374616d70944ae4d17a668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303030948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b018c08747269616c5f6964948c083836616532353866948c0464617465948c13323032342d30362d32355f31362d31392d3136948c1074696d655f746869735f697465725f7394474061b24aa58000008c0c74696d655f746f74616c5f7394474061b24aa58000008c03706964944ac60e01008c08686f73746e616d65948c06723369356e36948c076e6f64655f6970948c0c31302e3135392e382e313539948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b028c146465636f6465725f72745f6e756d5f6c61796572944b048c156465636f6465725f696e745f6e756d5f6c61796572944b048c0d656d62656464696e675f64696d944d00018c0a656e636f6465725f6666944d00088c0d6465636f6465725f72745f6666944d00048c0e6465636f6465725f696e745f6666944d00048c066e5f68656164944b108c0964726f705f7261746594473fe837d8a14d31ce8c026c7294473f313c1ba9d70f5a8c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f726594474061b24aa58000008c18697465726174696f6e735f73696e63655f726573746f7265944b0175756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f38366165323538665f375f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d313032342c6465636f6465725f696e745f6e756d5f6c617965723d342c6465636f6465725f72745f66663d313032342c6465636f6465725f72745f6e756d5f6c617965723d342c64726f705f5f323032342d30362d32355f31362d31352d31382f636865636b706f696e745f30303030303194682268257d946827897385945294682a682d2981947d9468308a104c28418a6a14e6b2a24745a6d15899067362756268317d94288c046c6f7373946836683c43086fb7db0d692c9f4094869452948c0974696d657374616d70944a7adb7a668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303031948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b028c08747269616c5f6964948c083836616532353866948c0464617465948c13323032342d30362d32355f31372d30302d3130948c1074696d655f746869735f697465725f7394474061b1aa6c0000008c0c74696d655f746f74616c5f7394474071b1fa88c000008c03706964944ae4fb01008c08686f73746e616d65948c06723369356e36948c076e6f64655f6970948c0c31302e3135392e382e313539948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b028c146465636f6465725f72745f6e756d5f6c61796572944b048c156465636f6465725f696e745f6e756d5f6c61796572944b048c0d656d62656464696e675f64696d944d00018c0a656e636f6465725f6666944d00088c0d6465636f6465725f72745f6666944d00048c0e6465636f6465725f696e745f6666944d00048c066e5f68656164944b108c0964726f705f7261746594473fe837d8a14d31ce8c026c7294473f313c1ba9d70f5a8c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f726594474061b1aa6c0000008c18697465726174696f6e735f73696e63655f726573746f7265944b01757562658c195f6c61746573745f636865636b706f696e745f726573756c7494686275622e\"\n }\n}"
+ ],
+ [
+ "{\n \"stub\": false,\n \"trainable_name\": \"train_model\",\n \"trial_id\": \"c4172973\",\n \"storage\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059572040000000000008c1b7261792e747261696e2e5f696e7465726e616c2e73746f72616765948c0e53746f72616765436f6e746578749493942981947d94288c12637573746f6d5f66735f70726f766964656494898c1273746f726167655f6c6f63616c5f70617468948c2b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c7473948c136578706572696d656e745f6469725f6e616d65948c0f746573745f6578706572696d656e74948c0e747269616c5f6469725f6e616d65948c96747261696e5f6d6f64656c5f63343137323937335f385f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d323034382c6465636f6465725f696e745f6e756d5f6c617965723d382c6465636f6465725f72745f66663d3531322c6465636f6465725f72745f6e756d5f6c617965723d322c64726f705f725f323032342d30362d32355f31362d31362d3534948c1863757272656e745f636865636b706f696e745f696e646578944b018c0b73796e635f636f6e666967948c097261792e747261696e948c0a53796e63436f6e6669679493942981947d94288c0b73796e635f706572696f64944d2c018c0c73796e635f74696d656f7574944d08078c0e73796e635f61727469666163747394898c1c73796e635f6172746966616374735f6f6e5f636865636b706f696e7494888c0a75706c6f61645f646972948c0a44455052454341544544948c0673796e6365729468188c1273796e635f6f6e5f636865636b706f696e7494681875628c1273746f726167655f66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c0f73746f726167655f66735f70617468948c452f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f7465737494681968008c115f46696c6573797374656d53796e6365729493942981947d9428681b682268134d2c0168144d08078c116c6173745f73796e635f75705f74696d65944741d99eb418846a298c136c6173745f73796e635f646f776e5f74696d659447fff00000000000008c0d5f73796e635f70726f63657373944e8c0c5f63757272656e745f636d649468008c125f75706c6f61645f746f5f66735f706174689493947d94288c0a6c6f63616c5f70617468948c3b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c74732f746573745f6578706572696d656e74948c0266739468228c0766735f70617468948c552f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e74948c076578636c756465945d948c0e2a2f636865636b706f696e745f2a9461758694756275622e\"\n },\n \"config\": {\n \"encoder_num_layer\": 2,\n \"decoder_rt_num_layer\": 2,\n \"decoder_int_num_layer\": 8,\n \"embedding_dim\": 256,\n \"encoder_ff\": 1024,\n \"decoder_rt_ff\": 512,\n \"decoder_int_ff\": 2048,\n \"n_head\": 2,\n \"drop_rate\": 0.4176788223006639,\n \"lr\": 0.00038150746043137153,\n \"batch_size\": 2048\n },\n \"_Trial__unresolved_config\": {\n \"encoder_num_layer\": 2,\n \"decoder_rt_num_layer\": 2,\n \"decoder_int_num_layer\": 8,\n \"embedding_dim\": 256,\n \"encoder_ff\": 1024,\n \"decoder_rt_ff\": 512,\n \"decoder_int_ff\": 2048,\n \"n_head\": 2,\n \"drop_rate\": 0.4176788223006639,\n \"lr\": 0.00038150746043137153,\n \"batch_size\": 2048\n },\n \"evaluated_params\": {\n \"batch_size\": 2048,\n \"decoder_int_ff\": 2048,\n \"decoder_int_num_layer\": 8,\n \"decoder_rt_ff\": 512,\n \"decoder_rt_num_layer\": 2,\n \"drop_rate\": 0.4176788223006639,\n \"embedding_dim\": 256,\n \"encoder_ff\": 1024,\n \"encoder_num_layer\": 2,\n \"lr\": 0.00038150746043137153,\n \"n_head\": 2\n },\n \"experiment_tag\": \"8_batch_size=2048,decoder_int_ff=2048,decoder_int_num_layer=8,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_rate=0.4177,embedding_dim=256,encoder_ff=1024,encoder_num_layer=2,lr=0.0004,n_head=2\",\n \"stopping_criterion\": {},\n \"_setup_default_resource\": true,\n \"_default_placement_group_factory\": \"80054e2e\",\n \"placement_group_factory\": \"800595ba000000000000008c237261792e74756e652e657865637574696f6e2e706c6163656d656e745f67726f757073948c15506c6163656d656e7447726f7570466163746f72799493942981947d94288c085f62756e646c6573945d947d94288c03435055944740540000000000008c034750559447401000000000000075618c155f686561645f62756e646c655f69735f656d70747994898c095f7374726174656779948c045041434b948c055f6172677394298c075f6b7761726773947d9475622e\",\n \"log_to_file\": [\n null,\n null\n ],\n \"max_failures\": 0,\n \"_default_result_or_future\": null,\n \"export_formats\": [],\n \"status\": \"PAUSED\",\n \"relative_logdir\": \"train_model_c4172973_8_batch_size=2048,decoder_int_ff=2048,decoder_int_num_layer=8,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_16-16-54\",\n \"trial_name_creator\": null,\n \"trial_dirname_creator\": null,\n \"custom_trial_name\": null,\n \"custom_dirname\": null,\n \"restore_path\": null,\n \"_restore_checkpoint_result\": null,\n \"_state_json\": null,\n \"results\": \"80054e2e\",\n \"extra_arg\": \"80054e2e\",\n \"_resources\": \"80054e2e\"\n}",
+ "{\n \"start_time\": 1719325156.2771058,\n \"num_failures\": 0,\n \"num_failures_after_restore\": 0,\n \"error_filename\": null,\n \"pickled_error_filename\": null,\n \"last_result\": {\n \"loss\": 261.25574487400803,\n \"timestamp\": 1719327690,\n \"checkpoint_dir_name\": \"checkpoint_000001\",\n \"should_checkpoint\": true,\n \"done\": false,\n \"training_iteration\": 2,\n \"trial_id\": \"c4172973\",\n \"date\": \"2024-06-25_17-01-30\",\n \"time_this_iter_s\": 79.34292578697205,\n \"time_total_s\": 157.75713872909546,\n \"pid\": 130020,\n \"hostname\": \"r3i5n6\",\n \"node_ip\": \"10.159.8.159\",\n \"config\": {\n \"encoder_num_layer\": 2,\n \"decoder_rt_num_layer\": 2,\n \"decoder_int_num_layer\": 8,\n \"embedding_dim\": 256,\n \"encoder_ff\": 1024,\n \"decoder_rt_ff\": 512,\n \"decoder_int_ff\": 2048,\n \"n_head\": 2,\n \"drop_rate\": 0.4176788223006639,\n \"lr\": 0.00038150746043137153,\n \"batch_size\": 2048\n },\n \"time_since_restore\": 79.34292578697205,\n \"iterations_since_restore\": 1,\n \"experiment_tag\": \"8_batch_size=2048,decoder_int_ff=2048,decoder_int_num_layer=8,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_rate=0.4177,embedding_dim=256,encoder_ff=1024,encoder_num_layer=2,lr=0.0004,n_head=2\"\n },\n \"last_result_time\": 1719327690.1038792,\n \"metric_analysis\": {\n \"loss\": {\n \"max\": 357.444681242695,\n \"min\": 261.25574487400803,\n \"avg\": 309.3502130583515,\n \"last\": 261.25574487400803,\n \"last-5-avg\": 309.3502130583515,\n \"last-10-avg\": 309.3502130583515\n },\n \"should_checkpoint\": {\n \"max\": true,\n \"min\": true,\n \"avg\": 1.0,\n \"last\": true,\n \"last-5-avg\": 1.0,\n \"last-10-avg\": 1.0\n },\n \"done\": {\n \"max\": false,\n \"min\": false,\n \"avg\": 0.0,\n \"last\": false,\n \"last-5-avg\": 0.0,\n \"last-10-avg\": 0.0\n },\n \"training_iteration\": {\n \"max\": 2,\n \"min\": 1,\n \"avg\": 1.5,\n \"last\": 2,\n \"last-5-avg\": 1.5,\n \"last-10-avg\": 1.5\n },\n \"time_this_iter_s\": {\n \"max\": 79.34292578697205,\n \"min\": 78.41421294212341,\n \"avg\": 78.87856936454773,\n \"last\": 79.34292578697205,\n \"last-5-avg\": 78.87856936454773,\n \"last-10-avg\": 78.87856936454773\n },\n \"time_total_s\": {\n \"max\": 157.75713872909546,\n \"min\": 78.41421294212341,\n \"avg\": 118.08567583560944,\n \"last\": 157.75713872909546,\n \"last-5-avg\": 118.08567583560944,\n \"last-10-avg\": 118.08567583560944\n },\n \"time_since_restore\": {\n \"max\": 79.34292578697205,\n \"min\": 78.41421294212341,\n \"avg\": 78.87856936454773,\n \"last\": 79.34292578697205,\n \"last-5-avg\": 78.87856936454773,\n \"last-10-avg\": 78.87856936454773\n },\n \"iterations_since_restore\": {\n \"max\": 1,\n \"min\": 1,\n \"avg\": 1.0,\n \"last\": 1,\n \"last-5-avg\": 1.0,\n \"last-10-avg\": 1.0\n }\n },\n \"_n_steps\": [\n 5,\n 10\n ],\n \"metric_n_steps\": {\n \"loss\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005959d000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294288c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b0074946243085128146a1d57764094869452946807680d4308bfdfef87175470409486945294652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005959d000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294288c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b0074946243085128146a1d57764094869452946807680d4308bfdfef87175470409486945294652e\"\n }\n },\n \"should_checkpoint\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059524000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294288888652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059524000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294288888652e\"\n }\n },\n \"done\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059524000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294288989652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059524000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294288989652e\"\n }\n },\n \"training_iteration\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059526000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294284b014b02652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059526000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294284b014b02652e\"\n }\n },\n \"time_this_iter_s\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059534000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294284740539a8277000000474053d5f27f000000652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059534000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294284740539a8277000000474053d5f27f000000652e\"\n }\n },\n \"time_total_s\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059534000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294284740539a8277000000474063b83a7b000000652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059534000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294284740539a8277000000474063b83a7b000000652e\"\n }\n },\n \"time_since_restore\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059534000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294284740539a8277000000474053d5f27f000000652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059534000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294284740539a8277000000474053d5f27f000000652e\"\n }\n },\n \"iterations_since_restore\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059526000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294284b014b01652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059526000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294284b014b01652e\"\n }\n }\n },\n \"checkpoint_manager\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"800595a8090000000000008c267261792e747261696e2e5f696e7465726e616c2e636865636b706f696e745f6d616e61676572948c125f436865636b706f696e744d616e616765729493942981947d94288c125f636865636b706f696e745f636f6e666967948c097261792e747261696e948c10436865636b706f696e74436f6e6669679493942981947d94288c0b6e756d5f746f5f6b656570944e8c1a636865636b706f696e745f73636f72655f617474726962757465944e8c16636865636b706f696e745f73636f72655f6f72646572948c036d6178948c14636865636b706f696e745f6672657175656e6379944b008c11636865636b706f696e745f61745f656e6494898c1a5f636865636b706f696e745f6b6565705f616c6c5f72616e6b73948c0a44455052454341544544948c1f5f636865636b706f696e745f75706c6f61645f66726f6d5f776f726b65727394681275628c135f636865636b706f696e745f726573756c7473945d94288c1b7261792e747261696e2e5f696e7465726e616c2e73657373696f6e948c0f5f547261696e696e67526573756c749493942981947d94288c0a636865636b706f696e749468068c0a436865636b706f696e749493942981947d94288c0470617468948cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f63343137323937335f385f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d323034382c6465636f6465725f696e745f6e756d5f6c617965723d382c6465636f6465725f72745f66663d3531322c6465636f6465725f72745f6e756d5f6c617965723d322c64726f705f725f323032342d30362d32355f31362d31362d35342f636865636b706f696e745f303030303030948c0a66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c055f75756964948c0475756964948c04555549449493942981947d948c03696e74948a11767c713a35b871b5de447f9814b0d9b300736275628c076d657472696373947d94288c046c6f7373948c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b0074946243085128146a1d57764094869452948c0974696d657374616d70944a32d27a668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303030948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b018c08747269616c5f6964948c086334313732393733948c0464617465948c13323032342d30362d32355f31362d32302d3334948c1074696d655f746869735f697465725f73944740539a82770000008c0c74696d655f746f74616c5f73944740539a82770000008c03706964944ac60e01008c08686f73746e616d65948c06723369356e36948c076e6f64655f6970948c0c31302e3135392e382e313539948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b028c146465636f6465725f72745f6e756d5f6c61796572944b028c156465636f6465725f696e745f6e756d5f6c61796572944b088c0d656d62656464696e675f64696d944d00018c0a656e636f6465725f6666944d00048c0d6465636f6465725f72745f6666944d00028c0e6465636f6465725f696e745f6666944d00088c066e5f68656164944b028c0964726f705f7261746594473fdabb3ff480d7668c026c7294473f3900a210d3dca08c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f7265944740539a82770000008c18697465726174696f6e735f73696e63655f726573746f7265944b0175756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f63343137323937335f385f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d323034382c6465636f6465725f696e745f6e756d5f6c617965723d382c6465636f6465725f72745f66663d3531322c6465636f6465725f72745f6e756d5f6c617965723d322c64726f705f725f323032342d30362d32355f31362d31362d35342f636865636b706f696e745f30303030303194682268257d946827897385945294682a682d2981947d9468308a116354e19f53c12892fe4f978c6a8c7a8b007362756268317d94288c046c6f7373946836683c4308bfdfef871754704094869452948c0974696d657374616d70944acadb7a668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303031948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b028c08747269616c5f6964948c086334313732393733948c0464617465948c13323032342d30362d32355f31372d30312d3330948c1074696d655f746869735f697465725f7394474053d5f27f0000008c0c74696d655f746f74616c5f7394474063b83a7b0000008c03706964944ae4fb01008c08686f73746e616d65948c06723369356e36948c076e6f64655f6970948c0c31302e3135392e382e313539948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b028c146465636f6465725f72745f6e756d5f6c61796572944b028c156465636f6465725f696e745f6e756d5f6c61796572944b088c0d656d62656464696e675f64696d944d00018c0a656e636f6465725f6666944d00048c0d6465636f6465725f72745f6666944d00028c0e6465636f6465725f696e745f6666944d00088c066e5f68656164944b028c0964726f705f7261746594473fdabb3ff480d7668c026c7294473f3900a210d3dca08c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f726594474053d5f27f0000008c18697465726174696f6e735f73696e63655f726573746f7265944b01757562658c195f6c61746573745f636865636b706f696e745f726573756c7494686275622e\"\n }\n}"
+ ],
+ [
+ "{\n \"stub\": false,\n \"trainable_name\": \"train_model\",\n \"trial_id\": \"1920f493\",\n \"storage\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059572040000000000008c1b7261792e747261696e2e5f696e7465726e616c2e73746f72616765948c0e53746f72616765436f6e746578749493942981947d94288c12637573746f6d5f66735f70726f766964656494898c1273746f726167655f6c6f63616c5f70617468948c2b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c7473948c136578706572696d656e745f6469725f6e616d65948c0f746573745f6578706572696d656e74948c0e747269616c5f6469725f6e616d65948c96747261696e5f6d6f64656c5f31393230663439335f395f62617463685f73697a653d313032342c6465636f6465725f696e745f66663d313032342c6465636f6465725f696e745f6e756d5f6c617965723d322c6465636f6465725f72745f66663d323034382c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f5f323032342d30362d32355f31362d31392d3136948c1863757272656e745f636865636b706f696e745f696e646578944b008c0b73796e635f636f6e666967948c097261792e747261696e948c0a53796e63436f6e6669679493942981947d94288c0b73796e635f706572696f64944d2c018c0c73796e635f74696d656f7574944d08078c0e73796e635f61727469666163747394898c1c73796e635f6172746966616374735f6f6e5f636865636b706f696e7494888c0a75706c6f61645f646972948c0a44455052454341544544948c0673796e6365729468188c1273796e635f6f6e5f636865636b706f696e7494681875628c1273746f726167655f66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c0f73746f726167655f66735f70617468948c452f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f7465737494681968008c115f46696c6573797374656d53796e6365729493942981947d9428681b682268134d2c0168144d08078c116c6173745f73796e635f75705f74696d65944741d99eb463df15148c136c6173745f73796e635f646f776e5f74696d659447fff00000000000008c0d5f73796e635f70726f63657373944e8c0c5f63757272656e745f636d649468008c125f75706c6f61645f746f5f66735f706174689493947d94288c0a6c6f63616c5f70617468948c3b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c74732f746573745f6578706572696d656e74948c0266739468228c0766735f70617468948c552f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e74948c076578636c756465945d948c0e2a2f636865636b706f696e745f2a9461758694756275622e\"\n },\n \"config\": {\n \"encoder_num_layer\": 4,\n \"decoder_rt_num_layer\": 1,\n \"decoder_int_num_layer\": 2,\n \"embedding_dim\": 64,\n \"encoder_ff\": 2048,\n \"decoder_rt_ff\": 2048,\n \"decoder_int_ff\": 1024,\n \"n_head\": 1,\n \"drop_rate\": 0.922082156004742,\n \"lr\": 0.0016525285062566051,\n \"batch_size\": 1024\n },\n \"_Trial__unresolved_config\": {\n \"encoder_num_layer\": 4,\n \"decoder_rt_num_layer\": 1,\n \"decoder_int_num_layer\": 2,\n \"embedding_dim\": 64,\n \"encoder_ff\": 2048,\n \"decoder_rt_ff\": 2048,\n \"decoder_int_ff\": 1024,\n \"n_head\": 1,\n \"drop_rate\": 0.922082156004742,\n \"lr\": 0.0016525285062566051,\n \"batch_size\": 1024\n },\n \"evaluated_params\": {\n \"batch_size\": 1024,\n \"decoder_int_ff\": 1024,\n \"decoder_int_num_layer\": 2,\n \"decoder_rt_ff\": 2048,\n \"decoder_rt_num_layer\": 1,\n \"drop_rate\": 0.922082156004742,\n \"embedding_dim\": 64,\n \"encoder_ff\": 2048,\n \"encoder_num_layer\": 4,\n \"lr\": 0.0016525285062566051,\n \"n_head\": 1\n },\n \"experiment_tag\": \"9_batch_size=1024,decoder_int_ff=1024,decoder_int_num_layer=2,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop_rate=0.9221,embedding_dim=64,encoder_ff=2048,encoder_num_layer=4,lr=0.0017,n_head=1\",\n \"stopping_criterion\": {},\n \"_setup_default_resource\": true,\n \"_default_placement_group_factory\": \"80054e2e\",\n \"placement_group_factory\": \"800595ba000000000000008c237261792e74756e652e657865637574696f6e2e706c6163656d656e745f67726f757073948c15506c6163656d656e7447726f7570466163746f72799493942981947d94288c085f62756e646c6573945d947d94288c03435055944740540000000000008c034750559447401000000000000075618c155f686561645f62756e646c655f69735f656d70747994898c095f7374726174656779948c045041434b948c055f6172677394298c075f6b7761726773947d9475622e\",\n \"log_to_file\": [\n null,\n null\n ],\n \"max_failures\": 0,\n \"_default_result_or_future\": null,\n \"export_formats\": [],\n \"status\": \"PAUSED\",\n \"relative_logdir\": \"train_model_1920f493_9_batch_size=1024,decoder_int_ff=1024,decoder_int_num_layer=2,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop__2024-06-25_16-19-16\",\n \"trial_name_creator\": null,\n \"trial_dirname_creator\": null,\n \"custom_trial_name\": null,\n \"custom_dirname\": null,\n \"restore_path\": null,\n \"_restore_checkpoint_result\": null,\n \"_state_json\": null,\n \"results\": \"80054e2e\",\n \"extra_arg\": \"80054e2e\",\n \"_resources\": \"80054e2e\"\n}",
+ "{\n \"start_time\": 1719325234.8395333,\n \"num_failures\": 0,\n \"num_failures_after_restore\": 0,\n \"error_filename\": null,\n \"pickled_error_filename\": null,\n \"last_result\": {\n \"loss\": 2102.196185254675,\n \"timestamp\": 1719325314,\n \"checkpoint_dir_name\": \"checkpoint_000000\",\n \"should_checkpoint\": true,\n \"done\": false,\n \"training_iteration\": 1,\n \"trial_id\": \"1920f493\",\n \"date\": \"2024-06-25_16-21-54\",\n \"time_this_iter_s\": 79.38143539428711,\n \"time_total_s\": 79.38143539428711,\n \"pid\": 69318,\n \"hostname\": \"r3i5n6\",\n \"node_ip\": \"10.159.8.159\",\n \"config\": {\n \"encoder_num_layer\": 4,\n \"decoder_rt_num_layer\": 1,\n \"decoder_int_num_layer\": 2,\n \"embedding_dim\": 64,\n \"encoder_ff\": 2048,\n \"decoder_rt_ff\": 2048,\n \"decoder_int_ff\": 1024,\n \"n_head\": 1,\n \"drop_rate\": 0.922082156004742,\n \"lr\": 0.0016525285062566051,\n \"batch_size\": 1024\n },\n \"time_since_restore\": 79.38143539428711,\n \"iterations_since_restore\": 1,\n \"experiment_tag\": \"9_batch_size=1024,decoder_int_ff=1024,decoder_int_num_layer=2,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop_rate=0.9221,embedding_dim=64,encoder_ff=2048,encoder_num_layer=4,lr=0.0017,n_head=1\"\n },\n \"last_result_time\": 1719325314.225798,\n \"metric_analysis\": {\n \"loss\": {\n \"max\": 2102.196185254675,\n \"min\": 2102.196185254675,\n \"avg\": 2102.196185254675,\n \"last\": 2102.196185254675,\n \"last-5-avg\": 2102.196185254675,\n \"last-10-avg\": 2102.196185254675\n },\n \"should_checkpoint\": {\n \"max\": true,\n \"min\": true,\n \"avg\": true,\n \"last\": true,\n \"last-5-avg\": true,\n \"last-10-avg\": true\n },\n \"done\": {\n \"max\": false,\n \"min\": false,\n \"avg\": false,\n \"last\": false,\n \"last-5-avg\": false,\n \"last-10-avg\": false\n },\n \"training_iteration\": {\n \"max\": 1,\n \"min\": 1,\n \"avg\": 1,\n \"last\": 1,\n \"last-5-avg\": 1,\n \"last-10-avg\": 1\n },\n \"time_this_iter_s\": {\n \"max\": 79.38143539428711,\n \"min\": 79.38143539428711,\n \"avg\": 79.38143539428711,\n \"last\": 79.38143539428711,\n \"last-5-avg\": 79.38143539428711,\n \"last-10-avg\": 79.38143539428711\n },\n \"time_total_s\": {\n \"max\": 79.38143539428711,\n \"min\": 79.38143539428711,\n \"avg\": 79.38143539428711,\n \"last\": 79.38143539428711,\n \"last-5-avg\": 79.38143539428711,\n \"last-10-avg\": 79.38143539428711\n },\n \"time_since_restore\": {\n \"max\": 79.38143539428711,\n \"min\": 79.38143539428711,\n \"avg\": 79.38143539428711,\n \"last\": 79.38143539428711,\n \"last-5-avg\": 79.38143539428711,\n \"last-10-avg\": 79.38143539428711\n },\n \"iterations_since_restore\": {\n \"max\": 1,\n \"min\": 1,\n \"avg\": 1,\n \"last\": 1,\n \"last-5-avg\": 1,\n \"last-10-avg\": 1\n }\n },\n \"_n_steps\": [\n 5,\n 10\n ],\n \"metric_n_steps\": {\n \"loss\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059589000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452948c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b00749462430893c96472646ca0409486945294612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059589000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452948c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b00749462430893c96472646ca0409486945294612e\"\n }\n },\n \"should_checkpoint\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059522000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b058694529488612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059522000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a8694529488612e\"\n }\n },\n \"done\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059522000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b058694529489612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059522000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a8694529489612e\"\n }\n },\n \"training_iteration\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059523000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452944b01612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059523000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452944b01612e\"\n }\n },\n \"time_this_iter_s\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294474053d86970000000612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294474053d86970000000612e\"\n }\n },\n \"time_total_s\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294474053d86970000000612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294474053d86970000000612e\"\n }\n },\n \"time_since_restore\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294474053d86970000000612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294474053d86970000000612e\"\n }\n },\n \"iterations_since_restore\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059523000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452944b01612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059523000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452944b01612e\"\n }\n }\n },\n \"checkpoint_manager\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059518060000000000008c267261792e747261696e2e5f696e7465726e616c2e636865636b706f696e745f6d616e61676572948c125f436865636b706f696e744d616e616765729493942981947d94288c125f636865636b706f696e745f636f6e666967948c097261792e747261696e948c10436865636b706f696e74436f6e6669679493942981947d94288c0b6e756d5f746f5f6b656570944e8c1a636865636b706f696e745f73636f72655f617474726962757465944e8c16636865636b706f696e745f73636f72655f6f72646572948c036d6178948c14636865636b706f696e745f6672657175656e6379944b008c11636865636b706f696e745f61745f656e6494898c1a5f636865636b706f696e745f6b6565705f616c6c5f72616e6b73948c0a44455052454341544544948c1f5f636865636b706f696e745f75706c6f61645f66726f6d5f776f726b65727394681275628c135f636865636b706f696e745f726573756c7473945d948c1b7261792e747261696e2e5f696e7465726e616c2e73657373696f6e948c0f5f547261696e696e67526573756c749493942981947d94288c0a636865636b706f696e749468068c0a436865636b706f696e749493942981947d94288c0470617468948cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f31393230663439335f395f62617463685f73697a653d313032342c6465636f6465725f696e745f66663d313032342c6465636f6465725f696e745f6e756d5f6c617965723d322c6465636f6465725f72745f66663d323034382c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f5f323032342d30362d32355f31362d31392d31362f636865636b706f696e745f303030303030948c0a66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c055f75756964948c0475756964948c04555549449493942981947d948c03696e74948a11fdcf99429d7f868e6d47eef0e42d098000736275628c076d657472696373947d94288c046c6f7373948c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b00749462430893c96472646ca04094869452948c0974696d657374616d70944a82d27a668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303030948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b018c08747269616c5f6964948c083139323066343933948c0464617465948c13323032342d30362d32355f31362d32312d3534948c1074696d655f746869735f697465725f7394474053d869700000008c0c74696d655f746f74616c5f7394474053d869700000008c03706964944ac60e01008c08686f73746e616d65948c06723369356e36948c076e6f64655f6970948c0c31302e3135392e382e313539948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b048c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b028c0d656d62656464696e675f64696d944b408c0a656e636f6465725f6666944d00088c0d6465636f6465725f72745f6666944d00088c0e6465636f6465725f696e745f6666944d00048c066e5f68656164944b018c0964726f705f7261746594473fed81b270087f478c026c7294473f5b1334f8f7117a8c0a62617463685f73697a65944d0004758c1274696d655f73696e63655f726573746f726594474053d869700000008c18697465726174696f6e735f73696e63655f726573746f7265944b01757562618c195f6c61746573745f636865636b706f696e745f726573756c7494681975622e\"\n }\n}"
+ ],
+ [
+ "{\n \"stub\": false,\n \"trainable_name\": \"train_model\",\n \"trial_id\": \"1de460a8\",\n \"storage\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059572040000000000008c1b7261792e747261696e2e5f696e7465726e616c2e73746f72616765948c0e53746f72616765436f6e746578749493942981947d94288c12637573746f6d5f66735f70726f766964656494898c1273746f726167655f6c6f63616c5f70617468948c2b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c7473948c136578706572696d656e745f6469725f6e616d65948c0f746573745f6578706572696d656e74948c0e747269616c5f6469725f6e616d65948c96747261696e5f6d6f64656c5f31646534363061385f31305f62617463685f73697a653d313032342c6465636f6465725f696e745f66663d313032342c6465636f6465725f696e745f6e756d5f6c617965723d382c6465636f6465725f72745f66663d3531322c6465636f6465725f72745f6e756d5f6c617965723d342c64726f705f5f323032342d30362d32355f31362d32302d3334948c1863757272656e745f636865636b706f696e745f696e646578944b018c0b73796e635f636f6e666967948c097261792e747261696e948c0a53796e63436f6e6669679493942981947d94288c0b73796e635f706572696f64944d2c018c0c73796e635f74696d656f7574944d08078c0e73796e635f61727469666163747394898c1c73796e635f6172746966616374735f6f6e5f636865636b706f696e7494888c0a75706c6f61645f646972948c0a44455052454341544544948c0673796e6365729468188c1273796e635f6f6e5f636865636b706f696e7494681875628c1273746f726167655f66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c0f73746f726167655f66735f70617468948c452f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f7465737494681968008c115f46696c6573797374656d53796e6365729493942981947d9428681b682268134d2c0168144d08078c116c6173745f73796e635f75705f74696d65944741d99eb463df15148c136c6173745f73796e635f646f776e5f74696d659447fff00000000000008c0d5f73796e635f70726f63657373944e8c0c5f63757272656e745f636d649468008c125f75706c6f61645f746f5f66735f706174689493947d94288c0a6c6f63616c5f70617468948c3b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c74732f746573745f6578706572696d656e74948c0266739468228c0766735f70617468948c552f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e74948c076578636c756465945d948c0e2a2f636865636b706f696e745f2a9461758694756275622e\"\n },\n \"config\": {\n \"encoder_num_layer\": 2,\n \"decoder_rt_num_layer\": 4,\n \"decoder_int_num_layer\": 8,\n \"embedding_dim\": 16,\n \"encoder_ff\": 2048,\n \"decoder_rt_ff\": 512,\n \"decoder_int_ff\": 1024,\n \"n_head\": 1,\n \"drop_rate\": 0.03175079062321118,\n \"lr\": 0.003436670599863372,\n \"batch_size\": 1024\n },\n \"_Trial__unresolved_config\": {\n \"encoder_num_layer\": 2,\n \"decoder_rt_num_layer\": 4,\n \"decoder_int_num_layer\": 8,\n \"embedding_dim\": 16,\n \"encoder_ff\": 2048,\n \"decoder_rt_ff\": 512,\n \"decoder_int_ff\": 1024,\n \"n_head\": 1,\n \"drop_rate\": 0.03175079062321118,\n \"lr\": 0.003436670599863372,\n \"batch_size\": 1024\n },\n \"evaluated_params\": {\n \"batch_size\": 1024,\n \"decoder_int_ff\": 1024,\n \"decoder_int_num_layer\": 8,\n \"decoder_rt_ff\": 512,\n \"decoder_rt_num_layer\": 4,\n \"drop_rate\": 0.03175079062321118,\n \"embedding_dim\": 16,\n \"encoder_ff\": 2048,\n \"encoder_num_layer\": 2,\n \"lr\": 0.003436670599863372,\n \"n_head\": 1\n },\n \"experiment_tag\": \"10_batch_size=1024,decoder_int_ff=1024,decoder_int_num_layer=8,decoder_rt_ff=512,decoder_rt_num_layer=4,drop_rate=0.0318,embedding_dim=16,encoder_ff=2048,encoder_num_layer=2,lr=0.0034,n_head=1\",\n \"stopping_criterion\": {},\n \"_setup_default_resource\": true,\n \"_default_placement_group_factory\": \"80054e2e\",\n \"placement_group_factory\": \"800595ba000000000000008c237261792e74756e652e657865637574696f6e2e706c6163656d656e745f67726f757073948c15506c6163656d656e7447726f7570466163746f72799493942981947d94288c085f62756e646c6573945d947d94288c03435055944740540000000000008c034750559447401000000000000075618c155f686561645f62756e646c655f69735f656d70747994898c095f7374726174656779948c045041434b948c055f6172677394298c075f6b7761726773947d9475622e\",\n \"log_to_file\": [\n null,\n null\n ],\n \"max_failures\": 0,\n \"_default_result_or_future\": null,\n \"export_formats\": [],\n \"status\": \"PAUSED\",\n \"relative_logdir\": \"train_model_1de460a8_10_batch_size=1024,decoder_int_ff=1024,decoder_int_num_layer=8,decoder_rt_ff=512,decoder_rt_num_layer=4,drop__2024-06-25_16-20-34\",\n \"trial_name_creator\": null,\n \"trial_dirname_creator\": null,\n \"custom_trial_name\": null,\n \"custom_dirname\": null,\n \"restore_path\": null,\n \"_restore_checkpoint_result\": null,\n \"_state_json\": null,\n \"results\": \"80054e2e\",\n \"extra_arg\": \"80054e2e\",\n \"_resources\": \"80054e2e\"\n}",
+ "{\n \"start_time\": 1719325314.396891,\n \"num_failures\": 0,\n \"num_failures_after_restore\": 0,\n \"error_filename\": null,\n \"pickled_error_filename\": null,\n \"last_result\": {\n \"loss\": 1955.3101080947035,\n \"timestamp\": 1719327763,\n \"checkpoint_dir_name\": \"checkpoint_000001\",\n \"should_checkpoint\": true,\n \"done\": false,\n \"training_iteration\": 2,\n \"trial_id\": \"1de460a8\",\n \"date\": \"2024-06-25_17-02-43\",\n \"time_this_iter_s\": 72.80412983894348,\n \"time_total_s\": 146.37021279335022,\n \"pid\": 130020,\n \"hostname\": \"r3i5n6\",\n \"node_ip\": \"10.159.8.159\",\n \"config\": {\n \"encoder_num_layer\": 2,\n \"decoder_rt_num_layer\": 4,\n \"decoder_int_num_layer\": 8,\n \"embedding_dim\": 16,\n \"encoder_ff\": 2048,\n \"decoder_rt_ff\": 512,\n \"decoder_int_ff\": 1024,\n \"n_head\": 1,\n \"drop_rate\": 0.03175079062321118,\n \"lr\": 0.003436670599863372,\n \"batch_size\": 1024\n },\n \"time_since_restore\": 72.80412983894348,\n \"iterations_since_restore\": 1,\n \"experiment_tag\": \"10_batch_size=1024,decoder_int_ff=1024,decoder_int_num_layer=8,decoder_rt_ff=512,decoder_rt_num_layer=4,drop_rate=0.0318,embedding_dim=16,encoder_ff=2048,encoder_num_layer=2,lr=0.0034,n_head=1\"\n },\n \"last_result_time\": 1719327763.0620155,\n \"metric_analysis\": {\n \"loss\": {\n \"max\": 1961.4144263079786,\n \"min\": 1955.3101080947035,\n \"avg\": 1958.3622672013412,\n \"last\": 1955.3101080947035,\n \"last-5-avg\": 1958.3622672013412,\n \"last-10-avg\": 1958.3622672013412\n },\n \"should_checkpoint\": {\n \"max\": true,\n \"min\": true,\n \"avg\": 1.0,\n \"last\": true,\n \"last-5-avg\": 1.0,\n \"last-10-avg\": 1.0\n },\n \"done\": {\n \"max\": false,\n \"min\": false,\n \"avg\": 0.0,\n \"last\": false,\n \"last-5-avg\": 0.0,\n \"last-10-avg\": 0.0\n },\n \"training_iteration\": {\n \"max\": 2,\n \"min\": 1,\n \"avg\": 1.5,\n \"last\": 2,\n \"last-5-avg\": 1.5,\n \"last-10-avg\": 1.5\n },\n \"time_this_iter_s\": {\n \"max\": 73.56608295440674,\n \"min\": 72.80412983894348,\n \"avg\": 73.18510639667511,\n \"last\": 72.80412983894348,\n \"last-5-avg\": 73.18510639667511,\n \"last-10-avg\": 73.18510639667511\n },\n \"time_total_s\": {\n \"max\": 146.37021279335022,\n \"min\": 73.56608295440674,\n \"avg\": 109.96814787387848,\n \"last\": 146.37021279335022,\n \"last-5-avg\": 109.96814787387848,\n \"last-10-avg\": 109.96814787387848\n },\n \"time_since_restore\": {\n \"max\": 73.56608295440674,\n \"min\": 72.80412983894348,\n \"avg\": 73.18510639667511,\n \"last\": 72.80412983894348,\n \"last-5-avg\": 73.18510639667511,\n \"last-10-avg\": 73.18510639667511\n },\n \"iterations_since_restore\": {\n \"max\": 1,\n \"min\": 1,\n \"avg\": 1.0,\n \"last\": 1,\n \"last-5-avg\": 1.0,\n \"last-10-avg\": 1.0\n }\n },\n \"_n_steps\": [\n 5,\n 10\n ],\n \"metric_n_steps\": {\n \"loss\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005959d000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294288c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b0074946243087bbd5e5fa8a59e4094869452946807680d4308e8f3f98c3d8d9e409486945294652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005959d000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294288c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b0074946243087bbd5e5fa8a59e4094869452946807680d4308e8f3f98c3d8d9e409486945294652e\"\n }\n },\n \"should_checkpoint\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059524000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294288888652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059524000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294288888652e\"\n }\n },\n \"done\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059524000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294288989652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059524000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294288989652e\"\n }\n },\n \"training_iteration\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059526000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294284b014b02652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059526000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294284b014b02652e\"\n }\n },\n \"time_this_iter_s\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059534000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b058694529428474052643ab40000004740523376dd000000652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059534000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a8694529428474052643ab40000004740523376dd000000652e\"\n }\n },\n \"time_total_s\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059534000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b058694529428474052643ab40000004740624bd8c8800000652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059534000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a8694529428474052643ab40000004740624bd8c8800000652e\"\n }\n },\n \"time_since_restore\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059534000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b058694529428474052643ab40000004740523376dd000000652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059534000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a8694529428474052643ab40000004740523376dd000000652e\"\n }\n },\n \"iterations_since_restore\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059526000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294284b014b01652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059526000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294284b014b01652e\"\n }\n }\n },\n \"checkpoint_manager\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"800595a5090000000000008c267261792e747261696e2e5f696e7465726e616c2e636865636b706f696e745f6d616e61676572948c125f436865636b706f696e744d616e616765729493942981947d94288c125f636865636b706f696e745f636f6e666967948c097261792e747261696e948c10436865636b706f696e74436f6e6669679493942981947d94288c0b6e756d5f746f5f6b656570944e8c1a636865636b706f696e745f73636f72655f617474726962757465944e8c16636865636b706f696e745f73636f72655f6f72646572948c036d6178948c14636865636b706f696e745f6672657175656e6379944b008c11636865636b706f696e745f61745f656e6494898c1a5f636865636b706f696e745f6b6565705f616c6c5f72616e6b73948c0a44455052454341544544948c1f5f636865636b706f696e745f75706c6f61645f66726f6d5f776f726b65727394681275628c135f636865636b706f696e745f726573756c7473945d94288c1b7261792e747261696e2e5f696e7465726e616c2e73657373696f6e948c0f5f547261696e696e67526573756c749493942981947d94288c0a636865636b706f696e749468068c0a436865636b706f696e749493942981947d94288c0470617468948cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f31646534363061385f31305f62617463685f73697a653d313032342c6465636f6465725f696e745f66663d313032342c6465636f6465725f696e745f6e756d5f6c617965723d382c6465636f6465725f72745f66663d3531322c6465636f6465725f72745f6e756d5f6c617965723d342c64726f705f5f323032342d30362d32355f31362d32302d33342f636865636b706f696e745f303030303030948c0a66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c055f75756964948c0475756964948c04555549449493942981947d948c03696e74948a1055810ffc8fb1d7ab1141f27ccac18639736275628c076d657472696373947d94288c046c6f7373948c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b0074946243087bbd5e5fa8a59e4094869452948c0974696d657374616d70944acbd27a668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303030948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b018c08747269616c5f6964948c083164653436306138948c0464617465948c13323032342d30362d32355f31362d32332d3037948c1074696d655f746869735f697465725f7394474052643ab40000008c0c74696d655f746f74616c5f7394474052643ab40000008c03706964944ac60e01008c08686f73746e616d65948c06723369356e36948c076e6f64655f6970948c0c31302e3135392e382e313539948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b028c146465636f6465725f72745f6e756d5f6c61796572944b048c156465636f6465725f696e745f6e756d5f6c61796572944b088c0d656d62656464696e675f64696d944b108c0a656e636f6465725f6666944d00088c0d6465636f6465725f72745f6666944d00028c0e6465636f6465725f696e745f6666944d00048c066e5f68656164944b018c0964726f705f7261746594473fa041a3beb29a908c026c7294473f6c27387aac57a48c0a62617463685f73697a65944d0004758c1274696d655f73696e63655f726573746f726594474052643ab40000008c18697465726174696f6e735f73696e63655f726573746f7265944b0175756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f31646534363061385f31305f62617463685f73697a653d313032342c6465636f6465725f696e745f66663d313032342c6465636f6465725f696e745f6e756d5f6c617965723d382c6465636f6465725f72745f66663d3531322c6465636f6465725f72745f6e756d5f6c617965723d342c64726f705f5f323032342d30362d32355f31362d32302d33342f636865636b706f696e745f30303030303194682268257d946827897385945294682a682d2981947d9468308a11b8b96603598218a15144e463ba24cf8e007362756268317d94288c046c6f7373946836683c4308e8f3f98c3d8d9e4094869452948c0974696d657374616d70944a13dc7a668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303031948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b028c08747269616c5f6964948c083164653436306138948c0464617465948c13323032342d30362d32355f31372d30322d3433948c1074696d655f746869735f697465725f73944740523376dd0000008c0c74696d655f746f74616c5f73944740624bd8c88000008c03706964944ae4fb01008c08686f73746e616d65948c06723369356e36948c076e6f64655f6970948c0c31302e3135392e382e313539948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b028c146465636f6465725f72745f6e756d5f6c61796572944b048c156465636f6465725f696e745f6e756d5f6c61796572944b088c0d656d62656464696e675f64696d944b108c0a656e636f6465725f6666944d00088c0d6465636f6465725f72745f6666944d00028c0e6465636f6465725f696e745f6666944d00048c066e5f68656164944b018c0964726f705f7261746594473fa041a3beb29a908c026c7294473f6c27387aac57a48c0a62617463685f73697a65944d0004758c1274696d655f73696e63655f726573746f7265944740523376dd0000008c18697465726174696f6e735f73696e63655f726573746f7265944b01757562658c195f6c61746573745f636865636b706f696e745f726573756c7494686275622e\"\n }\n}"
+ ],
+ [
+ "{\n \"stub\": false,\n \"trainable_name\": \"train_model\",\n \"trial_id\": \"1ef7c88b\",\n \"storage\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059572040000000000008c1b7261792e747261696e2e5f696e7465726e616c2e73746f72616765948c0e53746f72616765436f6e746578749493942981947d94288c12637573746f6d5f66735f70726f766964656494898c1273746f726167655f6c6f63616c5f70617468948c2b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c7473948c136578706572696d656e745f6469725f6e616d65948c0f746573745f6578706572696d656e74948c0e747269616c5f6469725f6e616d65948c96747261696e5f6d6f64656c5f31656637633838625f31315f62617463685f73697a653d313032342c6465636f6465725f696e745f66663d313032342c6465636f6465725f696e745f6e756d5f6c617965723d342c6465636f6465725f72745f66663d3531322c6465636f6465725f72745f6e756d5f6c617965723d342c64726f705f5f323032342d30362d32355f31362d32312d3534948c1863757272656e745f636865636b706f696e745f696e646578944b008c0b73796e635f636f6e666967948c097261792e747261696e948c0a53796e63436f6e6669679493942981947d94288c0b73796e635f706572696f64944d2c018c0c73796e635f74696d656f7574944d08078c0e73796e635f61727469666163747394898c1c73796e635f6172746966616374735f6f6e5f636865636b706f696e7494888c0a75706c6f61645f646972948c0a44455052454341544544948c0673796e6365729468188c1273796e635f6f6e5f636865636b706f696e7494681875628c1273746f726167655f66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c0f73746f726167655f66735f70617468948c452f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f7465737494681968008c115f46696c6573797374656d53796e6365729493942981947d9428681b682268134d2c0168144d08078c116c6173745f73796e635f75705f74696d65944741d99eb463df15148c136c6173745f73796e635f646f776e5f74696d659447fff00000000000008c0d5f73796e635f70726f63657373944e8c0c5f63757272656e745f636d649468008c125f75706c6f61645f746f5f66735f706174689493947d94288c0a6c6f63616c5f70617468948c3b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c74732f746573745f6578706572696d656e74948c0266739468228c0766735f70617468948c552f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e74948c076578636c756465945d948c0e2a2f636865636b706f696e745f2a9461758694756275622e\"\n },\n \"config\": {\n \"encoder_num_layer\": 8,\n \"decoder_rt_num_layer\": 4,\n \"decoder_int_num_layer\": 4,\n \"embedding_dim\": 1024,\n \"encoder_ff\": 1024,\n \"decoder_rt_ff\": 512,\n \"decoder_int_ff\": 1024,\n \"n_head\": 2,\n \"drop_rate\": 0.14711494497891564,\n \"lr\": 0.00018307057253269556,\n \"batch_size\": 1024\n },\n \"_Trial__unresolved_config\": {\n \"encoder_num_layer\": 8,\n \"decoder_rt_num_layer\": 4,\n \"decoder_int_num_layer\": 4,\n \"embedding_dim\": 1024,\n \"encoder_ff\": 1024,\n \"decoder_rt_ff\": 512,\n \"decoder_int_ff\": 1024,\n \"n_head\": 2,\n \"drop_rate\": 0.14711494497891564,\n \"lr\": 0.00018307057253269556,\n \"batch_size\": 1024\n },\n \"evaluated_params\": {\n \"batch_size\": 1024,\n \"decoder_int_ff\": 1024,\n \"decoder_int_num_layer\": 4,\n \"decoder_rt_ff\": 512,\n \"decoder_rt_num_layer\": 4,\n \"drop_rate\": 0.14711494497891564,\n \"embedding_dim\": 1024,\n \"encoder_ff\": 1024,\n \"encoder_num_layer\": 8,\n \"lr\": 0.00018307057253269556,\n \"n_head\": 2\n },\n \"experiment_tag\": \"11_batch_size=1024,decoder_int_ff=1024,decoder_int_num_layer=4,decoder_rt_ff=512,decoder_rt_num_layer=4,drop_rate=0.1471,embedding_dim=1024,encoder_ff=1024,encoder_num_layer=8,lr=0.0002,n_head=2\",\n \"stopping_criterion\": {},\n \"_setup_default_resource\": true,\n \"_default_placement_group_factory\": \"80054e2e\",\n \"placement_group_factory\": \"800595ba000000000000008c237261792e74756e652e657865637574696f6e2e706c6163656d656e745f67726f757073948c15506c6163656d656e7447726f7570466163746f72799493942981947d94288c085f62756e646c6573945d947d94288c03435055944740540000000000008c034750559447401000000000000075618c155f686561645f62756e646c655f69735f656d70747994898c095f7374726174656779948c045041434b948c055f6172677394298c075f6b7761726773947d9475622e\",\n \"log_to_file\": [\n null,\n null\n ],\n \"max_failures\": 0,\n \"_default_result_or_future\": null,\n \"export_formats\": [],\n \"status\": \"ERROR\",\n \"relative_logdir\": \"train_model_1ef7c88b_11_batch_size=1024,decoder_int_ff=1024,decoder_int_num_layer=4,decoder_rt_ff=512,decoder_rt_num_layer=4,drop__2024-06-25_16-21-54\",\n \"trial_name_creator\": null,\n \"trial_dirname_creator\": null,\n \"custom_trial_name\": null,\n \"custom_dirname\": null,\n \"restore_path\": null,\n \"_restore_checkpoint_result\": null,\n \"_state_json\": null,\n \"results\": \"80054e2e\",\n \"extra_arg\": \"80054e2e\",\n \"_resources\": \"80054e2e\"\n}",
+ "{\n \"start_time\": 1719325388.1127121,\n \"num_failures\": 1,\n \"num_failures_after_restore\": 0,\n \"error_filename\": \"error.txt\",\n \"pickled_error_filename\": \"error.pkl\",\n \"last_result\": {\n \"loss\": 1963.8436942513533,\n \"timestamp\": 1719326402,\n \"checkpoint_dir_name\": \"checkpoint_000000\",\n \"should_checkpoint\": true,\n \"done\": false,\n \"training_iteration\": 1,\n \"trial_id\": \"1ef7c88b\",\n \"date\": \"2024-06-25_16-40-02\",\n \"time_this_iter_s\": 1014.7334928512573,\n \"time_total_s\": 1014.7334928512573,\n \"pid\": 69318,\n \"hostname\": \"r3i5n6\",\n \"node_ip\": \"10.159.8.159\",\n \"config\": {\n \"encoder_num_layer\": 8,\n \"decoder_rt_num_layer\": 4,\n \"decoder_int_num_layer\": 4,\n \"embedding_dim\": 1024,\n \"encoder_ff\": 1024,\n \"decoder_rt_ff\": 512,\n \"decoder_int_ff\": 1024,\n \"n_head\": 2,\n \"drop_rate\": 0.14711494497891564,\n \"lr\": 0.00018307057253269556,\n \"batch_size\": 1024\n },\n \"time_since_restore\": 1014.7334928512573,\n \"iterations_since_restore\": 1\n },\n \"last_result_time\": 1719326402.8500564,\n \"metric_analysis\": {\n \"loss\": {\n \"max\": 1963.8436942513533,\n \"min\": 1963.8436942513533,\n \"avg\": 1963.8436942513533,\n \"last\": 1963.8436942513533,\n \"last-5-avg\": 1963.8436942513533,\n \"last-10-avg\": 1963.8436942513533\n },\n \"should_checkpoint\": {\n \"max\": true,\n \"min\": true,\n \"avg\": true,\n \"last\": true,\n \"last-5-avg\": true,\n \"last-10-avg\": true\n },\n \"done\": {\n \"max\": false,\n \"min\": false,\n \"avg\": false,\n \"last\": false,\n \"last-5-avg\": false,\n \"last-10-avg\": false\n },\n \"training_iteration\": {\n \"max\": 1,\n \"min\": 1,\n \"avg\": 1,\n \"last\": 1,\n \"last-5-avg\": 1,\n \"last-10-avg\": 1\n },\n \"time_this_iter_s\": {\n \"max\": 1014.7334928512573,\n \"min\": 1014.7334928512573,\n \"avg\": 1014.7334928512573,\n \"last\": 1014.7334928512573,\n \"last-5-avg\": 1014.7334928512573,\n \"last-10-avg\": 1014.7334928512573\n },\n \"time_total_s\": {\n \"max\": 1014.7334928512573,\n \"min\": 1014.7334928512573,\n \"avg\": 1014.7334928512573,\n \"last\": 1014.7334928512573,\n \"last-5-avg\": 1014.7334928512573,\n \"last-10-avg\": 1014.7334928512573\n },\n \"time_since_restore\": {\n \"max\": 1014.7334928512573,\n \"min\": 1014.7334928512573,\n \"avg\": 1014.7334928512573,\n \"last\": 1014.7334928512573,\n \"last-5-avg\": 1014.7334928512573,\n \"last-10-avg\": 1014.7334928512573\n },\n \"iterations_since_restore\": {\n \"max\": 1,\n \"min\": 1,\n \"avg\": 1,\n \"last\": 1,\n \"last-5-avg\": 1,\n \"last-10-avg\": 1\n }\n },\n \"_n_steps\": [\n 5,\n 10\n ],\n \"metric_n_steps\": {\n \"loss\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059589000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452948c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b0074946243088bc562f15faf9e409486945294612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059589000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452948c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b0074946243088bc562f15faf9e409486945294612e\"\n }\n },\n \"should_checkpoint\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059522000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b058694529488612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059522000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a8694529488612e\"\n }\n },\n \"done\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059522000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b058694529489612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059522000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a8694529489612e\"\n }\n },\n \"training_iteration\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059523000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452944b01612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059523000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452944b01612e\"\n }\n },\n \"time_this_iter_s\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b058694529447408fb5de31800000612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a8694529447408fb5de31800000612e\"\n }\n },\n \"time_total_s\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b058694529447408fb5de31800000612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a8694529447408fb5de31800000612e\"\n }\n },\n \"time_since_restore\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b058694529447408fb5de31800000612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a8694529447408fb5de31800000612e\"\n }\n },\n \"iterations_since_restore\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059523000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452944b01612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059523000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452944b01612e\"\n }\n }\n },\n \"checkpoint_manager\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059519060000000000008c267261792e747261696e2e5f696e7465726e616c2e636865636b706f696e745f6d616e61676572948c125f436865636b706f696e744d616e616765729493942981947d94288c125f636865636b706f696e745f636f6e666967948c097261792e747261696e948c10436865636b706f696e74436f6e6669679493942981947d94288c0b6e756d5f746f5f6b656570944e8c1a636865636b706f696e745f73636f72655f617474726962757465944e8c16636865636b706f696e745f73636f72655f6f72646572948c036d6178948c14636865636b706f696e745f6672657175656e6379944b008c11636865636b706f696e745f61745f656e6494898c1a5f636865636b706f696e745f6b6565705f616c6c5f72616e6b73948c0a44455052454341544544948c1f5f636865636b706f696e745f75706c6f61645f66726f6d5f776f726b65727394681275628c135f636865636b706f696e745f726573756c7473945d948c1b7261792e747261696e2e5f696e7465726e616c2e73657373696f6e948c0f5f547261696e696e67526573756c749493942981947d94288c0a636865636b706f696e749468068c0a436865636b706f696e749493942981947d94288c0470617468948cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f31656637633838625f31315f62617463685f73697a653d313032342c6465636f6465725f696e745f66663d313032342c6465636f6465725f696e745f6e756d5f6c617965723d342c6465636f6465725f72745f66663d3531322c6465636f6465725f72745f6e756d5f6c617965723d342c64726f705f5f323032342d30362d32355f31362d32312d35342f636865636b706f696e745f303030303030948c0a66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c055f75756964948c0475756964948c04555549449493942981947d948c03696e74948a117d30e4131f2c1ba2eb43a2955dff97f100736275628c076d657472696373947d94288c046c6f7373948c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b0074946243088bc562f15faf9e4094869452948c0974696d657374616d70944ac2d67a668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303030948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b018c08747269616c5f6964948c083165663763383862948c0464617465948c13323032342d30362d32355f31362d34302d3032948c1074696d655f746869735f697465725f739447408fb5de318000008c0c74696d655f746f74616c5f739447408fb5de318000008c03706964944ac60e01008c08686f73746e616d65948c06723369356e36948c076e6f64655f6970948c0c31302e3135392e382e313539948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b088c146465636f6465725f72745f6e756d5f6c61796572944b048c156465636f6465725f696e745f6e756d5f6c61796572944b048c0d656d62656464696e675f64696d944d00048c0a656e636f6465725f6666944d00048c0d6465636f6465725f72745f6666944d00028c0e6465636f6465725f696e745f6666944d00048c066e5f68656164944b028c0964726f705f7261746594473fc2d4a99ab7f8dc8c026c7294473f27fed43e6818098c0a62617463685f73697a65944d0004758c1274696d655f73696e63655f726573746f72659447408fb5de318000008c18697465726174696f6e735f73696e63655f726573746f7265944b01757562618c195f6c61746573745f636865636b706f696e745f726573756c7494681975622e\"\n }\n}"
+ ],
+ [
+ "{\n \"stub\": false,\n \"trainable_name\": \"train_model\",\n \"trial_id\": \"651349c4\",\n \"storage\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059572040000000000008c1b7261792e747261696e2e5f696e7465726e616c2e73746f72616765948c0e53746f72616765436f6e746578749493942981947d94288c12637573746f6d5f66735f70726f766964656494898c1273746f726167655f6c6f63616c5f70617468948c2b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c7473948c136578706572696d656e745f6469725f6e616d65948c0f746573745f6578706572696d656e74948c0e747269616c5f6469725f6e616d65948c96747261696e5f6d6f64656c5f36353133343963345f31325f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d323034382c6465636f6465725f696e745f6e756d5f6c617965723d382c6465636f6465725f72745f66663d323034382c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f323032342d30362d32355f31362d32332d3038948c1863757272656e745f636865636b706f696e745f696e646578944b008c0b73796e635f636f6e666967948c097261792e747261696e948c0a53796e63436f6e6669679493942981947d94288c0b73796e635f706572696f64944d2c018c0c73796e635f74696d656f7574944d08078c0e73796e635f61727469666163747394898c1c73796e635f6172746966616374735f6f6e5f636865636b706f696e7494888c0a75706c6f61645f646972948c0a44455052454341544544948c0673796e6365729468188c1273796e635f6f6e5f636865636b706f696e7494681875628c1273746f726167655f66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c0f73746f726167655f66735f70617468948c452f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f7465737494681968008c115f46696c6573797374656d53796e6365729493942981947d9428681b682268134d2c0168144d08078c116c6173745f73796e635f75705f74696d65944741d99eb4af484b348c136c6173745f73796e635f646f776e5f74696d659447fff00000000000008c0d5f73796e635f70726f63657373944e8c0c5f63757272656e745f636d649468008c125f75706c6f61645f746f5f66735f706174689493947d94288c0a6c6f63616c5f70617468948c3b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c74732f746573745f6578706572696d656e74948c0266739468228c0766735f70617468948c552f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e74948c076578636c756465945d948c0e2a2f636865636b706f696e745f2a9461758694756275622e\"\n },\n \"config\": {\n \"encoder_num_layer\": 8,\n \"decoder_rt_num_layer\": 1,\n \"decoder_int_num_layer\": 8,\n \"embedding_dim\": 16,\n \"encoder_ff\": 2048,\n \"decoder_rt_ff\": 2048,\n \"decoder_int_ff\": 2048,\n \"n_head\": 4,\n \"drop_rate\": 0.6683907000004579,\n \"lr\": 0.0001175178193869245,\n \"batch_size\": 2048\n },\n \"_Trial__unresolved_config\": {\n \"encoder_num_layer\": 8,\n \"decoder_rt_num_layer\": 1,\n \"decoder_int_num_layer\": 8,\n \"embedding_dim\": 16,\n \"encoder_ff\": 2048,\n \"decoder_rt_ff\": 2048,\n \"decoder_int_ff\": 2048,\n \"n_head\": 4,\n \"drop_rate\": 0.6683907000004579,\n \"lr\": 0.0001175178193869245,\n \"batch_size\": 2048\n },\n \"evaluated_params\": {\n \"batch_size\": 2048,\n \"decoder_int_ff\": 2048,\n \"decoder_int_num_layer\": 8,\n \"decoder_rt_ff\": 2048,\n \"decoder_rt_num_layer\": 1,\n \"drop_rate\": 0.6683907000004579,\n \"embedding_dim\": 16,\n \"encoder_ff\": 2048,\n \"encoder_num_layer\": 8,\n \"lr\": 0.0001175178193869245,\n \"n_head\": 4\n },\n \"experiment_tag\": \"12_batch_size=2048,decoder_int_ff=2048,decoder_int_num_layer=8,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop_rate=0.6684,embedding_dim=16,encoder_ff=2048,encoder_num_layer=8,lr=0.0001,n_head=4\",\n \"stopping_criterion\": {},\n \"_setup_default_resource\": true,\n \"_default_placement_group_factory\": \"80054e2e\",\n \"placement_group_factory\": \"800595ba000000000000008c237261792e74756e652e657865637574696f6e2e706c6163656d656e745f67726f757073948c15506c6163656d656e7447726f7570466163746f72799493942981947d94288c085f62756e646c6573945d947d94288c03435055944740540000000000008c034750559447401000000000000075618c155f686561645f62756e646c655f69735f656d70747994898c095f7374726174656779948c045041434b948c055f6172677394298c075f6b7761726773947d9475622e\",\n \"log_to_file\": [\n null,\n null\n ],\n \"max_failures\": 0,\n \"_default_result_or_future\": null,\n \"export_formats\": [],\n \"status\": \"PENDING\",\n \"relative_logdir\": \"train_model_651349c4_12_batch_size=2048,decoder_int_ff=2048,decoder_int_num_layer=8,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop_2024-06-25_16-23-08\",\n \"trial_name_creator\": null,\n \"trial_dirname_creator\": null,\n \"custom_trial_name\": null,\n \"custom_dirname\": null,\n \"restore_path\": null,\n \"_restore_checkpoint_result\": null,\n \"_state_json\": null,\n \"results\": \"80054e2e\",\n \"extra_arg\": \"80054e2e\",\n \"_resources\": \"80054e2e\"\n}",
+ "{\n \"start_time\": 1719326403.0031497,\n \"num_failures\": 0,\n \"num_failures_after_restore\": 0,\n \"error_filename\": null,\n \"pickled_error_filename\": null,\n \"last_result\": {\n \"loss\": 1981.618032380352,\n \"timestamp\": 1719326489,\n \"checkpoint_dir_name\": \"checkpoint_000000\",\n \"should_checkpoint\": true,\n \"done\": false,\n \"training_iteration\": 1,\n \"trial_id\": \"651349c4\",\n \"date\": \"2024-06-25_16-41-29\",\n \"time_this_iter_s\": 86.11733651161194,\n \"time_total_s\": 86.11733651161194,\n \"pid\": 69318,\n \"hostname\": \"r3i5n6\",\n \"node_ip\": \"10.159.8.159\",\n \"config\": {\n \"encoder_num_layer\": 8,\n \"decoder_rt_num_layer\": 1,\n \"decoder_int_num_layer\": 8,\n \"embedding_dim\": 16,\n \"encoder_ff\": 2048,\n \"decoder_rt_ff\": 2048,\n \"decoder_int_ff\": 2048,\n \"n_head\": 4,\n \"drop_rate\": 0.6683907000004579,\n \"lr\": 0.0001175178193869245,\n \"batch_size\": 2048\n },\n \"time_since_restore\": 86.11733651161194,\n \"iterations_since_restore\": 1,\n \"experiment_tag\": \"12_batch_size=2048,decoder_int_ff=2048,decoder_int_num_layer=8,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop_rate=0.6684,embedding_dim=16,encoder_ff=2048,encoder_num_layer=8,lr=0.0001,n_head=4\"\n },\n \"last_result_time\": 1719326489.1258004,\n \"metric_analysis\": {\n \"loss\": {\n \"max\": 1981.618032380352,\n \"min\": 1981.618032380352,\n \"avg\": 1981.618032380352,\n \"last\": 1981.618032380352,\n \"last-5-avg\": 1981.618032380352,\n \"last-10-avg\": 1981.618032380352\n },\n \"should_checkpoint\": {\n \"max\": true,\n \"min\": true,\n \"avg\": true,\n \"last\": true,\n \"last-5-avg\": true,\n \"last-10-avg\": true\n },\n \"done\": {\n \"max\": false,\n \"min\": false,\n \"avg\": false,\n \"last\": false,\n \"last-5-avg\": false,\n \"last-10-avg\": false\n },\n \"training_iteration\": {\n \"max\": 1,\n \"min\": 1,\n \"avg\": 1,\n \"last\": 1,\n \"last-5-avg\": 1,\n \"last-10-avg\": 1\n },\n \"time_this_iter_s\": {\n \"max\": 86.11733651161194,\n \"min\": 86.11733651161194,\n \"avg\": 86.11733651161194,\n \"last\": 86.11733651161194,\n \"last-5-avg\": 86.11733651161194,\n \"last-10-avg\": 86.11733651161194\n },\n \"time_total_s\": {\n \"max\": 86.11733651161194,\n \"min\": 86.11733651161194,\n \"avg\": 86.11733651161194,\n \"last\": 86.11733651161194,\n \"last-5-avg\": 86.11733651161194,\n \"last-10-avg\": 86.11733651161194\n },\n \"time_since_restore\": {\n \"max\": 86.11733651161194,\n \"min\": 86.11733651161194,\n \"avg\": 86.11733651161194,\n \"last\": 86.11733651161194,\n \"last-5-avg\": 86.11733651161194,\n \"last-10-avg\": 86.11733651161194\n },\n \"iterations_since_restore\": {\n \"max\": 1,\n \"min\": 1,\n \"avg\": 1,\n \"last\": 1,\n \"last-5-avg\": 1,\n \"last-10-avg\": 1\n }\n },\n \"_n_steps\": [\n 5,\n 10\n ],\n \"metric_n_steps\": {\n \"loss\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059589000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452948c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b007494624308ecf57add78f69e409486945294612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059589000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452948c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b007494624308ecf57add78f69e409486945294612e\"\n }\n },\n \"should_checkpoint\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059522000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b058694529488612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059522000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a8694529488612e\"\n }\n },\n \"done\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059522000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b058694529489612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059522000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a8694529489612e\"\n }\n },\n \"training_iteration\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059523000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452944b01612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059523000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452944b01612e\"\n }\n },\n \"time_this_iter_s\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294474055878271000000612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294474055878271000000612e\"\n }\n },\n \"time_total_s\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294474055878271000000612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294474055878271000000612e\"\n }\n },\n \"time_since_restore\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294474055878271000000612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294474055878271000000612e\"\n }\n },\n \"iterations_since_restore\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059523000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452944b01612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059523000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452944b01612e\"\n }\n }\n },\n \"checkpoint_manager\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059517060000000000008c267261792e747261696e2e5f696e7465726e616c2e636865636b706f696e745f6d616e61676572948c125f436865636b706f696e744d616e616765729493942981947d94288c125f636865636b706f696e745f636f6e666967948c097261792e747261696e948c10436865636b706f696e74436f6e6669679493942981947d94288c0b6e756d5f746f5f6b656570944e8c1a636865636b706f696e745f73636f72655f617474726962757465944e8c16636865636b706f696e745f73636f72655f6f72646572948c036d6178948c14636865636b706f696e745f6672657175656e6379944b008c11636865636b706f696e745f61745f656e6494898c1a5f636865636b706f696e745f6b6565705f616c6c5f72616e6b73948c0a44455052454341544544948c1f5f636865636b706f696e745f75706c6f61645f66726f6d5f776f726b65727394681275628c135f636865636b706f696e745f726573756c7473945d948c1b7261792e747261696e2e5f696e7465726e616c2e73657373696f6e948c0f5f547261696e696e67526573756c749493942981947d94288c0a636865636b706f696e749468068c0a436865636b706f696e749493942981947d94288c0470617468948cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f36353133343963345f31325f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d323034382c6465636f6465725f696e745f6e756d5f6c617965723d382c6465636f6465725f72745f66663d323034382c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f323032342d30362d32355f31362d32332d30382f636865636b706f696e745f303030303030948c0a66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c055f75756964948c0475756964948c04555549449493942981947d948c03696e74948a103cd44f9d38a2e7bec446dcaeaf332b13736275628c076d657472696373947d94288c046c6f7373948c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b007494624308ecf57add78f69e4094869452948c0974696d657374616d70944a19d77a668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303030948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b018c08747269616c5f6964948c083635313334396334948c0464617465948c13323032342d30362d32355f31362d34312d3239948c1074696d655f746869735f697465725f73944740558782710000008c0c74696d655f746f74616c5f73944740558782710000008c03706964944ac60e01008c08686f73746e616d65948c06723369356e36948c076e6f64655f6970948c0c31302e3135392e382e313539948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b088c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b088c0d656d62656464696e675f64696d944b108c0a656e636f6465725f6666944d00088c0d6465636f6465725f72745f6666944d00088c0e6465636f6465725f696e745f6666944d00088c066e5f68656164944b048c0964726f705f7261746594473fe56374e4ae7afe8c026c7294473f1ece7cc38c162c8c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f7265944740558782710000008c18697465726174696f6e735f73696e63655f726573746f7265944b01757562618c195f6c61746573745f636865636b706f696e745f726573756c7494681975622e\"\n }\n}"
+ ],
+ [
+ "{\n \"stub\": false,\n \"trainable_name\": \"train_model\",\n \"trial_id\": \"e566e32a\",\n \"storage\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059572040000000000008c1b7261792e747261696e2e5f696e7465726e616c2e73746f72616765948c0e53746f72616765436f6e746578749493942981947d94288c12637573746f6d5f66735f70726f766964656494898c1273746f726167655f6c6f63616c5f70617468948c2b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c7473948c136578706572696d656e745f6469725f6e616d65948c0f746573745f6578706572696d656e74948c0e747269616c5f6469725f6e616d65948c96747261696e5f6d6f64656c5f65353636653332615f31335f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d323034382c6465636f6465725f72745f6e756d5f6c617965723d322c64726f705f5f323032342d30362d32355f31362d34302d3033948c1863757272656e745f636865636b706f696e745f696e646578944b008c0b73796e635f636f6e666967948c097261792e747261696e948c0a53796e63436f6e6669679493942981947d94288c0b73796e635f706572696f64944d2c018c0c73796e635f74696d656f7574944d08078c0e73796e635f61727469666163747394898c1c73796e635f6172746966616374735f6f6e5f636865636b706f696e7494888c0a75706c6f61645f646972948c0a44455052454341544544948c0673796e6365729468188c1273796e635f6f6e5f636865636b706f696e7494681875628c1273746f726167655f66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c0f73746f726167655f66735f70617468948c452f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f7465737494681968008c115f46696c6573797374656d53796e6365729493942981947d9428681b682268134d2c0168144d08078c116c6173745f73796e635f75705f74696d65944741d99eb5915a973d8c136c6173745f73796e635f646f776e5f74696d659447fff00000000000008c0d5f73796e635f70726f63657373944e8c0c5f63757272656e745f636d649468008c125f75706c6f61645f746f5f66735f706174689493947d94288c0a6c6f63616c5f70617468948c3b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c74732f746573745f6578706572696d656e74948c0266739468228c0766735f70617468948c552f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e74948c076578636c756465945d948c0e2a2f636865636b706f696e745f2a9461758694756275622e\"\n },\n \"config\": {\n \"encoder_num_layer\": 8,\n \"decoder_rt_num_layer\": 2,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 16,\n \"encoder_ff\": 1024,\n \"decoder_rt_ff\": 2048,\n \"decoder_int_ff\": 512,\n \"n_head\": 4,\n \"drop_rate\": 0.7731110479994988,\n \"lr\": 0.03425650698356203,\n \"batch_size\": 2048\n },\n \"_Trial__unresolved_config\": {\n \"encoder_num_layer\": 8,\n \"decoder_rt_num_layer\": 2,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 16,\n \"encoder_ff\": 1024,\n \"decoder_rt_ff\": 2048,\n \"decoder_int_ff\": 512,\n \"n_head\": 4,\n \"drop_rate\": 0.7731110479994988,\n \"lr\": 0.03425650698356203,\n \"batch_size\": 2048\n },\n \"evaluated_params\": {\n \"batch_size\": 2048,\n \"decoder_int_ff\": 512,\n \"decoder_int_num_layer\": 1,\n \"decoder_rt_ff\": 2048,\n \"decoder_rt_num_layer\": 2,\n \"drop_rate\": 0.7731110479994988,\n \"embedding_dim\": 16,\n \"encoder_ff\": 1024,\n \"encoder_num_layer\": 8,\n \"lr\": 0.03425650698356203,\n \"n_head\": 4\n },\n \"experiment_tag\": \"13_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=2,drop_rate=0.7731,embedding_dim=16,encoder_ff=1024,encoder_num_layer=8,lr=0.0343,n_head=4\",\n \"stopping_criterion\": {},\n \"_setup_default_resource\": true,\n \"_default_placement_group_factory\": \"80054e2e\",\n \"placement_group_factory\": \"800595ba000000000000008c237261792e74756e652e657865637574696f6e2e706c6163656d656e745f67726f757073948c15506c6163656d656e7447726f7570466163746f72799493942981947d94288c085f62756e646c6573945d947d94288c03435055944740540000000000008c034750559447401000000000000075618c155f686561645f62756e646c655f69735f656d70747994898c095f7374726174656779948c045041434b948c055f6172677394298c075f6b7761726773947d9475622e\",\n \"log_to_file\": [\n null,\n null\n ],\n \"max_failures\": 0,\n \"_default_result_or_future\": null,\n \"export_formats\": [],\n \"status\": \"PAUSED\",\n \"relative_logdir\": \"train_model_e566e32a_13_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=2,drop__2024-06-25_16-40-03\",\n \"trial_name_creator\": null,\n \"trial_dirname_creator\": null,\n \"custom_trial_name\": null,\n \"custom_dirname\": null,\n \"restore_path\": null,\n \"_restore_checkpoint_result\": null,\n \"_state_json\": null,\n \"results\": \"80054e2e\",\n \"extra_arg\": \"80054e2e\",\n \"_resources\": \"80054e2e\"\n}",
+ "{\n \"start_time\": 1719326489.2697742,\n \"num_failures\": 0,\n \"num_failures_after_restore\": 0,\n \"error_filename\": null,\n \"pickled_error_filename\": null,\n \"last_result\": {\n \"loss\": 2129.4940118264026,\n \"timestamp\": 1719326571,\n \"checkpoint_dir_name\": \"checkpoint_000000\",\n \"should_checkpoint\": true,\n \"done\": false,\n \"training_iteration\": 1,\n \"trial_id\": \"e566e32a\",\n \"date\": \"2024-06-25_16-42-51\",\n \"time_this_iter_s\": 82.61908912658691,\n \"time_total_s\": 82.61908912658691,\n \"pid\": 69318,\n \"hostname\": \"r3i5n6\",\n \"node_ip\": \"10.159.8.159\",\n \"config\": {\n \"encoder_num_layer\": 8,\n \"decoder_rt_num_layer\": 2,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 16,\n \"encoder_ff\": 1024,\n \"decoder_rt_ff\": 2048,\n \"decoder_int_ff\": 512,\n \"n_head\": 4,\n \"drop_rate\": 0.7731110479994988,\n \"lr\": 0.03425650698356203,\n \"batch_size\": 2048\n },\n \"time_since_restore\": 82.61908912658691,\n \"iterations_since_restore\": 1,\n \"experiment_tag\": \"13_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=2,drop_rate=0.7731,embedding_dim=16,encoder_ff=1024,encoder_num_layer=8,lr=0.0343,n_head=4\"\n },\n \"last_result_time\": 1719326571.8931763,\n \"metric_analysis\": {\n \"loss\": {\n \"max\": 2129.4940118264026,\n \"min\": 2129.4940118264026,\n \"avg\": 2129.4940118264026,\n \"last\": 2129.4940118264026,\n \"last-5-avg\": 2129.4940118264026,\n \"last-10-avg\": 2129.4940118264026\n },\n \"should_checkpoint\": {\n \"max\": true,\n \"min\": true,\n \"avg\": true,\n \"last\": true,\n \"last-5-avg\": true,\n \"last-10-avg\": true\n },\n \"done\": {\n \"max\": false,\n \"min\": false,\n \"avg\": false,\n \"last\": false,\n \"last-5-avg\": false,\n \"last-10-avg\": false\n },\n \"training_iteration\": {\n \"max\": 1,\n \"min\": 1,\n \"avg\": 1,\n \"last\": 1,\n \"last-5-avg\": 1,\n \"last-10-avg\": 1\n },\n \"time_this_iter_s\": {\n \"max\": 82.61908912658691,\n \"min\": 82.61908912658691,\n \"avg\": 82.61908912658691,\n \"last\": 82.61908912658691,\n \"last-5-avg\": 82.61908912658691,\n \"last-10-avg\": 82.61908912658691\n },\n \"time_total_s\": {\n \"max\": 82.61908912658691,\n \"min\": 82.61908912658691,\n \"avg\": 82.61908912658691,\n \"last\": 82.61908912658691,\n \"last-5-avg\": 82.61908912658691,\n \"last-10-avg\": 82.61908912658691\n },\n \"time_since_restore\": {\n \"max\": 82.61908912658691,\n \"min\": 82.61908912658691,\n \"avg\": 82.61908912658691,\n \"last\": 82.61908912658691,\n \"last-5-avg\": 82.61908912658691,\n \"last-10-avg\": 82.61908912658691\n },\n \"iterations_since_restore\": {\n \"max\": 1,\n \"min\": 1,\n \"avg\": 1,\n \"last\": 1,\n \"last-5-avg\": 1,\n \"last-10-avg\": 1\n }\n },\n \"_n_steps\": [\n 5,\n 10\n ],\n \"metric_n_steps\": {\n \"loss\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059589000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452948c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b007494624308793c1eeffca2a0409486945294612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059589000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452948c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b007494624308793c1eeffca2a0409486945294612e\"\n }\n },\n \"should_checkpoint\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059522000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b058694529488612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059522000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a8694529488612e\"\n }\n },\n \"done\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059522000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b058694529489612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059522000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a8694529489612e\"\n }\n },\n \"training_iteration\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059523000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452944b01612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059523000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452944b01612e\"\n }\n },\n \"time_this_iter_s\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294474054a79f28000000612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294474054a79f28000000612e\"\n }\n },\n \"time_total_s\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294474054a79f28000000612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294474054a79f28000000612e\"\n }\n },\n \"time_since_restore\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294474054a79f28000000612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294474054a79f28000000612e\"\n }\n },\n \"iterations_since_restore\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059523000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452944b01612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059523000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452944b01612e\"\n }\n }\n },\n \"checkpoint_manager\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059517060000000000008c267261792e747261696e2e5f696e7465726e616c2e636865636b706f696e745f6d616e61676572948c125f436865636b706f696e744d616e616765729493942981947d94288c125f636865636b706f696e745f636f6e666967948c097261792e747261696e948c10436865636b706f696e74436f6e6669679493942981947d94288c0b6e756d5f746f5f6b656570944e8c1a636865636b706f696e745f73636f72655f617474726962757465944e8c16636865636b706f696e745f73636f72655f6f72646572948c036d6178948c14636865636b706f696e745f6672657175656e6379944b008c11636865636b706f696e745f61745f656e6494898c1a5f636865636b706f696e745f6b6565705f616c6c5f72616e6b73948c0a44455052454341544544948c1f5f636865636b706f696e745f75706c6f61645f66726f6d5f776f726b65727394681275628c135f636865636b706f696e745f726573756c7473945d948c1b7261792e747261696e2e5f696e7465726e616c2e73657373696f6e948c0f5f547261696e696e67526573756c749493942981947d94288c0a636865636b706f696e749468068c0a436865636b706f696e749493942981947d94288c0470617468948cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f65353636653332615f31335f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d323034382c6465636f6465725f72745f6e756d5f6c617965723d322c64726f705f5f323032342d30362d32355f31362d34302d30332f636865636b706f696e745f303030303030948c0a66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c055f75756964948c0475756964948c04555549449493942981947d948c03696e74948a108eef8c58375f918166442f09d4e1302f736275628c076d657472696373947d94288c046c6f7373948c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b007494624308793c1eeffca2a04094869452948c0974696d657374616d70944a6bd77a668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303030948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b018c08747269616c5f6964948c086535363665333261948c0464617465948c13323032342d30362d32355f31362d34322d3531948c1074696d655f746869735f697465725f7394474054a79f280000008c0c74696d655f746f74616c5f7394474054a79f280000008c03706964944ac60e01008c08686f73746e616d65948c06723369356e36948c076e6f64655f6970948c0c31302e3135392e382e313539948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b088c146465636f6465725f72745f6e756d5f6c61796572944b028c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b108c0a656e636f6465725f6666944d00048c0d6465636f6465725f72745f6666944d00088c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b048c0964726f705f7261746594473fe8bd53616ab1398c026c7294473fa18a11a256d5188c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f726594474054a79f280000008c18697465726174696f6e735f73696e63655f726573746f7265944b01757562618c195f6c61746573745f636865636b706f696e745f726573756c7494681975622e\"\n }\n}"
+ ],
+ [
+ "{\n \"stub\": false,\n \"trainable_name\": \"train_model\",\n \"trial_id\": \"8c6e7040\",\n \"storage\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059575040000000000008c1b7261792e747261696e2e5f696e7465726e616c2e73746f72616765948c0e53746f72616765436f6e746578749493942981947d94288c12637573746f6d5f66735f70726f766964656494898c1273746f726167655f6c6f63616c5f70617468948c2b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c7473948c136578706572696d656e745f6469725f6e616d65948c0f746573745f6578706572696d656e74948c0e747269616c5f6469725f6e616d65948c96747261696e5f6d6f64656c5f38633665373034305f31345f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d323034382c6465636f6465725f696e745f6e756d5f6c617965723d382c6465636f6465725f72745f66663d313032342c6465636f6465725f72745f6e756d5f6c617965723d382c64726f705f323032342d30362d32355f31362d34312d3239948c1863757272656e745f636865636b706f696e745f696e646578944affffffff8c0b73796e635f636f6e666967948c097261792e747261696e948c0a53796e63436f6e6669679493942981947d94288c0b73796e635f706572696f64944d2c018c0c73796e635f74696d656f7574944d08078c0e73796e635f61727469666163747394898c1c73796e635f6172746966616374735f6f6e5f636865636b706f696e7494888c0a75706c6f61645f646972948c0a44455052454341544544948c0673796e6365729468188c1273796e635f6f6e5f636865636b706f696e7494681875628c1273746f726167655f66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c0f73746f726167655f66735f70617468948c452f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f7465737494681968008c115f46696c6573797374656d53796e6365729493942981947d9428681b682268134d2c0168144d08078c116c6173745f73796e635f75705f74696d65944741d99eb5915a973d8c136c6173745f73796e635f646f776e5f74696d659447fff00000000000008c0d5f73796e635f70726f63657373944e8c0c5f63757272656e745f636d649468008c125f75706c6f61645f746f5f66735f706174689493947d94288c0a6c6f63616c5f70617468948c3b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c74732f746573745f6578706572696d656e74948c0266739468228c0766735f70617468948c552f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e74948c076578636c756465945d948c0e2a2f636865636b706f696e745f2a9461758694756275622e\"\n },\n \"config\": {\n \"encoder_num_layer\": 8,\n \"decoder_rt_num_layer\": 8,\n \"decoder_int_num_layer\": 8,\n \"embedding_dim\": 256,\n \"encoder_ff\": 2048,\n \"decoder_rt_ff\": 1024,\n \"decoder_int_ff\": 2048,\n \"n_head\": 1,\n \"drop_rate\": 0.6537122299874158,\n \"lr\": 0.010515037912087563,\n \"batch_size\": 2048\n },\n \"_Trial__unresolved_config\": {\n \"encoder_num_layer\": 8,\n \"decoder_rt_num_layer\": 8,\n \"decoder_int_num_layer\": 8,\n \"embedding_dim\": 256,\n \"encoder_ff\": 2048,\n \"decoder_rt_ff\": 1024,\n \"decoder_int_ff\": 2048,\n \"n_head\": 1,\n \"drop_rate\": 0.6537122299874158,\n \"lr\": 0.010515037912087563,\n \"batch_size\": 2048\n },\n \"evaluated_params\": {\n \"batch_size\": 2048,\n \"decoder_int_ff\": 2048,\n \"decoder_int_num_layer\": 8,\n \"decoder_rt_ff\": 1024,\n \"decoder_rt_num_layer\": 8,\n \"drop_rate\": 0.6537122299874158,\n \"embedding_dim\": 256,\n \"encoder_ff\": 2048,\n \"encoder_num_layer\": 8,\n \"lr\": 0.010515037912087563,\n \"n_head\": 1\n },\n \"experiment_tag\": \"14_batch_size=2048,decoder_int_ff=2048,decoder_int_num_layer=8,decoder_rt_ff=1024,decoder_rt_num_layer=8,drop_rate=0.6537,embedding_dim=256,encoder_ff=2048,encoder_num_layer=8,lr=0.0105,n_head=1\",\n \"stopping_criterion\": {},\n \"_setup_default_resource\": true,\n \"_default_placement_group_factory\": \"80054e2e\",\n \"placement_group_factory\": \"800595ba000000000000008c237261792e74756e652e657865637574696f6e2e706c6163656d656e745f67726f757073948c15506c6163656d656e7447726f7570466163746f72799493942981947d94288c085f62756e646c6573945d947d94288c03435055944740540000000000008c034750559447401000000000000075618c155f686561645f62756e646c655f69735f656d70747994898c095f7374726174656779948c045041434b948c055f6172677394298c075f6b7761726773947d9475622e\",\n \"log_to_file\": [\n null,\n null\n ],\n \"max_failures\": 0,\n \"_default_result_or_future\": null,\n \"export_formats\": [],\n \"status\": \"ERROR\",\n \"relative_logdir\": \"train_model_8c6e7040_14_batch_size=2048,decoder_int_ff=2048,decoder_int_num_layer=8,decoder_rt_ff=1024,decoder_rt_num_layer=8,drop_2024-06-25_16-41-29\",\n \"trial_name_creator\": null,\n \"trial_dirname_creator\": null,\n \"custom_trial_name\": null,\n \"custom_dirname\": null,\n \"restore_path\": null,\n \"_restore_checkpoint_result\": null,\n \"_state_json\": null,\n \"results\": \"80054e2e\",\n \"extra_arg\": \"80054e2e\",\n \"_resources\": \"80054e2e\"\n}",
+ "{\n \"start_time\": 1719326572.0475543,\n \"num_failures\": 1,\n \"num_failures_after_restore\": 0,\n \"error_filename\": \"error.txt\",\n \"pickled_error_filename\": \"error.pkl\",\n \"last_result\": {\n \"trial_id\": \"8c6e7040\"\n },\n \"last_result_time\": -Infinity,\n \"metric_analysis\": {},\n \"_n_steps\": [\n 5,\n 10\n ],\n \"metric_n_steps\": {},\n \"checkpoint_manager\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059584010000000000008c267261792e747261696e2e5f696e7465726e616c2e636865636b706f696e745f6d616e61676572948c125f436865636b706f696e744d616e616765729493942981947d94288c125f636865636b706f696e745f636f6e666967948c097261792e747261696e948c10436865636b706f696e74436f6e6669679493942981947d94288c0b6e756d5f746f5f6b656570944e8c1a636865636b706f696e745f73636f72655f617474726962757465944e8c16636865636b706f696e745f73636f72655f6f72646572948c036d6178948c14636865636b706f696e745f6672657175656e6379944b008c11636865636b706f696e745f61745f656e6494898c1a5f636865636b706f696e745f6b6565705f616c6c5f72616e6b73948c0a44455052454341544544948c1f5f636865636b706f696e745f75706c6f61645f66726f6d5f776f726b65727394681275628c135f636865636b706f696e745f726573756c7473945d948c195f6c61746573745f636865636b706f696e745f726573756c74944e75622e\"\n }\n}"
+ ],
+ [
+ "{\n \"stub\": false,\n \"trainable_name\": \"train_model\",\n \"trial_id\": \"25b42194\",\n \"storage\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059572040000000000008c1b7261792e747261696e2e5f696e7465726e616c2e73746f72616765948c0e53746f72616765436f6e746578749493942981947d94288c12637573746f6d5f66735f70726f766964656494898c1273746f726167655f6c6f63616c5f70617468948c2b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c7473948c136578706572696d656e745f6469725f6e616d65948c0f746573745f6578706572696d656e74948c0e747269616c5f6469725f6e616d65948c96747261696e5f6d6f64656c5f32356234323139345f31355f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d323034382c6465636f6465725f696e745f6e756d5f6c617965723d382c6465636f6465725f72745f66663d313032342c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f323032342d30362d32355f31362d34322d3532948c1863757272656e745f636865636b706f696e745f696e646578944b008c0b73796e635f636f6e666967948c097261792e747261696e948c0a53796e63436f6e6669679493942981947d94288c0b73796e635f706572696f64944d2c018c0c73796e635f74696d656f7574944d08078c0e73796e635f61727469666163747394898c1c73796e635f6172746966616374735f6f6e5f636865636b706f696e7494888c0a75706c6f61645f646972948c0a44455052454341544544948c0673796e6365729468188c1273796e635f6f6e5f636865636b706f696e7494681875628c1273746f726167655f66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c0f73746f726167655f66735f70617468948c452f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f7465737494681968008c115f46696c6573797374656d53796e6365729493942981947d9428681b682268134d2c0168144d08078c116c6173745f73796e635f75705f74696d65944741d99eb5915a973d8c136c6173745f73796e635f646f776e5f74696d659447fff00000000000008c0d5f73796e635f70726f63657373944e8c0c5f63757272656e745f636d649468008c125f75706c6f61645f746f5f66735f706174689493947d94288c0a6c6f63616c5f70617468948c3b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c74732f746573745f6578706572696d656e74948c0266739468228c0766735f70617468948c552f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e74948c076578636c756465945d948c0e2a2f636865636b706f696e745f2a9461758694756275622e\"\n },\n \"config\": {\n \"encoder_num_layer\": 1,\n \"decoder_rt_num_layer\": 1,\n \"decoder_int_num_layer\": 8,\n \"embedding_dim\": 1024,\n \"encoder_ff\": 512,\n \"decoder_rt_ff\": 1024,\n \"decoder_int_ff\": 2048,\n \"n_head\": 16,\n \"drop_rate\": 0.3256301932631026,\n \"lr\": 0.0018178629337167497,\n \"batch_size\": 2048\n },\n \"_Trial__unresolved_config\": {\n \"encoder_num_layer\": 1,\n \"decoder_rt_num_layer\": 1,\n \"decoder_int_num_layer\": 8,\n \"embedding_dim\": 1024,\n \"encoder_ff\": 512,\n \"decoder_rt_ff\": 1024,\n \"decoder_int_ff\": 2048,\n \"n_head\": 16,\n \"drop_rate\": 0.3256301932631026,\n \"lr\": 0.0018178629337167497,\n \"batch_size\": 2048\n },\n \"evaluated_params\": {\n \"batch_size\": 2048,\n \"decoder_int_ff\": 2048,\n \"decoder_int_num_layer\": 8,\n \"decoder_rt_ff\": 1024,\n \"decoder_rt_num_layer\": 1,\n \"drop_rate\": 0.3256301932631026,\n \"embedding_dim\": 1024,\n \"encoder_ff\": 512,\n \"encoder_num_layer\": 1,\n \"lr\": 0.0018178629337167497,\n \"n_head\": 16\n },\n \"experiment_tag\": \"15_batch_size=2048,decoder_int_ff=2048,decoder_int_num_layer=8,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop_rate=0.3256,embedding_dim=1024,encoder_ff=512,encoder_num_layer=1,lr=0.0018,n_head=16\",\n \"stopping_criterion\": {},\n \"_setup_default_resource\": true,\n \"_default_placement_group_factory\": \"80054e2e\",\n \"placement_group_factory\": \"800595ba000000000000008c237261792e74756e652e657865637574696f6e2e706c6163656d656e745f67726f757073948c15506c6163656d656e7447726f7570466163746f72799493942981947d94288c085f62756e646c6573945d947d94288c03435055944740540000000000008c034750559447401000000000000075618c155f686561645f62756e646c655f69735f656d70747994898c095f7374726174656779948c045041434b948c055f6172677394298c075f6b7761726773947d9475622e\",\n \"log_to_file\": [\n null,\n null\n ],\n \"max_failures\": 0,\n \"_default_result_or_future\": null,\n \"export_formats\": [],\n \"status\": \"PAUSED\",\n \"relative_logdir\": \"train_model_25b42194_15_batch_size=2048,decoder_int_ff=2048,decoder_int_num_layer=8,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop_2024-06-25_16-42-52\",\n \"trial_name_creator\": null,\n \"trial_dirname_creator\": null,\n \"custom_trial_name\": null,\n \"custom_dirname\": null,\n \"restore_path\": null,\n \"_restore_checkpoint_result\": null,\n \"_state_json\": null,\n \"results\": \"80054e2e\",\n \"extra_arg\": \"80054e2e\",\n \"_resources\": \"80054e2e\"\n}",
+ "{\n \"start_time\": 1719326598.1773813,\n \"num_failures\": 0,\n \"num_failures_after_restore\": 0,\n \"error_filename\": null,\n \"pickled_error_filename\": null,\n \"last_result\": {\n \"loss\": 5061.415592704232,\n \"timestamp\": 1719326792,\n \"checkpoint_dir_name\": \"checkpoint_000000\",\n \"should_checkpoint\": true,\n \"done\": false,\n \"training_iteration\": 1,\n \"trial_id\": \"25b42194\",\n \"date\": \"2024-06-25_16-46-32\",\n \"time_this_iter_s\": 194.36064219474792,\n \"time_total_s\": 194.36064219474792,\n \"pid\": 122466,\n \"hostname\": \"r3i5n6\",\n \"node_ip\": \"10.159.8.159\",\n \"config\": {\n \"encoder_num_layer\": 1,\n \"decoder_rt_num_layer\": 1,\n \"decoder_int_num_layer\": 8,\n \"embedding_dim\": 1024,\n \"encoder_ff\": 512,\n \"decoder_rt_ff\": 1024,\n \"decoder_int_ff\": 2048,\n \"n_head\": 16,\n \"drop_rate\": 0.3256301932631026,\n \"lr\": 0.0018178629337167497,\n \"batch_size\": 2048\n },\n \"time_since_restore\": 194.36064219474792,\n \"iterations_since_restore\": 1,\n \"experiment_tag\": \"15_batch_size=2048,decoder_int_ff=2048,decoder_int_num_layer=8,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop_rate=0.3256,embedding_dim=1024,encoder_ff=512,encoder_num_layer=1,lr=0.0018,n_head=16\"\n },\n \"last_result_time\": 1719326792.5431535,\n \"metric_analysis\": {\n \"loss\": {\n \"max\": 5061.415592704232,\n \"min\": 5061.415592704232,\n \"avg\": 5061.415592704232,\n \"last\": 5061.415592704232,\n \"last-5-avg\": 5061.415592704232,\n \"last-10-avg\": 5061.415592704232\n },\n \"should_checkpoint\": {\n \"max\": true,\n \"min\": true,\n \"avg\": true,\n \"last\": true,\n \"last-5-avg\": true,\n \"last-10-avg\": true\n },\n \"done\": {\n \"max\": false,\n \"min\": false,\n \"avg\": false,\n \"last\": false,\n \"last-5-avg\": false,\n \"last-10-avg\": false\n },\n \"training_iteration\": {\n \"max\": 1,\n \"min\": 1,\n \"avg\": 1,\n \"last\": 1,\n \"last-5-avg\": 1,\n \"last-10-avg\": 1\n },\n \"time_this_iter_s\": {\n \"max\": 194.36064219474792,\n \"min\": 194.36064219474792,\n \"avg\": 194.36064219474792,\n \"last\": 194.36064219474792,\n \"last-5-avg\": 194.36064219474792,\n \"last-10-avg\": 194.36064219474792\n },\n \"time_total_s\": {\n \"max\": 194.36064219474792,\n \"min\": 194.36064219474792,\n \"avg\": 194.36064219474792,\n \"last\": 194.36064219474792,\n \"last-5-avg\": 194.36064219474792,\n \"last-10-avg\": 194.36064219474792\n },\n \"time_since_restore\": {\n \"max\": 194.36064219474792,\n \"min\": 194.36064219474792,\n \"avg\": 194.36064219474792,\n \"last\": 194.36064219474792,\n \"last-5-avg\": 194.36064219474792,\n \"last-10-avg\": 194.36064219474792\n },\n \"iterations_since_restore\": {\n \"max\": 1,\n \"min\": 1,\n \"avg\": 1,\n \"last\": 1,\n \"last-5-avg\": 1,\n \"last-10-avg\": 1\n }\n },\n \"_n_steps\": [\n 5,\n 10\n ],\n \"metric_n_steps\": {\n \"loss\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059589000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452948c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b007494624308229148646ac5b3409486945294612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059589000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452948c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b007494624308229148646ac5b3409486945294612e\"\n }\n },\n \"should_checkpoint\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059522000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b058694529488612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059522000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a8694529488612e\"\n }\n },\n \"done\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059522000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b058694529489612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059522000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a8694529489612e\"\n }\n },\n \"training_iteration\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059523000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452944b01612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059523000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452944b01612e\"\n }\n },\n \"time_this_iter_s\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452944740684b8a61800000612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452944740684b8a61800000612e\"\n }\n },\n \"time_total_s\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452944740684b8a61800000612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452944740684b8a61800000612e\"\n }\n },\n \"time_since_restore\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452944740684b8a61800000612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452944740684b8a61800000612e\"\n }\n },\n \"iterations_since_restore\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059523000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452944b01612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059523000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452944b01612e\"\n }\n }\n },\n \"checkpoint_manager\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059519060000000000008c267261792e747261696e2e5f696e7465726e616c2e636865636b706f696e745f6d616e61676572948c125f436865636b706f696e744d616e616765729493942981947d94288c125f636865636b706f696e745f636f6e666967948c097261792e747261696e948c10436865636b706f696e74436f6e6669679493942981947d94288c0b6e756d5f746f5f6b656570944e8c1a636865636b706f696e745f73636f72655f617474726962757465944e8c16636865636b706f696e745f73636f72655f6f72646572948c036d6178948c14636865636b706f696e745f6672657175656e6379944b008c11636865636b706f696e745f61745f656e6494898c1a5f636865636b706f696e745f6b6565705f616c6c5f72616e6b73948c0a44455052454341544544948c1f5f636865636b706f696e745f75706c6f61645f66726f6d5f776f726b65727394681275628c135f636865636b706f696e745f726573756c7473945d948c1b7261792e747261696e2e5f696e7465726e616c2e73657373696f6e948c0f5f547261696e696e67526573756c749493942981947d94288c0a636865636b706f696e749468068c0a436865636b706f696e749493942981947d94288c0470617468948cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f32356234323139345f31355f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d323034382c6465636f6465725f696e745f6e756d5f6c617965723d382c6465636f6465725f72745f66663d313032342c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f323032342d30362d32355f31362d34322d35322f636865636b706f696e745f303030303030948c0a66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c055f75756964948c0475756964948c04555549449493942981947d948c03696e74948a1134c6fe3ca07c39bd6c4247e744cd77b300736275628c076d657472696373947d94288c046c6f7373948c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b007494624308229148646ac5b34094869452948c0974696d657374616d70944a48d87a668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303030948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b018c08747269616c5f6964948c083235623432313934948c0464617465948c13323032342d30362d32355f31362d34362d3332948c1074696d655f746869735f697465725f73944740684b8a618000008c0c74696d655f746f74616c5f73944740684b8a618000008c03706964944a62de01008c08686f73746e616d65948c06723369356e36948c076e6f64655f6970948c0c31302e3135392e382e313539948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b018c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b088c0d656d62656464696e675f64696d944d00048c0a656e636f6465725f6666944d00028c0d6465636f6465725f72745f6666944d00048c0e6465636f6465725f696e745f6666944d00088c066e5f68656164944b108c0964726f705f7261746594473fd4d72005a9ee8e8c026c7294473f5dc8ab7654c5908c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f7265944740684b8a618000008c18697465726174696f6e735f73696e63655f726573746f7265944b01757562618c195f6c61746573745f636865636b706f696e745f726573756c7494681975622e\"\n }\n}"
+ ],
+ [
+ "{\n \"stub\": false,\n \"trainable_name\": \"train_model\",\n \"trial_id\": \"b53cfe05\",\n \"storage\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059572040000000000008c1b7261792e747261696e2e5f696e7465726e616c2e73746f72616765948c0e53746f72616765436f6e746578749493942981947d94288c12637573746f6d5f66735f70726f766964656494898c1273746f726167655f6c6f63616c5f70617468948c2b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c7473948c136578706572696d656e745f6469725f6e616d65948c0f746573745f6578706572696d656e74948c0e747269616c5f6469725f6e616d65948c96747261696e5f6d6f64656c5f62353363666530355f31365f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d382c6465636f6465725f72745f66663d3531322c6465636f6465725f72745f6e756d5f6c617965723d322c64726f705f725f323032342d30362d32355f31362d34332d3138948c1863757272656e745f636865636b706f696e745f696e646578944b008c0b73796e635f636f6e666967948c097261792e747261696e948c0a53796e63436f6e6669679493942981947d94288c0b73796e635f706572696f64944d2c018c0c73796e635f74696d656f7574944d08078c0e73796e635f61727469666163747394898c1c73796e635f6172746966616374735f6f6e5f636865636b706f696e7494888c0a75706c6f61645f646972948c0a44455052454341544544948c0673796e6365729468188c1273796e635f6f6e5f636865636b706f696e7494681875628c1273746f726167655f66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c0f73746f726167655f66735f70617468948c452f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f7465737494681968008c115f46696c6573797374656d53796e6365729493942981947d9428681b682268134d2c0168144d08078c116c6173745f73796e635f75705f74696d65944741d99eb5dcbfed3f8c136c6173745f73796e635f646f776e5f74696d659447fff00000000000008c0d5f73796e635f70726f63657373944e8c0c5f63757272656e745f636d649468008c125f75706c6f61645f746f5f66735f706174689493947d94288c0a6c6f63616c5f70617468948c3b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c74732f746573745f6578706572696d656e74948c0266739468228c0766735f70617468948c552f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e74948c076578636c756465945d948c0e2a2f636865636b706f696e745f2a9461758694756275622e\"\n },\n \"config\": {\n \"encoder_num_layer\": 2,\n \"decoder_rt_num_layer\": 2,\n \"decoder_int_num_layer\": 8,\n \"embedding_dim\": 64,\n \"encoder_ff\": 512,\n \"decoder_rt_ff\": 512,\n \"decoder_int_ff\": 512,\n \"n_head\": 4,\n \"drop_rate\": 0.9936594412219328,\n \"lr\": 0.000836614555813242,\n \"batch_size\": 2048\n },\n \"_Trial__unresolved_config\": {\n \"encoder_num_layer\": 2,\n \"decoder_rt_num_layer\": 2,\n \"decoder_int_num_layer\": 8,\n \"embedding_dim\": 64,\n \"encoder_ff\": 512,\n \"decoder_rt_ff\": 512,\n \"decoder_int_ff\": 512,\n \"n_head\": 4,\n \"drop_rate\": 0.9936594412219328,\n \"lr\": 0.000836614555813242,\n \"batch_size\": 2048\n },\n \"evaluated_params\": {\n \"batch_size\": 2048,\n \"decoder_int_ff\": 512,\n \"decoder_int_num_layer\": 8,\n \"decoder_rt_ff\": 512,\n \"decoder_rt_num_layer\": 2,\n \"drop_rate\": 0.9936594412219328,\n \"embedding_dim\": 64,\n \"encoder_ff\": 512,\n \"encoder_num_layer\": 2,\n \"lr\": 0.000836614555813242,\n \"n_head\": 4\n },\n \"experiment_tag\": \"16_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=8,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_rate=0.9937,embedding_dim=64,encoder_ff=512,encoder_num_layer=2,lr=0.0008,n_head=4\",\n \"stopping_criterion\": {},\n \"_setup_default_resource\": true,\n \"_default_placement_group_factory\": \"80054e2e\",\n \"placement_group_factory\": \"800595ba000000000000008c237261792e74756e652e657865637574696f6e2e706c6163656d656e745f67726f757073948c15506c6163656d656e7447726f7570466163746f72799493942981947d94288c085f62756e646c6573945d947d94288c03435055944740540000000000008c034750559447401000000000000075618c155f686561645f62756e646c655f69735f656d70747994898c095f7374726174656779948c045041434b948c055f6172677394298c075f6b7761726773947d9475622e\",\n \"log_to_file\": [\n null,\n null\n ],\n \"max_failures\": 0,\n \"_default_result_or_future\": null,\n \"export_formats\": [],\n \"status\": \"PAUSED\",\n \"relative_logdir\": \"train_model_b53cfe05_16_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=8,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_16-43-18\",\n \"trial_name_creator\": null,\n \"trial_dirname_creator\": null,\n \"custom_trial_name\": null,\n \"custom_dirname\": null,\n \"restore_path\": null,\n \"_restore_checkpoint_result\": null,\n \"_state_json\": null,\n \"results\": \"80054e2e\",\n \"extra_arg\": \"80054e2e\",\n \"_resources\": \"80054e2e\"\n}",
+ "{\n \"start_time\": 1719326792.6860473,\n \"num_failures\": 0,\n \"num_failures_after_restore\": 0,\n \"error_filename\": null,\n \"pickled_error_filename\": null,\n \"last_result\": {\n \"loss\": 3022.543074480192,\n \"timestamp\": 1719326860,\n \"checkpoint_dir_name\": \"checkpoint_000000\",\n \"should_checkpoint\": true,\n \"done\": false,\n \"training_iteration\": 1,\n \"trial_id\": \"b53cfe05\",\n \"date\": \"2024-06-25_16-47-40\",\n \"time_this_iter_s\": 67.51874232292175,\n \"time_total_s\": 67.51874232292175,\n \"pid\": 122466,\n \"hostname\": \"r3i5n6\",\n \"node_ip\": \"10.159.8.159\",\n \"config\": {\n \"encoder_num_layer\": 2,\n \"decoder_rt_num_layer\": 2,\n \"decoder_int_num_layer\": 8,\n \"embedding_dim\": 64,\n \"encoder_ff\": 512,\n \"decoder_rt_ff\": 512,\n \"decoder_int_ff\": 512,\n \"n_head\": 4,\n \"drop_rate\": 0.9936594412219328,\n \"lr\": 0.000836614555813242,\n \"batch_size\": 2048\n },\n \"time_since_restore\": 67.51874232292175,\n \"iterations_since_restore\": 1,\n \"experiment_tag\": \"16_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=8,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_rate=0.9937,embedding_dim=64,encoder_ff=512,encoder_num_layer=2,lr=0.0008,n_head=4\"\n },\n \"last_result_time\": 1719326860.2093158,\n \"metric_analysis\": {\n \"loss\": {\n \"max\": 3022.543074480192,\n \"min\": 3022.543074480192,\n \"avg\": 3022.543074480192,\n \"last\": 3022.543074480192,\n \"last-5-avg\": 3022.543074480192,\n \"last-10-avg\": 3022.543074480192\n },\n \"should_checkpoint\": {\n \"max\": true,\n \"min\": true,\n \"avg\": true,\n \"last\": true,\n \"last-5-avg\": true,\n \"last-10-avg\": true\n },\n \"done\": {\n \"max\": false,\n \"min\": false,\n \"avg\": false,\n \"last\": false,\n \"last-5-avg\": false,\n \"last-10-avg\": false\n },\n \"training_iteration\": {\n \"max\": 1,\n \"min\": 1,\n \"avg\": 1,\n \"last\": 1,\n \"last-5-avg\": 1,\n \"last-10-avg\": 1\n },\n \"time_this_iter_s\": {\n \"max\": 67.51874232292175,\n \"min\": 67.51874232292175,\n \"avg\": 67.51874232292175,\n \"last\": 67.51874232292175,\n \"last-5-avg\": 67.51874232292175,\n \"last-10-avg\": 67.51874232292175\n },\n \"time_total_s\": {\n \"max\": 67.51874232292175,\n \"min\": 67.51874232292175,\n \"avg\": 67.51874232292175,\n \"last\": 67.51874232292175,\n \"last-5-avg\": 67.51874232292175,\n \"last-10-avg\": 67.51874232292175\n },\n \"time_since_restore\": {\n \"max\": 67.51874232292175,\n \"min\": 67.51874232292175,\n \"avg\": 67.51874232292175,\n \"last\": 67.51874232292175,\n \"last-5-avg\": 67.51874232292175,\n \"last-10-avg\": 67.51874232292175\n },\n \"iterations_since_restore\": {\n \"max\": 1,\n \"min\": 1,\n \"avg\": 1,\n \"last\": 1,\n \"last-5-avg\": 1,\n \"last-10-avg\": 1\n }\n },\n \"_n_steps\": [\n 5,\n 10\n ],\n \"metric_n_steps\": {\n \"loss\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059589000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452948c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b0074946243086fb7db0d169da7409486945294612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059589000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452948c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b0074946243086fb7db0d169da7409486945294612e\"\n }\n },\n \"should_checkpoint\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059522000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b058694529488612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059522000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a8694529488612e\"\n }\n },\n \"done\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059522000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b058694529489612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059522000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a8694529489612e\"\n }\n },\n \"training_iteration\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059523000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452944b01612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059523000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452944b01612e\"\n }\n },\n \"time_this_iter_s\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294474050e13313000000612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294474050e13313000000612e\"\n }\n },\n \"time_total_s\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294474050e13313000000612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294474050e13313000000612e\"\n }\n },\n \"time_since_restore\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294474050e13313000000612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294474050e13313000000612e\"\n }\n },\n \"iterations_since_restore\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059523000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452944b01612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059523000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452944b01612e\"\n }\n }\n },\n \"checkpoint_manager\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059517060000000000008c267261792e747261696e2e5f696e7465726e616c2e636865636b706f696e745f6d616e61676572948c125f436865636b706f696e744d616e616765729493942981947d94288c125f636865636b706f696e745f636f6e666967948c097261792e747261696e948c10436865636b706f696e74436f6e6669679493942981947d94288c0b6e756d5f746f5f6b656570944e8c1a636865636b706f696e745f73636f72655f617474726962757465944e8c16636865636b706f696e745f73636f72655f6f72646572948c036d6178948c14636865636b706f696e745f6672657175656e6379944b008c11636865636b706f696e745f61745f656e6494898c1a5f636865636b706f696e745f6b6565705f616c6c5f72616e6b73948c0a44455052454341544544948c1f5f636865636b706f696e745f75706c6f61645f66726f6d5f776f726b65727394681275628c135f636865636b706f696e745f726573756c7473945d948c1b7261792e747261696e2e5f696e7465726e616c2e73657373696f6e948c0f5f547261696e696e67526573756c749493942981947d94288c0a636865636b706f696e749468068c0a436865636b706f696e749493942981947d94288c0470617468948cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f62353363666530355f31365f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d382c6465636f6465725f72745f66663d3531322c6465636f6465725f72745f6e756d5f6c617965723d322c64726f705f725f323032342d30362d32355f31362d34332d31382f636865636b706f696e745f303030303030948c0a66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c055f75756964948c0475756964948c04555549449493942981947d948c03696e74948a100ded1d5120b71bb6d94d1874c6c85758736275628c076d657472696373947d94288c046c6f7373948c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b0074946243086fb7db0d169da74094869452948c0974696d657374616d70944a8cd87a668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303030948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b018c08747269616c5f6964948c086235336366653035948c0464617465948c13323032342d30362d32355f31362d34372d3430948c1074696d655f746869735f697465725f7394474050e133130000008c0c74696d655f746f74616c5f7394474050e133130000008c03706964944a62de01008c08686f73746e616d65948c06723369356e36948c076e6f64655f6970948c0c31302e3135392e382e313539948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b028c146465636f6465725f72745f6e756d5f6c61796572944b028c156465636f6465725f696e745f6e756d5f6c61796572944b088c0d656d62656464696e675f64696d944b408c0a656e636f6465725f6666944d00028c0d6465636f6465725f72745f6666944d00028c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b048c0964726f705f7261746594473fefcc0ee26d1d608c026c7294473f4b6a08140aa9ff8c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f726594474050e133130000008c18697465726174696f6e735f73696e63655f726573746f7265944b01757562618c195f6c61746573745f636865636b706f696e745f726573756c7494681975622e\"\n }\n}"
+ ],
+ [
+ "{\n \"stub\": false,\n \"trainable_name\": \"train_model\",\n \"trial_id\": \"45b8cfdd\",\n \"storage\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059575040000000000008c1b7261792e747261696e2e5f696e7465726e616c2e73746f72616765948c0e53746f72616765436f6e746578749493942981947d94288c12637573746f6d5f66735f70726f766964656494898c1273746f726167655f6c6f63616c5f70617468948c2b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c7473948c136578706572696d656e745f6469725f6e616d65948c0f746573745f6578706572696d656e74948c0e747269616c5f6469725f6e616d65948c96747261696e5f6d6f64656c5f34356238636664645f31375f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d323034382c6465636f6465725f696e745f6e756d5f6c617965723d322c6465636f6465725f72745f66663d313032342c6465636f6465725f72745f6e756d5f6c617965723d342c64726f705f323032342d30362d32355f31362d34362d3332948c1863757272656e745f636865636b706f696e745f696e646578944affffffff8c0b73796e635f636f6e666967948c097261792e747261696e948c0a53796e63436f6e6669679493942981947d94288c0b73796e635f706572696f64944d2c018c0c73796e635f74696d656f7574944d08078c0e73796e635f61727469666163747394898c1c73796e635f6172746966616374735f6f6e5f636865636b706f696e7494888c0a75706c6f61645f646972948c0a44455052454341544544948c0673796e6365729468188c1273796e635f6f6e5f636865636b706f696e7494681875628c1273746f726167655f66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c0f73746f726167655f66735f70617468948c452f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f7465737494681968008c115f46696c6573797374656d53796e6365729493942981947d9428681b682268134d2c0168144d08078c116c6173745f73796e635f75705f74696d65944741d99eb5dcbfed3f8c136c6173745f73796e635f646f776e5f74696d659447fff00000000000008c0d5f73796e635f70726f63657373944e8c0c5f63757272656e745f636d649468008c125f75706c6f61645f746f5f66735f706174689493947d94288c0a6c6f63616c5f70617468948c3b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c74732f746573745f6578706572696d656e74948c0266739468228c0766735f70617468948c552f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e74948c076578636c756465945d948c0e2a2f636865636b706f696e745f2a9461758694756275622e\"\n },\n \"config\": {\n \"encoder_num_layer\": 2,\n \"decoder_rt_num_layer\": 4,\n \"decoder_int_num_layer\": 2,\n \"embedding_dim\": 1024,\n \"encoder_ff\": 2048,\n \"decoder_rt_ff\": 1024,\n \"decoder_int_ff\": 2048,\n \"n_head\": 4,\n \"drop_rate\": 0.18931069188451255,\n \"lr\": 0.0002168775970949337,\n \"batch_size\": 2048\n },\n \"_Trial__unresolved_config\": {\n \"encoder_num_layer\": 2,\n \"decoder_rt_num_layer\": 4,\n \"decoder_int_num_layer\": 2,\n \"embedding_dim\": 1024,\n \"encoder_ff\": 2048,\n \"decoder_rt_ff\": 1024,\n \"decoder_int_ff\": 2048,\n \"n_head\": 4,\n \"drop_rate\": 0.18931069188451255,\n \"lr\": 0.0002168775970949337,\n \"batch_size\": 2048\n },\n \"evaluated_params\": {\n \"batch_size\": 2048,\n \"decoder_int_ff\": 2048,\n \"decoder_int_num_layer\": 2,\n \"decoder_rt_ff\": 1024,\n \"decoder_rt_num_layer\": 4,\n \"drop_rate\": 0.18931069188451255,\n \"embedding_dim\": 1024,\n \"encoder_ff\": 2048,\n \"encoder_num_layer\": 2,\n \"lr\": 0.0002168775970949337,\n \"n_head\": 4\n },\n \"experiment_tag\": \"17_batch_size=2048,decoder_int_ff=2048,decoder_int_num_layer=2,decoder_rt_ff=1024,decoder_rt_num_layer=4,drop_rate=0.1893,embedding_dim=1024,encoder_ff=2048,encoder_num_layer=2,lr=0.0002,n_head=4\",\n \"stopping_criterion\": {},\n \"_setup_default_resource\": true,\n \"_default_placement_group_factory\": \"80054e2e\",\n \"placement_group_factory\": \"800595ba000000000000008c237261792e74756e652e657865637574696f6e2e706c6163656d656e745f67726f757073948c15506c6163656d656e7447726f7570466163746f72799493942981947d94288c085f62756e646c6573945d947d94288c03435055944740540000000000008c034750559447401000000000000075618c155f686561645f62756e646c655f69735f656d70747994898c095f7374726174656779948c045041434b948c055f6172677394298c075f6b7761726773947d9475622e\",\n \"log_to_file\": [\n null,\n null\n ],\n \"max_failures\": 0,\n \"_default_result_or_future\": null,\n \"export_formats\": [],\n \"status\": \"ERROR\",\n \"relative_logdir\": \"train_model_45b8cfdd_17_batch_size=2048,decoder_int_ff=2048,decoder_int_num_layer=2,decoder_rt_ff=1024,decoder_rt_num_layer=4,drop_2024-06-25_16-46-32\",\n \"trial_name_creator\": null,\n \"trial_dirname_creator\": null,\n \"custom_trial_name\": null,\n \"custom_dirname\": null,\n \"restore_path\": null,\n \"_restore_checkpoint_result\": null,\n \"_state_json\": null,\n \"results\": \"80054e2e\",\n \"extra_arg\": \"80054e2e\",\n \"_resources\": \"80054e2e\"\n}",
+ "{\n \"start_time\": 1719326860.3577213,\n \"num_failures\": 1,\n \"num_failures_after_restore\": 0,\n \"error_filename\": \"error.txt\",\n \"pickled_error_filename\": \"error.pkl\",\n \"last_result\": {\n \"trial_id\": \"45b8cfdd\"\n },\n \"last_result_time\": -Infinity,\n \"metric_analysis\": {},\n \"_n_steps\": [\n 5,\n 10\n ],\n \"metric_n_steps\": {},\n \"checkpoint_manager\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059584010000000000008c267261792e747261696e2e5f696e7465726e616c2e636865636b706f696e745f6d616e61676572948c125f436865636b706f696e744d616e616765729493942981947d94288c125f636865636b706f696e745f636f6e666967948c097261792e747261696e948c10436865636b706f696e74436f6e6669679493942981947d94288c0b6e756d5f746f5f6b656570944e8c1a636865636b706f696e745f73636f72655f617474726962757465944e8c16636865636b706f696e745f73636f72655f6f72646572948c036d6178948c14636865636b706f696e745f6672657175656e6379944b008c11636865636b706f696e745f61745f656e6494898c1a5f636865636b706f696e745f6b6565705f616c6c5f72616e6b73948c0a44455052454341544544948c1f5f636865636b706f696e745f75706c6f61645f66726f6d5f776f726b65727394681275628c135f636865636b706f696e745f726573756c7473945d948c195f6c61746573745f636865636b706f696e745f726573756c74944e75622e\"\n }\n}"
+ ],
+ [
+ "{\n \"stub\": false,\n \"trainable_name\": \"train_model\",\n \"trial_id\": \"8e58579c\",\n \"storage\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059572040000000000008c1b7261792e747261696e2e5f696e7465726e616c2e73746f72616765948c0e53746f72616765436f6e746578749493942981947d94288c12637573746f6d5f66735f70726f766964656494898c1273746f726167655f6c6f63616c5f70617468948c2b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c7473948c136578706572696d656e745f6469725f6e616d65948c0f746573745f6578706572696d656e74948c0e747269616c5f6469725f6e616d65948c96747261696e5f6d6f64656c5f38653538353739635f31385f62617463685f73697a653d313032342c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d382c6465636f6465725f72745f66663d323034382c6465636f6465725f72745f6e756d5f6c617965723d382c64726f705f5f323032342d30362d32355f31362d34372d3430948c1863757272656e745f636865636b706f696e745f696e646578944b018c0b73796e635f636f6e666967948c097261792e747261696e948c0a53796e63436f6e6669679493942981947d94288c0b73796e635f706572696f64944d2c018c0c73796e635f74696d656f7574944d08078c0e73796e635f61727469666163747394898c1c73796e635f6172746966616374735f6f6e5f636865636b706f696e7494888c0a75706c6f61645f646972948c0a44455052454341544544948c0673796e6365729468188c1273796e635f6f6e5f636865636b706f696e7494681875628c1273746f726167655f66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c0f73746f726167655f66735f70617468948c452f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f7465737494681968008c115f46696c6573797374656d53796e6365729493942981947d9428681b682268134d2c0168144d08078c116c6173745f73796e635f75705f74696d65944741d99eb5dcbfed3f8c136c6173745f73796e635f646f776e5f74696d659447fff00000000000008c0d5f73796e635f70726f63657373944e8c0c5f63757272656e745f636d649468008c125f75706c6f61645f746f5f66735f706174689493947d94288c0a6c6f63616c5f70617468948c3b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c74732f746573745f6578706572696d656e74948c0266739468228c0766735f70617468948c552f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e74948c076578636c756465945d948c0e2a2f636865636b706f696e745f2a9461758694756275622e\"\n },\n \"config\": {\n \"encoder_num_layer\": 8,\n \"decoder_rt_num_layer\": 8,\n \"decoder_int_num_layer\": 8,\n \"embedding_dim\": 64,\n \"encoder_ff\": 1024,\n \"decoder_rt_ff\": 2048,\n \"decoder_int_ff\": 512,\n \"n_head\": 8,\n \"drop_rate\": 0.8999111579906955,\n \"lr\": 0.000663247583015271,\n \"batch_size\": 1024\n },\n \"_Trial__unresolved_config\": {\n \"encoder_num_layer\": 8,\n \"decoder_rt_num_layer\": 8,\n \"decoder_int_num_layer\": 8,\n \"embedding_dim\": 64,\n \"encoder_ff\": 1024,\n \"decoder_rt_ff\": 2048,\n \"decoder_int_ff\": 512,\n \"n_head\": 8,\n \"drop_rate\": 0.8999111579906955,\n \"lr\": 0.000663247583015271,\n \"batch_size\": 1024\n },\n \"evaluated_params\": {\n \"batch_size\": 1024,\n \"decoder_int_ff\": 512,\n \"decoder_int_num_layer\": 8,\n \"decoder_rt_ff\": 2048,\n \"decoder_rt_num_layer\": 8,\n \"drop_rate\": 0.8999111579906955,\n \"embedding_dim\": 64,\n \"encoder_ff\": 1024,\n \"encoder_num_layer\": 8,\n \"lr\": 0.000663247583015271,\n \"n_head\": 8\n },\n \"experiment_tag\": \"18_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=8,decoder_rt_ff=2048,decoder_rt_num_layer=8,drop_rate=0.8999,embedding_dim=64,encoder_ff=1024,encoder_num_layer=8,lr=0.0007,n_head=8\",\n \"stopping_criterion\": {},\n \"_setup_default_resource\": true,\n \"_default_placement_group_factory\": \"80054e2e\",\n \"placement_group_factory\": \"800595ba000000000000008c237261792e74756e652e657865637574696f6e2e706c6163656d656e745f67726f757073948c15506c6163656d656e7447726f7570466163746f72799493942981947d94288c085f62756e646c6573945d947d94288c03435055944740540000000000008c034750559447401000000000000075618c155f686561645f62756e646c655f69735f656d70747994898c095f7374726174656779948c045041434b948c055f6172677394298c075f6b7761726773947d9475622e\",\n \"log_to_file\": [\n null,\n null\n ],\n \"max_failures\": 0,\n \"_default_result_or_future\": null,\n \"export_formats\": [],\n \"status\": \"PAUSED\",\n \"relative_logdir\": \"train_model_8e58579c_18_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=8,decoder_rt_ff=2048,decoder_rt_num_layer=8,drop__2024-06-25_16-47-40\",\n \"trial_name_creator\": null,\n \"trial_dirname_creator\": null,\n \"custom_trial_name\": null,\n \"custom_dirname\": null,\n \"restore_path\": null,\n \"_restore_checkpoint_result\": null,\n \"_state_json\": null,\n \"results\": \"80054e2e\",\n \"extra_arg\": \"80054e2e\",\n \"_resources\": \"80054e2e\"\n}",
+ "{\n \"start_time\": 1719326883.2966387,\n \"num_failures\": 0,\n \"num_failures_after_restore\": 0,\n \"error_filename\": null,\n \"pickled_error_filename\": null,\n \"last_result\": {\n \"loss\": 2056.04917078694,\n \"timestamp\": 1719327182,\n \"checkpoint_dir_name\": \"checkpoint_000001\",\n \"should_checkpoint\": true,\n \"done\": false,\n \"training_iteration\": 2,\n \"trial_id\": \"8e58579c\",\n \"date\": \"2024-06-25_16-53-02\",\n \"time_this_iter_s\": 142.2777497768402,\n \"time_total_s\": 298.91499376296997,\n \"pid\": 130020,\n \"hostname\": \"r3i5n6\",\n \"node_ip\": \"10.159.8.159\",\n \"config\": {\n \"encoder_num_layer\": 8,\n \"decoder_rt_num_layer\": 8,\n \"decoder_int_num_layer\": 8,\n \"embedding_dim\": 64,\n \"encoder_ff\": 1024,\n \"decoder_rt_ff\": 2048,\n \"decoder_int_ff\": 512,\n \"n_head\": 8,\n \"drop_rate\": 0.8999111579906955,\n \"lr\": 0.000663247583015271,\n \"batch_size\": 1024\n },\n \"time_since_restore\": 298.91499376296997,\n \"iterations_since_restore\": 2,\n \"experiment_tag\": \"18_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=8,decoder_rt_ff=2048,decoder_rt_num_layer=8,drop_rate=0.8999,embedding_dim=64,encoder_ff=1024,encoder_num_layer=8,lr=0.0007,n_head=8\"\n },\n \"last_result_time\": 1719327182.2214158,\n \"metric_analysis\": {\n \"loss\": {\n \"max\": 2062.3793243648497,\n \"min\": 2056.04917078694,\n \"avg\": 2059.2142475758947,\n \"last\": 2056.04917078694,\n \"last-5-avg\": 2059.2142475758947,\n \"last-10-avg\": 2059.2142475758947\n },\n \"should_checkpoint\": {\n \"max\": true,\n \"min\": true,\n \"avg\": 1.0,\n \"last\": true,\n \"last-5-avg\": 1.0,\n \"last-10-avg\": 1.0\n },\n \"done\": {\n \"max\": false,\n \"min\": false,\n \"avg\": 0.0,\n \"last\": false,\n \"last-5-avg\": 0.0,\n \"last-10-avg\": 0.0\n },\n \"training_iteration\": {\n \"max\": 2,\n \"min\": 1,\n \"avg\": 1.5,\n \"last\": 2,\n \"last-5-avg\": 1.5,\n \"last-10-avg\": 1.5\n },\n \"time_this_iter_s\": {\n \"max\": 156.63724398612976,\n \"min\": 142.2777497768402,\n \"avg\": 149.45749688148499,\n \"last\": 142.2777497768402,\n \"last-5-avg\": 149.45749688148499,\n \"last-10-avg\": 149.45749688148499\n },\n \"time_total_s\": {\n \"max\": 298.91499376296997,\n \"min\": 156.63724398612976,\n \"avg\": 227.77611887454987,\n \"last\": 298.91499376296997,\n \"last-5-avg\": 227.77611887454987,\n \"last-10-avg\": 227.77611887454987\n },\n \"time_since_restore\": {\n \"max\": 298.91499376296997,\n \"min\": 156.63724398612976,\n \"avg\": 227.77611887454987,\n \"last\": 298.91499376296997,\n \"last-5-avg\": 227.77611887454987,\n \"last-10-avg\": 227.77611887454987\n },\n \"iterations_since_restore\": {\n \"max\": 2,\n \"min\": 1,\n \"avg\": 1.5,\n \"last\": 2,\n \"last-5-avg\": 1.5,\n \"last-10-avg\": 1.5\n }\n },\n \"_n_steps\": [\n 5,\n 10\n ],\n \"metric_n_steps\": {\n \"loss\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005959d000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294288c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b007494624308369bcd36c21ca04094869452946807680d4308a7d3e92c1910a0409486945294652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005959d000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294288c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b007494624308369bcd36c21ca04094869452946807680d4308a7d3e92c1910a0409486945294652e\"\n }\n },\n \"should_checkpoint\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059524000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294288888652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059524000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294288888652e\"\n }\n },\n \"done\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059524000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294288989652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059524000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294288989652e\"\n }\n },\n \"training_iteration\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059526000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294284b014b02652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059526000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294284b014b02652e\"\n }\n },\n \"time_this_iter_s\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059534000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452942847406394644d800000474061c8e353800000652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059534000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452942847406394644d800000474061c8e353800000652e\"\n }\n },\n \"time_total_s\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059534000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452942847406394644d800000474072aea3d0800000652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059534000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452942847406394644d800000474072aea3d0800000652e\"\n }\n },\n \"time_since_restore\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059534000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452942847406394644d800000474072aea3d0800000652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059534000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452942847406394644d800000474072aea3d0800000652e\"\n }\n },\n \"iterations_since_restore\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059526000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294284b014b02652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059526000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294284b014b02652e\"\n }\n }\n },\n \"checkpoint_manager\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"800595a4090000000000008c267261792e747261696e2e5f696e7465726e616c2e636865636b706f696e745f6d616e61676572948c125f436865636b706f696e744d616e616765729493942981947d94288c125f636865636b706f696e745f636f6e666967948c097261792e747261696e948c10436865636b706f696e74436f6e6669679493942981947d94288c0b6e756d5f746f5f6b656570944e8c1a636865636b706f696e745f73636f72655f617474726962757465944e8c16636865636b706f696e745f73636f72655f6f72646572948c036d6178948c14636865636b706f696e745f6672657175656e6379944b008c11636865636b706f696e745f61745f656e6494898c1a5f636865636b706f696e745f6b6565705f616c6c5f72616e6b73948c0a44455052454341544544948c1f5f636865636b706f696e745f75706c6f61645f66726f6d5f776f726b65727394681275628c135f636865636b706f696e745f726573756c7473945d94288c1b7261792e747261696e2e5f696e7465726e616c2e73657373696f6e948c0f5f547261696e696e67526573756c749493942981947d94288c0a636865636b706f696e749468068c0a436865636b706f696e749493942981947d94288c0470617468948cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f38653538353739635f31385f62617463685f73697a653d313032342c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d382c6465636f6465725f72745f66663d323034382c6465636f6465725f72745f6e756d5f6c617965723d382c64726f705f5f323032342d30362d32355f31362d34372d34302f636865636b706f696e745f303030303030948c0a66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c055f75756964948c0475756964948c04555549449493942981947d948c03696e74948a102fd40e2ad2aee0ab774f9b6b0d8e1422736275628c076d657472696373947d94288c046c6f7373948c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b007494624308369bcd36c21ca04094869452948c0974696d657374616d70944a3fd97a668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303030948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b018c08747269616c5f6964948c083865353835373963948c0464617465948c13323032342d30362d32355f31362d35302d3339948c1074696d655f746869735f697465725f739447406394644d8000008c0c74696d655f746f74616c5f739447406394644d8000008c03706964944ae4fb01008c08686f73746e616d65948c06723369356e36948c076e6f64655f6970948c0c31302e3135392e382e313539948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b088c146465636f6465725f72745f6e756d5f6c61796572944b088c156465636f6465725f696e745f6e756d5f6c61796572944b088c0d656d62656464696e675f64696d944b408c0a656e636f6465725f6666944d00048c0d6465636f6465725f72745f6666944d00088c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b088c0964726f705f7261746594473feccc127c1c04508c026c7294473f45bbb956cf4fa78c0a62617463685f73697a65944d0004758c1274696d655f73696e63655f726573746f72659447406394644d8000008c18697465726174696f6e735f73696e63655f726573746f7265944b0175756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f38653538353739635f31385f62617463685f73697a653d313032342c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d382c6465636f6465725f72745f66663d323034382c6465636f6465725f72745f6e756d5f6c617965723d382c64726f705f5f323032342d30362d32355f31362d34372d34302f636865636b706f696e745f30303030303194682268257d946827897385945294682a682d2981947d9468308a109082755a26183b9e5c416e75fbaec42e7362756268317d94288c046c6f7373946836683c4308a7d3e92c1910a04094869452948c0974696d657374616d70944aced97a668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303031948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b028c08747269616c5f6964948c083865353835373963948c0464617465948c13323032342d30362d32355f31362d35332d3032948c1074696d655f746869735f697465725f7394474061c8e3538000008c0c74696d655f746f74616c5f7394474072aea3d08000008c03706964944ae4fb01008c08686f73746e616d65948c06723369356e36948c076e6f64655f6970948c0c31302e3135392e382e313539948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b088c146465636f6465725f72745f6e756d5f6c61796572944b088c156465636f6465725f696e745f6e756d5f6c61796572944b088c0d656d62656464696e675f64696d944b408c0a656e636f6465725f6666944d00048c0d6465636f6465725f72745f6666944d00088c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b088c0964726f705f7261746594473feccc127c1c04508c026c7294473f45bbb956cf4fa78c0a62617463685f73697a65944d0004758c1274696d655f73696e63655f726573746f726594474072aea3d08000008c18697465726174696f6e735f73696e63655f726573746f7265944b02757562658c195f6c61746573745f636865636b706f696e745f726573756c7494686275622e\"\n }\n}"
+ ],
+ [
+ "{\n \"stub\": false,\n \"trainable_name\": \"train_model\",\n \"trial_id\": \"375ed3d7\",\n \"storage\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059572040000000000008c1b7261792e747261696e2e5f696e7465726e616c2e73746f72616765948c0e53746f72616765436f6e746578749493942981947d94288c12637573746f6d5f66735f70726f766964656494898c1273746f726167655f6c6f63616c5f70617468948c2b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c7473948c136578706572696d656e745f6469725f6e616d65948c0f746573745f6578706572696d656e74948c0e747269616c5f6469725f6e616d65948c96747261696e5f6d6f64656c5f33373565643364375f31395f62617463685f73697a653d313032342c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d322c6465636f6465725f72745f66663d313032342c6465636f6465725f72745f6e756d5f6c617965723d342c64726f705f5f323032342d30362d32355f31362d35362d3132948c1863757272656e745f636865636b706f696e745f696e646578944b008c0b73796e635f636f6e666967948c097261792e747261696e948c0a53796e63436f6e6669679493942981947d94288c0b73796e635f706572696f64944d2c018c0c73796e635f74696d656f7574944d08078c0e73796e635f61727469666163747394898c1c73796e635f6172746966616374735f6f6e5f636865636b706f696e7494888c0a75706c6f61645f646972948c0a44455052454341544544948c0673796e6365729468188c1273796e635f6f6e5f636865636b706f696e7494681875628c1273746f726167655f66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c0f73746f726167655f66735f70617468948c452f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f7465737494681968008c115f46696c6573797374656d53796e6365729493942981947d9428681b682268134d2c0168144d08078c116c6173745f73796e635f75705f74696d65944741d99eb673a554b88c136c6173745f73796e635f646f776e5f74696d659447fff00000000000008c0d5f73796e635f70726f63657373944e8c0c5f63757272656e745f636d649468008c125f75706c6f61645f746f5f66735f706174689493947d94288c0a6c6f63616c5f70617468948c3b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c74732f746573745f6578706572696d656e74948c0266739468228c0766735f70617468948c552f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e74948c076578636c756465945d948c0e2a2f636865636b706f696e745f2a9461758694756275622e\"\n },\n \"config\": {\n \"encoder_num_layer\": 1,\n \"decoder_rt_num_layer\": 4,\n \"decoder_int_num_layer\": 2,\n \"embedding_dim\": 16,\n \"encoder_ff\": 1024,\n \"decoder_rt_ff\": 1024,\n \"decoder_int_ff\": 512,\n \"n_head\": 1,\n \"drop_rate\": 0.29850788173794873,\n \"lr\": 0.017371208825505195,\n \"batch_size\": 1024\n },\n \"_Trial__unresolved_config\": {\n \"encoder_num_layer\": 1,\n \"decoder_rt_num_layer\": 4,\n \"decoder_int_num_layer\": 2,\n \"embedding_dim\": 16,\n \"encoder_ff\": 1024,\n \"decoder_rt_ff\": 1024,\n \"decoder_int_ff\": 512,\n \"n_head\": 1,\n \"drop_rate\": 0.29850788173794873,\n \"lr\": 0.017371208825505195,\n \"batch_size\": 1024\n },\n \"evaluated_params\": {\n \"batch_size\": 1024,\n \"decoder_int_ff\": 512,\n \"decoder_int_num_layer\": 2,\n \"decoder_rt_ff\": 1024,\n \"decoder_rt_num_layer\": 4,\n \"drop_rate\": 0.29850788173794873,\n \"embedding_dim\": 16,\n \"encoder_ff\": 1024,\n \"encoder_num_layer\": 1,\n \"lr\": 0.017371208825505195,\n \"n_head\": 1\n },\n \"experiment_tag\": \"19_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=2,decoder_rt_ff=1024,decoder_rt_num_layer=4,drop_rate=0.2985,embedding_dim=16,encoder_ff=1024,encoder_num_layer=1,lr=0.0174,n_head=1\",\n \"stopping_criterion\": {},\n \"_setup_default_resource\": true,\n \"_default_placement_group_factory\": \"80054e2e\",\n \"placement_group_factory\": \"800595ba000000000000008c237261792e74756e652e657865637574696f6e2e706c6163656d656e745f67726f757073948c15506c6163656d656e7447726f7570466163746f72799493942981947d94288c085f62756e646c6573945d947d94288c03435055944740540000000000008c034750559447401000000000000075618c155f686561645f62756e646c655f69735f656d70747994898c095f7374726174656779948c045041434b948c055f6172677394298c075f6b7761726773947d9475622e\",\n \"log_to_file\": [\n null,\n null\n ],\n \"max_failures\": 0,\n \"_default_result_or_future\": null,\n \"export_formats\": [],\n \"status\": \"RUNNING\",\n \"relative_logdir\": \"train_model_375ed3d7_19_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=2,decoder_rt_ff=1024,decoder_rt_num_layer=4,drop__2024-06-25_16-56-12\",\n \"trial_name_creator\": null,\n \"trial_dirname_creator\": null,\n \"custom_trial_name\": null,\n \"custom_dirname\": null,\n \"restore_path\": null,\n \"_restore_checkpoint_result\": null,\n \"_state_json\": null,\n \"results\": \"80054e2e\",\n \"extra_arg\": \"80054e2e\",\n \"_resources\": \"80054e2e\"\n}",
+ "{\n \"start_time\": 1719327790.388722,\n \"num_failures\": 0,\n \"num_failures_after_restore\": 0,\n \"error_filename\": null,\n \"pickled_error_filename\": null,\n \"last_result\": {\n \"loss\": 1971.010832058163,\n \"timestamp\": 1719327860,\n \"checkpoint_dir_name\": \"checkpoint_000000\",\n \"should_checkpoint\": true,\n \"done\": false,\n \"training_iteration\": 1,\n \"trial_id\": \"375ed3d7\",\n \"date\": \"2024-06-25_17-04-20\",\n \"time_this_iter_s\": 70.38656640052795,\n \"time_total_s\": 70.38656640052795,\n \"pid\": 168065,\n \"hostname\": \"r3i5n6\",\n \"node_ip\": \"10.159.8.159\",\n \"config\": {\n \"encoder_num_layer\": 1,\n \"decoder_rt_num_layer\": 4,\n \"decoder_int_num_layer\": 2,\n \"embedding_dim\": 16,\n \"encoder_ff\": 1024,\n \"decoder_rt_ff\": 1024,\n \"decoder_int_ff\": 512,\n \"n_head\": 1,\n \"drop_rate\": 0.29850788173794873,\n \"lr\": 0.017371208825505195,\n \"batch_size\": 1024\n },\n \"time_since_restore\": 70.38656640052795,\n \"iterations_since_restore\": 1,\n \"experiment_tag\": \"19_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=2,decoder_rt_ff=1024,decoder_rt_num_layer=4,drop_rate=0.2985,embedding_dim=16,encoder_ff=1024,encoder_num_layer=1,lr=0.0174,n_head=1\"\n },\n \"last_result_time\": 1719327860.7799456,\n \"metric_analysis\": {\n \"loss\": {\n \"max\": 1971.010832058163,\n \"min\": 1971.010832058163,\n \"avg\": 1971.010832058163,\n \"last\": 1971.010832058163,\n \"last-5-avg\": 1971.010832058163,\n \"last-10-avg\": 1971.010832058163\n },\n \"should_checkpoint\": {\n \"max\": true,\n \"min\": true,\n \"avg\": true,\n \"last\": true,\n \"last-5-avg\": true,\n \"last-10-avg\": true\n },\n \"done\": {\n \"max\": false,\n \"min\": false,\n \"avg\": false,\n \"last\": false,\n \"last-5-avg\": false,\n \"last-10-avg\": false\n },\n \"training_iteration\": {\n \"max\": 1,\n \"min\": 1,\n \"avg\": 1,\n \"last\": 1,\n \"last-5-avg\": 1,\n \"last-10-avg\": 1\n },\n \"time_this_iter_s\": {\n \"max\": 70.38656640052795,\n \"min\": 70.38656640052795,\n \"avg\": 70.38656640052795,\n \"last\": 70.38656640052795,\n \"last-5-avg\": 70.38656640052795,\n \"last-10-avg\": 70.38656640052795\n },\n \"time_total_s\": {\n \"max\": 70.38656640052795,\n \"min\": 70.38656640052795,\n \"avg\": 70.38656640052795,\n \"last\": 70.38656640052795,\n \"last-5-avg\": 70.38656640052795,\n \"last-10-avg\": 70.38656640052795\n },\n \"time_since_restore\": {\n \"max\": 70.38656640052795,\n \"min\": 70.38656640052795,\n \"avg\": 70.38656640052795,\n \"last\": 70.38656640052795,\n \"last-5-avg\": 70.38656640052795,\n \"last-10-avg\": 70.38656640052795\n },\n \"iterations_since_restore\": {\n \"max\": 1,\n \"min\": 1,\n \"avg\": 1,\n \"last\": 1,\n \"last-5-avg\": 1,\n \"last-10-avg\": 1\n }\n },\n \"_n_steps\": [\n 5,\n 10\n ],\n \"metric_n_steps\": {\n \"loss\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059589000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452948c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b0074946243083c1e8f170bcc9e409486945294612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059589000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452948c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b0074946243083c1e8f170bcc9e409486945294612e\"\n }\n },\n \"should_checkpoint\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059522000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b058694529488612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059522000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a8694529488612e\"\n }\n },\n \"done\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059522000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b058694529489612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059522000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a8694529489612e\"\n }\n },\n \"training_iteration\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059523000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452944b01612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059523000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452944b01612e\"\n }\n },\n \"time_this_iter_s\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b058694529447405198bd81000000612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a8694529447405198bd81000000612e\"\n }\n },\n \"time_total_s\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b058694529447405198bd81000000612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a8694529447405198bd81000000612e\"\n }\n },\n \"time_since_restore\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b058694529447405198bd81000000612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a8694529447405198bd81000000612e\"\n }\n },\n \"iterations_since_restore\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059523000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452944b01612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059523000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452944b01612e\"\n }\n }\n },\n \"checkpoint_manager\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059517060000000000008c267261792e747261696e2e5f696e7465726e616c2e636865636b706f696e745f6d616e61676572948c125f436865636b706f696e744d616e616765729493942981947d94288c125f636865636b706f696e745f636f6e666967948c097261792e747261696e948c10436865636b706f696e74436f6e6669679493942981947d94288c0b6e756d5f746f5f6b656570944e8c1a636865636b706f696e745f73636f72655f617474726962757465944e8c16636865636b706f696e745f73636f72655f6f72646572948c036d6178948c14636865636b706f696e745f6672657175656e6379944b008c11636865636b706f696e745f61745f656e6494898c1a5f636865636b706f696e745f6b6565705f616c6c5f72616e6b73948c0a44455052454341544544948c1f5f636865636b706f696e745f75706c6f61645f66726f6d5f776f726b65727394681275628c135f636865636b706f696e745f726573756c7473945d948c1b7261792e747261696e2e5f696e7465726e616c2e73657373696f6e948c0f5f547261696e696e67526573756c749493942981947d94288c0a636865636b706f696e749468068c0a436865636b706f696e749493942981947d94288c0470617468948cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f33373565643364375f31395f62617463685f73697a653d313032342c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d322c6465636f6465725f72745f66663d313032342c6465636f6465725f72745f6e756d5f6c617965723d342c64726f705f5f323032342d30362d32355f31362d35362d31322f636865636b706f696e745f303030303030948c0a66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c055f75756964948c0475756964948c04555549449493942981947d948c03696e74948a10396822c682368391904e67012945ac5a736275628c076d657472696373947d94288c046c6f7373948c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b0074946243083c1e8f170bcc9e4094869452948c0974696d657374616d70944a74dc7a668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303030948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b018c08747269616c5f6964948c083337356564336437948c0464617465948c13323032342d30362d32355f31372d30342d3230948c1074696d655f746869735f697465725f739447405198bd810000008c0c74696d655f746f74616c5f739447405198bd810000008c03706964944a819002008c08686f73746e616d65948c06723369356e36948c076e6f64655f6970948c0c31302e3135392e382e313539948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b018c146465636f6465725f72745f6e756d5f6c61796572944b048c156465636f6465725f696e745f6e756d5f6c61796572944b028c0d656d62656464696e675f64696d944b108c0a656e636f6465725f6666944d00048c0d6465636f6465725f72745f6666944d00048c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b018c0964726f705f7261746594473fd31ac0cd6a6a188c026c7294473f91c9c21730ac118c0a62617463685f73697a65944d0004758c1274696d655f73696e63655f726573746f72659447405198bd810000008c18697465726174696f6e735f73696e63655f726573746f7265944b01757562618c195f6c61746573745f636865636b706f696e745f726573756c7494681975622e\"\n }\n}"
+ ]
+ ],
+ "runner_data": {
+ "_earliest_stopping_actor": 35513.70259362,
+ "_actor_cleanup_timeout": 600,
+ "_actor_force_cleanup_timeout": 10,
+ "_reuse_actors": true,
+ "_buffer_length": 1,
+ "_buffer_min_time_s": 0.0,
+ "_buffer_max_time_s": 100.0,
+ "_max_pending_trials": 1,
+ "_metric": "loss",
+ "_total_time": 3296.2440366744995,
+ "_iteration": 34237,
+ "_has_errored": true,
+ "_fail_fast": false,
+ "_print_trial_errors": true,
+ "_cached_trial_decisions": {},
+ "_queued_trial_decisions": {},
+ "_should_stop_experiment": false,
+ "_stopper": {
+ "_type": "CLOUDPICKLE_FALLBACK",
+ "value": "8005952c000000000000008c157261792e74756e652e73746f707065722e6e6f6f70948c0b4e6f6f7053746f707065729493942981942e"
+ },
+ "_start_time": 1719324468.4160578,
+ "_last_checkpoint_time": -Infinity,
+ "_session_str": "2024-06-25_16-07-48",
+ "_checkpoint_period": "auto",
+ "_trial_checkpoint_config": {
+ "_type": "CLOUDPICKLE_FALLBACK",
+ "value": "800595f2000000000000008c097261792e747261696e948c10436865636b706f696e74436f6e6669679493942981947d94288c0b6e756d5f746f5f6b656570944e8c1a636865636b706f696e745f73636f72655f617474726962757465944e8c16636865636b706f696e745f73636f72655f6f72646572948c036d6178948c14636865636b706f696e745f6672657175656e6379944b008c11636865636b706f696e745f61745f656e6494898c1a5f636865636b706f696e745f6b6565705f616c6c5f72616e6b73948c0a44455052454341544544948c1f5f636865636b706f696e745f75706c6f61645f66726f6d5f776f726b65727394680c75622e"
+ },
+ "_resumed": false
+ },
+ "stats": {
+ "start_time": 1719324468.4160578,
+ "timestamp": -Infinity
+ }
+}
\ No newline at end of file
diff --git a/ray_results/test_experiment/experiment_state-2024-06-25_17-10-00.json b/ray_results/test_experiment/experiment_state-2024-06-25_17-10-00.json
new file mode 100644
index 0000000..97e43a7
--- /dev/null
+++ b/ray_results/test_experiment/experiment_state-2024-06-25_17-10-00.json
@@ -0,0 +1,168 @@
+{
+ "trial_data": [
+ [
+ "{\n \"stub\": false,\n \"trainable_name\": \"train_model\",\n \"trial_id\": \"5763f130\",\n \"storage\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059582030000000000008c1b7261792e747261696e2e5f696e7465726e616c2e73746f72616765948c0e53746f72616765436f6e746578749493942981947d94288c12637573746f6d5f66735f70726f766964656494898c1273746f726167655f6c6f63616c5f70617468948c2b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c7473948c136578706572696d656e745f6469725f6e616d65948c0f746573745f6578706572696d656e74948c0e747269616c5f6469725f6e616d65948c96747261696e5f6d6f64656c5f35373633663133305f315f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d323034382c6465636f6465725f72745f6e756d5f6c617965723d382c64726f705f725f323032342d30362d32355f31372d31302d3030948c1863757272656e745f636865636b706f696e745f696e646578944b008c0b73796e635f636f6e666967948c097261792e747261696e948c0a53796e63436f6e6669679493942981947d94288c0b73796e635f706572696f64944d2c018c0c73796e635f74696d656f7574944d08078c0e73796e635f61727469666163747394898c1c73796e635f6172746966616374735f6f6e5f636865636b706f696e7494888c0a75706c6f61645f646972948c0a44455052454341544544948c0673796e6365729468188c1273796e635f6f6e5f636865636b706f696e7494681875628c1273746f726167655f66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c0f73746f726167655f66735f70617468948c452f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f7465737494681968008c115f46696c6573797374656d53796e6365729493942981947d9428681b682268134d2c0168144d08078c116c6173745f73796e635f75705f74696d659447fff00000000000008c136c6173745f73796e635f646f776e5f74696d659447fff00000000000008c0d5f73796e635f70726f63657373944e8c0c5f63757272656e745f636d64944e756275622e\"\n },\n \"config\": {\n \"encoder_num_layer\": 4,\n \"decoder_rt_num_layer\": 8,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 256,\n \"encoder_ff\": 512,\n \"decoder_rt_ff\": 2048,\n \"decoder_int_ff\": 512,\n \"n_head\": 2,\n \"drop_rate\": 0.7224904221599877,\n \"lr\": 0.04561252173503944,\n \"batch_size\": 2048\n },\n \"_Trial__unresolved_config\": {\n \"encoder_num_layer\": 4,\n \"decoder_rt_num_layer\": 8,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 256,\n \"encoder_ff\": 512,\n \"decoder_rt_ff\": 2048,\n \"decoder_int_ff\": 512,\n \"n_head\": 2,\n \"drop_rate\": 0.7224904221599877,\n \"lr\": 0.04561252173503944,\n \"batch_size\": 2048\n },\n \"evaluated_params\": {\n \"batch_size\": 2048,\n \"decoder_int_ff\": 512,\n \"decoder_int_num_layer\": 1,\n \"decoder_rt_ff\": 2048,\n \"decoder_rt_num_layer\": 8,\n \"drop_rate\": 0.7224904221599877,\n \"embedding_dim\": 256,\n \"encoder_ff\": 512,\n \"encoder_num_layer\": 4,\n \"lr\": 0.04561252173503944,\n \"n_head\": 2\n },\n \"experiment_tag\": \"1_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=8,drop_rate=0.7225,embedding_dim=256,encoder_ff=512,encoder_num_layer=4,lr=0.0456,n_head=2\",\n \"stopping_criterion\": {},\n \"_setup_default_resource\": true,\n \"_default_placement_group_factory\": \"80054e2e\",\n \"placement_group_factory\": \"800595ba000000000000008c237261792e74756e652e657865637574696f6e2e706c6163656d656e745f67726f757073948c15506c6163656d656e7447726f7570466163746f72799493942981947d94288c085f62756e646c6573945d947d94288c03435055944740540000000000008c034750559447401000000000000075618c155f686561645f62756e646c655f69735f656d70747994898c095f7374726174656779948c045041434b948c055f6172677394298c075f6b7761726773947d9475622e\",\n \"log_to_file\": [\n null,\n null\n ],\n \"max_failures\": 0,\n \"_default_result_or_future\": null,\n \"export_formats\": [],\n \"status\": \"PAUSED\",\n \"relative_logdir\": \"train_model_5763f130_1_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=8,drop_r_2024-06-25_17-10-00\",\n \"trial_name_creator\": null,\n \"trial_dirname_creator\": null,\n \"custom_trial_name\": null,\n \"custom_dirname\": null,\n \"restore_path\": null,\n \"_restore_checkpoint_result\": null,\n \"_state_json\": null,\n \"results\": \"80054e2e\",\n \"extra_arg\": \"80054e2e\",\n \"_resources\": \"80054e2e\"\n}",
+ "{\n \"start_time\": 1719328213.2508516,\n \"num_failures\": 0,\n \"num_failures_after_restore\": 0,\n \"error_filename\": null,\n \"pickled_error_filename\": null,\n \"last_result\": {\n \"loss\": 4538.754975086122,\n \"timestamp\": 1719328455,\n \"checkpoint_dir_name\": \"checkpoint_000000\",\n \"should_checkpoint\": true,\n \"done\": false,\n \"training_iteration\": 1,\n \"trial_id\": \"5763f130\",\n \"date\": \"2024-06-25_17-14-16\",\n \"time_this_iter_s\": 242.80822491645813,\n \"time_total_s\": 242.80822491645813,\n \"pid\": 2195253,\n \"hostname\": \"r8i6n2\",\n \"node_ip\": \"10.159.28.60\",\n \"config\": {\n \"encoder_num_layer\": 4,\n \"decoder_rt_num_layer\": 8,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 256,\n \"encoder_ff\": 512,\n \"decoder_rt_ff\": 2048,\n \"decoder_int_ff\": 512,\n \"n_head\": 2,\n \"drop_rate\": 0.7224904221599877,\n \"lr\": 0.04561252173503944,\n \"batch_size\": 2048\n },\n \"time_since_restore\": 242.80822491645813,\n \"iterations_since_restore\": 1,\n \"experiment_tag\": \"1_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=8,drop_rate=0.7225,embedding_dim=256,encoder_ff=512,encoder_num_layer=4,lr=0.0456,n_head=2\"\n },\n \"last_result_time\": 1719328456.063987,\n \"metric_analysis\": {\n \"loss\": {\n \"max\": 4538.754975086122,\n \"min\": 4538.754975086122,\n \"avg\": 4538.754975086122,\n \"last\": 4538.754975086122,\n \"last-5-avg\": 4538.754975086122,\n \"last-10-avg\": 4538.754975086122\n },\n \"should_checkpoint\": {\n \"max\": true,\n \"min\": true,\n \"avg\": true,\n \"last\": true,\n \"last-5-avg\": true,\n \"last-10-avg\": true\n },\n \"done\": {\n \"max\": false,\n \"min\": false,\n \"avg\": false,\n \"last\": false,\n \"last-5-avg\": false,\n \"last-10-avg\": false\n },\n \"training_iteration\": {\n \"max\": 1,\n \"min\": 1,\n \"avg\": 1,\n \"last\": 1,\n \"last-5-avg\": 1,\n \"last-10-avg\": 1\n },\n \"time_this_iter_s\": {\n \"max\": 242.80822491645813,\n \"min\": 242.80822491645813,\n \"avg\": 242.80822491645813,\n \"last\": 242.80822491645813,\n \"last-5-avg\": 242.80822491645813,\n \"last-10-avg\": 242.80822491645813\n },\n \"time_total_s\": {\n \"max\": 242.80822491645813,\n \"min\": 242.80822491645813,\n \"avg\": 242.80822491645813,\n \"last\": 242.80822491645813,\n \"last-5-avg\": 242.80822491645813,\n \"last-10-avg\": 242.80822491645813\n },\n \"time_since_restore\": {\n \"max\": 242.80822491645813,\n \"min\": 242.80822491645813,\n \"avg\": 242.80822491645813,\n \"last\": 242.80822491645813,\n \"last-5-avg\": 242.80822491645813,\n \"last-10-avg\": 242.80822491645813\n },\n \"iterations_since_restore\": {\n \"max\": 1,\n \"min\": 1,\n \"avg\": 1,\n \"last\": 1,\n \"last-5-avg\": 1,\n \"last-10-avg\": 1\n }\n },\n \"_n_steps\": [\n 5,\n 10\n ],\n \"metric_n_steps\": {\n \"loss\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059589000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452948c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b00749462430830180c46c1bab1409486945294612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059589000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452948c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b00749462430830180c46c1bab1409486945294612e\"\n }\n },\n \"should_checkpoint\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059522000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b058694529488612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059522000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a8694529488612e\"\n }\n },\n \"done\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059522000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b058694529489612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059522000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a8694529489612e\"\n }\n },\n \"training_iteration\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059523000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452944b01612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059523000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452944b01612e\"\n }\n },\n \"time_this_iter_s\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b058694529447406e59dcfa800000612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a8694529447406e59dcfa800000612e\"\n }\n },\n \"time_total_s\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b058694529447406e59dcfa800000612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a8694529447406e59dcfa800000612e\"\n }\n },\n \"time_since_restore\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b058694529447406e59dcfa800000612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a8694529447406e59dcfa800000612e\"\n }\n },\n \"iterations_since_restore\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059523000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452944b01612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059523000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452944b01612e\"\n }\n }\n },\n \"checkpoint_manager\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059518060000000000008c267261792e747261696e2e5f696e7465726e616c2e636865636b706f696e745f6d616e61676572948c125f436865636b706f696e744d616e616765729493942981947d94288c125f636865636b706f696e745f636f6e666967948c097261792e747261696e948c10436865636b706f696e74436f6e6669679493942981947d94288c0b6e756d5f746f5f6b656570944e8c1a636865636b706f696e745f73636f72655f617474726962757465944e8c16636865636b706f696e745f73636f72655f6f72646572948c036d6178948c14636865636b706f696e745f6672657175656e6379944b008c11636865636b706f696e745f61745f656e6494898c1a5f636865636b706f696e745f6b6565705f616c6c5f72616e6b73948c0a44455052454341544544948c1f5f636865636b706f696e745f75706c6f61645f66726f6d5f776f726b65727394681275628c135f636865636b706f696e745f726573756c7473945d948c1b7261792e747261696e2e5f696e7465726e616c2e73657373696f6e948c0f5f547261696e696e67526573756c749493942981947d94288c0a636865636b706f696e749468068c0a436865636b706f696e749493942981947d94288c0470617468948cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f35373633663133305f315f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d323034382c6465636f6465725f72745f6e756d5f6c617965723d382c64726f705f725f323032342d30362d32355f31372d31302d30302f636865636b706f696e745f303030303030948c0a66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c055f75756964948c0475756964948c04555549449493942981947d948c03696e74948a10b0dd19201197f187aa4e48898e873e6f736275628c076d657472696373947d94288c046c6f7373948c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b00749462430830180c46c1bab14094869452948c0974696d657374616d70944ac7de7a668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303030948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b018c08747269616c5f6964948c083537363366313330948c0464617465948c13323032342d30362d32355f31372d31342d3136948c1074696d655f746869735f697465725f739447406e59dcfa8000008c0c74696d655f746f74616c5f739447406e59dcfa8000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b048c146465636f6465725f72745f6e756d5f6c61796572944b088c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944d00018c0a656e636f6465725f6666944d00028c0d6465636f6465725f72745f6666944d00088c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b028c0964726f705f7261746594473fe71ea43bdb36528c026c7294473fa75a864247b9098c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f72659447406e59dcfa8000008c18697465726174696f6e735f73696e63655f726573746f7265944b01757562618c195f6c61746573745f636865636b706f696e745f726573756c7494681975622e\"\n }\n}"
+ ],
+ [
+ "{\n \"stub\": false,\n \"trainable_name\": \"train_model\",\n \"trial_id\": \"a00d3259\",\n \"storage\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059572040000000000008c1b7261792e747261696e2e5f696e7465726e616c2e73746f72616765948c0e53746f72616765436f6e746578749493942981947d94288c12637573746f6d5f66735f70726f766964656494898c1273746f726167655f6c6f63616c5f70617468948c2b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c7473948c136578706572696d656e745f6469725f6e616d65948c0f746573745f6578706572696d656e74948c0e747269616c5f6469725f6e616d65948c96747261696e5f6d6f64656c5f61303064333235395f325f62617463685f73697a653d313032342c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d3531322c6465636f6465725f72745f6e756d5f6c617965723d342c64726f705f72615f323032342d30362d32355f31372d31302d3133948c1863757272656e745f636865636b706f696e745f696e646578944b078c0b73796e635f636f6e666967948c097261792e747261696e948c0a53796e63436f6e6669679493942981947d94288c0b73796e635f706572696f64944d2c018c0c73796e635f74696d656f7574944d08078c0e73796e635f61727469666163747394898c1c73796e635f6172746966616374735f6f6e5f636865636b706f696e7494888c0a75706c6f61645f646972948c0a44455052454341544544948c0673796e6365729468188c1273796e635f6f6e5f636865636b706f696e7494681875628c1273746f726167655f66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c0f73746f726167655f66735f70617468948c452f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f7465737494681968008c115f46696c6573797374656d53796e6365729493942981947d9428681b682268134d2c0168144d08078c116c6173745f73796e635f75705f74696d65944741d99eb7722637668c136c6173745f73796e635f646f776e5f74696d659447fff00000000000008c0d5f73796e635f70726f63657373944e8c0c5f63757272656e745f636d649468008c125f75706c6f61645f746f5f66735f706174689493947d94288c0a6c6f63616c5f70617468948c3b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c74732f746573745f6578706572696d656e74948c0266739468228c0766735f70617468948c552f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e74948c076578636c756465945d948c0e2a2f636865636b706f696e745f2a9461758694756275622e\"\n },\n \"config\": {\n \"encoder_num_layer\": 1,\n \"decoder_rt_num_layer\": 4,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 16,\n \"encoder_ff\": 1024,\n \"decoder_rt_ff\": 512,\n \"decoder_int_ff\": 512,\n \"n_head\": 4,\n \"drop_rate\": 0.22544213936627444,\n \"lr\": 0.0013157536168827595,\n \"batch_size\": 1024\n },\n \"_Trial__unresolved_config\": {\n \"encoder_num_layer\": 1,\n \"decoder_rt_num_layer\": 4,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 16,\n \"encoder_ff\": 1024,\n \"decoder_rt_ff\": 512,\n \"decoder_int_ff\": 512,\n \"n_head\": 4,\n \"drop_rate\": 0.22544213936627444,\n \"lr\": 0.0013157536168827595,\n \"batch_size\": 1024\n },\n \"evaluated_params\": {\n \"batch_size\": 1024,\n \"decoder_int_ff\": 512,\n \"decoder_int_num_layer\": 1,\n \"decoder_rt_ff\": 512,\n \"decoder_rt_num_layer\": 4,\n \"drop_rate\": 0.22544213936627444,\n \"embedding_dim\": 16,\n \"encoder_ff\": 1024,\n \"encoder_num_layer\": 1,\n \"lr\": 0.0013157536168827595,\n \"n_head\": 4\n },\n \"experiment_tag\": \"2_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=4,drop_rate=0.2254,embedding_dim=16,encoder_ff=1024,encoder_num_layer=1,lr=0.0013,n_head=4\",\n \"stopping_criterion\": {},\n \"_setup_default_resource\": true,\n \"_default_placement_group_factory\": \"80054e2e\",\n \"placement_group_factory\": \"800595ba000000000000008c237261792e74756e652e657865637574696f6e2e706c6163656d656e745f67726f757073948c15506c6163656d656e7447726f7570466163746f72799493942981947d94288c085f62756e646c6573945d947d94288c03435055944740540000000000008c034750559447401000000000000075618c155f686561645f62756e646c655f69735f656d70747994898c095f7374726174656779948c045041434b948c055f6172677394298c075f6b7761726773947d9475622e\",\n \"log_to_file\": [\n null,\n null\n ],\n \"max_failures\": 0,\n \"_default_result_or_future\": null,\n \"export_formats\": [],\n \"status\": \"PAUSED\",\n \"relative_logdir\": \"train_model_a00d3259_2_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=4,drop_ra_2024-06-25_17-10-13\",\n \"trial_name_creator\": null,\n \"trial_dirname_creator\": null,\n \"custom_trial_name\": null,\n \"custom_dirname\": null,\n \"restore_path\": null,\n \"_restore_checkpoint_result\": null,\n \"_state_json\": null,\n \"results\": \"80054e2e\",\n \"extra_arg\": \"80054e2e\",\n \"_resources\": \"80054e2e\"\n}",
+ "{\n \"start_time\": 1719328456.2145321,\n \"num_failures\": 0,\n \"num_failures_after_restore\": 0,\n \"error_filename\": null,\n \"pickled_error_filename\": null,\n \"last_result\": {\n \"loss\": 119.19234625748761,\n \"timestamp\": 1719332071,\n \"checkpoint_dir_name\": \"checkpoint_000007\",\n \"should_checkpoint\": true,\n \"done\": false,\n \"training_iteration\": 8,\n \"trial_id\": \"a00d3259\",\n \"date\": \"2024-06-25_18-14-31\",\n \"time_this_iter_s\": 58.22098970413208,\n \"time_total_s\": 537.5048625469208,\n \"pid\": 2195253,\n \"hostname\": \"r8i6n2\",\n \"node_ip\": \"10.159.28.60\",\n \"config\": {\n \"encoder_num_layer\": 1,\n \"decoder_rt_num_layer\": 4,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 16,\n \"encoder_ff\": 1024,\n \"decoder_rt_ff\": 512,\n \"decoder_int_ff\": 512,\n \"n_head\": 4,\n \"drop_rate\": 0.22544213936627444,\n \"lr\": 0.0013157536168827595,\n \"batch_size\": 1024\n },\n \"time_since_restore\": 246.92507433891296,\n \"iterations_since_restore\": 4,\n \"experiment_tag\": \"2_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=4,drop_rate=0.2254,embedding_dim=16,encoder_ff=1024,encoder_num_layer=1,lr=0.0013,n_head=4\"\n },\n \"last_result_time\": 1719332071.4645963,\n \"metric_analysis\": {\n \"loss\": {\n \"max\": 253.8278117743064,\n \"min\": 119.19234625748761,\n \"avg\": 155.9507788898438,\n \"last\": 119.19234625748761,\n \"last-5-avg\": 133.0517245735709,\n \"last-10-avg\": 155.9507788898438\n },\n \"should_checkpoint\": {\n \"max\": true,\n \"min\": true,\n \"avg\": 1.0,\n \"last\": true,\n \"last-5-avg\": 1.0,\n \"last-10-avg\": 1.0\n },\n \"done\": {\n \"max\": false,\n \"min\": false,\n \"avg\": 0.0,\n \"last\": false,\n \"last-5-avg\": 0.0,\n \"last-10-avg\": 0.0\n },\n \"training_iteration\": {\n \"max\": 8,\n \"min\": 1,\n \"avg\": 4.5,\n \"last\": 8,\n \"last-5-avg\": 6.0,\n \"last-10-avg\": 4.5\n },\n \"time_this_iter_s\": {\n \"max\": 88.16569352149963,\n \"min\": 58.202213287353516,\n \"avg\": 67.1881078183651,\n \"last\": 58.22098970413208,\n \"last-5-avg\": 61.9065589427948,\n \"last-10-avg\": 67.1881078183651\n },\n \"time_total_s\": {\n \"max\": 537.5048625469208,\n \"min\": 70.23847699165344,\n \"avg\": 316.16187810897827,\n \"last\": 537.5048625469208,\n \"last-5-avg\": 418.25562114715575,\n \"last-10-avg\": 316.16187810897827\n },\n \"time_since_restore\": {\n \"max\": 246.92507433891296,\n \"min\": 69.5678973197937,\n \"avg\": 127.14058080315588,\n \"last\": 246.92507433891296,\n \"last-5-avg\": 157.83051571846008,\n \"last-10-avg\": 127.1405808031559\n },\n \"iterations_since_restore\": {\n \"max\": 4,\n \"min\": 1,\n \"avg\": 1.875,\n \"last\": 4,\n \"last-5-avg\": 2.4,\n \"last-10-avg\": 1.875\n }\n },\n \"_n_steps\": [\n 5,\n 10\n ],\n \"metric_n_steps\": {\n \"loss\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"800595d6000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294288c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b0074946243080c068339a894614094869452946807680d4308bbdd6e5f00c0624094869452946807680d4308000000b8f85f604094869452946807680d430873b95c3efb1a5f4094869452946807680d4308b55aad664fcc5d409486945294652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005950f010000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294288c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b007494624308793c1e6f7dba6f4094869452946807680d430883c160c03b16654094869452946807680d43085fafd76b66fa634094869452946807680d43080c068339a894614094869452946807680d4308bbdd6e5f00c0624094869452946807680d4308000000b8f85f604094869452946807680d430873b95c3efb1a5f4094869452946807680d4308b55aad664fcc5d409486945294652e\"\n }\n },\n \"should_checkpoint\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059527000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294288888888888652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294288888888888888888652e\"\n }\n },\n \"done\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059527000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294288989898989652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294288989898989898989652e\"\n }\n },\n \"training_iteration\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952c000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294284b044b054b064b074b08652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059532000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294284b014b024b034b044b054b064b074b08652e\"\n }\n },\n \"time_this_iter_s\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005954f000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452942847404f4dc9c800000047405211878400000047404d19e22000000047404d1d2e4a00000047404d1c4964000000652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005956a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294284740518f433500000047405164586e0000004740560a9ab900000047404f4dc9c800000047405211878400000047404d19e22000000047404d1d2e4a00000047404d1c4964000000652e\"\n }\n },\n \"time_total_s\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005954f000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294284740722946d0000000474076ada8b100000047407a50e4f500000047407df48abe400000474080cc09f5600000652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005956a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294284740518f433500000047406179cdd180000047406c7f1b2e0000004740722946d0000000474076ada8b100000047407a50e4f500000047407df48abe400000474080cc09f5600000652e\"\n }\n },\n \"time_since_restore\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005954f000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b058694529428474062d8bfce8000004740521187840000004740604f3c4a0000004740679687dc80000047406edd9a35800000652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005956a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294284740518f433500000047405164586e0000004740560a9ab9000000474062d8bfce8000004740521187840000004740604f3c4a0000004740679687dc80000047406edd9a35800000652e\"\n }\n },\n \"iterations_since_restore\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952c000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294284b024b014b024b034b04652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059532000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294284b014b014b014b024b014b024b034b04652e\"\n }\n }\n },\n \"checkpoint_manager\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"800595f41e0000000000008c267261792e747261696e2e5f696e7465726e616c2e636865636b706f696e745f6d616e61676572948c125f436865636b706f696e744d616e616765729493942981947d94288c125f636865636b706f696e745f636f6e666967948c097261792e747261696e948c10436865636b706f696e74436f6e6669679493942981947d94288c0b6e756d5f746f5f6b656570944e8c1a636865636b706f696e745f73636f72655f617474726962757465944e8c16636865636b706f696e745f73636f72655f6f72646572948c036d6178948c14636865636b706f696e745f6672657175656e6379944b008c11636865636b706f696e745f61745f656e6494898c1a5f636865636b706f696e745f6b6565705f616c6c5f72616e6b73948c0a44455052454341544544948c1f5f636865636b706f696e745f75706c6f61645f66726f6d5f776f726b65727394681275628c135f636865636b706f696e745f726573756c7473945d94288c1b7261792e747261696e2e5f696e7465726e616c2e73657373696f6e948c0f5f547261696e696e67526573756c749493942981947d94288c0a636865636b706f696e749468068c0a436865636b706f696e749493942981947d94288c0470617468948cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f61303064333235395f325f62617463685f73697a653d313032342c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d3531322c6465636f6465725f72745f6e756d5f6c617965723d342c64726f705f72615f323032342d30362d32355f31372d31302d31332f636865636b706f696e745f303030303030948c0a66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c055f75756964948c0475756964948c04555549449493942981947d948c03696e74948a110e9e5dc1217ec9bebd476cec37a884c400736275628c076d657472696373947d94288c046c6f7373948c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b007494624308793c1e6f7dba6f4094869452948c0974696d657374616d70944a0edf7a668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303030948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b018c08747269616c5f6964948c086130306433323539948c0464617465948c13323032342d30362d32355f31372d31352d3236948c1074696d655f746869735f697465725f73944740518f43350000008c0c74696d655f746f74616c5f73944740518f43350000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b018c146465636f6465725f72745f6e756d5f6c61796572944b048c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b108c0a656e636f6465725f6666944d00048c0d6465636f6465725f72745f6666944d00028c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b048c0964726f705f7261746594473fccdb49bbdbf5488c026c7294473f558eabb04342508c0a62617463685f73697a65944d0004758c1274696d655f73696e63655f726573746f7265944740518f43350000008c18697465726174696f6e735f73696e63655f726573746f7265944b0175756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f61303064333235395f325f62617463685f73697a653d313032342c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d3531322c6465636f6465725f72745f6e756d5f6c617965723d342c64726f705f72615f323032342d30362d32355f31372d31302d31332f636865636b706f696e745f30303030303194682268257d946827897385945294682a682d2981947d9468308a11cc42bb71e00b7b97494800e3d3acd9b2007362756268317d94288c046c6f7373946836683c430883c160c03b16654094869452948c0974696d657374616d70944a1be77a668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303031948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b028c08747269616c5f6964948c086130306433323539948c0464617465948c13323032342d30362d32355f31372d34392d3437948c1074696d655f746869735f697465725f739447405164586e0000008c0c74696d655f746f74616c5f739447406179cdd18000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b018c146465636f6465725f72745f6e756d5f6c61796572944b048c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b108c0a656e636f6465725f6666944d00048c0d6465636f6465725f72745f6666944d00028c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b048c0964726f705f7261746594473fccdb49bbdbf5488c026c7294473f558eabb04342508c0a62617463685f73697a65944d0004758c1274696d655f73696e63655f726573746f72659447405164586e0000008c18697465726174696f6e735f73696e63655f726573746f7265944b0175756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f61303064333235395f325f62617463685f73697a653d313032342c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d3531322c6465636f6465725f72745f6e756d5f6c617965723d342c64726f705f72615f323032342d30362d32355f31372d31302d31332f636865636b706f696e745f30303030303294682268257d946827897385945294682a682d2981947d9468308a10141deb64e2f047a7ca4b06bf191963437362756268317d94288c046c6f7373946836683c43085fafd76b66fa634094869452948c0974696d657374616d70944a93e97a668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303032948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b038c08747269616c5f6964948c086130306433323539948c0464617465948c13323032342d30362d32355f31382d30302d3139948c1074696d655f746869735f697465725f73944740560a9ab90000008c0c74696d655f746f74616c5f739447406c7f1b2e0000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b018c146465636f6465725f72745f6e756d5f6c61796572944b048c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b108c0a656e636f6465725f6666944d00048c0d6465636f6465725f72745f6666944d00028c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b048c0964726f705f7261746594473fccdb49bbdbf5488c026c7294473f558eabb04342508c0a62617463685f73697a65944d0004758c1274696d655f73696e63655f726573746f7265944740560a9ab90000008c18697465726174696f6e735f73696e63655f726573746f7265944b0175756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f61303064333235395f325f62617463685f73697a653d313032342c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d3531322c6465636f6465725f72745f6e756d5f6c617965723d342c64726f705f72615f323032342d30362d32355f31372d31302d31332f636865636b706f696e745f30303030303394682268257d946827897385945294682a682d2981947d9468308a119f5735580a8fcb92b74321bfa361c4b9007362756268317d94288c046c6f7373946836683c43080c068339a894614094869452948c0974696d657374616d70944ad2e97a668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303033948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b048c08747269616c5f6964948c086130306433323539948c0464617465948c13323032342d30362d32355f31382d30312d3232948c1074696d655f746869735f697465725f739447404f4dc9c80000008c0c74696d655f746f74616c5f73944740722946d00000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b018c146465636f6465725f72745f6e756d5f6c61796572944b048c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b108c0a656e636f6465725f6666944d00048c0d6465636f6465725f72745f6666944d00028c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b048c0964726f705f7261746594473fccdb49bbdbf5488c026c7294473f558eabb04342508c0a62617463685f73697a65944d0004758c1274696d655f73696e63655f726573746f726594474062d8bfce8000008c18697465726174696f6e735f73696e63655f726573746f7265944b0275756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f61303064333235395f325f62617463685f73697a653d313032342c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d3531322c6465636f6465725f72745f6e756d5f6c617965723d342c64726f705f72615f323032342d30362d32355f31372d31302d31332f636865636b706f696e745f30303030303494682268257d946827897385945294682a682d2981947d9468308a1178391f217b7e0e8b8e484fe617cc4aa7007362756268317d94288c046c6f7373946836683c4308bbdd6e5f00c0624094869452948c0974696d657374616d70944a38ec7a668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303034948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b058c08747269616c5f6964948c086130306433323539948c0464617465948c13323032342d30362d32355f31382d31312d3336948c1074696d655f746869735f697465725f73944740521187840000008c0c74696d655f746f74616c5f7394474076ada8b10000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b018c146465636f6465725f72745f6e756d5f6c61796572944b048c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b108c0a656e636f6465725f6666944d00048c0d6465636f6465725f72745f6666944d00028c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b048c0964726f705f7261746594473fccdb49bbdbf5488c026c7294473f558eabb04342508c0a62617463685f73697a65944d0004758c1274696d655f73696e63655f726573746f7265944740521187840000008c18697465726174696f6e735f73696e63655f726573746f7265944b0175756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f61303064333235395f325f62617463685f73697a653d313032342c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d3531322c6465636f6465725f72745f6e756d5f6c617965723d342c64726f705f72615f323032342d30362d32355f31372d31302d31332f636865636b706f696e745f30303030303594682268257d946827897385945294682a682d2981947d9468308a11b01bc90fcfb3849519437ed644c87283007362756268317d94288c046c6f7373946836683c4308000000b8f85f604094869452948c0974696d657374616d70944a72ec7a668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303035948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b068c08747269616c5f6964948c086130306433323539948c0464617465948c13323032342d30362d32355f31382d31322d3334948c1074696d655f746869735f697465725f739447404d19e2200000008c0c74696d655f746f74616c5f739447407a50e4f50000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b018c146465636f6465725f72745f6e756d5f6c61796572944b048c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b108c0a656e636f6465725f6666944d00048c0d6465636f6465725f72745f6666944d00028c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b048c0964726f705f7261746594473fccdb49bbdbf5488c026c7294473f558eabb04342508c0a62617463685f73697a65944d0004758c1274696d655f73696e63655f726573746f7265944740604f3c4a0000008c18697465726174696f6e735f73696e63655f726573746f7265944b0275756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f61303064333235395f325f62617463685f73697a653d313032342c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d3531322c6465636f6465725f72745f6e756d5f6c617965723d342c64726f705f72615f323032342d30362d32355f31372d31302d31332f636865636b706f696e745f30303030303694682268257d946827897385945294682a682d2981947d9468308a100e2bc27ee9b7d8baec4b607719848f1a7362756268317d94288c046c6f7373946836683c430873b95c3efb1a5f4094869452948c0974696d657374616d70944aadec7a668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303036948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b078c08747269616c5f6964948c086130306433323539948c0464617465948c13323032342d30362d32355f31382d31332d3333948c1074696d655f746869735f697465725f739447404d1d2e4a0000008c0c74696d655f746f74616c5f739447407df48abe4000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b018c146465636f6465725f72745f6e756d5f6c61796572944b048c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b108c0a656e636f6465725f6666944d00048c0d6465636f6465725f72745f6666944d00028c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b048c0964726f705f7261746594473fccdb49bbdbf5488c026c7294473f558eabb04342508c0a62617463685f73697a65944d0004758c1274696d655f73696e63655f726573746f7265944740679687dc8000008c18697465726174696f6e735f73696e63655f726573746f7265944b0375756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f61303064333235395f325f62617463685f73697a653d313032342c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d3531322c6465636f6465725f72745f6e756d5f6c617965723d342c64726f705f72615f323032342d30362d32355f31372d31302d31332f636865636b706f696e745f30303030303794682268257d946827897385945294682a682d2981947d9468308a10ede47fd5253ab29d7647405ee5896a327362756268317d94288c046c6f7373946836683c4308b55aad664fcc5d4094869452948c0974696d657374616d70944ae7ec7a668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303037948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b088c08747269616c5f6964948c086130306433323539948c0464617465948c13323032342d30362d32355f31382d31342d3331948c1074696d655f746869735f697465725f739447404d1c49640000008c0c74696d655f746f74616c5f7394474080cc09f56000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b018c146465636f6465725f72745f6e756d5f6c61796572944b048c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b108c0a656e636f6465725f6666944d00048c0d6465636f6465725f72745f6666944d00028c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b048c0964726f705f7261746594473fccdb49bbdbf5488c026c7294473f558eabb04342508c0a62617463685f73697a65944d0004758c1274696d655f73696e63655f726573746f72659447406edd9a358000008c18697465726174696f6e735f73696e63655f726573746f7265944b04757562658c195f6c61746573745f636865636b706f696e745f726573756c74946a7c01000075622e\"\n }\n}"
+ ],
+ [
+ "{\n \"stub\": false,\n \"trainable_name\": \"train_model\",\n \"trial_id\": \"7ddcd025\",\n \"storage\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059572040000000000008c1b7261792e747261696e2e5f696e7465726e616c2e73746f72616765948c0e53746f72616765436f6e746578749493942981947d94288c12637573746f6d5f66735f70726f766964656494898c1273746f726167655f6c6f63616c5f70617468948c2b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c7473948c136578706572696d656e745f6469725f6e616d65948c0f746573745f6578706572696d656e74948c0e747269616c5f6469725f6e616d65948c96747261696e5f6d6f64656c5f37646463643032355f335f62617463685f73697a653d313032342c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d3531322c6465636f6465725f72745f6e756d5f6c617965723d382c64726f705f72615f323032342d30362d32355f31372d31342d3136948c1863757272656e745f636865636b706f696e745f696e646578944b008c0b73796e635f636f6e666967948c097261792e747261696e948c0a53796e63436f6e6669679493942981947d94288c0b73796e635f706572696f64944d2c018c0c73796e635f74696d656f7574944d08078c0e73796e635f61727469666163747394898c1c73796e635f6172746966616374735f6f6e5f636865636b706f696e7494888c0a75706c6f61645f646972948c0a44455052454341544544948c0673796e6365729468188c1273796e635f6f6e5f636865636b706f696e7494681875628c1273746f726167655f66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c0f73746f726167655f66735f70617468948c452f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f7465737494681968008c115f46696c6573797374656d53796e6365729493942981947d9428681b682268134d2c0168144d08078c116c6173745f73796e635f75705f74696d65944741d99eb7722637668c136c6173745f73796e635f646f776e5f74696d659447fff00000000000008c0d5f73796e635f70726f63657373944e8c0c5f63757272656e745f636d649468008c125f75706c6f61645f746f5f66735f706174689493947d94288c0a6c6f63616c5f70617468948c3b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c74732f746573745f6578706572696d656e74948c0266739468228c0766735f70617468948c552f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e74948c076578636c756465945d948c0e2a2f636865636b706f696e745f2a9461758694756275622e\"\n },\n \"config\": {\n \"encoder_num_layer\": 2,\n \"decoder_rt_num_layer\": 8,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 64,\n \"encoder_ff\": 2048,\n \"decoder_rt_ff\": 512,\n \"decoder_int_ff\": 512,\n \"n_head\": 1,\n \"drop_rate\": 0.474834603771515,\n \"lr\": 0.015369204694575145,\n \"batch_size\": 1024\n },\n \"_Trial__unresolved_config\": {\n \"encoder_num_layer\": 2,\n \"decoder_rt_num_layer\": 8,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 64,\n \"encoder_ff\": 2048,\n \"decoder_rt_ff\": 512,\n \"decoder_int_ff\": 512,\n \"n_head\": 1,\n \"drop_rate\": 0.474834603771515,\n \"lr\": 0.015369204694575145,\n \"batch_size\": 1024\n },\n \"evaluated_params\": {\n \"batch_size\": 1024,\n \"decoder_int_ff\": 512,\n \"decoder_int_num_layer\": 1,\n \"decoder_rt_ff\": 512,\n \"decoder_rt_num_layer\": 8,\n \"drop_rate\": 0.474834603771515,\n \"embedding_dim\": 64,\n \"encoder_ff\": 2048,\n \"encoder_num_layer\": 2,\n \"lr\": 0.015369204694575145,\n \"n_head\": 1\n },\n \"experiment_tag\": \"3_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=8,drop_rate=0.4748,embedding_dim=64,encoder_ff=2048,encoder_num_layer=2,lr=0.0154,n_head=1\",\n \"stopping_criterion\": {},\n \"_setup_default_resource\": true,\n \"_default_placement_group_factory\": \"80054e2e\",\n \"placement_group_factory\": \"800595ba000000000000008c237261792e74756e652e657865637574696f6e2e706c6163656d656e745f67726f757073948c15506c6163656d656e7447726f7570466163746f72799493942981947d94288c085f62756e646c6573945d947d94288c03435055944740540000000000008c034750559447401000000000000075618c155f686561645f62756e646c655f69735f656d70747994898c095f7374726174656779948c045041434b948c055f6172677394298c075f6b7761726773947d9475622e\",\n \"log_to_file\": [\n null,\n null\n ],\n \"max_failures\": 0,\n \"_default_result_or_future\": null,\n \"export_formats\": [],\n \"status\": \"PAUSED\",\n \"relative_logdir\": \"train_model_7ddcd025_3_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=8,drop_ra_2024-06-25_17-14-16\",\n \"trial_name_creator\": null,\n \"trial_dirname_creator\": null,\n \"custom_trial_name\": null,\n \"custom_dirname\": null,\n \"restore_path\": null,\n \"_restore_checkpoint_result\": null,\n \"_state_json\": null,\n \"results\": \"80054e2e\",\n \"extra_arg\": \"80054e2e\",\n \"_resources\": \"80054e2e\"\n}",
+ "{\n \"start_time\": 1719328526.6072588,\n \"num_failures\": 0,\n \"num_failures_after_restore\": 0,\n \"error_filename\": null,\n \"pickled_error_filename\": null,\n \"last_result\": {\n \"loss\": 2079.0407700425994,\n \"timestamp\": 1719328617,\n \"checkpoint_dir_name\": \"checkpoint_000000\",\n \"should_checkpoint\": true,\n \"done\": false,\n \"training_iteration\": 1,\n \"trial_id\": \"7ddcd025\",\n \"date\": \"2024-06-25_17-16-57\",\n \"time_this_iter_s\": 90.59219408035278,\n \"time_total_s\": 90.59219408035278,\n \"pid\": 2195253,\n \"hostname\": \"r8i6n2\",\n \"node_ip\": \"10.159.28.60\",\n \"config\": {\n \"encoder_num_layer\": 2,\n \"decoder_rt_num_layer\": 8,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 64,\n \"encoder_ff\": 2048,\n \"decoder_rt_ff\": 512,\n \"decoder_int_ff\": 512,\n \"n_head\": 1,\n \"drop_rate\": 0.474834603771515,\n \"lr\": 0.015369204694575145,\n \"batch_size\": 1024\n },\n \"time_since_restore\": 90.59219408035278,\n \"iterations_since_restore\": 1,\n \"experiment_tag\": \"3_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=8,drop_rate=0.4748,embedding_dim=64,encoder_ff=2048,encoder_num_layer=2,lr=0.0154,n_head=1\"\n },\n \"last_result_time\": 1719328617.2057276,\n \"metric_analysis\": {\n \"loss\": {\n \"max\": 2079.0407700425994,\n \"min\": 2079.0407700425994,\n \"avg\": 2079.0407700425994,\n \"last\": 2079.0407700425994,\n \"last-5-avg\": 2079.0407700425994,\n \"last-10-avg\": 2079.0407700425994\n },\n \"should_checkpoint\": {\n \"max\": true,\n \"min\": true,\n \"avg\": true,\n \"last\": true,\n \"last-5-avg\": true,\n \"last-10-avg\": true\n },\n \"done\": {\n \"max\": false,\n \"min\": false,\n \"avg\": false,\n \"last\": false,\n \"last-5-avg\": false,\n \"last-10-avg\": false\n },\n \"training_iteration\": {\n \"max\": 1,\n \"min\": 1,\n \"avg\": 1,\n \"last\": 1,\n \"last-5-avg\": 1,\n \"last-10-avg\": 1\n },\n \"time_this_iter_s\": {\n \"max\": 90.59219408035278,\n \"min\": 90.59219408035278,\n \"avg\": 90.59219408035278,\n \"last\": 90.59219408035278,\n \"last-5-avg\": 90.59219408035278,\n \"last-10-avg\": 90.59219408035278\n },\n \"time_total_s\": {\n \"max\": 90.59219408035278,\n \"min\": 90.59219408035278,\n \"avg\": 90.59219408035278,\n \"last\": 90.59219408035278,\n \"last-5-avg\": 90.59219408035278,\n \"last-10-avg\": 90.59219408035278\n },\n \"time_since_restore\": {\n \"max\": 90.59219408035278,\n \"min\": 90.59219408035278,\n \"avg\": 90.59219408035278,\n \"last\": 90.59219408035278,\n \"last-5-avg\": 90.59219408035278,\n \"last-10-avg\": 90.59219408035278\n },\n \"iterations_since_restore\": {\n \"max\": 1,\n \"min\": 1,\n \"avg\": 1,\n \"last\": 1,\n \"last-5-avg\": 1,\n \"last-10-avg\": 1\n }\n },\n \"_n_steps\": [\n 5,\n 10\n ],\n \"metric_n_steps\": {\n \"loss\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059589000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452948c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b0074946243083e9fcfdf143ea0409486945294612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059589000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452948c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b0074946243083e9fcfdf143ea0409486945294612e\"\n }\n },\n \"should_checkpoint\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059522000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b058694529488612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059522000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a8694529488612e\"\n }\n },\n \"done\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059522000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b058694529489612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059522000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a8694529489612e\"\n }\n },\n \"training_iteration\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059523000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452944b01612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059523000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452944b01612e\"\n }\n },\n \"time_this_iter_s\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294474056a5e682000000612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294474056a5e682000000612e\"\n }\n },\n \"time_total_s\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294474056a5e682000000612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294474056a5e682000000612e\"\n }\n },\n \"time_since_restore\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294474056a5e682000000612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294474056a5e682000000612e\"\n }\n },\n \"iterations_since_restore\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059523000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452944b01612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059523000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452944b01612e\"\n }\n }\n },\n \"checkpoint_manager\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059517060000000000008c267261792e747261696e2e5f696e7465726e616c2e636865636b706f696e745f6d616e61676572948c125f436865636b706f696e744d616e616765729493942981947d94288c125f636865636b706f696e745f636f6e666967948c097261792e747261696e948c10436865636b706f696e74436f6e6669679493942981947d94288c0b6e756d5f746f5f6b656570944e8c1a636865636b706f696e745f73636f72655f617474726962757465944e8c16636865636b706f696e745f73636f72655f6f72646572948c036d6178948c14636865636b706f696e745f6672657175656e6379944b008c11636865636b706f696e745f61745f656e6494898c1a5f636865636b706f696e745f6b6565705f616c6c5f72616e6b73948c0a44455052454341544544948c1f5f636865636b706f696e745f75706c6f61645f66726f6d5f776f726b65727394681275628c135f636865636b706f696e745f726573756c7473945d948c1b7261792e747261696e2e5f696e7465726e616c2e73657373696f6e948c0f5f547261696e696e67526573756c749493942981947d94288c0a636865636b706f696e749468068c0a436865636b706f696e749493942981947d94288c0470617468948cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f37646463643032355f335f62617463685f73697a653d313032342c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d3531322c6465636f6465725f72745f6e756d5f6c617965723d382c64726f705f72615f323032342d30362d32355f31372d31342d31362f636865636b706f696e745f303030303030948c0a66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c055f75756964948c0475756964948c04555549449493942981947d948c03696e74948a105ac6f07c076b1b94404e03bf2719e871736275628c076d657472696373947d94288c046c6f7373948c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b0074946243083e9fcfdf143ea04094869452948c0974696d657374616d70944a69df7a668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303030948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b018c08747269616c5f6964948c083764646364303235948c0464617465948c13323032342d30362d32355f31372d31362d3537948c1074696d655f746869735f697465725f7394474056a5e6820000008c0c74696d655f746f74616c5f7394474056a5e6820000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b028c146465636f6465725f72745f6e756d5f6c61796572944b088c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b408c0a656e636f6465725f6666944d00088c0d6465636f6465725f72745f6666944d00028c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b018c0964726f705f7261746594473fde63b0ad8d4c308c026c7294473f8f79e3bc3ad6c08c0a62617463685f73697a65944d0004758c1274696d655f73696e63655f726573746f726594474056a5e6820000008c18697465726174696f6e735f73696e63655f726573746f7265944b01757562618c195f6c61746573745f636865636b706f696e745f726573756c7494681975622e\"\n }\n}"
+ ],
+ [
+ "{\n \"stub\": false,\n \"trainable_name\": \"train_model\",\n \"trial_id\": \"7dfe67c7\",\n \"storage\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059572040000000000008c1b7261792e747261696e2e5f696e7465726e616c2e73746f72616765948c0e53746f72616765436f6e746578749493942981947d94288c12637573746f6d5f66735f70726f766964656494898c1273746f726167655f6c6f63616c5f70617468948c2b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c7473948c136578706572696d656e745f6469725f6e616d65948c0f746573745f6578706572696d656e74948c0e747269616c5f6469725f6e616d65948c96747261696e5f6d6f64656c5f37646665363763375f345f62617463685f73697a653d313032342c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d323034382c6465636f6465725f72745f6e756d5f6c617965723d322c64726f705f725f323032342d30362d32355f31372d31352d3236948c1863757272656e745f636865636b706f696e745f696e646578944b008c0b73796e635f636f6e666967948c097261792e747261696e948c0a53796e63436f6e6669679493942981947d94288c0b73796e635f706572696f64944d2c018c0c73796e635f74696d656f7574944d08078c0e73796e635f61727469666163747394898c1c73796e635f6172746966616374735f6f6e5f636865636b706f696e7494888c0a75706c6f61645f646972948c0a44455052454341544544948c0673796e6365729468188c1273796e635f6f6e5f636865636b706f696e7494681875628c1273746f726167655f66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c0f73746f726167655f66735f70617468948c452f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f7465737494681968008c115f46696c6573797374656d53796e6365729493942981947d9428681b682268134d2c0168144d08078c116c6173745f73796e635f75705f74696d65944741d99eb7bda3c2988c136c6173745f73796e635f646f776e5f74696d659447fff00000000000008c0d5f73796e635f70726f63657373944e8c0c5f63757272656e745f636d649468008c125f75706c6f61645f746f5f66735f706174689493947d94288c0a6c6f63616c5f70617468948c3b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c74732f746573745f6578706572696d656e74948c0266739468228c0766735f70617468948c552f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e74948c076578636c756465945d948c0e2a2f636865636b706f696e745f2a9461758694756275622e\"\n },\n \"config\": {\n \"encoder_num_layer\": 2,\n \"decoder_rt_num_layer\": 2,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 256,\n \"encoder_ff\": 2048,\n \"decoder_rt_ff\": 2048,\n \"decoder_int_ff\": 512,\n \"n_head\": 2,\n \"drop_rate\": 0.6066107852445892,\n \"lr\": 0.0011313709304158431,\n \"batch_size\": 1024\n },\n \"_Trial__unresolved_config\": {\n \"encoder_num_layer\": 2,\n \"decoder_rt_num_layer\": 2,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 256,\n \"encoder_ff\": 2048,\n \"decoder_rt_ff\": 2048,\n \"decoder_int_ff\": 512,\n \"n_head\": 2,\n \"drop_rate\": 0.6066107852445892,\n \"lr\": 0.0011313709304158431,\n \"batch_size\": 1024\n },\n \"evaluated_params\": {\n \"batch_size\": 1024,\n \"decoder_int_ff\": 512,\n \"decoder_int_num_layer\": 1,\n \"decoder_rt_ff\": 2048,\n \"decoder_rt_num_layer\": 2,\n \"drop_rate\": 0.6066107852445892,\n \"embedding_dim\": 256,\n \"encoder_ff\": 2048,\n \"encoder_num_layer\": 2,\n \"lr\": 0.0011313709304158431,\n \"n_head\": 2\n },\n \"experiment_tag\": \"4_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=2,drop_rate=0.6066,embedding_dim=256,encoder_ff=2048,encoder_num_layer=2,lr=0.0011,n_head=2\",\n \"stopping_criterion\": {},\n \"_setup_default_resource\": true,\n \"_default_placement_group_factory\": \"80054e2e\",\n \"placement_group_factory\": \"800595ba000000000000008c237261792e74756e652e657865637574696f6e2e706c6163656d656e745f67726f757073948c15506c6163656d656e7447726f7570466163746f72799493942981947d94288c085f62756e646c6573945d947d94288c03435055944740540000000000008c034750559447401000000000000075618c155f686561645f62756e646c655f69735f656d70747994898c095f7374726174656779948c045041434b948c055f6172677394298c075f6b7761726773947d9475622e\",\n \"log_to_file\": [\n null,\n null\n ],\n \"max_failures\": 0,\n \"_default_result_or_future\": null,\n \"export_formats\": [],\n \"status\": \"PAUSED\",\n \"relative_logdir\": \"train_model_7dfe67c7_4_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=2,drop_r_2024-06-25_17-15-26\",\n \"trial_name_creator\": null,\n \"trial_dirname_creator\": null,\n \"custom_trial_name\": null,\n \"custom_dirname\": null,\n \"restore_path\": null,\n \"_restore_checkpoint_result\": null,\n \"_state_json\": null,\n \"results\": \"80054e2e\",\n \"extra_arg\": \"80054e2e\",\n \"_resources\": \"80054e2e\"\n}",
+ "{\n \"start_time\": 1719328617.3522246,\n \"num_failures\": 0,\n \"num_failures_after_restore\": 0,\n \"error_filename\": null,\n \"pickled_error_filename\": null,\n \"last_result\": {\n \"loss\": 1979.0773276982345,\n \"timestamp\": 1719328732,\n \"checkpoint_dir_name\": \"checkpoint_000000\",\n \"should_checkpoint\": true,\n \"done\": false,\n \"training_iteration\": 1,\n \"trial_id\": \"7dfe67c7\",\n \"date\": \"2024-06-25_17-18-52\",\n \"time_this_iter_s\": 115.28918504714966,\n \"time_total_s\": 115.28918504714966,\n \"pid\": 2195253,\n \"hostname\": \"r8i6n2\",\n \"node_ip\": \"10.159.28.60\",\n \"config\": {\n \"encoder_num_layer\": 2,\n \"decoder_rt_num_layer\": 2,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 256,\n \"encoder_ff\": 2048,\n \"decoder_rt_ff\": 2048,\n \"decoder_int_ff\": 512,\n \"n_head\": 2,\n \"drop_rate\": 0.6066107852445892,\n \"lr\": 0.0011313709304158431,\n \"batch_size\": 1024\n },\n \"time_since_restore\": 115.28918504714966,\n \"iterations_since_restore\": 1,\n \"experiment_tag\": \"4_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=2,drop_rate=0.6066,embedding_dim=256,encoder_ff=2048,encoder_num_layer=2,lr=0.0011,n_head=2\"\n },\n \"last_result_time\": 1719328732.6464097,\n \"metric_analysis\": {\n \"loss\": {\n \"max\": 1979.0773276982345,\n \"min\": 1979.0773276982345,\n \"avg\": 1979.0773276982345,\n \"last\": 1979.0773276982345,\n \"last-5-avg\": 1979.0773276982345,\n \"last-10-avg\": 1979.0773276982345\n },\n \"should_checkpoint\": {\n \"max\": true,\n \"min\": true,\n \"avg\": true,\n \"last\": true,\n \"last-5-avg\": true,\n \"last-10-avg\": true\n },\n \"done\": {\n \"max\": false,\n \"min\": false,\n \"avg\": false,\n \"last\": false,\n \"last-5-avg\": false,\n \"last-10-avg\": false\n },\n \"training_iteration\": {\n \"max\": 1,\n \"min\": 1,\n \"avg\": 1,\n \"last\": 1,\n \"last-5-avg\": 1,\n \"last-10-avg\": 1\n },\n \"time_this_iter_s\": {\n \"max\": 115.28918504714966,\n \"min\": 115.28918504714966,\n \"avg\": 115.28918504714966,\n \"last\": 115.28918504714966,\n \"last-5-avg\": 115.28918504714966,\n \"last-10-avg\": 115.28918504714966\n },\n \"time_total_s\": {\n \"max\": 115.28918504714966,\n \"min\": 115.28918504714966,\n \"avg\": 115.28918504714966,\n \"last\": 115.28918504714966,\n \"last-5-avg\": 115.28918504714966,\n \"last-10-avg\": 115.28918504714966\n },\n \"time_since_restore\": {\n \"max\": 115.28918504714966,\n \"min\": 115.28918504714966,\n \"avg\": 115.28918504714966,\n \"last\": 115.28918504714966,\n \"last-5-avg\": 115.28918504714966,\n \"last-10-avg\": 115.28918504714966\n },\n \"iterations_since_restore\": {\n \"max\": 1,\n \"min\": 1,\n \"avg\": 1,\n \"last\": 1,\n \"last-5-avg\": 1,\n \"last-10-avg\": 1\n }\n },\n \"_n_steps\": [\n 5,\n 10\n ],\n \"metric_n_steps\": {\n \"loss\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059589000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452948c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b007494624308f8fbfd2e4fec9e409486945294612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059589000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452948c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b007494624308f8fbfd2e4fec9e409486945294612e\"\n }\n },\n \"should_checkpoint\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059522000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b058694529488612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059522000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a8694529488612e\"\n }\n },\n \"done\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059522000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b058694529489612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059522000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a8694529489612e\"\n }\n },\n \"training_iteration\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059523000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452944b01612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059523000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452944b01612e\"\n }\n },\n \"time_this_iter_s\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b058694529447405cd28202000000612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a8694529447405cd28202000000612e\"\n }\n },\n \"time_total_s\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b058694529447405cd28202000000612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a8694529447405cd28202000000612e\"\n }\n },\n \"time_since_restore\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b058694529447405cd28202000000612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a8694529447405cd28202000000612e\"\n }\n },\n \"iterations_since_restore\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059523000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452944b01612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059523000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452944b01612e\"\n }\n }\n },\n \"checkpoint_manager\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059518060000000000008c267261792e747261696e2e5f696e7465726e616c2e636865636b706f696e745f6d616e61676572948c125f436865636b706f696e744d616e616765729493942981947d94288c125f636865636b706f696e745f636f6e666967948c097261792e747261696e948c10436865636b706f696e74436f6e6669679493942981947d94288c0b6e756d5f746f5f6b656570944e8c1a636865636b706f696e745f73636f72655f617474726962757465944e8c16636865636b706f696e745f73636f72655f6f72646572948c036d6178948c14636865636b706f696e745f6672657175656e6379944b008c11636865636b706f696e745f61745f656e6494898c1a5f636865636b706f696e745f6b6565705f616c6c5f72616e6b73948c0a44455052454341544544948c1f5f636865636b706f696e745f75706c6f61645f66726f6d5f776f726b65727394681275628c135f636865636b706f696e745f726573756c7473945d948c1b7261792e747261696e2e5f696e7465726e616c2e73657373696f6e948c0f5f547261696e696e67526573756c749493942981947d94288c0a636865636b706f696e749468068c0a436865636b706f696e749493942981947d94288c0470617468948cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f37646665363763375f345f62617463685f73697a653d313032342c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d323034382c6465636f6465725f72745f6e756d5f6c617965723d322c64726f705f725f323032342d30362d32355f31372d31352d32362f636865636b706f696e745f303030303030948c0a66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c055f75756964948c0475756964948c04555549449493942981947d948c03696e74948a10d00a7865e6fde98160434a4fa02a8c2c736275628c076d657472696373947d94288c046c6f7373948c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b007494624308f8fbfd2e4fec9e4094869452948c0974696d657374616d70944adcdf7a668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303030948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b018c08747269616c5f6964948c083764666536376337948c0464617465948c13323032342d30362d32355f31372d31382d3532948c1074696d655f746869735f697465725f739447405cd282020000008c0c74696d655f746f74616c5f739447405cd282020000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b028c146465636f6465725f72745f6e756d5f6c61796572944b028c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944d00018c0a656e636f6465725f6666944d00088c0d6465636f6465725f72745f6666944d00088c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b028c0964726f705f7261746594473fe3695b0580d8308c026c7294473f52895049547a698c0a62617463685f73697a65944d0004758c1274696d655f73696e63655f726573746f72659447405cd282020000008c18697465726174696f6e735f73696e63655f726573746f7265944b01757562618c195f6c61746573745f636865636b706f696e745f726573756c7494681975622e\"\n }\n}"
+ ],
+ [
+ "{\n \"stub\": false,\n \"trainable_name\": \"train_model\",\n \"trial_id\": \"f1fbbaf2\",\n \"storage\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059572040000000000008c1b7261792e747261696e2e5f696e7465726e616c2e73746f72616765948c0e53746f72616765436f6e746578749493942981947d94288c12637573746f6d5f66735f70726f766964656494898c1273746f726167655f6c6f63616c5f70617468948c2b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c7473948c136578706572696d656e745f6469725f6e616d65948c0f746573745f6578706572696d656e74948c0e747269616c5f6469725f6e616d65948c96747261696e5f6d6f64656c5f66316662626166325f355f62617463685f73697a653d313032342c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d313032342c6465636f6465725f72745f6e756d5f6c617965723d322c64726f705f725f323032342d30362d32355f31372d31362d3537948c1863757272656e745f636865636b706f696e745f696e646578944b008c0b73796e635f636f6e666967948c097261792e747261696e948c0a53796e63436f6e6669679493942981947d94288c0b73796e635f706572696f64944d2c018c0c73796e635f74696d656f7574944d08078c0e73796e635f61727469666163747394898c1c73796e635f6172746966616374735f6f6e5f636865636b706f696e7494888c0a75706c6f61645f646972948c0a44455052454341544544948c0673796e6365729468188c1273796e635f6f6e5f636865636b706f696e7494681875628c1273746f726167655f66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c0f73746f726167655f66735f70617468948c452f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f7465737494681968008c115f46696c6573797374656d53796e6365729493942981947d9428681b682268134d2c0168144d08078c116c6173745f73796e635f75705f74696d65944741d99eb7bda3c2988c136c6173745f73796e635f646f776e5f74696d659447fff00000000000008c0d5f73796e635f70726f63657373944e8c0c5f63757272656e745f636d649468008c125f75706c6f61645f746f5f66735f706174689493947d94288c0a6c6f63616c5f70617468948c3b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c74732f746573745f6578706572696d656e74948c0266739468228c0766735f70617468948c552f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e74948c076578636c756465945d948c0e2a2f636865636b706f696e745f2a9461758694756275622e\"\n },\n \"config\": {\n \"encoder_num_layer\": 4,\n \"decoder_rt_num_layer\": 2,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 1024,\n \"encoder_ff\": 2048,\n \"decoder_rt_ff\": 1024,\n \"decoder_int_ff\": 512,\n \"n_head\": 1,\n \"drop_rate\": 0.24540187796715263,\n \"lr\": 0.0006485525268785047,\n \"batch_size\": 1024\n },\n \"_Trial__unresolved_config\": {\n \"encoder_num_layer\": 4,\n \"decoder_rt_num_layer\": 2,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 1024,\n \"encoder_ff\": 2048,\n \"decoder_rt_ff\": 1024,\n \"decoder_int_ff\": 512,\n \"n_head\": 1,\n \"drop_rate\": 0.24540187796715263,\n \"lr\": 0.0006485525268785047,\n \"batch_size\": 1024\n },\n \"evaluated_params\": {\n \"batch_size\": 1024,\n \"decoder_int_ff\": 512,\n \"decoder_int_num_layer\": 1,\n \"decoder_rt_ff\": 1024,\n \"decoder_rt_num_layer\": 2,\n \"drop_rate\": 0.24540187796715263,\n \"embedding_dim\": 1024,\n \"encoder_ff\": 2048,\n \"encoder_num_layer\": 4,\n \"lr\": 0.0006485525268785047,\n \"n_head\": 1\n },\n \"experiment_tag\": \"5_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop_rate=0.2454,embedding_dim=1024,encoder_ff=2048,encoder_num_layer=4,lr=0.0006,n_head=1\",\n \"stopping_criterion\": {},\n \"_setup_default_resource\": true,\n \"_default_placement_group_factory\": \"80054e2e\",\n \"placement_group_factory\": \"800595ba000000000000008c237261792e74756e652e657865637574696f6e2e706c6163656d656e745f67726f757073948c15506c6163656d656e7447726f7570466163746f72799493942981947d94288c085f62756e646c6573945d947d94288c03435055944740540000000000008c034750559447401000000000000075618c155f686561645f62756e646c655f69735f656d70747994898c095f7374726174656779948c045041434b948c055f6172677394298c075f6b7761726773947d9475622e\",\n \"log_to_file\": [\n null,\n null\n ],\n \"max_failures\": 0,\n \"_default_result_or_future\": null,\n \"export_formats\": [],\n \"status\": \"PAUSED\",\n \"relative_logdir\": \"train_model_f1fbbaf2_5_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop_r_2024-06-25_17-16-57\",\n \"trial_name_creator\": null,\n \"trial_dirname_creator\": null,\n \"custom_trial_name\": null,\n \"custom_dirname\": null,\n \"restore_path\": null,\n \"_restore_checkpoint_result\": null,\n \"_state_json\": null,\n \"results\": \"80054e2e\",\n \"extra_arg\": \"80054e2e\",\n \"_resources\": \"80054e2e\"\n}",
+ "{\n \"start_time\": 1719328732.7937407,\n \"num_failures\": 0,\n \"num_failures_after_restore\": 0,\n \"error_filename\": null,\n \"pickled_error_filename\": null,\n \"last_result\": {\n \"loss\": 1986.5954556202325,\n \"timestamp\": 1719329376,\n \"checkpoint_dir_name\": \"checkpoint_000000\",\n \"should_checkpoint\": true,\n \"done\": false,\n \"training_iteration\": 1,\n \"trial_id\": \"f1fbbaf2\",\n \"date\": \"2024-06-25_17-29-36\",\n \"time_this_iter_s\": 643.849014043808,\n \"time_total_s\": 643.849014043808,\n \"pid\": 2195253,\n \"hostname\": \"r8i6n2\",\n \"node_ip\": \"10.159.28.60\",\n \"config\": {\n \"encoder_num_layer\": 4,\n \"decoder_rt_num_layer\": 2,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 1024,\n \"encoder_ff\": 2048,\n \"decoder_rt_ff\": 1024,\n \"decoder_int_ff\": 512,\n \"n_head\": 1,\n \"drop_rate\": 0.24540187796715263,\n \"lr\": 0.0006485525268785047,\n \"batch_size\": 1024\n },\n \"time_since_restore\": 643.849014043808,\n \"iterations_since_restore\": 1,\n \"experiment_tag\": \"5_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop_rate=0.2454,embedding_dim=1024,encoder_ff=2048,encoder_num_layer=4,lr=0.0006,n_head=1\"\n },\n \"last_result_time\": 1719329376.6484032,\n \"metric_analysis\": {\n \"loss\": {\n \"max\": 1986.5954556202325,\n \"min\": 1986.5954556202325,\n \"avg\": 1986.5954556202325,\n \"last\": 1986.5954556202325,\n \"last-5-avg\": 1986.5954556202325,\n \"last-10-avg\": 1986.5954556202325\n },\n \"should_checkpoint\": {\n \"max\": true,\n \"min\": true,\n \"avg\": true,\n \"last\": true,\n \"last-5-avg\": true,\n \"last-10-avg\": true\n },\n \"done\": {\n \"max\": false,\n \"min\": false,\n \"avg\": false,\n \"last\": false,\n \"last-5-avg\": false,\n \"last-10-avg\": false\n },\n \"training_iteration\": {\n \"max\": 1,\n \"min\": 1,\n \"avg\": 1,\n \"last\": 1,\n \"last-5-avg\": 1,\n \"last-10-avg\": 1\n },\n \"time_this_iter_s\": {\n \"max\": 643.849014043808,\n \"min\": 643.849014043808,\n \"avg\": 643.849014043808,\n \"last\": 643.849014043808,\n \"last-5-avg\": 643.849014043808,\n \"last-10-avg\": 643.849014043808\n },\n \"time_total_s\": {\n \"max\": 643.849014043808,\n \"min\": 643.849014043808,\n \"avg\": 643.849014043808,\n \"last\": 643.849014043808,\n \"last-5-avg\": 643.849014043808,\n \"last-10-avg\": 643.849014043808\n },\n \"time_since_restore\": {\n \"max\": 643.849014043808,\n \"min\": 643.849014043808,\n \"avg\": 643.849014043808,\n \"last\": 643.849014043808,\n \"last-5-avg\": 643.849014043808,\n \"last-10-avg\": 643.849014043808\n },\n \"iterations_since_restore\": {\n \"max\": 1,\n \"min\": 1,\n \"avg\": 1,\n \"last\": 1,\n \"last-5-avg\": 1,\n \"last-10-avg\": 1\n }\n },\n \"_n_steps\": [\n 5,\n 10\n ],\n \"metric_n_steps\": {\n \"loss\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059589000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452948c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b007494624308793c1ebf610a9f409486945294612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059589000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452948c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b007494624308793c1ebf610a9f409486945294612e\"\n }\n },\n \"should_checkpoint\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059522000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b058694529488612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059522000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a8694529488612e\"\n }\n },\n \"done\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059522000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b058694529489612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059522000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a8694529489612e\"\n }\n },\n \"training_iteration\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059523000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452944b01612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059523000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452944b01612e\"\n }\n },\n \"time_this_iter_s\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452944740841ecac7e00000612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452944740841ecac7e00000612e\"\n }\n },\n \"time_total_s\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452944740841ecac7e00000612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452944740841ecac7e00000612e\"\n }\n },\n \"time_since_restore\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452944740841ecac7e00000612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452944740841ecac7e00000612e\"\n }\n },\n \"iterations_since_restore\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059523000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452944b01612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059523000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452944b01612e\"\n }\n }\n },\n \"checkpoint_manager\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059518060000000000008c267261792e747261696e2e5f696e7465726e616c2e636865636b706f696e745f6d616e61676572948c125f436865636b706f696e744d616e616765729493942981947d94288c125f636865636b706f696e745f636f6e666967948c097261792e747261696e948c10436865636b706f696e74436f6e6669679493942981947d94288c0b6e756d5f746f5f6b656570944e8c1a636865636b706f696e745f73636f72655f617474726962757465944e8c16636865636b706f696e745f73636f72655f6f72646572948c036d6178948c14636865636b706f696e745f6672657175656e6379944b008c11636865636b706f696e745f61745f656e6494898c1a5f636865636b706f696e745f6b6565705f616c6c5f72616e6b73948c0a44455052454341544544948c1f5f636865636b706f696e745f75706c6f61645f66726f6d5f776f726b65727394681275628c135f636865636b706f696e745f726573756c7473945d948c1b7261792e747261696e2e5f696e7465726e616c2e73657373696f6e948c0f5f547261696e696e67526573756c749493942981947d94288c0a636865636b706f696e749468068c0a436865636b706f696e749493942981947d94288c0470617468948cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f66316662626166325f355f62617463685f73697a653d313032342c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d313032342c6465636f6465725f72745f6e756d5f6c617965723d322c64726f705f725f323032342d30362d32355f31372d31362d35372f636865636b706f696e745f303030303030948c0a66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c055f75756964948c0475756964948c04555549449493942981947d948c03696e74948a107ceec938c58a36a286442505bc700a43736275628c076d657472696373947d94288c046c6f7373948c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b007494624308793c1ebf610a9f4094869452948c0974696d657374616d70944a60e27a668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303030948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b018c08747269616c5f6964948c086631666262616632948c0464617465948c13323032342d30362d32355f31372d32392d3336948c1074696d655f746869735f697465725f73944740841ecac7e000008c0c74696d655f746f74616c5f73944740841ecac7e000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b048c146465636f6465725f72745f6e756d5f6c61796572944b028c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944d00048c0a656e636f6465725f6666944d00088c0d6465636f6465725f72745f6666944d00048c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b018c0964726f705f7261746594473fcf6954281f79c48c026c7294473f454073f24361d28c0a62617463685f73697a65944d0004758c1274696d655f73696e63655f726573746f7265944740841ecac7e000008c18697465726174696f6e735f73696e63655f726573746f7265944b01757562618c195f6c61746573745f636865636b706f696e745f726573756c7494681975622e\"\n }\n}"
+ ],
+ [
+ "{\n \"stub\": false,\n \"trainable_name\": \"train_model\",\n \"trial_id\": \"5b0a0ae2\",\n \"storage\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059572040000000000008c1b7261792e747261696e2e5f696e7465726e616c2e73746f72616765948c0e53746f72616765436f6e746578749493942981947d94288c12637573746f6d5f66735f70726f766964656494898c1273746f726167655f6c6f63616c5f70617468948c2b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c7473948c136578706572696d656e745f6469725f6e616d65948c0f746573745f6578706572696d656e74948c0e747269616c5f6469725f6e616d65948c96747261696e5f6d6f64656c5f35623061306165325f365f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d323034382c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f725f323032342d30362d32355f31372d31382d3532948c1863757272656e745f636865636b706f696e745f696e646578944b018c0b73796e635f636f6e666967948c097261792e747261696e948c0a53796e63436f6e6669679493942981947d94288c0b73796e635f706572696f64944d2c018c0c73796e635f74696d656f7574944d08078c0e73796e635f61727469666163747394898c1c73796e635f6172746966616374735f6f6e5f636865636b706f696e7494888c0a75706c6f61645f646972948c0a44455052454341544544948c0673796e6365729468188c1273796e635f6f6e5f636865636b706f696e7494681875628c1273746f726167655f66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c0f73746f726167655f66735f70617468948c452f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f7465737494681968008c115f46696c6573797374656d53796e6365729493942981947d9428681b682268134d2c0168144d08078c116c6173745f73796e635f75705f74696d65944741d99eb7bda3c2988c136c6173745f73796e635f646f776e5f74696d659447fff00000000000008c0d5f73796e635f70726f63657373944e8c0c5f63757272656e745f636d649468008c125f75706c6f61645f746f5f66735f706174689493947d94288c0a6c6f63616c5f70617468948c3b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c74732f746573745f6578706572696d656e74948c0266739468228c0766735f70617468948c552f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e74948c076578636c756465945d948c0e2a2f636865636b706f696e745f2a9461758694756275622e\"\n },\n \"config\": {\n \"encoder_num_layer\": 8,\n \"decoder_rt_num_layer\": 1,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 16,\n \"encoder_ff\": 512,\n \"decoder_rt_ff\": 2048,\n \"decoder_int_ff\": 512,\n \"n_head\": 16,\n \"drop_rate\": 0.0063231604639778505,\n \"lr\": 0.0021079084039053323,\n \"batch_size\": 2048\n },\n \"_Trial__unresolved_config\": {\n \"encoder_num_layer\": 8,\n \"decoder_rt_num_layer\": 1,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 16,\n \"encoder_ff\": 512,\n \"decoder_rt_ff\": 2048,\n \"decoder_int_ff\": 512,\n \"n_head\": 16,\n \"drop_rate\": 0.0063231604639778505,\n \"lr\": 0.0021079084039053323,\n \"batch_size\": 2048\n },\n \"evaluated_params\": {\n \"batch_size\": 2048,\n \"decoder_int_ff\": 512,\n \"decoder_int_num_layer\": 1,\n \"decoder_rt_ff\": 2048,\n \"decoder_rt_num_layer\": 1,\n \"drop_rate\": 0.0063231604639778505,\n \"embedding_dim\": 16,\n \"encoder_ff\": 512,\n \"encoder_num_layer\": 8,\n \"lr\": 0.0021079084039053323,\n \"n_head\": 16\n },\n \"experiment_tag\": \"6_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop_rate=0.0063,embedding_dim=16,encoder_ff=512,encoder_num_layer=8,lr=0.0021,n_head=16\",\n \"stopping_criterion\": {},\n \"_setup_default_resource\": true,\n \"_default_placement_group_factory\": \"80054e2e\",\n \"placement_group_factory\": \"800595ba000000000000008c237261792e74756e652e657865637574696f6e2e706c6163656d656e745f67726f757073948c15506c6163656d656e7447726f7570466163746f72799493942981947d94288c085f62756e646c6573945d947d94288c03435055944740540000000000008c034750559447401000000000000075618c155f686561645f62756e646c655f69735f656d70747994898c095f7374726174656779948c045041434b948c055f6172677394298c075f6b7761726773947d9475622e\",\n \"log_to_file\": [\n null,\n null\n ],\n \"max_failures\": 0,\n \"_default_result_or_future\": null,\n \"export_formats\": [],\n \"status\": \"PAUSED\",\n \"relative_logdir\": \"train_model_5b0a0ae2_6_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop_r_2024-06-25_17-18-52\",\n \"trial_name_creator\": null,\n \"trial_dirname_creator\": null,\n \"custom_trial_name\": null,\n \"custom_dirname\": null,\n \"restore_path\": null,\n \"_restore_checkpoint_result\": null,\n \"_state_json\": null,\n \"results\": \"80054e2e\",\n \"extra_arg\": \"80054e2e\",\n \"_resources\": \"80054e2e\"\n}",
+ "{\n \"start_time\": 1719329376.7954574,\n \"num_failures\": 0,\n \"num_failures_after_restore\": 0,\n \"error_filename\": null,\n \"pickled_error_filename\": null,\n \"last_result\": {\n \"loss\": 1963.9745468980684,\n \"timestamp\": 1719330671,\n \"checkpoint_dir_name\": \"checkpoint_000001\",\n \"should_checkpoint\": true,\n \"done\": false,\n \"training_iteration\": 2,\n \"trial_id\": \"5b0a0ae2\",\n \"date\": \"2024-06-25_17-51-11\",\n \"time_this_iter_s\": 84.48489356040955,\n \"time_total_s\": 168.87538933753967,\n \"pid\": 2195253,\n \"hostname\": \"r8i6n2\",\n \"node_ip\": \"10.159.28.60\",\n \"config\": {\n \"encoder_num_layer\": 8,\n \"decoder_rt_num_layer\": 1,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 16,\n \"encoder_ff\": 512,\n \"decoder_rt_ff\": 2048,\n \"decoder_int_ff\": 512,\n \"n_head\": 16,\n \"drop_rate\": 0.0063231604639778505,\n \"lr\": 0.0021079084039053323,\n \"batch_size\": 2048\n },\n \"time_since_restore\": 84.48489356040955,\n \"iterations_since_restore\": 1,\n \"experiment_tag\": \"6_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop_rate=0.0063,embedding_dim=16,encoder_ff=512,encoder_num_layer=8,lr=0.0021,n_head=16\"\n },\n \"last_result_time\": 1719330671.9681833,\n \"metric_analysis\": {\n \"loss\": {\n \"max\": 1963.9745468980684,\n \"min\": 1952.9751620555487,\n \"avg\": 1958.4748544768086,\n \"last\": 1963.9745468980684,\n \"last-5-avg\": 1958.4748544768086,\n \"last-10-avg\": 1958.4748544768086\n },\n \"should_checkpoint\": {\n \"max\": true,\n \"min\": true,\n \"avg\": 1.0,\n \"last\": true,\n \"last-5-avg\": 1.0,\n \"last-10-avg\": 1.0\n },\n \"done\": {\n \"max\": false,\n \"min\": false,\n \"avg\": 0.0,\n \"last\": false,\n \"last-5-avg\": 0.0,\n \"last-10-avg\": 0.0\n },\n \"training_iteration\": {\n \"max\": 2,\n \"min\": 1,\n \"avg\": 1.5,\n \"last\": 2,\n \"last-5-avg\": 1.5,\n \"last-10-avg\": 1.5\n },\n \"time_this_iter_s\": {\n \"max\": 84.48489356040955,\n \"min\": 84.39049577713013,\n \"avg\": 84.43769466876984,\n \"last\": 84.48489356040955,\n \"last-5-avg\": 84.43769466876984,\n \"last-10-avg\": 84.43769466876984\n },\n \"time_total_s\": {\n \"max\": 168.87538933753967,\n \"min\": 84.39049577713013,\n \"avg\": 126.6329425573349,\n \"last\": 168.87538933753967,\n \"last-5-avg\": 126.6329425573349,\n \"last-10-avg\": 126.6329425573349\n },\n \"time_since_restore\": {\n \"max\": 84.48489356040955,\n \"min\": 84.39049577713013,\n \"avg\": 84.43769466876984,\n \"last\": 84.48489356040955,\n \"last-5-avg\": 84.43769466876984,\n \"last-10-avg\": 84.43769466876984\n },\n \"iterations_since_restore\": {\n \"max\": 1,\n \"min\": 1,\n \"avg\": 1.0,\n \"last\": 1,\n \"last-5-avg\": 1.0,\n \"last-10-avg\": 1.0\n }\n },\n \"_n_steps\": [\n 5,\n 10\n ],\n \"metric_n_steps\": {\n \"loss\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005959d000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294288c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b00749462430887c3e190e6839e4094869452946807680d43087d3e9fefe5af9e409486945294652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005959d000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294288c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b00749462430887c3e190e6839e4094869452946807680d43087d3e9fefe5af9e409486945294652e\"\n }\n },\n \"should_checkpoint\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059524000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294288888652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059524000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294288888652e\"\n }\n },\n \"done\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059524000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294288989652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059524000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294288989652e\"\n }\n },\n \"training_iteration\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059526000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294284b014b02652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059526000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294284b014b02652e\"\n }\n },\n \"time_this_iter_s\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059534000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452942847405518fde20000004740551f087f000000652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059534000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452942847405518fde20000004740551f087f000000652e\"\n }\n },\n \"time_total_s\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059534000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452942847405518fde20000004740651c0330800000652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059534000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452942847405518fde20000004740651c0330800000652e\"\n }\n },\n \"time_since_restore\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059534000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452942847405518fde20000004740551f087f000000652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059534000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452942847405518fde20000004740551f087f000000652e\"\n }\n },\n \"iterations_since_restore\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059526000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294284b014b01652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059526000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294284b014b01652e\"\n }\n }\n },\n \"checkpoint_manager\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"800595a5090000000000008c267261792e747261696e2e5f696e7465726e616c2e636865636b706f696e745f6d616e61676572948c125f436865636b706f696e744d616e616765729493942981947d94288c125f636865636b706f696e745f636f6e666967948c097261792e747261696e948c10436865636b706f696e74436f6e6669679493942981947d94288c0b6e756d5f746f5f6b656570944e8c1a636865636b706f696e745f73636f72655f617474726962757465944e8c16636865636b706f696e745f73636f72655f6f72646572948c036d6178948c14636865636b706f696e745f6672657175656e6379944b008c11636865636b706f696e745f61745f656e6494898c1a5f636865636b706f696e745f6b6565705f616c6c5f72616e6b73948c0a44455052454341544544948c1f5f636865636b706f696e745f75706c6f61645f66726f6d5f776f726b65727394681275628c135f636865636b706f696e745f726573756c7473945d94288c1b7261792e747261696e2e5f696e7465726e616c2e73657373696f6e948c0f5f547261696e696e67526573756c749493942981947d94288c0a636865636b706f696e749468068c0a436865636b706f696e749493942981947d94288c0470617468948cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f35623061306165325f365f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d323034382c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f725f323032342d30362d32355f31372d31382d35322f636865636b706f696e745f303030303030948c0a66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c055f75756964948c0475756964948c04555549449493942981947d948c03696e74948a1070db152d94d48fa82e42a314639aa52d736275628c076d657472696373947d94288c046c6f7373948c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b00749462430887c3e190e6839e4094869452948c0974696d657374616d70944ab5e27a668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303030948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b018c08747269616c5f6964948c083562306130616532948c0464617465948c13323032342d30362d32355f31372d33312d3031948c1074696d655f746869735f697465725f739447405518fde20000008c0c74696d655f746f74616c5f739447405518fde20000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b088c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b108c0a656e636f6465725f6666944d00028c0d6465636f6465725f72745f6666944d00088c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b108c0964726f705f7261746594473f79e6507666fe808c026c7294473f61449ab50c2e628c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f72659447405518fde20000008c18697465726174696f6e735f73696e63655f726573746f7265944b0175756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f35623061306165325f365f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d323034382c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f725f323032342d30362d32355f31372d31382d35322f636865636b706f696e745f30303030303194682268257d946827897385945294682a682d2981947d9468308a1137570e916e59b7a6e64247e3a7f2e0a5007362756268317d94288c046c6f7373946836683c43087d3e9fefe5af9e4094869452948c0974696d657374616d70944a6fe77a668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303031948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b028c08747269616c5f6964948c083562306130616532948c0464617465948c13323032342d30362d32355f31372d35312d3131948c1074696d655f746869735f697465725f73944740551f087f0000008c0c74696d655f746f74616c5f73944740651c03308000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b088c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b108c0a656e636f6465725f6666944d00028c0d6465636f6465725f72745f6666944d00088c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b108c0964726f705f7261746594473f79e6507666fe808c026c7294473f61449ab50c2e628c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f7265944740551f087f0000008c18697465726174696f6e735f73696e63655f726573746f7265944b01757562658c195f6c61746573745f636865636b706f696e745f726573756c7494686275622e\"\n }\n}"
+ ],
+ [
+ "{\n \"stub\": false,\n \"trainable_name\": \"train_model\",\n \"trial_id\": \"3f37f45c\",\n \"storage\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059572040000000000008c1b7261792e747261696e2e5f696e7465726e616c2e73746f72616765948c0e53746f72616765436f6e746578749493942981947d94288c12637573746f6d5f66735f70726f766964656494898c1273746f726167655f6c6f63616c5f70617468948c2b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c7473948c136578706572696d656e745f6469725f6e616d65948c0f746573745f6578706572696d656e74948c0e747269616c5f6469725f6e616d65948c96747261696e5f6d6f64656c5f33663337663435635f375f62617463685f73697a653d313032342c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d323034382c6465636f6465725f72745f6e756d5f6c617965723d322c64726f705f725f323032342d30362d32355f31372d32392d3336948c1863757272656e745f636865636b706f696e745f696e646578944b008c0b73796e635f636f6e666967948c097261792e747261696e948c0a53796e63436f6e6669679493942981947d94288c0b73796e635f706572696f64944d2c018c0c73796e635f74696d656f7574944d08078c0e73796e635f61727469666163747394898c1c73796e635f6172746966616374735f6f6e5f636865636b706f696e7494888c0a75706c6f61645f646972948c0a44455052454341544544948c0673796e6365729468188c1273796e635f6f6e5f636865636b706f696e7494681875628c1273746f726167655f66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c0f73746f726167655f66735f70617468948c452f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f7465737494681968008c115f46696c6573797374656d53796e6365729493942981947d9428681b682268134d2c0168144d08078c116c6173745f73796e635f75705f74696d65944741d99eb8547484f48c136c6173745f73796e635f646f776e5f74696d659447fff00000000000008c0d5f73796e635f70726f63657373944e8c0c5f63757272656e745f636d649468008c125f75706c6f61645f746f5f66735f706174689493947d94288c0a6c6f63616c5f70617468948c3b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c74732f746573745f6578706572696d656e74948c0266739468228c0766735f70617468948c552f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e74948c076578636c756465945d948c0e2a2f636865636b706f696e745f2a9461758694756275622e\"\n },\n \"config\": {\n \"encoder_num_layer\": 1,\n \"decoder_rt_num_layer\": 2,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 256,\n \"encoder_ff\": 512,\n \"decoder_rt_ff\": 2048,\n \"decoder_int_ff\": 512,\n \"n_head\": 4,\n \"drop_rate\": 0.6691316467787182,\n \"lr\": 0.004731694425031267,\n \"batch_size\": 1024\n },\n \"_Trial__unresolved_config\": {\n \"encoder_num_layer\": 1,\n \"decoder_rt_num_layer\": 2,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 256,\n \"encoder_ff\": 512,\n \"decoder_rt_ff\": 2048,\n \"decoder_int_ff\": 512,\n \"n_head\": 4,\n \"drop_rate\": 0.6691316467787182,\n \"lr\": 0.004731694425031267,\n \"batch_size\": 1024\n },\n \"evaluated_params\": {\n \"batch_size\": 1024,\n \"decoder_int_ff\": 512,\n \"decoder_int_num_layer\": 1,\n \"decoder_rt_ff\": 2048,\n \"decoder_rt_num_layer\": 2,\n \"drop_rate\": 0.6691316467787182,\n \"embedding_dim\": 256,\n \"encoder_ff\": 512,\n \"encoder_num_layer\": 1,\n \"lr\": 0.004731694425031267,\n \"n_head\": 4\n },\n \"experiment_tag\": \"7_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=2,drop_rate=0.6691,embedding_dim=256,encoder_ff=512,encoder_num_layer=1,lr=0.0047,n_head=4\",\n \"stopping_criterion\": {},\n \"_setup_default_resource\": true,\n \"_default_placement_group_factory\": \"80054e2e\",\n \"placement_group_factory\": \"800595ba000000000000008c237261792e74756e652e657865637574696f6e2e706c6163656d656e745f67726f757073948c15506c6163656d656e7447726f7570466163746f72799493942981947d94288c085f62756e646c6573945d947d94288c03435055944740540000000000008c034750559447401000000000000075618c155f686561645f62756e646c655f69735f656d70747994898c095f7374726174656779948c045041434b948c055f6172677394298c075f6b7761726773947d9475622e\",\n \"log_to_file\": [\n null,\n null\n ],\n \"max_failures\": 0,\n \"_default_result_or_future\": null,\n \"export_formats\": [],\n \"status\": \"PAUSED\",\n \"relative_logdir\": \"train_model_3f37f45c_7_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=2,drop_r_2024-06-25_17-29-36\",\n \"trial_name_creator\": null,\n \"trial_dirname_creator\": null,\n \"custom_trial_name\": null,\n \"custom_dirname\": null,\n \"restore_path\": null,\n \"_restore_checkpoint_result\": null,\n \"_state_json\": null,\n \"results\": \"80054e2e\",\n \"extra_arg\": \"80054e2e\",\n \"_resources\": \"80054e2e\"\n}",
+ "{\n \"start_time\": 1719329461.3491297,\n \"num_failures\": 0,\n \"num_failures_after_restore\": 0,\n \"error_filename\": null,\n \"pickled_error_filename\": null,\n \"last_result\": {\n \"loss\": 2052.4382574126475,\n \"timestamp\": 1719329544,\n \"checkpoint_dir_name\": \"checkpoint_000000\",\n \"should_checkpoint\": true,\n \"done\": false,\n \"training_iteration\": 1,\n \"trial_id\": \"3f37f45c\",\n \"date\": \"2024-06-25_17-32-25\",\n \"time_this_iter_s\": 83.68853616714478,\n \"time_total_s\": 83.68853616714478,\n \"pid\": 2195253,\n \"hostname\": \"r8i6n2\",\n \"node_ip\": \"10.159.28.60\",\n \"config\": {\n \"encoder_num_layer\": 1,\n \"decoder_rt_num_layer\": 2,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 256,\n \"encoder_ff\": 512,\n \"decoder_rt_ff\": 2048,\n \"decoder_int_ff\": 512,\n \"n_head\": 4,\n \"drop_rate\": 0.6691316467787182,\n \"lr\": 0.004731694425031267,\n \"batch_size\": 1024\n },\n \"time_since_restore\": 83.68853616714478,\n \"iterations_since_restore\": 1,\n \"experiment_tag\": \"7_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=2,drop_rate=0.6691,embedding_dim=256,encoder_ff=512,encoder_num_layer=1,lr=0.0047,n_head=4\"\n },\n \"last_result_time\": 1719329545.0414808,\n \"metric_analysis\": {\n \"loss\": {\n \"max\": 2052.4382574126475,\n \"min\": 2052.4382574126475,\n \"avg\": 2052.4382574126475,\n \"last\": 2052.4382574126475,\n \"last-5-avg\": 2052.4382574126475,\n \"last-10-avg\": 2052.4382574126475\n },\n \"should_checkpoint\": {\n \"max\": true,\n \"min\": true,\n \"avg\": true,\n \"last\": true,\n \"last-5-avg\": true,\n \"last-10-avg\": true\n },\n \"done\": {\n \"max\": false,\n \"min\": false,\n \"avg\": false,\n \"last\": false,\n \"last-5-avg\": false,\n \"last-10-avg\": false\n },\n \"training_iteration\": {\n \"max\": 1,\n \"min\": 1,\n \"avg\": 1,\n \"last\": 1,\n \"last-5-avg\": 1,\n \"last-10-avg\": 1\n },\n \"time_this_iter_s\": {\n \"max\": 83.68853616714478,\n \"min\": 83.68853616714478,\n \"avg\": 83.68853616714478,\n \"last\": 83.68853616714478,\n \"last-5-avg\": 83.68853616714478,\n \"last-10-avg\": 83.68853616714478\n },\n \"time_total_s\": {\n \"max\": 83.68853616714478,\n \"min\": 83.68853616714478,\n \"avg\": 83.68853616714478,\n \"last\": 83.68853616714478,\n \"last-5-avg\": 83.68853616714478,\n \"last-10-avg\": 83.68853616714478\n },\n \"time_since_restore\": {\n \"max\": 83.68853616714478,\n \"min\": 83.68853616714478,\n \"avg\": 83.68853616714478,\n \"last\": 83.68853616714478,\n \"last-5-avg\": 83.68853616714478,\n \"last-10-avg\": 83.68853616714478\n },\n \"iterations_since_restore\": {\n \"max\": 1,\n \"min\": 1,\n \"avg\": 1,\n \"last\": 1,\n \"last-5-avg\": 1,\n \"last-10-avg\": 1\n }\n },\n \"_n_steps\": [\n 5,\n 10\n ],\n \"metric_n_steps\": {\n \"loss\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059589000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452948c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b0074946243081a8d4663e008a0409486945294612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059589000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452948c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b0074946243081a8d4663e008a0409486945294612e\"\n }\n },\n \"should_checkpoint\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059522000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b058694529488612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059522000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a8694529488612e\"\n }\n },\n \"done\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059522000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b058694529489612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059522000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a8694529489612e\"\n }\n },\n \"training_iteration\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059523000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452944b01612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059523000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452944b01612e\"\n }\n },\n \"time_this_iter_s\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294474054ec10fa000000612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294474054ec10fa000000612e\"\n }\n },\n \"time_total_s\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294474054ec10fa000000612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294474054ec10fa000000612e\"\n }\n },\n \"time_since_restore\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294474054ec10fa000000612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294474054ec10fa000000612e\"\n }\n },\n \"iterations_since_restore\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059523000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452944b01612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059523000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452944b01612e\"\n }\n }\n },\n \"checkpoint_manager\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059519060000000000008c267261792e747261696e2e5f696e7465726e616c2e636865636b706f696e745f6d616e61676572948c125f436865636b706f696e744d616e616765729493942981947d94288c125f636865636b706f696e745f636f6e666967948c097261792e747261696e948c10436865636b706f696e74436f6e6669679493942981947d94288c0b6e756d5f746f5f6b656570944e8c1a636865636b706f696e745f73636f72655f617474726962757465944e8c16636865636b706f696e745f73636f72655f6f72646572948c036d6178948c14636865636b706f696e745f6672657175656e6379944b008c11636865636b706f696e745f61745f656e6494898c1a5f636865636b706f696e745f6b6565705f616c6c5f72616e6b73948c0a44455052454341544544948c1f5f636865636b706f696e745f75706c6f61645f66726f6d5f776f726b65727394681275628c135f636865636b706f696e745f726573756c7473945d948c1b7261792e747261696e2e5f696e7465726e616c2e73657373696f6e948c0f5f547261696e696e67526573756c749493942981947d94288c0a636865636b706f696e749468068c0a436865636b706f696e749493942981947d94288c0470617468948cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f33663337663435635f375f62617463685f73697a653d313032342c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d323034382c6465636f6465725f72745f6e756d5f6c617965723d322c64726f705f725f323032342d30362d32355f31372d32392d33362f636865636b706f696e745f303030303030948c0a66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c055f75756964948c0475756964948c04555549449493942981947d948c03696e74948a1138d348b093235c974043ee23a2e6638700736275628c076d657472696373947d94288c046c6f7373948c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b0074946243081a8d4663e008a04094869452948c0974696d657374616d70944a08e37a668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303030948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b018c08747269616c5f6964948c083366333766343563948c0464617465948c13323032342d30362d32355f31372d33322d3235948c1074696d655f746869735f697465725f7394474054ec10fa0000008c0c74696d655f746f74616c5f7394474054ec10fa0000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b018c146465636f6465725f72745f6e756d5f6c61796572944b028c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944d00018c0a656e636f6465725f6666944d00028c0d6465636f6465725f72745f6666944d00088c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b048c0964726f705f7261746594473fe56986c57443538c026c7294473f73618a8cf679de8c0a62617463685f73697a65944d0004758c1274696d655f73696e63655f726573746f726594474054ec10fa0000008c18697465726174696f6e735f73696e63655f726573746f7265944b01757562618c195f6c61746573745f636865636b706f696e745f726573756c7494681975622e\"\n }\n}"
+ ],
+ [
+ "{\n \"stub\": false,\n \"trainable_name\": \"train_model\",\n \"trial_id\": \"040fcf09\",\n \"storage\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059572040000000000008c1b7261792e747261696e2e5f696e7465726e616c2e73746f72616765948c0e53746f72616765436f6e746578749493942981947d94288c12637573746f6d5f66735f70726f766964656494898c1273746f726167655f6c6f63616c5f70617468948c2b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c7473948c136578706572696d656e745f6469725f6e616d65948c0f746573745f6578706572696d656e74948c0e747269616c5f6469725f6e616d65948c96747261696e5f6d6f64656c5f30343066636630395f385f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d3531322c6465636f6465725f72745f6e756d5f6c617965723d342c64726f705f72615f323032342d30362d32355f31372d33312d3031948c1863757272656e745f636865636b706f696e745f696e646578944b008c0b73796e635f636f6e666967948c097261792e747261696e948c0a53796e63436f6e6669679493942981947d94288c0b73796e635f706572696f64944d2c018c0c73796e635f74696d656f7574944d08078c0e73796e635f61727469666163747394898c1c73796e635f6172746966616374735f6f6e5f636865636b706f696e7494888c0a75706c6f61645f646972948c0a44455052454341544544948c0673796e6365729468188c1273796e635f6f6e5f636865636b706f696e7494681875628c1273746f726167655f66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c0f73746f726167655f66735f70617468948c452f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f7465737494681968008c115f46696c6573797374656d53796e6365729493942981947d9428681b682268134d2c0168144d08078c116c6173745f73796e635f75705f74696d65944741d99eb89fd0cfec8c136c6173745f73796e635f646f776e5f74696d659447fff00000000000008c0d5f73796e635f70726f63657373944e8c0c5f63757272656e745f636d649468008c125f75706c6f61645f746f5f66735f706174689493947d94288c0a6c6f63616c5f70617468948c3b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c74732f746573745f6578706572696d656e74948c0266739468228c0766735f70617468948c552f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e74948c076578636c756465945d948c0e2a2f636865636b706f696e745f2a9461758694756275622e\"\n },\n \"config\": {\n \"encoder_num_layer\": 1,\n \"decoder_rt_num_layer\": 4,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 16,\n \"encoder_ff\": 1024,\n \"decoder_rt_ff\": 512,\n \"decoder_int_ff\": 512,\n \"n_head\": 1,\n \"drop_rate\": 0.18759871111111892,\n \"lr\": 0.02750689103642202,\n \"batch_size\": 2048\n },\n \"_Trial__unresolved_config\": {\n \"encoder_num_layer\": 1,\n \"decoder_rt_num_layer\": 4,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 16,\n \"encoder_ff\": 1024,\n \"decoder_rt_ff\": 512,\n \"decoder_int_ff\": 512,\n \"n_head\": 1,\n \"drop_rate\": 0.18759871111111892,\n \"lr\": 0.02750689103642202,\n \"batch_size\": 2048\n },\n \"evaluated_params\": {\n \"batch_size\": 2048,\n \"decoder_int_ff\": 512,\n \"decoder_int_num_layer\": 1,\n \"decoder_rt_ff\": 512,\n \"decoder_rt_num_layer\": 4,\n \"drop_rate\": 0.18759871111111892,\n \"embedding_dim\": 16,\n \"encoder_ff\": 1024,\n \"encoder_num_layer\": 1,\n \"lr\": 0.02750689103642202,\n \"n_head\": 1\n },\n \"experiment_tag\": \"8_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=4,drop_rate=0.1876,embedding_dim=16,encoder_ff=1024,encoder_num_layer=1,lr=0.0275,n_head=1\",\n \"stopping_criterion\": {},\n \"_setup_default_resource\": true,\n \"_default_placement_group_factory\": \"80054e2e\",\n \"placement_group_factory\": \"800595ba000000000000008c237261792e74756e652e657865637574696f6e2e706c6163656d656e745f67726f757073948c15506c6163656d656e7447726f7570466163746f72799493942981947d94288c085f62756e646c6573945d947d94288c03435055944740540000000000008c034750559447401000000000000075618c155f686561645f62756e646c655f69735f656d70747994898c095f7374726174656779948c045041434b948c055f6172677394298c075f6b7761726773947d9475622e\",\n \"log_to_file\": [\n null,\n null\n ],\n \"max_failures\": 0,\n \"_default_result_or_future\": null,\n \"export_formats\": [],\n \"status\": \"PAUSED\",\n \"relative_logdir\": \"train_model_040fcf09_8_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=4,drop_ra_2024-06-25_17-31-01\",\n \"trial_name_creator\": null,\n \"trial_dirname_creator\": null,\n \"custom_trial_name\": null,\n \"custom_dirname\": null,\n \"restore_path\": null,\n \"_restore_checkpoint_result\": null,\n \"_state_json\": null,\n \"results\": \"80054e2e\",\n \"extra_arg\": \"80054e2e\",\n \"_resources\": \"80054e2e\"\n}",
+ "{\n \"start_time\": 1719329545.1895237,\n \"num_failures\": 0,\n \"num_failures_after_restore\": 0,\n \"error_filename\": null,\n \"pickled_error_filename\": null,\n \"last_result\": {\n \"loss\": 2019.4176592488927,\n \"timestamp\": 1719329610,\n \"checkpoint_dir_name\": \"checkpoint_000000\",\n \"should_checkpoint\": true,\n \"done\": false,\n \"training_iteration\": 1,\n \"trial_id\": \"040fcf09\",\n \"date\": \"2024-06-25_17-33-30\",\n \"time_this_iter_s\": 65.59718012809753,\n \"time_total_s\": 65.59718012809753,\n \"pid\": 2195253,\n \"hostname\": \"r8i6n2\",\n \"node_ip\": \"10.159.28.60\",\n \"config\": {\n \"encoder_num_layer\": 1,\n \"decoder_rt_num_layer\": 4,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 16,\n \"encoder_ff\": 1024,\n \"decoder_rt_ff\": 512,\n \"decoder_int_ff\": 512,\n \"n_head\": 1,\n \"drop_rate\": 0.18759871111111892,\n \"lr\": 0.02750689103642202,\n \"batch_size\": 2048\n },\n \"time_since_restore\": 65.59718012809753,\n \"iterations_since_restore\": 1,\n \"experiment_tag\": \"8_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=4,drop_rate=0.1876,embedding_dim=16,encoder_ff=1024,encoder_num_layer=1,lr=0.0275,n_head=1\"\n },\n \"last_result_time\": 1719329610.791616,\n \"metric_analysis\": {\n \"loss\": {\n \"max\": 2019.4176592488927,\n \"min\": 2019.4176592488927,\n \"avg\": 2019.4176592488927,\n \"last\": 2019.4176592488927,\n \"last-5-avg\": 2019.4176592488927,\n \"last-10-avg\": 2019.4176592488927\n },\n \"should_checkpoint\": {\n \"max\": true,\n \"min\": true,\n \"avg\": true,\n \"last\": true,\n \"last-5-avg\": true,\n \"last-10-avg\": true\n },\n \"done\": {\n \"max\": false,\n \"min\": false,\n \"avg\": false,\n \"last\": false,\n \"last-5-avg\": false,\n \"last-10-avg\": false\n },\n \"training_iteration\": {\n \"max\": 1,\n \"min\": 1,\n \"avg\": 1,\n \"last\": 1,\n \"last-5-avg\": 1,\n \"last-10-avg\": 1\n },\n \"time_this_iter_s\": {\n \"max\": 65.59718012809753,\n \"min\": 65.59718012809753,\n \"avg\": 65.59718012809753,\n \"last\": 65.59718012809753,\n \"last-5-avg\": 65.59718012809753,\n \"last-10-avg\": 65.59718012809753\n },\n \"time_total_s\": {\n \"max\": 65.59718012809753,\n \"min\": 65.59718012809753,\n \"avg\": 65.59718012809753,\n \"last\": 65.59718012809753,\n \"last-5-avg\": 65.59718012809753,\n \"last-10-avg\": 65.59718012809753\n },\n \"time_since_restore\": {\n \"max\": 65.59718012809753,\n \"min\": 65.59718012809753,\n \"avg\": 65.59718012809753,\n \"last\": 65.59718012809753,\n \"last-5-avg\": 65.59718012809753,\n \"last-10-avg\": 65.59718012809753\n },\n \"iterations_since_restore\": {\n \"max\": 1,\n \"min\": 1,\n \"avg\": 1,\n \"last\": 1,\n \"last-5-avg\": 1,\n \"last-10-avg\": 1\n }\n },\n \"_n_steps\": [\n 5,\n 10\n ],\n \"metric_n_steps\": {\n \"loss\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059589000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452948c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b00749462430877bbddaeab8d9f409486945294612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059589000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452948c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b00749462430877bbddaeab8d9f409486945294612e\"\n }\n },\n \"should_checkpoint\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059522000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b058694529488612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059522000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a8694529488612e\"\n }\n },\n \"done\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059522000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b058694529489612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059522000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a8694529489612e\"\n }\n },\n \"training_iteration\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059523000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452944b01612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059523000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452944b01612e\"\n }\n },\n \"time_this_iter_s\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294474050663833000000612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294474050663833000000612e\"\n }\n },\n \"time_total_s\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294474050663833000000612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294474050663833000000612e\"\n }\n },\n \"time_since_restore\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294474050663833000000612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294474050663833000000612e\"\n }\n },\n \"iterations_since_restore\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059523000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452944b01612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059523000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452944b01612e\"\n }\n }\n },\n \"checkpoint_manager\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059518060000000000008c267261792e747261696e2e5f696e7465726e616c2e636865636b706f696e745f6d616e61676572948c125f436865636b706f696e744d616e616765729493942981947d94288c125f636865636b706f696e745f636f6e666967948c097261792e747261696e948c10436865636b706f696e74436f6e6669679493942981947d94288c0b6e756d5f746f5f6b656570944e8c1a636865636b706f696e745f73636f72655f617474726962757465944e8c16636865636b706f696e745f73636f72655f6f72646572948c036d6178948c14636865636b706f696e745f6672657175656e6379944b008c11636865636b706f696e745f61745f656e6494898c1a5f636865636b706f696e745f6b6565705f616c6c5f72616e6b73948c0a44455052454341544544948c1f5f636865636b706f696e745f75706c6f61645f66726f6d5f776f726b65727394681275628c135f636865636b706f696e745f726573756c7473945d948c1b7261792e747261696e2e5f696e7465726e616c2e73657373696f6e948c0f5f547261696e696e67526573756c749493942981947d94288c0a636865636b706f696e749468068c0a436865636b706f696e749493942981947d94288c0470617468948cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f30343066636630395f385f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d3531322c6465636f6465725f72745f6e756d5f6c617965723d342c64726f705f72615f323032342d30362d32355f31372d33312d30312f636865636b706f696e745f303030303030948c0a66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c055f75756964948c0475756964948c04555549449493942981947d948c03696e74948a11fc7754c809abffae8d4f2132ea00c58200736275628c076d657472696373947d94288c046c6f7373948c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b00749462430877bbddaeab8d9f4094869452948c0974696d657374616d70944a4ae37a668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303030948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b018c08747269616c5f6964948c083034306663663039948c0464617465948c13323032342d30362d32355f31372d33332d3330948c1074696d655f746869735f697465725f73944740506638330000008c0c74696d655f746f74616c5f73944740506638330000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b018c146465636f6465725f72745f6e756d5f6c61796572944b048c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b108c0a656e636f6465725f6666944d00048c0d6465636f6465725f72745f6666944d00028c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b018c0964726f705f7261746594473fc8033c0c7f3ba48c026c7294473f9c2ac435aa0dc38c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f7265944740506638330000008c18697465726174696f6e735f73696e63655f726573746f7265944b01757562618c195f6c61746573745f636865636b706f696e745f726573756c7494681975622e\"\n }\n}"
+ ],
+ [
+ "{\n \"stub\": false,\n \"trainable_name\": \"train_model\",\n \"trial_id\": \"934c5104\",\n \"storage\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059572040000000000008c1b7261792e747261696e2e5f696e7465726e616c2e73746f72616765948c0e53746f72616765436f6e746578749493942981947d94288c12637573746f6d5f66735f70726f766964656494898c1273746f726167655f6c6f63616c5f70617468948c2b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c7473948c136578706572696d656e745f6469725f6e616d65948c0f746573745f6578706572696d656e74948c0e747269616c5f6469725f6e616d65948c96747261696e5f6d6f64656c5f39333463353130345f395f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d313032342c6465636f6465725f72745f6e756d5f6c617965723d322c64726f705f725f323032342d30362d32355f31372d33322d3235948c1863757272656e745f636865636b706f696e745f696e646578944b0d8c0b73796e635f636f6e666967948c097261792e747261696e948c0a53796e63436f6e6669679493942981947d94288c0b73796e635f706572696f64944d2c018c0c73796e635f74696d656f7574944d08078c0e73796e635f61727469666163747394898c1c73796e635f6172746966616374735f6f6e5f636865636b706f696e7494888c0a75706c6f61645f646972948c0a44455052454341544544948c0673796e6365729468188c1273796e635f6f6e5f636865636b706f696e7494681875628c1273746f726167655f66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c0f73746f726167655f66735f70617468948c452f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f7465737494681968008c115f46696c6573797374656d53796e6365729493942981947d9428681b682268134d2c0168144d08078c116c6173745f73796e635f75705f74696d65944741d99eb89fd0cfec8c136c6173745f73796e635f646f776e5f74696d659447fff00000000000008c0d5f73796e635f70726f63657373944e8c0c5f63757272656e745f636d649468008c125f75706c6f61645f746f5f66735f706174689493947d94288c0a6c6f63616c5f70617468948c3b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c74732f746573745f6578706572696d656e74948c0266739468228c0766735f70617468948c552f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e74948c076578636c756465945d948c0e2a2f636865636b706f696e745f2a9461758694756275622e\"\n },\n \"config\": {\n \"encoder_num_layer\": 1,\n \"decoder_rt_num_layer\": 2,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 256,\n \"encoder_ff\": 2048,\n \"decoder_rt_ff\": 1024,\n \"decoder_int_ff\": 512,\n \"n_head\": 8,\n \"drop_rate\": 0.2811887966312956,\n \"lr\": 0.0005389075253520159,\n \"batch_size\": 2048\n },\n \"_Trial__unresolved_config\": {\n \"encoder_num_layer\": 1,\n \"decoder_rt_num_layer\": 2,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 256,\n \"encoder_ff\": 2048,\n \"decoder_rt_ff\": 1024,\n \"decoder_int_ff\": 512,\n \"n_head\": 8,\n \"drop_rate\": 0.2811887966312956,\n \"lr\": 0.0005389075253520159,\n \"batch_size\": 2048\n },\n \"evaluated_params\": {\n \"batch_size\": 2048,\n \"decoder_int_ff\": 512,\n \"decoder_int_num_layer\": 1,\n \"decoder_rt_ff\": 1024,\n \"decoder_rt_num_layer\": 2,\n \"drop_rate\": 0.2811887966312956,\n \"embedding_dim\": 256,\n \"encoder_ff\": 2048,\n \"encoder_num_layer\": 1,\n \"lr\": 0.0005389075253520159,\n \"n_head\": 8\n },\n \"experiment_tag\": \"9_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop_rate=0.2812,embedding_dim=256,encoder_ff=2048,encoder_num_layer=1,lr=0.0005,n_head=8\",\n \"stopping_criterion\": {},\n \"_setup_default_resource\": true,\n \"_default_placement_group_factory\": \"80054e2e\",\n \"placement_group_factory\": \"800595ba000000000000008c237261792e74756e652e657865637574696f6e2e706c6163656d656e745f67726f757073948c15506c6163656d656e7447726f7570466163746f72799493942981947d94288c085f62756e646c6573945d947d94288c03435055944740540000000000008c034750559447401000000000000075618c155f686561645f62756e646c655f69735f656d70747994898c095f7374726174656779948c045041434b948c055f6172677394298c075f6b7761726773947d9475622e\",\n \"log_to_file\": [\n null,\n null\n ],\n \"max_failures\": 0,\n \"_default_result_or_future\": null,\n \"export_formats\": [],\n \"status\": \"TERMINATED\",\n \"relative_logdir\": \"train_model_934c5104_9_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop_r_2024-06-25_17-32-25\",\n \"trial_name_creator\": null,\n \"trial_dirname_creator\": null,\n \"custom_trial_name\": null,\n \"custom_dirname\": null,\n \"restore_path\": null,\n \"_restore_checkpoint_result\": null,\n \"_state_json\": null,\n \"results\": \"80054e2e\",\n \"extra_arg\": \"80054e2e\",\n \"_resources\": \"80054e2e\"\n}",
+ "{\n \"start_time\": 1719329610.941942,\n \"num_failures\": 0,\n \"num_failures_after_restore\": 0,\n \"error_filename\": null,\n \"pickled_error_filename\": null,\n \"last_result\": {\n \"loss\": 98.24798722154512,\n \"timestamp\": 1719332767,\n \"checkpoint_dir_name\": \"checkpoint_000013\",\n \"should_checkpoint\": true,\n \"done\": true,\n \"training_iteration\": 14,\n \"trial_id\": \"934c5104\",\n \"date\": \"2024-06-25_18-26-08\",\n \"time_this_iter_s\": 68.14825201034546,\n \"time_total_s\": 1037.901340007782,\n \"pid\": 2195253,\n \"hostname\": \"r8i6n2\",\n \"node_ip\": \"10.159.28.60\",\n \"config\": {\n \"encoder_num_layer\": 1,\n \"decoder_rt_num_layer\": 2,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 256,\n \"encoder_ff\": 2048,\n \"decoder_rt_ff\": 1024,\n \"decoder_int_ff\": 512,\n \"n_head\": 8,\n \"drop_rate\": 0.2811887966312956,\n \"lr\": 0.0005389075253520159,\n \"batch_size\": 2048\n },\n \"time_since_restore\": 696.4155676364899,\n \"iterations_since_restore\": 10,\n \"experiment_tag\": \"9_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop_rate=0.2812,embedding_dim=256,encoder_ff=2048,encoder_num_layer=1,lr=0.0005,n_head=8\"\n },\n \"last_result_time\": 1719332768.0763395,\n \"metric_analysis\": {\n \"loss\": {\n \"max\": 234.06637176753966,\n \"min\": 98.24798722154512,\n \"avg\": 132.7836964438817,\n \"last\": 98.24798722154512,\n \"last-5-avg\": 102.28564795546644,\n \"last-10-avg\": 110.75848803783026\n },\n \"should_checkpoint\": {\n \"max\": true,\n \"min\": true,\n \"avg\": 1.0,\n \"last\": true,\n \"last-5-avg\": 1.0,\n \"last-10-avg\": 1.0\n },\n \"done\": {\n \"max\": false,\n \"min\": false,\n \"avg\": 0.0,\n \"last\": false,\n \"last-5-avg\": 0.0,\n \"last-10-avg\": 0.0\n },\n \"training_iteration\": {\n \"max\": 14,\n \"min\": 1,\n \"avg\": 7.5,\n \"last\": 14,\n \"last-5-avg\": 12.0,\n \"last-10-avg\": 9.5\n },\n \"time_this_iter_s\": {\n \"max\": 95.2866895198822,\n \"min\": 67.7198805809021,\n \"avg\": 74.13581000055585,\n \"last\": 68.14825201034546,\n \"last-5-avg\": 68.35834956169128,\n \"last-10-avg\": 69.64155676364899\n },\n \"time_total_s\": {\n \"max\": 1037.901340007782,\n \"min\": 82.2380793094635,\n \"avg\": 582.5065506185805,\n \"last\": 1037.901340007782,\n \"last-5-avg\": 901.4340446949005,\n \"last-10-avg\": 730.7140613794327\n },\n \"time_since_restore\": {\n \"max\": 696.4155676364899,\n \"min\": 82.23219752311707,\n \"avg\": 309.21823942661285,\n \"last\": 696.4155676364899,\n \"last-5-avg\": 559.9482723236084,\n \"last-10-avg\": 389.22828900814056\n },\n \"iterations_since_restore\": {\n \"max\": 10,\n \"min\": 1,\n \"avg\": 4.285714285714286,\n \"last\": 10,\n \"last-5-avg\": 8.0,\n \"last-10-avg\": 5.5\n }\n },\n \"_n_steps\": [\n 5,\n 10\n ],\n \"metric_n_steps\": {\n \"loss\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"800595d6000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294288c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b00749462430832994c86a42a5b4094869452946807680d430863b1586c5982594094869452946807680d4308653299ac3316594094869452946807680d4308168bc5a25788594094869452946807680d43082e97cb05df8f58409486945294652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059535010000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294288c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b00749462430853a9547a09bf614094869452946807680d43084fa7d389576d5d4094869452946807680d430847a3d12863595c4094869452946807680d43089bcd661342ee5b4094869452946807680d43085d2e97abf6d65b4094869452946807680d430832994c86a42a5b4094869452946807680d430863b1586c5982594094869452946807680d4308653299ac3316594094869452946807680d4308168bc5a25788594094869452946807680d43082e97cb05df8f58409486945294652e\"\n }\n },\n \"should_checkpoint\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059527000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294288888888888652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952c000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452942888888888888888888888652e\"\n }\n },\n \"done\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059527000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294288989898989652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952c000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452942889898989898989898989652e\"\n }\n },\n \"training_iteration\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952c000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294284b0a4b0b4b0c4b0d4b0e652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059536000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294284b054b064b074b084b094b0a4b0b4b0c4b0d4b0e652e\"\n }\n },\n \"time_this_iter_s\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005954f000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294284740513f6b2000000047405103f21400000047405105d2960000004740511fff3f000000474051097cf6000000652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005957c000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452942847405499f0d5000000474051054b19000000474050ee128600000047405126f212000000474050f3ac240000004740513f6b2000000047405103f21400000047405105d2960000004740511fff3f000000474051097cf6000000652e\"\n }\n },\n \"time_total_s\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005954f000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b058694529428474087e8cdd5e0000047408a094c1860000047408c2a066b20000047408e4e0653000000474090379af8e00000652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005957c000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452942847407a7e41ee80000047407ebf94b4c000004740817d8cab200000474083a26aed600000474085c0e071e00000474087e8cdd5e0000047408a094c1860000047408c2a066b20000047408e4e0653000000474090379af8e00000652e\"\n }\n },\n \"time_since_restore\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005954f000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452942847407a79d5f280000047407ebad2778000004740817e238e800000474083a22376600000474085c35315200000652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005957c000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452942847405499f0d5000000474062cf9df700000047406b46a73a000000474071ed102180000047407629fb2a80000047407a79d5f280000047407ebad2778000004740817e238e800000474083a22376600000474085c35315200000652e\"\n }\n },\n \"iterations_since_restore\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952c000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294284b064b074b084b094b0a652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059536000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294284b014b024b034b044b054b064b074b084b094b0a652e\"\n }\n }\n },\n \"checkpoint_manager\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005954d340000000000008c267261792e747261696e2e5f696e7465726e616c2e636865636b706f696e745f6d616e61676572948c125f436865636b706f696e744d616e616765729493942981947d94288c125f636865636b706f696e745f636f6e666967948c097261792e747261696e948c10436865636b706f696e74436f6e6669679493942981947d94288c0b6e756d5f746f5f6b656570944e8c1a636865636b706f696e745f73636f72655f617474726962757465944e8c16636865636b706f696e745f73636f72655f6f72646572948c036d6178948c14636865636b706f696e745f6672657175656e6379944b008c11636865636b706f696e745f61745f656e6494898c1a5f636865636b706f696e745f6b6565705f616c6c5f72616e6b73948c0a44455052454341544544948c1f5f636865636b706f696e745f75706c6f61645f66726f6d5f776f726b65727394681275628c135f636865636b706f696e745f726573756c7473945d94288c1b7261792e747261696e2e5f696e7465726e616c2e73657373696f6e948c0f5f547261696e696e67526573756c749493942981947d94288c0a636865636b706f696e749468068c0a436865636b706f696e749493942981947d94288c0470617468948cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f39333463353130345f395f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d313032342c6465636f6465725f72745f6e756d5f6c617965723d322c64726f705f725f323032342d30362d32355f31372d33322d32352f636865636b706f696e745f303030303030948c0a66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c055f75756964948c0475756964948c04555549449493942981947d948c03696e74948a113ca5f21f44513f9f654790e892756aaa00736275628c076d657472696373947d94288c046c6f7373948c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b007494624308bd5eafb71f426d4094869452948c0974696d657374616d70944a9de37a668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303030948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b018c08747269616c5f6964948c083933346335313034948c0464617465948c13323032342d30362d32355f31372d33342d3533948c1074696d655f746869735f697465725f73944740548f3cb10000008c0c74696d655f746f74616c5f73944740548f3cb10000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b018c146465636f6465725f72745f6e756d5f6c61796572944b028c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944d00018c0a656e636f6465725f6666944d00088c0d6465636f6465725f72745f6666944d00048c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b088c0964726f705f7261746594473fd1feff4b621cd48c026c7294473f41a8af1935d3f68c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f7265944740548f3cb10000008c18697465726174696f6e735f73696e63655f726573746f7265944b0175756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f39333463353130345f395f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d313032342c6465636f6465725f72745f6e756d5f6c617965723d322c64726f705f725f323032342d30362d32355f31372d33322d32352f636865636b706f696e745f30303030303194682268257d946827897385945294682a682d2981947d9468308a117ca28ae951829cbc5b4c9022326fceab007362756268317d94288c046c6f7373946836683c43084fa7d3690fe9674094869452948c0974696d657374616d70944ac2e77a668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303031948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b028c08747269616c5f6964948c083933346335313034948c0464617465948c13323032342d30362d32355f31372d35322d3334948c1074696d655f746869735f697465725f73944740548edc530000008c0c74696d655f746f74616c5f73944740648f0c820000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b018c146465636f6465725f72745f6e756d5f6c61796572944b028c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944d00018c0a656e636f6465725f6666944d00088c0d6465636f6465725f72745f6666944d00048c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b088c0964726f705f7261746594473fd1feff4b621cd48c026c7294473f41a8af1935d3f68c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f7265944740548edc530000008c18697465726174696f6e735f73696e63655f726573746f7265944b0175756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f39333463353130345f395f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d313032342c6465636f6465725f72745f6e756d5f6c617965723d322c64726f705f725f323032342d30362d32355f31372d33322d32352f636865636b706f696e745f30303030303294682268257d946827897385945294682a682d2981947d9468308a11f8ffe8fb191278bdc64dd54f425d0895007362756268317d94288c046c6f7373946836683c4308592c166bdec1644094869452948c0974696d657374616d70944a31ea7a668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303032948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b038c08747269616c5f6964948c083933346335313034948c0464617465948c13323032342d30362d32355f31382d30322d3537948c1074696d655f746869735f697465725f7394474057d2591f0000008c0c74696d655f746f74616c5f73944740703c1c88c000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b018c146465636f6465725f72745f6e756d5f6c61796572944b028c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944d00018c0a656e636f6465725f6666944d00088c0d6465636f6465725f72745f6666944d00048c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b088c0964726f705f7261746594473fd1feff4b621cd48c026c7294473f41a8af1935d3f68c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f726594474057d2591f0000008c18697465726174696f6e735f73696e63655f726573746f7265944b0175756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f39333463353130345f395f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d313032342c6465636f6465725f72745f6e756d5f6c617965723d322c64726f705f725f323032342d30362d32355f31372d33322d32352f636865636b706f696e745f30303030303394682268257d946827897385945294682a682d2981947d9468308a102377a72e3654ecaa034308385248700f7362756268317d94288c046c6f7373946836683c4308028140b053ff634094869452948c0974696d657374616d70944a83ea7a668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303033948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b048c08747269616c5f6964948c083933346335313034948c0464617465948c13323032342d30362d32355f31382d30342d3139948c1074696d655f746869735f697465725f73944740546ea4c20000008c0c74696d655f746f74616c5f739447407557c5b94000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b018c146465636f6465725f72745f6e756d5f6c61796572944b028c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944d00018c0a656e636f6465725f6666944d00088c0d6465636f6465725f72745f6666944d00048c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b088c0964726f705f7261746594473fd1feff4b621cd48c026c7294473f41a8af1935d3f68c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f726594474066207ef08000008c18697465726174696f6e735f73696e63655f726573746f7265944b0275756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f39333463353130345f395f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d313032342c6465636f6465725f72745f6e756d5f6c617965723d322c64726f705f725f323032342d30362d32355f31372d33322d32352f636865636b706f696e745f30303030303494682268257d946827897385945294682a682d2981947d9468308a10e4fddd941927c4a8ec4f33d7e5fd991c7362756268317d94288c046c6f7373946836683c430853a9547a09bf614094869452948c0974696d657374616d70944a39ed7a668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303034948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b058c08747269616c5f6964948c083933346335313034948c0464617465948c13323032342d30362d32355f31382d31352d3534948c1074696d655f746869735f697465725f739447405499f0d50000008c0c74696d655f746f74616c5f739447407a7e41ee8000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b018c146465636f6465725f72745f6e756d5f6c61796572944b028c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944d00018c0a656e636f6465725f6666944d00088c0d6465636f6465725f72745f6666944d00048c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b088c0964726f705f7261746594473fd1feff4b621cd48c026c7294473f41a8af1935d3f68c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f72659447405499f0d50000008c18697465726174696f6e735f73696e63655f726573746f7265944b0175756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f39333463353130345f395f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d313032342c6465636f6465725f72745f6e756d5f6c617965723d322c64726f705f725f323032342d30362d32355f31372d33322d32352f636865636b706f696e745f30303030303594682268257d946827897385945294682a682d2981947d9468308a1121aa1fe7b3527aa95a439d1c8db5f0a5007362756268317d94288c046c6f7373946836683c43084fa7d389576d5d4094869452948c0974696d657374616d70944a7eed7a668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303035948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b068c08747269616c5f6964948c083933346335313034948c0464617465948c13323032342d30362d32355f31382d31372d3032948c1074696d655f746869735f697465725f7394474051054b190000008c0c74696d655f746f74616c5f739447407ebf94b4c000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b018c146465636f6465725f72745f6e756d5f6c61796572944b028c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944d00018c0a656e636f6465725f6666944d00088c0d6465636f6465725f72745f6666944d00048c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b088c0964726f705f7261746594473fd1feff4b621cd48c026c7294473f41a8af1935d3f68c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f726594474062cf9df70000008c18697465726174696f6e735f73696e63655f726573746f7265944b0275756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f39333463353130345f395f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d313032342c6465636f6465725f72745f6e756d5f6c617965723d322c64726f705f725f323032342d30362d32355f31372d33322d32352f636865636b706f696e745f30303030303694682268257d946827897385945294682a682d2981947d9468308a11c74c9ef6372bbdbe514cde6a5c8f09ff007362756268317d94288c046c6f7373946836683c430847a3d12863595c4094869452948c0974696d657374616d70944ac1ed7a668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303036948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b078c08747269616c5f6964948c083933346335313034948c0464617465948c13323032342d30362d32355f31382d31382d3039948c1074696d655f746869735f697465725f7394474050ee12860000008c0c74696d655f746f74616c5f73944740817d8cab2000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b018c146465636f6465725f72745f6e756d5f6c61796572944b028c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944d00018c0a656e636f6465725f6666944d00088c0d6465636f6465725f72745f6666944d00048c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b088c0964726f705f7261746594473fd1feff4b621cd48c026c7294473f41a8af1935d3f68c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f72659447406b46a73a0000008c18697465726174696f6e735f73696e63655f726573746f7265944b0375756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f39333463353130345f395f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d313032342c6465636f6465725f72745f6e756d5f6c617965723d322c64726f705f725f323032342d30362d32355f31372d33322d32352f636865636b706f696e745f30303030303794682268257d946827897385945294682a682d2981947d9468308a1053bcef571437bd9cc04adacd19b3473d7362756268317d94288c046c6f7373946836683c43089bcd661342ee5b4094869452948c0974696d657374616d70944a06ee7a668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303037948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b088c08747269616c5f6964948c083933346335313034948c0464617465948c13323032342d30362d32355f31382d31392d3138948c1074696d655f746869735f697465725f739447405126f2120000008c0c74696d655f746f74616c5f7394474083a26aed6000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b018c146465636f6465725f72745f6e756d5f6c61796572944b028c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944d00018c0a656e636f6465725f6666944d00088c0d6465636f6465725f72745f6666944d00048c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b088c0964726f705f7261746594473fd1feff4b621cd48c026c7294473f41a8af1935d3f68c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f726594474071ed10218000008c18697465726174696f6e735f73696e63655f726573746f7265944b0475756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f39333463353130345f395f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d313032342c6465636f6465725f72745f6e756d5f6c617965723d322c64726f705f725f323032342d30362d32355f31372d33322d32352f636865636b706f696e745f30303030303894682268257d946827897385945294682a682d2981947d9468308a1195212ae085dbfb82a54a82fe466abbc1007362756268317d94288c046c6f7373946836683c43085d2e97abf6d65b4094869452948c0974696d657374616d70944a4aee7a668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303038948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b098c08747269616c5f6964948c083933346335313034948c0464617465948c13323032342d30362d32355f31382d32302d3236948c1074696d655f746869735f697465725f7394474050f3ac240000008c0c74696d655f746f74616c5f7394474085c0e071e000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b018c146465636f6465725f72745f6e756d5f6c61796572944b028c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944d00018c0a656e636f6465725f6666944d00088c0d6465636f6465725f72745f6666944d00048c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b088c0964726f705f7261746594473fd1feff4b621cd48c026c7294473f41a8af1935d3f68c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f72659447407629fb2a8000008c18697465726174696f6e735f73696e63655f726573746f7265944b0575756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f39333463353130345f395f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d313032342c6465636f6465725f72745f6e756d5f6c617965723d322c64726f705f725f323032342d30362d32355f31372d33322d32352f636865636b706f696e745f30303030303994682268257d946827897385945294682a682d2981947d9468308a10ee3248a264cfe3a4c54fdc972eeda71b7362756268317d94288c046c6f7373946836683c430832994c86a42a5b4094869452948c0974696d657374616d70944a8eee7a668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303039948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b0a8c08747269616c5f6964948c083933346335313034948c0464617465948c13323032342d30362d32355f31382d32312d3335948c1074696d655f746869735f697465725f73944740513f6b200000008c0c74696d655f746f74616c5f7394474087e8cdd5e000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b018c146465636f6465725f72745f6e756d5f6c61796572944b028c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944d00018c0a656e636f6465725f6666944d00088c0d6465636f6465725f72745f6666944d00048c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b088c0964726f705f7261746594473fd1feff4b621cd48c026c7294473f41a8af1935d3f68c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f72659447407a79d5f28000008c18697465726174696f6e735f73696e63655f726573746f7265944b0675756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f39333463353130345f395f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d313032342c6465636f6465725f72745f6e756d5f6c617965723d322c64726f705f725f323032342d30362d32355f31372d33322d32352f636865636b706f696e745f30303030313094682268257d946827897385945294682a682d2981947d9468308a11ddb581b325cb91b0374ddf2264a944d4007362756268317d94288c046c6f7373946836683c430863b1586c5982594094869452948c0974696d657374616d70944ad3ee7a668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303130948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b0b8c08747269616c5f6964948c083933346335313034948c0464617465948c13323032342d30362d32355f31382d32322d3433948c1074696d655f746869735f697465725f739447405103f2140000008c0c74696d655f746f74616c5f739447408a094c186000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b018c146465636f6465725f72745f6e756d5f6c61796572944b028c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944d00018c0a656e636f6465725f6666944d00088c0d6465636f6465725f72745f6666944d00048c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b088c0964726f705f7261746594473fd1feff4b621cd48c026c7294473f41a8af1935d3f68c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f72659447407ebad2778000008c18697465726174696f6e735f73696e63655f726573746f7265944b0775756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f39333463353130345f395f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d313032342c6465636f6465725f72745f6e756d5f6c617965723d322c64726f705f725f323032342d30362d32355f31372d33322d32352f636865636b706f696e745f30303030313194682268257d946827897385945294682a682d2981947d9468308a10e7b8684994fe77a9a34e7f60393270667362756268317d94288c046c6f7373946836683c4308653299ac3316594094869452948c0974696d657374616d70944a17ef7a668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303131948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b0c8c08747269616c5f6964948c083933346335313034948c0464617465948c13323032342d30362d32355f31382d32332d3531948c1074696d655f746869735f697465725f739447405105d2960000008c0c74696d655f746f74616c5f739447408c2a066b2000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b018c146465636f6465725f72745f6e756d5f6c61796572944b028c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944d00018c0a656e636f6465725f6666944d00088c0d6465636f6465725f72745f6666944d00048c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b088c0964726f705f7261746594473fd1feff4b621cd48c026c7294473f41a8af1935d3f68c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f7265944740817e238e8000008c18697465726174696f6e735f73696e63655f726573746f7265944b0875756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f39333463353130345f395f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d313032342c6465636f6465725f72745f6e756d5f6c617965723d322c64726f705f725f323032342d30362d32355f31372d33322d32352f636865636b706f696e745f30303030313294682268257d946827897385945294682a682d2981947d9468308a1155973d114de961a7f24898e54966f6d9007362756268317d94288c046c6f7373946836683c4308168bc5a25788594094869452948c0974696d657374616d70944a5bef7a668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303132948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b0d8c08747269616c5f6964948c083933346335313034948c0464617465948c13323032342d30362d32355f31382d32342d3539948c1074696d655f746869735f697465725f73944740511fff3f0000008c0c74696d655f746f74616c5f739447408e4e06530000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b018c146465636f6465725f72745f6e756d5f6c61796572944b028c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944d00018c0a656e636f6465725f6666944d00088c0d6465636f6465725f72745f6666944d00048c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b088c0964726f705f7261746594473fd1feff4b621cd48c026c7294473f41a8af1935d3f68c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f726594474083a223766000008c18697465726174696f6e735f73696e63655f726573746f7265944b0975756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f39333463353130345f395f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d313032342c6465636f6465725f72745f6e756d5f6c617965723d322c64726f705f725f323032342d30362d32355f31372d33322d32352f636865636b706f696e745f30303030313394682268257d946827897385945294682a682d2981947d9468308a10cb0a04e2aef0c18bb140650baea389707362756268317d94288c046c6f7373946836683c43082e97cb05df8f584094869452948c0974696d657374616d70944a9fef7a668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303133948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b0e8c08747269616c5f6964948c083933346335313034948c0464617465948c13323032342d30362d32355f31382d32362d3038948c1074696d655f746869735f697465725f7394474051097cf60000008c0c74696d655f746f74616c5f7394474090379af8e000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b018c146465636f6465725f72745f6e756d5f6c61796572944b028c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944d00018c0a656e636f6465725f6666944d00088c0d6465636f6465725f72745f6666944d00048c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b088c0964726f705f7261746594473fd1feff4b621cd48c026c7294473f41a8af1935d3f68c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f726594474085c353152000008c18697465726174696f6e735f73696e63655f726573746f7265944b0a757562658c195f6c61746573745f636865636b706f696e745f726573756c74946a9602000075622e\"\n }\n}"
+ ],
+ [
+ "{\n \"stub\": false,\n \"trainable_name\": \"train_model\",\n \"trial_id\": \"481f1fff\",\n \"storage\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059572040000000000008c1b7261792e747261696e2e5f696e7465726e616c2e73746f72616765948c0e53746f72616765436f6e746578749493942981947d94288c12637573746f6d5f66735f70726f766964656494898c1273746f726167655f6c6f63616c5f70617468948c2b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c7473948c136578706572696d656e745f6469725f6e616d65948c0f746573745f6578706572696d656e74948c0e747269616c5f6469725f6e616d65948c96747261696e5f6d6f64656c5f34383166316666665f31305f62617463685f73697a653d313032342c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d313032342c6465636f6465725f72745f6e756d5f6c617965723d382c64726f705f5f323032342d30362d32355f31372d33332d3330948c1863757272656e745f636865636b706f696e745f696e646578944b018c0b73796e635f636f6e666967948c097261792e747261696e948c0a53796e63436f6e6669679493942981947d94288c0b73796e635f706572696f64944d2c018c0c73796e635f74696d656f7574944d08078c0e73796e635f61727469666163747394898c1c73796e635f6172746966616374735f6f6e5f636865636b706f696e7494888c0a75706c6f61645f646972948c0a44455052454341544544948c0673796e6365729468188c1273796e635f6f6e5f636865636b706f696e7494681875628c1273746f726167655f66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c0f73746f726167655f66735f70617468948c452f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f7465737494681968008c115f46696c6573797374656d53796e6365729493942981947d9428681b682268134d2c0168144d08078c116c6173745f73796e635f75705f74696d65944741d99eb89fd0cfec8c136c6173745f73796e635f646f776e5f74696d659447fff00000000000008c0d5f73796e635f70726f63657373944e8c0c5f63757272656e745f636d649468008c125f75706c6f61645f746f5f66735f706174689493947d94288c0a6c6f63616c5f70617468948c3b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c74732f746573745f6578706572696d656e74948c0266739468228c0766735f70617468948c552f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e74948c076578636c756465945d948c0e2a2f636865636b706f696e745f2a9461758694756275622e\"\n },\n \"config\": {\n \"encoder_num_layer\": 2,\n \"decoder_rt_num_layer\": 8,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 64,\n \"encoder_ff\": 1024,\n \"decoder_rt_ff\": 1024,\n \"decoder_int_ff\": 512,\n \"n_head\": 16,\n \"drop_rate\": 0.15995138350968297,\n \"lr\": 0.004656630423530019,\n \"batch_size\": 1024\n },\n \"_Trial__unresolved_config\": {\n \"encoder_num_layer\": 2,\n \"decoder_rt_num_layer\": 8,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 64,\n \"encoder_ff\": 1024,\n \"decoder_rt_ff\": 1024,\n \"decoder_int_ff\": 512,\n \"n_head\": 16,\n \"drop_rate\": 0.15995138350968297,\n \"lr\": 0.004656630423530019,\n \"batch_size\": 1024\n },\n \"evaluated_params\": {\n \"batch_size\": 1024,\n \"decoder_int_ff\": 512,\n \"decoder_int_num_layer\": 1,\n \"decoder_rt_ff\": 1024,\n \"decoder_rt_num_layer\": 8,\n \"drop_rate\": 0.15995138350968297,\n \"embedding_dim\": 64,\n \"encoder_ff\": 1024,\n \"encoder_num_layer\": 2,\n \"lr\": 0.004656630423530019,\n \"n_head\": 16\n },\n \"experiment_tag\": \"10_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=8,drop_rate=0.1600,embedding_dim=64,encoder_ff=1024,encoder_num_layer=2,lr=0.0047,n_head=16\",\n \"stopping_criterion\": {},\n \"_setup_default_resource\": true,\n \"_default_placement_group_factory\": \"80054e2e\",\n \"placement_group_factory\": \"800595ba000000000000008c237261792e74756e652e657865637574696f6e2e706c6163656d656e745f67726f757073948c15506c6163656d656e7447726f7570466163746f72799493942981947d94288c085f62756e646c6573945d947d94288c03435055944740540000000000008c034750559447401000000000000075618c155f686561645f62756e646c655f69735f656d70747994898c095f7374726174656779948c045041434b948c055f6172677394298c075f6b7761726773947d9475622e\",\n \"log_to_file\": [\n null,\n null\n ],\n \"max_failures\": 0,\n \"_default_result_or_future\": null,\n \"export_formats\": [],\n \"status\": \"PAUSED\",\n \"relative_logdir\": \"train_model_481f1fff_10_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=8,drop__2024-06-25_17-33-30\",\n \"trial_name_creator\": null,\n \"trial_dirname_creator\": null,\n \"custom_trial_name\": null,\n \"custom_dirname\": null,\n \"restore_path\": null,\n \"_restore_checkpoint_result\": null,\n \"_state_json\": null,\n \"results\": \"80054e2e\",\n \"extra_arg\": \"80054e2e\",\n \"_resources\": \"80054e2e\"\n}",
+ "{\n \"start_time\": 1719329693.3274555,\n \"num_failures\": 0,\n \"num_failures_after_restore\": 0,\n \"error_filename\": null,\n \"pickled_error_filename\": null,\n \"last_result\": {\n \"loss\": 1959.9800554410679,\n \"timestamp\": 1719330869,\n \"checkpoint_dir_name\": \"checkpoint_000001\",\n \"should_checkpoint\": true,\n \"done\": false,\n \"training_iteration\": 2,\n \"trial_id\": \"481f1fff\",\n \"date\": \"2024-06-25_17-54-29\",\n \"time_this_iter_s\": 114.88935971260071,\n \"time_total_s\": 228.74907898902893,\n \"pid\": 2195253,\n \"hostname\": \"r8i6n2\",\n \"node_ip\": \"10.159.28.60\",\n \"config\": {\n \"encoder_num_layer\": 2,\n \"decoder_rt_num_layer\": 8,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 64,\n \"encoder_ff\": 1024,\n \"decoder_rt_ff\": 1024,\n \"decoder_int_ff\": 512,\n \"n_head\": 16,\n \"drop_rate\": 0.15995138350968297,\n \"lr\": 0.004656630423530019,\n \"batch_size\": 1024\n },\n \"time_since_restore\": 114.88935971260071,\n \"iterations_since_restore\": 1,\n \"experiment_tag\": \"10_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=8,drop_rate=0.1600,embedding_dim=64,encoder_ff=1024,encoder_num_layer=2,lr=0.0047,n_head=16\"\n },\n \"last_result_time\": 1719330869.3889194,\n \"metric_analysis\": {\n \"loss\": {\n \"max\": 1961.035730076587,\n \"min\": 1959.9800554410679,\n \"avg\": 1960.5078927588274,\n \"last\": 1959.9800554410679,\n \"last-5-avg\": 1960.5078927588274,\n \"last-10-avg\": 1960.5078927588274\n },\n \"should_checkpoint\": {\n \"max\": true,\n \"min\": true,\n \"avg\": 1.0,\n \"last\": true,\n \"last-5-avg\": 1.0,\n \"last-10-avg\": 1.0\n },\n \"done\": {\n \"max\": false,\n \"min\": false,\n \"avg\": 0.0,\n \"last\": false,\n \"last-5-avg\": 0.0,\n \"last-10-avg\": 0.0\n },\n \"training_iteration\": {\n \"max\": 2,\n \"min\": 1,\n \"avg\": 1.5,\n \"last\": 2,\n \"last-5-avg\": 1.5,\n \"last-10-avg\": 1.5\n },\n \"time_this_iter_s\": {\n \"max\": 114.88935971260071,\n \"min\": 113.85971927642822,\n \"avg\": 114.37453949451447,\n \"last\": 114.88935971260071,\n \"last-5-avg\": 114.37453949451447,\n \"last-10-avg\": 114.37453949451447\n },\n \"time_total_s\": {\n \"max\": 228.74907898902893,\n \"min\": 113.85971927642822,\n \"avg\": 171.30439913272858,\n \"last\": 228.74907898902893,\n \"last-5-avg\": 171.30439913272858,\n \"last-10-avg\": 171.30439913272858\n },\n \"time_since_restore\": {\n \"max\": 114.88935971260071,\n \"min\": 113.85971927642822,\n \"avg\": 114.37453949451447,\n \"last\": 114.88935971260071,\n \"last-5-avg\": 114.37453949451447,\n \"last-10-avg\": 114.37453949451447\n },\n \"iterations_since_restore\": {\n \"max\": 1,\n \"min\": 1,\n \"avg\": 1.0,\n \"last\": 1,\n \"last-5-avg\": 1.0,\n \"last-10-avg\": 1.0\n }\n },\n \"_n_steps\": [\n 5,\n 10\n ],\n \"metric_n_steps\": {\n \"loss\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005959d000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294288c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b007494624308b3d96c9624a49e4094869452946807680d43089d4ea793eb9f9e409486945294652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005959d000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294288c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b007494624308b3d96c9624a49e4094869452946807680d43089d4ea793eb9f9e409486945294652e\"\n }\n },\n \"should_checkpoint\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059524000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294288888652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059524000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294288888652e\"\n }\n },\n \"done\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059524000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294288989652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059524000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294288989652e\"\n }\n },\n \"training_iteration\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059526000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294284b014b02652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059526000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294284b014b02652e\"\n }\n },\n \"time_this_iter_s\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059534000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452942847405c7705a400000047405cb8eb45000000652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059534000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452942847405c7705a400000047405cb8eb45000000652e\"\n }\n },\n \"time_total_s\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059534000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452942847405c7705a400000047406c97f874800000652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059534000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452942847405c7705a400000047406c97f874800000652e\"\n }\n },\n \"time_since_restore\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059534000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452942847405c7705a400000047405cb8eb45000000652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059534000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452942847405c7705a400000047405cb8eb45000000652e\"\n }\n },\n \"iterations_since_restore\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059526000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294284b014b01652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059526000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294284b014b01652e\"\n }\n }\n },\n \"checkpoint_manager\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"800595a5090000000000008c267261792e747261696e2e5f696e7465726e616c2e636865636b706f696e745f6d616e61676572948c125f436865636b706f696e744d616e616765729493942981947d94288c125f636865636b706f696e745f636f6e666967948c097261792e747261696e948c10436865636b706f696e74436f6e6669679493942981947d94288c0b6e756d5f746f5f6b656570944e8c1a636865636b706f696e745f73636f72655f617474726962757465944e8c16636865636b706f696e745f73636f72655f6f72646572948c036d6178948c14636865636b706f696e745f6672657175656e6379944b008c11636865636b706f696e745f61745f656e6494898c1a5f636865636b706f696e745f6b6565705f616c6c5f72616e6b73948c0a44455052454341544544948c1f5f636865636b706f696e745f75706c6f61645f66726f6d5f776f726b65727394681275628c135f636865636b706f696e745f726573756c7473945d94288c1b7261792e747261696e2e5f696e7465726e616c2e73657373696f6e948c0f5f547261696e696e67526573756c749493942981947d94288c0a636865636b706f696e749468068c0a436865636b706f696e749493942981947d94288c0470617468948cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f34383166316666665f31305f62617463685f73697a653d313032342c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d313032342c6465636f6465725f72745f6e756d5f6c617965723d382c64726f705f5f323032342d30362d32355f31372d33332d33302f636865636b706f696e745f303030303030948c0a66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c055f75756964948c0475756964948c04555549449493942981947d948c03696e74948a101a1ee3d5d45707b6114cb3034c146e7e736275628c076d657472696373947d94288c046c6f7373948c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b007494624308b3d96c9624a49e4094869452948c0974696d657374616d70944a0fe47a668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303030948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b018c08747269616c5f6964948c083438316631666666948c0464617465948c13323032342d30362d32355f31372d33362d3437948c1074696d655f746869735f697465725f739447405c7705a40000008c0c74696d655f746f74616c5f739447405c7705a40000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b028c146465636f6465725f72745f6e756d5f6c61796572944b088c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b408c0a656e636f6465725f6666944d00048c0d6465636f6465725f72745f6666944d00048c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b108c0964726f705f7261746594473fc47949748fe09c8c026c7294473f7312d4b60ed3e98c0a62617463685f73697a65944d0004758c1274696d655f73696e63655f726573746f72659447405c7705a40000008c18697465726174696f6e735f73696e63655f726573746f7265944b0175756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f34383166316666665f31305f62617463685f73697a653d313032342c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d313032342c6465636f6465725f72745f6e756d5f6c617965723d382c64726f705f5f323032342d30362d32355f31372d33332d33302f636865636b706f696e745f30303030303194682268257d946827897385945294682a682d2981947d9468308a1103a55102964a82888840273d3f2dbac6007362756268317d94288c046c6f7373946836683c43089d4ea793eb9f9e4094869452948c0974696d657374616d70944a35e87a668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303031948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b028c08747269616c5f6964948c083438316631666666948c0464617465948c13323032342d30362d32355f31372d35342d3239948c1074696d655f746869735f697465725f739447405cb8eb450000008c0c74696d655f746f74616c5f739447406c97f8748000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b028c146465636f6465725f72745f6e756d5f6c61796572944b088c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b408c0a656e636f6465725f6666944d00048c0d6465636f6465725f72745f6666944d00048c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b108c0964726f705f7261746594473fc47949748fe09c8c026c7294473f7312d4b60ed3e98c0a62617463685f73697a65944d0004758c1274696d655f73696e63655f726573746f72659447405cb8eb450000008c18697465726174696f6e735f73696e63655f726573746f7265944b01757562658c195f6c61746573745f636865636b706f696e745f726573756c7494686275622e\"\n }\n}"
+ ],
+ [
+ "{\n \"stub\": false,\n \"trainable_name\": \"train_model\",\n \"trial_id\": \"2465ee2c\",\n \"storage\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059572040000000000008c1b7261792e747261696e2e5f696e7465726e616c2e73746f72616765948c0e53746f72616765436f6e746578749493942981947d94288c12637573746f6d5f66735f70726f766964656494898c1273746f726167655f6c6f63616c5f70617468948c2b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c7473948c136578706572696d656e745f6469725f6e616d65948c0f746573745f6578706572696d656e74948c0e747269616c5f6469725f6e616d65948c96747261696e5f6d6f64656c5f32343635656532635f31315f62617463685f73697a653d313032342c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d313032342c6465636f6465725f72745f6e756d5f6c617965723d322c64726f705f5f323032342d30362d32355f31372d33342d3533948c1863757272656e745f636865636b706f696e745f696e646578944b008c0b73796e635f636f6e666967948c097261792e747261696e948c0a53796e63436f6e6669679493942981947d94288c0b73796e635f706572696f64944d2c018c0c73796e635f74696d656f7574944d08078c0e73796e635f61727469666163747394898c1c73796e635f6172746966616374735f6f6e5f636865636b706f696e7494888c0a75706c6f61645f646972948c0a44455052454341544544948c0673796e6365729468188c1273796e635f6f6e5f636865636b706f696e7494681875628c1273746f726167655f66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c0f73746f726167655f66735f70617468948c452f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f7465737494681968008c115f46696c6573797374656d53796e6365729493942981947d9428681b682268134d2c0168144d08078c116c6173745f73796e635f75705f74696d65944741d99eb89fd0cfec8c136c6173745f73796e635f646f776e5f74696d659447fff00000000000008c0d5f73796e635f70726f63657373944e8c0c5f63757272656e745f636d649468008c125f75706c6f61645f746f5f66735f706174689493947d94288c0a6c6f63616c5f70617468948c3b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c74732f746573745f6578706572696d656e74948c0266739468228c0766735f70617468948c552f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e74948c076578636c756465945d948c0e2a2f636865636b706f696e745f2a9461758694756275622e\"\n },\n \"config\": {\n \"encoder_num_layer\": 4,\n \"decoder_rt_num_layer\": 2,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 64,\n \"encoder_ff\": 2048,\n \"decoder_rt_ff\": 1024,\n \"decoder_int_ff\": 512,\n \"n_head\": 8,\n \"drop_rate\": 0.3089996310219745,\n \"lr\": 0.062282868368169514,\n \"batch_size\": 1024\n },\n \"_Trial__unresolved_config\": {\n \"encoder_num_layer\": 4,\n \"decoder_rt_num_layer\": 2,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 64,\n \"encoder_ff\": 2048,\n \"decoder_rt_ff\": 1024,\n \"decoder_int_ff\": 512,\n \"n_head\": 8,\n \"drop_rate\": 0.3089996310219745,\n \"lr\": 0.062282868368169514,\n \"batch_size\": 1024\n },\n \"evaluated_params\": {\n \"batch_size\": 1024,\n \"decoder_int_ff\": 512,\n \"decoder_int_num_layer\": 1,\n \"decoder_rt_ff\": 1024,\n \"decoder_rt_num_layer\": 2,\n \"drop_rate\": 0.3089996310219745,\n \"embedding_dim\": 64,\n \"encoder_ff\": 2048,\n \"encoder_num_layer\": 4,\n \"lr\": 0.062282868368169514,\n \"n_head\": 8\n },\n \"experiment_tag\": \"11_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop_rate=0.3090,embedding_dim=64,encoder_ff=2048,encoder_num_layer=4,lr=0.0623,n_head=8\",\n \"stopping_criterion\": {},\n \"_setup_default_resource\": true,\n \"_default_placement_group_factory\": \"80054e2e\",\n \"placement_group_factory\": \"800595ba000000000000008c237261792e74756e652e657865637574696f6e2e706c6163656d656e745f67726f757073948c15506c6163656d656e7447726f7570466163746f72799493942981947d94288c085f62756e646c6573945d947d94288c03435055944740540000000000008c034750559447401000000000000075618c155f686561645f62756e646c655f69735f656d70747994898c095f7374726174656779948c045041434b948c055f6172677394298c075f6b7761726773947d9475622e\",\n \"log_to_file\": [\n null,\n null\n ],\n \"max_failures\": 0,\n \"_default_result_or_future\": null,\n \"export_formats\": [],\n \"status\": \"PAUSED\",\n \"relative_logdir\": \"train_model_2465ee2c_11_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop__2024-06-25_17-34-53\",\n \"trial_name_creator\": null,\n \"trial_dirname_creator\": null,\n \"custom_trial_name\": null,\n \"custom_dirname\": null,\n \"restore_path\": null,\n \"_restore_checkpoint_result\": null,\n \"_state_json\": null,\n \"results\": \"80054e2e\",\n \"extra_arg\": \"80054e2e\",\n \"_resources\": \"80054e2e\"\n}",
+ "{\n \"start_time\": 1719329807.375336,\n \"num_failures\": 0,\n \"num_failures_after_restore\": 0,\n \"error_filename\": null,\n \"pickled_error_filename\": null,\n \"last_result\": {\n \"loss\": 2699.211756428396,\n \"timestamp\": 1719329892,\n \"checkpoint_dir_name\": \"checkpoint_000000\",\n \"should_checkpoint\": true,\n \"done\": false,\n \"training_iteration\": 1,\n \"trial_id\": \"2465ee2c\",\n \"date\": \"2024-06-25_17-38-12\",\n \"time_this_iter_s\": 85.10484099388123,\n \"time_total_s\": 85.10484099388123,\n \"pid\": 2195253,\n \"hostname\": \"r8i6n2\",\n \"node_ip\": \"10.159.28.60\",\n \"config\": {\n \"encoder_num_layer\": 4,\n \"decoder_rt_num_layer\": 2,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 64,\n \"encoder_ff\": 2048,\n \"decoder_rt_ff\": 1024,\n \"decoder_int_ff\": 512,\n \"n_head\": 8,\n \"drop_rate\": 0.3089996310219745,\n \"lr\": 0.062282868368169514,\n \"batch_size\": 1024\n },\n \"time_since_restore\": 85.10484099388123,\n \"iterations_since_restore\": 1,\n \"experiment_tag\": \"11_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop_rate=0.3090,embedding_dim=64,encoder_ff=2048,encoder_num_layer=4,lr=0.0623,n_head=8\"\n },\n \"last_result_time\": 1719329892.485186,\n \"metric_analysis\": {\n \"loss\": {\n \"max\": 2699.211756428396,\n \"min\": 2699.211756428396,\n \"avg\": 2699.211756428396,\n \"last\": 2699.211756428396,\n \"last-5-avg\": 2699.211756428396,\n \"last-10-avg\": 2699.211756428396\n },\n \"should_checkpoint\": {\n \"max\": true,\n \"min\": true,\n \"avg\": true,\n \"last\": true,\n \"last-5-avg\": true,\n \"last-10-avg\": true\n },\n \"done\": {\n \"max\": false,\n \"min\": false,\n \"avg\": false,\n \"last\": false,\n \"last-5-avg\": false,\n \"last-10-avg\": false\n },\n \"training_iteration\": {\n \"max\": 1,\n \"min\": 1,\n \"avg\": 1,\n \"last\": 1,\n \"last-5-avg\": 1,\n \"last-10-avg\": 1\n },\n \"time_this_iter_s\": {\n \"max\": 85.10484099388123,\n \"min\": 85.10484099388123,\n \"avg\": 85.10484099388123,\n \"last\": 85.10484099388123,\n \"last-5-avg\": 85.10484099388123,\n \"last-10-avg\": 85.10484099388123\n },\n \"time_total_s\": {\n \"max\": 85.10484099388123,\n \"min\": 85.10484099388123,\n \"avg\": 85.10484099388123,\n \"last\": 85.10484099388123,\n \"last-5-avg\": 85.10484099388123,\n \"last-10-avg\": 85.10484099388123\n },\n \"time_since_restore\": {\n \"max\": 85.10484099388123,\n \"min\": 85.10484099388123,\n \"avg\": 85.10484099388123,\n \"last\": 85.10484099388123,\n \"last-5-avg\": 85.10484099388123,\n \"last-10-avg\": 85.10484099388123\n },\n \"iterations_since_restore\": {\n \"max\": 1,\n \"min\": 1,\n \"avg\": 1,\n \"last\": 1,\n \"last-5-avg\": 1,\n \"last-10-avg\": 1\n }\n },\n \"_n_steps\": [\n 5,\n 10\n ],\n \"metric_n_steps\": {\n \"loss\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059589000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452948c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b0074946243085bad566b6c16a5409486945294612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059589000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452948c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b0074946243085bad566b6c16a5409486945294612e\"\n }\n },\n \"should_checkpoint\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059522000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b058694529488612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059522000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a8694529488612e\"\n }\n },\n \"done\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059522000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b058694529489612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059522000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a8694529489612e\"\n }\n },\n \"training_iteration\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059523000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452944b01612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059523000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452944b01612e\"\n }\n },\n \"time_this_iter_s\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b058694529447405546b5b7000000612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a8694529447405546b5b7000000612e\"\n }\n },\n \"time_total_s\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b058694529447405546b5b7000000612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a8694529447405546b5b7000000612e\"\n }\n },\n \"time_since_restore\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b058694529447405546b5b7000000612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a8694529447405546b5b7000000612e\"\n }\n },\n \"iterations_since_restore\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059523000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452944b01612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059523000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452944b01612e\"\n }\n }\n },\n \"checkpoint_manager\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059518060000000000008c267261792e747261696e2e5f696e7465726e616c2e636865636b706f696e745f6d616e61676572948c125f436865636b706f696e744d616e616765729493942981947d94288c125f636865636b706f696e745f636f6e666967948c097261792e747261696e948c10436865636b706f696e74436f6e6669679493942981947d94288c0b6e756d5f746f5f6b656570944e8c1a636865636b706f696e745f73636f72655f617474726962757465944e8c16636865636b706f696e745f73636f72655f6f72646572948c036d6178948c14636865636b706f696e745f6672657175656e6379944b008c11636865636b706f696e745f61745f656e6494898c1a5f636865636b706f696e745f6b6565705f616c6c5f72616e6b73948c0a44455052454341544544948c1f5f636865636b706f696e745f75706c6f61645f66726f6d5f776f726b65727394681275628c135f636865636b706f696e745f726573756c7473945d948c1b7261792e747261696e2e5f696e7465726e616c2e73657373696f6e948c0f5f547261696e696e67526573756c749493942981947d94288c0a636865636b706f696e749468068c0a436865636b706f696e749493942981947d94288c0470617468948cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f32343635656532635f31315f62617463685f73697a653d313032342c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d313032342c6465636f6465725f72745f6e756d5f6c617965723d322c64726f705f5f323032342d30362d32355f31372d33342d35332f636865636b706f696e745f303030303030948c0a66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c055f75756964948c0475756964948c04555549449493942981947d948c03696e74948a11d69238a42f7cd0b08f42f95515b8d7e000736275628c076d657472696373947d94288c046c6f7373948c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b0074946243085bad566b6c16a54094869452948c0974696d657374616d70944a64e47a668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303030948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b018c08747269616c5f6964948c083234363565653263948c0464617465948c13323032342d30362d32355f31372d33382d3132948c1074696d655f746869735f697465725f739447405546b5b70000008c0c74696d655f746f74616c5f739447405546b5b70000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b048c146465636f6465725f72745f6e756d5f6c61796572944b028c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b408c0a656e636f6465725f6666944d00088c0d6465636f6465725f72745f6666944d00048c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b088c0964726f705f7261746594473fd3c6a6636dc9e48c026c7294473fafe38a457c16bc8c0a62617463685f73697a65944d0004758c1274696d655f73696e63655f726573746f72659447405546b5b70000008c18697465726174696f6e735f73696e63655f726573746f7265944b01757562618c195f6c61746573745f636865636b706f696e745f726573756c7494681975622e\"\n }\n}"
+ ],
+ [
+ "{\n \"stub\": false,\n \"trainable_name\": \"train_model\",\n \"trial_id\": \"ad12346e\",\n \"storage\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059572040000000000008c1b7261792e747261696e2e5f696e7465726e616c2e73746f72616765948c0e53746f72616765436f6e746578749493942981947d94288c12637573746f6d5f66735f70726f766964656494898c1273746f726167655f6c6f63616c5f70617468948c2b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c7473948c136578706572696d656e745f6469725f6e616d65948c0f746573745f6578706572696d656e74948c0e747269616c5f6469725f6e616d65948c96747261696e5f6d6f64656c5f61643132333436655f31325f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d313032342c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f5f323032342d30362d32355f31372d33362d3437948c1863757272656e745f636865636b706f696e745f696e646578944b008c0b73796e635f636f6e666967948c097261792e747261696e948c0a53796e63436f6e6669679493942981947d94288c0b73796e635f706572696f64944d2c018c0c73796e635f74696d656f7574944d08078c0e73796e635f61727469666163747394898c1c73796e635f6172746966616374735f6f6e5f636865636b706f696e7494888c0a75706c6f61645f646972948c0a44455052454341544544948c0673796e6365729468188c1273796e635f6f6e5f636865636b706f696e7494681875628c1273746f726167655f66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c0f73746f726167655f66735f70617468948c452f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f7465737494681968008c115f46696c6573797374656d53796e6365729493942981947d9428681b682268134d2c0168144d08078c116c6173745f73796e635f75705f74696d65944741d99eb8eb5008998c136c6173745f73796e635f646f776e5f74696d659447fff00000000000008c0d5f73796e635f70726f63657373944e8c0c5f63757272656e745f636d649468008c125f75706c6f61645f746f5f66735f706174689493947d94288c0a6c6f63616c5f70617468948c3b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c74732f746573745f6578706572696d656e74948c0266739468228c0766735f70617468948c552f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e74948c076578636c756465945d948c0e2a2f636865636b706f696e745f2a9461758694756275622e\"\n },\n \"config\": {\n \"encoder_num_layer\": 1,\n \"decoder_rt_num_layer\": 1,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 1024,\n \"encoder_ff\": 1024,\n \"decoder_rt_ff\": 1024,\n \"decoder_int_ff\": 512,\n \"n_head\": 4,\n \"drop_rate\": 0.9358999268282086,\n \"lr\": 0.00020838214141938324,\n \"batch_size\": 2048\n },\n \"_Trial__unresolved_config\": {\n \"encoder_num_layer\": 1,\n \"decoder_rt_num_layer\": 1,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 1024,\n \"encoder_ff\": 1024,\n \"decoder_rt_ff\": 1024,\n \"decoder_int_ff\": 512,\n \"n_head\": 4,\n \"drop_rate\": 0.9358999268282086,\n \"lr\": 0.00020838214141938324,\n \"batch_size\": 2048\n },\n \"evaluated_params\": {\n \"batch_size\": 2048,\n \"decoder_int_ff\": 512,\n \"decoder_int_num_layer\": 1,\n \"decoder_rt_ff\": 1024,\n \"decoder_rt_num_layer\": 1,\n \"drop_rate\": 0.9358999268282086,\n \"embedding_dim\": 1024,\n \"encoder_ff\": 1024,\n \"encoder_num_layer\": 1,\n \"lr\": 0.00020838214141938324,\n \"n_head\": 4\n },\n \"experiment_tag\": \"12_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop_rate=0.9359,embedding_dim=1024,encoder_ff=1024,encoder_num_layer=1,lr=0.0002,n_head=4\",\n \"stopping_criterion\": {},\n \"_setup_default_resource\": true,\n \"_default_placement_group_factory\": \"80054e2e\",\n \"placement_group_factory\": \"800595ba000000000000008c237261792e74756e652e657865637574696f6e2e706c6163656d656e745f67726f757073948c15506c6163656d656e7447726f7570466163746f72799493942981947d94288c085f62756e646c6573945d947d94288c03435055944740540000000000008c034750559447401000000000000075618c155f686561645f62756e646c655f69735f656d70747994898c095f7374726174656779948c045041434b948c055f6172677394298c075f6b7761726773947d9475622e\",\n \"log_to_file\": [\n null,\n null\n ],\n \"max_failures\": 0,\n \"_default_result_or_future\": null,\n \"export_formats\": [],\n \"status\": \"PAUSED\",\n \"relative_logdir\": \"train_model_ad12346e_12_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_17-36-47\",\n \"trial_name_creator\": null,\n \"trial_dirname_creator\": null,\n \"custom_trial_name\": null,\n \"custom_dirname\": null,\n \"restore_path\": null,\n \"_restore_checkpoint_result\": null,\n \"_state_json\": null,\n \"results\": \"80054e2e\",\n \"extra_arg\": \"80054e2e\",\n \"_resources\": \"80054e2e\"\n}",
+ "{\n \"start_time\": 1719329892.630865,\n \"num_failures\": 0,\n \"num_failures_after_restore\": 0,\n \"error_filename\": null,\n \"pickled_error_filename\": null,\n \"last_result\": {\n \"loss\": 2126.6908641424707,\n \"timestamp\": 1719330099,\n \"checkpoint_dir_name\": \"checkpoint_000000\",\n \"should_checkpoint\": true,\n \"done\": false,\n \"training_iteration\": 1,\n \"trial_id\": \"ad12346e\",\n \"date\": \"2024-06-25_17-41-39\",\n \"time_this_iter_s\": 206.83131527900696,\n \"time_total_s\": 206.83131527900696,\n \"pid\": 2195253,\n \"hostname\": \"r8i6n2\",\n \"node_ip\": \"10.159.28.60\",\n \"config\": {\n \"encoder_num_layer\": 1,\n \"decoder_rt_num_layer\": 1,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 1024,\n \"encoder_ff\": 1024,\n \"decoder_rt_ff\": 1024,\n \"decoder_int_ff\": 512,\n \"n_head\": 4,\n \"drop_rate\": 0.9358999268282086,\n \"lr\": 0.00020838214141938324,\n \"batch_size\": 2048\n },\n \"time_since_restore\": 206.83131527900696,\n \"iterations_since_restore\": 1,\n \"experiment_tag\": \"12_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop_rate=0.9359,embedding_dim=1024,encoder_ff=1024,encoder_num_layer=1,lr=0.0002,n_head=4\"\n },\n \"last_result_time\": 1719330099.4666245,\n \"metric_analysis\": {\n \"loss\": {\n \"max\": 2126.6908641424707,\n \"min\": 2126.6908641424707,\n \"avg\": 2126.6908641424707,\n \"last\": 2126.6908641424707,\n \"last-5-avg\": 2126.6908641424707,\n \"last-10-avg\": 2126.6908641424707\n },\n \"should_checkpoint\": {\n \"max\": true,\n \"min\": true,\n \"avg\": true,\n \"last\": true,\n \"last-5-avg\": true,\n \"last-10-avg\": true\n },\n \"done\": {\n \"max\": false,\n \"min\": false,\n \"avg\": false,\n \"last\": false,\n \"last-5-avg\": false,\n \"last-10-avg\": false\n },\n \"training_iteration\": {\n \"max\": 1,\n \"min\": 1,\n \"avg\": 1,\n \"last\": 1,\n \"last-5-avg\": 1,\n \"last-10-avg\": 1\n },\n \"time_this_iter_s\": {\n \"max\": 206.83131527900696,\n \"min\": 206.83131527900696,\n \"avg\": 206.83131527900696,\n \"last\": 206.83131527900696,\n \"last-5-avg\": 206.83131527900696,\n \"last-10-avg\": 206.83131527900696\n },\n \"time_total_s\": {\n \"max\": 206.83131527900696,\n \"min\": 206.83131527900696,\n \"avg\": 206.83131527900696,\n \"last\": 206.83131527900696,\n \"last-5-avg\": 206.83131527900696,\n \"last-10-avg\": 206.83131527900696\n },\n \"time_since_restore\": {\n \"max\": 206.83131527900696,\n \"min\": 206.83131527900696,\n \"avg\": 206.83131527900696,\n \"last\": 206.83131527900696,\n \"last-5-avg\": 206.83131527900696,\n \"last-10-avg\": 206.83131527900696\n },\n \"iterations_since_restore\": {\n \"max\": 1,\n \"min\": 1,\n \"avg\": 1,\n \"last\": 1,\n \"last-5-avg\": 1,\n \"last-10-avg\": 1\n }\n },\n \"_n_steps\": [\n 5,\n 10\n ],\n \"metric_n_steps\": {\n \"loss\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059589000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452948c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b007494624308c8e3f1b8619da0409486945294612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059589000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452948c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b007494624308c8e3f1b8619da0409486945294612e\"\n }\n },\n \"should_checkpoint\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059522000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b058694529488612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059522000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a8694529488612e\"\n }\n },\n \"done\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059522000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b058694529489612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059522000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a8694529489612e\"\n }\n },\n \"training_iteration\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059523000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452944b01612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059523000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452944b01612e\"\n }\n },\n \"time_this_iter_s\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294474069da9a22800000612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294474069da9a22800000612e\"\n }\n },\n \"time_total_s\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294474069da9a22800000612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294474069da9a22800000612e\"\n }\n },\n \"time_since_restore\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294474069da9a22800000612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294474069da9a22800000612e\"\n }\n },\n \"iterations_since_restore\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059523000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452944b01612e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059523000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452944b01612e\"\n }\n }\n },\n \"checkpoint_manager\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059519060000000000008c267261792e747261696e2e5f696e7465726e616c2e636865636b706f696e745f6d616e61676572948c125f436865636b706f696e744d616e616765729493942981947d94288c125f636865636b706f696e745f636f6e666967948c097261792e747261696e948c10436865636b706f696e74436f6e6669679493942981947d94288c0b6e756d5f746f5f6b656570944e8c1a636865636b706f696e745f73636f72655f617474726962757465944e8c16636865636b706f696e745f73636f72655f6f72646572948c036d6178948c14636865636b706f696e745f6672657175656e6379944b008c11636865636b706f696e745f61745f656e6494898c1a5f636865636b706f696e745f6b6565705f616c6c5f72616e6b73948c0a44455052454341544544948c1f5f636865636b706f696e745f75706c6f61645f66726f6d5f776f726b65727394681275628c135f636865636b706f696e745f726573756c7473945d948c1b7261792e747261696e2e5f696e7465726e616c2e73657373696f6e948c0f5f547261696e696e67526573756c749493942981947d94288c0a636865636b706f696e749468068c0a436865636b706f696e749493942981947d94288c0470617468948cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f61643132333436655f31325f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d313032342c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f5f323032342d30362d32355f31372d33362d34372f636865636b706f696e745f303030303030948c0a66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c055f75756964948c0475756964948c04555549449493942981947d948c03696e74948a11d6a8427ced244b852d454a69b0fff7f400736275628c076d657472696373947d94288c046c6f7373948c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b007494624308c8e3f1b8619da04094869452948c0974696d657374616d70944a33e57a668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303030948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b018c08747269616c5f6964948c086164313233343665948c0464617465948c13323032342d30362d32355f31372d34312d3339948c1074696d655f746869735f697465725f7394474069da9a228000008c0c74696d655f746f74616c5f7394474069da9a228000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b018c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944d00048c0a656e636f6465725f6666944d00048c0d6465636f6465725f72745f6666944d00048c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b048c0964726f705f7261746594473fedf2e46741ca558c026c7294473f2b5024f705e0038c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f726594474069da9a228000008c18697465726174696f6e735f73696e63655f726573746f7265944b01757562618c195f6c61746573745f636865636b706f696e745f726573756c7494681975622e\"\n }\n}"
+ ],
+ [
+ "{\n \"stub\": false,\n \"trainable_name\": \"train_model\",\n \"trial_id\": \"25a9f43a\",\n \"storage\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059572040000000000008c1b7261792e747261696e2e5f696e7465726e616c2e73746f72616765948c0e53746f72616765436f6e746578749493942981947d94288c12637573746f6d5f66735f70726f766964656494898c1273746f726167655f6c6f63616c5f70617468948c2b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c7473948c136578706572696d656e745f6469725f6e616d65948c0f746573745f6578706572696d656e74948c0e747269616c5f6469725f6e616d65948c96747261696e5f6d6f64656c5f32356139663433615f31335f62617463685f73697a653d313032342c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d313032342c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f5f323032342d30362d32355f31372d33382d3132948c1863757272656e745f636865636b706f696e745f696e646578944b038c0b73796e635f636f6e666967948c097261792e747261696e948c0a53796e63436f6e6669679493942981947d94288c0b73796e635f706572696f64944d2c018c0c73796e635f74696d656f7574944d08078c0e73796e635f61727469666163747394898c1c73796e635f6172746966616374735f6f6e5f636865636b706f696e7494888c0a75706c6f61645f646972948c0a44455052454341544544948c0673796e6365729468188c1273796e635f6f6e5f636865636b706f696e7494681875628c1273746f726167655f66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c0f73746f726167655f66735f70617468948c452f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f7465737494681968008c115f46696c6573797374656d53796e6365729493942981947d9428681b682268134d2c0168144d08078c116c6173745f73796e635f75705f74696d65944741d99eb8eb5008998c136c6173745f73796e635f646f776e5f74696d659447fff00000000000008c0d5f73796e635f70726f63657373944e8c0c5f63757272656e745f636d649468008c125f75706c6f61645f746f5f66735f706174689493947d94288c0a6c6f63616c5f70617468948c3b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c74732f746573745f6578706572696d656e74948c0266739468228c0766735f70617468948c552f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e74948c076578636c756465945d948c0e2a2f636865636b706f696e745f2a9461758694756275622e\"\n },\n \"config\": {\n \"encoder_num_layer\": 4,\n \"decoder_rt_num_layer\": 1,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 64,\n \"encoder_ff\": 512,\n \"decoder_rt_ff\": 1024,\n \"decoder_int_ff\": 512,\n \"n_head\": 16,\n \"drop_rate\": 0.7474706763430264,\n \"lr\": 0.00045083297498032556,\n \"batch_size\": 1024\n },\n \"_Trial__unresolved_config\": {\n \"encoder_num_layer\": 4,\n \"decoder_rt_num_layer\": 1,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 64,\n \"encoder_ff\": 512,\n \"decoder_rt_ff\": 1024,\n \"decoder_int_ff\": 512,\n \"n_head\": 16,\n \"drop_rate\": 0.7474706763430264,\n \"lr\": 0.00045083297498032556,\n \"batch_size\": 1024\n },\n \"evaluated_params\": {\n \"batch_size\": 1024,\n \"decoder_int_ff\": 512,\n \"decoder_int_num_layer\": 1,\n \"decoder_rt_ff\": 1024,\n \"decoder_rt_num_layer\": 1,\n \"drop_rate\": 0.7474706763430264,\n \"embedding_dim\": 64,\n \"encoder_ff\": 512,\n \"encoder_num_layer\": 4,\n \"lr\": 0.00045083297498032556,\n \"n_head\": 16\n },\n \"experiment_tag\": \"13_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop_rate=0.7475,embedding_dim=64,encoder_ff=512,encoder_num_layer=4,lr=0.0005,n_head=16\",\n \"stopping_criterion\": {},\n \"_setup_default_resource\": true,\n \"_default_placement_group_factory\": \"80054e2e\",\n \"placement_group_factory\": \"800595ba000000000000008c237261792e74756e652e657865637574696f6e2e706c6163656d656e745f67726f757073948c15506c6163656d656e7447726f7570466163746f72799493942981947d94288c085f62756e646c6573945d947d94288c03435055944740540000000000008c034750559447401000000000000075618c155f686561645f62756e646c655f69735f656d70747994898c095f7374726174656779948c045041434b948c055f6172677394298c075f6b7761726773947d9475622e\",\n \"log_to_file\": [\n null,\n null\n ],\n \"max_failures\": 0,\n \"_default_result_or_future\": null,\n \"export_formats\": [],\n \"status\": \"PAUSED\",\n \"relative_logdir\": \"train_model_25a9f43a_13_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_17-38-12\",\n \"trial_name_creator\": null,\n \"trial_dirname_creator\": null,\n \"custom_trial_name\": null,\n \"custom_dirname\": null,\n \"restore_path\": null,\n \"_restore_checkpoint_result\": null,\n \"_state_json\": null,\n \"results\": \"80054e2e\",\n \"extra_arg\": \"80054e2e\",\n \"_resources\": \"80054e2e\"\n}",
+ "{\n \"start_time\": 1719330099.6124241,\n \"num_failures\": 0,\n \"num_failures_after_restore\": 0,\n \"error_filename\": null,\n \"pickled_error_filename\": null,\n \"last_result\": {\n \"loss\": 381.2108863169753,\n \"timestamp\": 1719331634,\n \"checkpoint_dir_name\": \"checkpoint_000003\",\n \"should_checkpoint\": true,\n \"done\": false,\n \"training_iteration\": 4,\n \"trial_id\": \"25a9f43a\",\n \"date\": \"2024-06-25_18-07-14\",\n \"time_this_iter_s\": 80.07035231590271,\n \"time_total_s\": 332.0298240184784,\n \"pid\": 2195253,\n \"hostname\": \"r8i6n2\",\n \"node_ip\": \"10.159.28.60\",\n \"config\": {\n \"encoder_num_layer\": 4,\n \"decoder_rt_num_layer\": 1,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 64,\n \"encoder_ff\": 512,\n \"decoder_rt_ff\": 1024,\n \"decoder_int_ff\": 512,\n \"n_head\": 16,\n \"drop_rate\": 0.7474706763430264,\n \"lr\": 0.00045083297498032556,\n \"batch_size\": 1024\n },\n \"time_since_restore\": 174.64248299598694,\n \"iterations_since_restore\": 2,\n \"experiment_tag\": \"13_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop_rate=0.7475,embedding_dim=64,encoder_ff=512,encoder_num_layer=4,lr=0.0005,n_head=16\"\n },\n \"last_result_time\": 1719331634.3944979,\n \"metric_analysis\": {\n \"loss\": {\n \"max\": 1284.8956039308564,\n \"min\": 381.2108863169753,\n \"avg\": 652.1886275584304,\n \"last\": 381.2108863169753,\n \"last-5-avg\": 652.1886275584304,\n \"last-10-avg\": 652.1886275584304\n },\n \"should_checkpoint\": {\n \"max\": true,\n \"min\": true,\n \"avg\": 1.0,\n \"last\": true,\n \"last-5-avg\": 1.0,\n \"last-10-avg\": 1.0\n },\n \"done\": {\n \"max\": false,\n \"min\": false,\n \"avg\": 0.0,\n \"last\": false,\n \"last-5-avg\": 0.0,\n \"last-10-avg\": 0.0\n },\n \"training_iteration\": {\n \"max\": 4,\n \"min\": 1,\n \"avg\": 2.5,\n \"last\": 4,\n \"last-5-avg\": 2.5,\n \"last-10-avg\": 2.5\n },\n \"time_this_iter_s\": {\n \"max\": 94.57213068008423,\n \"min\": 77.99471831321716,\n \"avg\": 83.0074560046196,\n \"last\": 80.07035231590271,\n \"last-5-avg\": 83.0074560046196,\n \"last-10-avg\": 83.0074560046196\n },\n \"time_total_s\": {\n \"max\": 332.0298240184784,\n \"min\": 79.39262270927429,\n \"avg\": 205.19231486320496,\n \"last\": 332.0298240184784,\n \"last-5-avg\": 205.19231486320496,\n \"last-10-avg\": 205.19231486320496\n },\n \"time_since_restore\": {\n \"max\": 174.64248299598694,\n \"min\": 77.99471831321716,\n \"avg\": 106.65048867464066,\n \"last\": 174.64248299598694,\n \"last-5-avg\": 106.65048867464066,\n \"last-10-avg\": 106.65048867464066\n },\n \"iterations_since_restore\": {\n \"max\": 2,\n \"min\": 1,\n \"avg\": 1.25,\n \"last\": 2,\n \"last-5-avg\": 1.25,\n \"last-10-avg\": 1.25\n }\n },\n \"_n_steps\": [\n 5,\n 10\n ],\n \"metric_n_steps\": {\n \"loss\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"800595c3000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294288c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b007494624308ca6432199513944094869452946807680d43087fbfdf1fe42d804094869452946807680d4308d4e9740a968e7a4094869452946807680d430853a954ca5fd377409486945294652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"800595c3000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294288c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b007494624308ca6432199513944094869452946807680d43087fbfdf1fe42d804094869452946807680d4308d4e9740a968e7a4094869452946807680d430853a954ca5fd377409486945294652e\"\n }\n },\n \"should_checkpoint\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059526000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452942888888888652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059526000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452942888888888652e\"\n }\n },\n \"done\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059526000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452942889898989652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059526000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452942889898989652e\"\n }\n },\n \"training_iteration\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294284b014b024b034b04652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294284b014b024b034b04652e\"\n }\n },\n \"time_this_iter_s\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059546000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b058694529428474053d920bb0000004740537fa977000000474057a49dca0000004740540480a7000000652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059546000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a8694529428474053d920bb0000004740537fa977000000474057a49dca0000004740540480a7000000652e\"\n }\n },\n \"time_total_s\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059546000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b058694529428474053d920bb000000474063ac651900000047406f7eb3fe000000474074c07a28c00000652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059546000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a8694529428474053d920bb000000474063ac651900000047406f7eb3fe000000474074c07a28c00000652e\"\n }\n },\n \"time_since_restore\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059546000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b058694529428474053d920bb0000004740537fa977000000474057a49dca000000474065d48f38800000652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059546000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a8694529428474053d920bb0000004740537fa977000000474057a49dca000000474065d48f38800000652e\"\n }\n },\n \"iterations_since_restore\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294284b014b014b014b02652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294284b014b014b014b02652e\"\n }\n }\n },\n \"checkpoint_manager\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"800595be100000000000008c267261792e747261696e2e5f696e7465726e616c2e636865636b706f696e745f6d616e61676572948c125f436865636b706f696e744d616e616765729493942981947d94288c125f636865636b706f696e745f636f6e666967948c097261792e747261696e948c10436865636b706f696e74436f6e6669679493942981947d94288c0b6e756d5f746f5f6b656570944e8c1a636865636b706f696e745f73636f72655f617474726962757465944e8c16636865636b706f696e745f73636f72655f6f72646572948c036d6178948c14636865636b706f696e745f6672657175656e6379944b008c11636865636b706f696e745f61745f656e6494898c1a5f636865636b706f696e745f6b6565705f616c6c5f72616e6b73948c0a44455052454341544544948c1f5f636865636b706f696e745f75706c6f61645f66726f6d5f776f726b65727394681275628c135f636865636b706f696e745f726573756c7473945d94288c1b7261792e747261696e2e5f696e7465726e616c2e73657373696f6e948c0f5f547261696e696e67526573756c749493942981947d94288c0a636865636b706f696e749468068c0a436865636b706f696e749493942981947d94288c0470617468948cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f32356139663433615f31335f62617463685f73697a653d313032342c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d313032342c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f5f323032342d30362d32355f31372d33382d31322f636865636b706f696e745f303030303030948c0a66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c055f75756964948c0475756964948c04555549449493942981947d948c03696e74948a10a129e095eddc25b4d044f32a0b7ba13f736275628c076d657472696373947d94288c046c6f7373948c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b007494624308ca6432199513944094869452948c0974696d657374616d70944a82e57a668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303030948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b018c08747269616c5f6964948c083235613966343361948c0464617465948c13323032342d30362d32355f31372d34322d3539948c1074696d655f746869735f697465725f7394474053d920bb0000008c0c74696d655f746f74616c5f7394474053d920bb0000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b048c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b408c0a656e636f6465725f6666944d00028c0d6465636f6465725f72745f6666944d00048c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b108c0964726f705f7261746594473fe7eb479fb397f58c026c7294473f3d8bb8e22cfb9f8c0a62617463685f73697a65944d0004758c1274696d655f73696e63655f726573746f726594474053d920bb0000008c18697465726174696f6e735f73696e63655f726573746f7265944b0175756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f32356139663433615f31335f62617463685f73697a653d313032342c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d313032342c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f5f323032342d30362d32355f31372d33382d31322f636865636b706f696e745f30303030303194682268257d946827897385945294682a682d2981947d9468308a11e92ab01417c513b2a345f8709b6dc5a0007362756268317d94288c046c6f7373946836683c43087fbfdf1fe42d804094869452948c0974696d657374616d70944a83e87a668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303031948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b028c08747269616c5f6964948c083235613966343361948c0464617465948c13323032342d30362d32355f31372d35352d3437948c1074696d655f746869735f697465725f73944740537fa9770000008c0c74696d655f746f74616c5f7394474063ac65190000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b048c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b408c0a656e636f6465725f6666944d00028c0d6465636f6465725f72745f6666944d00048c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b108c0964726f705f7261746594473fe7eb479fb397f58c026c7294473f3d8bb8e22cfb9f8c0a62617463685f73697a65944d0004758c1274696d655f73696e63655f726573746f7265944740537fa9770000008c18697465726174696f6e735f73696e63655f726573746f7265944b0175756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f32356139663433615f31335f62617463685f73697a653d313032342c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d313032342c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f5f323032342d30362d32355f31372d33382d31322f636865636b706f696e745f30303030303294682268257d946827897385945294682a682d2981947d9468308a1092c9f175bddcd68a0848129376eec62e7362756268317d94288c046c6f7373946836683c4308d4e9740a968e7a4094869452948c0974696d657374616d70944ae2ea7a668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303032948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b038c08747269616c5f6964948c083235613966343361948c0464617465948c13323032342d30362d32355f31382d30352d3534948c1074696d655f746869735f697465725f7394474057a49dca0000008c0c74696d655f746f74616c5f739447406f7eb3fe0000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b048c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b408c0a656e636f6465725f6666944d00028c0d6465636f6465725f72745f6666944d00048c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b108c0964726f705f7261746594473fe7eb479fb397f58c026c7294473f3d8bb8e22cfb9f8c0a62617463685f73697a65944d0004758c1274696d655f73696e63655f726573746f726594474057a49dca0000008c18697465726174696f6e735f73696e63655f726573746f7265944b0175756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f32356139663433615f31335f62617463685f73697a653d313032342c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d313032342c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f5f323032342d30362d32355f31372d33382d31322f636865636b706f696e745f30303030303394682268257d946827897385945294682a682d2981947d9468308a11fbd0d64e460991bc9d4e4220c9ebd9cb007362756268317d94288c046c6f7373946836683c430853a954ca5fd3774094869452948c0974696d657374616d70944a32eb7a668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303033948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b048c08747269616c5f6964948c083235613966343361948c0464617465948c13323032342d30362d32355f31382d30372d3134948c1074696d655f746869735f697465725f73944740540480a70000008c0c74696d655f746f74616c5f7394474074c07a28c000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b048c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b408c0a656e636f6465725f6666944d00028c0d6465636f6465725f72745f6666944d00048c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b108c0964726f705f7261746594473fe7eb479fb397f58c026c7294473f3d8bb8e22cfb9f8c0a62617463685f73697a65944d0004758c1274696d655f73696e63655f726573746f726594474065d48f388000008c18697465726174696f6e735f73696e63655f726573746f7265944b02757562658c195f6c61746573745f636865636b706f696e745f726573756c749468c075622e\"\n }\n}"
+ ],
+ [
+ "{\n \"stub\": false,\n \"trainable_name\": \"train_model\",\n \"trial_id\": \"110f0a27\",\n \"storage\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059572040000000000008c1b7261792e747261696e2e5f696e7465726e616c2e73746f72616765948c0e53746f72616765436f6e746578749493942981947d94288c12637573746f6d5f66735f70726f766964656494898c1273746f726167655f6c6f63616c5f70617468948c2b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c7473948c136578706572696d656e745f6469725f6e616d65948c0f746573745f6578706572696d656e74948c0e747269616c5f6469725f6e616d65948c96747261696e5f6d6f64656c5f31313066306132375f31345f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d313032342c6465636f6465725f72745f6e756d5f6c617965723d322c64726f705f5f323032342d30362d32355f31372d34312d3339948c1863757272656e745f636865636b706f696e745f696e646578944b018c0b73796e635f636f6e666967948c097261792e747261696e948c0a53796e63436f6e6669679493942981947d94288c0b73796e635f706572696f64944d2c018c0c73796e635f74696d656f7574944d08078c0e73796e635f61727469666163747394898c1c73796e635f6172746966616374735f6f6e5f636865636b706f696e7494888c0a75706c6f61645f646972948c0a44455052454341544544948c0673796e6365729468188c1273796e635f6f6e5f636865636b706f696e7494681875628c1273746f726167655f66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c0f73746f726167655f66735f70617468948c452f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f7465737494681968008c115f46696c6573797374656d53796e6365729493942981947d9428681b682268134d2c0168144d08078c116c6173745f73796e635f75705f74696d65944741d99eb936cf4b1e8c136c6173745f73796e635f646f776e5f74696d659447fff00000000000008c0d5f73796e635f70726f63657373944e8c0c5f63757272656e745f636d649468008c125f75706c6f61645f746f5f66735f706174689493947d94288c0a6c6f63616c5f70617468948c3b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c74732f746573745f6578706572696d656e74948c0266739468228c0766735f70617468948c552f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e74948c076578636c756465945d948c0e2a2f636865636b706f696e745f2a9461758694756275622e\"\n },\n \"config\": {\n \"encoder_num_layer\": 2,\n \"decoder_rt_num_layer\": 2,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 256,\n \"encoder_ff\": 2048,\n \"decoder_rt_ff\": 1024,\n \"decoder_int_ff\": 512,\n \"n_head\": 16,\n \"drop_rate\": 0.025073968761468723,\n \"lr\": 0.0010259835678295768,\n \"batch_size\": 2048\n },\n \"_Trial__unresolved_config\": {\n \"encoder_num_layer\": 2,\n \"decoder_rt_num_layer\": 2,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 256,\n \"encoder_ff\": 2048,\n \"decoder_rt_ff\": 1024,\n \"decoder_int_ff\": 512,\n \"n_head\": 16,\n \"drop_rate\": 0.025073968761468723,\n \"lr\": 0.0010259835678295768,\n \"batch_size\": 2048\n },\n \"evaluated_params\": {\n \"batch_size\": 2048,\n \"decoder_int_ff\": 512,\n \"decoder_int_num_layer\": 1,\n \"decoder_rt_ff\": 1024,\n \"decoder_rt_num_layer\": 2,\n \"drop_rate\": 0.025073968761468723,\n \"embedding_dim\": 256,\n \"encoder_ff\": 2048,\n \"encoder_num_layer\": 2,\n \"lr\": 0.0010259835678295768,\n \"n_head\": 16\n },\n \"experiment_tag\": \"14_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop_rate=0.0251,embedding_dim=256,encoder_ff=2048,encoder_num_layer=2,lr=0.0010,n_head=16\",\n \"stopping_criterion\": {},\n \"_setup_default_resource\": true,\n \"_default_placement_group_factory\": \"80054e2e\",\n \"placement_group_factory\": \"800595ba000000000000008c237261792e74756e652e657865637574696f6e2e706c6163656d656e745f67726f757073948c15506c6163656d656e7447726f7570466163746f72799493942981947d94288c085f62756e646c6573945d947d94288c03435055944740540000000000008c034750559447401000000000000075618c155f686561645f62756e646c655f69735f656d70747994898c095f7374726174656779948c045041434b948c055f6172677394298c075f6b7761726773947d9475622e\",\n \"log_to_file\": [\n null,\n null\n ],\n \"max_failures\": 0,\n \"_default_result_or_future\": null,\n \"export_formats\": [],\n \"status\": \"PAUSED\",\n \"relative_logdir\": \"train_model_110f0a27_14_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop__2024-06-25_17-41-39\",\n \"trial_name_creator\": null,\n \"trial_dirname_creator\": null,\n \"custom_trial_name\": null,\n \"custom_dirname\": null,\n \"restore_path\": null,\n \"_restore_checkpoint_result\": null,\n \"_state_json\": null,\n \"results\": \"80054e2e\",\n \"extra_arg\": \"80054e2e\",\n \"_resources\": \"80054e2e\"\n}",
+ "{\n \"start_time\": 1719330179.1827745,\n \"num_failures\": 0,\n \"num_failures_after_restore\": 0,\n \"error_filename\": null,\n \"pickled_error_filename\": null,\n \"last_result\": {\n \"loss\": 1952.48307740219,\n \"timestamp\": 1719331057,\n \"checkpoint_dir_name\": \"checkpoint_000001\",\n \"should_checkpoint\": true,\n \"done\": false,\n \"training_iteration\": 2,\n \"trial_id\": \"110f0a27\",\n \"date\": \"2024-06-25_17-57-37\",\n \"time_this_iter_s\": 110.16742134094238,\n \"time_total_s\": 220.83409094810486,\n \"pid\": 2195253,\n \"hostname\": \"r8i6n2\",\n \"node_ip\": \"10.159.28.60\",\n \"config\": {\n \"encoder_num_layer\": 2,\n \"decoder_rt_num_layer\": 2,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 256,\n \"encoder_ff\": 2048,\n \"decoder_rt_ff\": 1024,\n \"decoder_int_ff\": 512,\n \"n_head\": 16,\n \"drop_rate\": 0.025073968761468723,\n \"lr\": 0.0010259835678295768,\n \"batch_size\": 2048\n },\n \"time_since_restore\": 110.16742134094238,\n \"iterations_since_restore\": 1,\n \"experiment_tag\": \"14_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop_rate=0.0251,embedding_dim=256,encoder_ff=2048,encoder_num_layer=2,lr=0.0010,n_head=16\"\n },\n \"last_result_time\": 1719331057.906955,\n \"metric_analysis\": {\n \"loss\": {\n \"max\": 1956.1862581508367,\n \"min\": 1952.48307740219,\n \"avg\": 1954.3346677765135,\n \"last\": 1952.48307740219,\n \"last-5-avg\": 1954.3346677765135,\n \"last-10-avg\": 1954.3346677765135\n },\n \"should_checkpoint\": {\n \"max\": true,\n \"min\": true,\n \"avg\": 1.0,\n \"last\": true,\n \"last-5-avg\": 1.0,\n \"last-10-avg\": 1.0\n },\n \"done\": {\n \"max\": false,\n \"min\": false,\n \"avg\": 0.0,\n \"last\": false,\n \"last-5-avg\": 0.0,\n \"last-10-avg\": 0.0\n },\n \"training_iteration\": {\n \"max\": 2,\n \"min\": 1,\n \"avg\": 1.5,\n \"last\": 2,\n \"last-5-avg\": 1.5,\n \"last-10-avg\": 1.5\n },\n \"time_this_iter_s\": {\n \"max\": 110.66666960716248,\n \"min\": 110.16742134094238,\n \"avg\": 110.41704547405243,\n \"last\": 110.16742134094238,\n \"last-5-avg\": 110.41704547405243,\n \"last-10-avg\": 110.41704547405243\n },\n \"time_total_s\": {\n \"max\": 220.83409094810486,\n \"min\": 110.66666960716248,\n \"avg\": 165.75038027763367,\n \"last\": 220.83409094810486,\n \"last-5-avg\": 165.75038027763367,\n \"last-10-avg\": 165.75038027763367\n },\n \"time_since_restore\": {\n \"max\": 110.66666960716248,\n \"min\": 110.16742134094238,\n \"avg\": 110.41704547405243,\n \"last\": 110.16742134094238,\n \"last-5-avg\": 110.41704547405243,\n \"last-10-avg\": 110.41704547405243\n },\n \"iterations_since_restore\": {\n \"max\": 1,\n \"min\": 1,\n \"avg\": 1.0,\n \"last\": 1,\n \"last-5-avg\": 1.0,\n \"last-10-avg\": 1.0\n }\n },\n \"_n_steps\": [\n 5,\n 10\n ],\n \"metric_n_steps\": {\n \"loss\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005959d000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294288c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b007494624308d4e974babe909e4094869452946807680d43085fafd7abee819e409486945294652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005959d000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294288c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b007494624308d4e974babe909e4094869452946807680d43085fafd7abee819e409486945294652e\"\n }\n },\n \"should_checkpoint\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059524000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294288888652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059524000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294288888652e\"\n }\n },\n \"done\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059524000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294288989652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059524000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294288989652e\"\n }\n },\n \"training_iteration\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059526000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294284b014b02652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059526000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294284b014b02652e\"\n }\n },\n \"time_this_iter_s\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059534000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452942847405baaaab700000047405b8ab708000000652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059534000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452942847405baaaab700000047405b8ab708000000652e\"\n }\n },\n \"time_total_s\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059534000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452942847405baaaab700000047406b9ab0df800000652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059534000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452942847405baaaab700000047406b9ab0df800000652e\"\n }\n },\n \"time_since_restore\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059534000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452942847405baaaab700000047405b8ab708000000652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059534000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452942847405baaaab700000047405b8ab708000000652e\"\n }\n },\n \"iterations_since_restore\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059526000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294284b014b01652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059526000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294284b014b01652e\"\n }\n }\n },\n \"checkpoint_manager\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"800595a8090000000000008c267261792e747261696e2e5f696e7465726e616c2e636865636b706f696e745f6d616e61676572948c125f436865636b706f696e744d616e616765729493942981947d94288c125f636865636b706f696e745f636f6e666967948c097261792e747261696e948c10436865636b706f696e74436f6e6669679493942981947d94288c0b6e756d5f746f5f6b656570944e8c1a636865636b706f696e745f73636f72655f617474726962757465944e8c16636865636b706f696e745f73636f72655f6f72646572948c036d6178948c14636865636b706f696e745f6672657175656e6379944b008c11636865636b706f696e745f61745f656e6494898c1a5f636865636b706f696e745f6b6565705f616c6c5f72616e6b73948c0a44455052454341544544948c1f5f636865636b706f696e745f75706c6f61645f66726f6d5f776f726b65727394681275628c135f636865636b706f696e745f726573756c7473945d94288c1b7261792e747261696e2e5f696e7465726e616c2e73657373696f6e948c0f5f547261696e696e67526573756c749493942981947d94288c0a636865636b706f696e749468068c0a436865636b706f696e749493942981947d94288c0470617468948cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f31313066306132375f31345f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d313032342c6465636f6465725f72745f6e756d5f6c617965723d322c64726f705f5f323032342d30362d32355f31372d34312d33392f636865636b706f696e745f303030303030948c0a66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c055f75756964948c0475756964948c04555549449493942981947d948c03696e74948a11ca83eb928c757b9c114894f700e3c3b900736275628c076d657472696373947d94288c046c6f7373948c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b007494624308d4e974babe909e4094869452948c0974696d657374616d70944af1e57a668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303030948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b018c08747269616c5f6964948c083131306630613237948c0464617465948c13323032342d30362d32355f31372d34342d3439948c1074696d655f746869735f697465725f739447405baaaab70000008c0c74696d655f746f74616c5f739447405baaaab70000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b028c146465636f6465725f72745f6e756d5f6c61796572944b028c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944d00018c0a656e636f6465725f6666944d00088c0d6465636f6465725f72745f6666944d00048c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b108c0964726f705f7261746594473f99acfd8f3ee8808c026c7294473f50cf4977af16238c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f72659447405baaaab70000008c18697465726174696f6e735f73696e63655f726573746f7265944b0175756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f31313066306132375f31345f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d313032342c6465636f6465725f72745f6e756d5f6c617965723d322c64726f705f5f323032342d30362d32355f31372d34312d33392f636865636b706f696e745f30303030303194682268257d946827897385945294682a682d2981947d9468308a11aca660fb96d059b81742d1fa316ebf82007362756268317d94288c046c6f7373946836683c43085fafd7abee819e4094869452948c0974696d657374616d70944af1e87a668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303031948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b028c08747269616c5f6964948c083131306630613237948c0464617465948c13323032342d30362d32355f31372d35372d3337948c1074696d655f746869735f697465725f739447405b8ab7080000008c0c74696d655f746f74616c5f739447406b9ab0df8000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b028c146465636f6465725f72745f6e756d5f6c61796572944b028c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944d00018c0a656e636f6465725f6666944d00088c0d6465636f6465725f72745f6666944d00048c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b108c0964726f705f7261746594473f99acfd8f3ee8808c026c7294473f50cf4977af16238c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f72659447405b8ab7080000008c18697465726174696f6e735f73696e63655f726573746f7265944b01757562658c195f6c61746573745f636865636b706f696e745f726573756c7494686275622e\"\n }\n}"
+ ],
+ [
+ "{\n \"stub\": false,\n \"trainable_name\": \"train_model\",\n \"trial_id\": \"3510e84b\",\n \"storage\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059572040000000000008c1b7261792e747261696e2e5f696e7465726e616c2e73746f72616765948c0e53746f72616765436f6e746578749493942981947d94288c12637573746f6d5f66735f70726f766964656494898c1273746f726167655f6c6f63616c5f70617468948c2b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c7473948c136578706572696d656e745f6469725f6e616d65948c0f746573745f6578706572696d656e74948c0e747269616c5f6469725f6e616d65948c96747261696e5f6d6f64656c5f33353130653834625f31355f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d323034382c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f5f323032342d30362d32355f31372d34322d3539948c1863757272656e745f636865636b706f696e745f696e646578944b018c0b73796e635f636f6e666967948c097261792e747261696e948c0a53796e63436f6e6669679493942981947d94288c0b73796e635f706572696f64944d2c018c0c73796e635f74696d656f7574944d08078c0e73796e635f61727469666163747394898c1c73796e635f6172746966616374735f6f6e5f636865636b706f696e7494888c0a75706c6f61645f646972948c0a44455052454341544544948c0673796e6365729468188c1273796e635f6f6e5f636865636b706f696e7494681875628c1273746f726167655f66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c0f73746f726167655f66735f70617468948c452f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f7465737494681968008c115f46696c6573797374656d53796e6365729493942981947d9428681b682268134d2c0168144d08078c116c6173745f73796e635f75705f74696d65944741d99eb936cf4b1e8c136c6173745f73796e635f646f776e5f74696d659447fff00000000000008c0d5f73796e635f70726f63657373944e8c0c5f63757272656e745f636d649468008c125f75706c6f61645f746f5f66735f706174689493947d94288c0a6c6f63616c5f70617468948c3b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c74732f746573745f6578706572696d656e74948c0266739468228c0766735f70617468948c552f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e74948c076578636c756465945d948c0e2a2f636865636b706f696e745f2a9461758694756275622e\"\n },\n \"config\": {\n \"encoder_num_layer\": 4,\n \"decoder_rt_num_layer\": 1,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 16,\n \"encoder_ff\": 2048,\n \"decoder_rt_ff\": 2048,\n \"decoder_int_ff\": 512,\n \"n_head\": 4,\n \"drop_rate\": 0.3408951993486181,\n \"lr\": 0.009004430142657121,\n \"batch_size\": 2048\n },\n \"_Trial__unresolved_config\": {\n \"encoder_num_layer\": 4,\n \"decoder_rt_num_layer\": 1,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 16,\n \"encoder_ff\": 2048,\n \"decoder_rt_ff\": 2048,\n \"decoder_int_ff\": 512,\n \"n_head\": 4,\n \"drop_rate\": 0.3408951993486181,\n \"lr\": 0.009004430142657121,\n \"batch_size\": 2048\n },\n \"evaluated_params\": {\n \"batch_size\": 2048,\n \"decoder_int_ff\": 512,\n \"decoder_int_num_layer\": 1,\n \"decoder_rt_ff\": 2048,\n \"decoder_rt_num_layer\": 1,\n \"drop_rate\": 0.3408951993486181,\n \"embedding_dim\": 16,\n \"encoder_ff\": 2048,\n \"encoder_num_layer\": 4,\n \"lr\": 0.009004430142657121,\n \"n_head\": 4\n },\n \"experiment_tag\": \"15_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop_rate=0.3409,embedding_dim=16,encoder_ff=2048,encoder_num_layer=4,lr=0.0090,n_head=4\",\n \"stopping_criterion\": {},\n \"_setup_default_resource\": true,\n \"_default_placement_group_factory\": \"80054e2e\",\n \"placement_group_factory\": \"800595ba000000000000008c237261792e74756e652e657865637574696f6e2e706c6163656d656e745f67726f757073948c15506c6163656d656e7447726f7570466163746f72799493942981947d94288c085f62756e646c6573945d947d94288c03435055944740540000000000008c034750559447401000000000000075618c155f686561645f62756e646c655f69735f656d70747994898c095f7374726174656779948c045041434b948c055f6172677394298c075f6b7761726773947d9475622e\",\n \"log_to_file\": [\n null,\n null\n ],\n \"max_failures\": 0,\n \"_default_result_or_future\": null,\n \"export_formats\": [],\n \"status\": \"TERMINATED\",\n \"relative_logdir\": \"train_model_3510e84b_15_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop__2024-06-25_17-42-59\",\n \"trial_name_creator\": null,\n \"trial_dirname_creator\": null,\n \"custom_trial_name\": null,\n \"custom_dirname\": null,\n \"restore_path\": null,\n \"_restore_checkpoint_result\": null,\n \"_state_json\": null,\n \"results\": \"80054e2e\",\n \"extra_arg\": \"80054e2e\",\n \"_resources\": \"80054e2e\"\n}",
+ "{\n \"start_time\": 1719330290.0214992,\n \"num_failures\": 0,\n \"num_failures_after_restore\": 0,\n \"error_filename\": null,\n \"pickled_error_filename\": null,\n \"last_result\": {\n \"loss\": 1964.9452173428272,\n \"timestamp\": 1719331131,\n \"checkpoint_dir_name\": \"checkpoint_000001\",\n \"should_checkpoint\": true,\n \"done\": true,\n \"training_iteration\": 2,\n \"trial_id\": \"3510e84b\",\n \"date\": \"2024-06-25_17-58-51\",\n \"time_this_iter_s\": 73.41431927680969,\n \"time_total_s\": 146.1404869556427,\n \"pid\": 2195253,\n \"hostname\": \"r8i6n2\",\n \"node_ip\": \"10.159.28.60\",\n \"config\": {\n \"encoder_num_layer\": 4,\n \"decoder_rt_num_layer\": 1,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 16,\n \"encoder_ff\": 2048,\n \"decoder_rt_ff\": 2048,\n \"decoder_int_ff\": 512,\n \"n_head\": 4,\n \"drop_rate\": 0.3408951993486181,\n \"lr\": 0.009004430142657121,\n \"batch_size\": 2048\n },\n \"time_since_restore\": 73.41431927680969,\n \"iterations_since_restore\": 1,\n \"experiment_tag\": \"15_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop_rate=0.3409,embedding_dim=16,encoder_ff=2048,encoder_num_layer=4,lr=0.0090,n_head=4\"\n },\n \"last_result_time\": 1719331131.4748592,\n \"metric_analysis\": {\n \"loss\": {\n \"max\": 1964.9452173428272,\n \"min\": 1964.014622485544,\n \"avg\": 1964.4799199141855,\n \"last\": 1964.9452173428272,\n \"last-5-avg\": 1964.4799199141855,\n \"last-10-avg\": 1964.4799199141855\n },\n \"should_checkpoint\": {\n \"max\": true,\n \"min\": true,\n \"avg\": 1.0,\n \"last\": true,\n \"last-5-avg\": 1.0,\n \"last-10-avg\": 1.0\n },\n \"done\": {\n \"max\": true,\n \"min\": false,\n \"avg\": 0.5,\n \"last\": true,\n \"last-5-avg\": 0.5,\n \"last-10-avg\": 0.5\n },\n \"training_iteration\": {\n \"max\": 2,\n \"min\": 1,\n \"avg\": 1.5,\n \"last\": 2,\n \"last-5-avg\": 1.5,\n \"last-10-avg\": 1.5\n },\n \"time_this_iter_s\": {\n \"max\": 73.41431927680969,\n \"min\": 72.72616767883301,\n \"avg\": 73.07024347782135,\n \"last\": 73.41431927680969,\n \"last-5-avg\": 73.07024347782135,\n \"last-10-avg\": 73.07024347782135\n },\n \"time_total_s\": {\n \"max\": 146.1404869556427,\n \"min\": 72.72616767883301,\n \"avg\": 109.43332731723785,\n \"last\": 146.1404869556427,\n \"last-5-avg\": 109.43332731723785,\n \"last-10-avg\": 109.43332731723785\n },\n \"time_since_restore\": {\n \"max\": 73.41431927680969,\n \"min\": 72.72616767883301,\n \"avg\": 73.07024347782135,\n \"last\": 73.41431927680969,\n \"last-5-avg\": 73.07024347782135,\n \"last-10-avg\": 73.07024347782135\n },\n \"iterations_since_restore\": {\n \"max\": 1,\n \"min\": 1,\n \"avg\": 1.0,\n \"last\": 1,\n \"last-5-avg\": 1.0,\n \"last-10-avg\": 1.0\n }\n },\n \"_n_steps\": [\n 5,\n 10\n ],\n \"metric_n_steps\": {\n \"loss\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005959d000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294288c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b007494624308ca6432f90eb09e4094869452946807680d4308381c0ee7c7b39e409486945294652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005959d000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294288c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b007494624308ca6432f90eb09e4094869452946807680d4308381c0ee7c7b39e409486945294652e\"\n }\n },\n \"should_checkpoint\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059524000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294288888652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059524000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294288888652e\"\n }\n },\n \"done\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059524000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294288988652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059524000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294288988652e\"\n }\n },\n \"training_iteration\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059526000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294284b014b02652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059526000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294284b014b02652e\"\n }\n },\n \"time_this_iter_s\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059534000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294284740522e79880000004740525a8435000000652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059534000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294284740522e79880000004740525a8435000000652e\"\n }\n },\n \"time_total_s\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059534000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294284740522e7988000000474062447ede800000652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059534000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294284740522e7988000000474062447ede800000652e\"\n }\n },\n \"time_since_restore\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059534000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294284740522e79880000004740525a8435000000652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059534000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294284740522e79880000004740525a8435000000652e\"\n }\n },\n \"iterations_since_restore\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059526000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294284b014b01652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059526000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294284b014b01652e\"\n }\n }\n },\n \"checkpoint_manager\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"800595a4090000000000008c267261792e747261696e2e5f696e7465726e616c2e636865636b706f696e745f6d616e61676572948c125f436865636b706f696e744d616e616765729493942981947d94288c125f636865636b706f696e745f636f6e666967948c097261792e747261696e948c10436865636b706f696e74436f6e6669679493942981947d94288c0b6e756d5f746f5f6b656570944e8c1a636865636b706f696e745f73636f72655f617474726962757465944e8c16636865636b706f696e745f73636f72655f6f72646572948c036d6178948c14636865636b706f696e745f6672657175656e6379944b008c11636865636b706f696e745f61745f656e6494898c1a5f636865636b706f696e745f6b6565705f616c6c5f72616e6b73948c0a44455052454341544544948c1f5f636865636b706f696e745f75706c6f61645f66726f6d5f776f726b65727394681275628c135f636865636b706f696e745f726573756c7473945d94288c1b7261792e747261696e2e5f696e7465726e616c2e73657373696f6e948c0f5f547261696e696e67526573756c749493942981947d94288c0a636865636b706f696e749468068c0a436865636b706f696e749493942981947d94288c0470617468948cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f33353130653834625f31355f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d323034382c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f5f323032342d30362d32355f31372d34322d35392f636865636b706f696e745f303030303030948c0a66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c055f75756964948c0475756964948c04555549449493942981947d948c03696e74948a1063fb3088b10820947b4c52d91eb57649736275628c076d657472696373947d94288c046c6f7373948c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b007494624308ca6432f90eb09e4094869452948c0974696d657374616d70944a3ae67a668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303030948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b018c08747269616c5f6964948c083335313065383462948c0464617465948c13323032342d30362d32355f31372d34362d3032948c1074696d655f746869735f697465725f73944740522e79880000008c0c74696d655f746f74616c5f73944740522e79880000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b048c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b108c0a656e636f6465725f6666944d00088c0d6465636f6465725f72745f6666944d00088c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b048c0964726f705f7261746594473fd5d13a192434ae8c026c7294473f8270ea27dac9238c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f7265944740522e79880000008c18697465726174696f6e735f73696e63655f726573746f7265944b0175756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f33353130653834625f31355f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d323034382c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f5f323032342d30362d32355f31372d34322d35392f636865636b706f696e745f30303030303194682268257d946827897385945294682a682d2981947d9468308a1032d8d537008cf58e134bb9d8c30d880d7362756268317d94288c046c6f7373946836683c4308381c0ee7c7b39e4094869452948c0974696d657374616d70944a3be97a668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303031948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b028c08747269616c5f6964948c083335313065383462948c0464617465948c13323032342d30362d32355f31372d35382d3531948c1074696d655f746869735f697465725f73944740525a84350000008c0c74696d655f746f74616c5f7394474062447ede8000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b048c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b108c0a656e636f6465725f6666944d00088c0d6465636f6465725f72745f6666944d00088c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b048c0964726f705f7261746594473fd5d13a192434ae8c026c7294473f8270ea27dac9238c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f7265944740525a84350000008c18697465726174696f6e735f73696e63655f726573746f7265944b01757562658c195f6c61746573745f636865636b706f696e745f726573756c7494686275622e\"\n }\n}"
+ ],
+ [
+ "{\n \"stub\": false,\n \"trainable_name\": \"train_model\",\n \"trial_id\": \"a01c9b2d\",\n \"storage\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059572040000000000008c1b7261792e747261696e2e5f696e7465726e616c2e73746f72616765948c0e53746f72616765436f6e746578749493942981947d94288c12637573746f6d5f66735f70726f766964656494898c1273746f726167655f6c6f63616c5f70617468948c2b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c7473948c136578706572696d656e745f6469725f6e616d65948c0f746573745f6578706572696d656e74948c0e747269616c5f6469725f6e616d65948c96747261696e5f6d6f64656c5f61303163396232645f31365f62617463685f73697a653d313032342c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d313032342c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f5f323032342d30362d32355f31372d34342d3530948c1863757272656e745f636865636b706f696e745f696e646578944b038c0b73796e635f636f6e666967948c097261792e747261696e948c0a53796e63436f6e6669679493942981947d94288c0b73796e635f706572696f64944d2c018c0c73796e635f74696d656f7574944d08078c0e73796e635f61727469666163747394898c1c73796e635f6172746966616374735f6f6e5f636865636b706f696e7494888c0a75706c6f61645f646972948c0a44455052454341544544948c0673796e6365729468188c1273796e635f6f6e5f636865636b706f696e7494681875628c1273746f726167655f66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c0f73746f726167655f66735f70617468948c452f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f7465737494681968008c115f46696c6573797374656d53796e6365729493942981947d9428681b682268134d2c0168144d08078c116c6173745f73796e635f75705f74696d65944741d99eb936cf4b1e8c136c6173745f73796e635f646f776e5f74696d659447fff00000000000008c0d5f73796e635f70726f63657373944e8c0c5f63757272656e745f636d649468008c125f75706c6f61645f746f5f66735f706174689493947d94288c0a6c6f63616c5f70617468948c3b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c74732f746573745f6578706572696d656e74948c0266739468228c0766735f70617468948c552f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e74948c076578636c756465945d948c0e2a2f636865636b706f696e745f2a9461758694756275622e\"\n },\n \"config\": {\n \"encoder_num_layer\": 8,\n \"decoder_rt_num_layer\": 1,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 16,\n \"encoder_ff\": 1024,\n \"decoder_rt_ff\": 1024,\n \"decoder_int_ff\": 512,\n \"n_head\": 4,\n \"drop_rate\": 0.23338023981015343,\n \"lr\": 0.00010269515886648154,\n \"batch_size\": 1024\n },\n \"_Trial__unresolved_config\": {\n \"encoder_num_layer\": 8,\n \"decoder_rt_num_layer\": 1,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 16,\n \"encoder_ff\": 1024,\n \"decoder_rt_ff\": 1024,\n \"decoder_int_ff\": 512,\n \"n_head\": 4,\n \"drop_rate\": 0.23338023981015343,\n \"lr\": 0.00010269515886648154,\n \"batch_size\": 1024\n },\n \"evaluated_params\": {\n \"batch_size\": 1024,\n \"decoder_int_ff\": 512,\n \"decoder_int_num_layer\": 1,\n \"decoder_rt_ff\": 1024,\n \"decoder_rt_num_layer\": 1,\n \"drop_rate\": 0.23338023981015343,\n \"embedding_dim\": 16,\n \"encoder_ff\": 1024,\n \"encoder_num_layer\": 8,\n \"lr\": 0.00010269515886648154,\n \"n_head\": 4\n },\n \"experiment_tag\": \"16_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop_rate=0.2334,embedding_dim=16,encoder_ff=1024,encoder_num_layer=8,lr=0.0001,n_head=4\",\n \"stopping_criterion\": {},\n \"_setup_default_resource\": true,\n \"_default_placement_group_factory\": \"80054e2e\",\n \"placement_group_factory\": \"800595ba000000000000008c237261792e74756e652e657865637574696f6e2e706c6163656d656e745f67726f757073948c15506c6163656d656e7447726f7570466163746f72799493942981947d94288c085f62756e646c6573945d947d94288c03435055944740540000000000008c034750559447401000000000000075618c155f686561645f62756e646c655f69735f656d70747994898c095f7374726174656779948c045041434b948c055f6172677394298c075f6b7761726773947d9475622e\",\n \"log_to_file\": [\n null,\n null\n ],\n \"max_failures\": 0,\n \"_default_result_or_future\": null,\n \"export_formats\": [],\n \"status\": \"TERMINATED\",\n \"relative_logdir\": \"train_model_a01c9b2d_16_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_17-44-50\",\n \"trial_name_creator\": null,\n \"trial_dirname_creator\": null,\n \"custom_trial_name\": null,\n \"custom_dirname\": null,\n \"restore_path\": null,\n \"_restore_checkpoint_result\": null,\n \"_state_json\": null,\n \"results\": \"80054e2e\",\n \"extra_arg\": \"80054e2e\",\n \"_resources\": \"80054e2e\"\n}",
+ "{\n \"start_time\": 1719330362.8999166,\n \"num_failures\": 0,\n \"num_failures_after_restore\": 0,\n \"error_filename\": null,\n \"pickled_error_filename\": null,\n \"last_result\": {\n \"loss\": 290.2177711246521,\n \"timestamp\": 1719331824,\n \"checkpoint_dir_name\": \"checkpoint_000003\",\n \"should_checkpoint\": true,\n \"done\": true,\n \"training_iteration\": 4,\n \"trial_id\": \"a01c9b2d\",\n \"date\": \"2024-06-25_18-10-24\",\n \"time_this_iter_s\": 87.97965884208679,\n \"time_total_s\": 344.4578182697296,\n \"pid\": 2195253,\n \"hostname\": \"r8i6n2\",\n \"node_ip\": \"10.159.28.60\",\n \"config\": {\n \"encoder_num_layer\": 8,\n \"decoder_rt_num_layer\": 1,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 16,\n \"encoder_ff\": 1024,\n \"decoder_rt_ff\": 1024,\n \"decoder_int_ff\": 512,\n \"n_head\": 4,\n \"drop_rate\": 0.23338023981015343,\n \"lr\": 0.00010269515886648154,\n \"batch_size\": 1024\n },\n \"time_since_restore\": 189.79181909561157,\n \"iterations_since_restore\": 2,\n \"experiment_tag\": \"16_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop_rate=0.2334,embedding_dim=16,encoder_ff=1024,encoder_num_layer=8,lr=0.0001,n_head=4\"\n },\n \"last_result_time\": 1719331824.3503153,\n \"metric_analysis\": {\n \"loss\": {\n \"max\": 983.9757466954509,\n \"min\": 290.2177711246521,\n \"avg\": 534.5706713143296,\n \"last\": 290.2177711246521,\n \"last-5-avg\": 534.5706713143296,\n \"last-10-avg\": 534.5706713143296\n },\n \"should_checkpoint\": {\n \"max\": true,\n \"min\": true,\n \"avg\": 1.0,\n \"last\": true,\n \"last-5-avg\": 1.0,\n \"last-10-avg\": 1.0\n },\n \"done\": {\n \"max\": true,\n \"min\": false,\n \"avg\": 0.25,\n \"last\": true,\n \"last-5-avg\": 0.25,\n \"last-10-avg\": 0.25\n },\n \"training_iteration\": {\n \"max\": 4,\n \"min\": 1,\n \"avg\": 2.5,\n \"last\": 4,\n \"last-5-avg\": 2.5,\n \"last-10-avg\": 2.5\n },\n \"time_this_iter_s\": {\n \"max\": 101.81216025352478,\n \"min\": 70.83729195594788,\n \"avg\": 86.1144545674324,\n \"last\": 87.97965884208679,\n \"last-5-avg\": 86.1144545674324,\n \"last-10-avg\": 86.1144545674324\n },\n \"time_total_s\": {\n \"max\": 344.4578182697296,\n \"min\": 83.82870721817017,\n \"avg\": 209.85767102241516,\n \"last\": 344.4578182697296,\n \"last-5-avg\": 209.85767102241516,\n \"last-10-avg\": 209.85767102241516\n },\n \"time_since_restore\": {\n \"max\": 189.79181909561157,\n \"min\": 83.82870721817017,\n \"avg\": 132.52467143535614,\n \"last\": 189.79181909561157,\n \"last-5-avg\": 132.52467143535614,\n \"last-10-avg\": 132.52467143535614\n },\n \"iterations_since_restore\": {\n \"max\": 2,\n \"min\": 1,\n \"avg\": 1.5,\n \"last\": 2,\n \"last-5-avg\": 1.5,\n \"last-10-avg\": 1.5\n }\n },\n \"_n_steps\": [\n 5,\n 10\n ],\n \"metric_n_steps\": {\n \"loss\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"800595c3000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294288c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b00749462430822914854cebf8e4094869452946807680d4308552a95bad1567f4094869452946807680d4308028140809baa764094869452946807680d43084d2693fd7b2372409486945294652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"800595c3000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294288c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b00749462430822914854cebf8e4094869452946807680d4308552a95bad1567f4094869452946807680d4308028140809baa764094869452946807680d43084d2693fd7b2372409486945294652e\"\n }\n },\n \"should_checkpoint\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059526000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452942888888888652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059526000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452942888888888652e\"\n }\n },\n \"done\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059526000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452942889898988652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059526000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452942889898988652e\"\n }\n },\n \"training_iteration\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294284b014b024b034b04652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294284b014b024b034b04652e\"\n }\n },\n \"time_this_iter_s\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059546000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b058694529428474054f5098a000000474051b5963100000047405973fa6f000000474055feb2bb000000652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059546000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a8694529428474054f5098a000000474051b5963100000047405973fa6f000000474055feb2bb000000652e\"\n }\n },\n \"time_total_s\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059546000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b058694529428474054f5098a000000474063554fdd80000047407007a68a800000474075875339400000652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059546000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a8694529428474054f5098a000000474063554fdd80000047407007a68a800000474075875339400000652e\"\n }\n },\n \"time_since_restore\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059546000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b058694529428474054f5098a000000474063554fdd80000047405973fa6f000000474067b95695000000652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059546000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a8694529428474054f5098a000000474063554fdd80000047405973fa6f000000474067b95695000000652e\"\n }\n },\n \"iterations_since_restore\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294284b014b024b014b02652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294284b014b024b014b02652e\"\n }\n }\n },\n \"checkpoint_manager\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"800595be100000000000008c267261792e747261696e2e5f696e7465726e616c2e636865636b706f696e745f6d616e61676572948c125f436865636b706f696e744d616e616765729493942981947d94288c125f636865636b706f696e745f636f6e666967948c097261792e747261696e948c10436865636b706f696e74436f6e6669679493942981947d94288c0b6e756d5f746f5f6b656570944e8c1a636865636b706f696e745f73636f72655f617474726962757465944e8c16636865636b706f696e745f73636f72655f6f72646572948c036d6178948c14636865636b706f696e745f6672657175656e6379944b008c11636865636b706f696e745f61745f656e6494898c1a5f636865636b706f696e745f6b6565705f616c6c5f72616e6b73948c0a44455052454341544544948c1f5f636865636b706f696e745f75706c6f61645f66726f6d5f776f726b65727394681275628c135f636865636b706f696e745f726573756c7473945d94288c1b7261792e747261696e2e5f696e7465726e616c2e73657373696f6e948c0f5f547261696e696e67526573756c749493942981947d94288c0a636865636b706f696e749468068c0a436865636b706f696e749493942981947d94288c0470617468948cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f61303163396232645f31365f62617463685f73697a653d313032342c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d313032342c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f5f323032342d30362d32355f31372d34342d35302f636865636b706f696e745f303030303030948c0a66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c055f75756964948c0475756964948c04555549449493942981947d948c03696e74948a10c8ae3bd2cfa83498264691db42891e69736275628c076d657472696373947d94288c046c6f7373948c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b00749462430822914854cebf8e4094869452948c0974696d657374616d70944a8ee67a668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303030948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b018c08747269616c5f6964948c086130316339623264948c0464617465948c13323032342d30362d32355f31372d34372d3236948c1074696d655f746869735f697465725f7394474054f5098a0000008c0c74696d655f746f74616c5f7394474054f5098a0000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b088c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b108c0a656e636f6465725f6666944d00048c0d6465636f6465725f72745f6666944d00048c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b048c0964726f705f7261746594473fcddf6758c235208c026c7294473f1aebc16528f5c58c0a62617463685f73697a65944d0004758c1274696d655f73696e63655f726573746f726594474054f5098a0000008c18697465726174696f6e735f73696e63655f726573746f7265944b0175756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f61303163396232645f31365f62617463685f73697a653d313032342c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d313032342c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f5f323032342d30362d32355f31372d34342d35302f636865636b706f696e745f30303030303194682268257d946827897385945294682a682d2981947d9468308a10bb086198d988a581a94a4fbe433a685d7362756268317d94288c046c6f7373946836683c4308552a95bad1567f4094869452948c0974696d657374616d70944ad5e67a668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303031948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b028c08747269616c5f6964948c086130316339623264948c0464617465948c13323032342d30362d32355f31372d34382d3337948c1074696d655f746869735f697465725f7394474051b596310000008c0c74696d655f746f74616c5f7394474063554fdd8000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b088c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b108c0a656e636f6465725f6666944d00048c0d6465636f6465725f72745f6666944d00048c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b048c0964726f705f7261746594473fcddf6758c235208c026c7294473f1aebc16528f5c58c0a62617463685f73697a65944d0004758c1274696d655f73696e63655f726573746f726594474063554fdd8000008c18697465726174696f6e735f73696e63655f726573746f7265944b0275756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f61303163396232645f31365f62617463685f73697a653d313032342c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d313032342c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f5f323032342d30362d32355f31372d34342d35302f636865636b706f696e745f30303030303294682268257d946827897385945294682a682d2981947d9468308a119f83ae4eac3d058fa948287dc2caf5ed007362756268317d94288c046c6f7373946836683c4308028140809baa764094869452948c0974696d657374616d70944a98eb7a668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303032948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b038c08747269616c5f6964948c086130316339623264948c0464617465948c13323032342d30362d32355f31382d30382d3536948c1074696d655f746869735f697465725f739447405973fa6f0000008c0c74696d655f746f74616c5f739447407007a68a8000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b088c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b108c0a656e636f6465725f6666944d00048c0d6465636f6465725f72745f6666944d00048c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b048c0964726f705f7261746594473fcddf6758c235208c026c7294473f1aebc16528f5c58c0a62617463685f73697a65944d0004758c1274696d655f73696e63655f726573746f72659447405973fa6f0000008c18697465726174696f6e735f73696e63655f726573746f7265944b0175756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f61303163396232645f31365f62617463685f73697a653d313032342c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d313032342c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f5f323032342d30362d32355f31372d34342d35302f636865636b706f696e745f30303030303394682268257d946827897385945294682a682d2981947d9468308a115a8bbc5bf2511f88d34992ba4b8dfef7007362756268317d94288c046c6f7373946836683c43084d2693fd7b23724094869452948c0974696d657374616d70944af0eb7a668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303033948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b048c08747269616c5f6964948c086130316339623264948c0464617465948c13323032342d30362d32355f31382d31302d3234948c1074696d655f746869735f697465725f7394474055feb2bb0000008c0c74696d655f746f74616c5f73944740758753394000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b088c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b108c0a656e636f6465725f6666944d00048c0d6465636f6465725f72745f6666944d00048c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b048c0964726f705f7261746594473fcddf6758c235208c026c7294473f1aebc16528f5c58c0a62617463685f73697a65944d0004758c1274696d655f73696e63655f726573746f726594474067b956950000008c18697465726174696f6e735f73696e63655f726573746f7265944b02757562658c195f6c61746573745f636865636b706f696e745f726573756c749468c075622e\"\n }\n}"
+ ],
+ [
+ "{\n \"stub\": false,\n \"trainable_name\": \"train_model\",\n \"trial_id\": \"bea15841\",\n \"storage\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059572040000000000008c1b7261792e747261696e2e5f696e7465726e616c2e73746f72616765948c0e53746f72616765436f6e746578749493942981947d94288c12637573746f6d5f66735f70726f766964656494898c1273746f726167655f6c6f63616c5f70617468948c2b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c7473948c136578706572696d656e745f6469725f6e616d65948c0f746573745f6578706572696d656e74948c0e747269616c5f6469725f6e616d65948c96747261696e5f6d6f64656c5f62656131353834315f31375f62617463685f73697a653d313032342c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d313032342c6465636f6465725f72745f6e756d5f6c617965723d342c64726f705f5f323032342d30362d32355f31372d34362d3032948c1863757272656e745f636865636b706f696e745f696e646578944b018c0b73796e635f636f6e666967948c097261792e747261696e948c0a53796e63436f6e6669679493942981947d94288c0b73796e635f706572696f64944d2c018c0c73796e635f74696d656f7574944d08078c0e73796e635f61727469666163747394898c1c73796e635f6172746966616374735f6f6e5f636865636b706f696e7494888c0a75706c6f61645f646972948c0a44455052454341544544948c0673796e6365729468188c1273796e635f6f6e5f636865636b706f696e7494681875628c1273746f726167655f66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c0f73746f726167655f66735f70617468948c452f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f7465737494681968008c115f46696c6573797374656d53796e6365729493942981947d9428681b682268134d2c0168144d08078c116c6173745f73796e635f75705f74696d65944741d99eb9823a4e538c136c6173745f73796e635f646f776e5f74696d659447fff00000000000008c0d5f73796e635f70726f63657373944e8c0c5f63757272656e745f636d649468008c125f75706c6f61645f746f5f66735f706174689493947d94288c0a6c6f63616c5f70617468948c3b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c74732f746573745f6578706572696d656e74948c0266739468228c0766735f70617468948c552f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e74948c076578636c756465945d948c0e2a2f636865636b706f696e745f2a9461758694756275622e\"\n },\n \"config\": {\n \"encoder_num_layer\": 4,\n \"decoder_rt_num_layer\": 4,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 64,\n \"encoder_ff\": 1024,\n \"decoder_rt_ff\": 1024,\n \"decoder_int_ff\": 512,\n \"n_head\": 4,\n \"drop_rate\": 0.8101220088903347,\n \"lr\": 0.0005780102991375737,\n \"batch_size\": 1024\n },\n \"_Trial__unresolved_config\": {\n \"encoder_num_layer\": 4,\n \"decoder_rt_num_layer\": 4,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 64,\n \"encoder_ff\": 1024,\n \"decoder_rt_ff\": 1024,\n \"decoder_int_ff\": 512,\n \"n_head\": 4,\n \"drop_rate\": 0.8101220088903347,\n \"lr\": 0.0005780102991375737,\n \"batch_size\": 1024\n },\n \"evaluated_params\": {\n \"batch_size\": 1024,\n \"decoder_int_ff\": 512,\n \"decoder_int_num_layer\": 1,\n \"decoder_rt_ff\": 1024,\n \"decoder_rt_num_layer\": 4,\n \"drop_rate\": 0.8101220088903347,\n \"embedding_dim\": 64,\n \"encoder_ff\": 1024,\n \"encoder_num_layer\": 4,\n \"lr\": 0.0005780102991375737,\n \"n_head\": 4\n },\n \"experiment_tag\": \"17_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=4,drop_rate=0.8101,embedding_dim=64,encoder_ff=1024,encoder_num_layer=4,lr=0.0006,n_head=4\",\n \"stopping_criterion\": {},\n \"_setup_default_resource\": true,\n \"_default_placement_group_factory\": \"80054e2e\",\n \"placement_group_factory\": \"800595ba000000000000008c237261792e74756e652e657865637574696f6e2e706c6163656d656e745f67726f757073948c15506c6163656d656e7447726f7570466163746f72799493942981947d94288c085f62756e646c6573945d947d94288c03435055944740540000000000008c034750559447401000000000000075618c155f686561645f62756e646c655f69735f656d70747994898c095f7374726174656779948c045041434b948c055f6172677394298c075f6b7761726773947d9475622e\",\n \"log_to_file\": [\n null,\n null\n ],\n \"max_failures\": 0,\n \"_default_result_or_future\": null,\n \"export_formats\": [],\n \"status\": \"PAUSED\",\n \"relative_logdir\": \"train_model_bea15841_17_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=4,drop__2024-06-25_17-46-02\",\n \"trial_name_creator\": null,\n \"trial_dirname_creator\": null,\n \"custom_trial_name\": null,\n \"custom_dirname\": null,\n \"restore_path\": null,\n \"_restore_checkpoint_result\": null,\n \"_state_json\": null,\n \"results\": \"80054e2e\",\n \"extra_arg\": \"80054e2e\",\n \"_resources\": \"80054e2e\"\n}",
+ "{\n \"start_time\": 1719332768.2796977,\n \"num_failures\": 0,\n \"num_failures_after_restore\": 0,\n \"error_filename\": null,\n \"pickled_error_filename\": null,\n \"last_result\": {\n \"loss\": 2008.3096395177165,\n \"timestamp\": 1719332932,\n \"checkpoint_dir_name\": \"checkpoint_000001\",\n \"should_checkpoint\": true,\n \"done\": false,\n \"training_iteration\": 2,\n \"trial_id\": \"bea15841\",\n \"date\": \"2024-06-25_18-28-52\",\n \"time_this_iter_s\": 75.03496551513672,\n \"time_total_s\": 164.1780436038971,\n \"pid\": 2195253,\n \"hostname\": \"r8i6n2\",\n \"node_ip\": \"10.159.28.60\",\n \"config\": {\n \"encoder_num_layer\": 4,\n \"decoder_rt_num_layer\": 4,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 64,\n \"encoder_ff\": 1024,\n \"decoder_rt_ff\": 1024,\n \"decoder_int_ff\": 512,\n \"n_head\": 4,\n \"drop_rate\": 0.8101220088903347,\n \"lr\": 0.0005780102991375737,\n \"batch_size\": 1024\n },\n \"time_since_restore\": 164.1780436038971,\n \"iterations_since_restore\": 2,\n \"experiment_tag\": \"17_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=4,drop_rate=0.8101,embedding_dim=64,encoder_ff=1024,encoder_num_layer=4,lr=0.0006,n_head=4\"\n },\n \"last_result_time\": 1719332932.4666936,\n \"metric_analysis\": {\n \"loss\": {\n \"max\": 2012.1837220680056,\n \"min\": 2008.3096395177165,\n \"avg\": 2010.246680792861,\n \"last\": 2008.3096395177165,\n \"last-5-avg\": 2010.246680792861,\n \"last-10-avg\": 2010.246680792861\n },\n \"should_checkpoint\": {\n \"max\": true,\n \"min\": true,\n \"avg\": 1.0,\n \"last\": true,\n \"last-5-avg\": 1.0,\n \"last-10-avg\": 1.0\n },\n \"done\": {\n \"max\": false,\n \"min\": false,\n \"avg\": 0.0,\n \"last\": false,\n \"last-5-avg\": 0.0,\n \"last-10-avg\": 0.0\n },\n \"training_iteration\": {\n \"max\": 2,\n \"min\": 1,\n \"avg\": 1.5,\n \"last\": 2,\n \"last-5-avg\": 1.5,\n \"last-10-avg\": 1.5\n },\n \"time_this_iter_s\": {\n \"max\": 89.14307808876038,\n \"min\": 75.03496551513672,\n \"avg\": 82.08902180194855,\n \"last\": 75.03496551513672,\n \"last-5-avg\": 82.08902180194855,\n \"last-10-avg\": 82.08902180194855\n },\n \"time_total_s\": {\n \"max\": 164.1780436038971,\n \"min\": 89.14307808876038,\n \"avg\": 126.66056084632874,\n \"last\": 164.1780436038971,\n \"last-5-avg\": 126.66056084632874,\n \"last-10-avg\": 126.66056084632874\n },\n \"time_since_restore\": {\n \"max\": 164.1780436038971,\n \"min\": 89.14307808876038,\n \"avg\": 126.66056084632874,\n \"last\": 164.1780436038971,\n \"last-5-avg\": 126.66056084632874,\n \"last-10-avg\": 126.66056084632874\n },\n \"iterations_since_restore\": {\n \"max\": 2,\n \"min\": 1,\n \"avg\": 1.5,\n \"last\": 2,\n \"last-5-avg\": 1.5,\n \"last-10-avg\": 1.5\n }\n },\n \"_n_steps\": [\n 5,\n 10\n ],\n \"metric_n_steps\": {\n \"loss\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005959d000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294288c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b0074946243088d46a321bc709f4094869452946807680d4308914824123d619f409486945294652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005959d000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294288c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b0074946243088d46a321bc709f4094869452946807680d4308914824123d619f409486945294652e\"\n }\n },\n \"should_checkpoint\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059524000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294288888652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059524000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294288888652e\"\n }\n },\n \"done\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059524000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294288989652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059524000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294288989652e\"\n }\n },\n \"training_iteration\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059526000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294284b014b02652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059526000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294284b014b02652e\"\n }\n },\n \"time_this_iter_s\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059534000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b058694529428474056492831000000474052c23ce0000000652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059534000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a8694529428474056492831000000474052c23ce0000000652e\"\n }\n },\n \"time_total_s\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059534000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452942847405649283100000047406485b288800000652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059534000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452942847405649283100000047406485b288800000652e\"\n }\n },\n \"time_since_restore\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059534000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452942847405649283100000047406485b288800000652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059534000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452942847405649283100000047406485b288800000652e\"\n }\n },\n \"iterations_since_restore\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059526000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294284b014b02652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059526000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294284b014b02652e\"\n }\n }\n },\n \"checkpoint_manager\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"800595a4090000000000008c267261792e747261696e2e5f696e7465726e616c2e636865636b706f696e745f6d616e61676572948c125f436865636b706f696e744d616e616765729493942981947d94288c125f636865636b706f696e745f636f6e666967948c097261792e747261696e948c10436865636b706f696e74436f6e6669679493942981947d94288c0b6e756d5f746f5f6b656570944e8c1a636865636b706f696e745f73636f72655f617474726962757465944e8c16636865636b706f696e745f73636f72655f6f72646572948c036d6178948c14636865636b706f696e745f6672657175656e6379944b008c11636865636b706f696e745f61745f656e6494898c1a5f636865636b706f696e745f6b6565705f616c6c5f72616e6b73948c0a44455052454341544544948c1f5f636865636b706f696e745f75706c6f61645f66726f6d5f776f726b65727394681275628c135f636865636b706f696e745f726573756c7473945d94288c1b7261792e747261696e2e5f696e7465726e616c2e73657373696f6e948c0f5f547261696e696e67526573756c749493942981947d94288c0a636865636b706f696e749468068c0a436865636b706f696e749493942981947d94288c0470617468948cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f62656131353834315f31375f62617463685f73697a653d313032342c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d313032342c6465636f6465725f72745f6e756d5f6c617965723d342c64726f705f5f323032342d30362d32355f31372d34362d30322f636865636b706f696e745f303030303030948c0a66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c055f75756964948c0475756964948c04555549449493942981947d948c03696e74948a105352ddbd558402981243df8184916b6d736275628c076d657472696373947d94288c046c6f7373948c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b0074946243088d46a321bc709f4094869452948c0974696d657374616d70944af9ef7a668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303030948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b018c08747269616c5f6964948c086265613135383431948c0464617465948c13323032342d30362d32355f31382d32372d3337948c1074696d655f746869735f697465725f73944740564928310000008c0c74696d655f746f74616c5f73944740564928310000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b048c146465636f6465725f72745f6e756d5f6c61796572944b048c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b408c0a656e636f6465725f6666944d00048c0d6465636f6465725f72745f6666944d00048c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b048c0964726f705f7261746594473fe9ec84fdbe859a8c026c7294473f42f0b3aa7022498c0a62617463685f73697a65944d0004758c1274696d655f73696e63655f726573746f7265944740564928310000008c18697465726174696f6e735f73696e63655f726573746f7265944b0175756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f62656131353834315f31375f62617463685f73697a653d313032342c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d313032342c6465636f6465725f72745f6e756d5f6c617965723d342c64726f705f5f323032342d30362d32355f31372d34362d30322f636865636b706f696e745f30303030303194682268257d946827897385945294682a682d2981947d9468308a10be83920c3952d4a97344b9f324a4f52e7362756268317d94288c046c6f7373946836683c4308914824123d619f4094869452948c0974696d657374616d70944a44f07a668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303031948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b028c08747269616c5f6964948c086265613135383431948c0464617465948c13323032342d30362d32355f31382d32382d3532948c1074696d655f746869735f697465725f7394474052c23ce00000008c0c74696d655f746f74616c5f739447406485b2888000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b048c146465636f6465725f72745f6e756d5f6c61796572944b048c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b408c0a656e636f6465725f6666944d00048c0d6465636f6465725f72745f6666944d00048c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b048c0964726f705f7261746594473fe9ec84fdbe859a8c026c7294473f42f0b3aa7022498c0a62617463685f73697a65944d0004758c1274696d655f73696e63655f726573746f72659447406485b2888000008c18697465726174696f6e735f73696e63655f726573746f7265944b02757562658c195f6c61746573745f636865636b706f696e745f726573756c7494686275622e\"\n }\n}"
+ ],
+ [
+ "{\n \"stub\": false,\n \"trainable_name\": \"train_model\",\n \"trial_id\": \"c2c99680\",\n \"storage\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059572040000000000008c1b7261792e747261696e2e5f696e7465726e616c2e73746f72616765948c0e53746f72616765436f6e746578749493942981947d94288c12637573746f6d5f66735f70726f766964656494898c1273746f726167655f6c6f63616c5f70617468948c2b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c7473948c136578706572696d656e745f6469725f6e616d65948c0f746573745f6578706572696d656e74948c0e747269616c5f6469725f6e616d65948c96747261696e5f6d6f64656c5f63326339393638305f31385f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d313032342c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f5f323032342d30362d32355f31382d32362d3038948c1863757272656e745f636865636b706f696e745f696e646578944b038c0b73796e635f636f6e666967948c097261792e747261696e948c0a53796e63436f6e6669679493942981947d94288c0b73796e635f706572696f64944d2c018c0c73796e635f74696d656f7574944d08078c0e73796e635f61727469666163747394898c1c73796e635f6172746966616374735f6f6e5f636865636b706f696e7494888c0a75706c6f61645f646972948c0a44455052454341544544948c0673796e6365729468188c1273796e635f6f6e5f636865636b706f696e7494681875628c1273746f726167655f66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c0f73746f726167655f66735f70617468948c452f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f7465737494681968008c115f46696c6573797374656d53796e6365729493942981947d9428681b682268134d2c0168144d08078c116c6173745f73796e635f75705f74696d65944741d99ebbde5539d88c136c6173745f73796e635f646f776e5f74696d659447fff00000000000008c0d5f73796e635f70726f63657373944e8c0c5f63757272656e745f636d649468008c125f75706c6f61645f746f5f66735f706174689493947d94288c0a6c6f63616c5f70617468948c3b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c74732f746573745f6578706572696d656e74948c0266739468228c0766735f70617468948c552f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e74948c076578636c756465945d948c0e2a2f636865636b706f696e745f2a9461758694756275622e\"\n },\n \"config\": {\n \"encoder_num_layer\": 4,\n \"decoder_rt_num_layer\": 1,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 1024,\n \"encoder_ff\": 1024,\n \"decoder_rt_ff\": 1024,\n \"decoder_int_ff\": 512,\n \"n_head\": 1,\n \"drop_rate\": 0.5887703154000944,\n \"lr\": 0.0008721653331207079,\n \"batch_size\": 2048\n },\n \"_Trial__unresolved_config\": {\n \"encoder_num_layer\": 4,\n \"decoder_rt_num_layer\": 1,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 1024,\n \"encoder_ff\": 1024,\n \"decoder_rt_ff\": 1024,\n \"decoder_int_ff\": 512,\n \"n_head\": 1,\n \"drop_rate\": 0.5887703154000944,\n \"lr\": 0.0008721653331207079,\n \"batch_size\": 2048\n },\n \"evaluated_params\": {\n \"batch_size\": 2048,\n \"decoder_int_ff\": 512,\n \"decoder_int_num_layer\": 1,\n \"decoder_rt_ff\": 1024,\n \"decoder_rt_num_layer\": 1,\n \"drop_rate\": 0.5887703154000944,\n \"embedding_dim\": 1024,\n \"encoder_ff\": 1024,\n \"encoder_num_layer\": 4,\n \"lr\": 0.0008721653331207079,\n \"n_head\": 1\n },\n \"experiment_tag\": \"18_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop_rate=0.5888,embedding_dim=1024,encoder_ff=1024,encoder_num_layer=4,lr=0.0009,n_head=1\",\n \"stopping_criterion\": {},\n \"_setup_default_resource\": true,\n \"_default_placement_group_factory\": \"80054e2e\",\n \"placement_group_factory\": \"800595ba000000000000008c237261792e74756e652e657865637574696f6e2e706c6163656d656e745f67726f757073948c15506c6163656d656e7447726f7570466163746f72799493942981947d94288c085f62756e646c6573945d947d94288c03435055944740540000000000008c034750559447401000000000000075618c155f686561645f62756e646c655f69735f656d70747994898c095f7374726174656779948c045041434b948c055f6172677394298c075f6b7761726773947d9475622e\",\n \"log_to_file\": [\n null,\n null\n ],\n \"max_failures\": 0,\n \"_default_result_or_future\": null,\n \"export_formats\": [],\n \"status\": \"PAUSED\",\n \"relative_logdir\": \"train_model_c2c99680_18_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_18-26-08\",\n \"trial_name_creator\": null,\n \"trial_dirname_creator\": null,\n \"custom_trial_name\": null,\n \"custom_dirname\": null,\n \"restore_path\": null,\n \"_restore_checkpoint_result\": null,\n \"_state_json\": null,\n \"results\": \"80054e2e\",\n \"extra_arg\": \"80054e2e\",\n \"_resources\": \"80054e2e\"\n}",
+ "{\n \"start_time\": 1719332932.6360667,\n \"num_failures\": 0,\n \"num_failures_after_restore\": 0,\n \"error_filename\": null,\n \"pickled_error_filename\": null,\n \"last_result\": {\n \"loss\": 2023.2190220149482,\n \"timestamp\": 1719338188,\n \"checkpoint_dir_name\": \"checkpoint_000003\",\n \"should_checkpoint\": true,\n \"done\": false,\n \"training_iteration\": 4,\n \"trial_id\": \"c2c99680\",\n \"date\": \"2024-06-25_19-56-29\",\n \"time_this_iter_s\": 439.1456036567688,\n \"time_total_s\": 1790.0054636001587,\n \"pid\": 2195253,\n \"hostname\": \"r8i6n2\",\n \"node_ip\": \"10.159.28.60\",\n \"config\": {\n \"encoder_num_layer\": 4,\n \"decoder_rt_num_layer\": 1,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 1024,\n \"encoder_ff\": 1024,\n \"decoder_rt_ff\": 1024,\n \"decoder_int_ff\": 512,\n \"n_head\": 1,\n \"drop_rate\": 0.5887703154000944,\n \"lr\": 0.0008721653331207079,\n \"batch_size\": 2048\n },\n \"time_since_restore\": 893.1749258041382,\n \"iterations_since_restore\": 2,\n \"experiment_tag\": \"18_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop_rate=0.5888,embedding_dim=1024,encoder_ff=1024,encoder_num_layer=4,lr=0.0009,n_head=1\"\n },\n \"last_result_time\": 1719338189.219069,\n \"metric_analysis\": {\n \"loss\": {\n \"max\": 2023.2190220149482,\n \"min\": 1982.5428274560163,\n \"avg\": 2001.3316611943283,\n \"last\": 2023.2190220149482,\n \"last-5-avg\": 2001.3316611943283,\n \"last-10-avg\": 2001.3316611943283\n },\n \"should_checkpoint\": {\n \"max\": true,\n \"min\": true,\n \"avg\": 1.0,\n \"last\": true,\n \"last-5-avg\": 1.0,\n \"last-10-avg\": 1.0\n },\n \"done\": {\n \"max\": false,\n \"min\": false,\n \"avg\": 0.0,\n \"last\": false,\n \"last-5-avg\": 0.0,\n \"last-10-avg\": 0.0\n },\n \"training_iteration\": {\n \"max\": 4,\n \"min\": 1,\n \"avg\": 2.5,\n \"last\": 4,\n \"last-5-avg\": 2.5,\n \"last-10-avg\": 2.5\n },\n \"time_this_iter_s\": {\n \"max\": 456.8247263431549,\n \"min\": 439.1456036567688,\n \"avg\": 447.5013659000397,\n \"last\": 439.1456036567688,\n \"last-5-avg\": 447.5013659000397,\n \"last-10-avg\": 447.5013659000397\n },\n \"time_total_s\": {\n \"max\": 1790.0054636001587,\n \"min\": 456.8247263431549,\n \"avg\": 1123.630146920681,\n \"last\": 1790.0054636001587,\n \"last-5-avg\": 1123.630146920681,\n \"last-10-avg\": 1123.630146920681\n },\n \"time_since_restore\": {\n \"max\": 896.8305377960205,\n \"min\": 454.0293221473694,\n \"avg\": 675.2148780226707,\n \"last\": 893.1749258041382,\n \"last-5-avg\": 675.2148780226707,\n \"last-10-avg\": 675.2148780226707\n },\n \"iterations_since_restore\": {\n \"max\": 2,\n \"min\": 1,\n \"avg\": 1.5,\n \"last\": 2,\n \"last-5-avg\": 1.5,\n \"last-10-avg\": 1.5\n }\n },\n \"_n_steps\": [\n 5,\n 10\n ],\n \"metric_n_steps\": {\n \"loss\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"800595c3000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294288c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b00749462430824128924c40c9f4094869452946807680d4308d8ebf5da2bfa9e4094869452946807680d4308a9542a357e719f4094869452946807680d43083a9d4e47e09c9f409486945294652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"800595c3000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294288c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b00749462430824128924c40c9f4094869452946807680d4308d8ebf5da2bfa9e4094869452946807680d4308a9542a357e719f4094869452946807680d43083a9d4e47e09c9f409486945294652e\"\n }\n },\n \"should_checkpoint\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059526000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452942888888888652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059526000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452942888888888652e\"\n }\n },\n \"done\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059526000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452942889898989652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059526000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452942889898989652e\"\n }\n },\n \"training_iteration\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294284b014b024b034b04652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294284b014b024b034b04652e\"\n }\n },\n \"time_this_iter_s\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059546000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452942847407c8d321440000047407b8017cdc0000047407c60781a80000047407b725464800000652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059546000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452942847407c8d321440000047407b8017cdc0000047407c60781a80000047407b725464800000652e\"\n }\n },\n \"time_total_s\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059546000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452942847407c8d321440000047408c06a4f10000004740951b707f20000047409bf80598400000652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059546000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452942847407c8d321440000047408c06a4f10000004740951b707f20000047409bf80598400000652e\"\n }\n },\n \"time_since_restore\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059546000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452942847407c8d321440000047408c06a4f100000047407c60781a80000047408be9663f800000652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059546000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452942847407c8d321440000047408c06a4f100000047407c60781a80000047408be9663f800000652e\"\n }\n },\n \"iterations_since_restore\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294284b014b024b014b02652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294284b014b024b014b02652e\"\n }\n }\n },\n \"checkpoint_manager\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"800595c1100000000000008c267261792e747261696e2e5f696e7465726e616c2e636865636b706f696e745f6d616e61676572948c125f436865636b706f696e744d616e616765729493942981947d94288c125f636865636b706f696e745f636f6e666967948c097261792e747261696e948c10436865636b706f696e74436f6e6669679493942981947d94288c0b6e756d5f746f5f6b656570944e8c1a636865636b706f696e745f73636f72655f617474726962757465944e8c16636865636b706f696e745f73636f72655f6f72646572948c036d6178948c14636865636b706f696e745f6672657175656e6379944b008c11636865636b706f696e745f61745f656e6494898c1a5f636865636b706f696e745f6b6565705f616c6c5f72616e6b73948c0a44455052454341544544948c1f5f636865636b706f696e745f75706c6f61645f66726f6d5f776f726b65727394681275628c135f636865636b706f696e745f726573756c7473945d94288c1b7261792e747261696e2e5f696e7465726e616c2e73657373696f6e948c0f5f547261696e696e67526573756c749493942981947d94288c0a636865636b706f696e749468068c0a436865636b706f696e749493942981947d94288c0470617468948cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f63326339393638305f31385f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d313032342c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f5f323032342d30362d32355f31382d32362d30382f636865636b706f696e745f303030303030948c0a66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c055f75756964948c0475756964948c04555549449493942981947d948c03696e74948a101c8edb50b935b7a61d419595d7acd718736275628c076d657472696373947d94288c046c6f7373948c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b00749462430824128924c40c9f4094869452948c0974696d657374616d70944a0df27a668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303030948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b018c08747269616c5f6964948c086332633939363830948c0464617465948c13323032342d30362d32355f31382d33362d3239948c1074696d655f746869735f697465725f739447407c8d32144000008c0c74696d655f746f74616c5f739447407c8d32144000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b048c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944d00048c0a656e636f6465725f6666944d00048c0d6465636f6465725f72745f6666944d00048c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b018c0964726f705f7261746594473fe2d734d82ff7e58c026c7294473f4c9440ca8dff148c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f72659447407c8d32144000008c18697465726174696f6e735f73696e63655f726573746f7265944b0175756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f63326339393638305f31385f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d313032342c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f5f323032342d30362d32355f31382d32362d30382f636865636b706f696e745f30303030303194682268257d946827897385945294682a682d2981947d9468308a10e1b8940c1af27eabef4bf5010b4ec7667362756268317d94288c046c6f7373946836683c4308d8ebf5da2bfa9e4094869452948c0974696d657374616d70944ac5f37a668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303031948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b028c08747269616c5f6964948c086332633939363830948c0464617465948c13323032342d30362d32355f31382d34332d3439948c1074696d655f746869735f697465725f739447407b8017cdc000008c0c74696d655f746f74616c5f739447408c06a4f10000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b048c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944d00048c0a656e636f6465725f6666944d00048c0d6465636f6465725f72745f6666944d00048c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b018c0964726f705f7261746594473fe2d734d82ff7e58c026c7294473f4c9440ca8dff148c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f72659447408c06a4f10000008c18697465726174696f6e735f73696e63655f726573746f7265944b0275756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f63326339393638305f31385f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d313032342c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f5f323032342d30362d32355f31382d32362d30382f636865636b706f696e745f30303030303294682268257d946827897385945294682a682d2981947d9468308a1135863e593a8482982843ef83852573c9007362756268317d94288c046c6f7373946836683c4308a9542a357e719f4094869452948c0974696d657374616d70944a15037b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303032948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b038c08747269616c5f6964948c086332633939363830948c0464617465948c13323032342d30362d32355f31392d34392d3130948c1074696d655f746869735f697465725f739447407c60781a8000008c0c74696d655f746f74616c5f73944740951b707f2000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b048c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944d00048c0a656e636f6465725f6666944d00048c0d6465636f6465725f72745f6666944d00048c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b018c0964726f705f7261746594473fe2d734d82ff7e58c026c7294473f4c9440ca8dff148c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f72659447407c60781a8000008c18697465726174696f6e735f73696e63655f726573746f7265944b0175756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f63326339393638305f31385f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d313032342c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f5f323032342d30362d32355f31382d32362d30382f636865636b706f696e745f30303030303394682268257d946827897385945294682a682d2981947d9468308a1021b63c16e40a7298204def415d0d94067362756268317d94288c046c6f7373946836683c43083a9d4e47e09c9f4094869452948c0974696d657374616d70944acc047b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303033948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b048c08747269616c5f6964948c086332633939363830948c0464617465948c13323032342d30362d32355f31392d35362d3239948c1074696d655f746869735f697465725f739447407b7254648000008c0c74696d655f746f74616c5f739447409bf805984000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b048c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944d00048c0a656e636f6465725f6666944d00048c0d6465636f6465725f72745f6666944d00048c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b018c0964726f705f7261746594473fe2d734d82ff7e58c026c7294473f4c9440ca8dff148c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f72659447408be9663f8000008c18697465726174696f6e735f73696e63655f726573746f7265944b02757562658c195f6c61746573745f636865636b706f696e745f726573756c749468c075622e\"\n }\n}"
+ ],
+ [
+ "{\n \"stub\": false,\n \"trainable_name\": \"train_model\",\n \"trial_id\": \"43bba37c\",\n \"storage\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059572040000000000008c1b7261792e747261696e2e5f696e7465726e616c2e73746f72616765948c0e53746f72616765436f6e746578749493942981947d94288c12637573746f6d5f66735f70726f766964656494898c1273746f726167655f6c6f63616c5f70617468948c2b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c7473948c136578706572696d656e745f6469725f6e616d65948c0f746573745f6578706572696d656e74948c0e747269616c5f6469725f6e616d65948c96747261696e5f6d6f64656c5f34336262613337635f31395f62617463685f73697a653d313032342c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d313032342c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f5f323032342d30362d32355f31382d32382d3532948c1863757272656e745f636865636b706f696e745f696e646578944b078c0b73796e635f636f6e666967948c097261792e747261696e948c0a53796e63436f6e6669679493942981947d94288c0b73796e635f706572696f64944d2c018c0c73796e635f74696d656f7574944d08078c0e73796e635f61727469666163747394898c1c73796e635f6172746966616374735f6f6e5f636865636b706f696e7494888c0a75706c6f61645f646972948c0a44455052454341544544948c0673796e6365729468188c1273796e635f6f6e5f636865636b706f696e7494681875628c1273746f726167655f66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c0f73746f726167655f66735f70617468948c452f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f7465737494681968008c115f46696c6573797374656d53796e6365729493942981947d9428681b682268134d2c0168144d08078c116c6173745f73796e635f75705f74696d65944741d99ebbde5539d88c136c6173745f73796e635f646f776e5f74696d659447fff00000000000008c0d5f73796e635f70726f63657373944e8c0c5f63757272656e745f636d649468008c125f75706c6f61645f746f5f66735f706174689493947d94288c0a6c6f63616c5f70617468948c3b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c74732f746573745f6578706572696d656e74948c0266739468228c0766735f70617468948c552f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e74948c076578636c756465945d948c0e2a2f636865636b706f696e745f2a9461758694756275622e\"\n },\n \"config\": {\n \"encoder_num_layer\": 8,\n \"decoder_rt_num_layer\": 1,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 256,\n \"encoder_ff\": 1024,\n \"decoder_rt_ff\": 1024,\n \"decoder_int_ff\": 512,\n \"n_head\": 1,\n \"drop_rate\": 0.4036073673456261,\n \"lr\": 0.0007661776725543005,\n \"batch_size\": 1024\n },\n \"_Trial__unresolved_config\": {\n \"encoder_num_layer\": 8,\n \"decoder_rt_num_layer\": 1,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 256,\n \"encoder_ff\": 1024,\n \"decoder_rt_ff\": 1024,\n \"decoder_int_ff\": 512,\n \"n_head\": 1,\n \"drop_rate\": 0.4036073673456261,\n \"lr\": 0.0007661776725543005,\n \"batch_size\": 1024\n },\n \"evaluated_params\": {\n \"batch_size\": 1024,\n \"decoder_int_ff\": 512,\n \"decoder_int_num_layer\": 1,\n \"decoder_rt_ff\": 1024,\n \"decoder_rt_num_layer\": 1,\n \"drop_rate\": 0.4036073673456261,\n \"embedding_dim\": 256,\n \"encoder_ff\": 1024,\n \"encoder_num_layer\": 8,\n \"lr\": 0.0007661776725543005,\n \"n_head\": 1\n },\n \"experiment_tag\": \"19_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop_rate=0.4036,embedding_dim=256,encoder_ff=1024,encoder_num_layer=8,lr=0.0008,n_head=1\",\n \"stopping_criterion\": {},\n \"_setup_default_resource\": true,\n \"_default_placement_group_factory\": \"80054e2e\",\n \"placement_group_factory\": \"800595ba000000000000008c237261792e74756e652e657865637574696f6e2e706c6163656d656e745f67726f757073948c15506c6163656d656e7447726f7570466163746f72799493942981947d94288c085f62756e646c6573945d947d94288c03435055944740540000000000008c034750559447401000000000000075618c155f686561645f62756e646c655f69735f656d70747994898c095f7374726174656779948c045041434b948c055f6172677394298c075f6b7761726773947d9475622e\",\n \"log_to_file\": [\n null,\n null\n ],\n \"max_failures\": 0,\n \"_default_result_or_future\": null,\n \"export_formats\": [],\n \"status\": \"PAUSED\",\n \"relative_logdir\": \"train_model_43bba37c_19_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_18-28-52\",\n \"trial_name_creator\": null,\n \"trial_dirname_creator\": null,\n \"custom_trial_name\": null,\n \"custom_dirname\": null,\n \"restore_path\": null,\n \"_restore_checkpoint_result\": null,\n \"_state_json\": null,\n \"results\": \"80054e2e\",\n \"extra_arg\": \"80054e2e\",\n \"_resources\": \"80054e2e\"\n}",
+ "{\n \"start_time\": 1719333829.6209524,\n \"num_failures\": 0,\n \"num_failures_after_restore\": 0,\n \"error_filename\": null,\n \"pickled_error_filename\": null,\n \"last_result\": {\n \"loss\": 1962.8120477631337,\n \"timestamp\": 1719340670,\n \"checkpoint_dir_name\": \"checkpoint_000007\",\n \"should_checkpoint\": true,\n \"done\": false,\n \"training_iteration\": 8,\n \"trial_id\": \"43bba37c\",\n \"date\": \"2024-06-25_20-37-50\",\n \"time_this_iter_s\": 147.67359066009521,\n \"time_total_s\": 1221.8980810642242,\n \"pid\": 2195253,\n \"hostname\": \"r8i6n2\",\n \"node_ip\": \"10.159.28.60\",\n \"config\": {\n \"encoder_num_layer\": 8,\n \"decoder_rt_num_layer\": 1,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 256,\n \"encoder_ff\": 1024,\n \"decoder_rt_ff\": 1024,\n \"decoder_int_ff\": 512,\n \"n_head\": 1,\n \"drop_rate\": 0.4036073673456261,\n \"lr\": 0.0007661776725543005,\n \"batch_size\": 1024\n },\n \"time_since_restore\": 603.0734503269196,\n \"iterations_since_restore\": 4,\n \"experiment_tag\": \"19_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop_rate=0.4036,embedding_dim=256,encoder_ff=1024,encoder_num_layer=8,lr=0.0008,n_head=1\"\n },\n \"last_result_time\": 1719340670.7245317,\n \"metric_analysis\": {\n \"loss\": {\n \"max\": 1972.7257733682948,\n \"min\": 1961.9788803941622,\n \"avg\": 1966.0090920155444,\n \"last\": 1962.8120477631337,\n \"last-5-avg\": 1966.325843534695,\n \"last-10-avg\": 1966.0090920155444\n },\n \"should_checkpoint\": {\n \"max\": true,\n \"min\": true,\n \"avg\": 1.0,\n \"last\": true,\n \"last-5-avg\": 1.0,\n \"last-10-avg\": 1.0\n },\n \"done\": {\n \"max\": false,\n \"min\": false,\n \"avg\": 0.0,\n \"last\": false,\n \"last-5-avg\": 0.0,\n \"last-10-avg\": 0.0\n },\n \"training_iteration\": {\n \"max\": 8,\n \"min\": 1,\n \"avg\": 4.5,\n \"last\": 8,\n \"last-5-avg\": 6.0,\n \"last-10-avg\": 4.5\n },\n \"time_this_iter_s\": {\n \"max\": 161.8538646697998,\n \"min\": 147.0123507976532,\n \"avg\": 152.737260133028,\n \"last\": 147.67359066009521,\n \"last-5-avg\": 150.12170810699462,\n \"last-10-avg\": 152.73726013302803\n },\n \"time_total_s\": {\n \"max\": 1221.8980810642242,\n \"min\": 161.8538646697998,\n \"avg\": 695.531242519617,\n \"last\": 1221.8980810642242,\n \"last-5-avg\": 924.2763486385345,\n \"last-10-avg\": 695.5312425196171\n },\n \"time_since_restore\": {\n \"max\": 603.0734503269196,\n \"min\": 160.8864643573761,\n \"avg\": 308.6877292096615,\n \"last\": 603.0734503269196,\n \"last-5-avg\": 367.2716856956482,\n \"last-10-avg\": 308.6877292096615\n },\n \"iterations_since_restore\": {\n \"max\": 4,\n \"min\": 1,\n \"avg\": 2.0,\n \"last\": 4,\n \"last-5-avg\": 2.4,\n \"last-10-avg\": 2.0\n }\n },\n \"_n_steps\": [\n 5,\n 10\n ],\n \"metric_n_steps\": {\n \"loss\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"800595d6000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294288c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b00749462430889442231e7d29e4094869452946807680d4308fe7ebf1f56b69e4094869452946807680d4308381c0e7758b49e4094869452946807680d430802814000afb59e4094869452946807680d4308cce572893fab9e409486945294652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005950f010000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294288c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b0074946243087d3e9f5feaa79e4094869452946807680d430863b158bc38bb9e4094869452946807680d4308ee76bb0da3be9e4094869452946807680d430889442231e7d29e4094869452946807680d4308fe7ebf1f56b69e4094869452946807680d4308381c0e7758b49e4094869452946807680d430802814000afb59e4094869452946807680d4308cce572893fab9e409486945294652e\"\n }\n },\n \"should_checkpoint\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059527000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294288888888888652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294288888888888888888652e\"\n }\n },\n \"done\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059527000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294288989898989652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294288989898989898989652e\"\n }\n },\n \"training_iteration\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952c000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294284b044b054b064b074b08652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059532000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294284b014b024b034b044b054b064b074b08652e\"\n }\n },\n \"time_this_iter_s\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005954f000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b058694529428474062711f758000004740641c5dea80000047406260652d80000047406270088e800000474062758e0e000000652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005956a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294284740643b52dc0000004740627bdea280000047406432126c000000474062711f758000004740641c5dea80000047406260652d80000047406270088e800000474062758e0e000000652e\"\n }\n },\n \"time_total_s\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005954f000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294284740835698d80000004740885db052a0000047408cf5c99e000000474090c8e5e0d000004740931797a2900000652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005956a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294284740643b52dc0000004740735b98bf40000047407d74a1f54000004740835698d80000004740885db052a0000047408cf5c99e000000474090c8e5e0d000004740931797a2900000652e\"\n }\n },\n \"time_since_restore\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005954f000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294284740735198f0c000004740641c5dea8000004740733e618c00000047407c7665d3400000474082d8966d200000652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005956a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294284740643b52dc0000004740735b98bf40000047406432126c0000004740735198f0c000004740641c5dea8000004740733e618c00000047407c7665d3400000474082d8966d200000652e\"\n }\n },\n \"iterations_since_restore\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952c000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294284b024b014b024b034b04652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059532000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294284b014b024b014b024b014b024b034b04652e\"\n }\n }\n },\n \"checkpoint_manager\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"800595fb1e0000000000008c267261792e747261696e2e5f696e7465726e616c2e636865636b706f696e745f6d616e61676572948c125f436865636b706f696e744d616e616765729493942981947d94288c125f636865636b706f696e745f636f6e666967948c097261792e747261696e948c10436865636b706f696e74436f6e6669679493942981947d94288c0b6e756d5f746f5f6b656570944e8c1a636865636b706f696e745f73636f72655f617474726962757465944e8c16636865636b706f696e745f73636f72655f6f72646572948c036d6178948c14636865636b706f696e745f6672657175656e6379944b008c11636865636b706f696e745f61745f656e6494898c1a5f636865636b706f696e745f6b6565705f616c6c5f72616e6b73948c0a44455052454341544544948c1f5f636865636b706f696e745f75706c6f61645f66726f6d5f776f726b65727394681275628c135f636865636b706f696e745f726573756c7473945d94288c1b7261792e747261696e2e5f696e7465726e616c2e73657373696f6e948c0f5f547261696e696e67526573756c749493942981947d94288c0a636865636b706f696e749468068c0a436865636b706f696e749493942981947d94288c0470617468948cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f34336262613337635f31395f62617463685f73697a653d313032342c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d313032342c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f5f323032342d30362d32355f31382d32382d35322f636865636b706f696e745f303030303030948c0a66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c055f75756964948c0475756964948c04555549449493942981947d948c03696e74948a10c55bb4c2568adfa6d54834ade0808d70736275628c076d657472696373947d94288c046c6f7373948c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b0074946243087d3e9f5feaa79e4094869452948c0974696d657374616d70944a67f47a668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303030948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b018c08747269616c5f6964948c083433626261333763948c0464617465948c13323032342d30362d32355f31382d34362d3331948c1074696d655f746869735f697465725f73944740643b52dc0000008c0c74696d655f746f74616c5f73944740643b52dc0000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b088c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944d00018c0a656e636f6465725f6666944d00048c0d6465636f6465725f72745f6666944d00048c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b018c0964726f705f7261746594473fd9d4b3fecb24d28c026c7294473f491b2a05f539228c0a62617463685f73697a65944d0004758c1274696d655f73696e63655f726573746f7265944740643b52dc0000008c18697465726174696f6e735f73696e63655f726573746f7265944b0175756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f34336262613337635f31395f62617463685f73697a653d313032342c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d313032342c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f5f323032342d30362d32355f31382d32382d35322f636865636b706f696e745f30303030303194682268257d946827897385945294682a682d2981947d9468308a118c9b38b9a32c22987742418adc523bfb007362756268317d94288c046c6f7373946836683c430863b158bc38bb9e4094869452948c0974696d657374616d70944afbf47a668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303031948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b028c08747269616c5f6964948c083433626261333763948c0464617465948c13323032342d30362d32355f31382d34382d3539948c1074696d655f746869735f697465725f73944740627bdea28000008c0c74696d655f746f74616c5f73944740735b98bf4000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b088c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944d00018c0a656e636f6465725f6666944d00048c0d6465636f6465725f72745f6666944d00048c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b018c0964726f705f7261746594473fd9d4b3fecb24d28c026c7294473f491b2a05f539228c0a62617463685f73697a65944d0004758c1274696d655f73696e63655f726573746f7265944740735b98bf4000008c18697465726174696f6e735f73696e63655f726573746f7265944b0275756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f34336262613337635f31395f62617463685f73697a653d313032342c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d313032342c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f5f323032342d30362d32355f31382d32382d35322f636865636b706f696e745f30303030303294682268257d946827897385945294682a682d2981947d9468308a11556bfc02404457ad8f4a07c98b89dc83007362756268317d94288c046c6f7373946836683c4308ee76bb0da3be9e4094869452948c0974696d657374616d70944a6e057b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303032948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b038c08747269616c5f6964948c083433626261333763948c0464617465948c13323032342d30362d32355f31392d35392d3130948c1074696d655f746869735f697465725f739447406432126c0000008c0c74696d655f746f74616c5f739447407d74a1f54000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b088c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944d00018c0a656e636f6465725f6666944d00048c0d6465636f6465725f72745f6666944d00048c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b018c0964726f705f7261746594473fd9d4b3fecb24d28c026c7294473f491b2a05f539228c0a62617463685f73697a65944d0004758c1274696d655f73696e63655f726573746f72659447406432126c0000008c18697465726174696f6e735f73696e63655f726573746f7265944b0175756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f34336262613337635f31395f62617463685f73697a653d313032342c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d313032342c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f5f323032342d30362d32355f31382d32382d35322f636865636b706f696e745f30303030303394682268257d946827897385945294682a682d2981947d9468308a115a68d41d442743a97541c3eb24d52f92007362756268317d94288c046c6f7373946836683c430889442231e7d29e4094869452948c0974696d657374616d70944a02067b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303033948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b048c08747269616c5f6964948c083433626261333763948c0464617465948c13323032342d30362d32355f32302d30312d3338948c1074696d655f746869735f697465725f7394474062711f758000008c0c74696d655f746f74616c5f73944740835698d80000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b088c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944d00018c0a656e636f6465725f6666944d00048c0d6465636f6465725f72745f6666944d00048c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b018c0964726f705f7261746594473fd9d4b3fecb24d28c026c7294473f491b2a05f539228c0a62617463685f73697a65944d0004758c1274696d655f73696e63655f726573746f7265944740735198f0c000008c18697465726174696f6e735f73696e63655f726573746f7265944b0275756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f34336262613337635f31395f62617463685f73697a653d313032342c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d313032342c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f5f323032342d30362d32355f31382d32382d35322f636865636b706f696e745f30303030303494682268257d946827897385945294682a682d2981947d9468308a1024dd1969cdf9cbb17b4e4994e75700317362756268317d94288c046c6f7373946836683c4308fe7ebf1f56b69e4094869452948c0974696d657374616d70944ac40c7b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303034948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b058c08747269616c5f6964948c083433626261333763948c0464617465948c13323032342d30362d32355f32302d33302d3238948c1074696d655f746869735f697465725f73944740641c5dea8000008c0c74696d655f746f74616c5f73944740885db052a000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b088c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944d00018c0a656e636f6465725f6666944d00048c0d6465636f6465725f72745f6666944d00048c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b018c0964726f705f7261746594473fd9d4b3fecb24d28c026c7294473f491b2a05f539228c0a62617463685f73697a65944d0004758c1274696d655f73696e63655f726573746f7265944740641c5dea8000008c18697465726174696f6e735f73696e63655f726573746f7265944b0175756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f34336262613337635f31395f62617463685f73697a653d313032342c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d313032342c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f5f323032342d30362d32355f31382d32382d35322f636865636b706f696e745f30303030303594682268257d946827897385945294682a682d2981947d9468308a101be42d7b0b232c97b2421babd3c09e087362756268317d94288c046c6f7373946836683c4308381c0e7758b49e4094869452948c0974696d657374616d70944a570d7b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303035948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b068c08747269616c5f6964948c083433626261333763948c0464617465948c13323032342d30362d32355f32302d33322d3535948c1074696d655f746869735f697465725f739447406260652d8000008c0c74696d655f746f74616c5f739447408cf5c99e0000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b088c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944d00018c0a656e636f6465725f6666944d00048c0d6465636f6465725f72745f6666944d00048c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b018c0964726f705f7261746594473fd9d4b3fecb24d28c026c7294473f491b2a05f539228c0a62617463685f73697a65944d0004758c1274696d655f73696e63655f726573746f7265944740733e618c0000008c18697465726174696f6e735f73696e63655f726573746f7265944b0275756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f34336262613337635f31395f62617463685f73697a653d313032342c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d313032342c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f5f323032342d30362d32355f31382d32382d35322f636865636b706f696e745f30303030303694682268257d946827897385945294682a682d2981947d9468308a1055798b272905eb9bf84a5c121809c9137362756268317d94288c046c6f7373946836683c430802814000afb59e4094869452948c0974696d657374616d70944aea0d7b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303036948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b078c08747269616c5f6964948c083433626261333763948c0464617465948c13323032342d30362d32355f32302d33352d3233948c1074696d655f746869735f697465725f739447406270088e8000008c0c74696d655f746f74616c5f7394474090c8e5e0d000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b088c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944d00018c0a656e636f6465725f6666944d00048c0d6465636f6465725f72745f6666944d00048c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b018c0964726f705f7261746594473fd9d4b3fecb24d28c026c7294473f491b2a05f539228c0a62617463685f73697a65944d0004758c1274696d655f73696e63655f726573746f72659447407c7665d34000008c18697465726174696f6e735f73696e63655f726573746f7265944b0375756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f34336262613337635f31395f62617463685f73697a653d313032342c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d313032342c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f5f323032342d30362d32355f31382d32382d35322f636865636b706f696e745f30303030303794682268257d946827897385945294682a682d2981947d9468308a11fbb129ac075b9a9a344ecce4395bca9d007362756268317d94288c046c6f7373946836683c4308cce572893fab9e4094869452948c0974696d657374616d70944a7e0e7b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303037948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b088c08747269616c5f6964948c083433626261333763948c0464617465948c13323032342d30362d32355f32302d33372d3530948c1074696d655f746869735f697465725f7394474062758e0e0000008c0c74696d655f746f74616c5f73944740931797a29000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b088c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944d00018c0a656e636f6465725f6666944d00048c0d6465636f6465725f72745f6666944d00048c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b018c0964726f705f7261746594473fd9d4b3fecb24d28c026c7294473f491b2a05f539228c0a62617463685f73697a65944d0004758c1274696d655f73696e63655f726573746f726594474082d8966d2000008c18697465726174696f6e735f73696e63655f726573746f7265944b04757562658c195f6c61746573745f636865636b706f696e745f726573756c74946a7c01000075622e\"\n }\n}"
+ ],
+ [
+ "{\n \"stub\": false,\n \"trainable_name\": \"train_model\",\n \"trial_id\": \"6c8b476a\",\n \"storage\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059572040000000000008c1b7261792e747261696e2e5f696e7465726e616c2e73746f72616765948c0e53746f72616765436f6e746578749493942981947d94288c12637573746f6d5f66735f70726f766964656494898c1273746f726167655f6c6f63616c5f70617468948c2b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c7473948c136578706572696d656e745f6469725f6e616d65948c0f746573745f6578706572696d656e74948c0e747269616c5f6469725f6e616d65948c96747261696e5f6d6f64656c5f36633862343736615f32305f62617463685f73697a653d313032342c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d3531322c6465636f6465725f72745f6e756d5f6c617965723d342c64726f705f725f323032342d30362d32355f31382d34332d3439948c1863757272656e745f636865636b706f696e745f696e646578944b038c0b73796e635f636f6e666967948c097261792e747261696e948c0a53796e63436f6e6669679493942981947d94288c0b73796e635f706572696f64944d2c018c0c73796e635f74696d656f7574944d08078c0e73796e635f61727469666163747394898c1c73796e635f6172746966616374735f6f6e5f636865636b706f696e7494888c0a75706c6f61645f646972948c0a44455052454341544544948c0673796e6365729468188c1273796e635f6f6e5f636865636b706f696e7494681875628c1273746f726167655f66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c0f73746f726167655f66735f70617468948c452f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f7465737494681968008c115f46696c6573797374656d53796e6365729493942981947d9428681b682268134d2c0168144d08078c116c6173745f73796e635f75705f74696d65944741d99ebcc08782328c136c6173745f73796e635f646f776e5f74696d659447fff00000000000008c0d5f73796e635f70726f63657373944e8c0c5f63757272656e745f636d649468008c125f75706c6f61645f746f5f66735f706174689493947d94288c0a6c6f63616c5f70617468948c3b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c74732f746573745f6578706572696d656e74948c0266739468228c0766735f70617468948c552f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e74948c076578636c756465945d948c0e2a2f636865636b706f696e745f2a9461758694756275622e\"\n },\n \"config\": {\n \"encoder_num_layer\": 8,\n \"decoder_rt_num_layer\": 4,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 64,\n \"encoder_ff\": 1024,\n \"decoder_rt_ff\": 512,\n \"decoder_int_ff\": 512,\n \"n_head\": 16,\n \"drop_rate\": 0.5088357248512263,\n \"lr\": 0.010524289583233754,\n \"batch_size\": 1024\n },\n \"_Trial__unresolved_config\": {\n \"encoder_num_layer\": 8,\n \"decoder_rt_num_layer\": 4,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 64,\n \"encoder_ff\": 1024,\n \"decoder_rt_ff\": 512,\n \"decoder_int_ff\": 512,\n \"n_head\": 16,\n \"drop_rate\": 0.5088357248512263,\n \"lr\": 0.010524289583233754,\n \"batch_size\": 1024\n },\n \"evaluated_params\": {\n \"batch_size\": 1024,\n \"decoder_int_ff\": 512,\n \"decoder_int_num_layer\": 1,\n \"decoder_rt_ff\": 512,\n \"decoder_rt_num_layer\": 4,\n \"drop_rate\": 0.5088357248512263,\n \"embedding_dim\": 64,\n \"encoder_ff\": 1024,\n \"encoder_num_layer\": 8,\n \"lr\": 0.010524289583233754,\n \"n_head\": 16\n },\n \"experiment_tag\": \"20_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=4,drop_rate=0.5088,embedding_dim=64,encoder_ff=1024,encoder_num_layer=8,lr=0.0105,n_head=16\",\n \"stopping_criterion\": {},\n \"_setup_default_resource\": true,\n \"_default_placement_group_factory\": \"80054e2e\",\n \"placement_group_factory\": \"800595ba000000000000008c237261792e74756e652e657865637574696f6e2e706c6163656d656e745f67726f757073948c15506c6163656d656e7447726f7570466163746f72799493942981947d94288c085f62756e646c6573945d947d94288c03435055944740540000000000008c034750559447401000000000000075618c155f686561645f62756e646c655f69735f656d70747994898c095f7374726174656779948c045041434b948c055f6172677394298c075f6b7761726773947d9475622e\",\n \"log_to_file\": [\n null,\n null\n ],\n \"max_failures\": 0,\n \"_default_result_or_future\": null,\n \"export_formats\": [],\n \"status\": \"PAUSED\",\n \"relative_logdir\": \"train_model_6c8b476a_20_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=4,drop_r_2024-06-25_18-43-49\",\n \"trial_name_creator\": null,\n \"trial_dirname_creator\": null,\n \"custom_trial_name\": null,\n \"custom_dirname\": null,\n \"restore_path\": null,\n \"_restore_checkpoint_result\": null,\n \"_state_json\": null,\n \"results\": \"80054e2e\",\n \"extra_arg\": \"80054e2e\",\n \"_resources\": \"80054e2e\"\n}",
+ "{\n \"start_time\": 1719334139.5021868,\n \"num_failures\": 0,\n \"num_failures_after_restore\": 0,\n \"error_filename\": null,\n \"pickled_error_filename\": null,\n \"last_result\": {\n \"loss\": 1991.5076505405696,\n \"timestamp\": 1719338738,\n \"checkpoint_dir_name\": \"checkpoint_000003\",\n \"should_checkpoint\": true,\n \"done\": false,\n \"training_iteration\": 4,\n \"trial_id\": \"6c8b476a\",\n \"date\": \"2024-06-25_20-05-38\",\n \"time_this_iter_s\": 113.23560690879822,\n \"time_total_s\": 480.9753620624542,\n \"pid\": 2195253,\n \"hostname\": \"r8i6n2\",\n \"node_ip\": \"10.159.28.60\",\n \"config\": {\n \"encoder_num_layer\": 8,\n \"decoder_rt_num_layer\": 4,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 64,\n \"encoder_ff\": 1024,\n \"decoder_rt_ff\": 512,\n \"decoder_int_ff\": 512,\n \"n_head\": 16,\n \"drop_rate\": 0.5088357248512263,\n \"lr\": 0.010524289583233754,\n \"batch_size\": 1024\n },\n \"time_since_restore\": 239.98935389518738,\n \"iterations_since_restore\": 2,\n \"experiment_tag\": \"20_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=4,drop_rate=0.5088,embedding_dim=64,encoder_ff=1024,encoder_num_layer=8,lr=0.0105,n_head=16\"\n },\n \"last_result_time\": 1719338738.6413836,\n \"metric_analysis\": {\n \"loss\": {\n \"max\": 2006.8072533795214,\n \"min\": 1991.5076505405696,\n \"avg\": 1997.4594356506825,\n \"last\": 1991.5076505405696,\n \"last-5-avg\": 1997.4594356506827,\n \"last-10-avg\": 1997.4594356506827\n },\n \"should_checkpoint\": {\n \"max\": true,\n \"min\": true,\n \"avg\": 1.0,\n \"last\": true,\n \"last-5-avg\": 1.0,\n \"last-10-avg\": 1.0\n },\n \"done\": {\n \"max\": false,\n \"min\": false,\n \"avg\": 0.0,\n \"last\": false,\n \"last-5-avg\": 0.0,\n \"last-10-avg\": 0.0\n },\n \"training_iteration\": {\n \"max\": 4,\n \"min\": 1,\n \"avg\": 2.5,\n \"last\": 4,\n \"last-5-avg\": 2.5,\n \"last-10-avg\": 2.5\n },\n \"time_this_iter_s\": {\n \"max\": 127.51717782020569,\n \"min\": 113.23560690879822,\n \"avg\": 120.24384051561356,\n \"last\": 113.23560690879822,\n \"last-5-avg\": 120.24384051561356,\n \"last-10-avg\": 120.24384051561356\n },\n \"time_total_s\": {\n \"max\": 480.9753620624542,\n \"min\": 127.51717782020569,\n \"avg\": 304.3045758008957,\n \"last\": 480.9753620624542,\n \"last-5-avg\": 304.3045758008957,\n \"last-10-avg\": 304.3045758008957\n },\n \"time_since_restore\": {\n \"max\": 240.98600816726685,\n \"min\": 126.75374698638916,\n \"avg\": 183.81157171726227,\n \"last\": 239.98935389518738,\n \"last-5-avg\": 183.81157171726227,\n \"last-10-avg\": 183.81157171726227\n },\n \"iterations_since_restore\": {\n \"max\": 2,\n \"min\": 1,\n \"avg\": 1.5,\n \"last\": 2,\n \"last-5-avg\": 1.5,\n \"last-10-avg\": 1.5\n }\n },\n \"_n_steps\": [\n 5,\n 10\n ],\n \"metric_n_steps\": {\n \"loss\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"800595c3000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294288c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b0074946243088542a1a03a5b9f4094869452946807680d430828140a75fc2c9f4094869452946807680d4308f0f7fbed1a319f4094869452946807680d43082c168bd5071e9f409486945294652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"800595c3000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294288c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b0074946243088542a1a03a5b9f4094869452946807680d430828140a75fc2c9f4094869452946807680d4308f0f7fbed1a319f4094869452946807680d43082c168bd5071e9f409486945294652e\"\n }\n },\n \"should_checkpoint\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059526000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452942888888888652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059526000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452942888888888652e\"\n }\n },\n \"done\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059526000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452942889898989652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059526000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452942889898989652e\"\n }\n },\n \"training_iteration\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294284b014b024b034b04652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294284b014b024b034b04652e\"\n }\n },\n \"time_this_iter_s\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059546000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452942847405fe1197100000047405c5e015100000047405fb03d6400000047405c4f142f000000652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059546000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452942847405fe1197100000047405c5e015100000047405fb03d6400000047405c4f142f000000652e\"\n }\n },\n \"time_total_s\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059546000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452942847405fe1197100000047406e1f8d61000000474076fbd60980000047407e0f9b15400000652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059546000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452942847405fe1197100000047406e1f8d61000000474076fbd60980000047407e0f9b15400000652e\"\n }\n },\n \"time_since_restore\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059546000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452942847405fe1197100000047406e1f8d6100000047405fb03d6400000047406dffa8c9800000652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059546000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452942847405fe1197100000047406e1f8d6100000047405fb03d6400000047406dffa8c9800000652e\"\n }\n },\n \"iterations_since_restore\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294284b014b024b014b02652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952a000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294284b014b024b014b02652e\"\n }\n }\n },\n \"checkpoint_manager\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"800595be100000000000008c267261792e747261696e2e5f696e7465726e616c2e636865636b706f696e745f6d616e61676572948c125f436865636b706f696e744d616e616765729493942981947d94288c125f636865636b706f696e745f636f6e666967948c097261792e747261696e948c10436865636b706f696e74436f6e6669679493942981947d94288c0b6e756d5f746f5f6b656570944e8c1a636865636b706f696e745f73636f72655f617474726962757465944e8c16636865636b706f696e745f73636f72655f6f72646572948c036d6178948c14636865636b706f696e745f6672657175656e6379944b008c11636865636b706f696e745f61745f656e6494898c1a5f636865636b706f696e745f6b6565705f616c6c5f72616e6b73948c0a44455052454341544544948c1f5f636865636b706f696e745f75706c6f61645f66726f6d5f776f726b65727394681275628c135f636865636b706f696e745f726573756c7473945d94288c1b7261792e747261696e2e5f696e7465726e616c2e73657373696f6e948c0f5f547261696e696e67526573756c749493942981947d94288c0a636865636b706f696e749468068c0a436865636b706f696e749493942981947d94288c0470617468948cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f36633862343736615f32305f62617463685f73697a653d313032342c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d3531322c6465636f6465725f72745f6e756d5f6c617965723d342c64726f705f725f323032342d30362d32355f31382d34332d34392f636865636b706f696e745f303030303030948c0a66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c055f75756964948c0475756964948c04555549449493942981947d948c03696e74948a11803e749eadb9749b79460273979bdcab00736275628c076d657472696373947d94288c046c6f7373948c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b0074946243088542a1a03a5b9f4094869452948c0974696d657374616d70944a7bf57a668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303030948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b018c08747269616c5f6964948c083663386234373661948c0464617465948c13323032342d30362d32355f31382d35312d3037948c1074696d655f746869735f697465725f739447405fe119710000008c0c74696d655f746f74616c5f739447405fe119710000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b088c146465636f6465725f72745f6e756d5f6c61796572944b048c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b408c0a656e636f6465725f6666944d00048c0d6465636f6465725f72745f6666944d00028c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b108c0964726f705f7261746594473fe04861dba8b8168c026c7294473f858dc23c96c75d8c0a62617463685f73697a65944d0004758c1274696d655f73696e63655f726573746f72659447405fe119710000008c18697465726174696f6e735f73696e63655f726573746f7265944b0175756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f36633862343736615f32305f62617463685f73697a653d313032342c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d3531322c6465636f6465725f72745f6e756d5f6c617965723d342c64726f705f725f323032342d30362d32355f31382d34332d34392f636865636b706f696e745f30303030303194682268257d946827897385945294682a682d2981947d9468308a11fccc6197e14f14be294ffea124b8adb9007362756268317d94288c046c6f7373946836683c430828140a75fc2c9f4094869452948c0974696d657374616d70944aecf57a668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303031948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b028c08747269616c5f6964948c083663386234373661948c0464617465948c13323032342d30362d32355f31382d35332d3030948c1074696d655f746869735f697465725f739447405c5e01510000008c0c74696d655f746f74616c5f739447406e1f8d610000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b088c146465636f6465725f72745f6e756d5f6c61796572944b048c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b408c0a656e636f6465725f6666944d00048c0d6465636f6465725f72745f6666944d00028c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b108c0964726f705f7261746594473fe04861dba8b8168c026c7294473f858dc23c96c75d8c0a62617463685f73697a65944d0004758c1274696d655f73696e63655f726573746f72659447406e1f8d610000008c18697465726174696f6e735f73696e63655f726573746f7265944b0275756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f36633862343736615f32305f62617463685f73697a653d313032342c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d3531322c6465636f6465725f72745f6e756d5f6c617965723d342c64726f705f725f323032342d30362d32355f31382d34332d34392f636865636b706f696e745f30303030303294682268257d946827897385945294682a682d2981947d9468308a105ca15e0bab76d6a61c457fae0a0690687362756268317d94288c046c6f7373946836683c4308f0f7fbed1a319f4094869452948c0974696d657374616d70944a81067b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303032948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b038c08747269616c5f6964948c083663386234373661948c0464617465948c13323032342d30362d32355f32302d30332d3435948c1074696d655f746869735f697465725f739447405fb03d640000008c0c74696d655f746f74616c5f7394474076fbd6098000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b088c146465636f6465725f72745f6e756d5f6c61796572944b048c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b408c0a656e636f6465725f6666944d00048c0d6465636f6465725f72745f6666944d00028c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b108c0964726f705f7261746594473fe04861dba8b8168c026c7294473f858dc23c96c75d8c0a62617463685f73697a65944d0004758c1274696d655f73696e63655f726573746f72659447405fb03d640000008c18697465726174696f6e735f73696e63655f726573746f7265944b0175756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f36633862343736615f32305f62617463685f73697a653d313032342c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d3531322c6465636f6465725f72745f6e756d5f6c617965723d342c64726f705f725f323032342d30362d32355f31382d34332d34392f636865636b706f696e745f30303030303394682268257d946827897385945294682a682d2981947d9468308a10085a31f812052289c447ecf15e9e87147362756268317d94288c046c6f7373946836683c43082c168bd5071e9f4094869452948c0974696d657374616d70944af2067b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303033948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b048c08747269616c5f6964948c083663386234373661948c0464617465948c13323032342d30362d32355f32302d30352d3338948c1074696d655f746869735f697465725f739447405c4f142f0000008c0c74696d655f746f74616c5f739447407e0f9b154000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b088c146465636f6465725f72745f6e756d5f6c61796572944b048c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b408c0a656e636f6465725f6666944d00048c0d6465636f6465725f72745f6666944d00028c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b108c0964726f705f7261746594473fe04861dba8b8168c026c7294473f858dc23c96c75d8c0a62617463685f73697a65944d0004758c1274696d655f73696e63655f726573746f72659447406dffa8c98000008c18697465726174696f6e735f73696e63655f726573746f7265944b02757562658c195f6c61746573745f636865636b706f696e745f726573756c749468c075622e\"\n }\n}"
+ ],
+ [
+ "{\n \"stub\": false,\n \"trainable_name\": \"train_model\",\n \"trial_id\": \"6743cdee\",\n \"storage\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059572040000000000008c1b7261792e747261696e2e5f696e7465726e616c2e73746f72616765948c0e53746f72616765436f6e746578749493942981947d94288c12637573746f6d5f66735f70726f766964656494898c1273746f726167655f6c6f63616c5f70617468948c2b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c7473948c136578706572696d656e745f6469725f6e616d65948c0f746573745f6578706572696d656e74948c0e747269616c5f6469725f6e616d65948c96747261696e5f6d6f64656c5f36373433636465655f32315f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d323034382c6465636f6465725f72745f6e756d5f6c617965723d342c64726f705f5f323032342d30362d32355f31382d34382d3539948c1863757272656e745f636865636b706f696e745f696e646578944b018c0b73796e635f636f6e666967948c097261792e747261696e948c0a53796e63436f6e6669679493942981947d94288c0b73796e635f706572696f64944d2c018c0c73796e635f74696d656f7574944d08078c0e73796e635f61727469666163747394898c1c73796e635f6172746966616374735f6f6e5f636865636b706f696e7494888c0a75706c6f61645f646972948c0a44455052454341544544948c0673796e6365729468188c1273796e635f6f6e5f636865636b706f696e7494681875628c1273746f726167655f66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c0f73746f726167655f66735f70617468948c452f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f7465737494681968008c115f46696c6573797374656d53796e6365729493942981947d9428681b682268134d2c0168144d08078c116c6173745f73796e635f75705f74696d65944741d99ebd0beabbc18c136c6173745f73796e635f646f776e5f74696d659447fff00000000000008c0d5f73796e635f70726f63657373944e8c0c5f63757272656e745f636d649468008c125f75706c6f61645f746f5f66735f706174689493947d94288c0a6c6f63616c5f70617468948c3b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c74732f746573745f6578706572696d656e74948c0266739468228c0766735f70617468948c552f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e74948c076578636c756465945d948c0e2a2f636865636b706f696e745f2a9461758694756275622e\"\n },\n \"config\": {\n \"encoder_num_layer\": 2,\n \"decoder_rt_num_layer\": 4,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 256,\n \"encoder_ff\": 512,\n \"decoder_rt_ff\": 2048,\n \"decoder_int_ff\": 512,\n \"n_head\": 8,\n \"drop_rate\": 0.4436137502039823,\n \"lr\": 0.07179226263764402,\n \"batch_size\": 2048\n },\n \"_Trial__unresolved_config\": {\n \"encoder_num_layer\": 2,\n \"decoder_rt_num_layer\": 4,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 256,\n \"encoder_ff\": 512,\n \"decoder_rt_ff\": 2048,\n \"decoder_int_ff\": 512,\n \"n_head\": 8,\n \"drop_rate\": 0.4436137502039823,\n \"lr\": 0.07179226263764402,\n \"batch_size\": 2048\n },\n \"evaluated_params\": {\n \"batch_size\": 2048,\n \"decoder_int_ff\": 512,\n \"decoder_int_num_layer\": 1,\n \"decoder_rt_ff\": 2048,\n \"decoder_rt_num_layer\": 4,\n \"drop_rate\": 0.4436137502039823,\n \"embedding_dim\": 256,\n \"encoder_ff\": 512,\n \"encoder_num_layer\": 2,\n \"lr\": 0.07179226263764402,\n \"n_head\": 8\n },\n \"experiment_tag\": \"21_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=4,drop_rate=0.4436,embedding_dim=256,encoder_ff=512,encoder_num_layer=2,lr=0.0718,n_head=8\",\n \"stopping_criterion\": {},\n \"_setup_default_resource\": true,\n \"_default_placement_group_factory\": \"80054e2e\",\n \"placement_group_factory\": \"800595ba000000000000008c237261792e74756e652e657865637574696f6e2e706c6163656d656e745f67726f757073948c15506c6163656d656e7447726f7570466163746f72799493942981947d94288c085f62756e646c6573945d947d94288c03435055944740540000000000008c034750559447401000000000000075618c155f686561645f62756e646c655f69735f656d70747994898c095f7374726174656779948c045041434b948c055f6172677394298c075f6b7761726773947d9475622e\",\n \"log_to_file\": [\n null,\n null\n ],\n \"max_failures\": 0,\n \"_default_result_or_future\": null,\n \"export_formats\": [],\n \"status\": \"PAUSED\",\n \"relative_logdir\": \"train_model_6743cdee_21_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=4,drop__2024-06-25_18-48-59\",\n \"trial_name_creator\": null,\n \"trial_dirname_creator\": null,\n \"custom_trial_name\": null,\n \"custom_dirname\": null,\n \"restore_path\": null,\n \"_restore_checkpoint_result\": null,\n \"_state_json\": null,\n \"results\": \"80054e2e\",\n \"extra_arg\": \"80054e2e\",\n \"_resources\": \"80054e2e\"\n}",
+ "{\n \"start_time\": 1719334380.648466,\n \"num_failures\": 0,\n \"num_failures_after_restore\": 0,\n \"error_filename\": null,\n \"pickled_error_filename\": null,\n \"last_result\": {\n \"loss\": 2680.254179226132,\n \"timestamp\": 1719334648,\n \"checkpoint_dir_name\": \"checkpoint_000001\",\n \"should_checkpoint\": true,\n \"done\": false,\n \"training_iteration\": 2,\n \"trial_id\": \"6743cdee\",\n \"date\": \"2024-06-25_18-57-29\",\n \"time_this_iter_s\": 127.1407482624054,\n \"time_total_s\": 268.34629464149475,\n \"pid\": 2195253,\n \"hostname\": \"r8i6n2\",\n \"node_ip\": \"10.159.28.60\",\n \"config\": {\n \"encoder_num_layer\": 2,\n \"decoder_rt_num_layer\": 4,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 256,\n \"encoder_ff\": 512,\n \"decoder_rt_ff\": 2048,\n \"decoder_int_ff\": 512,\n \"n_head\": 8,\n \"drop_rate\": 0.4436137502039823,\n \"lr\": 0.07179226263764402,\n \"batch_size\": 2048\n },\n \"time_since_restore\": 268.34629464149475,\n \"iterations_since_restore\": 2,\n \"experiment_tag\": \"21_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=4,drop_rate=0.4436,embedding_dim=256,encoder_ff=512,encoder_num_layer=2,lr=0.0718,n_head=8\"\n },\n \"last_result_time\": 1719334649.0042667,\n \"metric_analysis\": {\n \"loss\": {\n \"max\": 3811.964426596334,\n \"min\": 2680.254179226132,\n \"avg\": 3246.1093029112326,\n \"last\": 2680.254179226132,\n \"last-5-avg\": 3246.1093029112326,\n \"last-10-avg\": 3246.1093029112326\n },\n \"should_checkpoint\": {\n \"max\": true,\n \"min\": true,\n \"avg\": 1.0,\n \"last\": true,\n \"last-5-avg\": 1.0,\n \"last-10-avg\": 1.0\n },\n \"done\": {\n \"max\": false,\n \"min\": false,\n \"avg\": 0.0,\n \"last\": false,\n \"last-5-avg\": 0.0,\n \"last-10-avg\": 0.0\n },\n \"training_iteration\": {\n \"max\": 2,\n \"min\": 1,\n \"avg\": 1.5,\n \"last\": 2,\n \"last-5-avg\": 1.5,\n \"last-10-avg\": 1.5\n },\n \"time_this_iter_s\": {\n \"max\": 141.20554637908936,\n \"min\": 127.1407482624054,\n \"avg\": 134.17314732074738,\n \"last\": 127.1407482624054,\n \"last-5-avg\": 134.17314732074738,\n \"last-10-avg\": 134.17314732074738\n },\n \"time_total_s\": {\n \"max\": 268.34629464149475,\n \"min\": 141.20554637908936,\n \"avg\": 204.77592051029205,\n \"last\": 268.34629464149475,\n \"last-5-avg\": 204.77592051029205,\n \"last-10-avg\": 204.77592051029205\n },\n \"time_since_restore\": {\n \"max\": 268.34629464149475,\n \"min\": 141.20554637908936,\n \"avg\": 204.77592051029205,\n \"last\": 268.34629464149475,\n \"last-5-avg\": 204.77592051029205,\n \"last-10-avg\": 204.77592051029205\n },\n \"iterations_since_restore\": {\n \"max\": 2,\n \"min\": 1,\n \"avg\": 1.5,\n \"last\": 2,\n \"last-5-avg\": 1.5,\n \"last-10-avg\": 1.5\n }\n },\n \"_n_steps\": [\n 5,\n 10\n ],\n \"metric_n_steps\": {\n \"loss\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005959d000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294288c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b0074946243084ba552c9edc7ad4094869452946807680d43081e8fc72382f0a4409486945294652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005959d000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294288c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b0074946243084ba552c9edc7ad4094869452946807680d43081e8fc72382f0a4409486945294652e\"\n }\n },\n \"should_checkpoint\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059524000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294288888652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059524000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294288888652e\"\n }\n },\n \"done\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059524000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294288989652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059524000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294288989652e\"\n }\n },\n \"training_iteration\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059526000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294284b014b02652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059526000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294284b014b02652e\"\n }\n },\n \"time_this_iter_s\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059534000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b058694529428474061a693d600000047405fc90205000000652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059534000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a8694529428474061a693d600000047405fc90205000000652e\"\n }\n },\n \"time_total_s\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059534000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b058694529428474061a693d6000000474070c58a6c400000652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059534000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a8694529428474061a693d6000000474070c58a6c400000652e\"\n }\n },\n \"time_since_restore\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059534000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b058694529428474061a693d6000000474070c58a6c400000652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059534000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a8694529428474061a693d6000000474070c58a6c400000652e\"\n }\n },\n \"iterations_since_restore\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059526000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294284b014b02652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059526000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294284b014b02652e\"\n }\n }\n },\n \"checkpoint_manager\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"800595a6090000000000008c267261792e747261696e2e5f696e7465726e616c2e636865636b706f696e745f6d616e61676572948c125f436865636b706f696e744d616e616765729493942981947d94288c125f636865636b706f696e745f636f6e666967948c097261792e747261696e948c10436865636b706f696e74436f6e6669679493942981947d94288c0b6e756d5f746f5f6b656570944e8c1a636865636b706f696e745f73636f72655f617474726962757465944e8c16636865636b706f696e745f73636f72655f6f72646572948c036d6178948c14636865636b706f696e745f6672657175656e6379944b008c11636865636b706f696e745f61745f656e6494898c1a5f636865636b706f696e745f6b6565705f616c6c5f72616e6b73948c0a44455052454341544544948c1f5f636865636b706f696e745f75706c6f61645f66726f6d5f776f726b65727394681275628c135f636865636b706f696e745f726573756c7473945d94288c1b7261792e747261696e2e5f696e7465726e616c2e73657373696f6e948c0f5f547261696e696e67526573756c749493942981947d94288c0a636865636b706f696e749468068c0a436865636b706f696e749493942981947d94288c0470617468948cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f36373433636465655f32315f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d323034382c6465636f6465725f72745f6e756d5f6c617965723d342c64726f705f5f323032342d30362d32355f31382d34382d35392f636865636b706f696e745f303030303030948c0a66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c055f75756964948c0475756964948c04555549449493942981947d948c03696e74948a1030324741ea99cb8426455c335ab8d75d736275628c076d657472696373947d94288c046c6f7373948c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b0074946243084ba552c9edc7ad4094869452948c0974696d657374616d70944a79f67a668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303030948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b018c08747269616c5f6964948c083637343363646565948c0464617465948c13323032342d30362d32355f31382d35352d3231948c1074696d655f746869735f697465725f7394474061a693d60000008c0c74696d655f746f74616c5f7394474061a693d60000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b028c146465636f6465725f72745f6e756d5f6c61796572944b048c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944d00018c0a656e636f6465725f6666944d00028c0d6465636f6465725f72745f6666944d00088c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b088c0964726f705f7261746594473fdc642aed4ba62c8c026c7294473fb260fa4c2270268c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f726594474061a693d60000008c18697465726174696f6e735f73696e63655f726573746f7265944b0175756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f36373433636465655f32315f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d323034382c6465636f6465725f72745f6e756d5f6c617965723d342c64726f705f5f323032342d30362d32355f31382d34382d35392f636865636b706f696e745f30303030303194682268257d946827897385945294682a682d2981947d9468308a10abdc9a77d96eacb24f45169ee215a2007362756268317d94288c046c6f7373946836683c43081e8fc72382f0a44094869452948c0974696d657374616d70944af8f67a668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303031948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b028c08747269616c5f6964948c083637343363646565948c0464617465948c13323032342d30362d32355f31382d35372d3239948c1074696d655f746869735f697465725f739447405fc902050000008c0c74696d655f746f74616c5f7394474070c58a6c4000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b028c146465636f6465725f72745f6e756d5f6c61796572944b048c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944d00018c0a656e636f6465725f6666944d00028c0d6465636f6465725f72745f6666944d00088c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b088c0964726f705f7261746594473fdc642aed4ba62c8c026c7294473fb260fa4c2270268c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f726594474070c58a6c4000008c18697465726174696f6e735f73696e63655f726573746f7265944b02757562658c195f6c61746573745f636865636b706f696e745f726573756c7494686275622e\"\n }\n}"
+ ],
+ [
+ "{\n \"stub\": false,\n \"trainable_name\": \"train_model\",\n \"trial_id\": \"0f9cc97d\",\n \"storage\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059572040000000000008c1b7261792e747261696e2e5f696e7465726e616c2e73746f72616765948c0e53746f72616765436f6e746578749493942981947d94288c12637573746f6d5f66735f70726f766964656494898c1273746f726167655f6c6f63616c5f70617468948c2b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c7473948c136578706572696d656e745f6469725f6e616d65948c0f746573745f6578706572696d656e74948c0e747269616c5f6469725f6e616d65948c96747261696e5f6d6f64656c5f30663963633937645f32325f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d3531322c6465636f6465725f72745f6e756d5f6c617965723d322c64726f705f725f323032342d30362d32355f31382d35332d3030948c1863757272656e745f636865636b706f696e745f696e646578944b0d8c0b73796e635f636f6e666967948c097261792e747261696e948c0a53796e63436f6e6669679493942981947d94288c0b73796e635f706572696f64944d2c018c0c73796e635f74696d656f7574944d08078c0e73796e635f61727469666163747394898c1c73796e635f6172746966616374735f6f6e5f636865636b706f696e7494888c0a75706c6f61645f646972948c0a44455052454341544544948c0673796e6365729468188c1273796e635f6f6e5f636865636b706f696e7494681875628c1273746f726167655f66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c0f73746f726167655f66735f70617468948c452f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f7465737494681968008c115f46696c6573797374656d53796e6365729493942981947d9428681b682268134d2c0168144d08078c116c6173745f73796e635f75705f74696d65944741d99ebd577452378c136c6173745f73796e635f646f776e5f74696d659447fff00000000000008c0d5f73796e635f70726f63657373944e8c0c5f63757272656e745f636d649468008c125f75706c6f61645f746f5f66735f706174689493947d94288c0a6c6f63616c5f70617468948c3b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c74732f746573745f6578706572696d656e74948c0266739468228c0766735f70617468948c552f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e74948c076578636c756465945d948c0e2a2f636865636b706f696e745f2a9461758694756275622e\"\n },\n \"config\": {\n \"encoder_num_layer\": 4,\n \"decoder_rt_num_layer\": 2,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 1024,\n \"encoder_ff\": 1024,\n \"decoder_rt_ff\": 512,\n \"decoder_int_ff\": 512,\n \"n_head\": 1,\n \"drop_rate\": 0.005508735010543275,\n \"lr\": 0.0011880094129501908,\n \"batch_size\": 2048\n },\n \"_Trial__unresolved_config\": {\n \"encoder_num_layer\": 4,\n \"decoder_rt_num_layer\": 2,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 1024,\n \"encoder_ff\": 1024,\n \"decoder_rt_ff\": 512,\n \"decoder_int_ff\": 512,\n \"n_head\": 1,\n \"drop_rate\": 0.005508735010543275,\n \"lr\": 0.0011880094129501908,\n \"batch_size\": 2048\n },\n \"evaluated_params\": {\n \"batch_size\": 2048,\n \"decoder_int_ff\": 512,\n \"decoder_int_num_layer\": 1,\n \"decoder_rt_ff\": 512,\n \"decoder_rt_num_layer\": 2,\n \"drop_rate\": 0.005508735010543275,\n \"embedding_dim\": 1024,\n \"encoder_ff\": 1024,\n \"encoder_num_layer\": 4,\n \"lr\": 0.0011880094129501908,\n \"n_head\": 1\n },\n \"experiment_tag\": \"22_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_rate=0.0055,embedding_dim=1024,encoder_ff=1024,encoder_num_layer=4,lr=0.0012,n_head=1\",\n \"stopping_criterion\": {},\n \"_setup_default_resource\": true,\n \"_default_placement_group_factory\": \"80054e2e\",\n \"placement_group_factory\": \"800595ba000000000000008c237261792e74756e652e657865637574696f6e2e706c6163656d656e745f67726f757073948c15506c6163656d656e7447726f7570466163746f72799493942981947d94288c085f62756e646c6573945d947d94288c03435055944740540000000000008c034750559447401000000000000075618c155f686561645f62756e646c655f69735f656d70747994898c095f7374726174656779948c045041434b948c055f6172677394298c075f6b7761726773947d9475622e\",\n \"log_to_file\": [\n null,\n null\n ],\n \"max_failures\": 0,\n \"_default_result_or_future\": null,\n \"export_formats\": [],\n \"status\": \"TERMINATED\",\n \"relative_logdir\": \"train_model_0f9cc97d_22_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_18-53-00\",\n \"trial_name_creator\": null,\n \"trial_dirname_creator\": null,\n \"custom_trial_name\": null,\n \"custom_dirname\": null,\n \"restore_path\": null,\n \"_restore_checkpoint_result\": null,\n \"_state_json\": null,\n \"results\": \"80054e2e\",\n \"extra_arg\": \"80054e2e\",\n \"_resources\": \"80054e2e\"\n}",
+ "{\n \"start_time\": 1719334649.1537328,\n \"num_failures\": 0,\n \"num_failures_after_restore\": 0,\n \"error_filename\": null,\n \"pickled_error_filename\": null,\n \"last_result\": {\n \"loss\": 1956.4829832062007,\n \"timestamp\": 1719345747,\n \"checkpoint_dir_name\": \"checkpoint_000013\",\n \"should_checkpoint\": true,\n \"done\": true,\n \"training_iteration\": 14,\n \"trial_id\": \"0f9cc97d\",\n \"date\": \"2024-06-25_22-02-27\",\n \"time_this_iter_s\": 506.25228786468506,\n \"time_total_s\": 7130.941356182098,\n \"pid\": 2195253,\n \"hostname\": \"r8i6n2\",\n \"node_ip\": \"10.159.28.60\",\n \"config\": {\n \"encoder_num_layer\": 4,\n \"decoder_rt_num_layer\": 2,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 1024,\n \"encoder_ff\": 1024,\n \"decoder_rt_ff\": 512,\n \"decoder_int_ff\": 512,\n \"n_head\": 1,\n \"drop_rate\": 0.005508735010543275,\n \"lr\": 0.0011880094129501908,\n \"batch_size\": 2048\n },\n \"time_since_restore\": 5076.989426612854,\n \"iterations_since_restore\": 10,\n \"experiment_tag\": \"22_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_rate=0.0055,embedding_dim=1024,encoder_ff=1024,encoder_num_layer=4,lr=0.0012,n_head=1\"\n },\n \"last_result_time\": 1719345747.9423904,\n \"metric_analysis\": {\n \"loss\": {\n \"max\": 1970.2912184347317,\n \"min\": 1951.9356997031866,\n \"avg\": 1956.8511505641873,\n \"last\": 1956.4829832062007,\n \"last-5-avg\": 1954.9646307517226,\n \"last-10-avg\": 1954.5448684331939\n },\n \"should_checkpoint\": {\n \"max\": true,\n \"min\": true,\n \"avg\": 1.0,\n \"last\": true,\n \"last-5-avg\": 1.0,\n \"last-10-avg\": 1.0\n },\n \"done\": {\n \"max\": false,\n \"min\": false,\n \"avg\": 0.0,\n \"last\": false,\n \"last-5-avg\": 0.0,\n \"last-10-avg\": 0.0\n },\n \"training_iteration\": {\n \"max\": 14,\n \"min\": 1,\n \"avg\": 7.5,\n \"last\": 14,\n \"last-5-avg\": 12.0,\n \"last-10-avg\": 9.5\n },\n \"time_this_iter_s\": {\n \"max\": 522.4360599517822,\n \"min\": 505.6786599159241,\n \"avg\": 509.352954013007,\n \"last\": 506.25228786468506,\n \"last-5-avg\": 506.09470949172976,\n \"last-10-avg\": 507.6989426612854\n },\n \"time_total_s\": {\n \"max\": 7130.941356182098,\n \"min\": 519.6532871723175,\n \"avg\": 3834.4753139189315,\n \"last\": 7130.941356182098,\n \"last-5-avg\": 6118.492047595978,\n \"last-10-avg\": 4853.56500992775\n },\n \"time_since_restore\": {\n \"max\": 5076.989426612854,\n \"min\": 519.6532871723175,\n \"avg\": 2220.8551683255605,\n \"last\": 5076.989426612854,\n \"last-5-avg\": 4064.5401180267336,\n \"last-10-avg\": 2799.613080358505\n },\n \"iterations_since_restore\": {\n \"max\": 10,\n \"min\": 1,\n \"avg\": 4.357142857142857,\n \"last\": 10,\n \"last-5-avg\": 8.0,\n \"last-10-avg\": 5.5\n }\n },\n \"_n_steps\": [\n 5,\n 10\n ],\n \"metric_n_steps\": {\n \"loss\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"800595d6000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294288c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b0074946243089fcfe773fa859e4094869452946807680d43088fc7e3911a849e4094869452946807680d4308f8fbfd3e21919e4094869452946807680d430887c3e110268e9e4094869452946807680d4308994c2693ee919e409486945294652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059535010000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294288c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b00749462430841201028be7f9e4094869452946807680d43083a9d4e27ca839e4094869452946807680d4308bfdfefd79d8b9e4094869452946807680d43089bcd6613ce8b9e4094869452946807680d430883c160508c8f9e4094869452946807680d43089fcfe773fa859e4094869452946807680d43088fc7e3911a849e4094869452946807680d4308f8fbfd3e21919e4094869452946807680d430887c3e110268e9e4094869452946807680d4308994c2693ee919e409486945294652e\"\n }\n },\n \"should_checkpoint\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059527000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294288888888888652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952c000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452942888888888888888888888652e\"\n }\n },\n \"done\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059527000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294288989898989652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952c000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452942889898989898989898989652e\"\n }\n },\n \"training_iteration\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952c000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294284b0a4b0b4b0c4b0d4b0e652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059536000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294284b054b064b074b084b094b0a4b0b4b0c4b0d4b0e652e\"\n }\n },\n \"time_this_iter_s\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005954f000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452942847407f9d4d0480000047407f9bdb4600000047407fa734e200000047407fa32d1b00000047407fa4095f000000652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005957c000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a8694529428474080537d0d00000047407fa48b5a40000047407fa336da80000047407f9ea8f140000047407f9adbca80000047407f9d4d0480000047407f9bdb4600000047407fa734e200000047407fa32d1b00000047407fa4095f000000652e\"\n }\n },\n \"time_total_s\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005954f000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294284740b3f24c929800004740b5ec0a46f800004740b7e67d951800004740b9e0b066c800004740bbdaf0fcb80000652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005957c000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294284740a420c6a69000004740a8155811d800004740ac09beed2800004740affd940b5000004740b1f877c25000004740b3f24c929800004740b5ec0a46f800004740b7e67d951800004740b9e0b066c800004740bbdaf0fcb80000652e\"\n }\n },\n \"time_since_restore\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005954f000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294284740a7d8b1c1e000004740abcc2d2aa000004740afc113c6e000004740b1dabcb52000004740b3d4fd4b100000652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005957c000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a8694529428474080537d0d00000047409012e15d100000474097fbaf13b0000047409fe359500000004740a3e508215000004740a7d8b1c1e000004740abcc2d2aa000004740afc113c6e000004740b1dabcb52000004740b3d4fd4b100000652e\"\n }\n },\n \"iterations_since_restore\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952c000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294284b064b074b084b094b0a652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059536000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294284b014b024b034b044b054b064b074b084b094b0a652e\"\n }\n }\n },\n \"checkpoint_manager\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005954a340000000000008c267261792e747261696e2e5f696e7465726e616c2e636865636b706f696e745f6d616e61676572948c125f436865636b706f696e744d616e616765729493942981947d94288c125f636865636b706f696e745f636f6e666967948c097261792e747261696e948c10436865636b706f696e74436f6e6669679493942981947d94288c0b6e756d5f746f5f6b656570944e8c1a636865636b706f696e745f73636f72655f617474726962757465944e8c16636865636b706f696e745f73636f72655f6f72646572948c036d6178948c14636865636b706f696e745f6672657175656e6379944b008c11636865636b706f696e745f61745f656e6494898c1a5f636865636b706f696e745f6b6565705f616c6c5f72616e6b73948c0a44455052454341544544948c1f5f636865636b706f696e745f75706c6f61645f66726f6d5f776f726b65727394681275628c135f636865636b706f696e745f726573756c7473945d94288c1b7261792e747261696e2e5f696e7465726e616c2e73657373696f6e948c0f5f547261696e696e67526573756c749493942981947d94288c0a636865636b706f696e749468068c0a436865636b706f696e749493942981947d94288c0470617468948cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f30663963633937645f32325f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d3531322c6465636f6465725f72745f6e756d5f6c617965723d322c64726f705f725f323032342d30362d32355f31382d35332d30302f636865636b706f696e745f303030303030948c0a66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c055f75756964948c0475756964948c04555549449493942981947d948c03696e74948a11bc8b62e8c3edbcb2c8485d28b207aeb100736275628c076d657472696373947d94288c046c6f7373948c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b007494624308d268349a08909e4094869452948c0974696d657374616d70944a00f97a668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303030948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b018c08747269616c5f6964948c083066396363393764948c0464617465948c13323032342d30362d32355f31392d30362d3038948c1074696d655f746869735f697465725f73944740803d39eea000008c0c74696d655f746f74616c5f73944740803d39eea000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b048c146465636f6465725f72745f6e756d5f6c61796572944b028c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944d00048c0a656e636f6465725f6666944d00048c0d6465636f6465725f72745f6666944d00028c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b018c0964726f705f7261746594473f769053cb66dd808c026c7294473f5376df64dc8c7c8c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f7265944740803d39eea000008c18697465726174696f6e735f73696e63655f726573746f7265944b0175756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f30663963633937645f32325f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d3531322c6465636f6465725f72745f6e756d5f6c617965723d322c64726f705f725f323032342d30362d32355f31382d35332d30302f636865636b706f696e745f30303030303194682268257d946827897385945294682a682d2981947d9468308a105e13c0c82c9e3191cf421fa8c4e1fc307362756268317d94288c046c6f7373946836683c4308a9542a352ac99e4094869452948c0974696d657374616d70944afafa7a668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303031948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b028c08747269616c5f6964948c083066396363393764948c0464617465948c13323032342d30362d32355f31392d31342d3334948c1074696d655f746869735f697465725f739447407f9ed96ec000008c0c74696d655f746f74616c5f73944740900653530000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b048c146465636f6465725f72745f6e756d5f6c61796572944b028c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944d00048c0a656e636f6465725f6666944d00048c0d6465636f6465725f72745f6666944d00028c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b018c0964726f705f7261746594473f769053cb66dd808c026c7294473f5376df64dc8c7c8c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f7265944740900653530000008c18697465726174696f6e735f73696e63655f726573746f7265944b0275756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f30663963633937645f32325f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d3531322c6465636f6465725f72745f6e756d5f6c617965723d322c64726f705f725f323032342d30362d32355f31382d35332d30302f636865636b706f696e745f30303030303294682268257d946827897385945294682a682d2981947d9468308a1047a51823437053b9eb45d1bfde1b56417362756268317d94288c046c6f7373946836683c4308d4e974fab0899e4094869452948c0974696d657374616d70944afc087b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303032948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b038c08747269616c5f6964948c083066396363393764948c0464617465948c13323032342d30362d32355f32302d31342d3231948c1074696d655f746869735f697465725f739447408051e4044000008c0c74696d655f746f74616c5f73944740982f45552000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b048c146465636f6465725f72745f6e756d5f6c61796572944b028c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944d00048c0a656e636f6465725f6666944d00048c0d6465636f6465725f72745f6666944d00028c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b018c0964726f705f7261746594473f769053cb66dd808c026c7294473f5376df64dc8c7c8c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f72659447408051e4044000008c18697465726174696f6e735f73696e63655f726573746f7265944b0175756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f30663963633937645f32325f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d3531322c6465636f6465725f72745f6e756d5f6c617965723d322c64726f705f725f323032342d30362d32355f31382d35332d30302f636865636b706f696e745f30303030303394682268257d946827897385945294682a682d2981947d9468308a11fe5eef75c2c2fb9c0b4a0a747000a89f007362756268317d94288c046c6f7373946836683c4308d4e974dafac69e4094869452948c0974696d657374616d70944af60a7b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303033948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b048c08747269616c5f6964948c083066396363393764948c0464617465948c13323032342d30362d32355f32302d32322d3437948c1074696d655f746869735f697465725f739447407fa225c60000008c0c74696d655f746f74616c5f73944740a00be7635000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b048c146465636f6465725f72745f6e756d5f6c61796572944b028c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944d00048c0a656e636f6465725f6666944d00048c0d6465636f6465725f72745f6666944d00028c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b018c0964726f705f7261746594473f769053cb66dd808c026c7294473f5376df64dc8c7c8c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f726594474090117b73a000008c18697465726174696f6e735f73696e63655f726573746f7265944b0275756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f30663963633937645f32325f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d3531322c6465636f6465725f72745f6e756d5f6c617965723d322c64726f705f725f323032342d30362d32355f31382d35332d30302f636865636b706f696e745f30303030303494682268257d946827897385945294682a682d2981947d9468308a10085c6792f9ded8bf46472e09e7339a767362756268317d94288c046c6f7373946836683c430841201028be7f9e4094869452948c0974696d657374616d70944a88107b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303034948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b058c08747269616c5f6964948c083066396363393764948c0464617465948c13323032342d30362d32355f32302d34362d3333948c1074696d655f746869735f697465725f7394474080537d0d0000008c0c74696d655f746f74616c5f73944740a420c6a69000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b048c146465636f6465725f72745f6e756d5f6c61796572944b028c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944d00048c0a656e636f6465725f6666944d00048c0d6465636f6465725f72745f6666944d00028c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b018c0964726f705f7261746594473f769053cb66dd808c026c7294473f5376df64dc8c7c8c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f726594474080537d0d0000008c18697465726174696f6e735f73696e63655f726573746f7265944b0175756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f30663963633937645f32325f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d3531322c6465636f6465725f72745f6e756d5f6c617965723d322c64726f705f725f323032342d30362d32355f31382d35332d30302f636865636b706f696e745f30303030303594682268257d946827897385945294682a682d2981947d9468308a1068d58297224d9a964f43f4e1c1d90f797362756268317d94288c046c6f7373946836683c43083a9d4e27ca839e4094869452948c0974696d657374616d70944a83127b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303035948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b068c08747269616c5f6964948c083066396363393764948c0464617465948c13323032342d30362d32355f32302d35342d3539948c1074696d655f746869735f697465725f739447407fa48b5a4000008c0c74696d655f746f74616c5f73944740a8155811d800008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b048c146465636f6465725f72745f6e756d5f6c61796572944b028c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944d00048c0a656e636f6465725f6666944d00048c0d6465636f6465725f72745f6666944d00028c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b018c0964726f705f7261746594473f769053cb66dd808c026c7294473f5376df64dc8c7c8c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f72659447409012e15d1000008c18697465726174696f6e735f73696e63655f726573746f7265944b0275756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f30663963633937645f32325f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d3531322c6465636f6465725f72745f6e756d5f6c617965723d322c64726f705f725f323032342d30362d32355f31382d35332d30302f636865636b706f696e745f30303030303694682268257d946827897385945294682a682d2981947d9468308a10d78dd9d3647e568eb845a94a734b6b4b7362756268317d94288c046c6f7373946836683c4308bfdfefd79d8b9e4094869452948c0974696d657374616d70944a7d147b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303036948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b078c08747269616c5f6964948c083066396363393764948c0464617465948c13323032342d30362d32355f32312d30332d3235948c1074696d655f746869735f697465725f739447407fa336da8000008c0c74696d655f746f74616c5f73944740ac09beed2800008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b048c146465636f6465725f72745f6e756d5f6c61796572944b028c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944d00048c0a656e636f6465725f6666944d00048c0d6465636f6465725f72745f6666944d00028c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b018c0964726f705f7261746594473f769053cb66dd808c026c7294473f5376df64dc8c7c8c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f726594474097fbaf13b000008c18697465726174696f6e735f73696e63655f726573746f7265944b0375756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f30663963633937645f32325f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d3531322c6465636f6465725f72745f6e756d5f6c617965723d322c64726f705f725f323032342d30362d32355f31382d35332d30302f636865636b706f696e745f30303030303794682268257d946827897385945294682a682d2981947d9468308a11e00c723a18786c94ee4853bb68c682c6007362756268317d94288c046c6f7373946836683c43089bcd6613ce8b9e4094869452948c0974696d657374616d70944a77167b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303037948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b088c08747269616c5f6964948c083066396363393764948c0464617465948c13323032342d30362d32355f32312d31312d3531948c1074696d655f746869735f697465725f739447407f9ea8f14000008c0c74696d655f746f74616c5f73944740affd940b5000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b048c146465636f6465725f72745f6e756d5f6c61796572944b028c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944d00048c0a656e636f6465725f6666944d00048c0d6465636f6465725f72745f6666944d00028c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b018c0964726f705f7261746594473f769053cb66dd808c026c7294473f5376df64dc8c7c8c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f72659447409fe359500000008c18697465726174696f6e735f73696e63655f726573746f7265944b0475756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f30663963633937645f32325f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d3531322c6465636f6465725f72745f6e756d5f6c617965723d322c64726f705f725f323032342d30362d32355f31382d35332d30302f636865636b706f696e745f30303030303894682268257d946827897385945294682a682d2981947d9468308a114c9987b028f3149ab84de430cd82a0aa007362756268317d94288c046c6f7373946836683c430883c160508c8f9e4094869452948c0974696d657374616d70944a71187b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303038948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b098c08747269616c5f6964948c083066396363393764948c0464617465948c13323032342d30362d32355f32312d32302d3137948c1074696d655f746869735f697465725f739447407f9adbca8000008c0c74696d655f746f74616c5f73944740b1f877c25000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b048c146465636f6465725f72745f6e756d5f6c61796572944b028c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944d00048c0a656e636f6465725f6666944d00048c0d6465636f6465725f72745f6666944d00028c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b018c0964726f705f7261746594473f769053cb66dd808c026c7294473f5376df64dc8c7c8c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f7265944740a3e508215000008c18697465726174696f6e735f73696e63655f726573746f7265944b0575756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f30663963633937645f32325f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d3531322c6465636f6465725f72745f6e756d5f6c617965723d322c64726f705f725f323032342d30362d32355f31382d35332d30302f636865636b706f696e745f30303030303994682268257d946827897385945294682a682d2981947d9468308a100e8a22064078bc9d22409ef1e05347527362756268317d94288c046c6f7373946836683c43089fcfe773fa859e4094869452948c0974696d657374616d70944a6a1a7b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303039948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b0a8c08747269616c5f6964948c083066396363393764948c0464617465948c13323032342d30362d32355f32312d32382d3433948c1074696d655f746869735f697465725f739447407f9d4d048000008c0c74696d655f746f74616c5f73944740b3f24c929800008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b048c146465636f6465725f72745f6e756d5f6c61796572944b028c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944d00048c0a656e636f6465725f6666944d00048c0d6465636f6465725f72745f6666944d00028c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b018c0964726f705f7261746594473f769053cb66dd808c026c7294473f5376df64dc8c7c8c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f7265944740a7d8b1c1e000008c18697465726174696f6e735f73696e63655f726573746f7265944b0675756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f30663963633937645f32325f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d3531322c6465636f6465725f72745f6e756d5f6c617965723d322c64726f705f725f323032342d30362d32355f31382d35332d30302f636865636b706f696e745f30303030313094682268257d946827897385945294682a682d2981947d9468308a11f07707441e5ecb8e5043ef37cdf3579f007362756268317d94288c046c6f7373946836683c43088fc7e3911a849e4094869452948c0974696d657374616d70944a641c7b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303130948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b0b8c08747269616c5f6964948c083066396363393764948c0464617465948c13323032342d30362d32355f32312d33372d3039948c1074696d655f746869735f697465725f739447407f9bdb460000008c0c74696d655f746f74616c5f73944740b5ec0a46f800008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b048c146465636f6465725f72745f6e756d5f6c61796572944b028c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944d00048c0a656e636f6465725f6666944d00048c0d6465636f6465725f72745f6666944d00028c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b018c0964726f705f7261746594473f769053cb66dd808c026c7294473f5376df64dc8c7c8c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f7265944740abcc2d2aa000008c18697465726174696f6e735f73696e63655f726573746f7265944b0775756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f30663963633937645f32325f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d3531322c6465636f6465725f72745f6e756d5f6c617965723d322c64726f705f725f323032342d30362d32355f31382d35332d30302f636865636b706f696e745f30303030313194682268257d946827897385945294682a682d2981947d9468308a1095be09bc9c1548877342e6ddc1583d0b7362756268317d94288c046c6f7373946836683c4308f8fbfd3e21919e4094869452948c0974696d657374616d70944a5f1e7b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303131948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b0c8c08747269616c5f6964948c083066396363393764948c0464617465948c13323032342d30362d32355f32312d34352d3335948c1074696d655f746869735f697465725f739447407fa734e20000008c0c74696d655f746f74616c5f73944740b7e67d951800008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b048c146465636f6465725f72745f6e756d5f6c61796572944b028c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944d00048c0a656e636f6465725f6666944d00048c0d6465636f6465725f72745f6666944d00028c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b018c0964726f705f7261746594473f769053cb66dd808c026c7294473f5376df64dc8c7c8c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f7265944740afc113c6e000008c18697465726174696f6e735f73696e63655f726573746f7265944b0875756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f30663963633937645f32325f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d3531322c6465636f6465725f72745f6e756d5f6c617965723d322c64726f705f725f323032342d30362d32355f31382d35332d30302f636865636b706f696e745f30303030313294682268257d946827897385945294682a682d2981947d9468308a10c47a732ff00386986d4175a68c1701057362756268317d94288c046c6f7373946836683c430887c3e110268e9e4094869452948c0974696d657374616d70944a59207b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303132948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b0d8c08747269616c5f6964948c083066396363393764948c0464617465948c13323032342d30362d32355f32312d35342d3031948c1074696d655f746869735f697465725f739447407fa32d1b0000008c0c74696d655f746f74616c5f73944740b9e0b066c800008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b048c146465636f6465725f72745f6e756d5f6c61796572944b028c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944d00048c0a656e636f6465725f6666944d00048c0d6465636f6465725f72745f6666944d00028c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b018c0964726f705f7261746594473f769053cb66dd808c026c7294473f5376df64dc8c7c8c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f7265944740b1dabcb52000008c18697465726174696f6e735f73696e63655f726573746f7265944b0975756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f30663963633937645f32325f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d3531322c6465636f6465725f72745f6e756d5f6c617965723d322c64726f705f725f323032342d30362d32355f31382d35332d30302f636865636b706f696e745f30303030313394682268257d946827897385945294682a682d2981947d9468308a10638046ba80165599f44a08949e6b85357362756268317d94288c046c6f7373946836683c4308994c2693ee919e4094869452948c0974696d657374616d70944a53227b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303133948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b0e8c08747269616c5f6964948c083066396363393764948c0464617465948c13323032342d30362d32355f32322d30322d3237948c1074696d655f746869735f697465725f739447407fa4095f0000008c0c74696d655f746f74616c5f73944740bbdaf0fcb800008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b048c146465636f6465725f72745f6e756d5f6c61796572944b028c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944d00048c0a656e636f6465725f6666944d00048c0d6465636f6465725f72745f6666944d00028c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b018c0964726f705f7261746594473f769053cb66dd808c026c7294473f5376df64dc8c7c8c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f7265944740b3d4fd4b1000008c18697465726174696f6e735f73696e63655f726573746f7265944b0a757562658c195f6c61746573745f636865636b706f696e745f726573756c74946a9602000075622e\"\n }\n}"
+ ],
+ [
+ "{\n \"stub\": false,\n \"trainable_name\": \"train_model\",\n \"trial_id\": \"5eb1e282\",\n \"storage\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059572040000000000008c1b7261792e747261696e2e5f696e7465726e616c2e73746f72616765948c0e53746f72616765436f6e746578749493942981947d94288c12637573746f6d5f66735f70726f766964656494898c1273746f726167655f6c6f63616c5f70617468948c2b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c7473948c136578706572696d656e745f6469725f6e616d65948c0f746573745f6578706572696d656e74948c0e747269616c5f6469725f6e616d65948c96747261696e5f6d6f64656c5f35656231653238325f32335f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d3531322c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f725f323032342d30362d32355f31382d35372d3239948c1863757272656e745f636865636b706f696e745f696e646578944b118c0b73796e635f636f6e666967948c097261792e747261696e948c0a53796e63436f6e6669679493942981947d94288c0b73796e635f706572696f64944d2c018c0c73796e635f74696d656f7574944d08078c0e73796e635f61727469666163747394898c1c73796e635f6172746966616374735f6f6e5f636865636b706f696e7494888c0a75706c6f61645f646972948c0a44455052454341544544948c0673796e6365729468188c1273796e635f6f6e5f636865636b706f696e7494681875628c1273746f726167655f66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c0f73746f726167655f66735f70617468948c452f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f7465737494681968008c115f46696c6573797374656d53796e6365729493942981947d9428681b682268134d2c0168144d08078c116c6173745f73796e635f75705f74696d65944741d99ebda2f220858c136c6173745f73796e635f646f776e5f74696d659447fff00000000000008c0d5f73796e635f70726f63657373944e8c0c5f63757272656e745f636d649468008c125f75706c6f61645f746f5f66735f706174689493947d94288c0a6c6f63616c5f70617468948c3b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c74732f746573745f6578706572696d656e74948c0266739468228c0766735f70617468948c552f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e74948c076578636c756465945d948c0e2a2f636865636b706f696e745f2a9461758694756275622e\"\n },\n \"config\": {\n \"encoder_num_layer\": 1,\n \"decoder_rt_num_layer\": 1,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 16,\n \"encoder_ff\": 1024,\n \"decoder_rt_ff\": 512,\n \"decoder_int_ff\": 512,\n \"n_head\": 4,\n \"drop_rate\": 0.2842529108982538,\n \"lr\": 0.00015364900402465778,\n \"batch_size\": 2048\n },\n \"_Trial__unresolved_config\": {\n \"encoder_num_layer\": 1,\n \"decoder_rt_num_layer\": 1,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 16,\n \"encoder_ff\": 1024,\n \"decoder_rt_ff\": 512,\n \"decoder_int_ff\": 512,\n \"n_head\": 4,\n \"drop_rate\": 0.2842529108982538,\n \"lr\": 0.00015364900402465778,\n \"batch_size\": 2048\n },\n \"evaluated_params\": {\n \"batch_size\": 2048,\n \"decoder_int_ff\": 512,\n \"decoder_int_num_layer\": 1,\n \"decoder_rt_ff\": 512,\n \"decoder_rt_num_layer\": 1,\n \"drop_rate\": 0.2842529108982538,\n \"embedding_dim\": 16,\n \"encoder_ff\": 1024,\n \"encoder_num_layer\": 1,\n \"lr\": 0.00015364900402465778,\n \"n_head\": 4\n },\n \"experiment_tag\": \"23_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=1,drop_rate=0.2843,embedding_dim=16,encoder_ff=1024,encoder_num_layer=1,lr=0.0002,n_head=4\",\n \"stopping_criterion\": {},\n \"_setup_default_resource\": true,\n \"_default_placement_group_factory\": \"80054e2e\",\n \"placement_group_factory\": \"800595ba000000000000008c237261792e74756e652e657865637574696f6e2e706c6163656d656e745f67726f757073948c15506c6163656d656e7447726f7570466163746f72799493942981947d94288c085f62756e646c6573945d947d94288c03435055944740540000000000008c034750559447401000000000000075618c155f686561645f62756e646c655f69735f656d70747994898c095f7374726174656779948c045041434b948c055f6172677394298c075f6b7761726773947d9475622e\",\n \"log_to_file\": [\n null,\n null\n ],\n \"max_failures\": 0,\n \"_default_result_or_future\": null,\n \"export_formats\": [],\n \"status\": \"TERMINATED\",\n \"relative_logdir\": \"train_model_5eb1e282_23_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=1,drop_r_2024-06-25_18-57-29\",\n \"trial_name_creator\": null,\n \"trial_dirname_creator\": null,\n \"custom_trial_name\": null,\n \"custom_dirname\": null,\n \"restore_path\": null,\n \"_restore_checkpoint_result\": null,\n \"_state_json\": null,\n \"results\": \"80054e2e\",\n \"extra_arg\": \"80054e2e\",\n \"_resources\": \"80054e2e\"\n}",
+ "{\n \"start_time\": 1719335674.8936594,\n \"num_failures\": 0,\n \"num_failures_after_restore\": 0,\n \"error_filename\": null,\n \"pickled_error_filename\": null,\n \"last_result\": {\n \"loss\": 183.96574522003414,\n \"timestamp\": 1719346234,\n \"checkpoint_dir_name\": \"checkpoint_000017\",\n \"should_checkpoint\": true,\n \"done\": true,\n \"training_iteration\": 18,\n \"trial_id\": \"5eb1e282\",\n \"date\": \"2024-06-25_22-10-34\",\n \"time_this_iter_s\": 47.40534067153931,\n \"time_total_s\": 894.9221906661987,\n \"pid\": 2195253,\n \"hostname\": \"r8i6n2\",\n \"node_ip\": \"10.159.28.60\",\n \"config\": {\n \"encoder_num_layer\": 1,\n \"decoder_rt_num_layer\": 1,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 16,\n \"encoder_ff\": 1024,\n \"decoder_rt_ff\": 512,\n \"decoder_int_ff\": 512,\n \"n_head\": 4,\n \"drop_rate\": 0.2842529108982538,\n \"lr\": 0.00015364900402465778,\n \"batch_size\": 2048\n },\n \"time_since_restore\": 486.2922348976135,\n \"iterations_since_restore\": 10,\n \"experiment_tag\": \"23_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=1,drop_rate=0.2843,embedding_dim=16,encoder_ff=1024,encoder_num_layer=1,lr=0.0002,n_head=4\"\n },\n \"last_result_time\": 1719346234.4968047,\n \"metric_analysis\": {\n \"loss\": {\n \"max\": 1030.0586158572219,\n \"min\": 183.96574522003414,\n \"avg\": 322.085730137996,\n \"last\": 183.96574522003414,\n \"last-5-avg\": 195.48832729069267,\n \"last-10-avg\": 213.65420443317083\n },\n \"should_checkpoint\": {\n \"max\": true,\n \"min\": true,\n \"avg\": 1.0,\n \"last\": true,\n \"last-5-avg\": 1.0,\n \"last-10-avg\": 1.0\n },\n \"done\": {\n \"max\": false,\n \"min\": false,\n \"avg\": 0.0,\n \"last\": false,\n \"last-5-avg\": 0.0,\n \"last-10-avg\": 0.0\n },\n \"training_iteration\": {\n \"max\": 18,\n \"min\": 1,\n \"avg\": 9.5,\n \"last\": 18,\n \"last-5-avg\": 16.0,\n \"last-10-avg\": 13.5\n },\n \"time_this_iter_s\": {\n \"max\": 61.87126088142395,\n \"min\": 46.741618633270264,\n \"avg\": 49.71789948145548,\n \"last\": 47.40534067153931,\n \"last-5-avg\": 47.24765267372131,\n \"last-10-avg\": 48.62922348976135\n },\n \"time_total_s\": {\n \"max\": 894.9221906661987,\n \"min\": 60.84843850135803,\n \"avg\": 485.0399955908457,\n \"last\": 894.9221906661987,\n \"last-5-avg\": 800.4915899276733,\n \"last-10-avg\": 682.4233734607697\n },\n \"time_since_restore\": {\n \"max\": 486.2922348976135,\n \"min\": 60.84843850135803,\n \"avg\": 221.83922640482584,\n \"last\": 486.2922348976135,\n \"last-5-avg\": 391.8616341590881,\n \"last-10-avg\": 273.79341769218445\n },\n \"iterations_since_restore\": {\n \"max\": 10,\n \"min\": 1,\n \"avg\": 4.388888888888888,\n \"last\": 10,\n \"last-5-avg\": 8.0,\n \"last-10-avg\": 5.5\n }\n },\n \"_n_steps\": [\n 5,\n 10\n ],\n \"metric_n_steps\": {\n \"loss\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"800595d6000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294288c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b007494624308180c06c30e90694094869452946807680d4308e4f178bcd9c1684094869452946807680d4308bfdfef77d742684094869452946807680d430847a3d1887a9a684094869452946807680d4308140a8562e7fe66409486945294652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059535010000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294288c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b007494624308369bcd96da0e704094869452946807680d4308241289a404126e4094869452946807680d430845229168ebcd6c4094869452946807680d43085128142a096d6b4094869452946807680d43082e97cb2588786a4094869452946807680d4308180c06c30e90694094869452946807680d4308e4f178bcd9c1684094869452946807680d4308bfdfef77d742684094869452946807680d430847a3d1887a9a684094869452946807680d4308140a8562e7fe66409486945294652e\"\n }\n },\n \"should_checkpoint\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059527000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294288888888888652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952c000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452942888888888888888888888652e\"\n }\n },\n \"done\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059527000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294288989898989652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952c000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452942889898989898989898989652e\"\n }\n },\n \"training_iteration\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952c000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294284b0e4b0f4b104b114b12652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059536000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294284b094b0a4b0b4b0c4b0d4b0e4b0f4b104b114b12652e\"\n }\n },\n \"time_this_iter_s\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005954f000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b058694529428474047e31db20000004740475eed5c00000047404794970400000047404793fb24000000474047b3e234000000652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005957c000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452942847404eb3e57200000047404792ce3800000047404797ce0e000000474047919be200000047404796caf0000000474047e31db20000004740475eed5c00000047404794970400000047404793fb24000000474047b3e234000000652e\"\n }\n },\n \"time_total_s\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005954f000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452942847408613aa8a00000047408789995fc0000047408902e2d000000047408a7c228240000047408bf760a5800000652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005957c000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452942847407d6090fac00000474080297560e00000474081a2f241c000004740831c0bffe000004740849578aee0000047408613aa8a00000047408789995fc0000047408902e2d000000047408a7c228240000047408bf760a5800000652e\"\n }\n },\n \"time_since_restore\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005954f000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294284740729d40c7800000474075891e730000004740787bb15380000047407b6e30b800000047407e64acfe800000652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005957c000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452942847404eb3e57200000047405b2359d500000047406377a06e0000004740695c076680000047406f41ba228000004740729d40c7800000474075891e730000004740787bb15380000047407b6e30b800000047407e64acfe800000652e\"\n }\n },\n \"iterations_since_restore\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952c000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294284b064b074b084b094b0a652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059536000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294284b014b024b034b044b054b064b074b084b094b0a652e\"\n }\n }\n },\n \"checkpoint_manager\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059572420000000000008c267261792e747261696e2e5f696e7465726e616c2e636865636b706f696e745f6d616e61676572948c125f436865636b706f696e744d616e616765729493942981947d94288c125f636865636b706f696e745f636f6e666967948c097261792e747261696e948c10436865636b706f696e74436f6e6669679493942981947d94288c0b6e756d5f746f5f6b656570944e8c1a636865636b706f696e745f73636f72655f617474726962757465944e8c16636865636b706f696e745f73636f72655f6f72646572948c036d6178948c14636865636b706f696e745f6672657175656e6379944b008c11636865636b706f696e745f61745f656e6494898c1a5f636865636b706f696e745f6b6565705f616c6c5f72616e6b73948c0a44455052454341544544948c1f5f636865636b706f696e745f75706c6f61645f66726f6d5f776f726b65727394681275628c135f636865636b706f696e745f726573756c7473945d94288c1b7261792e747261696e2e5f696e7465726e616c2e73657373696f6e948c0f5f547261696e696e67526573756c749493942981947d94288c0a636865636b706f696e749468068c0a436865636b706f696e749493942981947d94288c0470617468948cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f35656231653238325f32335f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d3531322c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f725f323032342d30362d32355f31382d35372d32392f636865636b706f696e745f303030303030948c0a66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c055f75756964948c0475756964948c04555549449493942981947d948c03696e74948a1195d20293345c8a99de4f658c432f75ee00736275628c076d657472696373947d94288c046c6f7373948c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b0074946243082e97cb053c18904094869452948c0974696d657374616d70944a37fb7a668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303030948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b018c08747269616c5f6964948c083565623165323832948c0464617465948c13323032342d30362d32355f31392d31352d3335948c1074696d655f746869735f697465725f739447404e6c99a20000008c0c74696d655f746f74616c5f739447404e6c99a20000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b018c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b108c0a656e636f6465725f6666944d00048c0d6465636f6465725f72745f6666944d00028c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b048c0964726f705f7261746594473fd231331f06738a8c026c7294473f24239ae50ae97e8c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f72659447404e6c99a20000008c18697465726174696f6e735f73696e63655f726573746f7265944b0175756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f35656231653238325f32335f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d3531322c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f725f323032342d30362d32355f31382d35372d32392f636865636b706f696e745f30303030303194682268257d946827897385945294682a682d2981947d9468308a1178af4dae568327b5074207261e5053d2007362756268317d94288c046c6f7373946836683c4308201008043b90814094869452948c0974696d657374616d70944a67fb7a668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303031948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b028c08747269616c5f6964948c083565623165323832948c0464617465948c13323032342d30362d32355f31392d31362d3233948c1074696d655f746869735f697465725f7394474047da1acc0000008c0c74696d655f746f74616c5f739447405b235a370000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b018c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b108c0a656e636f6465725f6666944d00048c0d6465636f6465725f72745f6666944d00028c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b048c0964726f705f7261746594473fd231331f06738a8c026c7294473f24239ae50ae97e8c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f72659447405b235a370000008c18697465726174696f6e735f73696e63655f726573746f7265944b0275756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f35656231653238325f32335f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d3531322c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f725f323032342d30362d32355f31382d35372d32392f636865636b706f696e745f30303030303294682268257d946827897385945294682a682d2981947d9468308a1188f94f4311288da1e14082c0ef818cb8007362756268317d94288c046c6f7373946836683c43084d2693a981ce7b4094869452948c0974696d657374616d70944a350b7b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303032948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b038c08747269616c5f6964948c083565623165323832948c0464617465948c13323032342d30362d32355f32302d32332d3439948c1074696d655f746869735f697465725f739447404eef857a0000008c0c74696d655f746f74616c5f73944740654d8e7a0000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b018c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b108c0a656e636f6465725f6666944d00048c0d6465636f6465725f72745f6666944d00028c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b048c0964726f705f7261746594473fd231331f06738a8c026c7294473f24239ae50ae97e8c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f72659447404eef857a0000008c18697465726174696f6e735f73696e63655f726573746f7265944b0175756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f35656231653238325f32335f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d3531322c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f725f323032342d30362d32355f31382d35372d32392f636865636b706f696e745f30303030303394682268257d946827897385945294682a682d2981947d9468308a102651890a9afad186f34145110bb9e22f7362756268317d94288c046c6f7373946836683c43085bad568b4f1c784094869452948c0974696d657374616d70944a640b7b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303033948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b048c08747269616c5f6964948c083565623165323832948c0464617465948c13323032342d30362d32355f32302d32342d3336948c1074696d655f746869735f697465725f7394474047b3f4240000008c0c74696d655f746f74616c5f739447406b3a8b830000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b018c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b108c0a656e636f6465725f6666944d00048c0d6465636f6465725f72745f6666944d00028c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b048c0964726f705f7261746594473fd231331f06738a8c026c7294473f24239ae50ae97e8c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f72659447405b51bccf0000008c18697465726174696f6e735f73696e63655f726573746f7265944b0275756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f35656231653238325f32335f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d3531322c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f725f323032342d30362d32355f31382d35372d32392f636865636b706f696e745f30303030303494682268257d946827897385945294682a682d2981947d9468308a11d26c790b9ccacf9ac248298dcb6f6bee007362756268317d94288c046c6f7373946836683c4308140a85a2cef1754094869452948c0974696d657374616d70944a940b7b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303034948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b058c08747269616c5f6964948c083565623165323832948c0464617465948c13323032342d30362d32355f32302d32352d3234948c1074696d655f746869735f697465725f7394474047fd1bc00000008c0c74696d655f746f74616c5f73944740709ce9398000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b018c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b108c0a656e636f6465725f6666944d00048c0d6465636f6465725f72745f6666944d00028c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b048c0964726f705f7261746594473fd231331f06738a8c026c7294473f24239ae50ae97e8c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f726594474063a825578000008c18697465726174696f6e735f73696e63655f726573746f7265944b0375756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f35656231653238325f32335f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d3531322c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f725f323032342d30362d32355f31382d35372d32392f636865636b706f696e745f30303030303594682268257d946827897385945294682a682d2981947d9468308a1055e673ebcf0b81bede4bfa3c05941c5e7362756268317d94288c046c6f7373946836683c4308f67abd3e7007744094869452948c0974696d657374616d70944ac40b7b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303035948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b068c08747269616c5f6964948c083565623165323832948c0464617465948c13323032342d30362d32355f32302d32362d3132948c1074696d655f746869735f697465725f7394474047b75a200000008c0c74696d655f746f74616c5f739447407393d47d8000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b018c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b108c0a656e636f6465725f6666944d00048c0d6465636f6465725f72745f6666944d00028c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b048c0964726f705f7261746594473fd231331f06738a8c026c7294473f24239ae50ae97e8c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f72659447406995fbdf8000008c18697465726174696f6e735f73696e63655f726573746f7265944b0475756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f35656231653238325f32335f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d3531322c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f725f323032342d30362d32355f31382d35372d32392f636865636b706f696e745f30303030303694682268257d946827897385945294682a682d2981947d9468308a108c30c8a2a1ec26aa4d45bb6fa954dc1d7362756268317d94288c046c6f7373946836683c43086fb7dbfd5d57724094869452948c0974696d657374616d70944af30b7b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303036948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b078c08747269616c5f6964948c083565623165323832948c0464617465948c13323032342d30362d32355f32302d32362d3539948c1074696d655f746869735f697465725f7394474047f5514c0000008c0c74696d655f746f74616c5f7394474076927ea70000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b018c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b108c0a656e636f6465725f6666944d00048c0d6465636f6465725f72745f6666944d00028c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b048c0964726f705f7261746594473fd231331f06738a8c026c7294473f24239ae50ae97e8c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f72659447406f9350328000008c18697465726174696f6e735f73696e63655f726573746f7265944b0575756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f35656231653238325f32335f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d3531322c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f725f323032342d30362d32355f31382d35372d32392f636865636b706f696e745f30303030303794682268257d946827897385945294682a682d2981947d9468308a1047ecf3292ecafbaf8043f4325d7111457362756268317d94288c046c6f7373946836683c4308e270384c3013714094869452948c0974696d657374616d70944a230c7b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303037948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b088c08747269616c5f6964948c083565623165323832948c0464617465948c13323032342d30362d32355f32302d32372d3437948c1074696d655f746869735f697465725f7394474047bcad2c0000008c0c74696d655f746f74616c5f73944740798a144c8000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b018c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b108c0a656e636f6465725f6666944d00048c0d6465636f6465725f72745f6666944d00028c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b048c0964726f705f7261746594473fd231331f06738a8c026c7294473f24239ae50ae97e8c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f726594474072c13dbec000008c18697465726174696f6e735f73696e63655f726573746f7265944b0675756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f35656231653238325f32335f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d3531322c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f725f323032342d30362d32355f31382d35372d32392f636865636b706f696e745f30303030303894682268257d946827897385945294682a682d2981947d9468308a1076123843a8ac16adcf4931a9534560767362756268317d94288c046c6f7373946836683c4308369bcd96da0e704094869452948c0974696d657374616d70944a91227b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303038948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b098c08747269616c5f6964948c083565623165323832948c0464617465948c13323032342d30362d32355f32322d30332d3239948c1074696d655f746869735f697465725f739447404eb3e5720000008c0c74696d655f746f74616c5f739447407d6090fac000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b018c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b108c0a656e636f6465725f6666944d00048c0d6465636f6465725f72745f6666944d00028c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b048c0964726f705f7261746594473fd231331f06738a8c026c7294473f24239ae50ae97e8c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f72659447404eb3e5720000008c18697465726174696f6e735f73696e63655f726573746f7265944b0175756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f35656231653238325f32335f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d3531322c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f725f323032342d30362d32355f31382d35372d32392f636865636b706f696e745f30303030303994682268257d946827897385945294682a682d2981947d9468308a11c8b346056ef247af62461e359ba9e3ed007362756268317d94288c046c6f7373946836683c4308241289a404126e4094869452948c0974696d657374616d70944ac0227b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303039948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b0a8c08747269616c5f6964948c083565623165323832948c0464617465948c13323032342d30362d32355f32322d30342d3136948c1074696d655f746869735f697465725f739447404792ce380000008c0c74696d655f746f74616c5f7394474080297560e000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b018c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b108c0a656e636f6465725f6666944d00048c0d6465636f6465725f72745f6666944d00028c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b048c0964726f705f7261746594473fd231331f06738a8c026c7294473f24239ae50ae97e8c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f72659447405b2359d50000008c18697465726174696f6e735f73696e63655f726573746f7265944b0275756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f35656231653238325f32335f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d3531322c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f725f323032342d30362d32355f31382d35372d32392f636865636b706f696e745f30303030313094682268257d946827897385945294682a682d2981947d9468308a114bae079ae08a7393eb42144a99fde4e9007362756268317d94288c046c6f7373946836683c430845229168ebcd6c4094869452948c0974696d657374616d70944aef227b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303130948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b0b8c08747269616c5f6964948c083565623165323832948c0464617465948c13323032342d30362d32355f32322d30352d3033948c1074696d655f746869735f697465725f739447404797ce0e0000008c0c74696d655f746f74616c5f7394474081a2f241c000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b018c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b108c0a656e636f6465725f6666944d00048c0d6465636f6465725f72745f6666944d00028c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b048c0964726f705f7261746594473fd231331f06738a8c026c7294473f24239ae50ae97e8c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f72659447406377a06e0000008c18697465726174696f6e735f73696e63655f726573746f7265944b0375756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f35656231653238325f32335f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d3531322c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f725f323032342d30362d32355f31382d35372d32392f636865636b706f696e745f30303030313194682268257d946827897385945294682a682d2981947d9468308a11f00e26ed3e9b8eaf074cdf1ffa7501dd007362756268317d94288c046c6f7373946836683c43085128142a096d6b4094869452948c0974696d657374616d70944a1f237b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303131948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b0c8c08747269616c5f6964948c083565623165323832948c0464617465948c13323032342d30362d32355f32322d30352d3531948c1074696d655f746869735f697465725f7394474047919be20000008c0c74696d655f746f74616c5f73944740831c0bffe000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b018c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b108c0a656e636f6465725f6666944d00048c0d6465636f6465725f72745f6666944d00028c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b048c0964726f705f7261746594473fd231331f06738a8c026c7294473f24239ae50ae97e8c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f7265944740695c07668000008c18697465726174696f6e735f73696e63655f726573746f7265944b0475756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f35656231653238325f32335f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d3531322c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f725f323032342d30362d32355f31382d35372d32392f636865636b706f696e745f30303030313294682268257d946827897385945294682a682d2981947d9468308a11fe873f03ff0f78ad294ca6bd93705bf8007362756268317d94288c046c6f7373946836683c43082e97cb2588786a4094869452948c0974696d657374616d70944a4e237b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303132948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b0d8c08747269616c5f6964948c083565623165323832948c0464617465948c13323032342d30362d32355f32322d30362d3338948c1074696d655f746869735f697465725f739447404796caf00000008c0c74696d655f746f74616c5f73944740849578aee000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b018c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b108c0a656e636f6465725f6666944d00048c0d6465636f6465725f72745f6666944d00028c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b048c0964726f705f7261746594473fd231331f06738a8c026c7294473f24239ae50ae97e8c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f72659447406f41ba228000008c18697465726174696f6e735f73696e63655f726573746f7265944b0575756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f35656231653238325f32335f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d3531322c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f725f323032342d30362d32355f31382d35372d32392f636865636b706f696e745f30303030313394682268257d946827897385945294682a682d2981947d9468308a1124e339c8fc2638a8374fbe2521a165f2007362756268317d94288c046c6f7373946836683c4308180c06c30e90694094869452948c0974696d657374616d70944a7e237b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303133948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b0e8c08747269616c5f6964948c083565623165323832948c0464617465948c13323032342d30362d32355f32322d30372d3236948c1074696d655f746869735f697465725f7394474047e31db20000008c0c74696d655f746f74616c5f739447408613aa8a0000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b018c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b108c0a656e636f6465725f6666944d00048c0d6465636f6465725f72745f6666944d00028c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b048c0964726f705f7261746594473fd231331f06738a8c026c7294473f24239ae50ae97e8c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f7265944740729d40c78000008c18697465726174696f6e735f73696e63655f726573746f7265944b0675756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f35656231653238325f32335f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d3531322c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f725f323032342d30362d32355f31382d35372d32392f636865636b706f696e745f30303030313494682268257d946827897385945294682a682d2981947d9468308a113d2d0972ce2ffa8ef048bd410c626484007362756268317d94288c046c6f7373946836683c4308e4f178bcd9c1684094869452948c0974696d657374616d70944aac237b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303134948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b0f8c08747269616c5f6964948c083565623165323832948c0464617465948c13323032342d30362d32355f32322d30382d3132948c1074696d655f746869735f697465725f73944740475eed5c0000008c0c74696d655f746f74616c5f739447408789995fc000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b018c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b108c0a656e636f6465725f6666944d00048c0d6465636f6465725f72745f6666944d00028c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b048c0964726f705f7261746594473fd231331f06738a8c026c7294473f24239ae50ae97e8c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f726594474075891e730000008c18697465726174696f6e735f73696e63655f726573746f7265944b0775756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f35656231653238325f32335f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d3531322c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f725f323032342d30362d32355f31382d35372d32392f636865636b706f696e745f30303030313594682268257d946827897385945294682a682d2981947d9468308a10a143eb21091fa9a2394734ed33e86c357362756268317d94288c046c6f7373946836683c4308bfdfef77d742684094869452948c0974696d657374616d70944adb237b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303135948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b108c08747269616c5f6964948c083565623165323832948c0464617465948c13323032342d30362d32355f32322d30382d3539948c1074696d655f746869735f697465725f73944740479497040000008c0c74696d655f746f74616c5f739447408902e2d00000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b018c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b108c0a656e636f6465725f6666944d00048c0d6465636f6465725f72745f6666944d00028c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b048c0964726f705f7261746594473fd231331f06738a8c026c7294473f24239ae50ae97e8c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f7265944740787bb1538000008c18697465726174696f6e735f73696e63655f726573746f7265944b0875756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f35656231653238325f32335f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d3531322c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f725f323032342d30362d32355f31382d35372d32392f636865636b706f696e745f30303030313694682268257d946827897385945294682a682d2981947d9468308a11b5a2db847af6a690174b20e00c776398007362756268317d94288c046c6f7373946836683c430847a3d1887a9a684094869452948c0974696d657374616d70944a0b247b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303136948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b118c08747269616c5f6964948c083565623165323832948c0464617465948c13323032342d30362d32355f32322d30392d3437948c1074696d655f746869735f697465725f739447404793fb240000008c0c74696d655f746f74616c5f739447408a7c22824000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b018c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b108c0a656e636f6465725f6666944d00048c0d6465636f6465725f72745f6666944d00028c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b048c0964726f705f7261746594473fd231331f06738a8c026c7294473f24239ae50ae97e8c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f72659447407b6e30b80000008c18697465726174696f6e735f73696e63655f726573746f7265944b0975756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f35656231653238325f32335f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d3531322c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f725f323032342d30362d32355f31382d35372d32392f636865636b706f696e745f30303030313794682268257d946827897385945294682a682d2981947d9468308a10f8f9a9758b204aba2e454a28a0bd42257362756268317d94288c046c6f7373946836683c4308140a8562e7fe664094869452948c0974696d657374616d70944a3a247b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303137948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b128c08747269616c5f6964948c083565623165323832948c0464617465948c13323032342d30362d32355f32322d31302d3334948c1074696d655f746869735f697465725f7394474047b3e2340000008c0c74696d655f746f74616c5f739447408bf760a58000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b018c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b108c0a656e636f6465725f6666944d00048c0d6465636f6465725f72745f6666944d00028c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b048c0964726f705f7261746594473fd231331f06738a8c026c7294473f24239ae50ae97e8c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f72659447407e64acfe8000008c18697465726174696f6e735f73696e63655f726573746f7265944b0a757562658c195f6c61746573745f636865636b706f696e745f726573756c74946a5203000075622e\"\n }\n}"
+ ],
+ [
+ "{\n \"stub\": false,\n \"trainable_name\": \"train_model\",\n \"trial_id\": \"6be79717\",\n \"storage\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059572040000000000008c1b7261792e747261696e2e5f696e7465726e616c2e73746f72616765948c0e53746f72616765436f6e746578749493942981947d94288c12637573746f6d5f66735f70726f766964656494898c1273746f726167655f6c6f63616c5f70617468948c2b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c7473948c136578706572696d656e745f6469725f6e616d65948c0f746573745f6578706572696d656e74948c0e747269616c5f6469725f6e616d65948c96747261696e5f6d6f64656c5f36626537393731375f32345f62617463685f73697a653d313032342c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d313032342c6465636f6465725f72745f6e756d5f6c617965723d322c64726f705f5f323032342d30362d32355f31392d31342d3334948c1863757272656e745f636865636b706f696e745f696e646578944b018c0b73796e635f636f6e666967948c097261792e747261696e948c0a53796e63436f6e6669679493942981947d94288c0b73796e635f706572696f64944d2c018c0c73796e635f74696d656f7574944d08078c0e73796e635f61727469666163747394898c1c73796e635f6172746966616374735f6f6e5f636865636b706f696e7494888c0a75706c6f61645f646972948c0a44455052454341544544948c0673796e6365729468188c1273796e635f6f6e5f636865636b706f696e7494681875628c1273746f726167655f66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c0f73746f726167655f66735f70617468948c452f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f7465737494681968008c115f46696c6573797374656d53796e6365729493942981947d9428681b682268134d2c0168144d08078c116c6173745f73796e635f75705f74696d65944741d99ebe858154548c136c6173745f73796e635f646f776e5f74696d659447fff00000000000008c0d5f73796e635f70726f63657373944e8c0c5f63757272656e745f636d649468008c125f75706c6f61645f746f5f66735f706174689493947d94288c0a6c6f63616c5f70617468948c3b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c74732f746573745f6578706572696d656e74948c0266739468228c0766735f70617468948c552f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e74948c076578636c756465945d948c0e2a2f636865636b706f696e745f2a9461758694756275622e\"\n },\n \"config\": {\n \"encoder_num_layer\": 1,\n \"decoder_rt_num_layer\": 2,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 64,\n \"encoder_ff\": 2048,\n \"decoder_rt_ff\": 1024,\n \"decoder_int_ff\": 512,\n \"n_head\": 8,\n \"drop_rate\": 0.7446531774853204,\n \"lr\": 0.006000257122673396,\n \"batch_size\": 1024\n },\n \"_Trial__unresolved_config\": {\n \"encoder_num_layer\": 1,\n \"decoder_rt_num_layer\": 2,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 64,\n \"encoder_ff\": 2048,\n \"decoder_rt_ff\": 1024,\n \"decoder_int_ff\": 512,\n \"n_head\": 8,\n \"drop_rate\": 0.7446531774853204,\n \"lr\": 0.006000257122673396,\n \"batch_size\": 1024\n },\n \"evaluated_params\": {\n \"batch_size\": 1024,\n \"decoder_int_ff\": 512,\n \"decoder_int_num_layer\": 1,\n \"decoder_rt_ff\": 1024,\n \"decoder_rt_num_layer\": 2,\n \"drop_rate\": 0.7446531774853204,\n \"embedding_dim\": 64,\n \"encoder_ff\": 2048,\n \"encoder_num_layer\": 1,\n \"lr\": 0.006000257122673396,\n \"n_head\": 8\n },\n \"experiment_tag\": \"24_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop_rate=0.7447,embedding_dim=64,encoder_ff=2048,encoder_num_layer=1,lr=0.0060,n_head=8\",\n \"stopping_criterion\": {},\n \"_setup_default_resource\": true,\n \"_default_placement_group_factory\": \"80054e2e\",\n \"placement_group_factory\": \"800595ba000000000000008c237261792e74756e652e657865637574696f6e2e706c6163656d656e745f67726f757073948c15506c6163656d656e7447726f7570466163746f72799493942981947d94288c085f62756e646c6573945d947d94288c03435055944740540000000000008c034750559447401000000000000075618c155f686561645f62756e646c655f69735f656d70747994898c095f7374726174656779948c045041434b948c055f6172677394298c075f6b7761726773947d9475622e\",\n \"log_to_file\": [\n null,\n null\n ],\n \"max_failures\": 0,\n \"_default_result_or_future\": null,\n \"export_formats\": [],\n \"status\": \"PAUSED\",\n \"relative_logdir\": \"train_model_6be79717_24_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop__2024-06-25_19-14-34\",\n \"trial_name_creator\": null,\n \"trial_dirname_creator\": null,\n \"custom_trial_name\": null,\n \"custom_dirname\": null,\n \"restore_path\": null,\n \"_restore_checkpoint_result\": null,\n \"_state_json\": null,\n \"results\": \"80054e2e\",\n \"extra_arg\": \"80054e2e\",\n \"_resources\": \"80054e2e\"\n}",
+ "{\n \"start_time\": 1719335783.6048226,\n \"num_failures\": 0,\n \"num_failures_after_restore\": 0,\n \"error_filename\": null,\n \"pickled_error_filename\": null,\n \"last_result\": {\n \"loss\": 2003.3902054433747,\n \"timestamp\": 1719335912,\n \"checkpoint_dir_name\": \"checkpoint_000001\",\n \"should_checkpoint\": true,\n \"done\": false,\n \"training_iteration\": 2,\n \"trial_id\": \"6be79717\",\n \"date\": \"2024-06-25_19-18-32\",\n \"time_this_iter_s\": 57.25079274177551,\n \"time_total_s\": 128.72498559951782,\n \"pid\": 2195253,\n \"hostname\": \"r8i6n2\",\n \"node_ip\": \"10.159.28.60\",\n \"config\": {\n \"encoder_num_layer\": 1,\n \"decoder_rt_num_layer\": 2,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 64,\n \"encoder_ff\": 2048,\n \"decoder_rt_ff\": 1024,\n \"decoder_int_ff\": 512,\n \"n_head\": 8,\n \"drop_rate\": 0.7446531774853204,\n \"lr\": 0.006000257122673396,\n \"batch_size\": 1024\n },\n \"time_since_restore\": 128.72498559951782,\n \"iterations_since_restore\": 2,\n \"experiment_tag\": \"24_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop_rate=0.7447,embedding_dim=64,encoder_ff=2048,encoder_num_layer=1,lr=0.0060,n_head=8\"\n },\n \"last_result_time\": 1719335912.3383458,\n \"metric_analysis\": {\n \"loss\": {\n \"max\": 2020.6805025836613,\n \"min\": 2003.3902054433747,\n \"avg\": 2012.035354013518,\n \"last\": 2003.3902054433747,\n \"last-5-avg\": 2012.035354013518,\n \"last-10-avg\": 2012.035354013518\n },\n \"should_checkpoint\": {\n \"max\": true,\n \"min\": true,\n \"avg\": 1.0,\n \"last\": true,\n \"last-5-avg\": 1.0,\n \"last-10-avg\": 1.0\n },\n \"done\": {\n \"max\": false,\n \"min\": false,\n \"avg\": 0.0,\n \"last\": false,\n \"last-5-avg\": 0.0,\n \"last-10-avg\": 0.0\n },\n \"training_iteration\": {\n \"max\": 2,\n \"min\": 1,\n \"avg\": 1.5,\n \"last\": 2,\n \"last-5-avg\": 1.5,\n \"last-10-avg\": 1.5\n },\n \"time_this_iter_s\": {\n \"max\": 71.47419285774231,\n \"min\": 57.25079274177551,\n \"avg\": 64.36249279975891,\n \"last\": 57.25079274177551,\n \"last-5-avg\": 64.36249279975891,\n \"last-10-avg\": 64.36249279975891\n },\n \"time_total_s\": {\n \"max\": 128.72498559951782,\n \"min\": 71.47419285774231,\n \"avg\": 100.09958922863007,\n \"last\": 128.72498559951782,\n \"last-5-avg\": 100.09958922863007,\n \"last-10-avg\": 100.09958922863007\n },\n \"time_since_restore\": {\n \"max\": 128.72498559951782,\n \"min\": 71.47419285774231,\n \"avg\": 100.09958922863007,\n \"last\": 128.72498559951782,\n \"last-5-avg\": 100.09958922863007,\n \"last-10-avg\": 100.09958922863007\n },\n \"iterations_since_restore\": {\n \"max\": 2,\n \"min\": 1,\n \"avg\": 1.5,\n \"last\": 2,\n \"last-5-avg\": 1.5,\n \"last-10-avg\": 1.5\n }\n },\n \"_n_steps\": [\n 5,\n 10\n ],\n \"metric_n_steps\": {\n \"loss\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005959d000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294288c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b007494624308ad56abd5b8929f4094869452946807680d4308100804928f4d9f409486945294652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005959d000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294288c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b007494624308ad56abd5b8929f4094869452946807680d4308100804928f4d9f409486945294652e\"\n }\n },\n \"should_checkpoint\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059524000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294288888652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059524000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294288888652e\"\n }\n },\n \"done\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059524000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294288989652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059524000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294288989652e\"\n }\n },\n \"training_iteration\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059526000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294284b014b02652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059526000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294284b014b02652e\"\n }\n },\n \"time_this_iter_s\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059534000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b058694529428474051de592d00000047404ca019fa000000652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059534000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a8694529428474051de592d00000047404ca019fa000000652e\"\n }\n },\n \"time_total_s\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059534000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b058694529428474051de592d000000474060173315000000652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059534000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a8694529428474051de592d000000474060173315000000652e\"\n }\n },\n \"time_since_restore\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059534000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b058694529428474051de592d000000474060173315000000652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059534000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a8694529428474051de592d000000474060173315000000652e\"\n }\n },\n \"iterations_since_restore\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059526000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294284b014b02652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059526000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294284b014b02652e\"\n }\n }\n },\n \"checkpoint_manager\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"800595a6090000000000008c267261792e747261696e2e5f696e7465726e616c2e636865636b706f696e745f6d616e61676572948c125f436865636b706f696e744d616e616765729493942981947d94288c125f636865636b706f696e745f636f6e666967948c097261792e747261696e948c10436865636b706f696e74436f6e6669679493942981947d94288c0b6e756d5f746f5f6b656570944e8c1a636865636b706f696e745f73636f72655f617474726962757465944e8c16636865636b706f696e745f73636f72655f6f72646572948c036d6178948c14636865636b706f696e745f6672657175656e6379944b008c11636865636b706f696e745f61745f656e6494898c1a5f636865636b706f696e745f6b6565705f616c6c5f72616e6b73948c0a44455052454341544544948c1f5f636865636b706f696e745f75706c6f61645f66726f6d5f776f726b65727394681275628c135f636865636b706f696e745f726573756c7473945d94288c1b7261792e747261696e2e5f696e7465726e616c2e73657373696f6e948c0f5f547261696e696e67526573756c749493942981947d94288c0a636865636b706f696e749468068c0a436865636b706f696e749493942981947d94288c0470617468948cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f36626537393731375f32345f62617463685f73697a653d313032342c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d313032342c6465636f6465725f72745f6e756d5f6c617965723d322c64726f705f5f323032342d30362d32355f31392d31342d33342f636865636b706f696e745f303030303030948c0a66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c055f75756964948c0475756964948c04555549449493942981947d948c03696e74948a117534ee679e09c7a75841a69709c52f9900736275628c076d657472696373947d94288c046c6f7373948c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b007494624308ad56abd5b8929f4094869452948c0974696d657374616d70944aaffb7a668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303030948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b018c08747269616c5f6964948c083662653739373137948c0464617465948c13323032342d30362d32355f31392d31372d3335948c1074696d655f746869735f697465725f7394474051de592d0000008c0c74696d655f746f74616c5f7394474051de592d0000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b018c146465636f6465725f72745f6e756d5f6c61796572944b028c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b408c0a656e636f6465725f6666944d00088c0d6465636f6465725f72745f6666944d00048c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b088c0964726f705f7261746594473fe7d432e6852e928c026c7294473f7893b9c1c067128c0a62617463685f73697a65944d0004758c1274696d655f73696e63655f726573746f726594474051de592d0000008c18697465726174696f6e735f73696e63655f726573746f7265944b0175756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f36626537393731375f32345f62617463685f73697a653d313032342c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d313032342c6465636f6465725f72745f6e756d5f6c617965723d322c64726f705f5f323032342d30362d32355f31392d31342d33342f636865636b706f696e745f30303030303194682268257d946827897385945294682a682d2981947d9468308a11ef0301a808ba10b9e348ae9d26f051ed007362756268317d94288c046c6f7373946836683c4308100804928f4d9f4094869452948c0974696d657374616d70944ae8fb7a668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303031948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b028c08747269616c5f6964948c083662653739373137948c0464617465948c13323032342d30362d32355f31392d31382d3332948c1074696d655f746869735f697465725f739447404ca019fa0000008c0c74696d655f746f74616c5f73944740601733150000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b018c146465636f6465725f72745f6e756d5f6c61796572944b028c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b408c0a656e636f6465725f6666944d00088c0d6465636f6465725f72745f6666944d00048c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b088c0964726f705f7261746594473fe7d432e6852e928c026c7294473f7893b9c1c067128c0a62617463685f73697a65944d0004758c1274696d655f73696e63655f726573746f7265944740601733150000008c18697465726174696f6e735f73696e63655f726573746f7265944b02757562658c195f6c61746573745f636865636b706f696e745f726573756c7494686275622e\"\n }\n}"
+ ],
+ [
+ "{\n \"stub\": false,\n \"trainable_name\": \"train_model\",\n \"trial_id\": \"82396cf3\",\n \"storage\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059572040000000000008c1b7261792e747261696e2e5f696e7465726e616c2e73746f72616765948c0e53746f72616765436f6e746578749493942981947d94288c12637573746f6d5f66735f70726f766964656494898c1273746f726167655f6c6f63616c5f70617468948c2b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c7473948c136578706572696d656e745f6469725f6e616d65948c0f746573745f6578706572696d656e74948c0e747269616c5f6469725f6e616d65948c96747261696e5f6d6f64656c5f38323339366366335f32355f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d3531322c6465636f6465725f72745f6e756d5f6c617965723d322c64726f705f725f323032342d30362d32355f31392d31362d3233948c1863757272656e745f636865636b706f696e745f696e646578944b018c0b73796e635f636f6e666967948c097261792e747261696e948c0a53796e63436f6e6669679493942981947d94288c0b73796e635f706572696f64944d2c018c0c73796e635f74696d656f7574944d08078c0e73796e635f61727469666163747394898c1c73796e635f6172746966616374735f6f6e5f636865636b706f696e7494888c0a75706c6f61645f646972948c0a44455052454341544544948c0673796e6365729468188c1273796e635f6f6e5f636865636b706f696e7494681875628c1273746f726167655f66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c0f73746f726167655f66735f70617468948c452f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f7465737494681968008c115f46696c6573797374656d53796e6365729493942981947d9428681b682268134d2c0168144d08078c116c6173745f73796e635f75705f74696d65944741d99ebed10938d48c136c6173745f73796e635f646f776e5f74696d659447fff00000000000008c0d5f73796e635f70726f63657373944e8c0c5f63757272656e745f636d649468008c125f75706c6f61645f746f5f66735f706174689493947d94288c0a6c6f63616c5f70617468948c3b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c74732f746573745f6578706572696d656e74948c0266739468228c0766735f70617468948c552f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e74948c076578636c756465945d948c0e2a2f636865636b706f696e745f2a9461758694756275622e\"\n },\n \"config\": {\n \"encoder_num_layer\": 4,\n \"decoder_rt_num_layer\": 2,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 1024,\n \"encoder_ff\": 2048,\n \"decoder_rt_ff\": 512,\n \"decoder_int_ff\": 512,\n \"n_head\": 4,\n \"drop_rate\": 0.9507029678825294,\n \"lr\": 0.00025487621898754583,\n \"batch_size\": 2048\n },\n \"_Trial__unresolved_config\": {\n \"encoder_num_layer\": 4,\n \"decoder_rt_num_layer\": 2,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 1024,\n \"encoder_ff\": 2048,\n \"decoder_rt_ff\": 512,\n \"decoder_int_ff\": 512,\n \"n_head\": 4,\n \"drop_rate\": 0.9507029678825294,\n \"lr\": 0.00025487621898754583,\n \"batch_size\": 2048\n },\n \"evaluated_params\": {\n \"batch_size\": 2048,\n \"decoder_int_ff\": 512,\n \"decoder_int_num_layer\": 1,\n \"decoder_rt_ff\": 512,\n \"decoder_rt_num_layer\": 2,\n \"drop_rate\": 0.9507029678825294,\n \"embedding_dim\": 1024,\n \"encoder_ff\": 2048,\n \"encoder_num_layer\": 4,\n \"lr\": 0.00025487621898754583,\n \"n_head\": 4\n },\n \"experiment_tag\": \"25_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_rate=0.9507,embedding_dim=1024,encoder_ff=2048,encoder_num_layer=4,lr=0.0003,n_head=4\",\n \"stopping_criterion\": {},\n \"_setup_default_resource\": true,\n \"_default_placement_group_factory\": \"80054e2e\",\n \"placement_group_factory\": \"800595ba000000000000008c237261792e74756e652e657865637574696f6e2e706c6163656d656e745f67726f757073948c15506c6163656d656e7447726f7570466163746f72799493942981947d94288c085f62756e646c6573945d947d94288c03435055944740540000000000008c034750559447401000000000000075618c155f686561645f62756e646c655f69735f656d70747994898c095f7374726174656779948c045041434b948c055f6172677394298c075f6b7761726773947d9475622e\",\n \"log_to_file\": [\n null,\n null\n ],\n \"max_failures\": 0,\n \"_default_result_or_future\": null,\n \"export_formats\": [],\n \"status\": \"PAUSED\",\n \"relative_logdir\": \"train_model_82396cf3_25_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_19-16-23\",\n \"trial_name_creator\": null,\n \"trial_dirname_creator\": null,\n \"custom_trial_name\": null,\n \"custom_dirname\": null,\n \"restore_path\": null,\n \"_restore_checkpoint_result\": null,\n \"_state_json\": null,\n \"results\": \"80054e2e\",\n \"extra_arg\": \"80054e2e\",\n \"_resources\": \"80054e2e\"\n}",
+ "{\n \"start_time\": 1719335912.4933662,\n \"num_failures\": 0,\n \"num_failures_after_restore\": 0,\n \"error_filename\": null,\n \"pickled_error_filename\": null,\n \"last_result\": {\n \"loss\": 2187.193873608206,\n \"timestamp\": 1719337142,\n \"checkpoint_dir_name\": \"checkpoint_000001\",\n \"should_checkpoint\": true,\n \"done\": false,\n \"training_iteration\": 2,\n \"trial_id\": \"82396cf3\",\n \"date\": \"2024-06-25_19-39-03\",\n \"time_this_iter_s\": 608.1205008029938,\n \"time_total_s\": 1230.5634942054749,\n \"pid\": 2195253,\n \"hostname\": \"r8i6n2\",\n \"node_ip\": \"10.159.28.60\",\n \"config\": {\n \"encoder_num_layer\": 4,\n \"decoder_rt_num_layer\": 2,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 1024,\n \"encoder_ff\": 2048,\n \"decoder_rt_ff\": 512,\n \"decoder_int_ff\": 512,\n \"n_head\": 4,\n \"drop_rate\": 0.9507029678825294,\n \"lr\": 0.00025487621898754583,\n \"batch_size\": 2048\n },\n \"time_since_restore\": 1230.5634942054749,\n \"iterations_since_restore\": 2,\n \"experiment_tag\": \"25_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_rate=0.9507,embedding_dim=1024,encoder_ff=2048,encoder_num_layer=4,lr=0.0003,n_head=4\"\n },\n \"last_result_time\": 1719337143.0662968,\n \"metric_analysis\": {\n \"loss\": {\n \"max\": 2187.193873608206,\n \"min\": 2178.1833553764764,\n \"avg\": 2182.688614492341,\n \"last\": 2187.193873608206,\n \"last-5-avg\": 2182.688614492341,\n \"last-10-avg\": 2182.688614492341\n },\n \"should_checkpoint\": {\n \"max\": true,\n \"min\": true,\n \"avg\": 1.0,\n \"last\": true,\n \"last-5-avg\": 1.0,\n \"last-10-avg\": 1.0\n },\n \"done\": {\n \"max\": false,\n \"min\": false,\n \"avg\": 0.0,\n \"last\": false,\n \"last-5-avg\": 0.0,\n \"last-10-avg\": 0.0\n },\n \"training_iteration\": {\n \"max\": 2,\n \"min\": 1,\n \"avg\": 1.5,\n \"last\": 2,\n \"last-5-avg\": 1.5,\n \"last-10-avg\": 1.5\n },\n \"time_this_iter_s\": {\n \"max\": 622.4429934024811,\n \"min\": 608.1205008029938,\n \"avg\": 615.2817471027374,\n \"last\": 608.1205008029938,\n \"last-5-avg\": 615.2817471027374,\n \"last-10-avg\": 615.2817471027374\n },\n \"time_total_s\": {\n \"max\": 1230.5634942054749,\n \"min\": 622.4429934024811,\n \"avg\": 926.503243803978,\n \"last\": 1230.5634942054749,\n \"last-5-avg\": 926.503243803978,\n \"last-10-avg\": 926.503243803978\n },\n \"time_since_restore\": {\n \"max\": 1230.5634942054749,\n \"min\": 622.4429934024811,\n \"avg\": 926.503243803978,\n \"last\": 1230.5634942054749,\n \"last-5-avg\": 926.503243803978,\n \"last-10-avg\": 926.503243803978\n },\n \"iterations_since_restore\": {\n \"max\": 2,\n \"min\": 1,\n \"avg\": 1.5,\n \"last\": 2,\n \"last-5-avg\": 1.5,\n \"last-10-avg\": 1.5\n }\n },\n \"_n_steps\": [\n 5,\n 10\n ],\n \"metric_n_steps\": {\n \"loss\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005959d000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294288c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b0074946243080683c1e05d04a14094869452946807680d43089bcd66436316a1409486945294652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005959d000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294288c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b0074946243080683c1e05d04a14094869452946807680d43089bcd66436316a1409486945294652e\"\n }\n },\n \"should_checkpoint\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059524000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294288888652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059524000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294288888652e\"\n }\n },\n \"done\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059524000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294288989652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059524000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294288989652e\"\n }\n },\n \"training_iteration\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059526000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294284b014b02652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059526000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294284b014b02652e\"\n }\n },\n \"time_this_iter_s\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059534000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b058694529428474083738b4020000047408300f6c9200000652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059534000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a8694529428474083738b4020000047408300f6c9200000652e\"\n }\n },\n \"time_total_s\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059534000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b058694529428474083738b402000004740933a4104a00000652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059534000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a8694529428474083738b402000004740933a4104a00000652e\"\n }\n },\n \"time_since_restore\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059534000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b058694529428474083738b402000004740933a4104a00000652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059534000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a8694529428474083738b402000004740933a4104a00000652e\"\n }\n },\n \"iterations_since_restore\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059526000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294284b014b02652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059526000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294284b014b02652e\"\n }\n }\n },\n \"checkpoint_manager\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"800595a7090000000000008c267261792e747261696e2e5f696e7465726e616c2e636865636b706f696e745f6d616e61676572948c125f436865636b706f696e744d616e616765729493942981947d94288c125f636865636b706f696e745f636f6e666967948c097261792e747261696e948c10436865636b706f696e74436f6e6669679493942981947d94288c0b6e756d5f746f5f6b656570944e8c1a636865636b706f696e745f73636f72655f617474726962757465944e8c16636865636b706f696e745f73636f72655f6f72646572948c036d6178948c14636865636b706f696e745f6672657175656e6379944b008c11636865636b706f696e745f61745f656e6494898c1a5f636865636b706f696e745f6b6565705f616c6c5f72616e6b73948c0a44455052454341544544948c1f5f636865636b706f696e745f75706c6f61645f66726f6d5f776f726b65727394681275628c135f636865636b706f696e745f726573756c7473945d94288c1b7261792e747261696e2e5f696e7465726e616c2e73657373696f6e948c0f5f547261696e696e67526573756c749493942981947d94288c0a636865636b706f696e749468068c0a436865636b706f696e749493942981947d94288c0470617468948cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f38323339366366335f32355f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d3531322c6465636f6465725f72745f6e756d5f6c617965723d322c64726f705f725f323032342d30362d32355f31392d31362d32332f636865636b706f696e745f303030303030948c0a66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c055f75756964948c0475756964948c04555549449493942981947d948c03696e74948a10b0521178b5d8959c1040a702210f1421736275628c076d657472696373947d94288c046c6f7373948c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b0074946243080683c1e05d04a14094869452948c0974696d657374616d70944a56fe7a668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303030948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b018c08747269616c5f6964948c083832333936636633948c0464617465948c13323032342d30362d32355f31392d32382d3534948c1074696d655f746869735f697465725f7394474083738b402000008c0c74696d655f746f74616c5f7394474083738b402000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b048c146465636f6465725f72745f6e756d5f6c61796572944b028c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944d00048c0a656e636f6465725f6666944d00088c0d6465636f6465725f72745f6666944d00028c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b048c0964726f705f7261746594473fee6c28a1687fd08c026c7294473f30b41d066b9e998c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f726594474083738b402000008c18697465726174696f6e735f73696e63655f726573746f7265944b0175756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f38323339366366335f32355f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d3531322c6465636f6465725f72745f6e756d5f6c617965723d322c64726f705f725f323032342d30362d32355f31392d31362d32332f636865636b706f696e745f30303030303194682268257d946827897385945294682a682d2981947d9468308a1109d9a6b3fe1045b69f4feea938dd89bf007362756268317d94288c046c6f7373946836683c43089bcd66436316a14094869452948c0974696d657374616d70944ab6007b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303031948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b028c08747269616c5f6964948c083832333936636633948c0464617465948c13323032342d30362d32355f31392d33392d3033948c1074696d655f746869735f697465725f739447408300f6c92000008c0c74696d655f746f74616c5f73944740933a4104a000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b048c146465636f6465725f72745f6e756d5f6c61796572944b028c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944d00048c0a656e636f6465725f6666944d00088c0d6465636f6465725f72745f6666944d00028c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b048c0964726f705f7261746594473fee6c28a1687fd08c026c7294473f30b41d066b9e998c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f7265944740933a4104a000008c18697465726174696f6e735f73696e63655f726573746f7265944b02757562658c195f6c61746573745f636865636b706f696e745f726573756c7494686275622e\"\n }\n}"
+ ],
+ [
+ "{\n \"stub\": false,\n \"trainable_name\": \"train_model\",\n \"trial_id\": \"e31d7ca7\",\n \"storage\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059572040000000000008c1b7261792e747261696e2e5f696e7465726e616c2e73746f72616765948c0e53746f72616765436f6e746578749493942981947d94288c12637573746f6d5f66735f70726f766964656494898c1273746f726167655f6c6f63616c5f70617468948c2b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c7473948c136578706572696d656e745f6469725f6e616d65948c0f746573745f6578706572696d656e74948c0e747269616c5f6469725f6e616d65948c96747261696e5f6d6f64656c5f65333164376361375f32365f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d323034382c6465636f6465725f72745f6e756d5f6c617965723d322c64726f705f5f323032342d30362d32355f31392d31382d3332948c1863757272656e745f636865636b706f696e745f696e646578944b018c0b73796e635f636f6e666967948c097261792e747261696e948c0a53796e63436f6e6669679493942981947d94288c0b73796e635f706572696f64944d2c018c0c73796e635f74696d656f7574944d08078c0e73796e635f61727469666163747394898c1c73796e635f6172746966616374735f6f6e5f636865636b706f696e7494888c0a75706c6f61645f646972948c0a44455052454341544544948c0673796e6365729468188c1273796e635f6f6e5f636865636b706f696e7494681875628c1273746f726167655f66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c0f73746f726167655f66735f70617468948c452f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f7465737494681968008c115f46696c6573797374656d53796e6365729493942981947d9428681b682268134d2c0168144d08078c116c6173745f73796e635f75705f74696d65944741d99ebed10938d48c136c6173745f73796e635f646f776e5f74696d659447fff00000000000008c0d5f73796e635f70726f63657373944e8c0c5f63757272656e745f636d649468008c125f75706c6f61645f746f5f66735f706174689493947d94288c0a6c6f63616c5f70617468948c3b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c74732f746573745f6578706572696d656e74948c0266739468228c0766735f70617468948c552f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e74948c076578636c756465945d948c0e2a2f636865636b706f696e745f2a9461758694756275622e\"\n },\n \"config\": {\n \"encoder_num_layer\": 4,\n \"decoder_rt_num_layer\": 2,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 64,\n \"encoder_ff\": 2048,\n \"decoder_rt_ff\": 2048,\n \"decoder_int_ff\": 512,\n \"n_head\": 4,\n \"drop_rate\": 0.7500609138892806,\n \"lr\": 0.0713230632683364,\n \"batch_size\": 2048\n },\n \"_Trial__unresolved_config\": {\n \"encoder_num_layer\": 4,\n \"decoder_rt_num_layer\": 2,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 64,\n \"encoder_ff\": 2048,\n \"decoder_rt_ff\": 2048,\n \"decoder_int_ff\": 512,\n \"n_head\": 4,\n \"drop_rate\": 0.7500609138892806,\n \"lr\": 0.0713230632683364,\n \"batch_size\": 2048\n },\n \"evaluated_params\": {\n \"batch_size\": 2048,\n \"decoder_int_ff\": 512,\n \"decoder_int_num_layer\": 1,\n \"decoder_rt_ff\": 2048,\n \"decoder_rt_num_layer\": 2,\n \"drop_rate\": 0.7500609138892806,\n \"embedding_dim\": 64,\n \"encoder_ff\": 2048,\n \"encoder_num_layer\": 4,\n \"lr\": 0.0713230632683364,\n \"n_head\": 4\n },\n \"experiment_tag\": \"26_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=2,drop_rate=0.7501,embedding_dim=64,encoder_ff=2048,encoder_num_layer=4,lr=0.0713,n_head=4\",\n \"stopping_criterion\": {},\n \"_setup_default_resource\": true,\n \"_default_placement_group_factory\": \"80054e2e\",\n \"placement_group_factory\": \"800595ba000000000000008c237261792e74756e652e657865637574696f6e2e706c6163656d656e745f67726f757073948c15506c6163656d656e7447726f7570466163746f72799493942981947d94288c085f62756e646c6573945d947d94288c03435055944740540000000000008c034750559447401000000000000075618c155f686561645f62756e646c655f69735f656d70747994898c095f7374726174656779948c045041434b948c055f6172677394298c075f6b7761726773947d9475622e\",\n \"log_to_file\": [\n null,\n null\n ],\n \"max_failures\": 0,\n \"_default_result_or_future\": null,\n \"export_formats\": [],\n \"status\": \"TERMINATED\",\n \"relative_logdir\": \"train_model_e31d7ca7_26_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=2,drop__2024-06-25_19-18-32\",\n \"trial_name_creator\": null,\n \"trial_dirname_creator\": null,\n \"custom_trial_name\": null,\n \"custom_dirname\": null,\n \"restore_path\": null,\n \"_restore_checkpoint_result\": null,\n \"_state_json\": null,\n \"results\": \"80054e2e\",\n \"extra_arg\": \"80054e2e\",\n \"_resources\": \"80054e2e\"\n}",
+ "{\n \"start_time\": 1719337143.2143147,\n \"num_failures\": 0,\n \"num_failures_after_restore\": 0,\n \"error_filename\": null,\n \"pickled_error_filename\": null,\n \"last_result\": {\n \"loss\": 2698.886155496432,\n \"timestamp\": 1719337295,\n \"checkpoint_dir_name\": \"checkpoint_000001\",\n \"should_checkpoint\": true,\n \"done\": true,\n \"training_iteration\": 2,\n \"trial_id\": \"e31d7ca7\",\n \"date\": \"2024-06-25_19-41-35\",\n \"time_this_iter_s\": 69.29531860351562,\n \"time_total_s\": 152.63765025138855,\n \"pid\": 2195253,\n \"hostname\": \"r8i6n2\",\n \"node_ip\": \"10.159.28.60\",\n \"config\": {\n \"encoder_num_layer\": 4,\n \"decoder_rt_num_layer\": 2,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 64,\n \"encoder_ff\": 2048,\n \"decoder_rt_ff\": 2048,\n \"decoder_int_ff\": 512,\n \"n_head\": 4,\n \"drop_rate\": 0.7500609138892806,\n \"lr\": 0.0713230632683364,\n \"batch_size\": 2048\n },\n \"time_since_restore\": 152.63765025138855,\n \"iterations_since_restore\": 2,\n \"experiment_tag\": \"26_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=2,drop_rate=0.7501,embedding_dim=64,encoder_ff=2048,encoder_num_layer=4,lr=0.0713,n_head=4\"\n },\n \"last_result_time\": 1719337295.8805537,\n \"metric_analysis\": {\n \"loss\": {\n \"max\": 3842.66859736405,\n \"min\": 2698.886155496432,\n \"avg\": 3270.777376430241,\n \"last\": 2698.886155496432,\n \"last-5-avg\": 3270.777376430241,\n \"last-10-avg\": 3270.777376430241\n },\n \"should_checkpoint\": {\n \"max\": true,\n \"min\": true,\n \"avg\": 1.0,\n \"last\": true,\n \"last-5-avg\": 1.0,\n \"last-10-avg\": 1.0\n },\n \"done\": {\n \"max\": true,\n \"min\": false,\n \"avg\": 0.5,\n \"last\": true,\n \"last-5-avg\": 0.5,\n \"last-10-avg\": 0.5\n },\n \"training_iteration\": {\n \"max\": 2,\n \"min\": 1,\n \"avg\": 1.5,\n \"last\": 2,\n \"last-5-avg\": 1.5,\n \"last-10-avg\": 1.5\n },\n \"time_this_iter_s\": {\n \"max\": 83.34233164787292,\n \"min\": 69.29531860351562,\n \"avg\": 76.31882512569427,\n \"last\": 69.29531860351562,\n \"last-5-avg\": 76.31882512569427,\n \"last-10-avg\": 76.31882512569427\n },\n \"time_total_s\": {\n \"max\": 152.63765025138855,\n \"min\": 83.34233164787292,\n \"avg\": 117.98999094963074,\n \"last\": 152.63765025138855,\n \"last-5-avg\": 117.98999094963074,\n \"last-10-avg\": 117.98999094963074\n },\n \"time_since_restore\": {\n \"max\": 152.63765025138855,\n \"min\": 83.34233164787292,\n \"avg\": 117.98999094963074,\n \"last\": 152.63765025138855,\n \"last-5-avg\": 117.98999094963074,\n \"last-10-avg\": 117.98999094963074\n },\n \"iterations_since_restore\": {\n \"max\": 2,\n \"min\": 1,\n \"avg\": 1.5,\n \"last\": 2,\n \"last-5-avg\": 1.5,\n \"last-10-avg\": 1.5\n }\n },\n \"_n_steps\": [\n 5,\n 10\n ],\n \"metric_n_steps\": {\n \"loss\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005959d000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294288c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b00749462430893c964525605ae4094869452946807680d4308b1582cb6c515a5409486945294652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005959d000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294288c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b00749462430893c964525605ae4094869452946807680d4308b1582cb6c515a5409486945294652e\"\n }\n },\n \"should_checkpoint\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059524000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294288888652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059524000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294288888652e\"\n }\n },\n \"done\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059524000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294288988652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059524000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294288988652e\"\n }\n },\n \"training_iteration\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059526000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294284b014b02652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059526000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294284b014b02652e\"\n }\n },\n \"time_this_iter_s\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059534000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b058694529428474054d5e8c300000047405152e680000000652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059534000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a8694529428474054d5e8c300000047405152e680000000652e\"\n }\n },\n \"time_total_s\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059534000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b058694529428474054d5e8c30000004740631467a1800000652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059534000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a8694529428474054d5e8c30000004740631467a1800000652e\"\n }\n },\n \"time_since_restore\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059534000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b058694529428474054d5e8c30000004740631467a1800000652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059534000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a8694529428474054d5e8c30000004740631467a1800000652e\"\n }\n },\n \"iterations_since_restore\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059526000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294284b014b02652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059526000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294284b014b02652e\"\n }\n }\n },\n \"checkpoint_manager\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"800595a5090000000000008c267261792e747261696e2e5f696e7465726e616c2e636865636b706f696e745f6d616e61676572948c125f436865636b706f696e744d616e616765729493942981947d94288c125f636865636b706f696e745f636f6e666967948c097261792e747261696e948c10436865636b706f696e74436f6e6669679493942981947d94288c0b6e756d5f746f5f6b656570944e8c1a636865636b706f696e745f73636f72655f617474726962757465944e8c16636865636b706f696e745f73636f72655f6f72646572948c036d6178948c14636865636b706f696e745f6672657175656e6379944b008c11636865636b706f696e745f61745f656e6494898c1a5f636865636b706f696e745f6b6565705f616c6c5f72616e6b73948c0a44455052454341544544948c1f5f636865636b706f696e745f75706c6f61645f66726f6d5f776f726b65727394681275628c135f636865636b706f696e745f726573756c7473945d94288c1b7261792e747261696e2e5f696e7465726e616c2e73657373696f6e948c0f5f547261696e696e67526573756c749493942981947d94288c0a636865636b706f696e749468068c0a436865636b706f696e749493942981947d94288c0470617468948cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f65333164376361375f32365f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d323034382c6465636f6465725f72745f6e756d5f6c617965723d322c64726f705f5f323032342d30362d32355f31392d31382d33322f636865636b706f696e745f303030303030948c0a66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c055f75756964948c0475756964948c04555549449493942981947d948c03696e74948a1069f0f04dc7e8f8a8d14373899995645a736275628c076d657472696373947d94288c046c6f7373948c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b00749462430893c964525605ae4094869452948c0974696d657374616d70944a0a017b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303030948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b018c08747269616c5f6964948c086533316437636137948c0464617465948c13323032342d30362d32355f31392d34302d3236948c1074696d655f746869735f697465725f7394474054d5e8c30000008c0c74696d655f746f74616c5f7394474054d5e8c30000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b048c146465636f6465725f72745f6e756d5f6c61796572944b028c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b408c0a656e636f6465725f6666944d00088c0d6465636f6465725f72745f6666944d00088c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b048c0964726f705f7261746594473fe8007fbee531d48c026c7294473fb2423a703023a28c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f726594474054d5e8c30000008c18697465726174696f6e735f73696e63655f726573746f7265944b0175756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f65333164376361375f32365f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d323034382c6465636f6465725f72745f6e756d5f6c617965723d322c64726f705f5f323032342d30362d32355f31392d31382d33322f636865636b706f696e745f30303030303194682268257d946827897385945294682a682d2981947d9468308a1116b3c5101b040fad064e3d871bb62198007362756268317d94288c046c6f7373946836683c4308b1582cb6c515a54094869452948c0974696d657374616d70944a4f017b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303031948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b028c08747269616c5f6964948c086533316437636137948c0464617465948c13323032342d30362d32355f31392d34312d3335948c1074696d655f746869735f697465725f739447405152e6800000008c0c74696d655f746f74616c5f73944740631467a18000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b048c146465636f6465725f72745f6e756d5f6c61796572944b028c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b408c0a656e636f6465725f6666944d00088c0d6465636f6465725f72745f6666944d00088c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b048c0964726f705f7261746594473fe8007fbee531d48c026c7294473fb2423a703023a28c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f7265944740631467a18000008c18697465726174696f6e735f73696e63655f726573746f7265944b02757562658c195f6c61746573745f636865636b706f696e745f726573756c7494686275622e\"\n }\n}"
+ ],
+ [
+ "{\n \"stub\": false,\n \"trainable_name\": \"train_model\",\n \"trial_id\": \"18c11cbc\",\n \"storage\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059572040000000000008c1b7261792e747261696e2e5f696e7465726e616c2e73746f72616765948c0e53746f72616765436f6e746578749493942981947d94288c12637573746f6d5f66735f70726f766964656494898c1273746f726167655f6c6f63616c5f70617468948c2b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c7473948c136578706572696d656e745f6469725f6e616d65948c0f746573745f6578706572696d656e74948c0e747269616c5f6469725f6e616d65948c96747261696e5f6d6f64656c5f31386331316362635f32375f62617463685f73697a653d313032342c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d323034382c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f5f323032342d30362d32355f31392d33392d3033948c1863757272656e745f636865636b706f696e745f696e646578944b138c0b73796e635f636f6e666967948c097261792e747261696e948c0a53796e63436f6e6669679493942981947d94288c0b73796e635f706572696f64944d2c018c0c73796e635f74696d656f7574944d08078c0e73796e635f61727469666163747394898c1c73796e635f6172746966616374735f6f6e5f636865636b706f696e7494888c0a75706c6f61645f646972948c0a44455052454341544544948c0673796e6365729468188c1273796e635f6f6e5f636865636b706f696e7494681875628c1273746f726167655f66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c0f73746f726167655f66735f70617468948c452f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f7465737494681968008c115f46696c6573797374656d53796e6365729493942981947d9428681b682268134d2c0168144d08078c116c6173745f73796e635f75705f74696d65944741d99ebfff24893c8c136c6173745f73796e635f646f776e5f74696d659447fff00000000000008c0d5f73796e635f70726f63657373944e8c0c5f63757272656e745f636d649468008c125f75706c6f61645f746f5f66735f706174689493947d94288c0a6c6f63616c5f70617468948c3b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c74732f746573745f6578706572696d656e74948c0266739468228c0766735f70617468948c552f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e74948c076578636c756465945d948c0e2a2f636865636b706f696e745f2a9461758694756275622e\"\n },\n \"config\": {\n \"encoder_num_layer\": 4,\n \"decoder_rt_num_layer\": 1,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 16,\n \"encoder_ff\": 512,\n \"decoder_rt_ff\": 2048,\n \"decoder_int_ff\": 512,\n \"n_head\": 8,\n \"drop_rate\": 0.3688667410196246,\n \"lr\": 0.0018036063294034547,\n \"batch_size\": 1024\n },\n \"_Trial__unresolved_config\": {\n \"encoder_num_layer\": 4,\n \"decoder_rt_num_layer\": 1,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 16,\n \"encoder_ff\": 512,\n \"decoder_rt_ff\": 2048,\n \"decoder_int_ff\": 512,\n \"n_head\": 8,\n \"drop_rate\": 0.3688667410196246,\n \"lr\": 0.0018036063294034547,\n \"batch_size\": 1024\n },\n \"evaluated_params\": {\n \"batch_size\": 1024,\n \"decoder_int_ff\": 512,\n \"decoder_int_num_layer\": 1,\n \"decoder_rt_ff\": 2048,\n \"decoder_rt_num_layer\": 1,\n \"drop_rate\": 0.3688667410196246,\n \"embedding_dim\": 16,\n \"encoder_ff\": 512,\n \"encoder_num_layer\": 4,\n \"lr\": 0.0018036063294034547,\n \"n_head\": 8\n },\n \"experiment_tag\": \"27_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop_rate=0.3689,embedding_dim=16,encoder_ff=512,encoder_num_layer=4,lr=0.0018,n_head=8\",\n \"stopping_criterion\": {},\n \"_setup_default_resource\": true,\n \"_default_placement_group_factory\": \"80054e2e\",\n \"placement_group_factory\": \"800595ba000000000000008c237261792e74756e652e657865637574696f6e2e706c6163656d656e745f67726f757073948c15506c6163656d656e7447726f7570466163746f72799493942981947d94288c085f62756e646c6573945d947d94288c03435055944740540000000000008c034750559447401000000000000075618c155f686561645f62756e646c655f69735f656d70747994898c095f7374726174656779948c045041434b948c055f6172677394298c075f6b7761726773947d9475622e\",\n \"log_to_file\": [\n null,\n null\n ],\n \"max_failures\": 0,\n \"_default_result_or_future\": null,\n \"export_formats\": [],\n \"status\": \"TERMINATED\",\n \"relative_logdir\": \"train_model_18c11cbc_27_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop__2024-06-25_19-39-03\",\n \"trial_name_creator\": null,\n \"trial_dirname_creator\": null,\n \"custom_trial_name\": null,\n \"custom_dirname\": null,\n \"restore_path\": null,\n \"_restore_checkpoint_result\": null,\n \"_state_json\": null,\n \"results\": \"80054e2e\",\n \"extra_arg\": \"80054e2e\",\n \"_resources\": \"80054e2e\"\n}",
+ "{\n \"start_time\": 1719346234.668183,\n \"num_failures\": 0,\n \"num_failures_after_restore\": 0,\n \"error_filename\": null,\n \"pickled_error_filename\": null,\n \"last_result\": {\n \"loss\": 118.33789474006713,\n \"timestamp\": 1719357871,\n \"checkpoint_dir_name\": \"checkpoint_000019\",\n \"should_checkpoint\": true,\n \"done\": true,\n \"training_iteration\": 20,\n \"trial_id\": \"18c11cbc\",\n \"date\": \"2024-06-26_01-24-31\",\n \"time_this_iter_s\": 59.315247774124146,\n \"time_total_s\": 1226.7515268325806,\n \"pid\": 2850562,\n \"hostname\": \"r8i6n2\",\n \"node_ip\": \"10.159.28.60\",\n \"config\": {\n \"encoder_num_layer\": 4,\n \"decoder_rt_num_layer\": 1,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 16,\n \"encoder_ff\": 512,\n \"decoder_rt_ff\": 2048,\n \"decoder_int_ff\": 512,\n \"n_head\": 8,\n \"drop_rate\": 0.3688667410196246,\n \"lr\": 0.0018036063294034547,\n \"batch_size\": 1024\n },\n \"time_since_restore\": 606.2512559890747,\n \"iterations_since_restore\": 10,\n \"experiment_tag\": \"27_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop_rate=0.3689,embedding_dim=16,encoder_ff=512,encoder_num_layer=4,lr=0.0018,n_head=8\"\n },\n \"last_result_time\": 1719357871.9733982,\n \"metric_analysis\": {\n \"loss\": {\n \"max\": 266.66766681821326,\n \"min\": 118.33789474006713,\n \"avg\": 143.6234760126729,\n \"last\": 118.33789474006713,\n \"last-5-avg\": 120.82310969134953,\n \"last-10-avg\": 123.46259958913008\n },\n \"should_checkpoint\": {\n \"max\": true,\n \"min\": true,\n \"avg\": 1.0,\n \"last\": true,\n \"last-5-avg\": 1.0,\n \"last-10-avg\": 1.0\n },\n \"done\": {\n \"max\": false,\n \"min\": false,\n \"avg\": 0.0,\n \"last\": false,\n \"last-5-avg\": 0.0,\n \"last-10-avg\": 0.0\n },\n \"training_iteration\": {\n \"max\": 20,\n \"min\": 1,\n \"avg\": 10.5,\n \"last\": 20,\n \"last-5-avg\": 18.0,\n \"last-10-avg\": 15.5\n },\n \"time_this_iter_s\": {\n \"max\": 74.13704776763916,\n \"min\": 58.23622250556946,\n \"avg\": 61.33757634162903,\n \"last\": 59.315247774124146,\n \"last-5-avg\": 59.27494025230408,\n \"last-10-avg\": 60.62512559890747\n },\n \"time_total_s\": {\n \"max\": 1226.7515268325806,\n \"min\": 74.13704776763916,\n \"avg\": 654.1219100713729,\n \"last\": 1226.7515268325806,\n \"last-5-avg\": 1108.3144157409668,\n \"last-10-avg\": 960.0894476175308\n },\n \"time_since_restore\": {\n \"max\": 606.2512559890747,\n \"min\": 72.75415802001953,\n \"avg\": 265.71221474409106,\n \"last\": 606.2512559890747,\n \"last-5-avg\": 487.81414489746095,\n \"last-10-avg\": 339.58917677402496\n },\n \"iterations_since_restore\": {\n \"max\": 10,\n \"min\": 1,\n \"avg\": 4.25,\n \"last\": 10,\n \"last-5-avg\": 8.0,\n \"last-10-avg\": 5.5\n }\n },\n \"_n_steps\": [\n 5,\n 10\n ],\n \"metric_n_steps\": {\n \"loss\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"800595d6000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294288c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b007494624308d8ebf5c27a5c5e4094869452946807680d4308613018cc31075f4094869452946807680d4308241289b4d7615e4094869452946807680d4308040281d040ac5d4094869452946807680d43080a854211a0955d409486945294652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059535010000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294288c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b00749462430897cbe50acf00604094869452946807680d4308e2703894b34f604094869452946807680d43089fcfe793d4635f4094869452946807680d430843a150183a005f4094869452946807680d43080a854241979b5e4094869452946807680d4308d8ebf5c27a5c5e4094869452946807680d4308613018cc31075f4094869452946807680d4308241289b4d7615e4094869452946807680d4308040281d040ac5d4094869452946807680d43080a854211a0955d409486945294652e\"\n }\n },\n \"should_checkpoint\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059527000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294288888888888652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952c000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452942888888888888888888888652e\"\n }\n },\n \"done\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059527000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294288989898989652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952c000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452942889898989898989898989652e\"\n }\n },\n \"training_iteration\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952c000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294284b104b114b124b134b14652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059536000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294284b0b4b0c4b0d4b0e4b0f4b104b114b124b134b14652e\"\n }\n },\n \"time_this_iter_s\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005954f000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452942847404dcea81600000047404d92bc7200000047404d8f0b3e00000047404d972c6600000047404da85a0a000000652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005957c000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452942847405230442000000047404dac917e00000047404da1a58400000047404dadc47e00000047404d93af3200000047404dcea81600000047404d92bc7200000047404d8f0b3e00000047404d972c6600000047404da85a0a000000652e\"\n }\n },\n \"time_total_s\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005954f000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452942847408eefee3e800000474090648d02d0000047409151055cc000004740923dbebff000004740932b0190400000652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005957c000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a8694529428474085aa091200000047408784d229e000004740895eec8220000047408b39c8ca00000047408d1303bd20000047408eefee3e800000474090648d02d0000047409151055cc000004740923dbebff000004740932b0190400000652e\"\n }\n },\n \"time_since_restore\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005954f000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452942847407717db6100000047407aca32ef40000047407e7c1457000000474081177cf1e00000474082f20292800000652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005957c000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452942847405230442000000047406083466f800000474067ebafd080000047406f5720f00000004740735e065e40000047407717db6100000047407aca32ef40000047407e7c1457000000474081177cf1e00000474082f20292800000652e\"\n }\n },\n \"iterations_since_restore\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952c000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294284b064b074b084b094b0a652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059536000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294284b014b024b034b044b054b064b074b084b094b0a652e\"\n }\n }\n },\n \"checkpoint_manager\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059589490000000000008c267261792e747261696e2e5f696e7465726e616c2e636865636b706f696e745f6d616e61676572948c125f436865636b706f696e744d616e616765729493942981947d94288c125f636865636b706f696e745f636f6e666967948c097261792e747261696e948c10436865636b706f696e74436f6e6669679493942981947d94288c0b6e756d5f746f5f6b656570944e8c1a636865636b706f696e745f73636f72655f617474726962757465944e8c16636865636b706f696e745f73636f72655f6f72646572948c036d6178948c14636865636b706f696e745f6672657175656e6379944b008c11636865636b706f696e745f61745f656e6494898c1a5f636865636b706f696e745f6b6565705f616c6c5f72616e6b73948c0a44455052454341544544948c1f5f636865636b706f696e745f75706c6f61645f66726f6d5f776f726b65727394681275628c135f636865636b706f696e745f726573756c7473945d94288c1b7261792e747261696e2e5f696e7465726e616c2e73657373696f6e948c0f5f547261696e696e67526573756c749493942981947d94288c0a636865636b706f696e749468068c0a436865636b706f696e749493942981947d94288c0470617468948cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f31386331316362635f32375f62617463685f73697a653d313032342c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d323034382c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f5f323032342d30362d32355f31392d33392d30332f636865636b706f696e745f303030303030948c0a66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c055f75756964948c0475756964948c04555549449493942981947d948c03696e74948a1086df3ed6b22978aaa74314407424f10b736275628c076d657472696373947d94288c046c6f7373948c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b0074946243089bcd66c3aeaa704094869452948c0974696d657374616d70944a84247b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303030948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b018c08747269616c5f6964948c083138633131636263948c0464617465948c13323032342d30362d32355f32322d31312d3438948c1074696d655f746869735f697465725f739447405288c5640000008c0c74696d655f746f74616c5f739447405288c5640000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b048c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b108c0a656e636f6465725f6666944d00028c0d6465636f6465725f72745f6666944d00088c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b088c0964726f705f7261746594473fd79b833f50ba9a8c026c7294473f5d8cdf8cc701de8c0a62617463685f73697a65944d0004758c1274696d655f73696e63655f726573746f72659447405288c5640000008c18697465726174696f6e735f73696e63655f726573746f7265944b0175756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f31386331316362635f32375f62617463685f73697a653d313032342c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d323034382c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f5f323032342d30362d32355f31392d33392d30332f636865636b706f696e745f30303030303194682268257d946827897385945294682a682d2981947d9468308a11a000894b3dda7f875b4e024ae8b9f2eb007362756268317d94288c046c6f7373946836683c430853a9548a88fb674094869452948c0974696d657374616d70944abf247b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303031948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b028c08747269616c5f6964948c083138633131636263948c0464617465948c13323032342d30362d32355f32322d31322d3437948c1074696d655f746869735f697465725f739447404d912d960000008c0c74696d655f746f74616c5f7394474060a8ae178000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b048c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b108c0a656e636f6465725f6666944d00028c0d6465636f6465725f72745f6666944d00088c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b088c0964726f705f7261746594473fd79b833f50ba9a8c026c7294473f5d8cdf8cc701de8c0a62617463685f73697a65944d0004758c1274696d655f73696e63655f726573746f726594474060a8ae178000008c18697465726174696f6e735f73696e63655f726573746f7265944b0275756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f31386331316362635f32375f62617463685f73697a653d313032342c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d323034382c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f5f323032342d30362d32355f31392d33392d30332f636865636b706f696e745f30303030303294682268257d946827897385945294682a682d2981947d9468308a11ef4b086337798fab724707bc8d770398007362756268317d94288c046c6f7373946836683c4308180c06031189654094869452948c0974696d657374616d70944afb247b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303032948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b038c08747269616c5f6964948c083138633131636263948c0464617465948c13323032342d30362d32355f32322d31332d3437948c1074696d655f746869735f697465725f739447404dc3c7900000008c0c74696d655f746f74616c5f7394474068199ffb8000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b048c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b108c0a656e636f6465725f6666944d00028c0d6465636f6465725f72745f6666944d00088c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b088c0964726f705f7261746594473fd79b833f50ba9a8c026c7294473f5d8cdf8cc701de8c0a62617463685f73697a65944d0004758c1274696d655f73696e63655f726573746f726594474068199ffb8000008c18697465726174696f6e735f73696e63655f726573746f7265944b0375756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f31386331316362635f32375f62617463685f73697a653d313032342c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d323034382c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f5f323032342d30362d32355f31392d33392d30332f636865636b706f696e745f30303030303394682268257d946827897385945294682a682d2981947d9468308a10c0cac13ebaaa329b3049d64165106b4b7362756268317d94288c046c6f7373946836683c430828140af519bc644094869452948c0974696d657374616d70944a37257b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303033948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b048c08747269616c5f6964948c083138633131636263948c0464617465948c13323032342d30362d32355f32322d31342d3437948c1074696d655f746869735f697465725f739447404dd443c80000008c0c74696d655f746f74616c5f739447406f8eb0ed8000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b048c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b108c0a656e636f6465725f6666944d00028c0d6465636f6465725f72745f6666944d00088c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b088c0964726f705f7261746594473fd79b833f50ba9a8c026c7294473f5d8cdf8cc701de8c0a62617463685f73697a65944d0004758c1274696d655f73696e63655f726573746f72659447406f8eb0ed8000008c18697465726174696f6e735f73696e63655f726573746f7265944b0475756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f31386331316362635f32375f62617463685f73697a653d313032342c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d323034382c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f5f323032342d30362d32355f31392d33392d30332f636865636b706f696e745f30303030303494682268257d946827897385945294682a682d2981947d9468308a116d61ff6fc4f453a2604803a0673145a3007362756268317d94288c046c6f7373946836683c4308dced763babae624094869452948c0974696d657374616d70944a73257b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303034948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b058c08747269616c5f6964948c083138633131636263948c0464617465948c13323032342d30362d32355f32322d31352d3437948c1074696d655f746869735f697465725f739447404e16ee200000008c0c74696d655f746f74616c5f73944740738a363ac000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b048c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b108c0a656e636f6465725f6666944d00028c0d6465636f6465725f72745f6666944d00088c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b088c0964726f705f7261746594473fd79b833f50ba9a8c026c7294473f5d8cdf8cc701de8c0a62617463685f73697a65944d0004758c1274696d655f73696e63655f726573746f7265944740738a363ac000008c18697465726174696f6e735f73696e63655f726573746f7265944b0575756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f31386331316362635f32375f62617463685f73697a653d313032342c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d323034382c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f5f323032342d30362d32355f31392d33392d30332f636865636b706f696e745f30303030303594682268257d946827897385945294682a682d2981947d9468308a10f26bae67b6730cb51f4cc81fe59f67167362756268317d94288c046c6f7373946836683c4308369bcd3ec5d9614094869452948c0974696d657374616d70944a5e457b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303035948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b068c08747269616c5f6964948c083138633131636263948c0464617465948c13323032342d30362d32365f30302d33312d3538948c1074696d655f746869735f697465725f73944740525eacfc0000008c0c74696d655f746f74616c5f739447407821e179c000008c03706964944a027f2b008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b048c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b108c0a656e636f6465725f6666944d00028c0d6465636f6465725f72745f6666944d00088c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b088c0964726f705f7261746594473fd79b833f50ba9a8c026c7294473f5d8cdf8cc701de8c0a62617463685f73697a65944d0004758c1274696d655f73696e63655f726573746f7265944740525eacfc0000008c18697465726174696f6e735f73696e63655f726573746f7265944b0175756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f31386331316362635f32375f62617463685f73697a653d313032342c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d323034382c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f5f323032342d30362d32355f31392d33392d30332f636865636b706f696e745f30303030303694682268257d946827897385945294682a682d2981947d9468308a10c04ca19083252abc3d4f3c39852a7e727362756268317d94288c046c6f7373946836683c4308c26030d0cd73624094869452948c0974696d657374616d70944a98457b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303036948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b078c08747269616c5f6964948c083138633131636263948c0464617465948c13323032342d30362d32365f30302d33322d3536948c1074696d655f746869735f697465725f739447404d1e3c8a0000008c0c74696d655f746f74616c5f739447407bc5a90b0000008c03706964944a027f2b008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b048c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b108c0a656e636f6465725f6666944d00028c0d6465636f6465725f72745f6666944d00088c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b088c0964726f705f7261746594473fd79b833f50ba9a8c026c7294473f5d8cdf8cc701de8c0a62617463685f73697a65944d0004758c1274696d655f73696e63655f726573746f72659447406076e5a08000008c18697465726174696f6e735f73696e63655f726573746f7265944b0275756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f31386331316362635f32375f62617463685f73697a653d313032342c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d323034382c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f5f323032342d30362d32355f31392d33392d30332f636865636b706f696e745f30303030303794682268257d946827897385945294682a682d2981947d9468308a10aa1b24d1c873da9cde48140f766316467362756268317d94288c046c6f7373946836683c4308b1582c7eef1d614094869452948c0974696d657374616d70944ad3457b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303037948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b088c08747269616c5f6964948c083138633131636263948c0464617465948c13323032342d30362d32365f30302d33332d3535948c1074696d655f746869735f697465725f739447404d8a674c0000008c0c74696d655f746f74616c5f739447407f76f5f48000008c03706964944a027f2b008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b048c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b108c0a656e636f6465725f6666944d00028c0d6465636f6465725f72745f6666944d00088c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b088c0964726f705f7261746594473fd79b833f50ba9a8c026c7294473f5d8cdf8cc701de8c0a62617463685f73697a65944d0004758c1274696d655f73696e63655f726573746f726594474067d97f738000008c18697465726174696f6e735f73696e63655f726573746f7265944b0375756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f31386331316362635f32375f62617463685f73697a653d313032342c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d323034382c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f5f323032342d30362d32355f31392d33392d30332f636865636b706f696e745f30303030303894682268257d946827897385945294682a682d2981947d9468308a109112222aaa942e97504068bbe7cbc45d7362756268317d94288c046c6f7373946836683c4308e27038f485ac604094869452948c0974696d657374616d70944a0d467b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303038948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b098c08747269616c5f6964948c083138633131636263948c0464617465948c13323032342d30362d32365f30302d33342d3533948c1074696d655f746869735f697465725f739447404d32a3dc0000008c0c74696d655f746f74616c5f73944740818ea5380000008c03706964944a027f2b008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b048c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b108c0a656e636f6465725f6666944d00028c0d6465636f6465725f72745f6666944d00088c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b088c0964726f705f7261746594473fd79b833f50ba9a8c026c7294473f5d8cdf8cc701de8c0a62617463685f73697a65944d0004758c1274696d655f73696e63655f726573746f72659447406f26286a8000008c18697465726174696f6e735f73696e63655f726573746f7265944b0475756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f31386331316362635f32375f62617463685f73697a653d313032342c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d323034382c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f5f323032342d30362d32355f31392d33392d30332f636865636b706f696e745f30303030303994682268257d946827897385945294682a682d2981947d9468308a119785aa279b54399f9a40e1bc294f0ff3007362756268317d94288c046c6f7373946836683c43080683c160395e604094869452948c0974696d657374616d70944a48467b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303039948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b0a8c08747269616c5f6964948c083138633131636263948c0464617465948c13323032342d30362d32365f30302d33352d3532948c1074696d655f746869735f697465725f739447404d55b5600000008c0c74696d655f746f74616c5f739447408364008e0000008c03706964944a027f2b008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b048c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b108c0a656e636f6465725f6666944d00028c0d6465636f6465725f72745f6666944d00088c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b088c0964726f705f7261746594473fd79b833f50ba9a8c026c7294473f5d8cdf8cc701de8c0a62617463685f73697a65944d0004758c1274696d655f73696e63655f726573746f7265944740733dcae14000008c18697465726174696f6e735f73696e63655f726573746f7265944b0575756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f31386331316362635f32375f62617463685f73697a653d313032342c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d323034382c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f5f323032342d30362d32355f31392d33392d30332f636865636b706f696e745f30303030313094682268257d946827897385945294682a682d2981947d9468308a11df99e3d8c62135b4a14e96ee86bb889b007362756268317d94288c046c6f7373946836683c430897cbe50acf00604094869452948c0974696d657374616d70944a9a4f7b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303130948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b0b8c08747269616c5f6964948c083138633131636263948c0464617465948c13323032342d30362d32365f30312d31352d3338948c1074696d655f746869735f697465725f73944740523044200000008c0c74696d655f746f74616c5f7394474085aa09120000008c03706964944a027f2b008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b048c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b108c0a656e636f6465725f6666944d00028c0d6465636f6465725f72745f6666944d00088c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b088c0964726f705f7261746594473fd79b833f50ba9a8c026c7294473f5d8cdf8cc701de8c0a62617463685f73697a65944d0004758c1274696d655f73696e63655f726573746f7265944740523044200000008c18697465726174696f6e735f73696e63655f726573746f7265944b0175756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f31386331316362635f32375f62617463685f73697a653d313032342c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d323034382c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f5f323032342d30362d32355f31392d33392d30332f636865636b706f696e745f30303030313194682268257d946827897385945294682a682d2981947d9468308a11c09cc703c01eecb8294d5a9db423d3a1007362756268317d94288c046c6f7373946836683c4308e2703894b34f604094869452948c0974696d657374616d70944ad54f7b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303131948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b0c8c08747269616c5f6964948c083138633131636263948c0464617465948c13323032342d30362d32365f30312d31362d3337948c1074696d655f746869735f697465725f739447404dac917e0000008c0c74696d655f746f74616c5f739447408784d229e000008c03706964944a027f2b008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b048c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b108c0a656e636f6465725f6666944d00028c0d6465636f6465725f72745f6666944d00088c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b088c0964726f705f7261746594473fd79b833f50ba9a8c026c7294473f5d8cdf8cc701de8c0a62617463685f73697a65944d0004758c1274696d655f73696e63655f726573746f72659447406083466f8000008c18697465726174696f6e735f73696e63655f726573746f7265944b0275756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f31386331316362635f32375f62617463685f73697a653d313032342c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d323034382c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f5f323032342d30362d32355f31392d33392d30332f636865636b706f696e745f30303030313294682268257d946827897385945294682a682d2981947d9468308a11508686547d02c2817449c27b146b35b3007362756268317d94288c046c6f7373946836683c43089fcfe793d4635f4094869452948c0974696d657374616d70944a11507b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303132948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b0d8c08747269616c5f6964948c083138633131636263948c0464617465948c13323032342d30362d32365f30312d31372d3337948c1074696d655f746869735f697465725f739447404da1a5840000008c0c74696d655f746f74616c5f73944740895eec822000008c03706964944a027f2b008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b048c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b108c0a656e636f6465725f6666944d00028c0d6465636f6465725f72745f6666944d00088c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b088c0964726f705f7261746594473fd79b833f50ba9a8c026c7294473f5d8cdf8cc701de8c0a62617463685f73697a65944d0004758c1274696d655f73696e63655f726573746f726594474067ebafd08000008c18697465726174696f6e735f73696e63655f726573746f7265944b0375756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f31386331316362635f32375f62617463685f73697a653d313032342c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d323034382c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f5f323032342d30362d32355f31392d33392d30332f636865636b706f696e745f30303030313394682268257d946827897385945294682a682d2981947d9468308a116cbb66f88fca9b852f40838bb78628f4007362756268317d94288c046c6f7373946836683c430843a150183a005f4094869452948c0974696d657374616d70944a4c507b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303133948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b0e8c08747269616c5f6964948c083138633131636263948c0464617465948c13323032342d30362d32365f30312d31382d3336948c1074696d655f746869735f697465725f739447404dadc47e0000008c0c74696d655f746f74616c5f739447408b39c8ca0000008c03706964944a027f2b008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b048c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b108c0a656e636f6465725f6666944d00028c0d6465636f6465725f72745f6666944d00088c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b088c0964726f705f7261746594473fd79b833f50ba9a8c026c7294473f5d8cdf8cc701de8c0a62617463685f73697a65944d0004758c1274696d655f73696e63655f726573746f72659447406f5720f00000008c18697465726174696f6e735f73696e63655f726573746f7265944b0475756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f31386331316362635f32375f62617463685f73697a653d313032342c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d323034382c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f5f323032342d30362d32355f31392d33392d30332f636865636b706f696e745f30303030313494682268257d946827897385945294682a682d2981947d9468308a11613624b5cce4d2b99b4d25d68d3f1d8b007362756268317d94288c046c6f7373946836683c43080a854241979b5e4094869452948c0974696d657374616d70944a87507b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303134948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b0f8c08747269616c5f6964948c083138633131636263948c0464617465948c13323032342d30362d32365f30312d31392d3335948c1074696d655f746869735f697465725f739447404d93af320000008c0c74696d655f746f74616c5f739447408d1303bd2000008c03706964944a027f2b008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b048c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b108c0a656e636f6465725f6666944d00028c0d6465636f6465725f72745f6666944d00088c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b088c0964726f705f7261746594473fd79b833f50ba9a8c026c7294473f5d8cdf8cc701de8c0a62617463685f73697a65944d0004758c1274696d655f73696e63655f726573746f7265944740735e065e4000008c18697465726174696f6e735f73696e63655f726573746f7265944b0575756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f31386331316362635f32375f62617463685f73697a653d313032342c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d323034382c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f5f323032342d30362d32355f31392d33392d30332f636865636b706f696e745f30303030313594682268257d946827897385945294682a682d2981947d9468308a10938eba92221ca199b44bc0aef16e165b7362756268317d94288c046c6f7373946836683c4308d8ebf5c27a5c5e4094869452948c0974696d657374616d70944ac3507b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303135948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b108c08747269616c5f6964948c083138633131636263948c0464617465948c13323032342d30362d32365f30312d32302d3335948c1074696d655f746869735f697465725f739447404dcea8160000008c0c74696d655f746f74616c5f739447408eefee3e8000008c03706964944a027f2b008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b048c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b108c0a656e636f6465725f6666944d00028c0d6465636f6465725f72745f6666944d00088c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b088c0964726f705f7261746594473fd79b833f50ba9a8c026c7294473f5d8cdf8cc701de8c0a62617463685f73697a65944d0004758c1274696d655f73696e63655f726573746f72659447407717db610000008c18697465726174696f6e735f73696e63655f726573746f7265944b0675756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f31386331316362635f32375f62617463685f73697a653d313032342c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d323034382c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f5f323032342d30362d32355f31392d33392d30332f636865636b706f696e745f30303030313694682268257d946827897385945294682a682d2981947d9468308a10d774e2adcfce7da9b04371b86194f5047362756268317d94288c046c6f7373946836683c4308613018cc31075f4094869452948c0974696d657374616d70944afe507b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303136948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b118c08747269616c5f6964948c083138633131636263948c0464617465948c13323032342d30362d32365f30312d32312d3334948c1074696d655f746869735f697465725f739447404d92bc720000008c0c74696d655f746f74616c5f7394474090648d02d000008c03706964944a027f2b008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b048c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b108c0a656e636f6465725f6666944d00028c0d6465636f6465725f72745f6666944d00088c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b088c0964726f705f7261746594473fd79b833f50ba9a8c026c7294473f5d8cdf8cc701de8c0a62617463685f73697a65944d0004758c1274696d655f73696e63655f726573746f72659447407aca32ef4000008c18697465726174696f6e735f73696e63655f726573746f7265944b0775756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f31386331316362635f32375f62617463685f73697a653d313032342c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d323034382c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f5f323032342d30362d32355f31392d33392d30332f636865636b706f696e745f30303030313794682268257d946827897385945294682a682d2981947d9468308a103a52109f48a6a791e44e1b0991e471477362756268317d94288c046c6f7373946836683c4308241289b4d7615e4094869452948c0974696d657374616d70944a39517b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303137948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b128c08747269616c5f6964948c083138633131636263948c0464617465948c13323032342d30362d32365f30312d32322d3333948c1074696d655f746869735f697465725f739447404d8f0b3e0000008c0c74696d655f746f74616c5f739447409151055cc000008c03706964944a027f2b008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b048c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b108c0a656e636f6465725f6666944d00028c0d6465636f6465725f72745f6666944d00088c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b088c0964726f705f7261746594473fd79b833f50ba9a8c026c7294473f5d8cdf8cc701de8c0a62617463685f73697a65944d0004758c1274696d655f73696e63655f726573746f72659447407e7c14570000008c18697465726174696f6e735f73696e63655f726573746f7265944b0875756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f31386331316362635f32375f62617463685f73697a653d313032342c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d323034382c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f5f323032342d30362d32355f31392d33392d30332f636865636b706f696e745f30303030313894682268257d946827897385945294682a682d2981947d9468308a1015dc357caba8f7a13b4ebd0ba57182407362756268317d94288c046c6f7373946836683c4308040281d040ac5d4094869452948c0974696d657374616d70944a74517b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303138948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b138c08747269616c5f6964948c083138633131636263948c0464617465948c13323032342d30362d32365f30312d32332d3332948c1074696d655f746869735f697465725f739447404d972c660000008c0c74696d655f746f74616c5f73944740923dbebff000008c03706964944a027f2b008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b048c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b108c0a656e636f6465725f6666944d00028c0d6465636f6465725f72745f6666944d00088c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b088c0964726f705f7261746594473fd79b833f50ba9a8c026c7294473f5d8cdf8cc701de8c0a62617463685f73697a65944d0004758c1274696d655f73696e63655f726573746f726594474081177cf1e000008c18697465726174696f6e735f73696e63655f726573746f7265944b0975756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f31386331316362635f32375f62617463685f73697a653d313032342c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d323034382c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f5f323032342d30362d32355f31392d33392d30332f636865636b706f696e745f30303030313994682268257d946827897385945294682a682d2981947d9468308a1147920dc8f570e8b19e4e6779415f42a8007362756268317d94288c046c6f7373946836683c43080a854211a0955d4094869452948c0974696d657374616d70944aaf517b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303139948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b148c08747269616c5f6964948c083138633131636263948c0464617465948c13323032342d30362d32365f30312d32342d3331948c1074696d655f746869735f697465725f739447404da85a0a0000008c0c74696d655f746f74616c5f73944740932b01904000008c03706964944a027f2b008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b048c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b108c0a656e636f6465725f6666944d00028c0d6465636f6465725f72745f6666944d00088c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b088c0964726f705f7261746594473fd79b833f50ba9a8c026c7294473f5d8cdf8cc701de8c0a62617463685f73697a65944d0004758c1274696d655f73696e63655f726573746f726594474082f202928000008c18697465726174696f6e735f73696e63655f726573746f7265944b0a757562658c195f6c61746573745f636865636b706f696e745f726573756c74946ab003000075622e\"\n }\n}"
+ ],
+ [
+ "{\n \"stub\": false,\n \"trainable_name\": \"train_model\",\n \"trial_id\": \"3ab2f301\",\n \"storage\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059572040000000000008c1b7261792e747261696e2e5f696e7465726e616c2e73746f72616765948c0e53746f72616765436f6e746578749493942981947d94288c12637573746f6d5f66735f70726f766964656494898c1273746f726167655f6c6f63616c5f70617468948c2b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c7473948c136578706572696d656e745f6469725f6e616d65948c0f746573745f6578706572696d656e74948c0e747269616c5f6469725f6e616d65948c96747261696e5f6d6f64656c5f33616232663330315f32385f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d323034382c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f5f323032342d30362d32355f32322d31302d3334948c1863757272656e745f636865636b706f696e745f696e646578944b138c0b73796e635f636f6e666967948c097261792e747261696e948c0a53796e63436f6e6669679493942981947d94288c0b73796e635f706572696f64944d2c018c0c73796e635f74696d656f7574944d08078c0e73796e635f61727469666163747394898c1c73796e635f6172746966616374735f6f6e5f636865636b706f696e7494888c0a75706c6f61645f646972948c0a44455052454341544544948c0673796e6365729468188c1273796e635f6f6e5f636865636b706f696e7494681875628c1273746f726167655f66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c0f73746f726167655f66735f70617468948c452f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f7465737494681968008c115f46696c6573797374656d53796e6365729493942981947d9428681b682268134d2c0168144d08078c116c6173745f73796e635f75705f74696d65944741d99ec8d7e1c03a8c136c6173745f73796e635f646f776e5f74696d659447fff00000000000008c0d5f73796e635f70726f63657373944e8c0c5f63757272656e745f636d649468008c125f75706c6f61645f746f5f66735f706174689493947d94288c0a6c6f63616c5f70617468948c3b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c74732f746573745f6578706572696d656e74948c0266739468228c0766735f70617468948c552f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e74948c076578636c756465945d948c0e2a2f636865636b706f696e745f2a9461758694756275622e\"\n },\n \"config\": {\n \"encoder_num_layer\": 2,\n \"decoder_rt_num_layer\": 1,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 16,\n \"encoder_ff\": 2048,\n \"decoder_rt_ff\": 2048,\n \"decoder_int_ff\": 512,\n \"n_head\": 1,\n \"drop_rate\": 0.034481700426631456,\n \"lr\": 0.0010186128360476897,\n \"batch_size\": 2048\n },\n \"_Trial__unresolved_config\": {\n \"encoder_num_layer\": 2,\n \"decoder_rt_num_layer\": 1,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 16,\n \"encoder_ff\": 2048,\n \"decoder_rt_ff\": 2048,\n \"decoder_int_ff\": 512,\n \"n_head\": 1,\n \"drop_rate\": 0.034481700426631456,\n \"lr\": 0.0010186128360476897,\n \"batch_size\": 2048\n },\n \"evaluated_params\": {\n \"batch_size\": 2048,\n \"decoder_int_ff\": 512,\n \"decoder_int_num_layer\": 1,\n \"decoder_rt_ff\": 2048,\n \"decoder_rt_num_layer\": 1,\n \"drop_rate\": 0.034481700426631456,\n \"embedding_dim\": 16,\n \"encoder_ff\": 2048,\n \"encoder_num_layer\": 2,\n \"lr\": 0.0010186128360476897,\n \"n_head\": 1\n },\n \"experiment_tag\": \"28_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop_rate=0.0345,embedding_dim=16,encoder_ff=2048,encoder_num_layer=2,lr=0.0010,n_head=1\",\n \"stopping_criterion\": {},\n \"_setup_default_resource\": true,\n \"_default_placement_group_factory\": \"80054e2e\",\n \"placement_group_factory\": \"800595ba000000000000008c237261792e74756e652e657865637574696f6e2e706c6163656d656e745f67726f757073948c15506c6163656d656e7447726f7570466163746f72799493942981947d94288c085f62756e646c6573945d947d94288c03435055944740540000000000008c034750559447401000000000000075618c155f686561645f62756e646c655f69735f656d70747994898c095f7374726174656779948c045041434b948c055f6172677394298c075f6b7761726773947d9475622e\",\n \"log_to_file\": [\n null,\n null\n ],\n \"max_failures\": 0,\n \"_default_result_or_future\": null,\n \"export_formats\": [],\n \"status\": \"TERMINATED\",\n \"relative_logdir\": \"train_model_3ab2f301_28_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop__2024-06-25_22-10-34\",\n \"trial_name_creator\": null,\n \"trial_dirname_creator\": null,\n \"custom_trial_name\": null,\n \"custom_dirname\": null,\n \"restore_path\": null,\n \"_restore_checkpoint_result\": null,\n \"_state_json\": null,\n \"results\": \"80054e2e\",\n \"extra_arg\": \"80054e2e\",\n \"_resources\": \"80054e2e\"\n}",
+ "{\n \"start_time\": 1719346547.487432,\n \"num_failures\": 0,\n \"num_failures_after_restore\": 0,\n \"error_filename\": null,\n \"pickled_error_filename\": null,\n \"last_result\": {\n \"loss\": 84.81797225832001,\n \"timestamp\": 1719358398,\n \"checkpoint_dir_name\": \"checkpoint_000019\",\n \"should_checkpoint\": true,\n \"done\": true,\n \"training_iteration\": 20,\n \"trial_id\": \"3ab2f301\",\n \"date\": \"2024-06-26_01-33-18\",\n \"time_this_iter_s\": 51.372178077697754,\n \"time_total_s\": 1072.982276916504,\n \"pid\": 2850562,\n \"hostname\": \"r8i6n2\",\n \"node_ip\": \"10.159.28.60\",\n \"config\": {\n \"encoder_num_layer\": 2,\n \"decoder_rt_num_layer\": 1,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 16,\n \"encoder_ff\": 2048,\n \"decoder_rt_ff\": 2048,\n \"decoder_int_ff\": 512,\n \"n_head\": 1,\n \"drop_rate\": 0.034481700426631456,\n \"lr\": 0.0010186128360476897,\n \"batch_size\": 2048\n },\n \"time_since_restore\": 525.9760255813599,\n \"iterations_since_restore\": 10,\n \"experiment_tag\": \"28_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop_rate=0.0345,embedding_dim=16,encoder_ff=2048,encoder_num_layer=2,lr=0.0010,n_head=1\"\n },\n \"last_result_time\": 1719358398.1699922,\n \"metric_analysis\": {\n \"loss\": {\n \"max\": 217.46355960875985,\n \"min\": 84.81797225832001,\n \"avg\": 111.01648353816957,\n \"last\": 84.81797225832001,\n \"last-5-avg\": 88.37300739438516,\n \"last-10-avg\": 90.88470410925197\n },\n \"should_checkpoint\": {\n \"max\": true,\n \"min\": true,\n \"avg\": 1.0,\n \"last\": true,\n \"last-5-avg\": 1.0,\n \"last-10-avg\": 1.0\n },\n \"done\": {\n \"max\": false,\n \"min\": false,\n \"avg\": 0.0,\n \"last\": false,\n \"last-5-avg\": 0.0,\n \"last-10-avg\": 0.0\n },\n \"training_iteration\": {\n \"max\": 20,\n \"min\": 1,\n \"avg\": 10.5,\n \"last\": 20,\n \"last-5-avg\": 18.0,\n \"last-10-avg\": 15.5\n },\n \"time_this_iter_s\": {\n \"max\": 65.41611504554749,\n \"min\": 50.784724712371826,\n \"avg\": 53.6491138458252,\n \"last\": 51.372178077697754,\n \"last-5-avg\": 51.40114240646362,\n \"last-10-avg\": 52.59760255813599\n },\n \"time_total_s\": {\n \"max\": 1072.982276916504,\n \"min\": 65.41611504554749,\n \"avg\": 574.7836294054985,\n \"last\": 1072.982276916504,\n \"last-5-avg\": 970.4215782165527,\n \"last-10-avg\": 842.11748483181\n },\n \"time_since_restore\": {\n \"max\": 525.9760255813599,\n \"min\": 64.502512216568,\n \"avg\": 232.20214306116105,\n \"last\": 525.9760255813599,\n \"last-5-avg\": 423.4153268814087,\n \"last-10-avg\": 295.11123349666593\n },\n \"iterations_since_restore\": {\n \"max\": 10,\n \"min\": 1,\n \"avg\": 4.25,\n \"last\": 10,\n \"last-5-avg\": 8.0,\n \"last-10-avg\": 5.5\n }\n },\n \"_n_steps\": [\n 5,\n 10\n ],\n \"metric_n_steps\": {\n \"loss\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"800595d6000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294288c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b007494624308abd56af55780574094869452946807680d4308f67abd5e1ade554094869452946807680d4308cce572191604564094869452946807680d430871381cae7ae0554094869452946807680d430843a150a8593455409486945294652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059535010000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294288c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b007494624308d0e7f3b9ce9f584094869452946807680d4308f2783c7ebf5f574094869452946807680d4308fcfd7e1f86b2574094869452946807680d43083e9fcfa7c9dd564094869452946807680d430843a15028fb2e564094869452946807680d4308abd56af55780574094869452946807680d4308f67abd5e1ade554094869452946807680d4308cce572191604564094869452946807680d430871381cae7ae0554094869452946807680d430843a150a8593455409486945294652e\"\n }\n },\n \"should_checkpoint\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059527000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294288888888888652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952c000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452942888888888888888888888652e\"\n }\n },\n \"done\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059527000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294288989898989652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952c000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452942889898989898989898989652e\"\n }\n },\n \"training_iteration\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952c000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294284b104b114b124b134b14652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059536000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294284b0b4b0c4b0d4b0e4b0f4b104b114b124b134b14652e\"\n }\n },\n \"time_this_iter_s\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005954f000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b058694529428474049ccd45e000000474049ebdd980000004740498cc3e20000004740498ba1cc000000474049afa388000000652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005957c000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294284740502029290000004740496e97f2000000474049d276600000004740499660bc0000004740496471dc000000474049ccd45e000000474049ebdd980000004740498cc3e20000004740498ba1cc000000474049afa388000000652e\"\n }\n },\n \"time_total_s\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005954f000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452942847408b1c9d4720000047408cbb5b20a0000047408e54275ec0000047408fece17b800000474090c3edda000000652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005957c000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294284740831c11f2a00000474084b2fb71c000004740865022d7c00000474087e988e38000004740897fd00140000047408b1c9d4720000047408cbb5b20a0000047408e54275ec0000047408fece17b800000474090c3edda000000652e\"\n }\n },\n \"time_since_restore\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005954f000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294284740740920f3400000474077469ca640000047407a78352280000047407da9a95c0000004740806fcee6800000652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005957c000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452942847405020292900000047405cd77522000000474064e0582900000047406b45f058000000474070cf86678000004740740920f3400000474077469ca640000047407a78352280000047407da9a95c0000004740806fcee6800000652e\"\n }\n },\n \"iterations_since_restore\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952c000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294284b064b074b084b094b0a652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059536000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294284b014b024b034b044b054b064b074b084b094b0a652e\"\n }\n }\n },\n \"checkpoint_manager\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059587490000000000008c267261792e747261696e2e5f696e7465726e616c2e636865636b706f696e745f6d616e61676572948c125f436865636b706f696e744d616e616765729493942981947d94288c125f636865636b706f696e745f636f6e666967948c097261792e747261696e948c10436865636b706f696e74436f6e6669679493942981947d94288c0b6e756d5f746f5f6b656570944e8c1a636865636b706f696e745f73636f72655f617474726962757465944e8c16636865636b706f696e745f73636f72655f6f72646572948c036d6178948c14636865636b706f696e745f6672657175656e6379944b008c11636865636b706f696e745f61745f656e6494898c1a5f636865636b706f696e745f6b6565705f616c6c5f72616e6b73948c0a44455052454341544544948c1f5f636865636b706f696e745f75706c6f61645f66726f6d5f776f726b65727394681275628c135f636865636b706f696e745f726573756c7473945d94288c1b7261792e747261696e2e5f696e7465726e616c2e73657373696f6e948c0f5f547261696e696e67526573756c749493942981947d94288c0a636865636b706f696e749468068c0a436865636b706f696e749493942981947d94288c0470617468948cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f33616232663330315f32385f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d323034382c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f5f323032342d30362d32355f32322d31302d33342f636865636b706f696e745f303030303030948c0a66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c055f75756964948c0475756964948c04555549449493942981947d948c03696e74948a10e8ed94ab740a45ae324a3111f342bd41736275628c076d657472696373947d94288c046c6f7373948c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b007494624308d8ebf57ad52e6b4094869452948c0974696d657374616d70944ab4257b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303030948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b018c08747269616c5f6964948c083361623266333031948c0464617465948c13323032342d30362d32355f32322d31362d3532948c1074696d655f746869735f697465725f73944740505aa1a10000008c0c74696d655f746f74616c5f73944740505aa1a10000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b028c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b108c0a656e636f6465725f6666944d00088c0d6465636f6465725f72745f6666944d00088c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b018c0964726f705f7261746594473fa1a795df4923d08c026c7294473f50b05f345becb88c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f7265944740505aa1a10000008c18697465726174696f6e735f73696e63655f726573746f7265944b0175756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f33616232663330315f32385f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d323034382c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f5f323032342d30362d32355f32322d31302d33342f636865636b706f696e745f30303030303194682268257d946827897385945294682a682d2981947d9468308a102fede7cdb9c96a8d2144477b65b563337362756268317d94288c046c6f7373946836683c43086130184c0a7b644094869452948c0974696d657374616d70944ae8257b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303031948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b028c08747269616c5f6964948c083361623266333031948c0464617465948c13323032342d30362d32355f32322d31372d3435948c1074696d655f746869735f697465725f739447404a336ca00000008c0c74696d655f746f74616c5f739447405d7457f10000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b028c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b108c0a656e636f6465725f6666944d00088c0d6465636f6465725f72745f6666944d00088c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b018c0964726f705f7261746594473fa1a795df4923d08c026c7294473f50b05f345becb88c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f72659447405d7457f10000008c18697465726174696f6e735f73696e63655f726573746f7265944b0275756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f33616232663330315f32385f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d323034382c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f5f323032342d30362d32355f32322d31302d33342f636865636b706f696e745f30303030303294682268257d946827897385945294682a682d2981947d9468308a1140a5e00664cf47862e44f66032ca9e9c007362756268317d94288c046c6f7373946836683c430893c964529e70614094869452948c0974696d657374616d70944a1d267b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303032948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b038c08747269616c5f6964948c083361623266333031948c0464617465948c13323032342d30362d32355f32322d31382d3337948c1074696d655f746869735f697465725f739447404a1a97f60000008c0c74696d655f746f74616c5f739447406540d1f60000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b028c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b108c0a656e636f6465725f6666944d00088c0d6465636f6465725f72745f6666944d00088c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b018c0964726f705f7261746594473fa1a795df4923d08c026c7294473f50b05f345becb88c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f72659447406540d1f60000008c18697465726174696f6e735f73696e63655f726573746f7265944b0375756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f33616232663330315f32385f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d323034382c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f5f323032342d30362d32355f32322d31302d33342f636865636b706f696e745f30303030303394682268257d946827897385945294682a682d2981947d9468308a1058db2733e21f6db671481e20128b04797362756268317d94288c046c6f7373946836683c43087fbfdf4ffd745f4094869452948c0974696d657374616d70944a52267b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303033948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b048c08747269616c5f6964948c083361623266333031948c0464617465948c13323032342d30362d32355f32322d31392d3330948c1074696d655f746869735f697465725f739447404a8bc79c0000008c0c74696d655f746f74616c5f739447406be3c3dd0000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b028c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b108c0a656e636f6465725f6666944d00088c0d6465636f6465725f72745f6666944d00088c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b018c0964726f705f7261746594473fa1a795df4923d08c026c7294473f50b05f345becb88c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f72659447406be3c3dd0000008c18697465726174696f6e735f73696e63655f726573746f7265944b0475756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f33616232663330315f32385f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d323034382c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f5f323032342d30362d32355f32322d31302d33342f636865636b706f696e745f30303030303494682268257d946827897385945294682a682d2981947d9468308a110132d6caf4940790b5401c1c6f6270bb007362756268317d94288c046c6f7373946836683c43088bc562f16a5d5d4094869452948c0974696d657374616d70944a87267b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303034948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b058c08747269616c5f6964948c083361623266333031948c0464617465948c13323032342d30362d32355f32322d32302d3233948c1074696d655f746869735f697465725f739447404a990f700000008c0c74696d655f746f74616c5f73944740714503dc8000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b028c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b108c0a656e636f6465725f6666944d00088c0d6465636f6465725f72745f6666944d00088c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b018c0964726f705f7261746594473fa1a795df4923d08c026c7294473f50b05f345becb88c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f7265944740714503dc8000008c18697465726174696f6e735f73696e63655f726573746f7265944b0575756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f33616232663330315f32385f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d323034382c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f5f323032342d30362d32355f32322d31302d33342f636865636b706f696e745f30303030303594682268257d946827897385945294682a682d2981947d9468308a113c614213f41e10ab864c5182885dc6f9007362756268317d94288c046c6f7373946836683c4308894422f1949e5c4094869452948c0974696d657374616d70944a89467b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303035948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b068c08747269616c5f6964948c083361623266333031948c0464617465948c13323032342d30362d32365f30302d33362d3537948c1074696d655f746869735f697465725f73944740505018840000008c0c74696d655f746f74616c5f73944740755909fd8000008c03706964944a027f2b008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b028c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b108c0a656e636f6465725f6666944d00088c0d6465636f6465725f72745f6666944d00088c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b018c0964726f705f7261746594473fa1a795df4923d08c026c7294473f50b05f345becb88c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f7265944740505018840000008c18697465726174696f6e735f73696e63655f726573746f7265944b0175756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f33616232663330315f32385f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d323034382c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f5f323032342d30362d32355f32322d31302d33342f636865636b706f696e745f30303030303694682268257d946827897385945294682a682d2981947d9468308a11b0b7fbae3a25b0b0e1426175363775cf007362756268317d94288c046c6f7373946836683c430893c964d2f24f5b4094869452948c0974696d657374616d70944abd467b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303036948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b078c08747269616c5f6964948c083361623266333031948c0464617465948c13323032342d30362d32365f30302d33372d3439948c1074696d655f746869735f697465725f7394474049bf61580000008c0c74696d655f746f74616c5f739447407890f6288000008c03706964944a027f2b008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b028c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b108c0a656e636f6465725f6666944d00088c0d6465636f6465725f72745f6666944d00088c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b018c0964726f705f7261746594473fa1a795df4923d08c026c7294473f50b05f345becb88c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f72659447405d2fc9300000008c18697465726174696f6e735f73696e63655f726573746f7265944b0275756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f33616232663330315f32385f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d323034382c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f5f323032342d30362d32355f32322d31302d33342f636865636b706f696e745f30303030303794682268257d946827897385945294682a682d2981947d9468308a110ee569d210e17ba781414ebdd6974df0007362756268317d94288c046c6f7373946836683c430843a15068e24c5a4094869452948c0974696d657374616d70944af0467b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303037948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b088c08747269616c5f6964948c083361623266333031948c0464617465948c13323032342d30362d32365f30302d33382d3430948c1074696d655f746869735f697465725f7394474049bf9ac40000008c0c74696d655f746f74616c5f739447407bc8e9810000008c03706964944a027f2b008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b028c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b108c0a656e636f6465725f6666944d00088c0d6465636f6465725f72745f6666944d00088c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b018c0964726f705f7261746594473fa1a795df4923d08c026c7294473f50b05f345becb88c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f72659447406507cb490000008c18697465726174696f6e735f73696e63655f726573746f7265944b0375756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f33616232663330315f32385f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d323034382c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f5f323032342d30362d32355f32322d31302d33342f636865636b706f696e745f30303030303894682268257d946827897385945294682a682d2981947d9468308a10da33af97d2baa297f64dc751c8d7d27a7362756268317d94288c046c6f7373946836683c4308100804a23aa55d4094869452948c0974696d657374616d70944a23477b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303038948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b098c08747269616c5f6964948c083361623266333031948c0464617465948c13323032342d30362d32365f30302d33392d3331948c1074696d655f746869735f697465725f739447404987e6840000008c0c74696d655f746f74616c5f739447407ef9e6518000008c03706964944a027f2b008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b028c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b108c0a656e636f6465725f6666944d00088c0d6465636f6465725f72745f6666944d00088c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b018c0964726f705f7261746594473fa1a795df4923d08c026c7294473f50b05f345becb88c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f72659447406b69c4ea0000008c18697465726174696f6e735f73696e63655f726573746f7265944b0475756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f33616232663330315f32385f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d323034382c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f5f323032342d30362d32355f32322d31302d33342f636865636b706f696e745f30303030303994682268257d946827897385945294682a682d2981947d9468308a1018334bbd4a62dc974f4049e30037562e7362756268317d94288c046c6f7373946836683c43082c168b25daf6584094869452948c0974696d657374616d70944a57477b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303039948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b0a8c08747269616c5f6964948c083361623266333031948c0464617465948c13323032342d30362d32365f30302d34302d3233948c1074696d655f746869735f697465725f7394474049b19a4c0000008c0c74696d655f746f74616c5f7394474081180ccd8000008c03706964944a027f2b008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b028c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b108c0a656e636f6465725f6666944d00088c0d6465636f6465725f72745f6666944d00088c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b018c0964726f705f7261746594473fa1a795df4923d08c026c7294473f50b05f345becb88c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f726594474070eb15be8000008c18697465726174696f6e735f73696e63655f726573746f7265944b0575756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f33616232663330315f32385f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d323034382c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f5f323032342d30362d32355f32322d31302d33342f636865636b706f696e745f30303030313094682268257d946827897385945294682a682d2981947d9468308a10d94fee9fcf871d9a2d4bc8368f829f327362756268317d94288c046c6f7373946836683c4308d0e7f3b9ce9f584094869452948c0974696d657374616d70944af0517b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303130948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b0b8c08747269616c5f6964948c083361623266333031948c0464617465948c13323032342d30362d32365f30312d32352d3336948c1074696d655f746869735f697465725f73944740502029290000008c0c74696d655f746f74616c5f73944740831c11f2a000008c03706964944a027f2b008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b028c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b108c0a656e636f6465725f6666944d00088c0d6465636f6465725f72745f6666944d00088c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b018c0964726f705f7261746594473fa1a795df4923d08c026c7294473f50b05f345becb88c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f7265944740502029290000008c18697465726174696f6e735f73696e63655f726573746f7265944b0175756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f33616232663330315f32385f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d323034382c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f5f323032342d30362d32355f32322d31302d33342f636865636b706f696e745f30303030313194682268257d946827897385945294682a682d2981947d9468308a101d55dd00b02e91ba9f4d9dc2d447b2387362756268317d94288c046c6f7373946836683c4308f2783c7ebf5f574094869452948c0974696d657374616d70944a23527b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303131948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b0c8c08747269616c5f6964948c083361623266333031948c0464617465948c13323032342d30362d32365f30312d32362d3237948c1074696d655f746869735f697465725f73944740496e97f20000008c0c74696d655f746f74616c5f7394474084b2fb71c000008c03706964944a027f2b008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b028c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b108c0a656e636f6465725f6666944d00088c0d6465636f6465725f72745f6666944d00088c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b018c0964726f705f7261746594473fa1a795df4923d08c026c7294473f50b05f345becb88c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f72659447405cd775220000008c18697465726174696f6e735f73696e63655f726573746f7265944b0275756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f33616232663330315f32385f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d323034382c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f5f323032342d30362d32355f32322d31302d33342f636865636b706f696e745f30303030313294682268257d946827897385945294682a682d2981947d9468308a108b607ff288319d88e9491b307a617c547362756268317d94288c046c6f7373946836683c4308fcfd7e1f86b2574094869452948c0974696d657374616d70944a57527b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303132948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b0d8c08747269616c5f6964948c083361623266333031948c0464617465948c13323032342d30362d32365f30312d32372d3139948c1074696d655f746869735f697465725f7394474049d276600000008c0c74696d655f746f74616c5f73944740865022d7c000008c03706964944a027f2b008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b028c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b108c0a656e636f6465725f6666944d00088c0d6465636f6465725f72745f6666944d00088c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b018c0964726f705f7261746594473fa1a795df4923d08c026c7294473f50b05f345becb88c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f726594474064e058290000008c18697465726174696f6e735f73696e63655f726573746f7265944b0375756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f33616232663330315f32385f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d323034382c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f5f323032342d30362d32355f32322d31302d33342f636865636b706f696e745f30303030313394682268257d946827897385945294682a682d2981947d9468308a10f8312aa627f59fa458496df894d64f087362756268317d94288c046c6f7373946836683c43083e9fcfa7c9dd564094869452948c0974696d657374616d70944a8a527b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303133948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b0e8c08747269616c5f6964948c083361623266333031948c0464617465948c13323032342d30362d32365f30312d32382d3130948c1074696d655f746869735f697465725f73944740499660bc0000008c0c74696d655f746f74616c5f7394474087e988e38000008c03706964944a027f2b008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b028c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b108c0a656e636f6465725f6666944d00088c0d6465636f6465725f72745f6666944d00088c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b018c0964726f705f7261746594473fa1a795df4923d08c026c7294473f50b05f345becb88c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f72659447406b45f0580000008c18697465726174696f6e735f73696e63655f726573746f7265944b0475756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f33616232663330315f32385f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d323034382c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f5f323032342d30362d32355f32322d31302d33342f636865636b706f696e745f30303030313494682268257d946827897385945294682a682d2981947d9468308a101c4d5ed7924e63920a4809ae3bcd0d197362756268317d94288c046c6f7373946836683c430843a15028fb2e564094869452948c0974696d657374616d70944abd527b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303134948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b0f8c08747269616c5f6964948c083361623266333031948c0464617465948c13323032342d30362d32365f30312d32392d3031948c1074696d655f746869735f697465725f73944740496471dc0000008c0c74696d655f746f74616c5f73944740897fd0014000008c03706964944a027f2b008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b028c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b108c0a656e636f6465725f6666944d00088c0d6465636f6465725f72745f6666944d00088c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b018c0964726f705f7261746594473fa1a795df4923d08c026c7294473f50b05f345becb88c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f726594474070cf86678000008c18697465726174696f6e735f73696e63655f726573746f7265944b0575756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f33616232663330315f32385f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d323034382c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f5f323032342d30362d32355f32322d31302d33342f636865636b706f696e745f30303030313594682268257d946827897385945294682a682d2981947d9468308a109380808904b901aafa42edbf9fe1513c7362756268317d94288c046c6f7373946836683c4308abd56af55780574094869452948c0974696d657374616d70944af0527b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303135948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b108c08747269616c5f6964948c083361623266333031948c0464617465948c13323032342d30362d32365f30312d32392d3532948c1074696d655f746869735f697465725f7394474049ccd45e0000008c0c74696d655f746f74616c5f739447408b1c9d472000008c03706964944a027f2b008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b028c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b108c0a656e636f6465725f6666944d00088c0d6465636f6465725f72745f6666944d00088c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b018c0964726f705f7261746594473fa1a795df4923d08c026c7294473f50b05f345becb88c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f7265944740740920f34000008c18697465726174696f6e735f73696e63655f726573746f7265944b0675756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f33616232663330315f32385f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d323034382c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f5f323032342d30362d32355f32322d31302d33342f636865636b706f696e745f30303030313694682268257d946827897385945294682a682d2981947d9468308a111c21b0db9c483bba1245d7f6ed0624d4007362756268317d94288c046c6f7373946836683c4308f67abd5e1ade554094869452948c0974696d657374616d70944a24537b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303136948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b118c08747269616c5f6964948c083361623266333031948c0464617465948c13323032342d30362d32365f30312d33302d3434948c1074696d655f746869735f697465725f7394474049ebdd980000008c0c74696d655f746f74616c5f739447408cbb5b20a000008c03706964944a027f2b008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b028c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b108c0a656e636f6465725f6666944d00088c0d6465636f6465725f72745f6666944d00088c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b018c0964726f705f7261746594473fa1a795df4923d08c026c7294473f50b05f345becb88c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f726594474077469ca64000008c18697465726174696f6e735f73696e63655f726573746f7265944b0775756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f33616232663330315f32385f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d323034382c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f5f323032342d30362d32355f32322d31302d33342f636865636b706f696e745f30303030313794682268257d946827897385945294682a682d2981947d9468308a10696a9c34166e88864a42e14741b4af2e7362756268317d94288c046c6f7373946836683c4308cce572191604564094869452948c0974696d657374616d70944a57537b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303137948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b128c08747269616c5f6964948c083361623266333031948c0464617465948c13323032342d30362d32365f30312d33312d3335948c1074696d655f746869735f697465725f73944740498cc3e20000008c0c74696d655f746f74616c5f739447408e54275ec000008c03706964944a027f2b008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b028c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b108c0a656e636f6465725f6666944d00088c0d6465636f6465725f72745f6666944d00088c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b018c0964726f705f7261746594473fa1a795df4923d08c026c7294473f50b05f345becb88c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f72659447407a7835228000008c18697465726174696f6e735f73696e63655f726573746f7265944b0875756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f33616232663330315f32385f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d323034382c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f5f323032342d30362d32355f32322d31302d33342f636865636b706f696e745f30303030313894682268257d946827897385945294682a682d2981947d9468308a11ab454fccca6de7be48460b4ecd666cfa007362756268317d94288c046c6f7373946836683c430871381cae7ae0554094869452948c0974696d657374616d70944a8a537b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303138948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b138c08747269616c5f6964948c083361623266333031948c0464617465948c13323032342d30362d32365f30312d33322d3236948c1074696d655f746869735f697465725f73944740498ba1cc0000008c0c74696d655f746f74616c5f739447408fece17b8000008c03706964944a027f2b008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b028c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b108c0a656e636f6465725f6666944d00088c0d6465636f6465725f72745f6666944d00088c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b018c0964726f705f7261746594473fa1a795df4923d08c026c7294473f50b05f345becb88c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f72659447407da9a95c0000008c18697465726174696f6e735f73696e63655f726573746f7265944b0975756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f33616232663330315f32385f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d323034382c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f5f323032342d30362d32355f32322d31302d33342f636865636b706f696e745f30303030313994682268257d946827897385945294682a682d2981947d9468308a11c2289f66b1a5fb8e3745534de55c89d4007362756268317d94288c046c6f7373946836683c430843a150a85934554094869452948c0974696d657374616d70944abe537b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303139948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b148c08747269616c5f6964948c083361623266333031948c0464617465948c13323032342d30362d32365f30312d33332d3138948c1074696d655f746869735f697465725f7394474049afa3880000008c0c74696d655f746f74616c5f7394474090c3edda0000008c03706964944a027f2b008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b028c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b108c0a656e636f6465725f6666944d00088c0d6465636f6465725f72745f6666944d00088c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b018c0964726f705f7261746594473fa1a795df4923d08c026c7294473f50b05f345becb88c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f7265944740806fcee68000008c18697465726174696f6e735f73696e63655f726573746f7265944b0a757562658c195f6c61746573745f636865636b706f696e745f726573756c74946ab003000075622e\"\n }\n}"
+ ],
+ [
+ "{\n \"stub\": false,\n \"trainable_name\": \"train_model\",\n \"trial_id\": \"40101b9a\",\n \"storage\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059572040000000000008c1b7261792e747261696e2e5f696e7465726e616c2e73746f72616765948c0e53746f72616765436f6e746578749493942981947d94288c12637573746f6d5f66735f70726f766964656494898c1273746f726167655f6c6f63616c5f70617468948c2b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c7473948c136578706572696d656e745f6469725f6e616d65948c0f746573745f6578706572696d656e74948c0e747269616c5f6469725f6e616d65948c96747261696e5f6d6f64656c5f34303130316239615f32395f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d3531322c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f725f323032342d30362d32355f32322d31352d3437948c1863757272656e745f636865636b706f696e745f696e646578944b048c0b73796e635f636f6e666967948c097261792e747261696e948c0a53796e63436f6e6669679493942981947d94288c0b73796e635f706572696f64944d2c018c0c73796e635f74696d656f7574944d08078c0e73796e635f61727469666163747394898c1c73796e635f6172746966616374735f6f6e5f636865636b706f696e7494888c0a75706c6f61645f646972948c0a44455052454341544544948c0673796e6365729468188c1273796e635f6f6e5f636865636b706f696e7494681875628c1273746f726167655f66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c0f73746f726167655f66735f70617468948c452f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f7465737494681968008c115f46696c6573797374656d53796e6365729493942981947d9428681b682268134d2c0168144d08078c116c6173745f73796e635f75705f74696d65944741d99ec923518d8b8c136c6173745f73796e635f646f776e5f74696d659447fff00000000000008c0d5f73796e635f70726f63657373944e8c0c5f63757272656e745f636d649468008c125f75706c6f61645f746f5f66735f706174689493947d94288c0a6c6f63616c5f70617468948c3b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c74732f746573745f6578706572696d656e74948c0266739468228c0766735f70617468948c552f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e74948c076578636c756465945d948c0e2a2f636865636b706f696e745f2a9461758694756275622e\"\n },\n \"config\": {\n \"encoder_num_layer\": 2,\n \"decoder_rt_num_layer\": 1,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 1024,\n \"encoder_ff\": 2048,\n \"decoder_rt_ff\": 512,\n \"decoder_int_ff\": 512,\n \"n_head\": 1,\n \"drop_rate\": 0.8983976265900149,\n \"lr\": 0.039339947723305395,\n \"batch_size\": 2048\n },\n \"_Trial__unresolved_config\": {\n \"encoder_num_layer\": 2,\n \"decoder_rt_num_layer\": 1,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 1024,\n \"encoder_ff\": 2048,\n \"decoder_rt_ff\": 512,\n \"decoder_int_ff\": 512,\n \"n_head\": 1,\n \"drop_rate\": 0.8983976265900149,\n \"lr\": 0.039339947723305395,\n \"batch_size\": 2048\n },\n \"evaluated_params\": {\n \"batch_size\": 2048,\n \"decoder_int_ff\": 512,\n \"decoder_int_num_layer\": 1,\n \"decoder_rt_ff\": 512,\n \"decoder_rt_num_layer\": 1,\n \"drop_rate\": 0.8983976265900149,\n \"embedding_dim\": 1024,\n \"encoder_ff\": 2048,\n \"encoder_num_layer\": 2,\n \"lr\": 0.039339947723305395,\n \"n_head\": 1\n },\n \"experiment_tag\": \"29_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=1,drop_rate=0.8984,embedding_dim=1024,encoder_ff=2048,encoder_num_layer=2,lr=0.0393,n_head=1\",\n \"stopping_criterion\": {},\n \"_setup_default_resource\": true,\n \"_default_placement_group_factory\": \"80054e2e\",\n \"placement_group_factory\": \"800595ba000000000000008c237261792e74756e652e657865637574696f6e2e706c6163656d656e745f67726f757073948c15506c6163656d656e7447726f7570466163746f72799493942981947d94288c085f62756e646c6573945d947d94288c03435055944740540000000000008c034750559447401000000000000075618c155f686561645f62756e646c655f69735f656d70747994898c095f7374726174656779948c045041434b948c055f6172677394298c075f6b7761726773947d9475622e\",\n \"log_to_file\": [\n null,\n null\n ],\n \"max_failures\": 0,\n \"_default_result_or_future\": null,\n \"export_formats\": [],\n \"status\": \"PAUSED\",\n \"relative_logdir\": \"train_model_40101b9a_29_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=1,drop_r_2024-06-25_22-15-47\",\n \"trial_name_creator\": null,\n \"trial_dirname_creator\": null,\n \"custom_trial_name\": null,\n \"custom_dirname\": null,\n \"restore_path\": null,\n \"_restore_checkpoint_result\": null,\n \"_state_json\": null,\n \"results\": \"80054e2e\",\n \"extra_arg\": \"80054e2e\",\n \"_resources\": \"80054e2e\"\n}",
+ "{\n \"start_time\": 1719346823.9898968,\n \"num_failures\": 0,\n \"num_failures_after_restore\": 0,\n \"error_filename\": null,\n \"pickled_error_filename\": null,\n \"last_result\": {\n \"loss\": 2031.5720868448573,\n \"timestamp\": 1719348338,\n \"checkpoint_dir_name\": \"checkpoint_000004\",\n \"should_checkpoint\": true,\n \"done\": false,\n \"training_iteration\": 5,\n \"trial_id\": \"40101b9a\",\n \"date\": \"2024-06-25_22-45-38\",\n \"time_this_iter_s\": 300.0205228328705,\n \"time_total_s\": 1514.9287722110748,\n \"pid\": 2195253,\n \"hostname\": \"r8i6n2\",\n \"node_ip\": \"10.159.28.60\",\n \"config\": {\n \"encoder_num_layer\": 2,\n \"decoder_rt_num_layer\": 1,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 1024,\n \"encoder_ff\": 2048,\n \"decoder_rt_ff\": 512,\n \"decoder_int_ff\": 512,\n \"n_head\": 1,\n \"drop_rate\": 0.8983976265900149,\n \"lr\": 0.039339947723305395,\n \"batch_size\": 2048\n },\n \"time_since_restore\": 1514.9287722110748,\n \"iterations_since_restore\": 5,\n \"experiment_tag\": \"29_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=1,drop_rate=0.8984,embedding_dim=1024,encoder_ff=2048,encoder_num_layer=2,lr=0.0393,n_head=1\"\n },\n \"last_result_time\": 1719348338.9425833,\n \"metric_analysis\": {\n \"loss\": {\n \"max\": 4106.712271622785,\n \"min\": 2031.5720868448573,\n \"avg\": 2792.4098417507375,\n \"last\": 2031.5720868448573,\n \"last-5-avg\": 2792.409841750738,\n \"last-10-avg\": 2792.409841750738\n },\n \"should_checkpoint\": {\n \"max\": true,\n \"min\": true,\n \"avg\": 1.0,\n \"last\": true,\n \"last-5-avg\": 1.0,\n \"last-10-avg\": 1.0\n },\n \"done\": {\n \"max\": false,\n \"min\": false,\n \"avg\": 0.0,\n \"last\": false,\n \"last-5-avg\": 0.0,\n \"last-10-avg\": 0.0\n },\n \"training_iteration\": {\n \"max\": 5,\n \"min\": 1,\n \"avg\": 3.0,\n \"last\": 5,\n \"last-5-avg\": 3.0,\n \"last-10-avg\": 3.0\n },\n \"time_this_iter_s\": {\n \"max\": 313.700572013855,\n \"min\": 300.0205228328705,\n \"avg\": 302.98575444221495,\n \"last\": 300.0205228328705,\n \"last-5-avg\": 302.98575444221495,\n \"last-10-avg\": 302.98575444221495\n },\n \"time_total_s\": {\n \"max\": 1514.9287722110748,\n \"min\": 313.700572013855,\n \"avg\": 914.4032258033753,\n \"last\": 1514.9287722110748,\n \"last-5-avg\": 914.4032258033752,\n \"last-10-avg\": 914.4032258033752\n },\n \"time_since_restore\": {\n \"max\": 1514.9287722110748,\n \"min\": 313.700572013855,\n \"avg\": 914.4032258033753,\n \"last\": 1514.9287722110748,\n \"last-5-avg\": 914.4032258033752,\n \"last-10-avg\": 914.4032258033752\n },\n \"iterations_since_restore\": {\n \"max\": 5,\n \"min\": 1,\n \"avg\": 3.0,\n \"last\": 5,\n \"last-5-avg\": 3.0,\n \"last-10-avg\": 3.0\n }\n },\n \"_n_steps\": [\n 5,\n 10\n ],\n \"metric_n_steps\": {\n \"loss\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"800595d6000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294288c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b007494624308bbdd6e57b60ab04094869452946807680d43083e9fcfa72d74a84094869452946807680d4308080402e157a1a34094869452946807680d43088d46a311020aa14094869452946807680d4308894422d149be9f409486945294652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"800595d6000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294288c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b007494624308bbdd6e57b60ab04094869452946807680d43083e9fcfa72d74a84094869452946807680d4308080402e157a1a34094869452946807680d43088d46a311020aa14094869452946807680d4308894422d149be9f409486945294652e\"\n }\n },\n \"should_checkpoint\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059527000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294288888888888652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059527000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294288888888888652e\"\n }\n },\n \"done\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059527000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294288989898989652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059527000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294288989898989652e\"\n }\n },\n \"training_iteration\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952c000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294284b014b024b034b044b05652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952c000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294284b014b024b034b044b05652e\"\n }\n },\n \"time_this_iter_s\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005954f000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294284740739b358b000000474072c7fbec800000474072c14525c00000474072ca1193400000474072c0540fc00000652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005954f000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294284740739b358b000000474072c7fbec800000474072c14525c00000474072ca1193400000474072c0540fc00000652e\"\n }\n },\n \"time_total_s\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005954f000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294284740739b358b0000004740833198bbc0000047408c923b4ea00000474092fba20c200000474097abb710100000652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005954f000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294284740739b358b0000004740833198bbc0000047408c923b4ea00000474092fba20c200000474097abb710100000652e\"\n }\n },\n \"time_since_restore\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005954f000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294284740739b358b0000004740833198bbc0000047408c923b4ea00000474092fba20c200000474097abb710100000652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005954f000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294284740739b358b0000004740833198bbc0000047408c923b4ea00000474092fba20c200000474097abb710100000652e\"\n }\n },\n \"iterations_since_restore\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952c000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294284b014b024b034b044b05652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952c000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294284b014b024b034b044b05652e\"\n }\n }\n },\n \"checkpoint_manager\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005954e140000000000008c267261792e747261696e2e5f696e7465726e616c2e636865636b706f696e745f6d616e61676572948c125f436865636b706f696e744d616e616765729493942981947d94288c125f636865636b706f696e745f636f6e666967948c097261792e747261696e948c10436865636b706f696e74436f6e6669679493942981947d94288c0b6e756d5f746f5f6b656570944e8c1a636865636b706f696e745f73636f72655f617474726962757465944e8c16636865636b706f696e745f73636f72655f6f72646572948c036d6178948c14636865636b706f696e745f6672657175656e6379944b008c11636865636b706f696e745f61745f656e6494898c1a5f636865636b706f696e745f6b6565705f616c6c5f72616e6b73948c0a44455052454341544544948c1f5f636865636b706f696e745f75706c6f61645f66726f6d5f776f726b65727394681275628c135f636865636b706f696e745f726573756c7473945d94288c1b7261792e747261696e2e5f696e7465726e616c2e73657373696f6e948c0f5f547261696e696e67526573756c749493942981947d94288c0a636865636b706f696e749468068c0a436865636b706f696e749493942981947d94288c0470617468948cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f34303130316239615f32395f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d3531322c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f725f323032342d30362d32355f32322d31352d34372f636865636b706f696e745f303030303030948c0a66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c055f75756964948c0475756964948c04555549449493942981947d948c03696e74948a10466204a4ea70b689744ef652d6f7e478736275628c076d657472696373947d94288c046c6f7373948c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b007494624308bbdd6e57b60ab04094869452948c0974696d657374616d70944ac1277b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303030948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b018c08747269616c5f6964948c083430313031623961948c0464617465948c13323032342d30362d32355f32322d32352d3337948c1074696d655f746869735f697465725f73944740739b358b0000008c0c74696d655f746f74616c5f73944740739b358b0000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b028c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944d00048c0a656e636f6465725f6666944d00088c0d6465636f6465725f72745f6666944d00028c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b018c0964726f705f7261746594473fecbfac612042a28c026c7294473fa4245d99cbbf9f8c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f7265944740739b358b0000008c18697465726174696f6e735f73696e63655f726573746f7265944b0175756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f34303130316239615f32395f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d3531322c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f725f323032342d30362d32355f32322d31352d34372f636865636b706f696e745f30303030303194682268257d946827897385945294682a682d2981947d9468308a109d9c39eefbe09e9c564bf1f9a07b40327362756268317d94288c046c6f7373946836683c43083e9fcfa72d74a84094869452948c0974696d657374616d70944aed287b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303031948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b028c08747269616c5f6964948c083430313031623961948c0464617465948c13323032342d30362d32355f32322d33302d3338948c1074696d655f746869735f697465725f7394474072c7fbec8000008c0c74696d655f746f74616c5f73944740833198bbc000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b028c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944d00048c0a656e636f6465725f6666944d00088c0d6465636f6465725f72745f6666944d00028c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b018c0964726f705f7261746594473fecbfac612042a28c026c7294473fa4245d99cbbf9f8c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f7265944740833198bbc000008c18697465726174696f6e735f73696e63655f726573746f7265944b0275756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f34303130316239615f32395f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d3531322c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f725f323032342d30362d32355f32322d31352d34372f636865636b706f696e745f30303030303294682268257d946827897385945294682a682d2981947d9468308a10c5592afb394617b1fe42039a7b0872297362756268317d94288c046c6f7373946836683c4308080402e157a1a34094869452948c0974696d657374616d70944a192a7b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303032948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b038c08747269616c5f6964948c083430313031623961948c0464617465948c13323032342d30362d32355f32322d33352d3338948c1074696d655f746869735f697465725f7394474072c14525c000008c0c74696d655f746f74616c5f739447408c923b4ea000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b028c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944d00048c0a656e636f6465725f6666944d00088c0d6465636f6465725f72745f6666944d00028c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b018c0964726f705f7261746594473fecbfac612042a28c026c7294473fa4245d99cbbf9f8c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f72659447408c923b4ea000008c18697465726174696f6e735f73696e63655f726573746f7265944b0375756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f34303130316239615f32395f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d3531322c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f725f323032342d30362d32355f32322d31352d34372f636865636b706f696e745f30303030303394682268257d946827897385945294682a682d2981947d9468308a103e648eca43fcfebd824735f6a5de520d7362756268317d94288c046c6f7373946836683c43088d46a311020aa14094869452948c0974696d657374616d70944a462b7b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303033948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b048c08747269616c5f6964948c083430313031623961948c0464617465948c13323032342d30362d32355f32322d34302d3338948c1074696d655f746869735f697465725f7394474072ca11934000008c0c74696d655f746f74616c5f7394474092fba20c2000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b028c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944d00048c0a656e636f6465725f6666944d00088c0d6465636f6465725f72745f6666944d00028c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b018c0964726f705f7261746594473fecbfac612042a28c026c7294473fa4245d99cbbf9f8c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f726594474092fba20c2000008c18697465726174696f6e735f73696e63655f726573746f7265944b0475756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f34303130316239615f32395f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d3531322c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f725f323032342d30362d32355f32322d31352d34372f636865636b706f696e745f30303030303494682268257d946827897385945294682a682d2981947d9468308a11e83d97333973ffa71d4c11b616163ec9007362756268317d94288c046c6f7373946836683c4308894422d149be9f4094869452948c0974696d657374616d70944a722c7b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303034948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b058c08747269616c5f6964948c083430313031623961948c0464617465948c13323032342d30362d32355f32322d34352d3338948c1074696d655f746869735f697465725f7394474072c0540fc000008c0c74696d655f746f74616c5f7394474097abb7101000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b028c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944d00048c0a656e636f6465725f6666944d00088c0d6465636f6465725f72745f6666944d00028c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b018c0964726f705f7261746594473fecbfac612042a28c026c7294473fa4245d99cbbf9f8c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f726594474097abb7101000008c18697465726174696f6e735f73696e63655f726573746f7265944b05757562658c195f6c61746573745f636865636b706f696e745f726573756c749468ef75622e\"\n }\n}"
+ ],
+ [
+ "{\n \"stub\": false,\n \"trainable_name\": \"train_model\",\n \"trial_id\": \"de1e9134\",\n \"storage\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059572040000000000008c1b7261792e747261696e2e5f696e7465726e616c2e73746f72616765948c0e53746f72616765436f6e746578749493942981947d94288c12637573746f6d5f66735f70726f766964656494898c1273746f726167655f6c6f63616c5f70617468948c2b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c7473948c136578706572696d656e745f6469725f6e616d65948c0f746573745f6578706572696d656e74948c0e747269616c5f6469725f6e616d65948c96747261696e5f6d6f64656c5f64653165393133345f33305f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d3531322c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f725f323032342d30362d32355f32322d32302d3233948c1863757272656e745f636865636b706f696e745f696e646578944b098c0b73796e635f636f6e666967948c097261792e747261696e948c0a53796e63436f6e6669679493942981947d94288c0b73796e635f706572696f64944d2c018c0c73796e635f74696d656f7574944d08078c0e73796e635f61727469666163747394898c1c73796e635f6172746966616374735f6f6e5f636865636b706f696e7494888c0a75706c6f61645f646972948c0a44455052454341544544948c0673796e6365729468188c1273796e635f6f6e5f636865636b706f696e7494681875628c1273746f726167655f66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c0f73746f726167655f66735f70617468948c452f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f7465737494681968008c115f46696c6573797374656d53796e6365729493942981947d9428681b682268134d2c0168144d08078c116c6173745f73796e635f75705f74696d65944741d99ec96ee4bc5f8c136c6173745f73796e635f646f776e5f74696d659447fff00000000000008c0d5f73796e635f70726f63657373944e8c0c5f63757272656e745f636d649468008c125f75706c6f61645f746f5f66735f706174689493947d94288c0a6c6f63616c5f70617468948c3b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c74732f746573745f6578706572696d656e74948c0266739468228c0766735f70617468948c552f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e74948c076578636c756465945d948c0e2a2f636865636b706f696e745f2a9461758694756275622e\"\n },\n \"config\": {\n \"encoder_num_layer\": 4,\n \"decoder_rt_num_layer\": 1,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 16,\n \"encoder_ff\": 1024,\n \"decoder_rt_ff\": 512,\n \"decoder_int_ff\": 512,\n \"n_head\": 1,\n \"drop_rate\": 0.12892236942851165,\n \"lr\": 0.005626677279957184,\n \"batch_size\": 2048\n },\n \"_Trial__unresolved_config\": {\n \"encoder_num_layer\": 4,\n \"decoder_rt_num_layer\": 1,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 16,\n \"encoder_ff\": 1024,\n \"decoder_rt_ff\": 512,\n \"decoder_int_ff\": 512,\n \"n_head\": 1,\n \"drop_rate\": 0.12892236942851165,\n \"lr\": 0.005626677279957184,\n \"batch_size\": 2048\n },\n \"evaluated_params\": {\n \"batch_size\": 2048,\n \"decoder_int_ff\": 512,\n \"decoder_int_num_layer\": 1,\n \"decoder_rt_ff\": 512,\n \"decoder_rt_num_layer\": 1,\n \"drop_rate\": 0.12892236942851165,\n \"embedding_dim\": 16,\n \"encoder_ff\": 1024,\n \"encoder_num_layer\": 4,\n \"lr\": 0.005626677279957184,\n \"n_head\": 1\n },\n \"experiment_tag\": \"30_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=1,drop_rate=0.1289,embedding_dim=16,encoder_ff=1024,encoder_num_layer=4,lr=0.0056,n_head=1\",\n \"stopping_criterion\": {},\n \"_setup_default_resource\": true,\n \"_default_placement_group_factory\": \"80054e2e\",\n \"placement_group_factory\": \"800595ba000000000000008c237261792e74756e652e657865637574696f6e2e706c6163656d656e745f67726f757073948c15506c6163656d656e7447726f7570466163746f72799493942981947d94288c085f62756e646c6573945d947d94288c03435055944740540000000000008c034750559447401000000000000075618c155f686561645f62756e646c655f69735f656d70747994898c095f7374726174656779948c045041434b948c055f6172677394298c075f6b7761726773947d9475622e\",\n \"log_to_file\": [\n null,\n null\n ],\n \"max_failures\": 0,\n \"_default_result_or_future\": null,\n \"export_formats\": [],\n \"status\": \"PAUSED\",\n \"relative_logdir\": \"train_model_de1e9134_30_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=1,drop_r_2024-06-25_22-20-23\",\n \"trial_name_creator\": null,\n \"trial_dirname_creator\": null,\n \"custom_trial_name\": null,\n \"custom_dirname\": null,\n \"restore_path\": null,\n \"_restore_checkpoint_result\": null,\n \"_state_json\": null,\n \"results\": \"80054e2e\",\n \"extra_arg\": \"80054e2e\",\n \"_resources\": \"80054e2e\"\n}",
+ "{\n \"start_time\": 1719348339.0969658,\n \"num_failures\": 0,\n \"num_failures_after_restore\": 0,\n \"error_filename\": null,\n \"pickled_error_filename\": null,\n \"last_result\": {\n \"loss\": 1957.9578991987573,\n \"timestamp\": 1719355501,\n \"checkpoint_dir_name\": \"checkpoint_000009\",\n \"should_checkpoint\": true,\n \"done\": false,\n \"training_iteration\": 10,\n \"trial_id\": \"de1e9134\",\n \"date\": \"2024-06-26_00-45-01\",\n \"time_this_iter_s\": 52.88777208328247,\n \"time_total_s\": 560.1831469535828,\n \"pid\": 2850562,\n \"hostname\": \"r8i6n2\",\n \"node_ip\": \"10.159.28.60\",\n \"config\": {\n \"encoder_num_layer\": 4,\n \"decoder_rt_num_layer\": 1,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 16,\n \"encoder_ff\": 1024,\n \"decoder_rt_ff\": 512,\n \"decoder_int_ff\": 512,\n \"n_head\": 1,\n \"drop_rate\": 0.12892236942851165,\n \"lr\": 0.005626677279957184,\n \"batch_size\": 2048\n },\n \"time_since_restore\": 278.3368046283722,\n \"iterations_since_restore\": 5,\n \"experiment_tag\": \"30_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=1,drop_rate=0.1289,embedding_dim=16,encoder_ff=1024,encoder_num_layer=4,lr=0.0056,n_head=1\"\n },\n \"last_result_time\": 1719355501.903357,\n \"metric_analysis\": {\n \"loss\": {\n \"max\": 1966.5979321096827,\n \"min\": 1954.8947446327509,\n \"avg\": 1957.8368239996003,\n \"last\": 1957.9578991987573,\n \"last-5-avg\": 1957.8378904327633,\n \"last-10-avg\": 1957.8368239996005\n },\n \"should_checkpoint\": {\n \"max\": true,\n \"min\": true,\n \"avg\": 1.0,\n \"last\": true,\n \"last-5-avg\": 1.0,\n \"last-10-avg\": 1.0\n },\n \"done\": {\n \"max\": false,\n \"min\": false,\n \"avg\": 0.0,\n \"last\": false,\n \"last-5-avg\": 0.0,\n \"last-10-avg\": 0.0\n },\n \"training_iteration\": {\n \"max\": 10,\n \"min\": 1,\n \"avg\": 5.5,\n \"last\": 10,\n \"last-5-avg\": 8.0,\n \"last-10-avg\": 5.5\n },\n \"time_this_iter_s\": {\n \"max\": 67.86075568199158,\n \"min\": 52.848015785217285,\n \"avg\": 56.01831469535828,\n \"last\": 52.88777208328247,\n \"last-5-avg\": 55.66736092567444,\n \"last-10-avg\": 56.01831469535828\n },\n \"time_total_s\": {\n \"max\": 560.1831469535828,\n \"min\": 67.86075568199158,\n \"avg\": 314.47027125358585,\n \"last\": 560.1831469535828,\n \"last-5-avg\": 454.1982753276825,\n \"last-10-avg\": 314.4702712535858\n },\n \"time_since_restore\": {\n \"max\": 281.84634232521057,\n \"min\": 66.45859718322754,\n \"avg\": 173.5471000909805,\n \"last\": 278.3368046283722,\n \"last-5-avg\": 172.35193300247192,\n \"last-10-avg\": 173.54710009098054\n },\n \"iterations_since_restore\": {\n \"max\": 5,\n \"min\": 1,\n \"avg\": 3.0,\n \"last\": 5,\n \"last-5-avg\": 3.0,\n \"last-10-avg\": 3.0\n }\n },\n \"_n_steps\": [\n 5,\n 10\n ],\n \"metric_n_steps\": {\n \"loss\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"800595d6000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294288c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b0074946243086532998c198c9e4094869452946807680d430847a3d14879a99e4094869452946807680d430845229188ed949e4094869452946807680d4308f2783cbe6c929e4094869452946807680d43081c0e87e3d4979e409486945294652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059535010000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294288c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b007494624308a9542af5cb929e4094869452946807680d430843a1504864ba9e4094869452946807680d4308bfdfef37948b9e4094869452946807680d430893c96472978f9e4094869452946807680d430863b1582c5b8c9e4094869452946807680d43086532998c198c9e4094869452946807680d430847a3d14879a99e4094869452946807680d430845229188ed949e4094869452946807680d4308f2783cbe6c929e4094869452946807680d43081c0e87e3d4979e409486945294652e\"\n }\n },\n \"should_checkpoint\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059527000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294288888888888652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952c000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452942888888888888888888888652e\"\n }\n },\n \"done\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059527000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294288989898989652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952c000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452942889898989898989898989652e\"\n }\n },\n \"training_iteration\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952c000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294284b064b074b084b094b0a652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059536000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294284b014b024b034b044b054b064b074b084b094b0a652e\"\n }\n },\n \"time_this_iter_s\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005954f000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294284740509d59a800000047404a6c8bc800000047404a7374e400000047404a9ec5ea00000047404a71a284000000652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005957c000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a8694529428474050f7169f00000047404ab327b400000047404ab51a8a00000047404ab256f400000047404ae38e820000004740509d59a800000047404a6c8bc800000047404a7374e400000047404a9ec5ea00000047404a71a284000000652e\"\n }\n },\n \"time_total_s\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005954f000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b058694529428474075c4e10840000047407912728140000047407c60e11dc0000047407fb4b9db000000474081817715c00000652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005957c000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a8694529428474050f7169f00000047405e50aa79000000474065d59bdf00000047406c82319c0000004740719d8a9e400000474075c4e10840000047407912728140000047407c60e11dc0000047407fb4b9db000000474081817715c00000652e\"\n }\n },\n \"time_since_restore\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005954f000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294284740509d59a800000047405dd39f8c00000047406586acff00000047406c2e5e7980000047407165638d400000652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005957c000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a8694529428474050f7169f00000047405e50aa79000000474065d59bdf00000047406c82319c0000004740719d8a9e4000004740509d59a800000047405dd39f8c00000047406586acff00000047406c2e5e7980000047407165638d400000652e\"\n }\n },\n \"iterations_since_restore\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952c000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294284b014b024b034b044b05652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059536000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294284b014b024b034b044b054b014b024b034b044b05652e\"\n }\n }\n },\n \"checkpoint_manager\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005950b260000000000008c267261792e747261696e2e5f696e7465726e616c2e636865636b706f696e745f6d616e61676572948c125f436865636b706f696e744d616e616765729493942981947d94288c125f636865636b706f696e745f636f6e666967948c097261792e747261696e948c10436865636b706f696e74436f6e6669679493942981947d94288c0b6e756d5f746f5f6b656570944e8c1a636865636b706f696e745f73636f72655f617474726962757465944e8c16636865636b706f696e745f73636f72655f6f72646572948c036d6178948c14636865636b706f696e745f6672657175656e6379944b008c11636865636b706f696e745f61745f656e6494898c1a5f636865636b706f696e745f6b6565705f616c6c5f72616e6b73948c0a44455052454341544544948c1f5f636865636b706f696e745f75706c6f61645f66726f6d5f776f726b65727394681275628c135f636865636b706f696e745f726573756c7473945d94288c1b7261792e747261696e2e5f696e7465726e616c2e73657373696f6e948c0f5f547261696e696e67526573756c749493942981947d94288c0a636865636b706f696e749468068c0a436865636b706f696e749493942981947d94288c0470617468948cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f64653165393133345f33305f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d3531322c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f725f323032342d30362d32355f32322d32302d32332f636865636b706f696e745f303030303030948c0a66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c055f75756964948c0475756964948c04555549449493942981947d948c03696e74948a117e99e52d3a1ad38e0d44aea7c54e25dc00736275628c076d657472696373947d94288c046c6f7373948c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b007494624308a9542af5cb929e4094869452948c0974696d657374616d70944ab62c7b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303030948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b018c08747269616c5f6964948c086465316539313334948c0464617465948c13323032342d30362d32355f32322d34362d3436948c1074696d655f746869735f697465725f7394474050f7169f0000008c0c74696d655f746f74616c5f7394474050f7169f0000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b048c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b108c0a656e636f6465725f6666944d00048c0d6465636f6465725f72745f6666944d00028c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b018c0964726f705f7261746594473fc0808738358a748c026c7294473f770bffae70e5368c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f726594474050f7169f0000008c18697465726174696f6e735f73696e63655f726573746f7265944b0175756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f64653165393133345f33305f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d3531322c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f725f323032342d30362d32355f32322d32302d32332f636865636b706f696e745f30303030303194682268257d946827897385945294682a682d2981947d9468308a11e0766b93104e69aa2a4ce8efe19787dc007362756268317d94288c046c6f7373946836683c430843a1504864ba9e4094869452948c0974696d657374616d70944aec2c7b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303031948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b028c08747269616c5f6964948c086465316539313334948c0464617465948c13323032342d30362d32355f32322d34372d3430948c1074696d655f746869735f697465725f739447404ab327b40000008c0c74696d655f746f74616c5f739447405e50aa790000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b048c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b108c0a656e636f6465725f6666944d00048c0d6465636f6465725f72745f6666944d00028c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b018c0964726f705f7261746594473fc0808738358a748c026c7294473f770bffae70e5368c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f72659447405e50aa790000008c18697465726174696f6e735f73696e63655f726573746f7265944b0275756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f64653165393133345f33305f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d3531322c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f725f323032342d30362d32355f32322d32302d32332f636865636b706f696e745f30303030303294682268257d946827897385945294682a682d2981947d9468308a1172f8bcad354c3998c04ca697a29c22d1007362756268317d94288c046c6f7373946836683c4308bfdfef37948b9e4094869452948c0974696d657374616d70944a212d7b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303032948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b038c08747269616c5f6964948c086465316539313334948c0464617465948c13323032342d30362d32355f32322d34382d3333948c1074696d655f746869735f697465725f739447404ab51a8a0000008c0c74696d655f746f74616c5f7394474065d59bdf0000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b048c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b108c0a656e636f6465725f6666944d00048c0d6465636f6465725f72745f6666944d00028c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b018c0964726f705f7261746594473fc0808738358a748c026c7294473f770bffae70e5368c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f726594474065d59bdf0000008c18697465726174696f6e735f73696e63655f726573746f7265944b0375756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f64653165393133345f33305f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d3531322c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f725f323032342d30362d32355f32322d32302d32332f636865636b706f696e745f30303030303394682268257d946827897385945294682a682d2981947d9468308a1029599fd908f38f99194dfe050a975e4b7362756268317d94288c046c6f7373946836683c430893c96472978f9e4094869452948c0974696d657374616d70944a572d7b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303033948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b048c08747269616c5f6964948c086465316539313334948c0464617465948c13323032342d30362d32355f32322d34392d3237948c1074696d655f746869735f697465725f739447404ab256f40000008c0c74696d655f746f74616c5f739447406c82319c0000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b048c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b108c0a656e636f6465725f6666944d00048c0d6465636f6465725f72745f6666944d00028c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b018c0964726f705f7261746594473fc0808738358a748c026c7294473f770bffae70e5368c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f72659447406c82319c0000008c18697465726174696f6e735f73696e63655f726573746f7265944b0475756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f64653165393133345f33305f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d3531322c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f725f323032342d30362d32355f32322d32302d32332f636865636b706f696e745f30303030303494682268257d946827897385945294682a682d2981947d9468308a108717ed70f9eab9802a4cfc2b28770c597362756268317d94288c046c6f7373946836683c430863b1582c5b8c9e4094869452948c0974696d657374616d70944a8c2d7b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303034948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b058c08747269616c5f6964948c086465316539313334948c0464617465948c13323032342d30362d32355f32322d35302d3230948c1074696d655f746869735f697465725f739447404ae38e820000008c0c74696d655f746f74616c5f73944740719d8a9e4000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b048c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b108c0a656e636f6465725f6666944d00048c0d6465636f6465725f72745f6666944d00028c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b018c0964726f705f7261746594473fc0808738358a748c026c7294473f770bffae70e5368c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f7265944740719d8a9e4000008c18697465726174696f6e735f73696e63655f726573746f7265944b0575756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f64653165393133345f33305f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d3531322c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f725f323032342d30362d32355f32322d32302d32332f636865636b706f696e745f30303030303594682268257d946827897385945294682a682d2981947d9468308a11a1d04c24c760538f2248b1b0ce71ab9a007362756268317d94288c046c6f7373946836683c43086532998c198c9e4094869452948c0974696d657374616d70944a99477b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303035948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b068c08747269616c5f6964948c086465316539313334948c0464617465948c13323032342d30362d32365f30302d34312d3330948c1074696d655f746869735f697465725f73944740509d59a80000008c0c74696d655f746f74616c5f7394474075c4e1084000008c03706964944a027f2b008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b048c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b108c0a656e636f6465725f6666944d00048c0d6465636f6465725f72745f6666944d00028c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b018c0964726f705f7261746594473fc0808738358a748c026c7294473f770bffae70e5368c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f7265944740509d59a80000008c18697465726174696f6e735f73696e63655f726573746f7265944b0175756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f64653165393133345f33305f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d3531322c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f725f323032342d30362d32355f32322d32302d32332f636865636b706f696e745f30303030303694682268257d946827897385945294682a682d2981947d9468308a100f0006703543e68bd642176c71a376337362756268317d94288c046c6f7373946836683c430847a3d14879a99e4094869452948c0974696d657374616d70944ace477b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303036948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b078c08747269616c5f6964948c086465316539313334948c0464617465948c13323032342d30362d32365f30302d34322d3232948c1074696d655f746869735f697465725f739447404a6c8bc80000008c0c74696d655f746f74616c5f73944740791272814000008c03706964944a027f2b008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b048c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b108c0a656e636f6465725f6666944d00048c0d6465636f6465725f72745f6666944d00028c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b018c0964726f705f7261746594473fc0808738358a748c026c7294473f770bffae70e5368c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f72659447405dd39f8c0000008c18697465726174696f6e735f73696e63655f726573746f7265944b0275756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f64653165393133345f33305f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d3531322c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f725f323032342d30362d32355f32322d32302d32332f636865636b706f696e745f30303030303794682268257d946827897385945294682a682d2981947d9468308a10846da0a32c220f9d3b4fdaea935a21477362756268317d94288c046c6f7373946836683c430845229188ed949e4094869452948c0974696d657374616d70944a03487b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303037948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b088c08747269616c5f6964948c086465316539313334948c0464617465948c13323032342d30362d32365f30302d34332d3135948c1074696d655f746869735f697465725f739447404a7374e40000008c0c74696d655f746f74616c5f739447407c60e11dc000008c03706964944a027f2b008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b048c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b108c0a656e636f6465725f6666944d00048c0d6465636f6465725f72745f6666944d00028c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b018c0964726f705f7261746594473fc0808738358a748c026c7294473f770bffae70e5368c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f72659447406586acff0000008c18697465726174696f6e735f73696e63655f726573746f7265944b0375756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f64653165393133345f33305f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d3531322c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f725f323032342d30362d32355f32322d32302d32332f636865636b706f696e745f30303030303894682268257d946827897385945294682a682d2981947d9468308a10f3fa414a50a891a01a4c61ba0c7c707b7362756268317d94288c046c6f7373946836683c4308f2783cbe6c929e4094869452948c0974696d657374616d70944a38487b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303038948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b098c08747269616c5f6964948c086465316539313334948c0464617465948c13323032342d30362d32365f30302d34342d3039948c1074696d655f746869735f697465725f739447404a9ec5ea0000008c0c74696d655f746f74616c5f739447407fb4b9db0000008c03706964944a027f2b008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b048c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b108c0a656e636f6465725f6666944d00048c0d6465636f6465725f72745f6666944d00028c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b018c0964726f705f7261746594473fc0808738358a748c026c7294473f770bffae70e5368c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f72659447406c2e5e798000008c18697465726174696f6e735f73696e63655f726573746f7265944b0475756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f64653165393133345f33305f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d3531322c6465636f6465725f72745f6e756d5f6c617965723d312c64726f705f725f323032342d30362d32355f32322d32302d32332f636865636b706f696e745f30303030303994682268257d946827897385945294682a682d2981947d9468308a1058e700d0973d4caa6949f5ee7039cd737362756268317d94288c046c6f7373946836683c43081c0e87e3d4979e4094869452948c0974696d657374616d70944a6d487b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303039948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b0a8c08747269616c5f6964948c086465316539313334948c0464617465948c13323032342d30362d32365f30302d34352d3031948c1074696d655f746869735f697465725f739447404a71a2840000008c0c74696d655f746f74616c5f7394474081817715c000008c03706964944a027f2b008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b048c146465636f6465725f72745f6e756d5f6c61796572944b018c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944b108c0a656e636f6465725f6666944d00048c0d6465636f6465725f72745f6666944d00028c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b018c0964726f705f7261746594473fc0808738358a748c026c7294473f770bffae70e5368c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f72659447407165638d4000008c18697465726174696f6e735f73696e63655f726573746f7265944b05757562658c195f6c61746573745f636865636b706f696e745f726573756c74946ada01000075622e\"\n }\n}"
+ ],
+ [
+ "{\n \"stub\": false,\n \"trainable_name\": \"train_model\",\n \"trial_id\": \"0f80a6d7\",\n \"storage\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059572040000000000008c1b7261792e747261696e2e5f696e7465726e616c2e73746f72616765948c0e53746f72616765436f6e746578749493942981947d94288c12637573746f6d5f66735f70726f766964656494898c1273746f726167655f6c6f63616c5f70617468948c2b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c7473948c136578706572696d656e745f6469725f6e616d65948c0f746573745f6578706572696d656e74948c0e747269616c5f6469725f6e616d65948c96747261696e5f6d6f64656c5f30663830613664375f33315f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d3531322c6465636f6465725f72745f6e756d5f6c617965723d322c64726f705f725f323032342d30362d32355f32322d34352d3339948c1863757272656e745f636865636b706f696e745f696e646578944b098c0b73796e635f636f6e666967948c097261792e747261696e948c0a53796e63436f6e6669679493942981947d94288c0b73796e635f706572696f64944d2c018c0c73796e635f74696d656f7574944d08078c0e73796e635f61727469666163747394898c1c73796e635f6172746966616374735f6f6e5f636865636b706f696e7494888c0a75706c6f61645f646972948c0a44455052454341544544948c0673796e6365729468188c1273796e635f6f6e5f636865636b706f696e7494681875628c1273746f726167655f66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c0f73746f726167655f66735f70617468948c452f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f7465737494681968008c115f46696c6573797374656d53796e6365729493942981947d9428681b682268134d2c0168144d08078c116c6173745f73796e635f75705f74696d65944741d99ecafb428f1a8c136c6173745f73796e635f646f776e5f74696d659447fff00000000000008c0d5f73796e635f70726f63657373944e8c0c5f63757272656e745f636d649468008c125f75706c6f61645f746f5f66735f706174689493947d94288c0a6c6f63616c5f70617468948c3b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c74732f746573745f6578706572696d656e74948c0266739468228c0766735f70617468948c552f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e74948c076578636c756465945d948c0e2a2f636865636b706f696e745f2a9461758694756275622e\"\n },\n \"config\": {\n \"encoder_num_layer\": 2,\n \"decoder_rt_num_layer\": 2,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 1024,\n \"encoder_ff\": 1024,\n \"decoder_rt_ff\": 512,\n \"decoder_int_ff\": 512,\n \"n_head\": 16,\n \"drop_rate\": 0.023920599186226577,\n \"lr\": 0.0024687399238039302,\n \"batch_size\": 2048\n },\n \"_Trial__unresolved_config\": {\n \"encoder_num_layer\": 2,\n \"decoder_rt_num_layer\": 2,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 1024,\n \"encoder_ff\": 1024,\n \"decoder_rt_ff\": 512,\n \"decoder_int_ff\": 512,\n \"n_head\": 16,\n \"drop_rate\": 0.023920599186226577,\n \"lr\": 0.0024687399238039302,\n \"batch_size\": 2048\n },\n \"evaluated_params\": {\n \"batch_size\": 2048,\n \"decoder_int_ff\": 512,\n \"decoder_int_num_layer\": 1,\n \"decoder_rt_ff\": 512,\n \"decoder_rt_num_layer\": 2,\n \"drop_rate\": 0.023920599186226577,\n \"embedding_dim\": 1024,\n \"encoder_ff\": 1024,\n \"encoder_num_layer\": 2,\n \"lr\": 0.0024687399238039302,\n \"n_head\": 16\n },\n \"experiment_tag\": \"31_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_rate=0.0239,embedding_dim=1024,encoder_ff=1024,encoder_num_layer=2,lr=0.0025,n_head=16\",\n \"stopping_criterion\": {},\n \"_setup_default_resource\": true,\n \"_default_placement_group_factory\": \"80054e2e\",\n \"placement_group_factory\": \"800595ba000000000000008c237261792e74756e652e657865637574696f6e2e706c6163656d656e745f67726f757073948c15506c6163656d656e7447726f7570466163746f72799493942981947d94288c085f62756e646c6573945d947d94288c03435055944740540000000000008c034750559447401000000000000075618c155f686561645f62756e646c655f69735f656d70747994898c095f7374726174656779948c045041434b948c055f6172677394298c075f6b7761726773947d9475622e\",\n \"log_to_file\": [\n null,\n null\n ],\n \"max_failures\": 0,\n \"_default_result_or_future\": null,\n \"export_formats\": [],\n \"status\": \"TERMINATED\",\n \"relative_logdir\": \"train_model_0f80a6d7_31_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_22-45-39\",\n \"trial_name_creator\": null,\n \"trial_dirname_creator\": null,\n \"custom_trial_name\": null,\n \"custom_dirname\": null,\n \"restore_path\": null,\n \"_restore_checkpoint_result\": null,\n \"_state_json\": null,\n \"results\": \"80054e2e\",\n \"extra_arg\": \"80054e2e\",\n \"_resources\": \"80054e2e\"\n}",
+ "{\n \"start_time\": 1719348621.1283247,\n \"num_failures\": 0,\n \"num_failures_after_restore\": 0,\n \"error_filename\": null,\n \"pickled_error_filename\": null,\n \"last_result\": {\n \"loss\": 1959.91755063515,\n \"timestamp\": 1719357265,\n \"checkpoint_dir_name\": \"checkpoint_000009\",\n \"should_checkpoint\": true,\n \"done\": true,\n \"training_iteration\": 10,\n \"trial_id\": \"0f80a6d7\",\n \"date\": \"2024-06-26_01-14-25\",\n \"time_this_iter_s\": 349.87603521347046,\n \"time_total_s\": 3527.171615600586,\n \"pid\": 2850562,\n \"hostname\": \"r8i6n2\",\n \"node_ip\": \"10.159.28.60\",\n \"config\": {\n \"encoder_num_layer\": 2,\n \"decoder_rt_num_layer\": 2,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 1024,\n \"encoder_ff\": 1024,\n \"decoder_rt_ff\": 512,\n \"decoder_int_ff\": 512,\n \"n_head\": 16,\n \"drop_rate\": 0.023920599186226577,\n \"lr\": 0.0024687399238039302,\n \"batch_size\": 2048\n },\n \"time_since_restore\": 1763.417979001999,\n \"iterations_since_restore\": 5,\n \"experiment_tag\": \"31_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_rate=0.0239,embedding_dim=1024,encoder_ff=1024,encoder_num_layer=2,lr=0.0025,n_head=16\"\n },\n \"last_result_time\": 1719357265.5030398,\n \"metric_analysis\": {\n \"loss\": {\n \"max\": 1986.2198524775467,\n \"min\": 1955.6729476808564,\n \"avg\": 1965.4819975124567,\n \"last\": 1959.91755063515,\n \"last-5-avg\": 1962.4524979238436,\n \"last-10-avg\": 1965.4819975124567\n },\n \"should_checkpoint\": {\n \"max\": true,\n \"min\": true,\n \"avg\": 1.0,\n \"last\": true,\n \"last-5-avg\": 1.0,\n \"last-10-avg\": 1.0\n },\n \"done\": {\n \"max\": true,\n \"min\": false,\n \"avg\": 0.1,\n \"last\": true,\n \"last-5-avg\": 0.2,\n \"last-10-avg\": 0.1\n },\n \"training_iteration\": {\n \"max\": 10,\n \"min\": 1,\n \"avg\": 5.5,\n \"last\": 10,\n \"last-5-avg\": 8.0,\n \"last-10-avg\": 5.5\n },\n \"time_this_iter_s\": {\n \"max\": 365.2865264415741,\n \"min\": 349.18421268463135,\n \"avg\": 352.71716156005857,\n \"last\": 349.87603521347046,\n \"last-5-avg\": 352.68359580039976,\n \"last-10-avg\": 352.71716156005857\n },\n \"time_total_s\": {\n \"max\": 3527.171615600586,\n \"min\": 363.18635082244873,\n \"avg\": 1945.676479148865,\n \"last\": 3527.171615600586,\n \"last-5-avg\": 2827.933716440201,\n \"last-10-avg\": 1945.6764791488647\n },\n \"time_since_restore\": {\n \"max\": 1763.753636598587,\n \"min\": 363.18635082244873,\n \"avg\": 1063.7996608495712,\n \"last\": 1763.417979001999,\n \"last-5-avg\": 1064.1800798416139,\n \"last-10-avg\": 1063.7996608495712\n },\n \"iterations_since_restore\": {\n \"max\": 5,\n \"min\": 1,\n \"avg\": 3.0,\n \"last\": 5,\n \"last-5-avg\": 3.0,\n \"last-10-avg\": 3.0\n }\n },\n \"_n_steps\": [\n 5,\n 10\n ],\n \"metric_n_steps\": {\n \"loss\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"800595d6000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294288c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b00749462430867b3d96c98cf9e4094869452946807680d4308ca643219b18e9e4094869452946807680d4308d0e7f339fd959e4094869452946807680d4308bd5eaf771abd9e4094869452946807680d430893c96492ab9f9e409486945294652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059535010000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294288c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b00749462430808040221e1089f4094869452946807680d4308fa7c3e7f32ee9e4094869452946807680d430808040241bdb39e4094869452946807680d4308d4e9745ab38e9e4094869452946807680d43080e87c3a1b6909e4094869452946807680d430867b3d96c98cf9e4094869452946807680d4308ca643219b18e9e4094869452946807680d4308d0e7f339fd959e4094869452946807680d4308bd5eaf771abd9e4094869452946807680d430893c96492ab9f9e409486945294652e\"\n }\n },\n \"should_checkpoint\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059527000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294288888888888652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952c000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452942888888888888888888888652e\"\n }\n },\n \"done\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059527000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294288989898988652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952c000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452942889898989898989898988652e\"\n }\n },\n \"training_iteration\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952c000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294284b064b074b084b094b0a652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059536000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294284b014b024b034b044b054b064b074b084b094b0a652e\"\n }\n },\n \"time_this_iter_s\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005954f000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b058694529428474076d4959cc00000474075d2f289000000474075db6648c00000474075d5bd5ec00000474075de043d800000652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005957c000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a8694529428474076b2fb4b000000474075e4b866c00000474075d9fe77c00000474075e767ddc00000474075e2f4de000000474076d4959cc00000474075d2f289000000474075db6648c00000474075d5bd5ec00000474075de043d800000652e\"\n }\n },\n \"time_total_s\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005954f000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294284740a0a214904000004740a35c72e16000004740a617dfaa7800004740a8d297565000004740ab8e57de000000652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005957c000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a8694529428474076b2fb4b0000004740864bd9d8e000004740909c6c8a600000474096164681d0000047409b8f03b95000004740a0a214904000004740a35c72e16000004740a617dfaa7800004740a8d297565000004740ab8e57de000000652e\"\n }\n },\n \"time_since_restore\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005954f000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b058694529428474076d4959cc0000047408653c412e00000474090a0bb9ba00000474096162af350000047409b8dac02b00000652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005957c000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a8694529428474076b2fb4b0000004740864bd9d8e000004740909c6c8a600000474096164681d0000047409b8f03b9500000474076d4959cc0000047408653c412e00000474090a0bb9ba00000474096162af350000047409b8dac02b00000652e\"\n }\n },\n \"iterations_since_restore\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952c000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294284b014b024b034b044b05652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059536000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294284b014b024b034b044b054b014b024b034b044b05652e\"\n }\n }\n },\n \"checkpoint_manager\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059514260000000000008c267261792e747261696e2e5f696e7465726e616c2e636865636b706f696e745f6d616e61676572948c125f436865636b706f696e744d616e616765729493942981947d94288c125f636865636b706f696e745f636f6e666967948c097261792e747261696e948c10436865636b706f696e74436f6e6669679493942981947d94288c0b6e756d5f746f5f6b656570944e8c1a636865636b706f696e745f73636f72655f617474726962757465944e8c16636865636b706f696e745f73636f72655f6f72646572948c036d6178948c14636865636b706f696e745f6672657175656e6379944b008c11636865636b706f696e745f61745f656e6494898c1a5f636865636b706f696e745f6b6565705f616c6c5f72616e6b73948c0a44455052454341544544948c1f5f636865636b706f696e745f75706c6f61645f66726f6d5f776f726b65727394681275628c135f636865636b706f696e745f726573756c7473945d94288c1b7261792e747261696e2e5f696e7465726e616c2e73657373696f6e948c0f5f547261696e696e67526573756c749493942981947d94288c0a636865636b706f696e749468068c0a436865636b706f696e749493942981947d94288c0470617468948cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f30663830613664375f33315f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d3531322c6465636f6465725f72745f6e756d5f6c617965723d322c64726f705f725f323032342d30362d32355f32322d34352d33392f636865636b706f696e745f303030303030948c0a66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c055f75756964948c0475756964948c04555549449493942981947d948c03696e74948a1161e27b7800fdc1924e4b3679cc099dd800736275628c076d657472696373947d94288c046c6f7373948c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b00749462430808040221e1089f4094869452948c0974696d657374616d70944af72e7b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303030948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b018c08747269616c5f6964948c083066383061366437948c0464617465948c13323032342d30362d32355f32322d35362d3234948c1074696d655f746869735f697465725f7394474076b2fb4b0000008c0c74696d655f746f74616c5f7394474076b2fb4b0000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b028c146465636f6465725f72745f6e756d5f6c61796572944b028c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944d00048c0a656e636f6465725f6666944d00048c0d6465636f6465725f72745f6666944d00028c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b108c0964726f705f7261746594473f987ea43cd280408c026c7294473f643952a785ac188c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f726594474076b2fb4b0000008c18697465726174696f6e735f73696e63655f726573746f7265944b0175756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f30663830613664375f33315f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d3531322c6465636f6465725f72745f6e756d5f6c617965723d322c64726f705f725f323032342d30362d32355f32322d34352d33392f636865636b706f696e745f30303030303194682268257d946827897385945294682a682d2981947d9468308a101bd69a0e219bcbaedd476198d47ebf7f7362756268317d94288c046c6f7373946836683c4308fa7c3e7f32ee9e4094869452948c0974696d657374616d70944a56307b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303031948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b028c08747269616c5f6964948c083066383061366437948c0464617465948c13323032342d30362d32355f32332d30322d3134948c1074696d655f746869735f697465725f7394474075e4b866c000008c0c74696d655f746f74616c5f73944740864bd9d8e000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b028c146465636f6465725f72745f6e756d5f6c61796572944b028c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944d00048c0a656e636f6465725f6666944d00048c0d6465636f6465725f72745f6666944d00028c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b108c0964726f705f7261746594473f987ea43cd280408c026c7294473f643952a785ac188c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f7265944740864bd9d8e000008c18697465726174696f6e735f73696e63655f726573746f7265944b0275756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f30663830613664375f33315f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d3531322c6465636f6465725f72745f6e756d5f6c617965723d322c64726f705f725f323032342d30362d32355f32322d34352d33392f636865636b706f696e745f30303030303294682268257d946827897385945294682a682d2981947d9468308a103c1ee8c2549795b4c943114c6667993c7362756268317d94288c046c6f7373946836683c430808040241bdb39e4094869452948c0974696d657374616d70944ab3317b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303032948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b038c08747269616c5f6964948c083066383061366437948c0464617465948c13323032342d30362d32355f32332d30382d3034948c1074696d655f746869735f697465725f7394474075d9fe77c000008c0c74696d655f746f74616c5f73944740909c6c8a6000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b028c146465636f6465725f72745f6e756d5f6c61796572944b028c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944d00048c0a656e636f6465725f6666944d00048c0d6465636f6465725f72745f6666944d00028c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b108c0964726f705f7261746594473f987ea43cd280408c026c7294473f643952a785ac188c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f7265944740909c6c8a6000008c18697465726174696f6e735f73696e63655f726573746f7265944b0375756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f30663830613664375f33315f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d3531322c6465636f6465725f72745f6e756d5f6c617965723d322c64726f705f725f323032342d30362d32355f32322d34352d33392f636865636b706f696e745f30303030303394682268257d946827897385945294682a682d2981947d9468308a10ec8e63351823f2892247c258a97ee73d7362756268317d94288c046c6f7373946836683c4308d4e9745ab38e9e4094869452948c0974696d657374616d70944a12337b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303033948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b048c08747269616c5f6964948c083066383061366437948c0464617465948c13323032342d30362d32355f32332d31332d3534948c1074696d655f746869735f697465725f7394474075e767ddc000008c0c74696d655f746f74616c5f7394474096164681d000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b028c146465636f6465725f72745f6e756d5f6c61796572944b028c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944d00048c0a656e636f6465725f6666944d00048c0d6465636f6465725f72745f6666944d00028c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b108c0964726f705f7261746594473f987ea43cd280408c026c7294473f643952a785ac188c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f726594474096164681d000008c18697465726174696f6e735f73696e63655f726573746f7265944b0475756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f30663830613664375f33315f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d3531322c6465636f6465725f72745f6e756d5f6c617965723d322c64726f705f725f323032342d30362d32355f32322d34352d33392f636865636b706f696e745f30303030303494682268257d946827897385945294682a682d2981947d9468308a1067d6c95952ced697984797b362b8927d7362756268317d94288c046c6f7373946836683c43080e87c3a1b6909e4094869452948c0974696d657374616d70944a70347b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303034948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b058c08747269616c5f6964948c083066383061366437948c0464617465948c13323032342d30362d32355f32332d31392d3434948c1074696d655f746869735f697465725f7394474075e2f4de0000008c0c74696d655f746f74616c5f739447409b8f03b95000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b028c146465636f6465725f72745f6e756d5f6c61796572944b028c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944d00048c0a656e636f6465725f6666944d00048c0d6465636f6465725f72745f6666944d00028c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b108c0964726f705f7261746594473f987ea43cd280408c026c7294473f643952a785ac188c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f72659447409b8f03b95000008c18697465726174696f6e735f73696e63655f726573746f7265944b0575756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f30663830613664375f33315f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d3531322c6465636f6465725f72745f6e756d5f6c617965723d322c64726f705f725f323032342d30362d32355f32322d34352d33392f636865636b706f696e745f30303030303594682268257d946827897385945294682a682d2981947d9468308a1188174b41c08d35aff24e950105dea5c6007362756268317d94288c046c6f7373946836683c430867b3d96c98cf9e4094869452948c0974696d657374616d70944ada497b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303035948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b068c08747269616c5f6964948c083066383061366437948c0464617465948c13323032342d30362d32365f30302d35312d3037948c1074696d655f746869735f697465725f7394474076d4959cc000008c0c74696d655f746f74616c5f73944740a0a214904000008c03706964944a027f2b008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b028c146465636f6465725f72745f6e756d5f6c61796572944b028c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944d00048c0a656e636f6465725f6666944d00048c0d6465636f6465725f72745f6666944d00028c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b108c0964726f705f7261746594473f987ea43cd280408c026c7294473f643952a785ac188c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f726594474076d4959cc000008c18697465726174696f6e735f73696e63655f726573746f7265944b0175756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f30663830613664375f33315f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d3531322c6465636f6465725f72745f6e756d5f6c617965723d322c64726f705f725f323032342d30362d32355f32322d34352d33392f636865636b706f696e745f30303030303694682268257d946827897385945294682a682d2981947d9468308a1057cce3d9822a14a9614bea04201a966b7362756268317d94288c046c6f7373946836683c4308ca643219b18e9e4094869452948c0974696d657374616d70944a384b7b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303036948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b078c08747269616c5f6964948c083066383061366437948c0464617465948c13323032342d30362d32365f30302d35362d3536948c1074696d655f746869735f697465725f7394474075d2f2890000008c0c74696d655f746f74616c5f73944740a35c72e16000008c03706964944a027f2b008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b028c146465636f6465725f72745f6e756d5f6c61796572944b028c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944d00048c0a656e636f6465725f6666944d00048c0d6465636f6465725f72745f6666944d00028c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b108c0964726f705f7261746594473f987ea43cd280408c026c7294473f643952a785ac188c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f72659447408653c412e000008c18697465726174696f6e735f73696e63655f726573746f7265944b0275756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f30663830613664375f33315f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d3531322c6465636f6465725f72745f6e756d5f6c617965723d322c64726f705f725f323032342d30362d32355f32322d34352d33392f636865636b706f696e745f30303030303794682268257d946827897385945294682a682d2981947d9468308a102a5c119bf60755beb2446b96b09e55537362756268317d94288c046c6f7373946836683c4308d0e7f339fd959e4094869452948c0974696d657374616d70944a954c7b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303037948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b088c08747269616c5f6964948c083066383061366437948c0464617465948c13323032342d30362d32365f30312d30322d3436948c1074696d655f746869735f697465725f7394474075db6648c000008c0c74696d655f746f74616c5f73944740a617dfaa7800008c03706964944a027f2b008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b028c146465636f6465725f72745f6e756d5f6c61796572944b028c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944d00048c0a656e636f6465725f6666944d00048c0d6465636f6465725f72745f6666944d00028c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b108c0964726f705f7261746594473f987ea43cd280408c026c7294473f643952a785ac188c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f726594474090a0bb9ba000008c18697465726174696f6e735f73696e63655f726573746f7265944b0375756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f30663830613664375f33315f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d3531322c6465636f6465725f72745f6e756d5f6c617965723d322c64726f705f725f323032342d30362d32355f32322d34352d33392f636865636b706f696e745f30303030303894682268257d946827897385945294682a682d2981947d9468308a11a9087b4978719e871e443c6bfcdaf6e0007362756268317d94288c046c6f7373946836683c4308bd5eaf771abd9e4094869452948c0974696d657374616d70944af34d7b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303038948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b098c08747269616c5f6964948c083066383061366437948c0464617465948c13323032342d30362d32365f30312d30382d3335948c1074696d655f746869735f697465725f7394474075d5bd5ec000008c0c74696d655f746f74616c5f73944740a8d297565000008c03706964944a027f2b008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b028c146465636f6465725f72745f6e756d5f6c61796572944b028c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944d00048c0a656e636f6465725f6666944d00048c0d6465636f6465725f72745f6666944d00028c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b108c0964726f705f7261746594473f987ea43cd280408c026c7294473f643952a785ac188c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f726594474096162af35000008c18697465726174696f6e735f73696e63655f726573746f7265944b0475756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f30663830613664375f33315f62617463685f73697a653d323034382c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d3531322c6465636f6465725f72745f6e756d5f6c617965723d322c64726f705f725f323032342d30362d32355f32322d34352d33392f636865636b706f696e745f30303030303994682268257d946827897385945294682a682d2981947d9468308a10bcc31758fd28c490b547d6435831d4687362756268317d94288c046c6f7373946836683c430893c96492ab9f9e4094869452948c0974696d657374616d70944a514f7b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303039948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b0a8c08747269616c5f6964948c083066383061366437948c0464617465948c13323032342d30362d32365f30312d31342d3235948c1074696d655f746869735f697465725f7394474075de043d8000008c0c74696d655f746f74616c5f73944740ab8e57de0000008c03706964944a027f2b008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b028c146465636f6465725f72745f6e756d5f6c61796572944b028c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944d00048c0a656e636f6465725f6666944d00048c0d6465636f6465725f72745f6666944d00028c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b108c0964726f705f7261746594473f987ea43cd280408c026c7294473f643952a785ac188c0a62617463685f73697a65944d0008758c1274696d655f73696e63655f726573746f72659447409b8dac02b000008c18697465726174696f6e735f73696e63655f726573746f7265944b05757562658c195f6c61746573745f636865636b706f696e745f726573756c74946ada01000075622e\"\n }\n}"
+ ],
+ [
+ "{\n \"stub\": false,\n \"trainable_name\": \"train_model\",\n \"trial_id\": \"fe083630\",\n \"storage\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059572040000000000008c1b7261792e747261696e2e5f696e7465726e616c2e73746f72616765948c0e53746f72616765436f6e746578749493942981947d94288c12637573746f6d5f66735f70726f766964656494898c1273746f726167655f6c6f63616c5f70617468948c2b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c7473948c136578706572696d656e745f6469725f6e616d65948c0f746573745f6578706572696d656e74948c0e747269616c5f6469725f6e616d65948c96747261696e5f6d6f64656c5f66653038333633305f33325f62617463685f73697a653d313032342c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d3531322c6465636f6465725f72745f6e756d5f6c617965723d322c64726f705f725f323032342d30362d32355f32322d35302d3231948c1863757272656e745f636865636b706f696e745f696e646578944b048c0b73796e635f636f6e666967948c097261792e747261696e948c0a53796e63436f6e6669679493942981947d94288c0b73796e635f706572696f64944d2c018c0c73796e635f74696d656f7574944d08078c0e73796e635f61727469666163747394898c1c73796e635f6172746966616374735f6f6e5f636865636b706f696e7494888c0a75706c6f61645f646972948c0a44455052454341544544948c0673796e6365729468188c1273796e635f6f6e5f636865636b706f696e7494681875628c1273746f726167655f66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c0f73746f726167655f66735f70617468948c452f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f7465737494681968008c115f46696c6573797374656d53796e6365729493942981947d9428681b682268134d2c0168144d08078c116c6173745f73796e635f75705f74696d65944741d99ecb46c1710d8c136c6173745f73796e635f646f776e5f74696d659447fff00000000000008c0d5f73796e635f70726f63657373944e8c0c5f63757272656e745f636d649468008c125f75706c6f61645f746f5f66735f706174689493947d94288c0a6c6f63616c5f70617468948c3b2f6c696e6b686f6d652f726563682f67656e6b6d7730312f756367383177732f7261795f726573756c74732f746573745f6578706572696d656e74948c0266739468228c0766735f70617468948c552f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e74948c076578636c756465945d948c0e2a2f636865636b706f696e745f2a9461758694756275622e\"\n },\n \"config\": {\n \"encoder_num_layer\": 8,\n \"decoder_rt_num_layer\": 2,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 1024,\n \"encoder_ff\": 1024,\n \"decoder_rt_ff\": 512,\n \"decoder_int_ff\": 512,\n \"n_head\": 2,\n \"drop_rate\": 0.7814344084863447,\n \"lr\": 0.030816743385966763,\n \"batch_size\": 1024\n },\n \"_Trial__unresolved_config\": {\n \"encoder_num_layer\": 8,\n \"decoder_rt_num_layer\": 2,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 1024,\n \"encoder_ff\": 1024,\n \"decoder_rt_ff\": 512,\n \"decoder_int_ff\": 512,\n \"n_head\": 2,\n \"drop_rate\": 0.7814344084863447,\n \"lr\": 0.030816743385966763,\n \"batch_size\": 1024\n },\n \"evaluated_params\": {\n \"batch_size\": 1024,\n \"decoder_int_ff\": 512,\n \"decoder_int_num_layer\": 1,\n \"decoder_rt_ff\": 512,\n \"decoder_rt_num_layer\": 2,\n \"drop_rate\": 0.7814344084863447,\n \"embedding_dim\": 1024,\n \"encoder_ff\": 1024,\n \"encoder_num_layer\": 8,\n \"lr\": 0.030816743385966763,\n \"n_head\": 2\n },\n \"experiment_tag\": \"32_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_rate=0.7814,embedding_dim=1024,encoder_ff=1024,encoder_num_layer=8,lr=0.0308,n_head=2\",\n \"stopping_criterion\": {},\n \"_setup_default_resource\": true,\n \"_default_placement_group_factory\": \"80054e2e\",\n \"placement_group_factory\": \"800595ba000000000000008c237261792e74756e652e657865637574696f6e2e706c6163656d656e745f67726f757073948c15506c6163656d656e7447726f7570466163746f72799493942981947d94288c085f62756e646c6573945d947d94288c03435055944740540000000000008c034750559447401000000000000075618c155f686561645f62756e646c655f69735f656d70747994898c095f7374726174656779948c045041434b948c055f6172677394298c075f6b7761726773947d9475622e\",\n \"log_to_file\": [\n null,\n null\n ],\n \"max_failures\": 0,\n \"_default_result_or_future\": null,\n \"export_formats\": [],\n \"status\": \"TERMINATED\",\n \"relative_logdir\": \"train_model_fe083630_32_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_22-50-21\",\n \"trial_name_creator\": null,\n \"trial_dirname_creator\": null,\n \"custom_trial_name\": null,\n \"custom_dirname\": null,\n \"restore_path\": null,\n \"_restore_checkpoint_result\": null,\n \"_state_json\": null,\n \"results\": \"80054e2e\",\n \"extra_arg\": \"80054e2e\",\n \"_resources\": \"80054e2e\"\n}",
+ "{\n \"start_time\": 1719350385.0636432,\n \"num_failures\": 0,\n \"num_failures_after_restore\": 0,\n \"error_filename\": null,\n \"pickled_error_filename\": null,\n \"last_result\": {\n \"loss\": 1961.6992687315453,\n \"timestamp\": 1719354625,\n \"checkpoint_dir_name\": \"checkpoint_000004\",\n \"should_checkpoint\": true,\n \"done\": false,\n \"training_iteration\": 5,\n \"trial_id\": \"fe083630\",\n \"date\": \"2024-06-26_00-30-25\",\n \"time_this_iter_s\": 845.8089480400085,\n \"time_total_s\": 4240.718232393265,\n \"pid\": 2195253,\n \"hostname\": \"r8i6n2\",\n \"node_ip\": \"10.159.28.60\",\n \"config\": {\n \"encoder_num_layer\": 8,\n \"decoder_rt_num_layer\": 2,\n \"decoder_int_num_layer\": 1,\n \"embedding_dim\": 1024,\n \"encoder_ff\": 1024,\n \"decoder_rt_ff\": 512,\n \"decoder_int_ff\": 512,\n \"n_head\": 2,\n \"drop_rate\": 0.7814344084863447,\n \"lr\": 0.030816743385966763,\n \"batch_size\": 1024\n },\n \"time_since_restore\": 4240.718232393265,\n \"iterations_since_restore\": 5,\n \"experiment_tag\": \"32_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_rate=0.7814,embedding_dim=1024,encoder_ff=1024,encoder_num_layer=8,lr=0.0308,n_head=2\"\n },\n \"last_result_time\": 1719354625.8083603,\n \"metric_analysis\": {\n \"loss\": {\n \"max\": 4034.1917479507565,\n \"min\": 1961.6992687315453,\n \"avg\": 2624.951602004645,\n \"last\": 1961.6992687315453,\n \"last-5-avg\": 2624.9516020046444,\n \"last-10-avg\": 2624.9516020046444\n },\n \"should_checkpoint\": {\n \"max\": true,\n \"min\": true,\n \"avg\": 1.0,\n \"last\": true,\n \"last-5-avg\": 1.0,\n \"last-10-avg\": 1.0\n },\n \"done\": {\n \"max\": false,\n \"min\": false,\n \"avg\": 0.0,\n \"last\": false,\n \"last-5-avg\": 0.0,\n \"last-10-avg\": 0.0\n },\n \"training_iteration\": {\n \"max\": 5,\n \"min\": 1,\n \"avg\": 3.0,\n \"last\": 5,\n \"last-5-avg\": 3.0,\n \"last-10-avg\": 3.0\n },\n \"time_this_iter_s\": {\n \"max\": 858.9566829204559,\n \"min\": 845.1621363162994,\n \"avg\": 848.143646478653,\n \"last\": 845.8089480400085,\n \"last-5-avg\": 848.143646478653,\n \"last-10-avg\": 848.143646478653\n },\n \"time_total_s\": {\n \"max\": 4240.718232393265,\n \"min\": 858.9566829204559,\n \"avg\": 2549.6743980884553,\n \"last\": 4240.718232393265,\n \"last-5-avg\": 2549.6743980884553,\n \"last-10-avg\": 2549.6743980884553\n },\n \"time_since_restore\": {\n \"max\": 4240.718232393265,\n \"min\": 858.9566829204559,\n \"avg\": 2549.6743980884553,\n \"last\": 4240.718232393265,\n \"last-5-avg\": 2549.6743980884553,\n \"last-10-avg\": 2549.6743980884553\n },\n \"iterations_since_restore\": {\n \"max\": 5,\n \"min\": 1,\n \"avg\": 3.0,\n \"last\": 5,\n \"last-5-avg\": 3.0,\n \"last-10-avg\": 3.0\n }\n },\n \"_n_steps\": [\n 5,\n 10\n ],\n \"metric_n_steps\": {\n \"loss\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"800595d6000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294288c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b0074946243082693c92c6284af4094869452946807680d4308ecf57a5da882a64094869452946807680d4308ee76bb157c7ea14094869452946807680d4308b55aade62e619f4094869452946807680d430869341a0dcca69e409486945294652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"800595d6000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294288c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b0074946243082693c92c6284af4094869452946807680d4308ecf57a5da882a64094869452946807680d4308ee76bb157c7ea14094869452946807680d4308b55aade62e619f4094869452946807680d430869341a0dcca69e409486945294652e\"\n }\n },\n \"should_checkpoint\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059527000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294288888888888652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059527000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294288888888888652e\"\n }\n },\n \"done\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059527000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294288989898989652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059527000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294288989898989652e\"\n }\n },\n \"training_iteration\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952c000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294284b014b024b034b044b05652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952c000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294284b014b024b034b044b05652e\"\n }\n },\n \"time_this_iter_s\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005954f000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452942847408ad7a74960000047408a694c0e20000047408a6c66b640000047408a69ec2920000047408a6e78b9c00000652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005954f000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452942847408ad7a74960000047408a694c0e20000047408a6c66b640000047408a69ec2920000047408a6e78b9c00000652e\"\n }\n },\n \"time_total_s\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005954f000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452942847408ad7a74960000047409aa079abc000004740a3eb56837000004740aa85d18db800004740b090b7de140000652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005954f000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452942847408ad7a74960000047409aa079abc000004740a3eb56837000004740aa85d18db800004740b090b7de140000652e\"\n }\n },\n \"time_since_restore\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005954f000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b05869452942847408ad7a74960000047409aa079abc000004740a3eb56837000004740aa85d18db800004740b090b7de140000652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005954f000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a869452942847408ad7a74960000047409aa079abc000004740a3eb56837000004740aa85d18db800004740b090b7de140000652e\"\n }\n },\n \"iterations_since_restore\": {\n \"5\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952c000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0586945294284b014b024b034b044b05652e\"\n },\n \"10\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005952c000000000000008c0b636f6c6c656374696f6e73948c056465717565949394294b0a86945294284b014b024b034b044b05652e\"\n }\n }\n },\n \"checkpoint_manager\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005954f140000000000008c267261792e747261696e2e5f696e7465726e616c2e636865636b706f696e745f6d616e61676572948c125f436865636b706f696e744d616e616765729493942981947d94288c125f636865636b706f696e745f636f6e666967948c097261792e747261696e948c10436865636b706f696e74436f6e6669679493942981947d94288c0b6e756d5f746f5f6b656570944e8c1a636865636b706f696e745f73636f72655f617474726962757465944e8c16636865636b706f696e745f73636f72655f6f72646572948c036d6178948c14636865636b706f696e745f6672657175656e6379944b008c11636865636b706f696e745f61745f656e6494898c1a5f636865636b706f696e745f6b6565705f616c6c5f72616e6b73948c0a44455052454341544544948c1f5f636865636b706f696e745f75706c6f61645f66726f6d5f776f726b65727394681275628c135f636865636b706f696e745f726573756c7473945d94288c1b7261792e747261696e2e5f696e7465726e616c2e73657373696f6e948c0f5f547261696e696e67526573756c749493942981947d94288c0a636865636b706f696e749468068c0a436865636b706f696e749493942981947d94288c0470617468948cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f66653038333633305f33325f62617463685f73697a653d313032342c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d3531322c6465636f6465725f72745f6e756d5f6c617965723d322c64726f705f725f323032342d30362d32355f32322d35302d32312f636865636b706f696e745f303030303030948c0a66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c055f75756964948c0475756964948c04555549449493942981947d948c03696e74948a112ec45172fa7983bffa44a44d0e218ae900736275628c076d657472696373947d94288c046c6f7373948c156e756d70792e636f72652e6d756c74696172726179948c067363616c61729493948c056e756d7079948c0564747970659493948c02663894898887945294284b038c013c944e4e4e4affffffff4affffffff4b0074946243082693c92c6284af4094869452948c0974696d657374616d70944acb377b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303030948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b018c08747269616c5f6964948c086665303833363330948c0464617465948c13323032342d30362d32355f32332d33342d3034948c1074696d655f746869735f697465725f739447408ad7a7496000008c0c74696d655f746f74616c5f739447408ad7a7496000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b088c146465636f6465725f72745f6e756d5f6c61796572944b028c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944d00048c0a656e636f6465725f6666944d00048c0d6465636f6465725f72745f6666944d00028c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b028c0964726f705f7261746594473fe90182bb8d5fe48c026c7294473f9f8e6ca40c3d078c0a62617463685f73697a65944d0004758c1274696d655f73696e63655f726573746f72659447408ad7a7496000008c18697465726174696f6e735f73696e63655f726573746f7265944b0175756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f66653038333633305f33325f62617463685f73697a653d313032342c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d3531322c6465636f6465725f72745f6e756d5f6c617965723d322c64726f705f725f323032342d30362d32355f32322d35302d32312f636865636b706f696e745f30303030303194682268257d946827897385945294682a682d2981947d9468308a10955cd877237e09ba474c31d05669167a7362756268317d94288c046c6f7373946836683c4308ecf57a5da882a64094869452948c0974696d657374616d70944a183b7b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303031948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b028c08747269616c5f6964948c086665303833363330948c0464617465948c13323032342d30362d32355f32332d34382d3039948c1074696d655f746869735f697465725f739447408a694c0e2000008c0c74696d655f746f74616c5f739447409aa079abc000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b088c146465636f6465725f72745f6e756d5f6c61796572944b028c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944d00048c0a656e636f6465725f6666944d00048c0d6465636f6465725f72745f6666944d00028c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b028c0964726f705f7261746594473fe90182bb8d5fe48c026c7294473f9f8e6ca40c3d078c0a62617463685f73697a65944d0004758c1274696d655f73696e63655f726573746f72659447409aa079abc000008c18697465726174696f6e735f73696e63655f726573746f7265944b0275756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f66653038333633305f33325f62617463685f73697a653d313032342c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d3531322c6465636f6465725f72745f6e756d5f6c617965723d322c64726f705f725f323032342d30362d32355f32322d35302d32312f636865636b706f696e745f30303030303294682268257d946827897385945294682a682d2981947d9468308a1034ac3c87313821b8c74393a0b5d49d377362756268317d94288c046c6f7373946836683c4308ee76bb157c7ea14094869452948c0974696d657374616d70944a663e7b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303032948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b038c08747269616c5f6964948c086665303833363330948c0464617465948c13323032342d30362d32365f30302d30322d3134948c1074696d655f746869735f697465725f739447408a6c66b64000008c0c74696d655f746f74616c5f73944740a3eb56837000008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b088c146465636f6465725f72745f6e756d5f6c61796572944b028c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944d00048c0a656e636f6465725f6666944d00048c0d6465636f6465725f72745f6666944d00028c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b028c0964726f705f7261746594473fe90182bb8d5fe48c026c7294473f9f8e6ca40c3d078c0a62617463685f73697a65944d0004758c1274696d655f73696e63655f726573746f7265944740a3eb56837000008c18697465726174696f6e735f73696e63655f726573746f7265944b0375756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f66653038333633305f33325f62617463685f73697a653d313032342c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d3531322c6465636f6465725f72745f6e756d5f6c617965723d322c64726f705f725f323032342d30362d32355f32322d35302d32312f636865636b706f696e745f30303030303394682268257d946827897385945294682a682d2981947d9468308a11cc60ea54f385009b774823ed0f83bb96007362756268317d94288c046c6f7373946836683c4308b55aade62e619f4094869452948c0974696d657374616d70944ab3417b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303033948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b048c08747269616c5f6964948c086665303833363330948c0464617465948c13323032342d30362d32365f30302d31362d3139948c1074696d655f746869735f697465725f739447408a69ec292000008c0c74696d655f746f74616c5f73944740aa85d18db800008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b088c146465636f6465725f72745f6e756d5f6c61796572944b028c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944d00048c0a656e636f6465725f6666944d00048c0d6465636f6465725f72745f6666944d00028c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b028c0964726f705f7261746594473fe90182bb8d5fe48c026c7294473f9f8e6ca40c3d078c0a62617463685f73697a65944d0004758c1274696d655f73696e63655f726573746f7265944740aa85d18db800008c18697465726174696f6e735f73696e63655f726573746f7265944b0475756268182981947d9428681b681d2981947d942868208cfe2f67706673776f726b2f726563682f7574652f756367383177732f74686573652f4c432d4d532d52542d70726564696374696f6e2f7261795f726573756c74735f746573742f746573745f6578706572696d656e742f747261696e5f6d6f64656c5f66653038333633305f33325f62617463685f73697a653d313032342c6465636f6465725f696e745f66663d3531322c6465636f6465725f696e745f6e756d5f6c617965723d312c6465636f6465725f72745f66663d3531322c6465636f6465725f72745f6e756d5f6c617965723d322c64726f705f725f323032342d30362d32355f32322d35302d32312f636865636b706f696e745f30303030303494682268257d946827897385945294682a682d2981947d9468308a105c998f80b0fad298484f48365ecb77397362756268317d94288c046c6f7373946836683c430869341a0dcca69e4094869452948c0974696d657374616d70944a01457b668c13636865636b706f696e745f6469725f6e616d65948c11636865636b706f696e745f303030303034948c1173686f756c645f636865636b706f696e7494888c04646f6e6594898c12747261696e696e675f697465726174696f6e944b058c08747269616c5f6964948c086665303833363330948c0464617465948c13323032342d30362d32365f30302d33302d3235948c1074696d655f746869735f697465725f739447408a6e78b9c000008c0c74696d655f746f74616c5f73944740b090b7de1400008c03706964944a357f21008c08686f73746e616d65948c06723869366e32948c076e6f64655f6970948c0c31302e3135392e32382e3630948c06636f6e666967947d94288c11656e636f6465725f6e756d5f6c61796572944b088c146465636f6465725f72745f6e756d5f6c61796572944b028c156465636f6465725f696e745f6e756d5f6c61796572944b018c0d656d62656464696e675f64696d944d00048c0a656e636f6465725f6666944d00048c0d6465636f6465725f72745f6666944d00028c0e6465636f6465725f696e745f6666944d00028c066e5f68656164944b028c0964726f705f7261746594473fe90182bb8d5fe48c026c7294473f9f8e6ca40c3d078c0a62617463685f73697a65944d0004758c1274696d655f73696e63655f726573746f7265944740b090b7de1400008c18697465726174696f6e735f73696e63655f726573746f7265944b05757562658c195f6c61746573745f636865636b706f696e745f726573756c749468ef75622e\"\n }\n}"
+ ]
+ ],
+ "runner_data": {
+ "_earliest_stopping_actor": 401634.092423264,
+ "_actor_cleanup_timeout": 600,
+ "_actor_force_cleanup_timeout": 10,
+ "_reuse_actors": true,
+ "_buffer_length": 1,
+ "_buffer_min_time_s": 0.0,
+ "_buffer_max_time_s": 100.0,
+ "_max_pending_trials": 1,
+ "_metric": "loss",
+ "_total_time": 30888.675180912018,
+ "_iteration": 300268,
+ "_has_errored": false,
+ "_fail_fast": false,
+ "_print_trial_errors": true,
+ "_cached_trial_decisions": {},
+ "_queued_trial_decisions": {},
+ "_should_stop_experiment": false,
+ "_stopper": {
+ "_type": "CLOUDPICKLE_FALLBACK",
+ "value": "8005952c000000000000008c157261792e74756e652e73746f707065722e6e6f6f70948c0b4e6f6f7053746f707065729493942981942e"
+ },
+ "_start_time": 1719328200.4031954,
+ "_last_checkpoint_time": -Infinity,
+ "_session_str": "2024-06-25_17-10-00",
+ "_checkpoint_period": "auto",
+ "_trial_checkpoint_config": {
+ "_type": "CLOUDPICKLE_FALLBACK",
+ "value": "800595f2000000000000008c097261792e747261696e948c10436865636b706f696e74436f6e6669679493942981947d94288c0b6e756d5f746f5f6b656570944e8c1a636865636b706f696e745f73636f72655f617474726962757465944e8c16636865636b706f696e745f73636f72655f6f72646572948c036d6178948c14636865636b706f696e745f6672657175656e6379944b008c11636865636b706f696e745f61745f656e6494898c1a5f636865636b706f696e745f6b6565705f616c6c5f72616e6b73948c0a44455052454341544544948c1f5f636865636b706f696e745f75706c6f61645f66726f6d5f776f726b65727394680c75622e"
+ },
+ "_resumed": false
+ },
+ "stats": {
+ "start_time": 1719328200.4031954,
+ "timestamp": -Infinity
+ }
+}
\ No newline at end of file
diff --git a/ray_results/test_experiment/search_gen_state-2024-06-25_15-19-46.json b/ray_results/test_experiment/search_gen_state-2024-06-25_15-19-46.json
new file mode 100644
index 0000000000000000000000000000000000000000..bd56b6ad2aec0c95c881d75662ca024e9d960928
GIT binary patch
literal 2463
zcmb_eO>Z1U5VajUyII>C+xY;Qh~!{I2zZxV5Q0O99LEytbx52bgcP-UdwRD=o|zu{
zBlaRie2EB4EpebZz=8k3k)OelD_4ZXKM1d9*1O{vi6F5TTU}LMRqws7n$PFH_h+Vx
z-(!E()tI!I;mUVTxkV!lO^Bvxo^b6uS6saxXR#JM@`tX$4|1;JlxN1fmlYkhjLmp!
za$5P<yT<0k`#b;I7d|9AOO(w>9N~(%A0w%oH;Tp?NoB+n?`qmeMF-rwmx}v?twrKc
zUq+MG*xKAJ{Wc^_2gZ*RtS}3wl=L;pm3)Luq4JecAW0-ce>pY6yVu)Eob3sj^0wkk
zv<E!fOZVTr-nMM;_Vs<;Mg>H1Z4;xtJ2z>QM6n_nO;P1p!?ht3lXp#{Vx$<m?6?=1
zRSFFgCOF&6CAvXJ5kK(nKgG|@>tV(SlUY9=cz0&pHz3@F1uX`{xWm#t5FsPqePh}!
z6!8YUNNaqzUUSXjgeqW68Hzu2myVwhnN_Os-?)a&6G@{`HqzeO{NUc^#?AYix4b(Y
zB0-c`1B#*0<Dp{B_9pQ5W5{Nx4NpOAKBP*?{T8_J?$UNZ{x<Bpg}nu2(XuwmvOwQs
z@9I`_lBP8Ghx%)O58xiL7RW3p+iVZ=zTTH=ucXWxK4!}Zu6cWVW4*h(e((Nzu6PtP
z6U(g1BME$Y@rqb<%Oq4&R=ZpFc!aTdRkXxqU@NW!IF}O&l8feTP6E(=>*jAi{k-+(
z$G^IB6JsRzvyu7fU*!XHi)9?N$hg*i$1RYt9MZ_6jukS(-pE3=q0xZHRoAGh_1*uV
z*7gZi&VU2aa<ws%5B?V<PbRAI1D4o``-gZ6*V+}yU-Q&H@wNxO)Cri)Az|P#t8>nZ
zc28k6crlL?7=s`d1JtBvqX7r}@I5_xBmtado-xb}JVcw4gbu?jxj3Flnp4F@s~lv|
z;s!N*AXN+vPF_-jjZQJ@9{Q`D>CQ}Nwo~VrLd5`t=<m4mmCzP>ki~te(((>;az28J
zLlw$Y3K4h9aVPFquN5DPk8qBt3rA656in*>x3EQ?_IMP<*#Pe}^~c?pI`4I-rn^(K
ze3GYegi*hL+`Bt91JCYMz2dxhR1{iy9{=W%e`w-)0#0W{a0<=;6FQxxN-1X|C3B*{
zPGDf}Hbzrr#Lp#iA8dD~+?*{$)R{)hcU-gZJIT>Z0wO;fG4kcsjn%h)|Krley7+4A
z#xFmF-;YK3@+f>;4vw)5A<p)w=DyR=LA;F~HB1m4e~Rj=tvrjUb?2}>q*ZwN1Y#$}
zW`K<61Lc(+yIgrCBZZ4qx3n={7bjLgxB%l~kMkV&tybbSlZq2+kgJF32<ycYR;i#^
z?@p!kz`OGmb6@dKEHpC=;8v@m*l}4{q^p&+QKTm{Nx)RLhxxMRE?3Q?vDQ~u$6;i1
z6?}l;O1UD#1|>{!idBv}FN_!GGluy@4nntKx%i!(Cm1YPxH8BTPmyB3Vb(058Pcb5
zVu5)mzlOiK7!hJYELo&|A>8{8uq+2K8*ayG96o%)IBAA>fIk|}zEgK|1<`1FK8YXk
zl0EIe^=>u5Z?TojFC$VlBXg5cU>c1-9U!0;_#oFL0U$nfHIi~P&CvW@@d4y_tZn@b
D`Jlzz
literal 0
HcmV?d00001
diff --git a/ray_results/test_experiment/search_gen_state-2024-06-25_16-07-48.json b/ray_results/test_experiment/search_gen_state-2024-06-25_16-07-48.json
new file mode 100644
index 0000000000000000000000000000000000000000..c109608d0177f83f265c982cca5e36e6eed43cbc
GIT binary patch
literal 2463
zcmb_eO>Z1U5VajUyII>Ce<T5!h~!{I2zZxV5Q0O9948X%bx52bE>Wwur+0hgndzZF
zVlPs}mx!>`f&<L~4*U(oFW@(D;KG#){~)}cS?`WxB!a|VY;{$2RlWDRYCfC$-k+H&
zevkbXS7Xv<hAZD$b&EzCnh;IXJmK1RF1va^&SEWi<PTkgALLxcDbI{|FDp818JqFe
z<h1gyca5!y_jmra&wWUCmMEK%IKmZiKSokFZxoF)lFEoD-qo~`iVnDUFBSI(TZ_b@
zzKkZVv9-Bd`fW;>4vZfsSYZ}UDd}sHEBOeSLgg!?K$1v?{&H%BcdxgTINK94<!!~8
zXb*U{m+rrLt!>%h?Q8qGjS7h3+9pPOw>oK)M6n_nO;P1p!?ht3lXp#{Vx$<m?6?=1
zRSFFgCOF&6CAvXJ5kK(nKgG|@>tV(SlUY9=cz0&pHz3@F1uX`{xWm#t5FsPqePh}!
z6!8YUNNaqzUUSXjgeqW68Hzu27muG1nN_OsU%Q6Q6G@{`HqzeQ`rzKy=8gMXH@!O@
zB0-c`2a2K4<Dp{R_9pQ5W5{Nx4NpOAKBP*?{T8_J?&5Yp{ub=Jg}nu2(XuwmvOwQs
z@9I`_lBP8Ghx#jj58xiL7RW3p+iDN;zTTH=ucXWxK4!}Zu6cWVbECVvaqs>{u6PtP
z6U(g1BME$Y@rqb<%Oq4&R=ZpFc!aTdRkXw<U@I;MIF}O&l8feTP6E*W_Kn|u`nmDV
zufMz1i7}G<*~t9#ukwMp#WD_BWL#^%;}*zR4r%03#|jx?Z)Bm`&}hKpifdHW`tE;F
zYx{&MXTX7Ix!M@X2mcF_Cll590ZVMe{X;y3tL=*9uX<{qc-w<s>IBT@kTCF=)j4NH
zyQeT3yqL!cj6o2K0cz5-(SQSf_?{j;k^s&!&lqL~9->W2LWg0NTo}(J&8cFdRSq&}
zag!Q8kSc}-CoieNMyD8c5B-(SbZ4eB+o^L*p<)0+^mp93N@$Bb$l|_KX?X`aITu02
zp$cUxg^0W5xD)rQ*NP9tM>t2+`J<>X3MTdcTi7B`dpwHbY=C!~`s40Po%gy^)7`0A
zKFQNK!l>Ur?%kc5foFHBUU6PLDhjPUkAL&XKQ!??0jDz}IECi_37yVTrIa&~k~vXe
zConK~3!|wr;^z{%54Jl~Zq61W>P(~MJFZ#yo#bdH0g<1M82RG%^|iPD`19iChWPUK
z^<RDnzaNY6<x%*y92{dALY(bU&3&h#gLoS~YM3B8{sh%kTX_~y>sGNnq*ZwN1Y#$}
zW`K<61Lc(+yIgrCBZZ4Kx3oE47bjLgxB%l~kMkV&tySVRlZq2+kgJF32<wFtR;i#^
z?@p!kz`Ju5b6@d~Ei^L>;8v@m*l}4{q-&M6QKTm{Nx)RLhxxMZE>+E=vDQ~u$6;i1
z6?}l;O1UD#1|>{!idBv}&yN@9Gluy@4nntKx%i!(Cm1YPxH8BTPmyB3Vb(058Pcb5
zVu5)mzlOiK7!hJYELo&|A>8{8uq+2K8*ayG96o%)IBAA>fIk|}zEgK|1<`1FK8YXk
zl0EIe^=>V|Z?TojFC$VlBXg5cU>c1-9U!0;_#oFL0U$nfHIi~P&CvW@@d4y_tZn@b
D*{Q|P
literal 0
HcmV?d00001
diff --git a/ray_results/test_experiment/search_gen_state-2024-06-25_17-10-00.json b/ray_results/test_experiment/search_gen_state-2024-06-25_17-10-00.json
new file mode 100644
index 0000000000000000000000000000000000000000..2a60067bb11a8389929614a20cceda74b2a2a3ce
GIT binary patch
literal 2449
zcmb_eO>Z1U5VajUyI$KH+i?Oi7RkW~6Yws%AOwdHIZh<T>kvCZT%uO*Oz-x{Gt)zV
z#9pL`FJWP+B@Q$PIPfPp@eB9^T)1-O!Vf?SuV>b~<Cp_T?8R1BRaez}udC*>neUEg
zCyL)ww_$6cFqvl31=sC@7MdnR(KJt(@+b9PoW+W>$eq{*JIa}iQ<iCGUzT*x(k5fA
z@oD8>XB*pN?{D2}pF5xI5|Jh&afB=4UW}x6PD>hRBoz@$oUJJ>B<(Y2Un=hRwibzf
zeHl$!BWrWN^xG0J9ced8upAmVrKG1wF2xgM@|7<S14$x6^p{i3oqfHX#MuECDQim>
z@^+tP2kGIPH`^xczkTyiwNU|)OqoP0XD^T2BvC9$MpIOINi(I%*yL<e%NQv}F1z-{
z&`61fNyu4vkPCE!jv{vC-hYOlol}D>B%#Q9ao^e5QQv@Y9~QLe4WkZAcTa?fT<48R
zJ72_W@FI+MoqEkSixVn=F%41tiCsH;LPS=n#(ixYCQk&7eA!5QXZwS_?XBAnx9>Q6
z(MN(Pz77<9p=U$+y6KML?ZuGIKxvkO*nB{x6o)Nv;q2OuNB%DC`-Qy)WYMzH(uAJA
zr_R=mVkAvz?oQNKZV%v|vgXMwDBEuL^PV~s@}Q*5Xf|R?bEa5(XKS;wzq$8tGnXui
zLmi8($|Lc7IsOV?uuH^OQ&ziQ_IQdhf0eiRHDJrHdpMU83X+TFO-?+}?%wU+e*E>P
zzsxW8^4J)O!)$1N_GkIP>_Qm_Erv`fw`=FgNDgV_P{%SEVy{J^THmP0W5YJ8YJK-V
zsI`4gl~dq=w`^^M<fH!u$@7V7>?ll3#N1;%g&XaP<Zn1?j-Ba3FLeTDb4VC?OskxQ
zMY|U;8XTX)35;G43m$6HHBp}de)wJ-J`xYk63ar&3_L`Wl7tTYEV(+GNs3V!@>V&>
zpv5h!SzpK)8XUi*MmRjhsC(=-g2`Yim=5X;Q>Yk#5dD3-QVDI5M_JqxGA-{wCo2(D
z?5mKvQV4%gjywLadM*Eue}r>HT{(^NqhMTrH7uB<WB&z~c3Bj~SsxEHbydHOU@>^F
zGcnnjm}cXE#wj?x-dXR?#1uR`6ZMMo!f7?=;otZ-PyK!M&jsNAO5GPT!Wo6i{|S#^
zu~NfqBt=dn`0$v|+{G}e4E32r9D?6q!p@jNHo+u%y>FX^-+5N15>WT)kb^Jo-CBL?
z_dnLQHu;zLZvFhd|NT&eFAc+YME?x0klu8cD&~TQ@^Wo-sSY_&@h7OR+RF1#8haW0
zLKum+P9Sem>;edP&XZi(s!Nr3G1N3)wToM$1#xcZ`wK824j9XE-)bdZg+ekyHF9+^
z4`ID}&MG7{>g<J-9yz;GG4~|<*gz};4{o(8imjIU)wx<(6h(SMlLSm<2bd`9_FC0E
z9ArI-1?-13limjiE|*Ku?@SU(MzOq6=atc_e9kbJh`#UEuM@YsdyG+n6)U_x@%Sh<
z8m7xUnjt+JCkB}N@@x2uixI)+`JzGE7yP~N0LyaZhJ&3rjs2&NA0SO15AcWM*adYv
zQxJ`&=aTpdE7?=-8)sKN{04it{1zfrGcq?r3QVKX#{vYj3?JkgCji7JwnkEhrb9G8
MQ+)EcU1M5*1Fts3tpET3
literal 0
HcmV?d00001
diff --git a/ray_results/test_experiment/searcher-state-2024-06-25_15-19-46.pkl b/ray_results/test_experiment/searcher-state-2024-06-25_15-19-46.pkl
new file mode 100644
index 0000000000000000000000000000000000000000..1413e7a92ee7ac42e62bf31f6637bfdebb340a7d
GIT binary patch
literal 6726
zcmcIo3tUXw_pg~|nwcKcJ58<#6;VVfqNj?ZDO6H&%;VIV9y4PeiA%W2^LQUcLLMO{
zJtP#aSLAvm<PlO*ktlg2kN*42Op5#G_xb<s|IX*LXYalCI^VVS-fQo@)>=Z@GLEID
z_$|V1Fs4i*;Kaf*5iI6NIB`;(Akz4J7sn9}V@4}_3@%5;Lt&|40fPIh(J);;!V~ck
z2`muG;K)co)4+^-(}eOkIF>UXk>HRzrqewl`NnM2i}W2Yoi7oIVF^cu;J$86bt%qH
za|$sXB#w*l`2t}S%ooJrkRHZBgh~Ze!u7lRy})P~DOLhdyh*1!V2y<sRS2U9hYvoN
zW3<RIqY<v-$Wg@fPzul=r4oOd5Dldf4LU?Z8NP0Hcby%i&2LaGf~F0oj_LpoU0=7P
zo%sR!Z5=2Rq5(9=QIUwFQ=~&T8jV0B0W@_2jRw%v3A8J(Y_x0jTTv}YuNzFA0AsB!
zITCh}3c%Fma!eDBL%^2sa7-gsB$eWrHp~^FT$m>kMhc>EHb#Y|Vh#_%F$*sRX*$uf
zQx40)(g{SuZVm$_hl;9T#5dzZmMV5tL&%9k0G<Yr#{sn-jLsJ$5``5YGDHGIYw<*}
zu?UZVBzgf7u;~cjTfmbMFw=;&!>oI2@dJV#5A0-=2+W2Pi(@u0ES{eL%Y|YAFD4d&
zy?O@yzLN~V(LtEmuf`__CLGXZufTy0>v0=M4N~6#_6RKYay&xWM?jyAQhb$`uBHZ&
zFk@gI;`v~yB#2#{JXR*)NF<#3I7XEM{WuaJ7NaOwK+Tr{l?Vzc(iva1QXvT<W9qIr
zo6Vlw1%8S@NKJ-wy=Y1yPu%e3KqGVn>huF?InD-rC>6CrZT_x0L;}i_l=`pe)*m8)
z8Iag0wPyxoC33Lhcc-_o|3q4$l&|?S`4D-74Wtf{APtBn>(KxrBak{9)$CD0!>0G_
zDthanT!a4;Nv2eb_A|A9u`IM6o&NE%(0?ZX*ULihQQ==%76vxtS7%Eu6hw+7ad2d;
zh$F+XAwQh1VCAq6G5Yu8wE|=Pl3}u0!g+@Be0`gxuUo5fk9(Vd%H~&n<6;w6-4i7v
z``agk^=tidne`xde>HV*2!60XJvOTK4IKJc7Fzpfiv7a=w0m^=$M&c5Gx@)^Kb;;G
z{*`6XMOUF~dKrRp+l_;ZO7deY|Lk?uDM_T>B&NpU=Vgro<^8Y{ym3x-&3(Owvx)bP
zFpkdj<-j;jY$JYYgmEx`z6Ov+&*cQ^eP<q!hT?)^_N)X&bwQ;mpv!w-Q2#$9<((zR
z41ahoVF5ou;Q$aW0v#fp2@-hfsi-!jiP9i4N(Ej36JnqYh=S@ubd&<A$uSe+Gs5}$
z{^cZqAT_|E4n8A12{M2n+!6}-oC;ty0W9H-&;b+!Wy&!+%tHmSd<i1NQA^@!#s<Q$
zUtmD2pxbr$&NqN1B9RP+RlFz#yffgQXQMPWN(UzxoH;hC%|>;=hOtp5TaHo`sNe-7
zz)?+QMN?LEWyMfdTFOdWS?MS%U1h~oRKR5-oJ~Y1lYUb`Io`^ac8egIDltS;C5mXO
z#1T!ENTR6{OEguYiKa?C(Nw8GG?i+gpee_62+xWyAbcE7tQ235WBTA)<qG(GL<sX@
zIdUl?1ur5-m2iZ7Q5=pLDZL0~*A7-XGdN}d1HYBvkjj8x1M_4eg+rm~A2ia%Wz-lv
znyId)h8e*yS0IC<5Fr9wHIby7kMJ7^$6M#SQ5^gBIYv?7DwxPI2=H1AdU-RTi70$V
zEjUslisQ;7BZ2e18MjIKldimqXC|$$64$%0)pW`(+Q}g2=`q{9TyjtNl6<TVd45?j
zA*sr^n(sqicSasW>a_omUH9j0`<4A@$*EoBae5gBi3I`snvyj5k}rl0x`i{hWj6oz
z+qg@4dY6(vr955K=SF_aj#HJ5!n=XPMnr8pXpro+{#Neqodt)dWY#S={T#J(;rulF
zrZd8Oq4{4KBjrcB3L|4G$OqjHQUsH9D0<Mj*SuqU8eBF%&iYy#Tu|`rpz&Sg*!r=Y
zmb^P(LuPwBQwtsDOZ`$tSXIAZjXXRik@5MXWPR7meX%Phc*Wg}+T&@|>N3||oMxKe
z{IZzQrVTk4H5e6`6rHVOHQBK#7g}$*R5)jD93DSB+#@54_cHO8Y|s{8=CDvw@ZPq_
z=?6?2Rz+N{TaC0lDl(>zi4l)%tMsieejgBT<$BL1^qTe7B)a}(#+C}nEJy#`j+}I=
z<5;@+$aLd$%G*m()Z0}f54GmB*=Nbv_jT_W?4Fb{%4A`2#MZ-mhp*JLExcmVIxyne
z)b0Cc>Q884az=-tGr5W6E^gbi(r26651VEV@YyG4Ev!Zoj`~IIshQnjnjFSjabjeu
z`tGVhu61tC<9Bv>T>BKxa6B??c&9j?+wwGUcB$;Q-#h7tZ{|htjK3O&Uw5lJZID}9
zUhR8+%CMy`N@CP=Gh*6z4%*cjz3q9#b(_>kIO<xkK5}|u;)Hi<4U<nM7ZkQQWghd{
z`m}UTms!jEq*KO;v;4+gXPVWYIak}~&I9;m3a6&_BzEXV6UshiVpBa~xnDy<^}4_<
zLwBa}5<gf~%_tA*m(Q}Ghi$wvaG!(WD7P=qS=#3v;gbhOM_70U99?)e*CSQ@by@bp
z&E-aSrY8MX`lt3I$0h5o9*%B3_<myQdx?E!h?e}gb%ax>!{F`pMO%t8XZ5Kqyss9(
zvI+?vW45&a(*4ra)(wnRe(D$7r_Uc~v3Y`--?@w;t(CLhZA?1!<k=kYW7=A?ec>S|
zcJ{m6*Y9<Qby-B(=MU4<W_Ypcu3k#MG*cq>NqQ!|)aOuXc6*n1pU$`^9+uWuTi%`T
zYq_?+Y#tVwYTWddRFXGFe|J`Xje18w#h_Bpq8oN6Mut{39*X+7_j&Q4%~Q+DGE9z5
z8+^fdUf}7z%-r?29`CqV%A>%P`#0M?KW$tjJJ=_~>`vdM!~XIZ72$CG3Ga`o)5JCN
z#`Q0**_};|ezo$};Db{66mMJhhGz>6g0OXDX#CZH=1;~?x;zKICx!W5E|3&>yT52-
zRUEZV)+qQ`@jO$QGT=@?(P|@)v9+DI=T@8xynNquy=~l?5cBK>K0CadgG$8(!b9v?
zlzO}VP(u5RLTBL=RR2+UWWlOEx8G&2dD*hQ)FVD8DsS9?BLmTW14jMk=nzCTRA2nC
zcy(vlygjW01_bT8fbM$wG{kjT$MVMILeh0<*9S_t`lNFMt4X%Wxgu+mt!eM4j%}b{
zIP241W>4ex%UKX=GE5^l>$FpjRZjiWxOMPMuL8MA#epXd^IWABMRWL{RxYYVvaQtU
z8xpM+E^dCYC_XE3NsOQ2&}GeQ1|Pccpp|1~yfdKC{63WBb?(L0*++sS22LM2?$M4%
z>BH==xb!cop5bcgvG~#Fwf2E^f@clk!;80#8MdBP>;G}rq)G5e`;ZUjF1y{z;;Txb
z3euO)#l`&PR9?Mf#-62x`gLIk*IoGh`?3A^r!PC7;56t+fJ6RAnbE71x^)w;H#Wb0
z#HngIDO~C}rNb^StD^Q*O@}1G?$7Oq`}L_}RplOV3>ma+T-P>r#C1(}pHDiI1NfCX
z>1Aslq$QZs$IcFV?CzHmgKJ(|n?3J>VPA6c=RYhH9yXfhYz#VDeX^?)dD~X-@rXYj
zw!C`#yOPPa&JRRnjj=WR?*^xt+U3)%9+sat17|(Exc#EVWwp3v<Nu<j+b7T3JL3uE
zjr+SW>$OYjWq*X4_kYrud3gP<VJov6U)KL&7<$c{-!=Qn8yk;<w{}jnDAJhJ#CVt2
zXI~LJ^P*$E?Xr{|jL^sCCAYn|Xjq+T5Ki3qYUtVLen*dPaj5jG%lf3Xe`uUm19?>7
zNx#l*y9~Yt-@F;UH*~adl=n2FoOln5Q#RHzynWI8xmVU)dC<RNs&=Q#(sTZe)|)K`
zY`w~ZS!cg^I8n4>z81M$6;7(K&=0?nCF8<--YhKHyRaa2ZFwKoaH?y`vVNNqxzDlJ
z%|oiL>VDV(C91!_&YaCRU%BOzS7eRf2Hk0|f_cU~VCmks^v+bLZ7?;BY_6?5Pm4}+
zI{RXD##kgVr}g)AhY7clakEWJ;<WXSf1a0<aa<=v*W}rwQ&Xncy=jQd=O&v^_ZiQ=
znHOhycd20vdE#la{tKEe?X~%Iq+Cb5b;NN~H6P5|x-011UJHTDzrHr*={*DTTZ;mQ
z2du7BnbvmBaNz3Dv!laO?}gd%PK@ff9@?s3{&CY>9XH{MgHXuvHLY86*)Ep3Z0mwu
zB#Fi)LwU>0uHt2(_a9E4dekMXsj?~DEF!6_-Dkt0QFaaY7HEbGBR(O;Z-Qz~Z?B$~
z#5%pVxZ{OLKiAzo{cCDQ{){WPWkdhimRn=d8LN42;S^)m-U}|FRbE3rapvTgo;Vc~
zU6z!{thr5<H-@BlRhW!@;rv#EZ16hdUBRT1P<r{@j>|VwC%t%PYh|`)&zFp44=#jU
zi#PCl%sEqbVZ@?i>5hfhby%K8>#pr;>APjREUk2SW&1YYz~&>B?q-?Mk=o|NJT|$!
z_Vn7kHQv_Qb*W_1qx9QGDXC$W^^RxchqlFRw3-;;g0xP!m7czM`#MRhgA2KJej4(=
z{mIMp#Wk1K{lyBX-*;#H{K-#}+oO*MRe9RK-M{x%@UD-G>kQ9Se^_wC<o06Qu)$|#
zIp)XXQ;o_|v)}aQ`g!sn58Sc%@`y~2#SG`8WyYm0Zx>vdeXsWB>yP0`a<gUmEAS+@
zKMu9i4O!wPXy0z`z9?r66K>WF&PLzt9`x{1W74e|uT873Ua=jFRr^+HEXZdLf5`2h
zcb~?du>r}=EVA>rKK9|ph1?O9{Yq?F@4IS-4#lG0L+JP-yVC6sY<5p`Dqa+3A6uU1
zeqab|&U16-R^M!Q>KlteEl7Q^*`_<~?;Sc{Z1m9-n@^NPFV-IPxX*!{2p_NG<8h<a
ztoSc1?2WbV@zvKcXQOK`bW)nfB7JK@Gwx`=JiNo|yb0NLD<dju&G_Z12cExe+Oczc
z-QAKYYWMUl!$0S1n(FS~A04r%ctgzjxz0<-dH(m)`rB(i%AL-z<c=K2b-C7<ToRB`
zF*r;RJ7$#=>*V5P+{eQzCDoQ4Uu7G2TrIt^r>1Ij%YoIeoLr}Q7-c!O^gVfN)4`*$
zfp;h1=qQXNAKjCRLdX0)5#=u?CF7X77cK+tBt{mCq&VaOVgr;Tg`<Ec3%Y6;McMim
zGRe7^vC19qafPL@C{hv4P$Pi14+;$wIw%ZKv_R1YMF$jJP?(@lB5*nIZWTeuKZVii
zjt>4SCA7aqLjTR@U^OVP6hZhPBdRHp3xy!<41xk1R1Y=SfKkOnJiu2F$Qv*w9LGr@
zX3LXHh;RmnXdo^Wi$GpnN(6OCJSSF8B(vE7G!CT9!KTCsggC}fC3#_y7^J9GX%&o0
zBzl40jtxV_To6l0K^E3dB$f%{K;~T{Rpoelf(NM`@c~E_dj|M<0&icH*rmpbqM`^G
zO`<{QkI|zL*%T!eq%;{l@OUy|DLOKcbr-@Q`^!_zf=Oh4iFC6v(~O#9bnZMjlA{P}
zKo(R7Oadk}oG0MRP%wi)u771))@RI+7*Ne0w1_Fk;*d6`1p*XU%83)lf?x`A$C%1c
z2?Q$u6GUDB;*K(hiD}315jdVBA@Z+8(pgUYc#48!v~L5^79gCmNeNX(H)<qRO;*(!
zKk2K;YX51RqGD7WJ3+4u@c(w~AgyFP+6w5SICgZ9^oGR}5mym5YN{3z941IT0{=m6
zrIc2;@9S<QyXk6bg>#&oQ|g*jd_RuU+->c*nx>quP>t)k<EYbjN^ztZunH*=$}06x
w=5j7SZR;@d-Q}*x<p3U|D3ORy*fZFABASf>#zTC8F^El3>?YBbbLDpb1I>3`<p2Nx
literal 0
HcmV?d00001
diff --git a/ray_results/test_experiment/searcher-state-2024-06-25_16-07-48.pkl b/ray_results/test_experiment/searcher-state-2024-06-25_16-07-48.pkl
new file mode 100644
index 0000000000000000000000000000000000000000..304ac7bdae36124fb18b38216267623b0f8202b8
GIT binary patch
literal 10106
zcmeHMcUV(N*AF3qgcd+h5yVbWL_w-8QP>5!5I_`kEinylAO%wdY@jH}dRJJvVs}AB
ztSGP`h^!6k0xH#BP*4OF#D<M;ZjwuJSKn9n+dsbN`SLtBGjq<&@0>IBoS8WC28lI6
z`CEqCAT+sDz!tLP5>_Nz$`;8`j7V`|yEt;(oY~5f4u>u0!Yr9!F(36KP!IzipDW?<
zr7VG1&I%0$G!kOklO|S(SVDFTUy4E+h(6Adc2zd%LF!SLh){V_NhC|kmh(|BdQQrm
z$kNw^h(2G$;q!O`aTtpy5TTGEqQe)f6i^B$B1U+*2bcyz6iNZgh<v9<MV1UiOU#1#
zY#s`c6bOYK>cVs9j&sAzgUP@qSPK{dYeN*6f@P>6qvJ)dZ+6<gYT_bT7czwP{0Kx?
zAJ7fF=qs-ikJhf;4AUS(0LF24bsgvA&g%t61YkrQOalX}ygulYQ@jNLV_;wu9E{;1
z^mxR54#0>2%*}<(cXW0O?FEKuM#8~J7}#UiYZ-nH6|k-!K?6r4VyZbzG~ZW8`$?@D
z*U7=v(cN`iFEk2nClT05#L#XO(_hBbg?6ElfNn}OcfwA?s$1!5-5hv)CzrA8u3jVc
z>NN~asayJ#U4P_mNx^#!GgJvCw02v3Fl09X(*TCLI6Ht&<2wDlY7_vb?6n52{9*Ru
zPd&kuLtt64^bFG14uE0f>Be_<b9Qy-;$TY6upv;6CkCb*&z3Udr52<Bd<ZZzm0&*<
z7X@GT&jMf+05;Cmna>-?bNvfg*LW(;tQv5#JUqNVt{DcVfsMeg7f<&ad(cQ}rUKDs
ziTGfJaZ!XMl*nW#qQ~M$U=E8b5r+!GP$r_ql0~w)d=wevp(F)iC3_WH0ak)QBF1YN
zt7@pJ%8F<zYP$kc6d{S(B0j()0eNhY8nWP+NWN6LsQGff6o}U4N`yi_7lXt~{!*}p
z`8-bnSB}9<$Cd(O-BXLt5bU{NVTMV7HEbb@*sxfUG0`lAI8wlk5b{|b-AwOZk^wk8
z7BTx8-zOkC%Yd1P0v(c38;AgDq=01z7D)m+PgUlFG7~0ysWvVG0U{!%V6{iafUc7A
z?L`WqT)>t}*)b@hB?J1gr9dn~R#pKiPYzVVD8x`VG--`e5~PV}OhlPXrcWpMDgPjX
z9OZaWR6_32(FC9oJQ1Gy87To}f_ktP><BykRdt96lqah6PsH_yh`<6O6V~dsKvRW0
zQ~9guDeOOy5>@iGza$^hOtFD9AR<J9DDrL%AWc4`!GyKDRUp~)wC+kz9aMerO(eNW
zEy|bF`bxJ@x^?=;-9r76{I7QlwOfUMrCW3m+pn!Hg;)?Ok&0NMLJ3=rBDSBcu9+(K
zj~Mr_%PSQb@YM=FX3=MKRO_qDN-uh=YK(hM0ri|u>Z--W&WC%dip;Ji*zDK+YBw7~
z3~)4oMey1DGi1WLUEt8a($RWfQtT`9Pp@02e?0&6za;<b^H0B9g@2`64B!p$#vT(v
zHSDG{^NVvLhMw%P)u~9to+KLSP}8|b0@ePo#*03{rM}t1V>A;x@36`7Zd(GF97UV4
z-xh3g(7xOU5UG1}f((024~V4PplIDaK~`^2Ta?fjdu~wwJ0#Vfr9k?8c3iRqylABZ
zfVl{mV}rSKx{wyE2Wi6;NE6lqr#%hQfps7<YyeSVGDJ`yeX(YkFWvj(#DE|IsG<Ry
zVQz^gfWW*=GH9*^V6_1(<|k1BlnzW&AXF9?76^G#z8Hn=v7;Fo#$tH~_zMNN>(I+L
zU`ZtsIg6#P3zNY)1NM0)Oku)Qu!6ywW5Rk&SRV`+6Q(g0Fj<KTPB07{)>b7HRYFxI
zI;uohmFTGweN|$hN@&UixJ+2=z!%G9T?(l9ADA*+1eR5c!Ln*mSXM0#%c@0US+!U!
zs}_xA)#9<NS_LetQUeBA1)`66Ry+ab<FJJ?G#N#Vz_!W}@OXSNiz{R+WPBMo5fLpZ
zTg;P)P{c&#MW~AQER{2ZBE~EbhF~>ha^TmnxN?coq0si5;o$B*&J_(e*B}rO6Bdgj
zkh8-0Vm@%yBvQN{<~LwYnSL)WigYzc$VyygCCvyv@LG&}crzf&SNe>)tWc>$#8HHX
zf><gQwMjckRc*ziJ1@rsc?ON7Y#UhST?%<RS6_B}TDlDKwPnt^fy7*Cs6Q$EAR5(F
zU|MZ?N56!?PPkR`d(H0joc?{>A?|O1b_2!DaobNm)fwYmr$2aIQt{o4!y&7*NmWk*
zSJFZc5=M8Pcos9sUHG_NXMK3eoxJLVX(cPr1dBU<qdtz@w$JvacE_Y?u@{YYYdhrL
zf5R!fntnX7;m}YUcD>Vhg;nruNzKo%ulJe#%*5k$`+#%ym8aw2%AM;R-}6mUen?Mz
zx%%ulLq6A4-k$4}rE|Wdlu)$c0q@e>0gg451n(_2MVpJ4G!fk&*pL3r<KwA`EXvP0
zS;cPsw|O5OQInYWJh(-wE#jN_R{8d8iW-ILZM?RDq`f?3Pm4?J+Kl-=b8e2ARxvUn
za{udPb|IPd<`1_99GU+IHz{%KMsm=2t7w<-di@tEk1kbc&K|kf>B;en4nF5c+bs~s
z^2gj?^4y-sNjtUCRc{0XHLHHxKJ?tug})f5vxx^f>K1An=N6_cOUpWKGjI=_vYhgX
znNj>8YQ~5<BaUYB%-72tYV$f8CuFS*suOxrB@(mepN8if?L3>>7FzIbTmKEdm9_bY
zHZHVG_IbuDX%v2peIiKK@hfNeqPKltPl>Cw(jrWDooH+{tRZ>>FNJ%$xaMx(H*)_?
z1qT~LoNk9{#Ro67cq|yUI3+tfLCPz7I&}P~ejA9-o6kJne5;RDOJhUEq-UH7?eE^t
z7M!`-_+yK2%i0w;on-qKUo!GY|0I9Dz@}ii&aJkDwCSfc4xzJK31Rdw{THtXq1$xd
zZgJk$abx+wM>lfwvzOdR3urQZm}A{Ge90_k-?>)w=?8YYq#utfKGxUvL3?JbRpH>Y
zn*Qw-yX#W*uWD*v4@#;F617kdnL#8`!ZC^E*3)M{ywPqb$ThUTX}+|~^zEpm(=R`r
z3JF|Qkm2z}T4>cWCDTK&sv>oP<ulzyuJa!BJ7oGMq9wn1Y;lg*=-lmw$!Ue9o_^$?
z4)yJ~>$PjDrtgrtQ3D50&YJqsXU$EY6VviDW}2Y8b!8S1bw1f==#HWOgPfN?SSdSD
z=qu8<Su^)m{lOH^iKClm>$QrVKgLaCBW*du4iX~>>$2YK_Vqi>V-7(A8~yjx1f4ND
z^<v(hS#~ek1BYfx^9)RovgaqaTJ;&PF+t}3@Vrhroh_PRu;$JV6N0tJo8UNj=ZQt6
zp&AQgt1>3r3QCujPY$|2=?IfO=@MZLC3n-p3GMP4v*VNJtrlzyxS}7jZd}=+%Nq6t
zadf?dBd*h%J2x!jkl7Xi*6&T8mcED?b2YW<qwT|F|6$kQq>ZVf(B&6D<S(B**dVjG
zEyqpv!n5I4YjYCuE-%?**vAbqn_5l|GhH(}2DLxmaE9G{a>}iX9*}?N_9*A-KJeZB
z^<Mo4TkUMULdyR00&-j<eh`&i)%NR8dn~j^zHo0!9kt3KIAzS)-{Ng=T+4j0f*`1`
zj;jy2$2q=JzeGfCoN}S8A^ll@p=IS#-_wZ%PpcND6?^lcIZsM%A6QGgP}FhgW0YA#
z_9Ls4hD8IHB*V8dIoD2HO%Cd-*+$+vrnIVj=<UmQ-Unq@47fMEV`=;#p{CLN+75H!
zl6Pl(eUH6m9}hIzR(V}>&cZo;Pur}Ep5#)q_5``{{kSOeMUyhLT*@D9YT}4x+uJJk
zJUG1n{7-$BOb#9PVw-zX%<9`g%@RKk1?>p&*C`~AI@A5{9bZh#5I?rxFf-f1?(L7W
z>ogiIZoG2Q(97PIY&pFQaSXHYr1!f!WMwWSzSFs?MjC6TH8Z@@?!w8(Ug0a&(Lx-C
znk7`9b3VAPf%8byZ47nJ*m)B&bpLD{pITvL-8t69<hkZSQ&COUk&O4h+fmj{J3zkI
zRJzMH=3<Ag-H9O?wBw#@>H<mZn6eQ=BG)Hfq){f#PRlZj7?!t`Ys>F!PwSX@r{LbB
za#6E^#MhZKB);vU<Oh!n^vZ>%WgE4rtn-sSL!vlFjZ1urBZ@=|H3;k0OgEX5u=l{k
zMI(kiZg4B!9JsBrtRyt*R8iT%yHhRCaH#z<kv;hb)=Vwi*xA;{+uHeK<3O9jm1WEF
zI@eOnD`vm^*?aY-7mBgbb4I@`Ef*cRa5+L}WKDXXKc0N};q-W}m(AUA>u(yLvT@8m
zve$ljl+OvO?}t{sEk~oh51ko4YHD>w{12_a+3*9+vzU6DUtNJ_-cNfL>3)n8J1+d}
zyeR100P_s<#Q1qtD`$onZl=1{e)4=;G355z^1DX!HjT@3ezJKEpKJa65^`y+cUH?`
zvf<G;)ffFM&U(I@anuMhAIa!vP?l3f+EuYNz^tM#XO_;`+3_jC<C~xQKiT3K|6Xdb
zm-@Taa7!wBZ#^eJ!8f^8?;`DO`C)C&eb<+<AAbGlx6~@prBZ_uh?Y+o7u2_;!^(Q+
zkzGZ1KGC;_kA8eCdoFWOW~j&`WYg8g;5^!k1#O#sy=F`Ij?w?4gxz5zE$HmuFyB1z
z=ZUXuU+#-HDk)mH<sSFom<bJ;YdcFXB=M<7WG!UfW5#haMhI&*2Z?mg(m$@)Pq6jz
zl69Epw$Fbs<4?-yQ%1v&yT`Pc?Hh!iEPf9=-}>Q(e|q)33FXFIY3kfp^FGYlJv3=&
zq@zbiVQpgB+vz4J&Tt-UA`mZNpsP)+B-@)})1b@yr5&Yp9-#>i2e|PY-%hx<QNnx{
zW8WEd?$r5$JvXBd*qYbnWbBj6Q|I*=a9>+cd1v&)yv@I-XPV8J9PXW><E%6FLfps;
z33GF%S4wBTz4GqzO6~sP&lismSYjLM1`X<9*}6w~th%wM>g}<$uWIQuc|duywYgo<
zn7#dDAD@L1qZ6~o23>YuJgNHbsFM5DW)C!?Gw(~+J!~`nDC!f}5cZ}Y)41hQURlA5
z++9nXB3Nz<1~@h;td2TpnH;LG<B!lZ&figaD!}mm{2M8GTXbYb77WilvP(=(fBJxy
z1t!Z<M8jr0w6hDG?(f4i_S`yR#`r1h!h{o7ehfZn+0;HS++zQOpH}nP@{#3R*OZ>7
z&8R@M!)mRyPjTOHWObJC-bDWcmyS+v8=RIz^{|@1Fnqh|!^dc$yLD=CsdzaWW-{~v
zZ>^Ds{k5?xUb}3G2%PFFx*Bgd%<9TyBgaptNLQ9?+V2?Aw0$MX;)bQu`ag!JneSL}
z&X+B<te-W_KHo#w`7m~UP4<45VKn`#AuONi%-yxuquvZoYsmMs;1woOJ5SJ_PPqKL
z|Efz*Ggr4??#w(qpjHNP_bv{G+I8&~rU`fHjhRy(6#1<6ja~AT>AueQZ!|oL3HY;e
zf7PcJ-!w1l%mtx^iy47gDbIes^lQKV=B7c%10C*jett8@*^xwP8FHlH0z2iFD+<#Q
zqQa^BDhl@a>qV4bByly0Xn3G<;7%f%krEl|mjUihsW4l{3IndJ421{?S(WRGnG_tv
zRPB#<`@%AoM0KUCH4nJ^;G=+#3O*h1>4HxWeEQ%s03Qu}<aww9xVOqE<e$Rm;^Dz}
zDWTUD3Vly=&>LiyjL+wRD^YE!LM#T)2p}r3fi2+yDTr1i77*~1g#NHGYzX&-O<+yf
zA_bwbMC@qHa;`#(#WpBJ0YPG<1iX33un3R8h%HoL4;4%xKm?vczzB&1VieI)-}tg5
zk>L43eaC`mVYj}(iAU_<NDc@rWZ+8HUJ@x6h`_6gRHnZ0?T#d5_E-aOYdqE8dn$1K
z)wjG_LP=N{2BVE-Fk2967+>zIqJsNPoo;w^H8ySZ<={m{%mP=zT%{E>>>3!mdsbaL
z!`29u6U7Q;D<d3mC8`gMU|||Y33zfCSP;PRt8Cl%0qKJ=)b@iMX0i~4^blPTt*~Tl
zQKS$=RuBWBsp2LOxd2QMh5?8g)rBUa7s2DR7O|z+Wh{0Bt-yX<<%c4aE=Kq$5L?-#
z%~FLr1fn|CRHvjbl+|Ryf8vwX6_smH(BnGz@75rsyBZC51o|k~AQjwzvm&JujxvVS
zR(GT_yw?C1)t@PjE7UusFjVDMI!+&u{HNDy(rvt~<`cT=O{{ubxIf~tnonV%@YG&v
z{jJz;cwCGs?V>ZHZkk7|7d^+ts;l9A3S)jw_KASnJKrZ3hrxy8<yLy(>(a<MDLJ2s
z#p@9WdGDr9P~_&~{O_~2TO1BN?VWGa4znj}*uV0L>8XPolA_dn!*3Id!+k!s6vm>7
ztF}8e*W=dU+Ne_sW5ZD^QO%2o-zOGF!^Pt%fnX8u^l8xb%FnFDrxch%M{5d`50`iG
zziCQ!iD9S|=q^_{_+?yk#b+um4o_R@WG+%)@Z{8YTB{a^cXGG&p>=UD>BTqHu-)Qt
z_2xwW6r6JT!nax54Ufa9DGZ$B6-$ZCUGcOF_A@@CzIN8(qaUv0Y7zdcPR7CUPW{|k
z0wZNh?jO6Rq<6vg7TZcM$%r1G&AHdb|1Pn3C*tZ57`NUY-LkLxv>Fyyt6NQaZ2a_(
zaDkfNO8>9K;>~e!I4oW!5I!x=ekkWQbXn8eS_&iJvO(#oryFtpx2@YQ>+lRim1(8B
z=Y+F^O1fsUnvbhXVJujQ)Fx4zzf-JQ3~ntBOCYR!wvRlxu38O?^VQNpVqNgwto{x@
z)4y?-!eKs#ZKW?5D5^Io!;gY)byO`{>Bs*(sp*zo^+jv>5b;}&KYWD22$C$88ke=G
z!ExbuN?|Mu&bzrV@oHC}eAC3jVQ|~=^1plz;bQQv?k@lTfdGbrflsXe%Lf8Jw)oWg
z=Q}fBfdB_zhCNYWFGiS4sd}YzvoLUdyFZhme5Lb>?e%cGdTK^MRb#A*r>YJ}DYt!o
z>q&||arZKV){7)7nkyc(Wk=6iLHlMrS#8y;R-lbZ_`r}wJ!_?2<VktL^q!<8CG%N(
zGkamJJ<Dlh9qvVHdA0a<^y^-vn2DCpKkV#<HS||+UijI(p74^RH}(5&&r*?8n~loS
ztg0qEGs9lZEesML3>V6JHs9crIekb+FS($hlUHx!F(CrGUdEre`#iDVjL4o?kJ~$t
pNx!JEz<`;ANhCZhP6Qzp_GA<Rf--FQ5FISoQASBvQNdBz{||AqB?$lk
literal 0
HcmV?d00001
diff --git a/ray_results/test_experiment/searcher-state-2024-06-25_17-10-00.pkl b/ray_results/test_experiment/searcher-state-2024-06-25_17-10-00.pkl
new file mode 100644
index 0000000000000000000000000000000000000000..862455da2971212f5b405296f4c3e232559a06d0
GIT binary patch
literal 13726
zcmeHO3p`ZY_aDz0uPAb>D?M;i;U*%&Oh{@%H<GJ#8Z*P0%!7G|O5_pgglhC~Q<NT3
zqExqpL|5;Zgi3Fvl2o_4mH6*7XU;ryySLw8_w)Nde)IVpd!Mz}Uf;F$UVHDg*Ng;>
z>~#|el24r2644j(Su{2z<Ux@%K8+&~$B7Xl#9;z5U5u!+c>;kL(S<^Ja41CQal=^Q
zVqZiP5=7GIOffRdLqhUL-)yjE5uYYx@whn00E|OQl`x9eiAOI7IF^Vym&RcNdFlX<
z21)}6j*VpUC7LjWOg@0tq4U^mCLI-tzWMPPOg@w0$)XEUF|&XW?NFMK4nqRgVxZ6{
zMTL3^v!T<3%y1qsjK&rtmJk#f8v}{Bkt})yn+bWy8@-j3064G_GWf6Br_YSp)z3Fo
z4AjXKTT%&Bl`J2Ex(|W0LL7p<Gr_wrY&aDgcmjc{iVWn6IFYen)c8yrj)*N}(fE8?
ztQgT00R3ou0E=iySU|}T0+mpTYS={a$}|ZQRSi*b5&Qc3PVWI9$sZ~~C=T@?VIX(O
z;uSz6csTq+FVYIJFW^J^F_?577*2@A6+&TQV%QiS)Y}qZhiVEif=Zx)nlEfCw?IWv
zawi6sf=M2zC>huQv1zeP{+|O-RiyH%D!#DGSBcld#q+<YH&unIit3I{Ln=NER8=q$
zz>KMK95u|0*^*C#;+Xv%DHSu;NI{#wwiQ*1s+MIbDb=YYp<FGh8WXiYNUoQ<rQ-Nr
zR#Xg|-+zfD#B|kS@OhCCSfOC4jQ*>oBI2^bczg~N#^%w4fVbCTnT<7jp{<{<77jAx
zFBzr}j;YbcmW#~ZspL+~EJdrOF1%94<qNZ?lJ8=SzOpW;GwU%qp-cvY#SMoTEY6?0
znr!$Ds1zT93UG~H6JX#A$H<_l|HuSrGr8EB`@(wr8^1;5>h;-g`{Szg)oF9(O8p~a
zrpg9e2M>-?a2w1?2F^ZIUK%m-gu*_DkyI5)`s(|h{PviIEugfi<RK3CpP6`A`@*G|
zD}C6!0=UxAP3R{FbtO$M1T$%jzwjZdx_l8+4V1iysxDc`mGUvskS^pL3Bno0h5T(u
z*xD8$hP_Tth{cFOk6AdEN`ix^>aYt{6^@~5!U&ZJPo=8Cwp10klBx}}sTy!8RSPzx
z5=4kG+6|2dluu7o5S0K}R6sXXAW990PzPc)KzB_bmI%a>0F)LGr43h#5G{xfv)Bwi
zlPiY%OHOJt2=bok$7bP?gi@#g@p(KU1W9>eLvYrB^8jqsU{?go9xP2?81aR3!Mypx
zmA)d_P$Ct!#R?a!n2HraSi#1M7_30BA{Q%4v7%B^h!7GQ|1h~iflL9c{id$~hd`@R
z7_=&dLaS0Zv?_%}t5R6BDuqU?Qh2l~RROJHYQUf>LiEr;i@`!e9vWL9&J-j4!CoB7
zVlbFoh|Z>o1WW-qIS@@gjmzM1#E6k3Ot8UTZ6GY55hDX2u+yU)0wLJ&A-a$!2`Y#_
z0k%#~c8+5B17ZY0p)4U3&g3$|p}^zgOlaVM255RptspYQ`yv_=p%Nyw2qp+y24G4^
z!WmFyO5#TyD2&hJgo?t#!12FMY`Ok~7Pfht4jSou;#z}e^~gE8T^I95H#G@1Sv)gx
zNod|z=Gy+^`m7@-XAdvF5GX9EILCNb{l08*a{Zgx#7R+4>@`=IcG~1VsCa+&ck|5S
z8abKn&6nVj)yup`MhvSR+Zm|k{!(Yu9zkh{dYa9HS@rWi=)9W$6Q6l#S>PVhzDqZ~
zM)TK*quw%Tx%A{QdQP?nYqSEPLk;&TBWr4o`es>cpZv+LY^ay(n6o8Y0@D`qYM_IO
zE#)dNhODUf3iMlGeoA0f$~k{vJNHnV*2^lDxv74<AlBj5r8Zp$8a=yff3+~Jf9agy
zdgeyqrzz*3?oct>lpd8o!bH<+$on~dn-dB{PMWkL3+61lzcJH1IAVVAyw~MhA3%9t
zIsF!owL=cX>xA#w(D0Dd7~R|%r>Xl#WvpHOmOI^7^@PMJB>JT#Dh@sKTunM!F7&(~
zxt?7@A!KA*Pf7_XEP4@J(7MoZufS<+`H|fgpT-&OH?Cwq8}@XQ!M?iS4>vAF1Q%!S
zk1xnS@cwPnim`jfo=zIkmh^G$sBn!tnxl}P9q$y{N1q+F_sp9+%)Oe|N3L}w?jbH6
zxMK3niXsoIx;sCtjjB1VQS|c!Z?fHHo@esaoLWk<(@@Q$RS4r=`IhFU--GR-=rT9n
z-KQ2Ufo)?yWxqY+r+y;O;>t`lQ@C-D{-}hrwR1kT?cLH)X{n(SxRV5rHL85PKj9dZ
z9TdEwe0asK@u8Q?Uqd0Dk8BP+dUR{!{LMS}tj<z-PCgc$uQ~DNHf_(jTi(W9>1_Py
zm$ll~+OQ__t-FYIA-t%r$a8=3t!pW4%ayC^8-JR3%a)PIP+dK<nGBgS9+*;p$uMwC
z^wqD<SJx4p(KB-wZu~?1R8*FiVO#D$^nJVqb3ur=E4O?{LlD7{vb5Fbg!7%Wsc*R9
z^--s|8W#&jkbDasCMS<%>HWH(@knNHh+}kDi~INmnVR*mzwqZ$e(>V=M)ubRdnRs}
zdGejt>>{W04jzM?a;`KVN$SVm6!M4uh=L8XV(ed8Cx3t2N*HAyz}bJQ#X5kh9g{ZN
zgZ(MLntzcK-Tup--R2?RMOk?-`H@J3?Bb&iJ-i=__$}I@xsl)V$EW+qIGe08`!I;2
zRk=AkxQjAjxsPj7D&N#)pXbL^!S!29&K>xz>+lP5e`A~Wh2sJU%grts-%r#t))}{S
ziDOS3bbd*{ZidnHO^$iBZuYsO8m1U*uno;>-Mxg6-eYm%xW|>-i&I`qFdb}_%DWMs
z7FfLdWdD}E=^<GW8e!{bTH6P8q}{d5xuh~^Nr<hFb9UjVc^|UF%k@`}NgGY?ayjhP
zFTrtfY(Qa9;gxeFBdY-{w#%!){O>169EciJ*w6h$#6`WPDcqoy<#j#-I*e5P*G?E!
z=4A6RE-dG@>6DWW4?dcV*>0}CX4%8|x6^e8G?fHzy40SfKBRk|Tkgul!>{&+PBA>#
zP!^PW?Cb(t3)006hi()NxlTE_jG&znS$o{T`oY^BrQ50v+9tJWH0*jSteJV9d;fNJ
z)8mC2djrP28M>_T#$o%q_k7n=+~A996;6w{Paa9i$%wQGa+#X6dPL)G;~(n00;1e%
zZcO7OT-99Vd>~lJe*d1`$n-jy6j3(NwEf35&(uO6b$jgI+m$zXOcnq3{_DApL{d|C
zLThV`r{hAmU)J=*ci5V*vLCd2((<%)J)*<Tb8SX@)rkv`Nez=V>dm>GwoK~<9#C#i
ztgp}34Y!3>=Z73#zQpZWQ4Qztfx)MZjcs&v=>};RDRC(`i9D5vjVV`7MN)4~Dh-Qq
z+BS38xsXEhn7nsCUL7rfbKs}bMs43Qc<eq6x9*ClIoD1SU%x6i=uTXlJ#FHzu|vwm
zIvI1)e!t<yC5$(FF0%BySZ&BNf4d^Ef_eYD#E$(5ZATh1YIU4abqjfmZMH){Y}0*x
z$ho?1UIb%!OkN}RMv8h*^C81|RSW0o9nh|z3x}PXP&W6G`|<93&qr4++16%0z4+5k
z!G<`PANeRL{m_TUR&gb5b2SRCL@g{yy6tK-aJ{)kxB4#jWkNgUSj#Z4Qzsdrg=Qz9
zy@!$w3MMBW-l)oKZrQn1^<JInv*Y79>%tuJ@@84u*@s%1HgW2%g<klu`PjA{T_W1m
zQdiw`DOn{=r6$Jgb>j5}%#iYvD@GSm?p&|6Qw>R)US6A1p?h%9b@!hvKB|dEy*W<G
zvVK+U;GH#X#lxPp`JEME_YUZ%JG+z=_Y<~1ogO<oNOxO$@o2t>>&sJ-xxco5UtNBd
z_EE$Vu$??7k^}CQ?eG}5^GM0{{0sdLsdt9d?h8p>RUCNUzEpRigMM4>lcMV*HjWKE
zJ9n%wWzE*MnD>jSFB~H-8ce#T8!zgxDA6(L-1gztmEvJ$kCI0E=!=t+^ggmp@2<H#
zx1LtKy@_Jp<K8c$tp&b(Qr{{$s^$@8Jo~BrlG3fKTFWxZ41NmQd?L5<mKo~^OuzSW
zPRp|b&ZwX@UN0u(t2iHg?LVU3M6aZfSYX{4-~7AX4)-MTl-6zAX1%Q&R$O+NsGk`X
zZ`d~D2S&3+z`ID#(-%4xq}$EUV|P6s&6%IEOr>ps=G}!{hivAWOGhvFUo?X37hC;i
z*|M?engJng){QQ+iybdtoIyK0IsI<ju}_;UV()FQY&t`<OXhjyAlDW}Z#CVS|EA(<
zomuLk^mVE0ws&bonSJ++xa7jr*GVUy){~wN5J!#MDPT4ZujV0ZghLkInyT$|QrpDr
zr9;@oYQE?Wy_5@24|uoiUG3FR^ZSkGa<r$NtD2l8j&s|S6Y{dy?X~vXw4Yit4rWeK
z=^n~@;I}o)?DCt$^m@BxrD3+SEA~~rKOElr^E;^SRKwk7vrMz?MC0unrX;x>cWtCr
zm$brb9B0+E+*Tc*ReKL<bGfj^M1}ff;Je$L@OeX%U4PjgcA$7})XwdPiwkPl7msTH
z9$n+4W8G!2c+ipYSvg_O#~%8(>+r`{`Ipiz|GMkh$AUcDo+C$pICp$(hxO@M;=ps4
z)Bi}ELPU<f+_rQOt8zLrp10@Z)f9N?%%{$~Gn*}i6-T!PKRvhfww29J504IBYIj)f
z?{Akr$nsj0Hzved%Yy8=YJx`KJL80xS3Don)g}+h{w{5(Tg3d0?=mLs@ARLLtb@!A
zEUoLwTce^qD6-($**D$fM76qw-abFmGIKYWdphi$!=e{GUR-r6Deb2Po}M8`J`LwO
zCr%6Wjw23r*_am|uVQccBu-<Axo&ic^Z252sB8a&sH$9|)o-n&EAGx~`OAZsZ{4@4
z((KR{qn!98{TU-|>XOqpTJ)QC{bu?Y=K}{+ce-yFCfqeJeX!W6{P~&Yp#k%3Vcq8r
zi@r~}b7S$qj_f7D&Mj5f@|K8UGelKnFV986qdw0>`9y-;5u)ND7J_gRQH$gW#8f|U
zF@R|TC>(^b;FW-AU^Ur&Nfe3<l*Z!nI9LGj!X#H2+YmAA0-mYh2?7rrJTc%wz>^D}
zQt(uQ$1p@J0@1DH2GUnCI{4M#zomq(>>Bjn=?+Fi0}?Qq43Hfq@<m)ONN}Jx0?kPD
z8o)rS`VfZ}gW5<J@zEQMm`Vay!AKs+>Iu+$95b567NHp*Um%JD5`JJRI4rIh(Uzup
zAzq|dmP<i2(Nr&p))6Z>G89}N1RxJ<!;2KMI3U5w7f7?b@+*VD2JHY+#qNII?jXjO
zrgk;iyzp>T3=yrMS|eKFOyLZS3No46a`EC6)IaouAT`B>K>n95u|gls{h|qHEY}Qc
zBU+)0pfH-`9s`n}dcX(>)o>AuA%uYiGedo@zAWrQ4ABOu{h)@L%obC15gjzqA)s+0
z+2H;}r6Bs)?Fd{^fJ|`N0wNl)oFAea!C*qsG(MVkMbpkA^h1|CVuU1XSg5T+AXu&s
z!tOK#RcQ%`k`G?0N#E6_C0^j~UuxHoGD;R3OCj<5XN!%hlOkT&A3#bLn-<7*Ly>%5
zsN|+dln$ii60ZVM(!CVNa5~AjV!cWV_D-VAa=1CE!1OWBC#B1{DK_ad5(b2{OWQ$C
z82&*?tv`j8b^lU@;{E1k`LvtoaabvzT-~nr>Cr#lZN%Ft-vpLK88>6~z7qOvnKe>e
z3RZTK*R4w$s_riv4MN{wWn#xQC1}iLJ&;+`8+Moa*>%Q?uglcx6D&T6o#fisVU_``
zOH#2UsX7E%c~jVw<1_0DuGZICi;I`qATMySgoN&}=$VgVWn$$lXpKa%z3+dw+x3>T
z-Jp+iD~0VO`+Epatzfm{uu?h-iG3cs?nryma!=O&RfGIRYjLTRf=ThZE^4Q7u~NFc
zUpwE)!AIk%ti4aLxY$neHIq<kPq%FDC{kR!By`7ye`o9<pZl+Zl?wT+z+p+0&37|}
z(_(9-*5Gt`DKOF6|LQD0$bFn!x!8wiIP{&r)%SvxdvbICG?%!G&!z38uX8C}d~b8*
z^x0=8hd+EJ7mIPq+nZ-andeR@$LIFzV0**i?{cwkHXPhK^%=ek$>~x~%Gp32XyfB^
zQrNE?<j=%NrQ!5WazdM1=BK@zWY*wtSlLO=pzf-&THEk#U}bYhD7H5r?0f|mD+MLU
z+9`>Jbsg`T9?I1ECZ0?n44-f^DE!U^Tx_q=AW<xuM_gA+=i%+2%`KCv`lVA>W<#$E
zKGb~&|DSZ~%B|{UE(L%8M-GH9!Q%A)Ut1)B@PBF3z#XS|iO(&8l9N*TDY^ZLbamo%
zyd+RId(O4^aq{IauHHXgK(g+=aOL#YcRx1SH{O?u#Um<NJLSA@cXYeKH5``kjrOXm
z=I6Q(UTen1QZW5+SP~`n^nKSYEm!gOzQM|jrx^db`zFo%u2d`zhnFNu&g{dEFV^4@
zS0B$I`G#+`llpuN1CI}7bg5pQWV?5{AG)JEq-yo?u*a<h$6rg>IsdF^nONmPCQt&H
z15I8($Afgj*9|g0SUAk*Wx7Cfn4PVB?s~xzT#V@*zclQ}=k~L6=ubWQa~Nf@o#eGP
zDUZfq`z*$w;5{Aa#r5mo$vyq6Jo)de?Id50hx0Aj^0|}3;jom7pyr`=fz@BMmPu9r
zS5IBsBEmm&>HLz-IQ`#r4*i=B1OnkdTO@&i&;7UFBBieUzjm);Qc3?=z33$34kFu|
zKCAp&js*hYABi;n&f32FGn1<EFFO_mAM>1JGs#K6$a0N1880cf!56y|&fky)27NmU
zWw4jcrrpWjm5(2FGB~W#Np7Eh^Z9_~@_TXLU~%z;ZSKQqd#mt-E*`?-QC26JG=S!v
zx9^!ut#1S?OrJ0@HQ|IzEN-P#FG|#OQbTYN9&dc3=&F-UNx14#uxz7j?v!CW$@c!!
z^G}tvNZY?Mgq4lu7b>0PhtTTsL>2scf)6sj4oH+J{|`i61WyWno>ga3wg2PK=;-1@
z_JxrK<i0u<Q-pvowgl)8@Z!MlA<?f=u%Dui8>H)1eXNwC@Ke-S7gJu`F`gpHZR@2w
zMs9hEq(44>vhi!*qDTTAt7zI=6-lWz_p%-(DRB*1?M1&!Qj)c(yrsBkwNl55`TLIy
zbLS|^I@ru!Z(g7zYwo_l2)6MKMXu8iUj9nURT6)eF@A8ip;(dY+7sci$}A;Wfa`s~
z5+%~eHQoUit1_^TSSkz~wb^5`k6j6jSCp7kz2L+oW2Pc0&TZ?9*8NHn{dU!}^q+()
za^?E&TK8<L5?6fHona5H6BN0i{f$K(%ayp+c$*J!r74YC)3Y}>#(q%ZqL&68pIEL$
zvNX~bby+EOG|j9tOYAIAL@W##GhxgbrH;c5Na~ZamMd}r%GyjHC0V~*2=kyD=VM&h
zRR$Y1UCkt?-C~+zch%*T68|2$BFVon;4D<XNRi~6>pS)CW+jl7o+E7^Uscl1%6L@+
z(Nu{GO!JKozbSD6ms#a!5`$}phH990hl2No`JE3@#08cZx_8V~B;AjkIAfe)lp^VT
zD}N=qNC^b(cyX1KgdN)$$gStmlZuK%6}zuG75Z{vh!QR^?#i@EB}>3GTFph2K>DTD
z<R4b{VV3<6z4JR|xajtZ;c5c+beg|F5f==X#f`T?iX@YMsSmDgz`KLrA%ydI4D=g%
f@MSjoliUdKb-3got+dgvnI+#(qpzY+k<EVr7r}j&
literal 0
HcmV?d00001
diff --git a/ray_results/test_experiment/train_model_040fcf09_8_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=4,drop_ra_2024-06-25_17-31-01/events.out.tfevents.1719329545.r8i6n2 b/ray_results/test_experiment/train_model_040fcf09_8_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=4,drop_ra_2024-06-25_17-31-01/events.out.tfevents.1719329545.r8i6n2
new file mode 100644
index 0000000000000000000000000000000000000000..a4b8f2b87bd1c314e64ca0fd1a3070aa0e558e5d
GIT binary patch
literal 1311
zcmah|OK1~O6rE0_buK1trcFQT$23u}voNh7m0C>vZK?}DxNuP()6A<G&Af>-lWN_x
zS_L;kK@<zNRZ}a9&`<;wL7{e0TokD*7b0|1U0J0Hf`Tu}<Rz)K&Tj6x=brb@J+GF_
ze#wDDexc8Mc;?yI`@LRgk0GmM`@ud;td1S+^Y5+&?7r0ax&1=r*z`iWucAb%hAMCx
za(~cD5FCu@rs?U{ejT_tx%9i$rrh}c770^HMhQSOcp1fv>WP?wB2gSU6W3K@c{pw`
zv}eSZDw6zqpYe+&6`f#8;kct8Zk7dYNv*r_h;Vt5QT*V8IwonU8b(%BHIZsz1DT$g
zwxNCBYFsDUZ2IWX&}V^r!021R5B0@#Qza3M3~XAufinYQw^m-Y+XI$9y*Vi)@&hE;
z2F+|hT2;1GotUV2O#0@>7q9P1m1O4(kB*!c7S5)U&#Bqcgaht6pbACfvLS0G3gYu|
zY|y(9tLv%*t~j8~hQyz|_=ESm0D5FA5~V5^u+QNa!KL7cuFyMEpHXiwt<{02utpLZ
zipl-hppYA!ImocPYoV&3RM_kUK{50=q8Cd`Q7+bcu%f8s6jD@;mgs3P$*O0RJ>bs$
z7Y-@~K9*JoC?zP0WyR-^+)yqShfHdM263Gbt)iquA~iz2`0o%%o1sx$Z^_bD2#Eh*
z;-x3C7Gm#j=F7bf!DZO{Tjb#*4-NVKnb<UG3Xo~Z)L@nL>Rm5+c){oQIK_J(^H5#j
ztthxBb`Naobj*dIGG`GjW&^Vv0`ID=F>EjlO>0=raLTP;9S{ama`W1|XDt<BC{OQ>
z7G66a?VOcvXAV;o1J>LchMK}WFkm<X<YUW#J7M|=*Ul>ISq_c2%+7b(k8tj3@|Y0F
dx!(ps@c!*RCQ8<Mnz6j^8~psn5^iPo;U74On5h5&
literal 0
HcmV?d00001
diff --git a/ray_results/test_experiment/train_model_040fcf09_8_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=4,drop_ra_2024-06-25_17-31-01/params.json b/ray_results/test_experiment/train_model_040fcf09_8_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=4,drop_ra_2024-06-25_17-31-01/params.json
new file mode 100644
index 0000000..b034105
--- /dev/null
+++ b/ray_results/test_experiment/train_model_040fcf09_8_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=4,drop_ra_2024-06-25_17-31-01/params.json
@@ -0,0 +1,13 @@
+{
+ "batch_size": 2048,
+ "decoder_int_ff": 512,
+ "decoder_int_num_layer": 1,
+ "decoder_rt_ff": 512,
+ "decoder_rt_num_layer": 4,
+ "drop_rate": 0.18759871111111892,
+ "embedding_dim": 16,
+ "encoder_ff": 1024,
+ "encoder_num_layer": 1,
+ "lr": 0.02750689103642202,
+ "n_head": 1
+}
\ No newline at end of file
diff --git a/ray_results/test_experiment/train_model_040fcf09_8_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=4,drop_ra_2024-06-25_17-31-01/params.pkl b/ray_results/test_experiment/train_model_040fcf09_8_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=4,drop_ra_2024-06-25_17-31-01/params.pkl
new file mode 100644
index 0000000000000000000000000000000000000000..cb40d5843f9214339d4eed4e4022ca573d6a2b4f
GIT binary patch
literal 224
zcmZo*oqCl40&1sd^a!TrCFiH47RBe4=EmnFR;CtB@n-B1NlAr>7L_1Nu=I$+Br@|*
zWEgvRQ*)D2Q&KYX(&JMybEkL<^l-t9PD`8O%fQmZ3)2fU6fDHl!v_-r83z?(%ZtxQ
zO-unfgEOTlzaYLSu_SehyZs4f8=iXWB|S_zML^yhts|zZcn|k*B_)<5XT%q0R;2<h
I;V9Ju0EC25ga7~l
literal 0
HcmV?d00001
diff --git a/ray_results/test_experiment/train_model_040fcf09_8_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=4,drop_ra_2024-06-25_17-31-01/progress.csv b/ray_results/test_experiment/train_model_040fcf09_8_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=4,drop_ra_2024-06-25_17-31-01/progress.csv
new file mode 100644
index 0000000..53f618e
--- /dev/null
+++ b/ray_results/test_experiment/train_model_040fcf09_8_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=4,drop_ra_2024-06-25_17-31-01/progress.csv
@@ -0,0 +1,2 @@
+loss,timestamp,checkpoint_dir_name,should_checkpoint,done,training_iteration,trial_id,date,time_this_iter_s,time_total_s,pid,hostname,node_ip,time_since_restore,iterations_since_restore
+2019.4176592488927,1719329610,checkpoint_000000,True,False,1,040fcf09,2024-06-25_17-33-30,65.59718012809753,65.59718012809753,2195253,r8i6n2,10.159.28.60,65.59718012809753,1
diff --git a/ray_results/test_experiment/train_model_040fcf09_8_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=4,drop_ra_2024-06-25_17-31-01/result.json b/ray_results/test_experiment/train_model_040fcf09_8_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=4,drop_ra_2024-06-25_17-31-01/result.json
new file mode 100644
index 0000000..ecbddee
--- /dev/null
+++ b/ray_results/test_experiment/train_model_040fcf09_8_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=4,drop_ra_2024-06-25_17-31-01/result.json
@@ -0,0 +1 @@
+{"loss": 2019.4176592488927, "timestamp": 1719329610, "checkpoint_dir_name": "checkpoint_000000", "should_checkpoint": true, "done": false, "training_iteration": 1, "trial_id": "040fcf09", "date": "2024-06-25_17-33-30", "time_this_iter_s": 65.59718012809753, "time_total_s": 65.59718012809753, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 1, "decoder_rt_num_layer": 4, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 1024, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 1, "drop_rate": 0.18759871111111892, "lr": 0.02750689103642202, "batch_size": 2048}, "time_since_restore": 65.59718012809753, "iterations_since_restore": 1}
diff --git a/ray_results/test_experiment/train_model_0f80a6d7_31_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_22-45-39/events.out.tfevents.1719348621.r8i6n2 b/ray_results/test_experiment/train_model_0f80a6d7_31_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_22-45-39/events.out.tfevents.1719348621.r8i6n2
new file mode 100644
index 0000000000000000000000000000000000000000..3e4fe552af2e0836befa83eef18b9fff71dcdd87
GIT binary patch
literal 1865
zcmb1OfPlsI-b$Qv^gWVK&%5a;#hX-=n3<>NT9%quVrArJQRt}*RkvY&xie?lMUj0V
zbpni9Vq79zyhVwX`X!}#srotj#l@nW?mt`>NSMkRK($Q$dLf=O{2E+~GM6Hk6kJPj
zMt*5dN_=ugYI1f#er8^YC<8--{f*Vt>rt#sxdGRYYGq1(UMf%v14EJi8!dgPmASF9
z0i0hDT9mjHxFq3LmSpCp#+PJd7RP6nq!z^&i{6^P)w$Px<pM*f_PNc?FFEsW!fjOH
zQsR<E)n1&Lmz)}3lv-SpUz7?qAn}Ysx;fN<=Pyo0ay~{Fpv9%Zr3g0wWNTtcW`16A
zJgPAup9coUd_nQ~d!`klXF!3?ge|bA6n=LJopJa(N?`wEhHD`vunoT}+(xmoodvER
zM_}J-FWZI^*g~vuEd&Dlu<CkeCDRkXPy)LXp`Bo0pN}(g$<#RV1|_inAPgWnuo)b7
zWV=P9_*~CPAQv3i%-90^)!eTx@ecLzD1n{l4A(+TV9VAle~Dt{AB27!fgQ-Ta0W_X
zce}u?BoNqN&aZQx?e=dnN?^0P!nG3&?3BkXF0s$QTA~DYCBgur1KYuYOIH>(WhZN{
zI|2@D7Hok%yY!2Tw$+I;l)z5Xf@>irustWotVRj!mk9kh0^7)7rXWgSmukbUBoNs4
z*6W>xt$xU&1om5mc7lOzvT1|Ms@at>D1n`*1Gk^(z;19@F*)NLiqH2%TFIOR6(g+J
z0()!g7nh<e6<3tNJ{$$tLQG(P{>bixVx?C!TtAM$UM8%=i4xcs5LyTX_Hyy{&i9P3
m&qWDr<Fk1Or>H;*N`<@$SbcfxR1H0MUWHz+vyz)-C|Dm0W@V
literal 0
HcmV?d00001
diff --git a/ray_results/test_experiment/train_model_0f80a6d7_31_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_22-45-39/events.out.tfevents.1719355502.r8i6n2 b/ray_results/test_experiment/train_model_0f80a6d7_31_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_22-45-39/events.out.tfevents.1719355502.r8i6n2
new file mode 100644
index 0000000000000000000000000000000000000000..fcfab8bfd030b3cca70231ad4044c1ae29c1a234
GIT binary patch
literal 2771
zcma);e{2(F7{|+12lpfcdbVS<4Az@Ota06@(KuOp+9OdUGVmXa!P~8OqvzVYdUxG~
zELzxxIwMxZfRGIsOH5?zHev>bgqW#h`Ue>>4T{DeiD5{Bix~?VNxc4gukY+=``^9K
z^L@Veex4_NuH2ITop|Fpn!LKd({-uu`=^|x%_=X+)f?JGS+B1<e%?Y?W%5Rjw);sf
z@LgLnZ+VHQ4D3*<@;kkHR2ID<MbjK#bj-0G4Ht)NGLjE(O>*R85|UJb8=Od@1(j$>
z-~vH0&>B%BS$9}0G2i$Pr&!E#Z!6N{mVzRShQwmo)7SW`H)9#zUeQRpLP$bGC9Fi2
zx)c^UJt%3Mq>C!2Ife$t{5ST`{8gKg_jYz(Cl8zQ`+)`*F4rVEAabgx>53|*3as;e
ze&O*<fos#>yhtt&BYzL912-y=bmetPku?r4lf3!$_80!eZtmMnUr51ToDcTf(OLGC
zt#J&4&F(>xVqo79Ztue^zc%#?z`ktV=EY!d?nIUaU>`V`@Skg(V=>qhro13*cV8X*
z<R^n280>~F<X<#w3$x|jM-}YmU$(~AUQSN-l6<iL7@lPh@*A2l*jGYGQVi@(7e{TF
zWgv|73&7s&{*1?9Pn(hgu<NcS{Jvl8A7HRwmXTvY*gxId&C+LUpT%HLngxo6%`j8r
zU7un%-+l60M+!EX5BAjbE%w_tUmCz*?;AvtVqo{b^L!g-xnc<E7l6ISd*&GocE2ep
z0Q<&J!td_sJ&(b5C6Hr5*xN=&SnA5d)fnu!S)gdxF{bS1?_Jo<->Yt#O2LMFu(#it
zWuLBlwHJdO@ghkvupc`4-~r6?wyDnxJLZenw|s@c-c^Gv3&5sl5`Nc?BZo2AH%)m#
z*iGRd*`D^%e=yihwaC9{*vFXnkL*1&vMBxEAMgF>74m`8=`x-HD;&3ioeM^Il@Dv2
zS9~oZs>c42^^I>?q1y_JGa>44f|?`fr2x&m9tawnVV78Gr72h{hyg_~_LQ+3xt12=
zY$Z5y&*W&B3-LQe)d<<3G#gSg+2vr*Im%5g1yWFz2xly#q?Mr+!_A@~Nb(L&kiy21
z@iwBIc27Awz?OZ^EmQ(TDCrJRB6C5J7b>kD8!V&n4+GC?SVi4s@z0Ya7rUndT-3sg
z+p`8LsJks@&swOWidekyiAUs2|Hl99uP2+b{rio8p9xrH_D>Tv&Dh+Wrt=11Np4X#
zo+IEaVfJyDa1Ihsmcy1JZA|Ra`hnLE`QW}RBI7VUFf|M~=XDLmVhpU#gB-;z47lbG
z2nrfvcmCd^6hpr(**oK)e6aiDtxpKP<jY`VfRf`3tjL{*c>!~5QQ#P3d^qKwRg^R1
t8|yh#wm$RWG;#XGRx+BUUkhGv{-3=l3u*GkjG5f;p@}7n8mcDi{{xa~ecu28
literal 0
HcmV?d00001
diff --git a/ray_results/test_experiment/train_model_0f80a6d7_31_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_22-45-39/params.json b/ray_results/test_experiment/train_model_0f80a6d7_31_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_22-45-39/params.json
new file mode 100644
index 0000000..4db5d85
--- /dev/null
+++ b/ray_results/test_experiment/train_model_0f80a6d7_31_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_22-45-39/params.json
@@ -0,0 +1,13 @@
+{
+ "batch_size": 2048,
+ "decoder_int_ff": 512,
+ "decoder_int_num_layer": 1,
+ "decoder_rt_ff": 512,
+ "decoder_rt_num_layer": 2,
+ "drop_rate": 0.023920599186226577,
+ "embedding_dim": 1024,
+ "encoder_ff": 1024,
+ "encoder_num_layer": 2,
+ "lr": 0.0024687399238039302,
+ "n_head": 16
+}
\ No newline at end of file
diff --git a/ray_results/test_experiment/train_model_0f80a6d7_31_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_22-45-39/params.pkl b/ray_results/test_experiment/train_model_0f80a6d7_31_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_22-45-39/params.pkl
new file mode 100644
index 0000000000000000000000000000000000000000..611a751f78351d0edeaa1fbe25feacf85cea253e
GIT binary patch
literal 225
zcmZo*oqCM{0&1sd^a!TrCFiH47RBe4=EmnFR;CtB@n-4~NlAr>7L}k%h{7Z?^H5|M
zdw5fGlTuSsGV{{oQ!;a>_%g8caKX$@O9S(HVVZ%af`yoR_+UaH^PpmEdGQ&ki78XO
z1$sDBit-EMixNvxr?}hCs9R!lsllO#DW?d?OR)@E-nvGjhbt+uBsn9#II}7hXbDHD
F9stTSQbqs(
literal 0
HcmV?d00001
diff --git a/ray_results/test_experiment/train_model_0f80a6d7_31_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_22-45-39/progress.csv b/ray_results/test_experiment/train_model_0f80a6d7_31_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_22-45-39/progress.csv
new file mode 100644
index 0000000..946ee52
--- /dev/null
+++ b/ray_results/test_experiment/train_model_0f80a6d7_31_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_22-45-39/progress.csv
@@ -0,0 +1,11 @@
+loss,timestamp,checkpoint_dir_name,should_checkpoint,done,training_iteration,trial_id,date,time_this_iter_s,time_total_s,pid,hostname,node_ip,time_since_restore,iterations_since_restore
+1986.2198524775467,1719348983,checkpoint_000000,True,False,1,0f80a6d7,2024-06-25_22-56-24,363.18635082244873,363.18635082244873,2195253,r8i6n2,10.159.28.60,363.18635082244873,1
+1979.5493135226993,1719349334,checkpoint_000001,True,False,2,0f80a6d7,2024-06-25_23-02-14,350.295019865036,713.4813706874847,2195253,r8i6n2,10.159.28.60,713.4813706874847,2
+1964.9348182978592,1719349683,checkpoint_000002,True,False,3,0f80a6d7,2024-06-25_23-08-04,349.6246259212494,1063.1059966087341,2195253,r8i6n2,10.159.28.60,1063.1059966087341,3
+1955.6751497523992,1719350034,checkpoint_000003,True,False,4,0f80a6d7,2024-06-25_23-13-54,350.46285796165466,1413.5688545703888,2195253,r8i6n2,10.159.28.60,1413.5688545703888,4
+1956.1783514548474,1719350384,checkpoint_000004,True,False,5,0f80a6d7,2024-06-25_23-19-44,350.18478202819824,1763.753636598587,2195253,r8i6n2,10.159.28.60,1763.753636598587,5
+1971.8988527312993,1719355866,checkpoint_000005,True,False,6,0f80a6d7,2024-06-26_00-51-07,365.2865264415741,2129.040163040161,2850562,r8i6n2,10.159.28.60,365.2865264415741,1
+1955.6729476808564,1719356216,checkpoint_000006,True,False,7,0f80a6d7,2024-06-26_00-56-56,349.18421268463135,2478.2243757247925,2850562,r8i6n2,10.159.28.60,714.4707391262054,2
+1957.4972913847196,1719356565,checkpoint_000007,True,False,8,0f80a6d7,2024-06-26_01-02-46,349.7124717235565,2827.936847448349,2850562,r8i6n2,10.159.28.60,1064.183210849762,3
+1967.2758471871923,1719356915,checkpoint_000008,True,False,9,0f80a6d7,2024-06-26_01-08-35,349.3587329387665,3177.2955803871155,2850562,r8i6n2,10.159.28.60,1413.5419437885284,4
+1959.91755063515,1719357265,checkpoint_000009,True,True,10,0f80a6d7,2024-06-26_01-14-25,349.87603521347046,3527.171615600586,2850562,r8i6n2,10.159.28.60,1763.417979001999,5
diff --git a/ray_results/test_experiment/train_model_0f80a6d7_31_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_22-45-39/result.json b/ray_results/test_experiment/train_model_0f80a6d7_31_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_22-45-39/result.json
new file mode 100644
index 0000000..5dc123f
--- /dev/null
+++ b/ray_results/test_experiment/train_model_0f80a6d7_31_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_22-45-39/result.json
@@ -0,0 +1,10 @@
+{"loss": 1986.2198524775467, "timestamp": 1719348983, "checkpoint_dir_name": "checkpoint_000000", "should_checkpoint": true, "done": false, "training_iteration": 1, "trial_id": "0f80a6d7", "date": "2024-06-25_22-56-24", "time_this_iter_s": 363.18635082244873, "time_total_s": 363.18635082244873, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 2, "decoder_rt_num_layer": 2, "decoder_int_num_layer": 1, "embedding_dim": 1024, "encoder_ff": 1024, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 16, "drop_rate": 0.023920599186226577, "lr": 0.0024687399238039302, "batch_size": 2048}, "time_since_restore": 363.18635082244873, "iterations_since_restore": 1}
+{"loss": 1979.5493135226993, "timestamp": 1719349334, "checkpoint_dir_name": "checkpoint_000001", "should_checkpoint": true, "done": false, "training_iteration": 2, "trial_id": "0f80a6d7", "date": "2024-06-25_23-02-14", "time_this_iter_s": 350.295019865036, "time_total_s": 713.4813706874847, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 2, "decoder_rt_num_layer": 2, "decoder_int_num_layer": 1, "embedding_dim": 1024, "encoder_ff": 1024, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 16, "drop_rate": 0.023920599186226577, "lr": 0.0024687399238039302, "batch_size": 2048}, "time_since_restore": 713.4813706874847, "iterations_since_restore": 2}
+{"loss": 1964.9348182978592, "timestamp": 1719349683, "checkpoint_dir_name": "checkpoint_000002", "should_checkpoint": true, "done": false, "training_iteration": 3, "trial_id": "0f80a6d7", "date": "2024-06-25_23-08-04", "time_this_iter_s": 349.6246259212494, "time_total_s": 1063.1059966087341, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 2, "decoder_rt_num_layer": 2, "decoder_int_num_layer": 1, "embedding_dim": 1024, "encoder_ff": 1024, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 16, "drop_rate": 0.023920599186226577, "lr": 0.0024687399238039302, "batch_size": 2048}, "time_since_restore": 1063.1059966087341, "iterations_since_restore": 3}
+{"loss": 1955.6751497523992, "timestamp": 1719350034, "checkpoint_dir_name": "checkpoint_000003", "should_checkpoint": true, "done": false, "training_iteration": 4, "trial_id": "0f80a6d7", "date": "2024-06-25_23-13-54", "time_this_iter_s": 350.46285796165466, "time_total_s": 1413.5688545703888, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 2, "decoder_rt_num_layer": 2, "decoder_int_num_layer": 1, "embedding_dim": 1024, "encoder_ff": 1024, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 16, "drop_rate": 0.023920599186226577, "lr": 0.0024687399238039302, "batch_size": 2048}, "time_since_restore": 1413.5688545703888, "iterations_since_restore": 4}
+{"loss": 1956.1783514548474, "timestamp": 1719350384, "checkpoint_dir_name": "checkpoint_000004", "should_checkpoint": true, "done": false, "training_iteration": 5, "trial_id": "0f80a6d7", "date": "2024-06-25_23-19-44", "time_this_iter_s": 350.18478202819824, "time_total_s": 1763.753636598587, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 2, "decoder_rt_num_layer": 2, "decoder_int_num_layer": 1, "embedding_dim": 1024, "encoder_ff": 1024, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 16, "drop_rate": 0.023920599186226577, "lr": 0.0024687399238039302, "batch_size": 2048}, "time_since_restore": 1763.753636598587, "iterations_since_restore": 5}
+{"loss": 1971.8988527312993, "timestamp": 1719355866, "checkpoint_dir_name": "checkpoint_000005", "should_checkpoint": true, "done": false, "training_iteration": 6, "trial_id": "0f80a6d7", "date": "2024-06-26_00-51-07", "time_this_iter_s": 365.2865264415741, "time_total_s": 2129.040163040161, "pid": 2850562, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 2, "decoder_rt_num_layer": 2, "decoder_int_num_layer": 1, "embedding_dim": 1024, "encoder_ff": 1024, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 16, "drop_rate": 0.023920599186226577, "lr": 0.0024687399238039302, "batch_size": 2048}, "time_since_restore": 365.2865264415741, "iterations_since_restore": 1}
+{"loss": 1955.6729476808564, "timestamp": 1719356216, "checkpoint_dir_name": "checkpoint_000006", "should_checkpoint": true, "done": false, "training_iteration": 7, "trial_id": "0f80a6d7", "date": "2024-06-26_00-56-56", "time_this_iter_s": 349.18421268463135, "time_total_s": 2478.2243757247925, "pid": 2850562, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 2, "decoder_rt_num_layer": 2, "decoder_int_num_layer": 1, "embedding_dim": 1024, "encoder_ff": 1024, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 16, "drop_rate": 0.023920599186226577, "lr": 0.0024687399238039302, "batch_size": 2048}, "time_since_restore": 714.4707391262054, "iterations_since_restore": 2}
+{"loss": 1957.4972913847196, "timestamp": 1719356565, "checkpoint_dir_name": "checkpoint_000007", "should_checkpoint": true, "done": false, "training_iteration": 8, "trial_id": "0f80a6d7", "date": "2024-06-26_01-02-46", "time_this_iter_s": 349.7124717235565, "time_total_s": 2827.936847448349, "pid": 2850562, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 2, "decoder_rt_num_layer": 2, "decoder_int_num_layer": 1, "embedding_dim": 1024, "encoder_ff": 1024, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 16, "drop_rate": 0.023920599186226577, "lr": 0.0024687399238039302, "batch_size": 2048}, "time_since_restore": 1064.183210849762, "iterations_since_restore": 3}
+{"loss": 1967.2758471871923, "timestamp": 1719356915, "checkpoint_dir_name": "checkpoint_000008", "should_checkpoint": true, "done": false, "training_iteration": 9, "trial_id": "0f80a6d7", "date": "2024-06-26_01-08-35", "time_this_iter_s": 349.3587329387665, "time_total_s": 3177.2955803871155, "pid": 2850562, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 2, "decoder_rt_num_layer": 2, "decoder_int_num_layer": 1, "embedding_dim": 1024, "encoder_ff": 1024, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 16, "drop_rate": 0.023920599186226577, "lr": 0.0024687399238039302, "batch_size": 2048}, "time_since_restore": 1413.5419437885284, "iterations_since_restore": 4}
+{"loss": 1959.91755063515, "timestamp": 1719357265, "checkpoint_dir_name": "checkpoint_000009", "should_checkpoint": true, "done": true, "training_iteration": 10, "trial_id": "0f80a6d7", "date": "2024-06-26_01-14-25", "time_this_iter_s": 349.87603521347046, "time_total_s": 3527.171615600586, "pid": 2850562, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 2, "decoder_rt_num_layer": 2, "decoder_int_num_layer": 1, "embedding_dim": 1024, "encoder_ff": 1024, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 16, "drop_rate": 0.023920599186226577, "lr": 0.0024687399238039302, "batch_size": 2048}, "time_since_restore": 1763.417979001999, "iterations_since_restore": 5}
diff --git a/ray_results/test_experiment/train_model_0f9cc97d_22_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_18-53-00/events.out.tfevents.1719334649.r8i6n2 b/ray_results/test_experiment/train_model_0f9cc97d_22_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_18-53-00/events.out.tfevents.1719334649.r8i6n2
new file mode 100644
index 0000000000000000000000000000000000000000..dc0dea9a1381339a860c4465b489b19537d51c90
GIT binary patch
literal 770
zcmb1OfPlsI-b$RVr#<)Wop;kwiZ`h!F*8rkwJbHS#L6iBYnPugRNaR8<<6WdZyP)8
z1F92X)Dq(o;o>bytkf?l%}dqK$uBMzb#3_K(&Kdgk^xl9)UOxfIddMuwJ38ba!J9p
z6ldg@=A^_YXQU=)7vyK=m54GhG}wPEye5ER<)z1P{is%^<maUVwJ<QG2ywLQL#@n>
zl?~v`d;-^^#HGL`3AeH&GdDH9BqOspKC>jXD85+q{7Xicl%uAshEVNuo10&9zC>tO
z;Zov~M%7-NnU|azUzA#0l3$bxHh^)$=5}+a0ncBYisX!Y3b$X2OM^=hZUD&E#FEVX
zyyAFNV?aKimE83a#pk^X*Y5)dHWRkM4)XlwQncsWSCqhBun4Y&n83c|5Xy;SrP^Y+
zejI__x$*ONl)zqr&_W=vPk;O4+<Kx#6(z9cmcVT!7}&Fo8eF<AOYTMq?0$p+L<cs5
J!|$V+y8wXS0^$Gw
literal 0
HcmV?d00001
diff --git a/ray_results/test_experiment/train_model_0f9cc97d_22_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_18-53-00/events.out.tfevents.1719338738.r8i6n2 b/ray_results/test_experiment/train_model_0f9cc97d_22_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_18-53-00/events.out.tfevents.1719338738.r8i6n2
new file mode 100644
index 0000000000000000000000000000000000000000..dce79638b4556a1330da3fa9372c75e4df06ad7f
GIT binary patch
literal 770
zcmb1OfPlsI-b$Pf%Qx*gIPa#T6mL>dVrHJ6YguYuiItIC<MUt2P<0#TmpgMFa&@vl
z1XL%$tR==J!o^#ZSgBu9nwP4dlV4mc+Tr`f#e0TAsR2~W)UOxfIs3ffT9mmIxuoD)
ziZk*{b5i1yGg6bY3-UAbN<<kL8tgX+S^Y+_@|Q1MKdO}}`FW{8Ees6i|ITsiL#@n>
zl?~ur;Rn~E#HGL`3AeH&GdDH9BqOspKC>jXD85)!p`Xb`v4DZY5UPD{bMs411An-7
z6)q(%X;kgSnR&^n@kOb{CHY0EU;|XYZ+K`9HQ@P+Q<0qO5C&**X>cjR4FK7iSdy8a
zR~(OO49MqA`x!D&e169J^j>gavtSGC>*n8Fx|h7=M+s~mU$_=x0^7S`6{?l95c+Wh
z_Ua={0Vsj3<OjEsKw#?${&mhU<av$~*b5Na2?q9^?gp1>{cZD60$bW2Za>k1&EU}d
H=WZ(iF|+xC
literal 0
HcmV?d00001
diff --git a/ray_results/test_experiment/train_model_0f9cc97d_22_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_18-53-00/events.out.tfevents.1719340670.r8i6n2 b/ray_results/test_experiment/train_model_0f9cc97d_22_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_18-53-00/events.out.tfevents.1719340670.r8i6n2
new file mode 100644
index 0000000000000000000000000000000000000000..f808e7d784a188e9088b8ff8d17f1f6ebef71267
GIT binary patch
literal 4596
zcma*rdr%Wc90%|i6v+w-kq`;5kbu=VW2B>XL@U8uW-4H7tJPs_IYV-oRC0$&K(W-)
zVI1xF!d9ymL97h6z8_PuV+CuAQmob5YKscm>iEF6;25xWg7y-^9zU9R$A8@JcfZ-s
z?H}2U67~M=dOuSoskv6S<J+p1bVXRM-Dsu98;U5}nff9+epsva%{#PXsa|4{&sBZL
z&I=!+i9(Sm%x)~!Ityt^o6k5LvWGp72wV5a<O#mW=B|sRq#+JR)F=@taKvF@3iHjR
z$wHYH7cf@ZDHDlG(-vmtjR%#^M4X3}W`?HNh)C34B!8;)RodQ;n=MID;fM+;Q8cb}
zT5S~Rv{)Ua)k)b&hwNU3n3y{+v?RqBuPiIOFR9_;Nl1laVBBG)O%!RT98Shgc^7zj
z-@EmbeGA;Z*PJa$QsaIN8iNw?0-jc*)5_2e5-#J}d1_VbX0Y@75AIZZv4j0#zaH-*
zMxS2S39#q?g(E_+SAN_y9aKKx@&T})eo!g_*u{V2N*?S5Nj>`I3y%K_u)DZ8FZQr4
zrNqp-srvyo(}DX1V~cdn^G`klJAW_Unb7Dt*+czdcf92ywyVcY1K6J@;D``x;&Iay
zP&rYF^8v6UOMX-X>~FXT4|ZL2kN(Z}YcBxoX^FUz7rVH%f!OuW+N%Kj7`K36Y@P1H
zrQc70o!6#!RC=*P{9!N2aS^8msmlQN#u+#w1bgO+^AAC#RFCrkupb{?EC<*XT!aVv
zl`gmbmZ>EfV2>hjBQJKd;u%qY+dUXyui_REj9sd87wI%$=b;%3V$XO^w!|N{sl-K0
z{$#{KfITS_M}%O@lR_qe%6cvz0DEgj<2Znwm4z#Lu-mS<^=XI4Dgkyg7w5%JACq7>
z(fRcQfSoZ5_Y20Z)|n=*83}gIoW1y|7aRG*{zdB|hODZr0N7iba6}09vh2!DpfdJ*
zoDYC4vo)Lq*tJ}Q2iwx_)@NmGZv)t)n{gvA_WH)@hPhb>jsWaxZUMpAr*sv2;<CZc
zSFh7WpY@#VVg9f$D_ul0Q!^i6Z(oliLa^`Zk9L8|(Hn3+0QSi-M_U2*9xlRzeYMT4
zk6Lpq1!9-uMqX^IF~=~kXBz{scXA5|#=fbuX|E@Ooqu$pZHpH>)E_qb&_x_rGy68c
zKJycf2*J*NX=*vB%(;Z~0kHMN(qw>rhl}uF@1D}5H{F`i0I(PSj2n5et3p;7HZ`4z
z0oa$g1q5S@(nE8*Yr)Q|t}Kpf@|^52f7mHYT*QP$;pYMN7r)|&5bWS5U+o2zOMb)o
z0N8~&@*@Ddi;M7JTiI_TnteCT0K1?CH}YcNxxC(xYVTA4?0eh-g0aKXTk6lr!OnMQ
zxi@*Shx@}W+~6X**G6mw*!ySUh!E_(;WGHue#)yj9{~GY=A-cdyPk{iU}q|O^hjBI
z5MWPw4L9;)x9_ep%pUvY9)P`{TR<?jGW`dJ$Uiiw_i|KTx$Pav#OxuPJ=mI>Aha_G
zMUs{Rqupq8kXmY40cB?|XtXn9br4z^gogTtq))`sZn0`OLb*n#$--WpmQ%_gl@tv(
zQzpjDUgX%zBbk@SK8r@O{?F(_8<}q`rtIuc1PbdLvioL_LXrI*^`9JuBxXBPK(e<r
zPZgV`Y`K)#Y^4{GW~+^T#C{rz?rrxr%TPq$|NR#VM&f)=J3_%UX`zf}WsoKUjgZ2d
zvnC$JN}qEx9uV=|d^K?>MjH5Lt4Tm{(&yb=HOWXV<-eJp?i7{LzV~;!<Fi!B^1k=?
zhZx-uqgd|!IVgvNz1orvr;!D=(s@iqy%?PkbMNt>Sg}uxqWalNB@v~qdFSSo7p0+*
zeMIcT-UHK_h7<$d8Xhavp?E*!csyK(Vg?=%JjjL<{l9xWMWjO`JnxQuP_p*A`9{$k
z?}q`$08cL0A$kAvFkpZ?ws@dg$NsY5|E%IUooTYhTYpaT{j$(hx9bf_d7pj?(jvwG
Y?J3&aE6<)W&V8^-95QH8+TrB?0Qr&CrvLx|
literal 0
HcmV?d00001
diff --git a/ray_results/test_experiment/train_model_0f9cc97d_22_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_18-53-00/params.json b/ray_results/test_experiment/train_model_0f9cc97d_22_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_18-53-00/params.json
new file mode 100644
index 0000000..3eb44ac
--- /dev/null
+++ b/ray_results/test_experiment/train_model_0f9cc97d_22_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_18-53-00/params.json
@@ -0,0 +1,13 @@
+{
+ "batch_size": 2048,
+ "decoder_int_ff": 512,
+ "decoder_int_num_layer": 1,
+ "decoder_rt_ff": 512,
+ "decoder_rt_num_layer": 2,
+ "drop_rate": 0.005508735010543275,
+ "embedding_dim": 1024,
+ "encoder_ff": 1024,
+ "encoder_num_layer": 4,
+ "lr": 0.0011880094129501908,
+ "n_head": 1
+}
\ No newline at end of file
diff --git a/ray_results/test_experiment/train_model_0f9cc97d_22_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_18-53-00/params.pkl b/ray_results/test_experiment/train_model_0f9cc97d_22_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_18-53-00/params.pkl
new file mode 100644
index 0000000000000000000000000000000000000000..0b14b147032aaa52de8463ec6daaf91430fbcb3f
GIT binary patch
literal 225
zcmZo*oqCM{0&1sd^a!TrCFiH47RBe4=EmnFR;CtB@n-1}NlAr>7L_1NF!hMSBr@|*
zWEgvRQ*)D2Q&KYX(&JMybEo(+u=H@j%uY)K^Lb&Kfu@3mn0okNLLl>?Vr+Tw8L5dW
zQ@j~_I8%!93*w6sOH!w}+m}rUKAm>Ap@%7_2*?XAyPtBWr>2K1DX}CuBfdDZDivr6
HN2wkF>qk?m
literal 0
HcmV?d00001
diff --git a/ray_results/test_experiment/train_model_0f9cc97d_22_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_18-53-00/progress.csv b/ray_results/test_experiment/train_model_0f9cc97d_22_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_18-53-00/progress.csv
new file mode 100644
index 0000000..56220b4
--- /dev/null
+++ b/ray_results/test_experiment/train_model_0f9cc97d_22_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_18-53-00/progress.csv
@@ -0,0 +1,15 @@
+loss,timestamp,checkpoint_dir_name,should_checkpoint,done,training_iteration,trial_id,date,time_this_iter_s,time_total_s,pid,hostname,node_ip,time_since_restore,iterations_since_restore
+1956.0084007443406,1719335168,checkpoint_000000,True,False,1,0f9cc97d,2024-06-25_19-06-08,519.6532871723175,519.6532871723175,2195253,r8i6n2,10.159.28.60,519.6532871723175,1
+1970.2912184347317,1719335674,checkpoint_000001,True,False,2,0f9cc97d,2024-06-25_19-14-34,505.92808413505554,1025.581371307373,2195253,r8i6n2,10.159.28.60,1025.581371307373,2
+1954.4228304164617,1719339260,checkpoint_000002,True,False,3,0f9cc97d,2024-06-25_20-14-21,522.2363362312317,1547.8177075386047,2195253,r8i6n2,10.159.28.60,522.2363362312317,1
+1969.7449739711492,1719339766,checkpoint_000003,True,False,4,0f9cc97d,2024-06-25_20-22-47,506.13422203063965,2053.9519295692444,2195253,r8i6n2,10.159.28.60,1028.3705582618713,2
+1951.9356997031866,1719341192,checkpoint_000004,True,False,5,0f9cc97d,2024-06-25_20-46-33,522.4360599517822,2576.3879895210266,2195253,r8i6n2,10.159.28.60,522.4360599517822,1
+1952.9474155696357,1719341699,checkpoint_000005,True,False,6,0f9cc97d,2024-06-25_20-54-59,506.28402161598206,3082.6720111370087,2195253,r8i6n2,10.159.28.60,1028.7200815677643,2
+1954.9041440468134,1719342205,checkpoint_000006,True,False,7,0f9cc97d,2024-06-25_21-03-25,506.20089197158813,3588.872903108597,2195253,r8i6n2,10.159.28.60,1534.9209735393524,3
+1954.9512458861343,1719342711,checkpoint_000007,True,False,8,0f9cc97d,2024-06-25_21-11-51,505.91624569892883,4094.7891488075256,2195253,r8i6n2,10.159.28.60,2040.8372192382812,4
+1955.8870253675566,1719343217,checkpoint_000008,True,False,9,0f9cc97d,2024-06-25_21-20-17,505.6786599159241,4600.46780872345,2195253,r8i6n2,10.159.28.60,2546.5158791542053,5
+1953.494582769439,1719343722,checkpoint_000009,True,False,10,0f9cc97d,2024-06-25_21-28-43,505.8313031196594,5106.299111843109,2195253,r8i6n2,10.159.28.60,3052.3471822738647,6
+1953.025947150283,1719344228,checkpoint_000010,True,False,11,0f9cc97d,2024-06-25_21-37-09,505.74103355407715,5612.040145397186,2195253,r8i6n2,10.159.28.60,3558.088215827942,7
+1956.2824668583908,1719344735,checkpoint_000011,True,False,12,0f9cc97d,2024-06-25_21-45-35,506.4504108428955,6118.490556240082,2195253,r8i6n2,10.159.28.60,4064.5386266708374,8
+1955.5371737742987,1719345241,checkpoint_000012,True,False,13,0f9cc97d,2024-06-25_21-54-01,506.19851207733154,6624.689068317413,2195253,r8i6n2,10.159.28.60,4570.737138748169,9
+1956.4829832062007,1719345747,checkpoint_000013,True,False,14,0f9cc97d,2024-06-25_22-02-27,506.25228786468506,7130.941356182098,2195253,r8i6n2,10.159.28.60,5076.989426612854,10
diff --git a/ray_results/test_experiment/train_model_0f9cc97d_22_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_18-53-00/result.json b/ray_results/test_experiment/train_model_0f9cc97d_22_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_18-53-00/result.json
new file mode 100644
index 0000000..a0b4919
--- /dev/null
+++ b/ray_results/test_experiment/train_model_0f9cc97d_22_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_18-53-00/result.json
@@ -0,0 +1,14 @@
+{"loss": 1956.0084007443406, "timestamp": 1719335168, "checkpoint_dir_name": "checkpoint_000000", "should_checkpoint": true, "done": false, "training_iteration": 1, "trial_id": "0f9cc97d", "date": "2024-06-25_19-06-08", "time_this_iter_s": 519.6532871723175, "time_total_s": 519.6532871723175, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 4, "decoder_rt_num_layer": 2, "decoder_int_num_layer": 1, "embedding_dim": 1024, "encoder_ff": 1024, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 1, "drop_rate": 0.005508735010543275, "lr": 0.0011880094129501908, "batch_size": 2048}, "time_since_restore": 519.6532871723175, "iterations_since_restore": 1}
+{"loss": 1970.2912184347317, "timestamp": 1719335674, "checkpoint_dir_name": "checkpoint_000001", "should_checkpoint": true, "done": false, "training_iteration": 2, "trial_id": "0f9cc97d", "date": "2024-06-25_19-14-34", "time_this_iter_s": 505.92808413505554, "time_total_s": 1025.581371307373, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 4, "decoder_rt_num_layer": 2, "decoder_int_num_layer": 1, "embedding_dim": 1024, "encoder_ff": 1024, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 1, "drop_rate": 0.005508735010543275, "lr": 0.0011880094129501908, "batch_size": 2048}, "time_since_restore": 1025.581371307373, "iterations_since_restore": 2}
+{"loss": 1954.4228304164617, "timestamp": 1719339260, "checkpoint_dir_name": "checkpoint_000002", "should_checkpoint": true, "done": false, "training_iteration": 3, "trial_id": "0f9cc97d", "date": "2024-06-25_20-14-21", "time_this_iter_s": 522.2363362312317, "time_total_s": 1547.8177075386047, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 4, "decoder_rt_num_layer": 2, "decoder_int_num_layer": 1, "embedding_dim": 1024, "encoder_ff": 1024, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 1, "drop_rate": 0.005508735010543275, "lr": 0.0011880094129501908, "batch_size": 2048}, "time_since_restore": 522.2363362312317, "iterations_since_restore": 1}
+{"loss": 1969.7449739711492, "timestamp": 1719339766, "checkpoint_dir_name": "checkpoint_000003", "should_checkpoint": true, "done": false, "training_iteration": 4, "trial_id": "0f9cc97d", "date": "2024-06-25_20-22-47", "time_this_iter_s": 506.13422203063965, "time_total_s": 2053.9519295692444, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 4, "decoder_rt_num_layer": 2, "decoder_int_num_layer": 1, "embedding_dim": 1024, "encoder_ff": 1024, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 1, "drop_rate": 0.005508735010543275, "lr": 0.0011880094129501908, "batch_size": 2048}, "time_since_restore": 1028.3705582618713, "iterations_since_restore": 2}
+{"loss": 1951.9356997031866, "timestamp": 1719341192, "checkpoint_dir_name": "checkpoint_000004", "should_checkpoint": true, "done": false, "training_iteration": 5, "trial_id": "0f9cc97d", "date": "2024-06-25_20-46-33", "time_this_iter_s": 522.4360599517822, "time_total_s": 2576.3879895210266, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 4, "decoder_rt_num_layer": 2, "decoder_int_num_layer": 1, "embedding_dim": 1024, "encoder_ff": 1024, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 1, "drop_rate": 0.005508735010543275, "lr": 0.0011880094129501908, "batch_size": 2048}, "time_since_restore": 522.4360599517822, "iterations_since_restore": 1}
+{"loss": 1952.9474155696357, "timestamp": 1719341699, "checkpoint_dir_name": "checkpoint_000005", "should_checkpoint": true, "done": false, "training_iteration": 6, "trial_id": "0f9cc97d", "date": "2024-06-25_20-54-59", "time_this_iter_s": 506.28402161598206, "time_total_s": 3082.6720111370087, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 4, "decoder_rt_num_layer": 2, "decoder_int_num_layer": 1, "embedding_dim": 1024, "encoder_ff": 1024, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 1, "drop_rate": 0.005508735010543275, "lr": 0.0011880094129501908, "batch_size": 2048}, "time_since_restore": 1028.7200815677643, "iterations_since_restore": 2}
+{"loss": 1954.9041440468134, "timestamp": 1719342205, "checkpoint_dir_name": "checkpoint_000006", "should_checkpoint": true, "done": false, "training_iteration": 7, "trial_id": "0f9cc97d", "date": "2024-06-25_21-03-25", "time_this_iter_s": 506.20089197158813, "time_total_s": 3588.872903108597, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 4, "decoder_rt_num_layer": 2, "decoder_int_num_layer": 1, "embedding_dim": 1024, "encoder_ff": 1024, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 1, "drop_rate": 0.005508735010543275, "lr": 0.0011880094129501908, "batch_size": 2048}, "time_since_restore": 1534.9209735393524, "iterations_since_restore": 3}
+{"loss": 1954.9512458861343, "timestamp": 1719342711, "checkpoint_dir_name": "checkpoint_000007", "should_checkpoint": true, "done": false, "training_iteration": 8, "trial_id": "0f9cc97d", "date": "2024-06-25_21-11-51", "time_this_iter_s": 505.91624569892883, "time_total_s": 4094.7891488075256, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 4, "decoder_rt_num_layer": 2, "decoder_int_num_layer": 1, "embedding_dim": 1024, "encoder_ff": 1024, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 1, "drop_rate": 0.005508735010543275, "lr": 0.0011880094129501908, "batch_size": 2048}, "time_since_restore": 2040.8372192382812, "iterations_since_restore": 4}
+{"loss": 1955.8870253675566, "timestamp": 1719343217, "checkpoint_dir_name": "checkpoint_000008", "should_checkpoint": true, "done": false, "training_iteration": 9, "trial_id": "0f9cc97d", "date": "2024-06-25_21-20-17", "time_this_iter_s": 505.6786599159241, "time_total_s": 4600.46780872345, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 4, "decoder_rt_num_layer": 2, "decoder_int_num_layer": 1, "embedding_dim": 1024, "encoder_ff": 1024, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 1, "drop_rate": 0.005508735010543275, "lr": 0.0011880094129501908, "batch_size": 2048}, "time_since_restore": 2546.5158791542053, "iterations_since_restore": 5}
+{"loss": 1953.494582769439, "timestamp": 1719343722, "checkpoint_dir_name": "checkpoint_000009", "should_checkpoint": true, "done": false, "training_iteration": 10, "trial_id": "0f9cc97d", "date": "2024-06-25_21-28-43", "time_this_iter_s": 505.8313031196594, "time_total_s": 5106.299111843109, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 4, "decoder_rt_num_layer": 2, "decoder_int_num_layer": 1, "embedding_dim": 1024, "encoder_ff": 1024, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 1, "drop_rate": 0.005508735010543275, "lr": 0.0011880094129501908, "batch_size": 2048}, "time_since_restore": 3052.3471822738647, "iterations_since_restore": 6}
+{"loss": 1953.025947150283, "timestamp": 1719344228, "checkpoint_dir_name": "checkpoint_000010", "should_checkpoint": true, "done": false, "training_iteration": 11, "trial_id": "0f9cc97d", "date": "2024-06-25_21-37-09", "time_this_iter_s": 505.74103355407715, "time_total_s": 5612.040145397186, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 4, "decoder_rt_num_layer": 2, "decoder_int_num_layer": 1, "embedding_dim": 1024, "encoder_ff": 1024, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 1, "drop_rate": 0.005508735010543275, "lr": 0.0011880094129501908, "batch_size": 2048}, "time_since_restore": 3558.088215827942, "iterations_since_restore": 7}
+{"loss": 1956.2824668583908, "timestamp": 1719344735, "checkpoint_dir_name": "checkpoint_000011", "should_checkpoint": true, "done": false, "training_iteration": 12, "trial_id": "0f9cc97d", "date": "2024-06-25_21-45-35", "time_this_iter_s": 506.4504108428955, "time_total_s": 6118.490556240082, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 4, "decoder_rt_num_layer": 2, "decoder_int_num_layer": 1, "embedding_dim": 1024, "encoder_ff": 1024, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 1, "drop_rate": 0.005508735010543275, "lr": 0.0011880094129501908, "batch_size": 2048}, "time_since_restore": 4064.5386266708374, "iterations_since_restore": 8}
+{"loss": 1955.5371737742987, "timestamp": 1719345241, "checkpoint_dir_name": "checkpoint_000012", "should_checkpoint": true, "done": false, "training_iteration": 13, "trial_id": "0f9cc97d", "date": "2024-06-25_21-54-01", "time_this_iter_s": 506.19851207733154, "time_total_s": 6624.689068317413, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 4, "decoder_rt_num_layer": 2, "decoder_int_num_layer": 1, "embedding_dim": 1024, "encoder_ff": 1024, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 1, "drop_rate": 0.005508735010543275, "lr": 0.0011880094129501908, "batch_size": 2048}, "time_since_restore": 4570.737138748169, "iterations_since_restore": 9}
+{"loss": 1956.4829832062007, "timestamp": 1719345747, "checkpoint_dir_name": "checkpoint_000013", "should_checkpoint": true, "done": false, "training_iteration": 14, "trial_id": "0f9cc97d", "date": "2024-06-25_22-02-27", "time_this_iter_s": 506.25228786468506, "time_total_s": 7130.941356182098, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 4, "decoder_rt_num_layer": 2, "decoder_int_num_layer": 1, "embedding_dim": 1024, "encoder_ff": 1024, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 1, "drop_rate": 0.005508735010543275, "lr": 0.0011880094129501908, "batch_size": 2048}, "time_since_restore": 5076.989426612854, "iterations_since_restore": 10}
diff --git a/ray_results/test_experiment/train_model_110f0a27_14_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop__2024-06-25_17-41-39/events.out.tfevents.1719330179.r8i6n2 b/ray_results/test_experiment/train_model_110f0a27_14_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop__2024-06-25_17-41-39/events.out.tfevents.1719330179.r8i6n2
new file mode 100644
index 0000000000000000000000000000000000000000..b82acd39abc29482c82924ea3f47329be3374488
GIT binary patch
literal 405
zcmb1OfPlsI-b$Qt%T6ckoOjbviZ`h!F*8rkwJbHS#LDRDRZnwesJadF%bhvvSC!R(
z)Cn+ZiE)W=@fIak>X($}rRwM87Z;0uYyILf$?wck1E`j%UoXUSvTlQGQRY(Ql7ed~
z&d4v#Nr_L+NKMWz$j{6x5oKU#uot|UYm8#0`3|^#R4Y^R^HPCY7#Q?zT^8y?t;~&;
z4dDEQ(4xepz$FQ{vLrJ%HNGSxvp7DpB(*5MSTrp3t`k4wwP%J<?Q@%(Uvkda3Aa&&
zONmPwReNz}UUF)DQEG8Xeo-pefT=rO+0CH_Jb!U2l9P8A+yE^u4K78v0U%owOEUBG
PisMm@0r~va{{#jAu}g&v
literal 0
HcmV?d00001
diff --git a/ray_results/test_experiment/train_model_110f0a27_14_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop__2024-06-25_17-41-39/events.out.tfevents.1719330947.r8i6n2 b/ray_results/test_experiment/train_model_110f0a27_14_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop__2024-06-25_17-41-39/events.out.tfevents.1719330947.r8i6n2
new file mode 100644
index 0000000000000000000000000000000000000000..79a594ff1a06e34d9edd77d26a0c5e4c29d15082
GIT binary patch
literal 1311
zcmah|T}TvB7#(MAb3RygC)ad!ZJjNMA$Bz%0+Dw6d&nRp2r7unxN~hMcke7Svzb2B
zf=E4;FFjb8l|hh#zDUvEgAj!;7DNz4MGs{W)kB35QFnG{XJ<`q-sU^s`OdxPJGYL>
zf8(e33EcD9&%ryFU+nOEx;055;XNlXG1|Abom;#avihdS`gU>Jul^3RubLHWp$0sf
z)E_Z21V@spuKO}o-*)?-hDKT}%7yPw5ciQ8C{RMs41SxUC)7+*M)3rW_oh{a7(Rx{
zM#HCW43|j$DYyBhlCnxLr7(=U_t}|<C7C)J=;AJ}u_*!sq287>loUorLeY_8U=8WM
z+Xr9nJTf*>-D=TC2M0fJ0k=)x2m&;e(shNzG19PZs2a`<h;9kL+hz@Tzc6>0yHp$?
z!e(f;2bijop{PVhrDM#SPv4)Jm@doD86F)u&JDG_+HaaIUU$J&7u29cTGFJHjw1ME
z8f)|}#OOThf*}{IwIKc{%l}}7N`P+3h$pDZ01mnY9x7!VS7mx<>ND!;p|yJOIcp@7
zLP@C~YZUT=rvPbIcOBF?O3r2vaI&VR5xtlxMY%ZDjb&LO$B?X~Xo;Q%le~IP*$3Xj
zzjKfq*rcfr;3g=6B{}F4y->xMhD>aNMt+qLEuokp5d+Y`uN(q#0|fZhmMm_95dZ%r
zW*=5(iM7AkFW1_+xx)V5vhbRP#^U~Ttm`xdNH-*EutIv&&Ic^qWs7^9V*QgW)H=N7
zId67h`}yCKBT-mau!t7(foX|>f63NtY&Zr@%UEvXObi;92E+!lG5Tni-M!sH`Bmob
zXdzTP)wg%PD{3C5BnEbKItJ^Vd9Z;iIiMI@Cm^hlRlDbF^xjOxe24W2AKw~3z`ZHB
c-wF}%|LZ*_a^88Gv7+zg0$Wk`Xz|deznXTFK>z>%
literal 0
HcmV?d00001
diff --git a/ray_results/test_experiment/train_model_110f0a27_14_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop__2024-06-25_17-41-39/params.json b/ray_results/test_experiment/train_model_110f0a27_14_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop__2024-06-25_17-41-39/params.json
new file mode 100644
index 0000000..b2dd97e
--- /dev/null
+++ b/ray_results/test_experiment/train_model_110f0a27_14_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop__2024-06-25_17-41-39/params.json
@@ -0,0 +1,13 @@
+{
+ "batch_size": 2048,
+ "decoder_int_ff": 512,
+ "decoder_int_num_layer": 1,
+ "decoder_rt_ff": 1024,
+ "decoder_rt_num_layer": 2,
+ "drop_rate": 0.025073968761468723,
+ "embedding_dim": 256,
+ "encoder_ff": 2048,
+ "encoder_num_layer": 2,
+ "lr": 0.0010259835678295768,
+ "n_head": 16
+}
\ No newline at end of file
diff --git a/ray_results/test_experiment/train_model_110f0a27_14_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop__2024-06-25_17-41-39/params.pkl b/ray_results/test_experiment/train_model_110f0a27_14_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop__2024-06-25_17-41-39/params.pkl
new file mode 100644
index 0000000000000000000000000000000000000000..b400c94d2014abf046ea4082ed13be2b65b4c668
GIT binary patch
literal 225
zcmZo*oqCM{0&1sd^a!TrCFiH47RBe4=EmnFR;CtB@n-4~NlAr>7L}k%h{7Z?^H5|M
zdw5fGlTuSsGV{{oQ!;a>_%bl|aKX$@O9S#bdU#=)fu@3mSbF$iLLl?NVoW`3dGQ&k
zi78XO1$sDBit-EMixNvxr?}hCT=Tcz?nOfnQ%(_(7jWLQe7%@*4_8uRNpePfab{I2
I$dXb$0PH1GAOHXW
literal 0
HcmV?d00001
diff --git a/ray_results/test_experiment/train_model_110f0a27_14_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop__2024-06-25_17-41-39/progress.csv b/ray_results/test_experiment/train_model_110f0a27_14_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop__2024-06-25_17-41-39/progress.csv
new file mode 100644
index 0000000..a5487a4
--- /dev/null
+++ b/ray_results/test_experiment/train_model_110f0a27_14_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop__2024-06-25_17-41-39/progress.csv
@@ -0,0 +1,3 @@
+loss,timestamp,checkpoint_dir_name,should_checkpoint,done,training_iteration,trial_id,date,time_this_iter_s,time_total_s,pid,hostname,node_ip,time_since_restore,iterations_since_restore
+1956.1862581508367,1719330289,checkpoint_000000,True,False,1,110f0a27,2024-06-25_17-44-49,110.66666960716248,110.66666960716248,2195253,r8i6n2,10.159.28.60,110.66666960716248,1
+1952.48307740219,1719331057,checkpoint_000001,True,False,2,110f0a27,2024-06-25_17-57-37,110.16742134094238,220.83409094810486,2195253,r8i6n2,10.159.28.60,110.16742134094238,1
diff --git a/ray_results/test_experiment/train_model_110f0a27_14_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop__2024-06-25_17-41-39/result.json b/ray_results/test_experiment/train_model_110f0a27_14_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop__2024-06-25_17-41-39/result.json
new file mode 100644
index 0000000..32a0ab9
--- /dev/null
+++ b/ray_results/test_experiment/train_model_110f0a27_14_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop__2024-06-25_17-41-39/result.json
@@ -0,0 +1,2 @@
+{"loss": 1956.1862581508367, "timestamp": 1719330289, "checkpoint_dir_name": "checkpoint_000000", "should_checkpoint": true, "done": false, "training_iteration": 1, "trial_id": "110f0a27", "date": "2024-06-25_17-44-49", "time_this_iter_s": 110.66666960716248, "time_total_s": 110.66666960716248, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 2, "decoder_rt_num_layer": 2, "decoder_int_num_layer": 1, "embedding_dim": 256, "encoder_ff": 2048, "decoder_rt_ff": 1024, "decoder_int_ff": 512, "n_head": 16, "drop_rate": 0.025073968761468723, "lr": 0.0010259835678295768, "batch_size": 2048}, "time_since_restore": 110.66666960716248, "iterations_since_restore": 1}
+{"loss": 1952.48307740219, "timestamp": 1719331057, "checkpoint_dir_name": "checkpoint_000001", "should_checkpoint": true, "done": false, "training_iteration": 2, "trial_id": "110f0a27", "date": "2024-06-25_17-57-37", "time_this_iter_s": 110.16742134094238, "time_total_s": 220.83409094810486, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 2, "decoder_rt_num_layer": 2, "decoder_int_num_layer": 1, "embedding_dim": 256, "encoder_ff": 2048, "decoder_rt_ff": 1024, "decoder_int_ff": 512, "n_head": 16, "drop_rate": 0.025073968761468723, "lr": 0.0010259835678295768, "batch_size": 2048}, "time_since_restore": 110.16742134094238, "iterations_since_restore": 1}
diff --git a/ray_results/test_experiment/train_model_18c11cbc_27_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop__2024-06-25_19-39-03/events.out.tfevents.1719346234.r8i6n2 b/ray_results/test_experiment/train_model_18c11cbc_27_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop__2024-06-25_19-39-03/events.out.tfevents.1719346234.r8i6n2
new file mode 100644
index 0000000000000000000000000000000000000000..5f06f7f639bc2052375f3401b6e605202feaf7d7
GIT binary patch
literal 1865
zcma*jZ%7ky9LMpql-SXhL@S#qO=p?^E@KsmwA-DEVM!rDGwWFMU{~ASc6VDy1BW9@
z=s|?`#5A>#nuDaQ1PK!<vVZu*2S#8%%Ag>Th?PdU#_#yT#@#)>_xHo^m4p5N?zA@)
z=&GJY4by#dc9Z^`=;Zi{+91ozRr@OAW5tEZ|3+?Kq;*3Ztmd`<vNKAy;#+XN=xih8
z0M8O0L6VHYlb7l4??T(&WYYa(hSBwJL`flT#!V=Z+(N+9%DCLD>w-_<c-e?yVVmKa
z{t}pcU_d>bY!!I+UxHzcgwIYSC%tF#j_b%xC@H{q;@i=r%z0Txc5@QL$*jmo#?rbj
zYIe~=R3`J@NaTxdLdh+70iFwUiQ`=?BeIe#h-~7(c+I5aVDiA{#n%qqfo!y2f*0dv
zbYN}ODRTlZG4RaV%S)DoKJaqOuN5MJy)G4Y%|{!(^rw9SU=zPlq6ORfc(oKvK2iEK
z*xQ1e>j8H43Yt{G_I7%x=%MBh0Q-@Wt77|Wv*=Uy78zh$R?)s@Y>X^FAIktQA1|9f
zoWNe63j1n(8EqQKy8*DL%2A>P`+elw9GI-DKz$nQ-M#Zsfc-{ERIsb_d1~zF!gYX6
z5ol2r8+%96yQGFufIX@l(2PxzM{IB7;N`~q)5j9nnW?a6J{8f!;n-<_of}1o7VIW^
zPy~~MN}mRMKQ(X`U^fn<Nfqq;L7r;zFWm&#Q%bIieI*d0Gxy{a0c>jw?Q6yklh0rB
z_rS}S`Pr5Pc2+9v=75o2F6?mt?6`muE!gdop|4;v<U@TL?9QQ%7XTacqeKPU|I|k<
jbo5vN_FW}c#eR6=7X7NLxCdaHM6|CNyPsU$@pI%a1;bs5
literal 0
HcmV?d00001
diff --git a/ray_results/test_experiment/train_model_18c11cbc_27_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop__2024-06-25_19-39-03/events.out.tfevents.1719354644.r8i6n2 b/ray_results/test_experiment/train_model_18c11cbc_27_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop__2024-06-25_19-39-03/events.out.tfevents.1719354644.r8i6n2
new file mode 100644
index 0000000000000000000000000000000000000000..aada72158deb33f4f2ccf3c71d50f792587e0963
GIT binary patch
literal 1865
zcmb1OfPlsI-b$R>PgGnl&b#R-#hX-=n3<>NT9%quVrAs$rq`$pRkvY&xihC+eoHt=
zodBDb7?%haZ&6~Ueo1Lws(wy>aj~e@IX>rig3@LNP%TrxUWn(+FNJGS=2GO6f@>+x
z$S=)FiBHZ*P0lXJ&&(?kWngHq{}<e(gJNY{Ib1)gl_~josX#3Z49blkgY}_S=Elkf
zaLQD`wJ32Za7n_gEXmAGjW5Z_ERN4CNiB*m7R~rN$!Yp!Cp$x^_PNc?FFEHRw5xC_
zaY>_UFV4(MPK_^0EiTC~N(CDbw#xU5In;pXFHS{r%2vYd*W%LPQiK}-vNf?JGe565
z9@Q9-&vgR}*P-~l<(qICII!8V1$O>CVdt+3W*R7g-Si!<g_yvelF?OxVkOrPxPBag
zy;Dha8A@OuM`$4s*iSxYIBm{-FM|@;?myu+5)5qJ?aa;`!hF|I0{Z~M0HOn%!Qo8N
z!&N9g|5bgn1RU5L*aExeJ%{sDMJ*|m!2VqW*FsES`?KC#fnsG#EnGj2z^-iEZHyAw
z9CdIl1OnSCILGOl_L*rYfnA5tPB5_loVIh8daP%L64;*+1`r+C4i2k+1-YX5e4j>f
zGdQq0u?4n43ahh}-tV_4fxTZ7u7#MuE{$S6i(;jR7F<7$z&>@=J`5$WFCerK2yFSO
znND9-S8JmLwwE^CMuLIuY*^{M-{^KIN?;#A7(jGjH#pc7yQ7xb6T{>tfdiWhTVS95
z$mBftid-j3U>^^MYau4E`y%_*Q3Bg90<IrNU@zfavmYg}FCw%M2<+8svz)|_by%SU
jwofG7MuLGI{%)qT3E#$zD1m(xVF1yAy}&{I|I;V{3GQ3<
literal 0
HcmV?d00001
diff --git a/ray_results/test_experiment/train_model_18c11cbc_27_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop__2024-06-25_19-39-03/events.out.tfevents.1719357265.r8i6n2 b/ray_results/test_experiment/train_model_18c11cbc_27_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop__2024-06-25_19-39-03/events.out.tfevents.1719357265.r8i6n2
new file mode 100644
index 0000000000000000000000000000000000000000..cff8c2b4224ff135612f5376adb41d5505a365a6
GIT binary patch
literal 4596
zcma*reM}Q)90%}H4A4hGC@pFc2SqnlouF>&%uKIWP(ZV&afzbMt@I9CY_HgBH(@9$
zSvDQpm;*N;2+EeYMV!+SUoeDerfx8&8__wP80SkhIGOl@I_IvXJjc(qyW_v!^L(Fg
zpU*uZc{*A)@Yhr}PpfFYTiEr>*0TmpbiT`MV<%_5!LaUGQ;WA>Zb}Wz+uiV{QIY*c
z-BsVb*x|Yaq(adyb8))c!!qf1C&#H*N63t572A{%h*aM_NhxOS!jV*zf;2e7S)Cp`
zO<AmrWo41m#=6xq**d-FoOUs&tlEw9u#$GN3?Grnsx-U0(gT$aQ_=#(#IJEgi;~e8
zT<Nws7|Lz6ag@!?xF}AYvf(4rJN>P)DS>!>dHHokvk)JTv?vkAIU8$XC>O)Ioi4_|
zKwE2f+l;^h*StS0Rb=eJ{W_F}Qt$%4R<qmYWH|~h<J<Y<QZ^Cn{KQ<>T|f4SVA%C*
z<i@;Z9ghKaPd1K7!M4sWS^_Fp<=}h>Y}JN)mjHIZ5D~$ypdI8SW!^o2?a9TBqS(_k
z!;G`lX$=6oM_52Iwv4E$=rDnu&$Jnbb@@(qbTI7C$2}q~H<Pab?8SvRA_Y6AuB9DR
zo)z*Tu)Dgwn*sLgD{-X=_M~VB`9xJn0qnCvTok*^q&F5WXw3xJ^X<4_GB!bk86Lg^
zc7DEI6Ys~435I<tte<S1Kei2EKi-NXQm`v06wCpY<qbF=0z036u?%3xG~$Q|_ITDo
za^q7+0PJQVE{Yv>pv3qkdUhwkPTPk2C1bB6vg^rSu=5YwFQ@piM+U<_c>WIg`{<30
z0K4T|9Fc+@$wZz4m4?GO9|C*k@{cD0?C*t$2zE)mot*agyEOni^9XJf#h%t%Yb-g{
zvL0X`5*Cn*-AL>$YCZ#Yeu(MJ@MFgY!#@4TZE{BU^a6n0U4SD}u%FwMuo+ZRD{wvp
z_U1o-oe!|D2@w%&gNG$2mQLLXuqi8U6vaN*w%@4TxPLFe?h+P|jNMK=$~e9e?7TB}
zi{6hN7YzG@m-|WUhKRWUyDtt$q+sv(yZ0QZ<dirc0(<TI3pWAmCqhI7`*O9N)YR0t
z0Cq_{ZWP7-WZ`LJ#Hb@J0Q<hMfMo28L{Dw=ZLsr(Yo=HI*veqoZ83dhan`3f0K3DB
zBT}%p^fI?V<!je*J_Poq<*EjNeL;wbVCSv2lXKSARs!s0H*ljUcGJZ^<DQsh69D$l
z!UB@9WrjN9y^CPy?H$jX{Mhlqup{$tl1CR!xdX88bmE8<?7xz#WT3M2N1P9VecO};
z|7S;@z!4E_%K<xCeboC9V3!MVQEYE(ROXH04=n)uk+6Vd>{!FmeaGU#&a1NjS?R}C
z1;Y+&?je7gYApuX-{jzk6m0MM#aBV)^jw?|fqgZNX$2?ykPs2UK1n)A=7j_rV81jE
zH;Q7bD@SFP7BYPRdxx-qWb9-^-W>Ci-C+Zlqssc4LPdXMl--A|`z#!73r8x-T4Z*a
z9UPU;tS(|)`~{75=e`?`-VR5Rfg$Asx$>S|8-q~3*=@1%SEmvtIb5qmF*IXw()>k^
zzdTX}1^lxyNImo!>v2$ab1~!MhvHE5;E*dYdo)rFc{Fr#G*ZwmXA#BU)_hfbmT}}W
zG;L#7P_)g#KjOa)!w$6jo7E_8@bl1xB9PqfYey)8rL2sZP7c?_p^-{>bJmSRW0n7N
zGkz-KyZP#pP@*#Q%~tmeN>cvc%~dx6r7Fd5rk>xHK41gy@9?edvlLlTf%kVwj=JS&
ztnmIghU56FEycObJg|)|aOSqjQL9{dk6+0(JLM>0h^<l)S9@@QzP#>?9>ot5@ec<M
zOfrEqPrWrf)<~dnLCEoVEP)cAJ|K8-9l`$+5%k^TDZP3$()aH82N%MQRdiRF^u8|x
zjsc#WL7-7X&jTLdi}(TI*y4dog8yY9_*unsdiB*$?z681epxuauW6COG^jrXr6bLM
X?HQIHkmt{s;66+%j|yYeS33U%kx|h*
literal 0
HcmV?d00001
diff --git a/ray_results/test_experiment/train_model_18c11cbc_27_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop__2024-06-25_19-39-03/params.json b/ray_results/test_experiment/train_model_18c11cbc_27_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop__2024-06-25_19-39-03/params.json
new file mode 100644
index 0000000..99a97a2
--- /dev/null
+++ b/ray_results/test_experiment/train_model_18c11cbc_27_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop__2024-06-25_19-39-03/params.json
@@ -0,0 +1,13 @@
+{
+ "batch_size": 1024,
+ "decoder_int_ff": 512,
+ "decoder_int_num_layer": 1,
+ "decoder_rt_ff": 2048,
+ "decoder_rt_num_layer": 1,
+ "drop_rate": 0.3688667410196246,
+ "embedding_dim": 16,
+ "encoder_ff": 512,
+ "encoder_num_layer": 4,
+ "lr": 0.0018036063294034547,
+ "n_head": 8
+}
\ No newline at end of file
diff --git a/ray_results/test_experiment/train_model_18c11cbc_27_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop__2024-06-25_19-39-03/params.pkl b/ray_results/test_experiment/train_model_18c11cbc_27_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop__2024-06-25_19-39-03/params.pkl
new file mode 100644
index 0000000000000000000000000000000000000000..ed17526d37e22fc1e652a7c64e27ec1665e71efa
GIT binary patch
literal 224
zcmZo*oqCl40&1sd^a!TrCFiH47RBe4=EmnFR;CtB@n-1}NlAr>7L_1NF!qSTBr@~R
zWO!3^lTuSsGV{{oQ!;a>cnkD!!HiBzo8rsB)WZwY3p5lg#L>eC69O3rRmPSVpOKoF
zGR2#thcl%pzaYLSu_SehyZ!aq&GrGiX7w=T6ajg$J@<Q#Gv4drN=htA&WJD0tV#u1
I!cwXS0Lwg6vj6}9
literal 0
HcmV?d00001
diff --git a/ray_results/test_experiment/train_model_18c11cbc_27_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop__2024-06-25_19-39-03/progress.csv b/ray_results/test_experiment/train_model_18c11cbc_27_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop__2024-06-25_19-39-03/progress.csv
new file mode 100644
index 0000000..0ed9c15
--- /dev/null
+++ b/ray_results/test_experiment/train_model_18c11cbc_27_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop__2024-06-25_19-39-03/progress.csv
@@ -0,0 +1,21 @@
+loss,timestamp,checkpoint_dir_name,should_checkpoint,done,training_iteration,trial_id,date,time_this_iter_s,time_total_s,pid,hostname,node_ip,time_since_restore,iterations_since_restore
+266.66766681821326,1719346308,checkpoint_000000,True,False,1,18c11cbc,2024-06-25_22-11-48,74.13704776763916,74.13704776763916,2195253,r8i6n2,10.159.28.60,74.13704776763916,1
+191.86041752372202,1719346367,checkpoint_000001,True,False,2,18c11cbc,2024-06-25_22-12-47,59.13420367240906,133.27125144004822,2195253,r8i6n2,10.159.28.60,133.27125144004822,2
+172.28332663708784,1719346427,checkpoint_000002,True,False,3,18c11cbc,2024-06-25_22-13-47,59.52952766418457,192.8007791042328,2195253,r8i6n2,10.159.28.60,192.8007791042328,3
+165.87816860168937,1719346487,checkpoint_000003,True,False,4,18c11cbc,2024-06-25_22-14-47,59.658318519592285,252.45909762382507,2195253,r8i6n2,10.159.28.60,252.45909762382507,4
+149.45840237835262,1719346547,checkpoint_000004,True,False,5,18c11cbc,2024-06-25_22-15-47,60.179141998291016,312.6382396221161,2195253,r8i6n2,10.159.28.60,312.6382396221161,5
+142.80532779843787,1719354718,checkpoint_000005,True,False,6,18c11cbc,2024-06-26_00-31-58,73.47930812835693,386.117547750473,2850562,r8i6n2,10.159.28.60,73.47930812835693,1
+147.61887368630238,1719354776,checkpoint_000006,True,False,7,18c11cbc,2024-06-26_00-32-56,58.23622250556946,444.3537702560425,2850562,r8i6n2,10.159.28.60,131.7155306339264,2
+136.93548496877114,1719354835,checkpoint_000007,True,False,8,18c11cbc,2024-06-26_00-33-55,59.08127737045288,503.43504762649536,2850562,r8i6n2,10.159.28.60,190.79680800437927,3
+133.39135180495856,1719354893,checkpoint_000008,True,False,9,18c11cbc,2024-06-26_00-34-53,58.395625591278076,561.8306732177734,2850562,r8i6n2,10.159.28.60,249.19243359565735,4
+130.94450414462352,1719354952,checkpoint_000009,True,False,10,18c11cbc,2024-06-26_00-35-52,58.66959762573242,620.5002708435059,2850562,r8i6n2,10.159.28.60,307.86203122138977,5
+128.02527375108613,1719357338,checkpoint_000010,True,False,11,18c11cbc,2024-06-26_01-15-38,72.75415802001953,693.2544288635254,2850562,r8i6n2,10.159.28.60,72.75415802001953,1
+130.49067126296637,1719357397,checkpoint_000011,True,False,12,18c11cbc,2024-06-26_01-16-37,59.34819006919861,752.602618932724,2850562,r8i6n2,10.159.28.60,132.10234808921814,2
+125.55984971654696,1719357457,checkpoint_000012,True,False,13,18c11cbc,2024-06-26_01-17-37,59.262863636016846,811.8654825687408,2850562,r8i6n2,10.159.28.60,191.36521172523499,3
+124.00354583620087,1719357516,checkpoint_000013,True,False,14,18c11cbc,2024-06-26_01-18-36,59.35755896568298,871.2230415344238,2850562,r8i6n2,10.159.28.60,250.72277069091797,4
+122.43110686775267,1719357575,checkpoint_000014,True,False,15,18c11cbc,2024-06-26_01-19-35,59.15378403663635,930.3768255710602,2850562,r8i6n2,10.159.28.60,309.8765547275543,5
+121.44499277129887,1719357635,checkpoint_000015,True,False,16,18c11cbc,2024-06-26_01-20-35,59.61450457572937,989.9913301467896,2850562,r8i6n2,10.159.28.60,369.4910593032837,6
+124.11241438257413,1719357694,checkpoint_000016,True,False,17,18c11cbc,2024-06-26_01-21-34,59.14637589454651,1049.137706041336,2850562,r8i6n2,10.159.28.60,428.6374351978302,7
+121.52879060159518,1719357753,checkpoint_000017,True,False,18,18c11cbc,2024-06-26_01-22-33,59.11753058433533,1108.2552366256714,2850562,r8i6n2,10.159.28.60,487.7549657821655,8
+118.6914559612124,1719357812,checkpoint_000018,True,False,19,18c11cbc,2024-06-26_01-23-32,59.181042432785034,1167.4362790584564,2850562,r8i6n2,10.159.28.60,546.9360082149506,9
+118.33789474006713,1719357871,checkpoint_000019,True,False,20,18c11cbc,2024-06-26_01-24-31,59.315247774124146,1226.7515268325806,2850562,r8i6n2,10.159.28.60,606.2512559890747,10
diff --git a/ray_results/test_experiment/train_model_18c11cbc_27_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop__2024-06-25_19-39-03/result.json b/ray_results/test_experiment/train_model_18c11cbc_27_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop__2024-06-25_19-39-03/result.json
new file mode 100644
index 0000000..db58b67
--- /dev/null
+++ b/ray_results/test_experiment/train_model_18c11cbc_27_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop__2024-06-25_19-39-03/result.json
@@ -0,0 +1,20 @@
+{"loss": 266.66766681821326, "timestamp": 1719346308, "checkpoint_dir_name": "checkpoint_000000", "should_checkpoint": true, "done": false, "training_iteration": 1, "trial_id": "18c11cbc", "date": "2024-06-25_22-11-48", "time_this_iter_s": 74.13704776763916, "time_total_s": 74.13704776763916, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 4, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 512, "decoder_rt_ff": 2048, "decoder_int_ff": 512, "n_head": 8, "drop_rate": 0.3688667410196246, "lr": 0.0018036063294034547, "batch_size": 1024}, "time_since_restore": 74.13704776763916, "iterations_since_restore": 1}
+{"loss": 191.86041752372202, "timestamp": 1719346367, "checkpoint_dir_name": "checkpoint_000001", "should_checkpoint": true, "done": false, "training_iteration": 2, "trial_id": "18c11cbc", "date": "2024-06-25_22-12-47", "time_this_iter_s": 59.13420367240906, "time_total_s": 133.27125144004822, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 4, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 512, "decoder_rt_ff": 2048, "decoder_int_ff": 512, "n_head": 8, "drop_rate": 0.3688667410196246, "lr": 0.0018036063294034547, "batch_size": 1024}, "time_since_restore": 133.27125144004822, "iterations_since_restore": 2}
+{"loss": 172.28332663708784, "timestamp": 1719346427, "checkpoint_dir_name": "checkpoint_000002", "should_checkpoint": true, "done": false, "training_iteration": 3, "trial_id": "18c11cbc", "date": "2024-06-25_22-13-47", "time_this_iter_s": 59.52952766418457, "time_total_s": 192.8007791042328, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 4, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 512, "decoder_rt_ff": 2048, "decoder_int_ff": 512, "n_head": 8, "drop_rate": 0.3688667410196246, "lr": 0.0018036063294034547, "batch_size": 1024}, "time_since_restore": 192.8007791042328, "iterations_since_restore": 3}
+{"loss": 165.87816860168937, "timestamp": 1719346487, "checkpoint_dir_name": "checkpoint_000003", "should_checkpoint": true, "done": false, "training_iteration": 4, "trial_id": "18c11cbc", "date": "2024-06-25_22-14-47", "time_this_iter_s": 59.658318519592285, "time_total_s": 252.45909762382507, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 4, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 512, "decoder_rt_ff": 2048, "decoder_int_ff": 512, "n_head": 8, "drop_rate": 0.3688667410196246, "lr": 0.0018036063294034547, "batch_size": 1024}, "time_since_restore": 252.45909762382507, "iterations_since_restore": 4}
+{"loss": 149.45840237835262, "timestamp": 1719346547, "checkpoint_dir_name": "checkpoint_000004", "should_checkpoint": true, "done": false, "training_iteration": 5, "trial_id": "18c11cbc", "date": "2024-06-25_22-15-47", "time_this_iter_s": 60.179141998291016, "time_total_s": 312.6382396221161, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 4, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 512, "decoder_rt_ff": 2048, "decoder_int_ff": 512, "n_head": 8, "drop_rate": 0.3688667410196246, "lr": 0.0018036063294034547, "batch_size": 1024}, "time_since_restore": 312.6382396221161, "iterations_since_restore": 5}
+{"loss": 142.80532779843787, "timestamp": 1719354718, "checkpoint_dir_name": "checkpoint_000005", "should_checkpoint": true, "done": false, "training_iteration": 6, "trial_id": "18c11cbc", "date": "2024-06-26_00-31-58", "time_this_iter_s": 73.47930812835693, "time_total_s": 386.117547750473, "pid": 2850562, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 4, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 512, "decoder_rt_ff": 2048, "decoder_int_ff": 512, "n_head": 8, "drop_rate": 0.3688667410196246, "lr": 0.0018036063294034547, "batch_size": 1024}, "time_since_restore": 73.47930812835693, "iterations_since_restore": 1}
+{"loss": 147.61887368630238, "timestamp": 1719354776, "checkpoint_dir_name": "checkpoint_000006", "should_checkpoint": true, "done": false, "training_iteration": 7, "trial_id": "18c11cbc", "date": "2024-06-26_00-32-56", "time_this_iter_s": 58.23622250556946, "time_total_s": 444.3537702560425, "pid": 2850562, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 4, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 512, "decoder_rt_ff": 2048, "decoder_int_ff": 512, "n_head": 8, "drop_rate": 0.3688667410196246, "lr": 0.0018036063294034547, "batch_size": 1024}, "time_since_restore": 131.7155306339264, "iterations_since_restore": 2}
+{"loss": 136.93548496877114, "timestamp": 1719354835, "checkpoint_dir_name": "checkpoint_000007", "should_checkpoint": true, "done": false, "training_iteration": 8, "trial_id": "18c11cbc", "date": "2024-06-26_00-33-55", "time_this_iter_s": 59.08127737045288, "time_total_s": 503.43504762649536, "pid": 2850562, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 4, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 512, "decoder_rt_ff": 2048, "decoder_int_ff": 512, "n_head": 8, "drop_rate": 0.3688667410196246, "lr": 0.0018036063294034547, "batch_size": 1024}, "time_since_restore": 190.79680800437927, "iterations_since_restore": 3}
+{"loss": 133.39135180495856, "timestamp": 1719354893, "checkpoint_dir_name": "checkpoint_000008", "should_checkpoint": true, "done": false, "training_iteration": 9, "trial_id": "18c11cbc", "date": "2024-06-26_00-34-53", "time_this_iter_s": 58.395625591278076, "time_total_s": 561.8306732177734, "pid": 2850562, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 4, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 512, "decoder_rt_ff": 2048, "decoder_int_ff": 512, "n_head": 8, "drop_rate": 0.3688667410196246, "lr": 0.0018036063294034547, "batch_size": 1024}, "time_since_restore": 249.19243359565735, "iterations_since_restore": 4}
+{"loss": 130.94450414462352, "timestamp": 1719354952, "checkpoint_dir_name": "checkpoint_000009", "should_checkpoint": true, "done": false, "training_iteration": 10, "trial_id": "18c11cbc", "date": "2024-06-26_00-35-52", "time_this_iter_s": 58.66959762573242, "time_total_s": 620.5002708435059, "pid": 2850562, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 4, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 512, "decoder_rt_ff": 2048, "decoder_int_ff": 512, "n_head": 8, "drop_rate": 0.3688667410196246, "lr": 0.0018036063294034547, "batch_size": 1024}, "time_since_restore": 307.86203122138977, "iterations_since_restore": 5}
+{"loss": 128.02527375108613, "timestamp": 1719357338, "checkpoint_dir_name": "checkpoint_000010", "should_checkpoint": true, "done": false, "training_iteration": 11, "trial_id": "18c11cbc", "date": "2024-06-26_01-15-38", "time_this_iter_s": 72.75415802001953, "time_total_s": 693.2544288635254, "pid": 2850562, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 4, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 512, "decoder_rt_ff": 2048, "decoder_int_ff": 512, "n_head": 8, "drop_rate": 0.3688667410196246, "lr": 0.0018036063294034547, "batch_size": 1024}, "time_since_restore": 72.75415802001953, "iterations_since_restore": 1}
+{"loss": 130.49067126296637, "timestamp": 1719357397, "checkpoint_dir_name": "checkpoint_000011", "should_checkpoint": true, "done": false, "training_iteration": 12, "trial_id": "18c11cbc", "date": "2024-06-26_01-16-37", "time_this_iter_s": 59.34819006919861, "time_total_s": 752.602618932724, "pid": 2850562, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 4, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 512, "decoder_rt_ff": 2048, "decoder_int_ff": 512, "n_head": 8, "drop_rate": 0.3688667410196246, "lr": 0.0018036063294034547, "batch_size": 1024}, "time_since_restore": 132.10234808921814, "iterations_since_restore": 2}
+{"loss": 125.55984971654696, "timestamp": 1719357457, "checkpoint_dir_name": "checkpoint_000012", "should_checkpoint": true, "done": false, "training_iteration": 13, "trial_id": "18c11cbc", "date": "2024-06-26_01-17-37", "time_this_iter_s": 59.262863636016846, "time_total_s": 811.8654825687408, "pid": 2850562, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 4, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 512, "decoder_rt_ff": 2048, "decoder_int_ff": 512, "n_head": 8, "drop_rate": 0.3688667410196246, "lr": 0.0018036063294034547, "batch_size": 1024}, "time_since_restore": 191.36521172523499, "iterations_since_restore": 3}
+{"loss": 124.00354583620087, "timestamp": 1719357516, "checkpoint_dir_name": "checkpoint_000013", "should_checkpoint": true, "done": false, "training_iteration": 14, "trial_id": "18c11cbc", "date": "2024-06-26_01-18-36", "time_this_iter_s": 59.35755896568298, "time_total_s": 871.2230415344238, "pid": 2850562, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 4, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 512, "decoder_rt_ff": 2048, "decoder_int_ff": 512, "n_head": 8, "drop_rate": 0.3688667410196246, "lr": 0.0018036063294034547, "batch_size": 1024}, "time_since_restore": 250.72277069091797, "iterations_since_restore": 4}
+{"loss": 122.43110686775267, "timestamp": 1719357575, "checkpoint_dir_name": "checkpoint_000014", "should_checkpoint": true, "done": false, "training_iteration": 15, "trial_id": "18c11cbc", "date": "2024-06-26_01-19-35", "time_this_iter_s": 59.15378403663635, "time_total_s": 930.3768255710602, "pid": 2850562, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 4, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 512, "decoder_rt_ff": 2048, "decoder_int_ff": 512, "n_head": 8, "drop_rate": 0.3688667410196246, "lr": 0.0018036063294034547, "batch_size": 1024}, "time_since_restore": 309.8765547275543, "iterations_since_restore": 5}
+{"loss": 121.44499277129887, "timestamp": 1719357635, "checkpoint_dir_name": "checkpoint_000015", "should_checkpoint": true, "done": false, "training_iteration": 16, "trial_id": "18c11cbc", "date": "2024-06-26_01-20-35", "time_this_iter_s": 59.61450457572937, "time_total_s": 989.9913301467896, "pid": 2850562, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 4, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 512, "decoder_rt_ff": 2048, "decoder_int_ff": 512, "n_head": 8, "drop_rate": 0.3688667410196246, "lr": 0.0018036063294034547, "batch_size": 1024}, "time_since_restore": 369.4910593032837, "iterations_since_restore": 6}
+{"loss": 124.11241438257413, "timestamp": 1719357694, "checkpoint_dir_name": "checkpoint_000016", "should_checkpoint": true, "done": false, "training_iteration": 17, "trial_id": "18c11cbc", "date": "2024-06-26_01-21-34", "time_this_iter_s": 59.14637589454651, "time_total_s": 1049.137706041336, "pid": 2850562, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 4, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 512, "decoder_rt_ff": 2048, "decoder_int_ff": 512, "n_head": 8, "drop_rate": 0.3688667410196246, "lr": 0.0018036063294034547, "batch_size": 1024}, "time_since_restore": 428.6374351978302, "iterations_since_restore": 7}
+{"loss": 121.52879060159518, "timestamp": 1719357753, "checkpoint_dir_name": "checkpoint_000017", "should_checkpoint": true, "done": false, "training_iteration": 18, "trial_id": "18c11cbc", "date": "2024-06-26_01-22-33", "time_this_iter_s": 59.11753058433533, "time_total_s": 1108.2552366256714, "pid": 2850562, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 4, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 512, "decoder_rt_ff": 2048, "decoder_int_ff": 512, "n_head": 8, "drop_rate": 0.3688667410196246, "lr": 0.0018036063294034547, "batch_size": 1024}, "time_since_restore": 487.7549657821655, "iterations_since_restore": 8}
+{"loss": 118.6914559612124, "timestamp": 1719357812, "checkpoint_dir_name": "checkpoint_000018", "should_checkpoint": true, "done": false, "training_iteration": 19, "trial_id": "18c11cbc", "date": "2024-06-26_01-23-32", "time_this_iter_s": 59.181042432785034, "time_total_s": 1167.4362790584564, "pid": 2850562, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 4, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 512, "decoder_rt_ff": 2048, "decoder_int_ff": 512, "n_head": 8, "drop_rate": 0.3688667410196246, "lr": 0.0018036063294034547, "batch_size": 1024}, "time_since_restore": 546.9360082149506, "iterations_since_restore": 9}
+{"loss": 118.33789474006713, "timestamp": 1719357871, "checkpoint_dir_name": "checkpoint_000019", "should_checkpoint": true, "done": false, "training_iteration": 20, "trial_id": "18c11cbc", "date": "2024-06-26_01-24-31", "time_this_iter_s": 59.315247774124146, "time_total_s": 1226.7515268325806, "pid": 2850562, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 4, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 512, "decoder_rt_ff": 2048, "decoder_int_ff": 512, "n_head": 8, "drop_rate": 0.3688667410196246, "lr": 0.0018036063294034547, "batch_size": 1024}, "time_since_restore": 606.2512559890747, "iterations_since_restore": 10}
diff --git a/ray_results/test_experiment/train_model_1920f493_9_batch_size=1024,decoder_int_ff=1024,decoder_int_num_layer=2,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop__2024-06-25_16-19-16/events.out.tfevents.1719325234.r3i5n6 b/ray_results/test_experiment/train_model_1920f493_9_batch_size=1024,decoder_int_ff=1024,decoder_int_num_layer=2,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop__2024-06-25_16-19-16/events.out.tfevents.1719325234.r3i5n6
new file mode 100644
index 0000000000000000000000000000000000000000..3d56d70c86b59b1df86c8bd559ee0c2dc402b4f1
GIT binary patch
literal 1311
zcmah|O-vI(7;Se6vR_i5ON+3ySQaE^4^#}%gpg1*M1#f<5B`(M(jDw-J6m@a5H4tl
z(SuPg9z1|39z2*p@b6&aK@S>FBzW;eG#d3_^rGm^S-PFJh`6`?-uJ#Y^S(EAzT7uE
zen6E@Ja}~V;q=?xN?^d&P14pgjEU2^W7*#{((Luk-x}E~oieV?xP8@rtrkKMu=UZn
zGemGaZBZI-Ns2wK@1n|fk8<P3B9inpr!+$oD4ar5)==6&$rMhW%~~dL!am=H?!}d3
zZ6%UNEzU2MG%SJ{#pnAF6NR`Zndy)ANs%n4s1SpCF6o#VjGUB7k?CL?(eS|+(|d+H
zM#kGc`qae47ip?MZvhn=O6k-jNsMev9m~e61GL1U%e%Y*pFY1nE-B}@z6RT%i4Sm9
zb;q;_MWtihn=kCVUO(^8&6%9Kc}9A<^s)6ZGh4eW!fg>kD3#T1Jws6(pU+~O-Gw;4
zS46lZ!X^)re+%+YL9GNB(4Az8sf^*6sLD`j;G|`+J2Rir;2^8jL%6s`hB7Fvk7Aoa
zK?vj_+v~1_P*JJ4IRKJjTUo>|=1MUx&J19~Fv)3Tm>E`Lr@<trzN#FCVE(^&P&o)`
zR~?|7pcK}Pn5YGzN-hnV)(8=KgAlEuxFOM^&>;Ui1lkse${Q_N+X~I{|ChLQFBB!-
z{+@rk(<zm`@%Hy#fMo$9h5b=XDN6yOj?N4=$)MHyOn|3CVUKfya$kVjB5!3WxNp27
zyKwPqH`L@UV#QowzITJNW@{XqNkHQ|mUFy10S#*d!okS|d$jBBp0_aVDtC9RFcw_-
z)lqYD^)Mwd;LUvr*j$_k4zN=)pb%RQuoI~N$SUvgCHlLL?C<g(;j1UJhowl~{dS0h
Y^0)Vx7&+%z#tOcdsX~Q+er`$p10+(8zW@LL
literal 0
HcmV?d00001
diff --git a/ray_results/test_experiment/train_model_1920f493_9_batch_size=1024,decoder_int_ff=1024,decoder_int_num_layer=2,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop__2024-06-25_16-19-16/params.json b/ray_results/test_experiment/train_model_1920f493_9_batch_size=1024,decoder_int_ff=1024,decoder_int_num_layer=2,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop__2024-06-25_16-19-16/params.json
new file mode 100644
index 0000000..3a11335
--- /dev/null
+++ b/ray_results/test_experiment/train_model_1920f493_9_batch_size=1024,decoder_int_ff=1024,decoder_int_num_layer=2,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop__2024-06-25_16-19-16/params.json
@@ -0,0 +1,13 @@
+{
+ "batch_size": 1024,
+ "decoder_int_ff": 1024,
+ "decoder_int_num_layer": 2,
+ "decoder_rt_ff": 2048,
+ "decoder_rt_num_layer": 1,
+ "drop_rate": 0.922082156004742,
+ "embedding_dim": 64,
+ "encoder_ff": 2048,
+ "encoder_num_layer": 4,
+ "lr": 0.0016525285062566051,
+ "n_head": 1
+}
\ No newline at end of file
diff --git a/ray_results/test_experiment/train_model_1920f493_9_batch_size=1024,decoder_int_ff=1024,decoder_int_num_layer=2,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop__2024-06-25_16-19-16/params.pkl b/ray_results/test_experiment/train_model_1920f493_9_batch_size=1024,decoder_int_ff=1024,decoder_int_num_layer=2,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop__2024-06-25_16-19-16/params.pkl
new file mode 100644
index 0000000000000000000000000000000000000000..0d692d3c5ce4a7a704acc89c2aa87714b70d3b2b
GIT binary patch
literal 224
zcmZo*oqCl40&1sd^a!TrCFiH47RBe4=EmnFR;CtB@n-1}NlAr>7L_1NF!qSTBr@|*
zWSDw*Q*)D2Q&KYX(&JMybEkMa^l-t9PD`8O%fQjY3)2fU6e`3A69O3r7GvpQ%ZtxQ
zO-z~M&Dg`4Qj}j1UzAvqI>p`oZR4f_j(YbVrko-mFIw2-$9KW19<HRslH`o|;>@a4
Jpd~D&dH}$$QjGur
literal 0
HcmV?d00001
diff --git a/ray_results/test_experiment/train_model_1920f493_9_batch_size=1024,decoder_int_ff=1024,decoder_int_num_layer=2,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop__2024-06-25_16-19-16/progress.csv b/ray_results/test_experiment/train_model_1920f493_9_batch_size=1024,decoder_int_ff=1024,decoder_int_num_layer=2,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop__2024-06-25_16-19-16/progress.csv
new file mode 100644
index 0000000..71eb8bc
--- /dev/null
+++ b/ray_results/test_experiment/train_model_1920f493_9_batch_size=1024,decoder_int_ff=1024,decoder_int_num_layer=2,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop__2024-06-25_16-19-16/progress.csv
@@ -0,0 +1,2 @@
+loss,timestamp,checkpoint_dir_name,should_checkpoint,done,training_iteration,trial_id,date,time_this_iter_s,time_total_s,pid,hostname,node_ip,time_since_restore,iterations_since_restore
+2102.196185254675,1719325314,checkpoint_000000,True,False,1,1920f493,2024-06-25_16-21-54,79.38143539428711,79.38143539428711,69318,r3i5n6,10.159.8.159,79.38143539428711,1
diff --git a/ray_results/test_experiment/train_model_1920f493_9_batch_size=1024,decoder_int_ff=1024,decoder_int_num_layer=2,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop__2024-06-25_16-19-16/result.json b/ray_results/test_experiment/train_model_1920f493_9_batch_size=1024,decoder_int_ff=1024,decoder_int_num_layer=2,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop__2024-06-25_16-19-16/result.json
new file mode 100644
index 0000000..fd14144
--- /dev/null
+++ b/ray_results/test_experiment/train_model_1920f493_9_batch_size=1024,decoder_int_ff=1024,decoder_int_num_layer=2,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop__2024-06-25_16-19-16/result.json
@@ -0,0 +1 @@
+{"loss": 2102.196185254675, "timestamp": 1719325314, "checkpoint_dir_name": "checkpoint_000000", "should_checkpoint": true, "done": false, "training_iteration": 1, "trial_id": "1920f493", "date": "2024-06-25_16-21-54", "time_this_iter_s": 79.38143539428711, "time_total_s": 79.38143539428711, "pid": 69318, "hostname": "r3i5n6", "node_ip": "10.159.8.159", "config": {"encoder_num_layer": 4, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 2, "embedding_dim": 64, "encoder_ff": 2048, "decoder_rt_ff": 2048, "decoder_int_ff": 1024, "n_head": 1, "drop_rate": 0.922082156004742, "lr": 0.0016525285062566051, "batch_size": 1024}, "time_since_restore": 79.38143539428711, "iterations_since_restore": 1}
diff --git a/ray_results/test_experiment/train_model_1de460a8_10_batch_size=1024,decoder_int_ff=1024,decoder_int_num_layer=8,decoder_rt_ff=512,decoder_rt_num_layer=4,drop__2024-06-25_16-20-34/events.out.tfevents.1719325314.r3i5n6 b/ray_results/test_experiment/train_model_1de460a8_10_batch_size=1024,decoder_int_ff=1024,decoder_int_num_layer=8,decoder_rt_ff=512,decoder_rt_num_layer=4,drop__2024-06-25_16-20-34/events.out.tfevents.1719325314.r3i5n6
new file mode 100644
index 0000000000000000000000000000000000000000..f44e779c8b68dd1ae1feb09c2f227703e7c7ffe6
GIT binary patch
literal 405
zcmb1OfPlsI-b$Pc5@s&gGVi9N6mL>dVrHJ6YguYuiIvfN2k$@1P<0#TmpgMV`uBGe
zNSy$qmKc`^7jIEwrG80iUaEdhesQs=v+h@y8{W4s89=p6{dysuQ;6jsOp7v?B9|0g
zOL0biX--Ofaz<)$c0qn-UWq6JLxX({TP8P(mCM-R`cbV+$<Ip#YGGhFdMeOGA8KW8
ztZV>h06ScZ5|;v(B;3l9%-q!Yl8nsa_{@^jqWEIbYl@Sdp0S#97(%trZEk+a`2wL`
zg-eM`8dZC7W?ph?d{Js~Nq$i(*nl-N*jvq^20VXpDw4B|18%<-mj;(2+yIcRi6xo&
QdByRl#(;c2KRtpI03XeTtpET3
literal 0
HcmV?d00001
diff --git a/ray_results/test_experiment/train_model_1de460a8_10_batch_size=1024,decoder_int_ff=1024,decoder_int_num_layer=8,decoder_rt_ff=512,decoder_rt_num_layer=4,drop__2024-06-25_16-20-34/events.out.tfevents.1719327690.r3i5n6 b/ray_results/test_experiment/train_model_1de460a8_10_batch_size=1024,decoder_int_ff=1024,decoder_int_num_layer=8,decoder_rt_ff=512,decoder_rt_num_layer=4,drop__2024-06-25_16-20-34/events.out.tfevents.1719327690.r3i5n6
new file mode 100644
index 0000000000000000000000000000000000000000..984a613d6411f46d57a8e1857294676fecce33b3
GIT binary patch
literal 405
zcmb1OfPlsI-b$RynI?SNHt(jR6mL>dVrHJ6YguYuiItJYw4yJ{P<0#TmpgMZygtmb
z9jH!#NlT1Ngp0Q*u~NUJG%r;@C%?E@^iAd$7o|zLQU*{hQ@>t_=XCu5*P_g&$R!2W
zQk;=rnv)WroRONGU67xdS0c*5&|sfvbn_*OmFGUe^`lyulAo6f)WX1U_RiHE`cNx#
zV`T$4(>}wsC~+xpNy4ox$;?fSFUiO(j?XMfEs8G|-9CGwQ%0@qdPAu8xy{WlIiDl6
zt8giCNuz2n&df_rjW0?qF3B%S1sjmuYZ7M;HQ@P+Q<0oWU*Ps@acOWV!VLh~npl#V
RpI01@Y7EHdi=&;60|3eahkyV8
literal 0
HcmV?d00001
diff --git a/ray_results/test_experiment/train_model_1de460a8_10_batch_size=1024,decoder_int_ff=1024,decoder_int_num_layer=8,decoder_rt_ff=512,decoder_rt_num_layer=4,drop__2024-06-25_16-20-34/params.json b/ray_results/test_experiment/train_model_1de460a8_10_batch_size=1024,decoder_int_ff=1024,decoder_int_num_layer=8,decoder_rt_ff=512,decoder_rt_num_layer=4,drop__2024-06-25_16-20-34/params.json
new file mode 100644
index 0000000..cb9dc53
--- /dev/null
+++ b/ray_results/test_experiment/train_model_1de460a8_10_batch_size=1024,decoder_int_ff=1024,decoder_int_num_layer=8,decoder_rt_ff=512,decoder_rt_num_layer=4,drop__2024-06-25_16-20-34/params.json
@@ -0,0 +1,13 @@
+{
+ "batch_size": 1024,
+ "decoder_int_ff": 1024,
+ "decoder_int_num_layer": 8,
+ "decoder_rt_ff": 512,
+ "decoder_rt_num_layer": 4,
+ "drop_rate": 0.03175079062321118,
+ "embedding_dim": 16,
+ "encoder_ff": 2048,
+ "encoder_num_layer": 2,
+ "lr": 0.003436670599863372,
+ "n_head": 1
+}
\ No newline at end of file
diff --git a/ray_results/test_experiment/train_model_1de460a8_10_batch_size=1024,decoder_int_ff=1024,decoder_int_num_layer=8,decoder_rt_ff=512,decoder_rt_num_layer=4,drop__2024-06-25_16-20-34/params.pkl b/ray_results/test_experiment/train_model_1de460a8_10_batch_size=1024,decoder_int_ff=1024,decoder_int_num_layer=8,decoder_rt_ff=512,decoder_rt_num_layer=4,drop__2024-06-25_16-20-34/params.pkl
new file mode 100644
index 0000000000000000000000000000000000000000..7407cc3806eeb80607eabed513e14637907ac226
GIT binary patch
literal 224
zcmZo*oqCl40&1sd^a!TrCFiH47RBe4=EmnFR;CtB@n-4~NlAr>7L_1Nu=I$+Br@|*
zWH@?wQ*)D2Q&KYX(&JMybEkL<^l-t9PD`8O%K%gX(+e~dEX35q2NMDr2Nq-LVatop
zNKH(c;?3B@nNpNr5MPv7k~+oReu3lSeVb-Y=wZq!0`hXyEvnXpFX`b*N-Rmvh%e5p
LN(EZNQmO|4zb;bl
literal 0
HcmV?d00001
diff --git a/ray_results/test_experiment/train_model_1de460a8_10_batch_size=1024,decoder_int_ff=1024,decoder_int_num_layer=8,decoder_rt_ff=512,decoder_rt_num_layer=4,drop__2024-06-25_16-20-34/progress.csv b/ray_results/test_experiment/train_model_1de460a8_10_batch_size=1024,decoder_int_ff=1024,decoder_int_num_layer=8,decoder_rt_ff=512,decoder_rt_num_layer=4,drop__2024-06-25_16-20-34/progress.csv
new file mode 100644
index 0000000..f2c7e56
--- /dev/null
+++ b/ray_results/test_experiment/train_model_1de460a8_10_batch_size=1024,decoder_int_ff=1024,decoder_int_num_layer=8,decoder_rt_ff=512,decoder_rt_num_layer=4,drop__2024-06-25_16-20-34/progress.csv
@@ -0,0 +1,3 @@
+loss,timestamp,checkpoint_dir_name,should_checkpoint,done,training_iteration,trial_id,date,time_this_iter_s,time_total_s,pid,hostname,node_ip,time_since_restore,iterations_since_restore
+1961.4144263079786,1719325387,checkpoint_000000,True,False,1,1de460a8,2024-06-25_16-23-07,73.56608295440674,73.56608295440674,69318,r3i5n6,10.159.8.159,73.56608295440674,1
+1955.3101080947035,1719327763,checkpoint_000001,True,False,2,1de460a8,2024-06-25_17-02-43,72.80412983894348,146.37021279335022,130020,r3i5n6,10.159.8.159,72.80412983894348,1
diff --git a/ray_results/test_experiment/train_model_1de460a8_10_batch_size=1024,decoder_int_ff=1024,decoder_int_num_layer=8,decoder_rt_ff=512,decoder_rt_num_layer=4,drop__2024-06-25_16-20-34/result.json b/ray_results/test_experiment/train_model_1de460a8_10_batch_size=1024,decoder_int_ff=1024,decoder_int_num_layer=8,decoder_rt_ff=512,decoder_rt_num_layer=4,drop__2024-06-25_16-20-34/result.json
new file mode 100644
index 0000000..58cd15d
--- /dev/null
+++ b/ray_results/test_experiment/train_model_1de460a8_10_batch_size=1024,decoder_int_ff=1024,decoder_int_num_layer=8,decoder_rt_ff=512,decoder_rt_num_layer=4,drop__2024-06-25_16-20-34/result.json
@@ -0,0 +1,2 @@
+{"loss": 1961.4144263079786, "timestamp": 1719325387, "checkpoint_dir_name": "checkpoint_000000", "should_checkpoint": true, "done": false, "training_iteration": 1, "trial_id": "1de460a8", "date": "2024-06-25_16-23-07", "time_this_iter_s": 73.56608295440674, "time_total_s": 73.56608295440674, "pid": 69318, "hostname": "r3i5n6", "node_ip": "10.159.8.159", "config": {"encoder_num_layer": 2, "decoder_rt_num_layer": 4, "decoder_int_num_layer": 8, "embedding_dim": 16, "encoder_ff": 2048, "decoder_rt_ff": 512, "decoder_int_ff": 1024, "n_head": 1, "drop_rate": 0.03175079062321118, "lr": 0.003436670599863372, "batch_size": 1024}, "time_since_restore": 73.56608295440674, "iterations_since_restore": 1}
+{"loss": 1955.3101080947035, "timestamp": 1719327763, "checkpoint_dir_name": "checkpoint_000001", "should_checkpoint": true, "done": false, "training_iteration": 2, "trial_id": "1de460a8", "date": "2024-06-25_17-02-43", "time_this_iter_s": 72.80412983894348, "time_total_s": 146.37021279335022, "pid": 130020, "hostname": "r3i5n6", "node_ip": "10.159.8.159", "config": {"encoder_num_layer": 2, "decoder_rt_num_layer": 4, "decoder_int_num_layer": 8, "embedding_dim": 16, "encoder_ff": 2048, "decoder_rt_ff": 512, "decoder_int_ff": 1024, "n_head": 1, "drop_rate": 0.03175079062321118, "lr": 0.003436670599863372, "batch_size": 1024}, "time_since_restore": 72.80412983894348, "iterations_since_restore": 1}
diff --git a/ray_results/test_experiment/train_model_1ef7c88b_11_batch_size=1024,decoder_int_ff=1024,decoder_int_num_layer=4,decoder_rt_ff=512,decoder_rt_num_layer=4,drop__2024-06-25_16-21-54/error.pkl b/ray_results/test_experiment/train_model_1ef7c88b_11_batch_size=1024,decoder_int_ff=1024,decoder_int_num_layer=4,decoder_rt_ff=512,decoder_rt_num_layer=4,drop__2024-06-25_16-21-54/error.pkl
new file mode 100644
index 0000000000000000000000000000000000000000..dc96d91fc2cdcb1d28ff67aa443125d6624678a0
GIT binary patch
literal 4759
zcmeHK&yN&E6z;(#z>0`yToWT!ST!@U)7>+(vpci0C=Bw$g;}yJL}IW~(_KB)PEU7R
zRn0PnO}Ic}(gzZoz=6b@G4Y?l#Df?A2ag_oRXx2si$>rec(6M&>8g5F_1;(Cd-dL}
z-S^T%hJMnwIT@Kw5G8KxJA;5uerM?$ct{2mivbO&j6&=LM2PIWteo!cC4M0NP-N+p
z9K{qm)5B|Iv`)m}Q_dq^UQOhxcYzKgKGGePEIpt$qc<EHOFvRm_fO7HOH9j%JIr(v
zmt^U|$Cm(!JK1KIB6r@S?RK+48ck=uZns;lM%%F$UAjPA*L9j%S7vJ<_YcNlj|iEi
zujV#kL8XvHDm&O~o610p{m_>fXUs?s<T!Xq=qSxWs0_WY;4uieKwxhta_PE)<2LMH
zUfH!`z`G0H*E{dsH*S`08aM|Y{%@8%)h-nsu5ro8y0^1*w?h&^0a(KO0xV9a1P#25
zA^1$Je(Z@La!4SO*ou>$MMC04p-ZaqsA1LZdc(Ba77ecpD;{Y$tNl2s*3G(U9|aM!
zRx=l^!0)MH7KN(A627F>m^cH{r@{i*tnFhm<59-mOcy54$|M5d0mMRwMjqA*o5yms
zjsu5;xdq?Kz0O;=j9=e5a(615{(Y};c=o|r_}q6F-mGI}m5Mqm1^LSuk5vHF)0aSt
zOdUf=hEyHH9@X&tKs8<z4oSDGsl^VZhj2lM9ptHRDqENdi0#Mt>x2Y_{vQC@FbTs5
zc&6wNW&oEDU_97xsYjAPs;;h@wxd0b@i1}|Z8w0h2Zkd@O9Eivx+38Ul5RSqT7@HH
zk@WIuBD<C;Z2Ljf1J>wmlRMd+HCy*mP)Bm;Wp9<)JKLO{0?y9eH{=#Sfw9I{TdkX=
zA9fkWPw>kh;4XVN-P6yP(`03{>69`aD9qkxr_%PbGHARQ=(h5(^TsXXx0%Ba<il@%
zod0>o{$<8~bkNRNIp+^MTAa*0TP=>@6kBAUvDRkBmb&btE<48du;ng0-en(O%h-v{
zYzzZHDkdQ)X}2w{lmLRg_e%GTZ%f*|3Y)#HOu3RapVgFn{gv-K`~CoDm_d&YBG5hE
zM^&uEL(zwbp&daP0<q;_9&zX}mLu%YAb^-tSpy;EENq^+g^gFh#;bp0Bdjr<uQ8te
z`6ttc@%9&ug&{cB($cwM9QckePeV*IB`1DZuE1z}h|00=F4r2iUAO0uA1~Kzv({{z
zE%iT#hyyVNgG964bep76uQ~I?Zq-`uVuO;lN9>xdpE<;7%$N1jioi6eOr!1R%cppr
zK^{a;Luf!po>zR7Nk}bi9O(rUUl1*i%%Q*!DQe8u0l*MxDQyeDEGQBKKaQ!3JRS|v
zR-Da)Xn8!SuzJ1*5uaNS(<S9039Ll=!GtNTmbQc^p|(TaM<(V}qGgnSOSr43EPt4{
z)WtQ2J{-3O8>+qI^feEjf&sb6bI0Pb!`%w>TjxCiO?@7`0#5l@K!!b+9M=G8K|k#4
zJ7PH>St_7X&ZbDC2g@2UxWGNXUzyXBx1D72>xpPh&V7RLe;Uw6BX=9Tg|-Gtn-?WA
zP891r2v?H^Bcx1z(RMxNHMK_WHhvgVuC)c1N%%M-C<>JGgIB)dg^&pR#ZmtGAa~lk
z5%Gb=sl&i~sg*eWR&7IAl2Jje^PTF2OVzdYYRswY>nzq90uM1<23XO3g3;|<YYl%M
zxwY<)X$Z3iIU;E3YV@C9fYN!ag(VF|#3xj3)^n~DYH3CD8iFJ;>T}|5tGpAfe>sQF
zH7=B}igb8F_xi$u4jemlze2dLh+$%SgF3(`sJdv_hwH?I;EM~{GD5ScLxMm+=#deE
zLq?sWM^P?{pa-ONZ!rVKP`XW&SLWlz)e`DlK6L_thoQ(rLtVI;Xr23TQ^9($7Ze?m
zqI$Ds+BUl2pES{#i<gm&7!inoj3fba2&C}79H}*GO>=P(o$*hq#s>2phU&EibD^y!
zBVWJ;NU4d=g-(#T5G+v`h1H9rbyWh66_^7XKrrT!Lxs>g!K%%sUNzTI#a<|vOtk80
zg9}C7@e)c6yRIj|O53Bo5gP9WNloaY+CmL7oI*7YPQ$N*>Dywc2hb#NIp#pPpt%P*
zP9e8*u-F{74Aj*KNx%r=zl4&xk1k$aUtQ~*#R_nI;{5s54(_aWPcPFOF=(Q%20{Bn
z8cMOW&J&mn_+O8Fl`;_BuPX=>p-Yq^s0I=xtNV}_ag<lx=mJ!ReX52KDl;i#v1AoB
zp&9YM1v7;OMVv}tpe%vPPZi!Klfz&Z3P_(vp_*O2J|?B8=(-ki;2rrh<kZO~B>7NL
bu>GIULP_vHlm2JY(=ka^{tU51^vpj2z5!Q_
literal 0
HcmV?d00001
diff --git a/ray_results/test_experiment/train_model_1ef7c88b_11_batch_size=1024,decoder_int_ff=1024,decoder_int_num_layer=4,decoder_rt_ff=512,decoder_rt_num_layer=4,drop__2024-06-25_16-21-54/error.txt b/ray_results/test_experiment/train_model_1ef7c88b_11_batch_size=1024,decoder_int_ff=1024,decoder_int_num_layer=4,decoder_rt_ff=512,decoder_rt_num_layer=4,drop__2024-06-25_16-21-54/error.txt
new file mode 100644
index 0000000..da55c0d
--- /dev/null
+++ b/ray_results/test_experiment/train_model_1ef7c88b_11_batch_size=1024,decoder_int_ff=1024,decoder_int_num_layer=4,decoder_rt_ff=512,decoder_rt_num_layer=4,drop__2024-06-25_16-21-54/error.txt
@@ -0,0 +1,26 @@
+Failure # 1 (occurred at 2024-06-25_17-03-02)
+[36mray::ImplicitFunc.train()[39m (pid=130020, ip=10.159.8.159, actor_id=595d5a321c4a0818d73ea9fa01000000, repr=train_model)
+ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+ File "/gpfslocalsup/pub/anaconda-py3/2023.09/envs/pytorch-gpu-2.2.0+py3.11.7/lib/python3.11/site-packages/ray/tune/trainable/trainable.py", line 342, in train
+ raise skipped from exception_cause(skipped)
+ File "/gpfslocalsup/pub/anaconda-py3/2023.09/envs/pytorch-gpu-2.2.0+py3.11.7/lib/python3.11/site-packages/ray/air/_internal/util.py", line 88, in run
+ self._ret = self._target(*self._args, **self._kwargs)
+ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+ File "/gpfslocalsup/pub/anaconda-py3/2023.09/envs/pytorch-gpu-2.2.0+py3.11.7/lib/python3.11/site-packages/ray/tune/trainable/function_trainable.py", line 249, in <lambda>
+ training_func=lambda: self._trainable_func(self.config),
+ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+ File "/gpfslocalsup/pub/anaconda-py3/2023.09/envs/pytorch-gpu-2.2.0+py3.11.7/lib/python3.11/site-packages/ray/tune/trainable/function_trainable.py", line 332, in _trainable_func
+ output = fn()
+ ^^^^
+ File "/gpfslocalsup/pub/anaconda-py3/2023.09/envs/pytorch-gpu-2.2.0+py3.11.7/lib/python3.11/site-packages/ray/tune/trainable/util.py", line 138, in inner
+ return trainable(config, **fn_kwargs)
+ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+ File "/gpfswork/rech/ute/ucg81ws/these/LC-MS-RT-prediction/main_ray_tune.py", line 98, in train_model
+ loss.backward()
+ File "/gpfslocalsup/pub/anaconda-py3/2023.09/envs/pytorch-gpu-2.2.0+py3.11.7/lib/python3.11/site-packages/torch/_tensor.py", line 522, in backward
+ torch.autograd.backward(
+ File "/gpfslocalsup/pub/anaconda-py3/2023.09/envs/pytorch-gpu-2.2.0+py3.11.7/lib/python3.11/site-packages/torch/autograd/__init__.py", line 266, in backward
+ Variable._execution_engine.run_backward( # Calls into the C++ engine to run the backward pass
+torch.cuda.OutOfMemoryError: CUDA out of memory. Tried to allocate 258.00 MiB. GPU 0 has a total capacity of 15.77 GiB of which 216.69 MiB is free. Including non-PyTorch memory, this process has 15.55 GiB memory in use. Of the allocated memory 15.02 GiB is allocated by PyTorch, and 161.15 MiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables)
diff --git a/ray_results/test_experiment/train_model_1ef7c88b_11_batch_size=1024,decoder_int_ff=1024,decoder_int_num_layer=4,decoder_rt_ff=512,decoder_rt_num_layer=4,drop__2024-06-25_16-21-54/events.out.tfevents.1719325388.r3i5n6 b/ray_results/test_experiment/train_model_1ef7c88b_11_batch_size=1024,decoder_int_ff=1024,decoder_int_num_layer=4,decoder_rt_ff=512,decoder_rt_num_layer=4,drop__2024-06-25_16-21-54/events.out.tfevents.1719325388.r3i5n6
new file mode 100644
index 0000000000000000000000000000000000000000..81eb448e22e971b303b3b631227ff8873a5741e2
GIT binary patch
literal 405
zcmb1OfPlsI-b$QT_1QOXnRnAsiZ`h!F*8rkwJbHS#L7rFe=WB%RNaR8<<6W6QQJ0b
z1*#KZ)Dq(o;o>bytkf?l%}dqK$uBMzWvKq@;?Z?^vH?`f)UOxfIiJVFwJ38ba!J9p
z6ldg@=A^_YXQU=)7vyK=m54GhG}u4nol}Kk<<um&epD+{^7B%GS{N8?cZhD$hgz8%
zD;vP6mkigU#HGL`3AeH&GdDH9BqOspKC>jXD85+q)4Ez0N$$`7hEVNuo10&9Za`>P
z;Zov~M%7-NnU|azUzA#0l3$bxHsF%m^&E4k0ncBYisZCQf!nXerNN~LHvnX7Vo7Fx
QUU59CF(99d-BMBm0E>x*4*&oF
literal 0
HcmV?d00001
diff --git a/ray_results/test_experiment/train_model_1ef7c88b_11_batch_size=1024,decoder_int_ff=1024,decoder_int_num_layer=4,decoder_rt_ff=512,decoder_rt_num_layer=4,drop__2024-06-25_16-21-54/events.out.tfevents.1719327763.r3i5n6 b/ray_results/test_experiment/train_model_1ef7c88b_11_batch_size=1024,decoder_int_ff=1024,decoder_int_num_layer=4,decoder_rt_ff=512,decoder_rt_num_layer=4,drop__2024-06-25_16-21-54/events.out.tfevents.1719327763.r3i5n6
new file mode 100644
index 0000000000000000000000000000000000000000..34cccb21c50303885980210d17dac2401e234568
GIT binary patch
literal 40
rcmb1OfPlsI-b$QrxX-d|pLf$yiZ`h!F*8rkwJbHS#L8%$ucI;m)R7Dp
literal 0
HcmV?d00001
diff --git a/ray_results/test_experiment/train_model_1ef7c88b_11_batch_size=1024,decoder_int_ff=1024,decoder_int_num_layer=4,decoder_rt_ff=512,decoder_rt_num_layer=4,drop__2024-06-25_16-21-54/params.json b/ray_results/test_experiment/train_model_1ef7c88b_11_batch_size=1024,decoder_int_ff=1024,decoder_int_num_layer=4,decoder_rt_ff=512,decoder_rt_num_layer=4,drop__2024-06-25_16-21-54/params.json
new file mode 100644
index 0000000..17c6a86
--- /dev/null
+++ b/ray_results/test_experiment/train_model_1ef7c88b_11_batch_size=1024,decoder_int_ff=1024,decoder_int_num_layer=4,decoder_rt_ff=512,decoder_rt_num_layer=4,drop__2024-06-25_16-21-54/params.json
@@ -0,0 +1,13 @@
+{
+ "batch_size": 1024,
+ "decoder_int_ff": 1024,
+ "decoder_int_num_layer": 4,
+ "decoder_rt_ff": 512,
+ "decoder_rt_num_layer": 4,
+ "drop_rate": 0.14711494497891564,
+ "embedding_dim": 1024,
+ "encoder_ff": 1024,
+ "encoder_num_layer": 8,
+ "lr": 0.00018307057253269556,
+ "n_head": 2
+}
\ No newline at end of file
diff --git a/ray_results/test_experiment/train_model_1ef7c88b_11_batch_size=1024,decoder_int_ff=1024,decoder_int_num_layer=4,decoder_rt_ff=512,decoder_rt_num_layer=4,drop__2024-06-25_16-21-54/params.pkl b/ray_results/test_experiment/train_model_1ef7c88b_11_batch_size=1024,decoder_int_ff=1024,decoder_int_num_layer=4,decoder_rt_ff=512,decoder_rt_num_layer=4,drop__2024-06-25_16-21-54/params.pkl
new file mode 100644
index 0000000000000000000000000000000000000000..761dbb8411c02b9bd6cb33d071a631c4d2c0b735
GIT binary patch
literal 225
zcmZo*oqCM{0&1sd^a!TrCFiH47RBe4=EmnFR;CtB@#g3eNlAr>7L_1Nu=I$+Br@~R
zWO!3^lTuSsGV{{oQ!;a>_%g8caKX$@O9S(HVVZ%af`yoR_+UaH^PtMu^5Qd66H}&m
zGxcz$6y+Dh7bTXYPI0$CbY<nN?LY4HFy#~hdFubJ*kwp?_HZR7mLzAy7iU(bf-EW3
F0{{@VQ~m$|
literal 0
HcmV?d00001
diff --git a/ray_results/test_experiment/train_model_1ef7c88b_11_batch_size=1024,decoder_int_ff=1024,decoder_int_num_layer=4,decoder_rt_ff=512,decoder_rt_num_layer=4,drop__2024-06-25_16-21-54/progress.csv b/ray_results/test_experiment/train_model_1ef7c88b_11_batch_size=1024,decoder_int_ff=1024,decoder_int_num_layer=4,decoder_rt_ff=512,decoder_rt_num_layer=4,drop__2024-06-25_16-21-54/progress.csv
new file mode 100644
index 0000000..aef8460
--- /dev/null
+++ b/ray_results/test_experiment/train_model_1ef7c88b_11_batch_size=1024,decoder_int_ff=1024,decoder_int_num_layer=4,decoder_rt_ff=512,decoder_rt_num_layer=4,drop__2024-06-25_16-21-54/progress.csv
@@ -0,0 +1,2 @@
+loss,timestamp,checkpoint_dir_name,should_checkpoint,done,training_iteration,trial_id,date,time_this_iter_s,time_total_s,pid,hostname,node_ip,time_since_restore,iterations_since_restore
+1963.8436942513533,1719326402,checkpoint_000000,True,False,1,1ef7c88b,2024-06-25_16-40-02,1014.7334928512573,1014.7334928512573,69318,r3i5n6,10.159.8.159,1014.7334928512573,1
diff --git a/ray_results/test_experiment/train_model_1ef7c88b_11_batch_size=1024,decoder_int_ff=1024,decoder_int_num_layer=4,decoder_rt_ff=512,decoder_rt_num_layer=4,drop__2024-06-25_16-21-54/result.json b/ray_results/test_experiment/train_model_1ef7c88b_11_batch_size=1024,decoder_int_ff=1024,decoder_int_num_layer=4,decoder_rt_ff=512,decoder_rt_num_layer=4,drop__2024-06-25_16-21-54/result.json
new file mode 100644
index 0000000..9a00620
--- /dev/null
+++ b/ray_results/test_experiment/train_model_1ef7c88b_11_batch_size=1024,decoder_int_ff=1024,decoder_int_num_layer=4,decoder_rt_ff=512,decoder_rt_num_layer=4,drop__2024-06-25_16-21-54/result.json
@@ -0,0 +1 @@
+{"loss": 1963.8436942513533, "timestamp": 1719326402, "checkpoint_dir_name": "checkpoint_000000", "should_checkpoint": true, "done": false, "training_iteration": 1, "trial_id": "1ef7c88b", "date": "2024-06-25_16-40-02", "time_this_iter_s": 1014.7334928512573, "time_total_s": 1014.7334928512573, "pid": 69318, "hostname": "r3i5n6", "node_ip": "10.159.8.159", "config": {"encoder_num_layer": 8, "decoder_rt_num_layer": 4, "decoder_int_num_layer": 4, "embedding_dim": 1024, "encoder_ff": 1024, "decoder_rt_ff": 512, "decoder_int_ff": 1024, "n_head": 2, "drop_rate": 0.14711494497891564, "lr": 0.00018307057253269556, "batch_size": 1024}, "time_since_restore": 1014.7334928512573, "iterations_since_restore": 1}
diff --git a/ray_results/test_experiment/train_model_2465ee2c_11_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop__2024-06-25_17-34-53/events.out.tfevents.1719329807.r8i6n2 b/ray_results/test_experiment/train_model_2465ee2c_11_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop__2024-06-25_17-34-53/events.out.tfevents.1719329807.r8i6n2
new file mode 100644
index 0000000000000000000000000000000000000000..a58bd4ff7d62f770216de0cf32d7ec8ad74425f3
GIT binary patch
literal 1311
zcmah|T}TvB7+q&fn@?8VY1ef9TgL*Khc<%<CD^r)z#fty$cMO$GuL)<=gu-SR)&xu
zK}HY082Dg6uJjO*LPSIn5h&G5LP1my{TNhF*+WM7(j9m9?yjlL+kEFc-?{gE=T`eN
zU;6xEN%;8t4gcWk+m5im*HUz{tMfD_PWztalCsOqUf=AsGY5rMuCd1Lt1ObMAP9a-
z8H_pu1V@vmZHLsmEuHgiqp;hfjQ?6d!gwj8G(!`F8O2VR14#|339R;~Or1C(pKmyJ
zXzt080?F3`;}=S5Cc%{A^R>SlosN2v#)<lFp&uAUf(X<wNk=y@auT|YbO&3=4&B*%
z>%jb$d&R9DeR5>vyAUj6^c^5UZ6V#(iHebhZO63m>VV0il~4P;0bjqp=n<~u2FTC?
zO>BUxsyMnyY*aYLy}8h_&^B9?nKL#yaZ2b}nMu25%hNpE;30?-DaBF@8%6Qi6t?JH
zh|_hEhYLKEdXV^s6PG!u9H3Wm)C5&Iha<csLb--jQ=@mLKBK-qTC0IjevJ$mD5(r$
zi$VeLXCce$u7+S<DZkkdf@Ya1L@(w_Q7$%mv8L(dB+_()mgs3P$*8X?haiys&mU9*
zT+&qsC?P0;6)nQc0jLlQLnb#sow!Me)==D#$n{VwZX5!+5$eUwmMrgtX7T?^oC+`g
zB;Nj>{+MbPCbIkc%)ti^>T>(Dv2D{7Alp%>!8++PyB>4!kjw3Hh6~^3peoN>Q3!l_
zZm2isU&UZs)*@QW1SS%P@S3eLY%C59>sZe4$~e@n4G05=<Mh$4yL;B+i$jIGJ6d>D
zHPm@&aVF*-rXU8axjPQq^Yg%f>#IO6whW-hmo~^M>sg8&A5q@#_a5Q1`{|>?Y}Wl&
bh(h>Z?=jIb&eM$Ld@q)`twk@>OUM2KVe^oP
literal 0
HcmV?d00001
diff --git a/ray_results/test_experiment/train_model_2465ee2c_11_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop__2024-06-25_17-34-53/params.json b/ray_results/test_experiment/train_model_2465ee2c_11_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop__2024-06-25_17-34-53/params.json
new file mode 100644
index 0000000..534a5f9
--- /dev/null
+++ b/ray_results/test_experiment/train_model_2465ee2c_11_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop__2024-06-25_17-34-53/params.json
@@ -0,0 +1,13 @@
+{
+ "batch_size": 1024,
+ "decoder_int_ff": 512,
+ "decoder_int_num_layer": 1,
+ "decoder_rt_ff": 1024,
+ "decoder_rt_num_layer": 2,
+ "drop_rate": 0.3089996310219745,
+ "embedding_dim": 64,
+ "encoder_ff": 2048,
+ "encoder_num_layer": 4,
+ "lr": 0.062282868368169514,
+ "n_head": 8
+}
\ No newline at end of file
diff --git a/ray_results/test_experiment/train_model_2465ee2c_11_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop__2024-06-25_17-34-53/params.pkl b/ray_results/test_experiment/train_model_2465ee2c_11_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop__2024-06-25_17-34-53/params.pkl
new file mode 100644
index 0000000000000000000000000000000000000000..bcc16a0a09e990a2b6020c319aed895424b217d0
GIT binary patch
literal 224
zcmZo*oqCl40&1sd^a!TrCFiH47RBe4=EmnFR;CtB@n-1}NlAr>7L_1NF!hMSBr@|*
zWEgvRQ*)D2Q&KYX(&JMybEkMa^l-t9PD`8O%fQjY3)2fU6fDHj!v_-r83z_)>S4=^
z&qz&7nc~gS!<kZ)Ul3oESdu!$-Tv~iWy!fGpY$;06ajhbA9uOdi0$d&N=htA&WJD0
MtV#u1!cwXS0RBW&e*gdg
literal 0
HcmV?d00001
diff --git a/ray_results/test_experiment/train_model_2465ee2c_11_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop__2024-06-25_17-34-53/progress.csv b/ray_results/test_experiment/train_model_2465ee2c_11_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop__2024-06-25_17-34-53/progress.csv
new file mode 100644
index 0000000..69e0cb3
--- /dev/null
+++ b/ray_results/test_experiment/train_model_2465ee2c_11_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop__2024-06-25_17-34-53/progress.csv
@@ -0,0 +1,2 @@
+loss,timestamp,checkpoint_dir_name,should_checkpoint,done,training_iteration,trial_id,date,time_this_iter_s,time_total_s,pid,hostname,node_ip,time_since_restore,iterations_since_restore
+2699.211756428396,1719329892,checkpoint_000000,True,False,1,2465ee2c,2024-06-25_17-38-12,85.10484099388123,85.10484099388123,2195253,r8i6n2,10.159.28.60,85.10484099388123,1
diff --git a/ray_results/test_experiment/train_model_2465ee2c_11_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop__2024-06-25_17-34-53/result.json b/ray_results/test_experiment/train_model_2465ee2c_11_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop__2024-06-25_17-34-53/result.json
new file mode 100644
index 0000000..ea44c5a
--- /dev/null
+++ b/ray_results/test_experiment/train_model_2465ee2c_11_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop__2024-06-25_17-34-53/result.json
@@ -0,0 +1 @@
+{"loss": 2699.211756428396, "timestamp": 1719329892, "checkpoint_dir_name": "checkpoint_000000", "should_checkpoint": true, "done": false, "training_iteration": 1, "trial_id": "2465ee2c", "date": "2024-06-25_17-38-12", "time_this_iter_s": 85.10484099388123, "time_total_s": 85.10484099388123, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 4, "decoder_rt_num_layer": 2, "decoder_int_num_layer": 1, "embedding_dim": 64, "encoder_ff": 2048, "decoder_rt_ff": 1024, "decoder_int_ff": 512, "n_head": 8, "drop_rate": 0.3089996310219745, "lr": 0.062282868368169514, "batch_size": 1024}, "time_since_restore": 85.10484099388123, "iterations_since_restore": 1}
diff --git a/ray_results/test_experiment/train_model_25a9f43a_13_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_17-38-12/events.out.tfevents.1719330099.r8i6n2 b/ray_results/test_experiment/train_model_25a9f43a_13_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_17-38-12/events.out.tfevents.1719330099.r8i6n2
new file mode 100644
index 0000000000000000000000000000000000000000..581d2e5eb098e8e17386809ef437026c18d7184c
GIT binary patch
literal 405
zcmb1OfPlsI-b$QV+RuG<&b#R-#hX-=n3<>NT9%quVr8^1|Me1OsJadF%bhu=aUV(m
zsS{w-65|r#;w?(7)GsN`OV!WGFD@2cIcI^(&*H7`44_)3e!URS`CJgLMVU*HOA4-~
zI3vF_CnY{PBQ-g@AU`v&M3jM{!Tx09_G>6s#)!c6qgt7gpO*^M!oZ;AntwqbYGrP$
zYyhXSC|rvYmjah0+{%*7+|>AzjLhQr%#zfi_+nAElk=R|QchhrgleDL-29UB5<<HQ
zmlBsWs`lc{yyVpQqSWG&{GwE_0S%iRi_D=0Jb!U2lCw+<Zod|n2A3k-0FbSTC7Jno
P#qp@dfPBusFDnNCaL|PU
literal 0
HcmV?d00001
diff --git a/ray_results/test_experiment/train_model_25a9f43a_13_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_17-38-12/events.out.tfevents.1719330869.r8i6n2 b/ray_results/test_experiment/train_model_25a9f43a_13_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_17-38-12/events.out.tfevents.1719330869.r8i6n2
new file mode 100644
index 0000000000000000000000000000000000000000..f004f6238575b423ce99b9281e20c0ba39f98197
GIT binary patch
literal 405
zcmb1OfPlsI-b$R7*{Qs{=G}CZ;!P?_%*@ksElbTSu`-&s;4G^$RNaR8<<6XI7C%)0
zsS{w*65|r#;w?(7)GsN`OV!WGFD@2U%x84*JITJy0IFr`*9-BStCqvHD03-tNx`)g
zXXKaWq{Js@q$Xz<<Y(rUh%zuV*e^5RUx#Al=9O^$s8*)r=cNL*Ffg$Ej(nmIwK6wW
zHh|N06<muFmjah0+{%*7+|>AzjLhQr%#zfi_+n9?zq6fKwcOVjLbcCrZhpym2%%kt
zONmPwReNz}UUF)DQEG8Xeo-pefQe@}#+pM7c>dy4B&XSGxcypO8eEES13<PWmSpDV
P7006*1M<0?qVsJ4hP{Lj
literal 0
HcmV?d00001
diff --git a/ray_results/test_experiment/train_model_25a9f43a_13_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_17-38-12/events.out.tfevents.1719331459.r8i6n2 b/ray_results/test_experiment/train_model_25a9f43a_13_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_17-38-12/events.out.tfevents.1719331459.r8i6n2
new file mode 100644
index 0000000000000000000000000000000000000000..0a564b32bc39785eb520b097cf4fdc0c50d185c7
GIT binary patch
literal 1676
zcma)6TSyd97~W>pnjhB7<XWv-x?0Fmt`-!Owp&XV6iMx=2N}klV>{}e*=9!VN`*v$
zAO=Zj<t3%3Afbn(+e3jwSVRQXOMx%8TING`!HCYdvuAhAOY=7W|9}5?&i8-+S!;=W
z182*fY<ua~zWY7z_c&~|iXh>_ea#4KJGV}tnzwmI-B53fk8P2z4C{5NR#yh3gG~`m
zxV0vX+>NrT+K03^zHf(ZBSi+JbLJJ#mSYOZgIsV>h#HWa8b#h8K>mi1EMd)Vv7Ga~
z_<8=NiSk;A;?0zzj1hrYEaE16yW5}))?`<*+m2C)6LMf3rPQP#;<bRJ@{)!WUbRo<
zKiWI_Wz<`2z;Cp+hS{MQJRh8pX@;v3_9I?FswOLFuEFWHhVTxf!T6`=N7yY2<##~=
z<kAK@tDs3TR(W$9{pM$1r(n~~M|O3M&WW9{kl25^AN!{E_kT8tJyk*>|0DLp4;3jU
z%H~pvUqtK@U(<Dy*t0RnGGZs@_wC(&aqx^uY;8AXTvqIpgN44sKf(u1Vuxc5{#$H|
z_i|Nt$<T_(&^m5()w2~<8NYS0U4vHWw?aA}2nmW1RCzZ#9zqH+4%W&pS>b{eRvIbp
z7sGvLoXG&Sg60p9k)A|3RwoC^BJ#^3F&Hs5Usp$B>%bl#!%acHQ8<AVl1c+xG^H5T
zYau-@6kluuR#fB=Ph6x^5iAPUB2kp^QC^gSBt%YwaD;u1*$!#Z|M*5pz%=UY07;k+
zAVJKry3!zpGjFtOJ!El9Z0tOUzLBnM$mAB^IM)Wq=9bzh*G9<Wmfx5|_3t{dF~28Y
zZ|-C#qVpSL;3ETBvH7V;RmsZnswNPDC0r+$J!aqm6Px25#&Mg0j5uvMHf`qW-QEL}
z;~rQYm52l*hAHxbW8TzguEPuK7m%FhQ@xNm-yk%3&P!g%1!qr7rah3N&yECoXE%=x
zx0QPI!<cM9iz~gbCf*M;L5N5qW?Pyd$9FE`Ra!IcN&Hf9tHgNmp4}U$W^<~I?I{K~
aIR4chVKJgS@mNgv1Shj<#nYbLQ-1-e3KOpY
literal 0
HcmV?d00001
diff --git a/ray_results/test_experiment/train_model_25a9f43a_13_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_17-38-12/params.json b/ray_results/test_experiment/train_model_25a9f43a_13_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_17-38-12/params.json
new file mode 100644
index 0000000..e292b13
--- /dev/null
+++ b/ray_results/test_experiment/train_model_25a9f43a_13_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_17-38-12/params.json
@@ -0,0 +1,13 @@
+{
+ "batch_size": 1024,
+ "decoder_int_ff": 512,
+ "decoder_int_num_layer": 1,
+ "decoder_rt_ff": 1024,
+ "decoder_rt_num_layer": 1,
+ "drop_rate": 0.7474706763430264,
+ "embedding_dim": 64,
+ "encoder_ff": 512,
+ "encoder_num_layer": 4,
+ "lr": 0.00045083297498032556,
+ "n_head": 16
+}
\ No newline at end of file
diff --git a/ray_results/test_experiment/train_model_25a9f43a_13_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_17-38-12/params.pkl b/ray_results/test_experiment/train_model_25a9f43a_13_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_17-38-12/params.pkl
new file mode 100644
index 0000000000000000000000000000000000000000..7004f92dc9779858835e886d54e70af12fbc782f
GIT binary patch
literal 224
zcmZo*oqCl40&1sd^a!TrCFiH47RBe4=EmnFR;CtB@n-1}NlAr>7L_1NF!qSTBr@~R
zWO!3^lTuSsGV{{oQ!;a>csulP!HiBzo8rsB)WZwY3p5lg#L~kD69O3rRmPSVpOKoF
zGR0e<hcl%pzaYLSu_SehyZ!Um?(;WK|JuWpQv~GMcJFwk^Lu^|S5jh0az=b{W>qTC
I5|&au01UWQGynhq
literal 0
HcmV?d00001
diff --git a/ray_results/test_experiment/train_model_25a9f43a_13_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_17-38-12/progress.csv b/ray_results/test_experiment/train_model_25a9f43a_13_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_17-38-12/progress.csv
new file mode 100644
index 0000000..47d8699
--- /dev/null
+++ b/ray_results/test_experiment/train_model_25a9f43a_13_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_17-38-12/progress.csv
@@ -0,0 +1,5 @@
+loss,timestamp,checkpoint_dir_name,should_checkpoint,done,training_iteration,trial_id,date,time_this_iter_s,time_total_s,pid,hostname,node_ip,time_since_restore,iterations_since_restore
+1284.8956039308564,1719330178,checkpoint_000000,True,False,1,25a9f43a,2024-06-25_17-42-59,79.39262270927429,79.39262270927429,2195253,r8i6n2,10.159.28.60,79.39262270927429,1
+517.7363889198604,1719330947,checkpoint_000001,True,False,2,25a9f43a,2024-06-25_17-55-47,77.99471831321716,157.38734102249146,2195253,r8i6n2,10.159.28.60,77.99471831321716,1
+424.9116310660295,1719331554,checkpoint_000002,True,False,3,25a9f43a,2024-06-25_18-05-54,94.57213068008423,251.95947170257568,2195253,r8i6n2,10.159.28.60,94.57213068008423,1
+381.2108863169753,1719331634,checkpoint_000003,True,False,4,25a9f43a,2024-06-25_18-07-14,80.07035231590271,332.0298240184784,2195253,r8i6n2,10.159.28.60,174.64248299598694,2
diff --git a/ray_results/test_experiment/train_model_25a9f43a_13_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_17-38-12/result.json b/ray_results/test_experiment/train_model_25a9f43a_13_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_17-38-12/result.json
new file mode 100644
index 0000000..e7cb0d2
--- /dev/null
+++ b/ray_results/test_experiment/train_model_25a9f43a_13_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_17-38-12/result.json
@@ -0,0 +1,4 @@
+{"loss": 1284.8956039308564, "timestamp": 1719330178, "checkpoint_dir_name": "checkpoint_000000", "should_checkpoint": true, "done": false, "training_iteration": 1, "trial_id": "25a9f43a", "date": "2024-06-25_17-42-59", "time_this_iter_s": 79.39262270927429, "time_total_s": 79.39262270927429, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 4, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 64, "encoder_ff": 512, "decoder_rt_ff": 1024, "decoder_int_ff": 512, "n_head": 16, "drop_rate": 0.7474706763430264, "lr": 0.00045083297498032556, "batch_size": 1024}, "time_since_restore": 79.39262270927429, "iterations_since_restore": 1}
+{"loss": 517.7363889198604, "timestamp": 1719330947, "checkpoint_dir_name": "checkpoint_000001", "should_checkpoint": true, "done": false, "training_iteration": 2, "trial_id": "25a9f43a", "date": "2024-06-25_17-55-47", "time_this_iter_s": 77.99471831321716, "time_total_s": 157.38734102249146, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 4, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 64, "encoder_ff": 512, "decoder_rt_ff": 1024, "decoder_int_ff": 512, "n_head": 16, "drop_rate": 0.7474706763430264, "lr": 0.00045083297498032556, "batch_size": 1024}, "time_since_restore": 77.99471831321716, "iterations_since_restore": 1}
+{"loss": 424.9116310660295, "timestamp": 1719331554, "checkpoint_dir_name": "checkpoint_000002", "should_checkpoint": true, "done": false, "training_iteration": 3, "trial_id": "25a9f43a", "date": "2024-06-25_18-05-54", "time_this_iter_s": 94.57213068008423, "time_total_s": 251.95947170257568, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 4, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 64, "encoder_ff": 512, "decoder_rt_ff": 1024, "decoder_int_ff": 512, "n_head": 16, "drop_rate": 0.7474706763430264, "lr": 0.00045083297498032556, "batch_size": 1024}, "time_since_restore": 94.57213068008423, "iterations_since_restore": 1}
+{"loss": 381.2108863169753, "timestamp": 1719331634, "checkpoint_dir_name": "checkpoint_000003", "should_checkpoint": true, "done": false, "training_iteration": 4, "trial_id": "25a9f43a", "date": "2024-06-25_18-07-14", "time_this_iter_s": 80.07035231590271, "time_total_s": 332.0298240184784, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 4, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 64, "encoder_ff": 512, "decoder_rt_ff": 1024, "decoder_int_ff": 512, "n_head": 16, "drop_rate": 0.7474706763430264, "lr": 0.00045083297498032556, "batch_size": 1024}, "time_since_restore": 174.64248299598694, "iterations_since_restore": 2}
diff --git a/ray_results/test_experiment/train_model_25b42194_15_batch_size=2048,decoder_int_ff=2048,decoder_int_num_layer=8,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop_2024-06-25_16-42-52/events.out.tfevents.1719326598.r3i5n6 b/ray_results/test_experiment/train_model_25b42194_15_batch_size=2048,decoder_int_ff=2048,decoder_int_num_layer=8,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop_2024-06-25_16-42-52/events.out.tfevents.1719326598.r3i5n6
new file mode 100644
index 0000000000000000000000000000000000000000..1de1084d20c575948087454049071e447b8ab3cf
GIT binary patch
literal 1311
zcmah|OK1~O6ip{ueHYT2X;RztV>(fh5l!nxC_<v8n+hUTP>ba;&Ac{KlQ&^z(vKU9
zMG?f(jSIonPyHYYcB2&(D}qp6XhA_-h^UAPq9Rst=bL2mlGL_lmwWEH=e~2#RJpRh
zi6c8Dq_m%Wdu#mF_JF6$P&B%^wU1D%c}vCbqx(Wm-{iUe7SuW=iMLtbsuH;peBd#Z
z!LXI4B%IJq)4ylqc<Xei_jsd2IXyRv(a1ec2|)t{IK_<X>4b`7aS}U}(lu)NU9O?X
z{-4igizNNioL?-d>Xa~w%N0@4>#!r4?5x|3uI4Ec1fiNsT3V7|E3TPXvxtG&#s_z_
zbfCwnCWk&YJp37LdcYOef&?|ibW@`-f(>F?x<N7ncBK_#t25x^rzZ!{<@^8{Hb4U(
zV5=&Yrc)CakFj4)o8SMKEXmFp9Xr#D`UgL(x6PI>x#5Bvd^ny`3?*seFd0Y@gS~}X
zZ71At%nd6XNc<y+zXYiapi8l0ai(&Z1l^JdWh#m3Dtj~Y8FzQHS~d6!Yc!q231yHN
z4Dx~}2N_Ox75EBDh0Pv7s-dSadzdZ7xFp#{R8^xrSk;oO#I_+ctDaHzgE#lTa8M}-
z30obYl;Sv1)Sz4TLb+HRGPxdV#U(;Ck79>Ju7etJ@es&spiW%s$nrV}iT^+1<lOTw
z)VbfoZx@@<o!tH23GhaM+Wh@YVwx-k*t8U8utvM}w)+Cy74mnyDFki^P+8!uh`b|T
zuPqP0d=Y_)oJFjd4NPMc0`soMvC$~hFJL*xzek{Een2>QKFU7Y1#iz=7{A=U9V?7J
z+S_-1rZZwMQxpT<%$8YQm<J9FWq^EaIlwlM7Rf5_2}boRJ!iK$AK}c@L_2z#bH53~
Z5ct=7Le;GEEMs}!mr26Pl2aW6+Fymrk)Z$p
literal 0
HcmV?d00001
diff --git a/ray_results/test_experiment/train_model_25b42194_15_batch_size=2048,decoder_int_ff=2048,decoder_int_num_layer=8,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop_2024-06-25_16-42-52/params.json b/ray_results/test_experiment/train_model_25b42194_15_batch_size=2048,decoder_int_ff=2048,decoder_int_num_layer=8,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop_2024-06-25_16-42-52/params.json
new file mode 100644
index 0000000..11cf8d5
--- /dev/null
+++ b/ray_results/test_experiment/train_model_25b42194_15_batch_size=2048,decoder_int_ff=2048,decoder_int_num_layer=8,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop_2024-06-25_16-42-52/params.json
@@ -0,0 +1,13 @@
+{
+ "batch_size": 2048,
+ "decoder_int_ff": 2048,
+ "decoder_int_num_layer": 8,
+ "decoder_rt_ff": 1024,
+ "decoder_rt_num_layer": 1,
+ "drop_rate": 0.3256301932631026,
+ "embedding_dim": 1024,
+ "encoder_ff": 512,
+ "encoder_num_layer": 1,
+ "lr": 0.0018178629337167497,
+ "n_head": 16
+}
\ No newline at end of file
diff --git a/ray_results/test_experiment/train_model_25b42194_15_batch_size=2048,decoder_int_ff=2048,decoder_int_num_layer=8,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop_2024-06-25_16-42-52/params.pkl b/ray_results/test_experiment/train_model_25b42194_15_batch_size=2048,decoder_int_ff=2048,decoder_int_num_layer=8,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop_2024-06-25_16-42-52/params.pkl
new file mode 100644
index 0000000000000000000000000000000000000000..488fd5b39c866d502307fe72e6f5acd8cb2e9bd7
GIT binary patch
literal 225
zcmZo*oqCM{0&1sd^a!TrCFiH47RBe4=EmnFR;CtB@n-B1NlAr>7L}k%h{7Z?^H5|s
zdU#WFlTuSsGV{{oQ!;a>_%g8caKX$@O9S$mdU#=)fu@3mSbF$iLLl?NVjMkedGQ&k
zi78XO1$sDBit-EMixNvxr?}f+xvs#v@?BpKQ%(_(7kgrLS;)}|JzPnNCCM4_#hF#9
JKub7E^#JqlQ_cVY
literal 0
HcmV?d00001
diff --git a/ray_results/test_experiment/train_model_25b42194_15_batch_size=2048,decoder_int_ff=2048,decoder_int_num_layer=8,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop_2024-06-25_16-42-52/progress.csv b/ray_results/test_experiment/train_model_25b42194_15_batch_size=2048,decoder_int_ff=2048,decoder_int_num_layer=8,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop_2024-06-25_16-42-52/progress.csv
new file mode 100644
index 0000000..ef0228f
--- /dev/null
+++ b/ray_results/test_experiment/train_model_25b42194_15_batch_size=2048,decoder_int_ff=2048,decoder_int_num_layer=8,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop_2024-06-25_16-42-52/progress.csv
@@ -0,0 +1,2 @@
+loss,timestamp,checkpoint_dir_name,should_checkpoint,done,training_iteration,trial_id,date,time_this_iter_s,time_total_s,pid,hostname,node_ip,time_since_restore,iterations_since_restore
+5061.415592704232,1719326792,checkpoint_000000,True,False,1,25b42194,2024-06-25_16-46-32,194.36064219474792,194.36064219474792,122466,r3i5n6,10.159.8.159,194.36064219474792,1
diff --git a/ray_results/test_experiment/train_model_25b42194_15_batch_size=2048,decoder_int_ff=2048,decoder_int_num_layer=8,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop_2024-06-25_16-42-52/result.json b/ray_results/test_experiment/train_model_25b42194_15_batch_size=2048,decoder_int_ff=2048,decoder_int_num_layer=8,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop_2024-06-25_16-42-52/result.json
new file mode 100644
index 0000000..2bed455
--- /dev/null
+++ b/ray_results/test_experiment/train_model_25b42194_15_batch_size=2048,decoder_int_ff=2048,decoder_int_num_layer=8,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop_2024-06-25_16-42-52/result.json
@@ -0,0 +1 @@
+{"loss": 5061.415592704232, "timestamp": 1719326792, "checkpoint_dir_name": "checkpoint_000000", "should_checkpoint": true, "done": false, "training_iteration": 1, "trial_id": "25b42194", "date": "2024-06-25_16-46-32", "time_this_iter_s": 194.36064219474792, "time_total_s": 194.36064219474792, "pid": 122466, "hostname": "r3i5n6", "node_ip": "10.159.8.159", "config": {"encoder_num_layer": 1, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 8, "embedding_dim": 1024, "encoder_ff": 512, "decoder_rt_ff": 1024, "decoder_int_ff": 2048, "n_head": 16, "drop_rate": 0.3256301932631026, "lr": 0.0018178629337167497, "batch_size": 2048}, "time_since_restore": 194.36064219474792, "iterations_since_restore": 1}
diff --git a/ray_results/test_experiment/train_model_3510e84b_15_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop__2024-06-25_17-42-59/events.out.tfevents.1719330290.r8i6n2 b/ray_results/test_experiment/train_model_3510e84b_15_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop__2024-06-25_17-42-59/events.out.tfevents.1719330290.r8i6n2
new file mode 100644
index 0000000000000000000000000000000000000000..42f4d6e380450ceb2064f42298987f704a563cff
GIT binary patch
literal 405
zcmb1OfPlsI-b$S8VU0CA=iPLa;!P?_%*@ksElbTSu`)WfY5Q|!sJadF%bhu+RX6m3
z)Cn+ZiE)W=@fIak>X($}rRwM87Z;0GG<<bwYuIq!0IFr`*9-BS`*h%1l(`hSq~KbL
zGxAGwQsR>{Qj@a_@-y>FL>U+w?B8bc{6ewviXL1)s+B4Ed8t4x3=BV<|4h|~TA3Rw
z8^D>V57(l^rNAW#x3VNNH#NQ_BeOU@vm~`BzF723@kFNsb5F$?LbcCrZhp!62%%kt
zONmPwReNz}UUF)DQEG8Xeo-pefDFecAIzZ!Jb!U2k~7QzZod|n2A3k-0FbSTC7Jno
P#qp@dfP6lC;-pgm2P}zS
literal 0
HcmV?d00001
diff --git a/ray_results/test_experiment/train_model_3510e84b_15_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop__2024-06-25_17-42-59/events.out.tfevents.1719331058.r8i6n2 b/ray_results/test_experiment/train_model_3510e84b_15_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop__2024-06-25_17-42-59/events.out.tfevents.1719331058.r8i6n2
new file mode 100644
index 0000000000000000000000000000000000000000..0e0d499863dcfe68eed6184f569508c2a3993d90
GIT binary patch
literal 1311
zcmah|T}TvB7}eP@=Zn>L+%?@*(^;dK2UjBqM40XG#h{`GA)>CcJ9pcu-8;+Nv9h2<
z5qc3Ki0DDfthat70wE!y9x5ZSsD}&+deM4G2%{jVJMQe=T~nL4`ObH~bMJS~ttrWV
zW9RqE;?wW_9S=v|?+AKhmS&KaJtr}-+qe4ncfSg|edE_o?Gi`3KJIh+)(c8C_`z#w
zy%D>I;7H1(G@y<w>~6aMr>oVaTwa();^a|I2}2VEIfW+8o|KN_NgO|sF%4n|N=o|F
zzFS?PBFSgnoL?-dn*=XX&Q(3W5^*KdN9qrV0~)8u&<M3$(l*i<*-3*U!^Re(0r~Bv
zoxjzWQ*ADNcwpd**b?LPW{@FNOs58kV`O1!n-*RikeVYy+uZ@Pb5n=J2l)XCY=I^|
zz){s~!z2_Hk8y7PW@_EgcxiUd#o?>R#g^&pIh30o81;Z3B{Q0(r74Qw(-~~ByAZqc
zf(HgYu-1j7UxM^QkShRUnjKFvl{2`}BTG=B<G87_JF`8bM1s|7Ay8N&J!zEEda=bI
zA9!<+<#yMAzo1mu>;+M`%nV`|bEFs-r(;;x4RQ?WMw*q_X)wvEFDeJXm-{aqR0cxI
zQ3og^D2X+_(WCgFQYsFa(g1bRDj`}zaYCZhLr7XV1j<IJmsT@b*#u$f|0d3$pR>gE
z@5#3t?IO<k_gR2X0@UUGqnJ{b0z_?%E!ZFlv-6PvlS1CddqVKG0M!M%l|<j&ZOygq
zuivRqm0J-jW&^W63c)3=aqM~&8kSi(#~C-YG$0)8i?T<%toOWSUWH1>J5~t4nA!X_
z)2}**DT)DaZjHi*!aQ(*ost3h*cJlv3R&en^XkN%!9<7q2%k@k9Td;zw%-O32>!c!
VO!VyLS;q2vUn2|Ur5_HL)4xW#nC1Wg
literal 0
HcmV?d00001
diff --git a/ray_results/test_experiment/train_model_3510e84b_15_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop__2024-06-25_17-42-59/params.json b/ray_results/test_experiment/train_model_3510e84b_15_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop__2024-06-25_17-42-59/params.json
new file mode 100644
index 0000000..b2801fd
--- /dev/null
+++ b/ray_results/test_experiment/train_model_3510e84b_15_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop__2024-06-25_17-42-59/params.json
@@ -0,0 +1,13 @@
+{
+ "batch_size": 2048,
+ "decoder_int_ff": 512,
+ "decoder_int_num_layer": 1,
+ "decoder_rt_ff": 2048,
+ "decoder_rt_num_layer": 1,
+ "drop_rate": 0.3408951993486181,
+ "embedding_dim": 16,
+ "encoder_ff": 2048,
+ "encoder_num_layer": 4,
+ "lr": 0.009004430142657121,
+ "n_head": 4
+}
\ No newline at end of file
diff --git a/ray_results/test_experiment/train_model_3510e84b_15_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop__2024-06-25_17-42-59/params.pkl b/ray_results/test_experiment/train_model_3510e84b_15_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop__2024-06-25_17-42-59/params.pkl
new file mode 100644
index 0000000000000000000000000000000000000000..c983efd0bd3463d016ccf3fdf090ad866722f9f5
GIT binary patch
literal 224
zcmZo*oqCl40&1sd^a!TrCFiH47RBe4=EmnFR;CtB@n-1}NlAr>7L_1NF!qSTBr@~R
zWO!3^lTuSsGV{{oQ!;a>cnkD!!HiBzo8rsB(ZdVV3p5le#0L`s83z_)>S4=^&qz&7
z0Xc&+r6|83z9_LIb&9+F)r(e=DkkfCm~x7MyrzOz>bFiR_i!a8mLzAy7iU(bf-EW3
F0|2ZKQm+62
literal 0
HcmV?d00001
diff --git a/ray_results/test_experiment/train_model_3510e84b_15_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop__2024-06-25_17-42-59/progress.csv b/ray_results/test_experiment/train_model_3510e84b_15_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop__2024-06-25_17-42-59/progress.csv
new file mode 100644
index 0000000..7c7780a
--- /dev/null
+++ b/ray_results/test_experiment/train_model_3510e84b_15_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop__2024-06-25_17-42-59/progress.csv
@@ -0,0 +1,3 @@
+loss,timestamp,checkpoint_dir_name,should_checkpoint,done,training_iteration,trial_id,date,time_this_iter_s,time_total_s,pid,hostname,node_ip,time_since_restore,iterations_since_restore
+1964.014622485544,1719330362,checkpoint_000000,True,False,1,3510e84b,2024-06-25_17-46-02,72.72616767883301,72.72616767883301,2195253,r8i6n2,10.159.28.60,72.72616767883301,1
+1964.9452173428272,1719331131,checkpoint_000001,True,True,2,3510e84b,2024-06-25_17-58-51,73.41431927680969,146.1404869556427,2195253,r8i6n2,10.159.28.60,73.41431927680969,1
diff --git a/ray_results/test_experiment/train_model_3510e84b_15_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop__2024-06-25_17-42-59/result.json b/ray_results/test_experiment/train_model_3510e84b_15_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop__2024-06-25_17-42-59/result.json
new file mode 100644
index 0000000..19328aa
--- /dev/null
+++ b/ray_results/test_experiment/train_model_3510e84b_15_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop__2024-06-25_17-42-59/result.json
@@ -0,0 +1,2 @@
+{"loss": 1964.014622485544, "timestamp": 1719330362, "checkpoint_dir_name": "checkpoint_000000", "should_checkpoint": true, "done": false, "training_iteration": 1, "trial_id": "3510e84b", "date": "2024-06-25_17-46-02", "time_this_iter_s": 72.72616767883301, "time_total_s": 72.72616767883301, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 4, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 2048, "decoder_rt_ff": 2048, "decoder_int_ff": 512, "n_head": 4, "drop_rate": 0.3408951993486181, "lr": 0.009004430142657121, "batch_size": 2048}, "time_since_restore": 72.72616767883301, "iterations_since_restore": 1}
+{"loss": 1964.9452173428272, "timestamp": 1719331131, "checkpoint_dir_name": "checkpoint_000001", "should_checkpoint": true, "done": true, "training_iteration": 2, "trial_id": "3510e84b", "date": "2024-06-25_17-58-51", "time_this_iter_s": 73.41431927680969, "time_total_s": 146.1404869556427, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 4, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 2048, "decoder_rt_ff": 2048, "decoder_int_ff": 512, "n_head": 4, "drop_rate": 0.3408951993486181, "lr": 0.009004430142657121, "batch_size": 2048}, "time_since_restore": 73.41431927680969, "iterations_since_restore": 1}
diff --git a/ray_results/test_experiment/train_model_375ed3d7_19_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=2,decoder_rt_ff=1024,decoder_rt_num_layer=4,drop__2024-06-25_16-56-12/events.out.tfevents.1719327790.r3i5n6 b/ray_results/test_experiment/train_model_375ed3d7_19_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=2,decoder_rt_ff=1024,decoder_rt_num_layer=4,drop__2024-06-25_16-56-12/events.out.tfevents.1719327790.r3i5n6
new file mode 100644
index 0000000000000000000000000000000000000000..08e5459ca585e45531b5d6b1109a0c8472d3c574
GIT binary patch
literal 405
zcmb1OfPlsI-b$QHuFc@yKJTWZ6mL>dVrHJ6YguYuiIq`o=Z_3!sJadF%bhuQ+%uE~
zsS{w-65|r#;w?(7)GsN`OV!WGFD@31O!($<BYxsL1E`j%UoXUSGCqN8QRY(Ql7ed~
z&d4v#Nr_L+NKMWz$j{6x5oKU#u&@5Y`5nc|j%RTFs8*)r=cNL*Ffh~?%6!m=TA3Rw
z8^EdZ9Ii!)OMy!gZe>YkZfbl<MrLt*W=U#Me6i@8qdiVjZd&~{gleDL-29Sr8A7`X
zmlBsWs`lc{yyVpQqSWG&{GwE_0T+!X{4j?a@chN8NKT~}aQn5mG`JMu27qi$EXmBz
PD~?As2ITXwSLWORyf%f$
literal 0
HcmV?d00001
diff --git a/ray_results/test_experiment/train_model_375ed3d7_19_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=2,decoder_rt_ff=1024,decoder_rt_num_layer=4,drop__2024-06-25_16-56-12/params.json b/ray_results/test_experiment/train_model_375ed3d7_19_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=2,decoder_rt_ff=1024,decoder_rt_num_layer=4,drop__2024-06-25_16-56-12/params.json
new file mode 100644
index 0000000..970fbc4
--- /dev/null
+++ b/ray_results/test_experiment/train_model_375ed3d7_19_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=2,decoder_rt_ff=1024,decoder_rt_num_layer=4,drop__2024-06-25_16-56-12/params.json
@@ -0,0 +1,13 @@
+{
+ "batch_size": 1024,
+ "decoder_int_ff": 512,
+ "decoder_int_num_layer": 2,
+ "decoder_rt_ff": 1024,
+ "decoder_rt_num_layer": 4,
+ "drop_rate": 0.29850788173794873,
+ "embedding_dim": 16,
+ "encoder_ff": 1024,
+ "encoder_num_layer": 1,
+ "lr": 0.017371208825505195,
+ "n_head": 1
+}
\ No newline at end of file
diff --git a/ray_results/test_experiment/train_model_375ed3d7_19_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=2,decoder_rt_ff=1024,decoder_rt_num_layer=4,drop__2024-06-25_16-56-12/params.pkl b/ray_results/test_experiment/train_model_375ed3d7_19_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=2,decoder_rt_ff=1024,decoder_rt_num_layer=4,drop__2024-06-25_16-56-12/params.pkl
new file mode 100644
index 0000000000000000000000000000000000000000..38d5b145945a4bf9e65a628967eafd993a61f6b3
GIT binary patch
literal 224
zcmZo*oqCl40&1sd^a!TrCFiH47RBe4=EmnFR;CtB@n-B1NlAr>7L_1Nu=I$+Br@|*
zWSDw*Q*)D2Q&KYX(&JMybEkL<^l-t9PD`8O%fQmZ3)2fU6e`3A69O3r7GvsR%ZtxQ
zO-unfgEOTlzaYLSu_SehyZvRU181|cBzl-~ih#U{Cl84mtP$+tN=htA&WJD0tV#u0
HQmO|4s+Ce-
literal 0
HcmV?d00001
diff --git a/ray_results/test_experiment/train_model_375ed3d7_19_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=2,decoder_rt_ff=1024,decoder_rt_num_layer=4,drop__2024-06-25_16-56-12/progress.csv b/ray_results/test_experiment/train_model_375ed3d7_19_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=2,decoder_rt_ff=1024,decoder_rt_num_layer=4,drop__2024-06-25_16-56-12/progress.csv
new file mode 100644
index 0000000..0f8fff3
--- /dev/null
+++ b/ray_results/test_experiment/train_model_375ed3d7_19_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=2,decoder_rt_ff=1024,decoder_rt_num_layer=4,drop__2024-06-25_16-56-12/progress.csv
@@ -0,0 +1,2 @@
+loss,timestamp,checkpoint_dir_name,should_checkpoint,done,training_iteration,trial_id,date,time_this_iter_s,time_total_s,pid,hostname,node_ip,time_since_restore,iterations_since_restore
+1971.010832058163,1719327860,checkpoint_000000,True,False,1,375ed3d7,2024-06-25_17-04-20,70.38656640052795,70.38656640052795,168065,r3i5n6,10.159.8.159,70.38656640052795,1
diff --git a/ray_results/test_experiment/train_model_375ed3d7_19_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=2,decoder_rt_ff=1024,decoder_rt_num_layer=4,drop__2024-06-25_16-56-12/result.json b/ray_results/test_experiment/train_model_375ed3d7_19_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=2,decoder_rt_ff=1024,decoder_rt_num_layer=4,drop__2024-06-25_16-56-12/result.json
new file mode 100644
index 0000000..08a067b
--- /dev/null
+++ b/ray_results/test_experiment/train_model_375ed3d7_19_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=2,decoder_rt_ff=1024,decoder_rt_num_layer=4,drop__2024-06-25_16-56-12/result.json
@@ -0,0 +1 @@
+{"loss": 1971.010832058163, "timestamp": 1719327860, "checkpoint_dir_name": "checkpoint_000000", "should_checkpoint": true, "done": false, "training_iteration": 1, "trial_id": "375ed3d7", "date": "2024-06-25_17-04-20", "time_this_iter_s": 70.38656640052795, "time_total_s": 70.38656640052795, "pid": 168065, "hostname": "r3i5n6", "node_ip": "10.159.8.159", "config": {"encoder_num_layer": 1, "decoder_rt_num_layer": 4, "decoder_int_num_layer": 2, "embedding_dim": 16, "encoder_ff": 1024, "decoder_rt_ff": 1024, "decoder_int_ff": 512, "n_head": 1, "drop_rate": 0.29850788173794873, "lr": 0.017371208825505195, "batch_size": 1024}, "time_since_restore": 70.38656640052795, "iterations_since_restore": 1}
diff --git a/ray_results/test_experiment/train_model_3ab2f301_28_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop__2024-06-25_22-10-34/events.out.tfevents.1719346547.r8i6n2 b/ray_results/test_experiment/train_model_3ab2f301_28_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop__2024-06-25_22-10-34/events.out.tfevents.1719346547.r8i6n2
new file mode 100644
index 0000000000000000000000000000000000000000..33636372c728ca899def9f9d20e3bcc1bdaf621c
GIT binary patch
literal 1865
zcma*jVMr5k7{~D|&AOA3i!GdK<rcJ7G&Qgk%<GO`NwjRx2G%v_#SZl@cBi$H8Z9s_
zBC25^nk}hBLGgvocNPiGl6*1AC`c+P^-U3CBwB5a|M9@a-Mzl=^W*=?#{PeguT~iG
zLd(ahNawpFIhty(i{tknZDe_=w8;Hi7ceFM8+dqu#w&@^`q+P2>nvLd9ij2Md{(J}
zXRURDDC+wfowQHLGZ!S1;L;R>S8Ya#iO45%P$GJShPoQY?P1-u=LL?J^cdzZe|;>}
z0Va=YQ4c3;1fKnuU|3tQSZ7U4)}PL^;h*KiNbDpGXj0<pSw`}3BEw0nml5@v>1)&~
zRdA~?k#__F^LX_Zw782f65C)da=e>mysRh*UN(N<MfB~tgNXxQzfCytS^0p4FcbOc
zKy1||aRM(g@J#IG`$i}dygYrk{ZbrzeKPFI857<9T?hf}G&4%nU>EhTw1CN0xle%|
z?0fzPU}x<?lM2{u`w6NrI~)PnH|1Ordo;92VcoP5V3%0XzG`fYJgfb57rgxCxT`si
zot6yy=k-i_cx|->V84l=L=E=L+)yu=JT-y(6xd<wY%ak5ASVjgqfr}mvMG8SU^`!-
zMMdm885VlFc&Zv;FUtp1W0R!qeP9l}ygcLC{Wx}dGVFz?E7W99e;dF)yb&d8u&-S7
zErZE1xle)Z==m`Suq!s9Nd@e(Gj^)dG2jH)V{)#Dy<<B=ckTT(1+YtSw67Z5Pu5PZ
zTm~<HFtqe2j=donwqtRDGUR6R0K0P-C2FwC$ayQ6%nzeJ1@^r)?E!%OL{1d2o5t<b
i>Y%F^U>iozqB6F*m5w$K9|G7n<O8a)yT~v8S?6E>cVz7V
literal 0
HcmV?d00001
diff --git a/ray_results/test_experiment/train_model_3ab2f301_28_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop__2024-06-25_22-10-34/events.out.tfevents.1719354952.r8i6n2 b/ray_results/test_experiment/train_model_3ab2f301_28_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop__2024-06-25_22-10-34/events.out.tfevents.1719354952.r8i6n2
new file mode 100644
index 0000000000000000000000000000000000000000..6bc61b20030cbda1b55aa35ee326d6af18723a77
GIT binary patch
literal 1865
zcmb1OfPlsI-b$Qb_iIeLIPa#T6mL>dVrHJ6YguYuiIq`V#ZzBpsJadF%bhvT&Zt}j
zQYXNsCB`Mf#aonEsb5l>m#UwWUtBD@;>#1Ko%hOT7(lg5{dysuQ(zuki!zrYmlRw}
zaYlY=PD*@oMrv|)L4Iali6{d@gT20Pzb%TD8Vlh1QLRkL&r1brVPJ4>OWB|gwK6wW
zHh^;<LW>fY0+%G*%9707)cBH&%;NaWlGLL3V$mZFO-_li9Fm4m?Q@%(UvgS4gxjdX
zrNkwTs=YWfFF7^7D7CmGzbF-KK&j89ZRSt|p1(L1$+;F`fEJeqmm=H%kgbU&nfZCe
z@u<dtd>(qpa3YG&9cD(Z2M0Dgw!ofVf6M8<+V}M+f$cO4u7#MuR%BXJfnw!vgnk@>
zJ;Oz43rb+O&W2k_Ah5ZA`#LFH+M9$D*vxa_+6e}>SKUh|r%Nh@D1lvpFo5X5W^m9G
zTPuO$^K-i__kaVN16yDVr(bfC-n(5LC9t{nz_kz)*fXUnKA~7S4WS=LU^Bf{JC73B
zGJD}x5(w;Te|(+ZF5ba_64=ua+6e}Bl%0li+d92Gl)&cR2e+T-z;<vbmXUdb;&T)3
ze<#3!&513ruWG(^I+S~15=vm3@xZkZ6WBWi=ZT_Nxd)*iM_?N@eXBwVY)4+Wl>`FY
z#NNm0d)4W7l)&DN&`vP0)%~KK4SxPzhZ5L&d~o}T4(tYps0VCoQGD)Ya{Veeu(_}W
z_Qh?-oi--4$)W^yfGJ!HF@e2z(u8yrD=#AS;|S~+(Mmp)zz#BlTS*|WFZK93F&1lY
lK?&?r2<-#|dtF4k^BTk3%qW3vWe&HW=)hj!z{NQ|2>?(_XnFtu
literal 0
HcmV?d00001
diff --git a/ray_results/test_experiment/train_model_3ab2f301_28_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop__2024-06-25_22-10-34/events.out.tfevents.1719357872.r8i6n2 b/ray_results/test_experiment/train_model_3ab2f301_28_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop__2024-06-25_22-10-34/events.out.tfevents.1719357872.r8i6n2
new file mode 100644
index 0000000000000000000000000000000000000000..72a5f6bff7d168e3aef88a728e9af5d4168b3f80
GIT binary patch
literal 4596
zcma*reM}Q)90%}nHEJK_tpY+7t*vge%dDcpq9&B9QxS?Uaa)|xv(PIP+AHfFFkgpZ
z5H!m)QRbFRd2_>e5QlU420F!;DQ@75a~o<lol%6jDYB^9wUp=jbFFv$$34&Y`S$tT
z6UftOS>I3fhbbCG+#H3yefzZ}RdBk?Y-1+8l}|I=w73oBL!N5A^XkeAjEdc7#;<nI
z3mc+~LJ=s~WnQ7@@)%lgcd~3`{`2q11$o<-P4q^}9$lss^aUKzqA^H?BW#v4&u*bI
zvgnMQT&IoUB4x6|#66ks+ys>kZ8#4rEl!5!BQjY@s^JH{x6-jRcD^FD9Y-`sjbd;m
zXLHaLmt|uq8%Mh+HZnPA9l5o)W78yWd|Oe`T}8VPAB!|-1dOvbCWEG2G|M?%v}XbG
zW_?D2cY!;9o?on(cLn$B&}(Q6UclXI=4?)erQkB|oomiUC4rrvw%siCU=Q_&{q$KQ
zX|}X%0oc9SI3fjm$Gfd3L1l3c&IiD5JluN;V29apL<GBuOCdAwl#Bt`B|=;jd)JZM
zq)Kz*eSrN;SU@ti%+PrH`&h7Z^}X|D9_(O$*ps##Be{RN#sci5`#2&6yZOwltDy3N
zkPm=8%93pY*o!)Gr3m)H&1vMFMMu^G?2AHN6uX&t)wpcq-rWG(@BsHq#x@viMWxHZ
z&O@_5t@2=p_`^0mI7G%jVHX4Jgd7}^g1v7FcLP*53;6)pb{!c4u#@e$QUrV8!BjGC
z7D52~xDXe`F4DheJk)X82(aTFxL-1Mp`j`N?P#!b^3j}H4|b?O>}v%F$f)U6vjH~!
z7)PXF-yIOY0aQlx;CujVMcnm|0Cu?$5y9T5O(6%hM#TW^=qI>Q6npaO5@Qe7IUHbb
z78a0<U14Y|ON9R*gZAl<da%R%VK)&+$($vB-v-#){Wu~8`_Z#E<^$|KLOuX?e#2KA
z0X9{OD@CxYt!boc`tRic`zIkTiaj{F!nnq@H638jt;7A2v6~Fm+EY8h&da*So%3La
z`@?SQ-cPPEt%rX`8oF^r3U+aWsR&di{Dt!Yu*a^V-UQgkgop@s=dyX^DwC-mU{8IB
z8%44In9^u0u-?)D>>6PK$=EjxX(idS!OnlFynn@mt@MX|KwU?+?yngSPBu}EBT}%x
zjZe^m%3>iO09(1}+EjpzcHl}8?9tX#@<2<+a)7-~h>K$HYHl<BTUxXdU_TZXkc=%$
zYCrY$46yTuN9%8Su!s4>z9O$Bf36>p2C#=8#}O&mmgoo7pz;eL9{~GaPbmqoV^82p
z5$uqqspPmXqiO*5W+5($onq=VUV7d*1Yi$s!u^u5!;<WSVhFJFPk!Cd<-v~dhkdbX
z7kTyUyh?yw_ZyB#!On~?j02VN9XKBVJBOOk3b2m}5fSW>YfNOt$D>~X*pqMIMp5j7
zwep!WCwx{0uy+azNXAwtwe*}*)dlrkjy7%kJX`T2vBiU}+aX7lauh*j<(ge)2TSSc
z4{~W2e?eooxutScEJuU9L&{!4`IyjzAe3(AGP3xq(@I(`*C<hlh0btV_=_BWd8DjX
z{#gu)^nJ$UIVih%1?}R8!clPlkjpze8b$a#@|_%v6c(2=m*Q`0?kYY@JJM;3#l~b(
z7Mp{A#D5x^>1+2iN22il-+dPvhzPs89if2?l|`E^YPl{Pg(~6ASvLxeRKDb9{6fTi
z^VP+o5z4?fTis|Bt9;qbRW}Z4mEt$k;+CfCjQ9PW?5dikP|LmV?-qeN2sBc7e=N<i
z{MD9XIWrGzW30}(#|d<V5Z+@gq1sEJC?8v;BK)#8xudM6G7$~yC*mLW9hmM!q<Z14
z;jsz>8s&!^kIM{b#ES<64;C8uZzBA@dpw2j4t2jf{=wn<OQukD^F1F190NRgq5%!}
zJr8(*|CIcIaBO`CGy%>kp3|MUP|l{!@P4y!_M7U3ifR4&laL;%{%=n+mOgp@j0x@o
NG{oSb#V1~0@E_8->16-_
literal 0
HcmV?d00001
diff --git a/ray_results/test_experiment/train_model_3ab2f301_28_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop__2024-06-25_22-10-34/params.json b/ray_results/test_experiment/train_model_3ab2f301_28_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop__2024-06-25_22-10-34/params.json
new file mode 100644
index 0000000..1e2acc4
--- /dev/null
+++ b/ray_results/test_experiment/train_model_3ab2f301_28_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop__2024-06-25_22-10-34/params.json
@@ -0,0 +1,13 @@
+{
+ "batch_size": 2048,
+ "decoder_int_ff": 512,
+ "decoder_int_num_layer": 1,
+ "decoder_rt_ff": 2048,
+ "decoder_rt_num_layer": 1,
+ "drop_rate": 0.034481700426631456,
+ "embedding_dim": 16,
+ "encoder_ff": 2048,
+ "encoder_num_layer": 2,
+ "lr": 0.0010186128360476897,
+ "n_head": 1
+}
\ No newline at end of file
diff --git a/ray_results/test_experiment/train_model_3ab2f301_28_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop__2024-06-25_22-10-34/params.pkl b/ray_results/test_experiment/train_model_3ab2f301_28_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop__2024-06-25_22-10-34/params.pkl
new file mode 100644
index 0000000000000000000000000000000000000000..5741d83de357f483edaa8b4c62494ccccb584429
GIT binary patch
literal 224
zcmZo*oqCl40&1sd^a!TrCFiH47RBe4=EmnFR;CtB@n-4~NlAr>7L_1NF!qSTBr@~R
zWO!3^lTuSsGV{{oQ!;a>cnkD!!HiBzo8rsB(ZdVV3p5le#0L`s83z_)>S4=^&qz&7
znc~gZ!<kZ)Ul3oESdu!$-G1TnsrNmVFZ3|w6ajew8{$o(-|Xn&N=htA&WJD0tV#u0
HQmO|4#TZj;
literal 0
HcmV?d00001
diff --git a/ray_results/test_experiment/train_model_3ab2f301_28_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop__2024-06-25_22-10-34/progress.csv b/ray_results/test_experiment/train_model_3ab2f301_28_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop__2024-06-25_22-10-34/progress.csv
new file mode 100644
index 0000000..65321cf
--- /dev/null
+++ b/ray_results/test_experiment/train_model_3ab2f301_28_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop__2024-06-25_22-10-34/progress.csv
@@ -0,0 +1,21 @@
+loss,timestamp,checkpoint_dir_name,should_checkpoint,done,training_iteration,trial_id,date,time_this_iter_s,time_total_s,pid,hostname,node_ip,time_since_restore,iterations_since_restore
+217.46355960875985,1719346612,checkpoint_000000,True,False,1,3ab2f301,2024-06-25_22-16-52,65.41611504554749,65.41611504554749,2195253,r8i6n2,10.159.28.60,65.41611504554749,1
+163.8450069878045,1719346664,checkpoint_000001,True,False,2,3ab2f301,2024-06-25_22-17-45,52.40175247192383,117.81786751747131,2195253,r8i6n2,10.159.28.60,117.81786751747131,2
+139.51932639775313,1719346717,checkpoint_000002,True,False,3,3ab2f301,2024-06-25_22-18-37,52.207762479782104,170.02562999725342,2195253,r8i6n2,10.159.28.60,170.02562999725342,3
+125.82796093798059,1719346770,checkpoint_000003,True,False,4,3ab2f301,2024-06-25_22-19-30,53.092029094696045,223.11765909194946,2195253,r8i6n2,10.159.28.60,223.11765909194946,4
+117.45965227742833,1719346823,checkpoint_000004,True,False,5,3ab2f301,2024-06-25_22-20-23,53.195783615112305,276.31344270706177,2195253,r8i6n2,10.159.28.60,276.31344270706177,5
+114.47784069391686,1719355017,checkpoint_000005,True,False,6,3ab2f301,2024-06-26_00-36-57,65.25149631500244,341.5649390220642,2850562,r8i6n2,10.159.28.60,65.25149631500244,1
+109.24919566957969,1719355069,checkpoint_000006,True,False,7,3ab2f301,2024-06-26_00-37-49,51.495158195495605,393.0600972175598,2850562,r8i6n2,10.159.28.60,116.74665451049805,2
+105.2013188159372,1719355120,checkpoint_000007,True,False,8,3ab2f301,2024-06-26_00-38-40,51.496910572052,444.5570077896118,2850562,r8i6n2,10.159.28.60,168.24356508255005,3
+118.58170366662694,1719355171,checkpoint_000008,True,False,9,3ab2f301,2024-06-26_00-39-31,51.06172227859497,495.6187300682068,2850562,r8i6n2,10.159.28.60,219.30528736114502,4
+99.85706461508443,1719355223,checkpoint_000009,True,False,10,3ab2f301,2024-06-26_00-40-23,51.387521266937256,547.006251335144,2850562,r8i6n2,10.159.28.60,270.6928086280823,5
+98.49699257677935,1719357936,checkpoint_000010,True,False,11,3ab2f301,2024-06-26_01-25-36,64.502512216568,611.508763551712,2850562,r8i6n2,10.159.28.60,64.502512216568,1
+93.49606281190407,1719357987,checkpoint_000011,True,False,12,3ab2f301,2024-06-26_01-26-27,50.864012002944946,662.372775554657,2850562,r8i6n2,10.159.28.60,115.36652421951294,2
+94.78943622018409,1719358039,checkpoint_000012,True,False,13,3ab2f301,2024-06-26_01-27-19,51.64423751831055,714.0170130729675,2850562,r8i6n2,10.159.28.60,167.0107617378235,3
+91.46543307567205,1719358090,checkpoint_000013,True,False,14,3ab2f301,2024-06-26_01-28-10,51.174827098846436,765.191840171814,2850562,r8i6n2,10.159.28.60,218.18558883666992,4
+88.73407943605439,1719358141,checkpoint_000014,True,False,15,3ab2f301,2024-06-26_01-29-01,50.784724712371826,815.9765648841858,2850562,r8i6n2,10.159.28.60,268.97031354904175,5
+94.00536857064314,1719358192,checkpoint_000015,True,False,16,3ab2f301,2024-06-26_01-29-52,51.60023093223572,867.5767958164215,2850562,r8i6n2,10.159.28.60,320.57054448127747,6
+87.47035950187623,1719358244,checkpoint_000016,True,False,17,3ab2f301,2024-06-26_01-30-44,51.84270000457764,919.4194958209991,2850562,r8i6n2,10.159.28.60,372.4132444858551,7
+88.06384884090875,1719358295,checkpoint_000017,True,False,18,3ab2f301,2024-06-26_01-31-35,51.09972786903381,970.519223690033,2850562,r8i6n2,10.159.28.60,423.5129723548889,8
+87.50748780017763,1719358346,checkpoint_000018,True,False,19,3ab2f301,2024-06-26_01-32-26,51.09087514877319,1021.6100988388062,2850562,r8i6n2,10.159.28.60,474.6038475036621,9
+84.81797225832001,1719358398,checkpoint_000019,True,False,20,3ab2f301,2024-06-26_01-33-18,51.372178077697754,1072.982276916504,2850562,r8i6n2,10.159.28.60,525.9760255813599,10
diff --git a/ray_results/test_experiment/train_model_3ab2f301_28_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop__2024-06-25_22-10-34/result.json b/ray_results/test_experiment/train_model_3ab2f301_28_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop__2024-06-25_22-10-34/result.json
new file mode 100644
index 0000000..044b635
--- /dev/null
+++ b/ray_results/test_experiment/train_model_3ab2f301_28_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop__2024-06-25_22-10-34/result.json
@@ -0,0 +1,20 @@
+{"loss": 217.46355960875985, "timestamp": 1719346612, "checkpoint_dir_name": "checkpoint_000000", "should_checkpoint": true, "done": false, "training_iteration": 1, "trial_id": "3ab2f301", "date": "2024-06-25_22-16-52", "time_this_iter_s": 65.41611504554749, "time_total_s": 65.41611504554749, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 2, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 2048, "decoder_rt_ff": 2048, "decoder_int_ff": 512, "n_head": 1, "drop_rate": 0.034481700426631456, "lr": 0.0010186128360476897, "batch_size": 2048}, "time_since_restore": 65.41611504554749, "iterations_since_restore": 1}
+{"loss": 163.8450069878045, "timestamp": 1719346664, "checkpoint_dir_name": "checkpoint_000001", "should_checkpoint": true, "done": false, "training_iteration": 2, "trial_id": "3ab2f301", "date": "2024-06-25_22-17-45", "time_this_iter_s": 52.40175247192383, "time_total_s": 117.81786751747131, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 2, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 2048, "decoder_rt_ff": 2048, "decoder_int_ff": 512, "n_head": 1, "drop_rate": 0.034481700426631456, "lr": 0.0010186128360476897, "batch_size": 2048}, "time_since_restore": 117.81786751747131, "iterations_since_restore": 2}
+{"loss": 139.51932639775313, "timestamp": 1719346717, "checkpoint_dir_name": "checkpoint_000002", "should_checkpoint": true, "done": false, "training_iteration": 3, "trial_id": "3ab2f301", "date": "2024-06-25_22-18-37", "time_this_iter_s": 52.207762479782104, "time_total_s": 170.02562999725342, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 2, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 2048, "decoder_rt_ff": 2048, "decoder_int_ff": 512, "n_head": 1, "drop_rate": 0.034481700426631456, "lr": 0.0010186128360476897, "batch_size": 2048}, "time_since_restore": 170.02562999725342, "iterations_since_restore": 3}
+{"loss": 125.82796093798059, "timestamp": 1719346770, "checkpoint_dir_name": "checkpoint_000003", "should_checkpoint": true, "done": false, "training_iteration": 4, "trial_id": "3ab2f301", "date": "2024-06-25_22-19-30", "time_this_iter_s": 53.092029094696045, "time_total_s": 223.11765909194946, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 2, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 2048, "decoder_rt_ff": 2048, "decoder_int_ff": 512, "n_head": 1, "drop_rate": 0.034481700426631456, "lr": 0.0010186128360476897, "batch_size": 2048}, "time_since_restore": 223.11765909194946, "iterations_since_restore": 4}
+{"loss": 117.45965227742833, "timestamp": 1719346823, "checkpoint_dir_name": "checkpoint_000004", "should_checkpoint": true, "done": false, "training_iteration": 5, "trial_id": "3ab2f301", "date": "2024-06-25_22-20-23", "time_this_iter_s": 53.195783615112305, "time_total_s": 276.31344270706177, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 2, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 2048, "decoder_rt_ff": 2048, "decoder_int_ff": 512, "n_head": 1, "drop_rate": 0.034481700426631456, "lr": 0.0010186128360476897, "batch_size": 2048}, "time_since_restore": 276.31344270706177, "iterations_since_restore": 5}
+{"loss": 114.47784069391686, "timestamp": 1719355017, "checkpoint_dir_name": "checkpoint_000005", "should_checkpoint": true, "done": false, "training_iteration": 6, "trial_id": "3ab2f301", "date": "2024-06-26_00-36-57", "time_this_iter_s": 65.25149631500244, "time_total_s": 341.5649390220642, "pid": 2850562, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 2, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 2048, "decoder_rt_ff": 2048, "decoder_int_ff": 512, "n_head": 1, "drop_rate": 0.034481700426631456, "lr": 0.0010186128360476897, "batch_size": 2048}, "time_since_restore": 65.25149631500244, "iterations_since_restore": 1}
+{"loss": 109.24919566957969, "timestamp": 1719355069, "checkpoint_dir_name": "checkpoint_000006", "should_checkpoint": true, "done": false, "training_iteration": 7, "trial_id": "3ab2f301", "date": "2024-06-26_00-37-49", "time_this_iter_s": 51.495158195495605, "time_total_s": 393.0600972175598, "pid": 2850562, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 2, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 2048, "decoder_rt_ff": 2048, "decoder_int_ff": 512, "n_head": 1, "drop_rate": 0.034481700426631456, "lr": 0.0010186128360476897, "batch_size": 2048}, "time_since_restore": 116.74665451049805, "iterations_since_restore": 2}
+{"loss": 105.2013188159372, "timestamp": 1719355120, "checkpoint_dir_name": "checkpoint_000007", "should_checkpoint": true, "done": false, "training_iteration": 8, "trial_id": "3ab2f301", "date": "2024-06-26_00-38-40", "time_this_iter_s": 51.496910572052, "time_total_s": 444.5570077896118, "pid": 2850562, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 2, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 2048, "decoder_rt_ff": 2048, "decoder_int_ff": 512, "n_head": 1, "drop_rate": 0.034481700426631456, "lr": 0.0010186128360476897, "batch_size": 2048}, "time_since_restore": 168.24356508255005, "iterations_since_restore": 3}
+{"loss": 118.58170366662694, "timestamp": 1719355171, "checkpoint_dir_name": "checkpoint_000008", "should_checkpoint": true, "done": false, "training_iteration": 9, "trial_id": "3ab2f301", "date": "2024-06-26_00-39-31", "time_this_iter_s": 51.06172227859497, "time_total_s": 495.6187300682068, "pid": 2850562, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 2, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 2048, "decoder_rt_ff": 2048, "decoder_int_ff": 512, "n_head": 1, "drop_rate": 0.034481700426631456, "lr": 0.0010186128360476897, "batch_size": 2048}, "time_since_restore": 219.30528736114502, "iterations_since_restore": 4}
+{"loss": 99.85706461508443, "timestamp": 1719355223, "checkpoint_dir_name": "checkpoint_000009", "should_checkpoint": true, "done": false, "training_iteration": 10, "trial_id": "3ab2f301", "date": "2024-06-26_00-40-23", "time_this_iter_s": 51.387521266937256, "time_total_s": 547.006251335144, "pid": 2850562, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 2, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 2048, "decoder_rt_ff": 2048, "decoder_int_ff": 512, "n_head": 1, "drop_rate": 0.034481700426631456, "lr": 0.0010186128360476897, "batch_size": 2048}, "time_since_restore": 270.6928086280823, "iterations_since_restore": 5}
+{"loss": 98.49699257677935, "timestamp": 1719357936, "checkpoint_dir_name": "checkpoint_000010", "should_checkpoint": true, "done": false, "training_iteration": 11, "trial_id": "3ab2f301", "date": "2024-06-26_01-25-36", "time_this_iter_s": 64.502512216568, "time_total_s": 611.508763551712, "pid": 2850562, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 2, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 2048, "decoder_rt_ff": 2048, "decoder_int_ff": 512, "n_head": 1, "drop_rate": 0.034481700426631456, "lr": 0.0010186128360476897, "batch_size": 2048}, "time_since_restore": 64.502512216568, "iterations_since_restore": 1}
+{"loss": 93.49606281190407, "timestamp": 1719357987, "checkpoint_dir_name": "checkpoint_000011", "should_checkpoint": true, "done": false, "training_iteration": 12, "trial_id": "3ab2f301", "date": "2024-06-26_01-26-27", "time_this_iter_s": 50.864012002944946, "time_total_s": 662.372775554657, "pid": 2850562, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 2, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 2048, "decoder_rt_ff": 2048, "decoder_int_ff": 512, "n_head": 1, "drop_rate": 0.034481700426631456, "lr": 0.0010186128360476897, "batch_size": 2048}, "time_since_restore": 115.36652421951294, "iterations_since_restore": 2}
+{"loss": 94.78943622018409, "timestamp": 1719358039, "checkpoint_dir_name": "checkpoint_000012", "should_checkpoint": true, "done": false, "training_iteration": 13, "trial_id": "3ab2f301", "date": "2024-06-26_01-27-19", "time_this_iter_s": 51.64423751831055, "time_total_s": 714.0170130729675, "pid": 2850562, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 2, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 2048, "decoder_rt_ff": 2048, "decoder_int_ff": 512, "n_head": 1, "drop_rate": 0.034481700426631456, "lr": 0.0010186128360476897, "batch_size": 2048}, "time_since_restore": 167.0107617378235, "iterations_since_restore": 3}
+{"loss": 91.46543307567205, "timestamp": 1719358090, "checkpoint_dir_name": "checkpoint_000013", "should_checkpoint": true, "done": false, "training_iteration": 14, "trial_id": "3ab2f301", "date": "2024-06-26_01-28-10", "time_this_iter_s": 51.174827098846436, "time_total_s": 765.191840171814, "pid": 2850562, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 2, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 2048, "decoder_rt_ff": 2048, "decoder_int_ff": 512, "n_head": 1, "drop_rate": 0.034481700426631456, "lr": 0.0010186128360476897, "batch_size": 2048}, "time_since_restore": 218.18558883666992, "iterations_since_restore": 4}
+{"loss": 88.73407943605439, "timestamp": 1719358141, "checkpoint_dir_name": "checkpoint_000014", "should_checkpoint": true, "done": false, "training_iteration": 15, "trial_id": "3ab2f301", "date": "2024-06-26_01-29-01", "time_this_iter_s": 50.784724712371826, "time_total_s": 815.9765648841858, "pid": 2850562, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 2, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 2048, "decoder_rt_ff": 2048, "decoder_int_ff": 512, "n_head": 1, "drop_rate": 0.034481700426631456, "lr": 0.0010186128360476897, "batch_size": 2048}, "time_since_restore": 268.97031354904175, "iterations_since_restore": 5}
+{"loss": 94.00536857064314, "timestamp": 1719358192, "checkpoint_dir_name": "checkpoint_000015", "should_checkpoint": true, "done": false, "training_iteration": 16, "trial_id": "3ab2f301", "date": "2024-06-26_01-29-52", "time_this_iter_s": 51.60023093223572, "time_total_s": 867.5767958164215, "pid": 2850562, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 2, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 2048, "decoder_rt_ff": 2048, "decoder_int_ff": 512, "n_head": 1, "drop_rate": 0.034481700426631456, "lr": 0.0010186128360476897, "batch_size": 2048}, "time_since_restore": 320.57054448127747, "iterations_since_restore": 6}
+{"loss": 87.47035950187623, "timestamp": 1719358244, "checkpoint_dir_name": "checkpoint_000016", "should_checkpoint": true, "done": false, "training_iteration": 17, "trial_id": "3ab2f301", "date": "2024-06-26_01-30-44", "time_this_iter_s": 51.84270000457764, "time_total_s": 919.4194958209991, "pid": 2850562, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 2, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 2048, "decoder_rt_ff": 2048, "decoder_int_ff": 512, "n_head": 1, "drop_rate": 0.034481700426631456, "lr": 0.0010186128360476897, "batch_size": 2048}, "time_since_restore": 372.4132444858551, "iterations_since_restore": 7}
+{"loss": 88.06384884090875, "timestamp": 1719358295, "checkpoint_dir_name": "checkpoint_000017", "should_checkpoint": true, "done": false, "training_iteration": 18, "trial_id": "3ab2f301", "date": "2024-06-26_01-31-35", "time_this_iter_s": 51.09972786903381, "time_total_s": 970.519223690033, "pid": 2850562, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 2, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 2048, "decoder_rt_ff": 2048, "decoder_int_ff": 512, "n_head": 1, "drop_rate": 0.034481700426631456, "lr": 0.0010186128360476897, "batch_size": 2048}, "time_since_restore": 423.5129723548889, "iterations_since_restore": 8}
+{"loss": 87.50748780017763, "timestamp": 1719358346, "checkpoint_dir_name": "checkpoint_000018", "should_checkpoint": true, "done": false, "training_iteration": 19, "trial_id": "3ab2f301", "date": "2024-06-26_01-32-26", "time_this_iter_s": 51.09087514877319, "time_total_s": 1021.6100988388062, "pid": 2850562, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 2, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 2048, "decoder_rt_ff": 2048, "decoder_int_ff": 512, "n_head": 1, "drop_rate": 0.034481700426631456, "lr": 0.0010186128360476897, "batch_size": 2048}, "time_since_restore": 474.6038475036621, "iterations_since_restore": 9}
+{"loss": 84.81797225832001, "timestamp": 1719358398, "checkpoint_dir_name": "checkpoint_000019", "should_checkpoint": true, "done": false, "training_iteration": 20, "trial_id": "3ab2f301", "date": "2024-06-26_01-33-18", "time_this_iter_s": 51.372178077697754, "time_total_s": 1072.982276916504, "pid": 2850562, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 2, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 2048, "decoder_rt_ff": 2048, "decoder_int_ff": 512, "n_head": 1, "drop_rate": 0.034481700426631456, "lr": 0.0010186128360476897, "batch_size": 2048}, "time_since_restore": 525.9760255813599, "iterations_since_restore": 10}
diff --git a/ray_results/test_experiment/train_model_3f37f45c_7_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=2,drop_r_2024-06-25_17-29-36/events.out.tfevents.1719329461.r8i6n2 b/ray_results/test_experiment/train_model_3f37f45c_7_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=2,drop_r_2024-06-25_17-29-36/events.out.tfevents.1719329461.r8i6n2
new file mode 100644
index 0000000000000000000000000000000000000000..c59d8753d6c3273090a67d066fdd18c09c512b16
GIT binary patch
literal 1311
zcmah|TSyd97#?TKnqPKxC)aenq-#P9(PjvRL^~-GXg9hTKE!F<Ikr=~XNH;C%3czr
zkm$je9wIDljUc+{Q-#q*3c5W+5EX<$5M5sKB_Vw2j5~XF*UaW^{_p?(@0{=dPC1wU
zQl~mXLQzNO&AXRAYz=z5O+_Q?+KypjH*fIY`mv?n?Hjv%yj6I0v1QWfD=Co5pcK5O
z(i^su1c&3gWd%IjxVDBz*Jszel=Ht{Bf%J9lzOOxAfs3@JsDR~G=`%`4P7I4faCfi
zFW;<voG1At#rXM>s!lMaaGbn9zCG+pCJt8b7P>|mB?Prl!6a=hfsq~4ETq}kL{?zn
z(6yGLuHnH(mp;_r|6O>WrLTq%ROZtyjYKgrv1RKfo*OXncFMQW9q{$ri!Nd3Ri-b&
z8mMCf996~EbYh|WG0x2&e%!ZftROvSaOlDjVYYKDb&r}Yje20j1EnZtD5jFIP#B*u
zuu1Pi?Dhc<oc6#n7ZQK-;tU`10dy;NG)7fU;#yBg1fPndx=Qa%eMUV!v{nIu+!{$H
zP+aN7CWZXq%|NExT@Ix=rQBvO2&$<Yh+fQ*qFkKl#;U53!${Q<v_wyXNm_kQIRO65
zf9{|{;Ny-uKp{aftf;jf$q&V1e#oR6s1g?m(L9P15~&(0#eauDS_ReOVoR19pkDm{
z5|@rAgT&q6<j*V3!lnc6{yy{YiHEA}{w!=+GzG}A6>6|Xdi3@OJWTM}J>KSnH+U$^
z@m3W4dt2?NXS6R7Sdp=a7Sn+#ks&y5YYZEfp=JTg89p6>%J~6dV4qAM?Si{!Eu0L+
z&hBVos(0qxS;dMthslcpYc^z9nVSa&P$3FrW6OX+nLfgWv&wp=BdI3i-DdX@KD(3J
hBkahy-w0s{{_Q;`YT9|4v8?Y)Lj3XqU(@sZe*j}Kl)?Z2
literal 0
HcmV?d00001
diff --git a/ray_results/test_experiment/train_model_3f37f45c_7_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=2,drop_r_2024-06-25_17-29-36/params.json b/ray_results/test_experiment/train_model_3f37f45c_7_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=2,drop_r_2024-06-25_17-29-36/params.json
new file mode 100644
index 0000000..9cba325
--- /dev/null
+++ b/ray_results/test_experiment/train_model_3f37f45c_7_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=2,drop_r_2024-06-25_17-29-36/params.json
@@ -0,0 +1,13 @@
+{
+ "batch_size": 1024,
+ "decoder_int_ff": 512,
+ "decoder_int_num_layer": 1,
+ "decoder_rt_ff": 2048,
+ "decoder_rt_num_layer": 2,
+ "drop_rate": 0.6691316467787182,
+ "embedding_dim": 256,
+ "encoder_ff": 512,
+ "encoder_num_layer": 1,
+ "lr": 0.004731694425031267,
+ "n_head": 4
+}
\ No newline at end of file
diff --git a/ray_results/test_experiment/train_model_3f37f45c_7_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=2,drop_r_2024-06-25_17-29-36/params.pkl b/ray_results/test_experiment/train_model_3f37f45c_7_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=2,drop_r_2024-06-25_17-29-36/params.pkl
new file mode 100644
index 0000000000000000000000000000000000000000..3d20730fcb1235a43c5ba169b8895406c66a4a50
GIT binary patch
literal 225
zcmZo*oqCM{0&1sd^a!TrCFiH47RBe4=EmnFR;CtB@n-B1NlAr>7L_1NF!hMSBr@|*
zWEgvRQ*)D2Q&KYX(&JMybEo(+F!pf4%uY)K@|k*gVVZ%af`vGG_+UaH^PtMu^5Qd6
z6H}&mv-EJL6y+Dh7bTXYPI0$?n%Q==#5uT!DW?d?D^Bd{`Br(ahbt+uBsn9#II}7h
JXbDTH9suW=Q>g#|
literal 0
HcmV?d00001
diff --git a/ray_results/test_experiment/train_model_3f37f45c_7_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=2,drop_r_2024-06-25_17-29-36/progress.csv b/ray_results/test_experiment/train_model_3f37f45c_7_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=2,drop_r_2024-06-25_17-29-36/progress.csv
new file mode 100644
index 0000000..3eef001
--- /dev/null
+++ b/ray_results/test_experiment/train_model_3f37f45c_7_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=2,drop_r_2024-06-25_17-29-36/progress.csv
@@ -0,0 +1,2 @@
+loss,timestamp,checkpoint_dir_name,should_checkpoint,done,training_iteration,trial_id,date,time_this_iter_s,time_total_s,pid,hostname,node_ip,time_since_restore,iterations_since_restore
+2052.4382574126475,1719329544,checkpoint_000000,True,False,1,3f37f45c,2024-06-25_17-32-25,83.68853616714478,83.68853616714478,2195253,r8i6n2,10.159.28.60,83.68853616714478,1
diff --git a/ray_results/test_experiment/train_model_3f37f45c_7_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=2,drop_r_2024-06-25_17-29-36/result.json b/ray_results/test_experiment/train_model_3f37f45c_7_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=2,drop_r_2024-06-25_17-29-36/result.json
new file mode 100644
index 0000000..e888aa0
--- /dev/null
+++ b/ray_results/test_experiment/train_model_3f37f45c_7_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=2,drop_r_2024-06-25_17-29-36/result.json
@@ -0,0 +1 @@
+{"loss": 2052.4382574126475, "timestamp": 1719329544, "checkpoint_dir_name": "checkpoint_000000", "should_checkpoint": true, "done": false, "training_iteration": 1, "trial_id": "3f37f45c", "date": "2024-06-25_17-32-25", "time_this_iter_s": 83.68853616714478, "time_total_s": 83.68853616714478, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 1, "decoder_rt_num_layer": 2, "decoder_int_num_layer": 1, "embedding_dim": 256, "encoder_ff": 512, "decoder_rt_ff": 2048, "decoder_int_ff": 512, "n_head": 4, "drop_rate": 0.6691316467787182, "lr": 0.004731694425031267, "batch_size": 1024}, "time_since_restore": 83.68853616714478, "iterations_since_restore": 1}
diff --git a/ray_results/test_experiment/train_model_40101b9a_29_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=1,drop_r_2024-06-25_22-15-47/events.out.tfevents.1719346823.r8i6n2 b/ray_results/test_experiment/train_model_40101b9a_29_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=1,drop_r_2024-06-25_22-15-47/events.out.tfevents.1719346823.r8i6n2
new file mode 100644
index 0000000000000000000000000000000000000000..910e4dab8aa5c8154370eb77fea90f04beacf996
GIT binary patch
literal 2771
zcma);e{2(F7{|LS1MVRMdaSs0>*zWN*eHc1Vnj`gZ4JSMjRhr|%;VO3M^D?kdUr$6
z80t`hWFsJPhJ?Y!7XL8APEF*Ou*@hT4aAWUnM9Uof@m<-#b1k)d2Mg+^_?wk|Lgla
z-{*Vp=Xr9^RhTlrzFki_>GhkZdcQh4y3SVCB6EVcGO!(q>e~AEI~w-Y>3M_4UiQ&r
z+rr6I-og^sVkn0)ncL}B+eG9JONwIs_~}>x{a$f*l`eVbkFQy}CW0k(PzyFJQ9@E%
zm}i3_6nrr%38HE>nPQ&F;N{N=%LgQ^CoOqNM4H58+Pn3^fLphWG}{~LAw%K>2UKE9
zRfr%~4G9V>s7PiN>%iy{-!H?@>{_kMyLa#YiGC!C9hZU=sz|vah(W~4NKqvjr3+jL
z#~*rFFYx2-OV83Li~=rL0kyb5%9T?ENmN*}OzP%+bGE!k+<g3G$D@~0u&G?IBfY-B
znswK260oOEVM#Ht|5oGmgyo*oSf2;>ihcXPBw)`wgCzxEk9oR$lb>99j)2{6$P2>0
z&{65%)P0d5V2>LGiiU0S{?OU!A#VP-{p~$z*ydcYhrk|an0NU@0`^-qSW*n^sZ$$o
z5|-8_Sf2-WyLzjVfZc6K3c$V}x_nPwU$mTnZFgYDg0R0m$otP;TQ)(!K4cUq8n)N_
zP5;$3#LdMMXGYSnXXS!zX`CA{AE_uMV88kSmJ|cqHf)a(mUj((9@vwn8y^v{+vC`>
z0PN1jF5mM_FHI1z|1snRVK4mdUH|AUca(q~9mM`c!;X13CilmQn<pDStxlv~Bc-`u
zH;n)5x14|NDgpaWBbF2c`{&{M5yDd3g!Oq~*R9&TgMghhBn4piP+h*Q%Wpdg*r6uu
zSP=H<=5zkPws$26*f)#<MZ@m#UP-JU8!XAZ`ww&<eu2K0RJ<b^Y*)V-dd*PIhN7Iz
zMHJSJc0`e^?H^Iyu-6Q4nqiI}VkRi&Ps%wTpoLR|A#F3ff*fWi1M_(llz44VX}gha
zZPm^y!J2(0wnf-5w-d=)$O2{4Az9C^fby)P?Bp__d0C3G+A>O6X<8I%K|C*r+gM(R
zXh+)9ATi^fcD90L`hRwz*+7L;?f|nzHiS6dVRl(y0YiQmxN4x9xyRz4AxSNEmmR8@
z{EOSQ1nkVc7PD&^)G>t@FP!UrL)82C*{{df(zf69{(Vou7zNcv{}iMs+U8~zmD2zV
zVym>_JOyVdqmOaQHbBAREVc}7IsSx$8yLFdfko4Zw8PB6Bt2l8(KQ^4d7&l;avV2#
zp=#!U;Gib1&E0#PqIoY!^-eoDHk3HrytF_4FvJ+(<O5!~FMA$vpeG$L#x^_P%rmPv
wC+T^8%hAMR`iIlSlYLF}%4zzm!40<mvqvJIA+ODt!F`sKnp@IS`o+Qj07ut<wg3PC
literal 0
HcmV?d00001
diff --git a/ray_results/test_experiment/train_model_40101b9a_29_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=1,drop_r_2024-06-25_22-15-47/params.json b/ray_results/test_experiment/train_model_40101b9a_29_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=1,drop_r_2024-06-25_22-15-47/params.json
new file mode 100644
index 0000000..0027cad
--- /dev/null
+++ b/ray_results/test_experiment/train_model_40101b9a_29_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=1,drop_r_2024-06-25_22-15-47/params.json
@@ -0,0 +1,13 @@
+{
+ "batch_size": 2048,
+ "decoder_int_ff": 512,
+ "decoder_int_num_layer": 1,
+ "decoder_rt_ff": 512,
+ "decoder_rt_num_layer": 1,
+ "drop_rate": 0.8983976265900149,
+ "embedding_dim": 1024,
+ "encoder_ff": 2048,
+ "encoder_num_layer": 2,
+ "lr": 0.039339947723305395,
+ "n_head": 1
+}
\ No newline at end of file
diff --git a/ray_results/test_experiment/train_model_40101b9a_29_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=1,drop_r_2024-06-25_22-15-47/params.pkl b/ray_results/test_experiment/train_model_40101b9a_29_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=1,drop_r_2024-06-25_22-15-47/params.pkl
new file mode 100644
index 0000000000000000000000000000000000000000..3c52c6e8e2388ddf5b799b58a8d80fcf752792dd
GIT binary patch
literal 225
zcmZo*oqCM{0&1sd^a!TrCFiH47RBe4=EmnFR;CtB@n-4~NlAr>7L_1NF!qSTBr@~R
zWO!3^lTuSsGV{{oQ!;a>_%g8caKX$@O9S#bdU#=)fu@3mn0okNLLl>?Vr+Tw8L5dW
zQ@j~_I8%!93*w6sOH!w}+rQbrCQ-p@Q4dp25s<e;C3fcN{quXck`hajGvbRgt5QLh
Hl<ENh<}y=u
literal 0
HcmV?d00001
diff --git a/ray_results/test_experiment/train_model_40101b9a_29_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=1,drop_r_2024-06-25_22-15-47/progress.csv b/ray_results/test_experiment/train_model_40101b9a_29_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=1,drop_r_2024-06-25_22-15-47/progress.csv
new file mode 100644
index 0000000..2534081
--- /dev/null
+++ b/ray_results/test_experiment/train_model_40101b9a_29_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=1,drop_r_2024-06-25_22-15-47/progress.csv
@@ -0,0 +1,6 @@
+loss,timestamp,checkpoint_dir_name,should_checkpoint,done,training_iteration,trial_id,date,time_this_iter_s,time_total_s,pid,hostname,node_ip,time_since_restore,iterations_since_restore
+4106.712271622785,1719347137,checkpoint_000000,True,False,1,40101b9a,2024-06-25_22-25-37,313.700572013855,313.700572013855,2195253,r8i6n2,10.159.28.60,313.700572013855,1
+3130.0891709215057,1719347437,checkpoint_000001,True,False,2,40101b9a,2024-06-25_22-30-38,300.49900484085083,614.1995768547058,2195253,r8i6n2,10.159.28.60,614.1995768547058,2
+2512.6716385488435,1719347737,checkpoint_000002,True,False,3,40101b9a,2024-06-25_22-35-38,300.07938170433044,914.2789585590363,2195253,r8i6n2,10.159.28.60,914.2789585590363,3
+2181.0040408156988,1719348038,checkpoint_000003,True,False,4,40101b9a,2024-06-25_22-40-38,300.6292908191681,1214.9082493782043,2195253,r8i6n2,10.159.28.60,1214.9082493782043,4
+2031.5720868448573,1719348338,checkpoint_000004,True,False,5,40101b9a,2024-06-25_22-45-38,300.0205228328705,1514.9287722110748,2195253,r8i6n2,10.159.28.60,1514.9287722110748,5
diff --git a/ray_results/test_experiment/train_model_40101b9a_29_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=1,drop_r_2024-06-25_22-15-47/result.json b/ray_results/test_experiment/train_model_40101b9a_29_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=1,drop_r_2024-06-25_22-15-47/result.json
new file mode 100644
index 0000000..cb94596
--- /dev/null
+++ b/ray_results/test_experiment/train_model_40101b9a_29_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=1,drop_r_2024-06-25_22-15-47/result.json
@@ -0,0 +1,5 @@
+{"loss": 4106.712271622785, "timestamp": 1719347137, "checkpoint_dir_name": "checkpoint_000000", "should_checkpoint": true, "done": false, "training_iteration": 1, "trial_id": "40101b9a", "date": "2024-06-25_22-25-37", "time_this_iter_s": 313.700572013855, "time_total_s": 313.700572013855, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 2, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 1024, "encoder_ff": 2048, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 1, "drop_rate": 0.8983976265900149, "lr": 0.039339947723305395, "batch_size": 2048}, "time_since_restore": 313.700572013855, "iterations_since_restore": 1}
+{"loss": 3130.0891709215057, "timestamp": 1719347437, "checkpoint_dir_name": "checkpoint_000001", "should_checkpoint": true, "done": false, "training_iteration": 2, "trial_id": "40101b9a", "date": "2024-06-25_22-30-38", "time_this_iter_s": 300.49900484085083, "time_total_s": 614.1995768547058, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 2, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 1024, "encoder_ff": 2048, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 1, "drop_rate": 0.8983976265900149, "lr": 0.039339947723305395, "batch_size": 2048}, "time_since_restore": 614.1995768547058, "iterations_since_restore": 2}
+{"loss": 2512.6716385488435, "timestamp": 1719347737, "checkpoint_dir_name": "checkpoint_000002", "should_checkpoint": true, "done": false, "training_iteration": 3, "trial_id": "40101b9a", "date": "2024-06-25_22-35-38", "time_this_iter_s": 300.07938170433044, "time_total_s": 914.2789585590363, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 2, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 1024, "encoder_ff": 2048, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 1, "drop_rate": 0.8983976265900149, "lr": 0.039339947723305395, "batch_size": 2048}, "time_since_restore": 914.2789585590363, "iterations_since_restore": 3}
+{"loss": 2181.0040408156988, "timestamp": 1719348038, "checkpoint_dir_name": "checkpoint_000003", "should_checkpoint": true, "done": false, "training_iteration": 4, "trial_id": "40101b9a", "date": "2024-06-25_22-40-38", "time_this_iter_s": 300.6292908191681, "time_total_s": 1214.9082493782043, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 2, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 1024, "encoder_ff": 2048, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 1, "drop_rate": 0.8983976265900149, "lr": 0.039339947723305395, "batch_size": 2048}, "time_since_restore": 1214.9082493782043, "iterations_since_restore": 4}
+{"loss": 2031.5720868448573, "timestamp": 1719348338, "checkpoint_dir_name": "checkpoint_000004", "should_checkpoint": true, "done": false, "training_iteration": 5, "trial_id": "40101b9a", "date": "2024-06-25_22-45-38", "time_this_iter_s": 300.0205228328705, "time_total_s": 1514.9287722110748, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 2, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 1024, "encoder_ff": 2048, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 1, "drop_rate": 0.8983976265900149, "lr": 0.039339947723305395, "batch_size": 2048}, "time_since_restore": 1514.9287722110748, "iterations_since_restore": 5}
diff --git a/ray_results/test_experiment/train_model_43bba37c_19_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_18-28-52/events.out.tfevents.1719333829.r8i6n2 b/ray_results/test_experiment/train_model_43bba37c_19_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_18-28-52/events.out.tfevents.1719333829.r8i6n2
new file mode 100644
index 0000000000000000000000000000000000000000..de4459e3351b53546d3fd1186a9b0a9a936caeae
GIT binary patch
literal 770
zcmb1OfPlsI-b$Qy+tNSonRnAsiZ`h!F*8rkwJbHS#LB2mS!0GWRNaR8<<6X2Cf}3X
z3sfh-s3pcF!o^#ZSgBu9nwP4dlV4mc8f^d7h5b~~Rs*P(sb4R|bLP*4Yf<J><dTAG
zDbC0*%}I$*&PYwpF38W!D-mU2Xs}PazHTjwl?P_S^`lyulAo6f)WX29<nY(G`cNx#
zV`T$4!{)%XC~+xpNy4ox$;?fSFUiO(j?XMfEs8G|oqkKvIr@jjGDE2Lxy{WlIWHo#
zt8giCNuz2n&df_rjW0?qF3B%S1sfop(|gSvYQXasry@DM=fdsR;?m$!gc|^|HL)Z!
zKd(3*)fkY^UoYjBNAY>#vTJtWz-Gc0*vD>ub@9r#7Dfr|`OD#2hzaaRSED<W;kC
z*N-ExJrB>mi4xe45LyTX_SbvD&d088*@hC>$t&SD5)ABPcV;<PDEq8J3G7=41Bec6
K28S$8z4HJe<pJOT
literal 0
HcmV?d00001
diff --git a/ray_results/test_experiment/train_model_43bba37c_19_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_18-28-52/events.out.tfevents.1719338189.r8i6n2 b/ray_results/test_experiment/train_model_43bba37c_19_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_18-28-52/events.out.tfevents.1719338189.r8i6n2
new file mode 100644
index 0000000000000000000000000000000000000000..d71c6d523313f49eca5605e6b3e008b1e341c65c
GIT binary patch
literal 770
zcmb1OfPlsI-b$Q)C&G;n&b#R-#hX-=n3<>NT9%quVr9g-g=xPsRNaR8<<6Y{iuXi=
z)Cn+aiE)W=@fIak>X($}rRwM87Z;04eEsSY^Ur0z0aVM>uNUGumsY~HD03-tNx`)g
zXXKaWq{Js@q$Xz<<Y(rUh%zuV*gviBevD$JK@D6#s+B4Ed8t4x3=E56HU;WKt;~&;
z4dA?l(4xepz$FQ{vLrJ%HNGSxvp7DpB(*5MSak9PMdyvrzo;5Qwa;yCe#seJ3%5~)
zONmPwReNz}UUF)DQEG8Xeo-pefVwGtzs;crJb!U2lJgS604**JE=9NjAX^hlGV}9_
z<57(P`Mmkanms5!e<3)(0UX#Y*aF*X`Zt%<LvDLe0$WfRu7#MuzR09ri(=(Wgnk@>
zogMXIGfH5qi@>cU5ZHe@gq=5h>CZ<A?1c#J1Oxj-&n#!Z*jxQ5fh{fyx1Z?1W^mXv
H-6sS9q+R~K
literal 0
HcmV?d00001
diff --git a/ray_results/test_experiment/train_model_43bba37c_19_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_18-28-52/events.out.tfevents.1719340067.r8i6n2 b/ray_results/test_experiment/train_model_43bba37c_19_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_18-28-52/events.out.tfevents.1719340067.r8i6n2
new file mode 100644
index 0000000000000000000000000000000000000000..cbac79e5035f208089df4716762053af9d5a158e
GIT binary patch
literal 2406
zcma)-e{2(F7{|+%{kTUv=ylzX?g!l!;yj~ui$<M4df1vM5I}`Lrr_n)duLDAyXNi&
z+y1~N#Dv7epovEQ7(-?<V}=ZuNCFAUWYNsTfJ)4aN-~LvLPju^XvFxoy}j3WwzU1P
z@AG`0@4cV*$@4V1^1ty{HuCJ-bC+sAJ2<~S;OR4DRlDbjT}ZQfx}Q6DE)ufyrp9(h
z+1A>Qu58}wDxm@D!DGn7VQWxB;Q`$={l`Aa#O|+Y+I_Drc`I{PVo!UBBm^B0AQCg7
z4-P0&Jb~gbr*u`b{4Q5I@~H34dCIbGHPO?Simo9nak=U{rmlx=%jC|st?Za1;UNgk
z#L`leNU{>DDXA7RB-6jR5R5+i!POmIwtRH&-pg#WmpHBk9$IL*scLZ~8OXGB1LX?b
zJod@(19pK+zs~itZ#V@6=!6bZAnPhys;-$5T_$^TbN2d5>gG3hEo{%ht}X`q%;8Mz
ztJ%hV6zoGoL{bUtyH1~6Pgy=VO!OsSkN)uK9t!r1BPj#>PPZ@0zq4hMf*lzlj%8ur
z{>Opn;^wL&6zq?k0u{q{iLa$w$Ecf!ew!J|!LBI=d+10ew)x@D*HEw@|D8xGfjxBa
z{5s0=tfMah`^TDSh=SdFg;<t>9ZUP7dr}kMQ?S2t<Yi%pr+<z%&i!<Sg1zY~@vj)R
zD5597nW1jZKL6oZ4t8xZ*#GoqVi);sRTS**9Yj(I><br9@s#BWM_&T=w((696zs=#
z63a5MQ<J{v{P~I3DcHvyd0E)gXBuKpcPIa*VBhxw@vj(mTD-B{b9rh-ejM!|J@68H
z;*Q19Sv<3a_uTNV8|tM*N;c%ADTUEc3K@7LYSxB*Zg|xVHFk*mi{buc_&R_-*@`Fd
zuo^)@H_t(xg5tV@M<*WFQhz@_YX*PenKqb|2IOI6;E)$QOCiI~Zi4!Pqrzklu!^Cl
zBs?FpR#=OYeMnJM?L|pZllTaK4bt-NIcGn3m;M(Pss?5t>kd$@NeLt?LAT(ARUEym
zgm!4<Zn2A&NwT{}XoD86^ez$BKpS_fT_LQ45LbQ|#Oi*zTeIJ9X7O+j%XixEx4^&!
z23no>Gm&ZHWg?lDi~*}!zrNuV11A~h9gi}BNd_7U*mA7*>$PA0JN@l^1bj<~@L|4T
zu0<fQ?A1tYzX<I`kdt^&gqGzELV{@#@BX51Pg1b=s_fh01OA~*mC*83?qjGnAjv5a
z8Vmb@1n^hlfYY{x0lq}9lALRiY3sK4AF+4&7stn+VSAS7cR?5e|7VXhB~Kprn8SS~
M&(yAHztJiF2f_dShX4Qo
literal 0
HcmV?d00001
diff --git a/ray_results/test_experiment/train_model_43bba37c_19_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_18-28-52/params.json b/ray_results/test_experiment/train_model_43bba37c_19_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_18-28-52/params.json
new file mode 100644
index 0000000..950914f
--- /dev/null
+++ b/ray_results/test_experiment/train_model_43bba37c_19_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_18-28-52/params.json
@@ -0,0 +1,13 @@
+{
+ "batch_size": 1024,
+ "decoder_int_ff": 512,
+ "decoder_int_num_layer": 1,
+ "decoder_rt_ff": 1024,
+ "decoder_rt_num_layer": 1,
+ "drop_rate": 0.4036073673456261,
+ "embedding_dim": 256,
+ "encoder_ff": 1024,
+ "encoder_num_layer": 8,
+ "lr": 0.0007661776725543005,
+ "n_head": 1
+}
\ No newline at end of file
diff --git a/ray_results/test_experiment/train_model_43bba37c_19_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_18-28-52/params.pkl b/ray_results/test_experiment/train_model_43bba37c_19_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_18-28-52/params.pkl
new file mode 100644
index 0000000000000000000000000000000000000000..841f248f50619e300739577d668357c35b3abf6d
GIT binary patch
literal 225
zcmZo*oqCM{0&1sd^a!TrCFiH47RBe4=EmnFR;CtB@#g3eNlAr>7L_1NF!qSTBr@~R
zWO!3^lTuSsGV{{oQ!;a>_%bl|aKX$@O9S#*dU#=)fu=%*_+UaH^T1+EJ#2aL8L5dW
zQ@j~_I8%!93*w6sOH!w}+uyvh`QK@kOFc|EML?dXv=-}EOQjyJq{NcsjQHZrs#K6A
GrFsDE%u><-
literal 0
HcmV?d00001
diff --git a/ray_results/test_experiment/train_model_43bba37c_19_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_18-28-52/progress.csv b/ray_results/test_experiment/train_model_43bba37c_19_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_18-28-52/progress.csv
new file mode 100644
index 0000000..ac45833
--- /dev/null
+++ b/ray_results/test_experiment/train_model_43bba37c_19_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_18-28-52/progress.csv
@@ -0,0 +1,9 @@
+loss,timestamp,checkpoint_dir_name,should_checkpoint,done,training_iteration,trial_id,date,time_this_iter_s,time_total_s,pid,hostname,node_ip,time_since_restore,iterations_since_restore
+1961.9788803941622,1719333991,checkpoint_000000,True,False,1,43bba37c,2024-06-25_18-46-31,161.8538646697998,161.8538646697998,2195253,r8i6n2,10.159.28.60,161.8538646697998,1
+1966.8054059847134,1719334139,checkpoint_000001,True,False,2,43bba37c,2024-06-25_18-48-59,147.8709270954132,309.724791765213,2195253,r8i6n2,10.159.28.60,309.724791765213,2
+1967.6592320720042,1719338350,checkpoint_000002,True,False,3,43bba37c,2024-06-25_19-59-10,161.5647487640381,471.2895405292511,2195253,r8i6n2,10.159.28.60,161.5647487640381,1
+1972.7257733682948,1719338498,checkpoint_000003,True,False,4,43bba37c,2024-06-25_20-01-38,147.5350902080536,618.8246307373047,2195253,r8i6n2,10.159.28.60,309.0998389720917,2
+1965.584105484129,1719340228,checkpoint_000004,True,False,5,43bba37c,2024-06-25_20-30-28,160.8864643573761,779.7110950946808,2195253,r8i6n2,10.159.28.60,160.8864643573761,1
+1965.0863916592334,1719340375,checkpoint_000005,True,False,6,43bba37c,2024-06-25_20-32-55,147.0123507976532,926.723445892334,2195253,r8i6n2,10.159.28.60,307.8988151550293,2
+1965.4208993986836,1719340522,checkpoint_000006,True,False,7,43bba37c,2024-06-25_20-35-23,147.50104451179504,1074.224490404129,2195253,r8i6n2,10.159.28.60,455.39985966682434,3
+1962.8120477631337,1719340670,checkpoint_000007,True,False,8,43bba37c,2024-06-25_20-37-50,147.67359066009521,1221.8980810642242,2195253,r8i6n2,10.159.28.60,603.0734503269196,4
diff --git a/ray_results/test_experiment/train_model_43bba37c_19_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_18-28-52/result.json b/ray_results/test_experiment/train_model_43bba37c_19_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_18-28-52/result.json
new file mode 100644
index 0000000..8a62215
--- /dev/null
+++ b/ray_results/test_experiment/train_model_43bba37c_19_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_18-28-52/result.json
@@ -0,0 +1,8 @@
+{"loss": 1961.9788803941622, "timestamp": 1719333991, "checkpoint_dir_name": "checkpoint_000000", "should_checkpoint": true, "done": false, "training_iteration": 1, "trial_id": "43bba37c", "date": "2024-06-25_18-46-31", "time_this_iter_s": 161.8538646697998, "time_total_s": 161.8538646697998, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 8, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 256, "encoder_ff": 1024, "decoder_rt_ff": 1024, "decoder_int_ff": 512, "n_head": 1, "drop_rate": 0.4036073673456261, "lr": 0.0007661776725543005, "batch_size": 1024}, "time_since_restore": 161.8538646697998, "iterations_since_restore": 1}
+{"loss": 1966.8054059847134, "timestamp": 1719334139, "checkpoint_dir_name": "checkpoint_000001", "should_checkpoint": true, "done": false, "training_iteration": 2, "trial_id": "43bba37c", "date": "2024-06-25_18-48-59", "time_this_iter_s": 147.8709270954132, "time_total_s": 309.724791765213, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 8, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 256, "encoder_ff": 1024, "decoder_rt_ff": 1024, "decoder_int_ff": 512, "n_head": 1, "drop_rate": 0.4036073673456261, "lr": 0.0007661776725543005, "batch_size": 1024}, "time_since_restore": 309.724791765213, "iterations_since_restore": 2}
+{"loss": 1967.6592320720042, "timestamp": 1719338350, "checkpoint_dir_name": "checkpoint_000002", "should_checkpoint": true, "done": false, "training_iteration": 3, "trial_id": "43bba37c", "date": "2024-06-25_19-59-10", "time_this_iter_s": 161.5647487640381, "time_total_s": 471.2895405292511, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 8, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 256, "encoder_ff": 1024, "decoder_rt_ff": 1024, "decoder_int_ff": 512, "n_head": 1, "drop_rate": 0.4036073673456261, "lr": 0.0007661776725543005, "batch_size": 1024}, "time_since_restore": 161.5647487640381, "iterations_since_restore": 1}
+{"loss": 1972.7257733682948, "timestamp": 1719338498, "checkpoint_dir_name": "checkpoint_000003", "should_checkpoint": true, "done": false, "training_iteration": 4, "trial_id": "43bba37c", "date": "2024-06-25_20-01-38", "time_this_iter_s": 147.5350902080536, "time_total_s": 618.8246307373047, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 8, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 256, "encoder_ff": 1024, "decoder_rt_ff": 1024, "decoder_int_ff": 512, "n_head": 1, "drop_rate": 0.4036073673456261, "lr": 0.0007661776725543005, "batch_size": 1024}, "time_since_restore": 309.0998389720917, "iterations_since_restore": 2}
+{"loss": 1965.584105484129, "timestamp": 1719340228, "checkpoint_dir_name": "checkpoint_000004", "should_checkpoint": true, "done": false, "training_iteration": 5, "trial_id": "43bba37c", "date": "2024-06-25_20-30-28", "time_this_iter_s": 160.8864643573761, "time_total_s": 779.7110950946808, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 8, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 256, "encoder_ff": 1024, "decoder_rt_ff": 1024, "decoder_int_ff": 512, "n_head": 1, "drop_rate": 0.4036073673456261, "lr": 0.0007661776725543005, "batch_size": 1024}, "time_since_restore": 160.8864643573761, "iterations_since_restore": 1}
+{"loss": 1965.0863916592334, "timestamp": 1719340375, "checkpoint_dir_name": "checkpoint_000005", "should_checkpoint": true, "done": false, "training_iteration": 6, "trial_id": "43bba37c", "date": "2024-06-25_20-32-55", "time_this_iter_s": 147.0123507976532, "time_total_s": 926.723445892334, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 8, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 256, "encoder_ff": 1024, "decoder_rt_ff": 1024, "decoder_int_ff": 512, "n_head": 1, "drop_rate": 0.4036073673456261, "lr": 0.0007661776725543005, "batch_size": 1024}, "time_since_restore": 307.8988151550293, "iterations_since_restore": 2}
+{"loss": 1965.4208993986836, "timestamp": 1719340522, "checkpoint_dir_name": "checkpoint_000006", "should_checkpoint": true, "done": false, "training_iteration": 7, "trial_id": "43bba37c", "date": "2024-06-25_20-35-23", "time_this_iter_s": 147.50104451179504, "time_total_s": 1074.224490404129, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 8, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 256, "encoder_ff": 1024, "decoder_rt_ff": 1024, "decoder_int_ff": 512, "n_head": 1, "drop_rate": 0.4036073673456261, "lr": 0.0007661776725543005, "batch_size": 1024}, "time_since_restore": 455.39985966682434, "iterations_since_restore": 3}
+{"loss": 1962.8120477631337, "timestamp": 1719340670, "checkpoint_dir_name": "checkpoint_000007", "should_checkpoint": true, "done": false, "training_iteration": 8, "trial_id": "43bba37c", "date": "2024-06-25_20-37-50", "time_this_iter_s": 147.67359066009521, "time_total_s": 1221.8980810642242, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 8, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 256, "encoder_ff": 1024, "decoder_rt_ff": 1024, "decoder_int_ff": 512, "n_head": 1, "drop_rate": 0.4036073673456261, "lr": 0.0007661776725543005, "batch_size": 1024}, "time_since_restore": 603.0734503269196, "iterations_since_restore": 4}
diff --git a/ray_results/test_experiment/train_model_45b8cfdd_17_batch_size=2048,decoder_int_ff=2048,decoder_int_num_layer=2,decoder_rt_ff=1024,decoder_rt_num_layer=4,drop_2024-06-25_16-46-32/error.pkl b/ray_results/test_experiment/train_model_45b8cfdd_17_batch_size=2048,decoder_int_ff=2048,decoder_int_num_layer=2,decoder_rt_ff=1024,decoder_rt_num_layer=4,drop_2024-06-25_16-46-32/error.pkl
new file mode 100644
index 0000000000000000000000000000000000000000..16db52c26d343e291b1302cac14cba752cf0ff4b
GIT binary patch
literal 8271
zcmeHM&u`qu6<)atY^$#8ra=`Thf@@18_xDP*TsZ4GNoB7LSuqk6>0p)|?l8+G
z8P3qs78c-(4HV!ZKxsIK0KF9f`p=|5554q%=&|UjZ-(TGyA~X)K&&9uF0|qdXNK?l
z=6i33^B%qS^d_pPPx4koCc5PXu^qbB$fM=!G<gG$$%tY;q8=52k1daIp8hG-k~4$Y
z^@Qv5G+7jrkV0j0ahptb2p|0>ih@Ynip7?5hmM12qAIFsa$Y`+?pZVxZXmm!EBBB?
zEQuhpm~O>3Ns|lnLqLls>3*6ZLak1-(d~4b?M|~bXp^=%Fq@>=a_X&S*Qup_k#0k~
z7l6YK;UZ0bm|?=43NDC{IM|Q&B|wZ_-xV0AtdgA1cyMChl9odw4QhXYa}sdDz?oiP
z(;Z32V|c!{d1|u)-&64Yaqs8PDi0SPR&Yi<{61W8WO<>euq*c}Y5#GWyk?P@Qy`W^
zLk@_OMNT~@We6@~V;DNz3oPRCI5fg|U=W{Jfp3%5aMCd9X1$@CYX<cna3h>3I#-8b
zyjs`mx_K3v=(U>OF+6u5Z?nLc6$W<&T@8sfB16gzpv^d#COw>_>?cXPJXbp5Ko2nH
zTQqR6GFZQsnROjB#PunCn@@XhKC1lV?Mt66rc3`iQ@MEg^Y`HL>}T*zdzG|Wm2)L2
z|31dK3BY>tJ+LB`)8LaamD6xq797`;h2!EO?)R0n*z3vbxRAq)c<TES3)6wIa~OXb
z6R)uUmmoGwd_Mr4N%;c|(DHeVN4qw4NbCt&)tB97%1aoJ13Ol@frQg=JFrwsAPiiW
z$C0F@pDfE#;mDZBgRC1*KTajLLoXPBYSd%nC+U-IQ`J&1M?_G|-qhIJ2bx_3&8|GF
zh^an7v6WAzrH2cDIaR6r75=ghXqWvoIX%pVQ(|S8lSK(0B+P!nZYFC-RZu<{*tYpa
z?~O;5e=cAATzv6|FPnc`HvhhCes#f2nU?8?T`gv&$+n6qxXC)~x2(ILvh_auRi9mB
zr`blIUGKABf1I)#`)N)CP|C{`EGV~K*jxY#&OBXsR{7(Cf>&jiA4@3rlh#q8WaDrC
zxp(&8;0(*?s|yHhPtH;qEAf~QAz~;;kU$`|T*x909fx9qE$Vp?bE@0WNIDB^GP}Td
zA25D&2u2uVDH|j2{o#L>E0wo?*JzKyvDVk$ABUc6x#AYYG+ji*^|dP8T^l1UbnT5=
zz20oMSCAWS)J(nBTGPAocLfm(VhV;9jX}+^TWfXVSY~@byH>l?vD$0YteNVwf+8A5
z8)|4NVCq$u(81^TFY!4-duVzKLIb+=s^X)FeQGFhWZ;#qAX;8pL7wYV)M(a$Kp&}A
z3JcK8DdHnH45^KrC>Wz@oXvu$mfw_Iy;?xTjSPtCf<`{^j99o{2}-xCkVLVsaLDyY
z#}O5117+6&)+#ErD_Kb{Tr22GUK;Jn^33cj9lV4BvL>&bi@C|22KwoIm#C>X!7E^v
zuX$uVu*r2rkZS1qL$yY1WOvpJs-&|?(Wsl6Vhk2=#~oHzRObUHDPMgpTjjoA)A(CW
zXrqz24W6>Ch*I!^ScI`uode;j3>YB^`M7b-c}=a6xsB`lG*Z@r%fwM05u^mPY~rP_
zI6iCyo^zBJPvk7#-5?qn5w#e2FEwIo*sbkyLomvzal5yA=ich}&T1G@+f}<*V+=aP
zuoz$|_mZMj+|P~7BDb<#_Qa7S-={*)_7=JmL}0Y62O)ntHzjIL4o5I+xy_aBVHA8q
zEdd+QBY==A3$|h|f^i8$tyWWBd0?Ny0Q$)wD6g}r1-&D@y`xdznlt_|yJxNbhIip=
z?{4_Mk#FPj|7DP^93|e3h+I-&4e>ne!qTqL=4#7g^7NK=r3lYQ*4r!gu$N(=ZmM}c
z9C<PO68rxS`*>#^B7^`(rBP|(>$Of<-D<L3KZ}f+S~OLhmGq3Fj~7%PJr4FtwG%0~
zzVlLs1>NW1lj9jNH`ZoCwG#7mod=<4Sz!h7$bvT-KH88!pntBm%*5N6US(}YbOHwn
zwk`Kxs=e$uUx~pE?2!qL=69wuZt_hPPTcQX$H4!o`*muq=hl+F;=RrXXRT{{=xYA%
z;y8E!+n$lOSDjs3N^dI9xy<p-^x;9L)5wAjE)c1(Q{FJyQ4m7p0rWFS=g}Q6xRZ7C
z5C*>Uj+{%gv>}uyb8rTc?{~{uzNP1_vmGI<kn-#%%>NhX6nPOgjUKV+3k5�Ed>3
z$193+8d5&Gs@2=sTX@bxnH+q7N)Imh0cr|Xs>pSKR>cXT9vt13Ivmzsja>?bN4h+e
z$b-sKYE0jn%dM@W-Ul~tAP83|aL`yK-E_1Qxp2xT0t9bAkT4Py)lE}3O?1b7M@R48
z{Q#MW5snD75yV3lf%J<jCUT5gOYd~hyY4%(u*+PFp>|hqugSi!^MbQWN_F(UZ+WrY
zXCXiESMN@CWcDt9KG4XZA-vODlymhUIjE@y%}kV%g-n-@wj5|or*_);5YmczQ+0rm
z3KTXPOi=zTIFI&i)P)>Yx3DaN<BsS706%Dj20%;<$C?=pHW==}>2%R<_Ga;Hc%la{
z$`sTv0jCj=>>Q%IAMI>y_uj*jZhYhR?X4c}ZS`+$(7g~0QHOe*4#(6Ne0?X1p);`m
zAaLbjFH!X>_oE|pk5UAAoLIJ1b<%lJmM=wjAg4E^@)klG6C&j6Mv=|agJ@_#Pi{c^
zO&&c%LJ4wqGG(eWG4?J)>d%b=U-n*oKq6N*;#HNAgzUvk<^!@J@jj$yAQ!~WefTp-
V=A7v0L`N^9Bbl*fh{b%M{}(aa%<TXG
literal 0
HcmV?d00001
diff --git a/ray_results/test_experiment/train_model_45b8cfdd_17_batch_size=2048,decoder_int_ff=2048,decoder_int_num_layer=2,decoder_rt_ff=1024,decoder_rt_num_layer=4,drop_2024-06-25_16-46-32/error.txt b/ray_results/test_experiment/train_model_45b8cfdd_17_batch_size=2048,decoder_int_ff=2048,decoder_int_num_layer=2,decoder_rt_ff=1024,decoder_rt_num_layer=4,drop_2024-06-25_16-46-32/error.txt
new file mode 100644
index 0000000..3129628
--- /dev/null
+++ b/ray_results/test_experiment/train_model_45b8cfdd_17_batch_size=2048,decoder_int_ff=2048,decoder_int_num_layer=2,decoder_rt_ff=1024,decoder_rt_num_layer=4,drop_2024-06-25_16-46-32/error.txt
@@ -0,0 +1,68 @@
+Failure # 1 (occurred at 2024-06-25_16-47-55)
+[36mray::ImplicitFunc.train()[39m (pid=122466, ip=10.159.8.159, actor_id=3b1fd592afc06be8c677c69e01000000, repr=train_model)
+ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+ File "/gpfslocalsup/pub/anaconda-py3/2023.09/envs/pytorch-gpu-2.2.0+py3.11.7/lib/python3.11/site-packages/ray/tune/trainable/trainable.py", line 342, in train
+ raise skipped from exception_cause(skipped)
+ File "/gpfslocalsup/pub/anaconda-py3/2023.09/envs/pytorch-gpu-2.2.0+py3.11.7/lib/python3.11/site-packages/ray/air/_internal/util.py", line 88, in run
+ self._ret = self._target(*self._args, **self._kwargs)
+ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+ File "/gpfslocalsup/pub/anaconda-py3/2023.09/envs/pytorch-gpu-2.2.0+py3.11.7/lib/python3.11/site-packages/ray/tune/trainable/function_trainable.py", line 249, in <lambda>
+ training_func=lambda: self._trainable_func(self.config),
+ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+ File "/gpfslocalsup/pub/anaconda-py3/2023.09/envs/pytorch-gpu-2.2.0+py3.11.7/lib/python3.11/site-packages/ray/tune/trainable/function_trainable.py", line 332, in _trainable_func
+ output = fn()
+ ^^^^
+ File "/gpfslocalsup/pub/anaconda-py3/2023.09/envs/pytorch-gpu-2.2.0+py3.11.7/lib/python3.11/site-packages/ray/tune/trainable/util.py", line 138, in inner
+ return trainable(config, **fn_kwargs)
+ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+ File "/gpfswork/rech/ute/ucg81ws/these/LC-MS-RT-prediction/main_ray_tune.py", line 66, in train_model
+ pred_rt = net.module.forward_rt(seq)
+ ^^^^^^^^^^^^^^^^^^^^^^^^^^
+ File "/gpfsdswork/projects/rech/ute/ucg81ws/these/LC-MS-RT-prediction/model_custom.py", line 111, in forward_rt
+ out_rt = self.decoder_RT(enc)
+ ^^^^^^^^^^^^^^^^^^^^
+ File "/gpfslocalsup/pub/anaconda-py3/2023.09/envs/pytorch-gpu-2.2.0+py3.11.7/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1511, in _wrapped_call_impl
+ return self._call_impl(*args, **kwargs)
+ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+ File "/gpfslocalsup/pub/anaconda-py3/2023.09/envs/pytorch-gpu-2.2.0+py3.11.7/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1520, in _call_impl
+ return forward_call(*args, **kwargs)
+ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+ File "/gpfslocalsup/pub/anaconda-py3/2023.09/envs/pytorch-gpu-2.2.0+py3.11.7/lib/python3.11/site-packages/torch/nn/modules/container.py", line 217, in forward
+ input = module(input)
+ ^^^^^^^^^^^^^
+ File "/gpfslocalsup/pub/anaconda-py3/2023.09/envs/pytorch-gpu-2.2.0+py3.11.7/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1511, in _wrapped_call_impl
+ return self._call_impl(*args, **kwargs)
+ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+ File "/gpfslocalsup/pub/anaconda-py3/2023.09/envs/pytorch-gpu-2.2.0+py3.11.7/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1520, in _call_impl
+ return forward_call(*args, **kwargs)
+ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+ File "/gpfslocalsup/pub/anaconda-py3/2023.09/envs/pytorch-gpu-2.2.0+py3.11.7/lib/python3.11/site-packages/torch/nn/modules/transformer.py", line 391, in forward
+ output = mod(output, src_mask=mask, is_causal=is_causal, src_key_padding_mask=src_key_padding_mask_for_layers)
+ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+ File "/gpfslocalsup/pub/anaconda-py3/2023.09/envs/pytorch-gpu-2.2.0+py3.11.7/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1511, in _wrapped_call_impl
+ return self._call_impl(*args, **kwargs)
+ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+ File "/gpfslocalsup/pub/anaconda-py3/2023.09/envs/pytorch-gpu-2.2.0+py3.11.7/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1520, in _call_impl
+ return forward_call(*args, **kwargs)
+ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+ File "/gpfslocalsup/pub/anaconda-py3/2023.09/envs/pytorch-gpu-2.2.0+py3.11.7/lib/python3.11/site-packages/torch/nn/modules/transformer.py", line 715, in forward
+ x = self.norm2(x + self._ff_block(x))
+ ^^^^^^^^^^^^^^^^^
+ File "/gpfslocalsup/pub/anaconda-py3/2023.09/envs/pytorch-gpu-2.2.0+py3.11.7/lib/python3.11/site-packages/torch/nn/modules/transformer.py", line 731, in _ff_block
+ return self.dropout2(x)
+ ^^^^^^^^^^^^^^^^
+ File "/gpfslocalsup/pub/anaconda-py3/2023.09/envs/pytorch-gpu-2.2.0+py3.11.7/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1511, in _wrapped_call_impl
+ return self._call_impl(*args, **kwargs)
+ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+ File "/gpfslocalsup/pub/anaconda-py3/2023.09/envs/pytorch-gpu-2.2.0+py3.11.7/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1520, in _call_impl
+ return forward_call(*args, **kwargs)
+ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+ File "/gpfslocalsup/pub/anaconda-py3/2023.09/envs/pytorch-gpu-2.2.0+py3.11.7/lib/python3.11/site-packages/torch/nn/modules/dropout.py", line 59, in forward
+ return F.dropout(input, self.p, self.training, self.inplace)
+ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+ File "/gpfslocalsup/pub/anaconda-py3/2023.09/envs/pytorch-gpu-2.2.0+py3.11.7/lib/python3.11/site-packages/torch/nn/functional.py", line 1268, in dropout
+ return _VF.dropout_(input, p, training) if inplace else _VF.dropout(input, p, training)
+ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+torch.cuda.OutOfMemoryError: CUDA out of memory. Tried to allocate 200.00 MiB. GPU 0 has a total capacity of 15.77 GiB of which 68.69 MiB is free. Including non-PyTorch memory, this process has 15.70 GiB memory in use. Of the allocated memory 15.24 GiB is allocated by PyTorch, and 83.18 MiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables)
diff --git a/ray_results/test_experiment/train_model_45b8cfdd_17_batch_size=2048,decoder_int_ff=2048,decoder_int_num_layer=2,decoder_rt_ff=1024,decoder_rt_num_layer=4,drop_2024-06-25_16-46-32/events.out.tfevents.1719326860.r3i5n6 b/ray_results/test_experiment/train_model_45b8cfdd_17_batch_size=2048,decoder_int_ff=2048,decoder_int_num_layer=2,decoder_rt_ff=1024,decoder_rt_num_layer=4,drop_2024-06-25_16-46-32/events.out.tfevents.1719326860.r3i5n6
new file mode 100644
index 0000000000000000000000000000000000000000..f667b75df449fba850966736612c1eb6f560a35b
GIT binary patch
literal 40
rcmb1OfPlsI-b$RC?ujXHn|ISuiZ`h!F*8rkwJbHS#L8&X<@GxO*XIqS
literal 0
HcmV?d00001
diff --git a/ray_results/test_experiment/train_model_45b8cfdd_17_batch_size=2048,decoder_int_ff=2048,decoder_int_num_layer=2,decoder_rt_ff=1024,decoder_rt_num_layer=4,drop_2024-06-25_16-46-32/params.json b/ray_results/test_experiment/train_model_45b8cfdd_17_batch_size=2048,decoder_int_ff=2048,decoder_int_num_layer=2,decoder_rt_ff=1024,decoder_rt_num_layer=4,drop_2024-06-25_16-46-32/params.json
new file mode 100644
index 0000000..dab584b
--- /dev/null
+++ b/ray_results/test_experiment/train_model_45b8cfdd_17_batch_size=2048,decoder_int_ff=2048,decoder_int_num_layer=2,decoder_rt_ff=1024,decoder_rt_num_layer=4,drop_2024-06-25_16-46-32/params.json
@@ -0,0 +1,13 @@
+{
+ "batch_size": 2048,
+ "decoder_int_ff": 2048,
+ "decoder_int_num_layer": 2,
+ "decoder_rt_ff": 1024,
+ "decoder_rt_num_layer": 4,
+ "drop_rate": 0.18931069188451255,
+ "embedding_dim": 1024,
+ "encoder_ff": 2048,
+ "encoder_num_layer": 2,
+ "lr": 0.0002168775970949337,
+ "n_head": 4
+}
\ No newline at end of file
diff --git a/ray_results/test_experiment/train_model_45b8cfdd_17_batch_size=2048,decoder_int_ff=2048,decoder_int_num_layer=2,decoder_rt_ff=1024,decoder_rt_num_layer=4,drop_2024-06-25_16-46-32/params.pkl b/ray_results/test_experiment/train_model_45b8cfdd_17_batch_size=2048,decoder_int_ff=2048,decoder_int_num_layer=2,decoder_rt_ff=1024,decoder_rt_num_layer=4,drop_2024-06-25_16-46-32/params.pkl
new file mode 100644
index 0000000000000000000000000000000000000000..9bbaaa64ed5a70faf93a5c97e5680d371d33f75c
GIT binary patch
literal 225
zcmZo*oqCM{0&1sd^a!TrCFiH47RBe4=EmnFR;CtB@n-4~NlAr>7L_1Nu=I$+Br@|*
zWSDw*Q*)D2Q&KYX(&JMybEo(+u=H@j%uY)K@;Q2VVVZ%af`wRm_+UaH^PtMu^5Qd6
z6H}&mv-EJL6y+Dh7bTXYPI0$CVI8V(wt7tuQ%(_(r;}?EKW~RX4_8uRNpePfab{I2
I$dXb$0JNx4hyVZp
literal 0
HcmV?d00001
diff --git a/ray_results/test_experiment/train_model_45b8cfdd_17_batch_size=2048,decoder_int_ff=2048,decoder_int_num_layer=2,decoder_rt_ff=1024,decoder_rt_num_layer=4,drop_2024-06-25_16-46-32/result.json b/ray_results/test_experiment/train_model_45b8cfdd_17_batch_size=2048,decoder_int_ff=2048,decoder_int_num_layer=2,decoder_rt_ff=1024,decoder_rt_num_layer=4,drop_2024-06-25_16-46-32/result.json
new file mode 100644
index 0000000..e69de29
diff --git a/ray_results/test_experiment/train_model_481f1fff_10_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=8,drop__2024-06-25_17-33-30/events.out.tfevents.1719329693.r8i6n2 b/ray_results/test_experiment/train_model_481f1fff_10_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=8,drop__2024-06-25_17-33-30/events.out.tfevents.1719329693.r8i6n2
new file mode 100644
index 0000000000000000000000000000000000000000..0020c2e157c4fe99b9dd259b94491ab4dc45a174
GIT binary patch
literal 405
zcmb1OfPlsI-b$QO??ayNn0M1riZ`h!F*8rkwJbHS#LDP^@ttgCsJadF%bht*tj;j+
z1gaBY)Dq(o;o>bytkf?l%}dqK$uBMzRaN}zqHQ1)X#mwS_3MRr&H^X67G*9)E-AQ{
z;*9*#oRs+FjMU`pg8a<95>W<*277^@8U83%PIrOpN3}8~KQ9%ig@HkEfyiZjsFk^~
zvH_gdu5c|%Tnb#0a4SnPb5r9>GBS(fGfPs7;)_LfcRY4#l>PVG5UPD{bMs5i-3aX}
zTuNNhsM?D&^O95Ji&BeA@{3Zz2HXwhT4WA2;Q5PFk(@?uaQn5mG`JMu27qi$EXmBz
PD~?As2ITWAavALasCa;p
literal 0
HcmV?d00001
diff --git a/ray_results/test_experiment/train_model_481f1fff_10_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=8,drop__2024-06-25_17-33-30/events.out.tfevents.1719330754.r8i6n2 b/ray_results/test_experiment/train_model_481f1fff_10_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=8,drop__2024-06-25_17-33-30/events.out.tfevents.1719330754.r8i6n2
new file mode 100644
index 0000000000000000000000000000000000000000..ead04b117dd3c7918b653da129dc0043476292d2
GIT binary patch
literal 1311
zcmah|T}TvB7#(NCnh#dnao2SHX&nt>f>wJlkYKmJhcrZjpaQv!JJ)t}_s%kRtc)N*
zf}n?d=|RY(?5V;OqOhQc5{#Y->p_T=D9c`wP{AIoJMQf4uBpx2eCIpgx%YhMmU5|Y
z_)4=ZzDo`*jNhI<6!5g0s!sMDJCBLgxPNSHDPH6BjSgKnB68EGJ<nL*4qhn%A9zf)
zCuDUK9Euy1`rH029$jteTdi{_H<#Wb@zqC{QUlcxa49rqbjLLmjp1ll($I<J=eXYR
z<!0eSj-)i>@^d9MgJ4GCI5N^O9&#iTZI!L!{R~BhAe6b1mY%@Kis=;T7B&&}pPl@A
z_*B>OgL;QP*w;5N*3P<$wID-zE}iNmijj$_WtezvK-b(4-vMX9x9@Mxhy$4c3haSu
zcYv*`TDn0f${l0hd}MCQJDQi8(?2-SDgF{Zwb*7W_XM~jfDgrzs;MR@3gL@MY_hu$
ztLd5mR|VMSK+*~?E%S0AK)Y&1V@%}|4hpgag&K|;8oM*|8Fh59S{eAWYot4Y;%W~z
z8RP{|8Zw>kQt)M!vYS01YNnAy>|(YQ<Kjd+)-;`*Lz<poC3YH2QtE5Ue(<LMvj-Ib
zAGg&33J8i}RSOD=7mB3ZkSSGAA#D+&brd@!N+py_n}<Nz1(niPOICJ6jr9K|mK0BP
z5@&xef8K2r8}gm~edb|?hl<SpD5jL908vY22J585XnMiJb3U`jM||KR4<%XNN}~7H
z><#O9&qNrC(-yH}Dll~s2&~(hi}gpKY6Ht%oQpvD`hd7#Z-hPC4R`Oh1S3#n?~WDb
zrwX6EmuT2NOim2k=GF-8%+7-guv0Q16WeS+-XyDT&q8?d@!ZNm=MlbsI($+*k#@fx
bLJ;`ZdrY*H^DJW--}7aDd*1K8*T4J)%axaV
literal 0
HcmV?d00001
diff --git a/ray_results/test_experiment/train_model_481f1fff_10_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=8,drop__2024-06-25_17-33-30/params.json b/ray_results/test_experiment/train_model_481f1fff_10_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=8,drop__2024-06-25_17-33-30/params.json
new file mode 100644
index 0000000..4e01c9e
--- /dev/null
+++ b/ray_results/test_experiment/train_model_481f1fff_10_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=8,drop__2024-06-25_17-33-30/params.json
@@ -0,0 +1,13 @@
+{
+ "batch_size": 1024,
+ "decoder_int_ff": 512,
+ "decoder_int_num_layer": 1,
+ "decoder_rt_ff": 1024,
+ "decoder_rt_num_layer": 8,
+ "drop_rate": 0.15995138350968297,
+ "embedding_dim": 64,
+ "encoder_ff": 1024,
+ "encoder_num_layer": 2,
+ "lr": 0.004656630423530019,
+ "n_head": 16
+}
\ No newline at end of file
diff --git a/ray_results/test_experiment/train_model_481f1fff_10_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=8,drop__2024-06-25_17-33-30/params.pkl b/ray_results/test_experiment/train_model_481f1fff_10_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=8,drop__2024-06-25_17-33-30/params.pkl
new file mode 100644
index 0000000000000000000000000000000000000000..079cac3087e864cbb4289e74c79f3ee3b44eee0c
GIT binary patch
literal 224
zcmZo*oqCl40&1sd^a!TrCFiH47RBe4=EmnFR;CtB@n-4~NlAr>7L_1NaP)}6Br@|*
zWEgvRQ*)D2Q&KYX(&JMybEkMa^l-t9PD`8O%fQmZ3)2fU6e`3A69O3r7GvsR%ZtxQ
zO-z~MEzrZ6Qj}j1UzAvqI>p`oNTp{<|ARR_OgTkBUa`=XZG4ws_HZR7mLzAy7iU(b
Jf-EW30|4e%Q~Ce^
literal 0
HcmV?d00001
diff --git a/ray_results/test_experiment/train_model_481f1fff_10_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=8,drop__2024-06-25_17-33-30/progress.csv b/ray_results/test_experiment/train_model_481f1fff_10_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=8,drop__2024-06-25_17-33-30/progress.csv
new file mode 100644
index 0000000..78438d9
--- /dev/null
+++ b/ray_results/test_experiment/train_model_481f1fff_10_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=8,drop__2024-06-25_17-33-30/progress.csv
@@ -0,0 +1,3 @@
+loss,timestamp,checkpoint_dir_name,should_checkpoint,done,training_iteration,trial_id,date,time_this_iter_s,time_total_s,pid,hostname,node_ip,time_since_restore,iterations_since_restore
+1961.035730076587,1719329807,checkpoint_000000,True,False,1,481f1fff,2024-06-25_17-36-47,113.85971927642822,113.85971927642822,2195253,r8i6n2,10.159.28.60,113.85971927642822,1
+1959.9800554410679,1719330869,checkpoint_000001,True,False,2,481f1fff,2024-06-25_17-54-29,114.88935971260071,228.74907898902893,2195253,r8i6n2,10.159.28.60,114.88935971260071,1
diff --git a/ray_results/test_experiment/train_model_481f1fff_10_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=8,drop__2024-06-25_17-33-30/result.json b/ray_results/test_experiment/train_model_481f1fff_10_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=8,drop__2024-06-25_17-33-30/result.json
new file mode 100644
index 0000000..e6d1117
--- /dev/null
+++ b/ray_results/test_experiment/train_model_481f1fff_10_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=8,drop__2024-06-25_17-33-30/result.json
@@ -0,0 +1,2 @@
+{"loss": 1961.035730076587, "timestamp": 1719329807, "checkpoint_dir_name": "checkpoint_000000", "should_checkpoint": true, "done": false, "training_iteration": 1, "trial_id": "481f1fff", "date": "2024-06-25_17-36-47", "time_this_iter_s": 113.85971927642822, "time_total_s": 113.85971927642822, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 2, "decoder_rt_num_layer": 8, "decoder_int_num_layer": 1, "embedding_dim": 64, "encoder_ff": 1024, "decoder_rt_ff": 1024, "decoder_int_ff": 512, "n_head": 16, "drop_rate": 0.15995138350968297, "lr": 0.004656630423530019, "batch_size": 1024}, "time_since_restore": 113.85971927642822, "iterations_since_restore": 1}
+{"loss": 1959.9800554410679, "timestamp": 1719330869, "checkpoint_dir_name": "checkpoint_000001", "should_checkpoint": true, "done": false, "training_iteration": 2, "trial_id": "481f1fff", "date": "2024-06-25_17-54-29", "time_this_iter_s": 114.88935971260071, "time_total_s": 228.74907898902893, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 2, "decoder_rt_num_layer": 8, "decoder_int_num_layer": 1, "embedding_dim": 64, "encoder_ff": 1024, "decoder_rt_ff": 1024, "decoder_int_ff": 512, "n_head": 16, "drop_rate": 0.15995138350968297, "lr": 0.004656630423530019, "batch_size": 1024}, "time_since_restore": 114.88935971260071, "iterations_since_restore": 1}
diff --git a/ray_results/test_experiment/train_model_52d8dfa1_2_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=4,drop_r_2024-06-25_16-08-04/events.out.tfevents.1719324574.r3i5n6 b/ray_results/test_experiment/train_model_52d8dfa1_2_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=4,drop_r_2024-06-25_16-08-04/events.out.tfevents.1719324574.r3i5n6
new file mode 100644
index 0000000000000000000000000000000000000000..0c8fa347ef81f7c40770382a058151a3d74cbe79
GIT binary patch
literal 1241
zcmah|Pe>GD6yKc<a~`d_lWV%`+PYg2b7(cFK+N5WI#@yo9fIOx-1)W{b-r0<M$I6y
zJmevwf+P?#@*4CHibxXdQk^;l(jh2v^O8|*&>`v@clO&^Gn?DI-}}Ayz2EP>dQah-
zK6g0A=lJp5!<+N_LjE2@Qpt{kgP2%tJAdxVdt02m>-pD+pMCQEmz`JR730thAwwEY
zSOWwnOhq5)mr+{5>AtM45-aHOj3gr`^4)QVe{X8&06#Iy_;pYVe#Vz|f+@x0`8|0k
zoNy@G$;eS&nPZd~M4^FEELFqEQdASE7B-L>Jpb*%{<C!#I$9n2<ml)k&pl`KEf9ln
zIo(uA8Y2UnmTutX0#ge=diOX5a-Uuu=f_F~MA#0^tbpBBvQ(XzsJx6ldB@ny;H0-u
z2j?!1^T&>S-1dknCtmfzWgpZcB`X<{W}*Z>oy7*7g;<?=4!&}*-hqS#PWZsZssMT<
zE3MEjXK>UP6QD}QX<eo>Q=L&JLt_mPbjQemhWe#pY)~iw{vu>J+4WHC3b~X0z{`f7
zMRYK`73E^B2g|ZbP9a&<Xo$WBlS239&Orzi|J{Wufa|xr15^;CU`dYp#3pDI*4P*^
z0M$bIMu-sz3;%Xp+zb(6El0(z5EuUMSfst{C2`D}Te{uG-zb{(o`ZKBG?vUVv1wAv
zk!eX(O_gNy&SxAvb(waT3*F(M&LyP42Zr8;`>ttAN!VD_g9Z!R5lum8g+&azl7gmH
z^kq2ZhF5L_14mNyLRU4KrPQRL+BS*?b}vp;UYgG&?Z=dP&XQYFu*q#c1L!L$P})%j
z+)B}lTYWKE&X?qyuFrbAb8#;oPj~b8i|V&R0z!XlkBMARo-R!3Sk}e3O7C8AL+>BH
Cp@cdB
literal 0
HcmV?d00001
diff --git a/ray_results/test_experiment/train_model_52d8dfa1_2_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=4,drop_r_2024-06-25_16-08-04/params.json b/ray_results/test_experiment/train_model_52d8dfa1_2_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=4,drop_r_2024-06-25_16-08-04/params.json
new file mode 100644
index 0000000..cef4024
--- /dev/null
+++ b/ray_results/test_experiment/train_model_52d8dfa1_2_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=4,drop_r_2024-06-25_16-08-04/params.json
@@ -0,0 +1,13 @@
+{
+ "batch_size": 2048,
+ "decoder_int_ff": 512,
+ "decoder_int_num_layer": 1,
+ "decoder_rt_ff": 1024,
+ "decoder_rt_num_layer": 4,
+ "drop_rate": 0.982125481793317,
+ "embedding_dim": 16,
+ "encoder_ff": 1024,
+ "encoder_num_layer": 8,
+ "lr": 0.002789281371289888,
+ "n_head": 8
+}
\ No newline at end of file
diff --git a/ray_results/test_experiment/train_model_52d8dfa1_2_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=4,drop_r_2024-06-25_16-08-04/params.pkl b/ray_results/test_experiment/train_model_52d8dfa1_2_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=4,drop_r_2024-06-25_16-08-04/params.pkl
new file mode 100644
index 0000000000000000000000000000000000000000..7c3abfbca8fa2251d2a0adc0e85159d217bcd515
GIT binary patch
literal 224
zcmZo*oqCl40&1sd^a!TrCFiH47RBe4=EmnFR;CtB@#g3eNlAr>7L_1Nu=I$+Br@|*
zWEgvRQ*)D2Q&KYX(&JMybEkL<^l-t9PD`8O%fQmZ3)2fU6e`3A69O3r7GvsR%ZtxQ
zO-unfgEOTlzaYLSu_SehyZ!szN!il3%X^q|ih#Vdn>`$}Uz+uBB_)<5XT%q0R;2<h
I;V9Ju0L05vJpcdz
literal 0
HcmV?d00001
diff --git a/ray_results/test_experiment/train_model_52d8dfa1_2_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=4,drop_r_2024-06-25_16-08-04/progress.csv b/ray_results/test_experiment/train_model_52d8dfa1_2_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=4,drop_r_2024-06-25_16-08-04/progress.csv
new file mode 100644
index 0000000..1e9afeb
--- /dev/null
+++ b/ray_results/test_experiment/train_model_52d8dfa1_2_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=4,drop_r_2024-06-25_16-08-04/progress.csv
@@ -0,0 +1,2 @@
+loss,timestamp,checkpoint_dir_name,should_checkpoint,done,training_iteration,trial_id,date,time_this_iter_s,time_total_s,pid,hostname,node_ip,time_since_restore,iterations_since_restore
+nan,1719324663,checkpoint_000000,True,False,1,52d8dfa1,2024-06-25_16-11-03,89.4755928516388,89.4755928516388,69318,r3i5n6,10.159.8.159,89.4755928516388,1
diff --git a/ray_results/test_experiment/train_model_52d8dfa1_2_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=4,drop_r_2024-06-25_16-08-04/result.json b/ray_results/test_experiment/train_model_52d8dfa1_2_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=4,drop_r_2024-06-25_16-08-04/result.json
new file mode 100644
index 0000000..624a4df
--- /dev/null
+++ b/ray_results/test_experiment/train_model_52d8dfa1_2_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=4,drop_r_2024-06-25_16-08-04/result.json
@@ -0,0 +1 @@
+{"loss": NaN, "timestamp": 1719324663, "checkpoint_dir_name": "checkpoint_000000", "should_checkpoint": true, "done": false, "training_iteration": 1, "trial_id": "52d8dfa1", "date": "2024-06-25_16-11-03", "time_this_iter_s": 89.4755928516388, "time_total_s": 89.4755928516388, "pid": 69318, "hostname": "r3i5n6", "node_ip": "10.159.8.159", "config": {"encoder_num_layer": 8, "decoder_rt_num_layer": 4, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 1024, "decoder_rt_ff": 1024, "decoder_int_ff": 512, "n_head": 8, "drop_rate": 0.982125481793317, "lr": 0.002789281371289888, "batch_size": 2048}, "time_since_restore": 89.4755928516388, "iterations_since_restore": 1}
diff --git a/ray_results/test_experiment/train_model_5763f130_1_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=8,drop_r_2024-06-25_17-10-00/events.out.tfevents.1719328213.r8i6n2 b/ray_results/test_experiment/train_model_5763f130_1_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=8,drop_r_2024-06-25_17-10-00/events.out.tfevents.1719328213.r8i6n2
new file mode 100644
index 0000000000000000000000000000000000000000..a15214dc49bed0fae69d1e250b80c268488a66a4
GIT binary patch
literal 1311
zcmah|OK1~O6ip_fbuOlDrb%s+CZ<yhItyc6C{!WQQiOJ)6gLX>G0D6(6O%V#W}>Yi
z7K$QB!Ho+S_1B683oWA7A}DkvE<`teP!Zg;u5?qO*oAMJ$;-#KW;gfTbI-fyo-{iO
zzsXB|BL8vicz*uYry~JRLYGytx91!tX4n1?(=(&(cHi9X^WFT#TK6NXufZubfe$>o
zJQ6kY1V=NPVff+wc+WL&sAHc^xv}yV@%QT(r5)NJz$iva%V!jnOyT6&oTd`f?{JL9
z(ko}4R!DAdWc*4=MI)F}9FCzq=0Mbz%npQ3@Gnag5hBpSBuzDok(p8rq?%YqhJX2u
z-kp9tKHX{4C&tEp@O?g}xC=xGR?-cXBr(#lVQM;FA29o{#eL8ou(bU86#uR?K!V-S
z#s*lbvZ-psK$T;x&HuP#t8>o6oa+-e)BKY!vy%(dY-!2`cU|B^shq6KSp!A!a1QHq
z7h?8ab-`s9)Y_1+$_c+Yu?`?1o5>Vaxqu@sQGhxHCpCrcOnpX!gS6HH{_+~hXHiBT
z!8(P!;3-17-Q5hnvQl}o2Y5x-a)>TwNl`A&Ca|KYWC$s0mX_#iFe#|7EBnD){4O6<
z16;;Z2dE(^g=Hn;lDtqaREA6nL#wbwh^i=7NTd)1h0Q}C?Szo9)sdwRXczu}#EHdN
zJ+be%`14*D@BC`t?>h(IIA|^1&%lO3Q-BOprUt8IQ0slc!E>&3$5}2g!$DJ-w*v3&
zKk)g`RCp-{jYW%S(GFf50##RI*qu0pH?W-H4RHuo2ZVv6ar&p-@b;`F5{G*0cC^4R
z96NgD^n;l7n2H#%=3E?hl;?o~REPql*fM}VU)&_CtmjuOvKD@J*!JSp{A52LEV|zb
aQ3(9&Jtj)Qd7813?{1OX=Il5*jQ;{4Wsi{n
literal 0
HcmV?d00001
diff --git a/ray_results/test_experiment/train_model_5763f130_1_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=8,drop_r_2024-06-25_17-10-00/params.json b/ray_results/test_experiment/train_model_5763f130_1_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=8,drop_r_2024-06-25_17-10-00/params.json
new file mode 100644
index 0000000..554644b
--- /dev/null
+++ b/ray_results/test_experiment/train_model_5763f130_1_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=8,drop_r_2024-06-25_17-10-00/params.json
@@ -0,0 +1,13 @@
+{
+ "batch_size": 2048,
+ "decoder_int_ff": 512,
+ "decoder_int_num_layer": 1,
+ "decoder_rt_ff": 2048,
+ "decoder_rt_num_layer": 8,
+ "drop_rate": 0.7224904221599877,
+ "embedding_dim": 256,
+ "encoder_ff": 512,
+ "encoder_num_layer": 4,
+ "lr": 0.04561252173503944,
+ "n_head": 2
+}
\ No newline at end of file
diff --git a/ray_results/test_experiment/train_model_5763f130_1_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=8,drop_r_2024-06-25_17-10-00/params.pkl b/ray_results/test_experiment/train_model_5763f130_1_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=8,drop_r_2024-06-25_17-10-00/params.pkl
new file mode 100644
index 0000000000000000000000000000000000000000..4395f1ff6fd0ff4d452b715ee4e214e7f9c36808
GIT binary patch
literal 225
zcmZo*oqCM{0&1sd^a!TrCFiH47RBe4=EmnFR;CtB@n-1}NlAr>7L_1NaP)}6Br@|*
zWEgvRQ*)D2Q&KYX(&JMybEo(+F!pf4%uY)K@|k*gVVZ%af`vGG_+UaH^PtMu^5Qd6
z6H}&mGxcz$6y+Dh7bTXYPI0$?F1N(`wpmaQQ%(_(w>+xN$$ckh4_8uRNpePfab{I2
J&=QVPJpjZnQS$%*
literal 0
HcmV?d00001
diff --git a/ray_results/test_experiment/train_model_5763f130_1_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=8,drop_r_2024-06-25_17-10-00/progress.csv b/ray_results/test_experiment/train_model_5763f130_1_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=8,drop_r_2024-06-25_17-10-00/progress.csv
new file mode 100644
index 0000000..c3d04e5
--- /dev/null
+++ b/ray_results/test_experiment/train_model_5763f130_1_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=8,drop_r_2024-06-25_17-10-00/progress.csv
@@ -0,0 +1,2 @@
+loss,timestamp,checkpoint_dir_name,should_checkpoint,done,training_iteration,trial_id,date,time_this_iter_s,time_total_s,pid,hostname,node_ip,time_since_restore,iterations_since_restore
+4538.754975086122,1719328455,checkpoint_000000,True,False,1,5763f130,2024-06-25_17-14-16,242.80822491645813,242.80822491645813,2195253,r8i6n2,10.159.28.60,242.80822491645813,1
diff --git a/ray_results/test_experiment/train_model_5763f130_1_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=8,drop_r_2024-06-25_17-10-00/result.json b/ray_results/test_experiment/train_model_5763f130_1_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=8,drop_r_2024-06-25_17-10-00/result.json
new file mode 100644
index 0000000..ab24623
--- /dev/null
+++ b/ray_results/test_experiment/train_model_5763f130_1_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=8,drop_r_2024-06-25_17-10-00/result.json
@@ -0,0 +1 @@
+{"loss": 4538.754975086122, "timestamp": 1719328455, "checkpoint_dir_name": "checkpoint_000000", "should_checkpoint": true, "done": false, "training_iteration": 1, "trial_id": "5763f130", "date": "2024-06-25_17-14-16", "time_this_iter_s": 242.80822491645813, "time_total_s": 242.80822491645813, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 4, "decoder_rt_num_layer": 8, "decoder_int_num_layer": 1, "embedding_dim": 256, "encoder_ff": 512, "decoder_rt_ff": 2048, "decoder_int_ff": 512, "n_head": 2, "drop_rate": 0.7224904221599877, "lr": 0.04561252173503944, "batch_size": 2048}, "time_since_restore": 242.80822491645813, "iterations_since_restore": 1}
diff --git a/ray_results/test_experiment/train_model_5b0a0ae2_6_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop_r_2024-06-25_17-18-52/events.out.tfevents.1719329376.r8i6n2 b/ray_results/test_experiment/train_model_5b0a0ae2_6_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop_r_2024-06-25_17-18-52/events.out.tfevents.1719329376.r8i6n2
new file mode 100644
index 0000000000000000000000000000000000000000..f6d895802d3803de895243eb6a9f615d9abfdf13
GIT binary patch
literal 405
zcmb1OfPlsI-b$P+?;Fk7G4H0M6mL>dVrHJ6YguYuiIvf{D8n_%P<0#TmpgNcYWb`M
zsS{w-65|r#;w?(7)GsN`OV!WGFD@1}mH*<RyYPpw0aVM>uNUGu4fNq!l(`hSq~KbL
zGxAGwQsR>{Qj@a_@-y>FL>U+w?A6i|El{kCHiYX(wK63?FBPbTfgy0#?>c>`mASF9
z0i53vT9mjHxFq3LmSpCp#+PJd7RP6nq!z^&i@ra;!ig_*ler;O``qT{mz?EBa2r*)
zl(?i(wHIgRC8x$0r52au7o~y?P+t5f(j02Q^B1QgIX@!|(BjhIQiK}-vNf?JGe565
O9@Q9-&)@cE^8x@LjDrmT
literal 0
HcmV?d00001
diff --git a/ray_results/test_experiment/train_model_5b0a0ae2_6_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop_r_2024-06-25_17-18-52/events.out.tfevents.1719330587.r8i6n2 b/ray_results/test_experiment/train_model_5b0a0ae2_6_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop_r_2024-06-25_17-18-52/events.out.tfevents.1719330587.r8i6n2
new file mode 100644
index 0000000000000000000000000000000000000000..4f8b36a8361df3d3b03014589393ace09171acf0
GIT binary patch
literal 1311
zcmah|O=uHA7)>{|bw5mN)}&37{-j%lvWKP>K`25}8$l~*^`;=hCOa`}li9GltyY8v
zics((f(M0KwSpc53nHS$A3PNGq(?0(c+g4@EtM1nE6z5XNm6ay+kWqR-<x^g8=oWp
zO^zHCdC!rLGmoyl>+!o{x~!6peM6X--8(PLt<<#HeNz*sd-=-wm07EAD<{=~2VA;5
z95M$94y821@P<Zm(fY&B^-h~|Irj$fOACzB2Cd*{6eFn(rW6!U;`l&DQ;F$yII`hi
z_gj2rlDn1|zg$w$2&NQ=<M{q3=R&q*`b1Md|Efq4AqWjj(p1wJnMu_^s)==Ec%wh=
z?43VQZFbr8@zK$5e9w2LxE(|Yl+z8B#4*ybVQM;F9gxcf26o#6zRtZq#!nXqNYD<g
zY=EUIo2o_(R6fSKd2V_1?Nmj6&c*R7r}&xo9}YdHW=pr6aKi~6l+4JwoHkGhpUGgI
z-i4TbV@^2lglZcSesaPxC)NPOWHX+mDra%fDGE@d;JBvHJ5!%gB0*~n;4Q6@!8A(A
z!&s+~8(alQx4V7dDJhjUyMR}8EraOAEGf#x=@?cNm7GM1nx-Xs8cg!)tIA$*7ye5J
zRRNc>)B&mpN@7_FIwd#M3gsb_nxRqHAVg~@R!F2K2ng$kKx%;|VWTBWJD^SY{}Nlq
zE=>@7e=inpcJuAC_WnL|@QH)Q;{FV57&HaQFlB16N)lS%Qx2xN;vVmD{<|F1m3S-g
z?(FZxQ2&?VFl;MWM2q>rbVk6xW@`+)9)ae6SkCZZ1OjUV!oX~VKH7iop0zB7q1M_R
zEyOyuJh&Q_!`5NSV!)c|LF-HNzyK;lfnsbKK#wo3lU3HU7>;hb{cMl@2wy&&Jj#a)
d?sq{5{C|6oiIR7oW~}IYrO0ir@bzXBe*p76osj?l
literal 0
HcmV?d00001
diff --git a/ray_results/test_experiment/train_model_5b0a0ae2_6_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop_r_2024-06-25_17-18-52/params.json b/ray_results/test_experiment/train_model_5b0a0ae2_6_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop_r_2024-06-25_17-18-52/params.json
new file mode 100644
index 0000000..e842615
--- /dev/null
+++ b/ray_results/test_experiment/train_model_5b0a0ae2_6_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop_r_2024-06-25_17-18-52/params.json
@@ -0,0 +1,13 @@
+{
+ "batch_size": 2048,
+ "decoder_int_ff": 512,
+ "decoder_int_num_layer": 1,
+ "decoder_rt_ff": 2048,
+ "decoder_rt_num_layer": 1,
+ "drop_rate": 0.0063231604639778505,
+ "embedding_dim": 16,
+ "encoder_ff": 512,
+ "encoder_num_layer": 8,
+ "lr": 0.0021079084039053323,
+ "n_head": 16
+}
\ No newline at end of file
diff --git a/ray_results/test_experiment/train_model_5b0a0ae2_6_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop_r_2024-06-25_17-18-52/params.pkl b/ray_results/test_experiment/train_model_5b0a0ae2_6_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop_r_2024-06-25_17-18-52/params.pkl
new file mode 100644
index 0000000000000000000000000000000000000000..736546c8843456938dfbfec2824c7e47ab0926d8
GIT binary patch
literal 224
zcmZo*oqCl40&1sd^a!TrCFiH47RBe4=EmnFR;CtB@#g3eNlAr>7L_1NF!qSTBr@~R
zWO!3^lTuSsGV{{oQ!;a>cnkD!!HiBzo8rsB)WZwY3p5lg#L>eC69O3rRmPSVpOKoF
z0y2R!r6|83z9_LIb&9)v<+FgYw0{jfOgTkBUZTsatvq^3JzPnNCCM4_#hF#9Kub7E
F^#HfIQi=co
literal 0
HcmV?d00001
diff --git a/ray_results/test_experiment/train_model_5b0a0ae2_6_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop_r_2024-06-25_17-18-52/progress.csv b/ray_results/test_experiment/train_model_5b0a0ae2_6_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop_r_2024-06-25_17-18-52/progress.csv
new file mode 100644
index 0000000..9422605
--- /dev/null
+++ b/ray_results/test_experiment/train_model_5b0a0ae2_6_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop_r_2024-06-25_17-18-52/progress.csv
@@ -0,0 +1,3 @@
+loss,timestamp,checkpoint_dir_name,should_checkpoint,done,training_iteration,trial_id,date,time_this_iter_s,time_total_s,pid,hostname,node_ip,time_since_restore,iterations_since_restore
+1952.9751620555487,1719329461,checkpoint_000000,True,False,1,5b0a0ae2,2024-06-25_17-31-01,84.39049577713013,84.39049577713013,2195253,r8i6n2,10.159.28.60,84.39049577713013,1
+1963.9745468980684,1719330671,checkpoint_000001,True,False,2,5b0a0ae2,2024-06-25_17-51-11,84.48489356040955,168.87538933753967,2195253,r8i6n2,10.159.28.60,84.48489356040955,1
diff --git a/ray_results/test_experiment/train_model_5b0a0ae2_6_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop_r_2024-06-25_17-18-52/result.json b/ray_results/test_experiment/train_model_5b0a0ae2_6_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop_r_2024-06-25_17-18-52/result.json
new file mode 100644
index 0000000..dd7065b
--- /dev/null
+++ b/ray_results/test_experiment/train_model_5b0a0ae2_6_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop_r_2024-06-25_17-18-52/result.json
@@ -0,0 +1,2 @@
+{"loss": 1952.9751620555487, "timestamp": 1719329461, "checkpoint_dir_name": "checkpoint_000000", "should_checkpoint": true, "done": false, "training_iteration": 1, "trial_id": "5b0a0ae2", "date": "2024-06-25_17-31-01", "time_this_iter_s": 84.39049577713013, "time_total_s": 84.39049577713013, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 8, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 512, "decoder_rt_ff": 2048, "decoder_int_ff": 512, "n_head": 16, "drop_rate": 0.0063231604639778505, "lr": 0.0021079084039053323, "batch_size": 2048}, "time_since_restore": 84.39049577713013, "iterations_since_restore": 1}
+{"loss": 1963.9745468980684, "timestamp": 1719330671, "checkpoint_dir_name": "checkpoint_000001", "should_checkpoint": true, "done": false, "training_iteration": 2, "trial_id": "5b0a0ae2", "date": "2024-06-25_17-51-11", "time_this_iter_s": 84.48489356040955, "time_total_s": 168.87538933753967, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 8, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 512, "decoder_rt_ff": 2048, "decoder_int_ff": 512, "n_head": 16, "drop_rate": 0.0063231604639778505, "lr": 0.0021079084039053323, "batch_size": 2048}, "time_since_restore": 84.48489356040955, "iterations_since_restore": 1}
diff --git a/ray_results/test_experiment/train_model_5eb1e282_23_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=1,drop_r_2024-06-25_18-57-29/events.out.tfevents.1719335674.r8i6n2 b/ray_results/test_experiment/train_model_5eb1e282_23_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=1,drop_r_2024-06-25_18-57-29/events.out.tfevents.1719335674.r8i6n2
new file mode 100644
index 0000000000000000000000000000000000000000..bb8893a139aa6e56fab04eaf274dd18721a436a5
GIT binary patch
literal 770
zcmb1OfPlsI-b$R8w07>>H}9sS6mL>dVrHJ6YguYuiIq{|we2d(P<0#TmpgOT?0$b1
zq)vcQON>i|i?=ATQop1$FI7J$zqnZR!NCR>orY7q22d?izg~#v{JS5nMVU*HOA4-~
zI3vF_CnY{PBQ-g@AU`v&M3jM{!Cvc9$Ym5OgAc*=qgt7gpO*^M!ocwLud|~*)XLmg
z*#OQr2rWun3S5$KD@!tSQ{ziAGK=FgOHzyCi$%|-6gwSc*tg9Ps(o&A^GnX0!*Cl_
zxRki0QMDIm<|U`b7o`@L<QJua4Y<2BlG7Y&!1EWUA~|0o4AA1z;8KJe0J1f)Br`v+
zI3Cp)kk2pFcru{)Jm2W<O>kf{VGHaVjXW+>-bimn3G5nUxE5jpyFN}L9>q!~6S#gH
zfvv)BqJ|RK9SAK10{hwpJ10rD7pqVLo52)rBf-GFBz4m%@%~2zl)$b+7(jGjGdRrX
H&aeXjKF|b1
literal 0
HcmV?d00001
diff --git a/ray_results/test_experiment/train_model_5eb1e282_23_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=1,drop_r_2024-06-25_18-57-29/events.out.tfevents.1719339767.r8i6n2 b/ray_results/test_experiment/train_model_5eb1e282_23_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=1,drop_r_2024-06-25_18-57-29/events.out.tfevents.1719339767.r8i6n2
new file mode 100644
index 0000000000000000000000000000000000000000..948ba6c57f934e4a1d8025f5603eda83973ad968
GIT binary patch
literal 2230
zcma*oeMl2=9LMp2PrE~*B70Z~Ix}UO(}DzRY~B%pn%g9({;*@YgI&5i+1-4AWupIB
zkwj#X=3_)Dfn^pYEhYQI9*lwtG}MY_h7V;XSdOf%@jJe-akszk{rm8Haojd3{{OC?
z%u{2F8E@T>TSs$LF~=MhhE3mBPP0Px&aATEs>Hx^x7sR5tZ@H<5#Mub!ZaK3^>~cK
zQl%A~EUhi&cs@2pm>~O>kB(&olE$T16gHlZl0-ZKSD}Qran4dJRcxb+OUgKg6=D^N
z8hx}qg#nXVBkJL#m1F6Di9+$K@6;J>VA5U`XTn~Ji5ic`H={{`vD1`bV|a=YXa~i|
zrnxGJs;RX<GXr@`UENpAXhMtIa5cUK<~+j|)0Bhe1<pbH4@gn=rey^V_%i+M2(};|
zputn{1ayFJ)gmw)%Tw?e-_08q<8#5yJ7$tc{n+8buw9=9$p`K6Zh-BYMTs2jR%=T(
zm|Pb75ZLRwnF@eiK8GeHushmxgtm1c6<{xjxg_?~*)gIrX*LyLTj$ZfY;1*LH=SSy
zH}4Ej_V}?Qf?+!wddQ(_a~HrKjzEbVZ0_dj4lrqqM12VCAMcAs0QMU(k-(mq))U8@
z-Sq%FKME~MVyEWBk+U~cB>?-0cz|qdgP})v%>-_KsUU2@j~y8dyLL}I`8@CT34s0C
zh!Q#2O+}4Yz+{;T^&zmGYV9Pz_KJxF_UnEfanN%i3Sd{7(V`^wWKTYs6<1OXu$ROG
zWMkJD(myC?z|B_*{&@Y^QNggk_n#-py$`(rTV+Ct9BfN-?^ZC`DfS_-pAzTbo1JDx
zlM>jcyY)m8ss9GByTn`)ySB+iQkvC|06Vb|?aRh)H{dPX;Rh588&?kb{vegXuxl^Z
zleXc3>i}C5juJW8(fSvQ0Q-*Ehrr(WNVo^yY+VGJl)z>@IwI43=q|v%FXoci^Vmgl
WZsqVhfW0dc?aRg<G<b4yy?+7!xZtM%
literal 0
HcmV?d00001
diff --git a/ray_results/test_experiment/train_model_5eb1e282_23_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=1,drop_r_2024-06-25_18-57-29/events.out.tfevents.1719345748.r8i6n2 b/ray_results/test_experiment/train_model_5eb1e282_23_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=1,drop_r_2024-06-25_18-57-29/events.out.tfevents.1719345748.r8i6n2
new file mode 100644
index 0000000000000000000000000000000000000000..cc958918d7198f3e194bf3add12949bb2505abe6
GIT binary patch
literal 4596
zcma*reN<Cr90%~ZDS96pvhfm(SBD%tOB_gL$ZTU*O3~x7$fF!j-R5?2OLm9b1(aT#
z8H$<4>G%>B;tQwLJjO}QPQJkLC4n+2?3@S_@rVeTl9rgrsk<?r%g;9N<-gtM`99x0
zpXdI;vq;I{-|m&O)kIJ;vAJP;YnDpB*lDscle3FyhMhL0X62FEMBlvH>JpMrgj#Am
z^Fjzs1PViPr)h<jEn;YGfy3nr|E1VXc7NV`F4Y&={C6`&$Rcqh5sgPG9C2A4MFkeh
zY^BXh3mrCw4VOsV`i#zma!@&4h4Zk|;$UblB9V|8i#BO}mG)O-=Mm@lh#DoJC|t?f
z>@>w%Z7#~j(oV`1KB>CMIC}ows5D=^YW3=FVrDdMoPg9Q8pd5V#!ORA+QmAYw08k=
z+4SX8eG7D5Z^|R;_ysg*A{vhu@U)s(n}cyta2e0ekJld>3wHkSQcA5C8~MXdswgD0
zJ0?8<*y_tTA_hBU&Z=iX<pDk)0DD`EWD3B}?7)=**v>6FW6y2P9f18U9~Z<vuWB{U
zK2Tl-uv4$#e$m(xLqhYB0kHD{#e<_>?6Ll^7di6Dtb6<31lV_#I3fnS?dA^$KxJtt
z&IiEGtKT{YV2=vJ5drMN6FQ^$`L0<2dlerS#Fib8Cf7I2tOMBh`2|E{8w|O7tq!pB
z_>@y;z1UCq!=7bbM7Bm9)B^0-$v7eg`=ew_1gQLs&j-NHJd|Druv4_SQUJUC1D)~L
zxoIYVUBkx(u`@5{kU7?mn*er1D()AJ?KXUc)|kM~>8l%Cyx4Mo*jcA@$&mxQM*{36
zoj4)}drN6$3#jbj^8v6sTAZ%{Y~~uS6u@5iwa%DP+W#ED?&jly*gwfj$+xQ3eG0H&
z>%#q_v3D5$%&^P@JHLO%-R{K>_J>{Uo<lC}e|9y%o_H2V#9%+nyXFLyhxvQ}?Dt-e
zTL-Z94Y*PO`&Ojh7?=4+GQd8>#|5#2dNz^s>o&X#u#+2czi8}w!>-O-d0^+)t5R=x
zvB&wt{xSFkav;6-I=~)XjU!^P^AGOPfXWI!9{{^(C6ffOBX{6R0qmuH(~Kqm5<via
z3m+H6{%hub@=n{>WPmN-iTg!kUp5@qepdo^US#aQ=fw{3hkY!<NS-d=mkqFsNgNS_
zeT;2Hpz<M~4}iU(E$S%1{$d8M6u|zbMrZ6_uvZDNHQ6{Wh`s*zQ{<|(C>LOF=NAx-
zEy?Q3FMA2>yl=x9X`|==Y=uAU3oSFqR&#MKz`nB)N5o)<y?8hsRI(r9d;sjHFLay*
z*!_G&0K2tXXZ+1xR|c@z3fw4&y=GY(xi#u$48ZQ;7Z8mdl2y6K*a&tW+E5+r#a8;m
z&g`2?-q_Y<1K4CEj)=i-U3lIODx3Ix0PHcRE8hm#bDMCb0JgGvx=}moVl=?6=i`Fd
z)}*`Su}RLW06Xn_+%FnCA*+P!jjtUscsZ)5s(gdEcxkT}TT>%LJ7p-0vKE@0CcBH$
z(#s2JCwD<(*qm}1DwCnnz9Hp5Qsq6VIvAnFCf01_u1=+Nf=sPM!4}%=uy7YS?(#_G
z=X1}ZQ26j?Op%=`Fs-1S+>ipvhlZTK*^wx0*rVZ-<%qC29fcHkTk}+LS=zpswpeV;
z63Sw;bC0-hLo<Wz-sW(m82UVXp&%qJ@U$Zo#86h+WJ!=|6lk0h-kddYC`S31oADzN
z&&^j8i=vf*Z?>9v6svsP%~g|x5|zR?(-SANlNjImoxHtk8u3)S@BFSvQM(ky@aN~E
zT`ulwOSxDR2W(^V9XThY=%|!G$1kO-{ZbS$%vMP#-ggDXr>Fg_N1;PR+{3{aCd7bL
zkDMAFa~n{cA96h2t4Gm~z94vz3y=4I_IL`HALltc?!i0VsB>H51ik0WfY$&|PBow>
zhrbVapu#)AzqWW_hk^TL!T-C8=k)3~Wi*?0zF!s^j_iJ!m^GxIhO|iae|ws-49auw
Q81Ft(EgduB6Zb9lKO>mjZU6uP
literal 0
HcmV?d00001
diff --git a/ray_results/test_experiment/train_model_5eb1e282_23_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=1,drop_r_2024-06-25_18-57-29/params.json b/ray_results/test_experiment/train_model_5eb1e282_23_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=1,drop_r_2024-06-25_18-57-29/params.json
new file mode 100644
index 0000000..a4aefb0
--- /dev/null
+++ b/ray_results/test_experiment/train_model_5eb1e282_23_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=1,drop_r_2024-06-25_18-57-29/params.json
@@ -0,0 +1,13 @@
+{
+ "batch_size": 2048,
+ "decoder_int_ff": 512,
+ "decoder_int_num_layer": 1,
+ "decoder_rt_ff": 512,
+ "decoder_rt_num_layer": 1,
+ "drop_rate": 0.2842529108982538,
+ "embedding_dim": 16,
+ "encoder_ff": 1024,
+ "encoder_num_layer": 1,
+ "lr": 0.00015364900402465778,
+ "n_head": 4
+}
\ No newline at end of file
diff --git a/ray_results/test_experiment/train_model_5eb1e282_23_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=1,drop_r_2024-06-25_18-57-29/params.pkl b/ray_results/test_experiment/train_model_5eb1e282_23_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=1,drop_r_2024-06-25_18-57-29/params.pkl
new file mode 100644
index 0000000000000000000000000000000000000000..18b52d2f95739444bc3641862802d17e21716ccb
GIT binary patch
literal 224
zcmZo*oqCl40&1sd^a!TrCFiH47RBe4=EmnFR;CtB@n-B1NlAr>7L}k%h{7Z?^U!2?
zQ*)D2Q&KYX(&JMybEkL<^l-t9PD`8O%fQmZ3)2fU6fDHl!v_-r83z?(%ZtxQO-z~M
z&C<h}Qj}j1UzAvqI>p`olA*CYTX9zpQ%(_(r=mRTDc8%o9<HRslH`o|;>@a4pd}oo
FdH{ZDQE31G
literal 0
HcmV?d00001
diff --git a/ray_results/test_experiment/train_model_5eb1e282_23_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=1,drop_r_2024-06-25_18-57-29/progress.csv b/ray_results/test_experiment/train_model_5eb1e282_23_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=1,drop_r_2024-06-25_18-57-29/progress.csv
new file mode 100644
index 0000000..0363c5e
--- /dev/null
+++ b/ray_results/test_experiment/train_model_5eb1e282_23_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=1,drop_r_2024-06-25_18-57-29/progress.csv
@@ -0,0 +1,19 @@
+loss,timestamp,checkpoint_dir_name,should_checkpoint,done,training_iteration,trial_id,date,time_this_iter_s,time_total_s,pid,hostname,node_ip,time_since_restore,iterations_since_restore
+1030.0586158572219,1719335735,checkpoint_000000,True,False,1,5eb1e282,2024-06-25_19-15-35,60.84843850135803,60.84843850135803,2195253,r8i6n2,10.159.28.60,60.84843850135803,1
+562.0288162832185,1719335783,checkpoint_000001,True,False,2,5eb1e282,2024-06-25_19-16-23,47.70394277572632,108.55238127708435,2195253,r8i6n2,10.159.28.60,108.55238127708435,2
+444.9066558597595,1719339829,checkpoint_000002,True,False,3,5eb1e282,2024-06-25_20-23-49,61.87126088142395,170.4236421585083,2195253,r8i6n2,10.159.28.60,61.87126088142395,1
+385.7694199930026,1719339876,checkpoint_000003,True,False,4,5eb1e282,2024-06-25_20-24-36,47.405888080596924,217.82953023910522,2195253,r8i6n2,10.159.28.60,109.27714896202087,2
+351.1129479595995,1719339924,checkpoint_000004,True,False,5,5eb1e282,2024-06-25_20-25-24,47.97740936279297,265.8069396018982,2195253,r8i6n2,10.159.28.60,157.25455832481384,3
+320.4649035836768,1719339972,checkpoint_000005,True,False,6,5eb1e282,2024-06-25_20-26-12,47.432437896728516,313.2393774986267,2195253,r8i6n2,10.159.28.60,204.68699622154236,4
+293.46044717623494,1719340019,checkpoint_000006,True,False,7,5eb1e282,2024-06-25_20-26-59,47.91654348373413,361.15592098236084,2195253,r8i6n2,10.159.28.60,252.6035397052765,5
+273.19929143950696,1719340067,checkpoint_000007,True,False,8,5eb1e282,2024-06-25_20-27-47,47.474034786224365,408.6299557685852,2195253,r8i6n2,10.159.28.60,300.07757449150085,6
+256.9283664733406,1719345809,checkpoint_000008,True,False,9,5eb1e282,2024-06-25_22-03-29,61.405439615249634,470.03539538383484,2195253,r8i6n2,10.159.28.60,61.405439615249634,1
+240.56306673785832,1719345856,checkpoint_000009,True,False,10,5eb1e282,2024-06-25_22-04-16,47.146918296813965,517.1823136806488,2195253,r8i6n2,10.159.28.60,108.5523579120636,2
+230.43498638483484,1719345903,checkpoint_000010,True,False,11,5eb1e282,2024-06-25_22-05-03,47.185975790023804,564.3682894706726,2195253,r8i6n2,10.159.28.60,155.7383337020874,3
+219.4073686975194,1719345951,checkpoint_000011,True,False,12,5eb1e282,2024-06-25_22-05-51,47.13756966590881,611.5058591365814,2195253,r8i6n2,10.159.28.60,202.87590336799622,4
+211.7666195846918,1719345998,checkpoint_000012,True,False,13,5eb1e282,2024-06-25_22-06-38,47.17806816101074,658.6839272975922,2195253,r8i6n2,10.159.28.60,250.05397152900696,5
+204.5018019788847,1719346046,checkpoint_000013,True,False,14,5eb1e282,2024-06-25_22-07-26,47.774343729019165,706.4582710266113,2195253,r8i6n2,10.159.28.60,297.8283152580261,6
+198.05782912847565,1719346092,checkpoint_000014,True,False,15,5eb1e282,2024-06-25_22-08-12,46.741618633270264,753.1998896598816,2195253,r8i6n2,10.159.28.60,344.5699338912964,7
+194.08880230760948,1719346139,checkpoint_000015,True,False,16,5eb1e282,2024-06-25_22-08-59,47.16085863113403,800.3607482910156,2195253,r8i6n2,10.159.28.60,391.7307925224304,8
+196.82745781845935,1719346187,checkpoint_000016,True,False,17,5eb1e282,2024-06-25_22-09-47,47.1561017036438,847.5168499946594,2195253,r8i6n2,10.159.28.60,438.8868942260742,9
+183.96574522003414,1719346234,checkpoint_000017,True,False,18,5eb1e282,2024-06-25_22-10-34,47.40534067153931,894.9221906661987,2195253,r8i6n2,10.159.28.60,486.2922348976135,10
diff --git a/ray_results/test_experiment/train_model_5eb1e282_23_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=1,drop_r_2024-06-25_18-57-29/result.json b/ray_results/test_experiment/train_model_5eb1e282_23_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=1,drop_r_2024-06-25_18-57-29/result.json
new file mode 100644
index 0000000..db1051e
--- /dev/null
+++ b/ray_results/test_experiment/train_model_5eb1e282_23_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=1,drop_r_2024-06-25_18-57-29/result.json
@@ -0,0 +1,18 @@
+{"loss": 1030.0586158572219, "timestamp": 1719335735, "checkpoint_dir_name": "checkpoint_000000", "should_checkpoint": true, "done": false, "training_iteration": 1, "trial_id": "5eb1e282", "date": "2024-06-25_19-15-35", "time_this_iter_s": 60.84843850135803, "time_total_s": 60.84843850135803, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 1, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 1024, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 4, "drop_rate": 0.2842529108982538, "lr": 0.00015364900402465778, "batch_size": 2048}, "time_since_restore": 60.84843850135803, "iterations_since_restore": 1}
+{"loss": 562.0288162832185, "timestamp": 1719335783, "checkpoint_dir_name": "checkpoint_000001", "should_checkpoint": true, "done": false, "training_iteration": 2, "trial_id": "5eb1e282", "date": "2024-06-25_19-16-23", "time_this_iter_s": 47.70394277572632, "time_total_s": 108.55238127708435, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 1, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 1024, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 4, "drop_rate": 0.2842529108982538, "lr": 0.00015364900402465778, "batch_size": 2048}, "time_since_restore": 108.55238127708435, "iterations_since_restore": 2}
+{"loss": 444.9066558597595, "timestamp": 1719339829, "checkpoint_dir_name": "checkpoint_000002", "should_checkpoint": true, "done": false, "training_iteration": 3, "trial_id": "5eb1e282", "date": "2024-06-25_20-23-49", "time_this_iter_s": 61.87126088142395, "time_total_s": 170.4236421585083, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 1, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 1024, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 4, "drop_rate": 0.2842529108982538, "lr": 0.00015364900402465778, "batch_size": 2048}, "time_since_restore": 61.87126088142395, "iterations_since_restore": 1}
+{"loss": 385.7694199930026, "timestamp": 1719339876, "checkpoint_dir_name": "checkpoint_000003", "should_checkpoint": true, "done": false, "training_iteration": 4, "trial_id": "5eb1e282", "date": "2024-06-25_20-24-36", "time_this_iter_s": 47.405888080596924, "time_total_s": 217.82953023910522, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 1, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 1024, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 4, "drop_rate": 0.2842529108982538, "lr": 0.00015364900402465778, "batch_size": 2048}, "time_since_restore": 109.27714896202087, "iterations_since_restore": 2}
+{"loss": 351.1129479595995, "timestamp": 1719339924, "checkpoint_dir_name": "checkpoint_000004", "should_checkpoint": true, "done": false, "training_iteration": 5, "trial_id": "5eb1e282", "date": "2024-06-25_20-25-24", "time_this_iter_s": 47.97740936279297, "time_total_s": 265.8069396018982, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 1, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 1024, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 4, "drop_rate": 0.2842529108982538, "lr": 0.00015364900402465778, "batch_size": 2048}, "time_since_restore": 157.25455832481384, "iterations_since_restore": 3}
+{"loss": 320.4649035836768, "timestamp": 1719339972, "checkpoint_dir_name": "checkpoint_000005", "should_checkpoint": true, "done": false, "training_iteration": 6, "trial_id": "5eb1e282", "date": "2024-06-25_20-26-12", "time_this_iter_s": 47.432437896728516, "time_total_s": 313.2393774986267, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 1, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 1024, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 4, "drop_rate": 0.2842529108982538, "lr": 0.00015364900402465778, "batch_size": 2048}, "time_since_restore": 204.68699622154236, "iterations_since_restore": 4}
+{"loss": 293.46044717623494, "timestamp": 1719340019, "checkpoint_dir_name": "checkpoint_000006", "should_checkpoint": true, "done": false, "training_iteration": 7, "trial_id": "5eb1e282", "date": "2024-06-25_20-26-59", "time_this_iter_s": 47.91654348373413, "time_total_s": 361.15592098236084, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 1, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 1024, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 4, "drop_rate": 0.2842529108982538, "lr": 0.00015364900402465778, "batch_size": 2048}, "time_since_restore": 252.6035397052765, "iterations_since_restore": 5}
+{"loss": 273.19929143950696, "timestamp": 1719340067, "checkpoint_dir_name": "checkpoint_000007", "should_checkpoint": true, "done": false, "training_iteration": 8, "trial_id": "5eb1e282", "date": "2024-06-25_20-27-47", "time_this_iter_s": 47.474034786224365, "time_total_s": 408.6299557685852, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 1, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 1024, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 4, "drop_rate": 0.2842529108982538, "lr": 0.00015364900402465778, "batch_size": 2048}, "time_since_restore": 300.07757449150085, "iterations_since_restore": 6}
+{"loss": 256.9283664733406, "timestamp": 1719345809, "checkpoint_dir_name": "checkpoint_000008", "should_checkpoint": true, "done": false, "training_iteration": 9, "trial_id": "5eb1e282", "date": "2024-06-25_22-03-29", "time_this_iter_s": 61.405439615249634, "time_total_s": 470.03539538383484, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 1, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 1024, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 4, "drop_rate": 0.2842529108982538, "lr": 0.00015364900402465778, "batch_size": 2048}, "time_since_restore": 61.405439615249634, "iterations_since_restore": 1}
+{"loss": 240.56306673785832, "timestamp": 1719345856, "checkpoint_dir_name": "checkpoint_000009", "should_checkpoint": true, "done": false, "training_iteration": 10, "trial_id": "5eb1e282", "date": "2024-06-25_22-04-16", "time_this_iter_s": 47.146918296813965, "time_total_s": 517.1823136806488, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 1, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 1024, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 4, "drop_rate": 0.2842529108982538, "lr": 0.00015364900402465778, "batch_size": 2048}, "time_since_restore": 108.5523579120636, "iterations_since_restore": 2}
+{"loss": 230.43498638483484, "timestamp": 1719345903, "checkpoint_dir_name": "checkpoint_000010", "should_checkpoint": true, "done": false, "training_iteration": 11, "trial_id": "5eb1e282", "date": "2024-06-25_22-05-03", "time_this_iter_s": 47.185975790023804, "time_total_s": 564.3682894706726, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 1, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 1024, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 4, "drop_rate": 0.2842529108982538, "lr": 0.00015364900402465778, "batch_size": 2048}, "time_since_restore": 155.7383337020874, "iterations_since_restore": 3}
+{"loss": 219.4073686975194, "timestamp": 1719345951, "checkpoint_dir_name": "checkpoint_000011", "should_checkpoint": true, "done": false, "training_iteration": 12, "trial_id": "5eb1e282", "date": "2024-06-25_22-05-51", "time_this_iter_s": 47.13756966590881, "time_total_s": 611.5058591365814, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 1, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 1024, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 4, "drop_rate": 0.2842529108982538, "lr": 0.00015364900402465778, "batch_size": 2048}, "time_since_restore": 202.87590336799622, "iterations_since_restore": 4}
+{"loss": 211.7666195846918, "timestamp": 1719345998, "checkpoint_dir_name": "checkpoint_000012", "should_checkpoint": true, "done": false, "training_iteration": 13, "trial_id": "5eb1e282", "date": "2024-06-25_22-06-38", "time_this_iter_s": 47.17806816101074, "time_total_s": 658.6839272975922, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 1, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 1024, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 4, "drop_rate": 0.2842529108982538, "lr": 0.00015364900402465778, "batch_size": 2048}, "time_since_restore": 250.05397152900696, "iterations_since_restore": 5}
+{"loss": 204.5018019788847, "timestamp": 1719346046, "checkpoint_dir_name": "checkpoint_000013", "should_checkpoint": true, "done": false, "training_iteration": 14, "trial_id": "5eb1e282", "date": "2024-06-25_22-07-26", "time_this_iter_s": 47.774343729019165, "time_total_s": 706.4582710266113, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 1, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 1024, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 4, "drop_rate": 0.2842529108982538, "lr": 0.00015364900402465778, "batch_size": 2048}, "time_since_restore": 297.8283152580261, "iterations_since_restore": 6}
+{"loss": 198.05782912847565, "timestamp": 1719346092, "checkpoint_dir_name": "checkpoint_000014", "should_checkpoint": true, "done": false, "training_iteration": 15, "trial_id": "5eb1e282", "date": "2024-06-25_22-08-12", "time_this_iter_s": 46.741618633270264, "time_total_s": 753.1998896598816, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 1, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 1024, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 4, "drop_rate": 0.2842529108982538, "lr": 0.00015364900402465778, "batch_size": 2048}, "time_since_restore": 344.5699338912964, "iterations_since_restore": 7}
+{"loss": 194.08880230760948, "timestamp": 1719346139, "checkpoint_dir_name": "checkpoint_000015", "should_checkpoint": true, "done": false, "training_iteration": 16, "trial_id": "5eb1e282", "date": "2024-06-25_22-08-59", "time_this_iter_s": 47.16085863113403, "time_total_s": 800.3607482910156, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 1, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 1024, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 4, "drop_rate": 0.2842529108982538, "lr": 0.00015364900402465778, "batch_size": 2048}, "time_since_restore": 391.7307925224304, "iterations_since_restore": 8}
+{"loss": 196.82745781845935, "timestamp": 1719346187, "checkpoint_dir_name": "checkpoint_000016", "should_checkpoint": true, "done": false, "training_iteration": 17, "trial_id": "5eb1e282", "date": "2024-06-25_22-09-47", "time_this_iter_s": 47.1561017036438, "time_total_s": 847.5168499946594, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 1, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 1024, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 4, "drop_rate": 0.2842529108982538, "lr": 0.00015364900402465778, "batch_size": 2048}, "time_since_restore": 438.8868942260742, "iterations_since_restore": 9}
+{"loss": 183.96574522003414, "timestamp": 1719346234, "checkpoint_dir_name": "checkpoint_000017", "should_checkpoint": true, "done": false, "training_iteration": 18, "trial_id": "5eb1e282", "date": "2024-06-25_22-10-34", "time_this_iter_s": 47.40534067153931, "time_total_s": 894.9221906661987, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 1, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 1024, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 4, "drop_rate": 0.2842529108982538, "lr": 0.00015364900402465778, "batch_size": 2048}, "time_since_restore": 486.2922348976135, "iterations_since_restore": 10}
diff --git a/ray_results/test_experiment/train_model_651349c4_12_batch_size=2048,decoder_int_ff=2048,decoder_int_num_layer=8,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop_2024-06-25_16-23-08/events.out.tfevents.1719326403.r3i5n6 b/ray_results/test_experiment/train_model_651349c4_12_batch_size=2048,decoder_int_ff=2048,decoder_int_num_layer=8,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop_2024-06-25_16-23-08/events.out.tfevents.1719326403.r3i5n6
new file mode 100644
index 0000000000000000000000000000000000000000..0cf7bb0efd26ac5cf2723e720f5a8dda9f21725f
GIT binary patch
literal 405
zcmb1OfPlsI-b$RdItMmvop;kwiZ`h!F*8rkwJbHS#L8%=&bs5uP<0#TmpgO*{_lPa
zq)vcQON>i|i?=ATQop1$FI7J$zqnZR_~!2}a_<6P8$h*8{dysu^BAiKOp7v?B9|0g
zOL0biX--Ofaz<)$c0qn-UWq6JLxX*DpjZcrl_?x>{is%^<maUVwJ<Q0SQJ?6L#@n>
zl?~whkI<sTrNAW#x3VNNH#NQ_BeOU@vm~`BzF1VmW{uO=pWo*iLbcCrZhpzx$_cko
zg-eM`8dZC7W?ph?d{Js~Nq$i(*nnQ{_utK-20VXpDw30#3vPfGmj;(2+yIcRi6xo&
QdByRl#(;co@JhrT02&a4L;wH)
literal 0
HcmV?d00001
diff --git a/ray_results/test_experiment/train_model_651349c4_12_batch_size=2048,decoder_int_ff=2048,decoder_int_num_layer=8,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop_2024-06-25_16-23-08/params.json b/ray_results/test_experiment/train_model_651349c4_12_batch_size=2048,decoder_int_ff=2048,decoder_int_num_layer=8,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop_2024-06-25_16-23-08/params.json
new file mode 100644
index 0000000..7745c6e
--- /dev/null
+++ b/ray_results/test_experiment/train_model_651349c4_12_batch_size=2048,decoder_int_ff=2048,decoder_int_num_layer=8,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop_2024-06-25_16-23-08/params.json
@@ -0,0 +1,13 @@
+{
+ "batch_size": 2048,
+ "decoder_int_ff": 2048,
+ "decoder_int_num_layer": 8,
+ "decoder_rt_ff": 2048,
+ "decoder_rt_num_layer": 1,
+ "drop_rate": 0.6683907000004579,
+ "embedding_dim": 16,
+ "encoder_ff": 2048,
+ "encoder_num_layer": 8,
+ "lr": 0.0001175178193869245,
+ "n_head": 4
+}
\ No newline at end of file
diff --git a/ray_results/test_experiment/train_model_651349c4_12_batch_size=2048,decoder_int_ff=2048,decoder_int_num_layer=8,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop_2024-06-25_16-23-08/params.pkl b/ray_results/test_experiment/train_model_651349c4_12_batch_size=2048,decoder_int_ff=2048,decoder_int_num_layer=8,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop_2024-06-25_16-23-08/params.pkl
new file mode 100644
index 0000000000000000000000000000000000000000..74a714b2d04cf1b94b7f8ea4dc828e85b345fe6f
GIT binary patch
literal 224
zcmZo*oqCl40&1sd^a!TrCFiH47RBe4=EmnFR;CtB@#g3eNlAr>7L_1NF!qSTBr@|*
zWH@?wQ*)D2Q&KYX(&JMybEkL<^l-t9PD`8O%K%gX(+e~dD#Qm90vQJtW6O)rNKH(c
z;?2^-nNpNr5MPv7k~+oR{%LZ_lXX@9dYE#GfIPW#HHUk|bb7dw5=)XZ;)^q@QbCrK
G>Hz@Yu2Z)F
literal 0
HcmV?d00001
diff --git a/ray_results/test_experiment/train_model_651349c4_12_batch_size=2048,decoder_int_ff=2048,decoder_int_num_layer=8,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop_2024-06-25_16-23-08/progress.csv b/ray_results/test_experiment/train_model_651349c4_12_batch_size=2048,decoder_int_ff=2048,decoder_int_num_layer=8,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop_2024-06-25_16-23-08/progress.csv
new file mode 100644
index 0000000..6bff8a1
--- /dev/null
+++ b/ray_results/test_experiment/train_model_651349c4_12_batch_size=2048,decoder_int_ff=2048,decoder_int_num_layer=8,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop_2024-06-25_16-23-08/progress.csv
@@ -0,0 +1,2 @@
+loss,timestamp,checkpoint_dir_name,should_checkpoint,done,training_iteration,trial_id,date,time_this_iter_s,time_total_s,pid,hostname,node_ip,time_since_restore,iterations_since_restore
+1981.618032380352,1719326489,checkpoint_000000,True,False,1,651349c4,2024-06-25_16-41-29,86.11733651161194,86.11733651161194,69318,r3i5n6,10.159.8.159,86.11733651161194,1
diff --git a/ray_results/test_experiment/train_model_651349c4_12_batch_size=2048,decoder_int_ff=2048,decoder_int_num_layer=8,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop_2024-06-25_16-23-08/result.json b/ray_results/test_experiment/train_model_651349c4_12_batch_size=2048,decoder_int_ff=2048,decoder_int_num_layer=8,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop_2024-06-25_16-23-08/result.json
new file mode 100644
index 0000000..717abb4
--- /dev/null
+++ b/ray_results/test_experiment/train_model_651349c4_12_batch_size=2048,decoder_int_ff=2048,decoder_int_num_layer=8,decoder_rt_ff=2048,decoder_rt_num_layer=1,drop_2024-06-25_16-23-08/result.json
@@ -0,0 +1 @@
+{"loss": 1981.618032380352, "timestamp": 1719326489, "checkpoint_dir_name": "checkpoint_000000", "should_checkpoint": true, "done": false, "training_iteration": 1, "trial_id": "651349c4", "date": "2024-06-25_16-41-29", "time_this_iter_s": 86.11733651161194, "time_total_s": 86.11733651161194, "pid": 69318, "hostname": "r3i5n6", "node_ip": "10.159.8.159", "config": {"encoder_num_layer": 8, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 8, "embedding_dim": 16, "encoder_ff": 2048, "decoder_rt_ff": 2048, "decoder_int_ff": 2048, "n_head": 4, "drop_rate": 0.6683907000004579, "lr": 0.0001175178193869245, "batch_size": 2048}, "time_since_restore": 86.11733651161194, "iterations_since_restore": 1}
diff --git a/ray_results/test_experiment/train_model_6743cdee_21_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=4,drop__2024-06-25_18-48-59/events.out.tfevents.1719334380.r8i6n2 b/ray_results/test_experiment/train_model_6743cdee_21_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=4,drop__2024-06-25_18-48-59/events.out.tfevents.1719334380.r8i6n2
new file mode 100644
index 0000000000000000000000000000000000000000..1561fef9a4f2ae211498717f45c884da72657154
GIT binary patch
literal 1676
zcma)6T}TvB6kcaTn-i<MlWV!At+Qn{Sep?AQtYG&kp-a`^bphR&b6I#?<_N;rBMl1
z1ohBMMUYx4Jq1Y#Jwz70^b1AQLs1k<SwVjUsr3-ub!YGHm}c`f=bZ0+_kQO)cdjG#
z^&PC|*{khK_TA`tzRu?g>#~ZM)wLmPR<Ag{RjdkFbpyS7YT0A2+Io_8?i`^A3c;nz
z?Lo5@qhO0>7+xI0bx%*<>0fR^PX2f-u?-0d2|zjcD8z_rtu2ZaiK58vxTa#$>u_|2
zSQLM4qdfbT;_Z}*h7o}{9EHA|dqImbw!L%<yXqZ<@Zg7HN@=PwB$-jwkW>@tlHu)L
z>8kDPa1T~l@b0dzkL+Luz5sYAvBM1&N06i=!_;&%-r)F1g?F{p;QfdD+t{jM$}hlT
zD5njQtg@+U*pTdPk~d$pIc(T&KC4C?8W)?HOl;4gMRj>M2RGZqZmOk_{}Frr_lK=E
z$~PJO6k@wZPjNP}W9up9G-A6q{I1!5V9ae3`%4Bst=Nr~M{1iMMe}W9D|M9rzr}Wl
zPlnw?139Uoo#;NZi|q)EHr^u67W$oV-U)?LG%o9M%#ea;Zyf2wIN02H)Cq^3kY}a1
zpA0v`@c95?*^ETVNIQ_<$#amepopdrgAr3p&CMiM4BqS*ZjDJTay!yV$^))+O1G-#
zLSa@YyVwP+qHA%9xF|_QuqYNrilXA3lA^{)h@1xD6#F={7d+|z>_)l3v?SR9a<LRe
zvf_6N9w^}K8!eQ<JnkPGI{}j1NTC!;xT!Zzm=C4gzcxx(2mx;Tjalno`T|??yZ!Y-
zHJeD!FTucT2IghvXCT8MD<>JIOaxYOv$k=Ff$L0Wj+YtVB?gMJwB=Y&<;D8Xg7Pv1
zv(plhV9GFl5quM-Msw#xD4Rrbns<m$GSMJ3*(s72a?;t;61NBi$=Q)WMV#OCYsZyP
z@-Q|V(BilVbF%$F6NHE)GPb1&a(sRYuhN=tp$CcU>(*E=-rZ|`4eVH2{VE88?{Do9
VR#M6nk7ab9!80>+J|&*b_yZmv2QmNv
literal 0
HcmV?d00001
diff --git a/ray_results/test_experiment/train_model_6743cdee_21_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=4,drop__2024-06-25_18-48-59/params.json b/ray_results/test_experiment/train_model_6743cdee_21_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=4,drop__2024-06-25_18-48-59/params.json
new file mode 100644
index 0000000..75a5d4a
--- /dev/null
+++ b/ray_results/test_experiment/train_model_6743cdee_21_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=4,drop__2024-06-25_18-48-59/params.json
@@ -0,0 +1,13 @@
+{
+ "batch_size": 2048,
+ "decoder_int_ff": 512,
+ "decoder_int_num_layer": 1,
+ "decoder_rt_ff": 2048,
+ "decoder_rt_num_layer": 4,
+ "drop_rate": 0.4436137502039823,
+ "embedding_dim": 256,
+ "encoder_ff": 512,
+ "encoder_num_layer": 2,
+ "lr": 0.07179226263764402,
+ "n_head": 8
+}
\ No newline at end of file
diff --git a/ray_results/test_experiment/train_model_6743cdee_21_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=4,drop__2024-06-25_18-48-59/params.pkl b/ray_results/test_experiment/train_model_6743cdee_21_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=4,drop__2024-06-25_18-48-59/params.pkl
new file mode 100644
index 0000000000000000000000000000000000000000..a1a7d11f1f3c5afd5ac4eb6ac8174ddc4a8fdef4
GIT binary patch
literal 225
zcmZo*oqCM{0&1sd^a!TrCFiH47RBe4=EmnFR;CtB@n-4~NlAr>7L_1Nu=I$+Br@|*
zWEgvRQ*)D2Q&KYX(&JMybEo(+F!pf4%uY)K@|k*gVVZ%af`vGG_+UaH^PtMu^5Qd6
z6H}&mbM$bg6y+Dh7bTXYPI0%tlcM$3dznrTQ%(_(w<+P5k5YkJ4_8uRNpePfab{I2
J&=QVPJpj%hQeFT6
literal 0
HcmV?d00001
diff --git a/ray_results/test_experiment/train_model_6743cdee_21_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=4,drop__2024-06-25_18-48-59/progress.csv b/ray_results/test_experiment/train_model_6743cdee_21_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=4,drop__2024-06-25_18-48-59/progress.csv
new file mode 100644
index 0000000..31ceed3
--- /dev/null
+++ b/ray_results/test_experiment/train_model_6743cdee_21_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=4,drop__2024-06-25_18-48-59/progress.csv
@@ -0,0 +1,3 @@
+loss,timestamp,checkpoint_dir_name,should_checkpoint,done,training_iteration,trial_id,date,time_this_iter_s,time_total_s,pid,hostname,node_ip,time_since_restore,iterations_since_restore
+3811.964426596334,1719334521,checkpoint_000000,True,False,1,6743cdee,2024-06-25_18-55-21,141.20554637908936,141.20554637908936,2195253,r8i6n2,10.159.28.60,141.20554637908936,1
+2680.254179226132,1719334648,checkpoint_000001,True,False,2,6743cdee,2024-06-25_18-57-29,127.1407482624054,268.34629464149475,2195253,r8i6n2,10.159.28.60,268.34629464149475,2
diff --git a/ray_results/test_experiment/train_model_6743cdee_21_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=4,drop__2024-06-25_18-48-59/result.json b/ray_results/test_experiment/train_model_6743cdee_21_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=4,drop__2024-06-25_18-48-59/result.json
new file mode 100644
index 0000000..c0df350
--- /dev/null
+++ b/ray_results/test_experiment/train_model_6743cdee_21_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=4,drop__2024-06-25_18-48-59/result.json
@@ -0,0 +1,2 @@
+{"loss": 3811.964426596334, "timestamp": 1719334521, "checkpoint_dir_name": "checkpoint_000000", "should_checkpoint": true, "done": false, "training_iteration": 1, "trial_id": "6743cdee", "date": "2024-06-25_18-55-21", "time_this_iter_s": 141.20554637908936, "time_total_s": 141.20554637908936, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 2, "decoder_rt_num_layer": 4, "decoder_int_num_layer": 1, "embedding_dim": 256, "encoder_ff": 512, "decoder_rt_ff": 2048, "decoder_int_ff": 512, "n_head": 8, "drop_rate": 0.4436137502039823, "lr": 0.07179226263764402, "batch_size": 2048}, "time_since_restore": 141.20554637908936, "iterations_since_restore": 1}
+{"loss": 2680.254179226132, "timestamp": 1719334648, "checkpoint_dir_name": "checkpoint_000001", "should_checkpoint": true, "done": false, "training_iteration": 2, "trial_id": "6743cdee", "date": "2024-06-25_18-57-29", "time_this_iter_s": 127.1407482624054, "time_total_s": 268.34629464149475, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 2, "decoder_rt_num_layer": 4, "decoder_int_num_layer": 1, "embedding_dim": 256, "encoder_ff": 512, "decoder_rt_ff": 2048, "decoder_int_ff": 512, "n_head": 8, "drop_rate": 0.4436137502039823, "lr": 0.07179226263764402, "batch_size": 2048}, "time_since_restore": 268.34629464149475, "iterations_since_restore": 2}
diff --git a/ray_results/test_experiment/train_model_6a62b85a_3_batch_size=2048,decoder_int_ff=1024,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop__2024-06-25_16-09-34/events.out.tfevents.1719324663.r3i5n6 b/ray_results/test_experiment/train_model_6a62b85a_3_batch_size=2048,decoder_int_ff=1024,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop__2024-06-25_16-09-34/events.out.tfevents.1719324663.r3i5n6
new file mode 100644
index 0000000000000000000000000000000000000000..7c047983b9f9067296085771161c520497f33864
GIT binary patch
literal 405
zcmb1OfPlsI-b$PYumAnKdEQM&Dc+=_#LPTB*Rs^S5-X#F@_$&Bq3SlwFL&nbJ|e`o
z1*lGdQA><Vgp0Q*u~NUJG%r;@C%?E@bhg=l=liYexDB9MrhdH;&w1x0T#GW7B9|0g
zOL0biX--Ofaz<)$c0qn-UWq6JLxX+cx*6|LtPDH@*N<vtN`77{PzwXYEY8OH`cNx#
zV`T$4KOnRyaVc;~!mTXH%uS6i$;d2@&n!tTiZ2#rUE1KJyFMk=5UPD{bMs5i^0ROo
zRk)P6q*1jOXXYiR#uud)m*f|vf(^LGll93QYQXasry@B&Aq>#s(%@2r8vwF3u_QA;
QuQ(pn7?97e9*>O%02gk9-v9sr
literal 0
HcmV?d00001
diff --git a/ray_results/test_experiment/train_model_6a62b85a_3_batch_size=2048,decoder_int_ff=1024,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop__2024-06-25_16-09-34/events.out.tfevents.1719327182.r3i5n6 b/ray_results/test_experiment/train_model_6a62b85a_3_batch_size=2048,decoder_int_ff=1024,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop__2024-06-25_16-09-34/events.out.tfevents.1719327182.r3i5n6
new file mode 100644
index 0000000000000000000000000000000000000000..8c1b7f32f44d3b52e1917d5eda8319a27b1190a7
GIT binary patch
literal 405
zcmb1OfPlsI-b$QV^QRYYn|ISuiZ`h!F*8rkwJbHS#L6fq#OsDKRNaR8<<6X29aEY?
z>I9gy#JEJbc#9G%^-D_gQuTB4i;G3o6K6ZK=Y)PUfNGig^+G(Sq&r-TGM6Hk6kJPj
zMt*5dN_=ugYI1f#er8^YC<8--{kw;ut5K|+=Ly%3YGq1(UMf%v1H-|~`C|G|D|2II
z12~Pm;98Wp6u2beR+ePurpA|KWERI~mZTQN7mJ1*tZ|BSy%=N&)jqem`6cI8gmx7!
zB`#@H?ZugS$*J*0sl_GvMX6u|a^o}@&7lT7e{m|3Q`Z}AzZRDUmm=H%kgbU&nfZCe
O@u<dtd@ge_E)4)u{($WO
literal 0
HcmV?d00001
diff --git a/ray_results/test_experiment/train_model_6a62b85a_3_batch_size=2048,decoder_int_ff=1024,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop__2024-06-25_16-09-34/params.json b/ray_results/test_experiment/train_model_6a62b85a_3_batch_size=2048,decoder_int_ff=1024,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop__2024-06-25_16-09-34/params.json
new file mode 100644
index 0000000..8c61cf1
--- /dev/null
+++ b/ray_results/test_experiment/train_model_6a62b85a_3_batch_size=2048,decoder_int_ff=1024,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop__2024-06-25_16-09-34/params.json
@@ -0,0 +1,13 @@
+{
+ "batch_size": 2048,
+ "decoder_int_ff": 1024,
+ "decoder_int_num_layer": 1,
+ "decoder_rt_ff": 1024,
+ "decoder_rt_num_layer": 2,
+ "drop_rate": 0.06860896590384413,
+ "embedding_dim": 16,
+ "encoder_ff": 512,
+ "encoder_num_layer": 1,
+ "lr": 0.00016561766448063182,
+ "n_head": 4
+}
\ No newline at end of file
diff --git a/ray_results/test_experiment/train_model_6a62b85a_3_batch_size=2048,decoder_int_ff=1024,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop__2024-06-25_16-09-34/params.pkl b/ray_results/test_experiment/train_model_6a62b85a_3_batch_size=2048,decoder_int_ff=1024,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop__2024-06-25_16-09-34/params.pkl
new file mode 100644
index 0000000000000000000000000000000000000000..56b64ee286f614698ef2640771033fd2c68c1269
GIT binary patch
literal 224
zcmZo*oqCl40&1sd^a!TrCFiH47RBe4=EmnFR;CtB@n-B1NlAr>7L_1NF!hMSBr@|*
zWEgvRQ*)D2Q&KYX(&JMybEkL<^l-t9PD`8O%fQsb3)2fU6fDHj!v_-r83z?(%ZtxQ
zO-z~M&C<h}Qj}j1UzAvqI>p_7<AmsfBj0xPFy#~hd8%7Yp6`@%=;2CAEJ@CYFV3t=
K1zN&Uss{kcrc-AC
literal 0
HcmV?d00001
diff --git a/ray_results/test_experiment/train_model_6a62b85a_3_batch_size=2048,decoder_int_ff=1024,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop__2024-06-25_16-09-34/progress.csv b/ray_results/test_experiment/train_model_6a62b85a_3_batch_size=2048,decoder_int_ff=1024,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop__2024-06-25_16-09-34/progress.csv
new file mode 100644
index 0000000..f1fef35
--- /dev/null
+++ b/ray_results/test_experiment/train_model_6a62b85a_3_batch_size=2048,decoder_int_ff=1024,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop__2024-06-25_16-09-34/progress.csv
@@ -0,0 +1,3 @@
+loss,timestamp,checkpoint_dir_name,should_checkpoint,done,training_iteration,trial_id,date,time_this_iter_s,time_total_s,pid,hostname,node_ip,time_since_restore,iterations_since_restore
+510.42659056656004,1719324728,checkpoint_000000,True,False,1,6a62b85a,2024-06-25_16-12-08,64.32230615615845,64.32230615615845,69318,r3i5n6,10.159.8.159,64.32230615615845,1
+310.75900604968933,1719327245,checkpoint_000001,True,False,2,6a62b85a,2024-06-25_16-54-05,63.18879699707031,127.51110315322876,130020,r3i5n6,10.159.8.159,63.18879699707031,1
diff --git a/ray_results/test_experiment/train_model_6a62b85a_3_batch_size=2048,decoder_int_ff=1024,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop__2024-06-25_16-09-34/result.json b/ray_results/test_experiment/train_model_6a62b85a_3_batch_size=2048,decoder_int_ff=1024,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop__2024-06-25_16-09-34/result.json
new file mode 100644
index 0000000..c222fa4
--- /dev/null
+++ b/ray_results/test_experiment/train_model_6a62b85a_3_batch_size=2048,decoder_int_ff=1024,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop__2024-06-25_16-09-34/result.json
@@ -0,0 +1,2 @@
+{"loss": 510.42659056656004, "timestamp": 1719324728, "checkpoint_dir_name": "checkpoint_000000", "should_checkpoint": true, "done": false, "training_iteration": 1, "trial_id": "6a62b85a", "date": "2024-06-25_16-12-08", "time_this_iter_s": 64.32230615615845, "time_total_s": 64.32230615615845, "pid": 69318, "hostname": "r3i5n6", "node_ip": "10.159.8.159", "config": {"encoder_num_layer": 1, "decoder_rt_num_layer": 2, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 512, "decoder_rt_ff": 1024, "decoder_int_ff": 1024, "n_head": 4, "drop_rate": 0.06860896590384413, "lr": 0.00016561766448063182, "batch_size": 2048}, "time_since_restore": 64.32230615615845, "iterations_since_restore": 1}
+{"loss": 310.75900604968933, "timestamp": 1719327245, "checkpoint_dir_name": "checkpoint_000001", "should_checkpoint": true, "done": false, "training_iteration": 2, "trial_id": "6a62b85a", "date": "2024-06-25_16-54-05", "time_this_iter_s": 63.18879699707031, "time_total_s": 127.51110315322876, "pid": 130020, "hostname": "r3i5n6", "node_ip": "10.159.8.159", "config": {"encoder_num_layer": 1, "decoder_rt_num_layer": 2, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 512, "decoder_rt_ff": 1024, "decoder_int_ff": 1024, "n_head": 4, "drop_rate": 0.06860896590384413, "lr": 0.00016561766448063182, "batch_size": 2048}, "time_since_restore": 63.18879699707031, "iterations_since_restore": 1}
diff --git a/ray_results/test_experiment/train_model_6be79717_24_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop__2024-06-25_19-14-34/events.out.tfevents.1719335783.r8i6n2 b/ray_results/test_experiment/train_model_6be79717_24_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop__2024-06-25_19-14-34/events.out.tfevents.1719335783.r8i6n2
new file mode 100644
index 0000000000000000000000000000000000000000..c4aacef1d7de666f23944a20c1ba73cc59c14404
GIT binary patch
literal 1676
zcma)6TSydP6dq?oodc^mxn9;wx)#JfxG97n)~=T_%tGoxBw`wOe%;C4nPp})4I;~g
z9wMZW@If=R%R{7ZC52d^Wf2M0gHM4EUNT68q$Hw$cW3|IF>muW=bZ0+|Nouu{ApC^
z8@zJD#`Ha%n;GkyDYcsG6<(C`kG3F5D=z#}+B%k}*Nxn}SkBm-6H9@*-7)qQNCvaQ
zw>h+C2|2v7s#@N3|EX9UY3e_yL%M%XbIkJ!0?C71uo8&sk(<2&=k_3XlV27k%|cP_
z&g5EDXP`V>N$^HWL6#7PP}J2a>sg0R={uKI!+eTBY>)%#gi;fIh|@fx%843MIMp(>
za;vPq{X=Pi4)5*gSY)b?6UKdDgA5~F6(u*~6r^ghf`ScJkDM|W=?&%=-qkS^kp^}+
z0J)?=fR)!oSyDM;o50N{i%woM-2DBZWi2Q+y^+|Ly=xVz9hKz<vA+)y$WFxmmi2PL
zK-v6&;5QN5I!O&1#QqzBY$NtGg~~oYIxiT+)*ceZZN=^xrOM-1ZcZA+UWzo>vDlRB
z@knj;NK9yGUA;XQ7{{?+XU4I!?L#KGZ-Qjb<L4FLr*aO|>PHGTj-*xHFhQpY;`9{z
zn`T#OTLM5muem*Vq?b^R$;Lv0fZVcx4Teq4H8kK@I#{A(QnQcq@@+`LsU$FmQ;J@l
z2FX#O=wdT4f+G7l?4ke_#v)%m5(H6d<OI=&L-;gE3b6;7Eszxck8TtTv^T&G5G!#W
z#0xnldlDqF#*MaTLngb$#;$_|HqxF28SLg8XWt81>{c6P-w%20_8YShf44-^=lA-@
zKr!PD&u^B7uQX&v=BFZ6#Vg0D8jl4Qr3SfbjD{C<WRAnM^$86rQQERh((U}txx&x$
zPS_Kc2nR!k$#H>o-PA~~%LUmRNKW$0PRLkq5Rz<n;TLkl*^?5=1&M*#;Q-q>{%Z72
zt21yIgAGV=jSEtv{Xi1^!9>KiB!Q1_+r+D+X4$#ynLSjZzj$w-4W4GMht)3t2U!2t
W9!Ww-dF-)>?z?Psd`wzgWy?SGJQD5z
literal 0
HcmV?d00001
diff --git a/ray_results/test_experiment/train_model_6be79717_24_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop__2024-06-25_19-14-34/params.json b/ray_results/test_experiment/train_model_6be79717_24_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop__2024-06-25_19-14-34/params.json
new file mode 100644
index 0000000..a573c35
--- /dev/null
+++ b/ray_results/test_experiment/train_model_6be79717_24_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop__2024-06-25_19-14-34/params.json
@@ -0,0 +1,13 @@
+{
+ "batch_size": 1024,
+ "decoder_int_ff": 512,
+ "decoder_int_num_layer": 1,
+ "decoder_rt_ff": 1024,
+ "decoder_rt_num_layer": 2,
+ "drop_rate": 0.7446531774853204,
+ "embedding_dim": 64,
+ "encoder_ff": 2048,
+ "encoder_num_layer": 1,
+ "lr": 0.006000257122673396,
+ "n_head": 8
+}
\ No newline at end of file
diff --git a/ray_results/test_experiment/train_model_6be79717_24_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop__2024-06-25_19-14-34/params.pkl b/ray_results/test_experiment/train_model_6be79717_24_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop__2024-06-25_19-14-34/params.pkl
new file mode 100644
index 0000000000000000000000000000000000000000..aad1abecc4a67ea997619c3837e518a0aae66cd4
GIT binary patch
literal 224
zcmZo*oqCl40&1sd^a!TrCFiH47RBe4=EmnFR;CtB@n-B1NlAr>7L_1NF!hMSBr@|*
zWEgvRQ*)D2Q&KYX(&JMybEkMa^l-t9PD`8O%fQjY3)2fU6fDHj!v_-r83z_)>S4=^
z&qz&7nc~gS!<kZ)Ul3oESdu!$-TwI%qi3yplX{qPih#U|$vY1oNEhniN=htA&WJD0
MtV#u1!cwXS0O&ze=Kufz
literal 0
HcmV?d00001
diff --git a/ray_results/test_experiment/train_model_6be79717_24_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop__2024-06-25_19-14-34/progress.csv b/ray_results/test_experiment/train_model_6be79717_24_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop__2024-06-25_19-14-34/progress.csv
new file mode 100644
index 0000000..555eff0
--- /dev/null
+++ b/ray_results/test_experiment/train_model_6be79717_24_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop__2024-06-25_19-14-34/progress.csv
@@ -0,0 +1,3 @@
+loss,timestamp,checkpoint_dir_name,should_checkpoint,done,training_iteration,trial_id,date,time_this_iter_s,time_total_s,pid,hostname,node_ip,time_since_restore,iterations_since_restore
+2020.6805025836613,1719335855,checkpoint_000000,True,False,1,6be79717,2024-06-25_19-17-35,71.47419285774231,71.47419285774231,2195253,r8i6n2,10.159.28.60,71.47419285774231,1
+2003.3902054433747,1719335912,checkpoint_000001,True,False,2,6be79717,2024-06-25_19-18-32,57.25079274177551,128.72498559951782,2195253,r8i6n2,10.159.28.60,128.72498559951782,2
diff --git a/ray_results/test_experiment/train_model_6be79717_24_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop__2024-06-25_19-14-34/result.json b/ray_results/test_experiment/train_model_6be79717_24_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop__2024-06-25_19-14-34/result.json
new file mode 100644
index 0000000..a6ff692
--- /dev/null
+++ b/ray_results/test_experiment/train_model_6be79717_24_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop__2024-06-25_19-14-34/result.json
@@ -0,0 +1,2 @@
+{"loss": 2020.6805025836613, "timestamp": 1719335855, "checkpoint_dir_name": "checkpoint_000000", "should_checkpoint": true, "done": false, "training_iteration": 1, "trial_id": "6be79717", "date": "2024-06-25_19-17-35", "time_this_iter_s": 71.47419285774231, "time_total_s": 71.47419285774231, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 1, "decoder_rt_num_layer": 2, "decoder_int_num_layer": 1, "embedding_dim": 64, "encoder_ff": 2048, "decoder_rt_ff": 1024, "decoder_int_ff": 512, "n_head": 8, "drop_rate": 0.7446531774853204, "lr": 0.006000257122673396, "batch_size": 1024}, "time_since_restore": 71.47419285774231, "iterations_since_restore": 1}
+{"loss": 2003.3902054433747, "timestamp": 1719335912, "checkpoint_dir_name": "checkpoint_000001", "should_checkpoint": true, "done": false, "training_iteration": 2, "trial_id": "6be79717", "date": "2024-06-25_19-18-32", "time_this_iter_s": 57.25079274177551, "time_total_s": 128.72498559951782, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 1, "decoder_rt_num_layer": 2, "decoder_int_num_layer": 1, "embedding_dim": 64, "encoder_ff": 2048, "decoder_rt_ff": 1024, "decoder_int_ff": 512, "n_head": 8, "drop_rate": 0.7446531774853204, "lr": 0.006000257122673396, "batch_size": 1024}, "time_since_restore": 128.72498559951782, "iterations_since_restore": 2}
diff --git a/ray_results/test_experiment/train_model_6c8b476a_20_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=4,drop_r_2024-06-25_18-43-49/events.out.tfevents.1719334139.r8i6n2 b/ray_results/test_experiment/train_model_6c8b476a_20_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=4,drop_r_2024-06-25_18-43-49/events.out.tfevents.1719334139.r8i6n2
new file mode 100644
index 0000000000000000000000000000000000000000..e229310bfa517cb8926df9f2af4925df493548a9
GIT binary patch
literal 770
zcmb1OfPlsI-b$Q%WFFY<op;kwiZ`h!F*8rkwJbHS#LDP^g{h-5RNaR8<<6X?g$Lt6
z>I4|I#JEJbc#9G%^-D_gQuTB4i;G3C-u&evnqM&40IFr`*9-BSPs-q0l(`hSq~KbL
zGxAGwQsR>{Qj@a_@-y>FL>U+w?Dxk@%s{cyq!O+l)ykCoyi}kT1_s9oa<TeQD|2II
z132Fyv?y^Ya7n_gEXmAGjW5Z_ERN4CNiB*m7Cpo9-)W7HPr4yg``qT{mz+6Oa2r*)
zl(?i(wHIgRC8x$0r52au7o~y?m~^to(j02Q^B1QgIUgYm(BjhIQiK}-vNf?JGe565
z9@Q9-&qV}eU!(Z^z!mvwa9}fG3+yN9KV9y9G0s5=><3rjT8Ihk#<s7mC{`w4gX_l;
z*w?%}a!~^NIYJA8z~=t&$SFr_x-3dyCtQcyNHDOo{}eb!{+;fI64)0I1`r+C3=Zcf
H={*1dU{C)#
literal 0
HcmV?d00001
diff --git a/ray_results/test_experiment/train_model_6c8b476a_20_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=4,drop_r_2024-06-25_18-43-49/events.out.tfevents.1719338498.r8i6n2 b/ray_results/test_experiment/train_model_6c8b476a_20_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=4,drop_r_2024-06-25_18-43-49/events.out.tfevents.1719338498.r8i6n2
new file mode 100644
index 0000000000000000000000000000000000000000..b9c37540f6834b97b3aa22e56e87c3716256be97
GIT binary patch
literal 1676
zcma)6TSydP6kcb=oMWr*<ZilN(zP4=(5eK12)n6C>5KH{gAKDgzwOxWjLeLw5u}(R
zMQC6bAFRwCG@>Nz0*a6?MNmjkB2tM277_`OBxLmO?(Dxi=562ReCIpg%y+)C>4xxc
z@IsZHo|_(OeQ|5%sLkY&IYB&h>?{)1;-aLTZ$IX1eIvJ<%4oj3=1#CL)#$K78kl6R
z#i=%m$my39#WLNsQhxQ~iSGp(<=U?)mj2UADEW{FHbPN+Qlp<|y*}hU(;x|=YB3mE
zU5jrfhINwlHwa%Z$x9-_6oX;;aML@dCK;&BuBPdmgkpzW$RLuc5J0Tz6BJfZk<2QV
z#kSSbBDpQSP^0&^w|}EAMd^FN4w-toB8XnZ%1BWq8HEN^E2#5`HeliF>l!+~kLWw#
zAmot&K~+u_BvE1YV}f698G{zxmp@D#9uJuvzn<ByrQhXEPcN72%$}kNWjkgsRPpz9
zlBa<1H!#~Z-Ye?N{t~5ZVfH}teCeRI)T%SvlSCA^G`nI|Dy#5y`*mi|Mh9%$Y=e8G
z`SJddxbV_?`upnX(f0>tCULSI_lz)Lgf!OIz{y-dVV$VC0m(QxqFUKygbpL@&>&`o
zVwNd;GJuCuy*|9sb12tnXCRqJUWvy6!=YyD>ToRsEU`7QF~It{79?ZH45kPqYu)LP
z7E_9CHUZ7cQUi-q6qLeT6!0LP7sS&nF9dK2zYP+@>LFzdm?O`zgA#!92h{-*MAnBm
zKG*0lLkgq6Xh#lYF`HcM8cOgY9odk{Y<zK!J&?_8c2SOfkk4#=F~h!2i>U2y^2gm`
z`e$T+^AvohAS=2*1t|(XIaX0QY_K5KNtF{6j8V}&KB8<7D6qzO%g|=)^PySp_?!!N
zMJ&R_a9|4DU|X{_g0;FKXC2E4p6iCpwE-a@Cg-icd(x8XhLqs$aH00)^^(ae9j@TZ
z=wd*ctKG0WHV*{o2?5dA#sc;YvPycET$WMBQ=<Ljy?Qoyl1}=dT~8r6!S=uRNaVxL
S<BUapkGE5aapyn%eew@XHW1(d
literal 0
HcmV?d00001
diff --git a/ray_results/test_experiment/train_model_6c8b476a_20_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=4,drop_r_2024-06-25_18-43-49/params.json b/ray_results/test_experiment/train_model_6c8b476a_20_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=4,drop_r_2024-06-25_18-43-49/params.json
new file mode 100644
index 0000000..cdc8e6a
--- /dev/null
+++ b/ray_results/test_experiment/train_model_6c8b476a_20_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=4,drop_r_2024-06-25_18-43-49/params.json
@@ -0,0 +1,13 @@
+{
+ "batch_size": 1024,
+ "decoder_int_ff": 512,
+ "decoder_int_num_layer": 1,
+ "decoder_rt_ff": 512,
+ "decoder_rt_num_layer": 4,
+ "drop_rate": 0.5088357248512263,
+ "embedding_dim": 64,
+ "encoder_ff": 1024,
+ "encoder_num_layer": 8,
+ "lr": 0.010524289583233754,
+ "n_head": 16
+}
\ No newline at end of file
diff --git a/ray_results/test_experiment/train_model_6c8b476a_20_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=4,drop_r_2024-06-25_18-43-49/params.pkl b/ray_results/test_experiment/train_model_6c8b476a_20_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=4,drop_r_2024-06-25_18-43-49/params.pkl
new file mode 100644
index 0000000000000000000000000000000000000000..25e06ac30468867cc36b1fb368e21f3aff3e8c36
GIT binary patch
literal 224
zcmZo*oqCl40&1sd^a!TrCFiH47RBe4=EmnFR;CtB@#g3eNlAr>7L_1Nu=I$+Br@|*
zWEgvRQ*)D2Q&KYX(&JMybEkMa^l-t9PD`8O%fQmZ3)2fU6fDHl!v_-r83z?(%ZtxQ
zO-z~MEzrZ6Qj}j1UzAvqI>p`ofk)!)6+6Uwm~x7Myw=`BHq(yB_HZR7mLzAy7iU(b
Jf-EW30|4Y!Q@{WK
literal 0
HcmV?d00001
diff --git a/ray_results/test_experiment/train_model_6c8b476a_20_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=4,drop_r_2024-06-25_18-43-49/progress.csv b/ray_results/test_experiment/train_model_6c8b476a_20_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=4,drop_r_2024-06-25_18-43-49/progress.csv
new file mode 100644
index 0000000..4ba5ff3
--- /dev/null
+++ b/ray_results/test_experiment/train_model_6c8b476a_20_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=4,drop_r_2024-06-25_18-43-49/progress.csv
@@ -0,0 +1,5 @@
+loss,timestamp,checkpoint_dir_name,should_checkpoint,done,training_iteration,trial_id,date,time_this_iter_s,time_total_s,pid,hostname,node_ip,time_since_restore,iterations_since_restore
+2006.8072533795214,1719334267,checkpoint_000000,True,False,1,6c8b476a,2024-06-25_18-51-07,127.51717782020569,127.51717782020569,2195253,r8i6n2,10.159.28.60,127.51717782020569,1
+1995.246540219765,1719334380,checkpoint_000001,True,False,2,6c8b476a,2024-06-25_18-53-00,113.46883034706116,240.98600816726685,2195253,r8i6n2,10.159.28.60,240.98600816726685,2
+1996.2762984628753,1719338625,checkpoint_000002,True,False,3,6c8b476a,2024-06-25_20-03-45,126.75374698638916,367.739755153656,2195253,r8i6n2,10.159.28.60,126.75374698638916,1
+1991.5076505405696,1719338738,checkpoint_000003,True,False,4,6c8b476a,2024-06-25_20-05-38,113.23560690879822,480.9753620624542,2195253,r8i6n2,10.159.28.60,239.98935389518738,2
diff --git a/ray_results/test_experiment/train_model_6c8b476a_20_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=4,drop_r_2024-06-25_18-43-49/result.json b/ray_results/test_experiment/train_model_6c8b476a_20_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=4,drop_r_2024-06-25_18-43-49/result.json
new file mode 100644
index 0000000..9a54df5
--- /dev/null
+++ b/ray_results/test_experiment/train_model_6c8b476a_20_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=4,drop_r_2024-06-25_18-43-49/result.json
@@ -0,0 +1,4 @@
+{"loss": 2006.8072533795214, "timestamp": 1719334267, "checkpoint_dir_name": "checkpoint_000000", "should_checkpoint": true, "done": false, "training_iteration": 1, "trial_id": "6c8b476a", "date": "2024-06-25_18-51-07", "time_this_iter_s": 127.51717782020569, "time_total_s": 127.51717782020569, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 8, "decoder_rt_num_layer": 4, "decoder_int_num_layer": 1, "embedding_dim": 64, "encoder_ff": 1024, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 16, "drop_rate": 0.5088357248512263, "lr": 0.010524289583233754, "batch_size": 1024}, "time_since_restore": 127.51717782020569, "iterations_since_restore": 1}
+{"loss": 1995.246540219765, "timestamp": 1719334380, "checkpoint_dir_name": "checkpoint_000001", "should_checkpoint": true, "done": false, "training_iteration": 2, "trial_id": "6c8b476a", "date": "2024-06-25_18-53-00", "time_this_iter_s": 113.46883034706116, "time_total_s": 240.98600816726685, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 8, "decoder_rt_num_layer": 4, "decoder_int_num_layer": 1, "embedding_dim": 64, "encoder_ff": 1024, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 16, "drop_rate": 0.5088357248512263, "lr": 0.010524289583233754, "batch_size": 1024}, "time_since_restore": 240.98600816726685, "iterations_since_restore": 2}
+{"loss": 1996.2762984628753, "timestamp": 1719338625, "checkpoint_dir_name": "checkpoint_000002", "should_checkpoint": true, "done": false, "training_iteration": 3, "trial_id": "6c8b476a", "date": "2024-06-25_20-03-45", "time_this_iter_s": 126.75374698638916, "time_total_s": 367.739755153656, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 8, "decoder_rt_num_layer": 4, "decoder_int_num_layer": 1, "embedding_dim": 64, "encoder_ff": 1024, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 16, "drop_rate": 0.5088357248512263, "lr": 0.010524289583233754, "batch_size": 1024}, "time_since_restore": 126.75374698638916, "iterations_since_restore": 1}
+{"loss": 1991.5076505405696, "timestamp": 1719338738, "checkpoint_dir_name": "checkpoint_000003", "should_checkpoint": true, "done": false, "training_iteration": 4, "trial_id": "6c8b476a", "date": "2024-06-25_20-05-38", "time_this_iter_s": 113.23560690879822, "time_total_s": 480.9753620624542, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 8, "decoder_rt_num_layer": 4, "decoder_int_num_layer": 1, "embedding_dim": 64, "encoder_ff": 1024, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 16, "drop_rate": 0.5088357248512263, "lr": 0.010524289583233754, "batch_size": 1024}, "time_since_restore": 239.98935389518738, "iterations_since_restore": 2}
diff --git a/ray_results/test_experiment/train_model_7ddcd025_3_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=8,drop_ra_2024-06-25_17-14-16/events.out.tfevents.1719328526.r8i6n2 b/ray_results/test_experiment/train_model_7ddcd025_3_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=8,drop_ra_2024-06-25_17-14-16/events.out.tfevents.1719328526.r8i6n2
new file mode 100644
index 0000000000000000000000000000000000000000..9bac5693357821843ba482583e0c8d0988a565f3
GIT binary patch
literal 1311
zcmah|K}Zx)7~Y)~YkpdFC)ad!ZFjY@IW#IkAjWQqhpa$41RdfvyYqC%cHay$W3^or
zB&b8=r9+UFZKQ{yy@X^DQ5We_2MG$Ys6c}tI}~*5&F<{mT{D~8{NMlm-+TY}y+&8z
z%Z?q6@Ux3|U(DTjdm!MA8?s9F^q;}R>gmqNR(G4zm%DkkkDqTn`p)jFbBhg74_-sg
zNY*I9Qc5#T|IDhp|J0><z0;vw{rMX46YY%B24M&=ikZ|#QwmBXapJVDsl@WTT;oyq
zwDhq;a$zUqS4t`x!Ia{11+%(GawO9ygM<9>U5pZe5HvAKOHE^BB~=rt7B-OS-zz;l
z_+@1_-sRAzCMLe}uS)cGh(L2C-Bd{eBLkb3X5jpQ#VUPipEF>2<>d)J*1_~e=zuUA
zV5`cOsu2@aj<Ik4W9D03&Rv)@IdyG>pXtqIAJJutcRVodfqInGWkXJzNW$lIY|y(9
zYv7^>E_h&*0|~!5;TIRF0f@_1B1u)w<B%sJK#hVEnnLeP_l$;yXsrqS<ux*zMkzUi
z4GQ_dTZ9a!yAkTkO6AR7;1xsD5xtl#MY%W~$BLqoVWg;OTB4`Hq@bQx_JgnZUp}ZB
zxRk99P)$%0%Sy;2`k+>*44K#pEy4yNT0^lzA_k#Z_;(1z?GO|;GFjXKZNmRee1B+U
zH*x%X{{40jzg+b169*qSXes$;V$-B4K&B<r1*>F88+gjWW3J@m11@lngNCx*3cRoL
z$mRRF#8MQt7FR@zg}|&vA+W|ZhK<Ldb)A(n+!cf7wE<xuC5PAdp0$KxP-}Zf3(wy4
zJo%itl0QsE3|RAE47Qc$fdRMjKq<Bin2ga!xPDey&uVmhYOK50d4vmd*<-wGzw>&!
bK!U*EyT?Q+Y@TMUwD+nAx7j`1cYVtrAT*PS
literal 0
HcmV?d00001
diff --git a/ray_results/test_experiment/train_model_7ddcd025_3_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=8,drop_ra_2024-06-25_17-14-16/params.json b/ray_results/test_experiment/train_model_7ddcd025_3_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=8,drop_ra_2024-06-25_17-14-16/params.json
new file mode 100644
index 0000000..ae356ec
--- /dev/null
+++ b/ray_results/test_experiment/train_model_7ddcd025_3_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=8,drop_ra_2024-06-25_17-14-16/params.json
@@ -0,0 +1,13 @@
+{
+ "batch_size": 1024,
+ "decoder_int_ff": 512,
+ "decoder_int_num_layer": 1,
+ "decoder_rt_ff": 512,
+ "decoder_rt_num_layer": 8,
+ "drop_rate": 0.474834603771515,
+ "embedding_dim": 64,
+ "encoder_ff": 2048,
+ "encoder_num_layer": 2,
+ "lr": 0.015369204694575145,
+ "n_head": 1
+}
\ No newline at end of file
diff --git a/ray_results/test_experiment/train_model_7ddcd025_3_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=8,drop_ra_2024-06-25_17-14-16/params.pkl b/ray_results/test_experiment/train_model_7ddcd025_3_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=8,drop_ra_2024-06-25_17-14-16/params.pkl
new file mode 100644
index 0000000000000000000000000000000000000000..f7324d1996709089de50c6898ab7840b47452eee
GIT binary patch
literal 224
zcmZo*oqCl40&1sd^a!TrCFiH47RBe4=EmnFR;CtB@n-4~NlAr>7L_1NaP)}6Br@|*
zWEgvRQ*)D2Q&KYX(&JMybEkMa^l-t9PD`8O%fQjY3)2fU6fDHl!v_-r83z?(%ZtxQ
zO-z~M&Dg`4Qj}j1UzAvqI>p`oUh;;uy*>s#OgTkBUVr7|JyzEa^l&95mLzAy7iU(b
K0xe-F)dK+N@Kh}T
literal 0
HcmV?d00001
diff --git a/ray_results/test_experiment/train_model_7ddcd025_3_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=8,drop_ra_2024-06-25_17-14-16/progress.csv b/ray_results/test_experiment/train_model_7ddcd025_3_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=8,drop_ra_2024-06-25_17-14-16/progress.csv
new file mode 100644
index 0000000..d5f9bcf
--- /dev/null
+++ b/ray_results/test_experiment/train_model_7ddcd025_3_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=8,drop_ra_2024-06-25_17-14-16/progress.csv
@@ -0,0 +1,2 @@
+loss,timestamp,checkpoint_dir_name,should_checkpoint,done,training_iteration,trial_id,date,time_this_iter_s,time_total_s,pid,hostname,node_ip,time_since_restore,iterations_since_restore
+2079.0407700425994,1719328617,checkpoint_000000,True,False,1,7ddcd025,2024-06-25_17-16-57,90.59219408035278,90.59219408035278,2195253,r8i6n2,10.159.28.60,90.59219408035278,1
diff --git a/ray_results/test_experiment/train_model_7ddcd025_3_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=8,drop_ra_2024-06-25_17-14-16/result.json b/ray_results/test_experiment/train_model_7ddcd025_3_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=8,drop_ra_2024-06-25_17-14-16/result.json
new file mode 100644
index 0000000..5c795ba
--- /dev/null
+++ b/ray_results/test_experiment/train_model_7ddcd025_3_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=8,drop_ra_2024-06-25_17-14-16/result.json
@@ -0,0 +1 @@
+{"loss": 2079.0407700425994, "timestamp": 1719328617, "checkpoint_dir_name": "checkpoint_000000", "should_checkpoint": true, "done": false, "training_iteration": 1, "trial_id": "7ddcd025", "date": "2024-06-25_17-16-57", "time_this_iter_s": 90.59219408035278, "time_total_s": 90.59219408035278, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 2, "decoder_rt_num_layer": 8, "decoder_int_num_layer": 1, "embedding_dim": 64, "encoder_ff": 2048, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 1, "drop_rate": 0.474834603771515, "lr": 0.015369204694575145, "batch_size": 1024}, "time_since_restore": 90.59219408035278, "iterations_since_restore": 1}
diff --git a/ray_results/test_experiment/train_model_7dfe67c7_4_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=2,drop_r_2024-06-25_17-15-26/events.out.tfevents.1719328617.r8i6n2 b/ray_results/test_experiment/train_model_7dfe67c7_4_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=2,drop_r_2024-06-25_17-15-26/events.out.tfevents.1719328617.r8i6n2
new file mode 100644
index 0000000000000000000000000000000000000000..7367e226129739c3987e7bb20d4b3e14b1e03ed1
GIT binary patch
literal 1311
zcmah|O=uHA7)>{|x*w)(woPr*AJbGp_h4EtLJ^WyJ=k1I5Q~;FZn9Ien#?Xc8@0U{
z2ul4q^q>ck+G@og6xv%6!GjbL58}atdJ$CABBCH#Jc!OVn@Lh@-P?Zed*7RR-<w8P
z{+m7-4T?{5`(MpndD-Ll#7#vb+x8s8#OmID?rcq<!|9uyJibf3)xUMo?yIek8=xLM
zrZN(;G6aWGI;Fmm_~P!m=z`JdP%eIdhQ!<er*uF&_&J3p^-M}di6l-OHFS+wK9?&S
z_VrGUl}Q>0Ilo*|)d^-4m+MA$2ttlzdZ@KuTtCPuK?pz-m$bArMpjazNVBkssIT_I
z``&l4%%?7gK9S2Uh^LG6O%R0Uayr#W0wWVsOE>Y-fSb|izz%1?hmTJVh`);iWY_}j
ze1NU0Sei~KDj#Ft{QTw0nc0f`obicEM?{#PoxaD+man_vsvGK2(ojq#O;HG+FtEw)
zLae?qH=J_AS_hJT3epcDSPc+YtVEKjjN*VhC_%M~6S~Uo%zQ?}!>rZ>zS0`Wq)|#4
z!6t*e;3+_+)7=R5C8g444~VL%8;D)ZmSS9-j$>8T$YG>vX;xyV!6dJ~r0fH4;lFfH
z6$mL?9iWP!Bv#abTlPYYR30+94O*mCLbQxxheU3LX6fG{kT*iBwAzy8&CntJe~Gty
zKK~-l{vLk0)-CoG_V-4B*8;Q@_eU|MECq;K3Nu(E!+PHX0qzUMJ<bUJy8<+ncq@tC
z&KKD=gWB6L)D<jZMJIR>@GsjM$EG6Cwu0pxcSWFic|bUrjj%_%;_i71<JZ``V+Cz=
z-`@T+v9NuZvKa7YBLW*r^S}XiN(K~T%YpF-dxR@zmG^uN!?#ECI~^|`&rQd~Lk0J{
bAO!xuy~jk&JI^v!^u01DtgCo_r)}~Nrizn)
literal 0
HcmV?d00001
diff --git a/ray_results/test_experiment/train_model_7dfe67c7_4_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=2,drop_r_2024-06-25_17-15-26/params.json b/ray_results/test_experiment/train_model_7dfe67c7_4_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=2,drop_r_2024-06-25_17-15-26/params.json
new file mode 100644
index 0000000..adf6e7c
--- /dev/null
+++ b/ray_results/test_experiment/train_model_7dfe67c7_4_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=2,drop_r_2024-06-25_17-15-26/params.json
@@ -0,0 +1,13 @@
+{
+ "batch_size": 1024,
+ "decoder_int_ff": 512,
+ "decoder_int_num_layer": 1,
+ "decoder_rt_ff": 2048,
+ "decoder_rt_num_layer": 2,
+ "drop_rate": 0.6066107852445892,
+ "embedding_dim": 256,
+ "encoder_ff": 2048,
+ "encoder_num_layer": 2,
+ "lr": 0.0011313709304158431,
+ "n_head": 2
+}
\ No newline at end of file
diff --git a/ray_results/test_experiment/train_model_7dfe67c7_4_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=2,drop_r_2024-06-25_17-15-26/params.pkl b/ray_results/test_experiment/train_model_7dfe67c7_4_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=2,drop_r_2024-06-25_17-15-26/params.pkl
new file mode 100644
index 0000000000000000000000000000000000000000..04ed4ecc4f6a23dc67e03a1661d603cfd8caa48c
GIT binary patch
literal 225
zcmZo*oqCM{0&1sd^a!TrCFiH47RBe4=EmnFR;CtB@n-4~NlAr>7L}k%h{7Z?^H5|M
zdw5fGlTuSsGV{{oQ!;a>_%bl|aKX$@O9S#bdU#=)fu=%*_+UaH^T1+EJ#2aL8L5dW
zAa`)46y+Dh7bTXYPI0$?oEgp9aKoU7DW?d?3+fE;45`ZO;Yvy@NzRBb&a6rWTEbGQ
F2LQQiQL_L5
literal 0
HcmV?d00001
diff --git a/ray_results/test_experiment/train_model_7dfe67c7_4_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=2,drop_r_2024-06-25_17-15-26/progress.csv b/ray_results/test_experiment/train_model_7dfe67c7_4_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=2,drop_r_2024-06-25_17-15-26/progress.csv
new file mode 100644
index 0000000..dc8c1f9
--- /dev/null
+++ b/ray_results/test_experiment/train_model_7dfe67c7_4_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=2,drop_r_2024-06-25_17-15-26/progress.csv
@@ -0,0 +1,2 @@
+loss,timestamp,checkpoint_dir_name,should_checkpoint,done,training_iteration,trial_id,date,time_this_iter_s,time_total_s,pid,hostname,node_ip,time_since_restore,iterations_since_restore
+1979.0773276982345,1719328732,checkpoint_000000,True,False,1,7dfe67c7,2024-06-25_17-18-52,115.28918504714966,115.28918504714966,2195253,r8i6n2,10.159.28.60,115.28918504714966,1
diff --git a/ray_results/test_experiment/train_model_7dfe67c7_4_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=2,drop_r_2024-06-25_17-15-26/result.json b/ray_results/test_experiment/train_model_7dfe67c7_4_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=2,drop_r_2024-06-25_17-15-26/result.json
new file mode 100644
index 0000000..a11df1a
--- /dev/null
+++ b/ray_results/test_experiment/train_model_7dfe67c7_4_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=2,drop_r_2024-06-25_17-15-26/result.json
@@ -0,0 +1 @@
+{"loss": 1979.0773276982345, "timestamp": 1719328732, "checkpoint_dir_name": "checkpoint_000000", "should_checkpoint": true, "done": false, "training_iteration": 1, "trial_id": "7dfe67c7", "date": "2024-06-25_17-18-52", "time_this_iter_s": 115.28918504714966, "time_total_s": 115.28918504714966, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 2, "decoder_rt_num_layer": 2, "decoder_int_num_layer": 1, "embedding_dim": 256, "encoder_ff": 2048, "decoder_rt_ff": 2048, "decoder_int_ff": 512, "n_head": 2, "drop_rate": 0.6066107852445892, "lr": 0.0011313709304158431, "batch_size": 1024}, "time_since_restore": 115.28918504714966, "iterations_since_restore": 1}
diff --git a/ray_results/test_experiment/train_model_82396cf3_25_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_19-16-23/events.out.tfevents.1719335912.r8i6n2 b/ray_results/test_experiment/train_model_82396cf3_25_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_19-16-23/events.out.tfevents.1719335912.r8i6n2
new file mode 100644
index 0000000000000000000000000000000000000000..026fe5133f8eec3c18e3c3ede008ba87963143ff
GIT binary patch
literal 1676
zcma)6TSydP6kcaTcMhz&lWV%VUREO+($xwAGu%j_tVk*%1DVF1Uw3kMMrOuRA5xG)
z=pkPuHu9E14}~ol^<deB9-=4drF`fnMD<W6bg>}lA9wcO9rLztbH4MPZ{|DS*(^)+
zH`aZ?LwEP(|Ct(^+2gV|DZD6^)t*6;R=I6ps=cMi=!*=W-AnH;zCZ8~_oXMXnUDc?
zh41ug?Go|^WmR>~d2F?>u4u2z4a&feX^uX@5=s#if{RepfZQGwIDY{7Tiaw&(wr7c
zPjzI*cFH9AZY$xNB?VbRm}0T8J@chrLo(FpuBSUn3B>~ikWC~tF@!iRAgY|GA%#<&
zC$Hwzd~bY^Qen^sdwW0Ajxs_o1`p(z>8dFC5vL$klNGc$V4*=hzS9`+@ze7L`f+>!
z3nfrU2I#81Cd!h^naAi~evnxhH+}iV%BQ7|bhD}D%yv!KYP-6qV<xl5cM-~Z%szDK
zOVA{_y^8QxFk2fxH*7L{GEP~;?7JyhHTynX=rx(WV>eM;(`?&|aLtX&hs#Z7-;57f
zx7ilooBpEuNK$lZ{e$7tbVKd0qq^Dbbt{ZoA%hFF@d_VOIWOvHLkbR#q}BCV;k*@6
z42bznF+V9!DnJvj`2%>RU8un7VIWmNep$c)!=dJyn{h20oQXB5J;Vk1PNZPS0rnWA
z7~NTrkx)u(wgW9FavO(Jq)TBg3N;}?5TzDQ5JR|x-v&uh^+jbTIAYI<gOY&?>goW=
z5*I+cP+(;pkj9uVn$3e;W|fOwLeVdhbwdub^2M?F;AU34D0VXxF>7B;gYv#YGWPdm
z?ph@sj_q%jg0~dp#`mWpRmCUAsTz+B7NusnZi<3QD!#|tl<O7+nF-!9v}3wEFj^A)
zS`8ax7U5zvFzG&UE!i5udVG+#jO7H!+?=HWAs{9fF28%yV(~$mzB^n9549${@03^T
zFJp=UY3}#Iro=oDU}O=9$2Jl0tdLdG^R0UR@0Qsr;~($Yy|E*7B<6kvc)|6*_ec_=
T&f|>5ec#}rY)KO@?IZsH+7btu
literal 0
HcmV?d00001
diff --git a/ray_results/test_experiment/train_model_82396cf3_25_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_19-16-23/params.json b/ray_results/test_experiment/train_model_82396cf3_25_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_19-16-23/params.json
new file mode 100644
index 0000000..3afc3a3
--- /dev/null
+++ b/ray_results/test_experiment/train_model_82396cf3_25_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_19-16-23/params.json
@@ -0,0 +1,13 @@
+{
+ "batch_size": 2048,
+ "decoder_int_ff": 512,
+ "decoder_int_num_layer": 1,
+ "decoder_rt_ff": 512,
+ "decoder_rt_num_layer": 2,
+ "drop_rate": 0.9507029678825294,
+ "embedding_dim": 1024,
+ "encoder_ff": 2048,
+ "encoder_num_layer": 4,
+ "lr": 0.00025487621898754583,
+ "n_head": 4
+}
\ No newline at end of file
diff --git a/ray_results/test_experiment/train_model_82396cf3_25_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_19-16-23/params.pkl b/ray_results/test_experiment/train_model_82396cf3_25_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_19-16-23/params.pkl
new file mode 100644
index 0000000000000000000000000000000000000000..5822f94cb1e2ebddcad8543851807b8549ac0c89
GIT binary patch
literal 225
zcmZo*oqCM{0&1sd^a!TrCFiH47RBe4=EmnFR;CtB@n-1}NlAr>7L_1NF!hMSBr@|*
zWEgvRQ*)D2Q&KYX(&JMybEo(+u=H@j%uY)K@;Q2VVVZ%af`yoR_+UaH^PpmEdGQ&k
zi76m=aHbUH7sMAOmZVN`w||$Tu`r|lLJw0;5s+uFMV2jl-pn4Zq{NcsjQHZrs#K6A
GrFsC*p;BA`
literal 0
HcmV?d00001
diff --git a/ray_results/test_experiment/train_model_82396cf3_25_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_19-16-23/progress.csv b/ray_results/test_experiment/train_model_82396cf3_25_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_19-16-23/progress.csv
new file mode 100644
index 0000000..1f3b6b4
--- /dev/null
+++ b/ray_results/test_experiment/train_model_82396cf3_25_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_19-16-23/progress.csv
@@ -0,0 +1,3 @@
+loss,timestamp,checkpoint_dir_name,should_checkpoint,done,training_iteration,trial_id,date,time_this_iter_s,time_total_s,pid,hostname,node_ip,time_since_restore,iterations_since_restore
+2178.1833553764764,1719336534,checkpoint_000000,True,False,1,82396cf3,2024-06-25_19-28-54,622.4429934024811,622.4429934024811,2195253,r8i6n2,10.159.28.60,622.4429934024811,1
+2187.193873608206,1719337142,checkpoint_000001,True,False,2,82396cf3,2024-06-25_19-39-03,608.1205008029938,1230.5634942054749,2195253,r8i6n2,10.159.28.60,1230.5634942054749,2
diff --git a/ray_results/test_experiment/train_model_82396cf3_25_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_19-16-23/result.json b/ray_results/test_experiment/train_model_82396cf3_25_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_19-16-23/result.json
new file mode 100644
index 0000000..6aabf4a
--- /dev/null
+++ b/ray_results/test_experiment/train_model_82396cf3_25_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_19-16-23/result.json
@@ -0,0 +1,2 @@
+{"loss": 2178.1833553764764, "timestamp": 1719336534, "checkpoint_dir_name": "checkpoint_000000", "should_checkpoint": true, "done": false, "training_iteration": 1, "trial_id": "82396cf3", "date": "2024-06-25_19-28-54", "time_this_iter_s": 622.4429934024811, "time_total_s": 622.4429934024811, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 4, "decoder_rt_num_layer": 2, "decoder_int_num_layer": 1, "embedding_dim": 1024, "encoder_ff": 2048, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 4, "drop_rate": 0.9507029678825294, "lr": 0.00025487621898754583, "batch_size": 2048}, "time_since_restore": 622.4429934024811, "iterations_since_restore": 1}
+{"loss": 2187.193873608206, "timestamp": 1719337142, "checkpoint_dir_name": "checkpoint_000001", "should_checkpoint": true, "done": false, "training_iteration": 2, "trial_id": "82396cf3", "date": "2024-06-25_19-39-03", "time_this_iter_s": 608.1205008029938, "time_total_s": 1230.5634942054749, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 4, "decoder_rt_num_layer": 2, "decoder_int_num_layer": 1, "embedding_dim": 1024, "encoder_ff": 2048, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 4, "drop_rate": 0.9507029678825294, "lr": 0.00025487621898754583, "batch_size": 2048}, "time_since_restore": 1230.5634942054749, "iterations_since_restore": 2}
diff --git a/ray_results/test_experiment/train_model_86ae258f_7_batch_size=2048,decoder_int_ff=1024,decoder_int_num_layer=4,decoder_rt_ff=1024,decoder_rt_num_layer=4,drop__2024-06-25_16-15-18/events.out.tfevents.1719325014.r3i5n6 b/ray_results/test_experiment/train_model_86ae258f_7_batch_size=2048,decoder_int_ff=1024,decoder_int_num_layer=4,decoder_rt_ff=1024,decoder_rt_num_layer=4,drop__2024-06-25_16-15-18/events.out.tfevents.1719325014.r3i5n6
new file mode 100644
index 0000000000000000000000000000000000000000..10b61213c97a5e881e015339b840ca4732ffdcd8
GIT binary patch
literal 405
zcmb1OfPlsI-b$QK>5D_R%)99*#hX-=n3<>NT9%quVr8`2rz}Jns&2#la%WC{4~|NZ
zIsry4F)k4<-lD`x{gTqWRQ;U%;$l&Klb<dp*}b?8pjxJWy%5it<qOxM%%#XB1=muX
zkzbmV5}%xrnw(vbpP5%8%D~WIzcW8AAH_<U0JwfsD^v3GQh{0+7<O+J3Dk#LnHwt`
zz<Cm(MTtv+OA>BnNoH<pd`U)TaeQV;YEgW#Xy_zfXP29vR)$dRbDNuAawY`AZB*e>
z;*v(yUYwbioEl%0T3nJ}lnOS0!Eb_<In;pXFHS{renc3c#ihZe2sZ#^Yhp=eeqM1r
Nsxcs+SF1gl0{{V&d-(tW
literal 0
HcmV?d00001
diff --git a/ray_results/test_experiment/train_model_86ae258f_7_batch_size=2048,decoder_int_ff=1024,decoder_int_num_layer=4,decoder_rt_ff=1024,decoder_rt_num_layer=4,drop__2024-06-25_16-15-18/events.out.tfevents.1719327469.r3i5n6 b/ray_results/test_experiment/train_model_86ae258f_7_batch_size=2048,decoder_int_ff=1024,decoder_int_num_layer=4,decoder_rt_ff=1024,decoder_rt_num_layer=4,drop__2024-06-25_16-15-18/events.out.tfevents.1719327469.r3i5n6
new file mode 100644
index 0000000000000000000000000000000000000000..dd67d6d6f8550c83efa236fe944e13de4dfd5c06
GIT binary patch
literal 405
zcmb1OfPlsI-b$QvemU*lHt(jR6mL>dVrHJ6YguYuiIq|HTfNQ7P<0#TmpgNwKE3Q7
zNSy$amKc`^7jIEwrG80iUaEdhesQs=NAgdXUm>4k4WL@4e!URS$#4m-MVU*HOA4-~
zI3vF_CnY{PBQ-g@AU`v&M3jM{!Cr6sR1Xv@U9Q3Pqgt7gpO*^M0tCEK<@!)7b7N%#
zIA0^QC~+xpNy4ox$;?fSFUiO(j?XMfEs8G|4esT2t`B;+!4Rr_ZgcZX&f@EE8&$ZJ
zxTI0F7iZ=rr^Xkh7MJ7~rGgEZ*7!iw9BRPx7pEdQ-y#gq;?m$!gc|^|HL)Z!Kd(3*
N)fkY^r5Rqi0{~j5h{gZ_
literal 0
HcmV?d00001
diff --git a/ray_results/test_experiment/train_model_86ae258f_7_batch_size=2048,decoder_int_ff=1024,decoder_int_num_layer=4,decoder_rt_ff=1024,decoder_rt_num_layer=4,drop__2024-06-25_16-15-18/params.json b/ray_results/test_experiment/train_model_86ae258f_7_batch_size=2048,decoder_int_ff=1024,decoder_int_num_layer=4,decoder_rt_ff=1024,decoder_rt_num_layer=4,drop__2024-06-25_16-15-18/params.json
new file mode 100644
index 0000000..ee925cb
--- /dev/null
+++ b/ray_results/test_experiment/train_model_86ae258f_7_batch_size=2048,decoder_int_ff=1024,decoder_int_num_layer=4,decoder_rt_ff=1024,decoder_rt_num_layer=4,drop__2024-06-25_16-15-18/params.json
@@ -0,0 +1,13 @@
+{
+ "batch_size": 2048,
+ "decoder_int_ff": 1024,
+ "decoder_int_num_layer": 4,
+ "decoder_rt_ff": 1024,
+ "decoder_rt_num_layer": 4,
+ "drop_rate": 0.7568171644612052,
+ "embedding_dim": 256,
+ "encoder_ff": 2048,
+ "encoder_num_layer": 2,
+ "lr": 0.0002629821336450532,
+ "n_head": 16
+}
\ No newline at end of file
diff --git a/ray_results/test_experiment/train_model_86ae258f_7_batch_size=2048,decoder_int_ff=1024,decoder_int_num_layer=4,decoder_rt_ff=1024,decoder_rt_num_layer=4,drop__2024-06-25_16-15-18/params.pkl b/ray_results/test_experiment/train_model_86ae258f_7_batch_size=2048,decoder_int_ff=1024,decoder_int_num_layer=4,decoder_rt_ff=1024,decoder_rt_num_layer=4,drop__2024-06-25_16-15-18/params.pkl
new file mode 100644
index 0000000000000000000000000000000000000000..289c7c0772501541dc96ebcea495ecc9910ae5c3
GIT binary patch
literal 225
zcmZo*oqCM{0&1sd^a!TrCFiH47RBe4=EmnFR;CtB@n-4~NlAr>7L_1Nu=I$+Br@~R
zWO!3^lTuSsGV{{oQ!;a>_%bl|aKX$@O9S#bdU#=)fu@3mSbF$iLLl>?Vr+Tw8L5dW
zQ@jOwI8%!93*w6sOH!w}+rKcsvC!A>Tn|%D5s+tSBfauEe^d`wQesJRMtpH*RVv7m
GQau3CWl}x>
literal 0
HcmV?d00001
diff --git a/ray_results/test_experiment/train_model_86ae258f_7_batch_size=2048,decoder_int_ff=1024,decoder_int_num_layer=4,decoder_rt_ff=1024,decoder_rt_num_layer=4,drop__2024-06-25_16-15-18/progress.csv b/ray_results/test_experiment/train_model_86ae258f_7_batch_size=2048,decoder_int_ff=1024,decoder_int_num_layer=4,decoder_rt_ff=1024,decoder_rt_num_layer=4,drop__2024-06-25_16-15-18/progress.csv
new file mode 100644
index 0000000..68b7d89
--- /dev/null
+++ b/ray_results/test_experiment/train_model_86ae258f_7_batch_size=2048,decoder_int_ff=1024,decoder_int_num_layer=4,decoder_rt_ff=1024,decoder_rt_num_layer=4,drop__2024-06-25_16-15-18/progress.csv
@@ -0,0 +1,3 @@
+loss,timestamp,checkpoint_dir_name,should_checkpoint,done,training_iteration,trial_id,date,time_this_iter_s,time_total_s,pid,hostname,node_ip,time_since_restore,iterations_since_restore
+1993.6307767132137,1719325156,checkpoint_000000,True,False,1,86ae258f,2024-06-25_16-19-16,141.57161211967468,141.57161211967468,69318,r3i5n6,10.159.8.159,141.57161211967468,1
+1995.102591927596,1719327610,checkpoint_000001,True,False,2,86ae258f,2024-06-25_17-00-10,141.5520534515381,283.12366557121277,130020,r3i5n6,10.159.8.159,141.5520534515381,1
diff --git a/ray_results/test_experiment/train_model_86ae258f_7_batch_size=2048,decoder_int_ff=1024,decoder_int_num_layer=4,decoder_rt_ff=1024,decoder_rt_num_layer=4,drop__2024-06-25_16-15-18/result.json b/ray_results/test_experiment/train_model_86ae258f_7_batch_size=2048,decoder_int_ff=1024,decoder_int_num_layer=4,decoder_rt_ff=1024,decoder_rt_num_layer=4,drop__2024-06-25_16-15-18/result.json
new file mode 100644
index 0000000..1a450f4
--- /dev/null
+++ b/ray_results/test_experiment/train_model_86ae258f_7_batch_size=2048,decoder_int_ff=1024,decoder_int_num_layer=4,decoder_rt_ff=1024,decoder_rt_num_layer=4,drop__2024-06-25_16-15-18/result.json
@@ -0,0 +1,2 @@
+{"loss": 1993.6307767132137, "timestamp": 1719325156, "checkpoint_dir_name": "checkpoint_000000", "should_checkpoint": true, "done": false, "training_iteration": 1, "trial_id": "86ae258f", "date": "2024-06-25_16-19-16", "time_this_iter_s": 141.57161211967468, "time_total_s": 141.57161211967468, "pid": 69318, "hostname": "r3i5n6", "node_ip": "10.159.8.159", "config": {"encoder_num_layer": 2, "decoder_rt_num_layer": 4, "decoder_int_num_layer": 4, "embedding_dim": 256, "encoder_ff": 2048, "decoder_rt_ff": 1024, "decoder_int_ff": 1024, "n_head": 16, "drop_rate": 0.7568171644612052, "lr": 0.0002629821336450532, "batch_size": 2048}, "time_since_restore": 141.57161211967468, "iterations_since_restore": 1}
+{"loss": 1995.102591927596, "timestamp": 1719327610, "checkpoint_dir_name": "checkpoint_000001", "should_checkpoint": true, "done": false, "training_iteration": 2, "trial_id": "86ae258f", "date": "2024-06-25_17-00-10", "time_this_iter_s": 141.5520534515381, "time_total_s": 283.12366557121277, "pid": 130020, "hostname": "r3i5n6", "node_ip": "10.159.8.159", "config": {"encoder_num_layer": 2, "decoder_rt_num_layer": 4, "decoder_int_num_layer": 4, "embedding_dim": 256, "encoder_ff": 2048, "decoder_rt_ff": 1024, "decoder_int_ff": 1024, "n_head": 16, "drop_rate": 0.7568171644612052, "lr": 0.0002629821336450532, "batch_size": 2048}, "time_since_restore": 141.5520534515381, "iterations_since_restore": 1}
diff --git a/ray_results/test_experiment/train_model_8c6e7040_14_batch_size=2048,decoder_int_ff=2048,decoder_int_num_layer=8,decoder_rt_ff=1024,decoder_rt_num_layer=8,drop_2024-06-25_16-41-29/error.pkl b/ray_results/test_experiment/train_model_8c6e7040_14_batch_size=2048,decoder_int_ff=2048,decoder_int_num_layer=8,decoder_rt_ff=1024,decoder_rt_num_layer=8,drop_2024-06-25_16-41-29/error.pkl
new file mode 100644
index 0000000000000000000000000000000000000000..2fc8daba03d604d911ffe4f4d7076265d28d6b64
GIT binary patch
literal 7437
zcmeHMUvC@75vN=QwpG`4Q=<wH*j?CdLE6&&;eV;73x}?gR5oSEN|2yP7t6an?uEy@
z%kG{e78c+a8z^uO0eXh>5TI{GfqsB~jTGoZU-}*T*vHQ9@hDPO9jidBpfw3mw|ldD
zGr#%G?B497H=drlRMMaHZBE996GVv{`_3?+v)5UA5s%1_Vlkuvl~IVDfC!QOJ}akZ
z`iUP%KNMNID915{&h+vY8E+FY{3YiRFK;Gt)4NAU5g+T0N|v5i52Jeyjin!{q332t
zs413Y#2schiA%Ed!r>`E;z_ojrKs6(TTa_{sJm9LH(aOfHEX2p)Z6V^jkt9?>&a{j
z<X&JL_K1*K`u*G{ET|NcNM#56!M-vOV?XpI#u+Q6=W`sqBy^PKAXJ9l@8JmuxIo}c
zH*)E=g5xnfzr1m3qXgem@cm);C(lX`7ao>y4m|umT<}zTq3Cc+4@z0@ahAT}kVH@b
zmhgc9i_=9x121C;E)#PQdm@M&5{M)=<D_qrkT_B3lGS)zGpkm$W>{+`4R?eYk2RdD
zgE(2O8dbx(4kCta8!a>N`|37}Le*gkU((f>I72d^!UWjN12-A*IAcFbo3p+$hyZv1
zvCyHBhqc1Sja;ppz#(Bw;oErHed|%_@0YH8zL+ij^GxaT)i2(I%d?-uH|v(NN=5lf
zLH=EgCn^By>32YjOt~Q>BdXkRS~WaBP>qwqA?furwb+~Ko4BCE9C@ny$`)n-V&^da
zEFnRm|IdJIn1o>jJX7=sGl0wIF&^%^)FVkCRaZ|9n`<v&Jc`^z+YKO`hTD;&B>^x9
zU6F7FNiSVits;=INc#COk$sXWYzINq2iEAvCQq^_TbAynppN9w%ib!pO9z}?1kSEJ
zE6FK8fw84er>%zze>_zx{Tcr94{(?LI6Xbcr_*F*SJOphJW!bZl-*9(j?19QWT4x|
zm)(nxO8;2C{Du7Tw|}btb=mscvh~#kD`VxHKkRzpnL67nTyUGU*soZ7KV$1X_VXUQ
z!A`SIkKOFCUwo3WTl?7r27pw|La?C2c41=yAUN}M;aTZ-3);L2yZTs}@?qLIt||Ha
z8-M7Y{TBqoGWzNQ0^QTIROL!M5(CH>IuMj0kXtU~8HbKyImQkR0?0X)Ef7+{!s^T|
zZ2S;x{P+kPVUDGIj>+hc{&KNYdi&S4<_H37ef|AW9Qcke??6s7Bqx4YuE5>35h};N
z+i9-V?Dh)s<Bn|^c4N(GtKStw97rh`1Sp}l?K*3%hSh=$+;8`-y65&i%eM4q1#uel
zj-FZZmj;z3bnyAjOMH$Y52AM<GoUN4Dm}_1q^35G^n;lzNS0StP~e9Y)#_CMFhp8P
z+X5&Hip0>5W9lN0M<X;%vw03JPi`u#UadjI=O(0dNqI;DGm(BUV@g|5Aw)b0wH<0X
zGBBqSbx?jSVX30Bd@XOOb!!D(o3w_zsy*j@&4ZU<KrZsiu{h-1X`Y|@dj^_n9ijrT
zxe<_2-z7ISKw8ib2YQL<<agE!s8p;e(&(FIjTo%oo<FFp=)ng;GJEy4Xw8oO8sTp>
zptV{aHh7A*21=V3B{EJF>paL-vj!uiOg?G54n>V!%frSGL&~+b5Hbm$WCTTla_)E)
zD_#iufae0`#g3e}cQ@iglT(L5^inf%25oy+n37RJ&AZ*zdk<E(wpU|LU0-iv%@Obr
z!&-n9-DeoxY#yYoJacQ^)kw@0_#u@>zO^vCh=bCqABg<fA(v>M9Bsk8=C07X$2|I!
zIuiDu$1p;zEZ9kej7Bpy*tV^WIM7eq0Q|{K&{5}72S#(ewOyv6b4dB4^q$xH8{UNL
zt-Be9=42N)`9Eu9V`3%V<wPwhu!aNy_F-k$)=t=#=j7?F^0gv8ADeHlIKo~AfvTnb
ze6;0-_m%Dcz1b)0b;uAB0+sR^$5-vvEW5R{{V>mrIa|uQIZx?1MxU%vlk9P@SF&GI
zx$!?wWmwQd0Wmq6BXezSE>&wXPuF=MN;z+=Ai*7Yp%KH5`T_n6y=5jrXL^;(4Cxr^
z2(GL4Us_%bJlu&v4;+v&<%f5sb7AsLHC~6`w+?~-)9`EAjpx>qz2dztgkY_f_t5pp
zyNl=HK5To2<-N-M+A{N|@?6NAtf!AoI<4Ad-%wbj)6Udx_5MzHHGesCc@#qq${SGs
z@OO0mE^p_OwgTioL-HbY4#9OYU=?q;WcH=suD5EM8k`1+#)ne@RZ1X--kBMy|EP_|
z^ZWjpvc9o-=v-hObw9X$3jx9?^3X^h7#L`q`%og05rVB76uTuwRm(Cg3*GbIG0?mB
zKR^~@L?8k(k_5;hP&)SISj}NKj8+T1>%XHKyUceOvYU0Ixuyogx&f6JN)7aW=md#c
zB~TcJtM|v->gZvjzzUKK1mTUup+e{<!K_V7&zkF~UizU-Z=g*Ngy~dKcQS=i!*1&V
zFjHZMi~3_Uc@|XDLKih`sH_#50m78;faM2Ls1L{_P?X7Wa6xeoYHNku{LSLo@WcSN
z`4oyzV~AY|Cp80f|Ksh=t?qkR;f-(Iy}Q}P-Ob*e4&94E5nYlHbTFc!6zkhOfx!U(
zPUNdn718~g_y$4`C`C|&PL!<fL$I8p{ICbzg9C{HRksi-Gbv-SZWc!oM#Kjuj1(rE
z9jL+?RKwxGL7n#)Opb!9aBkuAC{&|YcL?{Dkf`Y66ezvUm2IJvo9shv84B6#+(%cS
W68)M;uZi?hL{jHX46#J?jei5rK)uTV
literal 0
HcmV?d00001
diff --git a/ray_results/test_experiment/train_model_8c6e7040_14_batch_size=2048,decoder_int_ff=2048,decoder_int_num_layer=8,decoder_rt_ff=1024,decoder_rt_num_layer=8,drop_2024-06-25_16-41-29/error.txt b/ray_results/test_experiment/train_model_8c6e7040_14_batch_size=2048,decoder_int_ff=2048,decoder_int_num_layer=8,decoder_rt_ff=1024,decoder_rt_num_layer=8,drop_2024-06-25_16-41-29/error.txt
new file mode 100644
index 0000000..1f1f25f
--- /dev/null
+++ b/ray_results/test_experiment/train_model_8c6e7040_14_batch_size=2048,decoder_int_ff=2048,decoder_int_num_layer=8,decoder_rt_ff=1024,decoder_rt_num_layer=8,drop_2024-06-25_16-41-29/error.txt
@@ -0,0 +1,59 @@
+Failure # 1 (occurred at 2024-06-25_16-43-06)
+[36mray::ImplicitFunc.train()[39m (pid=69318, ip=10.159.8.159, actor_id=eae11dc975075dcb8b04fdbf01000000, repr=train_model)
+ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+ File "/gpfslocalsup/pub/anaconda-py3/2023.09/envs/pytorch-gpu-2.2.0+py3.11.7/lib/python3.11/site-packages/ray/tune/trainable/trainable.py", line 342, in train
+ raise skipped from exception_cause(skipped)
+ File "/gpfslocalsup/pub/anaconda-py3/2023.09/envs/pytorch-gpu-2.2.0+py3.11.7/lib/python3.11/site-packages/ray/air/_internal/util.py", line 88, in run
+ self._ret = self._target(*self._args, **self._kwargs)
+ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+ File "/gpfslocalsup/pub/anaconda-py3/2023.09/envs/pytorch-gpu-2.2.0+py3.11.7/lib/python3.11/site-packages/ray/tune/trainable/function_trainable.py", line 249, in <lambda>
+ training_func=lambda: self._trainable_func(self.config),
+ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+ File "/gpfslocalsup/pub/anaconda-py3/2023.09/envs/pytorch-gpu-2.2.0+py3.11.7/lib/python3.11/site-packages/ray/tune/trainable/function_trainable.py", line 332, in _trainable_func
+ output = fn()
+ ^^^^
+ File "/gpfslocalsup/pub/anaconda-py3/2023.09/envs/pytorch-gpu-2.2.0+py3.11.7/lib/python3.11/site-packages/ray/tune/trainable/util.py", line 138, in inner
+ return trainable(config, **fn_kwargs)
+ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+ File "/gpfswork/rech/ute/ucg81ws/these/LC-MS-RT-prediction/main_ray_tune.py", line 66, in train_model
+ pred_rt = net.module.forward_rt(seq)
+ ^^^^^^^^^^^^^^^^^^^^^^^^^^
+ File "/gpfsdswork/projects/rech/ute/ucg81ws/these/LC-MS-RT-prediction/model_custom.py", line 111, in forward_rt
+ out_rt = self.decoder_RT(enc)
+ ^^^^^^^^^^^^^^^^^^^^
+ File "/gpfslocalsup/pub/anaconda-py3/2023.09/envs/pytorch-gpu-2.2.0+py3.11.7/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1511, in _wrapped_call_impl
+ return self._call_impl(*args, **kwargs)
+ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+ File "/gpfslocalsup/pub/anaconda-py3/2023.09/envs/pytorch-gpu-2.2.0+py3.11.7/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1520, in _call_impl
+ return forward_call(*args, **kwargs)
+ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+ File "/gpfslocalsup/pub/anaconda-py3/2023.09/envs/pytorch-gpu-2.2.0+py3.11.7/lib/python3.11/site-packages/torch/nn/modules/container.py", line 217, in forward
+ input = module(input)
+ ^^^^^^^^^^^^^
+ File "/gpfslocalsup/pub/anaconda-py3/2023.09/envs/pytorch-gpu-2.2.0+py3.11.7/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1511, in _wrapped_call_impl
+ return self._call_impl(*args, **kwargs)
+ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+ File "/gpfslocalsup/pub/anaconda-py3/2023.09/envs/pytorch-gpu-2.2.0+py3.11.7/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1520, in _call_impl
+ return forward_call(*args, **kwargs)
+ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+ File "/gpfslocalsup/pub/anaconda-py3/2023.09/envs/pytorch-gpu-2.2.0+py3.11.7/lib/python3.11/site-packages/torch/nn/modules/transformer.py", line 391, in forward
+ output = mod(output, src_mask=mask, is_causal=is_causal, src_key_padding_mask=src_key_padding_mask_for_layers)
+ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+ File "/gpfslocalsup/pub/anaconda-py3/2023.09/envs/pytorch-gpu-2.2.0+py3.11.7/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1511, in _wrapped_call_impl
+ return self._call_impl(*args, **kwargs)
+ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+ File "/gpfslocalsup/pub/anaconda-py3/2023.09/envs/pytorch-gpu-2.2.0+py3.11.7/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1520, in _call_impl
+ return forward_call(*args, **kwargs)
+ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+ File "/gpfslocalsup/pub/anaconda-py3/2023.09/envs/pytorch-gpu-2.2.0+py3.11.7/lib/python3.11/site-packages/torch/nn/modules/transformer.py", line 715, in forward
+ x = self.norm2(x + self._ff_block(x))
+ ^^^^^^^^^^^^^^^^^
+ File "/gpfslocalsup/pub/anaconda-py3/2023.09/envs/pytorch-gpu-2.2.0+py3.11.7/lib/python3.11/site-packages/torch/nn/modules/transformer.py", line 730, in _ff_block
+ x = self.linear2(self.dropout(self.activation(self.linear1(x))))
+ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+ File "/gpfslocalsup/pub/anaconda-py3/2023.09/envs/pytorch-gpu-2.2.0+py3.11.7/lib/python3.11/site-packages/torch/nn/functional.py", line 1473, in relu
+ result = torch.relu(input)
+ ^^^^^^^^^^^^^^^^^
+torch.cuda.OutOfMemoryError: CUDA out of memory. Tried to allocate 200.00 MiB. GPU 0 has a total capacity of 15.77 GiB of which 164.69 MiB is free. Including non-PyTorch memory, this process has 15.60 GiB memory in use. Of the allocated memory 15.18 GiB is allocated by PyTorch, and 51.15 MiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables)
diff --git a/ray_results/test_experiment/train_model_8c6e7040_14_batch_size=2048,decoder_int_ff=2048,decoder_int_num_layer=8,decoder_rt_ff=1024,decoder_rt_num_layer=8,drop_2024-06-25_16-41-29/events.out.tfevents.1719326572.r3i5n6 b/ray_results/test_experiment/train_model_8c6e7040_14_batch_size=2048,decoder_int_ff=2048,decoder_int_num_layer=8,decoder_rt_ff=1024,decoder_rt_num_layer=8,drop_2024-06-25_16-41-29/events.out.tfevents.1719326572.r3i5n6
new file mode 100644
index 0000000000000000000000000000000000000000..bb8b5d2f12702c6f37d63b73939f9db314e074bf
GIT binary patch
literal 40
rcmb1OfPlsI-b$S6%*?m9&b#R-#hX-=n3<>NT9%quVr4YF@%Tgl!1N4d
literal 0
HcmV?d00001
diff --git a/ray_results/test_experiment/train_model_8c6e7040_14_batch_size=2048,decoder_int_ff=2048,decoder_int_num_layer=8,decoder_rt_ff=1024,decoder_rt_num_layer=8,drop_2024-06-25_16-41-29/params.json b/ray_results/test_experiment/train_model_8c6e7040_14_batch_size=2048,decoder_int_ff=2048,decoder_int_num_layer=8,decoder_rt_ff=1024,decoder_rt_num_layer=8,drop_2024-06-25_16-41-29/params.json
new file mode 100644
index 0000000..8af95a6
--- /dev/null
+++ b/ray_results/test_experiment/train_model_8c6e7040_14_batch_size=2048,decoder_int_ff=2048,decoder_int_num_layer=8,decoder_rt_ff=1024,decoder_rt_num_layer=8,drop_2024-06-25_16-41-29/params.json
@@ -0,0 +1,13 @@
+{
+ "batch_size": 2048,
+ "decoder_int_ff": 2048,
+ "decoder_int_num_layer": 8,
+ "decoder_rt_ff": 1024,
+ "decoder_rt_num_layer": 8,
+ "drop_rate": 0.6537122299874158,
+ "embedding_dim": 256,
+ "encoder_ff": 2048,
+ "encoder_num_layer": 8,
+ "lr": 0.010515037912087563,
+ "n_head": 1
+}
\ No newline at end of file
diff --git a/ray_results/test_experiment/train_model_8c6e7040_14_batch_size=2048,decoder_int_ff=2048,decoder_int_num_layer=8,decoder_rt_ff=1024,decoder_rt_num_layer=8,drop_2024-06-25_16-41-29/params.pkl b/ray_results/test_experiment/train_model_8c6e7040_14_batch_size=2048,decoder_int_ff=2048,decoder_int_num_layer=8,decoder_rt_ff=1024,decoder_rt_num_layer=8,drop_2024-06-25_16-41-29/params.pkl
new file mode 100644
index 0000000000000000000000000000000000000000..b03d70fc1f346840504450aed94126874aed0347
GIT binary patch
literal 225
zcmZo*oqCM{0&1sd^a!TrCFiH47RBe4=EmnFR;CtB@#g3eNlAr>7L}k%h{7Z?^U!2?
zQ*)D2Q&KYX(&JMybEo(+F!pf4%uY)K@;Q2VVVZ%af`wRm_+UaH^PtMu^5Qd66H}&m
zGxl(%6y+Dh7bTXYPI0$?^4j#JWME7WQ%(_(*V^%-e%cX<9<HRslH`o|;>@a4kR_#h
E0QH_zg#Z8m
literal 0
HcmV?d00001
diff --git a/ray_results/test_experiment/train_model_8c6e7040_14_batch_size=2048,decoder_int_ff=2048,decoder_int_num_layer=8,decoder_rt_ff=1024,decoder_rt_num_layer=8,drop_2024-06-25_16-41-29/result.json b/ray_results/test_experiment/train_model_8c6e7040_14_batch_size=2048,decoder_int_ff=2048,decoder_int_num_layer=8,decoder_rt_ff=1024,decoder_rt_num_layer=8,drop_2024-06-25_16-41-29/result.json
new file mode 100644
index 0000000..e69de29
diff --git a/ray_results/test_experiment/train_model_8e58579c_18_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=8,decoder_rt_ff=2048,decoder_rt_num_layer=8,drop__2024-06-25_16-47-40/events.out.tfevents.1719326883.r3i5n6 b/ray_results/test_experiment/train_model_8e58579c_18_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=8,decoder_rt_ff=2048,decoder_rt_num_layer=8,drop__2024-06-25_16-47-40/events.out.tfevents.1719326883.r3i5n6
new file mode 100644
index 0000000000000000000000000000000000000000..7864aece5d8ff7dc3e8e074eb73881f1d778e557
GIT binary patch
literal 770
zcmb1OfPlsI-b$Q;?=ER<n|ISuiZ`h!F*8rkwJbHS#L8%6yrqXSRNaR8<<6Y441fGV
z>I4|I#JEJbc#9G%^-D_gQuTB4i;G2to-w#~Pf2$*fNGig^+G)7Nfx*kWiCZ7DY%y6
zjQrA^l=$R~)a2}f{LH)(Q3i$vd%=STAEQ_q&JNd)YGq1(UMf%v14GgD*~j#uR_4aa
z25^2uXi?%);F5$}S(2HX8efu;Ssb5Pl3EmBEUL6v#+lPJWvL-l``qT{mz<>>a2r*)
zl(?i(wHIgRC8x$0r52au7o~y?IOp!tZw@u!`HNGLoZk=zXmM$9DZ&i^*_v3AnV(l2
zk7^9a=aUX8&qwjOkVaoIIIx+p1@_4X2G=RB8c8UDt)U6mLQG)4ERj8lV&x`;ejI@v
zFH~lS64+K+a4QJ}wzNB+bLjfTttf%L8=;+GV9S?Ib$08FGe8M!b8Wc&L<cs5Lukb6
F5CB4U@>Bo-
literal 0
HcmV?d00001
diff --git a/ray_results/test_experiment/train_model_8e58579c_18_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=8,decoder_rt_ff=2048,decoder_rt_num_layer=8,drop__2024-06-25_16-47-40/params.json b/ray_results/test_experiment/train_model_8e58579c_18_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=8,decoder_rt_ff=2048,decoder_rt_num_layer=8,drop__2024-06-25_16-47-40/params.json
new file mode 100644
index 0000000..6c35680
--- /dev/null
+++ b/ray_results/test_experiment/train_model_8e58579c_18_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=8,decoder_rt_ff=2048,decoder_rt_num_layer=8,drop__2024-06-25_16-47-40/params.json
@@ -0,0 +1,13 @@
+{
+ "batch_size": 1024,
+ "decoder_int_ff": 512,
+ "decoder_int_num_layer": 8,
+ "decoder_rt_ff": 2048,
+ "decoder_rt_num_layer": 8,
+ "drop_rate": 0.8999111579906955,
+ "embedding_dim": 64,
+ "encoder_ff": 1024,
+ "encoder_num_layer": 8,
+ "lr": 0.000663247583015271,
+ "n_head": 8
+}
\ No newline at end of file
diff --git a/ray_results/test_experiment/train_model_8e58579c_18_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=8,decoder_rt_ff=2048,decoder_rt_num_layer=8,drop__2024-06-25_16-47-40/params.pkl b/ray_results/test_experiment/train_model_8e58579c_18_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=8,decoder_rt_ff=2048,decoder_rt_num_layer=8,drop__2024-06-25_16-47-40/params.pkl
new file mode 100644
index 0000000000000000000000000000000000000000..6bbd9fa4c9b60d515a5378a4c5267e0d057fe47c
GIT binary patch
literal 224
zcmZo*oqCl40&1sd^a!TrCFiH47RBe4=EmnFR;CtB@#g3eNlAr>7L}k%h{7Z?^U!2?
zQ*)D2Q&KYX(&JMybEkMa^l-t9PD`8O%fQmZ3)2fU6fDHi!v_-r83z_)>S4=^&qz&7
z0Xc&+r6|83z9_LIb&9+Fn=?W+GAsc-OgTkBp6l+NVdwpq_i!a8mLzAy7iU(bf-EW3
F0|3NTQm+62
literal 0
HcmV?d00001
diff --git a/ray_results/test_experiment/train_model_8e58579c_18_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=8,decoder_rt_ff=2048,decoder_rt_num_layer=8,drop__2024-06-25_16-47-40/progress.csv b/ray_results/test_experiment/train_model_8e58579c_18_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=8,decoder_rt_ff=2048,decoder_rt_num_layer=8,drop__2024-06-25_16-47-40/progress.csv
new file mode 100644
index 0000000..1b028f4
--- /dev/null
+++ b/ray_results/test_experiment/train_model_8e58579c_18_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=8,decoder_rt_ff=2048,decoder_rt_num_layer=8,drop__2024-06-25_16-47-40/progress.csv
@@ -0,0 +1,3 @@
+loss,timestamp,checkpoint_dir_name,should_checkpoint,done,training_iteration,trial_id,date,time_this_iter_s,time_total_s,pid,hostname,node_ip,time_since_restore,iterations_since_restore
+2062.3793243648497,1719327039,checkpoint_000000,True,False,1,8e58579c,2024-06-25_16-50-39,156.63724398612976,156.63724398612976,130020,r3i5n6,10.159.8.159,156.63724398612976,1
+2056.04917078694,1719327182,checkpoint_000001,True,False,2,8e58579c,2024-06-25_16-53-02,142.2777497768402,298.91499376296997,130020,r3i5n6,10.159.8.159,298.91499376296997,2
diff --git a/ray_results/test_experiment/train_model_8e58579c_18_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=8,decoder_rt_ff=2048,decoder_rt_num_layer=8,drop__2024-06-25_16-47-40/result.json b/ray_results/test_experiment/train_model_8e58579c_18_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=8,decoder_rt_ff=2048,decoder_rt_num_layer=8,drop__2024-06-25_16-47-40/result.json
new file mode 100644
index 0000000..f6ba986
--- /dev/null
+++ b/ray_results/test_experiment/train_model_8e58579c_18_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=8,decoder_rt_ff=2048,decoder_rt_num_layer=8,drop__2024-06-25_16-47-40/result.json
@@ -0,0 +1,2 @@
+{"loss": 2062.3793243648497, "timestamp": 1719327039, "checkpoint_dir_name": "checkpoint_000000", "should_checkpoint": true, "done": false, "training_iteration": 1, "trial_id": "8e58579c", "date": "2024-06-25_16-50-39", "time_this_iter_s": 156.63724398612976, "time_total_s": 156.63724398612976, "pid": 130020, "hostname": "r3i5n6", "node_ip": "10.159.8.159", "config": {"encoder_num_layer": 8, "decoder_rt_num_layer": 8, "decoder_int_num_layer": 8, "embedding_dim": 64, "encoder_ff": 1024, "decoder_rt_ff": 2048, "decoder_int_ff": 512, "n_head": 8, "drop_rate": 0.8999111579906955, "lr": 0.000663247583015271, "batch_size": 1024}, "time_since_restore": 156.63724398612976, "iterations_since_restore": 1}
+{"loss": 2056.04917078694, "timestamp": 1719327182, "checkpoint_dir_name": "checkpoint_000001", "should_checkpoint": true, "done": false, "training_iteration": 2, "trial_id": "8e58579c", "date": "2024-06-25_16-53-02", "time_this_iter_s": 142.2777497768402, "time_total_s": 298.91499376296997, "pid": 130020, "hostname": "r3i5n6", "node_ip": "10.159.8.159", "config": {"encoder_num_layer": 8, "decoder_rt_num_layer": 8, "decoder_int_num_layer": 8, "embedding_dim": 64, "encoder_ff": 1024, "decoder_rt_ff": 2048, "decoder_int_ff": 512, "n_head": 8, "drop_rate": 0.8999111579906955, "lr": 0.000663247583015271, "batch_size": 1024}, "time_since_restore": 298.91499376296997, "iterations_since_restore": 2}
diff --git a/ray_results/test_experiment/train_model_934c5104_9_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop_r_2024-06-25_17-32-25/events.out.tfevents.1719329610.r8i6n2 b/ray_results/test_experiment/train_model_934c5104_9_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop_r_2024-06-25_17-32-25/events.out.tfevents.1719329610.r8i6n2
new file mode 100644
index 0000000000000000000000000000000000000000..6286bd7b76e0ed6b0f2508218e6d19bd2fb5afbb
GIT binary patch
literal 405
zcmb1OfPlsI-b$R??DkyRG4H0M6mL>dVrHJ6YguYuiIvga67?I(P<0#TmpgNQyYBrQ
zq)vcQON>i|i?=ATQop1$FI7J$zqnZRpFo!LbB9Zs22d?izg~#v47v-~qRgepB?Z?~
zoRMFelM<htk(!)cke``XBFezfU?0TP?~G#Q#QSjls8*)r=cNL*Ffde3Ska&lwK6wW
zHh@#{0bGj`mjah0+{%*7+|>AzjLhQr%#zfi_+rs#l}ntavNSC*gleDL-29SrCPKRk
zmlBsWs`lc{yyVpQqSWG&{GwE_0U?26_2y6mp1(L1$;tB&Zod|n2A3k-0FbSTC7Jno
O#qp@dfPHTL=@0-TmWUDn
literal 0
HcmV?d00001
diff --git a/ray_results/test_experiment/train_model_934c5104_9_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop_r_2024-06-25_17-32-25/events.out.tfevents.1719330672.r8i6n2 b/ray_results/test_experiment/train_model_934c5104_9_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop_r_2024-06-25_17-32-25/events.out.tfevents.1719330672.r8i6n2
new file mode 100644
index 0000000000000000000000000000000000000000..0e757bb36b88857565d42e05f02c84edb9ff24d3
GIT binary patch
literal 405
zcmb1OfPlsI-b$Q~3hZ}w&b#R-#hX-=n3<>NT9%quVr69NmC2$ERkvY&xijaYs%alU
z>I9gy#JEJbc#9G%^-D_gQuTB4i;G38J?x#gGcmFnK($Q$dLf=Ot{$#MnM;vN3a+I%
zBfm5!B|bSLH95N=KQpgHl!2kazU8aKTofx0G{W_xTA7lcmkQLvz_9gc|1^E5mASF9
z0i1zNa4kw)3S5$KD@!tSQ{ziAGK=FgOHzyCi$x!oEpd7s{qKSyRQufK=9ipj5!zL_
zl(?i(wHIgRC8x$0r52au7o~y?;JK#%+#G7a^B1QgIUSne_G@uza4EtK0NI*Yl9``Z
P9FJ-Y$meN%({2I)U%rP=
literal 0
HcmV?d00001
diff --git a/ray_results/test_experiment/train_model_934c5104_9_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop_r_2024-06-25_17-32-25/events.out.tfevents.1719331282.r8i6n2 b/ray_results/test_experiment/train_model_934c5104_9_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop_r_2024-06-25_17-32-25/events.out.tfevents.1719331282.r8i6n2
new file mode 100644
index 0000000000000000000000000000000000000000..09c5868db57d6f4925f92cad890b331bfe74ebf1
GIT binary patch
literal 770
zcmb1OfPlsI-b$REvP(*K&AaI+#hX-=n3<>NT9%quVr6von)@|nsJadF%bhvz{4DDM
zsS{w<65|r#;w?(7)GsN`OV!WGFD@4S%%|pjtz%n>0aVM>uNUGucm0QJQRY(Ql7ed~
z&d4v#Nr_L+NKMWz$j{6x5oKU#uy4=t?nJTDm#G}Pl_~josX#3Z49)Ade%FUunHwt`
z!1)HDMTtv+OA>BnNoH<pd`U)TaeQV;YEgW#=*da@oc_tXY8gVc&uwmg$(hRxw^4;l
ziAx$)dvRu7a%y~0YH>+^Q7YJg+tO2hn?ntF{^C?5=Tn3MT3i}jif{u!wkDQj=I0g1
zqZ$M9`9z6rUr>Cm`}^qvaA31w3+#En<eeAWEtEhBY~Md{EyM(N{Vh)!6f18b^rOZv
zJg`3tKX*Y1?8LutD+vU)TIph^-ptFhPy+iVLOa30{=sbMtaEniB$UAR`v<q5=)h)h
J=x}d(3IMzP1mgez
literal 0
HcmV?d00001
diff --git a/ray_results/test_experiment/train_model_934c5104_9_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop_r_2024-06-25_17-32-25/events.out.tfevents.1719332071.r8i6n2 b/ray_results/test_experiment/train_model_934c5104_9_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop_r_2024-06-25_17-32-25/events.out.tfevents.1719332071.r8i6n2
new file mode 100644
index 0000000000000000000000000000000000000000..49e30115265afe258f1d2759701312484da2e83f
GIT binary patch
literal 4596
zcma*reM}Q)90%~y0&<TQXa`6OC=~IfF(6Bj$%ZXgSe#B=bZ<D9TcPJ#rPt<KRF<h2
zva~{s;B<>PCKHz}nNx5&<VCm0oPuu{hz_@jvdpo~Y&fUUF=f|Mp6idb-tiyz-1qtR
z`8*FKPva$HKaHE0Xo&b9Z!T#6us=^NFJ_GvCcR)C%{X(OoPT5Cl~mt*O?%fHh<_8d
z6nfr^4%5b=7$j$n<r&U0hR!IpIUKRi-<KQOk}ewO`69dS_E5xNGLEF86r{!xhuKzE
zYNATabjez~&B8ciC6WqV$dJknD)Uou9#)!c49!I(lFYy6wPpA!t*<A&K>WZ*G$<LV
zaHZ2?r75S`;-D-}nx!1Eu8aH0zW1AVW%}avm6g8|D`()w8AyW?VBBG0N@$9u9Znld
zdnedbDI3i8P4LT5cM;LcPoPD!Q3{^G(`s~DY>b0~(|A@cJMwoSSos%H7v;qc4}{&Z
z=67=b5IYB8M}*>t7;NRtu5F;Qj?V|dR^RzP6JV!@;YtDQBCVSo**vleU?1e;g4n}v
z#~B6+I{E<i+;H448e5|8>v^pctbDPt%<jdW6bM^&c92XvzhV`@-dKzyVz94PS6u>?
zVI?>p1iO=}d<3v-_=o`ZPP>adW=oR+>?tPPD2SbtwbC%<&ecYMUCB=%8e6Z=zcBMf
zu=1!=A8herM+CxFumj}G&*%v!`&%3lgB`D%%!0~lJ|6_T?AE{~fE{@nR|;U)W^W}k
ztLT>jb`>8N#NJ}wZrDdmO$XSc`~;%0EA-aoBWu9Q*B`6d;l(BbVIOU|M!p);p8`(y
z$E`Rb20P{5qk2#|yA9`qVAt86>H^r!d_(}-Snnc>ZiQ3=>^beYQ4qU-S+ikJL_-0<
zuIDEZjoqL>7INkQSh>mC)9A%Ufv{)1)K6Y`c98{OyBHi1gRME91^;Kq*>FAx_VE~F
zC%|stBLdh(BU{NomNnP_w#tqh1+kkh^cps>X<-0+2S0&m?636&^iudYUGsV05ij=S
zK-g!M=g8shOUnRu%Rw9wgWY|ywHEB}!b3P81p7gEbvwX5%|`^VYp1)(Xtv@jfK4{x
zMnUYIm+u;KUHdKr>=XP1qOq^*&+k#hft6dkub%Q^PYHxw-+YFw3SIFDz;>U-5i!_y
z)#y*4GU^=82f?;p$({tT>-dNO_O5g{`P@VEI)JS>j~fNC6K_w>U#pnY46t4N1fsDe
zdGq9TgJ9*=s*%fHY<VE;<K#JVR>B)3z;09Hh#2hOT3c>{%G?B;4}yKIwyOzXck&Sd
z?0-hxr0$+27hsc#xKS8ek(5t7JWvj>+xZDZV@Kz;ewiu-E5Cg$bjXVx83_CSrf<m5
zpG$23d)9Rv5rbX0=jrL7@-Uwdf~||ayAfdL-N2Os*h5R)<l*-3_5$n<J}!v8Tb-UC
z*3mm3U}q2Fe$m*;d2>(obTx&HU5;w%cdsEvE=GH?wfkl0BN>XJ%yuJdv^uB^dV`&2
zxeFTOT<n&icVuXiZ%Fw-s=O!FL?Tpdbe5R8tJ7O_vP`2yktVvtX5ub#+~tv4y_$Qb
zLb3kOm@+F>YAmN&Zb*UT<3p_P?RXU9_sIWcIU-D~%}#N*HBS|nrLD!Z$z);Pq)Zko
z_lWy6G&9!jZH`5X@!$O?3P;jXPdh^43}vQ`rev8`fufY~=B%BD5|xj+8Bd6KZob+i
zl%NcLv(-*VNy^9FT(xN^RVjQk%~}-q9^>1;lYj2ZA@X{B`!^^>15%X8@1KKqIJm1V
z<!~A~U<<R_w)lh;wM+SZJS0^gkfJz0TP30BODtY8`sk7lJuyziJsdkQ$$F%ouxog%
zMvtZiAjjjwI+QT+fZ)LjJ@-vSz`e(>Nc1Smvv=Hsd%anzL!aj9JRb%e1N^dGkEZ&c
z2RzW=9pI0x|9~dQS;g-R>k3sl<<I!OSvb?uxSV)pTt5?KAoc(4X~r}r&z&*eJyau&
K2r=HyUGN_WMAj_;
literal 0
HcmV?d00001
diff --git a/ray_results/test_experiment/train_model_934c5104_9_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop_r_2024-06-25_17-32-25/params.json b/ray_results/test_experiment/train_model_934c5104_9_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop_r_2024-06-25_17-32-25/params.json
new file mode 100644
index 0000000..74d001d
--- /dev/null
+++ b/ray_results/test_experiment/train_model_934c5104_9_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop_r_2024-06-25_17-32-25/params.json
@@ -0,0 +1,13 @@
+{
+ "batch_size": 2048,
+ "decoder_int_ff": 512,
+ "decoder_int_num_layer": 1,
+ "decoder_rt_ff": 1024,
+ "decoder_rt_num_layer": 2,
+ "drop_rate": 0.2811887966312956,
+ "embedding_dim": 256,
+ "encoder_ff": 2048,
+ "encoder_num_layer": 1,
+ "lr": 0.0005389075253520159,
+ "n_head": 8
+}
\ No newline at end of file
diff --git a/ray_results/test_experiment/train_model_934c5104_9_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop_r_2024-06-25_17-32-25/params.pkl b/ray_results/test_experiment/train_model_934c5104_9_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop_r_2024-06-25_17-32-25/params.pkl
new file mode 100644
index 0000000000000000000000000000000000000000..bf09f8838ddc89a7bb42c709c4600a089ff8a403
GIT binary patch
literal 225
zcmZo*oqCM{0&1sd^a!TrCFiH47RBe4=EmnFR;CtB@n-B1NlAr>7L_1NF!hMSBr@|*
zWEgvRQ*)D2Q&KYX(&JMybEo(+F!pf4%uY)K@;Q2VVVZ%af`wRm_+UaH^T1+EJ#2aL
z8L5dWQ@lBPI8%!93*w6sOH!w}+h6?m-#ba>N)J;`5s>G&V!fp4<!?P)Nr@%N8S%xL
KRjD9LO7#HxFH`dX
literal 0
HcmV?d00001
diff --git a/ray_results/test_experiment/train_model_934c5104_9_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop_r_2024-06-25_17-32-25/progress.csv b/ray_results/test_experiment/train_model_934c5104_9_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop_r_2024-06-25_17-32-25/progress.csv
new file mode 100644
index 0000000..1935394
--- /dev/null
+++ b/ray_results/test_experiment/train_model_934c5104_9_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop_r_2024-06-25_17-32-25/progress.csv
@@ -0,0 +1,15 @@
+loss,timestamp,checkpoint_dir_name,should_checkpoint,done,training_iteration,trial_id,date,time_this_iter_s,time_total_s,pid,hostname,node_ip,time_since_restore,iterations_since_restore
+234.06637176753966,1719329693,checkpoint_000000,True,False,1,934c5104,2024-06-25_17-34-53,82.2380793094635,82.2380793094635,2195253,r8i6n2,10.159.28.60,82.2380793094635,1
+191.28313151682457,1719330754,checkpoint_000001,True,False,2,934c5104,2024-06-25_17-52-34,82.23219752311707,164.47027683258057,2195253,r8i6n2,10.159.28.60,82.23219752311707,1
+166.0584006722518,1719331377,checkpoint_000002,True,False,3,934c5104,2024-06-25_18-02-57,95.2866895198822,259.75696635246277,2195253,r8i6n2,10.159.28.60,95.2866895198822,1
+159.9789658794253,1719331459,checkpoint_000003,True,False,4,934c5104,2024-06-25_18-04-19,81.72880601882935,341.4857723712921,2195253,r8i6n2,10.159.28.60,177.01549553871155,2
+141.96990696464,1719332153,checkpoint_000004,True,False,5,934c5104,2024-06-25_18-15-54,82.40532422065735,423.89109659194946,2195253,r8i6n2,10.159.28.60,82.40532422065735,1
+117.70846791905682,1719332222,checkpoint_000005,True,False,6,934c5104,2024-06-25_18-17-02,68.08270859718323,491.9738051891327,2195253,r8i6n2,10.159.28.60,150.48803281784058,2
+113.39667721245233,1719332289,checkpoint_000006,True,False,7,934c5104,2024-06-25_18-18-09,67.7198805809021,559.6936857700348,2195253,r8i6n2,10.159.28.60,218.20791339874268,3
+111.7227829460084,1719332358,checkpoint_000007,True,False,8,934c5104,2024-06-25_18-19-18,68.60852479934692,628.3022105693817,2195253,r8i6n2,10.159.28.60,286.8164381980896,4
+111.3588055588129,1719332426,checkpoint_000008,True,False,9,934c5104,2024-06-25_18-20-26,67.80738162994385,696.1095921993256,2195253,r8i6n2,10.159.28.60,354.62381982803345,5
+108.66629178505244,1719332494,checkpoint_000009,True,False,10,934c5104,2024-06-25_18-21-35,68.99091339111328,765.1005055904388,2195253,r8i6n2,10.159.28.60,423.61473321914673,6
+102.03670796071451,1719332563,checkpoint_000010,True,False,11,934c5104,2024-06-25_18-22-43,68.06165027618408,833.1621558666229,2195253,r8i6n2,10.159.28.60,491.6763834953308,7
+100.34690394364004,1719332631,checkpoint_000011,True,False,12,934c5104,2024-06-25_18-23-51,68.09097814559937,901.2531340122223,2195253,r8i6n2,10.159.28.60,559.7673616409302,8
+102.1303488663801,1719332699,checkpoint_000012,True,False,13,934c5104,2024-06-25_18-24-59,68.49995398521423,969.7530879974365,2195253,r8i6n2,10.159.28.60,628.2673156261444,9
+98.24798722154512,1719332767,checkpoint_000013,True,False,14,934c5104,2024-06-25_18-26-08,68.14825201034546,1037.901340007782,2195253,r8i6n2,10.159.28.60,696.4155676364899,10
diff --git a/ray_results/test_experiment/train_model_934c5104_9_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop_r_2024-06-25_17-32-25/result.json b/ray_results/test_experiment/train_model_934c5104_9_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop_r_2024-06-25_17-32-25/result.json
new file mode 100644
index 0000000..dba1737
--- /dev/null
+++ b/ray_results/test_experiment/train_model_934c5104_9_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop_r_2024-06-25_17-32-25/result.json
@@ -0,0 +1,14 @@
+{"loss": 234.06637176753966, "timestamp": 1719329693, "checkpoint_dir_name": "checkpoint_000000", "should_checkpoint": true, "done": false, "training_iteration": 1, "trial_id": "934c5104", "date": "2024-06-25_17-34-53", "time_this_iter_s": 82.2380793094635, "time_total_s": 82.2380793094635, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 1, "decoder_rt_num_layer": 2, "decoder_int_num_layer": 1, "embedding_dim": 256, "encoder_ff": 2048, "decoder_rt_ff": 1024, "decoder_int_ff": 512, "n_head": 8, "drop_rate": 0.2811887966312956, "lr": 0.0005389075253520159, "batch_size": 2048}, "time_since_restore": 82.2380793094635, "iterations_since_restore": 1}
+{"loss": 191.28313151682457, "timestamp": 1719330754, "checkpoint_dir_name": "checkpoint_000001", "should_checkpoint": true, "done": false, "training_iteration": 2, "trial_id": "934c5104", "date": "2024-06-25_17-52-34", "time_this_iter_s": 82.23219752311707, "time_total_s": 164.47027683258057, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 1, "decoder_rt_num_layer": 2, "decoder_int_num_layer": 1, "embedding_dim": 256, "encoder_ff": 2048, "decoder_rt_ff": 1024, "decoder_int_ff": 512, "n_head": 8, "drop_rate": 0.2811887966312956, "lr": 0.0005389075253520159, "batch_size": 2048}, "time_since_restore": 82.23219752311707, "iterations_since_restore": 1}
+{"loss": 166.0584006722518, "timestamp": 1719331377, "checkpoint_dir_name": "checkpoint_000002", "should_checkpoint": true, "done": false, "training_iteration": 3, "trial_id": "934c5104", "date": "2024-06-25_18-02-57", "time_this_iter_s": 95.2866895198822, "time_total_s": 259.75696635246277, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 1, "decoder_rt_num_layer": 2, "decoder_int_num_layer": 1, "embedding_dim": 256, "encoder_ff": 2048, "decoder_rt_ff": 1024, "decoder_int_ff": 512, "n_head": 8, "drop_rate": 0.2811887966312956, "lr": 0.0005389075253520159, "batch_size": 2048}, "time_since_restore": 95.2866895198822, "iterations_since_restore": 1}
+{"loss": 159.9789658794253, "timestamp": 1719331459, "checkpoint_dir_name": "checkpoint_000003", "should_checkpoint": true, "done": false, "training_iteration": 4, "trial_id": "934c5104", "date": "2024-06-25_18-04-19", "time_this_iter_s": 81.72880601882935, "time_total_s": 341.4857723712921, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 1, "decoder_rt_num_layer": 2, "decoder_int_num_layer": 1, "embedding_dim": 256, "encoder_ff": 2048, "decoder_rt_ff": 1024, "decoder_int_ff": 512, "n_head": 8, "drop_rate": 0.2811887966312956, "lr": 0.0005389075253520159, "batch_size": 2048}, "time_since_restore": 177.01549553871155, "iterations_since_restore": 2}
+{"loss": 141.96990696464, "timestamp": 1719332153, "checkpoint_dir_name": "checkpoint_000004", "should_checkpoint": true, "done": false, "training_iteration": 5, "trial_id": "934c5104", "date": "2024-06-25_18-15-54", "time_this_iter_s": 82.40532422065735, "time_total_s": 423.89109659194946, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 1, "decoder_rt_num_layer": 2, "decoder_int_num_layer": 1, "embedding_dim": 256, "encoder_ff": 2048, "decoder_rt_ff": 1024, "decoder_int_ff": 512, "n_head": 8, "drop_rate": 0.2811887966312956, "lr": 0.0005389075253520159, "batch_size": 2048}, "time_since_restore": 82.40532422065735, "iterations_since_restore": 1}
+{"loss": 117.70846791905682, "timestamp": 1719332222, "checkpoint_dir_name": "checkpoint_000005", "should_checkpoint": true, "done": false, "training_iteration": 6, "trial_id": "934c5104", "date": "2024-06-25_18-17-02", "time_this_iter_s": 68.08270859718323, "time_total_s": 491.9738051891327, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 1, "decoder_rt_num_layer": 2, "decoder_int_num_layer": 1, "embedding_dim": 256, "encoder_ff": 2048, "decoder_rt_ff": 1024, "decoder_int_ff": 512, "n_head": 8, "drop_rate": 0.2811887966312956, "lr": 0.0005389075253520159, "batch_size": 2048}, "time_since_restore": 150.48803281784058, "iterations_since_restore": 2}
+{"loss": 113.39667721245233, "timestamp": 1719332289, "checkpoint_dir_name": "checkpoint_000006", "should_checkpoint": true, "done": false, "training_iteration": 7, "trial_id": "934c5104", "date": "2024-06-25_18-18-09", "time_this_iter_s": 67.7198805809021, "time_total_s": 559.6936857700348, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 1, "decoder_rt_num_layer": 2, "decoder_int_num_layer": 1, "embedding_dim": 256, "encoder_ff": 2048, "decoder_rt_ff": 1024, "decoder_int_ff": 512, "n_head": 8, "drop_rate": 0.2811887966312956, "lr": 0.0005389075253520159, "batch_size": 2048}, "time_since_restore": 218.20791339874268, "iterations_since_restore": 3}
+{"loss": 111.7227829460084, "timestamp": 1719332358, "checkpoint_dir_name": "checkpoint_000007", "should_checkpoint": true, "done": false, "training_iteration": 8, "trial_id": "934c5104", "date": "2024-06-25_18-19-18", "time_this_iter_s": 68.60852479934692, "time_total_s": 628.3022105693817, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 1, "decoder_rt_num_layer": 2, "decoder_int_num_layer": 1, "embedding_dim": 256, "encoder_ff": 2048, "decoder_rt_ff": 1024, "decoder_int_ff": 512, "n_head": 8, "drop_rate": 0.2811887966312956, "lr": 0.0005389075253520159, "batch_size": 2048}, "time_since_restore": 286.8164381980896, "iterations_since_restore": 4}
+{"loss": 111.3588055588129, "timestamp": 1719332426, "checkpoint_dir_name": "checkpoint_000008", "should_checkpoint": true, "done": false, "training_iteration": 9, "trial_id": "934c5104", "date": "2024-06-25_18-20-26", "time_this_iter_s": 67.80738162994385, "time_total_s": 696.1095921993256, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 1, "decoder_rt_num_layer": 2, "decoder_int_num_layer": 1, "embedding_dim": 256, "encoder_ff": 2048, "decoder_rt_ff": 1024, "decoder_int_ff": 512, "n_head": 8, "drop_rate": 0.2811887966312956, "lr": 0.0005389075253520159, "batch_size": 2048}, "time_since_restore": 354.62381982803345, "iterations_since_restore": 5}
+{"loss": 108.66629178505244, "timestamp": 1719332494, "checkpoint_dir_name": "checkpoint_000009", "should_checkpoint": true, "done": false, "training_iteration": 10, "trial_id": "934c5104", "date": "2024-06-25_18-21-35", "time_this_iter_s": 68.99091339111328, "time_total_s": 765.1005055904388, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 1, "decoder_rt_num_layer": 2, "decoder_int_num_layer": 1, "embedding_dim": 256, "encoder_ff": 2048, "decoder_rt_ff": 1024, "decoder_int_ff": 512, "n_head": 8, "drop_rate": 0.2811887966312956, "lr": 0.0005389075253520159, "batch_size": 2048}, "time_since_restore": 423.61473321914673, "iterations_since_restore": 6}
+{"loss": 102.03670796071451, "timestamp": 1719332563, "checkpoint_dir_name": "checkpoint_000010", "should_checkpoint": true, "done": false, "training_iteration": 11, "trial_id": "934c5104", "date": "2024-06-25_18-22-43", "time_this_iter_s": 68.06165027618408, "time_total_s": 833.1621558666229, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 1, "decoder_rt_num_layer": 2, "decoder_int_num_layer": 1, "embedding_dim": 256, "encoder_ff": 2048, "decoder_rt_ff": 1024, "decoder_int_ff": 512, "n_head": 8, "drop_rate": 0.2811887966312956, "lr": 0.0005389075253520159, "batch_size": 2048}, "time_since_restore": 491.6763834953308, "iterations_since_restore": 7}
+{"loss": 100.34690394364004, "timestamp": 1719332631, "checkpoint_dir_name": "checkpoint_000011", "should_checkpoint": true, "done": false, "training_iteration": 12, "trial_id": "934c5104", "date": "2024-06-25_18-23-51", "time_this_iter_s": 68.09097814559937, "time_total_s": 901.2531340122223, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 1, "decoder_rt_num_layer": 2, "decoder_int_num_layer": 1, "embedding_dim": 256, "encoder_ff": 2048, "decoder_rt_ff": 1024, "decoder_int_ff": 512, "n_head": 8, "drop_rate": 0.2811887966312956, "lr": 0.0005389075253520159, "batch_size": 2048}, "time_since_restore": 559.7673616409302, "iterations_since_restore": 8}
+{"loss": 102.1303488663801, "timestamp": 1719332699, "checkpoint_dir_name": "checkpoint_000012", "should_checkpoint": true, "done": false, "training_iteration": 13, "trial_id": "934c5104", "date": "2024-06-25_18-24-59", "time_this_iter_s": 68.49995398521423, "time_total_s": 969.7530879974365, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 1, "decoder_rt_num_layer": 2, "decoder_int_num_layer": 1, "embedding_dim": 256, "encoder_ff": 2048, "decoder_rt_ff": 1024, "decoder_int_ff": 512, "n_head": 8, "drop_rate": 0.2811887966312956, "lr": 0.0005389075253520159, "batch_size": 2048}, "time_since_restore": 628.2673156261444, "iterations_since_restore": 9}
+{"loss": 98.24798722154512, "timestamp": 1719332767, "checkpoint_dir_name": "checkpoint_000013", "should_checkpoint": true, "done": false, "training_iteration": 14, "trial_id": "934c5104", "date": "2024-06-25_18-26-08", "time_this_iter_s": 68.14825201034546, "time_total_s": 1037.901340007782, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 1, "decoder_rt_num_layer": 2, "decoder_int_num_layer": 1, "embedding_dim": 256, "encoder_ff": 2048, "decoder_rt_ff": 1024, "decoder_int_ff": 512, "n_head": 8, "drop_rate": 0.2811887966312956, "lr": 0.0005389075253520159, "batch_size": 2048}, "time_since_restore": 696.4155676364899, "iterations_since_restore": 10}
diff --git a/ray_results/test_experiment/train_model_975aa6a0_4_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=2,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop_r_2024-06-25_16-11-03/events.out.tfevents.1719324728.r3i5n6 b/ray_results/test_experiment/train_model_975aa6a0_4_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=2,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop_r_2024-06-25_16-11-03/events.out.tfevents.1719324728.r3i5n6
new file mode 100644
index 0000000000000000000000000000000000000000..751ead7bdc6e653c4adc03dec7f4ad59e03acae9
GIT binary patch
literal 1311
zcmah|OH30{6fIMO%uNb*O8K;aP(;lnT5#b9O*(!fXbfRt;zD1B&clwjZ<={i5H~g<
z8l#DUg$qqU5!|Q=EQ}_aVB$(#v2dXaV>D60L~w&2;=(s{`r0Dm?B<?(?s@Or^Qv5#
zZ}h^Eka!hpN*@lt*&Fb5nW|2<9_+!yYTY)tyW~=%(>HOecfU9nZ>qEV))mPWP!1kb
z?F(Beg2QoxQvcWEiw7!SeVS@^DAyOJkyu7Kr4bq+z$r9lq~aQi#&Gm(($I<Jce&CL
zWoodqK=Nja^9v<4gJ4E+xpr;3vNP;RCQjCNh+BF%B?R?Q%_S{8fsqx{Dbg)$BI-{Z
zPVXC;d3LYGp%3-<&x-eR^bHV#U?H9AB#M!Vsb!dWX~3!JHs20sz~`BlC&Z>+t}nx8
zXy5~ERn^iBLQ&xu`{siSlVcM_nK=VPH_nJZ=00wG#LSk*+%V#XauiFdrkbEAjL#*p
z$?ig|_RDU#=!UfpB>fVk??R{)pi8x)F{W}J*SkX!lxjF?Xzb3+XVl%zYSrM+uaQ&&
z#nnD+GRO;_EMz*}RZyN+%5V07sF_9*v5VPKjEfUpSkrWJ8fkihmDp)8$*3<W`@x(2
z&mU9_LflpdC?+U|RjuAFd!bA!44GU9HPR{}T1K%$BG*Dt`gaKA^-wFVwq&^p8m0d)
z@$Go~d*bZx`TU($ar?ZpzYhYu6QCxyKZ+@3DL~XxnZY{gHrgizcq-)fI4%Sp2vCvd
ztt5Ke7LU#jSAU6sFKZDiW&+c!Kw#O{I5wa_-3pd-{96Qq%LBr}w89?kio546j9+H&
zjuq0^em}W;C>XI1QxF5*+@U~aejYf$PRW2=Y&mdSVUKX-tnwb0@-ytK*yB9H7mr6f
g#ba6bTObU9zrDvq%Q(+6mh*j0NGK_KEiF#}0ZYY^AOHXW
literal 0
HcmV?d00001
diff --git a/ray_results/test_experiment/train_model_975aa6a0_4_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=2,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop_r_2024-06-25_16-11-03/params.json b/ray_results/test_experiment/train_model_975aa6a0_4_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=2,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop_r_2024-06-25_16-11-03/params.json
new file mode 100644
index 0000000..a21fd66
--- /dev/null
+++ b/ray_results/test_experiment/train_model_975aa6a0_4_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=2,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop_r_2024-06-25_16-11-03/params.json
@@ -0,0 +1,13 @@
+{
+ "batch_size": 2048,
+ "decoder_int_ff": 512,
+ "decoder_int_num_layer": 2,
+ "decoder_rt_ff": 1024,
+ "decoder_rt_num_layer": 1,
+ "drop_rate": 0.8156275326399022,
+ "embedding_dim": 64,
+ "encoder_ff": 1024,
+ "encoder_num_layer": 1,
+ "lr": 0.00010404955423531405,
+ "n_head": 16
+}
\ No newline at end of file
diff --git a/ray_results/test_experiment/train_model_975aa6a0_4_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=2,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop_r_2024-06-25_16-11-03/params.pkl b/ray_results/test_experiment/train_model_975aa6a0_4_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=2,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop_r_2024-06-25_16-11-03/params.pkl
new file mode 100644
index 0000000000000000000000000000000000000000..e196681f5f7a5e368943bf3fb54a03fefb5f87e9
GIT binary patch
literal 224
zcmZo*oqCl40&1sd^a!TrCFiH47RBe4=EmnFR;CtB@n-B1NlAr>7L}k%h{7Z?^H5}%
zdU#WFlTuSsGV{{oQ!;a>csulP!HiBzo8rsB(!&eW3p5le#0L`s83z_)>S4=^&qz&7
znc^+b!<kZ)Ul3oESdu!$-Tsy2yq7+|JbRdOihw+6x23!O^fmNwB_)<5XT%q0R;2<h
I;V9Ju0OV0q`~Uy|
literal 0
HcmV?d00001
diff --git a/ray_results/test_experiment/train_model_975aa6a0_4_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=2,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop_r_2024-06-25_16-11-03/progress.csv b/ray_results/test_experiment/train_model_975aa6a0_4_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=2,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop_r_2024-06-25_16-11-03/progress.csv
new file mode 100644
index 0000000..cc55625
--- /dev/null
+++ b/ray_results/test_experiment/train_model_975aa6a0_4_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=2,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop_r_2024-06-25_16-11-03/progress.csv
@@ -0,0 +1,2 @@
+loss,timestamp,checkpoint_dir_name,should_checkpoint,done,training_iteration,trial_id,date,time_this_iter_s,time_total_s,pid,hostname,node_ip,time_since_restore,iterations_since_restore
+2002.7163720318651,1719324792,checkpoint_000000,True,False,1,975aa6a0,2024-06-25_16-13-12,64.14145565032959,64.14145565032959,69318,r3i5n6,10.159.8.159,64.14145565032959,1
diff --git a/ray_results/test_experiment/train_model_975aa6a0_4_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=2,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop_r_2024-06-25_16-11-03/result.json b/ray_results/test_experiment/train_model_975aa6a0_4_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=2,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop_r_2024-06-25_16-11-03/result.json
new file mode 100644
index 0000000..28d61e9
--- /dev/null
+++ b/ray_results/test_experiment/train_model_975aa6a0_4_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=2,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop_r_2024-06-25_16-11-03/result.json
@@ -0,0 +1 @@
+{"loss": 2002.7163720318651, "timestamp": 1719324792, "checkpoint_dir_name": "checkpoint_000000", "should_checkpoint": true, "done": false, "training_iteration": 1, "trial_id": "975aa6a0", "date": "2024-06-25_16-13-12", "time_this_iter_s": 64.14145565032959, "time_total_s": 64.14145565032959, "pid": 69318, "hostname": "r3i5n6", "node_ip": "10.159.8.159", "config": {"encoder_num_layer": 1, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 2, "embedding_dim": 64, "encoder_ff": 1024, "decoder_rt_ff": 1024, "decoder_int_ff": 512, "n_head": 16, "drop_rate": 0.8156275326399022, "lr": 0.00010404955423531405, "batch_size": 2048}, "time_since_restore": 64.14145565032959, "iterations_since_restore": 1}
diff --git a/ray_results/test_experiment/train_model_a00d3259_2_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=4,drop_ra_2024-06-25_17-10-13/events.out.tfevents.1719328456.r8i6n2 b/ray_results/test_experiment/train_model_a00d3259_2_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=4,drop_ra_2024-06-25_17-10-13/events.out.tfevents.1719328456.r8i6n2
new file mode 100644
index 0000000000000000000000000000000000000000..4cc40b099c6d264f4885bfa7e1e600d9fbd88405
GIT binary patch
literal 405
zcmb1OfPlsI-b$Rkn|U{FpLf$yiZ`h!F*8rkwJbHS#L7thVwRdRRNaR8<<6W{E^`lq
z)Cn+ZiE)W=@fIak>X($}rRwM87Z;1ZzFg}(VT#*I1E`j%UoXUSTKmDZD03-tNx`)g
zXXKaWq{Js@q$Xz<<Y(rUh%zuV*ys2vpFpwlSs+|Ls+B4Ed8t4x3=Efx+9mX%R_4aa
z25@!<!L=xHDR4=`tt`pRO^q+f$SjV}EJ-bjFBX-m>T!D9^>wEqRQufK=9irO!Eo&=
zTuNNhsM?D&^O95Ji&BeA@{3Zz1}MDV{lgq;!1EWUA~`1_4AA1z;8KJe0J1f)Br`v+
PI3Cp)kk5_xDqjKsl5U1Y
literal 0
HcmV?d00001
diff --git a/ray_results/test_experiment/train_model_a00d3259_2_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=4,drop_ra_2024-06-25_17-10-13/events.out.tfevents.1719330517.r8i6n2 b/ray_results/test_experiment/train_model_a00d3259_2_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=4,drop_ra_2024-06-25_17-10-13/events.out.tfevents.1719330517.r8i6n2
new file mode 100644
index 0000000000000000000000000000000000000000..058684f608701ed90d6d293a6424317a3ea4cda4
GIT binary patch
literal 405
zcmb1OfPlsI-b$Q|-342B&b#R-#hX-=n3<>NT9%quVrBG9-s+SxRNaR8<<6YT<F6hA
zsS{w*65|r#;w?(7)GsN`OV!WGFD@3nw^74+S;5Hw1E`j%UoXUS{!N8zQRY(Ql7ed~
z&d4v#Nr_L+NKMWz$j{6x5oKU#u-7+J5J9nWNd{a$s+B4Ed8t4x3=9zwUDNcTR_4aa
z25?$u!nG)IDR4=`tt`pRO^q+f$SjV}EJ-bjFBUzl)a?{E|BIF(RQufK=9ipX5!zL_
zl(?i(wHIgRC8x$0r52au7o~y?*e^V5fjQKG=Pyo0a%yJ5?bqVc;8KJe0J1f)Br`v+
PI3Cp)kk7R$7A^z;OQeQ;
literal 0
HcmV?d00001
diff --git a/ray_results/test_experiment/train_model_a00d3259_2_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=4,drop_ra_2024-06-25_17-10-13/events.out.tfevents.1719331131.r8i6n2 b/ray_results/test_experiment/train_model_a00d3259_2_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=4,drop_ra_2024-06-25_17-10-13/events.out.tfevents.1719331131.r8i6n2
new file mode 100644
index 0000000000000000000000000000000000000000..7cf1b5ad836eef601320c037e2736229dd488702
GIT binary patch
literal 770
zcmb1OfPlsI-b$SO)1Uk8ns?JtiZ`h!F*8rkwJbHS#LDQ757SL$sJadF%bhtZ*M3R?
zsS{w<65|r#;w?(7)GsN`OV!WGFD@1}zAW!N<?-<z1E`j%UoXUSI&6Y#QRY(Ql7ed~
z&d4v#Nr_L+NKMWz$j{6x5oKU#us4ohB#&Zc=N7nrR4Y^R^HPCY7#QL$Pae>RTA3Rw
z8^CF>6|O~zOMy!gZe>YkZfbl<MrLt*W=U#Me6i@YkPS{xXLyJjLbcCrZhpzR8KGT;
zONmPwReNz}UUF)DQEG8Xeo-pe0Aq$^OLM3J&tII1<kZ;)w_l4(gG&)^0La$FlFa<P
z;&@bJKt5;tf5jig=NBK%ECB~L3%0;^TFT?hDz0LK64-Aa!?h3-*uO8>&Ox!V;t5<o
zj==s>y7V?mU^702YatNWetA_+>xG$)paga;LOa30{&!T&naOT>21;OmL>NGHU^6&W
IbCyT}0D0*CcK`qY
literal 0
HcmV?d00001
diff --git a/ray_results/test_experiment/train_model_a00d3259_2_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=4,drop_ra_2024-06-25_17-10-13/events.out.tfevents.1719331824.r8i6n2 b/ray_results/test_experiment/train_model_a00d3259_2_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=4,drop_ra_2024-06-25_17-10-13/events.out.tfevents.1719331824.r8i6n2
new file mode 100644
index 0000000000000000000000000000000000000000..2992aec175590f0f1e210addbbd192505204f9b3
GIT binary patch
literal 2406
zcma)-e`phD7{}wqRqq+8&9$jEZEc(G7g3s2u+v&dnwwH9vZ8FDUAbLzZ+kbByK;B!
z+6l8mWd0%2jqMMiPFx3y7A$UK)57Lt4iUk2jBTwGvHz&UsU385BIujs`d;3enB>2_
z&+~o1_kNy-=UL@Q{YLg}^Rd58S3f@g#Z<G`)v1e$TDQ3msb=H4(7VAswN~EPvG;lQ
z-sHN%ahzB5g1;0>z@>`=0kc;{ftY3(9;c&}AFl713tE!T=OzSJ{GCW@p$5D}V#r!=
zOcJ6pitg#r6xH-N9D||ZHNig0^5maHPg_cwim=4tI5GOl@qlF+@2G5Jo2Q7x2h~tU
zEKMbj1XETFK{1go7@np>Ly_3twM@MwKe~VaUG^ti{u210oR%Ak8byMR3{%rl@&YOP
zckp%Vg4=hl>|!@e6MsLv1~ue@gsW&OnraC2HHnvZpKIJiz5LFu3l&M&1({&~@`T}6
z9{G7Y1v}JEB>BMZ`0&*yl;tH`p9S`{hN%_`_Re>RWe(WxYjR|F-HTT!*ps$AC+wL4
zhX456{>v2X)(+yIH*80^Xfz;FFRyBTb4?QVvP`h2|Co;q&2IUcg1sq1B>BMpq~X&!
z%JPD(&jNdPR``s9y_qMLIbiE|<w)1rgUuA|i?%!`?9ew&{M$>fuBTvcc!T)o4Lckz
z?*I2M>gDa<-`JRhy*v}_y`A?Xrw+6<P_T`2M3N8ezSv0@WjSx_v%r?CZqZ|8-+5x0
z1NOEVInsV?eujem$d>1X{p-CxzIf#E1O;0iC;oZE9t_uA;{O?2k{U-tM~}S6PM;e7
zC;{95l@pFTp+t~-L|u#<LICymARUiH)oeZJgab}kW`(%_8SX#ER|wE4no$`Ks}E4M
z)5k%fgrb^+M<*WFLRS|)D+5pZnc5o{V&VYOamWp>g^+G#uY!`aqx57Ku#&Fz2zWjw
ztgsfvJCP(QYPTRMaeRco2B|6cq_YRy3;)wEDgY*ya0e(*1sRD_wbSp06&$^*{HvjY
zd(JLeBuVTVe<hT2*>{P*3M#qh?F#?PP|M}s1p_B0bk(Zg#Rp$DvKvoZ^_yYfCIc0A
z{S0Inc$o->DPq8i+NHJrz`!_T*YOPF9c7?2jV;HzTjs8hZ@qps1jP%8@L{T9io)Ps
zR5cPC48!US$VvQR2+9{X2nk|wP3GE@6zshsQ9FFl7U9QEAAT#8_!z1UNOCX?E7SXd
z1cs6UyKU10zAU{;avp{zZ}8ns)-J#N-N+8MZGnD01i<?&d!$M!^0>!r?n`~l@+Ci(
H?G8Kzxn1?c
literal 0
HcmV?d00001
diff --git a/ray_results/test_experiment/train_model_a00d3259_2_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=4,drop_ra_2024-06-25_17-10-13/params.json b/ray_results/test_experiment/train_model_a00d3259_2_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=4,drop_ra_2024-06-25_17-10-13/params.json
new file mode 100644
index 0000000..9155b49
--- /dev/null
+++ b/ray_results/test_experiment/train_model_a00d3259_2_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=4,drop_ra_2024-06-25_17-10-13/params.json
@@ -0,0 +1,13 @@
+{
+ "batch_size": 1024,
+ "decoder_int_ff": 512,
+ "decoder_int_num_layer": 1,
+ "decoder_rt_ff": 512,
+ "decoder_rt_num_layer": 4,
+ "drop_rate": 0.22544213936627444,
+ "embedding_dim": 16,
+ "encoder_ff": 1024,
+ "encoder_num_layer": 1,
+ "lr": 0.0013157536168827595,
+ "n_head": 4
+}
\ No newline at end of file
diff --git a/ray_results/test_experiment/train_model_a00d3259_2_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=4,drop_ra_2024-06-25_17-10-13/params.pkl b/ray_results/test_experiment/train_model_a00d3259_2_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=4,drop_ra_2024-06-25_17-10-13/params.pkl
new file mode 100644
index 0000000000000000000000000000000000000000..cf049c9f581d612bc599d1f8f0bbb72e6fd079c8
GIT binary patch
literal 224
zcmZo*oqCl40&1sd^a!TrCFiH47RBe4=EmnFR;CtB@n-B1NlAr>7L_1Nu=I$+Br@|*
zWEgvRQ*)D2Q&KYX(&JMybEkL<^l-t9PD`8O%fQmZ3)2fU6fDHl!v_-r83z?(%ZtxQ
zO-z~M&C<h}Qj}j1UzAvqI>p`o%x%xzx4(MyFy#~hd7*u)H#j>5^l&95mLzAy7iU(b
Jf-EW30|3{XQ?CF3
literal 0
HcmV?d00001
diff --git a/ray_results/test_experiment/train_model_a00d3259_2_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=4,drop_ra_2024-06-25_17-10-13/progress.csv b/ray_results/test_experiment/train_model_a00d3259_2_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=4,drop_ra_2024-06-25_17-10-13/progress.csv
new file mode 100644
index 0000000..3ec1538
--- /dev/null
+++ b/ray_results/test_experiment/train_model_a00d3259_2_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=4,drop_ra_2024-06-25_17-10-13/progress.csv
@@ -0,0 +1,9 @@
+loss,timestamp,checkpoint_dir_name,should_checkpoint,done,training_iteration,trial_id,date,time_this_iter_s,time_total_s,pid,hostname,node_ip,time_since_restore,iterations_since_restore
+253.8278117743064,1719328526,checkpoint_000000,True,False,1,a00d3259,2024-06-25_17-15-26,70.23847699165344,70.23847699165344,2195253,r8i6n2,10.159.28.60,70.23847699165344,1
+168.6947938813938,1719330587,checkpoint_000001,True,False,2,a00d3259,2024-06-25_17-49-47,69.5678973197937,139.80637431144714,2195253,r8i6n2,10.159.28.60,69.5678973197937,1
+159.82500259519563,1719331219,checkpoint_000002,True,False,3,a00d3259,2024-06-25_18-00-19,88.16569352149963,227.97206783294678,2195253,r8i6n2,10.159.28.60,88.16569352149963,1
+140.64553523626853,1719331282,checkpoint_000003,True,False,4,a00d3259,2024-06-25_18-01-22,62.607720375061035,290.5797882080078,2195253,r8i6n2,10.159.28.60,150.77341389656067,2
+150.0000455060343,1719331896,checkpoint_000004,True,False,5,a00d3259,2024-06-25_18-11-36,72.27389621734619,362.853684425354,2195253,r8i6n2,10.159.28.60,72.27389621734619,1
+130.9991111755371,1719331954,checkpoint_000005,True,False,6,a00d3259,2024-06-25_18-12-34,58.202213287353516,421.0558977127075,2195253,r8i6n2,10.159.28.60,130.4761095046997,2
+124.42158469252699,1719332013,checkpoint_000006,True,False,7,a00d3259,2024-06-25_18-13-33,58.22797513008118,479.2838728427887,2195253,r8i6n2,10.159.28.60,188.70408463478088,3
+119.19234625748761,1719332071,checkpoint_000007,True,False,8,a00d3259,2024-06-25_18-14-31,58.22098970413208,537.5048625469208,2195253,r8i6n2,10.159.28.60,246.92507433891296,4
diff --git a/ray_results/test_experiment/train_model_a00d3259_2_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=4,drop_ra_2024-06-25_17-10-13/result.json b/ray_results/test_experiment/train_model_a00d3259_2_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=4,drop_ra_2024-06-25_17-10-13/result.json
new file mode 100644
index 0000000..5806933
--- /dev/null
+++ b/ray_results/test_experiment/train_model_a00d3259_2_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=4,drop_ra_2024-06-25_17-10-13/result.json
@@ -0,0 +1,8 @@
+{"loss": 253.8278117743064, "timestamp": 1719328526, "checkpoint_dir_name": "checkpoint_000000", "should_checkpoint": true, "done": false, "training_iteration": 1, "trial_id": "a00d3259", "date": "2024-06-25_17-15-26", "time_this_iter_s": 70.23847699165344, "time_total_s": 70.23847699165344, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 1, "decoder_rt_num_layer": 4, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 1024, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 4, "drop_rate": 0.22544213936627444, "lr": 0.0013157536168827595, "batch_size": 1024}, "time_since_restore": 70.23847699165344, "iterations_since_restore": 1}
+{"loss": 168.6947938813938, "timestamp": 1719330587, "checkpoint_dir_name": "checkpoint_000001", "should_checkpoint": true, "done": false, "training_iteration": 2, "trial_id": "a00d3259", "date": "2024-06-25_17-49-47", "time_this_iter_s": 69.5678973197937, "time_total_s": 139.80637431144714, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 1, "decoder_rt_num_layer": 4, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 1024, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 4, "drop_rate": 0.22544213936627444, "lr": 0.0013157536168827595, "batch_size": 1024}, "time_since_restore": 69.5678973197937, "iterations_since_restore": 1}
+{"loss": 159.82500259519563, "timestamp": 1719331219, "checkpoint_dir_name": "checkpoint_000002", "should_checkpoint": true, "done": false, "training_iteration": 3, "trial_id": "a00d3259", "date": "2024-06-25_18-00-19", "time_this_iter_s": 88.16569352149963, "time_total_s": 227.97206783294678, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 1, "decoder_rt_num_layer": 4, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 1024, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 4, "drop_rate": 0.22544213936627444, "lr": 0.0013157536168827595, "batch_size": 1024}, "time_since_restore": 88.16569352149963, "iterations_since_restore": 1}
+{"loss": 140.64553523626853, "timestamp": 1719331282, "checkpoint_dir_name": "checkpoint_000003", "should_checkpoint": true, "done": false, "training_iteration": 4, "trial_id": "a00d3259", "date": "2024-06-25_18-01-22", "time_this_iter_s": 62.607720375061035, "time_total_s": 290.5797882080078, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 1, "decoder_rt_num_layer": 4, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 1024, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 4, "drop_rate": 0.22544213936627444, "lr": 0.0013157536168827595, "batch_size": 1024}, "time_since_restore": 150.77341389656067, "iterations_since_restore": 2}
+{"loss": 150.0000455060343, "timestamp": 1719331896, "checkpoint_dir_name": "checkpoint_000004", "should_checkpoint": true, "done": false, "training_iteration": 5, "trial_id": "a00d3259", "date": "2024-06-25_18-11-36", "time_this_iter_s": 72.27389621734619, "time_total_s": 362.853684425354, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 1, "decoder_rt_num_layer": 4, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 1024, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 4, "drop_rate": 0.22544213936627444, "lr": 0.0013157536168827595, "batch_size": 1024}, "time_since_restore": 72.27389621734619, "iterations_since_restore": 1}
+{"loss": 130.9991111755371, "timestamp": 1719331954, "checkpoint_dir_name": "checkpoint_000005", "should_checkpoint": true, "done": false, "training_iteration": 6, "trial_id": "a00d3259", "date": "2024-06-25_18-12-34", "time_this_iter_s": 58.202213287353516, "time_total_s": 421.0558977127075, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 1, "decoder_rt_num_layer": 4, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 1024, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 4, "drop_rate": 0.22544213936627444, "lr": 0.0013157536168827595, "batch_size": 1024}, "time_since_restore": 130.4761095046997, "iterations_since_restore": 2}
+{"loss": 124.42158469252699, "timestamp": 1719332013, "checkpoint_dir_name": "checkpoint_000006", "should_checkpoint": true, "done": false, "training_iteration": 7, "trial_id": "a00d3259", "date": "2024-06-25_18-13-33", "time_this_iter_s": 58.22797513008118, "time_total_s": 479.2838728427887, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 1, "decoder_rt_num_layer": 4, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 1024, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 4, "drop_rate": 0.22544213936627444, "lr": 0.0013157536168827595, "batch_size": 1024}, "time_since_restore": 188.70408463478088, "iterations_since_restore": 3}
+{"loss": 119.19234625748761, "timestamp": 1719332071, "checkpoint_dir_name": "checkpoint_000007", "should_checkpoint": true, "done": false, "training_iteration": 8, "trial_id": "a00d3259", "date": "2024-06-25_18-14-31", "time_this_iter_s": 58.22098970413208, "time_total_s": 537.5048625469208, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 1, "decoder_rt_num_layer": 4, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 1024, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 4, "drop_rate": 0.22544213936627444, "lr": 0.0013157536168827595, "batch_size": 1024}, "time_since_restore": 246.92507433891296, "iterations_since_restore": 4}
diff --git a/ray_results/test_experiment/train_model_a01c9b2d_16_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_17-44-50/events.out.tfevents.1719330362.r8i6n2 b/ray_results/test_experiment/train_model_a01c9b2d_16_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_17-44-50/events.out.tfevents.1719330362.r8i6n2
new file mode 100644
index 0000000000000000000000000000000000000000..12da38bddd4ec933a17d9c29821179e8a2d15e84
GIT binary patch
literal 770
zcmb1OfPlsI-b$RR6L<FQoOjbviZ`h!F*8rkwJbHS#LDQFtV)V9RNaR8<<6Yn<Tfq_
zsS{w-65|r#;w?(7)GsN`OV!WGFD@1>{#WXfz`P~S0IFr`*9-BSfl6>K%3O+EQgAKB
z8Tq9-De=h}sma*|`I&hoq6`cT_Gvp~l~JrzQ-SM8wK63?FBPbTfgymqccnhm%G_Al
z0L~2vElOMpT#|4rOEPm)<4ZC!i{mp(Qj6k?MSWH*ce<kTm%|XMeQtB}OHNHyxQ!}Y
zN?g*Y+KV&ul2hZ0Qj1ISi&DV`XjtrwH-{SV{KctA&KU>;w74|56yXMdY)vf5%+D*1
zM>PiIa}Jj@RTQ7Ea!lU}4s0fDf!(+5movkr-TWwlz19h?g_yurHTo2UVx@^QTtAM$
zR(!htAWC5GL1-Zm*qhe$I>~CDPDTlA6BoFR1OvN%m6Y?Rea}@<0(&XK0HOn%!9imZ
G;~xM)koYhF
literal 0
HcmV?d00001
diff --git a/ray_results/test_experiment/train_model_a01c9b2d_16_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_17-44-50/events.out.tfevents.1719331634.r8i6n2 b/ray_results/test_experiment/train_model_a01c9b2d_16_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_17-44-50/events.out.tfevents.1719331634.r8i6n2
new file mode 100644
index 0000000000000000000000000000000000000000..edaa9f5f5aab1cc489cdb86002b338f3871994da
GIT binary patch
literal 1676
zcma)6TSydP6kcb{ngd>TbiJ&X*0n&kY*o;}irrj#F`}rjBI0M<`E^HkXKZGyj7UR*
zh#u@gqX#oJ2qID<6BL3Lx?p#F5J@r$v>Rm)22l_F>(0*Zn74VGbIy0Z|NqW+{!CN&
z>ucF<XHv&|pATP~*lA6ykvT!!vg-sE)r!))-|n0((CP-dPkNZ1qKxTKT?*|;hcrl(
zxn`%@Bx0vuQWVSN!9mZO;L(|44bn098Zp6A9Z~@KVAUatPipe>$m_%2hDJ#cRg1~g
z>T*3EE;3N=Ez|KvN?sB%ftXBxhVXZ%Mj1GkyN{WULhN9JES*vn0vM@2K|z9wWu#cf
zx}R0Pw2o~q(coQeZJ!xbt}|``J7gQ-iXeJ1lCh#nGM;a6fPIp)RckQwY4iv)5pCdr
zLde$}gjhLMkVFL;+k|dz-gDn>xVgud{&!yN_{GG2pMBYLeA}0s2C<6-9kLp+t3S4%
zF;L!%;ujUW)phRO&sPSq%j<Q@6~ykG>#v-&Ua2sMeJcuIQS8Cd9iBran>r0*XEx~k
zs}|ejzBD)TW*{y+wDzv6^-SH{a2v-B8hU6*L%v2%<^l?G;!}-SCdLufstYunr(vy@
zVt-TYFUp<-P{XNS9~tRsY@_WgB=OiQ@x)-n)Tp+W#InE=8xxxX$j>!nnWW5+7)i-m
zbta_6gkp;mf#GGT5fK-Ks0bDZYB0|WVjbdz011)PU@^=-&ujs6<Uh7i0#N=CJ3xYn
ze3;{Hw8IR^tZ}0qd62^{v#|>xp^bFpLN>ef#yQqQF1y@DIW|H8yYj}&v?N><wfO~q
z^j0v}BlG(}!4w5K(fKJ@QOL?6MdgUVf><k6J)mHOiq5g0vfid3Jw{uWG26<5hwG+Z
zxF9tm5ebG3li~*Jf~o1bb~ofLBDtQoxgmR@LG)y+o4k;V&R#E>bwP4yb|mm<t`I-`
zYITJUW3Yi<T<nI7SU>0qa!Qhj+BP;}U&5<;&8#b~qqBOu_ToJq>f6r@Mbs|=Cs_a2
W9*cZfdE&9C?(ueNT^t&xeg6POOb}53
literal 0
HcmV?d00001
diff --git a/ray_results/test_experiment/train_model_a01c9b2d_16_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_17-44-50/params.json b/ray_results/test_experiment/train_model_a01c9b2d_16_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_17-44-50/params.json
new file mode 100644
index 0000000..6fc28c7
--- /dev/null
+++ b/ray_results/test_experiment/train_model_a01c9b2d_16_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_17-44-50/params.json
@@ -0,0 +1,13 @@
+{
+ "batch_size": 1024,
+ "decoder_int_ff": 512,
+ "decoder_int_num_layer": 1,
+ "decoder_rt_ff": 1024,
+ "decoder_rt_num_layer": 1,
+ "drop_rate": 0.23338023981015343,
+ "embedding_dim": 16,
+ "encoder_ff": 1024,
+ "encoder_num_layer": 8,
+ "lr": 0.00010269515886648154,
+ "n_head": 4
+}
\ No newline at end of file
diff --git a/ray_results/test_experiment/train_model_a01c9b2d_16_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_17-44-50/params.pkl b/ray_results/test_experiment/train_model_a01c9b2d_16_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_17-44-50/params.pkl
new file mode 100644
index 0000000000000000000000000000000000000000..450eb78874348bec9322c08bf9cba50d42dd5617
GIT binary patch
literal 224
zcmZo*oqCl40&1sd^a!TrCFiH47RBe4=EmnFR;CtB@#g3eNlAr>7L_1NF!qSTBr@~R
zWO!3^lTuSsGV{{oQ!;a>cnkD!!HiBzo8rsB(!&eW3p5le#0L`s83z_)>S4=^&qz&7
znc~gT!<kZ)Ul3oESdu!$-Tv(T^oT>I3O!6YML?d^>w~EpUyt^1B_)<5XT%q0R;7Y0
HDb)i2!;4dH
literal 0
HcmV?d00001
diff --git a/ray_results/test_experiment/train_model_a01c9b2d_16_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_17-44-50/progress.csv b/ray_results/test_experiment/train_model_a01c9b2d_16_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_17-44-50/progress.csv
new file mode 100644
index 0000000..962131d
--- /dev/null
+++ b/ray_results/test_experiment/train_model_a01c9b2d_16_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_17-44-50/progress.csv
@@ -0,0 +1,5 @@
+loss,timestamp,checkpoint_dir_name,should_checkpoint,done,training_iteration,trial_id,date,time_this_iter_s,time_total_s,pid,hostname,node_ip,time_since_restore,iterations_since_restore
+983.9757466954509,1719330446,checkpoint_000000,True,False,1,a01c9b2d,2024-06-25_17-47-26,83.82870721817017,83.82870721817017,2195253,r8i6n2,10.159.28.60,83.82870721817017,1
+501.4262033297321,1719330517,checkpoint_000001,True,False,2,a01c9b2d,2024-06-25_17-48-37,70.83729195594788,154.66599917411804,2195253,r8i6n2,10.159.28.60,154.66599917411804,2
+362.6629641074834,1719331736,checkpoint_000002,True,False,3,a01c9b2d,2024-06-25_18-08-56,101.81216025352478,256.4781594276428,2195253,r8i6n2,10.159.28.60,101.81216025352478,1
+290.2177711246521,1719331824,checkpoint_000003,True,True,4,a01c9b2d,2024-06-25_18-10-24,87.97965884208679,344.4578182697296,2195253,r8i6n2,10.159.28.60,189.79181909561157,2
diff --git a/ray_results/test_experiment/train_model_a01c9b2d_16_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_17-44-50/result.json b/ray_results/test_experiment/train_model_a01c9b2d_16_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_17-44-50/result.json
new file mode 100644
index 0000000..9899007
--- /dev/null
+++ b/ray_results/test_experiment/train_model_a01c9b2d_16_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_17-44-50/result.json
@@ -0,0 +1,4 @@
+{"loss": 983.9757466954509, "timestamp": 1719330446, "checkpoint_dir_name": "checkpoint_000000", "should_checkpoint": true, "done": false, "training_iteration": 1, "trial_id": "a01c9b2d", "date": "2024-06-25_17-47-26", "time_this_iter_s": 83.82870721817017, "time_total_s": 83.82870721817017, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 8, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 1024, "decoder_rt_ff": 1024, "decoder_int_ff": 512, "n_head": 4, "drop_rate": 0.23338023981015343, "lr": 0.00010269515886648154, "batch_size": 1024}, "time_since_restore": 83.82870721817017, "iterations_since_restore": 1}
+{"loss": 501.4262033297321, "timestamp": 1719330517, "checkpoint_dir_name": "checkpoint_000001", "should_checkpoint": true, "done": false, "training_iteration": 2, "trial_id": "a01c9b2d", "date": "2024-06-25_17-48-37", "time_this_iter_s": 70.83729195594788, "time_total_s": 154.66599917411804, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 8, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 1024, "decoder_rt_ff": 1024, "decoder_int_ff": 512, "n_head": 4, "drop_rate": 0.23338023981015343, "lr": 0.00010269515886648154, "batch_size": 1024}, "time_since_restore": 154.66599917411804, "iterations_since_restore": 2}
+{"loss": 362.6629641074834, "timestamp": 1719331736, "checkpoint_dir_name": "checkpoint_000002", "should_checkpoint": true, "done": false, "training_iteration": 3, "trial_id": "a01c9b2d", "date": "2024-06-25_18-08-56", "time_this_iter_s": 101.81216025352478, "time_total_s": 256.4781594276428, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 8, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 1024, "decoder_rt_ff": 1024, "decoder_int_ff": 512, "n_head": 4, "drop_rate": 0.23338023981015343, "lr": 0.00010269515886648154, "batch_size": 1024}, "time_since_restore": 101.81216025352478, "iterations_since_restore": 1}
+{"loss": 290.2177711246521, "timestamp": 1719331824, "checkpoint_dir_name": "checkpoint_000003", "should_checkpoint": true, "done": true, "training_iteration": 4, "trial_id": "a01c9b2d", "date": "2024-06-25_18-10-24", "time_this_iter_s": 87.97965884208679, "time_total_s": 344.4578182697296, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 8, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 1024, "decoder_rt_ff": 1024, "decoder_int_ff": 512, "n_head": 4, "drop_rate": 0.23338023981015343, "lr": 0.00010269515886648154, "batch_size": 1024}, "time_since_restore": 189.79181909561157, "iterations_since_restore": 2}
diff --git a/ray_results/test_experiment/train_model_a0e8b2af_1_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=8,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop_r_2024-06-25_16-07-48/events.out.tfevents.1719324484.r3i5n6 b/ray_results/test_experiment/train_model_a0e8b2af_1_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=8,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop_r_2024-06-25_16-07-48/events.out.tfevents.1719324484.r3i5n6
new file mode 100644
index 0000000000000000000000000000000000000000..c7b5f480763345b2895ae6385b6313c3e4b6ad63
GIT binary patch
literal 1311
zcmah|Pe>F|9A0O|njfvU(~jw`YrA8AbRah+h$Pr8k;*QShltSExbti$ci${CV`UKu
zB6tWABFKaN(IA4zNYI1n!9zqHDu}>C1chj)JXFvj=#4x3cGuMAHox!p{l53U-}fq9
znQ!d+35gfO2j5TJoj>gNbQ+3A>RT>iVl_2RwTrmM>3eqXQZwH&eq+(@+gc=-gAY80
z(i^dQ2#zFm(-h}U?`?V8(H7k2P;UQxhxp@@j8X$(@H2{;(0h_9iYIWqJFRQP5?!vo
z=xF`^K!If4G~*Xasye}x;&S=wX7@xK$y9srB){baqeu{fN+xM(DU7UyW+KhP1~SFw
z;pa#Gbie%C;Lu0<`@iwSIr>hJAW%p*H4?|jz^0`ecxAxc(7Bld&Va?G*Jt?94AYmP
z7Q$?Rt*Tg>PE1rd#=g1Q`1r%KqRgD3k>Lw`M|ffPBsE)p=!OSw@S#LnF_e^vBKUF|
z8}u&3Y8`aLfEzYBkno!mesNMMK&N8G6IA624!I=(N>v=!ReERYGwSN1wMr24YosTI
zl1eW&DC7lC7BZaf3h?EX@|!)ttA?IN^kTLY<>FK)R#lChN2->hC3+f6GU_YJB6zd^
z`GZP;OWNuHB?Kj~qK4eE7s`afkjd3hC2SC)RTMiUau5Q-`XP{aKv39d$?`6!5&pl#
ziEDTIiL<}C@1srpOm=@?IQYy#Rc?PKHcgrWWLgR}SR-9}>nsP;TyBr!oc{?2<$2x;
zyw`v8&C!A4Pf^&GwTKoofeFRHziMj?8;U{o8kRHM6@$R)fH1HxMj!2(yJs!SQ7E%_
zM+<`swU3@I--_CYDTo1UPRC$-ejXTbe+9_JmI3tm(mGjXJ)zjrk7H*KIgjww)YvKh
fW7ho!h=BiJ?=ev`&eM$Ld@q)`%|*3W=iB}Q%P^E|
literal 0
HcmV?d00001
diff --git a/ray_results/test_experiment/train_model_a0e8b2af_1_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=8,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop_r_2024-06-25_16-07-48/params.json b/ray_results/test_experiment/train_model_a0e8b2af_1_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=8,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop_r_2024-06-25_16-07-48/params.json
new file mode 100644
index 0000000..2acdbf5
--- /dev/null
+++ b/ray_results/test_experiment/train_model_a0e8b2af_1_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=8,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop_r_2024-06-25_16-07-48/params.json
@@ -0,0 +1,13 @@
+{
+ "batch_size": 1024,
+ "decoder_int_ff": 512,
+ "decoder_int_num_layer": 8,
+ "decoder_rt_ff": 1024,
+ "decoder_rt_num_layer": 1,
+ "drop_rate": 0.4534009171604666,
+ "embedding_dim": 256,
+ "encoder_ff": 2048,
+ "encoder_num_layer": 2,
+ "lr": 0.019472817375775914,
+ "n_head": 8
+}
\ No newline at end of file
diff --git a/ray_results/test_experiment/train_model_a0e8b2af_1_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=8,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop_r_2024-06-25_16-07-48/params.pkl b/ray_results/test_experiment/train_model_a0e8b2af_1_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=8,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop_r_2024-06-25_16-07-48/params.pkl
new file mode 100644
index 0000000000000000000000000000000000000000..b954d8aa1d31bdc54af695f028f87b2d395e5d26
GIT binary patch
literal 225
zcmZo*oqCM{0&1sd^a!TrCFiH47RBe4=EmnFR;CtB@n-4~NlAr>7L_1NF!qSTBr@|*
zWH@?wQ*)D2Q&KYX(&JMybEo(+F!pf4%uY)K@_|ZVnt`T*g;;v{U_v1Cz+y~2Y<ck+
zsfj65yg7O}Q;PBn;)@bXQm44v-(_iaKix0Y!<175<W2stZn?(OjvlV0#FFHU_~Oi}
KRG=j+rFsD2LQ`r0
literal 0
HcmV?d00001
diff --git a/ray_results/test_experiment/train_model_a0e8b2af_1_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=8,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop_r_2024-06-25_16-07-48/progress.csv b/ray_results/test_experiment/train_model_a0e8b2af_1_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=8,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop_r_2024-06-25_16-07-48/progress.csv
new file mode 100644
index 0000000..2ab7a96
--- /dev/null
+++ b/ray_results/test_experiment/train_model_a0e8b2af_1_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=8,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop_r_2024-06-25_16-07-48/progress.csv
@@ -0,0 +1,2 @@
+loss,timestamp,checkpoint_dir_name,should_checkpoint,done,training_iteration,trial_id,date,time_this_iter_s,time_total_s,pid,hostname,node_ip,time_since_restore,iterations_since_restore
+2773.484795037217,1719324574,checkpoint_000000,True,False,1,a0e8b2af,2024-06-25_16-09-34,89.29933595657349,89.29933595657349,69318,r3i5n6,10.159.8.159,89.29933595657349,1
diff --git a/ray_results/test_experiment/train_model_a0e8b2af_1_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=8,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop_r_2024-06-25_16-07-48/result.json b/ray_results/test_experiment/train_model_a0e8b2af_1_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=8,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop_r_2024-06-25_16-07-48/result.json
new file mode 100644
index 0000000..d494e65
--- /dev/null
+++ b/ray_results/test_experiment/train_model_a0e8b2af_1_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=8,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop_r_2024-06-25_16-07-48/result.json
@@ -0,0 +1 @@
+{"loss": 2773.484795037217, "timestamp": 1719324574, "checkpoint_dir_name": "checkpoint_000000", "should_checkpoint": true, "done": false, "training_iteration": 1, "trial_id": "a0e8b2af", "date": "2024-06-25_16-09-34", "time_this_iter_s": 89.29933595657349, "time_total_s": 89.29933595657349, "pid": 69318, "hostname": "r3i5n6", "node_ip": "10.159.8.159", "config": {"encoder_num_layer": 2, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 8, "embedding_dim": 256, "encoder_ff": 2048, "decoder_rt_ff": 1024, "decoder_int_ff": 512, "n_head": 8, "drop_rate": 0.4534009171604666, "lr": 0.019472817375775914, "batch_size": 1024}, "time_since_restore": 89.29933595657349, "iterations_since_restore": 1}
diff --git a/ray_results/test_experiment/train_model_ad12346e_12_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_17-36-47/events.out.tfevents.1719329892.r8i6n2 b/ray_results/test_experiment/train_model_ad12346e_12_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_17-36-47/events.out.tfevents.1719329892.r8i6n2
new file mode 100644
index 0000000000000000000000000000000000000000..f0a469aba51f5e59528842fed0cf9c272fd182d1
GIT binary patch
literal 1311
zcmah|T}TvB7+q&Jb3WMBom|uPr*$=u5#018uwYlk7mJXh2Z`k}?p)o;-8;+NvC@|$
z3F=LV9wh8fA9^U-w+Nw!=pm%QAR>c)2*D&u2`YN3JMQe=pUmcMzVAEVx%ZrNYhBrI
z{``@U6g$yY_wf2kPrx&5sygZDKZA+Y-8K0x)6wG0Wp149lh&5s9<}GH%9I+Y29K#u
zgspLc!%2fuzxT7df3kh-&H)E<_4_iCg0DEF1)3qiAv9r(Cp8pL;P_bD(23=Dxuzmt
zH{4f>C@;U}_+m=UAecd1F3DP73_FymSmU5nyTTzM2tpmFwDc54Rzjypx3Gz*|HaE=
zeZ6Y$sZIwzH$A;B&F0~)5Q6$*IMqoUBNJ20F!9!cE_Zn0ptE4@)6$USeZ%<`XoF_H
zz-Cn~-5?Ydud#1Fabe?7rYyT>W^VShgqkz+w%E#o2)9J2Mv1g)sws-X_*@#B>@LI_
zxFo_w5q3K*`IjL75JFynVbzK!7|SFMiXj=i8jc$pyED@njf}8b9rz1tWITnE>I61f
z%LkrZ%XFq|p}L?{IP3vQGmSK27qh7t7N>@>rs-r9X?lv4*l94yvTrf_!I%3lTvQH1
z(q;!JCn$kcEhs8JsFaICrZho=yhDgeAa+QUMyQwn9Rg(^G|D@ftn7yt`Tr&ktz3IT
z9RD6~-0qg1=luI9z<U82^8QgwDN6yOmdXUy$%rxVNPtBl@8f+Ta94nu0&Qi<cXFv`
zboJYp2<*v8#ERL#R7D|B;u`PGM4@RL$$5V>0`;W<;mxTid$ikn&qo-(()NxO#1FfE
zwhp#O?86krfDg07>@CbgNh2Ry-nbcMk8t~}@|n%Zz1Z)*!_Fgo`e6RJw3<`D6T%Sq
XTYF5jtnw^ldEMP1p`vWPqWsw($CH-V
literal 0
HcmV?d00001
diff --git a/ray_results/test_experiment/train_model_ad12346e_12_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_17-36-47/params.json b/ray_results/test_experiment/train_model_ad12346e_12_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_17-36-47/params.json
new file mode 100644
index 0000000..d8b9b66
--- /dev/null
+++ b/ray_results/test_experiment/train_model_ad12346e_12_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_17-36-47/params.json
@@ -0,0 +1,13 @@
+{
+ "batch_size": 2048,
+ "decoder_int_ff": 512,
+ "decoder_int_num_layer": 1,
+ "decoder_rt_ff": 1024,
+ "decoder_rt_num_layer": 1,
+ "drop_rate": 0.9358999268282086,
+ "embedding_dim": 1024,
+ "encoder_ff": 1024,
+ "encoder_num_layer": 1,
+ "lr": 0.00020838214141938324,
+ "n_head": 4
+}
\ No newline at end of file
diff --git a/ray_results/test_experiment/train_model_ad12346e_12_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_17-36-47/params.pkl b/ray_results/test_experiment/train_model_ad12346e_12_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_17-36-47/params.pkl
new file mode 100644
index 0000000000000000000000000000000000000000..8c10fae9a1db4cee70588af8af9e5116360d6216
GIT binary patch
literal 225
zcmZo*oqCM{0&1sd^a!TrCFiH47RBe4=EmnFR;CtB@n-B1NlAr>7L}k%h{7Z?^U!2?
zQ*)D2Q&KYX(&JMybEo(+u=H@j%uY)K^Lb&Kfu=%*_+UaH^T1+EJ#2aL8L5dWQ@mMv
zI8%!93*w6sOH!w}+rRzvB;D~;Xb)3P5s;@Hpz@vd0do&mQesJRMtpH*RVvUDj#51U
D<aJWp
literal 0
HcmV?d00001
diff --git a/ray_results/test_experiment/train_model_ad12346e_12_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_17-36-47/progress.csv b/ray_results/test_experiment/train_model_ad12346e_12_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_17-36-47/progress.csv
new file mode 100644
index 0000000..9c22dec
--- /dev/null
+++ b/ray_results/test_experiment/train_model_ad12346e_12_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_17-36-47/progress.csv
@@ -0,0 +1,2 @@
+loss,timestamp,checkpoint_dir_name,should_checkpoint,done,training_iteration,trial_id,date,time_this_iter_s,time_total_s,pid,hostname,node_ip,time_since_restore,iterations_since_restore
+2126.6908641424707,1719330099,checkpoint_000000,True,False,1,ad12346e,2024-06-25_17-41-39,206.83131527900696,206.83131527900696,2195253,r8i6n2,10.159.28.60,206.83131527900696,1
diff --git a/ray_results/test_experiment/train_model_ad12346e_12_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_17-36-47/result.json b/ray_results/test_experiment/train_model_ad12346e_12_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_17-36-47/result.json
new file mode 100644
index 0000000..e6771dc
--- /dev/null
+++ b/ray_results/test_experiment/train_model_ad12346e_12_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_17-36-47/result.json
@@ -0,0 +1 @@
+{"loss": 2126.6908641424707, "timestamp": 1719330099, "checkpoint_dir_name": "checkpoint_000000", "should_checkpoint": true, "done": false, "training_iteration": 1, "trial_id": "ad12346e", "date": "2024-06-25_17-41-39", "time_this_iter_s": 206.83131527900696, "time_total_s": 206.83131527900696, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 1, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 1024, "encoder_ff": 1024, "decoder_rt_ff": 1024, "decoder_int_ff": 512, "n_head": 4, "drop_rate": 0.9358999268282086, "lr": 0.00020838214141938324, "batch_size": 2048}, "time_since_restore": 206.83131527900696, "iterations_since_restore": 1}
diff --git a/ray_results/test_experiment/train_model_af6d9c38_5_batch_size=2048,decoder_int_ff=1024,decoder_int_num_layer=8,decoder_rt_ff=1024,decoder_rt_num_layer=8,drop__2024-06-25_16-12-08/events.out.tfevents.1719324792.r3i5n6 b/ray_results/test_experiment/train_model_af6d9c38_5_batch_size=2048,decoder_int_ff=1024,decoder_int_num_layer=8,decoder_rt_ff=1024,decoder_rt_num_layer=8,drop__2024-06-25_16-12-08/events.out.tfevents.1719324792.r3i5n6
new file mode 100644
index 0000000000000000000000000000000000000000..f0f7171be1e2e83b5650da58c10d891a20709dc8
GIT binary patch
literal 405
zcmb1OfPlsI-b$P&H|WW2nRnAsiZ`h!F*8rkwJbHS#L7s1r`%;_sJadF%bhue4z98V
zsS{w-65|r#;w?(7)GsN`OV!WGFD@2ktNP($D_QSr0M#<}>xFpE2gl%Al(`hSq~KbL
zGxAGwQsR>{Qj@a_@-y>FL>U+w>=k%YB~h$&Jqg#3YGq1(UMf%v14FJ==TUvAmASF9
z0i1UbT9mjHxFq3LmSpCp#+PJd7RP6nq!z^&i|XI|?X<VQ`-LG?``qT{mz;5@;5Mpo
zDRD`oYA?>rOHPe1N-ZwQFG>X)VC?yOkvY_W=Pyo0a^6H3pv9%Zr3g0wWNTtcW`16A
OJgPAupTF84TMYpDwS|TN
literal 0
HcmV?d00001
diff --git a/ray_results/test_experiment/train_model_af6d9c38_5_batch_size=2048,decoder_int_ff=1024,decoder_int_num_layer=8,decoder_rt_ff=1024,decoder_rt_num_layer=8,drop__2024-06-25_16-12-08/events.out.tfevents.1719327245.r3i5n6 b/ray_results/test_experiment/train_model_af6d9c38_5_batch_size=2048,decoder_int_ff=1024,decoder_int_num_layer=8,decoder_rt_ff=1024,decoder_rt_num_layer=8,drop__2024-06-25_16-12-08/events.out.tfevents.1719327245.r3i5n6
new file mode 100644
index 0000000000000000000000000000000000000000..1840bfa396cc6a9655fd8c325054ad02eb51df9d
GIT binary patch
literal 405
zcmb1OfPlsI-b$Q*s&kvS&AaI+#hX-=n3<>NT9%quVr8^NcG^>AsJadF%bhuY=ddmY
zsS{w*65|r#;w?(7)GsN`OV!WGFD@44KKb2+`$zIZ1E`j%UoXUST9(4KD03-tNx`)g
zXXKaWq{Js@q$Xz<<Y(rUh%zuV*n4k`+>c`A>~gq%R4Y^R^HPCY7#OnKzA@@Ut;~&;
z4dAq?fNN3WQs9z=TUnBsn;Kt|ky#v{S&~{5Uo3iZ#~-J8;uf0?q1xv*H^1b(jnJ;b
zrNkwTs=YWfFF7^7D7CmGzbF-KfJ|)ob#tfz&tII1<czO`+poo?!KDZ{0Ay=oNoIat
PaXhLqAfJ25OV0!V@;HOu
literal 0
HcmV?d00001
diff --git a/ray_results/test_experiment/train_model_af6d9c38_5_batch_size=2048,decoder_int_ff=1024,decoder_int_num_layer=8,decoder_rt_ff=1024,decoder_rt_num_layer=8,drop__2024-06-25_16-12-08/params.json b/ray_results/test_experiment/train_model_af6d9c38_5_batch_size=2048,decoder_int_ff=1024,decoder_int_num_layer=8,decoder_rt_ff=1024,decoder_rt_num_layer=8,drop__2024-06-25_16-12-08/params.json
new file mode 100644
index 0000000..a2c32c9
--- /dev/null
+++ b/ray_results/test_experiment/train_model_af6d9c38_5_batch_size=2048,decoder_int_ff=1024,decoder_int_num_layer=8,decoder_rt_ff=1024,decoder_rt_num_layer=8,drop__2024-06-25_16-12-08/params.json
@@ -0,0 +1,13 @@
+{
+ "batch_size": 2048,
+ "decoder_int_ff": 1024,
+ "decoder_int_num_layer": 8,
+ "decoder_rt_ff": 1024,
+ "decoder_rt_num_layer": 8,
+ "drop_rate": 0.5912116123671598,
+ "embedding_dim": 16,
+ "encoder_ff": 512,
+ "encoder_num_layer": 8,
+ "lr": 0.009202977404873457,
+ "n_head": 16
+}
\ No newline at end of file
diff --git a/ray_results/test_experiment/train_model_af6d9c38_5_batch_size=2048,decoder_int_ff=1024,decoder_int_num_layer=8,decoder_rt_ff=1024,decoder_rt_num_layer=8,drop__2024-06-25_16-12-08/params.pkl b/ray_results/test_experiment/train_model_af6d9c38_5_batch_size=2048,decoder_int_ff=1024,decoder_int_num_layer=8,decoder_rt_ff=1024,decoder_rt_num_layer=8,drop__2024-06-25_16-12-08/params.pkl
new file mode 100644
index 0000000000000000000000000000000000000000..4b516bb5606b09c8d54c523cf3bed0451adcc9d3
GIT binary patch
literal 224
zcmZo*oqCl40&1sd^a!TrCFiH47RBe4=EmnFR;CtB@#g3eNlAr>7L}k%h{7Z?^U!2?
zQ*)D2Q&KYX(&JMybEkL<^l-t9PD`8O%fQsb3)2fU6fDHj!v_-r83z?(%ZtxQO-uoq
zz?o8%Ul3oESdu!$-Tu*QlesN6Wj#zeML=HDO{VSFx@Y!qB_)<5XT%q0R;2<h;V9Ju
E0NMgm-v9sr
literal 0
HcmV?d00001
diff --git a/ray_results/test_experiment/train_model_af6d9c38_5_batch_size=2048,decoder_int_ff=1024,decoder_int_num_layer=8,decoder_rt_ff=1024,decoder_rt_num_layer=8,drop__2024-06-25_16-12-08/progress.csv b/ray_results/test_experiment/train_model_af6d9c38_5_batch_size=2048,decoder_int_ff=1024,decoder_int_num_layer=8,decoder_rt_ff=1024,decoder_rt_num_layer=8,drop__2024-06-25_16-12-08/progress.csv
new file mode 100644
index 0000000..7becd21
--- /dev/null
+++ b/ray_results/test_experiment/train_model_af6d9c38_5_batch_size=2048,decoder_int_ff=1024,decoder_int_num_layer=8,decoder_rt_ff=1024,decoder_rt_num_layer=8,drop__2024-06-25_16-12-08/progress.csv
@@ -0,0 +1,3 @@
+loss,timestamp,checkpoint_dir_name,should_checkpoint,done,training_iteration,trial_id,date,time_this_iter_s,time_total_s,pid,hostname,node_ip,time_since_restore,iterations_since_restore
+1987.8132929764395,1719324918,checkpoint_000000,True,False,1,af6d9c38,2024-06-25_16-15-18,125.93394875526428,125.93394875526428,69318,r3i5n6,10.159.8.159,125.93394875526428,1
+1982.2825418307086,1719327372,checkpoint_000001,True,False,2,af6d9c38,2024-06-25_16-56-12,126.36091089248657,252.29485964775085,130020,r3i5n6,10.159.8.159,126.36091089248657,1
diff --git a/ray_results/test_experiment/train_model_af6d9c38_5_batch_size=2048,decoder_int_ff=1024,decoder_int_num_layer=8,decoder_rt_ff=1024,decoder_rt_num_layer=8,drop__2024-06-25_16-12-08/result.json b/ray_results/test_experiment/train_model_af6d9c38_5_batch_size=2048,decoder_int_ff=1024,decoder_int_num_layer=8,decoder_rt_ff=1024,decoder_rt_num_layer=8,drop__2024-06-25_16-12-08/result.json
new file mode 100644
index 0000000..e9b68a0
--- /dev/null
+++ b/ray_results/test_experiment/train_model_af6d9c38_5_batch_size=2048,decoder_int_ff=1024,decoder_int_num_layer=8,decoder_rt_ff=1024,decoder_rt_num_layer=8,drop__2024-06-25_16-12-08/result.json
@@ -0,0 +1,2 @@
+{"loss": 1987.8132929764395, "timestamp": 1719324918, "checkpoint_dir_name": "checkpoint_000000", "should_checkpoint": true, "done": false, "training_iteration": 1, "trial_id": "af6d9c38", "date": "2024-06-25_16-15-18", "time_this_iter_s": 125.93394875526428, "time_total_s": 125.93394875526428, "pid": 69318, "hostname": "r3i5n6", "node_ip": "10.159.8.159", "config": {"encoder_num_layer": 8, "decoder_rt_num_layer": 8, "decoder_int_num_layer": 8, "embedding_dim": 16, "encoder_ff": 512, "decoder_rt_ff": 1024, "decoder_int_ff": 1024, "n_head": 16, "drop_rate": 0.5912116123671598, "lr": 0.009202977404873457, "batch_size": 2048}, "time_since_restore": 125.93394875526428, "iterations_since_restore": 1}
+{"loss": 1982.2825418307086, "timestamp": 1719327372, "checkpoint_dir_name": "checkpoint_000001", "should_checkpoint": true, "done": false, "training_iteration": 2, "trial_id": "af6d9c38", "date": "2024-06-25_16-56-12", "time_this_iter_s": 126.36091089248657, "time_total_s": 252.29485964775085, "pid": 130020, "hostname": "r3i5n6", "node_ip": "10.159.8.159", "config": {"encoder_num_layer": 8, "decoder_rt_num_layer": 8, "decoder_int_num_layer": 8, "embedding_dim": 16, "encoder_ff": 512, "decoder_rt_ff": 1024, "decoder_int_ff": 1024, "n_head": 16, "drop_rate": 0.5912116123671598, "lr": 0.009202977404873457, "batch_size": 2048}, "time_since_restore": 126.36091089248657, "iterations_since_restore": 1}
diff --git a/ray_results/test_experiment/train_model_b53cfe05_16_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=8,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_16-43-18/events.out.tfevents.1719326792.r3i5n6 b/ray_results/test_experiment/train_model_b53cfe05_16_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=8,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_16-43-18/events.out.tfevents.1719326792.r3i5n6
new file mode 100644
index 0000000000000000000000000000000000000000..84159e01221465ea023e494cc53832129b11d176
GIT binary patch
literal 1311
zcmah|Pe>F|9A0Nlogb{a)9$9LYwK=>OwekVPz2XP2t6bbfd_ld?mXMEoj1$8v9e2&
zL`4Ka1W9Or45AS7k{tpG4;8WpJJux<EIWA+gHH9voqfA&W^<e0_xpa|d*APSb*|hu
zeWfoV-u-$|nw$F69rPqjMI(oLE?{DH9L~19nvObsvkxyG6UzrYQM<3UOb&n#Jf<=l
zvqlJxr435`v#Xswm7mv*R)=zLV+n~fXE-Ga%@E`inleVxDoUns@_fe7h~;;=#=1`F
zCt6D+cPX4-DybR-Gm6XA?%OyXb0qb_P``M;K#4#Y>baz)=@?lljUvs$CZhfc@%qu$
z?1Oi04t;Wb{D&AwaK(KPfre5#)kqQ}6I07D@#cW{gWnd9I0L?~E)Iyd3j<_mfo49y
zR#hy`AQY93v2XsOed*?GS#Hjq$%$bxFtu{PHd}t`hR1I3p;SgO6`i6OzLdcxy9=>;
zZ@A%_8!8=0`Xxv|g-A6(La~x5rg9mF-4O|@Rh%?bc4y`@8X97?dhi$5$cT>8$|yD&
z<ONS2GM(-^@D-Jcn>`?^rjbGHVzw0HVm*OXRU_w+s_Cr6PJ>BKeN)*F-u!>@pb8Mu
zwmLusK`E@LVYlps8mTm7auYO4JA`Np#SV!af(Gf|A&~b%NZM)1@_vX)|6k&>^{a1*
zv%iJ4CmrJH{Qi~&_#!}KVSf};%2I%+r7(jvGGz453-CrL?D3@#oDm>U<gFxn&xZ8n
z>g-w<?9N-nin+kl#v!<6YaAPkL(?{vbKDh&hOGhNAR{+#zkA-o_%-(KSmF0`@Kp`>
zb=ik0i2-l!kHengJaB*sF`y7z4m^spN4R}fc~3Ylf9ziAbROY{*V89Ol6SuiVi5e>
WdrZ`v^DJWp-^(LHRoTMQ+^s+McZ?|j
literal 0
HcmV?d00001
diff --git a/ray_results/test_experiment/train_model_b53cfe05_16_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=8,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_16-43-18/params.json b/ray_results/test_experiment/train_model_b53cfe05_16_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=8,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_16-43-18/params.json
new file mode 100644
index 0000000..bbdf643
--- /dev/null
+++ b/ray_results/test_experiment/train_model_b53cfe05_16_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=8,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_16-43-18/params.json
@@ -0,0 +1,13 @@
+{
+ "batch_size": 2048,
+ "decoder_int_ff": 512,
+ "decoder_int_num_layer": 8,
+ "decoder_rt_ff": 512,
+ "decoder_rt_num_layer": 2,
+ "drop_rate": 0.9936594412219328,
+ "embedding_dim": 64,
+ "encoder_ff": 512,
+ "encoder_num_layer": 2,
+ "lr": 0.000836614555813242,
+ "n_head": 4
+}
\ No newline at end of file
diff --git a/ray_results/test_experiment/train_model_b53cfe05_16_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=8,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_16-43-18/params.pkl b/ray_results/test_experiment/train_model_b53cfe05_16_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=8,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_16-43-18/params.pkl
new file mode 100644
index 0000000000000000000000000000000000000000..523a2a3cdde8d8143f3e74b4e9e1f7a93eaaaf31
GIT binary patch
literal 224
zcmZo*oqCl40&1sd^a!TrCFiH47RBe4=EmnFR;CtB@n-4~NlAr>7L}k%h{7Z?^H5|s
zdU#WFlTuSsGV{{oQ!;a>csulP!HiBzo8rsB)WZwY3p5le#0L`s83z?(%ZtxQO-z~M
z&C<h}Qj}j1UzAvqI>p`o{TaSTxv~j8OgTkBo_7|92-nL0JzPnNCCM4_#hF#9Kub7E
F^#Hf^QYioc
literal 0
HcmV?d00001
diff --git a/ray_results/test_experiment/train_model_b53cfe05_16_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=8,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_16-43-18/progress.csv b/ray_results/test_experiment/train_model_b53cfe05_16_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=8,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_16-43-18/progress.csv
new file mode 100644
index 0000000..81cd225
--- /dev/null
+++ b/ray_results/test_experiment/train_model_b53cfe05_16_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=8,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_16-43-18/progress.csv
@@ -0,0 +1,2 @@
+loss,timestamp,checkpoint_dir_name,should_checkpoint,done,training_iteration,trial_id,date,time_this_iter_s,time_total_s,pid,hostname,node_ip,time_since_restore,iterations_since_restore
+3022.543074480192,1719326860,checkpoint_000000,True,False,1,b53cfe05,2024-06-25_16-47-40,67.51874232292175,67.51874232292175,122466,r3i5n6,10.159.8.159,67.51874232292175,1
diff --git a/ray_results/test_experiment/train_model_b53cfe05_16_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=8,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_16-43-18/result.json b/ray_results/test_experiment/train_model_b53cfe05_16_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=8,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_16-43-18/result.json
new file mode 100644
index 0000000..5643fc7
--- /dev/null
+++ b/ray_results/test_experiment/train_model_b53cfe05_16_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=8,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_16-43-18/result.json
@@ -0,0 +1 @@
+{"loss": 3022.543074480192, "timestamp": 1719326860, "checkpoint_dir_name": "checkpoint_000000", "should_checkpoint": true, "done": false, "training_iteration": 1, "trial_id": "b53cfe05", "date": "2024-06-25_16-47-40", "time_this_iter_s": 67.51874232292175, "time_total_s": 67.51874232292175, "pid": 122466, "hostname": "r3i5n6", "node_ip": "10.159.8.159", "config": {"encoder_num_layer": 2, "decoder_rt_num_layer": 2, "decoder_int_num_layer": 8, "embedding_dim": 64, "encoder_ff": 512, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 4, "drop_rate": 0.9936594412219328, "lr": 0.000836614555813242, "batch_size": 2048}, "time_since_restore": 67.51874232292175, "iterations_since_restore": 1}
diff --git a/ray_results/test_experiment/train_model_b577349d_6_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=2,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop_r_2024-06-25_16-13-12/events.out.tfevents.1719324918.r3i5n6 b/ray_results/test_experiment/train_model_b577349d_6_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=2,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop_r_2024-06-25_16-13-12/events.out.tfevents.1719324918.r3i5n6
new file mode 100644
index 0000000000000000000000000000000000000000..5b02452d78cd4c9ff5c103f3cc7a608047903a81
GIT binary patch
literal 405
zcmb1OfPlsI-b$QPK5e$$GVi9N6mL>dVrHJ6YguYuiIvflM?TAxq3SlwFL&k?wVD+Q
zQYXNuCB`Mf#aonEsb5l>m#UwWUtBEu`@$EOr<0Xs4WL@4e!URSDeVZ?qRgepB?Z?~
zoRMFelM<htk(!)cke``XBFezfVDF%ox)8<609UwvR4Y^R^HPCY7#Nm&^i0)<TA3Rw
z8^Fos2G^p*rNAW#x3VNNH#NQ_BeOU@vm~`BzF72*{(dK||JRlpLbcCrZhpx*1EF1o
zONmPwReNz}UUF)DQEG8Xeo-pefWKO`zs;crJb!U2l2h6pZod|n2A3k-0FbSTC7Jno
P#qp@dfPDUHZFM~WiwuP5
literal 0
HcmV?d00001
diff --git a/ray_results/test_experiment/train_model_b577349d_6_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=2,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop_r_2024-06-25_16-13-12/events.out.tfevents.1719327372.r3i5n6 b/ray_results/test_experiment/train_model_b577349d_6_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=2,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop_r_2024-06-25_16-13-12/events.out.tfevents.1719327372.r3i5n6
new file mode 100644
index 0000000000000000000000000000000000000000..06891b4c4f58089104309f635f7954f62a6f1cdf
GIT binary patch
literal 405
zcmb1OfPlsI-b$R__xKiXn|ISuiZ`h!F*8rkwJbHS#L6fyka>+VRNaR8<<6Y4Yb<ty
z)Cn+YiE)W=@fIak>X($}rRwM87Z;0KUHal;<GnxC0IFr`*9-BSTQ|bBD03-tNx`)g
zXXKaWq{Js@q$Xz<<Y(rUh%zuV*t=IH?nkjwcMDuUs+B4Ed8t4x3=G{xd{gzIR_4aa
z25=rmXi?%);F5$}S(2HX8efu;Ssb5Pl3EmBENb9&&`Da7L(vebeQtB}OHPlia2r*)
zl(?i(wHIgRC8x$0r52au7o~y?;A&r}U=B6l`HNGLoJSD`XmM$9DZ&i^*_v3AnV(l2
Nk7^9a=LJVpd;lzifc5|Y
literal 0
HcmV?d00001
diff --git a/ray_results/test_experiment/train_model_b577349d_6_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=2,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop_r_2024-06-25_16-13-12/params.json b/ray_results/test_experiment/train_model_b577349d_6_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=2,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop_r_2024-06-25_16-13-12/params.json
new file mode 100644
index 0000000..02e2b54
--- /dev/null
+++ b/ray_results/test_experiment/train_model_b577349d_6_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=2,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop_r_2024-06-25_16-13-12/params.json
@@ -0,0 +1,13 @@
+{
+ "batch_size": 2048,
+ "decoder_int_ff": 512,
+ "decoder_int_num_layer": 2,
+ "decoder_rt_ff": 1024,
+ "decoder_rt_num_layer": 1,
+ "drop_rate": 0.2945022566291742,
+ "embedding_dim": 64,
+ "encoder_ff": 2048,
+ "encoder_num_layer": 8,
+ "lr": 0.000496318687335349,
+ "n_head": 2
+}
\ No newline at end of file
diff --git a/ray_results/test_experiment/train_model_b577349d_6_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=2,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop_r_2024-06-25_16-13-12/params.pkl b/ray_results/test_experiment/train_model_b577349d_6_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=2,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop_r_2024-06-25_16-13-12/params.pkl
new file mode 100644
index 0000000000000000000000000000000000000000..e91d5dbb24c552bd6ce68238e51e96d05f3fa139
GIT binary patch
literal 224
zcmZo*oqCl40&1sd^a!TrCFiH47RBe4=EmnFR;CtB@#g3eNlAr>7L_1NF!qSTBr@|*
zWSDw*Q*)D2Q&KYX(&JMybEkMa^l-t9PD`8O%fQjY3)2fU6fDHj!v_-r83z_)>S4=^
z&qz&7nc~gV!<kZ)Ul3oESdu!$-Tu-|`F|$OGCfQ=ML?c|bB^=#<o7*XNr@%N8S%xL
KRjD9LO7#HG+EW|=
literal 0
HcmV?d00001
diff --git a/ray_results/test_experiment/train_model_b577349d_6_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=2,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop_r_2024-06-25_16-13-12/progress.csv b/ray_results/test_experiment/train_model_b577349d_6_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=2,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop_r_2024-06-25_16-13-12/progress.csv
new file mode 100644
index 0000000..82a172d
--- /dev/null
+++ b/ray_results/test_experiment/train_model_b577349d_6_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=2,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop_r_2024-06-25_16-13-12/progress.csv
@@ -0,0 +1,3 @@
+loss,timestamp,checkpoint_dir_name,should_checkpoint,done,training_iteration,trial_id,date,time_this_iter_s,time_total_s,pid,hostname,node_ip,time_since_restore,iterations_since_restore
+1958.5306944359006,1719325014,checkpoint_000000,True,False,1,b577349d,2024-06-25_16-16-54,95.59347629547119,95.59347629547119,69318,r3i5n6,10.159.8.159,95.59347629547119,1
+1958.5695377860482,1719327468,checkpoint_000001,True,False,2,b577349d,2024-06-25_16-57-48,96.6448974609375,192.2383737564087,130020,r3i5n6,10.159.8.159,96.6448974609375,1
diff --git a/ray_results/test_experiment/train_model_b577349d_6_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=2,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop_r_2024-06-25_16-13-12/result.json b/ray_results/test_experiment/train_model_b577349d_6_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=2,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop_r_2024-06-25_16-13-12/result.json
new file mode 100644
index 0000000..8befa38
--- /dev/null
+++ b/ray_results/test_experiment/train_model_b577349d_6_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=2,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop_r_2024-06-25_16-13-12/result.json
@@ -0,0 +1,2 @@
+{"loss": 1958.5306944359006, "timestamp": 1719325014, "checkpoint_dir_name": "checkpoint_000000", "should_checkpoint": true, "done": false, "training_iteration": 1, "trial_id": "b577349d", "date": "2024-06-25_16-16-54", "time_this_iter_s": 95.59347629547119, "time_total_s": 95.59347629547119, "pid": 69318, "hostname": "r3i5n6", "node_ip": "10.159.8.159", "config": {"encoder_num_layer": 8, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 2, "embedding_dim": 64, "encoder_ff": 2048, "decoder_rt_ff": 1024, "decoder_int_ff": 512, "n_head": 2, "drop_rate": 0.2945022566291742, "lr": 0.000496318687335349, "batch_size": 2048}, "time_since_restore": 95.59347629547119, "iterations_since_restore": 1}
+{"loss": 1958.5695377860482, "timestamp": 1719327468, "checkpoint_dir_name": "checkpoint_000001", "should_checkpoint": true, "done": false, "training_iteration": 2, "trial_id": "b577349d", "date": "2024-06-25_16-57-48", "time_this_iter_s": 96.6448974609375, "time_total_s": 192.2383737564087, "pid": 130020, "hostname": "r3i5n6", "node_ip": "10.159.8.159", "config": {"encoder_num_layer": 8, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 2, "embedding_dim": 64, "encoder_ff": 2048, "decoder_rt_ff": 1024, "decoder_int_ff": 512, "n_head": 2, "drop_rate": 0.2945022566291742, "lr": 0.000496318687335349, "batch_size": 2048}, "time_since_restore": 96.6448974609375, "iterations_since_restore": 1}
diff --git a/ray_results/test_experiment/train_model_bea15841_17_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=4,drop__2024-06-25_17-46-02/events.out.tfevents.1719332768.r8i6n2 b/ray_results/test_experiment/train_model_bea15841_17_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=4,drop__2024-06-25_17-46-02/events.out.tfevents.1719332768.r8i6n2
new file mode 100644
index 0000000000000000000000000000000000000000..2e003b6b19e54d5f43ca7c25feeb64658d56a1f6
GIT binary patch
literal 1676
zcma)6T}TvB6kcb;niH$;<eIKpHWtNVZq`F!)~;(|T9Fw&MA$U$+-;}anPq0Ilpw*R
z9(wS>f&!C%D2OOgBFwN~B6{iPCA7knK&2vI>`#jBxU+Y6%)fb?bIy0Zd%yFYn{J7G
zS59nW>8t(r;qHsQ#ZFtb!i!S=mLo{g3fErS+8f9<>e}0mme8HPeSh`3lmu5Aq=HT1
z>)cwcgxoc<syYUm#y5{Ws=U6&fSetB#nER<2_zSmfs;Vgpj=xcaQ-0j9}df+q&X~>
z22W*D;DL#9u#Di%l!7cF46#`9*X3<+8<e4{><YTRoIqH}feb>ai6O*kK~d#I4Jn-J
z@RfD=K3#aWqriZ-Ha3pXzhm&_z(S@Ou8NW$aSBp3SwRyGTJF7ExZY?mH2h*OeS90?
zcfl%HMjGg>ye7($%9-2fH{bH}W0C3R9evsM`?}cFRAP_N<C{ylU1=t<pS~lIxrn{z
zPTmF+<@WakKaJR3pSuQ3V!w$&W)XYHe$Kaa<I2}2v3GtTjI)Z}vU+*Rlk}JSO=3Tf
zHJG#57Vp<U%Bl8*$k3Ww&mW@O4*b~t5R2`)YK2QyNaccIUg1M3=SIiENWsRDwDQwd
zXtKh5BgOopnC}#u1W?Uu{vaM{J<74N3?vE2FALaU*wkDgfMXfph>uCNA+Cn6LkdpW
z!4^#^Ms+%*#)aaGZ9of(9Okf#bSjKRp=u-uqI8fG#1IbQ(;z9rKEdn&d-OlPQ6f+^
zIy*q3#03#A<XByHNM_6%?OFm^%nTbl38HVLD;qMI={L@`6tbC_Hp;aEa+%pTX83L2
zG0B+U<8PM>>Hg^a1}W&LAS*UM6{#v-IZoAhEU+jA<nkU0x~bS4Z&S{j6r{yz%h2`<
zO+9~3ei`w=qNqeT7%@zW7o3x(Msm$wSTcp=B#*I~lMO<W4PN|0PC0v0g7L}v>~Nrr
z4d3V-^?CHem~23bE4;8c-VY>!Pl*#T+r}r@X}n5mMm@z{w>A|SFW&RJS1ReTsQLxq
a2Iv3UBT0xTk3AOCeI83KNT93swf+MbfDNVq
literal 0
HcmV?d00001
diff --git a/ray_results/test_experiment/train_model_bea15841_17_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=4,drop__2024-06-25_17-46-02/params.json b/ray_results/test_experiment/train_model_bea15841_17_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=4,drop__2024-06-25_17-46-02/params.json
new file mode 100644
index 0000000..7fa69d6
--- /dev/null
+++ b/ray_results/test_experiment/train_model_bea15841_17_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=4,drop__2024-06-25_17-46-02/params.json
@@ -0,0 +1,13 @@
+{
+ "batch_size": 1024,
+ "decoder_int_ff": 512,
+ "decoder_int_num_layer": 1,
+ "decoder_rt_ff": 1024,
+ "decoder_rt_num_layer": 4,
+ "drop_rate": 0.8101220088903347,
+ "embedding_dim": 64,
+ "encoder_ff": 1024,
+ "encoder_num_layer": 4,
+ "lr": 0.0005780102991375737,
+ "n_head": 4
+}
\ No newline at end of file
diff --git a/ray_results/test_experiment/train_model_bea15841_17_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=4,drop__2024-06-25_17-46-02/params.pkl b/ray_results/test_experiment/train_model_bea15841_17_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=4,drop__2024-06-25_17-46-02/params.pkl
new file mode 100644
index 0000000000000000000000000000000000000000..70deec0b1ae58c8b4addc77a860c6182b5ffdb33
GIT binary patch
literal 224
zcmZo*oqCl40&1sd^a!TrCFiH47RBe4=EmnFR;CtB@n-1}NlAr>7L}k%h{7Z?^H5|M
zdw5fGlTuSsGV{{oQ!;a>csulP!HiBzo8rsB(!&eW3p5le#0L`s83z_)>S4=^&qz&7
z0Xc&+r6|83z9_LIb&9+F%Qr25_qERIVah22@|-?wUR9vv*~68TSdyF(Uz}N$3bLeB
F4*&_fRI300
literal 0
HcmV?d00001
diff --git a/ray_results/test_experiment/train_model_bea15841_17_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=4,drop__2024-06-25_17-46-02/progress.csv b/ray_results/test_experiment/train_model_bea15841_17_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=4,drop__2024-06-25_17-46-02/progress.csv
new file mode 100644
index 0000000..2fd8fe1
--- /dev/null
+++ b/ray_results/test_experiment/train_model_bea15841_17_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=4,drop__2024-06-25_17-46-02/progress.csv
@@ -0,0 +1,3 @@
+loss,timestamp,checkpoint_dir_name,should_checkpoint,done,training_iteration,trial_id,date,time_this_iter_s,time_total_s,pid,hostname,node_ip,time_since_restore,iterations_since_restore
+2012.1837220680056,1719332857,checkpoint_000000,True,False,1,bea15841,2024-06-25_18-27-37,89.14307808876038,89.14307808876038,2195253,r8i6n2,10.159.28.60,89.14307808876038,1
+2008.3096395177165,1719332932,checkpoint_000001,True,False,2,bea15841,2024-06-25_18-28-52,75.03496551513672,164.1780436038971,2195253,r8i6n2,10.159.28.60,164.1780436038971,2
diff --git a/ray_results/test_experiment/train_model_bea15841_17_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=4,drop__2024-06-25_17-46-02/result.json b/ray_results/test_experiment/train_model_bea15841_17_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=4,drop__2024-06-25_17-46-02/result.json
new file mode 100644
index 0000000..bbf7ae8
--- /dev/null
+++ b/ray_results/test_experiment/train_model_bea15841_17_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=4,drop__2024-06-25_17-46-02/result.json
@@ -0,0 +1,2 @@
+{"loss": 2012.1837220680056, "timestamp": 1719332857, "checkpoint_dir_name": "checkpoint_000000", "should_checkpoint": true, "done": false, "training_iteration": 1, "trial_id": "bea15841", "date": "2024-06-25_18-27-37", "time_this_iter_s": 89.14307808876038, "time_total_s": 89.14307808876038, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 4, "decoder_rt_num_layer": 4, "decoder_int_num_layer": 1, "embedding_dim": 64, "encoder_ff": 1024, "decoder_rt_ff": 1024, "decoder_int_ff": 512, "n_head": 4, "drop_rate": 0.8101220088903347, "lr": 0.0005780102991375737, "batch_size": 1024}, "time_since_restore": 89.14307808876038, "iterations_since_restore": 1}
+{"loss": 2008.3096395177165, "timestamp": 1719332932, "checkpoint_dir_name": "checkpoint_000001", "should_checkpoint": true, "done": false, "training_iteration": 2, "trial_id": "bea15841", "date": "2024-06-25_18-28-52", "time_this_iter_s": 75.03496551513672, "time_total_s": 164.1780436038971, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 4, "decoder_rt_num_layer": 4, "decoder_int_num_layer": 1, "embedding_dim": 64, "encoder_ff": 1024, "decoder_rt_ff": 1024, "decoder_int_ff": 512, "n_head": 4, "drop_rate": 0.8101220088903347, "lr": 0.0005780102991375737, "batch_size": 1024}, "time_since_restore": 164.1780436038971, "iterations_since_restore": 2}
diff --git a/ray_results/test_experiment/train_model_c2c99680_18_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_18-26-08/events.out.tfevents.1719332932.r8i6n2 b/ray_results/test_experiment/train_model_c2c99680_18_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_18-26-08/events.out.tfevents.1719332932.r8i6n2
new file mode 100644
index 0000000000000000000000000000000000000000..80186fdfd44a0eb3cc6b9b6fd09db2190d925fe2
GIT binary patch
literal 770
zcmb1OfPlsI-b$Pdt26}n%)99*#hX-=n3<>NT9%quVrBHIRFGd8s&2#la%WDCW3kO3
zbpni9Vq79zyhVwX`X!}#srotj#l@nEX+K;JXm)cNK($Q$dLf?k;AyxPWiCZ7DY%y6
zjQrA^l=$R~)a2}f{LH)(Q3i$v`;_1Nr=VD=a1O2?)ykCoyi}kT28J)UCqCAPTA3Rw
z8^E~-p+$*HflCr@Wl3gkYJ5pXW^sIGNorAivFOCiC(inck2(#Z+UGVmzvOf{54TZ;
zONmPwReNz}UUF)DQEG8Xeo-pefcvk~ZOowtJb!U2l5+>b04**JE=9NjAX^hlGV}9_
z<57(P`FxxA+UY1huc?Up2o7u}Y=Iqr@w<!f6Mh$zz}{L3*FsES&%J%%7K)WlRdD?{
z0-N2#Zaqq1pG9aP5ZL<|?l>1Md_4;#u>GsyHWCbM4bub{@&E3bD1m(%VF1yA&ET-0
HHf9O{_1yp8
literal 0
HcmV?d00001
diff --git a/ray_results/test_experiment/train_model_c2c99680_18_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_18-26-08/events.out.tfevents.1719337296.r8i6n2 b/ray_results/test_experiment/train_model_c2c99680_18_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_18-26-08/events.out.tfevents.1719337296.r8i6n2
new file mode 100644
index 0000000000000000000000000000000000000000..8cc354ae2695e10c3f3312f2dd0ea2ed51a45b18
GIT binary patch
literal 1676
zcma)6T}TvB6kcaUn?q_lx~A(-x+cXEZkiy8<d*iL5tY42$z|NRwxjNh%#45akYGZf
zf|XJ|Xqi0*`jSDCq!EZuB|%AqpqTZNl9Ysj-F3&iJLccK%{k{g-@V`Y&P}iczJZ&i
z4lZ{k?DFfr>C^V8N>z~Ltg{VR*7I}jzsmoTVbl#hYAoV<%5J^&*BxM;@el`5s?hAx
z>t*b!RW!}^vwgMrLu2Io;|8SX*E__aatg_Sbg)y1=2hxzMda~fPfeX7$-2#AX(?Qu
zd*e1yE?uE`Go`4=m_RI+g5i^WE`!onk#d1+ze*tvNP|R5sY^bLbg!f#NyjSEY-8+P
z(Y?$rHQRvqwzhuf_E%8&VQ@f_8LmmP2O|}0x}xIs24}q8LY~oJ;oD?6*A{Hxgrks7
z8~9lTT~cHXncMhpKG%_3ZMwM>cK*beU+nO$#AX)$7LVH7rc7dYG8D2GvGWFFmrayb
zmg2V&yJ6%MHi`Wp2-!vKfxDlJnis#`F^SEGQ^sAz&M8I3^Q*ltO=5Qh8|+zZi~Hit
z>W!hWz|gvSA69ey&yP%x6S197S?FgW4teVYRq$!Zg`4WIN{l1xCAV2<V<FN=@qZZp
zH{*x~s1$UMmyGl}PGcQBM2py?h{RyT)TpY8#1g?48k6gNs8(pkDoI%(Y9pl@)d>(6
z5(+Jj0!~zwIz(LLry^MFtHh!x$=8r5`ACSI2Fn5Vb!Hn_H~vE#MF3OlX9tLokrxYM
z8tb$|3~%0OXDTG~J8bMGh<_uUDUif(zj4k(kize@QO-=r;CJ7crn2-b*_hw>;*)%C
zZDW4349qZ)9GstqHI1ws(sY3cEXh?$$tVLOOmL2ajQtq{@gdssoOOQ1^(pt`Y$3#M
zNJN4G!z>koebdxvuFDOnTS!jx2i%af*&sC8;wCTTmb0fNggeGRI}(Un>zs6VpIbkS
z$p*BT9OhuCA2ua|wxx;3Zt~)7?NwT{RJd|%BCWu9@g`mjT;k$~jqS+>7uf&T9?N1t
SdE&94?%@t*U)YZtaq1rcjtyl1
literal 0
HcmV?d00001
diff --git a/ray_results/test_experiment/train_model_c2c99680_18_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_18-26-08/params.json b/ray_results/test_experiment/train_model_c2c99680_18_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_18-26-08/params.json
new file mode 100644
index 0000000..073d458
--- /dev/null
+++ b/ray_results/test_experiment/train_model_c2c99680_18_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_18-26-08/params.json
@@ -0,0 +1,13 @@
+{
+ "batch_size": 2048,
+ "decoder_int_ff": 512,
+ "decoder_int_num_layer": 1,
+ "decoder_rt_ff": 1024,
+ "decoder_rt_num_layer": 1,
+ "drop_rate": 0.5887703154000944,
+ "embedding_dim": 1024,
+ "encoder_ff": 1024,
+ "encoder_num_layer": 4,
+ "lr": 0.0008721653331207079,
+ "n_head": 1
+}
\ No newline at end of file
diff --git a/ray_results/test_experiment/train_model_c2c99680_18_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_18-26-08/params.pkl b/ray_results/test_experiment/train_model_c2c99680_18_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_18-26-08/params.pkl
new file mode 100644
index 0000000000000000000000000000000000000000..82eaeb8610755e83829654c9acc2d2d1ef3e9c33
GIT binary patch
literal 225
zcmZo*oqCM{0&1sd^a!TrCFiH47RBe4=EmnFR;CtB@n-1}NlAr>7L_1NF!qSTBr@~R
zWO!3^lTuSsGV{{oQ!;a>_%g8caKX$@O9S(HVVZ%aLWTHXLLl?NVoW`3dGQ&ki78XO
z8GATWit-EMixNvxr?}fcx^8kq|NGM(rko-m&u5Cmsowu0JzPnNCCM4_#hF#9Kub7E
F^#BA5R4o7i
literal 0
HcmV?d00001
diff --git a/ray_results/test_experiment/train_model_c2c99680_18_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_18-26-08/progress.csv b/ray_results/test_experiment/train_model_c2c99680_18_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_18-26-08/progress.csv
new file mode 100644
index 0000000..74c0983
--- /dev/null
+++ b/ray_results/test_experiment/train_model_c2c99680_18_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_18-26-08/progress.csv
@@ -0,0 +1,5 @@
+loss,timestamp,checkpoint_dir_name,should_checkpoint,done,training_iteration,trial_id,date,time_this_iter_s,time_total_s,pid,hostname,node_ip,time_since_restore,iterations_since_restore
+1987.1915456216166,1719333389,checkpoint_000000,True,False,1,c2c99680,2024-06-25_18-36-29,456.8247263431549,456.8247263431549,2195253,r8i6n2,10.159.28.60,456.8247263431549,1
+1982.5428274560163,1719333829,checkpoint_000001,True,False,2,c2c99680,2024-06-25_18-43-49,440.0058114528656,896.8305377960205,2195253,r8i6n2,10.159.28.60,896.8305377960205,2
+2012.3732496847317,1719337749,checkpoint_000002,True,False,3,c2c99680,2024-06-25_19-49-10,454.0293221473694,1350.85985994339,2195253,r8i6n2,10.159.28.60,454.0293221473694,1
+2023.2190220149482,1719338188,checkpoint_000003,True,False,4,c2c99680,2024-06-25_19-56-29,439.1456036567688,1790.0054636001587,2195253,r8i6n2,10.159.28.60,893.1749258041382,2
diff --git a/ray_results/test_experiment/train_model_c2c99680_18_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_18-26-08/result.json b/ray_results/test_experiment/train_model_c2c99680_18_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_18-26-08/result.json
new file mode 100644
index 0000000..98eb284
--- /dev/null
+++ b/ray_results/test_experiment/train_model_c2c99680_18_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_18-26-08/result.json
@@ -0,0 +1,4 @@
+{"loss": 1987.1915456216166, "timestamp": 1719333389, "checkpoint_dir_name": "checkpoint_000000", "should_checkpoint": true, "done": false, "training_iteration": 1, "trial_id": "c2c99680", "date": "2024-06-25_18-36-29", "time_this_iter_s": 456.8247263431549, "time_total_s": 456.8247263431549, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 4, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 1024, "encoder_ff": 1024, "decoder_rt_ff": 1024, "decoder_int_ff": 512, "n_head": 1, "drop_rate": 0.5887703154000944, "lr": 0.0008721653331207079, "batch_size": 2048}, "time_since_restore": 456.8247263431549, "iterations_since_restore": 1}
+{"loss": 1982.5428274560163, "timestamp": 1719333829, "checkpoint_dir_name": "checkpoint_000001", "should_checkpoint": true, "done": false, "training_iteration": 2, "trial_id": "c2c99680", "date": "2024-06-25_18-43-49", "time_this_iter_s": 440.0058114528656, "time_total_s": 896.8305377960205, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 4, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 1024, "encoder_ff": 1024, "decoder_rt_ff": 1024, "decoder_int_ff": 512, "n_head": 1, "drop_rate": 0.5887703154000944, "lr": 0.0008721653331207079, "batch_size": 2048}, "time_since_restore": 896.8305377960205, "iterations_since_restore": 2}
+{"loss": 2012.3732496847317, "timestamp": 1719337749, "checkpoint_dir_name": "checkpoint_000002", "should_checkpoint": true, "done": false, "training_iteration": 3, "trial_id": "c2c99680", "date": "2024-06-25_19-49-10", "time_this_iter_s": 454.0293221473694, "time_total_s": 1350.85985994339, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 4, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 1024, "encoder_ff": 1024, "decoder_rt_ff": 1024, "decoder_int_ff": 512, "n_head": 1, "drop_rate": 0.5887703154000944, "lr": 0.0008721653331207079, "batch_size": 2048}, "time_since_restore": 454.0293221473694, "iterations_since_restore": 1}
+{"loss": 2023.2190220149482, "timestamp": 1719338188, "checkpoint_dir_name": "checkpoint_000003", "should_checkpoint": true, "done": false, "training_iteration": 4, "trial_id": "c2c99680", "date": "2024-06-25_19-56-29", "time_this_iter_s": 439.1456036567688, "time_total_s": 1790.0054636001587, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 4, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 1024, "encoder_ff": 1024, "decoder_rt_ff": 1024, "decoder_int_ff": 512, "n_head": 1, "drop_rate": 0.5887703154000944, "lr": 0.0008721653331207079, "batch_size": 2048}, "time_since_restore": 893.1749258041382, "iterations_since_restore": 2}
diff --git a/ray_results/test_experiment/train_model_c4172973_8_batch_size=2048,decoder_int_ff=2048,decoder_int_num_layer=8,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_16-16-54/events.out.tfevents.1719325156.r3i5n6 b/ray_results/test_experiment/train_model_c4172973_8_batch_size=2048,decoder_int_ff=2048,decoder_int_num_layer=8,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_16-16-54/events.out.tfevents.1719325156.r3i5n6
new file mode 100644
index 0000000000000000000000000000000000000000..857d34b750d8b2e30b5df5659eea38d004cdbc52
GIT binary patch
literal 405
zcmb1OfPlsI-b$P-n*}Sk%)99*#hX-=n3<>NT9%quVrAr_#jrvds&2#la%WEQsx>_z
zbpni9Vq79zyhVwX`X!}#srotj#l@npcWiR@tbW#R0M#<}>xFpE@J6^6WiCZ7DY%y6
zjQrA^l=$R~)a2}f{LH)(Q3i$v`)_{?Poh}K+6LE;YGq1(UMf%v1H(k8BrAQWmASF9
z0i4GXT9mjHxFq3LmSpCp#+PJd7RP6nq!z^&i;7&C<Mb|(Z;>HX``qT{mz*x`a2r*)
zl(?i(wHIgRC8x$0r52au7o~y?5G}~6HHRAT{KctA&VvX8w74|56yXMdY)vf5%+D*1
NM>PiIbK#(+S^$`TgLeP`
literal 0
HcmV?d00001
diff --git a/ray_results/test_experiment/train_model_c4172973_8_batch_size=2048,decoder_int_ff=2048,decoder_int_num_layer=8,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_16-16-54/events.out.tfevents.1719327610.r3i5n6 b/ray_results/test_experiment/train_model_c4172973_8_batch_size=2048,decoder_int_ff=2048,decoder_int_num_layer=8,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_16-16-54/events.out.tfevents.1719327610.r3i5n6
new file mode 100644
index 0000000000000000000000000000000000000000..be46ae5ad11034bb8f06f1cfcc1d9f6026229e6e
GIT binary patch
literal 405
zcmb1OfPlsI-b$QpavSb#n|ISuiZ`h!F*8rkwJbHS#LDQ!bXiYjsJadF%bhuWXSRI;
zsS{w*65|r#;w?(7)GsN`OV!WGFD@3{v!Kb@&Cxx^0IFr`*9-BS)91mpD03-tNx`)g
zXXKaWq{Js@q$Xz<<Y(rUh%zuV*t=GDo<OnEZy{Vis+B4Ed8t4x3=9+8RSxMxt;~&;
z4d8r>(4xepz$FQ{vLrJ%HNGSxvp7DpB(*5MSaizzc}|xXvxOT%wa;yCe#u#~2yUYa
zmlBsWs`lc{yyVpQqSWG&{GwE_0rONk!_1)uJb!U2lJhIV04**JE=9NjAX^hlGV}9_
O<57(P`CL=H&>jHp&WA7n
literal 0
HcmV?d00001
diff --git a/ray_results/test_experiment/train_model_c4172973_8_batch_size=2048,decoder_int_ff=2048,decoder_int_num_layer=8,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_16-16-54/params.json b/ray_results/test_experiment/train_model_c4172973_8_batch_size=2048,decoder_int_ff=2048,decoder_int_num_layer=8,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_16-16-54/params.json
new file mode 100644
index 0000000..f7623f2
--- /dev/null
+++ b/ray_results/test_experiment/train_model_c4172973_8_batch_size=2048,decoder_int_ff=2048,decoder_int_num_layer=8,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_16-16-54/params.json
@@ -0,0 +1,13 @@
+{
+ "batch_size": 2048,
+ "decoder_int_ff": 2048,
+ "decoder_int_num_layer": 8,
+ "decoder_rt_ff": 512,
+ "decoder_rt_num_layer": 2,
+ "drop_rate": 0.4176788223006639,
+ "embedding_dim": 256,
+ "encoder_ff": 1024,
+ "encoder_num_layer": 2,
+ "lr": 0.00038150746043137153,
+ "n_head": 2
+}
\ No newline at end of file
diff --git a/ray_results/test_experiment/train_model_c4172973_8_batch_size=2048,decoder_int_ff=2048,decoder_int_num_layer=8,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_16-16-54/params.pkl b/ray_results/test_experiment/train_model_c4172973_8_batch_size=2048,decoder_int_ff=2048,decoder_int_num_layer=8,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_16-16-54/params.pkl
new file mode 100644
index 0000000000000000000000000000000000000000..22dbfe027d19fb71871939982c85229f0b51a0ef
GIT binary patch
literal 225
zcmZo*oqCM{0&1sd^a!TrCFiH47RBe4=EmnFR;CtB@n-4~NlAr>7L}k%h{7Z?^H5|s
zdU#WFlTuSsGV{{oQ!;a>_%bl|aKX$@O9S#*dU#=)fu@3mn0okNLLl?NVjMkedGQ&k
zi76m=aHbUH7sMAOmZVN`x4*U9{!7F4v>v9MA|TI_VUfV)I}3Wak`hajGvbRgt5SiM
IaFpr+0Q0m{ng9R*
literal 0
HcmV?d00001
diff --git a/ray_results/test_experiment/train_model_c4172973_8_batch_size=2048,decoder_int_ff=2048,decoder_int_num_layer=8,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_16-16-54/progress.csv b/ray_results/test_experiment/train_model_c4172973_8_batch_size=2048,decoder_int_ff=2048,decoder_int_num_layer=8,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_16-16-54/progress.csv
new file mode 100644
index 0000000..7ed4de8
--- /dev/null
+++ b/ray_results/test_experiment/train_model_c4172973_8_batch_size=2048,decoder_int_ff=2048,decoder_int_num_layer=8,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_16-16-54/progress.csv
@@ -0,0 +1,3 @@
+loss,timestamp,checkpoint_dir_name,should_checkpoint,done,training_iteration,trial_id,date,time_this_iter_s,time_total_s,pid,hostname,node_ip,time_since_restore,iterations_since_restore
+357.444681242695,1719325234,checkpoint_000000,True,False,1,c4172973,2024-06-25_16-20-34,78.41421294212341,78.41421294212341,69318,r3i5n6,10.159.8.159,78.41421294212341,1
+261.25574487400803,1719327690,checkpoint_000001,True,False,2,c4172973,2024-06-25_17-01-30,79.34292578697205,157.75713872909546,130020,r3i5n6,10.159.8.159,79.34292578697205,1
diff --git a/ray_results/test_experiment/train_model_c4172973_8_batch_size=2048,decoder_int_ff=2048,decoder_int_num_layer=8,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_16-16-54/result.json b/ray_results/test_experiment/train_model_c4172973_8_batch_size=2048,decoder_int_ff=2048,decoder_int_num_layer=8,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_16-16-54/result.json
new file mode 100644
index 0000000..cee08e9
--- /dev/null
+++ b/ray_results/test_experiment/train_model_c4172973_8_batch_size=2048,decoder_int_ff=2048,decoder_int_num_layer=8,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_16-16-54/result.json
@@ -0,0 +1,2 @@
+{"loss": 357.444681242695, "timestamp": 1719325234, "checkpoint_dir_name": "checkpoint_000000", "should_checkpoint": true, "done": false, "training_iteration": 1, "trial_id": "c4172973", "date": "2024-06-25_16-20-34", "time_this_iter_s": 78.41421294212341, "time_total_s": 78.41421294212341, "pid": 69318, "hostname": "r3i5n6", "node_ip": "10.159.8.159", "config": {"encoder_num_layer": 2, "decoder_rt_num_layer": 2, "decoder_int_num_layer": 8, "embedding_dim": 256, "encoder_ff": 1024, "decoder_rt_ff": 512, "decoder_int_ff": 2048, "n_head": 2, "drop_rate": 0.4176788223006639, "lr": 0.00038150746043137153, "batch_size": 2048}, "time_since_restore": 78.41421294212341, "iterations_since_restore": 1}
+{"loss": 261.25574487400803, "timestamp": 1719327690, "checkpoint_dir_name": "checkpoint_000001", "should_checkpoint": true, "done": false, "training_iteration": 2, "trial_id": "c4172973", "date": "2024-06-25_17-01-30", "time_this_iter_s": 79.34292578697205, "time_total_s": 157.75713872909546, "pid": 130020, "hostname": "r3i5n6", "node_ip": "10.159.8.159", "config": {"encoder_num_layer": 2, "decoder_rt_num_layer": 2, "decoder_int_num_layer": 8, "embedding_dim": 256, "encoder_ff": 1024, "decoder_rt_ff": 512, "decoder_int_ff": 2048, "n_head": 2, "drop_rate": 0.4176788223006639, "lr": 0.00038150746043137153, "batch_size": 2048}, "time_since_restore": 79.34292578697205, "iterations_since_restore": 1}
diff --git a/ray_results/test_experiment/train_model_dd950274_1_batch_size=2048,decoder_int_ff=1024,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_15-19-46/events.out.tfevents.1719321603.r7i5n7 b/ray_results/test_experiment/train_model_dd950274_1_batch_size=2048,decoder_int_ff=1024,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_15-19-46/events.out.tfevents.1719321603.r7i5n7
new file mode 100644
index 0000000000000000000000000000000000000000..fa935586fae83d743c2e340fe35b13aabe51b2ae
GIT binary patch
literal 405
zcmb1OfPlsI-b$S7DvvR2oOjbviZ`h!F*8rkwJbHS#L8&;v#avTP<0#TmpgOHs>q6i
z)Cn+ZiE)W=@fIak>X($}rRwM87Z;1Ju=wTj#Ozd&0aVM>uNUGuTeRU?l(`hSq~KbL
zGxAGwQsR>{Qj@a_@-y>FL>U+w>{nd(ltHmF-vF*3)ykCoyi}kT28JoC<!tn!R_4aa
z25|Zs!nG)IDR4=`tt`pRO^q+f$SjV}EJ-bjFBUz$X_HfEcjpR2sP?(d%`Z9cBDAY;
zDRD`oYA?>rOHPe1N-ZwQFG>X);5;exzd6)^=Pyo0awZzV?bqVc;8KJe0J1f)Br`v+
PI3Cp)kk1>IZ2JxX^Fn|+
literal 0
HcmV?d00001
diff --git a/ray_results/test_experiment/train_model_dd950274_1_batch_size=2048,decoder_int_ff=1024,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_15-19-46/params.json b/ray_results/test_experiment/train_model_dd950274_1_batch_size=2048,decoder_int_ff=1024,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_15-19-46/params.json
new file mode 100644
index 0000000..6e93c3a
--- /dev/null
+++ b/ray_results/test_experiment/train_model_dd950274_1_batch_size=2048,decoder_int_ff=1024,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_15-19-46/params.json
@@ -0,0 +1,13 @@
+{
+ "batch_size": 2048,
+ "decoder_int_ff": 1024,
+ "decoder_int_num_layer": 1,
+ "decoder_rt_ff": 1024,
+ "decoder_rt_num_layer": 1,
+ "drop_rate": 0.7776595209025089,
+ "embedding_dim": 16,
+ "encoder_ff": 1024,
+ "encoder_num_layer": 8,
+ "lr": 0.0006106789085082022,
+ "n_head": 16
+}
\ No newline at end of file
diff --git a/ray_results/test_experiment/train_model_dd950274_1_batch_size=2048,decoder_int_ff=1024,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_15-19-46/params.pkl b/ray_results/test_experiment/train_model_dd950274_1_batch_size=2048,decoder_int_ff=1024,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_15-19-46/params.pkl
new file mode 100644
index 0000000000000000000000000000000000000000..f685246d35e73936eedd2db61601ee2b0a9426ae
GIT binary patch
literal 224
zcmZo*oqCl40&1sd^a!TrCFiH47RBe4=EmnFR;CtB@#g3eNlAr>7L_1NF!qSTBr@~R
zWO!3^lTuSsGV{{oQ!;a>cnkD!!HiBzo8rsB(!&eW3p5le#0L`s83z?(%ZtxQO-uoq
zz?o8%Ul3oESdu!$-TuX+X%=P)n|qjYihw*9rhPx+S!#N?k`hajGvbRgt5SiMaFpr+
E0J-B*tpET3
literal 0
HcmV?d00001
diff --git a/ray_results/test_experiment/train_model_dd950274_1_batch_size=2048,decoder_int_ff=1024,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_15-19-46/progress.csv b/ray_results/test_experiment/train_model_dd950274_1_batch_size=2048,decoder_int_ff=1024,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_15-19-46/progress.csv
new file mode 100644
index 0000000..28c9255
--- /dev/null
+++ b/ray_results/test_experiment/train_model_dd950274_1_batch_size=2048,decoder_int_ff=1024,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_15-19-46/progress.csv
@@ -0,0 +1,2 @@
+loss,timestamp,checkpoint_dir_name,should_checkpoint,done,training_iteration,trial_id,date,time_this_iter_s,time_total_s,pid,hostname,node_ip,time_since_restore,iterations_since_restore
+2001.7704808993603,1719321692,checkpoint_000000,True,False,1,dd950274,2024-06-25_15-21-32,89.34920597076416,89.34920597076416,355319,r7i5n7,10.159.24.56,89.34920597076416,1
diff --git a/ray_results/test_experiment/train_model_dd950274_1_batch_size=2048,decoder_int_ff=1024,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_15-19-46/result.json b/ray_results/test_experiment/train_model_dd950274_1_batch_size=2048,decoder_int_ff=1024,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_15-19-46/result.json
new file mode 100644
index 0000000..c20e0ec
--- /dev/null
+++ b/ray_results/test_experiment/train_model_dd950274_1_batch_size=2048,decoder_int_ff=1024,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=1,drop__2024-06-25_15-19-46/result.json
@@ -0,0 +1 @@
+{"loss": 2001.7704808993603, "timestamp": 1719321692, "checkpoint_dir_name": "checkpoint_000000", "should_checkpoint": true, "done": false, "training_iteration": 1, "trial_id": "dd950274", "date": "2024-06-25_15-21-32", "time_this_iter_s": 89.34920597076416, "time_total_s": 89.34920597076416, "pid": 355319, "hostname": "r7i5n7", "node_ip": "10.159.24.56", "config": {"encoder_num_layer": 8, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 1024, "decoder_rt_ff": 1024, "decoder_int_ff": 1024, "n_head": 16, "drop_rate": 0.7776595209025089, "lr": 0.0006106789085082022, "batch_size": 2048}, "time_since_restore": 89.34920597076416, "iterations_since_restore": 1}
diff --git a/ray_results/test_experiment/train_model_de1e9134_30_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=1,drop_r_2024-06-25_22-20-23/events.out.tfevents.1719348339.r8i6n2 b/ray_results/test_experiment/train_model_de1e9134_30_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=1,drop_r_2024-06-25_22-20-23/events.out.tfevents.1719348339.r8i6n2
new file mode 100644
index 0000000000000000000000000000000000000000..7870273a07987a5f6341b5403eed9658bab51da4
GIT binary patch
literal 1865
zcmb1OfPlsI-b$Q2ddFl=&%5a;#hX-=n3<>NT9%quVr5h)Rnw&mRkvY&xija&`FnLi
z>I4|I#JEJbc#9G%^-D_gQuTB4i;G1QrhReIlVbW}0M#<}>xFpE!sT!+%3O+EQgAKB
z8Tq9-De=h}sma*|`I&hoq6`cT_ATP3Gf}KOuoA8x)ykCoyi}kT1_m$oR5yL7mASF9
z0h|r1;98Wp6u2beR+ePurpA|KWERI~mZTQN7mIG)(e6}WyKS-|RQufK=9ir6tKr&J
zxRki0QMDIm<|U`b7o`@L<QJua4OkjbC14IU;Q5PFk(_f72550<a4EtK0NI*Yl9``Z
z9FJ-Y$mjl98&gnx?$#%64GwH3Y=NzG`K!w#&Cq<5zz*$)Yau4Et@Zv%pjdeip&v(J
zr`7LLLJ91&32-Y31h(DGP^S>?KWk6|`yN6&!N3k_{p7T0)rn}7zz&}Xx1Z?1W^jm_
ztY3@b^T&xr9^k-c#unI%V!pVzEKr<)64+0Y;97_Y>;>i@D^RS=Oor>n5!m8K7D%83
z_9uiE0)c&HMW~b6En(D@ot*-=kziooU!&)|FH46LC9v-!3?MqN9UN+}x*SIFx#0cN
zq2Rz~!4}wiYQDHAd(2yc64-(d;97_YY~y=@uTZR9jL?rGuz6puaX|@ey@zls2?X|z
zNuf?A^X@H1N!cq9+6e}BuV9LEzuw*-D1j~a2yQ>of!*NXbv$_qiqDIG+9rbon-yDN
z-%R@A63<?nixSvXzu;Pk3GA<W!8cH>{Eg6$Bd~+EXIh{HcHM8dl>`F2L?+xxJt0g3
kC9uCEv=a>M&^JBKy{cMsPy##i58QsD1ABpkj^dHo0Oj^yO8@`>
literal 0
HcmV?d00001
diff --git a/ray_results/test_experiment/train_model_de1e9134_30_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=1,drop_r_2024-06-25_22-20-23/events.out.tfevents.1719355223.r8i6n2 b/ray_results/test_experiment/train_model_de1e9134_30_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=1,drop_r_2024-06-25_22-20-23/events.out.tfevents.1719355223.r8i6n2
new file mode 100644
index 0000000000000000000000000000000000000000..56e3055f9bf9fe7ae68cd9d7b51dcf121c27b383
GIT binary patch
literal 2771
zcma);eQXnT7{|No+_*>g(xb(#d+FARAyp_4re?vcv|WhXKg8h+f4Jk?`_-Pb*XHgf
z%s;l0jVP!I@nw+=6dY!W;vi^5VloYx$wC~AA&JHqZ~?0O0~RTh%|*Sw{jR^+()Pdp
zp6B~~@ArA0+;dgB<ZrmI!%F|Qb>_;Y4}acbF7rs7Ag*hB6^TmA#un-9r%kE6Qy=Yi
z(KB6<oAJEL65A>;LYc(%*p+S(*#jY2HvO<O=AQUxyKj9;^8Vc^mfqpTk|t;bGnUBy
zP<Mc5y?*4~6%GlaV$$g%t%nR%QNnT;!g|t@4~a;X=yXq4KIgHgEQ2rAZKEk4mRO+y
zs<EXa1QDzF1(_8TB(bt-a(2ME|HT6X8&dKk{r!K?eVY6cutE(fmj%&_SP98WNJ5DM
z?KiiEo=g>p{(gQ3{ijyI25X=Z7l^xZiVzZImMjy$`P$~CFNm8L|Mk_K1nkmmu+Mkg
zb@#sYd>aA#p?g?T2<(3U&@5s3j;7B6J9c*Zdjht30bAyQ-Nko0J<sgb5wHg}d0yB{
zhi^GAEbdnb*z_XyFBrDY;S!IJ5jUT`H>aD9PxgvzumdA8_rjx-Q3Ce-0+tj4d-JuK
zQo^!t5$kinu2|DQNWd=n8%y%Q9vSL%{`8)un1J1@$@9YYH$U#0ik_|}VBgUS6b#$p
z;BHvA5jU?r;V4hQrnABJy&H4iYUtQPz^?xsOA3L#(Q@S+VL77dbHM&!-T1cz>`kNC
zG7oI~{C4LsbMXZN_NXS$3p*xpuIafdBLRE$N$g)R?1-b+QdB|QyneE$CIK6=!5$fl
zxv$^8_9X$^asf*Uf!#Iu^(Dgc6HT83_P&U_jeu>xh%NKLHoo2I3=i*}AYgx{$@9Xl
zee6xw<>;>u5U{ON*uP-dM;+JCY#KjRlzjIeJaXt2x}$pMkC)Zgi0!x@j_JY3`oo;W
z1!dNb_Jon7?jKQUe?t%Z^iY}#F$)wkPg%<WJe=b7tDE69)S$OAP|hQ7h*$TNx*J)a
zPd%#!Q|6i29b^Ms50cc70m{-LDV1FX#*CxP<T9XnDHLYaWfZqkwJ7L8JTHhZv%C;g
zkJP6@V$wa~Yyw02e`cW)paOAsfD)1QBaUy-+YC^_kRJxNdZ=aYv-p=u;)~r@2Q^IY
z#ci`d9dp0MY+DUYO#a2YcHFikruuj8_Qx&sP`ZCJ6kMgCR_mXPWLe$ZtgLV<U_tbS
z+Q%sPhSK_Yk}{v5U{wZNhBhb%nxA?-@?9&eOe0bclLIr?3g%^9!?B11>a!rn@k$5O
zEFTaYRK<<idyi98?}~Wu)PpB(gh&5<aChQih%vy)>mBfL<~-oQ!9+kCTO2s*P(Pfq
y&nnKDYdtph%bv}t52uMU!_U&E)AToh9nAk{k3>F6UY#+Gd$E;TQM8i3I`AI=N`XTF
literal 0
HcmV?d00001
diff --git a/ray_results/test_experiment/train_model_de1e9134_30_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=1,drop_r_2024-06-25_22-20-23/params.json b/ray_results/test_experiment/train_model_de1e9134_30_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=1,drop_r_2024-06-25_22-20-23/params.json
new file mode 100644
index 0000000..4f6c973
--- /dev/null
+++ b/ray_results/test_experiment/train_model_de1e9134_30_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=1,drop_r_2024-06-25_22-20-23/params.json
@@ -0,0 +1,13 @@
+{
+ "batch_size": 2048,
+ "decoder_int_ff": 512,
+ "decoder_int_num_layer": 1,
+ "decoder_rt_ff": 512,
+ "decoder_rt_num_layer": 1,
+ "drop_rate": 0.12892236942851165,
+ "embedding_dim": 16,
+ "encoder_ff": 1024,
+ "encoder_num_layer": 4,
+ "lr": 0.005626677279957184,
+ "n_head": 1
+}
\ No newline at end of file
diff --git a/ray_results/test_experiment/train_model_de1e9134_30_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=1,drop_r_2024-06-25_22-20-23/params.pkl b/ray_results/test_experiment/train_model_de1e9134_30_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=1,drop_r_2024-06-25_22-20-23/params.pkl
new file mode 100644
index 0000000000000000000000000000000000000000..f5d49d3c3306a54c36545122768fe638be6711c8
GIT binary patch
literal 224
zcmZo*oqCl40&1sd^a!TrCFiH47RBe4=EmnFR;CtB@n-1}NlAr>7L_1NF!qSTBr@~R
zWO!3^lTuSsGV{{oQ!;a>cnkD!!HiBzo8rsB(!&eW3p5lg#MHwF69O3r6=Tbb&qz&7
znc~gZ!<kZ)Ul3oESdu!$-TpvByM<|2Ne@#_5s+8T{eNA-Q?nkfq{NcsjQHZrs#KsQ
H9Hn{ytT9r?
literal 0
HcmV?d00001
diff --git a/ray_results/test_experiment/train_model_de1e9134_30_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=1,drop_r_2024-06-25_22-20-23/progress.csv b/ray_results/test_experiment/train_model_de1e9134_30_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=1,drop_r_2024-06-25_22-20-23/progress.csv
new file mode 100644
index 0000000..771cf59
--- /dev/null
+++ b/ray_results/test_experiment/train_model_de1e9134_30_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=1,drop_r_2024-06-25_22-20-23/progress.csv
@@ -0,0 +1,11 @@
+loss,timestamp,checkpoint_dir_name,should_checkpoint,done,training_iteration,trial_id,date,time_this_iter_s,time_total_s,pid,hostname,node_ip,time_since_restore,iterations_since_restore
+1956.6991774191067,1719348406,checkpoint_000000,True,False,1,de1e9134,2024-06-25_22-46-46,67.86075568199158,67.86075568199158,2195253,r8i6n2,10.159.28.60,67.86075568199158,1
+1966.5979321096827,1719348460,checkpoint_000001,True,False,2,de1e9134,2024-06-25_22-47-40,53.399649143218994,121.26040482521057,2195253,r8i6n2,10.159.28.60,121.26040482521057,2
+1954.8947446327509,1719348513,checkpoint_000002,True,False,3,de1e9134,2024-06-25_22-48-33,53.41487240791321,174.67527723312378,2195253,r8i6n2,10.159.28.60,174.67527723312378,3
+1955.8978973148376,1719348567,checkpoint_000003,True,False,4,de1e9134,2024-06-25_22-49-27,53.3932785987854,228.06855583190918,2195253,r8i6n2,10.159.28.60,228.06855583190918,4
+1955.0890363558071,1719348620,checkpoint_000004,True,False,5,de1e9134,2024-06-25_22-50-20,53.77778649330139,281.84634232521057,2195253,r8i6n2,10.159.28.60,281.84634232521057,5
+1955.0249504029282,1719355289,checkpoint_000005,True,False,6,de1e9134,2024-06-26_00-41-30,66.45859718322754,348.3049395084381,2850562,r8i6n2,10.159.28.60,66.45859718322754,1
+1962.3684418445498,1719355342,checkpoint_000006,True,False,7,de1e9134,2024-06-26_00-42-22,52.848015785217285,401.1529552936554,2850562,r8i6n2,10.159.28.60,119.30661296844482,2
+1957.2319662739912,1719355395,checkpoint_000007,True,False,8,de1e9134,2024-06-26_00-43-15,52.90200471878052,454.0549600124359,2850562,r8i6n2,10.159.28.60,172.20861768722534,3
+1956.60619444359,1719355448,checkpoint_000008,True,False,9,de1e9134,2024-06-26_00-44-09,53.24041485786438,507.2953748703003,2850562,r8i6n2,10.159.28.60,225.44903254508972,4
+1957.9578991987573,1719355501,checkpoint_000009,True,False,10,de1e9134,2024-06-26_00-45-01,52.88777208328247,560.1831469535828,2850562,r8i6n2,10.159.28.60,278.3368046283722,5
diff --git a/ray_results/test_experiment/train_model_de1e9134_30_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=1,drop_r_2024-06-25_22-20-23/result.json b/ray_results/test_experiment/train_model_de1e9134_30_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=1,drop_r_2024-06-25_22-20-23/result.json
new file mode 100644
index 0000000..2ecc83b
--- /dev/null
+++ b/ray_results/test_experiment/train_model_de1e9134_30_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=1,drop_r_2024-06-25_22-20-23/result.json
@@ -0,0 +1,10 @@
+{"loss": 1956.6991774191067, "timestamp": 1719348406, "checkpoint_dir_name": "checkpoint_000000", "should_checkpoint": true, "done": false, "training_iteration": 1, "trial_id": "de1e9134", "date": "2024-06-25_22-46-46", "time_this_iter_s": 67.86075568199158, "time_total_s": 67.86075568199158, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 4, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 1024, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 1, "drop_rate": 0.12892236942851165, "lr": 0.005626677279957184, "batch_size": 2048}, "time_since_restore": 67.86075568199158, "iterations_since_restore": 1}
+{"loss": 1966.5979321096827, "timestamp": 1719348460, "checkpoint_dir_name": "checkpoint_000001", "should_checkpoint": true, "done": false, "training_iteration": 2, "trial_id": "de1e9134", "date": "2024-06-25_22-47-40", "time_this_iter_s": 53.399649143218994, "time_total_s": 121.26040482521057, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 4, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 1024, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 1, "drop_rate": 0.12892236942851165, "lr": 0.005626677279957184, "batch_size": 2048}, "time_since_restore": 121.26040482521057, "iterations_since_restore": 2}
+{"loss": 1954.8947446327509, "timestamp": 1719348513, "checkpoint_dir_name": "checkpoint_000002", "should_checkpoint": true, "done": false, "training_iteration": 3, "trial_id": "de1e9134", "date": "2024-06-25_22-48-33", "time_this_iter_s": 53.41487240791321, "time_total_s": 174.67527723312378, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 4, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 1024, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 1, "drop_rate": 0.12892236942851165, "lr": 0.005626677279957184, "batch_size": 2048}, "time_since_restore": 174.67527723312378, "iterations_since_restore": 3}
+{"loss": 1955.8978973148376, "timestamp": 1719348567, "checkpoint_dir_name": "checkpoint_000003", "should_checkpoint": true, "done": false, "training_iteration": 4, "trial_id": "de1e9134", "date": "2024-06-25_22-49-27", "time_this_iter_s": 53.3932785987854, "time_total_s": 228.06855583190918, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 4, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 1024, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 1, "drop_rate": 0.12892236942851165, "lr": 0.005626677279957184, "batch_size": 2048}, "time_since_restore": 228.06855583190918, "iterations_since_restore": 4}
+{"loss": 1955.0890363558071, "timestamp": 1719348620, "checkpoint_dir_name": "checkpoint_000004", "should_checkpoint": true, "done": false, "training_iteration": 5, "trial_id": "de1e9134", "date": "2024-06-25_22-50-20", "time_this_iter_s": 53.77778649330139, "time_total_s": 281.84634232521057, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 4, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 1024, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 1, "drop_rate": 0.12892236942851165, "lr": 0.005626677279957184, "batch_size": 2048}, "time_since_restore": 281.84634232521057, "iterations_since_restore": 5}
+{"loss": 1955.0249504029282, "timestamp": 1719355289, "checkpoint_dir_name": "checkpoint_000005", "should_checkpoint": true, "done": false, "training_iteration": 6, "trial_id": "de1e9134", "date": "2024-06-26_00-41-30", "time_this_iter_s": 66.45859718322754, "time_total_s": 348.3049395084381, "pid": 2850562, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 4, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 1024, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 1, "drop_rate": 0.12892236942851165, "lr": 0.005626677279957184, "batch_size": 2048}, "time_since_restore": 66.45859718322754, "iterations_since_restore": 1}
+{"loss": 1962.3684418445498, "timestamp": 1719355342, "checkpoint_dir_name": "checkpoint_000006", "should_checkpoint": true, "done": false, "training_iteration": 7, "trial_id": "de1e9134", "date": "2024-06-26_00-42-22", "time_this_iter_s": 52.848015785217285, "time_total_s": 401.1529552936554, "pid": 2850562, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 4, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 1024, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 1, "drop_rate": 0.12892236942851165, "lr": 0.005626677279957184, "batch_size": 2048}, "time_since_restore": 119.30661296844482, "iterations_since_restore": 2}
+{"loss": 1957.2319662739912, "timestamp": 1719355395, "checkpoint_dir_name": "checkpoint_000007", "should_checkpoint": true, "done": false, "training_iteration": 8, "trial_id": "de1e9134", "date": "2024-06-26_00-43-15", "time_this_iter_s": 52.90200471878052, "time_total_s": 454.0549600124359, "pid": 2850562, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 4, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 1024, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 1, "drop_rate": 0.12892236942851165, "lr": 0.005626677279957184, "batch_size": 2048}, "time_since_restore": 172.20861768722534, "iterations_since_restore": 3}
+{"loss": 1956.60619444359, "timestamp": 1719355448, "checkpoint_dir_name": "checkpoint_000008", "should_checkpoint": true, "done": false, "training_iteration": 9, "trial_id": "de1e9134", "date": "2024-06-26_00-44-09", "time_this_iter_s": 53.24041485786438, "time_total_s": 507.2953748703003, "pid": 2850562, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 4, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 1024, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 1, "drop_rate": 0.12892236942851165, "lr": 0.005626677279957184, "batch_size": 2048}, "time_since_restore": 225.44903254508972, "iterations_since_restore": 4}
+{"loss": 1957.9578991987573, "timestamp": 1719355501, "checkpoint_dir_name": "checkpoint_000009", "should_checkpoint": true, "done": false, "training_iteration": 10, "trial_id": "de1e9134", "date": "2024-06-26_00-45-01", "time_this_iter_s": 52.88777208328247, "time_total_s": 560.1831469535828, "pid": 2850562, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 4, "decoder_rt_num_layer": 1, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 1024, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 1, "drop_rate": 0.12892236942851165, "lr": 0.005626677279957184, "batch_size": 2048}, "time_since_restore": 278.3368046283722, "iterations_since_restore": 5}
diff --git a/ray_results/test_experiment/train_model_e31d7ca7_26_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=2,drop__2024-06-25_19-18-32/events.out.tfevents.1719337143.r8i6n2 b/ray_results/test_experiment/train_model_e31d7ca7_26_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=2,drop__2024-06-25_19-18-32/events.out.tfevents.1719337143.r8i6n2
new file mode 100644
index 0000000000000000000000000000000000000000..80ebb7b865552e87926cf6142fbd51dbf047ce91
GIT binary patch
literal 1676
zcma)6OH30{6dk4nof`{uq<qT1Xe48#V2A{QX)U4&nn=V%G11A;dC*Zh)65J4ZfJre
z=t>vHu)q&Bu8bliA`yvUMMz9sxX_J!A`#u-LJh9;l}=wfSU$6xd+)jDy>rifxm5HE
z_nmSwq0x7T-roOo%wcU)1xc=GyoO}G?(q1VGkeO6y3q&S4a~`pVQ-`^Glk2BEU>CV
zuS@Tek*ibDH2XLgY%HoM4pkbEyT9J^jPRL2%Agb+1fuzsu1=Bn`H}BxP?2QaPEjHE
z`1kG(6Xo0j!J8>XMMfAxQDSZCvdf?hTq<m4e2WC)gc8Ujl)4l^yzZAYUeb}uYxWb*
zpLj3794oFe;KTj>-<U6P_&#t#z8S7bvJdep(sV^d>kX{04zAW14Zg0-wKC!o;pd<n
zN=btVtDsAYtnubHk(&?9Rri=~zIcAEWnFAKnb;N2${S~-dd?*F#03J`iP#I1YsXEL
zm8}GyRP2!Z&XK=2O=7=@L$(pS`ra*XPyKw@B=&)egmGK3kA`y_2BlD=N$jC`gB^=a
zd4^kNb4F95L%TgZbcM0ZM$f@Lw!k9`Wbyu>pb7zvccGpjQn7Jly=l+_0~Xk2q}V?+
z`<r&A1GEXc&yPoX9hF#|EToIbr-;~K*wlP`JC5anJuxPC1^7;(7pXX918Xd$8r8Xw
zl@LlSwgMxnN|47cicn!J3bY|nl;jRxlma+}PlMzr`#Q57Y_b2uMrlBIM%V$;WZsVi
zvBbjJAcHk;G*<)#>=qll0TS6rt`PFs%{PuKhC+6$jpFu08N2<)4Af5kl#Th#td7(%
zHL>|E(J)U#L41B1(loqsyrv6SU`cLQnx<)ZO~>c>oOV2=Av-}^ma%=PTAuEsR@|^R
zCJ_!s4U_2s$A+nq+(Qo(C6S!uF*bjrK}a&>!7pUe*^?5=0~wLo;lM~)Ww6dW=8hc3
zWCK#%?17v_Kad1IB~HX`o0xEJ;#E@f!+oR3H&t)Ec(W7Xv&`w3`c>cp$N$<RS&S-=
RJr>tJ)k*J8IX5++{sVz|4C(*?
literal 0
HcmV?d00001
diff --git a/ray_results/test_experiment/train_model_e31d7ca7_26_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=2,drop__2024-06-25_19-18-32/params.json b/ray_results/test_experiment/train_model_e31d7ca7_26_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=2,drop__2024-06-25_19-18-32/params.json
new file mode 100644
index 0000000..d0fef20
--- /dev/null
+++ b/ray_results/test_experiment/train_model_e31d7ca7_26_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=2,drop__2024-06-25_19-18-32/params.json
@@ -0,0 +1,13 @@
+{
+ "batch_size": 2048,
+ "decoder_int_ff": 512,
+ "decoder_int_num_layer": 1,
+ "decoder_rt_ff": 2048,
+ "decoder_rt_num_layer": 2,
+ "drop_rate": 0.7500609138892806,
+ "embedding_dim": 64,
+ "encoder_ff": 2048,
+ "encoder_num_layer": 4,
+ "lr": 0.0713230632683364,
+ "n_head": 4
+}
\ No newline at end of file
diff --git a/ray_results/test_experiment/train_model_e31d7ca7_26_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=2,drop__2024-06-25_19-18-32/params.pkl b/ray_results/test_experiment/train_model_e31d7ca7_26_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=2,drop__2024-06-25_19-18-32/params.pkl
new file mode 100644
index 0000000000000000000000000000000000000000..97d8b946d49e4185cba644765668d2b857017c28
GIT binary patch
literal 224
zcmZo*oqCl40&1sd^a!TrCFiH47RBe4=EmnFR;CtB@n-1}NlAr>7L_1NF!hMSBr@|*
zWEgvRQ*)D2Q&KYX(&JMybEkMa^l-t9PD`8O%fQjY3)2fU6e`3A69O3r7GvsR%ZtxQ
zO-unfgEOTlzaYLSu_SehyZsA>`h8Cguk<kG6ajggoU95Alo$1IB_)<5XT%q0R;7Y0
HDb)i2%zIM0
literal 0
HcmV?d00001
diff --git a/ray_results/test_experiment/train_model_e31d7ca7_26_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=2,drop__2024-06-25_19-18-32/progress.csv b/ray_results/test_experiment/train_model_e31d7ca7_26_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=2,drop__2024-06-25_19-18-32/progress.csv
new file mode 100644
index 0000000..2fa4e7c
--- /dev/null
+++ b/ray_results/test_experiment/train_model_e31d7ca7_26_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=2,drop__2024-06-25_19-18-32/progress.csv
@@ -0,0 +1,3 @@
+loss,timestamp,checkpoint_dir_name,should_checkpoint,done,training_iteration,trial_id,date,time_this_iter_s,time_total_s,pid,hostname,node_ip,time_since_restore,iterations_since_restore
+3842.66859736405,1719337226,checkpoint_000000,True,False,1,e31d7ca7,2024-06-25_19-40-26,83.34233164787292,83.34233164787292,2195253,r8i6n2,10.159.28.60,83.34233164787292,1
+2698.886155496432,1719337295,checkpoint_000001,True,True,2,e31d7ca7,2024-06-25_19-41-35,69.29531860351562,152.63765025138855,2195253,r8i6n2,10.159.28.60,152.63765025138855,2
diff --git a/ray_results/test_experiment/train_model_e31d7ca7_26_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=2,drop__2024-06-25_19-18-32/result.json b/ray_results/test_experiment/train_model_e31d7ca7_26_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=2,drop__2024-06-25_19-18-32/result.json
new file mode 100644
index 0000000..da281f7
--- /dev/null
+++ b/ray_results/test_experiment/train_model_e31d7ca7_26_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=2,drop__2024-06-25_19-18-32/result.json
@@ -0,0 +1,2 @@
+{"loss": 3842.66859736405, "timestamp": 1719337226, "checkpoint_dir_name": "checkpoint_000000", "should_checkpoint": true, "done": false, "training_iteration": 1, "trial_id": "e31d7ca7", "date": "2024-06-25_19-40-26", "time_this_iter_s": 83.34233164787292, "time_total_s": 83.34233164787292, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 4, "decoder_rt_num_layer": 2, "decoder_int_num_layer": 1, "embedding_dim": 64, "encoder_ff": 2048, "decoder_rt_ff": 2048, "decoder_int_ff": 512, "n_head": 4, "drop_rate": 0.7500609138892806, "lr": 0.0713230632683364, "batch_size": 2048}, "time_since_restore": 83.34233164787292, "iterations_since_restore": 1}
+{"loss": 2698.886155496432, "timestamp": 1719337295, "checkpoint_dir_name": "checkpoint_000001", "should_checkpoint": true, "done": true, "training_iteration": 2, "trial_id": "e31d7ca7", "date": "2024-06-25_19-41-35", "time_this_iter_s": 69.29531860351562, "time_total_s": 152.63765025138855, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 4, "decoder_rt_num_layer": 2, "decoder_int_num_layer": 1, "embedding_dim": 64, "encoder_ff": 2048, "decoder_rt_ff": 2048, "decoder_int_ff": 512, "n_head": 4, "drop_rate": 0.7500609138892806, "lr": 0.0713230632683364, "batch_size": 2048}, "time_since_restore": 152.63765025138855, "iterations_since_restore": 2}
diff --git a/ray_results/test_experiment/train_model_e566e32a_13_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=2,drop__2024-06-25_16-40-03/events.out.tfevents.1719326489.r3i5n6 b/ray_results/test_experiment/train_model_e566e32a_13_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=2,drop__2024-06-25_16-40-03/events.out.tfevents.1719326489.r3i5n6
new file mode 100644
index 0000000000000000000000000000000000000000..3b93ff1cc2833af8ac754fc78faee86a4f4d3a97
GIT binary patch
literal 1311
zcmah|Pe>F|9A0P5njfsX)2@G3S9fRAVuDsJkVM$c(1TvIK<E&Uap&2N?YvoLM$--@
zNK|%_po0f1wQhlSs0b;@LrAAC6?F-MP^qX(fw$i5&c59>wYkmj`+dLfz3=zE8dvsP
zyxJ%7104gepWOX;D&QG1WR)E29>c`yXzLj-UyC?>%l9sw=5GeKKG}VhB~lgmz+=c0
zQEQaoXi7It|9bVl?*7KXZ!Hex&hNK~#}P(}Ko|myVkY&`l!6jToVb|QRbu&FuF2T+
zhs~8D$@fi+Uo5HU1XGI3)pp`|IO<4h=Y##c*32j(grJs5TB?STl~hfnTG&9Qf2;Fh
z*Q?dk(N>2(H#N1%f6vnofe3ZQbW<e>j0|jAx`8tTs=N5vc4xr)#+!4zdYI`;a0J3^
zfUPQ9s!mK)JjT9xPxJcHa!Gd1%-rk<-}n4$^HXZJ^uP`CZt$UGS~g_OL{U7R#s<9$
zv3hT~;hG!vJCLx&34b`T9AHSc5=p9Z8He1W0Obl!=nB0v^%)Hh(^@U~3u|OlLn(Oz
z8x-<_CkGi$cMbRoN`=iH;1xqpBYH7gigK|wgcU_47m%WAv_wyXNmf0h><4e|zi?0)
za4B0Ipp2j-mX(lO@<N4B95Sf^>V-W*w2fkiL<&NkuzLukMhFUfEm>-Ui17a<{%D^n
zCC>g<e=c<Jw{rXY!og<_>ht?Ev1!s2Ak&hm!73TndtY*}!sYk4#04I4P*vcqz<W<V
z9+A(!*oeWwoJF*l4NPSm0^7F6u={an*uip!hvHDTJs=F6jMGQE<L+6DD-IR*?r5Rr
z-Qa?EdOl_!rYHuixg`$Og?V7WOa{oumI3tm;x1WbJ-=dCy4A5x=Mk<wTRg)r=G<?E
aC<OlX9up<&Jk40%_fnBNP;$HXdfQ)rij%Sc
literal 0
HcmV?d00001
diff --git a/ray_results/test_experiment/train_model_e566e32a_13_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=2,drop__2024-06-25_16-40-03/params.json b/ray_results/test_experiment/train_model_e566e32a_13_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=2,drop__2024-06-25_16-40-03/params.json
new file mode 100644
index 0000000..f5b422f
--- /dev/null
+++ b/ray_results/test_experiment/train_model_e566e32a_13_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=2,drop__2024-06-25_16-40-03/params.json
@@ -0,0 +1,13 @@
+{
+ "batch_size": 2048,
+ "decoder_int_ff": 512,
+ "decoder_int_num_layer": 1,
+ "decoder_rt_ff": 2048,
+ "decoder_rt_num_layer": 2,
+ "drop_rate": 0.7731110479994988,
+ "embedding_dim": 16,
+ "encoder_ff": 1024,
+ "encoder_num_layer": 8,
+ "lr": 0.03425650698356203,
+ "n_head": 4
+}
\ No newline at end of file
diff --git a/ray_results/test_experiment/train_model_e566e32a_13_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=2,drop__2024-06-25_16-40-03/params.pkl b/ray_results/test_experiment/train_model_e566e32a_13_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=2,drop__2024-06-25_16-40-03/params.pkl
new file mode 100644
index 0000000000000000000000000000000000000000..0828fd20418e00661a7e5a866774b997b9408aaf
GIT binary patch
literal 224
zcmZo*oqCl40&1sd^a!TrCFiH47RBe4=EmnFR;CtB@#g3eNlAr>7L_1NF!hMSBr@|*
zWEgvRQ*)D2Q&KYX(&JMybEkL<^l-t9PD`8O%fQmZ3)2fU6fDHi!v_-r83z_)>S4=^
z&qz&7nc~gT!<kZ)Ul3oESdu!$-TuYi;KZzrmOV^4ML^!dF2O}%S0#G5k`hajGvbRg
Mt5SiMaFpr+0Jxn}wg3PC
literal 0
HcmV?d00001
diff --git a/ray_results/test_experiment/train_model_e566e32a_13_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=2,drop__2024-06-25_16-40-03/progress.csv b/ray_results/test_experiment/train_model_e566e32a_13_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=2,drop__2024-06-25_16-40-03/progress.csv
new file mode 100644
index 0000000..151739c
--- /dev/null
+++ b/ray_results/test_experiment/train_model_e566e32a_13_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=2,drop__2024-06-25_16-40-03/progress.csv
@@ -0,0 +1,2 @@
+loss,timestamp,checkpoint_dir_name,should_checkpoint,done,training_iteration,trial_id,date,time_this_iter_s,time_total_s,pid,hostname,node_ip,time_since_restore,iterations_since_restore
+2129.4940118264026,1719326571,checkpoint_000000,True,False,1,e566e32a,2024-06-25_16-42-51,82.61908912658691,82.61908912658691,69318,r3i5n6,10.159.8.159,82.61908912658691,1
diff --git a/ray_results/test_experiment/train_model_e566e32a_13_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=2,drop__2024-06-25_16-40-03/result.json b/ray_results/test_experiment/train_model_e566e32a_13_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=2,drop__2024-06-25_16-40-03/result.json
new file mode 100644
index 0000000..fd337bc
--- /dev/null
+++ b/ray_results/test_experiment/train_model_e566e32a_13_batch_size=2048,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=2048,decoder_rt_num_layer=2,drop__2024-06-25_16-40-03/result.json
@@ -0,0 +1 @@
+{"loss": 2129.4940118264026, "timestamp": 1719326571, "checkpoint_dir_name": "checkpoint_000000", "should_checkpoint": true, "done": false, "training_iteration": 1, "trial_id": "e566e32a", "date": "2024-06-25_16-42-51", "time_this_iter_s": 82.61908912658691, "time_total_s": 82.61908912658691, "pid": 69318, "hostname": "r3i5n6", "node_ip": "10.159.8.159", "config": {"encoder_num_layer": 8, "decoder_rt_num_layer": 2, "decoder_int_num_layer": 1, "embedding_dim": 16, "encoder_ff": 1024, "decoder_rt_ff": 2048, "decoder_int_ff": 512, "n_head": 4, "drop_rate": 0.7731110479994988, "lr": 0.03425650698356203, "batch_size": 2048}, "time_since_restore": 82.61908912658691, "iterations_since_restore": 1}
diff --git a/ray_results/test_experiment/train_model_f1fbbaf2_5_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop_r_2024-06-25_17-16-57/events.out.tfevents.1719328732.r8i6n2 b/ray_results/test_experiment/train_model_f1fbbaf2_5_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop_r_2024-06-25_17-16-57/events.out.tfevents.1719328732.r8i6n2
new file mode 100644
index 0000000000000000000000000000000000000000..66fb04c000a2f00097bcaaea4d9f438d49a1a197
GIT binary patch
literal 1311
zcmah|OH30{6fIMO%+(e;QiSp=gBqNL6*eY>q*Wj$5RLIe;=(+qorfK5-!$_Eq_~hs
z4DMt@Vq@@!OB30l(YPQc7)->N5W|YZ_+3d%M0DkAJAG{taCURgJ@>qK?s?UY>^F6_
zR~9FqxBZ&Avvk<!3Y)4<4s;J<Vzsv|ExOB^?Y`-|mpjEz#{OGRSYJhnQU#UZGSy_j
z8X!0jGbr`CPpx&Ou8pl7v?&um-y-qoAg44#6ZklVMvZ}(h9XfMxs)(;VtE~ok>K3W
z!sjB%GfB=bmedS_8O7oFb7cNzz?O`kt3M&`80Hii{7}OsEj^Br71b%yEo>s{?OXMC
z1vO8i)uvC5j(!s-^7K6*Lv1mg>Lh}ZiK%6nI5R+cJ$Cz$J>bjN`LkmA6|S#93pDWo
zX;szI4MI`znDotk$6NcROR{suCsP;2ovG=mr_5~Sp%d;qp%O(Cs;R~)3gDpxHrZW>
z)pNrM*PXE4hNRzuv?|DMfUs&sqD*BN`<=1`ZVg8ajoq2~jQaaotp>b>H8K!KF*S)z
z26@1hgG{@-8Y&A)h0QJyHPc8ScCoY+<KlQ2Yno0jAWe_65<3khS@n#v7d*ND!a=1V
z#M0^jr36K>s`;IY2g;@5kSPsNCv6d;b(C~SlzOO@HV=W)2=&rdOIG$mv-JNZ&c0th
zPwf58et*y|9?R|Tg8<6{)aCa_F{La8h*~N$SSS5P&kF&b3Hd!f7JQEcs4DPQ5<M@I
zjV-73g&^$8S;UIjz^nwpw{B}38xKLl29|TYA_TST1H!?P5PP&6?w+?eLQtOG9V@(3
zJAcqncjhofG2qSYFuM!$z=3-iARk)}FowKIR(a1#aBeZRufu+XuV$uBis795tq=g;
Yzuse_Wu0dk%lp1f7RpL`-+b)&3vL^eLjV8(
literal 0
HcmV?d00001
diff --git a/ray_results/test_experiment/train_model_f1fbbaf2_5_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop_r_2024-06-25_17-16-57/params.json b/ray_results/test_experiment/train_model_f1fbbaf2_5_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop_r_2024-06-25_17-16-57/params.json
new file mode 100644
index 0000000..65f2092
--- /dev/null
+++ b/ray_results/test_experiment/train_model_f1fbbaf2_5_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop_r_2024-06-25_17-16-57/params.json
@@ -0,0 +1,13 @@
+{
+ "batch_size": 1024,
+ "decoder_int_ff": 512,
+ "decoder_int_num_layer": 1,
+ "decoder_rt_ff": 1024,
+ "decoder_rt_num_layer": 2,
+ "drop_rate": 0.24540187796715263,
+ "embedding_dim": 1024,
+ "encoder_ff": 2048,
+ "encoder_num_layer": 4,
+ "lr": 0.0006485525268785047,
+ "n_head": 1
+}
\ No newline at end of file
diff --git a/ray_results/test_experiment/train_model_f1fbbaf2_5_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop_r_2024-06-25_17-16-57/params.pkl b/ray_results/test_experiment/train_model_f1fbbaf2_5_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop_r_2024-06-25_17-16-57/params.pkl
new file mode 100644
index 0000000000000000000000000000000000000000..d2806b583545ebac95768737b473342db3d6b524
GIT binary patch
literal 225
zcmZo*oqCM{0&1sd^a!TrCFiH47RBe4=EmnFR;CtB@n-1}NlAr>7L_1NF!hMSBr@|*
zWEgvRQ*)D2Q&KYX(&JMybEo(+u=H@j%uY)K@;Q2VVVZ%af`wRm_+UaH^T1+EJ#2aL
z8L5dWQ@j~_I8%!93*w6sOH!w}+n>)2(U7k^(!-Qf1mw9o6n}C~ywt;$lvt9S5nr5H
Kl?t-7R1W~c)lx(N
literal 0
HcmV?d00001
diff --git a/ray_results/test_experiment/train_model_f1fbbaf2_5_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop_r_2024-06-25_17-16-57/progress.csv b/ray_results/test_experiment/train_model_f1fbbaf2_5_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop_r_2024-06-25_17-16-57/progress.csv
new file mode 100644
index 0000000..3d8d7bb
--- /dev/null
+++ b/ray_results/test_experiment/train_model_f1fbbaf2_5_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop_r_2024-06-25_17-16-57/progress.csv
@@ -0,0 +1,2 @@
+loss,timestamp,checkpoint_dir_name,should_checkpoint,done,training_iteration,trial_id,date,time_this_iter_s,time_total_s,pid,hostname,node_ip,time_since_restore,iterations_since_restore
+1986.5954556202325,1719329376,checkpoint_000000,True,False,1,f1fbbaf2,2024-06-25_17-29-36,643.849014043808,643.849014043808,2195253,r8i6n2,10.159.28.60,643.849014043808,1
diff --git a/ray_results/test_experiment/train_model_f1fbbaf2_5_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop_r_2024-06-25_17-16-57/result.json b/ray_results/test_experiment/train_model_f1fbbaf2_5_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop_r_2024-06-25_17-16-57/result.json
new file mode 100644
index 0000000..a73f3b5
--- /dev/null
+++ b/ray_results/test_experiment/train_model_f1fbbaf2_5_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop_r_2024-06-25_17-16-57/result.json
@@ -0,0 +1 @@
+{"loss": 1986.5954556202325, "timestamp": 1719329376, "checkpoint_dir_name": "checkpoint_000000", "should_checkpoint": true, "done": false, "training_iteration": 1, "trial_id": "f1fbbaf2", "date": "2024-06-25_17-29-36", "time_this_iter_s": 643.849014043808, "time_total_s": 643.849014043808, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 4, "decoder_rt_num_layer": 2, "decoder_int_num_layer": 1, "embedding_dim": 1024, "encoder_ff": 2048, "decoder_rt_ff": 1024, "decoder_int_ff": 512, "n_head": 1, "drop_rate": 0.24540187796715263, "lr": 0.0006485525268785047, "batch_size": 1024}, "time_since_restore": 643.849014043808, "iterations_since_restore": 1}
diff --git a/ray_results/test_experiment/train_model_f74eb513_2_batch_size=1024,decoder_int_ff=1024,decoder_int_num_layer=8,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop__2024-06-25_15-20-03/events.out.tfevents.1719321692.r7i5n7 b/ray_results/test_experiment/train_model_f74eb513_2_batch_size=1024,decoder_int_ff=1024,decoder_int_num_layer=8,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop__2024-06-25_15-20-03/events.out.tfevents.1719321692.r7i5n7
new file mode 100644
index 0000000..e69de29
diff --git a/ray_results/test_experiment/train_model_f74eb513_2_batch_size=1024,decoder_int_ff=1024,decoder_int_num_layer=8,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop__2024-06-25_15-20-03/params.json b/ray_results/test_experiment/train_model_f74eb513_2_batch_size=1024,decoder_int_ff=1024,decoder_int_num_layer=8,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop__2024-06-25_15-20-03/params.json
new file mode 100644
index 0000000..bb0521f
--- /dev/null
+++ b/ray_results/test_experiment/train_model_f74eb513_2_batch_size=1024,decoder_int_ff=1024,decoder_int_num_layer=8,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop__2024-06-25_15-20-03/params.json
@@ -0,0 +1,13 @@
+{
+ "batch_size": 1024,
+ "decoder_int_ff": 1024,
+ "decoder_int_num_layer": 8,
+ "decoder_rt_ff": 1024,
+ "decoder_rt_num_layer": 2,
+ "drop_rate": 0.5887719140440579,
+ "embedding_dim": 1024,
+ "encoder_ff": 2048,
+ "encoder_num_layer": 8,
+ "lr": 0.009580025887992769,
+ "n_head": 8
+}
\ No newline at end of file
diff --git a/ray_results/test_experiment/train_model_f74eb513_2_batch_size=1024,decoder_int_ff=1024,decoder_int_num_layer=8,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop__2024-06-25_15-20-03/params.pkl b/ray_results/test_experiment/train_model_f74eb513_2_batch_size=1024,decoder_int_ff=1024,decoder_int_num_layer=8,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop__2024-06-25_15-20-03/params.pkl
new file mode 100644
index 0000000000000000000000000000000000000000..ce79ddfb021ca60b7aa0c2634f11fac094cd75c0
GIT binary patch
literal 225
zcmZo*oqCM{0&1sd^a!TrCFiH47RBe4=EmnFR;CtB@#g3eNlAr>7L_1NF!hMSBr@|*
zWH@?wQ*)D2Q&KYX(&JMybEo(+u=H@j%uY)K@_|ZVnt`T*g;;v{U_v1Cpki!!@foR!
zDIj-nrWEBD#1|!&q)u_Se{|i#sQC4#9;TckAg_7eMt{MNKYF;55=)XZ;)^q@QbCrM
G>Hz=^E>v~^
literal 0
HcmV?d00001
diff --git a/ray_results/test_experiment/train_model_f74eb513_2_batch_size=1024,decoder_int_ff=1024,decoder_int_num_layer=8,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop__2024-06-25_15-20-03/result.json b/ray_results/test_experiment/train_model_f74eb513_2_batch_size=1024,decoder_int_ff=1024,decoder_int_num_layer=8,decoder_rt_ff=1024,decoder_rt_num_layer=2,drop__2024-06-25_15-20-03/result.json
new file mode 100644
index 0000000..e69de29
diff --git a/ray_results/test_experiment/train_model_fe083630_32_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_22-50-21/events.out.tfevents.1719350385.r8i6n2 b/ray_results/test_experiment/train_model_fe083630_32_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_22-50-21/events.out.tfevents.1719350385.r8i6n2
new file mode 100644
index 0000000000000000000000000000000000000000..d70d04e52ed2d24c2a20a14d61d62a92e59ca943
GIT binary patch
literal 2771
zcma);e@q)?7{}W?Hg=CItQ{S!h0zvGmTFosIy0Q<8QKup3`Wp6q8>-@g`T!|_3m)8
z7-h00L{V{Oe=G*afQ3KQjTo5&_9HQ7{z#_m53_7BE?bs3;%^Bi{@C;mj`#XvySD$`
z`#j(0d++CY(&s8{OTW>Vw+EQcwb9CR?_PY+<LFR$Q3}PLN0L^5w=+H7T4Ux-9NN{u
z6iwRajl6aC;07oIhr;)SwQdQ8ld`J1oi#7U?))ussn(PnSeWIQd#ACa2K?Z`5;Y-r
zCj~B^K=EhOvM6b8o2@s}T-NgrVL5OT>q$#NmXI#7*(R^~r^2RX>Zz(nn7}D434jka
zVoOa-Ax=w(Dko}4;Z%41smG%eTYDa=Gvz~leRIs1CBGd4;3eg%D8&({AXSqUv{>NA
zlf~Zq%mUwE|8gheJ&pZ?uo?WgfZ@t(qAaN#S;n||sI6-&aq|Xw$Jwt8*i<&y4eonl
z<;SZX1neDYEGY!`b#96#EH7C49I!8p4(}mg$DhNNd0@9UwM1iW=Wi0Qzq90dVV|SF
ziN4qI#0LcIRt5VP4BN(*9O+*mZhrZPV+X%BVAI)PUzxuxHhj{tjetG(BbF2b`)%&&
z4+%^4C#=r_J9J{ePr&}mlH`HCCEgM(e@&_&V5fh^j(K5!)K(qy-DRI9U|+Qg6bzeX
zYrFQ(5;q?l`8PRhz%I%Ldl&O}w7oI%83B7}6iW($y?3zr0Aaa#4C`~i{-^ibD+KK0
zmLv~szPTm3W`1`O0lW4HcFYTVAk-TBeP(he0ejpkP%!LXR<iA3iJOm{P}%ba>{Z!d
zw{Ka9zBRtRkAOWjg(Zc+4u(p<AS~Hwtj_^kx&Prg0`{ktBoFMfYD@IZ#~fz}*!45m
zF)!>h`}W2}`Y21lK57*x81^uGNh|hG*q8qM2Zs(m!`vv<7GVd+XgEwm8J9@&3ZGKB
zFnS@46n+0lT2ns_`)DXOL#{t5*B?}%1fYZ0;t72->_$F1;DQnX#brU?Q~GY?Iy?2V
zjo{8ale$w}lJ7x^9&&<XIi#4`6;PIOl$q=RMo{E5r!OPJO4p)P2NDEP>f!`3r61{U
zgQO+*MQ1lSm!C5Wtp+M-xC5+~xCG(_A02c;sf+wD2yTK(*DV(R3W>4UgH_;l<zC#u
zYN&GEYB2}zfEri+#k=a&m-;2Mf1ms^QqR<_Gy8Xyg3A<CTK!Xzs_L7YQ#D=(EJ~en
z(`gE3D65ahD9`&8Y{+2iVw~5w*Joa_&qZMUG9vxZ{8(VYv!ZJ_*2}`CEXZ-($AWj|
zfZ(7m_Gj-sPO-62YV=M&D1Kn=Sn`cS5#wcuF~G@#ER<)?!-{}4wwZxIj#<SyzeaAZ
qZ`i-hWbygOqmMF;%k=9Y44(h9N0P8aUY{|Gdr^Q|V{g28*!drlOKxfa
literal 0
HcmV?d00001
diff --git a/ray_results/test_experiment/train_model_fe083630_32_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_22-50-21/params.json b/ray_results/test_experiment/train_model_fe083630_32_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_22-50-21/params.json
new file mode 100644
index 0000000..f528e7d
--- /dev/null
+++ b/ray_results/test_experiment/train_model_fe083630_32_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_22-50-21/params.json
@@ -0,0 +1,13 @@
+{
+ "batch_size": 1024,
+ "decoder_int_ff": 512,
+ "decoder_int_num_layer": 1,
+ "decoder_rt_ff": 512,
+ "decoder_rt_num_layer": 2,
+ "drop_rate": 0.7814344084863447,
+ "embedding_dim": 1024,
+ "encoder_ff": 1024,
+ "encoder_num_layer": 8,
+ "lr": 0.030816743385966763,
+ "n_head": 2
+}
\ No newline at end of file
diff --git a/ray_results/test_experiment/train_model_fe083630_32_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_22-50-21/params.pkl b/ray_results/test_experiment/train_model_fe083630_32_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_22-50-21/params.pkl
new file mode 100644
index 0000000000000000000000000000000000000000..eabd75552a221282c9ed756addafb311b81036a0
GIT binary patch
literal 225
zcmZo*oqCM{0&1sd^a!TrCFiH47RBe4=EmnFR;CtB@#g3eNlAr>7L_1NF!hMSBr@|*
zWEgvRQ*)D2Q&KYX(&JMybEo(+u=H@j%uY)K^Lb&Kfu@3mn0okNLLl>?Vr+Tw8L5dW
zQ@ojaI8%!93*w6sOH!w}+rMON+T9!fq=zY|2*{h?m$QV&mc557DX}CuBfdDZDivf&
GsU85;s8Uw|
literal 0
HcmV?d00001
diff --git a/ray_results/test_experiment/train_model_fe083630_32_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_22-50-21/progress.csv b/ray_results/test_experiment/train_model_fe083630_32_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_22-50-21/progress.csv
new file mode 100644
index 0000000..4242a15
--- /dev/null
+++ b/ray_results/test_experiment/train_model_fe083630_32_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_22-50-21/progress.csv
@@ -0,0 +1,6 @@
+loss,timestamp,checkpoint_dir_name,should_checkpoint,done,training_iteration,trial_id,date,time_this_iter_s,time_total_s,pid,hostname,node_ip,time_since_restore,iterations_since_restore
+4034.1917479507565,1719351243,checkpoint_000000,True,False,1,fe083630,2024-06-25_23-34-04,858.9566829204559,858.9566829204559,2195253,r8i6n2,10.159.28.60,858.9566829204559,1
+2881.328838198204,1719352088,checkpoint_000001,True,False,2,fe083630,2024-06-25_23-48-09,845.1621363162994,1704.1188192367554,2195253,r8i6n2,10.159.28.60,1704.1188192367554,2
+2239.2423533041647,1719352934,checkpoint_000002,True,False,3,fe083630,2024-06-26_00-02-14,845.5501523017883,2549.6689715385437,2195253,r8i6n2,10.159.28.60,2549.6689715385437,3
+2008.2958018385518,1719353779,checkpoint_000003,True,False,4,fe083630,2024-06-26_00-16-19,845.2403128147125,3394.909284353256,2195253,r8i6n2,10.159.28.60,3394.909284353256,4
+1961.6992687315453,1719354625,checkpoint_000004,True,False,5,fe083630,2024-06-26_00-30-25,845.8089480400085,4240.718232393265,2195253,r8i6n2,10.159.28.60,4240.718232393265,5
diff --git a/ray_results/test_experiment/train_model_fe083630_32_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_22-50-21/result.json b/ray_results/test_experiment/train_model_fe083630_32_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_22-50-21/result.json
new file mode 100644
index 0000000..68c099d
--- /dev/null
+++ b/ray_results/test_experiment/train_model_fe083630_32_batch_size=1024,decoder_int_ff=512,decoder_int_num_layer=1,decoder_rt_ff=512,decoder_rt_num_layer=2,drop_r_2024-06-25_22-50-21/result.json
@@ -0,0 +1,5 @@
+{"loss": 4034.1917479507565, "timestamp": 1719351243, "checkpoint_dir_name": "checkpoint_000000", "should_checkpoint": true, "done": false, "training_iteration": 1, "trial_id": "fe083630", "date": "2024-06-25_23-34-04", "time_this_iter_s": 858.9566829204559, "time_total_s": 858.9566829204559, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 8, "decoder_rt_num_layer": 2, "decoder_int_num_layer": 1, "embedding_dim": 1024, "encoder_ff": 1024, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 2, "drop_rate": 0.7814344084863447, "lr": 0.030816743385966763, "batch_size": 1024}, "time_since_restore": 858.9566829204559, "iterations_since_restore": 1}
+{"loss": 2881.328838198204, "timestamp": 1719352088, "checkpoint_dir_name": "checkpoint_000001", "should_checkpoint": true, "done": false, "training_iteration": 2, "trial_id": "fe083630", "date": "2024-06-25_23-48-09", "time_this_iter_s": 845.1621363162994, "time_total_s": 1704.1188192367554, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 8, "decoder_rt_num_layer": 2, "decoder_int_num_layer": 1, "embedding_dim": 1024, "encoder_ff": 1024, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 2, "drop_rate": 0.7814344084863447, "lr": 0.030816743385966763, "batch_size": 1024}, "time_since_restore": 1704.1188192367554, "iterations_since_restore": 2}
+{"loss": 2239.2423533041647, "timestamp": 1719352934, "checkpoint_dir_name": "checkpoint_000002", "should_checkpoint": true, "done": false, "training_iteration": 3, "trial_id": "fe083630", "date": "2024-06-26_00-02-14", "time_this_iter_s": 845.5501523017883, "time_total_s": 2549.6689715385437, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 8, "decoder_rt_num_layer": 2, "decoder_int_num_layer": 1, "embedding_dim": 1024, "encoder_ff": 1024, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 2, "drop_rate": 0.7814344084863447, "lr": 0.030816743385966763, "batch_size": 1024}, "time_since_restore": 2549.6689715385437, "iterations_since_restore": 3}
+{"loss": 2008.2958018385518, "timestamp": 1719353779, "checkpoint_dir_name": "checkpoint_000003", "should_checkpoint": true, "done": false, "training_iteration": 4, "trial_id": "fe083630", "date": "2024-06-26_00-16-19", "time_this_iter_s": 845.2403128147125, "time_total_s": 3394.909284353256, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 8, "decoder_rt_num_layer": 2, "decoder_int_num_layer": 1, "embedding_dim": 1024, "encoder_ff": 1024, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 2, "drop_rate": 0.7814344084863447, "lr": 0.030816743385966763, "batch_size": 1024}, "time_since_restore": 3394.909284353256, "iterations_since_restore": 4}
+{"loss": 1961.6992687315453, "timestamp": 1719354625, "checkpoint_dir_name": "checkpoint_000004", "should_checkpoint": true, "done": false, "training_iteration": 5, "trial_id": "fe083630", "date": "2024-06-26_00-30-25", "time_this_iter_s": 845.8089480400085, "time_total_s": 4240.718232393265, "pid": 2195253, "hostname": "r8i6n2", "node_ip": "10.159.28.60", "config": {"encoder_num_layer": 8, "decoder_rt_num_layer": 2, "decoder_int_num_layer": 1, "embedding_dim": 1024, "encoder_ff": 1024, "decoder_rt_ff": 512, "decoder_int_ff": 512, "n_head": 2, "drop_rate": 0.7814344084863447, "lr": 0.030816743385966763, "batch_size": 1024}, "time_since_restore": 4240.718232393265, "iterations_since_restore": 5}
diff --git a/ray_results/test_experiment/tuner.pkl b/ray_results/test_experiment/tuner.pkl
index cf4c3d302f96c6d70b0fcc0431588892ca366842..064732323e8c0aae443330fb73506baaa5ff28c9 100644
GIT binary patch
delta 27
icmZpYX_DEH%*e<wIfc=Lkz?~r#yf0`3Y*I~dRPH#+6SQk
delta 27
icmZpYX_DEH%*e<xIfc=Lk!ABt#yf0`Oq<I%dRPH!rU!cf
--
GitLab