Training in progress, step 2000

.gitattributes

@@ -25,3 +25,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zstandard filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+wandb/run-20220127_121247-o8za41vn/run-o8za41vn.wandb filter=lfs diff=lfs merge=lfs -text

pytorch_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:4d80289eb039576f75e350675c60903ba8fcc6cc2d64287efc8fe26ccf4abf09
+oid sha256:fa0be572201ab8a60ce90f69e54b99fea4e6ff0901184575f6db6c67356e2eb8
 size 1262075377

runs/Jan27_12-08-15_brutasse/events.out.tfevents.1643285566.brutasse.15970.0

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a2ea9ccd8501b33656ca7e02d6de78e045a6244eede76ffeff590957e662a184
-size 7987
+oid sha256:205456cee4092e6691509aa608a1f11934e606eb28c6271d8ffb45034208fafa
+size 9090

wandb/run-20220127_121247-o8za41vn/files/output.log

@@ -1143,3 +1143,390 @@ Configuration saved in ./checkpoint-1500/config.json
 Model weights saved in ./checkpoint-1500/pytorch_model.bin
 Configuration saved in ./checkpoint-1500/preprocessor_config.json
 {'eval_loss': 1.147773265838623, 'eval_wer': 0.8787680953732614, 'eval_runtime': 146.3922, 'eval_samples_per_second': 33.082, 'eval_steps_per_second': 4.14, 'epoch': 4.11}
+Configuration saved in ./preprocessor_config.json
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  9%|████████████▊                                                                                                                                     | 1599/18250 [41:56<4:42:20,  1.02s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  9%|█████████████▌                                                                                                                                    | 1699/18250 [43:57<4:41:02,  1.02s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 10%|██████████████▍                                                                                                                                   | 1800/18250 [45:58<4:33:04,  1.00it/s]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 10%|███████████████▏                                                                                                                                  | 1901/18250 [48:03<5:39:28,  1.25s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 11%|███████████████▉                                                                                                                                  | 1999/18250 [50:01<5:43:52,  1.27s/it]
+ 11%|████████████████                                                                                                                                  | 2000/18250 [50:03<5:36:00,  1.24s/it]The following columns in the evaluation set  don't have a corresponding argument in `Wav2Vec2ForCTC.forward` and have been ignored: input_length.
+***** Running Evaluation *****
+  Num examples = 4843
+  Batch size = 8
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+Configuration saved in ./checkpoint-2000/config.json
+{'eval_loss': 0.6748705506324768, 'eval_wer': 0.6546693159239285, 'eval_runtime': 146.3732, 'eval_samples_per_second': 33.087, 'eval_steps_per_second': 4.14, 'epoch': 5.48}
+Model weights saved in ./checkpoint-2000/pytorch_model.bin
+Configuration saved in ./checkpoint-2000/preprocessor_config.json
+Configuration saved in ./preprocessor_config.json
+01/27/2022 13:06:22 - WARNING - huggingface_hub.repository - Adding files tracked by Git LFS: ['wandb/run-20220127_121247-o8za41vn/run-o8za41vn.wandb']. This may take a bit of time if the files are large.

wandb/run-20220127_121247-o8za41vn/logs/debug-internal.log

@@ -2826,3 +2826,541 @@
 2022-01-27 12:52:38,173 DEBUG   SenderThread:16113 [sender.py:send():182] send: stats
 2022-01-27 12:52:44,038 DEBUG   HandlerThread:16113 [handler.py:handle_request():126] handle_request: stop_status
 2022-01-27 12:52:44,038 DEBUG   SenderThread:16113 [sender.py:send_request():196] send_request: stop_status
+2022-01-27 12:52:53,562 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:52:55,563 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:52:57,563 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:52:59,170 DEBUG   HandlerThread:16113 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-27 12:52:59,170 DEBUG   SenderThread:16113 [sender.py:send_request():196] send_request: stop_status
+2022-01-27 12:52:59,563 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:53:01,564 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:53:03,564 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:53:05,565 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:53:07,565 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:53:08,450 DEBUG   SenderThread:16113 [sender.py:send():182] send: stats
+2022-01-27 12:53:09,566 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:53:11,566 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:53:14,344 DEBUG   HandlerThread:16113 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-27 12:53:14,345 DEBUG   SenderThread:16113 [sender.py:send_request():196] send_request: stop_status
+2022-01-27 12:53:14,567 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:53:16,567 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:53:18,568 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:53:20,568 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:53:22,569 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:53:24,569 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:53:26,570 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:53:28,570 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:53:29,430 DEBUG   HandlerThread:16113 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-27 12:53:29,430 DEBUG   SenderThread:16113 [sender.py:send_request():196] send_request: stop_status
+2022-01-27 12:53:30,571 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:53:32,571 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:53:34,572 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:53:36,572 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:53:38,572 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:53:38,704 DEBUG   SenderThread:16113 [sender.py:send():182] send: stats
+2022-01-27 12:53:40,573 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:53:42,573 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:53:44,522 DEBUG   HandlerThread:16113 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-27 12:53:44,522 DEBUG   SenderThread:16113 [sender.py:send_request():196] send_request: stop_status
+2022-01-27 12:53:44,574 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:53:46,574 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:53:48,574 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:53:50,575 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:53:52,575 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:53:54,575 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:53:56,576 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:53:58,576 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:53:59,593 DEBUG   HandlerThread:16113 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-27 12:53:59,593 DEBUG   SenderThread:16113 [sender.py:send_request():196] send_request: stop_status
+2022-01-27 12:54:00,576 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:54:02,577 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:54:04,577 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:54:06,577 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:54:08,578 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:54:08,945 DEBUG   SenderThread:16113 [sender.py:send():182] send: stats
+2022-01-27 12:54:10,578 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:54:12,578 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:54:14,578 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:54:14,687 DEBUG   HandlerThread:16113 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-27 12:54:14,687 DEBUG   SenderThread:16113 [sender.py:send_request():196] send_request: stop_status
+2022-01-27 12:54:16,579 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:54:18,579 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:54:20,579 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:54:22,580 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:54:24,580 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:54:26,580 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:54:28,581 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:54:29,794 DEBUG   HandlerThread:16113 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-27 12:54:29,794 DEBUG   SenderThread:16113 [sender.py:send_request():196] send_request: stop_status
+2022-01-27 12:54:30,581 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:54:32,581 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:54:34,582 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:54:36,582 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:54:38,582 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:54:39,194 DEBUG   SenderThread:16113 [sender.py:send():182] send: stats
+2022-01-27 12:54:40,583 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:54:42,583 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:54:44,583 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:54:44,885 DEBUG   HandlerThread:16113 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-27 12:54:44,886 DEBUG   SenderThread:16113 [sender.py:send_request():196] send_request: stop_status
+2022-01-27 12:54:46,584 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:54:48,584 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:54:50,580 DEBUG   SenderThread:16113 [sender.py:send():182] send: history
+2022-01-27 12:54:50,628 DEBUG   SenderThread:16113 [sender.py:send():182] send: summary
+2022-01-27 12:54:50,654 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:54:50,655 INFO    SenderThread:16113 [sender.py:_save_file():873] saving file wandb-summary.json with policy end
+2022-01-27 12:54:51,655 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/wandb-summary.json
+2022-01-27 12:54:52,655 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:54:54,655 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:54:56,656 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:54:58,656 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:54:59,959 DEBUG   HandlerThread:16113 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-27 12:54:59,960 DEBUG   SenderThread:16113 [sender.py:send_request():196] send_request: stop_status
+2022-01-27 12:55:00,656 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:55:02,657 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:55:04,657 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:55:06,657 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:55:08,658 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:55:09,445 DEBUG   SenderThread:16113 [sender.py:send():182] send: stats
+2022-01-27 12:55:10,658 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:55:12,659 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:55:14,659 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:55:15,099 DEBUG   HandlerThread:16113 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-27 12:55:15,100 DEBUG   SenderThread:16113 [sender.py:send_request():196] send_request: stop_status
+2022-01-27 12:55:16,659 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:55:18,660 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:55:20,660 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:55:22,661 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:55:24,661 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:55:26,661 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:55:28,662 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:55:30,193 DEBUG   HandlerThread:16113 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-27 12:55:30,193 DEBUG   SenderThread:16113 [sender.py:send_request():196] send_request: stop_status
+2022-01-27 12:55:30,662 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:55:32,662 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:55:35,663 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:55:37,663 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:55:39,663 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:55:39,690 DEBUG   SenderThread:16113 [sender.py:send():182] send: stats
+2022-01-27 12:55:41,664 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:55:43,664 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:55:45,310 DEBUG   HandlerThread:16113 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-27 12:55:45,310 DEBUG   SenderThread:16113 [sender.py:send_request():196] send_request: stop_status
+2022-01-27 12:55:45,664 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:55:47,665 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:55:49,665 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:55:51,665 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:55:53,666 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:55:55,666 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:55:57,666 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:55:59,667 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:56:00,394 DEBUG   HandlerThread:16113 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-27 12:56:00,394 DEBUG   SenderThread:16113 [sender.py:send_request():196] send_request: stop_status
+2022-01-27 12:56:01,667 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:56:03,668 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:56:05,668 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:56:07,668 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:56:09,669 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:56:09,964 DEBUG   SenderThread:16113 [sender.py:send():182] send: stats
+2022-01-27 12:56:11,669 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:56:13,669 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:56:15,516 DEBUG   HandlerThread:16113 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-27 12:56:15,517 DEBUG   SenderThread:16113 [sender.py:send_request():196] send_request: stop_status
+2022-01-27 12:56:15,670 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:56:17,670 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:56:19,670 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:56:21,671 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:56:23,671 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:56:25,672 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:56:27,672 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:56:29,672 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:56:30,594 DEBUG   HandlerThread:16113 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-27 12:56:30,594 DEBUG   SenderThread:16113 [sender.py:send_request():196] send_request: stop_status
+2022-01-27 12:56:31,673 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:56:33,673 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:56:35,673 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:56:37,674 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:56:39,674 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:56:40,233 DEBUG   SenderThread:16113 [sender.py:send():182] send: stats
+2022-01-27 12:56:41,674 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:56:43,674 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:56:45,675 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:56:45,706 DEBUG   HandlerThread:16113 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-27 12:56:45,706 DEBUG   SenderThread:16113 [sender.py:send_request():196] send_request: stop_status
+2022-01-27 12:56:47,675 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:56:49,675 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:56:51,188 DEBUG   SenderThread:16113 [sender.py:send():182] send: history
+2022-01-27 12:56:51,242 DEBUG   SenderThread:16113 [sender.py:send():182] send: summary
+2022-01-27 12:56:51,264 INFO    SenderThread:16113 [sender.py:_save_file():873] saving file wandb-summary.json with policy end
+2022-01-27 12:56:51,676 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/wandb-summary.json
+2022-01-27 12:56:51,676 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:56:53,676 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:56:55,676 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:56:57,677 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:56:59,677 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:57:00,838 DEBUG   HandlerThread:16113 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-27 12:57:00,839 DEBUG   SenderThread:16113 [sender.py:send_request():196] send_request: stop_status
+2022-01-27 12:57:01,678 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:57:03,678 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:57:05,678 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:57:07,678 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:57:09,679 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:57:10,502 DEBUG   SenderThread:16113 [sender.py:send():182] send: stats
+2022-01-27 12:57:11,679 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:57:13,679 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:57:15,680 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:57:15,912 DEBUG   HandlerThread:16113 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-27 12:57:15,912 DEBUG   SenderThread:16113 [sender.py:send_request():196] send_request: stop_status
+2022-01-27 12:57:17,680 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:57:19,680 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:57:21,681 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:57:23,681 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:57:25,681 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:57:27,682 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:57:29,682 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:57:30,982 DEBUG   HandlerThread:16113 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-27 12:57:30,982 DEBUG   SenderThread:16113 [sender.py:send_request():196] send_request: stop_status
+2022-01-27 12:57:31,682 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:57:33,682 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:57:35,683 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:57:37,683 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:57:39,683 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:57:40,765 DEBUG   SenderThread:16113 [sender.py:send():182] send: stats
+2022-01-27 12:57:41,684 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:57:43,684 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:57:45,684 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:57:46,108 DEBUG   HandlerThread:16113 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-27 12:57:46,108 DEBUG   SenderThread:16113 [sender.py:send_request():196] send_request: stop_status
+2022-01-27 12:57:47,685 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:57:50,685 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:57:52,685 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:57:54,686 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:57:56,686 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:57:58,686 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:58:00,687 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:58:01,177 DEBUG   HandlerThread:16113 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-27 12:58:01,178 DEBUG   SenderThread:16113 [sender.py:send_request():196] send_request: stop_status
+2022-01-27 12:58:02,687 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:58:04,687 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:58:06,688 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:58:08,688 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:58:10,688 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:58:11,022 DEBUG   SenderThread:16113 [sender.py:send():182] send: stats
+2022-01-27 12:58:12,689 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:58:14,689 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:58:16,260 DEBUG   HandlerThread:16113 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-27 12:58:16,261 DEBUG   SenderThread:16113 [sender.py:send_request():196] send_request: stop_status
+2022-01-27 12:58:16,689 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:58:18,690 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:58:20,690 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:58:22,690 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:58:24,691 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:58:26,691 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:58:28,691 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:58:30,692 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:58:31,348 DEBUG   HandlerThread:16113 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-27 12:58:31,348 DEBUG   SenderThread:16113 [sender.py:send_request():196] send_request: stop_status
+2022-01-27 12:58:32,692 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:58:34,692 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:58:36,693 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:58:38,693 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:58:40,693 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:58:41,279 DEBUG   SenderThread:16113 [sender.py:send():182] send: stats
+2022-01-27 12:58:42,694 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:58:44,694 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:58:46,433 DEBUG   HandlerThread:16113 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-27 12:58:46,433 DEBUG   SenderThread:16113 [sender.py:send_request():196] send_request: stop_status
+2022-01-27 12:58:46,694 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:58:48,695 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:58:50,695 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:58:51,337 DEBUG   SenderThread:16113 [sender.py:send():182] send: history
+2022-01-27 12:58:51,378 DEBUG   SenderThread:16113 [sender.py:send():182] send: summary
+2022-01-27 12:58:51,400 INFO    SenderThread:16113 [sender.py:_save_file():873] saving file wandb-summary.json with policy end
+2022-01-27 12:58:51,695 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/wandb-summary.json
+2022-01-27 12:58:52,695 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:58:53,696 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:58:54,696 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:58:56,696 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:58:58,696 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:59:00,697 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:59:01,516 DEBUG   HandlerThread:16113 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-27 12:59:01,517 DEBUG   SenderThread:16113 [sender.py:send_request():196] send_request: stop_status
+2022-01-27 12:59:02,697 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:59:04,697 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:59:06,698 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:59:08,698 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:59:10,698 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:59:11,597 DEBUG   SenderThread:16113 [sender.py:send():182] send: stats
+2022-01-27 12:59:12,699 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:59:14,699 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:59:16,654 DEBUG   HandlerThread:16113 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-27 12:59:16,654 DEBUG   SenderThread:16113 [sender.py:send_request():196] send_request: stop_status
+2022-01-27 12:59:16,700 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:59:18,700 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:59:20,700 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:59:22,701 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:59:24,701 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:59:26,702 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:59:28,702 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:59:30,702 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:59:31,965 DEBUG   HandlerThread:16113 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-27 12:59:31,965 DEBUG   SenderThread:16113 [sender.py:send_request():196] send_request: stop_status
+2022-01-27 12:59:32,703 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:59:34,703 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:59:36,703 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:59:38,704 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:59:40,704 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:59:41,841 DEBUG   SenderThread:16113 [sender.py:send():182] send: stats
+2022-01-27 12:59:42,704 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:59:44,705 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:59:46,705 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:59:47,118 DEBUG   HandlerThread:16113 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-27 12:59:47,118 DEBUG   SenderThread:16113 [sender.py:send_request():196] send_request: stop_status
+2022-01-27 12:59:48,705 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:59:50,706 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:59:52,706 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:59:54,706 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:59:56,706 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:59:58,707 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 13:00:01,707 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 13:00:02,206 DEBUG   HandlerThread:16113 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-27 13:00:02,207 DEBUG   SenderThread:16113 [sender.py:send_request():196] send_request: stop_status
+2022-01-27 13:00:03,708 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 13:00:05,708 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 13:00:07,708 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 13:00:09,708 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 13:00:11,709 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 13:00:12,093 DEBUG   SenderThread:16113 [sender.py:send():182] send: stats
+2022-01-27 13:00:13,709 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 13:00:15,709 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 13:00:17,315 DEBUG   HandlerThread:16113 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-27 13:00:17,315 DEBUG   SenderThread:16113 [sender.py:send_request():196] send_request: stop_status
+2022-01-27 13:00:17,710 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 13:00:19,710 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 13:00:21,710 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 13:00:23,711 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 13:00:25,711 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 13:00:27,711 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 13:00:29,712 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 13:00:31,712 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 13:00:32,396 DEBUG   HandlerThread:16113 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-27 13:00:32,397 DEBUG   SenderThread:16113 [sender.py:send_request():196] send_request: stop_status
+2022-01-27 13:00:33,712 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 13:00:35,713 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 13:00:37,713 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 13:00:39,713 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 13:00:41,714 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 13:00:42,344 DEBUG   SenderThread:16113 [sender.py:send():182] send: stats
+2022-01-27 13:00:43,714 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 13:00:45,714 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 13:00:47,541 DEBUG   HandlerThread:16113 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-27 13:00:47,541 DEBUG   SenderThread:16113 [sender.py:send_request():196] send_request: stop_status
+2022-01-27 13:00:47,715 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 13:00:49,715 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 13:00:51,715 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 13:00:53,716 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 13:00:55,716 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 13:00:55,740 DEBUG   SenderThread:16113 [sender.py:send():182] send: history
+2022-01-27 13:00:55,782 DEBUG   SenderThread:16113 [sender.py:send():182] send: summary
+2022-01-27 13:00:55,808 INFO    SenderThread:16113 [sender.py:_save_file():873] saving file wandb-summary.json with policy end
+2022-01-27 13:00:56,716 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/wandb-summary.json
+2022-01-27 13:00:57,716 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 13:00:59,717 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 13:01:01,717 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 13:01:02,613 DEBUG   HandlerThread:16113 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-27 13:01:02,613 DEBUG   SenderThread:16113 [sender.py:send_request():196] send_request: stop_status
+2022-01-27 13:01:03,717 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 13:01:05,718 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 13:01:07,718 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 13:01:09,718 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 13:01:11,719 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 13:01:12,610 DEBUG   SenderThread:16113 [sender.py:send():182] send: stats
+2022-01-27 13:01:13,719 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 13:01:15,719 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 13:01:17,719 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 13:01:17,737 DEBUG   HandlerThread:16113 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-27 13:01:17,737 DEBUG   SenderThread:16113 [sender.py:send_request():196] send_request: stop_status
+2022-01-27 13:01:19,720 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 13:01:21,720 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 13:01:23,720 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 13:01:25,721 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 13:01:27,721 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 13:01:29,721 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 13:01:31,722 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 13:01:32,861 DEBUG   HandlerThread:16113 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-27 13:01:32,861 DEBUG   SenderThread:16113 [sender.py:send_request():196] send_request: stop_status
+2022-01-27 13:01:33,722 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 13:01:35,722 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 13:01:37,723 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 13:01:39,723 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 13:01:41,723 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 13:01:42,863 DEBUG   SenderThread:16113 [sender.py:send():182] send: stats
+2022-01-27 13:01:43,724 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 13:01:45,724 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 13:01:47,724 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 13:01:47,944 DEBUG   HandlerThread:16113 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-27 13:01:47,944 DEBUG   SenderThread:16113 [sender.py:send_request():196] send_request: stop_status
+2022-01-27 13:01:49,725 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 13:01:51,725 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 13:01:53,725 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 13:01:55,725 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 13:01:57,726 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 13:01:59,726 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 13:02:01,726 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 13:02:03,018 DEBUG   HandlerThread:16113 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-27 13:02:03,019 DEBUG   SenderThread:16113 [sender.py:send_request():196] send_request: stop_status
+2022-01-27 13:02:03,727 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 13:02:06,727 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 13:02:08,727 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 13:02:10,728 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 13:02:12,728 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 13:02:13,131 DEBUG   SenderThread:16113 [sender.py:send():182] send: stats
+2022-01-27 13:02:14,729 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 13:02:16,729 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 13:02:18,091 DEBUG   HandlerThread:16113 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-27 13:02:18,091 DEBUG   SenderThread:16113 [sender.py:send_request():196] send_request: stop_status
+2022-01-27 13:02:18,729 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 13:02:20,730 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 13:02:22,730 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 13:02:24,730 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 13:02:26,731 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 13:02:28,731 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 13:02:30,731 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 13:02:32,731 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 13:02:33,237 DEBUG   HandlerThread:16113 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-27 13:02:33,237 DEBUG   SenderThread:16113 [sender.py:send_request():196] send_request: stop_status
+2022-01-27 13:02:34,732 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 13:02:36,732 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 13:02:38,732 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 13:02:40,733 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 13:02:42,733 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 13:02:43,390 DEBUG   SenderThread:16113 [sender.py:send():182] send: stats
+2022-01-27 13:02:44,733 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 13:02:46,734 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 13:02:48,314 DEBUG   HandlerThread:16113 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-27 13:02:48,314 DEBUG   SenderThread:16113 [sender.py:send_request():196] send_request: stop_status
+2022-01-27 13:02:48,734 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 13:02:50,734 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 13:02:52,735 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 13:02:54,735 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 13:02:56,297 DEBUG   SenderThread:16113 [sender.py:send():182] send: history
+2022-01-27 13:02:56,349 DEBUG   SenderThread:16113 [sender.py:send():182] send: summary
+2022-01-27 13:02:56,371 INFO    SenderThread:16113 [sender.py:_save_file():873] saving file wandb-summary.json with policy end
+2022-01-27 13:02:56,735 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/wandb-summary.json
+2022-01-27 13:02:56,735 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 13:02:58,736 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 13:03:00,736 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 13:03:02,736 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 13:03:03,442 DEBUG   HandlerThread:16113 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-27 13:03:03,442 DEBUG   SenderThread:16113 [sender.py:send_request():196] send_request: stop_status
+2022-01-27 13:03:04,737 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 13:03:06,737 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 13:03:08,737 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 13:03:10,738 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 13:03:12,738 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 13:03:13,661 DEBUG   SenderThread:16113 [sender.py:send():182] send: stats
+2022-01-27 13:03:14,738 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 13:03:16,739 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 13:03:18,545 DEBUG   HandlerThread:16113 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-27 13:03:18,545 DEBUG   SenderThread:16113 [sender.py:send_request():196] send_request: stop_status
+2022-01-27 13:03:18,739 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 13:03:20,740 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 13:03:22,740 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 13:03:24,740 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 13:03:26,741 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 13:03:28,741 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 13:03:30,741 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 13:03:32,742 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 13:03:33,621 DEBUG   HandlerThread:16113 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-27 13:03:33,621 DEBUG   SenderThread:16113 [sender.py:send_request():196] send_request: stop_status
+2022-01-27 13:03:34,742 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 13:03:36,742 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 13:03:38,743 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 13:03:40,743 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 13:03:42,743 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 13:03:43,920 DEBUG   SenderThread:16113 [sender.py:send():182] send: stats
+2022-01-27 13:03:45,744 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 13:03:47,744 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 13:03:48,732 DEBUG   HandlerThread:16113 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-27 13:03:48,732 DEBUG   SenderThread:16113 [sender.py:send_request():196] send_request: stop_status
+2022-01-27 13:03:49,745 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 13:03:51,745 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 13:03:53,745 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 13:03:55,746 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 13:03:57,746 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 13:03:59,746 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 13:04:01,747 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 13:04:03,747 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 13:04:03,817 DEBUG   HandlerThread:16113 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-27 13:04:03,817 DEBUG   SenderThread:16113 [sender.py:send_request():196] send_request: stop_status
+2022-01-27 13:04:05,748 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 13:04:07,748 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 13:04:09,748 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 13:04:11,749 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 13:04:13,749 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 13:04:14,178 DEBUG   SenderThread:16113 [sender.py:send():182] send: stats
+2022-01-27 13:04:15,749 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 13:04:17,750 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 13:04:18,924 DEBUG   HandlerThread:16113 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-27 13:04:18,924 DEBUG   SenderThread:16113 [sender.py:send_request():196] send_request: stop_status
+2022-01-27 13:04:19,750 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 13:04:21,750 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 13:04:23,751 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 13:04:25,751 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 13:04:27,751 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 13:04:29,752 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 13:04:31,752 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 13:04:33,752 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 13:04:34,077 DEBUG   HandlerThread:16113 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-27 13:04:34,078 DEBUG   SenderThread:16113 [sender.py:send_request():196] send_request: stop_status
+2022-01-27 13:04:35,753 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 13:04:37,753 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 13:04:39,753 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 13:04:41,754 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 13:04:43,754 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 13:04:44,432 DEBUG   SenderThread:16113 [sender.py:send():182] send: stats
+2022-01-27 13:04:45,754 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 13:04:47,755 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 13:04:49,201 DEBUG   HandlerThread:16113 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-27 13:04:49,201 DEBUG   SenderThread:16113 [sender.py:send_request():196] send_request: stop_status
+2022-01-27 13:04:49,755 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 13:04:51,755 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 13:04:53,756 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 13:04:55,756 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 13:04:57,756 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 13:04:59,757 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 13:05:01,757 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 13:05:03,757 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 13:05:04,347 DEBUG   HandlerThread:16113 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-27 13:05:04,347 DEBUG   SenderThread:16113 [sender.py:send_request():196] send_request: stop_status
+2022-01-27 13:05:05,758 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 13:05:07,758 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 13:05:09,758 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 13:05:11,759 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 13:05:13,759 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 13:05:14,683 DEBUG   SenderThread:16113 [sender.py:send():182] send: stats
+2022-01-27 13:05:15,759 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 13:05:17,760 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 13:05:19,435 DEBUG   HandlerThread:16113 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-27 13:05:19,435 DEBUG   SenderThread:16113 [sender.py:send_request():196] send_request: stop_status
+2022-01-27 13:05:19,760 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 13:05:21,760 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 13:05:22,677 DEBUG   SenderThread:16113 [sender.py:send():182] send: history
+2022-01-27 13:05:22,677 DEBUG   SenderThread:16113 [sender.py:send():182] send: summary
+2022-01-27 13:05:22,705 INFO    SenderThread:16113 [sender.py:_save_file():873] saving file wandb-summary.json with policy end
+2022-01-27 13:05:22,761 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/wandb-summary.json
+2022-01-27 13:05:23,761 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 13:05:24,762 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 13:05:25,762 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 13:05:34,580 DEBUG   HandlerThread:16113 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-27 13:05:34,580 DEBUG   SenderThread:16113 [sender.py:send_request():196] send_request: stop_status
+2022-01-27 13:05:44,983 DEBUG   SenderThread:16113 [sender.py:send():182] send: stats
+2022-01-27 13:05:49,655 DEBUG   HandlerThread:16113 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-27 13:05:49,655 DEBUG   SenderThread:16113 [sender.py:send_request():196] send_request: stop_status
+2022-01-27 13:06:00,775 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 13:06:04,729 DEBUG   HandlerThread:16113 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-27 13:06:04,729 DEBUG   SenderThread:16113 [sender.py:send_request():196] send_request: stop_status
+2022-01-27 13:06:15,304 DEBUG   SenderThread:16113 [sender.py:send():182] send: stats
+2022-01-27 13:06:19,811 DEBUG   HandlerThread:16113 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-27 13:06:19,812 DEBUG   SenderThread:16113 [sender.py:send_request():196] send_request: stop_status
+2022-01-27 13:06:23,781 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 13:06:24,782 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log