Upload folder using huggingface_hub

.gitattributes

@@ -84,3 +84,11 @@ Bible/25k/ta-en/preprocess_ta-en/train.ta-en.en.idx filter=lfs diff=lfs merge=lf
 Bible/25k/ta-en/preprocess_ta-en/train.ta-en.ta.idx filter=lfs diff=lfs merge=lfs -text
 Bible/25k/ta-en/preprocess_ta-en/train.tam_Taml-eng_Latn.eng_Latn.idx filter=lfs diff=lfs merge=lfs -text
 Bible/25k/ta-en/preprocess_ta-en/train.tam_Taml-eng_Latn.tam_Taml.idx filter=lfs diff=lfs merge=lfs -text
+MultiDomain/1k/en-si/preprocess_en-si/train.en-si.en.idx filter=lfs diff=lfs merge=lfs -text
+MultiDomain/1k/en-si/preprocess_en-si/train.en-si.si.idx filter=lfs diff=lfs merge=lfs -text
+MultiDomain/1k/en-ta/preprocess_en-ta/train.en-ta.en.idx filter=lfs diff=lfs merge=lfs -text
+MultiDomain/1k/en-ta/preprocess_en-ta/train.en-ta.ta.idx filter=lfs diff=lfs merge=lfs -text
+MultiDomain/1k/si-en/preprocess_si-en/train.si-en.en.idx filter=lfs diff=lfs merge=lfs -text
+MultiDomain/1k/si-en/preprocess_si-en/train.si-en.si.idx filter=lfs diff=lfs merge=lfs -text
+MultiDomain/1k/ta-en/preprocess_ta-en/train.ta-en.en.idx filter=lfs diff=lfs merge=lfs -text
+MultiDomain/1k/ta-en/preprocess_ta-en/train.ta-en.ta.idx filter=lfs diff=lfs merge=lfs -text

MultiDomain/1k/en-si/cross_domain_outputs/Bible/en-si.csv

@@ -0,0 +1,2 @@
+model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet
+_,16.73323720094863,14.45985662707628,0.3776804526380206,5.103039904588903,4.013575418200901,0.4178069984018803

MultiDomain/1k/en-si/cross_domain_outputs/Bible/preprocess_en-si/preprocess.log

@@ -0,0 +1,6 @@
+Namespace(aim_repo=None, aim_run_hash=None, align_suffix=None, alignfile=None, all_gather_list_size=16384, amp=False, amp_batch_retries=2, amp_init_scale=128, amp_scale_window=None, azureml_logging=False, bf16=False, bpe=None, cpu=False, criterion='cross_entropy', dataset_impl='mmap', destdir='../trained_models/Fairseq/MultiDomain/1k/en-si/cross_domain_outputs/Bible/preprocess_en-si/', dict_only=False, empty_cache_freq=0, fp16=False, fp16_init_scale=128, fp16_no_flatten_grads=False, fp16_scale_tolerance=0.0, fp16_scale_window=None, joined_dictionary=False, log_file=None, log_format=None, log_interval=100, lr_scheduler='fixed', memory_efficient_bf16=False, memory_efficient_fp16=False, min_loss_scale=0.0001, model_parallel_size=1, no_progress_bar=False, nwordssrc=-1, nwordstgt=-1, on_cpu_convert_precision=False, only_source=False, optimizer=None, padding_factor=8, plasma_path='/tmp/plasma', profile=False, quantization_config_path=None, reset_logging=False, scoring='bleu', seed=1, source_lang='en', srcdict='../trained_models/Fairseq/MultiDomain/1k/en-si/preprocess_en-si/dict.en.txt', suppress_crashes=False, target_lang='si', task='translation', tensorboard_logdir=None, testpref='../trained_models/Fairseq/MultiDomain/1k/en-si/cross_domain_outputs/Bible/encoded/test.bpe', tgtdict='../trained_models/Fairseq/MultiDomain/1k/en-si/preprocess_en-si/dict.si.txt', threshold_loss_scale=None, thresholdsrc=0, thresholdtgt=0, tokenizer=None, tpu=False, trainpref=None, use_plasma_view=False, user_dir=None, validpref=None, wandb_project=None, workers=16)
+[en] Dictionary: 23264 types
+[en] ../trained_models/Fairseq/MultiDomain/1k/en-si/cross_domain_outputs/Bible/encoded/test.bpe.en: 1000 sents, 34649 tokens, 0.133% replaced (by <unk>)
+[si] Dictionary: 23264 types
+[si] ../trained_models/Fairseq/MultiDomain/1k/en-si/cross_domain_outputs/Bible/encoded/test.bpe.si: 1000 sents, 29635 tokens, 0.199% replaced (by <unk>)
+Wrote preprocessed data to ../trained_models/Fairseq/MultiDomain/1k/en-si/cross_domain_outputs/Bible/preprocess_en-si/

MultiDomain/1k/en-si/cross_domain_outputs/Bible/preprocess_en-si/test.en-si.en.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:701da304b77bc596b561099876e9c4ef3dbe7aa95afdf92c939a96a069a36146
+size 69298

MultiDomain/1k/en-si/cross_domain_outputs/Bible/preprocess_en-si/test.en-si.si.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8c9a37030c8fbe68d2e5435e8a9e4afee3d7bcc634031b5e9babc662362663f6
+size 59270

MultiDomain/1k/en-si/cross_domain_outputs/FloresPlus/en-si.csv

@@ -0,0 +1,2 @@
+model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet
+_,10.865665465562982,9.975229580498818,0.40499193955174684,0.7102767038389898,0.9726355507059521,0.3313076307503838

MultiDomain/1k/en-si/cross_domain_outputs/FloresPlus/preprocess_en-si/preprocess.log

@@ -0,0 +1,6 @@
+Namespace(aim_repo=None, aim_run_hash=None, align_suffix=None, alignfile=None, all_gather_list_size=16384, amp=False, amp_batch_retries=2, amp_init_scale=128, amp_scale_window=None, azureml_logging=False, bf16=False, bpe=None, cpu=False, criterion='cross_entropy', dataset_impl='mmap', destdir='../trained_models/Fairseq/MultiDomain/1k/en-si/cross_domain_outputs/FloresPlus/preprocess_en-si/', dict_only=False, empty_cache_freq=0, fp16=False, fp16_init_scale=128, fp16_no_flatten_grads=False, fp16_scale_tolerance=0.0, fp16_scale_window=None, joined_dictionary=False, log_file=None, log_format=None, log_interval=100, lr_scheduler='fixed', memory_efficient_bf16=False, memory_efficient_fp16=False, min_loss_scale=0.0001, model_parallel_size=1, no_progress_bar=False, nwordssrc=-1, nwordstgt=-1, on_cpu_convert_precision=False, only_source=False, optimizer=None, padding_factor=8, plasma_path='/tmp/plasma', profile=False, quantization_config_path=None, reset_logging=False, scoring='bleu', seed=1, source_lang='en', srcdict='../trained_models/Fairseq/MultiDomain/1k/en-si/preprocess_en-si/dict.en.txt', suppress_crashes=False, target_lang='si', task='translation', tensorboard_logdir=None, testpref='../trained_models/Fairseq/MultiDomain/1k/en-si/cross_domain_outputs/FloresPlus/encoded/test.bpe', tgtdict='../trained_models/Fairseq/MultiDomain/1k/en-si/preprocess_en-si/dict.si.txt', threshold_loss_scale=None, thresholdsrc=0, thresholdtgt=0, tokenizer=None, tpu=False, trainpref=None, use_plasma_view=False, user_dir=None, validpref=None, wandb_project=None, workers=16)
+[en] Dictionary: 23264 types
+[en] ../trained_models/Fairseq/MultiDomain/1k/en-si/cross_domain_outputs/FloresPlus/encoded/test.bpe.en: 1012 sents, 33552 tokens, 0.396% replaced (by <unk>)
+[si] Dictionary: 23264 types
+[si] ../trained_models/Fairseq/MultiDomain/1k/en-si/cross_domain_outputs/FloresPlus/encoded/test.bpe.si: 1012 sents, 33124 tokens, 0.42% replaced (by <unk>)
+Wrote preprocessed data to ../trained_models/Fairseq/MultiDomain/1k/en-si/cross_domain_outputs/FloresPlus/preprocess_en-si/

MultiDomain/1k/en-si/cross_domain_outputs/FloresPlus/preprocess_en-si/test.en-si.en.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:15bbba6e8b6983e01f9b536ca247f8820070811f9c062e0d57ab8cc1e83f9c79
+size 67104

MultiDomain/1k/en-si/cross_domain_outputs/FloresPlus/preprocess_en-si/test.en-si.si.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b17c48079a9d04107f8c898fea2c8295365debaa400fcb44e32159821eaa8644
+size 66248

MultiDomain/1k/en-si/cross_domain_outputs/Government/en-si.csv

@@ -0,0 +1,2 @@
+model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet
+_,15.6984926445148,14.131403415511357,0.8885684224862633,2.1717531390719693,3.4343527865873704,0.4419743254184723

MultiDomain/1k/en-si/cross_domain_outputs/Government/preprocess_en-si/preprocess.log

@@ -0,0 +1,6 @@
+Namespace(aim_repo=None, aim_run_hash=None, align_suffix=None, alignfile=None, all_gather_list_size=16384, amp=False, amp_batch_retries=2, amp_init_scale=128, amp_scale_window=None, azureml_logging=False, bf16=False, bpe=None, cpu=False, criterion='cross_entropy', dataset_impl='mmap', destdir='../trained_models/Fairseq/MultiDomain/1k/en-si/cross_domain_outputs/Government/preprocess_en-si/', dict_only=False, empty_cache_freq=0, fp16=False, fp16_init_scale=128, fp16_no_flatten_grads=False, fp16_scale_tolerance=0.0, fp16_scale_window=None, joined_dictionary=False, log_file=None, log_format=None, log_interval=100, lr_scheduler='fixed', memory_efficient_bf16=False, memory_efficient_fp16=False, min_loss_scale=0.0001, model_parallel_size=1, no_progress_bar=False, nwordssrc=-1, nwordstgt=-1, on_cpu_convert_precision=False, only_source=False, optimizer=None, padding_factor=8, plasma_path='/tmp/plasma', profile=False, quantization_config_path=None, reset_logging=False, scoring='bleu', seed=1, source_lang='en', srcdict='../trained_models/Fairseq/MultiDomain/1k/en-si/preprocess_en-si/dict.en.txt', suppress_crashes=False, target_lang='si', task='translation', tensorboard_logdir=None, testpref='../trained_models/Fairseq/MultiDomain/1k/en-si/cross_domain_outputs/Government/encoded/test.bpe', tgtdict='../trained_models/Fairseq/MultiDomain/1k/en-si/preprocess_en-si/dict.si.txt', threshold_loss_scale=None, thresholdsrc=0, thresholdtgt=0, tokenizer=None, tpu=False, trainpref=None, use_plasma_view=False, user_dir=None, validpref=None, wandb_project=None, workers=16)
+[en] Dictionary: 23264 types
+[en] ../trained_models/Fairseq/MultiDomain/1k/en-si/cross_domain_outputs/Government/encoded/test.bpe.en: 1000 sents, 28520 tokens, 0.365% replaced (by <unk>)
+[si] Dictionary: 23264 types
+[si] ../trained_models/Fairseq/MultiDomain/1k/en-si/cross_domain_outputs/Government/encoded/test.bpe.si: 1000 sents, 26476 tokens, 0.23% replaced (by <unk>)
+Wrote preprocessed data to ../trained_models/Fairseq/MultiDomain/1k/en-si/cross_domain_outputs/Government/preprocess_en-si/

MultiDomain/1k/en-si/cross_domain_outputs/Government/preprocess_en-si/test.en-si.en.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:215b0e0a2427a4544a56a8c7d6b61233ebc8c8e9441888eb48181c5e449ff041
+size 57040

MultiDomain/1k/en-si/cross_domain_outputs/Government/preprocess_en-si/test.en-si.si.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6ebd02d5d0905a0a63e2a2147a76a169009117587f2efd893f802e783e3b7dad
+size 52952

MultiDomain/1k/en-si/cross_domain_outputs/Health/en-si.csv

@@ -0,0 +1,2 @@
+model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet
+_,11.538442751987944,10.563203319867807,0.5494976498763212,1.214040770876914,1.968449174345545,0.36016756209094875

MultiDomain/1k/en-si/cross_domain_outputs/Health/preprocess_en-si/preprocess.log

@@ -0,0 +1,6 @@
+Namespace(aim_repo=None, aim_run_hash=None, align_suffix=None, alignfile=None, all_gather_list_size=16384, amp=False, amp_batch_retries=2, amp_init_scale=128, amp_scale_window=None, azureml_logging=False, bf16=False, bpe=None, cpu=False, criterion='cross_entropy', dataset_impl='mmap', destdir='../trained_models/Fairseq/MultiDomain/1k/en-si/cross_domain_outputs/Health/preprocess_en-si/', dict_only=False, empty_cache_freq=0, fp16=False, fp16_init_scale=128, fp16_no_flatten_grads=False, fp16_scale_tolerance=0.0, fp16_scale_window=None, joined_dictionary=False, log_file=None, log_format=None, log_interval=100, lr_scheduler='fixed', memory_efficient_bf16=False, memory_efficient_fp16=False, min_loss_scale=0.0001, model_parallel_size=1, no_progress_bar=False, nwordssrc=-1, nwordstgt=-1, on_cpu_convert_precision=False, only_source=False, optimizer=None, padding_factor=8, plasma_path='/tmp/plasma', profile=False, quantization_config_path=None, reset_logging=False, scoring='bleu', seed=1, source_lang='en', srcdict='../trained_models/Fairseq/MultiDomain/1k/en-si/preprocess_en-si/dict.en.txt', suppress_crashes=False, target_lang='si', task='translation', tensorboard_logdir=None, testpref='../trained_models/Fairseq/MultiDomain/1k/en-si/cross_domain_outputs/Health/encoded/test.bpe', tgtdict='../trained_models/Fairseq/MultiDomain/1k/en-si/preprocess_en-si/dict.si.txt', threshold_loss_scale=None, thresholdsrc=0, thresholdtgt=0, tokenizer=None, tpu=False, trainpref=None, use_plasma_view=False, user_dir=None, validpref=None, wandb_project=None, workers=16)
+[en] Dictionary: 23264 types
+[en] ../trained_models/Fairseq/MultiDomain/1k/en-si/cross_domain_outputs/Health/encoded/test.bpe.en: 890 sents, 35175 tokens, 0.472% replaced (by <unk>)
+[si] Dictionary: 23264 types
+[si] ../trained_models/Fairseq/MultiDomain/1k/en-si/cross_domain_outputs/Health/encoded/test.bpe.si: 890 sents, 39286 tokens, 0.44% replaced (by <unk>)
+Wrote preprocessed data to ../trained_models/Fairseq/MultiDomain/1k/en-si/cross_domain_outputs/Health/preprocess_en-si/

MultiDomain/1k/en-si/cross_domain_outputs/Health/preprocess_en-si/test.en-si.en.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:41de44b38c1f3bfde34c66b097e7659c6425d77da09aafdbbd5bd94600606b53
+size 70350

MultiDomain/1k/en-si/cross_domain_outputs/Health/preprocess_en-si/test.en-si.si.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2d09aa0f49ea937575c15d2bd84a36142e6fff1da1b117849b40df3dbc550dbf
+size 78572

MultiDomain/1k/en-si/cross_domain_outputs/Jailbreak/en-si.csv

@@ -0,0 +1,2 @@
+model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet
+_,10.615437491099465,10.062918780110547,0.6398951992714951,0.8961787402529273,1.0664467813926035,0.4098448149561882