From 433fe95691ece1c116ed954e8fb86e3e4c518c52 Mon Sep 17 00:00:00 2001 From: abePclWaseda Date: Sat, 9 Nov 2024 13:40:02 +0900 Subject: [PATCH 1/5] initial commit From 9f5aaa778c0a22d98129b6579bcf09380ed36ce3 Mon Sep 17 00:00:00 2001 From: abePclWaseda Date: Sat, 9 Nov 2024 16:23:47 +0900 Subject: [PATCH 2/5] =?UTF-8?q?=E3=81=9D=E3=82=8C=E3=81=9E=E3=82=8C?= =?UTF-8?q?=E3=81=AE=20p(speech2text=5Fkwargs)=20=E8=AA=BF=E6=9F=BB?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- asr.json | 36 ++++++++++++++++++++++++++++++++++++ lm.json | 40 ++++++++++++++++++++++++++++++++++++++++ tryModelMerge.py | 23 +++++++++++++++++++++++ 3 files changed, 99 insertions(+) create mode 100644 asr.json create mode 100644 lm.json create mode 100644 tryModelMerge.py diff --git a/asr.json b/asr.json new file mode 100644 index 0000000..4c968e8 --- /dev/null +++ b/asr.json @@ -0,0 +1,36 @@ +{ + "asr_train_config": "exp/asr_train_asr_conformer_lr2e-3_warmup15k_amp_nondeterministic_raw_en_hugging_face_openai-community-gpt2_sp/config.yaml", + "asr_model_file": "exp/asr_train_asr_conformer_lr2e-3_warmup15k_amp_nondeterministic_raw_en_hugging_face_openai-community-gpt2_sp/valid.acc.ave.pth", + "transducer_conf": null, + "lm_train_config": null, + "lm_file": null, + "ngram_file": null, + "token_type": null, + "bpemodel": null, + "device": "cuda", + "maxlenratio": 0.0, + "minlenratio": 0.0, + "dtype": "float32", + "beam_size": 20, + "ctc_weight": 0.3, + "lm_weight": 0.0, + "ngram_weight": 0.9, + "penalty": 0.0, + "nbest": 1, + "normalize_length": false, + "streaming": false, + "enh_s2t_task": false, + "multi_asr": false, + "quantize_asr_model": false, + "quantize_lm": false, + "quantize_modules": [ + "Linear" + ], + "quantize_dtype": "qint8", + "hugging_face_decoder": false, + "hugging_face_decoder_conf": {}, + "time_sync": false, + "prompt_token_file": null, + "lang_prompt_token": null, + "nlp_prompt_token": null +} \ No newline at end of file diff --git a/lm.json b/lm.json new file mode 100644 index 0000000..974c45e --- /dev/null +++ b/lm.json @@ -0,0 +1,40 @@ +{ + "asr_train_config": "exp/asr_train_asr_conformer_lr2e-3_warmup15k_amp_nondeterministic_raw_en_hugging_face_openai-community-gpt2_sp/config.yaml", + "asr_model_file": "exp/asr_train_asr_conformer_lr2e-3_warmup15k_amp_nondeterministic_raw_en_hugging_face_openai-community-gpt2_sp/valid.acc.ave.pth", + "transducer_conf": null, + "lm_train_config": "exp/lm_train_transformer_gpt2_en_hugging_face/config.yaml", + "lm_file": "exp/lm_train_transformer_gpt2_en_hugging_face/valid.loss.ave.pth", + "ngram_file": null, + "token_type": null, + "bpemodel": null, + "device": "cuda", + "maxlenratio": 0.0, + "minlenratio": 0.0, + "dtype": "float32", + "beam_size": 20, + "ctc_weight": 0.3, + "lm_weight": 0.1, + "ngram_weight": 0.9, + "penalty": 0.0, + "nbest": 1, + "normalize_length": false, + "streaming": false, + "enh_s2t_task": false, + "multi_asr": false, + "quantize_asr_model": false, + "quantize_lm": false, + "quantize_modules": [ + "Linear" + ], + "quantize_dtype": "qint8", + "hugging_face_decoder": false, + "hugging_face_decoder_conf": {}, + "time_sync": false, + "prompt_token_file": null, + "lang_prompt_token": null, + "nlp_prompt_token": null, + "partial_ar": false, + "threshold_probability": 0.99, + "max_seq_len": 5, + "max_mask_parallel": -1 +} \ No newline at end of file diff --git a/tryModelMerge.py b/tryModelMerge.py new file mode 100644 index 0000000..972c72d --- /dev/null +++ b/tryModelMerge.py @@ -0,0 +1,23 @@ +from ..espnet.espnet2.bin.asr_inference import Speech2Text + +# (Pdb) p(speech2text_kwargs) +# {'asr_train_config': 'exp/asr_train_asr_conformer_lr2e-3_warmup15k_amp_nondeterministic_raw_en_hugging_face_openai-community-gpt2_sp/config.yaml', 'asr_model_file': 'exp/asr_train_asr_conformer_lr2e-3_warmup15k_amp_nondeterministic_raw_en_hugging_face_openai-community-gpt2_sp/valid.acc.ave.pth', 'transducer_conf': None, 'lm_train_config': None, 'lm_file': None, 'ngram_file': None, 'token_type': None, 'bpemodel': None, 'device': 'cuda', 'maxlenratio': 0.0, 'minlenratio': 0.0, 'dtype': 'float32', 'beam_size': 20, 'ctc_weight': 0.3, 'lm_weight': 0.0, 'ngram_weight': 0.9, 'penalty': 0.0, 'nbest': 1, 'normalize_length': False, 'streaming': False, 'enh_s2t_task': False, 'multi_asr': False, 'quantize_asr_model': False, 'quantize_lm': False, 'quantize_modules': ['Linear'], 'quantize_dtype': 'qint8', 'hugging_face_decoder': False, 'hugging_face_decoder_conf': {}, 'time_sync': False, 'prompt_token_file': None, 'lang_prompt_token': None, 'nlp_prompt_token': None} +speech2text = Speech2Text( + asr_train_config="/path/to/your/config.yaml", + asr_model_file="/path/to/your/inference_asr_model", + beam_size=10, + ctc_weight=0.3, + lm_weight=0.5 +) + +# (Pdb) p(speech2text_kwargs) +# {'asr_train_config': 'exp/asr_train_asr_conformer_lr2e-3_warmup15k_amp_nondeterministic_raw_en_hugging_face_openai-community-gpt2_sp/config.yaml', 'asr_model_file': 'exp/asr_train_asr_conformer_lr2e-3_warmup15k_amp_nondeterministic_raw_en_hugging_face_openai-community-gpt2_sp/valid.acc.ave.pth', 'transducer_conf': None, 'lm_train_config': 'exp/lm_train_transformer_gpt2_en_hugging_face/config.yaml', 'lm_file': 'exp/lm_train_transformer_gpt2_en_hugging_face/valid.loss.ave.pth', 'ngram_file': None, 'token_type': None, 'bpemodel': None, 'device': 'cuda', 'maxlenratio': 0.0, 'minlenratio': 0.0, 'dtype': 'float32', 'beam_size': 20, 'ctc_weight': 0.3, 'lm_weight': 0.1, 'ngram_weight': 0.9, 'penalty': 0.0, 'nbest': 1, 'normalize_length': False, 'streaming': False, 'enh_s2t_task': False, 'multi_asr': False, 'quantize_asr_model': False, 'quantize_lm': False, 'quantize_modules': ['Linear'], 'quantize_dtype': 'qint8', 'hugging_face_decoder': False, 'hugging_face_decoder_conf': {}, 'time_sync': False, 'prompt_token_file': None, 'lang_prompt_token': None, 'nlp_prompt_token': None, 'partial_ar': False, 'threshold_probability': 0.99, 'max_seq_len': 5, 'max_mask_parallel': -1} +speech2text_for_lm = Speech2Text( + asr_train_config="/path/to/asr_config.yaml", # 音声認識モデルの設定ファイル + asr_model_file="/path/to/asr_model.pth", # 音声認識モデルの重み + lm_train_config="/path/to/lm_config.yaml", # 言語モデルの設定ファイル + lm_file="/path/to/lm.pth", # 学習済み言語モデルの重み + beam_size=10, + ctc_weight=0.3, + lm_weight=0.5 +) \ No newline at end of file From 6bfb12a54806032bac252c7029a5eaf4ee0d5e4f Mon Sep 17 00:00:00 2001 From: abePclWaseda Date: Sun, 10 Nov 2024 00:00:21 +0900 Subject: [PATCH 3/5] =?UTF-8?q?asr=20=E5=81=B4=E3=81=AE=E5=BC=95=E6=95=B0?= =?UTF-8?q?=E8=A8=AD=E5=AE=9A?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tryModelMerge.py | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/tryModelMerge.py b/tryModelMerge.py index 972c72d..d9bb714 100644 --- a/tryModelMerge.py +++ b/tryModelMerge.py @@ -2,12 +2,24 @@ # (Pdb) p(speech2text_kwargs) # {'asr_train_config': 'exp/asr_train_asr_conformer_lr2e-3_warmup15k_amp_nondeterministic_raw_en_hugging_face_openai-community-gpt2_sp/config.yaml', 'asr_model_file': 'exp/asr_train_asr_conformer_lr2e-3_warmup15k_amp_nondeterministic_raw_en_hugging_face_openai-community-gpt2_sp/valid.acc.ave.pth', 'transducer_conf': None, 'lm_train_config': None, 'lm_file': None, 'ngram_file': None, 'token_type': None, 'bpemodel': None, 'device': 'cuda', 'maxlenratio': 0.0, 'minlenratio': 0.0, 'dtype': 'float32', 'beam_size': 20, 'ctc_weight': 0.3, 'lm_weight': 0.0, 'ngram_weight': 0.9, 'penalty': 0.0, 'nbest': 1, 'normalize_length': False, 'streaming': False, 'enh_s2t_task': False, 'multi_asr': False, 'quantize_asr_model': False, 'quantize_lm': False, 'quantize_modules': ['Linear'], 'quantize_dtype': 'qint8', 'hugging_face_decoder': False, 'hugging_face_decoder_conf': {}, 'time_sync': False, 'prompt_token_file': None, 'lang_prompt_token': None, 'nlp_prompt_token': None} -speech2text = Speech2Text( - asr_train_config="/path/to/your/config.yaml", - asr_model_file="/path/to/your/inference_asr_model", - beam_size=10, +speech2text = Speech2Text.from_pretrained( + asr_train_config="/mnt/kiso-qnap/abe/b4/espnet/egs2/librispeech_100/asr1/exp/asr_train_asr_conformer_lr2e-3_warmup15k_amp_nondeterministic_raw_en_hugging_face_openai-community-gpt2_sp/config.yaml", + asr_model_file="/mnt/kiso-qnap/abe/b4/espnet/egs2/librispeech_100/asr1/exp/asr_train_asr_conformer_lr2e-3_warmup15k_amp_nondeterministic_raw_en_hugging_face_openai-community-gpt2_sp/valid.acc.ave.pth", + transducer_conf=None, + lm_train_config=None, + lm_file=None, + ngram_file=None, + token_type=None, + bpemodel=None, + device="cuda", + maxlenratio=0.0, + minlenratio=0.0, + beam_size=20, ctc_weight=0.3, - lm_weight=0.5 + lm_weight=0.0, + ngram_weight=0.9, + penalty=0.0, + nbest=1, ) # (Pdb) p(speech2text_kwargs) @@ -19,5 +31,5 @@ lm_file="/path/to/lm.pth", # 学習済み言語モデルの重み beam_size=10, ctc_weight=0.3, - lm_weight=0.5 -) \ No newline at end of file + lm_weight=0.5, +) From 8399d560f7f1710eb7ac7ccc24ce0c36827dfcb4 Mon Sep 17 00:00:00 2001 From: abePclWaseda Date: Sun, 10 Nov 2024 00:09:47 +0900 Subject: [PATCH 4/5] =?UTF-8?q?lm=20=E5=81=B4=E3=81=AE=E5=BC=95=E6=95=B0?= =?UTF-8?q?=E8=A8=AD=E5=AE=9A?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tryModelMerge.py | 27 ++++++++++++++++++++++----- 1 file changed, 22 insertions(+), 5 deletions(-) diff --git a/tryModelMerge.py b/tryModelMerge.py index d9bb714..8d1984e 100644 --- a/tryModelMerge.py +++ b/tryModelMerge.py @@ -24,12 +24,29 @@ # (Pdb) p(speech2text_kwargs) # {'asr_train_config': 'exp/asr_train_asr_conformer_lr2e-3_warmup15k_amp_nondeterministic_raw_en_hugging_face_openai-community-gpt2_sp/config.yaml', 'asr_model_file': 'exp/asr_train_asr_conformer_lr2e-3_warmup15k_amp_nondeterministic_raw_en_hugging_face_openai-community-gpt2_sp/valid.acc.ave.pth', 'transducer_conf': None, 'lm_train_config': 'exp/lm_train_transformer_gpt2_en_hugging_face/config.yaml', 'lm_file': 'exp/lm_train_transformer_gpt2_en_hugging_face/valid.loss.ave.pth', 'ngram_file': None, 'token_type': None, 'bpemodel': None, 'device': 'cuda', 'maxlenratio': 0.0, 'minlenratio': 0.0, 'dtype': 'float32', 'beam_size': 20, 'ctc_weight': 0.3, 'lm_weight': 0.1, 'ngram_weight': 0.9, 'penalty': 0.0, 'nbest': 1, 'normalize_length': False, 'streaming': False, 'enh_s2t_task': False, 'multi_asr': False, 'quantize_asr_model': False, 'quantize_lm': False, 'quantize_modules': ['Linear'], 'quantize_dtype': 'qint8', 'hugging_face_decoder': False, 'hugging_face_decoder_conf': {}, 'time_sync': False, 'prompt_token_file': None, 'lang_prompt_token': None, 'nlp_prompt_token': None, 'partial_ar': False, 'threshold_probability': 0.99, 'max_seq_len': 5, 'max_mask_parallel': -1} -speech2text_for_lm = Speech2Text( - asr_train_config="/path/to/asr_config.yaml", # 音声認識モデルの設定ファイル - asr_model_file="/path/to/asr_model.pth", # 音声認識モデルの重み - lm_train_config="/path/to/lm_config.yaml", # 言語モデルの設定ファイル - lm_file="/path/to/lm.pth", # 学習済み言語モデルの重み +speech2text_for_lm = Speech2Text.from_pretrained( + asr_train_config="/mnt/kiso-qnap2/yuabe/b4/espnet/egs2/librispeech_100/asr1/exp/asr_train_asr_conformer_lr2e-3_warmup15k_amp_nondeterministic_raw_en_hugging_face_openai-community-gpt2_sp/config.yaml", + asr_model_file="/mnt/kiso-qnap2/yuabe/b4/espnet/egs2/librispeech_100/asr1/exp/asr_train_asr_conformer_lr2e-3_warmup15k_amp_nondeterministic_raw_en_hugging_face_openai-community-gpt2_sp/valid.acc.ave.pth", + transducer_conf=None, + lm_train_config="/mnt/kiso-qnap2/yuabe/b4/espnet/egs2/librispeech_100/asr1/exp/lm_train_transformer_gpt2_en_hugging_face/config.yaml", + lm_file="/mnt/kiso-qnap2/yuabe/b4/espnet/egs2/librispeech_100/asr1/exp/lm_train_transformer_gpt2_en_hugging_face/valid.loss.ave.pth", beam_size=10, ctc_weight=0.3, lm_weight=0.5, + ngram_file=None, + token_type=None, + bpemodel=None, + device="cuda", + maxlenratio=0.0, + minlenratio=0.0, + beam_size=20, + ctc_weight=0.3, + lm_weight=0.0, + ngram_weight=0.9, + penalty=0.0, + nbest=1, + partial_ar=False, + threshold_probability=0.99, + max_seq_len=5, + max_mask_parallel=-1 ) From b6c512fdb9d8def5d203481f9746b604f0f4d697 Mon Sep 17 00:00:00 2001 From: abePclWaseda Date: Sun, 10 Nov 2024 00:44:55 +0900 Subject: [PATCH 5/5] =?UTF-8?q?=E4=B8=8D=E8=A6=81=E3=81=AA=E5=BC=95?= =?UTF-8?q?=E6=95=B0=E5=89=8A=E9=99=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tryModelMerge.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/tryModelMerge.py b/tryModelMerge.py index 8d1984e..6e8fe07 100644 --- a/tryModelMerge.py +++ b/tryModelMerge.py @@ -30,9 +30,6 @@ transducer_conf=None, lm_train_config="/mnt/kiso-qnap2/yuabe/b4/espnet/egs2/librispeech_100/asr1/exp/lm_train_transformer_gpt2_en_hugging_face/config.yaml", lm_file="/mnt/kiso-qnap2/yuabe/b4/espnet/egs2/librispeech_100/asr1/exp/lm_train_transformer_gpt2_en_hugging_face/valid.loss.ave.pth", - beam_size=10, - ctc_weight=0.3, - lm_weight=0.5, ngram_file=None, token_type=None, bpemodel=None,