From bff0992dda3a1a6d8e0c0ceb0b3fcc7c7a23d0c4 Mon Sep 17 00:00:00 2001 From: abePclWaseda Date: Sun, 6 Apr 2025 14:27:44 +0900 Subject: [PATCH 1/3] initial commit From 24c2b64bdcbcdd937a0f5bcb73da76fbb1a3a3f6 Mon Sep 17 00:00:00 2001 From: abePclWaseda Date: Sun, 6 Apr 2025 14:41:07 +0900 Subject: [PATCH 2/3] =?UTF-8?q?whisper=20=E3=81=AB=20LoRA=20=E5=AE=9F?= =?UTF-8?q?=E8=A3=85?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- egs2/TEMPLATE/asr1/db.sh | 2 +- .../conf/tuning/train_asr_whisper_full.yaml | 61 ++++++++++++++++--- egs2/librispeech_100/asr1/run.sh | 11 ++-- 3 files changed, 62 insertions(+), 12 deletions(-) diff --git a/egs2/TEMPLATE/asr1/db.sh b/egs2/TEMPLATE/asr1/db.sh index 217c32c39a..d256ef0fdc 100755 --- a/egs2/TEMPLATE/asr1/db.sh +++ b/egs2/TEMPLATE/asr1/db.sh @@ -63,7 +63,7 @@ HUI_ACG=downloads HUB4_SPANISH= LABOROTV= TEDXJP= -LIBRISPEECH=downloads +LIBRISPEECH=/mnt/aoni04/higuchi/data LIBRILIGHT_LIMITED= LIBRILIGHT=downloads FSC= diff --git a/egs2/librispeech_100/asr1/conf/tuning/train_asr_whisper_full.yaml b/egs2/librispeech_100/asr1/conf/tuning/train_asr_whisper_full.yaml index 01e9f3444a..c5cd9b1f6e 100644 --- a/egs2/librispeech_100/asr1/conf/tuning/train_asr_whisper_full.yaml +++ b/egs2/librispeech_100/asr1/conf/tuning/train_asr_whisper_full.yaml @@ -2,7 +2,7 @@ normalize: null encoder: whisper encoder_conf: - whisper_model: medium + whisper_model: tiny dropout_rate: 0.0 use_specaug: true specaug_conf: @@ -23,8 +23,12 @@ encoder_conf: decoder: whisper decoder_conf: - whisper_model: medium + whisper_model: tiny dropout_rate: 0.0 +preprocessor: default +preprocessor_conf: + whisper_language: "en" + whisper_task: "transcribe" model_conf: ctc_weight: 0.0 @@ -44,9 +48,9 @@ num_workers: 4 sort_in_batch: descending # how to sort data in making batch sort_batch: descending # how to sort created batches batch_type: numel -batch_bins: 12000000 # good for single GPU w/ 40G mem -accum_grad: 4 -max_epoch: 3 +batch_bins: 3000000 # good for single GPU w/ 40G mem +accum_grad: 16 +max_epoch: 2 patience: none init: none best_model_criterion: @@ -59,10 +63,53 @@ use_amp: true cudnn_deterministic: false cudnn_benchmark: false +# LoRA finetune related +use_adapter: true +adapter: lora +adapter_conf: + rank: 8 + alpha: 16 + dropout_rate: 0.05 + target_modules: ["decoder.decoders.blocks.0.attn.query","decoder.decoders.blocks.0.attn.key","decoder.decoders.blocks.0.attn.value","decoder.decoders.blocks.0.attn.out" + ,"decoder.decoders.blocks.1.attn.query","decoder.decoders.blocks.1.attn.key","decoder.decoders.blocks.1.attn.value","decoder.decoders.blocks.1.attn.out" + ,"decoder.decoders.blocks.2.attn.query","decoder.decoders.blocks.2.attn.key","decoder.decoders.blocks.2.attn.value","decoder.decoders.blocks.2.attn.out" + ,"decoder.decoders.blocks.3.attn.query","decoder.decoders.blocks.3.attn.key","decoder.decoders.blocks.3.attn.value","decoder.decoders.blocks.3.attn.out" + ,"decoder.decoders.blocks.4.attn.query","decoder.decoders.blocks.4.attn.key","decoder.decoders.blocks.4.attn.value","decoder.decoders.blocks.4.attn.out" + ,"decoder.decoders.blocks.5.attn.query","decoder.decoders.blocks.5.attn.key","decoder.decoders.blocks.5.attn.value","decoder.decoders.blocks.5.attn.out" + ,"decoder.decoders.blocks.6.attn.query","decoder.decoders.blocks.6.attn.key","decoder.decoders.blocks.6.attn.value","decoder.decoders.blocks.6.attn.out" + ,"decoder.decoders.blocks.7.attn.query","decoder.decoders.blocks.7.attn.key","decoder.decoders.blocks.7.attn.value","decoder.decoders.blocks.7.attn.out" + ,"decoder.decoders.blocks.8.attn.query","decoder.decoders.blocks.8.attn.key","decoder.decoders.blocks.8.attn.value","decoder.decoders.blocks.8.attn.out" + ,"decoder.decoders.blocks.9.attn.query","decoder.decoders.blocks.9.attn.key","decoder.decoders.blocks.9.attn.value","decoder.decoders.blocks.9.attn.out" + ,"decoder.decoders.blocks.10.attn.query","decoder.decoders.blocks.10.attn.key","decoder.decoders.blocks.10.attn.value","decoder.decoders.blocks.10.attn.out" + ,"decoder.decoders.blocks.11.attn.query","decoder.decoders.blocks.11.attn.key","decoder.decoders.blocks.11.attn.value","decoder.decoders.blocks.11.attn.out" + ,"decoder.decoders.blocks.12.attn.query","decoder.decoders.blocks.12.attn.key","decoder.decoders.blocks.12.attn.value","decoder.decoders.blocks.12.attn.out" + ,"decoder.decoders.blocks.13.attn.query","decoder.decoders.blocks.13.attn.key","decoder.decoders.blocks.13.attn.value","decoder.decoders.blocks.13.attn.out" + ,"decoder.decoders.blocks.14.attn.query","decoder.decoders.blocks.14.attn.key","decoder.decoders.blocks.14.attn.value","decoder.decoders.blocks.14.attn.out" + ,"decoder.decoders.blocks.15.attn.query","decoder.decoders.blocks.15.attn.key","decoder.decoders.blocks.15.attn.value","decoder.decoders.blocks.15.attn.out" + ,"decoder.decoders.blocks.16.attn.query","decoder.decoders.blocks.16.attn.key","decoder.decoders.blocks.16.attn.value","decoder.decoders.blocks.16.attn.out" + ,"decoder.decoders.blocks.17.attn.query","decoder.decoders.blocks.17.attn.key","decoder.decoders.blocks.17.attn.value","decoder.decoders.blocks.17.attn.out" + ,"decoder.decoders.blocks.18.attn.query","decoder.decoders.blocks.18.attn.key","decoder.decoders.blocks.18.attn.value","decoder.decoders.blocks.18.attn.out" + ,"decoder.decoders.blocks.19.attn.query","decoder.decoders.blocks.19.attn.key","decoder.decoders.blocks.19.attn.value","decoder.decoders.blocks.19.attn.out" + ,"decoder.decoders.blocks.20.attn.query","decoder.decoders.blocks.20.attn.key","decoder.decoders.blocks.20.attn.value","decoder.decoders.blocks.20.attn.out" + ,"decoder.decoders.blocks.21.attn.query","decoder.decoders.blocks.21.attn.key","decoder.decoders.blocks.21.attn.value","decoder.decoders.blocks.21.attn.out" + ,"decoder.decoders.blocks.22.attn.query","decoder.decoders.blocks.22.attn.key","decoder.decoders.blocks.22.attn.value","decoder.decoders.blocks.22.attn.out" + ,"decoder.decoders.blocks.23.attn.query","decoder.decoders.blocks.23.attn.key","decoder.decoders.blocks.23.attn.value","decoder.decoders.blocks.23.attn.out" + ,"decoder.decoders.blocks.24.attn.query","decoder.decoders.blocks.24.attn.key","decoder.decoders.blocks.24.attn.value","decoder.decoders.blocks.24.attn.out" + ,"decoder.decoders.blocks.25.attn.query","decoder.decoders.blocks.25.attn.key","decoder.decoders.blocks.25.attn.value","decoder.decoders.blocks.25.attn.out" + ,"decoder.decoders.blocks.26.attn.query","decoder.decoders.blocks.26.attn.key","decoder.decoders.blocks.26.attn.value","decoder.decoders.blocks.26.attn.out" + ,"decoder.decoders.blocks.27.attn.query","decoder.decoders.blocks.27.attn.key","decoder.decoders.blocks.27.attn.value","decoder.decoders.blocks.27.attn.out" + ,"decoder.decoders.blocks.28.attn.query","decoder.decoders.blocks.28.attn.key","decoder.decoders.blocks.28.attn.value","decoder.decoders.blocks.28.attn.out" + ,"decoder.decoders.blocks.29.attn.query","decoder.decoders.blocks.29.attn.key","decoder.decoders.blocks.29.attn.value","decoder.decoders.blocks.29.attn.out" + ,"decoder.decoders.blocks.30.attn.query","decoder.decoders.blocks.30.attn.key","decoder.decoders.blocks.30.attn.value","decoder.decoders.blocks.30.attn.out" + ,"decoder.decoders.blocks.31.attn.query","decoder.decoders.blocks.31.attn.key","decoder.decoders.blocks.31.attn.value","decoder.decoders.blocks.31.attn.out"] # only decoder self-attention + +unused_parameters: true +freeze_param: ["encoder","decoder"] + optim: adamw grad_clip: 1.0 optim_conf: - lr: 1.0e-05 + lr: 5.0e-04 weight_decay: 0.01 betas: - 0.9 @@ -70,4 +117,4 @@ optim_conf: eps: 1.0e-06 scheduler: warmuplr scheduler_conf: - warmup_steps: 1500 + warmup_steps: 1500 \ No newline at end of file diff --git a/egs2/librispeech_100/asr1/run.sh b/egs2/librispeech_100/asr1/run.sh index 5a3f02510c..8cced6bc6f 100755 --- a/egs2/librispeech_100/asr1/run.sh +++ b/egs2/librispeech_100/asr1/run.sh @@ -8,14 +8,17 @@ set -o pipefail train_set="train_clean_100" valid_set="dev" test_sets="test_clean test_other dev_clean dev_other" - -asr_config=conf/train_asr.yaml -inference_config=conf/decode_asr.yaml +asr_config=conf/tuning/train_asr_whisper_full.yaml +inference_config=conf/tuning/decode_asr_whisper_noctc_greedy.yaml ./asr.sh \ + --token_type whisper_multilingual \ + --asr_speech_fold_length 256 \ + --asr_text_fold_length 150 \ + --cleaner whisper_basic \ --lang en \ --ngpu 1 \ - --nj 16 \ + --nj 8 \ --gpu_inference true \ --inference_nj 2 \ --nbpe 5000 \ From cc9224b0882a75cfe2946f82f4144a79f54b47f2 Mon Sep 17 00:00:00 2001 From: abePclWaseda Date: Sun, 6 Apr 2025 20:48:18 +0900 Subject: [PATCH 3/3] =?UTF-8?q?=E3=83=90=E3=83=83=E3=83=81=E3=82=B5?= =?UTF-8?q?=E3=82=A4=E3=82=BA=E5=A4=89=E6=9B=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../asr1/conf/tuning/train_asr_whisper_full.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/egs2/librispeech_100/asr1/conf/tuning/train_asr_whisper_full.yaml b/egs2/librispeech_100/asr1/conf/tuning/train_asr_whisper_full.yaml index c5cd9b1f6e..d717655ef9 100644 --- a/egs2/librispeech_100/asr1/conf/tuning/train_asr_whisper_full.yaml +++ b/egs2/librispeech_100/asr1/conf/tuning/train_asr_whisper_full.yaml @@ -48,8 +48,8 @@ num_workers: 4 sort_in_batch: descending # how to sort data in making batch sort_batch: descending # how to sort created batches batch_type: numel -batch_bins: 3000000 # good for single GPU w/ 40G mem -accum_grad: 16 +batch_bins: 750000 # good for single GPU w/ 40G mem +accum_grad: 64 max_epoch: 2 patience: none init: none