Skip to content

Commit

Permalink
Merge pull request #89 from marian-nmt/romang/valid-reset-all
Browse files Browse the repository at this point in the history
Add test for --valid-reset-all
  • Loading branch information
snukky authored Mar 28, 2023
2 parents 0a00605 + c24df19 commit 4faba70
Show file tree
Hide file tree
Showing 3 changed files with 92 additions and 0 deletions.
1 change: 1 addition & 0 deletions tests/training/restoring/validation/.gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ valid_stalled
valid_add
valid_newbest
valid_reset_add
valid_reset_all
train.*
valid.*
dev.bpe.de
Expand Down
61 changes: 61 additions & 0 deletions tests/training/restoring/validation/test_valid_reset_all.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,61 @@
#!/bin/bash -x

#####################################################################
# SUMMARY: Test the option for resetting all validation metrics after the training is resumed
# AUTHOR: snukky
# TAGS: restore valid valid-script valid-reset
#####################################################################

# Exit on error
set -e

# Test code goes here
rm -rf valid_reset_all valid_reset_all_?.*log valid_script_?.temp
mkdir -p valid_reset_all

test -s valid.mini.bpe.en || head -n 8 $MRT_DATA/europarl.de-en/toy.bpe.en > valid.mini.bpe.en
test -s valid.mini.bpe.de || head -n 8 $MRT_DATA/europarl.de-en/toy.bpe.de > valid.mini.bpe.de

extra_opts="--no-shuffle --seed 2222 --maxi-batch 1 --maxi-batch-sort none --quiet-translation"
extra_opts="$extra_opts --dim-emb 64 --dim-rnn 128 --mini-batch 16 --optimizer sgd"
extra_opts="$extra_opts --cost-type ce-mean --disp-label-counts false"


# Train a model for a while and stop
$MRT_MARIAN/marian $extra_opts \
-m valid_reset_all/model.npz -t $MRT_DATA/europarl.de-en/toy.bpe.{en,de} -v vocab.en.yml vocab.de.yml \
--disp-freq 10 --valid-freq 20 --after-batches 140 --early-stopping 5 \
--valid-metrics translation valid-script cross-entropy --valid-script-path ./valid_script_ab.sh \
--valid-sets valid.mini.bpe.{de,en} \
--overwrite --keep-best --clip-norm 0 \
--log valid_reset_all_1.log

test -e valid_reset_all/model.npz
test -e valid_reset_all/model.npz.yml
test -e valid_reset_all_1.log

cp valid_reset_all/model.npz.progress.yml valid_reset_all/model.npz.progress.yml.bak

cat valid_reset_all_1.log | $MRT_TOOLS/strip-timestamps.sh | grep -P "\[valid\]" > valid_reset_all.out


# Restart training with --valid-reset-all
$MRT_MARIAN/marian $extra_opts \
-m valid_reset_all/model.npz -t $MRT_DATA/europarl.de-en/toy.bpe.{en,de} -v vocab.en.yml vocab.de.yml \
--disp-freq 10 --valid-freq 20 --after-batches 200 --early-stopping 5 --valid-reset-all \
--valid-metrics translation valid-script cross-entropy --valid-script-path ./valid_script_ab.sh \
--valid-sets valid.mini.bpe.{de,en} \
--overwrite --keep-best --clip-norm 0 \
--log valid_reset_all_2.log

test -e valid_reset_all/model.npz
test -e valid_reset_all_2.log

cat valid_reset_all_2.log | $MRT_TOOLS/strip-timestamps.sh | grep -P "\[valid\]" >> valid_reset_all.out


# Compare with the expected output
$MRT_TOOLS/diff.sh valid_reset_all.out valid_reset_all.expected > valid_reset_all.diff

# Exit with success code
exit 0
30 changes: 30 additions & 0 deletions tests/training/restoring/validation/valid_reset_all.expected
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
[valid] Ep. 1 : Up. 20 : translation : 333.5 : new best
[valid] Ep. 1 : Up. 20 : valid-script : 222.3 : new best
[valid] Ep. 1 : Up. 20 : cross-entropy : 249.337 : new best
[valid] Ep. 1 : Up. 40 : translation : 333.4 : stalled 1 times (last best: 333.5)
[valid] Ep. 1 : Up. 40 : valid-script : 222.2 : stalled 1 times (last best: 222.3)
[valid] Ep. 1 : Up. 40 : cross-entropy : 248.218 : new best
[valid] Ep. 1 : Up. 60 : translation : 333.3 : stalled 2 times (last best: 333.5)
[valid] Ep. 1 : Up. 60 : valid-script : 222.1 : stalled 2 times (last best: 222.3)
[valid] Ep. 1 : Up. 60 : cross-entropy : 246.856 : new best
[valid] Ep. 1 : Up. 80 : translation : 333.2 : stalled 3 times (last best: 333.5)
[valid] Ep. 1 : Up. 80 : valid-script : 222.6 : new best
[valid] Ep. 1 : Up. 80 : cross-entropy : 245.247 : new best
[valid] Ep. 1 : Up. 100 : translation : 333.1 : stalled 4 times (last best: 333.5)
[valid] Ep. 1 : Up. 100 : valid-script : 222.5 : stalled 1 times (last best: 222.6)
[valid] Ep. 1 : Up. 100 : cross-entropy : 243.37 : new best
[valid] Ep. 1 : Up. 120 : translation : 333.9 : new best
[valid] Ep. 1 : Up. 120 : valid-script : 222.4 : stalled 2 times (last best: 222.6)
[valid] Ep. 1 : Up. 120 : cross-entropy : 240.802 : new best
[valid] Ep. 1 : Up. 140 : translation : 333.8 : stalled 1 times (last best: 333.9)
[valid] Ep. 1 : Up. 140 : valid-script : 222.3 : stalled 3 times (last best: 222.6)
[valid] Ep. 1 : Up. 140 : cross-entropy : 237.65 : new best
[valid] Ep. 1 : Up. 160 : translation : 333.7 : new best
[valid] Ep. 1 : Up. 160 : valid-script : 222.2 : new best
[valid] Ep. 1 : Up. 160 : cross-entropy : 233.833 : new best
[valid] Ep. 2 : Up. 180 : translation : 333.6 : stalled 1 times (last best: 333.7)
[valid] Ep. 2 : Up. 180 : valid-script : 222.1 : stalled 1 times (last best: 222.2)
[valid] Ep. 2 : Up. 180 : cross-entropy : 230.035 : new best
[valid] Ep. 2 : Up. 200 : translation : 333.5 : stalled 2 times (last best: 333.7)
[valid] Ep. 2 : Up. 200 : valid-script : 222.6 : new best
[valid] Ep. 2 : Up. 200 : cross-entropy : 227.982 : new best

0 comments on commit 4faba70

Please sign in to comment.