-
Notifications
You must be signed in to change notification settings - Fork 9
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #89 from marian-nmt/romang/valid-reset-all
Add test for --valid-reset-all
- Loading branch information
Showing
3 changed files
with
92 additions
and
0 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -6,6 +6,7 @@ valid_stalled | |
valid_add | ||
valid_newbest | ||
valid_reset_add | ||
valid_reset_all | ||
train.* | ||
valid.* | ||
dev.bpe.de | ||
|
61 changes: 61 additions & 0 deletions
61
tests/training/restoring/validation/test_valid_reset_all.sh
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,61 @@ | ||
#!/bin/bash -x | ||
|
||
##################################################################### | ||
# SUMMARY: Test the option for resetting all validation metrics after the training is resumed | ||
# AUTHOR: snukky | ||
# TAGS: restore valid valid-script valid-reset | ||
##################################################################### | ||
|
||
# Exit on error | ||
set -e | ||
|
||
# Test code goes here | ||
rm -rf valid_reset_all valid_reset_all_?.*log valid_script_?.temp | ||
mkdir -p valid_reset_all | ||
|
||
test -s valid.mini.bpe.en || head -n 8 $MRT_DATA/europarl.de-en/toy.bpe.en > valid.mini.bpe.en | ||
test -s valid.mini.bpe.de || head -n 8 $MRT_DATA/europarl.de-en/toy.bpe.de > valid.mini.bpe.de | ||
|
||
extra_opts="--no-shuffle --seed 2222 --maxi-batch 1 --maxi-batch-sort none --quiet-translation" | ||
extra_opts="$extra_opts --dim-emb 64 --dim-rnn 128 --mini-batch 16 --optimizer sgd" | ||
extra_opts="$extra_opts --cost-type ce-mean --disp-label-counts false" | ||
|
||
|
||
# Train a model for a while and stop | ||
$MRT_MARIAN/marian $extra_opts \ | ||
-m valid_reset_all/model.npz -t $MRT_DATA/europarl.de-en/toy.bpe.{en,de} -v vocab.en.yml vocab.de.yml \ | ||
--disp-freq 10 --valid-freq 20 --after-batches 140 --early-stopping 5 \ | ||
--valid-metrics translation valid-script cross-entropy --valid-script-path ./valid_script_ab.sh \ | ||
--valid-sets valid.mini.bpe.{de,en} \ | ||
--overwrite --keep-best --clip-norm 0 \ | ||
--log valid_reset_all_1.log | ||
|
||
test -e valid_reset_all/model.npz | ||
test -e valid_reset_all/model.npz.yml | ||
test -e valid_reset_all_1.log | ||
|
||
cp valid_reset_all/model.npz.progress.yml valid_reset_all/model.npz.progress.yml.bak | ||
|
||
cat valid_reset_all_1.log | $MRT_TOOLS/strip-timestamps.sh | grep -P "\[valid\]" > valid_reset_all.out | ||
|
||
|
||
# Restart training with --valid-reset-all | ||
$MRT_MARIAN/marian $extra_opts \ | ||
-m valid_reset_all/model.npz -t $MRT_DATA/europarl.de-en/toy.bpe.{en,de} -v vocab.en.yml vocab.de.yml \ | ||
--disp-freq 10 --valid-freq 20 --after-batches 200 --early-stopping 5 --valid-reset-all \ | ||
--valid-metrics translation valid-script cross-entropy --valid-script-path ./valid_script_ab.sh \ | ||
--valid-sets valid.mini.bpe.{de,en} \ | ||
--overwrite --keep-best --clip-norm 0 \ | ||
--log valid_reset_all_2.log | ||
|
||
test -e valid_reset_all/model.npz | ||
test -e valid_reset_all_2.log | ||
|
||
cat valid_reset_all_2.log | $MRT_TOOLS/strip-timestamps.sh | grep -P "\[valid\]" >> valid_reset_all.out | ||
|
||
|
||
# Compare with the expected output | ||
$MRT_TOOLS/diff.sh valid_reset_all.out valid_reset_all.expected > valid_reset_all.diff | ||
|
||
# Exit with success code | ||
exit 0 |
30 changes: 30 additions & 0 deletions
30
tests/training/restoring/validation/valid_reset_all.expected
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,30 @@ | ||
[valid] Ep. 1 : Up. 20 : translation : 333.5 : new best | ||
[valid] Ep. 1 : Up. 20 : valid-script : 222.3 : new best | ||
[valid] Ep. 1 : Up. 20 : cross-entropy : 249.337 : new best | ||
[valid] Ep. 1 : Up. 40 : translation : 333.4 : stalled 1 times (last best: 333.5) | ||
[valid] Ep. 1 : Up. 40 : valid-script : 222.2 : stalled 1 times (last best: 222.3) | ||
[valid] Ep. 1 : Up. 40 : cross-entropy : 248.218 : new best | ||
[valid] Ep. 1 : Up. 60 : translation : 333.3 : stalled 2 times (last best: 333.5) | ||
[valid] Ep. 1 : Up. 60 : valid-script : 222.1 : stalled 2 times (last best: 222.3) | ||
[valid] Ep. 1 : Up. 60 : cross-entropy : 246.856 : new best | ||
[valid] Ep. 1 : Up. 80 : translation : 333.2 : stalled 3 times (last best: 333.5) | ||
[valid] Ep. 1 : Up. 80 : valid-script : 222.6 : new best | ||
[valid] Ep. 1 : Up. 80 : cross-entropy : 245.247 : new best | ||
[valid] Ep. 1 : Up. 100 : translation : 333.1 : stalled 4 times (last best: 333.5) | ||
[valid] Ep. 1 : Up. 100 : valid-script : 222.5 : stalled 1 times (last best: 222.6) | ||
[valid] Ep. 1 : Up. 100 : cross-entropy : 243.37 : new best | ||
[valid] Ep. 1 : Up. 120 : translation : 333.9 : new best | ||
[valid] Ep. 1 : Up. 120 : valid-script : 222.4 : stalled 2 times (last best: 222.6) | ||
[valid] Ep. 1 : Up. 120 : cross-entropy : 240.802 : new best | ||
[valid] Ep. 1 : Up. 140 : translation : 333.8 : stalled 1 times (last best: 333.9) | ||
[valid] Ep. 1 : Up. 140 : valid-script : 222.3 : stalled 3 times (last best: 222.6) | ||
[valid] Ep. 1 : Up. 140 : cross-entropy : 237.65 : new best | ||
[valid] Ep. 1 : Up. 160 : translation : 333.7 : new best | ||
[valid] Ep. 1 : Up. 160 : valid-script : 222.2 : new best | ||
[valid] Ep. 1 : Up. 160 : cross-entropy : 233.833 : new best | ||
[valid] Ep. 2 : Up. 180 : translation : 333.6 : stalled 1 times (last best: 333.7) | ||
[valid] Ep. 2 : Up. 180 : valid-script : 222.1 : stalled 1 times (last best: 222.2) | ||
[valid] Ep. 2 : Up. 180 : cross-entropy : 230.035 : new best | ||
[valid] Ep. 2 : Up. 200 : translation : 333.5 : stalled 2 times (last best: 333.7) | ||
[valid] Ep. 2 : Up. 200 : valid-script : 222.6 : new best | ||
[valid] Ep. 2 : Up. 200 : cross-entropy : 227.982 : new best |