demo-hyper-param-tuning.config

#!returnn.py
# kate: syntax python;
# base of this is demo-tf-native-lstm2.12ax.config

use_tensorflow = True

from HyperParamTuning import HyperParam
import os
from returnn.util.basic import get_login_username
demo_name, _ = os.path.splitext(__file__)
print("Hello, experiment: %s" % demo_name)

#task = "train"
task = "hyper_param_tuning"
train = {"class": "Task12AXDataset", "num_seqs": 1000}
dev = {"class": "Task12AXDataset", "num_seqs": 100, "fixed_random_seed": 1}

num_inputs = 9
num_outputs = 2
batching = "random"
batch_size = 5000
max_seqs = 10
chunking = "0"

network = {
"fw0": {
    "class": "rec", "unit": "nativelstm2",
    "dropout": HyperParam(float, [0, 1], default=0),
    "n_out": HyperParam(int, [1, 100], default=20),
    "L2": HyperParam(float, [0, 1], log=True, default=0.01)
    },
"output": {
    "class": "softmax", "loss": "ce", "from": ["fw0"],
    "dropout": HyperParam(float, [0, 1], default=0),
    "L2": HyperParam(float, [0, 1], log=True, default=0.01)}
}

# training
optimizer = {"class": "adam"}
optimizer_epsilon = HyperParam(float, [1e-16, 1], log=True, default=1e-16)
decouple_constraints = HyperParam(bool)
learning_rate = HyperParam(float, [1e-6, 1], log=True, default=0.01)
gradient_clip = HyperParam(float, [1e-3, 100], log=True, default=0)
gradient_clip_norm = HyperParam(float, [1e-3, 100], log=True, default=0)
gradient_noise = HyperParam(float, [0, 1], default=0)

model = "/tmp/%s/returnn/%s/model" % (get_login_username(), demo_name)  # https://github.com/tensorflow/tensorflow/issues/6537
num_epochs = 100
save_interval = 1

# hyper param tuning
hyper_param_tuning = {
    "num_train_steps": 500,
    "num_tune_iterations": 100,
    "num_individuals": 30,
    "num_threads": 30
}

# log
log_verbosity = 3