bert_large_lamb_nvidia_data.json

{
    "name": "bing_bert_large_lamb_seq",
    "bert_token_file": "bert-large-uncased",
    "bert_model_file": "bert-large-uncased",
    "bert_model_config": {
        "vocab_size_or_config_json_file": 119547,
        "hidden_size": 1024,
        "num_hidden_layers": 24,
        "num_attention_heads": 16,
        "intermediate_size": 4096,
        "hidden_act": "gelu",
        "hidden_dropout_prob": 0.1,
        "attention_probs_dropout_prob": 0.1,
        "max_position_embeddings": 512,
        "type_vocab_size": 2,
        "initializer_range": 0.02
    },
    "data": {
        "flags": {
            "pretrain_dataset": true,
            "pretrain_type": "wiki_bc"
        },
        "mixed_seq_datasets": {
            "128": {
                "pretrain_dataset": "data/128"
            },
            "512": {
                "pretrain_dataset": "data/512"
            }
        }
    },
    "mixed_seq_training": {
        "128": {
            "num_epochs": 16,
            "warmup_proportion": 0.06,
            "learning_rate": 11e-3,
            "num_workers": 4,
            "async_worker": true,
            "decay_rate": 0.90,
            "decay_step": 250,
            "total_training_steps": 7500
        },
        "512": {
            "num_epochs": 20,
            "warmup_proportion": 0.02,
            "learning_rate": 2e-3,
            "num_workers": 4,
            "async_worker": true,
            "decay_rate": 0.90,
            "decay_step": 150,
            "total_training_steps": 7500
        }
    },
    "validation": {
        "path": "validation_set/"
    }
}