From 81597ddd9675073227ca85b9b4ebc8b67381ffca Mon Sep 17 00:00:00 2001 From: chenyu Date: Fri, 14 Feb 2025 19:10:35 -0500 Subject: [PATCH] increase lr for bert (#9098) had one run that converged better https://wandb.ai/chenyuxyz/MLPerf-BERT/runs/u66tv2hh/overview --- examples/mlperf/model_train.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/mlperf/model_train.py b/examples/mlperf/model_train.py index b40bf59239..932b90bb1f 100644 --- a/examples/mlperf/model_train.py +++ b/examples/mlperf/model_train.py @@ -647,7 +647,7 @@ def train_bert(): # ** hyperparameters ** BS = config["GLOBAL_BATCH_SIZE"] = getenv("BS", 11 * len(GPUS) if dtypes.default_float in (dtypes.float16, dtypes.bfloat16) else 8 * len(GPUS)) EVAL_BS = config["EVAL_BS"] = getenv("EVAL_BS", 1 * len(GPUS)) - max_lr = config["OPT_BASE_LEARNING_RATE"] = getenv("OPT_BASE_LEARNING_RATE", 0.0001 * math.sqrt(BS/66)) + max_lr = config["OPT_BASE_LEARNING_RATE"] = getenv("OPT_BASE_LEARNING_RATE", 0.00011 * math.sqrt(BS/66)) train_steps = config["TRAIN_STEPS"] = getenv("TRAIN_STEPS", 3630000 // BS) warmup_steps = config["NUM_WARMUP_STEPS"] = getenv("NUM_WARMUP_STEPS", 1)