Fix[critical] AdamW epsilon default value

rioyokotalab · Jul 16, 2024 · 93f317e · 93f317e
1 parent ade3a65
commit 93f317e
Showing 1 changed file with 1 addition and 1 deletion.
diff --git a/src/llama_recipes/arguments.py b/src/llama_recipes/arguments.py
@@ -306,7 +306,7 @@ def _add_regularization_args(parser: argparse.ArgumentParser) -> argparse.Argume
         help='Second coefficient for computing running averages of gradient and its square'
     )
     group.add_argument(
-        '--adam-eps', type=float, default=1e-06,
+        '--adam-eps', type=float, default=1e-08,
         help='Term added to the denominator to improve numerical stability'
     )