58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502 | def parse_args(args: Optional[List[str]] = None) -> argparse.Namespace:
"""Parse arguments for Evo2 model training."""
parser = argparse.ArgumentParser(
description="Train a Hyena model using NeMo 2.0.",
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
)
data_group = parser.add_mutually_exclusive_group(required=True)
data_group.add_argument(
"-d",
"--dataset-config",
type=str,
help="Path to the blended / weighted training dataset configuration YAML.",
)
data_group.add_argument(
"--mock-data",
action="store_true",
help="Train with Mock data (for testing/debugging), either set this or provide a dataset config.",
)
parser.add_argument(
"--dataset-dir",
type=str,
help="Absolute path to the dataset directory. Defaults to using the absolute or relative paths (dataset_prefix) specified in the dataset config YAML.",
)
parser.add_argument("--num-nodes", type=int, default=1, help="Number of nodes to use for training, defaults to 1.")
parser.add_argument("--devices", type=int, default=1, help="Number of devices to use for training, defaults to 1.")
parser.add_argument("--seq-length", type=int, default=8192, help="Training sequence length")
parser.add_argument(
"--tensor-parallel-size", type=int, default=1, help="Order of tensor parallelism. Defaults to 1."
)
parser.add_argument(
"--pipeline-model-parallel-size", type=int, default=1, help="Order of pipeline parallelism. Defaults to 1."
)
parser.add_argument(
"--context-parallel-size", type=int, default=1, help="Order of context parallelism. Defaults to 1."
)
parser.add_argument(
"--create-tensorboard-logger", action="store_true", default=False, help="Create a tensorboard logger."
)
parser.add_argument("--wandb-entity", type=str, default=None, help="The team posting this run")
parser.add_argument("--wandb-project", type=str, default=None, help="Wandb project name ")
parser.add_argument("--wandb-tags", nargs="+", type=str, default=None, help="Tags associated with this run")
parser.add_argument(
"--wandb-group", type=str, default=None, help="A unique string shared by all runs in a given group"
)
parser.add_argument(
"--wandb-job-type",
type=str,
default=None,
help="A unique string representing a type of run, which is useful when you're grouping runs together into larger experiments using group.",
)
parser.add_argument(
"--wandb-run-name",
type=str,
default=None,
help="A unique string representing the name of the wandb run. If not provided, the name will be generated from the model and training specifications.",
)
parser.add_argument(
"--wandb-id", type=str, default=None, help="Sets the version, mainly used to resume a previous run"
)
parser.add_argument(
"--wandb-anonymous", action="store_true", help="Enable or explicitly disable anonymous logging"
)
parser.add_argument(
"--wandb-log-model", action="store_true", help="Save checkpoints in wandb dir to upload on W&B servers"
)
parser.add_argument("--wandb-offline", action="store_true", help="Use wandb in offline mode")
parser.add_argument("--sequence-parallel", action="store_true", help="Set to enable sequence parallelism.")
parser.add_argument("--fp8", action="store_true", help="Set to enable FP8")
parser.add_argument("--micro-batch-size", type=int, default=1, help="Micro-batch size for data-parallel training.")
parser.add_argument(
"--global-batch-size",
type=int,
default=None,
help="Global batch size for training. If set to None, infer it from the TP, CP, and PP parameters.",
)
parser.add_argument(
"--grad-acc-batches", type=int, default=1, help="Number of batches to accumulate gradients over."
)
parser.add_argument(
"--max-steps",
type=int,
help="Number of training optimizer update steps. This controls the total number of steps as well as the "
"shape of the learning rate curve.",
default=500000,
)
parser.add_argument(
"--constant-steps",
type=int,
help="Number of steps to keep the learning rate constant before annealing. This controls the "
"shape of the learning rate curve.",
default=80000,
)
parser.add_argument(
"--early-stop-on-step",
type=int,
help="Stop training on this step, if set. This may be useful for testing or debugging purposes.",
)
parser.add_argument(
"--val-check-interval", type=int, help="Number of steps between validation measurements and model checkpoints."
)
parser.add_argument("--grad-reduce-in-fp32", action="store_true", default=False, help="Gradient reduce in FP32.")
parser.add_argument(
"--fp8-wgrad",
action="store_true",
default=False,
help="Faster option that is maybe less accurate (TBD) when using fp8.",
)
parser.add_argument("--use-megatron-comm-overlap-llama3-8k", action="store_true", default=False)
parser.add_argument(
"--tp-comm-overlap-backend",
type=str,
choices=["nccl", "mpi", "gloo"],
default="nccl",
help="TP communication backend to use. Defaults to 'nccl'.",
)
parser.add_argument("--align-param-gather", action="store_true", default=False)
# parser.add_argument("--straggler-detection", action="store_true", default=False)
parser.add_argument(
"--model-size",
type=str,
choices=sorted(list(HYENA_MODEL_OPTIONS.keys()) + list(MAMBA_MODEL_OPTIONS.keys())),
default="7b",
help="Model size/configuration to use. Options depend on the selected model-type.",
)
parser.add_argument(
"--add-bias-output",
action="store_true",
default=False,
help="Add bias to the output layer to enable learning a simple prior.",
)
parser.add_argument(
"--result-dir", type=Path, required=False, default=Path("./results"), help="Path to the result directory."
)
parser.add_argument("--experiment-name", type=str, required=False, default="evo2", help="Name of the experiment.")
parser.add_argument(
"--limit-val-batches",
type=int,
default=20,
help="Number of validation steps",
)
parser.add_argument(
"--log-every-n-steps",
type=int,
default=1,
required=False,
help="Number of steps between logging.",
)
parser.add_argument(
"--ckpt-dir",
type=str,
default=None,
help="Directory to restore an initial checkpoint from. Use this for supervised fine-tuning.",
)
parser.add_argument(
"--use-precision-aware-optimizer",
action="store_true",
default=False,
help="Use precision aware optimizer that stores main weights in FP32 when doing mixed precision training.",
)
parser.add_argument(
"--bf16-main-grads",
action="store_true",
default=False,
help="Use bf16 for main gradients, only use this with --use-precision-aware-optimizer.",
)
parser.add_argument("--wd", type=float, default=0.01, help="Weight decay for optimizer.")
parser.add_argument(
"--adam-beta1",
type=float,
default=0.9,
help="Adam optimizer beta1 parameter.",
)
parser.add_argument(
"--adam-beta2",
type=float,
default=0.95,
help="Adam optimizer beta2 parameter.",
)
parser.add_argument(
"--adam-eps",
type=float,
default=1e-8,
help="Adam optimizer epsilon parameter. The inverse of this value (1/eps) represents the maximum adaptive learning rate per parameter.",
)
parser.add_argument(
"--restore-optimizer-from-ckpt",
action="store_true",
help="Restore optimizer state from initial checkpoint. Defaults to False.",
)
parser.add_argument(
"--average-in-collective",
action="store_true",
default=False,
help="Avaerage optimizer state in collective rather than dividing by dp size and summing.",
)
parser.add_argument("--seed", type=int, default=1234, help="Set random seed for training.")
parser.add_argument("--workers", type=int, default=8, help="Number of workers to use for data loading.")
parser.add_argument(
"--gc-interval",
type=int,
default=0,
help="Set to a value > 0 if you want to synchronize garbage collection, will do gc every gc-interval steps.",
)
parser.add_argument(
"--enable-preemption",
action="store_true",
default=False,
help="Enable preemption hooks. If enabled this will save a checkpoint whenever slurm exits.",
)
parser.add_argument(
"--ckpt-async-save",
action="store_true",
default=False,
)
parser.add_argument(
"--ckpt-format",
type=str,
choices=["torch_dist", "zarr"],
default="torch_dist",
help="Specify checkpoint format to use. Defaults to 'torch_dist', as 'zarr' is deprecated. Only use if "
"resuming training from a zarr checkpoint.",
)
parser.add_argument(
"--eod-pad-in-loss-mask",
action="store_true",
default=False,
help="Do not predict EOD/Pad tokens (typical default, but not default in original evo2).",
)
parser.add_argument(
"--cross-entropy-loss-fusion",
action="store_true",
default=False,
help="Use the faster, but maybe less accurate fused form of cross entropy, "
"which also has bf16 grads internally.",
)
parser.add_argument(
"--no-fp32-residual-connection",
action="store_true",
default=False,
help="If set, turn off fp32 residual connections which may be faster but may impact accuracy.",
)
parser.add_argument(
"--debug-ddp-parity-freq",
type=int,
default=0,
help="Set to value > 0 to debug DDP weight parity between ranks.",
)
parser.add_argument(
"--hybrid-override-pattern",
type=str,
help="Override the hybrid override pattern in the config (specifies hyena layer ordering and type).",
)
parser.add_argument(
"--num-layers", type=int, help="If set, override the number of layers specified in the requested config."
)
parser.add_argument(
"--create-tflops-callback",
action="store_true",
default=False,
help="Enable tflops calculation callback for Hyena / Evo2. Defaults to False.",
)
parser.add_argument(
"--log-parameters-and-shapes",
action="store_true",
default=False,
help="Log training parameters shapes and dtypes for debugging.",
)
parser.add_argument("--lr", type=float, default=3e-4, help="Learning rate.")
parser.add_argument("--min-lr", type=float, default=3e-5, help="Min learning rate in cosine annealing.")
parser.add_argument("--warmup-steps", type=int, default=2500, help="Number of warmup steps in cosine annealing")
# NSYS profiling/tooling arguments
parser.add_argument(
"--nsys-profiling",
action="store_true",
default=False,
help="Enable targeted `nsys` profiling on the training loop for a defined step range. To actually get profiling"
" output you must run the whole program with `nsys`. For example: "
" `nsys profile -s none -o output_report_name -t cuda,nvtx --force-overwrite true "
"--capture-range=cudaProfilerApi --capture-range-end=stop [regular python command here]`",
)
# start, end, rank
parser.add_argument(
"--nsys-start-step",
type=int,
required=False,
default=0,
help="Start nsys profiling after this step.",
)
parser.add_argument(
"--spike-no-more-embedding-init",
action="store_true",
default=False,
help="If set, the embeddings are initialized with a Normal(0, 1.0) distribution rather "
"than the default Normal(0, 0.02). This may help avoid loss spiking during training. Consider using this with "
"--no-weight-decay-embeddings to avoid shrinking the embeddings to 0 by skipping weight decay on these layers, "
"or with --use-targeted-variance-loss to maintain a 1.0 variance during training even with weight decay.",
)
parser.add_argument(
"--no-weight-decay-embeddings",
action="store_true",
default=False,
help="If set, do not apply weight decay to the embeddings.",
)
parser.add_argument(
"--use-targeted-variance-loss",
action="store_true",
default=False,
help="Use targeted variance loss.",
)
parser.add_argument(
"--nsys-end-step",
type=int,
required=False,
help="End nsys profiling after this step.",
)
parser.add_argument(
"--no-renormalize-loss",
action="store_true",
default=False,
help="Do not renormalize the loss weights.",
)
parser.add_argument(
"--mamba-lowercase-loss-weight",
type=float,
default=0.1,
help="Loss weight for the Mamba model for lowercase bases, if you are using a Mamba model. "
"Default is 0.1 like the Evo2 paper. Set to 1.0 to disable differential loss weighting.",
)
# rank as list of integers
parser.add_argument(
"--nsys-ranks",
type=int,
nargs="+",
required=False,
default=[0],
help="Enable nsys profiling for these ranks.",
)
parser.add_argument(
"--activation-checkpoint-recompute-num-layers",
type=int,
help="If set, override the default value set in the config.",
)
parser.add_argument(
"--disable-checkpointing",
action="store_false",
default=True,
dest="create_checkpoint_callback",
help="Disable creating a ModelCheckpoint callback.",
)
parser.add_argument(
"--clip-grad",
type=float,
default=1.0,
help="Grad clip value. Note that when using DDP this may need to be inflated.",
)
parser.add_argument(
"--seq-len-interpolation-factor",
type=float,
help="Adjusts the linear scaling of ROPE (Rotary Position Embedding) for context extension. "
"Set this factor relative to your base context length e.g., for an original context length of 8192 and "
"an extended context length of 524288, use 524288/8192 = 64.",
)
parser.add_argument(
"--overlap-param-gather",
action="store_true",
default=False,
help="Overlap the parameter gather with the optimizer step. This is currently disabled due to a NeMo bug "
"when using DDP. Making this an option defaulting to False is a temporary solution until the bug is fixed.",
)
parser.add_argument(
"--overlap-grad-reduce",
action="store_true",
default=False,
help="Overlap the gradient reduce with the optimizer step.",
)
parser.add_argument(
"--hidden-dropout",
type=float,
default=0.0,
help="Dropout probability for the hyena layers",
)
parser.add_argument(
"--log-num-zeros-in-grad",
action="store_true",
default=False,
help="Log the number of zeros in the gradient.",
)
parser.add_argument(
"--attention-dropout",
type=float,
default=0.0,
help="Dropout probability for the attention layers.",
)
parser.add_argument(
"--use-b2b-causal-conv1d",
action="store_true",
help="Use back-to-back causal convolution CUDA kernel for hyena short conv layers for improved performance.",
)
parser.add_argument(
"--save-top-k",
type=int,
default=5,
help="Number of best checkpoints to keep. Set to -1 to save all checkpoints.",
)
parser.add_argument(
"--metric-to-monitor-for-checkpoints",
type=str,
default="val_loss",
help="Metric to monitor for checkpoints.",
)
parser.add_argument(
"--save-last-checkpoint",
action="store_true",
default=True,
help="Save the last checkpoint.",
)
parser.add_argument(
"--no-save-last-checkpoint",
action="store_false",
dest="save_last_checkpoint",
default=True,
help="Disable saving the last checkpoint.",
)
parser.add_argument(
"--no-calculate-per-token-loss",
action="store_true",
default=False,
help="Calculate a simpler mean across the microbatch of the loss prior to DDP reduction rather than the global"
" per-token mean loss. Use this if speed is critical and if you do not need token masking in your loss.",
)
parser.add_argument(
"--no-check-for-nan-in-grad",
action="store_true",
default=False,
help="Skip checking for NaNs in gradients. Only use this for debugging purposes.",
)
recompute_group = parser.add_mutually_exclusive_group(required=False)
recompute_group.add_argument("--no-activation-checkpointing", action="store_true", default=False)
recompute_group.add_argument("--selective-activation-checkpointing", action="store_true", default=False)
return parser.parse_args(args=args)
|