-
Notifications
You must be signed in to change notification settings - Fork 1.1k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge branch 'main' into readme-tests
- Loading branch information
Showing
17 changed files
with
819 additions
and
59 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,95 @@ | ||
|
|
||
| # The path to the base model's checkpoint directory to load for finetuning. (type: <class 'Path'>, default: checkpoints/stabilityai/stablelm-base-alpha-3b) | ||
| checkpoint_dir: checkpoints/meta-llama/Meta-Llama-3-8B | ||
|
|
||
| # Directory in which to save checkpoints and logs. (type: <class 'Path'>, default: out/finetune/full) | ||
| out_dir: out/finetune/full-llama-3-8b | ||
|
|
||
| # The precision to use for finetuning. Possible choices: "bf16-true", "bf16-mixed", "32-true". (type: Optional[str], default: null) | ||
| precision: bf16-true | ||
|
|
||
| # How many devices/GPUs to use (type: Union[int, str], default: 1) | ||
| devices: 4 | ||
|
|
||
| # Path to a checkpoint directory to resume from in case training was interrupted, or ``True`` to resume | ||
| # from the latest checkpoint in ``out_dir``. (type: Union[bool, Path], default: False) | ||
| resume: false | ||
|
|
||
| # Data-related arguments. If not provided, the default is ``litgpt.data.Alpaca``. | ||
| data: | ||
| class_path: litgpt.data.Alpaca2k | ||
| init_args: | ||
| mask_prompt: false | ||
| prompt_style: alpaca | ||
| ignore_index: -100 | ||
| seed: 42 | ||
| num_workers: 4 | ||
|
|
||
| # Training-related arguments. See ``litgpt.args.TrainArgs`` for details | ||
| train: | ||
|
|
||
| # Number of optimizer steps between saving checkpoints (type: Optional[int], default: 1000) | ||
| save_interval: 200 | ||
|
|
||
| # Number of iterations between logging calls (type: int, default: 1) | ||
| log_interval: 1 | ||
|
|
||
| # Number of samples between optimizer steps across data-parallel ranks (type: int, default: 64) | ||
| global_batch_size: 64 | ||
|
|
||
| # Number of samples per data-parallel rank (type: int, default: 1) | ||
| micro_batch_size: 4 | ||
|
|
||
| # Number of iterations with learning rate warmup active (type: int, default: 100) | ||
| lr_warmup_steps: 25 | ||
|
|
||
| # Number of epochs to train on (type: Optional[int], default: 5) | ||
| epochs: 1 | ||
|
|
||
| # Total number of tokens to train on (type: Optional[int], default: null) | ||
| max_tokens: | ||
|
|
||
| # Limits the number of optimizer steps to run. (type: Optional[int], default: null) | ||
| max_steps: | ||
|
|
||
| # Limits the length of samples. Off by default (type: Optional[int], default: null) | ||
| max_seq_length: 512 | ||
|
|
||
| # Whether to tie the embedding weights with the language modeling head weights. (type: Optional[bool], default: null) | ||
| tie_embeddings: | ||
|
|
||
| # (type: float, default: 0.003) | ||
| learning_rate: 0.0002 | ||
|
|
||
| # (type: float, default: 0.02) | ||
| weight_decay: 0.1 | ||
|
|
||
| # (type: float, default: 0.9) | ||
| beta1: 0.9 | ||
|
|
||
| # (type: float, default: 0.95) | ||
| beta2: 0.95 | ||
|
|
||
| # (type: Optional[float], default: null) | ||
| max_norm: | ||
|
|
||
| # (type: float, default: 6e-05) | ||
| min_lr: 6.0e-05 | ||
|
|
||
| # Evaluation-related arguments. See ``litgpt.args.EvalArgs`` for details | ||
| eval: | ||
|
|
||
| # Number of optimizer steps between evaluation calls (type: int, default: 600) | ||
| interval: 25 | ||
|
|
||
| # Number of tokens to generate (type: Optional[int], default: 100) | ||
| max_new_tokens: 100 | ||
|
|
||
| # Number of iterations (type: int, default: 100) | ||
| max_iters: 100 | ||
|
|
||
| # The name of the logger to send metrics to. (type: Literal['wandb', 'tensorboard', 'csv'], default: csv) | ||
| logger_name: csv | ||
|
|
||
| # The random seed to use for reproducibility. (type: int, default: 1337) | ||
| seed: 1337 |
Oops, something went wrong.