launch.json

{
    // Use IntelliSense to learn about possible attributes.
    // Hover to view descriptions of existing attributes.
    // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
    "version": "0.2.0",
    "configurations": [
        {
            "name": "Bert Debug",
            "type": "python",
            "request": "launch",
            "env": {
                "CUDA_DEVICE_MAX_CONNECTIONS": "1",
            },
            "module": "torch.distributed.run",
            "args" : ["/workspace/megatron/pretrain_bert.py", 
                      "--num-layers", "12",
                      "--hidden-size", "1024",
                      "--num-attention-heads", "16",
                      "--seq-length", "512",
                      "--max-position-embeddings", "512",
                      "--micro-batch-size", "4",
                      "--global-batch-size", "8",
                      "--lr", "0.0001",
                      "--train-iters", "20000",
                      "--lr-decay-iters", "990000",
                      "--lr-decay-style", "linear",
                      "--min-lr", "0.00001",
                      "--weight-decay", "1e-2",
                      "--lr-warmup-fraction", ".01",
                      "--clip-grad", "1.0",
                      "--fp16", 

                      "--data-path", "/workspace/dataset/europarl/europarl-bert_text_sentence",
                      "--vocab-file", "/workspace/checkpoints/BERT-345M-uncased/bert-large-uncased-vocab.txt",
                      "--split", "949,50,1",

                      "--log-interval", "100",
                      "--save-interval", "500",
                      "--eval-interval", "500",
                      "--eval-iters", "10",
                      "--log-throughput",

                      "--save", "/workspace/checkpoints/debug",
                      "--load", "/workspace/checkpoints/debug",
        ],
            "console": "integratedTerminal",
            "justMyCode": true
        }
    ]
}