pip install transformers torch torch-amp

import torch
from transformers import AutoModelForSequenceClassification
from torch.cuda.amp import autocast, GradScaler
from torch.nn.parallel import DistributedDataParallel as DDP
from torch.distributed import init_process_group, broadcast

# Initialize FSDP with FP16 precision
model = AutoModelForSequenceClassification.from_pretrained('your_model_name')
model = torch.nn.parallel.FSDP(model, device_ids=[0], process_group=None)

# Initialize ShardedGradScaler
scaler = GradScaler()

# Move model to device
model.to('cuda')

# Initialize optimizer and scheduler
optimizer = torch.optim.Adam(model.parameters(), lr=1e-5)
scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=10)

# Train model
for epoch in range(10):
    # Zero gradients
    optimizer.zero_grad()

    # Forward pass with autocast
    with autocast():
        outputs = model(input_ids, attention_mask)
        loss = outputs.loss

    # Backward pass with scaler
    scaler.scale(loss).backward()
    scaler.step(optimizer)
    scaler.update()

    # Update scheduler
    scheduler.step()