transformers - 💡(How to fix) Fix First-class fine-tuning support for Mamba / Mamba-2 SSMs — architecture is production-ready, but the training path in Transformers isn't [5 comments, 4 participants]

import torch
from transformers import MambaForPreTraining, MambaTokenizer
from transformers import Trainer, TrainingArguments
from torch.utils.data import Dataset

# Load pre-trained Mamba model and tokenizer
model = MambaForPreTraining.from_pretrained("mamba-base")
tokenizer = MambaTokenizer.from_pretrained("mamba-base")

# Define custom dataset class
class MambaDataset(Dataset):
    def __init__(self, texts, tokenizer):
        self.texts = texts
        self.tokenizer = tokenizer

    def __getitem__(self, idx):
        text = self.texts[idx]
        inputs = self.tokenizer(text, return_tensors="pt")
        return inputs

# Define custom DataCollatorForLanguageModeling class
class MambaDataCollator:
    def __call__(self, features):
        input_ids = torch.cat([f["input_ids"] for f in features])
        attention_mask = torch.cat([f["attention_mask"] for f in features])
        labels = torch.cat([f["labels"] for f in features])
        return {"input_ids": input_ids, "attention_mask": attention_mask, "labels": labels}

# Create dataset and data collator
dataset = MambaDataset(["Example text"], tokenizer)
data_collator = MambaDataCollator()

# Define training arguments
training_args = TrainingArguments(
    output_dir="./results",
    num_train_epochs=3,
    per_device_train_batch_size=16,
    per_device_eval_batch_size=64,
    evaluation_strategy="epoch",
    learning_rate=1e-5,
    save_total_limit=2,
    save_steps=500,
    load_best_model_at_end=True,
    metric_for_best_model="accuracy",
    greater_is_better=True,
    save_on_each_node=True,
)

# Create trainer
trainer = Trainer(
    model=model,
    args=training_args,
    train_dataset=dataset,
    eval_dataset=dataset,
    compute_metrics=lambda pred: {"accuracy": torch.sum(pred.label_ids == pred.predictions.argmax(-1))},
    data_collator=data_collator,
)

# Train model
trainer.train()