pytorch - 💡(How to fix) Fix RuntimeError: Function 'ScaledDotProductEfficientAttentionBackward0' returned nan values in its 0th output when using attn_mask in SDPA backward [2 comments, 2 participants]

import torch
import torch.nn.functional as F

# Define a function to clip attention weights
def clip_attention_weights(weights):
    return torch.clamp(weights, min=-100, max=100)

# Define a function to scale input values
def scale_input_values(q, k, v):
    q_scale = q.abs().max()
    k_scale = k.abs().max()
    v_scale = v.abs().max()
    q = q / q_scale
    k = k / k_scale
    v = v / v_scale
    return q, k, v

# Modify the original code
with sdpa_kernel(SDPBackend.EFFICIENT_ATTENTION):
    q, k, v = scale_input_values(q, k, v)
    out = F.scaled_dot_product_attention(
        q, k, v,
        attn_mask=attn_mask,
        dropout_p=min(dropout_p, 0.1),  # reduce dropout probability
        is_causal=causal
    )
    attention_weights = out[1]  # get attention weights
    attention_weights = clip_attention_weights(attention_weights)  # clip attention weights

# Verify that the gradients are not NaN
q_grad = q.grad
k_grad = k.grad
assert not torch.isnan(q_grad).any().item(), "q grad is nan"
assert not torch.isnan(k_grad).any().item(), "k grad is nan"