transformers - 💡(How to fix) Fix Direct build of layer/attention silently produces non-causal output

def eager_attention_forward(module, query, key, value, attention_mask, scaling, dropout=0.0, **kwargs):
    ...
    attn_weights = torch.matmul(query, key_states.transpose(2, 3)) * scaling
    if attention_mask is not None:
        attn_weights = attn_weights + attention_mask
    elif getattr(module, "is_causal", False):           # apply causal mask when none is supplied
        L, S = query.shape[-2], key_states.shape[-2]
        causal = torch.ones(L, S, dtype=torch.bool, device=query.device).tril(diagonal=S - L)
        attn_weights = attn_weights.masked_fill(~causal, float("-inf"))
    ...

---

config.to_dict(include_runtime=True)         # keep _attn_implementation, etc.
config.to_json_string(include_runtime=True)

---

def _remove_keys_not_serialized(self, d, include_runtime: bool = False):
    keys_to_remove = ["_is_quantized", "_auto_class", "_commit_hash", ...]
    if not include_runtime:
        keys_to_remove += ["_attn_implementation_internal", "_experts_implementation_internal"]
    for key_to_remove in keys_to_remove:
        d.pop(key_to_remove, None)
    ...

---

cfg = AutoConfig.from_pretrained("Qwen/Qwen3-4B")
cfg._attn_implementation            # -> None  (never set without a model)
layer = Qwen3DecoderLayer(cfg, 0)   # dispatch falls back to eager

---

import torch
from transformers import Qwen3Config
from transformers.models.qwen3.modeling_qwen3 import Qwen3Attention, Qwen3RotaryEmbedding

cfg = Qwen3Config(hidden_size=64, num_attention_heads=4, num_key_value_heads=2,
                  head_dim=16, intermediate_size=128, num_hidden_layers=1, vocab_size=100)

assert cfg._attn_implementation is None 
attn = Qwen3Attention(cfg, layer_idx=0).eval()
assert attn.is_causal is True   

B, T = 1, 8
x = torch.randn(B, T, cfg.hidden_size)
pe = Qwen3RotaryEmbedding(cfg)(x, torch.arange(T).unsqueeze(0))

with torch.no_grad():
    out_none, _ = attn(x, position_embeddings=pe, attention_mask=None)  # None -> eager, NOT causal
    cfg._attn_implementation = "sdpa"
    out_sdpa, _ = attn(x, position_embeddings=pe, attention_mask=None)  # sdpa, is_causal=True

print((out_none - out_sdpa).norm())   # large: bidirectional vs causal