pytorch - 💡(How to fix) Fix RuntimeError: pidfd_getfd: Bad file descriptor [1 comments, 1 participants]

#!/usr/bin/env python3
import torch
import torch.multiprocessing as mp
from queue import Empty
import time
import os


def producer(queue, device_id=0):
    """Producer process that creates GPU tensors and puts them in the queue."""
    print(f"Producer: Starting on GPU {device_id}")
    print(f"Producer PYTORCH_CUDA_ALLOC_CONF: {os.environ.get('PYTORCH_CUDA_ALLOC_CONF', 'not set')}")
    print(f"Producer PYTORCH_ALLOC_CONF: {os.environ.get('PYTORCH_ALLOC_CONF', 'not set')}")

    # Set CUDA device
    torch.cuda.set_device(device_id)
    device = torch.device(f'cuda:{device_id}')

    # Create some GPU tensors
    for i in range(5):
        # Create a tensor on GPU
        tensor = torch.randn(3, 4, device=device) * (i + 1)
        print(f"Producer: Created tensor {i} with shape {tensor.shape} on {tensor.device}")
        print(f"Producer: Tensor {i} mean value: {tensor.mean().item():.4f}")

        # Share tensor directly via CUDA IPC (zero-copy)
        queue.put((i, tensor))
        print(f"Producer: Put tensor {i} in queue")
        time.sleep(0.5)

    # Signal completion
    queue.put(None)
    print("Producer: Finished")


def consumer(queue, device_id=0):
    """Consumer process that retrieves GPU tensors from the queue."""
    print(f"Consumer: Starting on GPU {device_id}")
    print(f"Consumer PYTORCH_CUDA_ALLOC_CONF: {os.environ.get('PYTORCH_CUDA_ALLOC_CONF', 'not set')}")
    print(f"Consumer PYTORCH_ALLOC_CONF: {os.environ.get('PYTORCH_ALLOC_CONF', 'not set')}")

    # Set CUDA device
    torch.cuda.set_device(device_id)

    while True:
        try:
            # Get tensor from queue (zero-copy CUDA IPC)
            item = queue.get(timeout=5)

            if item is None:
                print("Consumer: Received termination signal")
                break

            idx, tensor = item
            print(f"Consumer: Received tensor {idx} with shape {tensor.shape} on {tensor.device}")
            print(f"Consumer: Tensor {idx} mean value: {tensor.mean().item():.4f}")

            # Perform some operation on the tensor
            result = tensor * 2
            print(f"Consumer: Processed tensor {idx}, new mean: {result.mean().item():.4f}")

        except Empty:
            print("Consumer: Queue timeout")
            break

    print("Consumer: Finished")


def main():
    """Main function to set up and run the multiprocessing example."""
    # Check if CUDA is available
    if not torch.cuda.is_available():
        print("CUDA is not available. This example requires GPU support.")
        return

    print(f"CUDA available: {torch.cuda.is_available()}")
    print(f"Number of GPUs: {torch.cuda.device_count()}")

    # Check current allocator configuration
    cuda_conf = os.environ.get('PYTORCH_CUDA_ALLOC_CONF', 'not set')
    alloc_conf = os.environ.get('PYTORCH_ALLOC_CONF', 'not set')
    print(f"Main PYTORCH_CUDA_ALLOC_CONF: {cuda_conf}")
    print(f"Main PYTORCH_ALLOC_CONF: {alloc_conf}")

    # Set multiprocessing start method to 'spawn' (required for CUDA)
    mp.set_start_method('spawn', force=True)

    # Create a multiprocessing queue
    queue = mp.Queue()

    # Determine which GPU to use
    device_id = 0

    # Create producer and consumer processes
    print("Starting processes...")
    producer_process = mp.Process(target=producer, args=(queue, device_id))
    consumer_process = mp.Process(target=consumer, args=(queue, device_id))

    # Start processes
    producer_process.start()
    consumer_process.start()

    # Wait for processes to complete
    producer_process.join()
    consumer_process.join()

    print("\nAll processes completed!")


if __name__ == '__main__':
    main()

---

$ srun --container-image=nvcr.io/nvidia/pytorch:26.03-py3 --container-mounts ~/test_gpu_tensor_ipc.py:/test_gpu_tensor_ipc.py env PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True python3 /test_gpu_tensor_ipc.py
pyxis: importing docker image: nvcr.io/nvidia/pytorch:26.03-py3
pyxis: imported docker image: nvcr.io/nvidia/pytorch:26.03-py3
CUDA available: True
Number of GPUs: 4
Main PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True
Main PYTORCH_ALLOC_CONF: not set
Starting processes...
Process Process-2:
Traceback (most recent call last):
  File "/usr/lib/python3.12/multiprocessing/process.py", line 314, in _bootstrap
    self.run()
  File "/usr/lib/python3.12/multiprocessing/process.py", line 108, in run
    self._target(*self._args, **self._kwargs)
  File "/test_gpu_tensor_ipc.py", line 48, in consumer
    item = queue.get(timeout=5)
           ^^^^^^^^^^^^^^^^^^^^
  File "/usr/lib/python3.12/multiprocessing/queues.py", line 122, in get
    return _ForkingPickler.loads(res)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/usr/local/lib/python3.12/dist-packages/torch/multiprocessing/reductions.py", line 180, in rebuild_cuda_tensor
    storage = storage_cls._new_shared_cuda(
              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/usr/local/lib/python3.12/dist-packages/torch/storage.py", line 1464, in _new_shared_cuda
    return torch.UntypedStorage._new_shared_cuda(*args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
RuntimeError: pidfd_getfd: Bad file descriptor
Consumer: Starting on GPU 0
Consumer PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True
Consumer PYTORCH_ALLOC_CONF: not set
Producer: Starting on GPU 0
Producer PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True
Producer PYTORCH_ALLOC_CONF: not set
Producer: Created tensor 0 with shape torch.Size([3, 4]) on cuda:0
Producer: Tensor 0 mean value: 0.4615
Producer: Put tensor 0 in queue
Producer: Created tensor 1 with shape torch.Size([3, 4]) on cuda:0
Producer: Tensor 1 mean value: 0.3834
Producer: Put tensor 1 in queue
Producer: Created tensor 2 with shape torch.Size([3, 4]) on cuda:0
Producer: Tensor 2 mean value: -0.6317
Producer: Put tensor 2 in queue
Producer: Created tensor 3 with shape torch.Size([3, 4]) on cuda:0
Producer: Tensor 3 mean value: -0.1617
Producer: Put tensor 3 in queue
Producer: Created tensor 4 with shape torch.Size([3, 4]) on cuda:0
Producer: Tensor 4 mean value: -0.4775
Producer: Put tensor 4 in queue
Producer: Finished
[W402 20:55:40.208040611 CudaIPCTypes.cpp:16] Producer process has been terminated before all shared CUDA tensors released. See Note [Sharing CUDA tensors]

All processes completed

---

[pid 1587003] pidfd_getfd(61, 2097152, 0) = -1 EBADF (Bad file descriptor)

---

torch.empty(1, device=f'cuda:{device_id}')

---

$ python collect_env.py
Collecting environment information...
PyTorch version: 2.11.0a0+a6c236b9fd.nv26.03.46836102
Is debug build: False
CUDA used to build PyTorch: 13.2
ROCM used to build PyTorch: N/A

OS: Ubuntu 24.04.4 LTS (aarch64)
GCC version: (Ubuntu 13.3.0-6ubuntu2~24.04.1) 13.3.0
Clang version: Could not collect
CMake version: version 3.31.6
Libc version: glibc-2.39