niktverd@niktverd-osx gonka-pow % ssh 0ug08d9byqj5kf-64411322@ssh.runpod.io -i ~/.ssh/id_ed25519

-- RUNPOD.IO --
Enjoy your Pod #0ug08d9byqj5kf ^_^

root@dd9e44eb4d34:/app# sudo systemctl status nvidia-fabricmanager
System has not been booted with systemd as init system (PID 1). Can't operate.
Failed to connect to bus: Host is down
root@dd9e44eb4d34:/app# cat /usr/local/cuda/version.txt
cat: /usr/local/cuda/version.txt: No such file or directory
root@dd9e44eb4d34:/app# nvidia-smi
Fri Jan 23 02:15:19 2026       
+-----------------------------------------------------------------------------------------+
| NVIDIA-SMI 575.57.08              Driver Version: 575.57.08      CUDA Version: 12.9     |
|-----------------------------------------+------------------------+----------------------+
| GPU  Name                 Persistence-M | Bus-Id          Disp.A | Volatile Uncorr. ECC |
| Fan  Temp   Perf          Pwr:Usage/Cap |           Memory-Usage | GPU-Util  Compute M. |
|                                         |                        |               MIG M. |
|=========================================+========================+======================|
|   0  NVIDIA H100 PCIe               On  |   00000000:A1:00.0 Off |                    0 |
| N/A   34C    P0             46W /  310W |       0MiB /  81559MiB |      0%      Default |
|                                         |                        |             Disabled |
+-----------------------------------------+------------------------+----------------------+
                                                                                         
+-----------------------------------------------------------------------------------------+
| Processes:                                                                              |
|  GPU   GI   CI              PID   Type   Process name                        GPU Memory |
|        ID   ID                                                               Usage      |
|=========================================================================================|
|  No running processes found                                                             |
+-----------------------------------------------------------------------------------------+
root@dd9e44eb4d34:/app# ls -la /dev/nvidia*
crw-rw-rw- 1 nobody nogroup 195, 254 Nov 11 14:17 /dev/nvidia-modeset
crw-rw-rw- 1 nobody nogroup 510,   0 Nov 11 14:17 /dev/nvidia-uvm
crw-rw-rw- 1 nobody nogroup 510,   1 Nov 11 14:17 /dev/nvidia-uvm-tools
crw-rw-rw- 1 nobody nogroup 195,   6 Nov 11 14:17 /dev/nvidia6
crw-rw-rw- 1 nobody nogroup 195, 255 Nov 11 14:17 /dev/nvidiactl

/dev/nvidia-caps:
total 0
drwxr-xr-x 2 root root  40 Jan 23 02:11 .
drwxr-xr-x 7 root root 480 Jan 23 02:11 ..
root@dd9e44eb4d34:/app# python3 -c "import torch; print(torch.version.cuda); print(torch.cuda.is_available()); print(torch.cuda.device_count())"
12.6
/app/packages/api/.venv/lib/python3.12/site-packages/torch/cuda/__init__.py:182: UserWarning: CUDA initialization: CUDA unknown error - this may be due to an incorrectly set up environment, e.g. changing env variable CUDA_VISIBLE_DEVICES after program start. Setting the available devices to be zero. (Triggered internally at /pytorch/c10/cuda/CUDAFunctions.cpp:119.)
  return torch._C._cuda_getDeviceCount() > 0
False
1
root@dd9e44eb4d34:/app# export CUDA_VISIBLE_DEVICES=6
python3 -c "import torch; print(torch.cuda.is_available()); print(torch.cuda.device_count())"
/app/packages/api/.venv/lib/python3.12/site-packages/torch/cuda/__init__.py:182: UserWarning: CUDA initialization: CUDA unknown error - this may be due to an incorrectly set up environment, e.g. changing env variable CUDA_VISIBLE_DEVICES after program start. Setting the available devices to be zero. (Triggered internally at /pytorch/c10/cuda/CUDAFunctions.cpp:119.)
  return torch._C._cuda_getDeviceCount() > 0
False
0
root@dd9e44eb4d34:/app# CUDA_VISIBLE_DEVICES=6 python3 -c "import torch; print(torch.cuda.is_available()); print(torch.cuda.device_count())"
/app/packages/api/.venv/lib/python3.12/site-packages/torch/cuda/__init__.py:182: UserWarning: CUDA initialization: CUDA unknown error - this may be due to an incorrectly set up environment, e.g. changing env variable CUDA_VISIBLE_DEVICES after program start. Setting the available devices to be zero. (Triggered internally at /pytorch/c10/cuda/CUDAFunctions.cpp:119.)
  return torch._C._cuda_getDeviceCount() > 0
False
0
root@dd9e44eb4d34:/app# python3 -c "
import os
os.environ['CUDA_VISIBLE_DEVICES'] = '6'
import torch
print('CUDA available:', torch.cuda.is_available())
print('Device count:', torch.cuda.device_count())
if torch.cuda.is_available():
    print('Device name:', torch.cuda.get_device_name(0))
"
/app/packages/api/.venv/lib/python3.12/site-packages/torch/cuda/__init__.py:182: UserWarning: CUDA initialization: CUDA unknown error - this may be due to an incorrectly set up environment, e.g. changing env variable CUDA_VISIBLE_DEVICES after program start. Setting the available devices to be zero. (Triggered internally at /pytorch/c10/cuda/CUDAFunctions.cpp:119.)
  return torch._C._cuda_getDeviceCount() > 0
CUDA available: False
Device count: 0
root@dd9e44eb4d34:/app# Connection to 100.65.19.34 closed.
Connection to ssh.runpod.io closed.
niktverd@niktverd-osx gonka-pow % ssh 0ug08d9byqj5kf-64411322@ssh.runpod.io -i ~/.ssh/id_ed25519

-- RUNPOD.IO --
Enjoy your Pod #0ug08d9byqj5kf ^_^

root@d34fcbd9dd10:/app# ls -la /dev/nvidia*
crw-rw-rw- 1 nobody nogroup 195, 254 Nov 11 14:17 /dev/nvidia-modeset
crw-rw-rw- 1 nobody nogroup 510,   0 Nov 11 14:17 /dev/nvidia-uvm
crw-rw-rw- 1 nobody nogroup 510,   1 Nov 11 14:17 /dev/nvidia-uvm-tools
crw-rw-rw- 1 nobody nogroup 195,   6 Nov 11 14:17 /dev/nvidia6
crw-rw-rw- 1 nobody nogroup 195, 255 Nov 11 14:17 /dev/nvidiactl

/dev/nvidia-caps:
total 0
drwxr-xr-x 2 root root  40 Jan 23 02:18 .
drwxr-xr-x 7 root root 480 Jan 23 02:18 ..
root@d34fcbd9dd10:/app# python3 -c "import torch; print(torch.version.cuda); print(torch.cuda.is_available()); print(torch.cuda.device_count())"
12.6
/app/packages/api/.venv/lib/python3.12/site-packages/torch/cuda/__init__.py:182: UserWarning: CUDA initialization: CUDA unknown error - this may be due to an incorrectly set up environment, e.g. changing env variable CUDA_VISIBLE_DEVICES after program start. Setting the available devices to be zero. (Triggered internally at /pytorch/c10/cuda/CUDAFunctions.cpp:119.)
  return torch._C._cuda_getDeviceCount() > 0
False
0
root@d34fcbd9dd10:/app# env | grep CUDA_VISIBLE_DEVICES
CUDA_VISIBLE_DEVICES=6
root@d34fcbd9dd10:/app#