│ 214  │   │ 215  │   │ 216 async def load(model: InferenceModel) -> │   │ │   │ /usr/local/lib/python3.11/concurrent/futures/th │   │ read.py:58 in run │   │ │   │ /usr/src/immich_ml/models/base.py:61 in predict │   │ │   │  58 │ │ self.load() │   │  59 │ │ if model_kwargs: │   │  60 │ │ │ self.configure(**model_kwargs │   │ ❱  61 │ │ return self._predict(*inputs, **m │   │  62 │  │   │  63 │ @abstractmethod │   │  64 │ def _predict(self, *inputs: Any, **mo │   │ │   │ /usr/src/immich_ml/models/clip/visual.py:31 in │   │ _predict │   │ │   │ 28 │  │   │ 29 │ def _predict(self, inputs: Image.Image │   │ 30 │ │ image = decode_pil(inputs) │   │ ❱ 31 │ │ res: NDArray[np.float32] = self.se │   │ 32 │ │ return serialize_np_array(res) │   │ 33 │  │   │ 34 │ @abstractmethod │   │ │   │ /usr/src/immich_ml/sessions/ort.py:49 in run │   │ │   │  46 │ │ input_feed: dict[str, NDArray[np. │   │  47 │ │ run_options: Any = None, │   │  48 │ ) -> list[NDArray[np.float32]]: │   │ ❱  49 │ │ outputs: list[NDArray[np.float32] │   │ run_options) │   │  50 │ │ return outputs │   │  51 │  │   │  52 │ @property │   │ │   │ /opt/venv/lib/python3.11/site-packages/onnxrunt │   │ ime/capi/onnxruntime_inference_collection.py:22 │   │ 0 in run │   │ │   │  217 │ │ if not output_names: │   │  218 │ │ │ output_names = [output.name │   │  219 │ │ try: │   │ ❱  220 │ │ │ return self._sess.run(output │   │  221 │ │ except C.EPFail as err: │   │  222 │ │ │ if self._enable_fallback: │   │  223 │ │ │ │ print(f"EP Error: {err!s │   ╰─────────────────────────────────────────────────╯   Fail: [ONNXRuntimeError] : 1 : FAIL : Non-zero   status code returned while running Conv node.   Name:'/visual/trunk/patch_embed/proj/Conv' Status   Message: CUDNN failure 5000:   CUDNN_STATUS_EXECUTION_FAILED ; GPU=0 ;   hostname=4dcfeb265dd4 ;   file=/onnxruntime_src/onnxruntime/core/providers/cu   da/nn/conv.cc ; line=455 ;   expr=cudnnConvolutionForward(cudnn_handle, &alpha,   s_.x_tensor, s_.x_data, s_.w_desc, s_.w_data,   s_.conv_desc, s_.algo, workspace.get(),   s_.workspace_bytes, &beta, s_.y_tensor, s_.y_data); 2025-10-16 02:08:32.594088898 [E:onnxruntime:Default, cuda_call.cc:118 CudaCall] CUDNN failure 5000: CUDNN_STATUS_EXECUTION_FAILED ; GPU=0 ; hostname=4dcfeb265dd4 ; file=/onnxruntime_src/onnxruntime/core/providers/cuda/nn/conv.cc ; line=455 ; expr=cudnnConvolutionForward(cudnn_handle, &alpha, s_.x_tensor, s_.x_data, s_.w_desc, s_.w_data, s_.conv_desc, s_.algo, workspace.get(), s_.workspace_bytes, &beta, s_.y_tensor, s_.y_data);  2025-10-16 02:08:32.594196639 [E:onnxruntime:, sequential_executor.cc:516 ExecuteKernel] Non-zero status code returned while running Conv node. Name:'/visual/trunk/patch_embed/proj/Conv' Status Message: CUDNN failure 5000: CUDNN_STATUS_EXECUTION_FAILED ; GPU=0 ; hostname=4dcfeb265dd4 ; file=/onnxruntime_src/onnxruntime/core/providers/cuda/nn/conv.cc ; line=455 ; expr=cudnnConvolutionForward(cudnn_handle, &alpha, s_.x_tensor, s_.x_data, s_.w_desc, s_.w_data, s_.conv_desc, s_.algo, workspace.get(), s_.workspace_bytes, &beta, s_.y_tensor, s_.y_data);  2025-10-16 02:08:32.644365998 [E:onnxruntime:Default, cuda_call.cc:118 CudaCall] CUDNN failure 5000: CUDNN_STATUS_EXECUTION_FAILED ; GPU=0 ; hostname=4dcfeb265dd4 ; file=/onnxruntime_src/onnxruntime/core/providers/cuda/nn/conv.cc ; line=455 ; expr=cudnnConvolutionForward(cudnn_handle, &alpha, s_.x_tensor, s_.x_data, s_.w_desc, s_.w_data, s_.conv_desc, s_.algo, workspace.get(), s_.workspace_bytes, &beta, s_.y_tensor, s_.y_data);  2025-10-16 02:08:32.644438682 [E:onnxruntime:, sequential_executor.cc:516 ExecuteKernel] Non-zero status code returned while running Conv node. Name:'/visual/trunk/patch_embed/proj/Conv' Status Message: CUDNN failure 5000: CUDNN_STATUS_EXECUTION_FAILED ; GPU=0 ; hostname=4dcfeb265dd4 ; file=/onnxruntime_src/onnxruntime/core/providers/cuda/nn/conv.cc ; line=455 ; expr=cudnnConvolutionForward(cudnn_handle, &alpha, s_.x_tensor, s_.x_data, s_.w_desc, s_.w_data, s_.conv_desc, s_.algo, workspace.get(), s_.workspace_bytes, &beta, s_.y_tensor, s_.y_data);  [10/16/25 02:08:32] ERROR  Exception in ASGI application     ╭─────── Traceback (most recent call last) ───────╮   │ /usr/src/immich_ml/main.py:177 in predict │   │ │   │ 174 │ │ inputs = text │   │ 175 │ else: │   │ 176 │ │ raise HTTPException(400, "Either  │   │ ❱ 177 │ response = await run_inference(inputs │   │ 178 │ return ORJSONResponse(response) │   │ 179  │   │ 180  │   │ │   │ /usr/src/immich_ml/main.py:200 in run_inference │   │ │   │ 197 │ │ response[entry["task"]] = output │   │ 198 │  │   │ 199 │ without_deps, with_deps = entries │   │ ❱ 200 │ await asyncio.gather(*[_run_inference │   │ 201 │ if with_deps: │   │ 202 │ │ await asyncio.gather(*[_run_infer │   │ 203 │ if isinstance(payload, Image): │   │ │   │ /usr/src/immich_ml/main.py:195 in │   │ _run_inference │   │ │   │ 192 │ │ │ │ message = f"Task {entry[' │   │ output of {dep}" │   │ 193 │ │ │ │ raise HTTPException(400, │   │ 194 │ │ model = await load(model) │   │ ❱ 195 │ │ output = await run(model.predict, │   │ 196 │ │ outputs[model.identity] = output │   │ 197 │ │ response[entry["task"]] = output │   │ 198  │   │ │   │ /usr/src/immich_ml/main.py:213 in run │   │ │   │ 210 │ if thread_pool is None: │   │ 211 │ │ return func(*args, **kwargs) │   │ 212 │ partial_func = partial(func, *args, * │   │ ❱ 213 │ return await asyncio.get_running_loop │   │ 214  │   │ 215  │   │ 216 async def load(model: InferenceModel) -> │   │ │   │ /usr/local/lib/python3.11/concurrent/futures/th │   │ read.py:58 in run │   │ │   │ /usr/src/immich_ml/models/base.py:61 in predict │   │ │   │  58 │ │ self.load() │   │  59 │ │ if model_kwargs: │   │  60 │ │ │ self.configure(**model_kwargs │   │ ❱  61 │ │ return self._predict(*inputs, **m │   │  62 │  │   │  63 │ @abstractmethod │   │  64 │ def _predict(self, *inputs: Any, **mo │   │ │   │ /usr/src/immich_ml/models/clip/visual.py:31 in │   │ _predict │   │ │   │ 28 │  │   │ 29 │ def _predict(self, inputs: Image.Image │   │ 30 │ │ image = decode_pil(inputs) │   │ ❱ 31 │ │ res: NDArray[np.float32] = self.se │   │ 32 │ │ return serialize_np_array(res) │   │ 33 │  │   │ 34 │ @abstractmethod │   │ │   │ /usr/src/immich_ml/sessions/ort.py:49 in run │   │ │   │  46 │ │ input_feed: dict[str, NDArray[np. │   │  47 │ │ run_options: Any = None, │   │  48 │ ) -> list[NDArray[np.float32]]: │   │ ❱  49 │ │ outputs: list[NDArray[np.float32] │   │ run_options) │   │  50 │ │ return outputs │   │  51 │  │   │  52 │ @property │   │ │   │ /opt/venv/lib/python3.11/site-packages/onnxrunt │   │ ime/capi/onnxruntime_inference_collection.py:22 │   │ 0 in run │   │ │   │  217 │ │ if not output_names: │   │  218 │ │ │ output_names = [output.name │   │  219 │ │ try: │   │ ❱  220 │ │ │ return self._sess.run(output │   │  221 │ │ except C.EPFail as err: │   │  222 │ │ │ if self._enable_fallback: │   │  223 │ │ │ │ print(f"EP Error: {err!s │   ╰─────────────────────────────────────────────────╯   Fail: [ONNXRuntimeError] : 1 : FAIL : Non-zero   status code returned while running Conv node.   Name:'/visual/trunk/patch_embed/proj/Conv' Status   Message: CUDNN failure 5000:   CUDNN_STATUS_EXECUTION_FAILED ; GPU=0 ;   hostname=4dcfeb265dd4 ;   file=/onnxruntime_src/onnxruntime/core/providers/cu   da/nn/conv.cc ; line=455 ;   expr=cudnnConvolutionForward(cudnn_handle, &alpha,   s_.x_tensor, s_.x_data, s_.w_desc, s_.w_data,   s_.conv_desc, s_.algo, workspace.get(),   s_.workspace_bytes, &beta, s_.y_tensor, s_.y_data); [10/16/25 02:08:32] ERROR  Exception in ASGI application     ╭─────── Traceback (most recent call last) ───────╮   │ /usr/src/immich_ml/main.py:177 in predict │   │ │   │ 174 │ │ inputs = text │   │ 175 │ else: │   │ 176 │ │ raise HTTPException(400, "Either  │   │ ❱ 177 │ response = await run_inference(inputs │   │ 178 │ return ORJSONResponse(response) │   │ 179  │   │ 180  │   │ │   │ /usr/src/immich_ml/main.py:200 in run_inference │   │ │   │ 197 │ │ response[entry["task"]] = output │   │ 198 │  │   │ 199 │ without_deps, with_deps = entries │   │ ❱ 200 │ await asyncio.gather(*[_run_inference │   │ 201 │ if with_deps: │   │ 202 │ │ await asyncio.gather(*[_run_infer │   │ 203 │ if isinstance(payload, Image): │   │ │   │ /usr/src/immich_ml/main.py:195 in │   │ _run_inference │   │ │   │ 192 │ │ │ │ message = f"Task {entry[' │   │ output of {dep}" │   │ 193 │ │ │ │ raise HTTPException(400, │   │ 194 │ │ model = await load(model) │   │ ❱ 195 │ │ output = await run(model.predict, │   │ 196 │ │ outputs[model.identity] = output │   │ 197 │ │ response[entry["task"]] = output │   │ 198  │   │ │   │ /usr/src/immich_ml/main.py:213 in run │   │ │   │ 210 │ if thread_pool is None: │   │ 211 │ │ return func(*args, **kwargs) │   │ 212 │ partial_func = partial(func, *args, * │   │ ❱ 213 │ return await asyncio.get_running_loop │   │ 214  │   │ 215  │   │ 216 async def load(model: InferenceModel) -> │   │ │   │ /usr/local/lib/python3.11/concurrent/futures/th │   │ read.py:58 in run │   │ │   │ /usr/src/immich_ml/models/base.py:61 in predict │   │ │   │  58 │ │ self.load() │   │  59 │ │ if model_kwargs: │   │  60 │ │ │ self.configure(**model_kwargs │   │ ❱  61 │ │ return self._predict(*inputs, **m │   │  62 │  │   │  63 │ @abstractmethod │   │  64 │ def _predict(self, *inputs: Any, **mo │   │ │   │ /usr/src/immich_ml/models/clip/visual.py:31 in │   │ _predict │   │ │   │ 28 │  │   │ 29 │ def _predict(self, inputs: Image.Image │   │ 30 │ │ image = decode_pil(inputs) │   │ ❱ 31 │ │ res: NDArray[np.float32] = self.se │   │ 32 │ │ return serialize_np_array(res) │   │ 33 │  │   │ 34 │ @abstractmethod │   │ │   │ /usr/src/immich_ml/sessions/ort.py:49 in run │   │ │   │  46 │ │ input_feed: dict[str, NDArray[np. │   │  47 │ │ run_options: Any = None, │   │  48 │ ) -> list[NDArray[np.float32]]: │   │ ❱  49 │ │ outputs: list[NDArray[np.float32] │   │ run_options) │   │  50 │ │ return outputs │   │  51 │  │   │  52 │ @property │   │ │   │ /opt/venv/lib/python3.11/site-packages/onnxrunt │   │ ime/capi/onnxruntime_inference_collection.py:22 │   │ 0 in run │   │ │   │  217 │ │ if not output_names: │   │  218 │ │ │ output_names = [output.name │   │  219 │ │ try: │   │ ❱  220 │ │ │ return self._sess.run(output │   │  221 │ │ except C.EPFail as err: │   │  222 │ │ │ if self._enable_fallback: │   │  223 │ │ │ │ print(f"EP Error: {err!s │   ╰─────────────────────────────────────────────────╯   Fail: [ONNXRuntimeError] : 1 : FAIL : Non-zero   status code returned while running Conv node.   Name:'/visual/trunk/patch_embed/proj/Conv' Status   Message: CUDNN failure 5000:   CUDNN_STATUS_EXECUTION_FAILED ; GPU=0 ;   hostname=4dcfeb265dd4 ;   file=/onnxruntime_src/onnxruntime/core/providers/cu   da/nn/conv.cc ; line=455 ;   expr=cudnnConvolutionForward(cudnn_handle, &alpha,   s_.x_tensor, s_.x_data, s_.w_desc, s_.w_data,   s_.conv_desc, s_.algo, workspace.get(),   s_.workspace_bytes, &beta, s_.y_tensor, s_.y_data); 2025-10-16 02:08:33.650519015 [E:onnxruntime:Default, cuda_call.cc:118 CudaCall] CUDNN failure 5000: CUDNN_STATUS_EXECUTION_FAILED ; GPU=0 ; hostname=4dcfeb265dd4 ; file=/onnxruntime_src/onnxruntime/core/providers/cuda/nn/conv.cc ; line=455 ; expr=cudnnConvolutionForward(cudnn_handle, &alpha, s_.x_tensor, s_.x_data, s_.w_desc, s_.w_data, s_.conv_desc, s_.algo, workspace.get(), s_.workspace_bytes, &beta, s_.y_tensor, s_.y_data);  2025-10-16 02:08:33.650626149 [E:onnxruntime:, sequential_executor.cc:516 ExecuteKernel] Non-zero status code returned while running Conv node. Name:'/visual/trunk/patch_embed/proj/Conv' Status Message: CUDNN failure 5000: CUDNN_STATUS_EXECUTION_FAILED ; GPU=0 ; hostname=4dcfeb265dd4 ; file=/onnxruntime_src/onnxruntime/core/providers/cuda/nn/conv.cc ; line=455 ; expr=cudnnConvolutionForward(cudnn_handle, &alpha, s_.x_tensor, s_.x_data, s_.w_desc, s_.w_data, s_.conv_desc, s_.algo, workspace.get(), s_.workspace_bytes, &beta, s_.y_tensor, s_.y_data);  2025-10-16 02:08:33.674274893 [E:onnxruntime:Default, cuda_call.cc:118 CudaCall] CUDNN failure 5000: CUDNN_STATUS_EXECUTION_FAILED ; GPU=0 ; hostname=4dcfeb265dd4 ; file=/onnxruntime_src/onnxruntime/core/providers/cuda/nn/conv.cc ; line=455 ; expr=cudnnConvolutionForward(cudnn_handle, &alpha, s_.x_tensor, s_.x_data, s_.w_desc, s_.w_data, s_.conv_desc, s_.algo, workspace.get(), s_.workspace_bytes, &beta, s_.y_tensor, s_.y_data);  2025-10-16 02:08:33.674352917 [E:onnxruntime:, sequential_executor.cc:516 ExecuteKernel] Non-zero status code returned while running Conv node. Name:'/visual/trunk/patch_embed/proj/Conv' Status Message: CUDNN failure 5000: CUDNN_STATUS_EXECUTION_FAILED ; GPU=0 ; hostname=4dcfeb265dd4 ; file=/onnxruntime_src/onnxruntime/core/providers/cuda/nn/conv.cc ; line=455 ; expr=cudnnConvolutionForward(cudnn_handle, &alpha, s_.x_tensor, s_.x_data, s_.w_desc, s_.w_data, s_.conv_desc, s_.algo, workspace.get(), s_.workspace_bytes, &beta, s_.y_tensor, s_.y_data);  [10/16/25 02:08:33] ERROR  Exception in ASGI application     ╭─────── Traceback (most recent call last) ───────╮   │ /usr/src/immich_ml/main.py:177 in predict │   │ │   │ 174 │ │ inputs = text │   │ 175 │ else: │   │ 176 │ │ raise HTTPException(400, "Either  │   │ ❱ 177 │ response = await run_inference(inputs │   │ 178 │ return ORJSONResponse(response) │   │ 179  │   │ 180  │   │ │   │ /usr/src/immich_ml/main.py:200 in run_inference │   │ │   │ 197 │ │ response[entry["task"]] = output │   │ 198 │  │   │ 199 │ without_deps, with_deps = entries │   │ ❱ 200 │ await asyncio.gather(*[_run_inference │   │ 201 │ if with_deps: │   │ 202 │ │ await asyncio.gather(*[_run_infer │   │ 203 │ if isinstance(payload, Image): │   │ │   │ /usr/src/immich_ml/main.py:195 in │   │ _run_inference │   │ │   │ 192 │ │ │ │ message = f"Task {entry[' │   │ output of {dep}" │   │ 193 │ │ │ │ raise HTTPException(400, │   │ 194 │ │ model = await load(model) │   │ ❱ 195 │ │ output = await run(model.predict, │   │ 196 │ │ outputs[model.identity] = output │   │ 197 │ │ response[entry["task"]] = output │   │ 198  │   │ │   │ /usr/src/immich_ml/main.py:213 in run │   │ │   │ 210 │ if thread_pool is None: │   │ 211 │ │ return func(*args, **kwargs) │   │ 212 │ partial_func = partial(func, *args, * │   │ ❱ 213 │ return await asyncio.get_running_loop │   │ 214  │   │ 215  │   │ 216 async def load(model: InferenceModel) -> │   │ │   │ /usr/local/lib/python3.11/concurrent/futures/th │   │ read.py:58 in run │   │ │   │ /usr/src/immich_ml/models/base.py:61 in predict │   │ │   │  58 │ │ self.load() │   │  59 │ │ if model_kwargs: │   │  60 │ │ │ self.configure(**model_kwargs │   │ ❱  61 │ │ return self._predict(*inputs, **m │   │  62 │  │   │  63 │ @abstractmethod │   │  64 │ def _predict(self, *inputs: Any, **mo │   │ │   │ /usr/src/immich_ml/models/clip/visual.py:31 in │   │ _predict │   │ │   │ 28 │  │   │ 29 │ def _predict(self, inputs: Image.Image │   │ 30 │ │ image = decode_pil(inputs) │   │ ❱ 31 │ │ res: NDArray[np.float32] = self.se │   │ 32 │ │ return serialize_np_array(res) │   │ 33 │  │   │ 34 │ @abstractmethod │   │ │   │ /usr/src/immich_ml/sessions/ort.py:49 in run │   │ │   │  46 │ │ input_feed: dict[str, NDArray[np. │   │  47 │ │ run_options: Any = None, │   │  48 │ ) -> list[NDArray[np.float32]]: │   │ ❱  49 │ │ outputs: list[NDArray[np.float32] │   │ run_options) │   │  50 │ │ return outputs │   │  51 │  │   │  52 │ @property │   │ │   │ /opt/venv/lib/python3.11/site-packages/onnxrunt │   │ ime/capi/onnxruntime_inference_collection.py:22 │   │ 0 in run │   │ │   │  217 │ │ if not output_names: │   │  218 │ │ │ output_names = [output.name │   │  219 │ │ try: │   │ ❱  220 │ │ │ return self._sess.run(output │   │  221 │ │ except C.EPFail as err: │   │  222 │ │ │ if self._enable_fallback: │   │  223 │ │ │ │ print(f"EP Error: {err!s │   ╰─────────────────────────────────────────────────╯   Fail: [ONNXRuntimeError] : 1 : FAIL : Non-zero   status code returned while running Conv node.   Name:'/visual/trunk/patch_embed/proj/Conv' Status   Message: CUDNN failure 5000:   CUDNN_STATUS_EXECUTION_FAILED ; GPU=0 ;   hostname=4dcfeb265dd4 ;   file=/onnxruntime_src/onnxruntime/core/providers/cu   da/nn/conv.cc ; line=455 ;   expr=cudnnConvolutionForward(cudnn_handle, &alpha,   s_.x_tensor, s_.x_data, s_.w_desc, s_.w_data,   s_.conv_desc, s_.algo, workspace.get(),   s_.workspace_bytes, &beta, s_.y_tensor, s_.y_data); [10/16/25 02:08:34] ERROR  Exception in ASGI application     ╭─────── Traceback (most recent call last) ───────╮   │ /usr/src/immich_ml/main.py:177 in predict │   │ │   │ 174 │ │ inputs = text │   │ 175 │ else: │   │ 176 │ │ raise HTTPException(400, "Either  │   │ ❱ 177 │ response = await run_inference(inputs │   │ 178 │ return ORJSONResponse(response) │   │ 179  │   │ 180  │   │ │   │ /usr/src/immich_ml/main.py:200 in run_inference │   │ │   │ 197 │ │ response[entry["task"]] = output │   │ 198 │  │   │ 199 │ without_deps, with_deps = entries │   │ ❱ 200 │ await asyncio.gather(*[_run_inference │   │ 201 │ if with_deps: │   │ 202 │ │ await asyncio.gather(*[_run_infer │   │ 203 │ if isinstance(payload, Image): │   │ │   │ /usr/src/immich_ml/main.py:195 in │   │ _run_inference │   │ │   │ 192 │ │ │ │ message = f"Task {entry[' │   │ output of {dep}" │   │ 193 │ │ │ │ raise HTTPException(400, │   │ 194 │ │ model = await load(model) │   │ ❱ 195 │ │ output = await run(model.predict, │   │ 196 │ │ outputs[model.identity] = output │   │ 197 │ │ response[entry["task"]] = output │   │ 198  │   │ │   │ /usr/src/immich_ml/main.py:213 in run │   │ │   │ 210 │ if thread_pool is None: │   │ 211 │ │ return func(*args, **kwargs) │   │ 212 │ partial_func = partial(func, *args, * │   │ ❱ 213 │ return await asyncio.get_running_loop │   │ 214  │   │ 215  │   │ 216 async def load(model: InferenceModel) -> │   │ │   │ /usr/local/lib/python3.11/concurrent/futures/th │   │ read.py:58 in run │   │ │   │ /usr/src/immich_ml/models/base.py:61 in predict │   │ │   │  58 │ │ self.load() │   │  59 │ │ if model_kwargs: │   │  60 │ │ │ self.configure(**model_kwargs │   │ ❱  61 │ │ return self._predict(*inputs, **m │   │  62 │  │   │  63 │ @abstractmethod │   │  64 │ def _predict(self, *inputs: Any, **mo │   │ │   │ /usr/src/immich_ml/models/clip/visual.py:31 in │   │ _predict │   │ │   │ 28 │  │   │ 29 │ def _predict(self, inputs: Image.Image │   │ 30 │ │ image = decode_pil(inputs) │   │ ❱ 31 │ │ res: NDArray[np.float32] = self.se │   │ 32 │ │ return serialize_np_array(res) │   │ 33 │  │   │ 34 │ @abstractmethod │   │ │   │ /usr/src/immich_ml/sessions/ort.py:49 in run │   │ │   │  46 │ │ input_feed: dict[str, NDArray[np. │   │  47 │ │ run_options: Any = None, │   │  48 │ ) -> list[NDArray[np.float32]]: │   │ ❱  49 │ │ outputs: list[NDArray[np.float32] │   │ run_options) │   │  50 │ │ return outputs │   │  51 │  │   │  52 │ @property │   │ │   │ /opt/venv/lib/python3.11/site-packages/onnxrunt │   │ ime/capi/onnxruntime_inference_collection.py:22 │   │ 0 in run │   │ │   │  217 │ │ if not output_names: │   │  218 │ │ │ output_names = [output.name │   │  219 │ │ try: │   │ ❱  220 │ │ │ return self._sess.run(output │   │  221 │ │ except C.EPFail as err: │   │  222 │ │ │ if self._enable_fallback: │   │  223 │ │ │ │ print(f"EP Error: {err!s │   ╰─────────────────────────────────────────────────╯   Fail: [ONNXRuntimeError] : 1 : FAIL : Non-zero   status code returned while running Conv node.   Name:'/visual/trunk/patch_embed/proj/Conv' Status   Message: CUDNN failure 5000:   CUDNN_STATUS_EXECUTION_FAILED ; GPU=0 ;   hostname=4dcfeb265dd4 ;   file=/onnxruntime_src/onnxruntime/core/providers/cu   da/nn/conv.cc ; line=455 ;   expr=cudnnConvolutionForward(cudnn_handle, &alpha,   s_.x_tensor, s_.x_data, s_.w_desc, s_.w_data,   s_.conv_desc, s_.algo, workspace.get(),   s_.workspace_bytes, &beta, s_.y_tensor, s_.y_data); [10/16/25 02:13:39] INFO  Shutting down due to inactivity. [10/16/25 02:13:39] INFO  Shutting down [10/16/25 02:13:39] INFO  Waiting for application shutdown. [10/16/25 02:13:39] INFO  Application shutdown complete. [10/16/25 02:13:39] INFO  Finished server process [9] [10/16/25 02:13:39] ERROR  Worker (pid:9) was sent SIGINT! [10/16/25 02:13:39] INFO  Booting worker with pid: 185 [10/16/25 02:13:43] INFO  Started server process [185] [10/16/25 02:13:43] INFO  Waiting for application startup. [10/16/25 02:13:43] INFO  Created in-memory cache with unloading after 300s   of inactivity. [10/16/25 02:13:43] INFO  Initialized request thread pool with 16 threads. [10/16/25 02:13:43] INFO  Application startup complete.