immich_machine_learning | [10/07/25 11:58:53] INFO Downloading textual model 'ViT-B-32__openai'. This immich_machine_learning | may take a while. Fetching 11 files: 36%|███▋ | 4/11 [00:00<00:00, 15.28it/s] immich_machine_learning | [10/07/25 11:58:53] ERROR Exception in ASGI application immich_machine_learning | immich_machine_learning | ╭─────── Traceback (most recent call last) ───────╮ immich_machine_learning | │ /usr/src/immich_ml/main.py:177 in predict │ immich_server | [Nest] 31 - 10/07/2025, 11:58:53 AM WARN [Api:MachineLearningRepository~41xvt1f8] Machine learning request to "http://immich-machine-learning:3003" failed with status 500: Internal Server Error immich_machine_learning | │ │ immich_server | [Nest] 31 - 10/07/2025, 11:58:53 AM LOG [Api:MachineLearningRepository~41xvt1f8] Machine learning server became unhealthy (http://immich-machine-learning:3003). immich_machine_learning | │ 174 │ │ inputs = text │ immich_server | [Nest] 31 - 10/07/2025, 11:58:53 AM ERROR [Api:ErrorInterceptor~41xvt1f8] Unknown error: Error: Machine learning request '{"clip":{"textual":{"modelName":"ViT-B-32__openai","options":{"language":"cs"}}}}' failed for all URLs immich_machine_learning | │ 175 │ else: │ immich_server | Error: Machine learning request '{"clip":{"textual":{"modelName":"ViT-B-32__openai","options":{"language":"cs"}}}}' failed for all URLs immich_machine_learning | │ 176 │ │ raise HTTPException(400, "Either │ immich_server | at MachineLearningRepository.predict (/usr/src/app/server/dist/repositories/machine-learning.repository.js:115:15) immich_machine_learning | │ ❱ 177 │ response = await run_inference(inputs │ immich_server | at process.processTicksAndRejections (node:internal/process/task_queues:105:5) immich_machine_learning | │ 178 │ return ORJSONResponse(response) │ immich_server | at async MachineLearningRepository.encodeText (/usr/src/app/server/dist/repositories/machine-learning.repository.js:138:26) immich_machine_learning | │ 179 │ immich_server | at async SearchService.searchSmart (/usr/src/app/server/dist/services/search.service.js:94:29) immich_machine_learning | │ 180 │ immich_machine_learning | │ │ immich_machine_learning | │ /usr/src/immich_ml/main.py:200 in run_inference │ immich_machine_learning | │ │ immich_machine_learning | │ 197 │ │ response[entry["task"]] = output │ immich_machine_learning | │ 198 │ │ immich_machine_learning | │ 199 │ without_deps, with_deps = entries │ immich_machine_learning | │ ❱ 200 │ await asyncio.gather(*[_run_inference │ immich_machine_learning | │ 201 │ if with_deps: │ immich_machine_learning | │ 202 │ │ await asyncio.gather(*[_run_infer │ immich_machine_learning | │ 203 │ if isinstance(payload, Image): │ immich_machine_learning | │ │ immich_machine_learning | │ /usr/src/immich_ml/main.py:194 in │ immich_machine_learning | │ _run_inference │ immich_machine_learning | │ │ immich_machine_learning | │ 191 │ │ │ except KeyError: │ immich_machine_learning | │ 192 │ │ │ │ message = f"Task {entry[' │ immich_machine_learning | │ output of {dep}" │ immich_machine_learning | │ 193 │ │ │ │ raise HTTPException(400, │ immich_machine_learning | │ ❱ 194 │ │ model = await load(model) │ immich_machine_learning | │ 195 │ │ output = await run(model.predict, │ immich_machine_learning | │ 196 │ │ outputs[model.identity] = output │ immich_machine_learning | │ 197 │ │ response[entry["task"]] = output │ immich_machine_learning | │ │ immich_machine_learning | │ /usr/src/immich_ml/main.py:238 in load │ immich_machine_learning | │ │ immich_machine_learning | │ 235 │ │ return model │ immich_machine_learning | │ 236 │ │ immich_machine_learning | │ 237 │ try: │ immich_machine_learning | │ ❱ 238 │ │ return await run(_load, model) │ immich_machine_learning | │ 239 │ except (OSError, InvalidProtobuf, Bad │ immich_machine_learning | │ 240 │ │ log.warning(f"Failed to load {mod │ immich_machine_learning | │ '{model.model_name}'. Clearing cache.") │ immich_machine_learning | │ 241 │ │ model.clear_cache() │ immich_machine_learning | │ │ immich_machine_learning | │ /usr/src/immich_ml/main.py:213 in run │ immich_machine_learning | │ │ immich_machine_learning | │ 210 │ if thread_pool is None: │ immich_machine_learning | │ 211 │ │ return func(*args, **kwargs) │ immich_machine_learning | │ 212 │ partial_func = partial(func, *args, * │ immich_machine_learning | │ ❱ 213 │ return await asyncio.get_running_loop │ immich_machine_learning | │ 214 │ immich_machine_learning | │ 215 │ immich_machine_learning | │ 216 async def load(model: InferenceModel) -> │ immich_machine_learning | │ │ immich_machine_learning | │ /usr/local/lib/python3.11/concurrent/futures/th │ immich_machine_learning | │ read.py:58 in run │ immich_machine_learning | │ │ immich_machine_learning | │ /usr/src/immich_ml/main.py:225 in _load │ immich_machine_learning | │ │ immich_machine_learning | │ 222 │ │ │ raise HTTPException(500, f"Fa │ immich_machine_learning | │ 223 │ │ with lock: │ immich_machine_learning | │ 224 │ │ │ try: │ immich_machine_learning | │ ❱ 225 │ │ │ │ model.load() │ immich_machine_learning | │ 226 │ │ │ except FileNotFoundError as e │ immich_machine_learning | │ 227 │ │ │ │ if model.model_format == │ immich_machine_learning | │ 228 │ │ │ │ │ raise e │ immich_machine_learning | │ │ immich_machine_learning | │ /usr/src/immich_ml/models/base.py:51 in load │ immich_machine_learning | │ │ immich_machine_learning | │ 48 │ │ │ return │ immich_machine_learning | │ 49 │ │ self.load_attempts += 1 │ immich_machine_learning | │ 50 │ │ │ immich_machine_learning | │ ❱ 51 │ │ self.download() │ immich_machine_learning | │ 52 │ │ attempt = f"Attempt #{self.load_a │ immich_machine_learning | │ else "Loading" │ immich_machine_learning | │ 53 │ │ log.info(f"{attempt} {self.model_ │ immich_machine_learning | │ '{self.model_name}' to memory") │ immich_machine_learning | │ 54 │ │ self.session = self._load() │ immich_machine_learning | │ │ immich_machine_learning | │ /usr/src/immich_ml/models/base.py:44 in │ immich_machine_learning | │ download │ immich_machine_learning | │ │ immich_machine_learning | │ 41 │ │ │ log.info( │ immich_machine_learning | │ 42 │ │ │ │ f"Downloading {self.model │ immich_machine_learning | │ '{self.model_name}'. This may take a whil │ immich_machine_learning | │ 43 │ │ │ ) │ immich_machine_learning | │ ❱ 44 │ │ │ self._download() │ immich_machine_learning | │ 45 │ │ immich_machine_learning | │ 46 │ def load(self) -> None: │ immich_machine_learning | │ 47 │ │ if self.loaded: │ immich_machine_learning | │ │ immich_machine_learning | │ /usr/src/immich_ml/models/base.py:76 in │ immich_machine_learning | │ _download │ immich_machine_learning | │ │ immich_machine_learning | │ 73 │ │ │ ModelFormat.RKNN: ["*.armnn"] │ immich_machine_learning | │ 74 │ │ } │ immich_machine_learning | │ 75 │ │ │ immich_machine_learning | │ ❱ 76 │ │ snapshot_download( │ immich_machine_learning | │ 77 │ │ │ f"immich-app/{clean_name(self │ immich_machine_learning | │ 78 │ │ │ cache_dir=self.cache_dir, │ immich_machine_learning | │ 79 │ │ │ local_dir=self.cache_dir, │ immich_machine_learning | │ │ immich_machine_learning | │ /opt/venv/lib/python3.11/site-packages/huggingf │ immich_machine_learning | │ ace_hub/utils/_validators.py:114 in _inner_fn │ immich_machine_learning | │ │ immich_machine_learning | │ 111 │ │ if check_use_auth_token: │ immich_machine_learning | │ 112 │ │ │ kwargs = smoothly_deprecate_u │ immich_machine_learning | │ has_token=has_token, kwargs=kwargs) │ immich_machine_learning | │ 113 │ │ │ immich_machine_learning | │ ❱ 114 │ │ return fn(*args, **kwargs) │ immich_machine_learning | │ 115 │ │ immich_machine_learning | │ 116 │ return _inner_fn # type: ignore │ immich_machine_learning | │ 117 │ immich_machine_learning | │ │ immich_machine_learning | │ /opt/venv/lib/python3.11/site-packages/huggingf │ immich_machine_learning | │ ace_hub/_snapshot_download.py:332 in │ immich_machine_learning | │ snapshot_download │ immich_machine_learning | │ │ immich_machine_learning | │ 329 │ │ for file in filtered_repo_files: │ immich_machine_learning | │ 330 │ │ │ _inner_hf_hub_download(file) │ immich_machine_learning | │ 331 │ else: │ immich_machine_learning | │ ❱ 332 │ │ thread_map( │ immich_machine_learning | │ 333 │ │ │ _inner_hf_hub_download, │ immich_machine_learning | │ 334 │ │ │ filtered_repo_files, │ immich_machine_learning | │ 335 │ │ │ desc=tqdm_desc, │ immich_machine_learning | │ │ immich_machine_learning | │ /opt/venv/lib/python3.11/site-packages/tqdm/con │ immich_machine_learning | │ trib/concurrent.py:69 in thread_map │ immich_machine_learning | │ │ immich_machine_learning | │ 66 │ │ [default: max(32, cpu_count() + 4 │ immich_machine_learning | │ 67 │ """ │ immich_machine_learning | │ 68 │ from concurrent.futures import Thread │ immich_machine_learning | │ ❱ 69 │ return _executor_map(ThreadPoolExecut │ immich_machine_learning | │ 70 │ immich_machine_learning | │ 71 │ immich_machine_learning | │ 72 def process_map(fn, *iterables, **tqdm_kw │ immich_machine_learning | │ │ immich_machine_learning | │ /opt/venv/lib/python3.11/site-packages/tqdm/con │ immich_machine_learning | │ trib/concurrent.py:51 in _executor_map │ immich_machine_learning | │ │ immich_machine_learning | │ 48 │ │ # share lock in case workers are │ immich_machine_learning | │ 49 │ │ with PoolExecutor(max_workers=max │ immich_machine_learning | │ 50 │ │ │ │ │ │ initargs=(lk,)) │ immich_machine_learning | │ ❱ 51 │ │ │ return list(tqdm_class(ex.map │ immich_machine_learning | │ **kwargs)) │ immich_machine_learning | │ 52 │ immich_machine_learning | │ 53 │ immich_machine_learning | │ 54 def thread_map(fn, *iterables, **tqdm_kwa │ immich_machine_learning | │ │ immich_machine_learning | │ /opt/venv/lib/python3.11/site-packages/tqdm/std │ immich_machine_learning | │ .py:1181 in __iter__ │ immich_machine_learning | │ │ immich_machine_learning | │ 1178 │ │ time = self._time │ immich_machine_learning | │ 1179 │ │ │ immich_machine_learning | │ 1180 │ │ try: │ immich_machine_learning | │ ❱ 1181 │ │ │ for obj in iterable: │ immich_machine_learning | │ 1182 │ │ │ │ yield obj │ immich_machine_learning | │ 1183 │ │ │ │ # Update and possibly pr │ immich_machine_learning | │ 1184 │ │ │ │ # Note: does not call se │ immich_machine_learning | │ │ immich_machine_learning | │ /usr/local/lib/python3.11/concurrent/futures/_b │ immich_machine_learning | │ ase.py:619 in result_iterator │ immich_machine_learning | │ │ immich_machine_learning | │ /usr/local/lib/python3.11/concurrent/futures/_b │ immich_machine_learning | │ ase.py:317 in _result_or_cancel │ immich_machine_learning | │ │ immich_machine_learning | │ /usr/local/lib/python3.11/concurrent/futures/_b │ immich_machine_learning | │ ase.py:456 in result │ immich_machine_learning | │ │ immich_machine_learning | │ /usr/local/lib/python3.11/concurrent/futures/_b │ immich_machine_learning | │ ase.py:401 in __get_result │ immich_machine_learning | │ │ immich_machine_learning | │ /usr/local/lib/python3.11/concurrent/futures/th │ immich_machine_learning | │ read.py:58 in run │ immich_machine_learning | │ │ immich_machine_learning | │ /opt/venv/lib/python3.11/site-packages/huggingf │ immich_machine_learning | │ ace_hub/_snapshot_download.py:306 in │ immich_machine_learning | │ _inner_hf_hub_download │ immich_machine_learning | │ │ immich_machine_learning | │ 303 │ # so no network call happens if we al │ immich_machine_learning | │ 304 │ # have the file locally. │ immich_machine_learning | │ 305 │ def _inner_hf_hub_download(repo_file: │ immich_machine_learning | │ ❱ 306 │ │ return hf_hub_download( │ immich_machine_learning | │ 307 │ │ │ repo_id, │ immich_machine_learning | │ 308 │ │ │ filename=repo_file, │ immich_machine_learning | │ 309 │ │ │ repo_type=repo_type, │ immich_machine_learning | │ │ immich_machine_learning | │ /opt/venv/lib/python3.11/site-packages/huggingf │ immich_machine_learning | │ ace_hub/utils/_validators.py:114 in _inner_fn │ immich_machine_learning | │ │ immich_machine_learning | │ 111 │ │ if check_use_auth_token: │ immich_machine_learning | │ 112 │ │ │ kwargs = smoothly_deprecate_u │ immich_machine_learning | │ has_token=has_token, kwargs=kwargs) │ immich_machine_learning | │ 113 │ │ │ immich_machine_learning | │ ❱ 114 │ │ return fn(*args, **kwargs) │ immich_machine_learning | │ 115 │ │ immich_machine_learning | │ 116 │ return _inner_fn # type: ignore │ immich_machine_learning | │ 117 │ immich_machine_learning | │ │ immich_machine_learning | │ /opt/venv/lib/python3.11/site-packages/huggingf │ immich_machine_learning | │ ace_hub/file_download.py:990 in hf_hub_download │ immich_machine_learning | │ │ immich_machine_learning | │ 987 │ │ │ │ "For more details, check │ immich_machine_learning | │ https://huggingface.co/docs/huggingface_ │ immich_machine_learning | │ cal-folder." │ immich_machine_learning | │ 988 │ │ │ ) │ immich_machine_learning | │ 989 │ │ │ immich_machine_learning | │ ❱ 990 │ │ return _hf_hub_download_to_local │ immich_machine_learning | │ 991 │ │ │ # Destination │ immich_machine_learning | │ 992 │ │ │ local_dir=local_dir, │ immich_machine_learning | │ 993 │ │ │ # File info │ immich_machine_learning | │ │ immich_machine_learning | │ /opt/venv/lib/python3.11/site-packages/huggingf │ immich_machine_learning | │ ace_hub/file_download.py:1300 in │ immich_machine_learning | │ _hf_hub_download_to_local_dir │ immich_machine_learning | │ │ immich_machine_learning | │ 1297 │ # Otherwise, let's download the file │ immich_machine_learning | │ 1298 │ with WeakFileLock(paths.lock_path): │ immich_machine_learning | │ 1299 │ │ paths.file_path.unlink(missing_o │ immich_machine_learning | │ ❱ 1300 │ │ _download_to_tmp_and_move( │ immich_machine_learning | │ 1301 │ │ │ incomplete_path=paths.incomp │ immich_machine_learning | │ 1302 │ │ │ destination_path=paths.file_ │ immich_machine_learning | │ 1303 │ │ │ url_to_download=url_to_downl │ immich_machine_learning | │ │ immich_machine_learning | │ /opt/venv/lib/python3.11/site-packages/huggingf │ immich_machine_learning | │ ace_hub/file_download.py:1723 in │ immich_machine_learning | │ _download_to_tmp_and_move │ immich_machine_learning | │ │ immich_machine_learning | │ 1720 │ │ │ immich_machine_learning | │ 1721 │ │ if xet_file_data is not None and │ immich_machine_learning | │ 1722 │ │ │ logger.debug("Xet Storage is │ immich_machine_learning | │ Xet Storage..") │ immich_machine_learning | │ ❱ 1723 │ │ │ xet_get( │ immich_machine_learning | │ 1724 │ │ │ │ incomplete_path=incomple │ immich_machine_learning | │ 1725 │ │ │ │ xet_file_data=xet_file_d │ immich_machine_learning | │ 1726 │ │ │ │ headers=headers, │ immich_machine_learning | │ │ immich_machine_learning | │ /opt/venv/lib/python3.11/site-packages/huggingf │ immich_machine_learning | │ ace_hub/file_download.py:629 in xet_get │ immich_machine_learning | │ │ immich_machine_learning | │ 626 │ │ def progress_updater(progress_by │ immich_machine_learning | │ 627 │ │ │ progress.update(progress_byt │ immich_machine_learning | │ 628 │ │ │ immich_machine_learning | │ ❱ 629 │ │ download_files( │ immich_machine_learning | │ 630 │ │ │ xet_download_info, │ immich_machine_learning | │ 631 │ │ │ endpoint=connection_info.end │ immich_machine_learning | │ 632 │ │ │ token_info=(connection_info. │ immich_machine_learning | │ connection_info.expiration_unix_epoch), │ immich_machine_learning | ╰─────────────────────────────────────────────────╯ immich_machine_learning | RuntimeError: Data processing error: I/O error: immich_machine_learning | Permission denied (os error 13) immich_server | [Nest] 31 - 10/07/2025, 11:58:57 AM LOG [Api:MachineLearningRepository] Machine learning server became healthy (http://immich-machine-learning:3003).