release: v0.4.3

Author: jstzwj

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "olah"
-version = "0.4.2"
+version = "0.4.3"
 description = "Self-hosted lightweight huggingface mirror."
 readme = "README.md"
 requires-python = ">=3.8"

src/olah/configs.py

@@ -92,13 +92,9 @@ def __init__(self, path: Optional[str] = None) -> None:
         self.hf_netloc: str = "huggingface.co"
         self.hf_lfs_netloc: str = "cdn-lfs.huggingface.co"
 
-        self.mirror_scheme: str = "http" if self.ssl_key is None else "https"
-        self.mirror_netloc: str = (
-            f"{self.host if self._is_specific_addr(self.host) else 'localhost'}:{self.port}"
-        )
-        self.mirror_lfs_netloc: str = (
-            f"{self.host if self._is_specific_addr(self.host) else 'localhost'}:{self.port}"
-        )
+        self.mirror_scheme: str = self.default_mirror_scheme()
+        self.mirror_netloc: str = self.default_mirror_netloc()
+        self.mirror_lfs_netloc: str = self.default_mirror_netloc()
 
         self.mirrors_path: List[str] = []
 
@@ -116,6 +112,20 @@ def _is_specific_addr(self, host: Union[List[str], str]) -> bool:
         else:
             return False
 
+    def default_mirror_scheme(self, ssl_key: Optional[str] = None) -> str:
+        ssl_key = self.ssl_key if ssl_key is None else ssl_key
+        return "http" if ssl_key is None else "https"
+
+    def default_mirror_netloc(
+        self,
+        host: Optional[Union[List[str], str]] = None,
+        port: Optional[int] = None,
+    ) -> str:
+        host = self.host if host is None else host
+        port = self.port if port is None else port
+        default_host = host if self._is_specific_addr(host) else "localhost"
+        return f"{default_host}:{port}"
+
     def hf_url_base(self) -> str:
         return f"{self.hf_scheme}://{self.hf_netloc}"
 
@@ -139,6 +149,9 @@ def read_toml(self, path: str) -> None:
 
         if "basic" in config:
             basic = config["basic"]
+            previous_default_mirror_scheme = self.default_mirror_scheme()
+            previous_default_mirror_netloc = self.default_mirror_netloc()
+
             self.host = basic.get("host", self.host)
             self.port = basic.get("port", self.port)
             self.ssl_key = self.empty_str(basic.get("ssl-key", self.ssl_key))
@@ -151,11 +164,20 @@ def read_toml(self, path: str) -> None:
             self.hf_netloc = basic.get("hf-netloc", self.hf_netloc)
             self.hf_lfs_netloc = basic.get("hf-lfs-netloc", self.hf_lfs_netloc)
 
-            self.mirror_scheme = basic.get("mirror-scheme", self.mirror_scheme)
-            self.mirror_netloc = basic.get("mirror-netloc", self.mirror_netloc)
-            self.mirror_lfs_netloc = basic.get(
-                "mirror-lfs-netloc", self.mirror_lfs_netloc
-            )
+            if "mirror-scheme" in basic:
+                self.mirror_scheme = basic["mirror-scheme"]
+            elif self.mirror_scheme == previous_default_mirror_scheme:
+                self.mirror_scheme = self.default_mirror_scheme()
+
+            if "mirror-netloc" in basic:
+                self.mirror_netloc = basic["mirror-netloc"]
+            elif self.mirror_netloc == previous_default_mirror_netloc:
+                self.mirror_netloc = self.default_mirror_netloc()
+
+            if "mirror-lfs-netloc" in basic:
+                self.mirror_lfs_netloc = basic["mirror-lfs-netloc"]
+            elif self.mirror_lfs_netloc == previous_default_mirror_netloc:
+                self.mirror_lfs_netloc = self.default_mirror_netloc()
 
             self.mirrors_path = basic.get("mirrors-path", self.mirrors_path)
 

src/olah/proxy/files.py

@@ -218,7 +218,7 @@ async def _get_file_range_from_remote(
                 yield raw_chunk
                 chunk_bytes += len(raw_chunk)
 
-        if is_compressed:
+        if is_compressed or "content-length" not in response.headers:
             response_content_length = chunk_bytes
         else:
             response_content_length = int(response.headers["content-length"])
@@ -420,7 +420,7 @@ async def _resource_etag(hf_url: str, authorization: Optional[str]=None, offline
                 ret_etag = response.headers["etag"]
             else:
                 ret_etag = f'"{content_hash[:32]}-10"'
-        except httpx.TimeoutException:
+        except httpx.HTTPError:
             ret_etag = None
     return ret_etag
 
@@ -438,24 +438,28 @@ async def _remote_file_metadata(
     headers = {}
     if authorization is not None:
         headers["authorization"] = authorization
-    async with httpx.AsyncClient() as client:
-        response = await client.request(
-            method="HEAD",
-            url=hf_url,
-            headers=headers,
-            timeout=WORKER_API_TIMEOUT,
-            follow_redirects=True,
-        )
+    try:
+        async with httpx.AsyncClient() as client:
+            response = await client.request(
+                method="HEAD",
+                url=hf_url,
+                headers=headers,
+                timeout=WORKER_API_TIMEOUT,
+                follow_redirects=True,
+            )
+    except (httpx.HTTPError, ValueError):
+        return None
     if response.status_code >= 400:
         return None
 
     content_length = response.headers.get("content-length")
     if content_length is None:
         return None
-    return RemoteFileMetadata(
-        file_size=int(content_length),
-        etag=response.headers.get("etag"),
-    )
+    try:
+        file_size = int(content_length)
+    except ValueError:
+        return None
+    return RemoteFileMetadata(file_size=file_size, etag=response.headers.get("etag"))
 
 async def _file_realtime_stream(
     app,

src/olah/server.py

@@ -65,6 +65,22 @@
 logger = None
 
 
+def normalize_server_host(host: Union[str, Sequence[str]]) -> str:
+    if isinstance(host, str):
+        host_candidates = host.split(",")
+    else:
+        host_candidates = list(host)
+
+    normalized_hosts = [str(item).strip() for item in host_candidates if str(item).strip()]
+    if len(normalized_hosts) == 0:
+        raise ValueError("Host cannot be empty.")
+    if len(normalized_hosts) > 1:
+        raise ValueError(
+            "Multiple hosts are not supported by uvicorn.run. Please run one Olah instance per host."
+        )
+    return normalized_hosts[0]
+
+
 async def check_connection(url: str) -> bool:
     try:
         async with httpx.AsyncClient(follow_redirects=True) as client:
@@ -74,7 +90,7 @@ async def check_connection(url: str) -> bool:
                 timeout=10,
             )
         return 200 <= response.status_code < 400
-    except httpx.TimeoutException:
+    except httpx.HTTPError:
         return False
 
 
@@ -193,6 +209,9 @@ def is_default_value(namespace, arg_name):
         config = OlahConfig(args.config)
     else:
         config = OlahConfig()
+        mirror_scheme_is_default = is_default_value(args, "mirror_scheme")
+        mirror_netloc_is_default = is_default_value(args, "mirror_netloc")
+        mirror_lfs_netloc_is_default = is_default_value(args, "mirror_lfs_netloc")
 
         if not is_default_value(args, "host"):
             config.host = args.host
@@ -222,8 +241,28 @@ def is_default_value(namespace, arg_name):
             config.cache_size_limit = convert_to_bytes(args.cache_size_limit)
         if not is_default_value(args, "cache_clean_strategy"):
             config.cache_clean_strategy = args.cache_clean_strategy
-        elif not args.has_lfs_site and not is_default_value(args, "mirror_netloc"):
-            config.mirror_lfs_netloc = args.mirror_netloc
+
+        config.host = normalize_server_host(config.host)
+
+        if mirror_scheme_is_default:
+            config.mirror_scheme = config.default_mirror_scheme()
+        else:
+            config.mirror_scheme = args.mirror_scheme
+
+        if mirror_netloc_is_default:
+            config.mirror_netloc = config.default_mirror_netloc()
+        else:
+            config.mirror_netloc = args.mirror_netloc
+
+        if mirror_lfs_netloc_is_default:
+            if not args.has_lfs_site and not mirror_netloc_is_default:
+                config.mirror_lfs_netloc = config.mirror_netloc
+            else:
+                config.mirror_lfs_netloc = config.default_mirror_netloc()
+        else:
+            config.mirror_lfs_netloc = args.mirror_lfs_netloc
+
+    config.host = normalize_server_host(config.host)
 
     if is_default_value(args, "host"):
         args.host = config.host
@@ -254,10 +293,8 @@ def is_default_value(namespace, arg_name):
     if is_default_value(args, "cache_clean_strategy"):
         args.cache_clean_strategy = config.cache_clean_strategy
 
-    if "," in args.host:
-        args.host = args.host.split(",")
-
-    args.mirror_scheme = config.mirror_scheme = "http" if args.ssl_key is None else "https"
+    args.host = normalize_server_host(args.host)
+    config.host = args.host
 
     print(args)
     if config.cache_size_limit is not None:

src/olah/server_api_routes.py

@@ -271,13 +271,16 @@ async def commits_proxy_common(
 async def whoami_v2(request: Request):
     new_headers = {k.lower(): v for k, v in request.headers.items()}
     new_headers["host"] = request.app.state.app_settings.config.hf_netloc
-    async with httpx.AsyncClient() as client:
-        response = await client.request(
-            method="GET",
-            url=urljoin(request.app.state.app_settings.config.hf_url_base(), "/api/whoami-v2"),
-            headers=new_headers,
-            timeout=10,
-        )
+    try:
+        async with httpx.AsyncClient() as client:
+            response = await client.request(
+                method="GET",
+                url=urljoin(request.app.state.app_settings.config.hf_url_base(), "/api/whoami-v2"),
+                headers=new_headers,
+                timeout=10,
+            )
+    except httpx.HTTPError:
+        return Response(status_code=504)
     response_headers = {k.lower(): v for k, v in response.headers.items()}
     response_headers.pop("content-encoding", None)
     response_headers.pop("content-length", None)

src/olah/server_file_routes.py

@@ -7,6 +7,7 @@
 
 import glob
 import os
+import re
 import traceback
 from typing import Literal
 
@@ -36,6 +37,13 @@ def _get_logger(app: FastAPI):
     return getattr(app.state, "logger", None) or _NullLogger()
 
 
+def _repo_name_from_cache_path(repo_path: str) -> str:
+    parts = [part for part in re.split(r"[\\/]+", repo_path) if part]
+    if len(parts) < 2:
+        return repo_path
+    return get_org_repo(parts[-2], parts[-1])
+
+
 async def file_head_common(
     app: FastAPI,
     repo_type: str,
@@ -345,9 +353,9 @@ async def repos(request: Request):
     datasets_repos = glob.glob(os.path.join(request.app.state.app_settings.config.repos_path, "api/datasets/*/*"))
     models_repos = glob.glob(os.path.join(request.app.state.app_settings.config.repos_path, "api/models/*/*"))
     spaces_repos = glob.glob(os.path.join(request.app.state.app_settings.config.repos_path, "api/spaces/*/*"))
-    datasets_repos = [get_org_repo(*repo.split("/")[-2:]) for repo in datasets_repos]
-    models_repos = [get_org_repo(*repo.split("/")[-2:]) for repo in models_repos]
-    spaces_repos = [get_org_repo(*repo.split("/")[-2:]) for repo in spaces_repos]
+    datasets_repos = [_repo_name_from_cache_path(repo) for repo in datasets_repos]
+    models_repos = [_repo_name_from_cache_path(repo) for repo in models_repos]
+    spaces_repos = [_repo_name_from_cache_path(repo) for repo in spaces_repos]
 
     return request.app.state.templates.TemplateResponse(
         "repos.html",

src/olah/server_upstream.py

@@ -20,8 +20,6 @@ def refresh_cache(self) -> bool:
 
 
 async def get_latest_commit(app, repo: RepoRef, authorization: Optional[str]) -> Optional[str]:
-    if app.state.app_settings.config.offline:
-        return "main"
     return await get_newest_commit_hf(
         app,
         repo.repo_type,

src/olah/utils/repo_utils.py

@@ -213,7 +213,7 @@ async def get_newest_commit_hf(
                 return await get_newest_commit_hf_offline(app, repo_type, org, repo)
             obj = json.loads(response.text)
         return obj.get("sha", None)
-    except httpx.TimeoutException as e:
+    except (httpx.HTTPError, ValueError, OSError):
         return await get_newest_commit_hf_offline(app, repo_type, org, repo)
 
 
@@ -293,7 +293,7 @@ async def get_commit_hf(
                 return await get_commit_hf_offline(app, repo_type, org, repo, commit)
             obj = json.loads(response.text)
         return obj.get("sha", None)
-    except:
+    except (httpx.HTTPError, ValueError, OSError):
         return await get_commit_hf_offline(app, repo_type, org, repo, commit)
 
 
@@ -335,7 +335,15 @@ async def check_commit_hf(
     headers = {}
     if authorization is not None:
         headers["authorization"] = authorization
-    async with httpx.AsyncClient() as client:
-        response = await client.request(method="HEAD", url=url, headers=headers, timeout=WORKER_API_TIMEOUT)
-        status_code = response.status_code
+    try:
+        async with httpx.AsyncClient() as client:
+            response = await client.request(
+                method="HEAD",
+                url=url,
+                headers=headers,
+                timeout=WORKER_API_TIMEOUT,
+            )
+            status_code = response.status_code
+    except httpx.HTTPError:
+        return False
     return status_code in [200, 307]

src/olah/utils/url_utils.py

@@ -143,7 +143,9 @@ def parse_range_params(range_header: str) -> Tuple[str, List[Tuple[Optional[int]
 def get_all_ranges(file_size: int, unit: str, ranges: List[Tuple[Optional[int], Optional[int]]], suffix: Optional[int]) -> List[Tuple[int, int]]:
     all_ranges: List[Tuple[int, int]] = []
     if suffix is not None:
-        all_ranges.append((file_size - suffix, file_size))
+        if file_size <= 0 or suffix <= 0:
+            return all_ranges
+        all_ranges.append((max(0, file_size - suffix), file_size))
     else:
         for r in ranges:
             r_start = r[0] if r[0] is not None else 0
@@ -259,4 +261,4 @@ def clean_path(path: str) -> str:
     path = path.replace("\\", "/")
     while "//" in path:
         path = path.replace("//", "/")
-    return path
+    return path

tests/test_configs.py

@@ -82,6 +82,43 @@ def test_olah_config_reads_toml_and_normalizes_empty_ssl_fields(tmp_path):
     assert config.cache.allow("team/project") is True
 
 
+def test_olah_config_derives_mirror_netloc_from_updated_host_and_port(tmp_path):
+    config_path = tmp_path / "config.toml"
+    config_path.write_text(
+        textwrap.dedent(
+            """
+            [basic]
+            host = "0.0.0.0"
+            port = 9001
+            """
+        ),
+        encoding="utf-8",
+    )
+
+    config = OlahConfig(str(config_path))
+
+    assert config.mirror_netloc == "localhost:9001"
+    assert config.mirror_lfs_netloc == "localhost:9001"
+
+
+def test_olah_config_derives_https_mirror_scheme_when_ssl_is_configured(tmp_path):
+    config_path = tmp_path / "config.toml"
+    config_path.write_text(
+        textwrap.dedent(
+            """
+            [basic]
+            ssl-key = "/tmp/key.pem"
+            ssl-cert = "/tmp/cert.pem"
+            """
+        ),
+        encoding="utf-8",
+    )
+
+    config = OlahConfig(str(config_path))
+
+    assert config.mirror_scheme == "https"
+
+
 def test_is_specific_addr_only_accepts_real_single_host():
     config = OlahConfig()