Skip to content

Commit b81115d

Browse files
committed
fix(setup): plug download FD leak, fix progress count, relax device sanitize
- _CanaryDownloadThread / ModelDownloadThread: wrap the HTTP response in a 'with' so the socket is closed on every path (success, cancel, incomplete), not only the skip path. - ModelDownloadThread: count progress over all files and bump 'done' once per file, so a re-downloaded truncated file can no longer show "2/1". - _sanitize_extra_devices: stop stripping ':' '/' '(' ')' which occur in real device names (silently broke #10 substring matching); only strip chars that are dangerous inside the double-quoted value ('"' '`' '$' '\\').
1 parent 2d1bbdf commit b81115d

1 file changed

Lines changed: 77 additions & 77 deletions

File tree

dictee-setup.py

Lines changed: 77 additions & 77 deletions
Original file line numberDiff line numberDiff line change
@@ -357,12 +357,15 @@ def _sanitize_conf_value(val):
357357
def _sanitize_extra_devices(val):
358358
"""Sanitize DICTEE_PTT_EXTRA_DEVICES (quoted in dictee.conf).
359359

360-
Device names contain spaces (e.g. 'LogiOps Virtual Input'), so we
361-
can't strip them like _sanitize_conf_value does. Keep only chars
362-
that can appear in a real device name; the value is wrapped in
363-
double quotes when written, so spaces survive bash sourcing.
360+
Device names contain spaces (e.g. 'LogiOps Virtual Input') and may
361+
contain ':' '/' '(' ')' (e.g. 'UNIW0001:00 093A:0274', 'HDA NVidia
362+
HDMI/DP'). A strict whitelist would drop those characters and silently
363+
break the substring matching dictee-ptt does on device names. The value
364+
is wrapped in double quotes when written, so we only strip what is
365+
actually dangerous inside a double-quoted bash / systemd value:
366+
'"' '`' '$' '\\' and newlines.
364367
"""
365-
return re.sub(r'[^A-Za-z0-9 ,\-_.]', '', str(val))
368+
return re.sub(r'["`$\\\r\n]', '', str(val))
366369

367370

368371
def save_config(backend, lang_source, lang_target, clipboard=True,
@@ -1700,43 +1703,42 @@ def run(self):
17001703
return
17011704
dest = os.path.join(model_dir, fname)
17021705
url = f"{base_url}/{fname}"
1703-
resp = urllib.request.urlopen(url, timeout=1800)
1704-
total_size = int(resp.headers.get("Content-Length", 0))
1705-
# Skip uniquement si le fichier local est COMPLET (taille ==
1706-
# Content-Length). Un fichier tronqué d'un essai précédent est
1707-
# ainsi re-téléchargé au lieu d'être pris pour valide.
1708-
if os.path.exists(dest) and total_size > 0 and \
1709-
os.path.getsize(dest) == total_size:
1710-
resp.close()
1711-
self.progress.emit(f"{fname} ✓ ({i}/{len(self.files)})")
1712-
continue
1713-
self.progress.emit(f"{fname} ({i}/{len(self.files)})…")
1714-
tmp = dest + ".part"
1715-
downloaded = 0
1716-
with open(tmp, "wb") as f:
1717-
while True:
1718-
if self._cancelled:
1719-
f.close()
1720-
os.remove(tmp)
1721-
self.done.emit(False, _("Cancelled"))
1722-
return
1723-
chunk = resp.read(256 * 1024)
1724-
if not chunk:
1725-
break
1726-
f.write(chunk)
1727-
downloaded += len(chunk)
1728-
if total_size > 0:
1729-
pct = int(downloaded * 100 / total_size)
1730-
size_mb = downloaded / (1024 * 1024)
1731-
total_mb = total_size / (1024 * 1024)
1732-
self.progress.emit(
1733-
f"{fname} {pct}% ({size_mb:.0f}/{total_mb:.0f} Mo)")
1734-
if total_size > 0 and downloaded != total_size:
1735-
os.remove(tmp)
1736-
raise IOError(
1737-
f"{fname}: téléchargement incomplet "
1738-
f"({downloaded}/{total_size} octets) — réessayez")
1739-
os.rename(tmp, dest)
1706+
with urllib.request.urlopen(url, timeout=1800) as resp:
1707+
total_size = int(resp.headers.get("Content-Length", 0))
1708+
# Skip uniquement si le fichier local est COMPLET (taille ==
1709+
# Content-Length). Un fichier tronqué d'un essai précédent est
1710+
# ainsi re-téléchargé au lieu d'être pris pour valide.
1711+
if os.path.exists(dest) and total_size > 0 and \
1712+
os.path.getsize(dest) == total_size:
1713+
self.progress.emit(f"{fname} ✓ ({i}/{len(self.files)})")
1714+
continue
1715+
self.progress.emit(f"{fname} ({i}/{len(self.files)})…")
1716+
tmp = dest + ".part"
1717+
downloaded = 0
1718+
with open(tmp, "wb") as f:
1719+
while True:
1720+
if self._cancelled:
1721+
f.close()
1722+
os.remove(tmp)
1723+
self.done.emit(False, _("Cancelled"))
1724+
return
1725+
chunk = resp.read(256 * 1024)
1726+
if not chunk:
1727+
break
1728+
f.write(chunk)
1729+
downloaded += len(chunk)
1730+
if total_size > 0:
1731+
pct = int(downloaded * 100 / total_size)
1732+
size_mb = downloaded / (1024 * 1024)
1733+
total_mb = total_size / (1024 * 1024)
1734+
self.progress.emit(
1735+
f"{fname} {pct}% ({size_mb:.0f}/{total_mb:.0f} Mo)")
1736+
if total_size > 0 and downloaded != total_size:
1737+
os.remove(tmp)
1738+
raise IOError(
1739+
f"{fname}: téléchargement incomplet "
1740+
f"({downloaded}/{total_size} octets) — réessayez")
1741+
os.rename(tmp, dest)
17401742
# Generate tokenizer.json if missing (needed for decodercontext)
17411743
tokenizer_path = os.path.join(model_dir, "tokenizer.json")
17421744
if not os.path.exists(tokenizer_path):
@@ -2055,48 +2057,46 @@ def run(self):
20552057
model_dir = model_dir.replace(MODEL_DIR,
20562058
os.path.join(os.path.expanduser("~/.local/share/dictee")))
20572059
os.makedirs(model_dir, exist_ok=True)
2058-
total_files = len([f for f in self.model["files"]
2059-
if not os.path.isfile(os.path.join(model_dir, f[1]))])
2060+
total_files = len(self.model["files"])
20602061
done = 0
20612062
for url, filename in self.model["files"]:
20622063
if self._cancelled:
20632064
self.done.emit(False, _("Cancelled"))
20642065
return
20652066
dest = os.path.join(model_dir, filename)
2066-
resp = urllib.request.urlopen(url, timeout=1800)
2067-
total_size = int(resp.headers.get("Content-Length", 0))
2068-
# Skip uniquement si le fichier local est COMPLET (cf. Canary).
2069-
if os.path.isfile(dest) and total_size > 0 and \
2070-
os.path.getsize(dest) == total_size:
2071-
resp.close()
2072-
continue
2073-
self.progress.emit(_("Downloading {name}…").format(name=filename))
2074-
downloaded = 0
2075-
tmp = dest + ".part"
2076-
with open(tmp, "wb") as f:
2077-
while True:
2078-
if self._cancelled:
2079-
f.close()
2067+
with urllib.request.urlopen(url, timeout=1800) as resp:
2068+
total_size = int(resp.headers.get("Content-Length", 0))
2069+
# Skip uniquement si le fichier local est COMPLET (cf. Canary).
2070+
skip = (os.path.isfile(dest) and total_size > 0 and
2071+
os.path.getsize(dest) == total_size)
2072+
if not skip:
2073+
self.progress.emit(_("Downloading {name}…").format(name=filename))
2074+
downloaded = 0
2075+
tmp = dest + ".part"
2076+
with open(tmp, "wb") as f:
2077+
while True:
2078+
if self._cancelled:
2079+
f.close()
2080+
os.remove(tmp)
2081+
self.done.emit(False, _("Cancelled"))
2082+
return
2083+
chunk = resp.read(256 * 1024)
2084+
if not chunk:
2085+
break
2086+
f.write(chunk)
2087+
downloaded += len(chunk)
2088+
if total_size > 0:
2089+
pct = int(downloaded * 100 / total_size)
2090+
size_mb = downloaded / (1024 * 1024)
2091+
total_mb = total_size / (1024 * 1024)
2092+
self.progress.emit(
2093+
f"{filename} {pct}% ({size_mb:.0f}/{total_mb:.0f} Mo)")
2094+
if total_size > 0 and downloaded != total_size:
20802095
os.remove(tmp)
2081-
self.done.emit(False, _("Cancelled"))
2082-
return
2083-
chunk = resp.read(256 * 1024)
2084-
if not chunk:
2085-
break
2086-
f.write(chunk)
2087-
downloaded += len(chunk)
2088-
if total_size > 0:
2089-
pct = int(downloaded * 100 / total_size)
2090-
size_mb = downloaded / (1024 * 1024)
2091-
total_mb = total_size / (1024 * 1024)
2092-
self.progress.emit(
2093-
f"{filename} {pct}% ({size_mb:.0f}/{total_mb:.0f} Mo)")
2094-
if total_size > 0 and downloaded != total_size:
2095-
os.remove(tmp)
2096-
raise IOError(
2097-
f"{filename}: téléchargement incomplet "
2098-
f"({downloaded}/{total_size} octets) — réessayez")
2099-
os.rename(tmp, dest)
2096+
raise IOError(
2097+
f"{filename}: téléchargement incomplet "
2098+
f"({downloaded}/{total_size} octets) — réessayez")
2099+
os.rename(tmp, dest)
21002100
done += 1
21012101
if total_files > 1:
21022102
self.progress.emit(_("Downloaded {done}/{total}").format(

0 commit comments

Comments
 (0)