FabNum/Code
4
0
Fork 0
Code Issues 2 Pull Requests Actions Packages Projects 1 Releases Wiki Activity

Update index.py

Browse Source
This commit is contained in:
Stéphan Peccini 2025-05-18 18:29:44 +02:00
parent f0f87b64f4
commit 3f2f13b65f
1 changed files with 3 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
index.py
View File

@ -45,8 +45,10 @@ def gather_files(root: Path):
def main(): def main():
docs, meta = [], [] docs, meta = [], []
files_count = 0
for fp in gather_files(ROOT): for fp in gather_files(ROOT):
files_count += 1
text = fp.read_text(encoding="utf-8", errors="ignore") text = fp.read_text(encoding="utf-8", errors="ignore")
for i, chunk in enumerate(split(text)): for i, chunk in enumerate(split(text)):
docs.append(chunk) docs.append(chunk)
@ -55,7 +57,7 @@ def main():
if not docs: if not docs:
raise SystemExit("Aucun fichier trouvé dans /app/Fiches. Vérifiez le montage ou les extensions.") raise SystemExit("Aucun fichier trouvé dans /app/Fiches. Vérifiez le montage ou les extensions.")
print(f"Découpé {len(docs)} passages, génération des embeddings…") print(f"Traité {files_count} fichiers, découpé {len(docs)} passages, génération des embeddings…")
model = BGEM3FlagModel(MODEL_NAME, device="cpu") model = BGEM3FlagModel(MODEL_NAME, device="cpu")
emb = model.encode(docs, batch_size=64) # pas de normalisation interne emb = model.encode(docs, batch_size=64) # pas de normalisation interne