Änderungen von rag_samples.py — Hybrid search, reranker y HyDE en Python

atareao hat die Gist bearbeitet 17 hours ago. Zu Änderung gehen

3 files changed, 151 insertions

rag_hybrid.py(Datei erstellt)

		@@ -0,0 +1,51 @@
1	+	#!/usr/bin/env python3
2	+	"""rag_hybrid.py — Hybrid search: FTS5 + cosine similarity"""
3	+	import sqlite3, numpy as np, math, os
4	+	from numpy.linalg import norm
5	+
6	+	DB_PATH = os.path.expanduser("~/.cerebro/rag_conocimiento.db")
7	+
8	+	def blob_to_vector(blob: bytes) -> np.ndarray:
9	+	return np.frombuffer(blob, dtype=np.float32).copy()
10	+
11	+	def cosine_similarity(a: np.ndarray, b: np.ndarray) -> float:
12	+	return float(np.dot(a, b))
13	+
14	+	def get_embedding_ollama(text: str) -> np.ndarray:
15	+	import requests
16	+	resp = requests.post("http://localhost:11434/api/embeddings",
17	+	json={"model": "bge-m3", "input": text})
18	+	return np.array(resp.json()["embeddings"][0], dtype=np.float32)
19	+
20	+	def hybrid_search(query: str, alpha: float = 0.4, top_k: int = 10):
21	+	"""
22	+	alpha: peso para FTS5, (1-alpha) para cosine similarity.
23	+	0.6 = nombres propios, 0.2 = conceptual, 0.4 = equilibrio
24	+	"""
25	+	conn = sqlite3.connect(DB_PATH)
26	+	query_emb = get_embedding_ollama(query)
27	+	query_emb = query_emb / (norm(query_emb) + 1e-10)
28	+
29	+	cursor = conn.execute("""
30	+	SELECT c.id, c.contenido, c.embedding,
31	+	bm25(chunks_fts, 0.0, 0.0, 5.0, 5.0) as fts_score
32	+	FROM chunks_fts
33	+	JOIN chunks c ON chunks_fts.rowid = c.id
34	+	WHERE chunks_fts MATCH ?
35	+	ORDER BY fts_score DESC
36	+	LIMIT ?
37	+	""", (query, top_k * 2))
38	+
39	+	results = []
40	+	for row in cursor:
41	+	chunk_id, contenido, emb_blob, fts_score = row
42	+	fts_norm = 1.0 / (1.0 + math.exp(-fts_score / 10.0))
43	+	emb_vec = blob_to_vector(emb_blob)
44	+	emb_vec = emb_vec / (norm(emb_vec) + 1e-10)
45	+	cos_sim = cosine_similarity(query_emb, emb_vec)
46	+	score_total = alpha * fts_norm + (1.0 - alpha) * cos_sim
47	+	results.append((score_total, contenido[:200], fts_norm, cos_sim))
48	+
49	+	results.sort(key=lambda x: x[0], reverse=True)
50	+	conn.close()
51	+	return results[:top_k]

rag_hyde.py(Datei erstellt)

		@@ -0,0 +1,77 @@
1	+	#!/usr/bin/env python3
2	+	"""rag_hyde.py — HyDE: Hypothetical Document Embeddings"""
3	+	import requests, numpy as np, sqlite3, os, math
4	+	from numpy.linalg import norm
5	+
6	+	DB_PATH = os.path.expanduser("~/.cerebro/rag_conocimiento.db")
7	+	OLLAMA_URL = "http://localhost:11434/api"
8	+
9	+	def generate_hypothetical_document(query: str, model: str ="llama3.2") -> str:
10	+	prompt = f"""Genera un texto breve y factual que responda a la siguiente pregunta.
11	+	El texto debe ser informativo, objetivo y escrito en español.
12	+
13	+	Pregunta: {query}
14	+
15	+	Texto informativo:"""
16	+	resp = requests.post(f"{OLLAMA_URL}/generate",
17	+	json={"model": model, "prompt": prompt, "stream": False})
18	+	return resp.json()["response"]
19	+
20	+	def get_embedding(text: str, model: str ="bge-m3") -> np.ndarray:
21	+	resp = requests.post(f"{OLLAMA_URL}/embed",
22	+	json={"model": model, "input": text})
23	+	return np.array(resp.json()["embeddings"][0], dtype=np.float32)
24	+
25	+	def blob_to_vector(blob: bytes) -> np.ndarray:
26	+	return np.frombuffer(blob, dtype=np.float32).copy()
27	+
28	+	def cosine_similarity(a: np.ndarray, b: np.ndarray) -> float:
29	+	return float(np.dot(a, b))
30	+
31	+	def hyde_search(query: str, top_k: int = 5):
32	+	hypothetic_doc = generate_hypothetical_document(query)
33	+	print(f"[HyDE] Documento generado: {hypothetic_doc[:100]}...")
34	+	hyde_emb = get_embedding(hypothetic_doc)
35	+	hyde_emb = hyde_emb / (norm(hyde_emb) + 1e-10)
36	+
37	+	conn = sqlite3.connect(DB_PATH)
38	+	cursor = conn.execute("SELECT id, contenido, embedding FROM chunks")
39	+	results = []
40	+	for row in cursor:
41	+	contenido, emb_blob = row[1], row[2]
42	+	emb_vec = blob_to_vector(emb_blob)
43	+	emb_vec = emb_vec / (norm(emb_vec) + 1e-10)
44	+	cos_sim = cosine_similarity(hyde_emb, emb_vec)
45	+	results.append((cos_sim, contenido[:200]))
46	+
47	+	results.sort(key=lambda x: x[0], reverse=True)
48	+	conn.close()
49	+	return results[:top_k]
50	+
51	+	def hyde_hybrid_search(query: str, alpha: float = 0.3, top_k: int = 5):
52	+	"""HyDE + Hybrid search combinado. Alpha bajo porque HyDE ya captura semántica."""
53	+	hyde_doc = generate_hypothetical_document(query)
54	+	hyde_emb = get_embedding(hyde_doc)
55	+	hyde_emb = hyde_emb / (norm(hyde_emb) + 1e-10)
56	+
57	+	conn = sqlite3.connect(DB_PATH)
58	+	cursor = conn.execute("""
59	+	SELECT c.contenido, c.embedding,
60	+	bm25(chunks_fts, 0, 0, 5, 5) as fts_score
61	+	FROM chunks_fts JOIN chunks c ON chunks_fts.rowid = c.id
62	+	WHERE chunks_fts MATCH ?
63	+	""", (query,))
64	+
65	+	results = []
66	+	for row in cursor:
67	+	contenido, emb_blob, fts_score = row
68	+	emb_vec = blob_to_vector(emb_blob)
69	+	emb_vec = emb_vec / (norm(emb_vec) + 1e-10)
70	+	cos_sim = cosine_similarity(hyde_emb, emb_vec)
71	+	fts_norm = 1.0 / (1.0 + math.exp(-fts_score / 10.0))
72	+	score_total = alpha * fts_norm + (1.0 - alpha) * cos_sim
73	+	results.append((score_total, contenido[:200]))
74	+
75	+	results.sort(reverse=True)
76	+	conn.close()
77	+	return results[:top_k]

rag_reranker.py(Datei erstellt)

		@@ -0,0 +1,23 @@
1	+	#!/usr/bin/env python3
2	+	"""rag_reranker.py — Re-ranking con cross-encoder"""
3	+	from sentence_transformers import CrossEncoder
4	+
5	+	class Reranker:
6	+	def __init__(self, model_name="BAAI/bge-reranker-v2-m3", use_fp16=True):
7	+	self.model = CrossEncoder(model_name, max_length=512, device="cpu")
8	+
9	+	def rerank(self, query: str, candidates: list[dict], top_k: int = 5) -> list[dict]:
10	+	pairs = [(query, c["contenido"]) for c in candidates]
11	+	scores = self.model.predict(pairs)
12	+	for i, score in enumerate(scores):
13	+	candidates[i]["rerank_score"] = float(score)
14	+	candidates.sort(key=lambda x: x["rerank_score"], reverse=True)
15	+	return candidates[:top_k]
16	+
17	+	def search_with_rerank(query: str, alpha: float = 0.4,
18	+	top_k_hybrid: int = 20, top_k_final: int = 5):
19	+	from rag_hybrid import hybrid_search
20	+	candidates = hybrid_search(query, alpha=alpha, top_k=top_k_hybrid)
21	+	candidates_dict = [{"contenido": c[1], "score": c[0]} for c in candidates]
22	+	reranker = Reranker()
23	+	return reranker.rerank(query, candidates_dict, top_k=top_k_final)

Neuer Älter