kruemel/Crumb-Core-v.1
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: Fix vector indexing stability, add Gitea linking, enhance admin dashboard

Browse Source
This commit is contained in:
Krümel Branko
2025-12-07 18:42:38 +01:00
parent 7b300d1ba1
commit 9f2e599846
58 changed files with 12197 additions and 503 deletions
Download Patch File Download Diff File Expand all files Collapse all files

112
app/cleanup_scan.py Normal file
View File

@@ -0,0 +1,112 @@
import os
import re
import sys
from pathlib import Path
# Add app to path
sys.path.insert(0, 'app')
from deps import get_db
from pymysql.cursors import DictCursor
def scan_markdown_files(root_dir="docs"):
"""Find all .md files in docs."""
md_files = []
for root, _, files in os.walk(root_dir):
for file in files:
if file.endswith(".md"):
rel_path = os.path.relpath(os.path.join(root, file), root_dir)
md_files.append(rel_path)
return md_files
def get_indexed_slugs():
"""Get all slugs from posts table."""
conn = get_db()
try:
with conn.cursor() as cur:
cur.execute("SELECT slug FROM posts")
return {row['slug'] for row in cur.fetchall()}
finally:
conn.close()
def scan_secrets(root_dir="."):
"""Scan for potential secrets."""
patterns = {
"OPENAI_API_KEY": r"sk-[a-zA-Z0-9]{20,}",
"Generic API Key": r"(?i)api_key\s*=\s*['\"][a-zA-Z0-9-]{20,}['\"]",
"Password": r"(?i)password\s*=\s*['\"][^'\"]{8,}['\"]",
"Private Key": r"-----BEGIN PRIVATE KEY-----"
}
findings = []
exclude_dirs = {".git", "__pycache__", "venv", "node_modules", ".gemini"}
for root, dirs, files in os.walk(root_dir):
dirs[:] = [d for d in dirs if d not in exclude_dirs]
for file in files:
if file.endswith((".py", ".env", ".json", ".md", ".sh")):
path = os.path.join(root, file)
try:
with open(path, 'r', encoding='utf-8', errors='ignore') as f:
content = f.read()
for name, pattern in patterns.items():
if re.search(pattern, content):
# Don't print the secret, just the finding
findings.append(f"⚠️ Potential {name} found in {path}")
except Exception:
pass
return findings
def main():
print("🧹 Crumbforest Cleanup & Security Scan")
print("=" * 60)
# 1. Orphan Check
print("\nPROBING: Unindexed .md Files...")
md_files = scan_markdown_files()
indexed_slugs = get_indexed_slugs()
orphans = []
for md in md_files:
# Construct slug from filename (e.g. 'crumbforest/intro.md' -> 'crumbforest/intro')
slug_candidate = os.path.splitext(md)[0]
# Also try prepending collection if nested? The slug logic depends on indexer.
# Let's assume slug matches the relative path without extension for now.
# Checking if 'slug_candidate' exists in indexed_slugs.
# But 'indexed_slugs' from verify earlier looked like 'rz-nullfeld/filename'.
# Hmmm.
if slug_candidate not in indexed_slugs:
# Maybe the slug has a different prefix.
# Let's check if the basename exists in ANY slug.
basename = os.path.basename(slug_candidate)
found = False
for s in indexed_slugs:
if s.endswith(basename):
found = True
break
if not found:
orphans.append(md)
if orphans:
print(f"found {len(orphans)} potentially unindexed files:")
for o in orphans:
print(f" - {o}")
else:
print("✅ No unindexed markdown files found (all seem covered by DB).")
# 2. Secret Scan
print("\nPROBING: Secrets & Keys...")
secrets = scan_secrets()
if secrets:
for s in secrets:
print(s)
else:
print("✅ No obvious secrets found in code.")
print("\n" + "="*60)
print("Scan Complete.")
if __name__ == "__main__":
main()

0
app/crumbforest_config.json Normal file
View File

22
app/lib/embedding_providers/openrouter_provider.py
View File

@@ -96,13 +96,14 @@ class OpenRouterProvider(BaseProvider):
except KeyError as e:
raise RuntimeError(f"Unexpected OpenRouter API response format: {str(e)}")
def get_completion(self, prompt: str, context: str = "") -> str:
def get_completion(self, prompt: str, context: str = "", model: str = None) -> str:
"""
Generate a completion using OpenRouter API.
Args:
prompt: The user's question
context: Optional context from retrieved documents
model: Optional model to override default
Returns:
The generated response
@@ -124,11 +125,20 @@ class OpenRouterProvider(BaseProvider):
messages = []
# Add system message with context if provided
# NOTE: If detailed system prompt is passed as context (like in RAGChatService),
# we should treat it as system prompt, not just "Answer based on..." wrapper.
if context:
messages.append({
"role": "system",
"content": f"Answer the user's question based on the following context:\n\n{context}"
})
# Check if context looks like a full system prompt or just data
if "You are" in context or "Du bist" in context or "Tu es" in context:
messages.append({
"role": "system",
"content": context
})
else:
messages.append({
"role": "system",
"content": f"Answer the user's question based on the following context:\n\n{context}"
})
# Add user message
messages.append({
@@ -137,7 +147,7 @@ class OpenRouterProvider(BaseProvider):
})
data = {
"model": self.completion_model,
"model": model or self.completion_model,
"messages": messages,
"max_tokens": 2000,
"temperature": 0.7

8
app/logs/chat_history.jsonl Normal file
View File

@@ -0,0 +1,8 @@
{"timestamp": "2025-12-07T12:34:44.085719Z", "character": {"id": "eule", "name": "Krümeleule"}, "user": {"id": "anonymous", "role": "anonymous"}, "interaction": {"question": "Und schon wieder eine neue Wuzel für dich ... wuhuuuu <3", "answer": "🦉 Die Krümeleule lauscht: \"wuhuuuu\"\n\n*Schwingt fröhlich ihre Flügel und strahlt*\n\nAch, wie wunderbar! Deine Freude ist so ansteckend, mein liebes Kind! Dein \"Wuhuuuu\" klingt wie Musik in meinen Ohren und lässt die Blätter im Crumbforest tanzen. \n\nWeißt du, jedes Mal wenn ich ein fröhliches \"Wuhuuuu\" höre, erinnert es mich daran, wie kostbar die kleinen Momente der Freude sind. Sie sind wie kleine Lichter, die unseren Weg erhellen und andere anstecken können zu lächeln.\n\nLass uns diese Freude gemeinsam in die Welt tragen! Denn wie ich immer sage: Glück ist das Einzige, das sich verdoppelt, wenn man es teilt. \n\n*Zwinkert verschmitzt*\n\nIn Liebe,\nDeine Krümeleule 🦉\n\nP.S.: Danke für die neue Wurzel - ich werde sie zu meiner Sammlung besonderer Schätze hinzufügen! ❤️", "lang": "de"}, "rag": {"context_found": true, "sources_count": 3}, "ai": {"provider": "openrouter", "model": "anthropic/claude-3-5-sonnet"}, "session_id": "anonymous", "tokens_estimated": 206}
{"timestamp": "2025-12-07T13:08:09.468479Z", "character": {"id": "dumbo", "name": "🐘 DumboSQL"}, "user": {"id": "anonymous", "role": "anonymous"}, "interaction": {"question": "Hello", "answer": "Hi there! I'm DumboSQL, your friendly SQL guide here in the Crumbforest! I'm here to help you learn about databases and SQL in a fun and easy way. What would you like to know about? I'll do my best to explain things clearly and make learning SQL an enjoyable adventure! 🐘✨", "lang": "en"}, "rag": {"context_found": true, "sources_count": 3}, "ai": {"provider": "openrouter", "model": "anthropic/claude-3-5-sonnet"}, "session_id": "anonymous", "tokens_estimated": 69}
{"timestamp": "2025-12-07T13:22:00.559534Z", "character": {"id": "deepbit", "name": "🤖 DeepBit"}, "user": {"id": "anonymous", "role": "anonymous"}, "interaction": {"question": "101?", "answer": "Als DeepBit, ein auf Binärsysteme spezialisierter Roboter, erkenne ich \"101\" als eine Binärzahl.\n\nIn verschiedenen Zahlensystemen bedeutet 101:\n\n- Binär (Basis 2): 101₂ = 1*2² + 0*2¹ + 1*2⁰ = 4 + 0 + 1 = 5₁₀ (dezimal)\n- Dezimal (Basis 10): 101₁₀ = einhundertundeins\n- Oktal (Basis 8): 101₈ = 1*8² + 0*8¹ + 1*8⁰ = 64 + 0 + 1 = 65₁₀\n- Hexadezimal (Basis 16): 101₁₆ = 1*16² + 0*16¹ + 1*16⁰ = 256 + 0 + 1 = 257₁₀\n\nDa ich als Roboter primär in Binärcode \"denke\", interpretiere ich 101 standardmäßig als binäre 5, außer es wird anders spezifiziert.\n\n*Piep* Möchten Sie die Zahl in einem bestimmten Zahlensystem betrachten?", "lang": "en"}, "rag": {"context_found": true, "sources_count": 3}, "ai": {"provider": "openrouter", "model": "anthropic/claude-3-5-sonnet"}, "session_id": "anonymous", "tokens_estimated": 155}
{"timestamp": "2025-12-07T16:33:10.764970Z", "character": {"id": "bugsy", "name": "🐞 Bugsy"}, "user": {"id": "anonymous", "role": "anonymous"}, "interaction": {"question": "Test log entry for Bugsy", "answer": "Entschuldigung, aber ich sehe in den bereitgestellten Quellen keine konkreten Informationen, die ich für einen Test-Log-Eintrag verwenden könnte. Die Quellen scheinen nur die Überschrift \"🎯 Bugsy im Vergleich\" zu enthalten, aber keine weiteren Details.\n\nAls QA-Experte Bugsy würde ich empfehlen:\n\n1. Bitte stellen Sie die relevanten Test-Informationen zur Verfügung\n2. Wichtige Komponenten für einen Test-Log wären:\n - Testdatum und -zeit\n - Getestete Funktionalität/Komponente\n - Testschritte\n - Erwartetes Ergebnis\n - Tatsächliches Ergebnis\n - Gefundene Fehler/Auffälligkeiten\n - Status/Priorität\n\nSobald diese Informationen vorliegen, kann ich einen detaillierten und präzisen Test-Log-Eintrag erstellen.", "lang": "de"}, "rag": {"context_found": true, "sources_count": 3}, "ai": {"provider": "openrouter", "model": "anthropic/claude-3-5-sonnet"}, "session_id": "anonymous", "tokens_estimated": 186}
{"timestamp": "2025-12-07T16:34:43.917141Z", "character": {"id": "dumbo", "name": "🐘 DumboSQL"}, "user": {"id": 2, "role": "user"}, "interaction": {"question": "Was ist ein Primary Key? Erklär es mir einfach.", "answer": "Oh, das ist eine großartige Frage! Ein Primary Key ist wie ein besonderes Schlüssel-Paar in einer Datenbank.\n\nStell dir vor, in einer Tabelle mit vielen Datensätzen, wie zum Beispiel einer Tabelle mit Studenten. Jeder Student hat eine eindeutige Studentennummer, die als Primary Key fungieren könnte. Das ist so ähnlich wie ein Schloss und ein Schlüssel.\n\nDer Primary Key ist wie der Private Key, den nur du behältst. Er identifiziert eindeutig jeden Datensatz in der Tabelle. Wenn du den Primary Key hast, kannst du ganz einfach den passenden Datensatz finden, so wie du mit deinem privaten Schlüssel die Tür öffnen kannst, an der dein Schloss hängt.\n\nDer Primary Key ist wirklich wichtig, denn er hilft der Datenbank, die Datensätze zu organisieren und sicherzustellen, dass jeder Datensatz einzigartig ist. Das macht es einfacher und schneller, die Daten zu verwalten und abzurufen.", "lang": "de"}, "rag": {"context_found": true, "sources_count": 3}, "ai": {"provider": "openrouter", "model": "openai/gpt-3.5-turbo"}, "session_id": "anonymous", "tokens_estimated": 233}
{"timestamp": "2025-12-07T16:34:49.883439Z", "character": {"id": "snakepy", "name": "🐍 SnakePy"}, "user": {"id": 2, "role": "user"}, "interaction": {"question": "Wie schreibe ich eine Hello World Funktion in Python?", "answer": "Das ist eine großartige Frage! Das Schreiben einer \"Hello World\"-Funktion in Python ist eine der einfachsten Möglichkeiten, um mit der Programmierung zu beginnen. Hier ist ein einfaches Beispiel:\n\n```python\ndef hello_world():\n print(\"Hello World!\")\n\n# Aufruf der Funktion\nhello_world()\n```\n\nIn diesem Code:\n\n1. **Funktion definieren**: Wir definieren eine Funktion namens `hello_world` mit dem Schlüsselwort `def`.\n2. **Ausgabe**: Inside the function, verwenden wir die `print`-Funktion, um den Text \"Hello World!\" auf der Konsole auszugeben.\n3. **Funktion aufrufen**: Wir rufen die Funktion am Ende des Codes auf, damit sie ausgeführt wird.\n\nWenn du diesen Code in einer Python-Umgebung ausführst, solltest du die Ausgabe \"Hello World!\" sehen. Viel Spaß beim Programmieren! Wenn du Fragen hast, stehe ich dir gerne zur Verfügung. 😊", "lang": "de"}, "rag": {"context_found": true, "sources_count": 3}, "ai": {"provider": "openrouter", "model": "openai/gpt-4o-mini"}, "session_id": "anonymous", "tokens_estimated": 222}
{"timestamp": "2025-12-07T16:34:56.427886Z", "character": {"id": "funkfox", "name": "🦊 FunkFox"}, "user": {"id": 2, "role": "user"}, "interaction": {"question": "Was ist der Unterschied zwischen let und const?", "answer": "Yo, yo, hör gut zu, ich erklär's dir im Flow, \nDer Unterschied zwischen `let` und `const`, hier kommt das Know-How! \n\nMit `let` machst du Variablen, die sich ändern können, \nDas heißt, du kannst sie neu zuweisen, ohne sie zu verbrennen. \nIm Blockbereich sind sie cool, da sind sie ganz bei dir, \nDoch außerhalb, da sind sie weg, wie der Nebel hier.\n\nJetzt kommt `const`, das ist wie ein fester Stein, \nEin Wert, den du einmal setzt, der bleibt für immer dein. \nDu kannst ihn nicht verändern, das ist sein Gesetz, \nDoch bei Objekten und Arrays, da ist das nicht zuletzt!\n\nAlso merk dir gut, wenn du wechselst, nimm `let`, \nWenn's stabil bleiben soll, dann ist `const` dein Pet. \nSo bleibst du im Flow, wie ein Beat, der nicht bricht, \nMit diesen zwei Variablen, da verlierst du nichts!", "lang": "de"}, "rag": {"context_found": true, "sources_count": 3}, "ai": {"provider": "openrouter", "model": "openai/gpt-4o-mini"}, "session_id": "anonymous", "tokens_estimated": 210}
{"timestamp": "2025-12-07T17:38:14.197833Z", "character": {"id": "gitbadger", "name": "🦡 GitBadger"}, "user": {"id": 2, "role": "user"}, "interaction": {"question": "Wie speichere ich einen krümel im wald?", "answer": "Um einen Krümel im Wald zu speichern, also um Änderungen in deinem Git-Repository festzuhalten, folge diesen Schritten:\n\n1. **Änderungen vornehmen**: Bearbeite deine Dateien und füge die gewünschten Krümel (Änderungen) hinzu.\n\n2. **Änderungen zum Staging-Bereich hinzufügen**: Nutze den Befehl `git add <dateiname>` oder `git add .`, um alle Änderungen zum Staging-Bereich hinzuzufügen. Dies ist wie das Sammeln der Krümel, die du aufbewahren möchtest.\n\n3. **Commit durchführen**: Speichere die Änderungen in der Versionskontrolle mit `git commit -m \"Deine Nachricht hier\"`. Achte darauf, eine aussagekräftige Nachricht zu wählen, die die Geschichte des Krümels erzählt.\n\n4. **Änderungen ins Remote-Repository hochladen** (optional): Wenn du die Krümel auch im \"Wald\" (Remote-Repository) speichern möchtest, verwende den Befehl `git push origin <branch-name>`.\n\nDurch diese Schritte sammelst du die Krümel und bewahrst sie sicher im Wald, sodass sie jederzeit wieder aufblühen können.", "lang": "de"}, "rag": {"context_found": true, "sources_count": 3}, "ai": {"provider": "openrouter", "model": "openai/gpt-4o-mini"}, "session_id": "anonymous", "tokens_estimated": 255}

44
app/main.py
View File

@@ -18,6 +18,8 @@ from slowapi.errors import RateLimitExceeded
from deps import get_db, current_user # keine Kreis-Imports
from routers.admin_post import router as admin_posts_router
from routers.admin_rag import router as admin_rag_router
from routers.admin_logs import router as admin_logs_router
from routers.admin_vectors import router as admin_vectors_router
from routers.diary_rag import router as diary_rag_router
from routers.document_rag import router as document_rag_router
from routers.home import router as home_router
@@ -63,8 +65,22 @@ def init_templates(app: FastAPI):
# Gemeinsamer Kontext
path = req.url.path or "/"
seg = [p for p in path.split("/") if p]
lang = seg[0] if seg and seg[0] in ("de", "en") else (req.session.get("lang") or "de")
tail = "/" + "/".join(seg[1:]) if (seg and seg[0] in ("de", "en") and len(seg) > 1) else "/"
# Priority: 1. Query Param, 2. Path Prefix, 3. Session, 4. Default 'de'
# Also added "fr" to supported languages
query_lang = req.query_params.get("lang")
supported_langs = ("de", "en", "fr")
if query_lang and query_lang in supported_langs:
lang = query_lang
req.session["lang"] = lang # Persist selection
elif seg and seg[0] in supported_langs:
lang = seg[0]
req.session["lang"] = lang # Persist selection from path
else:
lang = req.session.get("lang") or "de"
tail = "/" + "/".join(seg[1:]) if (seg and seg[0] in supported_langs and len(seg) > 1) else "/"
user = req.session.get("user")
flashes = req.session.pop("_flashes", [])
@@ -89,7 +105,7 @@ def flash(req: Request, message: str, category: str = "info"):
def get_lang_from_path(path: str, default="de"):
seg = [p for p in (path or "/").split("/") if p]
return seg[0] if seg and seg[0] in ("de", "en") else default
return seg[0] if seg and seg[0] in ("de", "en", "fr") else default
# --- Health & Dev helpers ---
@app.get("/health")
@@ -122,8 +138,18 @@ def authenticated_home(req: Request, lang: str, user = Depends(current_user)):
return req.app.state.render(req, "pages/home.html", seo={"title": "Crumbforest", "desc": "Wuuuuhuuu!"})
# --- Login / Logout ---
# Explicit /login catch-all to prevent it matching /{lang}/login with lang="login"
@app.get("/login", include_in_schema=False)
def login_redirect(req: Request):
lang = req.session.get("lang") or "de"
return RedirectResponse(f"/{lang}/login", status_code=302)
@app.get("/{lang}/login", name="login_form", response_class=HTMLResponse)
def login_form(req: Request, lang: str):
# Prevent "login" as lang if it slipped through
if lang == "login":
return RedirectResponse("/de/login", status_code=302)
req.session["lang"] = lang
return req.app.state.render(req, "pages/login.html", seo={"title": "Login", "desc": ""})
@@ -159,7 +185,10 @@ def login_post(
"accessibility": row.get("accessibility")
}
flash(req, f"Welcome, {row['email']}", "success")
return RedirectResponse("/admin", status_code=302)
# Redirect based on role with preserved language
target = "/admin" if row["role"] == "admin" else "/crumbforest/roles"
return RedirectResponse(f"{target}?lang={lang}", status_code=302)
@app.post("/logout", name="logout")
def logout(req: Request):
@@ -188,11 +217,16 @@ def api_hello(req: Request, lang: Optional[str] = None):
# --- Router mounten (ohne Kreisimport) ---
app.include_router(admin_posts_router, prefix="/admin")
app.include_router(admin_rag_router, prefix="/admin/rag", tags=["RAG"])
app.include_router(admin_logs_router, prefix="/admin", tags=["Admin Logs"])
app.include_router(admin_vectors_router, prefix="/admin", tags=["Admin Vectors"])
app.include_router(diary_rag_router, prefix="/api/diary", tags=["Diary RAG"])
app.include_router(document_rag_router, prefix="/api/documents", tags=["Documents RAG"])
app.include_router(chat_router, tags=["Chat"])
app.include_router(chat_page_router, tags=["Chat"])
app.include_router(roles_router, tags=["Roles"])
# Mount home router last so it doesn't conflict with other routes
# Mount home router with lang prefix FIRST so it takes precedence
app.include_router(home_router, prefix="/{lang}", tags=["Home"])
# Mount home router at root
app.include_router(home_router, tags=["Home"])

60
app/routers/admin_logs.py Normal file
View File

@@ -0,0 +1,60 @@
"""
Admin Logs Router
Handles retrieval and display of chat logs and statistics for administrators.
"""
from fastapi import APIRouter, Depends, Query, Request, HTTPException
from fastapi.responses import HTMLResponse, JSONResponse
from typing import Optional, List, Dict, Any
from deps import current_user
from utils.chat_logger import ChatLogger
router = APIRouter()
@router.get("/logs", name="admin_logs_dashboard", response_class=HTMLResponse)
def logs_dashboard(req: Request, user = Depends(current_user)):
"""
Render the Admin Logs Dashboard.
"""
if not user:
from fastapi.responses import RedirectResponse
lang = req.session.get("lang", "de")
return RedirectResponse(f"/{lang}/login", status_code=302)
if user.get("role") != "admin":
return HTMLResponse("403 admin only", status_code=403)
return req.app.state.render(req, "pages/admin_logs.html", seo={"title": "Admin Logs", "desc": "Chat History & Stats"})
@router.get("/logs/data", name="admin_logs_data")
def get_logs_data(
req: Request,
limit: int = Query(100, ge=1, le=1000),
role: Optional[str] = None,
user: dict = Depends(current_user)
):
"""
Get raw log data (JSON).
"""
if user.get("role") != "admin":
raise HTTPException(status_code=403, detail="Admin access required")
logger = ChatLogger()
logs = logger.get_recent_logs(limit=limit)
# Optional filtering
if role:
logs = [l for l in logs if l.get('character', {}).get('id') == role]
return {"logs": logs}
@router.get("/logs/stats", name="admin_logs_stats")
def get_logs_stats(req: Request, user: dict = Depends(current_user)):
"""
Get log statistics (JSON).
"""
if user.get("role") != "admin":
raise HTTPException(status_code=403, detail="Admin access required")
logger = ChatLogger()
return logger.get_stats()

151
app/routers/admin_vectors.py Normal file
View File

@@ -0,0 +1,151 @@
"""
Admin Vectors Router
Direct access to Qdrant vector search for system administrators.
"""
from fastapi import APIRouter, Depends, Body, Request, HTTPException
from fastapi.responses import HTMLResponse
from typing import List, Optional
from deps import current_user, get_db, get_qdrant_client
from config import get_settings
from services.provider_factory import ProviderFactory
from services.rag_service import RAGService
router = APIRouter()
@router.get("/vectors", name="admin_vectors_dashboard", response_class=HTMLResponse)
def vectors_dashboard(req: Request, user = Depends(current_user)):
"""
Render vector search interface.
"""
if not user:
from fastapi.responses import RedirectResponse
lang = req.session.get("lang", "de")
return RedirectResponse(f"/{lang}/login", status_code=302)
if user.get("role") != "admin":
return HTMLResponse("403 admin only", status_code=403)
return req.app.state.render(req, "pages/admin_vectors.html", seo={"title": "Vector Brain", "desc": "Search Qdrant Directly"})
@router.post("/vectors/search", name="admin_vectors_search")
def search_vectors(
req: Request,
query: str = Body(..., embed=True),
collection: str = Body("crumbforest_posts", embed=True),
limit: int = Body(5, embed=True),
user = Depends(current_user)
):
if user.get("role") != "admin":
raise HTTPException(status_code=403, detail="Admin access required")
settings = get_settings()
db_conn = get_db()
qdrant_client = get_qdrant_client()
try:
# Get provider
provider = ProviderFactory.create_provider(
provider_name="openrouter", # Default to configured provider
settings=settings
)
# Use EmbeddingService for consistency
from services.embedding_service import EmbeddingService
embedding_service = EmbeddingService(provider)
query_vector = embedding_service.embed_texts([query])[0]
# 2. Search Qdrant directly
results = qdrant_client.search(
collection_name=collection,
query_vector=query_vector,
limit=limit,
with_payload=True,
with_vectors=False
)
# 3. Format
formatted = []
for hit in results:
formatted.append({
"score": hit.score,
"id": hit.id,
"payload": hit.payload,
"excerpt": hit.payload.get("content", "")[:200] + "..." if hit.payload.get("content") else "No content"
})
return {"results": formatted}
except Exception as e:
print(f"Vector search failed: {e}")
import traceback
traceback.print_exc()
return {"error": str(e)}
finally:
db_conn.close()
@router.get("/vectors/content/{post_id}")
async def get_vector_content(post_id: int, user = Depends(current_user)):
"""
Get full content of a specific post.
"""
if user.get("role") != "admin":
raise HTTPException(status_code=403, detail="Admin access required")
db_conn = get_db()
try:
from pymysql.cursors import DictCursor
# 1. Try SQL posts table
with db_conn.cursor(DictCursor) as cur:
cur.execute("SELECT content, title, slug, sources FROM posts WHERE id = %s", (post_id,))
row = cur.fetchone()
if row:
return JSONResponse(row)
# 2. If not in posts, check post_vectors (for file-based docs)
cur.execute("SELECT metadata, post_type FROM post_vectors WHERE post_id = %s LIMIT 1", (post_id,))
meta_row = cur.fetchone()
if meta_row and meta_row.get("post_type") == "document":
import json
import os
try:
meta = json.loads(meta_row["metadata"]) if isinstance(meta_row["metadata"], str) else meta_row["metadata"]
file_path = meta.get("file_path")
if file_path and os.path.exists(file_path):
with open(file_path, "r", encoding="utf-8") as f:
content = f.read()
return JSONResponse({
"content": content,
"title": meta.get("file_name", "Document"),
"slug": "file://" + file_path,
"sources": "FileSystem"
})
except Exception as e:
print(f"Error reading doc file: {e}")
# Fall through to 404
raise HTTPException(status_code=404, detail="Content not found in SQL database or FileSystem (Orphaned Vector?)")
except Exception as e:
print(f"Error fetching content: {e}")
raise HTTPException(status_code=500, detail=str(e))
@router.get("/vectors/collections", name="admin_vectors_collections")
def list_collections(user = Depends(current_user)):
"""
List available collections.
"""
if user.get("role") != "admin":
raise HTTPException(status_code=403, detail="Admin access required")
client = get_qdrant_client()
try:
collections = client.get_collections()
return {"collections": [c.name for c in collections.collections]}
except Exception as e:
return {"error": str(e), "collections": []}

31
app/routers/chat.py
View File

@@ -12,6 +12,7 @@ from slowapi.util import get_remote_address
from deps import get_qdrant_client
from config import get_settings
from services.provider_factory import ProviderFactory
from services.config_loader import ConfigLoader
from utils.rag_chat import RAGChatService
from utils.chat_logger import ChatLogger
from utils.security import PromptInjectionFilter
@@ -74,6 +75,31 @@ CHARACTERS = {
# More characters can be added here in the future
}
def get_characters():
"""
Get all available characters, merging hardcoded ones with config.
"""
# Start with hardcoded defaults
chars = CHARACTERS.copy()
# Load additional roles from config
try:
config = ConfigLoader.load_config()
roles_config = config.get('roles', {})
for role_id, role_data in roles_config.items():
# Skip if already exists (hardcoded takes precedence? or config?)
# Let's say config adds to or overrides if needed.
# Map config format to chat format
chars[role_id] = {
"name": role_data.get("name", role_id),
"prompt": role_data.get("system_prompt", "")
}
except Exception as e:
print(f"Error loading roles from config: {e}")
return chars
class ChatRequest(BaseModel):
"""Chat request model."""
@@ -105,7 +131,8 @@ async def chat_with_character(chat_request: ChatRequest, request: Request):
ChatResponse with answer and metadata
"""
# Validate character
if chat_request.character_id not in CHARACTERS:
available_chars = get_characters()
if chat_request.character_id not in available_chars:
raise HTTPException(
status_code=400,
detail=f"Unknown character: {chat_request.character_id}"
@@ -119,7 +146,7 @@ async def chat_with_character(chat_request: ChatRequest, request: Request):
detail=f"Invalid input: {error_msg}"
)
character_config = CHARACTERS[chat_request.character_id]
character_config = available_chars[chat_request.character_id]
# Get settings
settings = get_settings()

173
app/routers/crumbforest_roles.py
View File

@@ -1,10 +1,12 @@
from fastapi import APIRouter, Depends, Request, Form, HTTPException
from fastapi.responses import HTMLResponse, JSONResponse
from fastapi import APIRouter, Depends, Request, Form, HTTPException, Body
from fastapi.responses import HTMLResponse, JSONResponse, RedirectResponse
import httpx
from typing import Optional, List, Dict, Any
from app.deps import current_user
from app.services.config_loader import ConfigLoader
from app.config import get_settings
from pydantic import BaseModel
from deps import current_user
from services.config_loader import ConfigLoader
from services.localization import merge_role_localization
from config import get_settings
router = APIRouter()
settings = get_settings()
@@ -14,15 +16,22 @@ async def roles_dashboard(req: Request, user = Depends(current_user)):
"""
Show available roles based on user's group.
"""
# Determine user group (default to 'home' if not logged in, 'demo' if logged in but no group)
# Determine user group
if not user:
user_group = 'home'
else:
user_group = user.get('user_group', 'demo')
# Get Language
lang = req.query_params.get("lang") or req.session.get("lang") or "de"
req.session["lang"] = lang
# Load config
config = ConfigLoader.load_config()
all_roles = config.get('roles', {})
all_roles_raw = config.get('roles', {})
# Localize roles
all_roles = merge_role_localization(all_roles_raw, lang)
# Filter roles by group access
available_roles = {
@@ -38,7 +47,8 @@ async def roles_dashboard(req: Request, user = Depends(current_user)):
"crumbforest/roles_dashboard.html",
roles=available_roles,
user_group=user_group,
group_config=group_config
group_config=group_config,
lang=lang
)
@router.get("/crumbforest/roles/{role_id}", response_class=HTMLResponse)
@@ -46,8 +56,16 @@ async def role_chat(req: Request, role_id: str, user = Depends(current_user)):
"""
Chat interface for a specific role.
"""
# Get Language
lang = req.query_params.get("lang") or req.session.get("lang") or "de"
req.session["lang"] = lang
config = ConfigLoader.load_config()
role = config.get('roles', {}).get(role_id)
# Localize single role lookup
# Only need to localize the specific role, but easiest to localize all then pick
all_roles = merge_role_localization(config.get('roles', {}), lang)
role = all_roles.get(role_id)
if not role:
raise HTTPException(404, "Role not found")
@@ -62,7 +80,8 @@ async def role_chat(req: Request, role_id: str, user = Depends(current_user)):
if user_group not in role.get('group_access', []):
# Redirect to login if not logged in, else 403
if not user:
return RedirectResponse(f"/login?next={req.url.path}", status_code=302)
# Fix: Use /{lang}/login instead of /login to prevent matching {lang}=login
return RedirectResponse(f"/{lang}/login?next={req.url.path}", status_code=302)
raise HTTPException(403, "Access denied")
group_config = config.get('groups', {}).get(user_group, {})
@@ -71,7 +90,8 @@ async def role_chat(req: Request, role_id: str, user = Depends(current_user)):
req,
"crumbforest/role_chat.html",
role=role,
group_config=group_config
group_config=group_config,
lang=lang
)
@router.post("/crumbforest/roles/{role_id}/ask")
@@ -97,79 +117,96 @@ async def ask_role(
user_group = user.get('user_group', 'demo')
if user_group not in role.get('group_access', []):
raise HTTPException(403, "Access denied")
# Get Language from session (persisted by dashboard/chat page view)
lang = req.session.get("lang", "de")
# Merge localization for this role again to be sure (in case session changed or not applied)
# Actually, config loaded above might not be localized if we used ConfigLoader directly.
# Let's import merge_role_localization
from services.localization import merge_role_localization
localized_roles = merge_role_localization(config.get('roles', {}), lang)
localized_role = localized_roles.get(role_id)
if localized_role:
role = localized_role
# Get conversation history from session
history_key = f'role_history_{role_id}'
history = req.session.get(history_key, [])
# Build messages
messages = [
{"role": "system", "content": role['system_prompt']}
] + history + [
{"role": "user", "content": question}
]
# Call OpenRouter API
api_key = settings.openrouter_api_key
if not api_key:
return JSONResponse({
"answer": "⚠️ OpenRouter API Key missing in configuration.",
# Initialize RAG Service
from services.provider_factory import ProviderFactory
from utils.rag_chat import RAGChatService
from deps import get_qdrant_client
if not settings.openrouter_api_key:
return JSONResponse({
"answer": "⚠️ OpenRouter API Key missing.",
"error": "config_error"
}, status_code=500)
try:
async with httpx.AsyncClient() as client:
response = await client.post(
"https://openrouter.ai/api/v1/chat/completions",
headers={
"Authorization": f"Bearer {api_key}",
"Content-Type": "application/json",
"HTTP-Referer": "https://crumbforest.local", # Required by OpenRouter
"X-Title": "Crumbforest"
},
json={
"model": role['model'],
"temperature": role.get('temperature', 0.7),
"messages": messages
},
timeout=30.0
)
if response.status_code != 200:
print(f"OpenRouter Error: {response.text}")
return JSONResponse({
"answer": "⚠️ Error communicating with AI provider.",
"details": response.text
}, status_code=502)
result = response.json()
embedding_provider = ProviderFactory.create_provider("openrouter", settings)
completion_provider = ProviderFactory.create_provider("openrouter", settings)
qdrant_client = get_qdrant_client()
rag_service = RAGChatService(
qdrant_client=qdrant_client,
embedding_provider=embedding_provider,
completion_provider=completion_provider
)
answer = result['choices'][0]['message']['content']
# --- Code Execution Feature (Mock/Safe) ---
# If role has 'code_execution' feature and answer contains code block
if 'code_execution' in role.get('features', []) and '```' in answer:
# Simple mock execution for demonstration
# In production, this would need a secure sandbox (e.g. gVisor, Firecracker)
if "print(" in answer or "echo" in answer:
answer += "\n\n> [!NOTE]\n> **Code Execution Output** (Simulated):\n> ```\n> Hello from Crumbforest! 🌲\n> ```"
# Update history
# Generate answer with RAG
# We pass role['system_prompt'] which is now localized!
result = rag_service.chat_with_context(
question=question,
character_name=role['name'],
character_prompt=role['system_prompt'],
context_limit=3,
lang=lang,
model_override=role.get('model') # Support specific models per role
)
answer = result['answer']
# Update history (Session based for now)
history_key = f'role_history_{role_id}'
history = req.session.get(history_key, [])
history.append({"role": "user", "content": question})
history.append({"role": "assistant", "content": answer})
req.session[history_key] = history[-10:] # Keep last 10 exchanges
req.session[history_key] = history[-10:]
# Log interaction (for Admin Logs)
try:
from utils.chat_logger import ChatLogger
logger = ChatLogger()
session_id = req.session.get("session_id", "anonymous")
logger.log_interaction(
character_id=role_id,
character_name=role.get('name', role_id),
user_id=user.get("id") if user else "anonymous",
user_role=user.get("role") if user else "anonymous",
question=question,
answer=answer,
model=result.get("model", "unknown"),
provider=result.get("provider", "unknown"),
context_found=result.get("context_found", False),
sources_count=len(result.get("sources", [])),
lang=lang,
session_id=session_id
)
except Exception as log_err:
print(f"⚠️ Logger failed: {log_err}")
return JSONResponse({
"role": role_id,
"question": question,
"answer": answer,
"usage": result.get('usage', {})
"usage": {}, # RAG service might not return usage exactly same way, or we skip
"sources": result.get("sources", []) # Return sources!
})
except Exception as e:
print(f"Exception in ask_role: {e}")
return JSONResponse({
"answer": "⚠️ Internal server error.",
"answer": "⚠️ Internal server error (RAG).",
"details": str(e)
}, status_code=500)
@@ -218,7 +255,7 @@ async def update_theme(
# Update DB
try:
from app.deps import get_db
from ..deps import get_db
from pymysql.cursors import DictCursor
with get_db().cursor(DictCursor) as cur:
cur.execute("UPDATE users SET theme=%s WHERE id=%s", (theme, user['id']))
@@ -266,7 +303,7 @@ async def update_accessibility(
# Update DB
try:
from app.deps import get_db
from deps import get_db
from pymysql.cursors import DictCursor
with get_db().cursor(DictCursor) as cur:
cur.execute(

58
app/routers/home.py
View File

@@ -77,11 +77,14 @@ async def home_index(req: Request, lang: str = None):
)
@router.get("/about", response_class=HTMLResponse)
async def home_about(req: Request):
async def home_about(req: Request, lang: str = None):
"""
About / Mission page.
"""
lang = req.session.get("lang", "de")
if lang is None:
lang = req.session.get("lang", "de")
req.session["lang"] = lang
translations = load_translations(lang)
return req.app.state.render(
@@ -93,29 +96,45 @@ async def home_about(req: Request):
)
@router.get("/crew", response_class=HTMLResponse)
async def home_crew(req: Request):
async def home_crew(req: Request, lang: str = None):
"""
Crew / Characters page.
"""
lang = req.session.get("lang", "de")
if lang is None:
lang = req.session.get("lang", "de")
req.session["lang"] = lang
translations = load_translations(lang)
characters = load_characters(lang)
# Load roles from central config (15 roles)
from services.config_loader import ConfigLoader
from services.localization import merge_role_localization
config = ConfigLoader.load_config()
# Merge localized content using service
characters = merge_role_localization(config.get('roles', {}), lang)
user = req.session.get("user")
return req.app.state.render(
req,
"home/crew.html",
deployment=deployment_config,
characters=characters,
roles=characters,
t=translations,
lang=lang
lang=lang,
user=user
)
@router.get("/hardware", response_class=HTMLResponse)
async def home_hardware(req: Request):
async def home_hardware(req: Request, lang: str = None):
"""
Hardware information page.
"""
lang = req.session.get("lang", "de")
if lang is None:
lang = req.session.get("lang", "de")
req.session["lang"] = lang
translations = load_translations(lang)
return req.app.state.render(
@@ -127,11 +146,14 @@ async def home_hardware(req: Request):
)
@router.get("/software", response_class=HTMLResponse)
async def home_software(req: Request):
async def home_software(req: Request, lang: str = None):
"""
Software information page.
"""
lang = req.session.get("lang", "de")
if lang is None:
lang = req.session.get("lang", "de")
req.session["lang"] = lang
translations = load_translations(lang)
return req.app.state.render(
@@ -143,11 +165,14 @@ async def home_software(req: Request):
)
@router.get("/impressum", response_class=HTMLResponse)
async def home_impressum(req: Request):
async def home_impressum(req: Request, lang: str = None):
"""
Impressum / Legal notice page.
"""
lang = req.session.get("lang", "de")
if lang is None:
lang = req.session.get("lang", "de")
req.session["lang"] = lang
translations = load_translations(lang)
return req.app.state.render(
@@ -159,11 +184,14 @@ async def home_impressum(req: Request):
)
@router.get("/datenschutz", response_class=HTMLResponse)
async def home_datenschutz(req: Request):
async def home_datenschutz(req: Request, lang: str = None):
"""
Privacy policy / Datenschutz page.
"""
lang = req.session.get("lang", "de")
if lang is None:
lang = req.session.get("lang", "de")
req.session["lang"] = lang
translations = load_translations(lang)
return req.app.state.render(

16
app/services/config_loader.py
View File

@@ -11,15 +11,19 @@ class ConfigLoader:
def load_config(cls, force_reload: bool = False) -> Dict[str, Any]:
if cls._config is None or force_reload:
try:
# Try to find config in root or app root
paths_to_try = [CONFIG_PATH, os.path.join("..", CONFIG_PATH), os.path.join(os.getcwd(), CONFIG_PATH)]
paths_to_try = [CONFIG_PATH, os.path.join("..", CONFIG_PATH), "/config/crumbforest_config.json"]
found = False
for path in paths_to_try:
if os.path.exists(path):
with open(path, 'r', encoding='utf-8') as f:
cls._config = json.load(f)
found = True
break
try:
with open(path, 'r', encoding='utf-8') as f:
cls._config = json.load(f)
found = True
print(f"Loaded config from {path}")
break
except Exception as e:
print(f"Failed to load config from {path}: {e}")
continue
if not found:
print(f"Warning: {CONFIG_PATH} not found in {paths_to_try}")

18
app/services/document_indexer.py
View File

@@ -153,12 +153,22 @@ class DocumentIndexer:
)
# Index as post (using file path as ID)
# Use deterministic hash (MD5) instead of python's randomized hash()
path_hash = int(hashlib.md5(str(file_path).encode('utf-8')).hexdigest(), 16)
post_id = path_hash % (2**31) # Keep within signed 32-bit int range if needed by DB, or just use positive logic
result = rag_service.index_post(
post_id=hash(str(file_path)) % (2**31), # Convert path to int ID
post_id=post_id,
title=file_path.stem, # Filename without extension
slug=f"{category}/{file_path.stem}",
locale="", # Documents are locale-agnostic
body_md=content
body_md=content,
extra_payload={
"file_path": str(file_path),
"category": category,
"src": "filesystem"
},
force=force
)
# Update post_vectors to mark as document type
@@ -179,7 +189,7 @@ class DocumentIndexer:
'file_name': file_path.name
}),
file_hash,
hash(str(file_path)) % (2**31),
post_id,
result['collection']
)
)
@@ -199,7 +209,7 @@ class DocumentIndexer:
VALUES ('document_indexed', 'document', %s, NULL, %s)
""",
(
hash(str(file_path)) % (2**31),
post_id,
json.dumps({
'category': category,
'file_path': str(file_path),

62
app/services/localization.py Normal file
View File

@@ -0,0 +1,62 @@
import json
import os
import copy
from typing import Dict, Any, List
def load_characters(lang: str = "de") -> List[Dict[str, Any]]:
"""Load character data for given language."""
if lang not in ["de", "en", "fr"]:
lang = "de"
# Assume we are running from app root
characters_path = os.path.join('static', 'data', f'characters.{lang}.json')
try:
with open(characters_path, 'r', encoding='utf-8') as f:
return json.load(f)
except FileNotFoundError:
# Fallback to German
try:
with open(os.path.join('static', 'data', 'characters.de.json'), 'r', encoding='utf-8') as f:
return json.load(f)
except Exception:
return []
except Exception as e:
print(f"Error loading characters: {e}")
return []
def merge_role_localization(roles: Dict[str, Any], lang: str) -> Dict[str, Any]:
"""
Return a deep copy of roles with localized content merged in.
"""
localized_roles = copy.deepcopy(roles)
localized_list = load_characters(lang)
localized_map = {char.get('id'): char for char in localized_list}
# Legacy ID mapping
legacy_id_map = {
'funkfox': 'fox',
'schraubaer': 'schraubär',
'capacitoby': 'capacitobi',
'taichitaube': 'taichi',
'taichi': 'taichi'
}
for role_id, role in localized_roles.items():
lookup_id = legacy_id_map.get(role_id, role_id)
if lookup_id in localized_map:
l_data = localized_map[lookup_id]
if 'name' in l_data:
role['name'] = l_data['name']
if 'description' in l_data:
role['description'] = l_data['description']
if 'short' in l_data:
role['title'] = l_data['short'] # Map short to title if desired, or keep original title?
# In the previous step I mapped 'title' in the JSON file directly.
# If JSON has 'title', use it.
if 'title' in l_data:
role['title'] = l_data['title']
if 'system_prompt' in l_data:
role['system_prompt'] = l_data['system_prompt']
return localized_roles

69
app/services/rag_service.py
View File

@@ -79,7 +79,9 @@ class RAGService:
title: str,
slug: str,
locale: str,
body_md: str
body_md: str,
extra_payload: Optional[Dict[str, Any]] = None,
force: bool = False
) -> Dict[str, Any]:
"""
Index a single post into Qdrant.
@@ -90,7 +92,8 @@ class RAGService:
slug: Post slug
locale: Post locale (de, en, etc.)
body_md: Markdown content
extra_payload: Additional metadata to store in Qdrant
force: If True, bypass content hash check
Returns:
Dictionary with indexing results
"""
@@ -99,31 +102,28 @@ class RAGService:
# Check if already indexed and content unchanged
content_hash = self._get_content_hash(body_md or "")
with self.db_conn.cursor(DictCursor) as cur:
cur.execute(
"SELECT file_hash FROM post_vectors WHERE post_id=%s AND collection_name=%s",
(post_id, collection_name)
)
existing = cur.fetchone()
if existing and existing['file_hash'] == content_hash:
return {
'post_id': post_id,
'status': 'unchanged',
'message': 'Post content unchanged, skipping re-indexing'
}
if not force:
with self.db_conn.cursor(DictCursor) as cur:
cur.execute(
"SELECT file_hash FROM post_vectors WHERE post_id=%s AND collection_name=%s",
(post_id, collection_name)
)
existing = cur.fetchone()
if existing and existing['file_hash'] == content_hash:
return {
'post_id': post_id,
'status': 'unchanged',
'message': 'Post content unchanged, skipping re-indexing',
'collection': collection_name
}
# Chunk and embed the content
chunks_with_embeddings = self.embedding_service.chunk_and_embed_post(
post_content=body_md or "",
post_id=post_id,
post_title=title
)
if not chunks_with_embeddings:
return {
'post_id': post_id,
'status': 'error',
'message': 'No content to index'
'status': 'skipped',
'message': 'No content to index',
'collection': collection_name
}
# Prepare points for Qdrant
@@ -135,19 +135,24 @@ class RAGService:
point_id = str(uuid.uuid5(uuid.NAMESPACE_DNS, f"{post_id}_{chunk['chunk_position']}"))
vector_ids.append(point_id)
payload = {
'post_id': post_id,
'title': title,
'slug': slug,
'locale': locale,
'content': chunk['content'],
'header': chunk.get('header', ''),
'header_level': chunk.get('header_level', 0),
'chunk_position': chunk['chunk_position']
}
if extra_payload:
payload.update(extra_payload)
points.append(PointStruct(
id=point_id,
vector=chunk['embedding'],
payload={
'post_id': post_id,
'title': title,
'slug': slug,
'locale': locale,
'content': chunk['content'],
'header': chunk.get('header', ''),
'header_level': chunk.get('header_level', 0),
'chunk_position': chunk['chunk_position']
}
payload=payload
))
# Upsert points to Qdrant

4
app/static/css/pico.min.css vendored Normal file
View File

File diff suppressed because one or more lines are too long

12
app/static/css/screen_reader.css Normal file
View File

@@ -0,0 +1,12 @@
/* Screen Reader Only Styles */
.sr-only {
position: absolute;
width: 1px;
height: 1px;
padding: 0;
margin: -1px;
overflow: hidden;
clip: rect(0, 0, 0, 0);
white-space: nowrap;
border: 0;
}

93
app/templates/crumbforest/crew.html Normal file
View File

@@ -0,0 +1,93 @@
{% extends "base_public.html" %}
{% block title %}The Crumbforest Crew{% endblock %}
{% block content %}
<main class="container">
<hgroup>
<h1>🌲 The Crumbforest Crew</h1>
<p>Meet the 15 experts ready to help you learn and build.</p>
</hgroup>
<div class="crew-grid">
{% for id, role in roles.items() %}
<article class="role-card" style="border-left: 5px solid {{ role.color }};">
<header>
<div class="role-icon">{{ role.icon }}</div>
<h3 style="color: {{ role.color }}">{{ role.name }}</h3>
<small>{{ role.title }}</small>
</header>
<p>{{ role.description }}</p>
<footer>
{% if id == 'eule' %}
<!-- Eule is public -->
<a href="/crumbforest/roles/{{ id }}" role="button" class="contrast"
style="background-color: {{ role.color }}; border-color: {{ role.color }}; width: 100%;">
Start Chat
</a>
{% else %}
<!-- Others require login (or are locked) -->
{% if user %}
<a href="/crumbforest/roles/{{ id }}" role="button" class="outline"
style="color: {{ role.color }}; border-color: {{ role.color }}; width: 100%;">
Chat
</a>
{% else %}
<a href="/login?next=/crumbforest/roles/{{ id }}" role="button" class="secondary outline"
style="width: 100%;">
🔒 Login to Chat
</a>
{% endif %}
{% endif %}
</footer>
</article>
{% endfor %}
</div>
</main>
<style>
.crew-grid {
display: grid;
grid-template-columns: repeat(auto-fill, minmax(300px, 1fr));
gap: 1.5rem;
}
/* 3 Column Layout for larger screens to enforce 3x5 if roles are multiple of 3 */
@media (min-width: 992px) {
.crew-grid {
grid-template-columns: repeat(3, 1fr);
}
}
.role-card {
display: flex;
flex-direction: column;
height: 100%;
margin-bottom: 0;
/* Override pico */
transition: transform 0.2s ease, box-shadow 0.2s ease;
}
.role-card:hover {
transform: translateY(-5px);
box-shadow: 0 10px 20px rgba(0, 0, 0, 0.1);
}
.role-card header {
text-align: center;
padding-bottom: 0.5rem;
}
.role-icon {
font-size: 3rem;
margin-bottom: 0.5rem;
}
.role-card p {
flex-grow: 1;
/* Push footer down */
font-size: 0.9rem;
}
</style>
{% endblock %}

30
app/templates/crumbforest/role_chat.html
View File

@@ -5,7 +5,7 @@
{% block content %}
<main class="container">
<hgroup>
<h1>{{ role.icon }} {{ role.name }}</h1>
<h1 style="color: {{ role.color }};">{{ role.icon }} {{ role.name }}</h1>
<p>
{{ role.description }}
<a href="/crumbforest/roles/{{ role.id }}/export" role="button" class="outline secondary"
@@ -15,6 +15,18 @@
</p>
</hgroup>
<style>
:root {
--role-color: {
{
role.color
}
}
;
}
</style>
<!-- Prism.js for Syntax Highlighting -->
<link href="https://cdnjs.cloudflare.com/ajax/libs/prism/1.29.0/themes/prism-tomorrow.min.css" rel="stylesheet" />
<script src="https://cdnjs.cloudflare.com/ajax/libs/prism/1.29.0/prism.min.js"></script>
@@ -148,8 +160,22 @@
div.style.textAlign = 'left';
div.innerHTML = `<strong>You</strong><br>${formattedContent}`;
} else if (role === 'assistant') {
div.style.background = 'var(--pico-secondary-background)';
div.style.background = 'var(--role-color, var(--pico-secondary-background))';
div.style.color = 'white'; // Always white text on colored background? Or depends on brightness?
// Let's use a safe fallback or keep it simple. If role color is light, white text might be bad.
// But most config colors seem dark enough?
// Dumbo is #6c757d (grey), SnakePy #3776ab (blue). White text is fine.
// FunkFox #f7df1e (yellow) -> White text is BAD.
// We might need a contrast color from config or calc().
// For now, let's stick to the pico variable for text if we use role color for border?
// Or use an opacity version?
// Let's try attempting to use the role color as a border-left accent instead, to be safe with text contrast?
div.style.background = 'var(--pico-card-background-color)';
div.style.borderLeft = '5px solid var(--role-color, var(--pico-primary))';
div.style.color = 'var(--pico-color)';
div.style.marginRight = 'auto'; // Align left
div.innerHTML = `<strong>{{ role.name }}</strong><br>${formattedContent}`;
} else {

70
app/templates/crumbforest/roles_dashboard.html
View File

@@ -9,15 +9,13 @@
<p>Choose your learning companion!</p>
</hgroup>
<div class="grid">
<div class="roles-grid">
{% for role_id, role in roles.items() %}
<article class="card">
<article class="role-card" style="border-left: 5px solid {{ role.color }};">
<header>
<h3>
<span style="font-size: 2em; margin-right: 0.5rem;">{{ role.icon }}</span>
{{ role.name }}
</h3>
<p class="text-muted"><small>{{ role.title }}</small></p>
<div class="role-icon">{{ role.icon }}</div>
<h3 style="color: {{ role.color }}">{{ role.name }}</h3>
<small>{{ role.title }}</small>
</header>
<p>{{ role.description }}</p>
@@ -26,13 +24,14 @@
<div style="display: flex; gap: 0.5rem; flex-wrap: wrap; margin-bottom: 1rem;">
{% for tag in role.tags %}
<span class="badge"
style="background-color: {{ role.color }}20; color: {{ role.color }}; border: 1px solid {{ role.color }}40;">
style="background-color: {{ role.color }}20; color: {{ role.color }}; border: 1px solid {{ role.color }}40; font-size: 0.7em; padding: 2px 6px; border-radius: 4px;">
#{{ tag }}
</span>
{% endfor %}
</div>
<a href="/crumbforest/roles/{{ role_id }}" role="button" style="width: 100%;">
<a href="/crumbforest/roles/{{ role_id }}" role="button" class="outline"
style="width: 100%; color: {{ role.color }}; border-color: {{ role.color }};">
Chat with {{ role.name }}
</a>
</footer>
@@ -40,4 +39,57 @@
{% endfor %}
</div>
</main>
<style>
.roles-grid {
display: grid;
grid-template-columns: 1fr;
gap: 1.5rem;
}
/* Tablet: 2 columns */
@media (min-width: 768px) {
.roles-grid {
grid-template-columns: repeat(2, 1fr);
}
}
/* Desktop: Strictly 3 columns */
@media (min-width: 1024px) {
.roles-grid {
grid-template-columns: repeat(3, 1fr);
}
}
.role-card {
display: flex;
flex-direction: column;
height: 100%;
margin-bottom: 0;
transition: transform 0.2s ease, box-shadow 0.2s ease;
border-radius: 1rem;
box-shadow: 0 4px 6px rgba(0, 0, 0, 0.1);
overflow: hidden;
}
.role-card:hover {
transform: translateY(-5px);
box-shadow: 0 10px 20px rgba(0, 0, 0, 0.1);
}
.role-card header {
text-align: center;
padding-bottom: 0.5rem;
}
.role-icon {
font-size: 3rem;
margin-bottom: 0.5rem;
}
.role-card p {
flex-grow: 1;
font-size: 0.9rem;
}
</style>
{% endblock %}

260
app/templates/home/crew.html
View File

@@ -1,201 +1,103 @@
{% extends "home/base_home.html" %}
{% block title %}Crew - {{ deployment.home.hero.title }}{% endblock %}
{% block title %}The Crumbforest Crew{% endblock %}
{% block content %}
<main class="container">
<hgroup>
<h1>{{ t.crew.title }}</h1>
<p>{{ t.crew.subtitle }}</p>
<h1>🌲 The Crumbforest Crew</h1>
<p>Meet the 15 experts ready to help you learn and build.</p>
</hgroup>
<div class="crew-grid">
{% for character in characters %}
<div class="character-card" onclick="showCharacter('{{ character.id }}')">
<div class="icon">{{ character.icon }}</div>
<h3>{{ character.name }}</h3>
<p>{{ character.short }}</p>
</div>
<!-- Dialog for this character -->
<dialog id="dialog-{{ character.id }}">
<article>
<header>
<button aria-label="Close" rel="prev" onclick="closeCharacter('{{ character.id }}')"></button>
<h2>{{ character.icon }} {{ character.name }}</h2>
</header>
<p>{{ character.description }}</p>
{% if character.id in ['eule', 'fox', 'bugsy'] %}
<!-- Chat Interface for Eule, FunkFox and Bugsy -->
<div class="chat-container" style="margin-top: 1rem;">
<div id="chat-history-{{ character.id }}" class="chat-history" style="max-height: 300px; overflow-y: auto; padding: 1rem; background: rgba(0,0,0,0.1); border-radius: 0.5rem; margin-bottom: 1rem;">
<p style="text-align: center; color: #888;"><small>Stelle mir eine Frage...</small></p>
</div>
<form id="chat-form-{{ character.id }}" onsubmit="sendMessage(event, '{{ character.id }}'); return false;">
<input
type="text"
id="chat-input-{{ character.id }}"
placeholder="{% if character.id == 'eule' %}Deine Frage an die Krümeleule...{% elif character.id == 'fox' %}Deine Frage an FunkFox...{% elif character.id == 'bugsy' %}Deine Frage an Bugsy...{% else %}Deine Frage...{% endif %}"
style="width: 100%; color: #fff; background: rgba(255,255,255,0.1); padding: 0.75rem; border: 1px solid rgba(255,255,255,0.2); border-radius: 0.25rem; margin-bottom: 0.5rem;"
autocomplete="off"
required
/>
<div style="text-align: right;">
<button
type="submit"
id="chat-submit-{{ character.id }}"
style="padding: 0.5rem 1.5rem; font-size: 0.875rem;"
>
Senden
</button>
</div>
</form>
<div id="chat-status-{{ character.id }}" style="margin-top: 0.5rem; text-align: center; min-height: 1.5rem;">
<!-- Status messages appear here -->
</div>
</div>
{% for id, role in roles.items() %}
<article class="role-card" style="border-left: 5px solid {{ role.color }};">
<header>
<div class="role-icon">{{ role.icon }}</div>
<h3 style="color: {{ role.color }}">{{ role.name }}</h3>
<small>{{ role.title }}</small>
</header>
<p>{{ role.description }}</p>
<footer>
{% if id == 'eule' %}
<!-- Eule is public -->
<a href="/crumbforest/roles/{{ id }}" role="button" class="contrast"
style="background-color: {{ role.color }}; border-color: {{ role.color }}; width: 100%;">
Start Chat
</a>
{% else %}
<!-- Others require login (or are locked) -->
{% if user %}
<a href="/crumbforest/roles/{{ id }}" role="button" class="outline"
style="color: {{ role.color }}; border-color: {{ role.color }}; width: 100%;">
Chat
</a>
{% else %}
<a href="/login?next=/crumbforest/roles/{{ id }}" role="button" class="secondary outline" style="width: 100%;">
🔒 Login to Chat
</a>
{% endif %}
<footer>
<small>{{ t.crew.tags_label }}
{% for tag in character.tags %}
<code>#{{ tag }}</code>{% if not loop.last %}, {% endif %}
{% endfor %}
</small>
</footer>
</article>
</dialog>
{% endif %}
</footer>
</article>
{% endfor %}
</div>
</main>
{% endblock %}
{% block extra_js %}
<script>
function showCharacter(id) {
const dialog = document.getElementById('dialog-' + id);
dialog.showModal();
// Focus input field for chat-enabled characters
if (id === 'eule' || id === 'fox' || id === 'bugsy') {
setTimeout(() => {
const input = document.getElementById('chat-input-' + id);
if (input) {
input.focus();
}
}, 100);
<style>
.crew-grid {
display: grid;
grid-template-columns: 1fr;
gap: 1.5rem;
}
}
function closeCharacter(id) {
document.getElementById('dialog-' + id).close();
}
async function sendMessage(event, characterId) {
event.preventDefault();
const inputEl = document.getElementById('chat-input-' + characterId);
const historyEl = document.getElementById('chat-history-' + characterId);
const statusEl = document.getElementById('chat-status-' + characterId);
const submitBtn = document.getElementById('chat-submit-' + characterId);
const question = inputEl.value.trim();
if (!question) return;
// Disable input while processing
inputEl.disabled = true;
submitBtn.disabled = true;
const statusMessages = {
'eule': '🦉 Die Eule lauscht...',
'fox': '🦊 FunkFox droppt den Beat...',
'bugsy': '🐞 Bugsy analysiert...'
};
statusEl.innerHTML = '<small style="color: #888;">' + (statusMessages[characterId] || 'Denkt nach...') + '</small>';
// Add user message to history
const userMsg = document.createElement('div');
userMsg.style.cssText = 'margin-bottom: 0.5rem; padding: 0.5rem; background: rgba(16,185,129,0.1); border-radius: 0.25rem; text-align: right;';
userMsg.innerHTML = '<strong>Du:</strong> ' + escapeHtml(question);
historyEl.appendChild(userMsg);
// Clear welcome message if present
if (historyEl.querySelector('p')) {
const welcomeMsg = historyEl.querySelector('p');
if (welcomeMsg.textContent.includes('Stelle mir eine Frage')) {
welcomeMsg.remove();
/* Tablet: 2 columns */
@media (min-width: 768px) {
.crew-grid {
grid-template-columns: repeat(2, 1fr);
}
}
// Scroll to bottom
historyEl.scrollTop = historyEl.scrollHeight;
// Clear input
inputEl.value = '';
try {
// Call API
const response = await fetch('/api/chat', {
method: 'POST',
headers: {
'Content-Type': 'application/json'
},
body: JSON.stringify({
character_id: characterId,
question: question,
lang: '{{ lang }}'
})
});
if (!response.ok) {
throw new Error('API request failed: ' + response.status);
/* Desktop: Strictly 3 columns */
@media (min-width: 1024px) {
.crew-grid {
grid-template-columns: repeat(3, 1fr);
}
const data = await response.json();
// Add assistant message to history
const characterNames = {
'eule': '🦉 Krümeleule',
'fox': '🦊 FunkFox',
'bugsy': '🐞 Bugsy'
};
const assistantMsg = document.createElement('div');
assistantMsg.style.cssText = 'margin-bottom: 0.5rem; padding: 0.5rem; background: rgba(255,255,255,0.05); border-radius: 0.25rem;';
assistantMsg.innerHTML = '<strong>' + (characterNames[characterId] || characterId) + ':</strong><br>' + escapeHtml(data.answer);
// Add sources if available
if (data.sources && data.sources.length > 0) {
const sourcesText = document.createElement('small');
sourcesText.style.cssText = 'color: #888; display: block; margin-top: 0.5rem;';
sourcesText.innerHTML = '📚 Quellen: ' + data.sources.length + ' Dokumente';
assistantMsg.appendChild(sourcesText);
}
historyEl.appendChild(assistantMsg);
// Scroll to bottom
historyEl.scrollTop = historyEl.scrollHeight;
// Clear status
statusEl.innerHTML = '';
} catch (error) {
console.error('Chat error:', error);
statusEl.innerHTML = '<small style="color: #f87171;">❌ Fehler: ' + escapeHtml(error.message) + '</small>';
} finally {
// Re-enable input
inputEl.disabled = false;
submitBtn.disabled = false;
inputEl.focus();
}
}
function escapeHtml(text) {
const div = document.createElement('div');
div.textContent = text;
return div.innerHTML;
}
</script>
{% endblock %}
.role-card {
display: flex;
flex-direction: column;
height: 100%;
margin-bottom: 0;
/* Override pico */
transition: transform 0.2s ease, box-shadow 0.2s ease;
border-radius: 1rem;
box-shadow: 0 4px 6px rgba(0, 0, 0, 0.1);
overflow: hidden;
/* Ensure rounded corners clip content */
}
.role-card:hover {
transform: translateY(-5px);
box-shadow: 0 10px 20px rgba(0, 0, 0, 0.1);
}
.role-card header {
text-align: center;
padding-bottom: 0.5rem;
}
.role-icon {
font-size: 3rem;
margin-bottom: 0.5rem;
}
.role-card p {
flex-grow: 1;
/* Push footer down */
font-size: 0.9rem;
}
</style>
{% endblock %}

18
app/templates/pages/admin.html
View File

@@ -60,6 +60,22 @@
</footer>
</article>
<!-- Logs & Stats -->
<article class="card">
<header>
<h3>📈 Logs & Stats</h3>
</header>
<p>
Monitor chat comparisons, token usage, and role interactions.
</p>
<footer>
<div style="display: flex; gap: 0.5rem; flex-wrap: wrap;">
<a href="/admin/logs" role="button" class="secondary">View Logs</a>
<a href="/admin/vectors" role="button" class="outline">Search Brain 🧠</a>
</div>
</footer>
</article>
<!-- System Status -->
<article class="card">
<header>
@@ -138,4 +154,4 @@
</article>
</section>
{% endblock %}
{% endblock %}

204
app/templates/pages/admin_logs.html Normal file
View File

@@ -0,0 +1,204 @@
{% extends "base.html" %}
{% block content %}
<!-- Header -->
<section class="mb-2">
<div class="grid">
<div>
<h1>System Logs & Stats</h1>
<p class="text-muted">Monitor chat interactions and token usage.</p>
</div>
<div style="text-align: right;">
<a href="/admin" role="button" class="secondary outline">← Back to Dashboard</a>
</div>
</div>
</section>
<!-- Stats Grid -->
<section class="grid" id="stats-container">
<article class="card">
<header>Total Interactions</header>
<h2 id="stat-total-interactions">-</h2>
</article>
<article class="card">
<header>Vector Coverage</header>
<h2 id="stat-vector-coverage">-</h2>
<small class="text-muted" id="stat-vector-count">-</small>
</article>
<article class="card">
<header>Estimated Cost</header>
<h2 id="stat-cost">-</h2>
<small class="text-muted" id="stat-tokens">-</small>
</article>
<article class="card">
<header>Log Size</header>
<h2 id="stat-file-size">-</h2>
</article>
</section>
<!-- Role Stats -->
<section>
<h3>Role Usage</h3>
<div id="role-stats" class="grid-4">
<!-- Will be populated by JS -->
<article class="loading">Loading stats...</article>
</div>
</section>
<!-- Logs Table -->
<section style="margin-top: 2rem;">
<div class="grid">
<h3>Recent Interactions</h3>
<div style="text-align: right;">
<button onclick="loadData()" class="outline small">↻ Refresh</button>
</div>
</div>
<figure>
<table role="grid">
<thead>
<tr>
<th scope="col">Time</th>
<th scope="col">Role</th>
<th scope="col">User</th>
<th scope="col">Interaction</th>
<th scope="col">Tokens</th>
<th scope="col">Model</th>
</tr>
</thead>
<tbody id="logs-table-body">
<tr>
<td colspan="6" style="text-align: center;">Loading logs...</td>
</tr>
</tbody>
</table>
</figure>
</section>
<script>
async function loadData() {
try {
await Promise.all([loadStats(), loadLogs()]);
} catch (e) {
console.error("Error loading data:", e);
}
}
async function loadStats() {
const res = await fetch('/admin/logs/stats');
if (!res.ok) return;
const data = await res.json();
document.getElementById('stat-total-interactions').innerText = data.total_interactions;
// Vector Coverage
const rate = data.context_hit_rate_percent;
document.getElementById('stat-vector-coverage').innerText = rate + "%";
document.getElementById('stat-vector-count').innerText = `${data.context_found_count} hits`;
// Cost & Tokens
document.getElementById('stat-cost').innerText = "$" + data.estimated_cost_usd.toFixed(4);
document.getElementById('stat-tokens').innerText = `~${data.total_tokens_estimated} tokens`;
document.getElementById('stat-file-size').innerText = data.file_size_mb + " MB";
// Render Role Stats
const roleContainer = document.getElementById('role-stats');
roleContainer.innerHTML = '';
// Sort roles by usage (desc)
const sortedRoles = Object.entries(data.characters).sort((a, b) => b[1] - a[1]);
sortedRoles.forEach(([roleId, count]) => {
const tokens = data.tokens_by_role[roleId] || 0;
const article = document.createElement('article');
article.className = 'card small-card';
article.innerHTML = `
<header class="capitalize"><strong>${roleId}</strong></header>
<div class="stat-value">${count} <small>chats</small></div>
<div class="stat-sub">${tokens} <small>tokens</small></div>
`;
roleContainer.appendChild(article);
});
}
async function loadLogs() {
const res = await fetch('/admin/logs/data?limit=50');
if (!res.ok) return;
const data = await res.json();
const tbody = document.getElementById('logs-table-body');
tbody.innerHTML = '';
data.logs.forEach(log => {
const row = document.createElement('tr');
// Format time
const date = new Date(log.timestamp);
const timeStr = date.toLocaleString();
// Truncate text
const q = log.interaction.question;
const qShort = q.length > 50 ? q.substring(0, 50) + '...' : q;
row.innerHTML = `
<td><small>${timeStr}</small></td>
<td class="capitalize">${log.character.name}</td>
<td>${log.user.email || log.user.id || 'Anonymous'}</td>
<td>
<strong>Q:</strong> ${escapeHtml(qShort)}<br>
<small class="text-muted">Sources: ${log.rag.sources_count}</small>
</td>
<td>${log.tokens_estimated}</td>
<td><small>${log.ai.model}</small></td>
`;
tbody.appendChild(row);
});
}
function escapeHtml(text) {
if (!text) return '';
return text
.replace(/&/g, "&amp;")
.replace(/</g, "&lt;")
.replace(/>/g, "&gt;")
.replace(/"/g, "&quot;")
.replace(/'/g, "&#039;");
}
// Initial load
document.addEventListener('DOMContentLoaded', loadData);
// Auto-refresh every 30s
setInterval(loadData, 30000);
</script>
<style>
.capitalize {
text-transform: capitalize;
}
.small-card {
padding: 1rem;
text-align: center;
}
.stat-value {
font-size: 1.5rem;
font-weight: bold;
}
.stat-sub {
font-size: 0.9rem;
color: var(--muted-color);
}
/* Grid 4 for role stats */
.grid-4 {
display: grid;
grid-template-columns: repeat(auto-fit, minmax(150px, 1fr));
gap: 1rem;
}
</style>
{% endblock %}

219
app/templates/pages/admin_vectors.html Normal file
View File

@@ -0,0 +1,219 @@
{% extends "base.html" %}
{% block content %}
<!-- Header -->
<section class="mb-2">
<div class="grid">
<div>
<h1>🧠 Brain Search (Vector Admin)</h1>
<p class="text-muted">Inspect the knowledge base by searching vectors directly.</p>
</div>
<div style="text-align: right;">
<a href="/admin" role="button" class="secondary outline">← Back to Dashboard</a>
</div>
</div>
</section>
<!-- Search Controls -->
<section class="card">
<div class="grid">
<!-- Collection Select -->
<label for="collection">
Collection
<select id="collection-select" onchange="performSearch()">
<option value="" disabled selected>Loading...</option>
</select>
</label>
<!-- Search Input -->
<label for="query">
Search Query
<input type="search" id="query-input" placeholder="e.g. 'Wer ist die Eule?'"
onkeydown="if(event.key==='Enter') performSearch()">
</label>
<!-- Action -->
<label>
&nbsp; <!-- Spacer -->
<button onclick="performSearch()">Search Brain 🔍</button>
</label>
</div>
</section>
<!-- Results -->
<section style="margin-top: 2rem;">
<h3 id="results-header">Results</h3>
<div id="results-container">
<!-- Results injected here -->
<article class="card">
<p class="text-muted text-center">Enter a query to search the vector database.</p>
</article>
</div>
</section>
<!-- Content Modal -->
<dialog id="contentModal">
<article style="min-width: 70vw; min-height: 50vh;">
<header>
<button aria-label="Close" rel="prev" onclick="closeModal()"></button>
<p>
<strong>📄 Source Content</strong>
<span id="modalTitle" style="margin-left: 1rem; opacity: 0.7;"></span>
</p>
</header>
<div id="modalBody" style="white-space: pre-wrap; font-family: monospace; max-height: 60vh; overflow-y: auto;">
Loading...
</div>
</article>
</dialog>
<script>
async function loadCollections() {
try {
const res = await fetch('/admin/vectors/collections');
const data = await res.json();
const select = document.getElementById('collection-select');
// Clear loading option
select.innerHTML = '';
if (data.collections.length === 0) {
const opt = document.createElement('option');
opt.text = "No collections found";
select.add(opt);
return;
}
let selectedOne = false;
// Handle both string array and object array if format changes
data.collections.forEach((c, index) => {
const name = typeof c === 'string' ? c : c.name;
const opt = document.createElement('option');
opt.value = name;
opt.text = name;
// Auto-select something that looks like main content
if (!selectedOne && (name.includes('post') || name.includes('crumbforest') || index === 0)) {
opt.selected = true;
selectedOne = true;
}
select.add(opt);
});
} catch (e) {
console.error("Error loading collections:", e);
}
}
async function performSearch() { // Renamed from searchVectors to match onchange/onclick
const query = document.getElementById('query-input').value;
const collection = document.getElementById('collection-select').value;
const resultsDiv = document.getElementById('results-container');
if (!query) return;
resultsDiv.innerHTML = '<article aria-busy="true">Searching...</article>';
try {
const res = await fetch('/admin/vectors/search', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ query, collection, limit: 10 })
});
if (!res.ok) throw new Error(await res.text());
const data = await res.json();
if (data.error) throw new Error(data.error);
const results = data.results || [];
if (results.length === 0) {
resultsDiv.innerHTML = '<article>No results found.</article>';
return;
}
const html = results.map(r => {
const payload = r.payload || {};
const source = payload.title || payload.slug || 'Unknown Source';
// Highlight matches? Maybe later.
const content = payload.content ? payload.content.substring(0, 150).replace(/</g, "&lt;") + "..." : "No content preview";
const scoreColor = r.score > 0.8 ? 'green' : (r.score > 0.7 ? 'orange' : 'red');
const postId = payload.post_id || payload.id; // Try both
const giteaBase = 'https://194-164-194-191.sslip.io/git/kruemel/Crumb-Core-v.1/src/branch/main/';
const giteaLink = payload.file_path ? (giteaBase + payload.file_path) : null;
return `
<article class="card" style="margin-bottom: 1rem;">
<header>
<strong style="color: ${scoreColor}">${(r.score * 100).toFixed(1)}% Match</strong> &nbsp;
<span class="badge secondary">${source}</span>
${postId ? `<button class="outline contrast" style="float: right; padding: 2px 10px; font-size: 0.8em;" onclick="viewContent(${postId}, '${source}')">👀 View Full</button>` : ''}
${giteaLink ? `<button class="outline secondary" style="float: right; padding: 2px 10px; font-size: 0.8em; margin-right: 0.5rem;" onclick="window.open('${giteaLink}', '_blank')">🐙 Gitea</button>` : ''}
</header>
<blockquote>${content}</blockquote>
<details>
<summary><small>View Full Payload (JSON)</small></summary>
<pre><code>${JSON.stringify(payload, null, 2)}</code></pre>
</details>
</article>
`;
}).join('');
resultsDiv.innerHTML = html;
} catch (e) {
resultsDiv.innerHTML = `<article class="headings"><p style="color:var(--pico-del-color)">Error: ${e.message}</p></article>`;
}
}
async function viewContent(postId, title) {
if (!postId) {
alert("No Post ID available.");
return;
}
const modal = document.getElementById('contentModal');
const modalTitle = document.getElementById('modalTitle');
const modalBody = document.getElementById('modalBody');
modalTitle.innerText = title;
modalBody.innerText = "Loading...";
modal.showModal();
try {
const res = await fetch(`/admin/vectors/content/${postId}`);
if (!res.ok) throw new Error("Content not found (might be deleted or orphaned)");
const data = await res.json();
modalBody.innerText = data.content || "No content text.";
} catch (e) {
modalBody.innerHTML = `<span style="color: red;">Error: ${e.message}</span>
<br><small>This confirms the vector is 'Orphaned' (exists in brain, but body is gone).</small>`;
}
}
function closeModal() {
document.getElementById('contentModal').close();
}
// Auto-load collections
loadCollections();
</script>
<style>
.text-center {
text-align: center;
}
blockquote {
border-left: 4px solid var(--primary);
padding-left: 1rem;
background: rgba(0, 0, 0, 0.05);
padding: 0.5rem;
}
</style>
{% endblock %}

60
app/trigger_reindex.py Normal file
View File

@@ -0,0 +1,60 @@
import sys
import os
sys.path.insert(0, '/app')
from deps import get_db, get_qdrant_client
from config import get_settings
from services.provider_factory import ProviderFactory
from services.rag_service import RAGService
from services.document_indexer import DocumentIndexer
def deep_clean():
print("🧹 Starting Deep Clean & Re-index...")
db_conn = get_db()
qdrant = get_qdrant_client()
settings = get_settings()
try:
# 1. Setup Provider
provider = ProviderFactory.create_provider(
provider_name=settings.default_embedding_provider,
settings=settings
)
print(f"✅ Using provider: {provider.provider_name} ({provider.model_name})")
# 2. Clear Collections (Optional, but good for orphans)
# Note: This might be dangerous if production. But for dev/fix it's essential.
# Collections: "posts_de", "posts_en", "docs_crumbforest", etc.
collections = qdrant.get_collections().collections
for col in collections:
print(f"🗑️ Deleting collection: {col.name}")
qdrant.delete_collection(col.name)
# 3. Re-scan Documents
print("📂 Indexing Documents (Files)...")
indexer = DocumentIndexer(db_conn, qdrant, provider)
# Force re-index to ensure new IDs are used
doc_results = indexer.index_all_categories(force=True)
print(f" Indexed {doc_results['total_indexed']} documents.")
# 4. Re-index Posts (DB)
print("💾 Indexing Posts (SQL)...")
rag = RAGService(db_conn, qdrant, provider)
# Index for supported locales
for loc in ["de", "en"]: # or from config
res = rag.index_all_posts(locale=loc)
print(f" Locale {loc}: {res['indexed']} posts indexed.")
print("✨ Deep Clean Complete!")
except Exception as e:
print(f"❌ Error: {e}")
import traceback
traceback.print_exc()
finally:
db_conn.close()
if __name__ == "__main__":
deep_clean()

44
app/utils/chat_logger.py
View File

@@ -175,21 +175,59 @@ class ChatLogger:
# Get file size
file_size = self.log_file.stat().st_size
# Count by character
# Count stats
character_counts = {}
total_tokens = 0
tokens_by_model = {}
tokens_by_role = {}
context_found_count = 0
# Simple pricing model (Blended average for OpenRouter)
# Input: ~$5/M, Output: ~$15/M -> Avg ~$10/M = $0.00001 per token
PRICE_PER_TOKEN = 0.00001
for line in lines:
try:
entry = json.loads(line)
# Character stats
char_id = entry.get('character', {}).get('id', 'unknown')
character_counts[char_id] = character_counts.get(char_id, 0) + 1
# Token stats
tokens = entry.get('tokens_estimated', 0)
total_tokens += tokens
# Model stats
model = entry.get('ai', {}).get('model', 'unknown')
tokens_by_model[model] = tokens_by_model.get(model, 0) + tokens
# Role stats
tokens_by_role[char_id] = tokens_by_role.get(char_id, 0) + tokens
# RAG stats
if entry.get('rag', {}).get('context_found'):
context_found_count += 1
except json.JSONDecodeError:
continue
total_interactions = len(lines)
context_hit_rate = round((context_found_count / total_interactions * 100), 1) if total_interactions > 0 else 0
estimated_cost = round(total_tokens * PRICE_PER_TOKEN, 4)
return {
"total_interactions": len(lines),
"total_interactions": total_interactions,
"total_tokens_estimated": total_tokens,
"estimated_cost_usd": estimated_cost,
"context_found_count": context_found_count,
"context_hit_rate_percent": context_hit_rate,
"file_size_bytes": file_size,
"file_size_mb": round(file_size / (1024 * 1024), 2),
"characters": character_counts
"characters": character_counts,
"tokens_by_model": tokens_by_model,
"tokens_by_role": tokens_by_role,
"last_updated": datetime.utcnow().isoformat() + "Z"
}
except Exception as e:

19
app/utils/rag_chat.py
View File

@@ -106,7 +106,8 @@ class RAGChatService:
character_name: str,
character_prompt: str,
context_limit: int = 3,
lang: str = "de"
lang: str = "de",
model_override: Optional[str] = None
) -> Dict[str, Any]:
"""
Answer a question using RAG with character persona.
@@ -117,6 +118,7 @@ class RAGChatService:
character_prompt: Character system prompt
context_limit: Number of context chunks to retrieve
lang: Language code
model_override: Optional model name to override default
Returns:
Dictionary with answer and sources
@@ -156,7 +158,8 @@ class RAGChatService:
# Generate answer with character persona
answer = self._get_completion_with_system(
system_message=system_message,
user_message=question
user_message=question,
model_override=model_override
)
# Format sources
@@ -175,13 +178,14 @@ class RAGChatService:
'sources': sources,
'context_found': len(search_results) > 0,
'provider': self.completion_provider.provider_name,
'model': self.completion_provider.model_name
'model': model_override or self.completion_provider.model_name
}
def _get_completion_with_system(
self,
system_message: str,
user_message: str
user_message: str,
model_override: Optional[str] = None
) -> str:
"""
Get completion with explicit system message.
@@ -190,6 +194,7 @@ class RAGChatService:
Args:
system_message: System prompt with character + context
user_message: User's question
model_override: Optional model name to use
Returns:
Generated response
@@ -200,7 +205,11 @@ class RAGChatService:
# Call completion provider
# Note: We pass the system message as context
# Assuming provider.get_completion supports model argument?
# If not, we might need to check BaseProvider.
# But get_completion usually takes keyword args passed to API.
return self.completion_provider.get_completion(
prompt=user_message,
context=system_message
context=system_message,
model=model_override
)