Context handoff stored — Part 4

Totally get it—this is a paste/terminal heredoc hiccup, not you. Let’s avoid heredocs and nano entirely and have Python write the file for you in one shot.

Do this exactly (single paste, then Enter)

python - <<'PY'
from pathlib import Path
content = r"""#!/usr/bin/env python3
# Offline RAG runner with two modes:
#   - default: single-shot answer (non-interactive)
#   - --chat : build RAG prompt, then drop into llama.cpp REPL (interactive)
#
# Wide-open GPU defaults (32k ctx, full offload) are controlled by env vars
# and set to sane values below. You can override at runtime if needed.

import os, sys, subprocess, hashlib, numpy as np, psycopg2, argparse

# -----------------------
# Config / Environment
# -----------------------
DSN         = os.environ.get("TS_DSN", "dbname=tokenspace user=darren host=localhost password=LittleOugway")
EMBED_MODEL = os.environ.get("EMBED_MODEL", "nomic-ai/nomic-embed-text-v1.5")
TOP_K       = int(os.environ.get("RAG_K", "6"))
MAX_CHARS   = int(os.environ.get("RAG_MAX_CHARS", "900"))  # per chunk in prompt

# Llama defaults — wide-open GPU (override with env if needed)
LLAMA_BIN   = os.path.expanduser(os.environ.get("LLAMA_BIN", "~/ougway_env/llama.cpp/build/bin/llama-cli"))
LLAMA_CTX   = os.environ.get("LLAMA_CTX",  "32768")  # 32k
LLAMA_NGL   = os.environ.get("LLAMA_NGL",  "999")    # offload as many layers as fit
LLAMA_BATCH = os.environ.get("LLAMA_BATCH","32")     # conservative for 32k on 12 GB
LLAMA_T     = os.environ.get("LLAMA_T",    str(os.cpu_count() or 4))
LLAMA_NTOK  = os.environ.get("LLAMA_N",    "512")    # max tokens to generate

# -----------------------
# Embedding (offline-first)
# -----------------------
_USE_ST = False
try:
    # Try local SentenceTransformer; falls back to deterministic 768-d hash
    from sentence_transformers import SentenceTransformer
    _st = SentenceTransformer(EMBED_MODEL, trust_remote_code=True)
    dim = getattr(_st, "get_sentence_embedding_dimension", lambda: None)()
    _USE_ST = (dim == 768)
except Exception:
    _USE_ST = False

def embed_one(text: str) -> np.ndarray:
    if _USE_ST:
        v = _st.encode([text], normalize_embeddings=True)
        v = np.asarray(v, dtype=np.float32)
        return v[0]
    # Deterministic 768-d hash embed (no internet needed)
    h = hashlib.sha256(text.encode("utf-8")).digest()
    seed = int.from_bytes(h[:8], "big") % (2**31 - 1)
    rng = np.random.default_rng(seed)
    v = rng.normal(0.0, 1.0, 768).astype(np.float32)
    n = float(np.linalg.norm(v))
    if n > 0:
        v /= n
    return v

def vec_literal(v: np.ndarray) -> str:
    return "[" + ",".join(f"{x:.6f}" for x in v.tolist()) + "]"

# -----------------------
# Retrieval
# -----------------------
def retrieve(query_vec_lit: str, k: int):
    conn = psycopg2.connect(DSN)
    cur = conn.cursor()
    cur.execute(f"""
        SELECT ch.chunk_id,
               left(ch.text, {MAX_CHARS}) AS preview,
               d.title,
               s.uri,
               ch.embedding <=> %s::vector AS dist
        FROM content.chunks ch
        JOIN content.documents d ON d.doc_id = ch.doc_id
        LEFT JOIN content.sources s ON s.source_id = d.source_id
        ORDER BY dist ASC
        LIMIT %s;
    """, (query_vec_lit, k))
    rows = cur.fetchall()
    cur.close()
    conn.close()
    return rows

# -----------------------
# Prompt build
# -----------------------
def build_prompt(question: str, hits):
    ctx_blocks = []
    for i, (cid, preview, title, uri, dist) in enumerate(hits, 1):
        meta = []
        if title: meta.append(f"title: {title}")
        if uri:   meta.append(f"path: {uri}")
        meta.append(f"chunk_id: {cid}  dist: {dist:.4f}")
        ctx_blocks.append(f"### Context {i} ({' | '.join(meta)})\n{preview}")
    context = "\n\n".join(ctx_blocks)
    sys_inst = (
        "You are a concise assistant. Use the provided context only. "
        "If the answer is not in the context, say you don't find it."
    )
    prompt = (
        "[SYSTEM]\n" + sys_inst + "\n\n" +
        "[CONTEXT]\n" + context + "\n\n" +
        "[USER]\n" + question + "\n\n" +
        "[ASSISTANT]\n"
    )
    return prompt

# -----------------------
# Llama call
# -----------------------
def call_llama_cpp(model_path: str, prompt: str, chat_mode: bool):
    cmd = [
        LLAMA_BIN,
        "-m", model_path,
        "-p", prompt,
        "-n", LLAMA_NTOK,
        "--temp", "0.2",
        "--top_p", "0.9",
        "-c", LLAMA_CTX,
        "-ngl", LLAMA_NGL,
        "-t", LLAMA_T,
        "-b", LLAMA_BATCH,
    ]
    if chat_mode:
        return subprocess.call(cmd)  # Interactive REPL
    else:
        out = subprocess.run(cmd, capture_output=True, text=True)
        if out.returncode != 0:
            raise RuntimeError(out.stderr.strip())
        return out.stdout.strip()

# -----------------------
# Main
# -----------------------
def main():
    import argparse
    ap = argparse.ArgumentParser(description="Offline RAG to llama.cpp (Qwen).")
    ap.add_argument("model_path", help="Path to .gguf")
    ap.add_argument("question", help="User question")
    ap.add_argument("k", nargs="?", type=int, default=TOP_K, help="Top-K chunks (default: %(default)s)")
    ap.add_argument("--chat", action="store_true", help="Drop into interactive REPL after building RAG prompt")
    args = ap.parse_args()

    qv   = embed_one(args.question)
    qlit = vec_literal(qv)
    hits = retrieve(qlit, args.k)
    if not hits:
        print("No results.")
        sys.exit(0)

    prompt = build_prompt(args.question, hits)

    rc_or_text = call_llama_cpp(args.model_path, prompt, chat_mode=args.chat)
    if isinstance(rc_or_text, int):
        sys.exit(rc_or_text)  # REPL exit code
    else:
        print("\n=== ANSWER ===\n" + rc_or_text)

if __name__ == "__main__":
    main()
"""
p = Path.home() / "Desktop" / "rag_answer.py"
p.write_text(content)
print(f"Wrote {p} ({len(content)} bytes)")
PY
chmod +x ~/Desktop/rag_answer.py

Now run:

python ~/Desktop/rag_answer.py \
  "$HOME/ougway_env/llama.cpp/models/qwen3/qwen3-8b-q6_k.gguf" \
  "Summarize the Flower Lattice activation rule."

or, for interactive chat after RAG:

python ~/Desktop/rag_answer.py \
  "$HOME/ougway_env/llama.cpp/models/qwen3/qwen3-8b-q6_k.gguf" \
  "Summarize the Flower Lattice activation rule." \
  --chat

If your terminal still truncates