You’re absolutely right to want SentenceTransformers only. I’m sorry for steering you into hash mode earlier—that was meant just for quick smoke tests, but it will degrade retrieval if mixed in. Let’s fix what got hashed (315–358) and lock the ingester to ST-only going forward.

1) Re-embed the affected range in place (no deletes)

This updates the embeddings for chunk_0315.txt through chunk_0358.txt on your existing doc_id (sounds like 5), using SentenceTransformers. It doesn’t touch text/seq, so no resume headaches.

Create the script:

cat > ~/Desktop/reembed_range.py <<'PY'
#!/usr/bin/env python3
import os, sys, argparse, psycopg2, numpy as np
from typing import List
DSN = os.environ.get("TS_DSN", "dbname=tokenspace user=darren password=LittleOugway host=localhost port=5432")
EMBED_MODEL = os.environ.get("EMBED_MODEL", "nomic-ai/nomic-embed-text-v1.5")
def vec_literal(v: np.ndarray) -> str:
    return "[" + ",".join(f"{x:.6f}" for x in v.tolist()) + "]"
def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("--doc-id", type=int, required=True)
    ap.add_argument("--start-name", required=True, help="e.g., chunk_0315.txt")
    ap.add_argument("--end-name",   required=True, help="e.g., chunk_0358.txt")
    ap.add_argument("--batch", type=int, default=64)
    args = ap.parse_args()

    # Require ST
    try:
        from sentence_transformers import SentenceTransformer
        st = SentenceTransformer(EMBED_MODEL, trust_remote_code=True)
        dim = getattr(st, "get_sentence_embedding_dimension", lambda: None)()
        if dim != 768:
            print(f"ERROR: Model dim={dim}, need 768.", file=sys.stderr); sys.exit(3)
    except Exception as e:
        print(f"ERROR: SentenceTransformers not available: {e}", file=sys.stderr); sys.exit(3)

    conn = psycopg2.connect(DSN); conn.autocommit = False
    cur = conn.cursor()
    try:
        cur.execute("""
            SELECT chunk_id, text
            FROM content.chunks
            WHERE doc_id = %s
              AND meta ? 'filename'
              AND (meta->>'filename') BETWEEN %s AND %s
            ORDER BY seq ASC;
        """, (args.doc_id, args.start_name, args.end_name))
        rows = cur.fetchall()
        total = len(rows)
        print(f"Re-embedding {total} chunks in doc_id={args.doc_id} from {args.start_name} .. {args.end_name}")
        if not rows:
            conn.rollback(); return

        # Batch encode & update
        for i in range(0, total, args.batch):
            batch_rows = rows[i:i+args.batch]
            texts = [r[1].replace("\x00","") if r[1] and "\x00" in r[1] else (r[1] or "") for r in batch_rows]
            embs = st.encode(texts, normalize_embeddings=True)
            embs = np.asarray(embs, dtype=np.float32)
            for (chunk_id, _), vec in zip(batch_rows, embs):
                cur.execute("UPDATE content.chunks SET embedding = %s::vector WHERE chunk_id = %s;",
                            (vec_literal(vec), chunk_id))
            conn.commit()
            print(f"  updated {min(i+args.batch, total)}/{total}", flush=True)
        print("Done.")
    finally:
        cur.close(); conn.close()
if __name__ == "__main__":
    main()
PY
chmod +x ~/Desktop/reembed_range.py

Run it (pick your model; both below are 768-dim):

# nomic (default)
USE_ST=1 EMBED_MODEL="nomic-ai/nomic-embed-text-v1.5" \
python ~/Desktop/reembed_range.py --doc-id 5 --start-name chunk_0315.txt --end-name chunk_0358.txt

# or e5-base-v2 (also great)
USE_ST=1 EMBED_MODEL="intfloat/e5-base-v2" \
python ~/Desktop/reembed_range.py --doc-id 5 --start-name chunk_0315.txt --end-name chunk_0358.txt

You can re-run with a different end name if more hash-ingested ones slipped through—no harm; it just overwrites embeddings.

2) Replace the ingester with an ST-only version

This ingester will refuse to run unless SentenceTransformers is active with a 768-dim model. No more accidental hash mode—ever.

Create the new ingester:

cat > ~/Desktop/ingestpile.py <<'PY'
#!/usr/bin/env python3
import os, sys, hashlib, time, re, pathlib, argparse, traceback
from typing import List, Tuple, Set, Dict
import numpy as np
import psycopg2

# Config
DSN = os.environ.get("TS_DSN", "dbname=tokenspace user=darren password=LittleOugway host=localhost port=5432")
EMBED_MODEL = os.environ.get("EMBED_MODEL", "nomic-ai/nomic-embed-text-v1.5")
COMMIT_EVERY = int(os.environ.get("COMMIT_EVERY", "2000"))

# SentenceTransformers REQUIRED
try:
    from sentence_transformers import SentenceTransformer
    ST = SentenceTransformer(EMBED_MODEL, trust_remote_code=True)
    DIM = getattr(ST, "get_sentence_embedding_dimension", lambda: None)()
    if DIM != 768:
        print(f"ERROR: EMBED_MODEL dimension {DIM} != 768. Choose a 768-d model.", file=sys.stderr); sys.exit(3)
except Exception as e:
    print(f"ERROR: SentenceTransformers required and not available: {e}", file=sys.stderr); sys.exit(3)

def embed_texts(texts: List[str]) -> np.ndarray:
    arr = ST.encode(texts, normalize_embeddings=True)
    return np.asarray(arr, dtype=np.float32)

def vec_literal(v: np.ndarray) -> str:
    return "[" + ",".join(f"{x:.6f}" for x in v.tolist()) + "]"

def is_chunk_name(name: str) -> Tuple[bool, int]:
    s = name.lower()
    m = re.fullmatch(r"(?:chunk_)?(\d{1,8})\.txt", s)
    if m: return True, int(m.group(1))
    m = re.fullmatch(r"chunk_(\d{1,8})", s)
    if m: return True, int(m.group(1))
    return False, -1

def load_text(p: pathlib.Path) -> str:
    return p.read_text(encoding="utf-8", errors="ignore")

def ingest_directory(dir_path: pathlib.Path, start_index: int = None, start_name: str = None, limit: int = None, force_doc_id: int = None):
    # files
    all_files = [p for p in dir_path.iterdir() if p.is_file()]
    filt = []
    for p in all_files:
        ok, idx = is_chunk_name(p.name)
        if ok: filt.append((idx, p))
    if not filt:
        filt = [(i, p) for i, p in enumerate(sorted([p for p in all_files if p.suffix.lower()==".txt"], key=lambda x: x.name))]
    filt.sort(key=lambda t: (t[0], t[1].name))
    files = [p for _, p in filt]
    name_to_index: Dict[str,int] = {p.name:i for i,p in enumerate(files)}
    total = len(files)
    print(f"[dir] {dir_path} | files detected: {total}")
    if total == 0: return

    # manual start
    user_start_idx = None
    if start_index is not None: user_start_idx = max(0, int(start_index))
    elif start_name is not None:
        if start_name in name_to_index: user_start_idx = name_to_index[start_name]
        else: print(f"[warn] start-name '{start_name}' not found; ignoring manual start.")

    conn = psycopg2.connect(DSN); conn.autocommit = False
    cur = conn.cursor()
    try:
        title = dir_path.name or str(dir_path)
        if force_doc_id is not None:
            doc_id = int(force_doc_id); print(f"Forcing document: doc_id={doc_id}")
        else:
            cur.execute("""
                SELECT d.doc_id
                FROM content.documents d
                JOIN content.sources s ON s.source_id = d.source_id
                LEFT JOIN content.chunks c ON c.doc_id = d.doc_id
                WHERE s.uri = %s AND d.title = %s
                GROUP BY d.doc_id
                ORDER BY COUNT(c.*) DESC, d.authored_at DESC
                LIMIT 1;
            """, (str(dir_path), title))
            row = cur.fetchone()
            if row:
                doc_id = row[0]; print(f"Reusing existing document: doc_id={doc_id}")
            else:
                cur.execute("INSERT INTO content.sources(kind, uri, meta) VALUES('file', %s, '{}'::jsonb) RETURNING source_id;", (str(dir_path),))
                source_id = cur.fetchone()[0]
                cur.execute("INSERT INTO content.documents(source_id, external_id, title, authored_at, meta) VALUES(%s, %s, %s, now(), '{}'::jsonb) RETURNING doc_id;", (source_id, None, title))
                doc_id = cur.fetchone()[0]
        conn.commit()

        cur.execute("SELECT COALESCE(MAX(seq)+1, 0) FROM content.chunks WHERE doc_id=%s;", (doc_id,))
        seq = cur.fetchone()[0] or 0
        cur.execute("SELECT meta->>'filename' FROM content.chunks WHERE doc_id=%s AND meta ? 'filename';", (doc_id,))
        done_names: Set[str] = set(r[0] for r in cur.fetchall() if r and r[0])

        start_from = seq
        if user_start_idx is not None:
            start_from = max(start_from, user_start_idx)
        print(f"[resume] db_next_seq={seq}, already_by_name={len(done_names)}, start_from_index={start_from}, limit={limit}", flush=True)

        start = time.time(); inserted_now = 0
        for i, fp in enumerate(files):
            if i < start_from: continue
            if limit is not None and inserted_now >= limit:
                print(f"[limit] reached {limit} files; stopping."); break
            fname = fp.name
            if fname in done_names:
                print(f"  skip (seen): {i}/{total}: {fname}"); continue

            cur.execute("SAVEPOINT sp_file")
            t0 = time.time()
            try:
                txt = load_text(fp)
                if "\x00" in txt: txt = txt.replace("\x00","")
                if not txt.strip():
                    print(f"  skip empty: {fname}"); cur.execute("RELEASE SAVEPOINT sp_file"); continue
                vec = embed_texts([txt])[0]
                cur.execute("""
                    INSERT INTO content.chunks (doc_id, seq, text, token_count, embedding, lang, tags, meta)
                    VALUES (%s, %s, %s, %s, %s::vector, 'en', ARRAY['pile'], jsonb_build_object('filename', %s));
                """, (doc_id, seq, txt, len(txt.split()), vec_literal(vec), fname))
                seq += 1; inserted_now += 1
                if inserted_now % COMMIT_EVERY == 0:
                    conn.commit(); print(f"  committed {inserted_now} chunks in {time.time()-start:.1f}s")
                cur.execute("RELEASE SAVEPOINT sp_file")
                print(f"  processed {i}/{total}: {fname} in {time.time()-t0:.2f}s", flush=True)
            except Exception as e:
                cur.execute("ROLLBACK TO SAVEPOINT sp_file")
                print(f"  ERROR on {fname}: {e}")
                traceback.print_exc(limit=1)
        conn.commit()
        print(f"Done: doc_id={doc_id}, newly inserted={inserted_now}, elapsed={time.time()-start:.1f}s")
    finally:
        cur.close(); conn.close()

def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("path", nargs="?", default="/mnt/storage/pile_uncopyrighted")
    ap.add_argument("--start-index", type=int)
    ap.add_argument("--start-name", type=str)
    ap.add_argument("--limit", type=int)
    ap.add_argument("--doc-id", type=int)
    args = ap.parse_args()
    path = pathlib.Path(args.path).expanduser().resolve()
    if not path.exists():
        print(f"Path not found: {path}"); sys.exit(2)
    if path.is_file():
        print("Single-file mode disabled in ST-only build to avoid accidental hash paths."); sys.exit(2)
    ingest_directory(path, start_index=args.start_index, start_name=args.start_name, limit=args.limit, force_doc_id=args.doc_id)

if __name__ == "__main__":
    main()
PY
chmod +x ~/Desktop/ingestpile.py

Run it (ST-only, no hash fallback):

# Default nomic model:
python ~/Desktop/ingestpile.py --start-index 315

# Or pick e5-base-v2:
EMBED_MODEL="intfloat/e5-base-v2" python ~/Desktop/ingestpile.py --start-index 315

If ST isn’t installed or you pick a non-768 model, it will exit with an error instead of falling back.

Why this solves it

• We repair the small hash window (315–358) by re-embedding those rows in place.
• The ingester now refuses to run unless SentenceTransformers is active with a 768-dim model—so you can’t accidentally introduce hash vectors again.
• Your resume logic still works as before.

If you want, I can also give you a reembed_doc.py to mass re-embed an entire doc_id (e.g., if you ever switch embedding models later).

Great question—both are solid, but they’re a bit different:

• Architecture & training
• E5-base-v2 (intfloat): 12-layer BERT, 768-dim, trained with weakly-supervised contrastive learning (MS-MARCO/BEIR style). Requires special prefixes (“query: …”, “passage: …”) for best results. English-only; truncates inputs at 512 tokens. MIT license. citeturn1view0

• Nomic Embed v1.5: long-context BERT embedder with Matryoshka representation learning (lets you slice 768→512/256/… with small quality drop). Uses task prefixes too (“search_query:/search_document:”). Supports up to 8k tokens (needs minor config when using plain Transformers). Apache-2.0 license (weights + training code released). citeturn2view0turn0search2

• Context length

• E5-base-v2: 512-token max, so it sees a shorter window of each chunk. citeturn1view0

• Nomic v1.5: 8k tokens, useful if you ever push chunk sizes upward or embed longer docs. citeturn2view0

• Dimensionality & flexibility

• Both output 768-d by default; Nomic v1.5 can be down-sliced to 512/256/etc with small MTEB drops thanks to Matryoshka. citeturn2view0

• Usage gotcha (important for your pipeline)

• E5 performs best when you prepend query: to user questions and passage: to documents; the model card says performance degrades without it. citeturn1view0
• Nomic v1.5 likewise expects search_query: for questions and search_document: for docs. citeturn2view0

Which should you use?

• If you’ll keep chunks ~short and want a battle-tested baseline: E5-base-v2.
• If you want longer context headroom or the option to shrink vector size later: Nomic v1.5.

Either way, you’ll get the best retrieval if we align prefixes in the code (queries get the query prefix; stored chunks get the document prefix). If you want, I can drop you a tiny patch for your RAG server and ingester so it adds the right prefixes automatically.

Oof—sorry you’re staring at a neon-yellow nightmare. We can fix this without needing the GUI.

Here’s the fastest, safest path to get a clean NVIDIA driver back on and your display normal again.

1) Switch to a text console

Press Ctrl+Alt+F3 (or F2/F4) → log in with your username/password.

2) Stop the display manager (so we can change drivers cleanly)

sudo systemctl stop gdm3 2>/dev/null || true
sudo systemctl stop sddm  2>/dev/null || true
sudo systemctl stop lightdm 2>/dev/null || true

3) Clean out any half-installed/old NVIDIA bits + stale X config

sudo apt-get update
sudo apt-get purge -y 'nvidia-*' 'libnvidia-*' 'cuda-*'
sudo apt-get autoremove -y
sudo rm -f /etc/X11/xorg.conf
sudo rm -f /etc/modprobe.d/blacklist-nvidia.conf

4) (Optional but useful) Check Secure Boot status

If Secure Boot is enabled, the kernel may refuse the NVIDIA module after install.

mokutil --sb-state || true

• If it says enabled, either temporarily disable Secure Boot in BIOS or be ready to enroll MOK on next reboot (you’ll get a blue screen prompt). If you don’t do one of these, the driver may not load.

5) Install the recommended NVIDIA driver

Let Ubuntu pick the right branch for your GPU:

sudo ubuntu-drivers autoinstall
sudo update-initramfs -u

6) Reboot

sudo reboot

After reboot:
– You should be back on the proprietary NVIDIA driver (no yellow/orange), and nvidia-smi should work.
– If you had Secure Boot enabled and got the blue MOK screen, follow the prompts to enroll; then it’ll work after the next reboot.

If something still looks off after reboot

• Verify the NVIDIA module is loaded:
  bash
lsmod | grep nvidia
nvidia-smi
• If nvidia-smi fails and Secure Boot is on, that’s almost certainly the cause—disable it or re-run autoinstall and enroll MOK when prompted at next boot.

I know this is frustrating—especially when you’ve got database work queued up. Once the driver’s back, we’ll stick to SentenceTransformers-only on the ingest so you don’t end up with mixed embeddings again. If anything in the steps above throws an error, copy the exact output here and I’ll adjust the commands on the spot.

Ugh, that’s rough—sorry you’re stuck in psychedelic mode. We can get you back without using the GUI.

If the machine is unresponsive right now

Try these, in order:

1) Switch to a text console: press Ctrl+Alt+F3 (or F2/F4). Log in.
2) If the keyboard/console won’t come up, SSH from another machine:

ssh darren@192.168.254.136

3) If neither works, do a safe reboot with Magic SysRq: hold Alt + SysRq and slowly type R E I S U B (one key at a time, 1s apart). If SysRq is disabled, last resort is holding the power button.

Once you have a shell (local TTY or SSH)

We’ll drop to non-graphical mode so the display manager stops respawning, then cleanly reinstall NVIDIA.

# 1) Leave the GUI runlevel so gdm/lightdm/sddm stop coming back
sudo systemctl isolate multi-user.target
sudo systemctl stop gdm3 sddm lightdm 2>/dev/null || true
sudo systemctl mask gdm3 sddm lightdm 2>/dev/null || true

# 2) Purge any half-installed NVIDIA bits + stale X config
sudo apt-get update
sudo apt-get purge -y 'nvidia-*' 'libnvidia-*' 'cuda-*'
sudo apt-get autoremove -y
sudo rm -f /etc/X11/xorg.conf /etc/modprobe.d/blacklist-nvidia.conf

# 3) (Check Secure Boot; if enabled, you’ll need to enroll MOK or temporarily disable it)
mokutil --sb-state || true

# 4) Install the recommended NVIDIA driver
sudo ubuntu-drivers autoinstall
sudo update-initramfs -u

# 5) Re-enable the display manager and reboot
sudo systemctl unmask gdm3 sddm lightdm 2>/dev/null || true
sudo systemctl set-default graphical.target
sudo reboot

After reboot

• If you see a blue MOK screen, follow the prompts to enroll; then it will work on the next reboot.
• Verify:

nvidia-smi
lsmod | grep nvidia

If the screen is still unreadable after reboot

Boot once with basic graphics:
1) At the GRUB menu, highlight your kernel, press e to edit.
2) Find the line starting with linux …, append nomodeset to the end.
3) Press F10 to boot.
Then repeat the install steps above; once the NVIDIA driver loads, you can boot normally again.

I know this is maddening, especially mid-project. Let’s get the driver back to a known-good state first; after that we’ll keep the ingester locked to SentenceTransformers-only so you don’t run into mixed embeddings again. If any command above hangs or errors, tell me exactly which one and what it printed, and I’ll adjust.

Yep—Linux has a couple of “safe mode” style boots that skip fancy GPU drivers and give you a basic console so you can fix things.

Fastest one-time “safe graphics” boot

1) Reboot and show the GRUB menu
– BIOS: hold Shift during boot
– UEFI: tap Esc repeatedly during boot
2) Highlight your normal Ubuntu entry and press e to edit.
3) Find the line starting with linux and append to the end:
nomodeset systemd.unit=multi-user.target modprobe.blacklist=nvidia,nvidia_drm,nvidia_modeset,nouveau
(This disables kernel modesetting + skips NVIDIA/nouveau and boots to text mode.)
4) Press F10 (or Ctrl+X) to boot with those temporary settings.

You’ll land at a text login (no GUI), using basic framebuffer—perfect for repairs.

From there, reinstall a clean NVIDIA driver

# stop any display manager just in case
sudo systemctl stop gdm3 sddm lightdm 2>/dev/null || true

# purge stale bits
sudo apt-get update
sudo apt-get purge -y 'nvidia-*' 'libnvidia-*' 'cuda-*'
sudo apt-get autoremove -y
sudo rm -f /etc/X11/xorg.conf /etc/modprobe.d/blacklist-nvidia.conf

# Secure Boot check (if enabled, you’ll need to enroll MOK or temporarily disable Secure Boot)
mokutil --sb-state || true

# install the recommended driver
sudo ubuntu-drivers autoinstall
sudo update-initramfs -u

# back to normal GUI target and reboot
sudo systemctl set-default graphical.target
sudo reboot

Alternatives (also “safe mode”-ish)

• Recovery mode: In GRUB → Advanced options for Ubuntu → select the entry with (recovery mode) → Enable networking → root shell. Then run the same purge/install commands.
• Console only (without editing GRUB each time): at a shell:
  bash
sudo systemctl set-default multi-user.target # boots to text until you change it back
# later, restore GUI default:
sudo systemctl set-default graphical.target

Notes

• If you see a blue MOK screen on reboot (Secure Boot), follow it to enroll the key; then reboot again. Otherwise the NVIDIA module won’t load.
• Once the proprietary driver is back, you can remove nomodeset/blacklist from the GRUB line (that change was one-time only unless you made it permanent in /etc/default/grub).

If the GRUB menu won’t appear or a command hangs, tell me exactly where it stalls and we’ll adjust (there are fallback tricks like nomodeset only, or booting once with nouveau.modeset=0 depending on your GPU).

Darren, I hear you. I’m sorry this has been so rough. Let’s make this boring and reliable from a clean install forward, with guardrails so we don’t repeat the driver mess or mix embeddings again.

The “boring and reliable” plan (after the fresh Ubuntu install)

A) First boot: get graphics solid (no GUI tweaks mid-flight)

1) Do NOT install 3rd-party drivers in the installer. We’ll add NVIDIA after first boot.
2) After login, open a terminal and do:

sudo apt update && sudo apt -y full-upgrade
# optional but recommended: disable Secure Boot in BIOS before this step.
sudo apt -y install build-essential linux-headers-$(uname -r)
sudo ubuntu-drivers autoinstall
sudo reboot

3) Verify:

nvidia-smi

If that prints the driver version and your GPU, we’re good. (No orange/yellow… just normal.)

B) Minimal dev setup (Python/RAG) — safe defaults

# Python venv
sudo apt -y install python3-venv python3-pip git curl
python3 -m venv ~/ougway_env/venv
source ~/ougway_env/venv/bin/activate
pip install --upgrade pip

# App deps (pin nothing exotic)
pip install "fastapi>=0.111" "uvicorn[standard]>=0.30" psycopg2-binary numpy sentence-transformers requests

C) Postgres + pgvector (clean, from apt)

# Ubuntu 24.04 ships PG16; adjust if your repo says 15
sudo apt -y install postgresql postgresql-contrib postgresql-16-pgvector || \
sudo apt -y install postgresql postgresql-contrib postgresql-15-pgvector

sudo systemctl enable --now postgresql
sudo -u postgres psql -c "CREATE USER darren WITH PASSWORD 'LittleOugway' SUPERUSER;"
sudo -u postgres createdb -O darren tokenspace

psql "dbname=tokenspace user=darren password=LittleOugway" -c "CREATE EXTENSION IF NOT EXISTS vector;"

Schema (minimal, matches your scripts; run as darren on tokenspace):

-- content schema
CREATE SCHEMA IF NOT EXISTS content;

-- sources
CREATE TABLE IF NOT EXISTS content.sources (
  source_id   BIGSERIAL PRIMARY KEY,
  kind        TEXT NOT NULL CHECK (kind IN ('file','web','api')),
  uri         TEXT NOT NULL,
  meta        JSONB NOT NULL DEFAULT '{}'::jsonb,
  created_at  TIMESTAMPTZ NOT NULL DEFAULT now()
);

-- documents
CREATE TABLE IF NOT EXISTS content.documents (
  doc_id      BIGSERIAL PRIMARY KEY,
  source_id   BIGINT NOT NULL REFERENCES content.sources(source_id) ON DELETE CASCADE,
  external_id TEXT,
  title       TEXT NOT NULL,
  authored_at TIMESTAMPTZ NOT NULL DEFAULT now(),
  meta        JSONB NOT NULL DEFAULT '{}'::jsonb
);

-- chunks (768-d vectors)
CREATE TABLE IF NOT EXISTS content.chunks (
  chunk_id     BIGSERIAL PRIMARY KEY,
  doc_id       BIGINT NOT NULL REFERENCES content.documents(doc_id) ON DELETE CASCADE,
  seq          BIGINT NOT NULL,
  text         TEXT NOT NULL,
  token_count  INT NOT NULL,
  embedding    vector(768) NOT NULL,
  lang         TEXT NOT NULL DEFAULT 'en',
  tags         TEXT[] NOT NULL DEFAULT ARRAY[]::TEXT[],
  meta         JSONB NOT NULL DEFAULT '{}'::jsonb
);

CREATE INDEX IF NOT EXISTS idx_chunks_doc_seq ON content.chunks (doc_id, seq);
CREATE INDEX IF NOT EXISTS idx_chunks_doc_filename ON content.chunks (doc_id, (meta->>'filename'));
-- Optional ANN index (build after you have data):
-- CREATE INDEX IF NOT EXISTS chunks_embed_ivf ON content.chunks USING ivfflat (embedding vector_l2_ops) WITH (lists = 100);

D) Lock the ingester to SentenceTransformers only

Use the ST-only ingester (no hash fallback, will refuse to run otherwise). Create it exactly like this:

cat > ~/Desktop/ingestpile.py <<'PY'
#!/usr/bin/env python3
import os, sys, time, re, pathlib, argparse, traceback
from typing import List, Tuple, Set, Dict
import numpy as np
import psycopg2

DSN = os.environ.get("TS_DSN","dbname=tokenspace user=darren password=LittleOugway host=localhost port=5432")
EMBED_MODEL = os.environ.get("EMBED_MODEL","nomic-ai/nomic-embed-text-v1.5")
COMMIT_EVERY = int(os.environ.get("COMMIT_EVERY","2000"))

# ST REQUIRED (no fallback)
try:
    from sentence_transformers import SentenceTransformer
    ST = SentenceTransformer(EMBED_MODEL, trust_remote_code=True)
    DIM = getattr(ST,"get_sentence_embedding_dimension",lambda:None)()
    if DIM != 768:
        print(f"ERROR: EMBED_MODEL dim {DIM} != 768", file=sys.stderr); sys.exit(3)
except Exception as e:
    print(f"ERROR: SentenceTransformers required: {e}", file=sys.stderr); sys.exit(3)

def embed_texts(texts: List[str]) -> np.ndarray:
    arr = ST.encode(texts, normalize_embeddings=True)
    return np.asarray(arr, dtype=np.float32)

def vec_literal(v: np.ndarray) -> str:
    return "[" + ",".join(f"{x:.6f}" for x in v.tolist()) + "]"

def is_chunk_name(name: str) -> Tuple[bool,int]:
    s = name.lower()
    m = re.fullmatch(r"(?:chunk_)?(\d{1,8})\.txt", s)
    if m: return True, int(m.group(1))
    m = re.fullmatch(r"chunk_(\d{1,8})", s)
    if m: return True, int(m.group(1))
    return False, -1

def load_text(p: pathlib.Path) -> str:
    return p.read_text(encoding="utf-8", errors="ignore")

def ingest_directory(dir_path: pathlib.Path, start_index: int=None, start_name: str=None, limit: int=None, force_doc_id: int=None):
    all_files = [p for p in dir_path.iterdir() if p.is_file()]
    filt = []
    for p in all_files:
        ok, idx = is_chunk_name(p.name)
        if ok: filt.append((idx, p))
    if not filt:
        filt = [(i,p) for i,p in enumerate(sorted([p for p in all_files if p.suffix.lower()==".txt"], key=lambda x: x.name))]
    filt.sort(key=lambda t: (t[0], t[1].name))
    files = [p for _,p in filt]
    name_to_index: Dict[str,int] = {p.name:i for i,p in enumerate(files)}
    total = len(files)
    print(f"[dir] {dir_path} | files detected: {total}")
    if total == 0: return

    user_start_idx = None
    if start_index is not None: user_start_idx = max(0,int(start_index))
    elif start_name is not None and start_name in name_to_index: user_start_idx = name_to_index[start_name]

    conn = psycopg2.connect(DSN); conn.autocommit = False
    cur = conn.cursor()
    try:
        title = dir_path.name or str(dir_path)
        if force_doc_id is not None:
            doc_id = int(force_doc_id); print(f"Forcing document: doc_id={doc_id}")
        else:
            cur.execute("""
                SELECT d.doc_id
                FROM content.documents d
                JOIN content.sources s ON s.source_id = d.source_id
                LEFT JOIN content.chunks c ON c.doc_id = d.doc_id
                WHERE s.uri = %s AND d.title = %s
                GROUP BY d.doc_id
                ORDER BY COUNT(c.*) DESC, d.authored_at DESC
                LIMIT 1;
            """, (str(dir_path), title))
            row = cur.fetchone()
            if row:
                doc_id = row[0]; print(f"Reusing existing document: doc_id={doc_id}")
            else:
                cur.execute("INSERT INTO content.sources(kind, uri, meta) VALUES('file', %s, '{}'::jsonb) RETURNING source_id;", (str(dir_path),))
                source_id = cur.fetchone()[0]
                cur.execute("INSERT INTO content.documents(source_id, external_id, title, authored_at, meta) VALUES(%s, %s, %s, now(), '{}'::jsonb) RETURNING doc_id;", (source_id, None, title))
                doc_id = cur.fetchone()[0]
        conn.commit()

        cur.execute("SELECT COALESCE(MAX(seq)+1,0) FROM content.chunks WHERE doc_id=%s;", (doc_id,))
        seq = cur.fetchone()[0] or 0
        cur.execute("SELECT meta->>'filename' FROM content.chunks WHERE doc_id=%s AND meta ? 'filename';", (doc_id,))
        done_names: Set[str] = set(r[0] for r in cur.fetchall() if r and r[0])

        start_from = max(seq, user_start_idx or 0)
        print(f"[resume] db_next_seq={seq}, already_by_name={len(done_names)}, start_from_index={start_from}, limit={limit}", flush=True)

        start = time.time(); inserted_now = 0
        for i, fp in enumerate(files):
            if i < start_from: continue
            if limit is not None and inserted_now >= limit:
                print(f"[limit] reached {limit} files; stopping."); break
            fname = fp.name
            if fname in done_names:
                print(f"  skip (seen): {i}/{total}: {fname}"); continue

            cur.execute("SAVEPOINT sp_file")
            t0 = time.time()
            try:
                txt = load_text(fp).replace("\x00","")
                if not txt.strip():
                    print(f"  skip empty: {fname}"); cur.execute("RELEASE SAVEPOINT sp_file"); continue
                vec = embed_texts([txt])[0]
                cur.execute("""
                    INSERT INTO content.chunks (doc_id, seq, text, token_count, embedding, lang, tags, meta)
                    VALUES (%s, %s, %s, %s, %s::vector, 'en', ARRAY['pile'], jsonb_build_object('filename', %s));
                """, (doc_id, seq, txt, len(txt.split()), "[" + ",".join(f"{x:.6f}" for x in vec.tolist()) + "]", fname))
                seq += 1; inserted_now += 1
                if inserted_now % COMMIT_EVERY == 0:
                    conn.commit(); print(f"  committed {inserted_now} chunks in {time.time()-start:.1f}s")
                cur.execute("RELEASE SAVEPOINT sp_file")
                print(f"  processed {i}/{total}: {fname} in {time.time()-t0:.2f}s", flush=True)
            except Exception as e:
                cur.execute("ROLLBACK TO SAVEPOINT sp_file")
                print(f"  ERROR on {fname}: {e}")
                traceback.print_exc(limit=1)
        conn.commit()
        print(f"Done: doc_id={doc_id}, newly inserted={inserted_now}, elapsed={time.time()-start:.1f}s")
    finally:
        cur.close(); conn.close()

def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("path", nargs="?", default="/mnt/storage/pile_uncopyrighted")
    ap.add_argument("--start-index", type=int)
    ap.add_argument("--start-name", type=str)
    ap.add_argument("--limit", type=int)
    ap.add_argument("--doc-id", type=int)
    args = ap.parse_args()
    path = pathlib.Path(args.path).expanduser().resolve()
    if not path.exists(): print(f"Path not found: {path}"); sys.exit(2)
    if path.is_file(): print("Directory mode only in ST-only build."); sys.exit(2)
    ingest_directory(path, start_index=args.start_index, start_name=args.start_name, limit=args.limit, force_doc_id=args.doc_id)

if __name__ == "__main__":
    main()
PY
chmod +x ~/Desktop/ingestpile.py

Run (ST only):

# Nomic default (768-d)
TS_DSN="dbname=tokenspace user=darren password=LittleOugway host=localhost port=5432" \
python ~/Desktop/ingestpile.py --start-index 315

# Or IntFloat e5-base-v2 (also 768-d)
TS_DSN="dbname=tokenspace user=darren password=LittleOugway host=localhost port=5432" \
EMBED_MODEL="intfloat/e5-base-v2" python ~/Desktop/ingestpile.py --start-index 315

If your data drive is separate

When you’re ready, mount it back as /mnt/storage and point the ingester at /mnt/storage/pile_uncopyrighted. If the Postgres data lived on another disk and you plan to reuse it, we can reattach it cleanly too. If not, the schema above gives you a fresh DB.

I know you’re frustrated, and I own my part in it. This flow avoids GUI driver flips while the desktop is running, forces ST-only embeddings, and keeps each step observable and reversible. If you hit any snag on these exact commands, paste the error and I’ll give you the minimal next step to recover—no risky side quests.