Keep the local index fresh from the change feed and improve ranking

- Track a feed cursor in the local index: reindex takes the head cursor
  before listing and commits it with the snapshot; reads refresh from
  /changes (bounded, throttled) and apply creates, updates, archives,
  restores and hard deletes atomically with the cursor. Existing
  cursorless indexes bootstrap tracking on first read. Failures answer
  locally with explicit freshness warnings; 400 triggers a full
  reindex, 404 falls back to timestamp sync.
- Rank by a weighted score: exact identity (including aliases and
  paths), term coverage including body, intent, bm25, authority, a
  status penalty for superseded/completed records and mild recency.
- Return query-centred FTS5 snippets (never for E2E nodes).
- FTS schema v2: porter stemming, prefix index, title column, rowid
  deletes, two-character terms, domain words no longer stopwords;
  v1 indexes upgrade in place.
- Hyphenated alias/path literals match as adjacent-token phrases and
  rank as exact identity.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>

tests/test_mcp_index_feed_sync.py

@@ -0,0 +1,404 @@
+"""Change-feed driven local index freshness against a disposable HTTP provider.
+
+The provider implements the node list, single-node read and change-feed
+routes with synthetic data. The real request, index and search code run.
+"""
+
+import copy
+import importlib.util
+import json
+import pathlib
+import re
+import threading
+from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
+from urllib.parse import parse_qs, urlsplit
+
+import pytest
+
+SCRIPT = pathlib.Path(__file__).resolve().parents[1] / "resources/scripts/napseer_mcp_server.py"
+
+
+def make_node(name, minute, *, content=None, archived=False):
+    stamp = f"2026-09-01T12:{minute:02d}:00Z"
+    return {
+        "id": name, "project_id": "project-1", "full_path": f"/notes/{name}",
+        "folder_path": "/notes", "name": name, "type": "note",
+        "metadata": {}, "tags": [], "aliases": [], "links": [],
+        "content_text": content if content is not None else name,
+        "updated_at": stamp, "archived_at": stamp if archived else None,
+    }
+
+
+class Provider:
+    def __init__(self):
+        self.nodes = {}
+        self.events = []
+        self.requests = []
+        self.fail = {}  # route kind -> HTTP status
+        self.on_list = None
+
+    def put(self, node, kind):
+        self.nodes[node["id"]] = copy.deepcopy(node)
+        self.events.append({"type": kind, "node_id": node["id"], "full_path": node["full_path"],
+                            "new_revision": node["updated_at"]})
+
+    def delete(self, node_id):
+        node = self.nodes.pop(node_id)
+        self.events.append({"type": "node_deleted", "node_id": node_id, "full_path": node["full_path"],
+                            "new_revision": None})
+
+    def count(self, kind):
+        return sum(1 for item in self.requests if item[0] == kind)
+
+
+@pytest.fixture
+def feed(tmp_path, monkeypatch):
+    provider = Provider()
+
+    class Handler(BaseHTTPRequestHandler):
+        def log_message(self, *_args):
+            pass
+
+        def reply(self, status, payload):
+            self.send_response(status)
+            self.send_header("Content-Type", "application/json")
+            self.end_headers()
+            self.wfile.write(json.dumps(payload).encode())
+
+        def do_GET(self):
+            url = urlsplit(self.path)
+            query = parse_qs(url.query)
+            if url.path.endswith("/changes"):
+                kind = "changes"
+            elif url.path.endswith("/nodes"):
+                kind = "list"
+            else:
+                kind = "get"
+            provider.requests.append((kind, url.path, query))
+            if kind in provider.fail:
+                status = provider.fail[kind]
+                return self.reply(status, {"error": {"code": "test_failure", "message": "injected"}})
+            if kind == "changes":
+                after = query.get("after", ["head"])[0]
+                if after == "head":
+                    start = len(provider.events)
+                elif re.fullmatch(r"c\d+", after):
+                    start = int(after[1:])
+                else:
+                    return self.reply(400, {"error": {"code": "invalid_cursor", "message": "bad cursor"}})
+                limit = int(query.get("limit", ["100"])[0])
+                page = provider.events[start:start + limit]
+                items = [{**event, "cursor": f"c{start + index + 1}", "project_id": "project-1",
+                          "changed_fields": [], "created_at": "2026-09-01T12:00:00Z"}
+                         for index, event in enumerate(page)]
+                end = start + len(page)
+                return self.reply(200, {"items": items, "next_cursor": f"c{end}",
+                                        "has_more": end < len(provider.events)})
+            if kind == "list":
+                if provider.on_list is not None:
+                    hook, provider.on_list = provider.on_list, None
+                    hook()
+                rows = [copy.deepcopy(n) for n in provider.nodes.values() if not n["archived_at"]]
+                rows.sort(key=lambda n: (n["updated_at"], n["id"]), reverse=True)
+                after = query.get("updated_after", [""])[0]
+                if after:
+                    rows = [n for n in provider.nodes.values() if n["updated_at"] > after]
+                return self.reply(200, {"items": rows, "next_cursor": None})
+            node_id = url.path.rsplit("/", 1)[-1]
+            if node_id not in provider.nodes:
+                return self.reply(404, {"error": {"code": "not_found", "message": "missing"}})
+            return self.reply(200, copy.deepcopy(provider.nodes[node_id]))
+
+    server = ThreadingHTTPServer(("127.0.0.1", 0), Handler)
+    thread = threading.Thread(target=server.serve_forever, daemon=True)
+    thread.start()
+    monkeypatch.setenv("NAPSEER_TOOL_PROFILES", "all")
+    monkeypatch.syspath_prepend(str(SCRIPT.parent))
+    spec = importlib.util.spec_from_file_location("feed_sync_test_worker", SCRIPT)
+    mod = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(mod)
+    mod.AUTH_DIR = tmp_path / "project-state"
+    mod.INDEX_PATH = mod.AUTH_DIR / "index.sqlite"
+    mod.INDEX_LOCK_PATH = mod.AUTH_DIR / "index.lock"
+    mod.BASE_URL = f"http://127.0.0.1:{server.server_port}"
+    mod.TOKEN = "disposable-test-token"
+    mod.AUTH = {"account_id": "test-account"}
+    mod.DEFAULT_PROJECT_ID = "project-1"
+    mod.CONFIGURED_TOOL_PROFILES = {"all"}
+    mod.refresh_public_auth_state = lambda: None
+    mod.require_unlocked = lambda _name: None
+    mod.resolve_project_id = lambda args: args.get("project_id", "project-1")
+    mod.project_encryption_state = lambda _project_id: "standard"
+    try:
+        yield mod, provider
+    finally:
+        server.shutdown()
+        server.server_close()
+        thread.join(timeout=2)
+
+
+def local_ids(mod, text):
+    return [item["id"] for item in mod.search_local_index({"q": text, "limit": 10})["items"]]
+
+
+def seed(mod, provider, *names):
+    for index, name in enumerate(names):
+        provider.put(make_node(name, index), "node_created")
+    result = mod.reindex_project({})
+    assert result["graph_complete"] is True
+    return result
+
+
+def test_reindex_takes_head_cursor_before_listing_and_replays_idempotently(feed):
+    mod, provider = feed
+    provider.put(make_node("alpha", 1), "node_created")
+
+    def concurrent_write():
+        provider.put(make_node("alpha", 2, content="alpha concurrent revision"), "node_updated")
+
+    provider.on_list = concurrent_write
+    mod.reindex_project({})
+
+    kinds = [item[0] for item in provider.requests]
+    assert kinds[:2] == ["changes", "list"]
+    assert provider.requests[0][2]["after"] == ["head"]
+    assert mod.local_index_feed_state("project-1")["feed_cursor"] == "c1"
+    assert local_ids(mod, "concurrent") == ["alpha"]
+
+    result = mod.sync_local_index_from_feed("project-1")
+
+    assert (result["events"], result["skipped_current"], result["indexed"]) == (1, 1, 0)
+    assert provider.count("get") == 0  # revision already indexed: no refetch
+    assert mod.local_index_feed_state("project-1")["feed_cursor"] == "c2"
+
+
+def test_feed_applies_create_update_archive_restore_and_delete(feed):
+    mod, provider = feed
+    seed(mod, provider, "keepme", "archiveme", "deleteme", "restoreme")
+    provider.put(make_node("restoreme", 5, archived=True), "node_archived")
+    provider.put(make_node("freshnode", 10, content="brandnewterm"), "node_created")
+    provider.put(make_node("keepme", 11, content="keepme revisedterm"), "node_updated")
+    provider.put(make_node("archiveme", 12, archived=True), "node_archived")
+    provider.delete("deleteme")
+    provider.put(make_node("restoreme", 13), "node_restored")
+
+    result = mod.sync_local_index_from_feed("project-1")
+
+    assert result["status"] == "synced"
+    assert local_ids(mod, "brandnewterm") == ["freshnode"]
+    assert local_ids(mod, "revisedterm") == ["keepme"]
+    assert local_ids(mod, "archiveme") == []
+    assert local_ids(mod, "deleteme") == []
+    assert local_ids(mod, "restoreme") == ["restoreme"]
+    with mod.index_connect() as conn:
+        ids = {row[0] for row in conn.execute("SELECT node_id FROM local_index_nodes")}
+    # Archived rows match a full rebuild, which lists only active nodes.
+    assert ids == {"keepme", "freshnode", "restoreme"}
+    assert mod.local_index_feed_state("project-1")["feed_cursor"] == f"c{len(provider.events)}"
+
+
+def test_feed_failure_never_advances_cursor_and_retry_recovers(feed):
+    mod, provider = feed
+    seed(mod, provider, "stable")
+    before = mod.local_index_feed_state("project-1")["feed_cursor"]
+    provider.put(make_node("newcomer", 10, content="latecomerterm"), "node_created")
+    provider.fail["get"] = 503
+
+    with pytest.raises(mod.SafeToolError):
+        mod.sync_local_index_from_feed("project-1")
+
+    assert mod.local_index_feed_state("project-1")["feed_cursor"] == before
+    assert local_ids(mod, "latecomerterm") == []
+    del provider.fail["get"]
+    mod.sync_local_index_from_feed("project-1")
+    assert local_ids(mod, "latecomerterm") == ["newcomer"]
+
+
+def test_invalid_feed_cursor_triggers_full_reindex(feed):
+    mod, provider = feed
+    seed(mod, provider, "stable")
+    with mod.index_connect() as conn:
+        conn.execute("UPDATE local_index_state SET feed_cursor = 'expired-cursor'")
+    provider.nodes["hardgone"] = make_node("hardgone", 3)  # present only in the full listing
+
+    result = mod.sync_local_index({})
+
+    assert result["mode"] == "full_reindex"
+    assert result["reason"] == "feed_cursor_invalid"
+    assert local_ids(mod, "hardgone") == ["hardgone"]
+    assert mod.local_index_feed_state("project-1")["feed_cursor"] == f"c{len(provider.events)}"
+
+
+def test_old_server_without_feed_falls_back_to_timestamp_sync(feed):
+    mod, provider = feed
+    seed(mod, provider, "stable")
+    provider.fail["changes"] = 404
+    provider.nodes["later"] = make_node("later", 30, content="timestampterm")
+
+    result = mod.sync_local_index({})
+
+    assert result["mode"] == "incremental"
+    assert result["feed"] == "unsupported"
+    assert local_ids(mod, "timestampterm") == ["later"]
+
+
+def test_reindex_on_old_server_without_feed_still_rebuilds(feed):
+    mod, provider = feed
+    provider.fail["changes"] = 404
+    provider.put(make_node("legacy", 1), "node_created")
+
+    result = mod.reindex_project({})
+
+    assert result["graph_complete"] is True
+    assert local_ids(mod, "legacy") == ["legacy"]
+    assert mod.local_index_feed_state("project-1")["feed_cursor"] is None
+
+
+def test_search_auto_syncs_from_feed_and_throttles(feed):
+    mod, provider = feed
+    seed(mod, provider, "stable")
+    provider.put(make_node("autosync", 10, content="autosyncterm"), "node_created")
+
+    first = mod.call_tool_impl("nap_discover", {"q": "autosyncterm", "mode": "local"})
+    provider.put(make_node("throttled", 11, content="throttledterm"), "node_created")
+    second = mod.call_tool_impl("nap_discover", {"q": "throttledterm", "mode": "local"})
+
+    assert [item["node_id"] for item in first["items"]] == ["autosync"]
+    assert first["freshness"]["stale"] is False
+    assert first["freshness"]["feed_synced_at"]
+    assert second["items"] == []  # within the throttle window no second feed read
+    assert provider.count("changes") == 2  # reindex head + one automatic sync
+    assert second["freshness"]["feed_sync"] == "throttled"
+
+
+def test_search_answers_locally_and_marks_stale_when_feed_unavailable(feed):
+    mod, provider = feed
+    seed(mod, provider, "stable")
+    provider.fail["changes"] = 503
+
+    result = mod.call_tool_impl("nap_discover", {"q": "stable", "mode": "local"})
+
+    assert [item["node_id"] for item in result["items"]] == ["stable"]
+    assert result["freshness"]["stale"] is True
+    assert result["freshness"]["reason"] == "server_unavailable"
+    assert result["warnings"][0]["code"] == "server_unavailable"
+
+
+def test_context_local_read_auto_syncs_from_feed(feed):
+    mod, provider = feed
+    seed(mod, provider, "stable")
+    provider.put(make_node("contextnew", 10), "node_created")
+
+    result = mod.call_tool_impl("nap_context", {"paths": ["/notes/contextnew"], "mode": "hybrid"})
+
+    assert provider.count("changes") == 2
+    assert "contextnew" in json.dumps(result)
+    assert result["freshness"]["feed_sync"] == "synced"
+
+
+def cursorless_complete_index(mod, provider, *names):
+    for index, name in enumerate(names):
+        node = make_node(name, index)
+        provider.put(node, "node_created")
+        mod.index_node(node)
+    mod.set_local_graph_index_complete("project-1", True)
+
+
+def test_cursorless_index_bootstraps_on_first_search_then_refreshes(feed):
+    mod, provider = feed
+    cursorless_complete_index(mod, provider, "manual")
+    provider.nodes["gapnode"] = make_node("gapnode", 20, content="gapterm")  # changed before tracking
+
+    first = mod.call_tool_impl("nap_discover", {"q": "gapterm", "mode": "local"})
+
+    assert [kind for kind, *_ in provider.requests] == ["changes", "list"]
+    assert provider.requests[0][2]["after"] == ["head"]
+    assert [item["node_id"] for item in first["items"]] == ["gapnode"]
+    assert first["freshness"]["feed_sync"] == "tracking_initialized"
+    assert not first.get("warnings")
+    assert mod.local_index_feed_state("project-1")["feed_cursor"] == "c1"
+
+    provider.put(make_node("manual", 30, content="manual remoteupdateterm"), "node_updated")
+    throttled = mod.call_tool_impl("nap_discover", {"q": "remoteupdateterm", "mode": "local"})
+    assert throttled["items"] == [] and len(provider.requests) == 2  # no HTTP inside the window
+
+    mod.FEED_AUTO_SYNC_STATE.clear()  # the throttle window elapses
+    later = mod.call_tool_impl("nap_discover", {"q": "remoteupdateterm", "mode": "local"})
+    assert [item["node_id"] for item in later["items"]] == ["manual"]
+    assert later["freshness"]["feed_sync"] == "synced"
+
+
+def test_bootstrap_failure_answers_locally_warns_and_does_not_retry(feed):
+    mod, provider = feed
+    cursorless_complete_index(mod, provider, "manual")
+    provider.fail["changes"] = 503
+
+    first = mod.call_tool_impl("nap_discover", {"q": "manual", "mode": "local"})
+    second = mod.call_tool_impl("nap_discover", {"q": "manual", "mode": "local"})
+
+    for result in (first, second):
+        assert [item["node_id"] for item in result["items"]] == ["manual"]
+        assert [w["code"] for w in result["warnings"]] == ["index_not_tracking"]
+        assert result["warnings"][0]["next_action"] == "nap_index_sync"
+        assert result["freshness"]["stale"] is True
+    assert provider.count("changes") == 1  # one attempt per retry window
+    assert provider.count("list") == 0
+    assert mod.local_index_feed_state("project-1")["feed_cursor"] is None
+
+
+def test_bootstrap_budget_exhaustion_stays_untracked_and_applies_nothing(feed, monkeypatch):
+    mod, provider = feed
+    cursorless_complete_index(mod, provider, "manual")
+    provider.nodes["gapnode"] = make_node("gapnode", 20, content="gapterm")
+    monkeypatch.setattr(mod, "FEED_AUTO_SYNC_MAX_PAGES", 0)
+
+    result = mod.call_tool_impl("nap_discover", {"q": "gapterm", "mode": "local"})
+
+    assert result["items"] == []
+    assert [w["code"] for w in result["warnings"]] == ["index_not_tracking"]
+    assert mod.local_index_feed_state("project-1")["feed_cursor"] is None
+
+
+def test_bootstrap_on_old_server_marks_unsupported_without_retry(feed):
+    mod, provider = feed
+    cursorless_complete_index(mod, provider, "manual")
+    provider.fail["changes"] = 404
+
+    first = mod.call_tool_impl("nap_discover", {"q": "manual", "mode": "local"})
+    mod.call_tool_impl("nap_discover", {"q": "manual", "mode": "local"})
+
+    assert first["freshness"]["feed_sync"] == "unsupported"
+    assert not first.get("warnings")
+    assert provider.count("changes") == 1 and provider.count("list") == 0
+
+
+def test_feed_sync_is_bounded_by_pages_and_resumes(feed):
+    mod, provider = feed
+    seed(mod, provider, "stable")
+    start = len(provider.events)
+    for index in range(3):
+        provider.put(make_node(f"bounded{index}", 10 + index), "node_created")
+
+    first = mod.sync_local_index_from_feed("project-1", max_pages=2, page_limit=1)
+
+    assert (first["status"], first["pages"], first["has_more"]) == ("partial", 2, True)
+    assert mod.local_index_feed_state("project-1")["feed_cursor"] == f"c{start + 2}"
+    second = mod.sync_local_index_from_feed("project-1", max_pages=5, page_limit=1)
+    assert second["status"] == "synced"
+    assert local_ids(mod, "bounded2") == ["bounded2"]
+
+
+def test_explicit_sync_initializes_feed_tracking_for_cursorless_index(feed):
+    mod, provider = feed
+    provider.put(make_node("manual", 1), "node_created")
+    mod.index_node(make_node("manual", 1))
+    mod.set_local_graph_index_complete("project-1", True)
+
+    result = mod.sync_local_index({})
+
+    assert result["mode"] == "incremental"
+    assert result["feed"] == "initialized"
+    assert mod.local_index_feed_state("project-1")["feed_cursor"] == "c1"
+    provider.delete("manual")
+    mod.sync_local_index({})
+    assert local_ids(mod, "manual") == []  # hard delete observed through the feed