From cbdc1840559b933d02acee28c10a74d681953103 Mon Sep 17 00:00:00 2001 From: HumbleTurtle Date: Sat, 23 May 2026 01:54:09 -0300 Subject: [PATCH] Add incremental local index sync --- resources/scripts/napseer_mcp_server.py | 94 ++++++++++++++++++++- tests/test_mcp_memory_ergonomics_helpers.py | 63 ++++++++++++++ 2 files changed, 156 insertions(+), 1 deletion(-) diff --git a/resources/scripts/napseer_mcp_server.py b/resources/scripts/napseer_mcp_server.py index b72ca08..128af04 100755 --- a/resources/scripts/napseer_mcp_server.py +++ b/resources/scripts/napseer_mcp_server.py @@ -473,6 +473,7 @@ PUBLIC_TOOLS = { "nap_lock_release", "nap_lock_ls", "nap_reindex_project", + "nap_index_sync", "nap_grep_local", "nap_index_status", "nap_index_clear", @@ -5483,6 +5484,19 @@ def normalize_datetime_filter(value, *, end_of_day=False): return text +def iso_minus_seconds(value, seconds): + text = str(value or "").strip() + if not text: + return None + try: + parsed = datetime.fromisoformat(text.replace("Z", "+00:00")) + except ValueError: + return text + return (parsed - timedelta(seconds=max(0, int(seconds)))).astimezone(timezone.utc).replace( + microsecond=0 + ).isoformat().replace("+00:00", "Z") + + def normalize_discovery_ranges(args): normalized = dict(args or {}) if normalized.get("after") and not normalized.get("updated_after"): @@ -6132,12 +6146,18 @@ def local_index_diagnostics(project_id, conn=None): def read(connection): count = connection.execute("SELECT COUNT(*) AS count FROM local_index_nodes WHERE project_id = ?", (project_id,)).fetchone()["count"] latest = connection.execute("SELECT MAX(indexed_at) AS latest FROM local_index_nodes WHERE project_id = ?", (project_id,)).fetchone()["latest"] + latest_node_updated_at = connection.execute( + "SELECT MAX(updated_at) AS latest FROM local_index_nodes WHERE project_id = ?", + (project_id,), + ).fetchone()["latest"] diagnostics.update( { "count": count, "latest_indexed_at": latest, + "latest_node_updated_at": latest_node_updated_at, "used": False, "reindex_recommended": count == 0, + "sync_recommended": count == 0, } ) return diagnostics @@ -6256,6 +6276,59 @@ def reindex_project(args): return {"project_id": project_id, "indexed": len(nodes), "index_path": str(INDEX_PATH)} +def sync_local_index(args): + args = normalize_discovery_ranges(args or {}) + project_id = resolve_project_id(args) + limit = max(1, min(int(args.get("limit", 200)), 200)) + include_archived = as_bool(args.get("include_archived"), True) + lookback_seconds = normalize_int(args.get("lookback_seconds"), 5, 0, 3600) + status_before = local_index_diagnostics(project_id) + latest_node_updated_at = status_before.get("latest_node_updated_at") + updated_after = args.get("updated_after") or iso_minus_seconds(latest_node_updated_at, lookback_seconds) + nodes = [] + cursor = None + pages = 0 + while True: + query = { + "limit": limit, + "view": "full", + "include_archived": include_archived, + } + if updated_after: + query["updated_after"] = updated_after + if cursor: + query["cursor"] = cursor + page = request_json("GET", f"/v1/projects/{project_id}/nodes?{rest_query_string(query)}") + pages += 1 + nodes.extend(filter_project_nodes(page.get("items", []))) + cursor = page.get("next_cursor") + if not cursor: + break + indexed = 0 + removed = 0 + for node in nodes: + if node.get("archived_at"): + remove_indexed_node(node.get("id")) + removed += 1 + continue + index_node(node) + indexed += 1 + return { + "project_id": project_id, + "status": "synced", + "mode": "incremental", + "updated_after": updated_after, + "lookback_seconds": lookback_seconds, + "include_archived": include_archived, + "pages": pages, + "seen": len(nodes), + "indexed": indexed, + "removed": removed, + "index_path": str(INDEX_PATH), + "index": local_index_diagnostics(project_id), + } + + def index_status(args): project_id = resolve_project_id(args) return local_index_diagnostics(project_id) @@ -11066,6 +11139,21 @@ def raw_tools(): "additionalProperties": False, }, }, + { + "name": "nap_index_sync", + "description": "Incrementally synchronize the local SQLite FTS index from remote nodes changed since the latest indexed node update. Archived nodes are removed from the local index.", + "inputSchema": { + "type": "object", + "properties": { + "updated_after": {"type": "string", "description": "Optional RFC3339 timestamp or YYYY-MM-DD lower bound. Defaults to the local index high-water mark with a short overlap."}, + "since": {"type": "string", "description": "Alias for updated_after."}, + "limit": {"type": "integer", "minimum": 1, "maximum": 200, "default": 200}, + "lookback_seconds": {"type": "integer", "minimum": 0, "maximum": 3600, "default": 5}, + "include_archived": {"type": "boolean", "default": True}, + }, + "additionalProperties": False, + }, + }, { "name": "nap_grep_local", "description": "Search the local SQLite FTS index with optional updated_at range filters. Encrypted nodes are indexed by safe path, name, tags, aliases, and links only.", @@ -11999,6 +12087,7 @@ TOOL_CATEGORIES = { ], "maintenance": [ "nap_reindex_project", + "nap_index_sync", "nap_index_status", "nap_index_clear", "nap_update_status", @@ -12067,6 +12156,7 @@ READ_ONLY_TOOLS = { LOCAL_FILE_TOOLS = { "nap_update_self", "nap_reindex_project", + "nap_index_sync", "nap_index_clear", "nap_gateway_setup", "nap_gateway_configure", @@ -12127,7 +12217,7 @@ def tool_auth_mode(name): return "public" if category == "gateway": return "local-gateway" - if name in {"nap_grep_local", "nap_index_status", "nap_index_clear", "nap_reindex_project", "nap_ui_open", "nap_update_self"}: + if name in {"nap_grep_local", "nap_index_status", "nap_index_clear", "nap_reindex_project", "nap_index_sync", "nap_ui_open", "nap_update_self"}: return "local-only" if category in {"explore", "update", "coordination", "agent", "project", "schedules", "maintenance"}: return "project-auth" @@ -12384,6 +12474,8 @@ def call_tool_impl(name, args): return list_project_locks(args) if name == "nap_reindex_project": return reindex_project(args) + if name == "nap_index_sync": + return sync_local_index(args) if name == "nap_grep_local": return search_local_index(args) if name == "nap_index_status": diff --git a/tests/test_mcp_memory_ergonomics_helpers.py b/tests/test_mcp_memory_ergonomics_helpers.py index 573e628..5bc269e 100644 --- a/tests/test_mcp_memory_ergonomics_helpers.py +++ b/tests/test_mcp_memory_ergonomics_helpers.py @@ -4,6 +4,7 @@ import importlib.util import pathlib import sys +import tempfile import pytest @@ -811,6 +812,68 @@ def test_rest_query_string_serializes_booleans_lowercase(mod): assert "status=doing" in query +def test_index_status_reports_sync_high_water_mark(mod): + with tempfile.TemporaryDirectory() as tmp: + state_dir = pathlib.Path(tmp) + mod.AUTH_DIR = state_dir + mod.INDEX_PATH = state_dir / "index.sqlite" + mod.INDEX_LOCK_PATH = state_dir / "index.lock" + mod.index_node(node("/notes/old", "active", content="old body")) + + status = mod.index_status({}) + assert status["count"] == 1 + assert status["latest_node_updated_at"] == "2026-05-06T12:00:00Z" + assert status["sync_recommended"] is False + + +def test_index_sync_fetches_incremental_changes_and_removes_archived_nodes(mod): + with tempfile.TemporaryDirectory() as tmp: + state_dir = pathlib.Path(tmp) + mod.AUTH_DIR = state_dir + mod.INDEX_PATH = state_dir / "index.sqlite" + mod.INDEX_LOCK_PATH = state_dir / "index.lock" + existing = node("/notes/existing", "active", content="existing body") + existing["id"] = "existing-node" + existing["updated_at"] = "2026-05-06T12:00:00Z" + mod.index_node(existing) + + fresh = node("/notes/fresh", "active", content="fresh sync needle") + fresh["id"] = "fresh-node" + fresh["updated_at"] = "2026-05-06T12:05:00Z" + archived = node("/notes/existing", "active", content="archived body", archived_at="2026-05-06T12:06:00Z") + archived["id"] = "existing-node" + archived["updated_at"] = "2026-05-06T12:06:00Z" + calls = [] + + def fake_request(method, path, payload=None, **kwargs): + calls.append(path) + assert method == "GET" + assert "view=full" in path + assert "include_archived=true" in path + assert "updated_after=2026-05-06T11%3A59%3A55Z" in path + return {"items": [fresh, archived], "next_cursor": None} + + mod.request_json = fake_request + result = mod.sync_local_index({"lookback_seconds": 5}) + assert result["status"] == "synced" + assert result["seen"] == 2 + assert result["indexed"] == 1 + assert result["removed"] == 1 + assert calls + local = mod.search_local_index({"q": "fresh needle"}) + assert [item["id"] for item in local["items"]] == ["fresh-node"] + assert mod.search_local_index({"q": "existing"})["items"] == [] + + +def test_tools_include_index_sync_descriptor(mod): + tool = next(item for item in mod.tools() if item["name"] == "nap_index_sync") + assert "Incrementally synchronize" in tool["description"] + assert tool["inputSchema"]["properties"]["lookback_seconds"]["default"] == 5 + contract = mod.tool_contract_metadata("nap_index_sync") + assert contract["side_effect"] == "local-file" + assert contract["auth_mode"] == "local-only" + + def test_kanban_list_grouping_and_filters(mod): doing = node( "/kanban/doing/fix-wrapper",