This is page 17 of 23. Use http://codebase.md/basicmachines-co/basic-memory?lines=true&page={x} to view the full context.
# Directory Structure
```
├── .claude
│ ├── agents
│ │ ├── python-developer.md
│ │ └── system-architect.md
│ └── commands
│ ├── release
│ │ ├── beta.md
│ │ ├── changelog.md
│ │ ├── release-check.md
│ │ └── release.md
│ ├── spec.md
│ └── test-live.md
├── .dockerignore
├── .github
│ ├── dependabot.yml
│ ├── ISSUE_TEMPLATE
│ │ ├── bug_report.md
│ │ ├── config.yml
│ │ ├── documentation.md
│ │ └── feature_request.md
│ └── workflows
│ ├── claude-code-review.yml
│ ├── claude-issue-triage.yml
│ ├── claude.yml
│ ├── dev-release.yml
│ ├── docker.yml
│ ├── pr-title.yml
│ ├── release.yml
│ └── test.yml
├── .gitignore
├── .python-version
├── CHANGELOG.md
├── CITATION.cff
├── CLA.md
├── CLAUDE.md
├── CODE_OF_CONDUCT.md
├── CONTRIBUTING.md
├── docker-compose.yml
├── Dockerfile
├── docs
│ ├── ai-assistant-guide-extended.md
│ ├── character-handling.md
│ ├── cloud-cli.md
│ └── Docker.md
├── justfile
├── LICENSE
├── llms-install.md
├── pyproject.toml
├── README.md
├── SECURITY.md
├── smithery.yaml
├── specs
│ ├── SPEC-1 Specification-Driven Development Process.md
│ ├── SPEC-10 Unified Deployment Workflow and Event Tracking.md
│ ├── SPEC-11 Basic Memory API Performance Optimization.md
│ ├── SPEC-12 OpenTelemetry Observability.md
│ ├── SPEC-13 CLI Authentication with Subscription Validation.md
│ ├── SPEC-14 Cloud Git Versioning & GitHub Backup.md
│ ├── SPEC-14- Cloud Git Versioning & GitHub Backup.md
│ ├── SPEC-15 Configuration Persistence via Tigris for Cloud Tenants.md
│ ├── SPEC-16 MCP Cloud Service Consolidation.md
│ ├── SPEC-17 Semantic Search with ChromaDB.md
│ ├── SPEC-18 AI Memory Management Tool.md
│ ├── SPEC-19 Sync Performance and Memory Optimization.md
│ ├── SPEC-2 Slash Commands Reference.md
│ ├── SPEC-3 Agent Definitions.md
│ ├── SPEC-4 Notes Web UI Component Architecture.md
│ ├── SPEC-5 CLI Cloud Upload via WebDAV.md
│ ├── SPEC-6 Explicit Project Parameter Architecture.md
│ ├── SPEC-7 POC to spike Tigris Turso for local access to cloud data.md
│ ├── SPEC-8 TigrisFS Integration.md
│ ├── SPEC-9 Multi-Project Bidirectional Sync Architecture.md
│ ├── SPEC-9 Signed Header Tenant Information.md
│ └── SPEC-9-1 Follow-Ups- Conflict, Sync, and Observability.md
├── src
│ └── basic_memory
│ ├── __init__.py
│ ├── alembic
│ │ ├── alembic.ini
│ │ ├── env.py
│ │ ├── migrations.py
│ │ ├── script.py.mako
│ │ └── versions
│ │ ├── 3dae7c7b1564_initial_schema.py
│ │ ├── 502b60eaa905_remove_required_from_entity_permalink.py
│ │ ├── 5fe1ab1ccebe_add_projects_table.py
│ │ ├── 647e7a75e2cd_project_constraint_fix.py
│ │ ├── 9d9c1cb7d8f5_add_mtime_and_size_columns_to_entity_.py
│ │ ├── a1b2c3d4e5f6_fix_project_foreign_keys.py
│ │ ├── b3c3938bacdb_relation_to_name_unique_index.py
│ │ ├── cc7172b46608_update_search_index_schema.py
│ │ └── e7e1f4367280_add_scan_watermark_tracking_to_project.py
│ ├── api
│ │ ├── __init__.py
│ │ ├── app.py
│ │ ├── routers
│ │ │ ├── __init__.py
│ │ │ ├── directory_router.py
│ │ │ ├── importer_router.py
│ │ │ ├── knowledge_router.py
│ │ │ ├── management_router.py
│ │ │ ├── memory_router.py
│ │ │ ├── project_router.py
│ │ │ ├── prompt_router.py
│ │ │ ├── resource_router.py
│ │ │ ├── search_router.py
│ │ │ └── utils.py
│ │ └── template_loader.py
│ ├── cli
│ │ ├── __init__.py
│ │ ├── app.py
│ │ ├── auth.py
│ │ ├── commands
│ │ │ ├── __init__.py
│ │ │ ├── cloud
│ │ │ │ ├── __init__.py
│ │ │ │ ├── api_client.py
│ │ │ │ ├── bisync_commands.py
│ │ │ │ ├── cloud_utils.py
│ │ │ │ ├── core_commands.py
│ │ │ │ ├── mount_commands.py
│ │ │ │ ├── rclone_config.py
│ │ │ │ ├── rclone_installer.py
│ │ │ │ ├── upload_command.py
│ │ │ │ └── upload.py
│ │ │ ├── command_utils.py
│ │ │ ├── db.py
│ │ │ ├── import_chatgpt.py
│ │ │ ├── import_claude_conversations.py
│ │ │ ├── import_claude_projects.py
│ │ │ ├── import_memory_json.py
│ │ │ ├── mcp.py
│ │ │ ├── project.py
│ │ │ ├── status.py
│ │ │ ├── sync.py
│ │ │ └── tool.py
│ │ └── main.py
│ ├── config.py
│ ├── db.py
│ ├── deps.py
│ ├── file_utils.py
│ ├── ignore_utils.py
│ ├── importers
│ │ ├── __init__.py
│ │ ├── base.py
│ │ ├── chatgpt_importer.py
│ │ ├── claude_conversations_importer.py
│ │ ├── claude_projects_importer.py
│ │ ├── memory_json_importer.py
│ │ └── utils.py
│ ├── markdown
│ │ ├── __init__.py
│ │ ├── entity_parser.py
│ │ ├── markdown_processor.py
│ │ ├── plugins.py
│ │ ├── schemas.py
│ │ └── utils.py
│ ├── mcp
│ │ ├── __init__.py
│ │ ├── async_client.py
│ │ ├── project_context.py
│ │ ├── prompts
│ │ │ ├── __init__.py
│ │ │ ├── ai_assistant_guide.py
│ │ │ ├── continue_conversation.py
│ │ │ ├── recent_activity.py
│ │ │ ├── search.py
│ │ │ └── utils.py
│ │ ├── resources
│ │ │ ├── ai_assistant_guide.md
│ │ │ └── project_info.py
│ │ ├── server.py
│ │ └── tools
│ │ ├── __init__.py
│ │ ├── build_context.py
│ │ ├── canvas.py
│ │ ├── chatgpt_tools.py
│ │ ├── delete_note.py
│ │ ├── edit_note.py
│ │ ├── list_directory.py
│ │ ├── move_note.py
│ │ ├── project_management.py
│ │ ├── read_content.py
│ │ ├── read_note.py
│ │ ├── recent_activity.py
│ │ ├── search.py
│ │ ├── utils.py
│ │ ├── view_note.py
│ │ └── write_note.py
│ ├── models
│ │ ├── __init__.py
│ │ ├── base.py
│ │ ├── knowledge.py
│ │ ├── project.py
│ │ └── search.py
│ ├── repository
│ │ ├── __init__.py
│ │ ├── entity_repository.py
│ │ ├── observation_repository.py
│ │ ├── project_info_repository.py
│ │ ├── project_repository.py
│ │ ├── relation_repository.py
│ │ ├── repository.py
│ │ └── search_repository.py
│ ├── schemas
│ │ ├── __init__.py
│ │ ├── base.py
│ │ ├── cloud.py
│ │ ├── delete.py
│ │ ├── directory.py
│ │ ├── importer.py
│ │ ├── memory.py
│ │ ├── project_info.py
│ │ ├── prompt.py
│ │ ├── request.py
│ │ ├── response.py
│ │ ├── search.py
│ │ └── sync_report.py
│ ├── services
│ │ ├── __init__.py
│ │ ├── context_service.py
│ │ ├── directory_service.py
│ │ ├── entity_service.py
│ │ ├── exceptions.py
│ │ ├── file_service.py
│ │ ├── initialization.py
│ │ ├── link_resolver.py
│ │ ├── project_service.py
│ │ ├── search_service.py
│ │ └── service.py
│ ├── sync
│ │ ├── __init__.py
│ │ ├── background_sync.py
│ │ ├── sync_service.py
│ │ └── watch_service.py
│ ├── templates
│ │ └── prompts
│ │ ├── continue_conversation.hbs
│ │ └── search.hbs
│ └── utils.py
├── test-int
│ ├── BENCHMARKS.md
│ ├── cli
│ │ ├── test_project_commands_integration.py
│ │ ├── test_sync_commands_integration.py
│ │ └── test_version_integration.py
│ ├── conftest.py
│ ├── mcp
│ │ ├── test_build_context_underscore.py
│ │ ├── test_build_context_validation.py
│ │ ├── test_chatgpt_tools_integration.py
│ │ ├── test_default_project_mode_integration.py
│ │ ├── test_delete_note_integration.py
│ │ ├── test_edit_note_integration.py
│ │ ├── test_list_directory_integration.py
│ │ ├── test_move_note_integration.py
│ │ ├── test_project_management_integration.py
│ │ ├── test_project_state_sync_integration.py
│ │ ├── test_read_content_integration.py
│ │ ├── test_read_note_integration.py
│ │ ├── test_search_integration.py
│ │ ├── test_single_project_mcp_integration.py
│ │ └── test_write_note_integration.py
│ ├── test_db_wal_mode.py
│ ├── test_disable_permalinks_integration.py
│ └── test_sync_performance_benchmark.py
├── tests
│ ├── __init__.py
│ ├── api
│ │ ├── conftest.py
│ │ ├── test_async_client.py
│ │ ├── test_continue_conversation_template.py
│ │ ├── test_directory_router.py
│ │ ├── test_importer_router.py
│ │ ├── test_knowledge_router.py
│ │ ├── test_management_router.py
│ │ ├── test_memory_router.py
│ │ ├── test_project_router_operations.py
│ │ ├── test_project_router.py
│ │ ├── test_prompt_router.py
│ │ ├── test_relation_background_resolution.py
│ │ ├── test_resource_router.py
│ │ ├── test_search_router.py
│ │ ├── test_search_template.py
│ │ ├── test_template_loader_helpers.py
│ │ └── test_template_loader.py
│ ├── cli
│ │ ├── conftest.py
│ │ ├── test_bisync_commands.py
│ │ ├── test_cli_tools.py
│ │ ├── test_cloud_authentication.py
│ │ ├── test_cloud_utils.py
│ │ ├── test_ignore_utils.py
│ │ ├── test_import_chatgpt.py
│ │ ├── test_import_claude_conversations.py
│ │ ├── test_import_claude_projects.py
│ │ ├── test_import_memory_json.py
│ │ └── test_upload.py
│ ├── conftest.py
│ ├── db
│ │ └── test_issue_254_foreign_key_constraints.py
│ ├── importers
│ │ ├── test_importer_base.py
│ │ └── test_importer_utils.py
│ ├── markdown
│ │ ├── __init__.py
│ │ ├── test_date_frontmatter_parsing.py
│ │ ├── test_entity_parser_error_handling.py
│ │ ├── test_entity_parser.py
│ │ ├── test_markdown_plugins.py
│ │ ├── test_markdown_processor.py
│ │ ├── test_observation_edge_cases.py
│ │ ├── test_parser_edge_cases.py
│ │ ├── test_relation_edge_cases.py
│ │ └── test_task_detection.py
│ ├── mcp
│ │ ├── conftest.py
│ │ ├── test_obsidian_yaml_formatting.py
│ │ ├── test_permalink_collision_file_overwrite.py
│ │ ├── test_prompts.py
│ │ ├── test_resources.py
│ │ ├── test_tool_build_context.py
│ │ ├── test_tool_canvas.py
│ │ ├── test_tool_delete_note.py
│ │ ├── test_tool_edit_note.py
│ │ ├── test_tool_list_directory.py
│ │ ├── test_tool_move_note.py
│ │ ├── test_tool_read_content.py
│ │ ├── test_tool_read_note.py
│ │ ├── test_tool_recent_activity.py
│ │ ├── test_tool_resource.py
│ │ ├── test_tool_search.py
│ │ ├── test_tool_utils.py
│ │ ├── test_tool_view_note.py
│ │ ├── test_tool_write_note.py
│ │ └── tools
│ │ └── test_chatgpt_tools.py
│ ├── Non-MarkdownFileSupport.pdf
│ ├── repository
│ │ ├── test_entity_repository_upsert.py
│ │ ├── test_entity_repository.py
│ │ ├── test_entity_upsert_issue_187.py
│ │ ├── test_observation_repository.py
│ │ ├── test_project_info_repository.py
│ │ ├── test_project_repository.py
│ │ ├── test_relation_repository.py
│ │ ├── test_repository.py
│ │ ├── test_search_repository_edit_bug_fix.py
│ │ └── test_search_repository.py
│ ├── schemas
│ │ ├── test_base_timeframe_minimum.py
│ │ ├── test_memory_serialization.py
│ │ ├── test_memory_url_validation.py
│ │ ├── test_memory_url.py
│ │ ├── test_schemas.py
│ │ └── test_search.py
│ ├── Screenshot.png
│ ├── services
│ │ ├── test_context_service.py
│ │ ├── test_directory_service.py
│ │ ├── test_entity_service_disable_permalinks.py
│ │ ├── test_entity_service.py
│ │ ├── test_file_service.py
│ │ ├── test_initialization.py
│ │ ├── test_link_resolver.py
│ │ ├── test_project_removal_bug.py
│ │ ├── test_project_service_operations.py
│ │ ├── test_project_service.py
│ │ └── test_search_service.py
│ ├── sync
│ │ ├── test_character_conflicts.py
│ │ ├── test_sync_service_incremental.py
│ │ ├── test_sync_service.py
│ │ ├── test_sync_wikilink_issue.py
│ │ ├── test_tmp_files.py
│ │ ├── test_watch_service_edge_cases.py
│ │ ├── test_watch_service_reload.py
│ │ └── test_watch_service.py
│ ├── test_config.py
│ ├── test_db_migration_deduplication.py
│ ├── test_deps.py
│ ├── test_production_cascade_delete.py
│ └── utils
│ ├── test_file_utils.py
│ ├── test_frontmatter_obsidian_compatible.py
│ ├── test_parse_tags.py
│ ├── test_permalink_formatting.py
│ ├── test_utf8_handling.py
│ └── test_validate_project_path.py
├── uv.lock
├── v0.15.0-RELEASE-DOCS.md
└── v15-docs
├── api-performance.md
├── background-relations.md
├── basic-memory-home.md
├── bug-fixes.md
├── chatgpt-integration.md
├── cloud-authentication.md
├── cloud-bisync.md
├── cloud-mode-usage.md
├── cloud-mount.md
├── default-project-mode.md
├── env-file-removal.md
├── env-var-overrides.md
├── explicit-project-parameter.md
├── gitignore-integration.md
├── project-root-env-var.md
├── README.md
└── sqlite-performance.md
```
# Files
--------------------------------------------------------------------------------
/specs/SPEC-16 MCP Cloud Service Consolidation.md:
--------------------------------------------------------------------------------
```markdown
1 | ---
2 | title: 'SPEC-16: MCP Cloud Service Consolidation'
3 | type: spec
4 | permalink: specs/spec-16-mcp-cloud-service-consolidation
5 | tags:
6 | - architecture
7 | - mcp
8 | - cloud
9 | - performance
10 | - deployment
11 | status: in-progress
12 | ---
13 |
14 | ## Status Update
15 |
16 | **Phase 0 (Basic Memory Refactor): ✅ COMPLETE**
17 | - basic-memory PR #344: async_client context manager pattern implemented
18 | - All 17 MCP tools updated to use `async with get_client() as client:`
19 | - CLI commands updated to use context manager
20 | - Removed `inject_auth_header()` and `headers.py` (~100 lines deleted)
21 | - Factory pattern enables clean dependency injection
22 | - Tests passing, typecheck clean
23 |
24 | **Phase 0 Integration: ✅ COMPLETE**
25 | - basic-memory-cloud updated to use async-client-context-manager branch
26 | - Implemented `tenant_direct_client_factory()` with proper context manager pattern
27 | - Removed module-level client override hacks
28 | - Removed unnecessary `/proxy` prefix stripping (tools pass relative URLs)
29 | - Typecheck and lint passing with proper noqa hints
30 | - MCP tools confirmed working via inspector (local testing)
31 |
32 | **Phase 1 (Code Consolidation): ✅ COMPLETE**
33 | - MCP server mounted on Cloud FastAPI app at /mcp endpoint
34 | - AuthKitProvider configured with WorkOS settings
35 | - Combined lifespans (Cloud + MCP) working correctly
36 | - JWT context middleware integrated
37 | - All routes and MCP tools functional
38 |
39 | **Phase 2 (Direct Tenant Transport): ✅ COMPLETE**
40 | - TenantDirectTransport implemented with custom httpx transport
41 | - Per-request JWT extraction via FastMCP DI
42 | - Tenant lookup and signed header generation working
43 | - Direct routing to tenant APIs (eliminating HTTP hop)
44 | - Transport tests passing (11/11)
45 |
46 | **Phase 3 (Testing & Validation): ✅ COMPLETE**
47 | - Typecheck and lint passing across all services
48 | - MCP OAuth authentication working in preview environment
49 | - Tenant isolation via signed headers verified
50 | - Fixed BM_TENANT_HEADER_SECRET mismatch between environments
51 | - MCP tools successfully calling tenant APIs in preview
52 |
53 | **Phase 4 (Deployment Configuration): ✅ COMPLETE**
54 | - Updated apps/cloud/fly.template.toml with MCP environment variables
55 | - Added HTTP/2 backend support for better MCP performance
56 | - Added OAuth protected resource health check
57 | - Removed MCP from preview deployment workflow
58 | - Successfully deployed to preview environment (PR #113)
59 | - All services operational at pr-113-basic-memory-cloud.fly.dev
60 |
61 | **Next Steps:**
62 | - Phase 5: Cleanup (remove apps/mcp directory)
63 | - Phase 6: Production rollout and performance measurement
64 |
65 | # SPEC-16: MCP Cloud Service Consolidation
66 |
67 | ## Why
68 |
69 | ### Original Architecture Constraints (Now Removed)
70 |
71 | The current architecture deploys MCP Gateway and Cloud Service as separate Fly.io apps:
72 |
73 | **Current Flow:**
74 | ```
75 | LLM Client → MCP Gateway (OAuth) → Cloud Proxy (JWT + header signing) → Tenant API (JWT + header validation)
76 | apps/mcp apps/cloud /proxy apps/api
77 | ```
78 |
79 | This separation was originally necessary because:
80 | 1. **Stateful SSE requirement** - MCP needed server-sent events with session state for active project tracking
81 | 2. **fastmcp.run limitation** - The FastMCP demo helper didn't support worker processes
82 |
83 | ### Why These Constraints No Longer Apply
84 |
85 | 1. **State externalized** - Project state moved from in-memory to LLM context (external state)
86 | 2. **HTTP transport enabled** - Switched from SSE to stateless HTTP for MCP tools
87 | 3. **Worker support added** - Converted from `fastmcp.run()` to `uvicorn.run()` with workers
88 |
89 | ### Current Problems
90 |
91 | - **Unnecessary HTTP hop** - MCP tools call Cloud /proxy endpoint which calls tenant API
92 | - **Higher latency** - Extra network round trip for every MCP operation
93 | - **Increased costs** - Two separate Fly.io apps instead of one
94 | - **Complex deployment** - Two services to deploy, monitor, and maintain
95 | - **Resource waste** - Separate database connections, HTTP clients, telemetry overhead
96 |
97 | ## What
98 |
99 | ### Services Affected
100 |
101 | 1. **apps/mcp** - MCP Gateway service (to be merged)
102 | 2. **apps/cloud** - Cloud service (will receive MCP functionality)
103 | 3. **basic-memory** - Update `async_client.py` to use direct calls
104 | 4. **Deployment** - Consolidate Fly.io deployment to single app
105 |
106 | ### Components Changed
107 |
108 | **Merged:**
109 | - MCP middleware and telemetry into Cloud app
110 | - MCP tools mounted on Cloud FastAPI instance
111 | - ProxyService used directly by MCP tools (not via HTTP)
112 |
113 | **Kept:**
114 | - `/proxy` endpoint (still needed by web UI)
115 | - All existing Cloud routes (provisioning, webhooks, etc.)
116 | - Dual validation in tenant API (JWT + signed headers)
117 |
118 | **Removed:**
119 | - apps/mcp directory
120 | - Separate MCP Fly.io deployment
121 | - HTTP calls from MCP tools to /proxy endpoint
122 |
123 | ## How (High Level)
124 |
125 | ### 1. Mount FastMCP on Cloud FastAPI App
126 |
127 | ```python
128 | # apps/cloud/src/basic_memory_cloud/main.py
129 |
130 | from basic_memory.mcp.server import mcp
131 | from basic_memory_cloud_mcp.middleware import TelemetryMiddleware
132 |
133 | # Configure MCP OAuth
134 | auth_provider = AuthKitProvider(
135 | authkit_domain=settings.authkit_domain,
136 | base_url=settings.authkit_base_url,
137 | required_scopes=[],
138 | )
139 | mcp.auth = auth_provider
140 | mcp.add_middleware(TelemetryMiddleware())
141 |
142 | # Mount MCP at /mcp endpoint
143 | mcp_app = mcp.http_app(path="/mcp", stateless_http=True)
144 | app.mount("/mcp", mcp_app)
145 |
146 | # Existing Cloud routes stay at root
147 | app.include_router(proxy_router)
148 | app.include_router(provisioning_router)
149 | # ... etc
150 | ```
151 |
152 | ### 2. Direct Tenant Transport (No HTTP Hop)
153 |
154 | Instead of calling `/proxy`, MCP tools call tenant APIs directly via custom httpx transport.
155 |
156 | **Important:** No URL prefix stripping needed. The transport receives relative URLs like `/main/resource/notes/my-note` which are correctly routed to tenant APIs. The `/proxy` prefix only exists for web UI requests to the proxy router, not for MCP tools using the custom transport.
157 |
158 | ```python
159 | # apps/cloud/src/basic_memory_cloud/transports/tenant_direct.py
160 |
161 | from httpx import AsyncBaseTransport, Request, Response
162 | from fastmcp.server.dependencies import get_http_headers
163 | import jwt
164 |
165 | class TenantDirectTransport(AsyncBaseTransport):
166 | """Direct transport to tenant APIs, bypassing /proxy endpoint."""
167 |
168 | async def handle_async_request(self, request: Request) -> Response:
169 | # 1. Get JWT from current MCP request (via FastMCP DI)
170 | http_headers = get_http_headers()
171 | auth_header = http_headers.get("authorization") or http_headers.get("Authorization")
172 | token = auth_header.replace("Bearer ", "")
173 | claims = jwt.decode(token, options={"verify_signature": False})
174 | workos_user_id = claims["sub"]
175 |
176 | # 2. Look up tenant for user
177 | tenant = await tenant_service.get_tenant_by_user_id(workos_user_id)
178 |
179 | # 3. Build tenant app URL with signed headers
180 | fly_app_name = f"{settings.tenant_prefix}-{tenant.id}"
181 | target_url = f"https://{fly_app_name}.fly.dev{request.url.path}"
182 |
183 | headers = dict(request.headers)
184 | signer = create_signer(settings.bm_tenant_header_secret)
185 | headers.update(signer.sign_tenant_headers(tenant.id))
186 |
187 | # 4. Make direct call to tenant API
188 | response = await self.client.request(
189 | method=request.method, url=target_url,
190 | headers=headers, content=request.content
191 | )
192 | return response
193 | ```
194 |
195 | Then configure basic-memory's client factory before mounting MCP:
196 |
197 | ```python
198 | # apps/cloud/src/basic_memory_cloud/main.py
199 |
200 | from contextlib import asynccontextmanager
201 | from basic_memory.mcp import async_client
202 | from basic_memory_cloud.transports.tenant_direct import TenantDirectTransport
203 |
204 | # Configure factory for basic-memory's async_client
205 | @asynccontextmanager
206 | async def tenant_direct_client_factory():
207 | """Factory for creating clients with tenant direct transport."""
208 | client = httpx.AsyncClient(
209 | transport=TenantDirectTransport(),
210 | base_url="http://direct",
211 | )
212 | try:
213 | yield client
214 | finally:
215 | await client.aclose()
216 |
217 | # Set factory BEFORE importing MCP tools
218 | async_client.set_client_factory(tenant_direct_client_factory)
219 |
220 | # NOW import - tools will use our factory
221 | import basic_memory.mcp.tools
222 | import basic_memory.mcp.prompts
223 | from basic_memory.mcp.server import mcp
224 |
225 | # Mount MCP - tools use direct transport via factory
226 | app.mount("/mcp", mcp_app)
227 | ```
228 |
229 | **Key benefits:**
230 | - Clean dependency injection via factory pattern
231 | - Per-request tenant resolution via FastMCP DI
232 | - Proper resource cleanup (client.aclose() guaranteed)
233 | - Eliminates HTTP hop entirely
234 | - /proxy endpoint remains for web UI
235 |
236 | ### 3. Keep /proxy Endpoint for Web UI
237 |
238 | The existing `/proxy` HTTP endpoint remains functional for:
239 | - Web UI requests
240 | - Future external API consumers
241 | - Backward compatibility
242 |
243 | ### 4. Security: Maintain Dual Validation
244 |
245 | **Do NOT remove JWT validation from tenant API.** Keep defense in depth:
246 |
247 | ```python
248 | # apps/api - Keep both validations
249 | 1. JWT validation (from WorkOS token)
250 | 2. Signed header validation (from Cloud/MCP)
251 | ```
252 |
253 | This ensures if the Cloud service is compromised, attackers still cannot access tenant APIs without valid JWTs.
254 |
255 | ### 5. Deployment Changes
256 |
257 | **Before:**
258 | - `apps/mcp/fly.template.toml` → MCP Gateway deployment
259 | - `apps/cloud/fly.template.toml` → Cloud Service deployment
260 |
261 | **After:**
262 | - Remove `apps/mcp/fly.template.toml`
263 | - Update `apps/cloud/fly.template.toml` to expose port 8000 for both /mcp and /proxy
264 | - Update deployment scripts to deploy single consolidated app
265 |
266 |
267 | ## Basic Memory Dependency: Async Client Refactor
268 |
269 | ### Problem
270 | The current `basic_memory.mcp.async_client` creates a module-level `client` at import time:
271 | ```python
272 | client = create_client() # Runs immediately when module is imported
273 | ```
274 |
275 | This prevents dependency injection - by the time we can override it, tools have already imported it.
276 |
277 | ### Solution: Context Manager Pattern with Auth at Client Creation
278 |
279 | Refactor basic-memory to use httpx's context manager pattern instead of module-level client.
280 |
281 | **Key principle:** Authentication happens at client creation time, not per-request.
282 |
283 | ```python
284 | # basic_memory/src/basic_memory/mcp/async_client.py
285 | from contextlib import asynccontextmanager
286 | from httpx import AsyncClient, ASGITransport, Timeout
287 |
288 | # Optional factory override for dependency injection
289 | _client_factory = None
290 |
291 | def set_client_factory(factory):
292 | """Override the default client factory (for cloud app, testing, etc)."""
293 | global _client_factory
294 | _client_factory = factory
295 |
296 | @asynccontextmanager
297 | async def get_client():
298 | """Get an AsyncClient as a context manager.
299 |
300 | Usage:
301 | async with get_client() as client:
302 | response = await client.get(...)
303 | """
304 | if _client_factory:
305 | # Cloud app: custom transport handles everything
306 | async with _client_factory() as client:
307 | yield client
308 | else:
309 | # Default: create based on config
310 | config = ConfigManager().config
311 | timeout = Timeout(connect=10.0, read=30.0, write=30.0, pool=30.0)
312 |
313 | if config.cloud_mode_enabled:
314 | # CLI cloud mode: inject auth when creating client
315 | from basic_memory.cli.auth import CLIAuth
316 |
317 | auth = CLIAuth(
318 | client_id=config.cloud_client_id,
319 | authkit_domain=config.cloud_domain
320 | )
321 | token = await auth.get_valid_token()
322 |
323 | if not token:
324 | raise RuntimeError(
325 | "Cloud mode enabled but not authenticated. "
326 | "Run 'basic-memory cloud login' first."
327 | )
328 |
329 | # Auth header set ONCE at client creation
330 | async with AsyncClient(
331 | base_url=f"{config.cloud_host}/proxy",
332 | headers={"Authorization": f"Bearer {token}"},
333 | timeout=timeout
334 | ) as client:
335 | yield client
336 | else:
337 | # Local mode: ASGI transport
338 | async with AsyncClient(
339 | transport=ASGITransport(app=fastapi_app),
340 | base_url="http://test",
341 | timeout=timeout
342 | ) as client:
343 | yield client
344 | ```
345 |
346 | **Tool Updates:**
347 | ```python
348 | # Before: from basic_memory.mcp.async_client import client
349 | from basic_memory.mcp.async_client import get_client
350 |
351 | async def read_note(...):
352 | # Before: response = await call_get(client, path, ...)
353 | async with get_client() as client:
354 | response = await call_get(client, path, ...)
355 | # ... use response
356 | ```
357 |
358 | **Cloud Usage:**
359 | ```python
360 | from contextlib import asynccontextmanager
361 | from basic_memory.mcp import async_client
362 |
363 | @asynccontextmanager
364 | async def tenant_direct_client():
365 | """Factory for creating clients with tenant direct transport."""
366 | client = httpx.AsyncClient(
367 | transport=TenantDirectTransport(),
368 | base_url="http://direct",
369 | )
370 | try:
371 | yield client
372 | finally:
373 | await client.aclose()
374 |
375 | # Before importing MCP tools:
376 | async_client.set_client_factory(tenant_direct_client)
377 |
378 | # Now import - tools will use our factory
379 | import basic_memory.mcp.tools
380 | ```
381 |
382 | ### Benefits
383 | - **No module-level state** - client created only when needed
384 | - **Proper cleanup** - context manager ensures `aclose()` is called
385 | - **Easy dependency injection** - factory pattern allows custom clients
386 | - **httpx best practices** - follows official recommendations
387 | - **Works for all modes** - stdio, cloud, testing
388 |
389 | ### Architecture Simplification: Auth at Client Creation
390 |
391 | **Key design principle:** Authentication happens when creating the client, not on every request.
392 |
393 | **Three modes, three approaches:**
394 |
395 | 1. **Local mode (ASGI)**
396 | - No auth needed
397 | - Direct in-process calls via ASGITransport
398 |
399 | 2. **CLI cloud mode (HTTP)**
400 | - Auth token from CLIAuth (stored in ~/.basic-memory/basic-memory-cloud.json)
401 | - Injected as default header when creating AsyncClient
402 | - Single auth check at client creation time
403 |
404 | 3. **Cloud app mode (Custom Transport)**
405 | - TenantDirectTransport handles everything
406 | - Extracts JWT from FastMCP context per-request
407 | - No interaction with inject_auth_header() logic
408 |
409 | **What this removes:**
410 | - `src/basic_memory/mcp/tools/headers.py` - entire file deleted
411 | - `inject_auth_header()` calls in all request helpers (call_get, call_post, etc.)
412 | - Per-request header manipulation complexity
413 | - Circular dependency concerns between async_client and auth logic
414 |
415 | **Benefits:**
416 | - Cleaner separation of concerns
417 | - Simpler request helper functions
418 | - Auth happens at the right layer (client creation)
419 | - Cloud app transport is completely independent
420 |
421 | ### Refactor Summary
422 |
423 | This refactor achieves:
424 |
425 | **Simplification:**
426 | - Removes ~100 lines of per-request header injection logic
427 | - Deletes entire `headers.py` module
428 | - Auth happens once at client creation, not per-request
429 |
430 | **Decoupling:**
431 | - Cloud app's custom transport is completely independent
432 | - No interaction with basic-memory's auth logic
433 | - Each mode (local, CLI cloud, cloud app) has clean separation
434 |
435 | **Better Design:**
436 | - Follows httpx best practices (context managers)
437 | - Proper resource cleanup (client.aclose() guaranteed)
438 | - Easier testing via factory injection
439 | - No circular import risks
440 |
441 | **Three Distinct Modes:**
442 | 1. Local: ASGI transport, no auth
443 | 2. CLI cloud: HTTP transport with CLIAuth token injection
444 | 3. Cloud app: Custom transport with per-request tenant routing
445 |
446 | ### Implementation Plan Summary
447 | 1. Create branch `async-client-context-manager` in basic-memory
448 | 2. Update `async_client.py` with context manager pattern and CLIAuth integration
449 | 3. Remove `inject_auth_header()` from all request helpers
450 | 4. Delete `src/basic_memory/mcp/tools/headers.py`
451 | 5. Update all MCP tools to use `async with get_client() as client:`
452 | 6. Update CLI commands to use context manager and remove manual auth
453 | 7. Remove `api_url` config field
454 | 8. Update tests
455 | 9. Update basic-memory-cloud to use branch: `basic-memory @ git+https://github.com/basicmachines-co/basic-memory.git@async-client-context-manager`
456 |
457 | Detailed breakdown in Phase 0 tasks below.
458 |
459 | ### Implementation Notes
460 |
461 | **Potential Issues & Solutions:**
462 |
463 | 1. **Circular Import** (async_client imports CLIAuth)
464 | - **Risk:** CLIAuth might import something from async_client
465 | - **Solution:** Use lazy import inside `get_client()` function
466 | - **Already done:** Import is inside the function, not at module level
467 |
468 | 2. **Test Fixtures**
469 | - **Risk:** Tests using module-level client will break
470 | - **Solution:** Update fixtures to use factory pattern
471 | - **Example:**
472 | ```python
473 | @pytest.fixture
474 | def mock_client_factory():
475 | @asynccontextmanager
476 | async def factory():
477 | async with AsyncClient(...) as client:
478 | yield client
479 | return factory
480 | ```
481 |
482 | 3. **Performance**
483 | - **Risk:** Creating client per tool call might be expensive
484 | - **Reality:** httpx is designed for this pattern, connection pooling at transport level
485 | - **Mitigation:** Monitor performance, can optimize later if needed
486 |
487 | 4. **CLI Cloud Commands Edge Cases**
488 | - **Risk:** Token expires mid-operation
489 | - **Solution:** CLIAuth.get_valid_token() already handles refresh
490 | - **Validation:** Test cloud login → use tools → token refresh flow
491 |
492 | 5. **Backward Compatibility**
493 | - **Risk:** External code importing `client` directly
494 | - **Solution:** Keep `create_client()` and `client` for one version, deprecate
495 | - **Timeline:** Remove in next major version
496 |
497 | ## Implementation Tasks
498 |
499 | ### Phase 0: Basic Memory Refactor (Prerequisite)
500 |
501 | #### 0.1 Core Refactor - async_client.py
502 | - [x] Create branch `async-client-context-manager` in basic-memory repo
503 | - [x] Implement `get_client()` context manager
504 | - [x] Implement `set_client_factory()` for dependency injection
505 | - [x] Add CLI cloud mode auth injection (CLIAuth integration)
506 | - [x] Remove `api_url` config field (legacy, unused)
507 | - [x] Keep `create_client()` temporarily for backward compatibility (deprecate later)
508 |
509 | #### 0.2 Simplify Request Helpers - tools/utils.py
510 | - [x] Remove `inject_auth_header()` calls from `call_get()`
511 | - [x] Remove `inject_auth_header()` calls from `call_post()`
512 | - [x] Remove `inject_auth_header()` calls from `call_put()`
513 | - [x] Remove `inject_auth_header()` calls from `call_patch()`
514 | - [x] Remove `inject_auth_header()` calls from `call_delete()`
515 | - [x] Delete `src/basic_memory/mcp/tools/headers.py` entirely
516 | - [x] Update imports in utils.py
517 |
518 | #### 0.3 Update MCP Tools (~16 files)
519 | Convert from `from async_client import client` to `async with get_client() as client:`
520 |
521 | - [x] `tools/write_note.py` (34/34 tests passing)
522 | - [x] `tools/read_note.py` (21/21 tests passing)
523 | - [x] `tools/view_note.py` (12/12 tests passing - no changes needed, delegates to read_note)
524 | - [x] `tools/delete_note.py` (2/2 tests passing)
525 | - [x] `tools/read_content.py` (20/20 tests passing)
526 | - [x] `tools/list_directory.py` (11/11 tests passing)
527 | - [x] `tools/move_note.py` (34/34 tests passing, 90% coverage)
528 | - [x] `tools/search.py` (16/16 tests passing, 96% coverage)
529 | - [x] `tools/recent_activity.py` (4/4 tests passing, 82% coverage)
530 | - [x] `tools/project_management.py` (3 functions: list_memory_projects, create_memory_project, delete_project - typecheck passed)
531 | - [x] `tools/edit_note.py` (17/17 tests passing)
532 | - [x] `tools/canvas.py` (5/5 tests passing)
533 | - [x] `tools/build_context.py` (6/6 tests passing)
534 | - [x] `tools/sync_status.py` (typecheck passed)
535 | - [x] `prompts/continue_conversation.py` (typecheck passed)
536 | - [x] `prompts/search.py` (typecheck passed)
537 | - [x] `resources/project_info.py` (typecheck passed)
538 |
539 | #### 0.4 Update CLI Commands (~3 files)
540 | Remove manual auth header passing, use context manager:
541 |
542 | - [x] `cli/commands/project.py` - removed get_authenticated_headers() calls, use context manager
543 | - [x] `cli/commands/status.py` - use context manager
544 | - [x] `cli/commands/command_utils.py` - use context manager
545 |
546 | #### 0.5 Update Config
547 | - [x] Remove `api_url` field from `BasicMemoryConfig` in config.py
548 | - [x] Update any lingering references/docs (added deprecation notice to v15-docs/cloud-mode-usage.md)
549 |
550 | #### 0.6 Testing
551 | - [-] Update test fixtures to use factory pattern
552 | - [x] Run full test suite in basic-memory
553 | - [x] Verify cloud_mode_enabled works with CLIAuth injection
554 | - [x] Run typecheck and linting
555 |
556 | #### 0.7 Cloud Integration Prep
557 | - [x] Update basic-memory-cloud pyproject.toml to use branch
558 | - [x] Implement factory pattern in cloud app main.py
559 | - [x] Remove `/proxy` prefix stripping logic (not needed - tools pass relative URLs)
560 |
561 | #### 0.8 Phase 0 Validation
562 |
563 | **Before merging async-client-context-manager branch:**
564 |
565 | - [x] All tests pass locally
566 | - [x] Typecheck passes (pyright/mypy)
567 | - [x] Linting passes (ruff)
568 | - [x] Manual test: local mode works (ASGI transport)
569 | - [x] Manual test: cloud login → cloud mode works (HTTP transport with auth)
570 | - [x] No import of `inject_auth_header` anywhere
571 | - [x] `headers.py` file deleted
572 | - [x] `api_url` config removed
573 | - [x] Tool functions properly scoped (client inside async with)
574 | - [ ] CLI commands properly scoped (client inside async with)
575 |
576 | **Integration validation:**
577 | - [x] basic-memory-cloud can import and use factory pattern
578 | - [x] TenantDirectTransport works without touching header injection
579 | - [x] No circular imports or lazy import issues
580 | - [x] MCP tools work via inspector (local testing confirmed)
581 |
582 | ### Phase 1: Code Consolidation
583 | - [x] Create feature branch `consolidate-mcp-cloud`
584 | - [x] Update `apps/cloud/src/basic_memory_cloud/config.py`:
585 | - [x] Add `authkit_base_url` field (already has authkit_domain)
586 | - [x] Workers config already exists ✓
587 | - [x] Update `apps/cloud/src/basic_memory_cloud/telemetry.py`:
588 | - [x] Add `logfire.instrument_mcp()` to existing setup
589 | - [x] Skip complex two-phase setup - use Cloud's simpler approach
590 | - [x] Create `apps/cloud/src/basic_memory_cloud/middleware/jwt_context.py`:
591 | - [x] FastAPI middleware to extract JWT claims from Authorization header
592 | - [x] Add tenant context (workos_user_id) to logfire baggage
593 | - [x] Simpler than FastMCP middleware version
594 | - [x] Update `apps/cloud/src/basic_memory_cloud/main.py`:
595 | - [x] Import FastMCP server from basic-memory
596 | - [x] Configure AuthKitProvider with WorkOS settings
597 | - [x] No FastMCP telemetry middleware needed (using FastAPI middleware instead)
598 | - [x] Create MCP ASGI app: `mcp_app = mcp.http_app(path='/mcp', stateless_http=True)`
599 | - [x] Combine lifespans (Cloud + MCP) using nested async context managers
600 | - [x] Mount MCP: `app.mount("/mcp", mcp_app)`
601 | - [x] Add JWT context middleware to FastAPI app
602 | - [x] Run typecheck - passes ✓
603 |
604 | ### Phase 2: Direct Tenant Transport
605 | - [x] Create `apps/cloud/src/basic_memory_cloud/transports/tenant_direct.py`:
606 | - [x] Implement `TenantDirectTransport(AsyncBaseTransport)`
607 | - [x] Use FastMCP DI (`get_http_headers()`) to extract JWT per-request
608 | - [x] Decode JWT to get `workos_user_id`
609 | - [x] Look up/create tenant via `TenantRepository.get_or_create_tenant_for_workos_user()`
610 | - [x] Build tenant app URL and add signed headers
611 | - [x] Make direct httpx call to tenant API
612 | - [x] No `/proxy` prefix stripping needed (tools pass relative URLs like `/main/resource/...`)
613 | - [x] Update `apps/cloud/src/basic_memory_cloud/main.py`:
614 | - [x] Refactored to use factory pattern instead of module-level override
615 | - [x] Implement `tenant_direct_client_factory()` context manager
616 | - [x] Call `async_client.set_client_factory()` before importing MCP tools
617 | - [x] Clean imports, proper noqa hints for lint
618 | - [x] Basic-memory refactor integrated (PR #344)
619 | - [x] Run typecheck - passes ✓
620 | - [x] Run lint - passes ✓
621 |
622 | ### Phase 3: Testing & Validation
623 | - [x] Run `just typecheck` in apps/cloud
624 | - [x] Run `just check` in project
625 | - [x] Run `just fix` - all lint errors fixed ✓
626 | - [x] Write comprehensive transport tests (11 tests passing) ✓
627 | - [x] Test MCP tools locally with consolidated service (inspector confirmed working)
628 | - [x] Verify OAuth authentication works (requires full deployment)
629 | - [x] Verify tenant isolation via signed headers (requires full deployment)
630 | - [x] Test /proxy endpoint still works for web UI
631 | - [ ] Measure latency before/after consolidation
632 | - [ ] Check telemetry traces span correctly
633 |
634 | ### Phase 4: Deployment Configuration
635 | - [x] Update `apps/cloud/fly.template.toml`:
636 | - [x] Merged MCP-specific environment variables (AUTHKIT_BASE_URL, FASTMCP_LOG_LEVEL, BASIC_MEMORY_*)
637 | - [x] Added HTTP/2 backend support (`h2_backend = true`) for better MCP performance
638 | - [x] Added health check for MCP OAuth endpoint (`/.well-known/oauth-protected-resource`)
639 | - [x] Port 8000 already exposed - serves both Cloud routes and /mcp endpoint
640 | - [x] Workers configured (UVICORN_WORKERS = 4)
641 | - [x] Update `.env.example`:
642 | - [x] Consolidated MCP Gateway section into Cloud app section
643 | - [x] Added AUTHKIT_BASE_URL, FASTMCP_LOG_LEVEL, BASIC_MEMORY_HOME
644 | - [x] Added LOG_LEVEL to Development Settings
645 | - [x] Documented that MCP now served at /mcp on Cloud service (port 8000)
646 | - [x] Test deployment to preview environment (PR #113)
647 | - [x] OAuth authentication verified
648 | - [x] MCP tools successfully calling tenant APIs
649 | - [x] Fixed BM_TENANT_HEADER_SECRET synchronization issue
650 |
651 | ### Phase 5: Cleanup
652 | - [x] Remove `apps/mcp/` directory entirely
653 | - [x] Remove MCP-specific fly.toml and deployment configs
654 | - [x] Update repository documentation
655 | - [x] Update CLAUDE.md with new architecture
656 | - [-] Archive old MCP deployment configs (if needed)
657 |
658 | ### Phase 6: Production Rollout
659 | - [ ] Deploy to development and validate
660 | - [ ] Monitor metrics and logs
661 | - [ ] Deploy to production
662 | - [ ] Verify production functionality
663 | - [ ] Document performance improvements
664 |
665 | ## Migration Plan
666 |
667 | ### Phase 1: Preparation
668 | 1. Create feature branch `consolidate-mcp-cloud`
669 | 2. Update basic-memory async_client.py for direct ProxyService calls
670 | 3. Update apps/cloud/main.py to mount MCP
671 |
672 | ### Phase 2: Testing
673 | 1. Local testing with consolidated app
674 | 2. Deploy to development environment
675 | 3. Run full test suite
676 | 4. Performance benchmarking
677 |
678 | ### Phase 3: Deployment
679 | 1. Deploy to development
680 | 2. Validate all functionality
681 | 3. Deploy to production
682 | 4. Monitor for issues
683 |
684 | ### Phase 4: Cleanup
685 | 1. Remove apps/mcp directory
686 | 2. Update documentation
687 | 3. Update deployment scripts
688 | 4. Archive old MCP deployment configs
689 |
690 | ## Rollback Plan
691 |
692 | If issues arise:
693 | 1. Revert feature branch
694 | 2. Redeploy separate apps/mcp and apps/cloud services
695 | 3. Restore previous fly.toml configurations
696 | 4. Document issues encountered
697 |
698 | The well-organized code structure makes splitting back out feasible if future scaling needs diverge.
699 |
700 | ## How to Evaluate
701 |
702 | ### 1. Functional Testing
703 |
704 | **MCP Tools:**
705 | - [ ] All 17 MCP tools work via consolidated /mcp endpoint
706 | - [x] OAuth authentication validates correctly
707 | - [x] Tenant isolation maintained via signed headers
708 | - [x] Project management tools function correctly
709 |
710 | **Cloud Routes:**
711 | - [x] /proxy endpoint still works for web UI
712 | - [x] /provisioning routes functional
713 | - [x] /webhooks routes functional
714 | - [x] /tenants routes functional
715 |
716 | **API Validation:**
717 | - [x] Tenant API validates both JWT and signed headers
718 | - [x] Unauthorized requests rejected appropriately
719 | - [x] Multi-tenant isolation verified
720 |
721 | ### 2. Performance Testing
722 |
723 | **Latency Reduction:**
724 | - [x] Measure MCP tool latency before consolidation
725 | - [x] Measure MCP tool latency after consolidation
726 | - [x] Verify reduction from eliminated HTTP hop (expected: 20-50ms improvement)
727 |
728 | **Resource Usage:**
729 | - [x] Single app uses less total memory than two apps
730 | - [x] Database connection pooling more efficient
731 | - [x] HTTP client overhead reduced
732 |
733 | ### 3. Deployment Testing
734 |
735 | **Fly.io Deployment:**
736 | - [x] Single app deploys successfully
737 | - [x] Health checks pass for consolidated service
738 | - [x] No apps/mcp deployment required
739 | - [x] Environment variables configured correctly
740 |
741 | **Local Development:**
742 | - [x] `just setup` works with consolidated architecture
743 | - [x] Local testing shows MCP tools working
744 | - [x] No regression in developer experience
745 |
746 | ### 4. Security Validation
747 |
748 | **Defense in Depth:**
749 | - [x] Tenant API still validates JWT tokens
750 | - [x] Tenant API still validates signed headers
751 | - [x] No access possible with only signed headers (JWT required)
752 | - [x] No access possible with only JWT (signed headers required)
753 |
754 | **Authorization:**
755 | - [x] Users can only access their own tenant data
756 | - [x] Cross-tenant requests rejected
757 | - [x] Admin operations require proper authentication
758 |
759 | ### 5. Observability
760 |
761 | **Telemetry:**
762 | - [x] OpenTelemetry traces span across MCP → ProxyService → Tenant API
763 | - [x] Logfire shows consolidated traces correctly
764 | - [x] Error tracking and debugging still functional
765 | - [x] Performance metrics accurate
766 |
767 | **Logging:**
768 | - [x] Structured logs show proper context (tenant_id, operation, etc.)
769 | - [x] Error logs contain actionable information
770 | - [x] Log volume reasonable for single app
771 |
772 | ## Success Criteria
773 |
774 | 1. **Functionality**: All MCP tools and Cloud routes work identically to before
775 | 2. **Performance**: Measurable latency reduction (>20ms average)
776 | 3. **Cost**: Single Fly.io app instead of two (50% infrastructure reduction)
777 | 4. **Security**: Dual validation maintained, no security regression
778 | 5. **Deployment**: Simplified deployment process, single app to manage
779 | 6. **Observability**: Telemetry and logging work correctly
780 |
781 |
782 |
783 | ## Notes
784 |
785 | ### Future Considerations
786 |
787 | - **Independent scaling**: If MCP and Cloud need different scaling profiles in future, code organization supports splitting back out
788 | - **Regional deployment**: Consolidated app can still be deployed to multiple regions
789 | - **Edge caching**: Could add edge caching layer in front of consolidated service
790 |
791 | ### Dependencies
792 |
793 | - SPEC-9: Signed Header Tenant Information (already implemented)
794 | - SPEC-12: OpenTelemetry Observability (telemetry must work across merged services)
795 |
796 | ### Related Work
797 |
798 | - basic-memory v0.13.x: MCP server implementation
799 | - FastMCP documentation: Mounting on existing FastAPI apps
800 | - Fly.io multi-service patterns
801 |
```
--------------------------------------------------------------------------------
/tests/mcp/test_tool_move_note.py:
--------------------------------------------------------------------------------
```python
1 | """Tests for the move_note MCP tool."""
2 |
3 | import pytest
4 | from unittest.mock import patch
5 |
6 | from basic_memory.mcp.tools.move_note import move_note, _format_move_error_response
7 | from basic_memory.mcp.tools.write_note import write_note
8 | from basic_memory.mcp.tools.read_note import read_note
9 |
10 |
11 | @pytest.mark.asyncio
12 | async def test_move_note_success(app, client, test_project):
13 | """Test successfully moving a note to a new location."""
14 | # Create initial note
15 | await write_note.fn(
16 | project=test_project.name,
17 | title="Test Note",
18 | folder="source",
19 | content="# Test Note\nOriginal content here.",
20 | )
21 |
22 | # Move note
23 | result = await move_note.fn(
24 | project=test_project.name,
25 | identifier="source/test-note",
26 | destination_path="target/MovedNote.md",
27 | )
28 |
29 | assert isinstance(result, str)
30 | assert "✅ Note moved successfully" in result
31 |
32 | # Verify original location no longer exists
33 | try:
34 | await read_note.fn(test_project.name, "source/test-note")
35 | assert False, "Original note should not exist after move"
36 | except Exception:
37 | pass # Expected - note should not exist at original location
38 |
39 | # Verify note exists at new location with same content
40 | content = await read_note.fn("target/moved-note", project=test_project.name)
41 | assert "# Test Note" in content
42 | assert "Original content here" in content
43 | assert "permalink: target/moved-note" in content
44 |
45 |
46 | @pytest.mark.asyncio
47 | async def test_move_note_with_folder_creation(client, test_project):
48 | """Test moving note creates necessary folders."""
49 | # Create initial note
50 | await write_note.fn(
51 | project=test_project.name,
52 | title="Deep Note",
53 | folder="",
54 | content="# Deep Note\nContent in root folder.",
55 | )
56 |
57 | # Move to deeply nested path
58 | result = await move_note.fn(
59 | project=test_project.name,
60 | identifier="deep-note",
61 | destination_path="deeply/nested/folder/DeepNote.md",
62 | )
63 |
64 | assert isinstance(result, str)
65 | assert "✅ Note moved successfully" in result
66 |
67 | # Verify note exists at new location
68 | content = await read_note.fn("deeply/nested/folder/deep-note", project=test_project.name)
69 | assert "# Deep Note" in content
70 | assert "Content in root folder" in content
71 |
72 |
73 | @pytest.mark.asyncio
74 | async def test_move_note_with_observations_and_relations(app, client, test_project):
75 | """Test moving note preserves observations and relations."""
76 | # Create note with complex semantic content
77 | await write_note.fn(
78 | project=test_project.name,
79 | title="Complex Entity",
80 | folder="source",
81 | content="""# Complex Entity
82 |
83 | ## Observations
84 | - [note] Important observation #tag1
85 | - [feature] Key feature #feature
86 |
87 | ## Relations
88 | - relation to [[SomeOtherEntity]]
89 | - depends on [[Dependency]]
90 |
91 | Some additional content.
92 | """,
93 | )
94 |
95 | # Move note
96 | result = await move_note.fn(
97 | project=test_project.name,
98 | identifier="source/complex-entity",
99 | destination_path="target/MovedComplex.md",
100 | )
101 |
102 | assert isinstance(result, str)
103 | assert "✅ Note moved successfully" in result
104 |
105 | # Verify moved note preserves all content
106 | content = await read_note.fn("target/moved-complex", project=test_project.name)
107 | assert "Important observation #tag1" in content
108 | assert "Key feature #feature" in content
109 | assert "[[SomeOtherEntity]]" in content
110 | assert "[[Dependency]]" in content
111 | assert "Some additional content" in content
112 |
113 |
114 | @pytest.mark.asyncio
115 | async def test_move_note_by_title(client, test_project):
116 | """Test moving note using title as identifier."""
117 | # Create note with unique title
118 | await write_note.fn(
119 | project=test_project.name,
120 | title="UniqueTestTitle",
121 | folder="source",
122 | content="# UniqueTestTitle\nTest content.",
123 | )
124 |
125 | # Move using title as identifier
126 | result = await move_note.fn(
127 | project=test_project.name,
128 | identifier="UniqueTestTitle",
129 | destination_path="target/MovedByTitle.md",
130 | )
131 |
132 | assert isinstance(result, str)
133 | assert "✅ Note moved successfully" in result
134 |
135 | # Verify note exists at new location
136 | content = await read_note.fn("target/moved-by-title", project=test_project.name)
137 | assert "# UniqueTestTitle" in content
138 | assert "Test content" in content
139 |
140 |
141 | @pytest.mark.asyncio
142 | async def test_move_note_by_file_path(client, test_project):
143 | """Test moving note using file path as identifier."""
144 | # Create initial note
145 | await write_note.fn(
146 | project=test_project.name,
147 | title="PathTest",
148 | folder="source",
149 | content="# PathTest\nContent for path test.",
150 | )
151 |
152 | # Move using file path as identifier
153 | result = await move_note.fn(
154 | project=test_project.name,
155 | identifier="source/PathTest.md",
156 | destination_path="target/MovedByPath.md",
157 | )
158 |
159 | assert isinstance(result, str)
160 | assert "✅ Note moved successfully" in result
161 |
162 | # Verify note exists at new location
163 | content = await read_note.fn("target/moved-by-path", project=test_project.name)
164 | assert "# PathTest" in content
165 | assert "Content for path test" in content
166 |
167 |
168 | @pytest.mark.asyncio
169 | async def test_move_note_nonexistent_note(client, test_project):
170 | """Test moving a note that doesn't exist."""
171 | result = await move_note.fn(
172 | project=test_project.name,
173 | identifier="nonexistent/note",
174 | destination_path="target/SomeFile.md",
175 | )
176 |
177 | # Should return user-friendly error message string
178 | assert isinstance(result, str)
179 | assert "# Move Failed - Note Not Found" in result
180 | assert "could not be found for moving" in result
181 | assert "Search for the note first" in result
182 |
183 |
184 | @pytest.mark.asyncio
185 | async def test_move_note_invalid_destination_path(client, test_project):
186 | """Test moving note with invalid destination path."""
187 | # Create initial note
188 | await write_note.fn(
189 | project=test_project.name,
190 | title="TestNote",
191 | folder="source",
192 | content="# TestNote\nTest content.",
193 | )
194 |
195 | # Test absolute path (should be rejected by validation)
196 | result = await move_note.fn(
197 | project=test_project.name,
198 | identifier="source/test-note",
199 | destination_path="/absolute/path.md",
200 | )
201 |
202 | # Should return user-friendly error message string
203 | assert isinstance(result, str)
204 | assert "# Move Failed" in result
205 | assert "/absolute/path.md" in result or "Invalid" in result or "path" in result
206 |
207 |
208 | @pytest.mark.asyncio
209 | async def test_move_note_missing_file_extension(client, test_project):
210 | """Test moving note without file extension in destination path."""
211 | # Create initial note
212 | await write_note.fn(
213 | project=test_project.name,
214 | title="ExtensionTest",
215 | folder="source",
216 | content="# Extension Test\nTesting extension validation.",
217 | )
218 |
219 | # Test path without extension
220 | result = await move_note.fn(
221 | project=test_project.name,
222 | identifier="source/extension-test",
223 | destination_path="target/renamed-note",
224 | )
225 |
226 | # Should return error about missing extension
227 | assert isinstance(result, str)
228 | assert "# Move Failed - File Extension Required" in result
229 | assert "must include a file extension" in result
230 | assert ".md" in result
231 | assert "renamed-note.md" in result # Should suggest adding .md
232 |
233 | # Test path with empty extension (edge case)
234 | result = await move_note.fn(
235 | project=test_project.name,
236 | identifier="source/extension-test",
237 | destination_path="target/renamed-note.",
238 | )
239 |
240 | assert isinstance(result, str)
241 | assert "# Move Failed - File Extension Required" in result
242 | assert "must include a file extension" in result
243 |
244 | # Test that note still exists at original location
245 | content = await read_note.fn("source/extension-test", project=test_project.name)
246 | assert "# Extension Test" in content
247 | assert "Testing extension validation" in content
248 |
249 |
250 | @pytest.mark.asyncio
251 | async def test_move_note_file_extension_mismatch(client, test_project):
252 | """Test that moving note with different extension is blocked."""
253 | # Create initial note with .md extension
254 | await write_note.fn(
255 | project=test_project.name,
256 | title="MarkdownNote",
257 | folder="source",
258 | content="# Markdown Note\nThis is a markdown file.",
259 | )
260 |
261 | # Try to move with .txt extension
262 | result = await move_note.fn(
263 | project=test_project.name,
264 | identifier="source/markdown-note",
265 | destination_path="target/renamed-note.txt",
266 | )
267 |
268 | # Should return error about extension mismatch
269 | assert isinstance(result, str)
270 | assert "# Move Failed - File Extension Mismatch" in result
271 | assert "does not match the source file extension" in result
272 | assert ".md" in result
273 | assert ".txt" in result
274 | assert "renamed-note.md" in result # Should suggest correct extension
275 |
276 | # Test that note still exists at original location with original extension
277 | content = await read_note.fn("source/markdown-note", project=test_project.name)
278 | assert "# Markdown Note" in content
279 | assert "This is a markdown file" in content
280 |
281 |
282 | @pytest.mark.asyncio
283 | async def test_move_note_preserves_file_extension(client, test_project):
284 | """Test that moving note with matching extension succeeds."""
285 | # Create initial note with .md extension
286 | await write_note.fn(
287 | project=test_project.name,
288 | title="PreserveExtension",
289 | folder="source",
290 | content="# Preserve Extension\nTesting that extension is preserved.",
291 | )
292 |
293 | # Move with same .md extension
294 | result = await move_note.fn(
295 | project=test_project.name,
296 | identifier="source/preserve-extension",
297 | destination_path="target/preserved-note.md",
298 | )
299 |
300 | # Should succeed
301 | assert isinstance(result, str)
302 | assert "✅ Note moved successfully" in result
303 |
304 | # Verify note exists at new location with same extension
305 | content = await read_note.fn("target/preserved-note", project=test_project.name)
306 | assert "# Preserve Extension" in content
307 | assert "Testing that extension is preserved" in content
308 |
309 | # Verify old location no longer exists
310 | try:
311 | await read_note.fn("source/preserve-extension")
312 | assert False, "Original note should not exist after move"
313 | except Exception:
314 | pass # Expected
315 |
316 |
317 | @pytest.mark.asyncio
318 | async def test_move_note_destination_exists(client, test_project):
319 | """Test moving note to existing destination."""
320 | # Create source note
321 | await write_note.fn(
322 | project=test_project.name,
323 | title="SourceNote",
324 | folder="source",
325 | content="# SourceNote\nSource content.",
326 | )
327 |
328 | # Create destination note
329 | await write_note.fn(
330 | project=test_project.name,
331 | title="DestinationNote",
332 | folder="target",
333 | content="# DestinationNote\nDestination content.",
334 | )
335 |
336 | # Try to move source to existing destination
337 | result = await move_note.fn(
338 | project=test_project.name,
339 | identifier="source/source-note",
340 | destination_path="target/DestinationNote.md",
341 | )
342 |
343 | # Should return user-friendly error message string
344 | assert isinstance(result, str)
345 | assert "# Move Failed" in result
346 | assert "already exists" in result or "Destination" in result
347 |
348 |
349 | @pytest.mark.asyncio
350 | async def test_move_note_same_location(client, test_project):
351 | """Test moving note to the same location."""
352 | # Create initial note
353 | await write_note.fn(
354 | project=test_project.name,
355 | title="SameLocationTest",
356 | folder="test",
357 | content="# SameLocationTest\nContent here.",
358 | )
359 |
360 | # Try to move to same location
361 | result = await move_note.fn(
362 | project=test_project.name,
363 | identifier="test/same-location-test",
364 | destination_path="test/SameLocationTest.md",
365 | )
366 |
367 | # Should return user-friendly error message string
368 | assert isinstance(result, str)
369 | assert "# Move Failed" in result
370 | assert "already exists" in result or "same" in result or "Destination" in result
371 |
372 |
373 | @pytest.mark.asyncio
374 | async def test_move_note_rename_only(client, test_project):
375 | """Test moving note within same folder (rename operation)."""
376 | # Create initial note
377 | await write_note.fn(
378 | project=test_project.name,
379 | title="OriginalName",
380 | folder="test",
381 | content="# OriginalName\nContent to rename.",
382 | )
383 |
384 | # Rename within same folder
385 | await move_note.fn(
386 | project=test_project.name,
387 | identifier="test/original-name",
388 | destination_path="test/NewName.md",
389 | )
390 |
391 | # Verify original is gone
392 | try:
393 | await read_note.fn("test/original-name", project=test_project.name)
394 | assert False, "Original note should not exist after rename"
395 | except Exception:
396 | pass # Expected
397 |
398 | # Verify new name exists with same content
399 | content = await read_note.fn("test/new-name", project=test_project.name)
400 | assert "# OriginalName" in content # Title in content remains same
401 | assert "Content to rename" in content
402 | assert "permalink: test/new-name" in content
403 |
404 |
405 | @pytest.mark.asyncio
406 | async def test_move_note_complex_filename(client, test_project):
407 | """Test moving note with spaces in filename."""
408 | # Create note with spaces in name
409 | await write_note.fn(
410 | project=test_project.name,
411 | title="Meeting Notes 2025",
412 | folder="meetings",
413 | content="# Meeting Notes 2025\nMeeting content with dates.",
414 | )
415 |
416 | # Move to new location
417 | result = await move_note.fn(
418 | project=test_project.name,
419 | identifier="meetings/meeting-notes-2025",
420 | destination_path="archive/2025/meetings/Meeting Notes 2025.md",
421 | )
422 |
423 | assert isinstance(result, str)
424 | assert "✅ Note moved successfully" in result
425 |
426 | # Verify note exists at new location with correct content
427 | content = await read_note.fn(
428 | "archive/2025/meetings/meeting-notes-2025", project=test_project.name
429 | )
430 | assert "# Meeting Notes 2025" in content
431 | assert "Meeting content with dates" in content
432 |
433 |
434 | @pytest.mark.asyncio
435 | async def test_move_note_with_tags(app, client, test_project):
436 | """Test moving note with tags preserves tags."""
437 | # Create note with tags
438 | await write_note.fn(
439 | project=test_project.name,
440 | title="Tagged Note",
441 | folder="source",
442 | content="# Tagged Note\nContent with tags.",
443 | tags=["important", "work", "project"],
444 | )
445 |
446 | # Move note
447 | result = await move_note.fn(
448 | project=test_project.name,
449 | identifier="source/tagged-note",
450 | destination_path="target/MovedTaggedNote.md",
451 | )
452 |
453 | assert isinstance(result, str)
454 | assert "✅ Note moved successfully" in result
455 |
456 | # Verify tags are preserved in correct YAML format
457 | content = await read_note.fn("target/moved-tagged-note", project=test_project.name)
458 | assert "- important" in content
459 | assert "- work" in content
460 | assert "- project" in content
461 |
462 |
463 | @pytest.mark.asyncio
464 | async def test_move_note_empty_string_destination(client, test_project):
465 | """Test moving note with empty destination path."""
466 | # Create initial note
467 | await write_note.fn(
468 | project=test_project.name,
469 | title="TestNote",
470 | folder="source",
471 | content="# TestNote\nTest content.",
472 | )
473 |
474 | # Test empty destination path
475 | result = await move_note.fn(
476 | project=test_project.name,
477 | identifier="source/test-note",
478 | destination_path="",
479 | )
480 |
481 | # Should return user-friendly error message string
482 | assert isinstance(result, str)
483 | assert "# Move Failed" in result
484 | assert "empty" in result or "Invalid" in result or "path" in result
485 |
486 |
487 | @pytest.mark.asyncio
488 | async def test_move_note_parent_directory_path(client, test_project):
489 | """Test moving note with parent directory in destination path."""
490 | # Create initial note
491 | await write_note.fn(
492 | project=test_project.name,
493 | title="TestNote",
494 | folder="source",
495 | content="# TestNote\nTest content.",
496 | )
497 |
498 | # Test parent directory path
499 | result = await move_note.fn(
500 | project=test_project.name,
501 | identifier="source/test-note",
502 | destination_path="../parent/file.md",
503 | )
504 |
505 | # Should return user-friendly error message string
506 | assert isinstance(result, str)
507 | assert "# Move Failed" in result
508 | assert "parent" in result or "Invalid" in result or "path" in result or ".." in result
509 |
510 |
511 | @pytest.mark.asyncio
512 | async def test_move_note_identifier_variations(client, test_project):
513 | """Test that various identifier formats work for moving."""
514 | # Create a note to test different identifier formats
515 | await write_note.fn(
516 | project=test_project.name,
517 | title="Test Document",
518 | folder="docs",
519 | content="# Test Document\nContent for testing identifiers.",
520 | )
521 |
522 | # Test with permalink identifier
523 | result = await move_note.fn(
524 | project=test_project.name,
525 | identifier="docs/test-document",
526 | destination_path="moved/TestDocument.md",
527 | )
528 |
529 | assert isinstance(result, str)
530 | assert "✅ Note moved successfully" in result
531 |
532 | # Verify it moved correctly
533 | content = await read_note.fn("moved/test-document", project=test_project.name)
534 | assert "# Test Document" in content
535 | assert "Content for testing identifiers" in content
536 |
537 |
538 | @pytest.mark.asyncio
539 | async def test_move_note_preserves_frontmatter(app, client, test_project):
540 | """Test that moving preserves custom frontmatter."""
541 | # Create note with custom frontmatter by first creating it normally
542 | await write_note.fn(
543 | project=test_project.name,
544 | title="Custom Frontmatter Note",
545 | folder="source",
546 | content="# Custom Frontmatter Note\nContent with custom metadata.",
547 | )
548 |
549 | # Move the note
550 | result = await move_note.fn(
551 | project=test_project.name,
552 | identifier="source/custom-frontmatter-note",
553 | destination_path="target/MovedCustomNote.md",
554 | )
555 |
556 | assert isinstance(result, str)
557 | assert "✅ Note moved successfully" in result
558 |
559 | # Verify the moved note has proper frontmatter structure
560 | content = await read_note.fn("target/moved-custom-note", project=test_project.name)
561 | assert "title: Custom Frontmatter Note" in content
562 | assert "type: note" in content
563 | assert "permalink: target/moved-custom-note" in content
564 | assert "# Custom Frontmatter Note" in content
565 | assert "Content with custom metadata" in content
566 |
567 |
568 | class TestMoveNoteErrorFormatting:
569 | """Test move note error formatting for better user experience."""
570 |
571 | def test_format_move_error_invalid_path(self):
572 | """Test formatting for invalid path errors."""
573 | result = _format_move_error_response("invalid path format", "test-note", "/invalid/path.md")
574 |
575 | assert "# Move Failed - Invalid Destination Path" in result
576 | assert "The destination path '/invalid/path.md' is not valid" in result
577 | assert "Relative paths only" in result
578 | assert "Include file extension" in result
579 |
580 | def test_format_move_error_permission_denied(self):
581 | """Test formatting for permission errors."""
582 | result = _format_move_error_response("permission denied", "test-note", "target/file.md")
583 |
584 | assert "# Move Failed - Permission Error" in result
585 | assert "You don't have permission to move 'test-note'" in result
586 | assert "Check file permissions" in result
587 | assert "Check file locks" in result
588 |
589 | def test_format_move_error_source_missing(self):
590 | """Test formatting for source file missing errors."""
591 | result = _format_move_error_response("source file missing", "test-note", "target/file.md")
592 |
593 | assert "# Move Failed - Source File Missing" in result
594 | assert "The source file for 'test-note' was not found on disk" in result
595 | assert "database and filesystem are out of sync" in result
596 |
597 | def test_format_move_error_server_error(self):
598 | """Test formatting for server errors."""
599 | result = _format_move_error_response("server error occurred", "test-note", "target/file.md")
600 |
601 | assert "# Move Failed - System Error" in result
602 | assert "A system error occurred while moving 'test-note'" in result
603 | assert "Try again" in result
604 | assert "Check disk space" in result
605 |
606 |
607 | class TestMoveNoteSecurityValidation:
608 | """Test move note security validation features."""
609 |
610 | @pytest.mark.asyncio
611 | async def test_move_note_blocks_path_traversal_unix(self, client, test_project):
612 | """Test that Unix-style path traversal attacks are blocked."""
613 | # Create initial note
614 | await write_note.fn(
615 | project=test_project.name,
616 | title="Test Note",
617 | folder="source",
618 | content="# Test Note\nTest content for security testing.",
619 | )
620 |
621 | # Test various Unix-style path traversal patterns
622 | attack_paths = [
623 | "../secrets.txt",
624 | "../../etc/passwd",
625 | "../../../root/.ssh/id_rsa",
626 | "notes/../../../etc/shadow",
627 | "folder/../../outside/file.md",
628 | "../../../../etc/hosts",
629 | ]
630 |
631 | for attack_path in attack_paths:
632 | result = await move_note.fn(
633 | project=test_project.name,
634 | identifier="source/test-note",
635 | destination_path=attack_path,
636 | )
637 |
638 | assert isinstance(result, str)
639 | assert "# Move Failed - Security Validation Error" in result
640 | assert "paths must stay within project boundaries" in result
641 | assert attack_path in result
642 | assert "Try again with a safe path" in result
643 |
644 | @pytest.mark.asyncio
645 | async def test_move_note_blocks_path_traversal_windows(self, client, test_project):
646 | """Test that Windows-style path traversal attacks are blocked."""
647 | # Create initial note
648 | await write_note.fn(
649 | project=test_project.name,
650 | title="Test Note",
651 | folder="source",
652 | content="# Test Note\nTest content for security testing.",
653 | )
654 |
655 | # Test various Windows-style path traversal patterns
656 | attack_paths = [
657 | "..\\secrets.txt",
658 | "..\\..\\Windows\\System32\\config\\SAM",
659 | "notes\\..\\..\\..\\Windows\\System32",
660 | "\\\\server\\share\\file.txt",
661 | "..\\..\\Users\\user\\.env",
662 | "\\\\..\\..\\Windows",
663 | ]
664 |
665 | for attack_path in attack_paths:
666 | result = await move_note.fn(
667 | project=test_project.name,
668 | identifier="source/test-note",
669 | destination_path=attack_path,
670 | )
671 |
672 | assert isinstance(result, str)
673 | assert "# Move Failed - Security Validation Error" in result
674 | assert "paths must stay within project boundaries" in result
675 | assert attack_path in result
676 |
677 | @pytest.mark.asyncio
678 | async def test_move_note_blocks_absolute_paths(self, client, test_project):
679 | """Test that absolute paths are blocked."""
680 | # Create initial note
681 | await write_note.fn(
682 | project=test_project.name,
683 | title="Test Note",
684 | folder="source",
685 | content="# Test Note\nTest content for security testing.",
686 | )
687 |
688 | # Test various absolute path patterns
689 | attack_paths = [
690 | "/etc/passwd",
691 | "/home/user/.env",
692 | "/var/log/auth.log",
693 | "/root/.ssh/id_rsa",
694 | "C:\\Windows\\System32\\config\\SAM",
695 | "C:\\Users\\user\\.env",
696 | "D:\\secrets\\config.json",
697 | "/tmp/malicious.txt",
698 | ]
699 |
700 | for attack_path in attack_paths:
701 | result = await move_note.fn(
702 | project=test_project.name,
703 | identifier="source/test-note",
704 | destination_path=attack_path,
705 | )
706 |
707 | assert isinstance(result, str)
708 | assert "# Move Failed - Security Validation Error" in result
709 | assert "paths must stay within project boundaries" in result
710 | assert attack_path in result
711 |
712 | @pytest.mark.asyncio
713 | async def test_move_note_blocks_home_directory_access(self, client, test_project):
714 | """Test that home directory access patterns are blocked."""
715 | # Create initial note
716 | await write_note.fn(
717 | project=test_project.name,
718 | title="Test Note",
719 | folder="source",
720 | content="# Test Note\nTest content for security testing.",
721 | )
722 |
723 | # Test various home directory access patterns
724 | attack_paths = [
725 | "~/secrets.txt",
726 | "~/.env",
727 | "~/.ssh/id_rsa",
728 | "~/Documents/passwords.txt",
729 | "~\\AppData\\secrets",
730 | "~\\Desktop\\config.ini",
731 | ]
732 |
733 | for attack_path in attack_paths:
734 | result = await move_note.fn(
735 | project=test_project.name,
736 | identifier="source/test-note",
737 | destination_path=attack_path,
738 | )
739 |
740 | assert isinstance(result, str)
741 | assert "# Move Failed - Security Validation Error" in result
742 | assert "paths must stay within project boundaries" in result
743 | assert attack_path in result
744 |
745 | @pytest.mark.asyncio
746 | async def test_move_note_blocks_mixed_attack_patterns(self, client, test_project):
747 | """Test that mixed legitimate/attack patterns are blocked."""
748 | # Create initial note
749 | await write_note.fn(
750 | project=test_project.name,
751 | title="Test Note",
752 | folder="source",
753 | content="# Test Note\nTest content for security testing.",
754 | )
755 |
756 | # Test mixed patterns that start legitimate but contain attacks
757 | attack_paths = [
758 | "notes/../../../etc/passwd",
759 | "docs/../../.env",
760 | "legitimate/path/../../.ssh/id_rsa",
761 | "project/folder/../../../Windows/System32",
762 | "valid/folder/../../home/user/.bashrc",
763 | ]
764 |
765 | for attack_path in attack_paths:
766 | result = await move_note.fn(
767 | project=test_project.name,
768 | identifier="source/test-note",
769 | destination_path=attack_path,
770 | )
771 |
772 | assert isinstance(result, str)
773 | assert "# Move Failed - Security Validation Error" in result
774 | assert "paths must stay within project boundaries" in result
775 |
776 | @pytest.mark.asyncio
777 | async def test_move_note_allows_safe_paths(self, client, test_project):
778 | """Test that legitimate paths are still allowed."""
779 | # Create initial note
780 | await write_note.fn(
781 | project=test_project.name,
782 | title="Test Note",
783 | folder="source",
784 | content="# Test Note\nTest content for security testing.",
785 | )
786 |
787 | # Test various safe path patterns
788 | safe_paths = [
789 | "notes/meeting.md",
790 | "docs/readme.txt",
791 | "projects/2025/planning.md",
792 | "archive/old-notes/backup.md",
793 | "deep/nested/directory/structure/file.txt",
794 | "folder/subfolder/document.md",
795 | ]
796 |
797 | for safe_path in safe_paths:
798 | result = await move_note.fn(
799 | project=test_project.name,
800 | identifier="source/test-note",
801 | destination_path=safe_path,
802 | )
803 |
804 | # Should succeed or fail for legitimate reasons (not security)
805 | assert isinstance(result, str)
806 | # Should NOT contain security error message
807 | assert "Security Validation Error" not in result
808 |
809 | # If it fails, it should be for other reasons like "already exists" or API errors
810 | if "Move Failed" in result:
811 | assert "paths must stay within project boundaries" not in result
812 |
813 | @pytest.mark.asyncio
814 | async def test_move_note_security_logging(self, client, test_project, caplog):
815 | """Test that security violations are properly logged."""
816 | # Create initial note
817 | await write_note.fn(
818 | project=test_project.name,
819 | title="Test Note",
820 | folder="source",
821 | content="# Test Note\nTest content for security testing.",
822 | )
823 |
824 | # Attempt path traversal attack
825 | result = await move_note.fn(
826 | project=test_project.name,
827 | identifier="source/test-note",
828 | destination_path="../../../etc/passwd",
829 | )
830 |
831 | assert "# Move Failed - Security Validation Error" in result
832 |
833 | # Check that security violation was logged
834 | # Note: This test may need adjustment based on the actual logging setup
835 | # The security validation should generate a warning log entry
836 |
837 | @pytest.mark.asyncio
838 | async def test_move_note_empty_path_security(self, client, test_project):
839 | """Test that empty destination path is handled securely."""
840 | # Create initial note
841 | await write_note.fn(
842 | project=test_project.name,
843 | title="Test Note",
844 | folder="source",
845 | content="# Test Note\nTest content for security testing.",
846 | )
847 |
848 | # Test empty destination path (should be allowed as it resolves to project root)
849 | result = await move_note.fn(
850 | project=test_project.name,
851 | identifier="source/test-note",
852 | destination_path="",
853 | )
854 |
855 | assert isinstance(result, str)
856 | # Empty path should not trigger security error (it's handled by pathlib validation)
857 | # But may fail for other API-related reasons
858 |
859 | @pytest.mark.asyncio
860 | async def test_move_note_current_directory_references_security(self, client, test_project):
861 | """Test that current directory references are handled securely."""
862 | # Create initial note
863 | await write_note.fn(
864 | project=test_project.name,
865 | title="Test Note",
866 | folder="source",
867 | content="# Test Note\nTest content for security testing.",
868 | )
869 |
870 | # Test current directory references (should be safe)
871 | safe_paths = [
872 | "./notes/file.md",
873 | "folder/./file.md",
874 | "./folder/subfolder/file.md",
875 | ]
876 |
877 | for safe_path in safe_paths:
878 | result = await move_note.fn(
879 | project=test_project.name,
880 | identifier="source/test-note",
881 | destination_path=safe_path,
882 | )
883 |
884 | assert isinstance(result, str)
885 | # Should NOT contain security error message
886 | assert "Security Validation Error" not in result
887 |
888 |
889 | class TestMoveNoteErrorHandling:
890 | """Test move note exception handling."""
891 |
892 | @pytest.mark.asyncio
893 | async def test_move_note_exception_handling(self):
894 | """Test exception handling in move_note."""
895 | with patch("basic_memory.mcp.tools.move_note.get_active_project") as mock_get_project:
896 | mock_get_project.return_value.project_url = "http://test"
897 | mock_get_project.return_value.name = "test-project"
898 |
899 | with patch(
900 | "basic_memory.mcp.tools.move_note.call_post",
901 | side_effect=Exception("entity not found"),
902 | ):
903 | result = await move_note.fn("test-note", "target/file.md", project="test-project")
904 |
905 | assert isinstance(result, str)
906 | assert "# Move Failed - Note Not Found" in result
907 |
908 | @pytest.mark.asyncio
909 | async def test_move_note_permission_error_handling(self):
910 | """Test permission error handling in move_note."""
911 | with patch("basic_memory.mcp.tools.move_note.get_active_project") as mock_get_project:
912 | mock_get_project.return_value.project_url = "http://test"
913 | mock_get_project.return_value.name = "test-project"
914 |
915 | with patch(
916 | "basic_memory.mcp.tools.move_note.call_post",
917 | side_effect=Exception("permission denied"),
918 | ):
919 | result = await move_note.fn("test-note", "target/file.md", project="test-project")
920 |
921 | assert isinstance(result, str)
922 | assert "# Move Failed - Permission Error" in result
923 |
```
--------------------------------------------------------------------------------
/specs/SPEC-8 TigrisFS Integration.md:
--------------------------------------------------------------------------------
```markdown
1 | ---
2 | title: 'SPEC-8: TigrisFS Integration for Tenant API'
3 | Date: September 22, 2025
4 | Status: Phase 3.6 Complete - Tenant Mount API Endpoints Ready for CLI Implementation
5 | Priority: High
6 | Goal: Replace Fly volumes with Tigris bucket provisioning in production tenant API
7 | permalink: spec-8-tigris-fs-integration
8 | ---
9 |
10 | ## Executive Summary
11 |
12 | Based on SPEC-7 Phase 4 POC testing, this spec outlines productizing the TigrisFS/rclone implementation in the Basic Memory Cloud tenant API.
13 | We're moving from proof-of-concept to production integration, replacing Fly volume storage with Tigris bucket-per-tenant architecture.
14 |
15 | ## Current Architecture (Fly Volumes)
16 |
17 | ### Tenant Provisioning Flow
18 | ```python
19 | # apps/cloud/src/basic_memory_cloud/workflows/tenant_provisioning.py
20 | async def provision_tenant_infrastructure(tenant_id: str):
21 | # 1. Create Fly app
22 | # 2. Create Fly volume ← REPLACE THIS
23 | # 3. Deploy API container with volume mount
24 | # 4. Configure health checks
25 | ```
26 |
27 | ### Storage Implementation
28 | - Each tenant gets dedicated Fly volume (1GB-10GB)
29 | - Volume mounted at `/app/data` in API container
30 | - Local filesystem storage with Basic Memory indexing
31 | - No global caching or edge distribution
32 |
33 | ## Proposed Architecture (Tigris Buckets)
34 |
35 | ### New Tenant Provisioning Flow
36 | ```python
37 | async def provision_tenant_infrastructure(tenant_id: str):
38 | # 1. Create Fly app
39 | # 2. Create Tigris bucket with admin credentials ← NEW
40 | # 3. Store bucket name in tenant record ← NEW
41 | # 4. Deploy API container with TigrisFS mount using admin credentials
42 | # 5. Configure health checks
43 | ```
44 |
45 | ### Storage Implementation
46 | - Each tenant gets dedicated Tigris bucket
47 | - TigrisFS mounts bucket at `/app/data` in API container
48 | - Global edge caching and distribution
49 | - Configurable cache TTL for sync performance
50 |
51 | ## Implementation Plan
52 |
53 | ### Phase 1: Bucket Provisioning Service
54 |
55 | **✅ IMPLEMENTED: StorageClient with Admin Credentials**
56 | ```python
57 | # apps/cloud/src/basic_memory_cloud/clients/storage_client.py
58 | class StorageClient:
59 | async def create_tenant_bucket(self, tenant_id: UUID) -> TigrisBucketCredentials
60 | async def delete_tenant_bucket(self, tenant_id: UUID, bucket_name: str) -> bool
61 | async def list_buckets(self) -> list[TigrisBucketResponse]
62 | async def test_tenant_credentials(self, credentials: TigrisBucketCredentials) -> bool
63 | ```
64 |
65 | **Simplified Architecture Using Admin Credentials:**
66 | - Single admin access key with full Tigris permissions (configured in console)
67 | - No tenant-specific IAM user creation needed
68 | - Bucket-per-tenant isolation for logical separation
69 | - Admin credentials shared across all tenant operations
70 |
71 | **Integrate with Provisioning workflow:**
72 | ```python
73 | # Update tenant_provisioning.py
74 | async def provision_tenant_infrastructure(tenant_id: str):
75 | storage_client = StorageClient(settings.aws_access_key_id, settings.aws_secret_access_key)
76 | bucket_creds = await storage_client.create_tenant_bucket(tenant_id)
77 | await store_bucket_name(tenant_id, bucket_creds.bucket_name)
78 | await deploy_api_with_tigris(tenant_id, bucket_creds)
79 | ```
80 |
81 | ### Phase 2: Simplified Bucket Management
82 |
83 | **✅ SIMPLIFIED: Admin Credentials + Bucket Names Only**
84 |
85 | Since we use admin credentials for all operations, we only need to track bucket names per tenant:
86 |
87 | 1. **Primary Storage (Fly Secrets)**
88 | ```bash
89 | flyctl secrets set -a basic-memory-{tenant_id} \
90 | AWS_ACCESS_KEY_ID="{admin_access_key}" \
91 | AWS_SECRET_ACCESS_KEY="{admin_secret_key}" \
92 | AWS_ENDPOINT_URL_S3="https://fly.storage.tigris.dev" \
93 | AWS_REGION="auto" \
94 | BUCKET_NAME="basic-memory-{tenant_id}"
95 | ```
96 |
97 | 2. **Database Storage (Bucket Name Only)**
98 | ```python
99 | # apps/cloud/src/basic_memory_cloud/models/tenant.py
100 | class Tenant(BaseModel):
101 | # ... existing fields
102 | tigris_bucket_name: Optional[str] = None # Just store bucket name
103 | tigris_region: str = "auto"
104 | created_at: datetime
105 | ```
106 |
107 | **Benefits of Simplified Approach:**
108 | - No credential encryption/decryption needed
109 | - Admin credentials managed centrally in environment
110 | - Only bucket names stored in database (not sensitive)
111 | - Simplified backup/restore scenarios
112 | - Reduced security attack surface
113 |
114 | ### Phase 3: API Container Updates
115 |
116 | **Update API container configuration:**
117 | ```dockerfile
118 | # apps/api/Dockerfile
119 | # Add TigrisFS installation
120 | RUN curl -L https://github.com/tigrisdata/tigrisfs/releases/latest/download/tigrisfs-linux-amd64 \
121 | -o /usr/local/bin/tigrisfs && chmod +x /usr/local/bin/tigrisfs
122 | ```
123 |
124 | **Startup script integration:**
125 | ```bash
126 | # apps/api/tigrisfs-startup.sh (already exists)
127 | # Mount TigrisFS → Start Basic Memory API
128 | exec python -m basic_memory_cloud_api.main
129 | ```
130 |
131 | **Fly.toml environment (optimized for < 5s startup):**
132 | ```toml
133 | # apps/api/fly.tigris-production.toml
134 | [env]
135 | TIGRISFS_MEMORY_LIMIT = '1024' # Reduced for faster init
136 | TIGRISFS_MAX_FLUSHERS = '16' # Fewer threads for faster startup
137 | TIGRISFS_STAT_CACHE_TTL = '30s' # Balance sync speed vs startup
138 | TIGRISFS_LAZY_INIT = 'true' # Enable lazy loading
139 | BASIC_MEMORY_HOME = '/app/data'
140 |
141 | # Suspend optimization for wake-on-network
142 | [machine]
143 | auto_stop_machines = "suspend" # Faster than full stop
144 | auto_start_machines = true
145 | min_machines_running = 0
146 | ```
147 |
148 | ### Phase 4: Local Access Features
149 |
150 | **CLI automation for local mounting:**
151 | ```python
152 | # New CLI command: basic-memory cloud mount
153 | async def setup_local_mount(tenant_id: str):
154 | # 1. Fetch bucket credentials from cloud API
155 | # 2. Configure rclone with scoped IAM policy
156 | # 3. Mount via rclone nfsmount (macOS) or FUSE (Linux)
157 | # 4. Start Basic Memory sync watcher
158 | ```
159 |
160 | **Local mount configuration:**
161 | ```bash
162 | # rclone config for tenant
163 | rclone mount basic-memory-{tenant_id}: ~/basic-memory-{tenant_id} \
164 | --nfs-mount \
165 | --vfs-cache-mode writes \
166 | --cache-dir ~/.cache/rclone/basic-memory-{tenant_id}
167 | ```
168 |
169 | ### Phase 5: TigrisFS Cache Sync Solutions
170 |
171 | **Problem**: When files are uploaded via CLI/bisync, the tenant API container doesn't see them immediately due to TigrisFS cache (30s TTL) and lack of inotify events on mounted filesystems.
172 |
173 | **Multi-Layer Solution:**
174 |
175 | **Layer 1: API Sync Endpoint** (Immediate)
176 | ```python
177 | # POST /sync - Force TigrisFS cache refresh
178 | # Callable by CLI after uploads
179 | subprocess.run(["sync", "fsync /app/data"], check=True)
180 | ```
181 |
182 | **Layer 2: Tigris Webhook Integration** (Real-time)
183 | https://www.tigrisdata.com/docs/buckets/object-notifications/#webhook
184 | ```python
185 | # Webhook endpoint for bucket changes
186 | @app.post("/webhooks/tigris/{tenant_id}")
187 | async def handle_bucket_notification(tenant_id: str, event: TigrisEvent):
188 | if event.eventName in ["OBJECT_CREATED_PUT", "OBJECT_DELETED"]:
189 | await notify_container_sync(tenant_id, event.object.key)
190 | ```
191 |
192 | **Layer 3: CLI Sync Notification** (User-triggered)
193 | ```bash
194 | # CLI calls container sync endpoint after successful bisync
195 | basic-memory cloud bisync # Automatically notifies container
196 | curl -X POST https://basic-memory-{tenant-id}.fly.dev/sync
197 | ```
198 |
199 | **Layer 4: Periodic Sync Fallback** (Safety net)
200 | ```python
201 | # Background task: fsync /app/data every 30s as fallback
202 | # Ensures eventual consistency even if other layers fail
203 | ```
204 |
205 | **Implementation Priority:**
206 | 1. Layer 1 (API endpoint) - Quick testing capability
207 | 2. Layer 3 (CLI integration) - Improved UX
208 | 3. Layer 4 (Periodic fallback) - Safety net
209 | 4. Layer 2 (Webhooks) - Production real-time sync
210 |
211 |
212 | ## Performance Targets
213 |
214 | ### Sync Latency
215 | - **Target**: < 5 seconds local→cloud→container
216 | - **Configuration**: `TIGRISFS_STAT_CACHE_TTL = '5s'`
217 | - **Monitoring**: Track sync metrics in production
218 |
219 | ### Container Startup
220 | - **Target**: < 5 seconds including TigrisFS mount
221 | - **Fast retry**: 0.5s intervals for mount verification
222 | - **Fallback**: Container fails fast if mount fails
223 |
224 | ### Memory Usage
225 | - **TigrisFS cache**: 2GB memory limit per container
226 | - **Concurrent uploads**: 32 flushers max
227 | - **VM sizing**: shared-cpu-2x (2048mb) minimum
228 |
229 | ## Security Considerations
230 |
231 | ### Bucket Isolation
232 | - Each tenant has dedicated bucket
233 | - IAM policies prevent cross-tenant access
234 | - No shared bucket with subdirectories
235 |
236 | ### Credential Security
237 | - Fly secrets for runtime access
238 | - Encrypted database backup for disaster recovery
239 | - Credential rotation capability
240 |
241 | ### Data Residency
242 | - Tigris global edge caching
243 | - SOC2 Type II compliance
244 | - Encryption at rest and in transit
245 |
246 | ## Operational Benefits
247 |
248 | ### Scalability
249 | - Horizontal scaling with stateless API containers
250 | - Global edge distribution
251 | - Better resource utilization
252 |
253 | ### Reliability
254 | - No cold starts between tenants
255 | - Built-in redundancy and caching
256 | - Simplified backup strategy
257 |
258 | ### Cost Efficiency
259 | - Pay-per-use storage pricing
260 | - Shared infrastructure benefits
261 | - Reduced operational overhead
262 |
263 | ## Risk Mitigation
264 |
265 | ### Data Loss Prevention
266 | - Dual credential storage (Fly + database)
267 | - Automated backup workflows to R2/S3
268 | - Tigris built-in redundancy
269 |
270 | ### Performance Degradation
271 | - Configurable cache settings per tenant
272 | - Monitoring and alerting on sync latency
273 | - Fallback to volume storage if needed
274 |
275 | ### Security Vulnerabilities
276 | - Bucket-per-tenant isolation
277 | - Regular credential rotation
278 | - Security scanning and monitoring
279 |
280 | ## Success Metrics
281 |
282 | ### Technical Metrics
283 | - Sync latency P50 < 5 seconds
284 | - Container startup time < 5 seconds
285 | - Zero data loss incidents
286 | - 99.9% uptime per tenant
287 |
288 | ### Business Metrics
289 | - Reduced infrastructure costs vs volumes
290 | - Improved user experience with faster sync
291 | - Enhanced enterprise security posture
292 | - Simplified operational overhead
293 |
294 | ## Open Questions
295 |
296 | 1. **Tigris rate limits**: What are the API limits for bucket creation?
297 | 2. **Cost analysis**: What's the break-even point vs Fly volumes?
298 | 3. **Regional preferences**: Should enterprise customers choose regions?
299 | 4. **Backup retention**: How long to keep automated backups?
300 |
301 | ## Implementation Checklist
302 |
303 | ### Phase 1: Bucket Provisioning Service ✅ COMPLETED
304 | - [x] **Research Tigris bucket API** - Document bucket creation and S3 API compatibility
305 | - [x] **Create StorageClient class** - Implemented with admin credentials and comprehensive integration tests
306 | - [x] **Test bucket creation** - Full test suite validates API integration with real Tigris environment
307 | - [x] **Add bucket provisioning to DBOS workflow** - Integrated StorageClient with tenant_provisioning.py
308 |
309 | ### Phase 2: Simplified Bucket Management ✅ COMPLETED
310 | - [x] **Update Tenant model** with tigris_bucket_name field (replaced fly_volume_id)
311 | - [x] **Implement bucket name storage** - Database migration and model updates completed
312 | - [x] **Test bucket provisioning integration** - Full test suite validates workflow from tenant creation to bucket assignment
313 | - [x] **Remove volume logic from all tests** - Complete migration from volume-based to bucket-based architecture
314 |
315 | ### Phase 3: API Container Integration ✅ COMPLETED
316 | - [x] **Update Dockerfile** to install TigrisFS binary in API container with configurable version
317 | - [x] **Optimize tigrisfs-startup.sh** with production-ready security and reliability improvements
318 | - [x] **Create production-ready container** with proper signal handling and mount validation
319 | - [x] **Implement security fixes** based on Claude code review (conditional debug, credential protection)
320 | - [x] **Add proper process supervision** with cleanup traps and error handling
321 | - [x] **Remove debug artifacts** - Cleaned up all debug Dockerfiles and test scripts
322 |
323 | ### Phase 3.5: IAM Access Key Management ✅ COMPLETED
324 | - [x] **Research Tigris IAM API** - Documented create_policy, attach_user_policy, delete_access_key operations
325 | - [x] **Implement bucket-scoped credential generation** - StorageClient.create_tenant_access_keys() with IAM policies
326 | - [x] **Add comprehensive security test suite** - 5 security-focused integration tests covering all attack vectors
327 | - [x] **Verify cross-bucket access prevention** - Scoped credentials can ONLY access their designated bucket
328 | - [x] **Test credential lifecycle management** - Create, validate, delete, and revoke access keys
329 | - [x] **Validate admin vs scoped credential isolation** - Different access patterns and security boundaries
330 | - [x] **Test multi-tenant isolation** - Multiple tenants cannot access each other's buckets
331 |
332 | ### Phase 3.6: Tenant Mount API Endpoints ✅ COMPLETED
333 | - [x] **Implement GET /tenant/mount/info** - Returns mount info without exposing credentials
334 | - [x] **Implement POST /tenant/mount/credentials** - Creates new bucket-scoped credentials for CLI mounting
335 | - [x] **Implement DELETE /tenant/mount/credentials/{cred_id}** - Revoke specific credentials with proper cleanup
336 | - [x] **Implement GET /tenant/mount/credentials** - List active credentials without exposing secrets
337 | - [x] **Add TenantMountCredentials database model** - Tracks credential metadata (no secret storage)
338 | - [x] **Create comprehensive test suite** - 28 tests covering all scenarios including multi-session support
339 | - [x] **Implement multi-session credential flow** - Multiple active credentials per tenant supported
340 | - [x] **Secure credential handling** - Secret keys never stored, returned once only for immediate use
341 | - [x] **Add dependency injection for StorageClient** - Clean integration with existing API architecture
342 | - [x] **Fix Tigris configuration for cloud service** - Added AWS environment variables to fly.template.toml
343 | - [x] **Update tenant machine configurations** - Include AWS credentials for TigrisFS mounting with clear credential strategy
344 |
345 | **Security Test Results:**
346 | ```
347 | ✅ Cross-bucket access prevention - PASS
348 | ✅ Deleted credentials access revoked - PASS
349 | ✅ Invalid credentials rejected - PASS
350 | ✅ Admin vs scoped credential isolation - PASS
351 | ✅ Multiple scoped credentials isolation - PASS
352 | ```
353 |
354 | **Implementation Details:**
355 | - Uses Tigris IAM managed policies (create_policy + attach_user_policy)
356 | - Bucket-scoped S3 policies with Actions: GetObject, PutObject, DeleteObject, ListBucket
357 | - Resource ARNs limited to specific bucket: `arn:aws:s3:::bucket-name` and `arn:aws:s3:::bucket-name/*`
358 | - Access keys follow Tigris format: `tid_` prefix with secure random suffix
359 | - Complete cleanup on deletion removes both access keys and associated policies
360 |
361 | ### Phase 4: Local Access CLI
362 | - [x] **Design local mount CLI command** for automated rclone configuration
363 | - [x] **Implement credential fetching** from cloud API for local setup
364 | - [x] **Create rclone config automation** for tenant-specific bucket mounting
365 | - [x] **Test local→cloud→container sync** with optimized cache settings
366 | - [x] **Document local access setup** for beta users
367 |
368 | ### Phase 5: Webhook Integration (Future)
369 | - [ ] **Research Tigris webhook API** for object notifications and payload format
370 | - [ ] **Design webhook endpoint** for real-time sync notifications
371 | - [ ] **Implement notification handling** to trigger Basic Memory sync events
372 | - [ ] **Test webhook delivery** and sync latency improvements
373 |
374 | ## Success Metrics
375 | - [ ] **Container startup < 5 seconds** including TigrisFS mount and Basic Memory init
376 | - [ ] **Sync latency < 5 seconds** for local→cloud→container file changes
377 | - [ ] **Zero data loss** during bucket provisioning and credential management
378 | - [ ] **100% test coverage** for new TigrisBucketService and credential functions
379 | - [ ] **Beta deployment** with internal users validating local-cloud workflow
380 |
381 |
382 |
383 | ## Implementation Notes
384 |
385 | ## Phase 4.1: Bidirectional Sync with rclone bisync (NEW)
386 |
387 | ### Problem Statement
388 | During testing, we discovered that some applications (particularly Obsidian) don't detect file changes over NFS mounts. Rather than building a custom sync daemon, we can leverage `rclone bisync` - rclone's built-in bidirectional synchronization feature.
389 |
390 | ### Solution: rclone bisync
391 | Use rclone's proven bidirectional sync instead of custom implementation:
392 |
393 | **Core Architecture:**
394 | ```bash
395 | # rclone bisync handles all the complexity
396 | rclone bisync ~/basic-memory-{tenant_id} basic-memory-{tenant_id}:{bucket_name} \
397 | --create-empty-src-dirs \
398 | --conflict-resolve newer \
399 | --resilient \
400 | --check-access
401 | ```
402 |
403 | **Key Benefits:**
404 | - ✅ **Battle-tested**: Production-proven rclone functionality
405 | - ✅ **MIT licensed**: Open source with permissive licensing
406 | - ✅ **No custom code**: Zero maintenance burden for sync logic
407 | - ✅ **Built-in safety**: max-delete protection, conflict resolution
408 | - ✅ **Simple installation**: Works with Homebrew rclone (no FUSE needed)
409 | - ✅ **File watcher compatible**: Works with Obsidian and all applications
410 | - ✅ **Offline support**: Can work offline and sync when connected
411 |
412 | ### bisync Conflict Resolution Options
413 |
414 | **Built-in conflict strategies:**
415 | ```bash
416 | --conflict-resolve none # Keep both files with .conflict suffixes (safest)
417 | --conflict-resolve newer # Always pick the most recently modified file
418 | --conflict-resolve larger # Choose based on file size
419 | --conflict-resolve path1 # Always prefer local changes
420 | --conflict-resolve path2 # Always prefer cloud changes
421 | ```
422 |
423 | ### Sync Profiles Using bisync
424 |
425 | **Profile configurations:**
426 | ```python
427 | BISYNC_PROFILES = {
428 | "safe": {
429 | "conflict_resolve": "none", # Keep both versions
430 | "max_delete": 10, # Prevent mass deletion
431 | "check_access": True, # Verify sync integrity
432 | "description": "Safe mode with conflict preservation"
433 | },
434 | "balanced": {
435 | "conflict_resolve": "newer", # Auto-resolve to newer file
436 | "max_delete": 25,
437 | "check_access": True,
438 | "description": "Balanced mode (recommended default)"
439 | },
440 | "fast": {
441 | "conflict_resolve": "newer",
442 | "max_delete": 50,
443 | "check_access": False, # Skip verification for speed
444 | "description": "Fast mode for rapid iteration"
445 | }
446 | }
447 | ```
448 |
449 | ### CLI Commands
450 |
451 | **Manual sync commands:**
452 | ```bash
453 | basic-memory cloud bisync # Manual bidirectional sync
454 | basic-memory cloud bisync --dry-run # Preview changes
455 | basic-memory cloud bisync --profile safe # Use specific profile
456 | basic-memory cloud bisync --resync # Force full baseline resync
457 | ```
458 |
459 | **Watch mode (Step 1):**
460 | ```bash
461 | basic-memory cloud bisync --watch # Long-running process, sync every 60s
462 | basic-memory cloud bisync --watch --interval 30s # Custom interval
463 | ```
464 |
465 | **System integration (Step 2 - Future):**
466 | ```bash
467 | basic-memory cloud bisync-service install # Install as system service
468 | basic-memory cloud bisync-service start # Start background service
469 | basic-memory cloud bisync-service status # Check service status
470 | ```
471 |
472 | ### Implementation Strategy
473 |
474 | **Phase 4.1.1: Core bisync Implementation**
475 | - [ ] Implement `run_bisync()` function wrapping rclone bisync
476 | - [ ] Add profile-based configuration (safe/balanced/fast)
477 | - [ ] Create conflict resolution and safety options
478 | - [ ] Test with sample files and conflict scenarios
479 |
480 | **Phase 4.1.2: Watch Mode**
481 | - [ ] Add `--watch` flag for continuous sync
482 | - [ ] Implement configurable sync intervals
483 | - [ ] Add graceful shutdown and signal handling
484 | - [ ] Create status monitoring and progress indicators
485 |
486 | **Phase 4.1.3: User Experience**
487 | - [ ] Add conflict reporting and resolution guidance
488 | - [ ] Implement dry-run preview functionality
489 | - [ ] Create troubleshooting and diagnostic commands
490 | - [ ] Add filtering configuration (.gitignore-style)
491 |
492 | **Phase 4.1.4: System Integration (Future)**
493 | - [ ] Generate platform-specific service files (launchd/systemd)
494 | - [ ] Add service management commands
495 | - [ ] Implement automatic startup and recovery
496 | - [ ] Create monitoring and logging integration
497 |
498 | ### Technical Implementation
499 |
500 | **Core bisync wrapper:**
501 | ```python
502 | def run_bisync(
503 | tenant_id: str,
504 | bucket_name: str,
505 | profile: str = "balanced",
506 | dry_run: bool = False
507 | ) -> bool:
508 | """Run rclone bisync with specified profile."""
509 |
510 | local_path = Path.home() / f"basic-memory-{tenant_id}"
511 | remote_path = f"basic-memory-{tenant_id}:{bucket_name}"
512 | profile_config = BISYNC_PROFILES[profile]
513 |
514 | cmd = [
515 | "rclone", "bisync",
516 | str(local_path), remote_path,
517 | "--create-empty-src-dirs",
518 | "--resilient",
519 | f"--conflict-resolve={profile_config['conflict_resolve']}",
520 | f"--max-delete={profile_config['max_delete']}",
521 | "--filters-file", "~/.basic-memory/bisync-filters.txt"
522 | ]
523 |
524 | if profile_config.get("check_access"):
525 | cmd.append("--check-access")
526 |
527 | if dry_run:
528 | cmd.append("--dry-run")
529 |
530 | return subprocess.run(cmd, check=True).returncode == 0
531 | ```
532 |
533 | **Default filter file (~/.basic-memory/bisync-filters.txt):**
534 | ```
535 | - .DS_Store
536 | - .git/**
537 | - __pycache__/**
538 | - *.pyc
539 | - .pytest_cache/**
540 | - node_modules/**
541 | - .conflict-*
542 | - Thumbs.db
543 | - desktop.ini
544 | ```
545 |
546 | **Advantages Over Custom Daemon:**
547 | - ✅ **Zero maintenance**: No custom sync logic to debug/maintain
548 | - ✅ **Production proven**: Used by thousands in production
549 | - ✅ **Safety features**: Built-in max-delete, conflict handling, recovery
550 | - ✅ **Filtering**: Advanced exclude patterns and rules
551 | - ✅ **Performance**: Optimized for various storage backends
552 | - ✅ **Community support**: Extensive documentation and community
553 |
554 | ## Phase 4.2: NFS Mount Support (Direct Access)
555 |
556 | ### Solution: rclone nfsmount
557 | Keep the existing NFS mount functionality for users who prefer direct file access:
558 |
559 | **Core Architecture:**
560 | ```bash
561 | # rclone nfsmount provides transparent file access
562 | rclone nfsmount basic-memory-{tenant_id}:{bucket_name} ~/basic-memory-{tenant_id} \
563 | --vfs-cache-mode writes \
564 | --dir-cache-time 10s \
565 | --daemon
566 | ```
567 |
568 | **Key Benefits:**
569 | - ✅ **Real-time access**: Files appear immediately as they're created/modified
570 | - ✅ **Transparent**: Works with any application that reads/writes files
571 | - ✅ **Low latency**: Direct access without sync delays
572 | - ✅ **Simple**: No periodic sync commands needed
573 | - ✅ **Homebrew compatible**: Works with Homebrew rclone (no FUSE required)
574 |
575 | **Limitations:**
576 | - ❌ **File watcher compatibility**: Some apps (Obsidian) don't detect changes over NFS
577 | - ❌ **Network dependency**: Requires active connection to cloud storage
578 | - ❌ **Potential conflicts**: Simultaneous edits from multiple locations can cause issues
579 |
580 | ### Mount Profiles (Existing)
581 |
582 | **Already implemented profiles from SPEC-7 testing:**
583 | ```python
584 | MOUNT_PROFILES = {
585 | "fast": {
586 | "cache_time": "5s",
587 | "poll_interval": "3s",
588 | "description": "Ultra-fast development (5s sync)"
589 | },
590 | "balanced": {
591 | "cache_time": "10s",
592 | "poll_interval": "5s",
593 | "description": "Fast development (10-15s sync, recommended)"
594 | },
595 | "safe": {
596 | "cache_time": "15s",
597 | "poll_interval": "10s",
598 | "description": "Conflict-aware mount with backup",
599 | "extra_args": ["--conflict-suffix", ".conflict-{DateTimeExt}"]
600 | }
601 | }
602 | ```
603 |
604 | ### CLI Commands (Existing)
605 |
606 | **Mount commands already implemented:**
607 | ```bash
608 | basic-memory cloud mount # Mount with balanced profile
609 | basic-memory cloud mount --profile fast # Ultra-fast caching
610 | basic-memory cloud mount --profile safe # Conflict detection
611 | basic-memory cloud unmount # Clean unmount
612 | basic-memory cloud mount-status # Show mount status
613 | ```
614 |
615 | ## User Choice: Mount vs Bisync
616 |
617 | ### When to Use Each Approach
618 |
619 | | Use Case | Recommended Solution | Why |
620 | |----------|---------------------|-----|
621 | | **Obsidian users** | `bisync` | File watcher support for live preview |
622 | | **CLI/vim/emacs users** | `mount` | Direct file access, lower latency |
623 | | **Offline work** | `bisync` | Can work offline, sync when connected |
624 | | **Real-time collaboration** | `mount` | Immediate visibility of changes |
625 | | **Multiple machines** | `bisync` | Better conflict handling |
626 | | **Single machine** | `mount` | Simpler, more transparent |
627 | | **Development work** | Either | Both work well, user preference |
628 | | **Large files** | `mount` | Streaming access vs full download |
629 |
630 | ### Installation Simplicity
631 |
632 | **Both approaches now use simple Homebrew installation:**
633 | ```bash
634 | # Single installation command for both approaches
635 | brew install rclone
636 |
637 | # No macFUSE, no system modifications needed
638 | # Works immediately with both mount and bisync
639 | ```
640 |
641 | ### Implementation Status
642 |
643 | **Phase 4.1: bisync** (NEW)
644 | - [ ] Implement bisync command wrapper
645 | - [ ] Add watch mode with configurable intervals
646 | - [ ] Create conflict resolution workflows
647 | - [ ] Add filtering and safety options
648 |
649 | **Phase 4.2: mount** (EXISTING - ✅ IMPLEMENTED)
650 | - [x] NFS mount commands with profile support
651 | - [x] Mount management and cleanup
652 | - [x] Process monitoring and health checks
653 | - [x] Credential integration with cloud API
654 |
655 | **Both approaches share:**
656 | - [x] Credential management via cloud API
657 | - [x] Secure rclone configuration
658 | - [x] Tenant isolation and bucket scoping
659 | - [x] Simple Homebrew rclone installation
660 |
661 |
662 | Key Features:
663 |
664 | 1. Cross-Platform rclone Installation (rclone_installer.py):
665 | - macOS: Homebrew → official script fallback
666 | - Linux: snap → apt → official script fallback
667 | - Windows: winget → chocolatey → scoop fallback
668 | - Automatic version detection and verification
669 |
670 | 2. Smart rclone Configuration (rclone_config.py):
671 | - Automatic tenant-specific config generation
672 | - Three optimized mount profiles from your SPEC-7 testing:
673 | - fast: 5s sync (ultra-performance)
674 | - balanced: 10-15s sync (recommended default)
675 | - safe: 15s sync + conflict detection
676 | - Backup existing configs before modification
677 |
678 | 3. Robust Mount Management (mount_commands.py):
679 | - Automatic tenant credential generation
680 | - Mount path management (~/basic-memory-{tenant-id})
681 | - Process lifecycle management (prevent duplicate mounts)
682 | - Orphaned process cleanup
683 | - Mount verification and health checking
684 |
685 | 4. Clean Architecture (api_client.py):
686 | - Separated API client to avoid circular imports
687 | - Reuses existing authentication infrastructure
688 | - Consistent error handling and logging
689 |
690 | User Experience:
691 |
692 | One-Command Setup:
693 | basic-memory cloud setup
694 | ```bash
695 | # 1. Installs rclone automatically
696 | # 2. Authenticates with existing login
697 | # 3. Generates secure credentials
698 | # 4. Configures rclone
699 | # 5. Performs initial mount
700 | ```
701 |
702 | Profile-Based Mounting:
703 | basic-memory cloud mount --profile fast # 5s sync
704 | basic-memory cloud mount --profile balanced # 15s sync (default)
705 | basic-memory cloud mount --profile safe # conflict detection
706 |
707 | Status Monitoring:
708 | basic-memory cloud mount-status
709 | ```bash
710 | # Shows: tenant info, mount path, sync profile, rclone processes
711 | ```
712 | ### local mount api
713 |
714 | Endpoint 1: Get Tenant Info for user
715 | Purpose: Get tenant details for mounting
716 | - pass in jwt
717 | - service returns mount info
718 |
719 | **✅ IMPLEMENTED API Specification:**
720 |
721 | **Endpoint 1: GET /tenant/mount/info**
722 | - Purpose: Get tenant mount information without exposing credentials
723 | - Authentication: JWT token (tenant_id extracted from claims)
724 |
725 | Request:
726 | ```
727 | GET /tenant/mount/info
728 | Authorization: Bearer {jwt_token}
729 | ```
730 |
731 | Response:
732 | ```json
733 | {
734 | "tenant_id": "434252dd-d83b-4b20-bf70-8a950ff875c4",
735 | "bucket_name": "basic-memory-434252dd",
736 | "has_credentials": true,
737 | "credentials_created_at": "2025-09-22T16:48:50.414694"
738 | }
739 | ```
740 |
741 | **Endpoint 2: POST /tenant/mount/credentials**
742 | - Purpose: Generate NEW bucket-scoped S3 credentials for rclone mounting
743 | - Authentication: JWT token (tenant_id extracted from claims)
744 | - Multi-session: Creates new credentials without revoking existing ones
745 |
746 | Request:
747 | ```
748 | POST /tenant/mount/credentials
749 | Authorization: Bearer {jwt_token}
750 | Content-Type: application/json
751 | ```
752 | *Note: No request body needed - tenant_id extracted from JWT*
753 |
754 | Response:
755 | ```json
756 | {
757 | "tenant_id": "434252dd-d83b-4b20-bf70-8a950ff875c4",
758 | "bucket_name": "basic-memory-434252dd",
759 | "access_key": "test_access_key_12345",
760 | "secret_key": "test_secret_key_abcdef",
761 | "endpoint_url": "https://fly.storage.tigris.dev",
762 | "region": "auto"
763 | }
764 | ```
765 |
766 | **🔒 Security Notes:**
767 | - Secret key returned ONCE only - never stored in database
768 | - Credentials are bucket-scoped (cannot access other tenants' buckets)
769 | - Multiple active credentials supported per tenant (work laptop + personal machine)
770 |
771 | Implementation Notes
772 |
773 | Security:
774 | - Both endpoints require JWT authentication
775 | - Extract tenant_id from JWT claims (not request body)
776 | - Generate scoped credentials (not admin credentials)
777 | - Credentials should have bucket-specific access only
778 |
779 | Integration Points:
780 | - Use your existing StorageClient from SPEC-8 implementation
781 | - Leverage existing JWT middleware for tenant extraction
782 | - Return same credential format as your Tigris bucket provisioning
783 |
784 | Error Handling:
785 | - 401 if not authenticated
786 | - 403 if tenant doesn't exist
787 | - 500 if credential generation fails
788 |
789 | **🔄 Design Decisions:**
790 |
791 | 1. **Secure Credential Flow (No Secret Storage)**
792 |
793 | Based on CLI flow analysis, we follow security best practices:
794 | - ✅ API generates both access_key + secret_key via Tigris IAM
795 | - ✅ Returns both in API response for immediate use
796 | - ✅ CLI uses credentials immediately to configure rclone
797 | - ✅ Database stores only metadata (access_key + policy_arn for cleanup)
798 | - ✅ rclone handles secure local credential storage
799 | - ❌ **Never store secret_key in database (even encrypted)**
800 |
801 | 2. **CLI Credential Flow**
802 | ```bash
803 | # CLI calls API
804 | POST /tenant/mount/credentials → {access_key, secret_key, ...}
805 |
806 | # CLI immediately configures rclone
807 | rclone config create basic-memory-{tenant_id} s3 \
808 | access_key_id={access_key} \
809 | secret_access_key={secret_key} \
810 | endpoint=https://fly.storage.tigris.dev
811 |
812 | # Database tracks metadata only
813 | INSERT INTO tenant_mount_credentials (tenant_id, access_key, policy_arn, ...)
814 | ```
815 |
816 | 3. **Multiple Sessions Supported**
817 |
818 | - Users can have multiple active credential sets (work laptop, personal machine, etc.)
819 | - Each credential generation creates a new Tigris access key
820 | - List active credentials via API (shows access_key but never secret)
821 |
822 | 4. **Failure Handling & Cleanup**
823 |
824 | - **Happy Path**: Credentials created → Used immediately → rclone configured
825 | - **Orphaned Credentials**: Background job revokes unused credentials
826 | - **API Failure Recovery**: Retry Tigris deletion with stored policy_arn
827 | - **Status Tracking**: Track tigris_deletion_status (pending/completed/failed)
828 |
829 | 5. **Event Sourcing & Audit**
830 |
831 | - MountCredentialCreatedEvent
832 | - MountCredentialRevokedEvent
833 | - MountCredentialOrphanedEvent (for cleanup)
834 | - Full audit trail for security compliance
835 |
836 | 6. **Tenant/Bucket Validation**
837 |
838 | - Verify tenant exists and has valid bucket before credential generation
839 | - Use existing StorageClient to validate bucket access
840 | - Prevent credential generation for inactive/invalid tenants
841 |
842 | 📋 **Implemented API Endpoints:**
843 |
844 | ```
845 | ✅ IMPLEMENTED:
846 | GET /tenant/mount/info # Get tenant/bucket info (no credentials exposed)
847 | POST /tenant/mount/credentials # Generate new credentials (returns secret once)
848 | GET /tenant/mount/credentials # List active credentials (no secrets)
849 | DELETE /tenant/mount/credentials/{cred_id} # Revoke specific credentials
850 | ```
851 |
852 | **API Implementation Status:**
853 | - ✅ **GET /tenant/mount/info**: Returns tenant_id, bucket_name, has_credentials, credentials_created_at
854 | - ✅ **POST /tenant/mount/credentials**: Creates new bucket-scoped access keys, returns access_key + secret_key once
855 | - ✅ **GET /tenant/mount/credentials**: Lists active credentials without exposing secret keys
856 | - ✅ **DELETE /tenant/mount/credentials/{cred_id}**: Revokes specific credentials with proper Tigris IAM cleanup
857 | - ✅ **Multi-session support**: Multiple active credentials per tenant (work laptop + personal machine)
858 | - ✅ **Security**: Secret keys never stored in database, returned once only for immediate use
859 | - ✅ **Comprehensive test suite**: 28 tests covering all scenarios including error handling and multi-session flows
860 | - ✅ **Dependency injection**: Clean integration with existing FastAPI architecture
861 | - ✅ **Production-ready configuration**: Tigris credentials properly configured for tenant machines
862 |
863 | 🗄️ **Secure Database Schema:**
864 |
865 | ```sql
866 | CREATE TABLE tenant_mount_credentials (
867 | id UUID PRIMARY KEY,
868 | tenant_id UUID REFERENCES tenant(id),
869 | access_key VARCHAR(255) NOT NULL,
870 | -- secret_key REMOVED - never store secrets (security best practice)
871 | policy_arn VARCHAR(255) NOT NULL, -- For Tigris IAM cleanup
872 | tigris_deletion_status VARCHAR(20) DEFAULT 'pending', -- Track cleanup
873 | created_at TIMESTAMP DEFAULT NOW(),
874 | updated_at TIMESTAMP DEFAULT NOW(),
875 | revoked_at TIMESTAMP NULL,
876 | last_used_at TIMESTAMP NULL, -- Track usage for orphan cleanup
877 | description VARCHAR(255) DEFAULT 'CLI mount credentials'
878 | );
879 | ```
880 |
881 | **Security Benefits:**
882 | - ✅ Database breach cannot expose secrets
883 | - ✅ Follows "secrets don't persist" security principle
884 | - ✅ Meets compliance requirements (SOC2, etc.)
885 | - ✅ Reduced attack surface
886 | - ✅ CLI gets credentials once and stores securely via rclone
887 |
```
--------------------------------------------------------------------------------
/src/basic_memory/services/entity_service.py:
--------------------------------------------------------------------------------
```python
1 | """Service for managing entities in the database."""
2 |
3 | from pathlib import Path
4 | from typing import List, Optional, Sequence, Tuple, Union
5 |
6 | import frontmatter
7 | import yaml
8 | from loguru import logger
9 | from sqlalchemy.exc import IntegrityError
10 |
11 | from basic_memory.config import ProjectConfig, BasicMemoryConfig
12 | from basic_memory.file_utils import (
13 | has_frontmatter,
14 | parse_frontmatter,
15 | remove_frontmatter,
16 | dump_frontmatter,
17 | )
18 | from basic_memory.markdown import EntityMarkdown
19 | from basic_memory.markdown.entity_parser import EntityParser
20 | from basic_memory.markdown.utils import entity_model_from_markdown, schema_to_markdown
21 | from basic_memory.models import Entity as EntityModel
22 | from basic_memory.models import Observation, Relation
23 | from basic_memory.models.knowledge import Entity
24 | from basic_memory.repository import ObservationRepository, RelationRepository
25 | from basic_memory.repository.entity_repository import EntityRepository
26 | from basic_memory.schemas import Entity as EntitySchema
27 | from basic_memory.schemas.base import Permalink
28 | from basic_memory.services import BaseService, FileService
29 | from basic_memory.services.exceptions import EntityCreationError, EntityNotFoundError
30 | from basic_memory.services.link_resolver import LinkResolver
31 | from basic_memory.utils import generate_permalink
32 |
33 |
34 | class EntityService(BaseService[EntityModel]):
35 | """Service for managing entities in the database."""
36 |
37 | def __init__(
38 | self,
39 | entity_parser: EntityParser,
40 | entity_repository: EntityRepository,
41 | observation_repository: ObservationRepository,
42 | relation_repository: RelationRepository,
43 | file_service: FileService,
44 | link_resolver: LinkResolver,
45 | app_config: Optional[BasicMemoryConfig] = None,
46 | ):
47 | super().__init__(entity_repository)
48 | self.observation_repository = observation_repository
49 | self.relation_repository = relation_repository
50 | self.entity_parser = entity_parser
51 | self.file_service = file_service
52 | self.link_resolver = link_resolver
53 | self.app_config = app_config
54 |
55 | async def detect_file_path_conflicts(
56 | self, file_path: str, skip_check: bool = False
57 | ) -> List[Entity]:
58 | """Detect potential file path conflicts for a given file path.
59 |
60 | This checks for entities with similar file paths that might cause conflicts:
61 | - Case sensitivity differences (Finance/file.md vs finance/file.md)
62 | - Character encoding differences
63 | - Hyphen vs space differences
64 | - Unicode normalization differences
65 |
66 | Args:
67 | file_path: The file path to check for conflicts
68 | skip_check: If True, skip the check and return empty list (optimization for bulk operations)
69 |
70 | Returns:
71 | List of entities that might conflict with the given file path
72 | """
73 | if skip_check:
74 | return []
75 |
76 | from basic_memory.utils import detect_potential_file_conflicts
77 |
78 | conflicts = []
79 |
80 | # Get all existing file paths
81 | all_entities = await self.repository.find_all()
82 | existing_paths = [entity.file_path for entity in all_entities]
83 |
84 | # Use the enhanced conflict detection utility
85 | conflicting_paths = detect_potential_file_conflicts(file_path, existing_paths)
86 |
87 | # Find the entities corresponding to conflicting paths
88 | for entity in all_entities:
89 | if entity.file_path in conflicting_paths:
90 | conflicts.append(entity)
91 |
92 | return conflicts
93 |
94 | async def resolve_permalink(
95 | self,
96 | file_path: Permalink | Path,
97 | markdown: Optional[EntityMarkdown] = None,
98 | skip_conflict_check: bool = False,
99 | ) -> str:
100 | """Get or generate unique permalink for an entity.
101 |
102 | Priority:
103 | 1. If markdown has permalink and it's not used by another file -> use as is
104 | 2. If markdown has permalink but it's used by another file -> make unique
105 | 3. For existing files, keep current permalink from db
106 | 4. Generate new unique permalink from file path
107 |
108 | Enhanced to detect and handle character-related conflicts.
109 | """
110 | file_path_str = Path(file_path).as_posix()
111 |
112 | # Check for potential file path conflicts before resolving permalink
113 | conflicts = await self.detect_file_path_conflicts(
114 | file_path_str, skip_check=skip_conflict_check
115 | )
116 | if conflicts:
117 | logger.warning(
118 | f"Detected potential file path conflicts for '{file_path_str}': "
119 | f"{[entity.file_path for entity in conflicts]}"
120 | )
121 |
122 | # If markdown has explicit permalink, try to validate it
123 | if markdown and markdown.frontmatter.permalink:
124 | desired_permalink = markdown.frontmatter.permalink
125 | existing = await self.repository.get_by_permalink(desired_permalink)
126 |
127 | # If no conflict or it's our own file, use as is
128 | if not existing or existing.file_path == file_path_str:
129 | return desired_permalink
130 |
131 | # For existing files, try to find current permalink
132 | existing = await self.repository.get_by_file_path(file_path_str)
133 | if existing:
134 | return existing.permalink
135 |
136 | # New file - generate permalink
137 | if markdown and markdown.frontmatter.permalink:
138 | desired_permalink = markdown.frontmatter.permalink
139 | else:
140 | desired_permalink = generate_permalink(file_path_str)
141 |
142 | # Make unique if needed - enhanced to handle character conflicts
143 | permalink = desired_permalink
144 | suffix = 1
145 | while await self.repository.get_by_permalink(permalink):
146 | permalink = f"{desired_permalink}-{suffix}"
147 | suffix += 1
148 | logger.debug(f"creating unique permalink: {permalink}")
149 |
150 | return permalink
151 |
152 | async def create_or_update_entity(self, schema: EntitySchema) -> Tuple[EntityModel, bool]:
153 | """Create new entity or update existing one.
154 | Returns: (entity, is_new) where is_new is True if a new entity was created
155 | """
156 | logger.debug(
157 | f"Creating or updating entity: {schema.file_path}, permalink: {schema.permalink}"
158 | )
159 |
160 | # Try to find existing entity using strict resolution (no fuzzy search)
161 | # This prevents incorrectly matching similar file paths like "Node A.md" and "Node C.md"
162 | existing = await self.link_resolver.resolve_link(schema.file_path, strict=True)
163 | if not existing and schema.permalink:
164 | existing = await self.link_resolver.resolve_link(schema.permalink, strict=True)
165 |
166 | if existing:
167 | logger.debug(f"Found existing entity: {existing.file_path}")
168 | return await self.update_entity(existing, schema), False
169 | else:
170 | # Create new entity
171 | return await self.create_entity(schema), True
172 |
173 | async def create_entity(self, schema: EntitySchema) -> EntityModel:
174 | """Create a new entity and write to filesystem."""
175 | logger.debug(f"Creating entity: {schema.title}")
176 |
177 | # Get file path and ensure it's a Path object
178 | file_path = Path(schema.file_path)
179 |
180 | if await self.file_service.exists(file_path):
181 | raise EntityCreationError(
182 | f"file for entity {schema.folder}/{schema.title} already exists: {file_path}"
183 | )
184 |
185 | # Parse content frontmatter to check for user-specified permalink and entity_type
186 | content_markdown = None
187 | if schema.content and has_frontmatter(schema.content):
188 | content_frontmatter = parse_frontmatter(schema.content)
189 |
190 | # If content has entity_type/type, use it to override the schema entity_type
191 | if "type" in content_frontmatter:
192 | schema.entity_type = content_frontmatter["type"]
193 |
194 | if "permalink" in content_frontmatter:
195 | # Create a minimal EntityMarkdown object for permalink resolution
196 | from basic_memory.markdown.schemas import EntityFrontmatter
197 |
198 | frontmatter_metadata = {
199 | "title": schema.title,
200 | "type": schema.entity_type,
201 | "permalink": content_frontmatter["permalink"],
202 | }
203 | frontmatter_obj = EntityFrontmatter(metadata=frontmatter_metadata)
204 | content_markdown = EntityMarkdown(
205 | frontmatter=frontmatter_obj,
206 | content="", # content not needed for permalink resolution
207 | observations=[],
208 | relations=[],
209 | )
210 |
211 | # Get unique permalink (prioritizing content frontmatter) unless disabled
212 | if self.app_config and self.app_config.disable_permalinks:
213 | # Use empty string as sentinel to indicate permalinks are disabled
214 | # The permalink property will return None when it sees empty string
215 | schema._permalink = ""
216 | else:
217 | # Generate and set permalink
218 | permalink = await self.resolve_permalink(file_path, content_markdown)
219 | schema._permalink = permalink
220 |
221 | post = await schema_to_markdown(schema)
222 |
223 | # write file
224 | final_content = dump_frontmatter(post)
225 | checksum = await self.file_service.write_file(file_path, final_content)
226 |
227 | # parse entity from file
228 | entity_markdown = await self.entity_parser.parse_file(file_path)
229 |
230 | # create entity
231 | created = await self.create_entity_from_markdown(file_path, entity_markdown)
232 |
233 | # add relations
234 | entity = await self.update_entity_relations(created.file_path, entity_markdown)
235 |
236 | # Set final checksum to mark complete
237 | return await self.repository.update(entity.id, {"checksum": checksum})
238 |
239 | async def update_entity(self, entity: EntityModel, schema: EntitySchema) -> EntityModel:
240 | """Update an entity's content and metadata."""
241 | logger.debug(
242 | f"Updating entity with permalink: {entity.permalink} content-type: {schema.content_type}"
243 | )
244 |
245 | # Convert file path string to Path
246 | file_path = Path(entity.file_path)
247 |
248 | # Read existing frontmatter from the file if it exists
249 | existing_markdown = await self.entity_parser.parse_file(file_path)
250 |
251 | # Parse content frontmatter to check for user-specified permalink and entity_type
252 | content_markdown = None
253 | if schema.content and has_frontmatter(schema.content):
254 | content_frontmatter = parse_frontmatter(schema.content)
255 |
256 | # If content has entity_type/type, use it to override the schema entity_type
257 | if "type" in content_frontmatter:
258 | schema.entity_type = content_frontmatter["type"]
259 |
260 | if "permalink" in content_frontmatter:
261 | # Create a minimal EntityMarkdown object for permalink resolution
262 | from basic_memory.markdown.schemas import EntityFrontmatter
263 |
264 | frontmatter_metadata = {
265 | "title": schema.title,
266 | "type": schema.entity_type,
267 | "permalink": content_frontmatter["permalink"],
268 | }
269 | frontmatter_obj = EntityFrontmatter(metadata=frontmatter_metadata)
270 | content_markdown = EntityMarkdown(
271 | frontmatter=frontmatter_obj,
272 | content="", # content not needed for permalink resolution
273 | observations=[],
274 | relations=[],
275 | )
276 |
277 | # Check if we need to update the permalink based on content frontmatter (unless disabled)
278 | new_permalink = entity.permalink # Default to existing
279 | if self.app_config and not self.app_config.disable_permalinks:
280 | if content_markdown and content_markdown.frontmatter.permalink:
281 | # Resolve permalink with the new content frontmatter
282 | resolved_permalink = await self.resolve_permalink(file_path, content_markdown)
283 | if resolved_permalink != entity.permalink:
284 | new_permalink = resolved_permalink
285 | # Update the schema to use the new permalink
286 | schema._permalink = new_permalink
287 |
288 | # Create post with new content from schema
289 | post = await schema_to_markdown(schema)
290 |
291 | # Merge new metadata with existing metadata
292 | existing_markdown.frontmatter.metadata.update(post.metadata)
293 |
294 | # Ensure the permalink in the metadata is the resolved one
295 | if new_permalink != entity.permalink:
296 | existing_markdown.frontmatter.metadata["permalink"] = new_permalink
297 |
298 | # Create a new post with merged metadata
299 | merged_post = frontmatter.Post(post.content, **existing_markdown.frontmatter.metadata)
300 |
301 | # write file
302 | final_content = dump_frontmatter(merged_post)
303 | checksum = await self.file_service.write_file(file_path, final_content)
304 |
305 | # parse entity from file
306 | entity_markdown = await self.entity_parser.parse_file(file_path)
307 |
308 | # update entity in db
309 | entity = await self.update_entity_and_observations(file_path, entity_markdown)
310 |
311 | # add relations
312 | await self.update_entity_relations(file_path.as_posix(), entity_markdown)
313 |
314 | # Set final checksum to match file
315 | entity = await self.repository.update(entity.id, {"checksum": checksum})
316 |
317 | return entity
318 |
319 | async def delete_entity(self, permalink_or_id: str | int) -> bool:
320 | """Delete entity and its file."""
321 | logger.debug(f"Deleting entity: {permalink_or_id}")
322 |
323 | try:
324 | # Get entity first for file deletion
325 | if isinstance(permalink_or_id, str):
326 | entity = await self.get_by_permalink(permalink_or_id)
327 | else:
328 | entities = await self.get_entities_by_id([permalink_or_id])
329 | if len(entities) != 1: # pragma: no cover
330 | logger.error(
331 | "Entity lookup error", entity_id=permalink_or_id, found_count=len(entities)
332 | )
333 | raise ValueError(
334 | f"Expected 1 entity with ID {permalink_or_id}, got {len(entities)}"
335 | )
336 | entity = entities[0]
337 |
338 | # Delete file first
339 | await self.file_service.delete_entity_file(entity)
340 |
341 | # Delete from DB (this will cascade to observations/relations)
342 | return await self.repository.delete(entity.id)
343 |
344 | except EntityNotFoundError:
345 | logger.info(f"Entity not found: {permalink_or_id}")
346 | return True # Already deleted
347 |
348 | async def get_by_permalink(self, permalink: str) -> EntityModel:
349 | """Get entity by type and name combination."""
350 | logger.debug(f"Getting entity by permalink: {permalink}")
351 | db_entity = await self.repository.get_by_permalink(permalink)
352 | if not db_entity:
353 | raise EntityNotFoundError(f"Entity not found: {permalink}")
354 | return db_entity
355 |
356 | async def get_entities_by_id(self, ids: List[int]) -> Sequence[EntityModel]:
357 | """Get specific entities and their relationships."""
358 | logger.debug(f"Getting entities: {ids}")
359 | return await self.repository.find_by_ids(ids)
360 |
361 | async def get_entities_by_permalinks(self, permalinks: List[str]) -> Sequence[EntityModel]:
362 | """Get specific nodes and their relationships."""
363 | logger.debug(f"Getting entities permalinks: {permalinks}")
364 | return await self.repository.find_by_permalinks(permalinks)
365 |
366 | async def delete_entity_by_file_path(self, file_path: Union[str, Path]) -> None:
367 | """Delete entity by file path."""
368 | await self.repository.delete_by_file_path(str(file_path))
369 |
370 | async def create_entity_from_markdown(
371 | self, file_path: Path, markdown: EntityMarkdown
372 | ) -> EntityModel:
373 | """Create entity and observations only.
374 |
375 | Creates the entity with null checksum to indicate sync not complete.
376 | Relations will be added in second pass.
377 |
378 | Uses UPSERT approach to handle permalink/file_path conflicts cleanly.
379 | """
380 | logger.debug(f"Creating entity: {markdown.frontmatter.title} file_path: {file_path}")
381 | model = entity_model_from_markdown(file_path, markdown)
382 |
383 | # Mark as incomplete because we still need to add relations
384 | model.checksum = None
385 |
386 | # Use UPSERT to handle conflicts cleanly
387 | try:
388 | return await self.repository.upsert_entity(model)
389 | except Exception as e:
390 | logger.error(f"Failed to upsert entity for {file_path}: {e}")
391 | raise EntityCreationError(f"Failed to create entity: {str(e)}") from e
392 |
393 | async def update_entity_and_observations(
394 | self, file_path: Path, markdown: EntityMarkdown
395 | ) -> EntityModel:
396 | """Update entity fields and observations.
397 |
398 | Updates everything except relations and sets null checksum
399 | to indicate sync not complete.
400 | """
401 | logger.debug(f"Updating entity and observations: {file_path}")
402 |
403 | db_entity = await self.repository.get_by_file_path(file_path.as_posix())
404 |
405 | # Clear observations for entity
406 | await self.observation_repository.delete_by_fields(entity_id=db_entity.id)
407 |
408 | # add new observations
409 | observations = [
410 | Observation(
411 | entity_id=db_entity.id,
412 | content=obs.content,
413 | category=obs.category,
414 | context=obs.context,
415 | tags=obs.tags,
416 | )
417 | for obs in markdown.observations
418 | ]
419 | await self.observation_repository.add_all(observations)
420 |
421 | # update values from markdown
422 | db_entity = entity_model_from_markdown(file_path, markdown, db_entity)
423 |
424 | # checksum value is None == not finished with sync
425 | db_entity.checksum = None
426 |
427 | # update entity
428 | return await self.repository.update(
429 | db_entity.id,
430 | db_entity,
431 | )
432 |
433 | async def update_entity_relations(
434 | self,
435 | path: str,
436 | markdown: EntityMarkdown,
437 | ) -> EntityModel:
438 | """Update relations for entity"""
439 | logger.debug(f"Updating relations for entity: {path}")
440 |
441 | db_entity = await self.repository.get_by_file_path(path)
442 |
443 | # Clear existing relations first
444 | await self.relation_repository.delete_outgoing_relations_from_entity(db_entity.id)
445 |
446 | # Batch resolve all relation targets in parallel
447 | if markdown.relations:
448 | import asyncio
449 |
450 | # Create tasks for all relation lookups
451 | lookup_tasks = [
452 | self.link_resolver.resolve_link(rel.target) for rel in markdown.relations
453 | ]
454 |
455 | # Execute all lookups in parallel
456 | resolved_entities = await asyncio.gather(*lookup_tasks, return_exceptions=True)
457 |
458 | # Process results and create relation records
459 | relations_to_add = []
460 | for rel, resolved in zip(markdown.relations, resolved_entities):
461 | # Handle exceptions from gather and None results
462 | target_entity: Optional[Entity] = None
463 | if not isinstance(resolved, Exception):
464 | # Type narrowing: resolved is Optional[Entity] here, not Exception
465 | target_entity = resolved # type: ignore
466 |
467 | # if the target is found, store the id
468 | target_id = target_entity.id if target_entity else None
469 | # if the target is found, store the title, otherwise add the target for a "forward link"
470 | target_name = target_entity.title if target_entity else rel.target
471 |
472 | # Create the relation
473 | relation = Relation(
474 | from_id=db_entity.id,
475 | to_id=target_id,
476 | to_name=target_name,
477 | relation_type=rel.type,
478 | context=rel.context,
479 | )
480 | relations_to_add.append(relation)
481 |
482 | # Batch insert all relations
483 | if relations_to_add:
484 | try:
485 | await self.relation_repository.add_all(relations_to_add)
486 | except IntegrityError:
487 | # Some relations might be duplicates - fall back to individual inserts
488 | logger.debug("Batch relation insert failed, trying individual inserts")
489 | for relation in relations_to_add:
490 | try:
491 | await self.relation_repository.add(relation)
492 | except IntegrityError:
493 | # Unique constraint violation - relation already exists
494 | logger.debug(
495 | f"Skipping duplicate relation {relation.relation_type} from {db_entity.permalink}"
496 | )
497 | continue
498 |
499 | return await self.repository.get_by_file_path(path)
500 |
501 | async def edit_entity(
502 | self,
503 | identifier: str,
504 | operation: str,
505 | content: str,
506 | section: Optional[str] = None,
507 | find_text: Optional[str] = None,
508 | expected_replacements: int = 1,
509 | ) -> EntityModel:
510 | """Edit an existing entity's content using various operations.
511 |
512 | Args:
513 | identifier: Entity identifier (permalink, title, etc.)
514 | operation: The editing operation (append, prepend, find_replace, replace_section)
515 | content: The content to add or use for replacement
516 | section: For replace_section operation - the markdown header
517 | find_text: For find_replace operation - the text to find and replace
518 | expected_replacements: For find_replace operation - expected number of replacements (default: 1)
519 |
520 | Returns:
521 | The updated entity model
522 |
523 | Raises:
524 | EntityNotFoundError: If the entity cannot be found
525 | ValueError: If required parameters are missing for the operation or replacement count doesn't match expected
526 | """
527 | logger.debug(f"Editing entity: {identifier}, operation: {operation}")
528 |
529 | # Find the entity using the link resolver with strict mode for destructive operations
530 | entity = await self.link_resolver.resolve_link(identifier, strict=True)
531 | if not entity:
532 | raise EntityNotFoundError(f"Entity not found: {identifier}")
533 |
534 | # Read the current file content
535 | file_path = Path(entity.file_path)
536 | current_content, _ = await self.file_service.read_file(file_path)
537 |
538 | # Apply the edit operation
539 | new_content = self.apply_edit_operation(
540 | current_content, operation, content, section, find_text, expected_replacements
541 | )
542 |
543 | # Write the updated content back to the file
544 | checksum = await self.file_service.write_file(file_path, new_content)
545 |
546 | # Parse the updated file to get new observations/relations
547 | entity_markdown = await self.entity_parser.parse_file(file_path)
548 |
549 | # Update entity and its relationships
550 | entity = await self.update_entity_and_observations(file_path, entity_markdown)
551 | await self.update_entity_relations(file_path.as_posix(), entity_markdown)
552 |
553 | # Set final checksum to match file
554 | entity = await self.repository.update(entity.id, {"checksum": checksum})
555 |
556 | return entity
557 |
558 | def apply_edit_operation(
559 | self,
560 | current_content: str,
561 | operation: str,
562 | content: str,
563 | section: Optional[str] = None,
564 | find_text: Optional[str] = None,
565 | expected_replacements: int = 1,
566 | ) -> str:
567 | """Apply the specified edit operation to the current content."""
568 |
569 | if operation == "append":
570 | # Ensure proper spacing
571 | if current_content and not current_content.endswith("\n"):
572 | return current_content + "\n" + content
573 | return current_content + content # pragma: no cover
574 |
575 | elif operation == "prepend":
576 | # Handle frontmatter-aware prepending
577 | return self._prepend_after_frontmatter(current_content, content)
578 |
579 | elif operation == "find_replace":
580 | if not find_text:
581 | raise ValueError("find_text is required for find_replace operation")
582 | if not find_text.strip():
583 | raise ValueError("find_text cannot be empty or whitespace only")
584 |
585 | # Count actual occurrences
586 | actual_count = current_content.count(find_text)
587 |
588 | # Validate count matches expected
589 | if actual_count != expected_replacements:
590 | if actual_count == 0:
591 | raise ValueError(f"Text to replace not found: '{find_text}'")
592 | else:
593 | raise ValueError(
594 | f"Expected {expected_replacements} occurrences of '{find_text}', "
595 | f"but found {actual_count}"
596 | )
597 |
598 | return current_content.replace(find_text, content)
599 |
600 | elif operation == "replace_section":
601 | if not section:
602 | raise ValueError("section is required for replace_section operation")
603 | if not section.strip():
604 | raise ValueError("section cannot be empty or whitespace only")
605 | return self.replace_section_content(current_content, section, content)
606 |
607 | else:
608 | raise ValueError(f"Unsupported operation: {operation}")
609 |
610 | def replace_section_content(
611 | self, current_content: str, section_header: str, new_content: str
612 | ) -> str:
613 | """Replace content under a specific markdown section header.
614 |
615 | This method uses a simple, safe approach: when replacing a section, it only
616 | replaces the immediate content under that header until it encounters the next
617 | header of ANY level. This means:
618 |
619 | - Replacing "# Header" replaces content until "## Subsection" (preserves subsections)
620 | - Replacing "## Section" replaces content until "### Subsection" (preserves subsections)
621 | - More predictable and safer than trying to consume entire hierarchies
622 |
623 | Args:
624 | current_content: The current markdown content
625 | section_header: The section header to find and replace (e.g., "## Section Name")
626 | new_content: The new content to replace the section with
627 |
628 | Returns:
629 | The updated content with the section replaced
630 |
631 | Raises:
632 | ValueError: If multiple sections with the same header are found
633 | """
634 | # Normalize the section header (ensure it starts with #)
635 | if not section_header.startswith("#"):
636 | section_header = "## " + section_header
637 |
638 | # First pass: count matching sections to check for duplicates
639 | lines = current_content.split("\n")
640 | matching_sections = []
641 |
642 | for i, line in enumerate(lines):
643 | if line.strip() == section_header.strip():
644 | matching_sections.append(i)
645 |
646 | # Handle multiple sections error
647 | if len(matching_sections) > 1:
648 | raise ValueError(
649 | f"Multiple sections found with header '{section_header}'. "
650 | f"Section replacement requires unique headers."
651 | )
652 |
653 | # If no section found, append it
654 | if len(matching_sections) == 0:
655 | logger.info(f"Section '{section_header}' not found, appending to end of document")
656 | separator = "\n\n" if current_content and not current_content.endswith("\n\n") else ""
657 | return current_content + separator + section_header + "\n" + new_content
658 |
659 | # Replace the single matching section
660 | result_lines = []
661 | section_line_idx = matching_sections[0]
662 |
663 | i = 0
664 | while i < len(lines):
665 | line = lines[i]
666 |
667 | # Check if this is our target section header
668 | if i == section_line_idx:
669 | # Add the section header and new content
670 | result_lines.append(line)
671 | result_lines.append(new_content)
672 | i += 1
673 |
674 | # Skip the original section content until next header or end
675 | while i < len(lines):
676 | next_line = lines[i]
677 | # Stop consuming when we hit any header (preserve subsections)
678 | if next_line.startswith("#"):
679 | # We found another header - continue processing from here
680 | break
681 | i += 1
682 | # Continue processing from the next header (don't increment i again)
683 | continue
684 |
685 | # Add all other lines (including subsequent sections)
686 | result_lines.append(line)
687 | i += 1
688 |
689 | return "\n".join(result_lines)
690 |
691 | def _prepend_after_frontmatter(self, current_content: str, content: str) -> str:
692 | """Prepend content after frontmatter, preserving frontmatter structure."""
693 |
694 | # Check if file has frontmatter
695 | if has_frontmatter(current_content):
696 | try:
697 | # Parse and separate frontmatter from body
698 | frontmatter_data = parse_frontmatter(current_content)
699 | body_content = remove_frontmatter(current_content)
700 |
701 | # Prepend content to the body
702 | if content and not content.endswith("\n"):
703 | new_body = content + "\n" + body_content
704 | else:
705 | new_body = content + body_content
706 |
707 | # Reconstruct file with frontmatter + prepended body
708 | yaml_fm = yaml.dump(frontmatter_data, sort_keys=False, allow_unicode=True)
709 | return f"---\n{yaml_fm}---\n\n{new_body.strip()}"
710 |
711 | except Exception as e: # pragma: no cover
712 | logger.warning(
713 | f"Failed to parse frontmatter during prepend: {e}"
714 | ) # pragma: no cover
715 | # Fall back to simple prepend if frontmatter parsing fails # pragma: no cover
716 |
717 | # No frontmatter or parsing failed - do simple prepend # pragma: no cover
718 | if content and not content.endswith("\n"): # pragma: no cover
719 | return content + "\n" + current_content # pragma: no cover
720 | return content + current_content # pragma: no cover
721 |
722 | async def move_entity(
723 | self,
724 | identifier: str,
725 | destination_path: str,
726 | project_config: ProjectConfig,
727 | app_config: BasicMemoryConfig,
728 | ) -> EntityModel:
729 | """Move entity to new location with database consistency.
730 |
731 | Args:
732 | identifier: Entity identifier (title, permalink, or memory:// URL)
733 | destination_path: New path relative to project root
734 | project_config: Project configuration for file operations
735 | app_config: App configuration for permalink update settings
736 |
737 | Returns:
738 | Success message with move details
739 |
740 | Raises:
741 | EntityNotFoundError: If the entity cannot be found
742 | ValueError: If move operation fails due to validation or filesystem errors
743 | """
744 | logger.debug(f"Moving entity: {identifier} to {destination_path}")
745 |
746 | # 1. Resolve identifier to entity with strict mode for destructive operations
747 | entity = await self.link_resolver.resolve_link(identifier, strict=True)
748 | if not entity:
749 | raise EntityNotFoundError(f"Entity not found: {identifier}")
750 |
751 | current_path = entity.file_path
752 | old_permalink = entity.permalink
753 |
754 | # 2. Validate destination path format first
755 | if not destination_path or destination_path.startswith("/") or not destination_path.strip():
756 | raise ValueError(f"Invalid destination path: {destination_path}")
757 |
758 | # 3. Validate paths
759 | source_file = project_config.home / current_path
760 | destination_file = project_config.home / destination_path
761 |
762 | # Validate source exists
763 | if not source_file.exists():
764 | raise ValueError(f"Source file not found: {current_path}")
765 |
766 | # Check if destination already exists
767 | if destination_file.exists():
768 | raise ValueError(f"Destination already exists: {destination_path}")
769 |
770 | try:
771 | # 4. Create destination directory if needed
772 | destination_file.parent.mkdir(parents=True, exist_ok=True)
773 |
774 | # 5. Move physical file
775 | source_file.rename(destination_file)
776 | logger.info(f"Moved file: {current_path} -> {destination_path}")
777 |
778 | # 6. Prepare database updates
779 | updates = {"file_path": destination_path}
780 |
781 | # 7. Update permalink if configured or if entity has null permalink (unless disabled)
782 | if not app_config.disable_permalinks and (
783 | app_config.update_permalinks_on_move or old_permalink is None
784 | ):
785 | # Generate new permalink from destination path
786 | new_permalink = await self.resolve_permalink(destination_path)
787 |
788 | # Update frontmatter with new permalink
789 | await self.file_service.update_frontmatter(
790 | destination_path, {"permalink": new_permalink}
791 | )
792 |
793 | updates["permalink"] = new_permalink
794 | if old_permalink is None:
795 | logger.info(
796 | f"Generated permalink for entity with null permalink: {new_permalink}"
797 | )
798 | else:
799 | logger.info(f"Updated permalink: {old_permalink} -> {new_permalink}")
800 |
801 | # 8. Recalculate checksum
802 | new_checksum = await self.file_service.compute_checksum(destination_path)
803 | updates["checksum"] = new_checksum
804 |
805 | # 9. Update database
806 | updated_entity = await self.repository.update(entity.id, updates)
807 | if not updated_entity:
808 | raise ValueError(f"Failed to update entity in database: {entity.id}")
809 |
810 | return updated_entity
811 |
812 | except Exception as e:
813 | # Rollback: try to restore original file location if move succeeded
814 | if destination_file.exists() and not source_file.exists():
815 | try:
816 | destination_file.rename(source_file)
817 | logger.info(f"Rolled back file move: {destination_path} -> {current_path}")
818 | except Exception as rollback_error: # pragma: no cover
819 | logger.error(f"Failed to rollback file move: {rollback_error}")
820 |
821 | # Re-raise the original error with context
822 | raise ValueError(f"Move failed: {str(e)}") from e
823 |
```
--------------------------------------------------------------------------------
/src/basic_memory/services/project_service.py:
--------------------------------------------------------------------------------
```python
1 | """Project management service for Basic Memory."""
2 |
3 | import asyncio
4 | import json
5 | import os
6 | import shutil
7 | from datetime import datetime
8 | from pathlib import Path
9 | from typing import Dict, Optional, Sequence
10 |
11 | from loguru import logger
12 | from sqlalchemy import text
13 |
14 | from basic_memory.models import Project
15 | from basic_memory.repository.project_repository import ProjectRepository
16 | from basic_memory.schemas import (
17 | ActivityMetrics,
18 | ProjectInfoResponse,
19 | ProjectStatistics,
20 | SystemStatus,
21 | )
22 | from basic_memory.config import WATCH_STATUS_JSON, ConfigManager, get_project_config, ProjectConfig
23 | from basic_memory.utils import generate_permalink
24 |
25 |
26 | config = ConfigManager().config
27 |
28 |
29 | class ProjectService:
30 | """Service for managing Basic Memory projects."""
31 |
32 | repository: ProjectRepository
33 |
34 | def __init__(self, repository: ProjectRepository):
35 | """Initialize the project service."""
36 | super().__init__()
37 | self.repository = repository
38 |
39 | @property
40 | def config_manager(self) -> ConfigManager:
41 | """Get a ConfigManager instance.
42 |
43 | Returns:
44 | Fresh ConfigManager instance for each access
45 | """
46 | return ConfigManager()
47 |
48 | @property
49 | def config(self) -> ProjectConfig:
50 | """Get the current project configuration.
51 |
52 | Returns:
53 | Current project configuration
54 | """
55 | return get_project_config()
56 |
57 | @property
58 | def projects(self) -> Dict[str, str]:
59 | """Get all configured projects.
60 |
61 | Returns:
62 | Dict mapping project names to their file paths
63 | """
64 | return self.config_manager.projects
65 |
66 | @property
67 | def default_project(self) -> str:
68 | """Get the name of the default project.
69 |
70 | Returns:
71 | The name of the default project
72 | """
73 | return self.config_manager.default_project
74 |
75 | @property
76 | def current_project(self) -> str:
77 | """Get the name of the currently active project.
78 |
79 | Returns:
80 | The name of the current project
81 | """
82 | return os.environ.get("BASIC_MEMORY_PROJECT", self.config_manager.default_project)
83 |
84 | async def list_projects(self) -> Sequence[Project]:
85 | """List all projects without loading entity relationships.
86 |
87 | Returns only basic project fields (name, path, etc.) without
88 | eager loading the entities relationship which could load thousands
89 | of entities for large knowledge bases.
90 | """
91 | return await self.repository.find_all(use_load_options=False)
92 |
93 | async def get_project(self, name: str) -> Optional[Project]:
94 | """Get the file path for a project by name or permalink."""
95 | return await self.repository.get_by_name(name) or await self.repository.get_by_permalink(
96 | name
97 | )
98 |
99 | def _check_nested_paths(self, path1: str, path2: str) -> bool:
100 | """Check if two paths are nested (one is a prefix of the other).
101 |
102 | Args:
103 | path1: First path to compare
104 | path2: Second path to compare
105 |
106 | Returns:
107 | True if one path is nested within the other, False otherwise
108 |
109 | Examples:
110 | _check_nested_paths("/foo", "/foo/bar") # True (child under parent)
111 | _check_nested_paths("/foo/bar", "/foo") # True (parent over child)
112 | _check_nested_paths("/foo", "/bar") # False (siblings)
113 | """
114 | # Normalize paths to ensure proper comparison
115 | p1 = Path(path1).resolve()
116 | p2 = Path(path2).resolve()
117 |
118 | # Check if either path is a parent of the other
119 | try:
120 | # Check if p2 is under p1
121 | p2.relative_to(p1)
122 | return True
123 | except ValueError:
124 | # Not nested in this direction, check the other
125 | try:
126 | # Check if p1 is under p2
127 | p1.relative_to(p2)
128 | return True
129 | except ValueError:
130 | # Not nested in either direction
131 | return False
132 |
133 | async def add_project(self, name: str, path: str, set_default: bool = False) -> None:
134 | """Add a new project to the configuration and database.
135 |
136 | Args:
137 | name: The name of the project
138 | path: The file path to the project directory
139 | set_default: Whether to set this project as the default
140 |
141 | Raises:
142 | ValueError: If the project already exists or path collides with existing project
143 | """
144 | # If project_root is set, constrain all projects to that directory
145 | project_root = self.config_manager.config.project_root
146 | if project_root:
147 | base_path = Path(project_root)
148 |
149 | # In cloud mode (when project_root is set), ignore user's path completely
150 | # and use sanitized project name as the directory name
151 | # This ensures flat structure: /app/data/test-bisync instead of /app/data/documents/test bisync
152 | sanitized_name = generate_permalink(name)
153 |
154 | # Construct path using sanitized project name only
155 | resolved_path = (base_path / sanitized_name).resolve().as_posix()
156 |
157 | # Verify the resolved path is actually under project_root
158 | if not resolved_path.startswith(base_path.resolve().as_posix()):
159 | raise ValueError(
160 | f"BASIC_MEMORY_PROJECT_ROOT is set to {project_root}. "
161 | f"All projects must be created under this directory. Invalid path: {path}"
162 | )
163 |
164 | # Check for case-insensitive path collisions with existing projects
165 | existing_projects = await self.list_projects()
166 | for existing in existing_projects:
167 | if (
168 | existing.path.lower() == resolved_path.lower()
169 | and existing.path != resolved_path
170 | ):
171 | raise ValueError(
172 | f"Path collision detected: '{resolved_path}' conflicts with existing project "
173 | f"'{existing.name}' at '{existing.path}'. "
174 | f"In cloud mode, paths are normalized to lowercase to prevent case-sensitivity issues."
175 | )
176 | else:
177 | resolved_path = Path(os.path.abspath(os.path.expanduser(path))).as_posix()
178 |
179 | # Check for nested paths with existing projects
180 | existing_projects = await self.list_projects()
181 | for existing in existing_projects:
182 | if self._check_nested_paths(resolved_path, existing.path):
183 | # Determine which path is nested within which for appropriate error message
184 | p_new = Path(resolved_path).resolve()
185 | p_existing = Path(existing.path).resolve()
186 |
187 | # Check if new path is nested under existing project
188 | if p_new.is_relative_to(p_existing):
189 | raise ValueError(
190 | f"Cannot create project at '{resolved_path}': "
191 | f"path is nested within existing project '{existing.name}' at '{existing.path}'. "
192 | f"Projects cannot share directory trees."
193 | )
194 | else:
195 | # Existing project is nested under new path
196 | raise ValueError(
197 | f"Cannot create project at '{resolved_path}': "
198 | f"existing project '{existing.name}' at '{existing.path}' is nested within this path. "
199 | f"Projects cannot share directory trees."
200 | )
201 |
202 | # First add to config file (this will validate the project doesn't exist)
203 | project_config = self.config_manager.add_project(name, resolved_path)
204 |
205 | # Then add to database
206 | project_data = {
207 | "name": name,
208 | "path": resolved_path,
209 | "permalink": generate_permalink(project_config.name),
210 | "is_active": True,
211 | # Don't set is_default=False to avoid UNIQUE constraint issues
212 | # Let it default to NULL, only set to True when explicitly making default
213 | }
214 | created_project = await self.repository.create(project_data)
215 |
216 | # If this should be the default project, ensure only one default exists
217 | if set_default:
218 | await self.repository.set_as_default(created_project.id)
219 | self.config_manager.set_default_project(name)
220 | logger.info(f"Project '{name}' set as default")
221 |
222 | logger.info(f"Project '{name}' added at {resolved_path}")
223 |
224 | async def remove_project(self, name: str, delete_notes: bool = False) -> None:
225 | """Remove a project from configuration and database.
226 |
227 | Args:
228 | name: The name of the project to remove
229 | delete_notes: If True, delete the project directory from filesystem
230 |
231 | Raises:
232 | ValueError: If the project doesn't exist or is the default project
233 | """
234 | if not self.repository: # pragma: no cover
235 | raise ValueError("Repository is required for remove_project")
236 |
237 | # Get project path before removing from config
238 | project = await self.get_project(name)
239 | project_path = project.path if project else None
240 |
241 | # First remove from config (this will validate the project exists and is not default)
242 | self.config_manager.remove_project(name)
243 |
244 | # Then remove from database using robust lookup
245 | if project:
246 | await self.repository.delete(project.id)
247 |
248 | logger.info(f"Project '{name}' removed from configuration and database")
249 |
250 | # Optionally delete the project directory
251 | if delete_notes and project_path:
252 | try:
253 | path_obj = Path(project_path)
254 | if path_obj.exists() and path_obj.is_dir():
255 | await asyncio.to_thread(shutil.rmtree, project_path)
256 | logger.info(f"Deleted project directory: {project_path}")
257 | else:
258 | logger.warning(
259 | f"Project directory not found or not a directory: {project_path}"
260 | )
261 | except Exception as e:
262 | logger.warning(f"Failed to delete project directory {project_path}: {e}")
263 |
264 | async def set_default_project(self, name: str) -> None:
265 | """Set the default project in configuration and database.
266 |
267 | Args:
268 | name: The name of the project to set as default
269 |
270 | Raises:
271 | ValueError: If the project doesn't exist
272 | """
273 | if not self.repository: # pragma: no cover
274 | raise ValueError("Repository is required for set_default_project")
275 |
276 | # First update config file (this will validate the project exists)
277 | self.config_manager.set_default_project(name)
278 |
279 | # Then update database using the same lookup logic as get_project
280 | project = await self.get_project(name)
281 | if project:
282 | await self.repository.set_as_default(project.id)
283 | else:
284 | logger.error(f"Project '{name}' exists in config but not in database")
285 |
286 | logger.info(f"Project '{name}' set as default in configuration and database")
287 |
288 | async def _ensure_single_default_project(self) -> None:
289 | """Ensure only one project has is_default=True.
290 |
291 | This method validates the database state and fixes any issues where
292 | multiple projects might have is_default=True or no project is marked as default.
293 | """
294 | if not self.repository:
295 | raise ValueError(
296 | "Repository is required for _ensure_single_default_project"
297 | ) # pragma: no cover
298 |
299 | # Get all projects with is_default=True
300 | db_projects = await self.repository.find_all()
301 | default_projects = [p for p in db_projects if p.is_default is True]
302 |
303 | if len(default_projects) > 1: # pragma: no cover
304 | # Multiple defaults found - fix by keeping the first one and clearing others
305 | # This is defensive code that should rarely execute due to business logic enforcement
306 | logger.warning( # pragma: no cover
307 | f"Found {len(default_projects)} projects with is_default=True, fixing..."
308 | )
309 | keep_default = default_projects[0] # pragma: no cover
310 |
311 | # Clear all defaults first, then set only the first one as default
312 | await self.repository.set_as_default(keep_default.id) # pragma: no cover
313 |
314 | logger.info(
315 | f"Fixed default project conflicts, kept '{keep_default.name}' as default"
316 | ) # pragma: no cover
317 |
318 | elif len(default_projects) == 0: # pragma: no cover
319 | # No default project - set the config default as default
320 | # This is defensive code for edge cases where no default exists
321 | config_default = self.config_manager.default_project # pragma: no cover
322 | config_project = await self.repository.get_by_name(config_default) # pragma: no cover
323 | if config_project: # pragma: no cover
324 | await self.repository.set_as_default(config_project.id) # pragma: no cover
325 | logger.info(
326 | f"Set '{config_default}' as default project (was missing)"
327 | ) # pragma: no cover
328 |
329 | async def synchronize_projects(self) -> None: # pragma: no cover
330 | """Synchronize projects between database and configuration.
331 |
332 | Ensures that all projects in the configuration file exist in the database
333 | and vice versa. This should be called during initialization to reconcile
334 | any differences between the two sources.
335 | """
336 | if not self.repository:
337 | raise ValueError("Repository is required for synchronize_projects")
338 |
339 | logger.info("Synchronizing projects between database and configuration")
340 |
341 | # Get all projects from database
342 | db_projects = await self.repository.get_active_projects()
343 | db_projects_by_permalink = {p.permalink: p for p in db_projects}
344 |
345 | # Get all projects from configuration and normalize names if needed
346 | config_projects = self.config_manager.projects.copy()
347 | updated_config = {}
348 | config_updated = False
349 |
350 | for name, path in config_projects.items():
351 | # Generate normalized name (what the database expects)
352 | normalized_name = generate_permalink(name)
353 |
354 | if normalized_name != name:
355 | logger.info(f"Normalizing project name in config: '{name}' -> '{normalized_name}'")
356 | config_updated = True
357 |
358 | updated_config[normalized_name] = path
359 |
360 | # Update the configuration if any changes were made
361 | if config_updated:
362 | config = self.config_manager.load_config()
363 | config.projects = updated_config
364 | self.config_manager.save_config(config)
365 | logger.info("Config updated with normalized project names")
366 |
367 | # Use the normalized config for further processing
368 | config_projects = updated_config
369 |
370 | # Add projects that exist in config but not in DB
371 | for name, path in config_projects.items():
372 | if name not in db_projects_by_permalink:
373 | logger.info(f"Adding project '{name}' to database")
374 | project_data = {
375 | "name": name,
376 | "path": path,
377 | "permalink": generate_permalink(name),
378 | "is_active": True,
379 | # Don't set is_default here - let the enforcement logic handle it
380 | }
381 | await self.repository.create(project_data)
382 |
383 | # Remove projects that exist in DB but not in config
384 | # Config is the source of truth - if a project was deleted from config,
385 | # it should be deleted from DB too (fixes issue #193)
386 | for name, project in db_projects_by_permalink.items():
387 | if name not in config_projects:
388 | logger.info(
389 | f"Removing project '{name}' from database (deleted from config, source of truth)"
390 | )
391 | await self.repository.delete(project.id)
392 |
393 | # Ensure database default project state is consistent
394 | await self._ensure_single_default_project()
395 |
396 | # Make sure default project is synchronized between config and database
397 | db_default = await self.repository.get_default_project()
398 | config_default = self.config_manager.default_project
399 |
400 | if db_default and db_default.name != config_default:
401 | # Update config to match DB default
402 | logger.info(f"Updating default project in config to '{db_default.name}'")
403 | self.config_manager.set_default_project(db_default.name)
404 | elif not db_default and config_default:
405 | # Update DB to match config default (if the project exists)
406 | project = await self.repository.get_by_name(config_default)
407 | if project:
408 | logger.info(f"Updating default project in database to '{config_default}'")
409 | await self.repository.set_as_default(project.id)
410 |
411 | logger.info("Project synchronization complete")
412 |
413 | async def move_project(self, name: str, new_path: str) -> None:
414 | """Move a project to a new location.
415 |
416 | Args:
417 | name: The name of the project to move
418 | new_path: The new absolute path for the project
419 |
420 | Raises:
421 | ValueError: If the project doesn't exist or repository isn't initialized
422 | """
423 | if not self.repository:
424 | raise ValueError("Repository is required for move_project")
425 |
426 | # Resolve to absolute path
427 | resolved_path = Path(os.path.abspath(os.path.expanduser(new_path))).as_posix()
428 |
429 | # Validate project exists in config
430 | if name not in self.config_manager.projects:
431 | raise ValueError(f"Project '{name}' not found in configuration")
432 |
433 | # Create the new directory if it doesn't exist
434 | Path(resolved_path).mkdir(parents=True, exist_ok=True)
435 |
436 | # Update in configuration
437 | config = self.config_manager.load_config()
438 | old_path = config.projects[name]
439 | config.projects[name] = resolved_path
440 | self.config_manager.save_config(config)
441 |
442 | # Update in database using robust lookup
443 | project = await self.get_project(name)
444 | if project:
445 | await self.repository.update_path(project.id, resolved_path)
446 | logger.info(f"Moved project '{name}' from {old_path} to {resolved_path}")
447 | else:
448 | logger.error(f"Project '{name}' exists in config but not in database")
449 | # Restore the old path in config since DB update failed
450 | config.projects[name] = old_path
451 | self.config_manager.save_config(config)
452 | raise ValueError(f"Project '{name}' not found in database")
453 |
454 | async def update_project( # pragma: no cover
455 | self, name: str, updated_path: Optional[str] = None, is_active: Optional[bool] = None
456 | ) -> None:
457 | """Update project information in both config and database.
458 |
459 | Args:
460 | name: The name of the project to update
461 | updated_path: Optional new path for the project
462 | is_active: Optional flag to set project active status
463 |
464 | Raises:
465 | ValueError: If project doesn't exist or repository isn't initialized
466 | """
467 | if not self.repository:
468 | raise ValueError("Repository is required for update_project")
469 |
470 | # Validate project exists in config
471 | if name not in self.config_manager.projects:
472 | raise ValueError(f"Project '{name}' not found in configuration")
473 |
474 | # Get project from database using robust lookup
475 | project = await self.get_project(name)
476 | if not project:
477 | logger.error(f"Project '{name}' exists in config but not in database")
478 | return
479 |
480 | # Update path if provided
481 | if updated_path:
482 | resolved_path = Path(os.path.abspath(os.path.expanduser(updated_path))).as_posix()
483 |
484 | # Update in config
485 | config = self.config_manager.load_config()
486 | config.projects[name] = resolved_path
487 | self.config_manager.save_config(config)
488 |
489 | # Update in database
490 | project.path = resolved_path
491 | await self.repository.update(project.id, project)
492 |
493 | logger.info(f"Updated path for project '{name}' to {resolved_path}")
494 |
495 | # Update active status if provided
496 | if is_active is not None:
497 | project.is_active = is_active
498 | await self.repository.update(project.id, project)
499 | logger.info(f"Set active status for project '{name}' to {is_active}")
500 |
501 | # If project was made inactive and it was the default, we need to pick a new default
502 | if is_active is False and project.is_default:
503 | # Find another active project
504 | active_projects = await self.repository.get_active_projects()
505 | if active_projects:
506 | new_default = active_projects[0]
507 | await self.repository.set_as_default(new_default.id)
508 | self.config_manager.set_default_project(new_default.name)
509 | logger.info(
510 | f"Changed default project to '{new_default.name}' as '{name}' was deactivated"
511 | )
512 |
513 | async def get_project_info(self, project_name: Optional[str] = None) -> ProjectInfoResponse:
514 | """Get comprehensive information about the specified Basic Memory project.
515 |
516 | Args:
517 | project_name: Name of the project to get info for. If None, uses the current config project.
518 |
519 | Returns:
520 | Comprehensive project information and statistics
521 | """
522 | if not self.repository: # pragma: no cover
523 | raise ValueError("Repository is required for get_project_info")
524 |
525 | # Use specified project or fall back to config project
526 | project_name = project_name or self.config.project
527 | # Get project path from configuration
528 | name, project_path = self.config_manager.get_project(project_name)
529 | if not name: # pragma: no cover
530 | raise ValueError(f"Project '{project_name}' not found in configuration")
531 |
532 | assert project_path is not None
533 | project_permalink = generate_permalink(project_name)
534 |
535 | # Get project from database to get project_id
536 | db_project = await self.repository.get_by_permalink(project_permalink)
537 | if not db_project: # pragma: no cover
538 | raise ValueError(f"Project '{project_name}' not found in database")
539 |
540 | # Get statistics for the specified project
541 | statistics = await self.get_statistics(db_project.id)
542 |
543 | # Get activity metrics for the specified project
544 | activity = await self.get_activity_metrics(db_project.id)
545 |
546 | # Get system status
547 | system = self.get_system_status()
548 |
549 | # Get enhanced project information from database
550 | db_projects = await self.repository.get_active_projects()
551 | db_projects_by_permalink = {p.permalink: p for p in db_projects}
552 |
553 | # Get default project info
554 | default_project = self.config_manager.default_project
555 |
556 | # Convert config projects to include database info
557 | enhanced_projects = {}
558 | for name, path in self.config_manager.projects.items():
559 | config_permalink = generate_permalink(name)
560 | db_project = db_projects_by_permalink.get(config_permalink)
561 | enhanced_projects[name] = {
562 | "path": path,
563 | "active": db_project.is_active if db_project else True,
564 | "id": db_project.id if db_project else None,
565 | "is_default": (name == default_project),
566 | "permalink": db_project.permalink if db_project else name.lower().replace(" ", "-"),
567 | }
568 |
569 | # Construct the response
570 | return ProjectInfoResponse(
571 | project_name=project_name,
572 | project_path=project_path,
573 | available_projects=enhanced_projects,
574 | default_project=default_project,
575 | statistics=statistics,
576 | activity=activity,
577 | system=system,
578 | )
579 |
580 | async def get_statistics(self, project_id: int) -> ProjectStatistics:
581 | """Get statistics about the specified project.
582 |
583 | Args:
584 | project_id: ID of the project to get statistics for (required).
585 | """
586 | if not self.repository: # pragma: no cover
587 | raise ValueError("Repository is required for get_statistics")
588 |
589 | # Get basic counts
590 | entity_count_result = await self.repository.execute_query(
591 | text("SELECT COUNT(*) FROM entity WHERE project_id = :project_id"),
592 | {"project_id": project_id},
593 | )
594 | total_entities = entity_count_result.scalar() or 0
595 |
596 | observation_count_result = await self.repository.execute_query(
597 | text(
598 | "SELECT COUNT(*) FROM observation o JOIN entity e ON o.entity_id = e.id WHERE e.project_id = :project_id"
599 | ),
600 | {"project_id": project_id},
601 | )
602 | total_observations = observation_count_result.scalar() or 0
603 |
604 | relation_count_result = await self.repository.execute_query(
605 | text(
606 | "SELECT COUNT(*) FROM relation r JOIN entity e ON r.from_id = e.id WHERE e.project_id = :project_id"
607 | ),
608 | {"project_id": project_id},
609 | )
610 | total_relations = relation_count_result.scalar() or 0
611 |
612 | unresolved_count_result = await self.repository.execute_query(
613 | text(
614 | "SELECT COUNT(*) FROM relation r JOIN entity e ON r.from_id = e.id WHERE r.to_id IS NULL AND e.project_id = :project_id"
615 | ),
616 | {"project_id": project_id},
617 | )
618 | total_unresolved = unresolved_count_result.scalar() or 0
619 |
620 | # Get entity counts by type
621 | entity_types_result = await self.repository.execute_query(
622 | text(
623 | "SELECT entity_type, COUNT(*) FROM entity WHERE project_id = :project_id GROUP BY entity_type"
624 | ),
625 | {"project_id": project_id},
626 | )
627 | entity_types = {row[0]: row[1] for row in entity_types_result.fetchall()}
628 |
629 | # Get observation counts by category
630 | category_result = await self.repository.execute_query(
631 | text(
632 | "SELECT o.category, COUNT(*) FROM observation o JOIN entity e ON o.entity_id = e.id WHERE e.project_id = :project_id GROUP BY o.category"
633 | ),
634 | {"project_id": project_id},
635 | )
636 | observation_categories = {row[0]: row[1] for row in category_result.fetchall()}
637 |
638 | # Get relation counts by type
639 | relation_types_result = await self.repository.execute_query(
640 | text(
641 | "SELECT r.relation_type, COUNT(*) FROM relation r JOIN entity e ON r.from_id = e.id WHERE e.project_id = :project_id GROUP BY r.relation_type"
642 | ),
643 | {"project_id": project_id},
644 | )
645 | relation_types = {row[0]: row[1] for row in relation_types_result.fetchall()}
646 |
647 | # Find most connected entities (most outgoing relations) - project filtered
648 | connected_result = await self.repository.execute_query(
649 | text("""
650 | SELECT e.id, e.title, e.permalink, COUNT(r.id) AS relation_count, e.file_path
651 | FROM entity e
652 | JOIN relation r ON e.id = r.from_id
653 | WHERE e.project_id = :project_id
654 | GROUP BY e.id
655 | ORDER BY relation_count DESC
656 | LIMIT 10
657 | """),
658 | {"project_id": project_id},
659 | )
660 | most_connected = [
661 | {
662 | "id": row[0],
663 | "title": row[1],
664 | "permalink": row[2],
665 | "relation_count": row[3],
666 | "file_path": row[4],
667 | }
668 | for row in connected_result.fetchall()
669 | ]
670 |
671 | # Count isolated entities (no relations) - project filtered
672 | isolated_result = await self.repository.execute_query(
673 | text("""
674 | SELECT COUNT(e.id)
675 | FROM entity e
676 | LEFT JOIN relation r1 ON e.id = r1.from_id
677 | LEFT JOIN relation r2 ON e.id = r2.to_id
678 | WHERE e.project_id = :project_id AND r1.id IS NULL AND r2.id IS NULL
679 | """),
680 | {"project_id": project_id},
681 | )
682 | isolated_count = isolated_result.scalar() or 0
683 |
684 | return ProjectStatistics(
685 | total_entities=total_entities,
686 | total_observations=total_observations,
687 | total_relations=total_relations,
688 | total_unresolved_relations=total_unresolved,
689 | entity_types=entity_types,
690 | observation_categories=observation_categories,
691 | relation_types=relation_types,
692 | most_connected_entities=most_connected,
693 | isolated_entities=isolated_count,
694 | )
695 |
696 | async def get_activity_metrics(self, project_id: int) -> ActivityMetrics:
697 | """Get activity metrics for the specified project.
698 |
699 | Args:
700 | project_id: ID of the project to get activity metrics for (required).
701 | """
702 | if not self.repository: # pragma: no cover
703 | raise ValueError("Repository is required for get_activity_metrics")
704 |
705 | # Get recently created entities (project filtered)
706 | created_result = await self.repository.execute_query(
707 | text("""
708 | SELECT id, title, permalink, entity_type, created_at, file_path
709 | FROM entity
710 | WHERE project_id = :project_id
711 | ORDER BY created_at DESC
712 | LIMIT 10
713 | """),
714 | {"project_id": project_id},
715 | )
716 | recently_created = [
717 | {
718 | "id": row[0],
719 | "title": row[1],
720 | "permalink": row[2],
721 | "entity_type": row[3],
722 | "created_at": row[4],
723 | "file_path": row[5],
724 | }
725 | for row in created_result.fetchall()
726 | ]
727 |
728 | # Get recently updated entities (project filtered)
729 | updated_result = await self.repository.execute_query(
730 | text("""
731 | SELECT id, title, permalink, entity_type, updated_at, file_path
732 | FROM entity
733 | WHERE project_id = :project_id
734 | ORDER BY updated_at DESC
735 | LIMIT 10
736 | """),
737 | {"project_id": project_id},
738 | )
739 | recently_updated = [
740 | {
741 | "id": row[0],
742 | "title": row[1],
743 | "permalink": row[2],
744 | "entity_type": row[3],
745 | "updated_at": row[4],
746 | "file_path": row[5],
747 | }
748 | for row in updated_result.fetchall()
749 | ]
750 |
751 | # Get monthly growth over the last 6 months
752 | # Calculate the start of 6 months ago
753 | now = datetime.now()
754 | six_months_ago = datetime(
755 | now.year - (1 if now.month <= 6 else 0), ((now.month - 6) % 12) or 12, 1
756 | )
757 |
758 | # Query for monthly entity creation (project filtered)
759 | entity_growth_result = await self.repository.execute_query(
760 | text("""
761 | SELECT
762 | strftime('%Y-%m', created_at) AS month,
763 | COUNT(*) AS count
764 | FROM entity
765 | WHERE created_at >= :six_months_ago AND project_id = :project_id
766 | GROUP BY month
767 | ORDER BY month
768 | """),
769 | {"six_months_ago": six_months_ago.isoformat(), "project_id": project_id},
770 | )
771 | entity_growth = {row[0]: row[1] for row in entity_growth_result.fetchall()}
772 |
773 | # Query for monthly observation creation (project filtered)
774 | observation_growth_result = await self.repository.execute_query(
775 | text("""
776 | SELECT
777 | strftime('%Y-%m', entity.created_at) AS month,
778 | COUNT(*) AS count
779 | FROM observation
780 | INNER JOIN entity ON observation.entity_id = entity.id
781 | WHERE entity.created_at >= :six_months_ago AND entity.project_id = :project_id
782 | GROUP BY month
783 | ORDER BY month
784 | """),
785 | {"six_months_ago": six_months_ago.isoformat(), "project_id": project_id},
786 | )
787 | observation_growth = {row[0]: row[1] for row in observation_growth_result.fetchall()}
788 |
789 | # Query for monthly relation creation (project filtered)
790 | relation_growth_result = await self.repository.execute_query(
791 | text("""
792 | SELECT
793 | strftime('%Y-%m', entity.created_at) AS month,
794 | COUNT(*) AS count
795 | FROM relation
796 | INNER JOIN entity ON relation.from_id = entity.id
797 | WHERE entity.created_at >= :six_months_ago AND entity.project_id = :project_id
798 | GROUP BY month
799 | ORDER BY month
800 | """),
801 | {"six_months_ago": six_months_ago.isoformat(), "project_id": project_id},
802 | )
803 | relation_growth = {row[0]: row[1] for row in relation_growth_result.fetchall()}
804 |
805 | # Combine all monthly growth data
806 | monthly_growth = {}
807 | for month in set(
808 | list(entity_growth.keys())
809 | + list(observation_growth.keys())
810 | + list(relation_growth.keys())
811 | ):
812 | monthly_growth[month] = {
813 | "entities": entity_growth.get(month, 0),
814 | "observations": observation_growth.get(month, 0),
815 | "relations": relation_growth.get(month, 0),
816 | "total": (
817 | entity_growth.get(month, 0)
818 | + observation_growth.get(month, 0)
819 | + relation_growth.get(month, 0)
820 | ),
821 | }
822 |
823 | return ActivityMetrics(
824 | recently_created=recently_created,
825 | recently_updated=recently_updated,
826 | monthly_growth=monthly_growth,
827 | )
828 |
829 | def get_system_status(self) -> SystemStatus:
830 | """Get system status information."""
831 | import basic_memory
832 |
833 | # Get database information
834 | db_path = self.config_manager.config.database_path
835 | db_size = db_path.stat().st_size if db_path.exists() else 0
836 | db_size_readable = f"{db_size / (1024 * 1024):.2f} MB"
837 |
838 | # Get watch service status if available
839 | watch_status = None
840 | watch_status_path = Path.home() / ".basic-memory" / WATCH_STATUS_JSON
841 | if watch_status_path.exists():
842 | try:
843 | watch_status = json.loads(watch_status_path.read_text(encoding="utf-8"))
844 | except Exception: # pragma: no cover
845 | pass
846 |
847 | return SystemStatus(
848 | version=basic_memory.__version__,
849 | database_path=str(db_path),
850 | database_size=db_size_readable,
851 | watch_status=watch_status,
852 | timestamp=datetime.now(),
853 | )
854 |
```