This is page 9 of 23. Use http://codebase.md/basicmachines-co/basic-memory?lines=true&page={x} to view the full context.
# Directory Structure
```
├── .claude
│ ├── agents
│ │ ├── python-developer.md
│ │ └── system-architect.md
│ └── commands
│ ├── release
│ │ ├── beta.md
│ │ ├── changelog.md
│ │ ├── release-check.md
│ │ └── release.md
│ ├── spec.md
│ └── test-live.md
├── .dockerignore
├── .github
│ ├── dependabot.yml
│ ├── ISSUE_TEMPLATE
│ │ ├── bug_report.md
│ │ ├── config.yml
│ │ ├── documentation.md
│ │ └── feature_request.md
│ └── workflows
│ ├── claude-code-review.yml
│ ├── claude-issue-triage.yml
│ ├── claude.yml
│ ├── dev-release.yml
│ ├── docker.yml
│ ├── pr-title.yml
│ ├── release.yml
│ └── test.yml
├── .gitignore
├── .python-version
├── CHANGELOG.md
├── CITATION.cff
├── CLA.md
├── CLAUDE.md
├── CODE_OF_CONDUCT.md
├── CONTRIBUTING.md
├── docker-compose.yml
├── Dockerfile
├── docs
│ ├── ai-assistant-guide-extended.md
│ ├── character-handling.md
│ ├── cloud-cli.md
│ └── Docker.md
├── justfile
├── LICENSE
├── llms-install.md
├── pyproject.toml
├── README.md
├── SECURITY.md
├── smithery.yaml
├── specs
│ ├── SPEC-1 Specification-Driven Development Process.md
│ ├── SPEC-10 Unified Deployment Workflow and Event Tracking.md
│ ├── SPEC-11 Basic Memory API Performance Optimization.md
│ ├── SPEC-12 OpenTelemetry Observability.md
│ ├── SPEC-13 CLI Authentication with Subscription Validation.md
│ ├── SPEC-14 Cloud Git Versioning & GitHub Backup.md
│ ├── SPEC-14- Cloud Git Versioning & GitHub Backup.md
│ ├── SPEC-15 Configuration Persistence via Tigris for Cloud Tenants.md
│ ├── SPEC-16 MCP Cloud Service Consolidation.md
│ ├── SPEC-17 Semantic Search with ChromaDB.md
│ ├── SPEC-18 AI Memory Management Tool.md
│ ├── SPEC-19 Sync Performance and Memory Optimization.md
│ ├── SPEC-2 Slash Commands Reference.md
│ ├── SPEC-20 Simplified Project-Scoped Rclone Sync.md
│ ├── SPEC-3 Agent Definitions.md
│ ├── SPEC-4 Notes Web UI Component Architecture.md
│ ├── SPEC-5 CLI Cloud Upload via WebDAV.md
│ ├── SPEC-6 Explicit Project Parameter Architecture.md
│ ├── SPEC-7 POC to spike Tigris Turso for local access to cloud data.md
│ ├── SPEC-8 TigrisFS Integration.md
│ ├── SPEC-9 Multi-Project Bidirectional Sync Architecture.md
│ ├── SPEC-9 Signed Header Tenant Information.md
│ └── SPEC-9-1 Follow-Ups- Conflict, Sync, and Observability.md
├── src
│ └── basic_memory
│ ├── __init__.py
│ ├── alembic
│ │ ├── alembic.ini
│ │ ├── env.py
│ │ ├── migrations.py
│ │ ├── script.py.mako
│ │ └── versions
│ │ ├── 3dae7c7b1564_initial_schema.py
│ │ ├── 502b60eaa905_remove_required_from_entity_permalink.py
│ │ ├── 5fe1ab1ccebe_add_projects_table.py
│ │ ├── 647e7a75e2cd_project_constraint_fix.py
│ │ ├── 9d9c1cb7d8f5_add_mtime_and_size_columns_to_entity_.py
│ │ ├── a1b2c3d4e5f6_fix_project_foreign_keys.py
│ │ ├── b3c3938bacdb_relation_to_name_unique_index.py
│ │ ├── cc7172b46608_update_search_index_schema.py
│ │ └── e7e1f4367280_add_scan_watermark_tracking_to_project.py
│ ├── api
│ │ ├── __init__.py
│ │ ├── app.py
│ │ ├── routers
│ │ │ ├── __init__.py
│ │ │ ├── directory_router.py
│ │ │ ├── importer_router.py
│ │ │ ├── knowledge_router.py
│ │ │ ├── management_router.py
│ │ │ ├── memory_router.py
│ │ │ ├── project_router.py
│ │ │ ├── prompt_router.py
│ │ │ ├── resource_router.py
│ │ │ ├── search_router.py
│ │ │ └── utils.py
│ │ └── template_loader.py
│ ├── cli
│ │ ├── __init__.py
│ │ ├── app.py
│ │ ├── auth.py
│ │ ├── commands
│ │ │ ├── __init__.py
│ │ │ ├── cloud
│ │ │ │ ├── __init__.py
│ │ │ │ ├── api_client.py
│ │ │ │ ├── bisync_commands.py
│ │ │ │ ├── cloud_utils.py
│ │ │ │ ├── core_commands.py
│ │ │ │ ├── rclone_commands.py
│ │ │ │ ├── rclone_config.py
│ │ │ │ ├── rclone_installer.py
│ │ │ │ ├── upload_command.py
│ │ │ │ └── upload.py
│ │ │ ├── command_utils.py
│ │ │ ├── db.py
│ │ │ ├── import_chatgpt.py
│ │ │ ├── import_claude_conversations.py
│ │ │ ├── import_claude_projects.py
│ │ │ ├── import_memory_json.py
│ │ │ ├── mcp.py
│ │ │ ├── project.py
│ │ │ ├── status.py
│ │ │ └── tool.py
│ │ └── main.py
│ ├── config.py
│ ├── db.py
│ ├── deps.py
│ ├── file_utils.py
│ ├── ignore_utils.py
│ ├── importers
│ │ ├── __init__.py
│ │ ├── base.py
│ │ ├── chatgpt_importer.py
│ │ ├── claude_conversations_importer.py
│ │ ├── claude_projects_importer.py
│ │ ├── memory_json_importer.py
│ │ └── utils.py
│ ├── markdown
│ │ ├── __init__.py
│ │ ├── entity_parser.py
│ │ ├── markdown_processor.py
│ │ ├── plugins.py
│ │ ├── schemas.py
│ │ └── utils.py
│ ├── mcp
│ │ ├── __init__.py
│ │ ├── async_client.py
│ │ ├── project_context.py
│ │ ├── prompts
│ │ │ ├── __init__.py
│ │ │ ├── ai_assistant_guide.py
│ │ │ ├── continue_conversation.py
│ │ │ ├── recent_activity.py
│ │ │ ├── search.py
│ │ │ └── utils.py
│ │ ├── resources
│ │ │ ├── ai_assistant_guide.md
│ │ │ └── project_info.py
│ │ ├── server.py
│ │ └── tools
│ │ ├── __init__.py
│ │ ├── build_context.py
│ │ ├── canvas.py
│ │ ├── chatgpt_tools.py
│ │ ├── delete_note.py
│ │ ├── edit_note.py
│ │ ├── list_directory.py
│ │ ├── move_note.py
│ │ ├── project_management.py
│ │ ├── read_content.py
│ │ ├── read_note.py
│ │ ├── recent_activity.py
│ │ ├── search.py
│ │ ├── utils.py
│ │ ├── view_note.py
│ │ └── write_note.py
│ ├── models
│ │ ├── __init__.py
│ │ ├── base.py
│ │ ├── knowledge.py
│ │ ├── project.py
│ │ └── search.py
│ ├── repository
│ │ ├── __init__.py
│ │ ├── entity_repository.py
│ │ ├── observation_repository.py
│ │ ├── project_info_repository.py
│ │ ├── project_repository.py
│ │ ├── relation_repository.py
│ │ ├── repository.py
│ │ └── search_repository.py
│ ├── schemas
│ │ ├── __init__.py
│ │ ├── base.py
│ │ ├── cloud.py
│ │ ├── delete.py
│ │ ├── directory.py
│ │ ├── importer.py
│ │ ├── memory.py
│ │ ├── project_info.py
│ │ ├── prompt.py
│ │ ├── request.py
│ │ ├── response.py
│ │ ├── search.py
│ │ └── sync_report.py
│ ├── services
│ │ ├── __init__.py
│ │ ├── context_service.py
│ │ ├── directory_service.py
│ │ ├── entity_service.py
│ │ ├── exceptions.py
│ │ ├── file_service.py
│ │ ├── initialization.py
│ │ ├── link_resolver.py
│ │ ├── project_service.py
│ │ ├── search_service.py
│ │ └── service.py
│ ├── sync
│ │ ├── __init__.py
│ │ ├── background_sync.py
│ │ ├── sync_service.py
│ │ └── watch_service.py
│ ├── templates
│ │ └── prompts
│ │ ├── continue_conversation.hbs
│ │ └── search.hbs
│ └── utils.py
├── test-int
│ ├── BENCHMARKS.md
│ ├── cli
│ │ ├── test_project_commands_integration.py
│ │ └── test_version_integration.py
│ ├── conftest.py
│ ├── mcp
│ │ ├── test_build_context_underscore.py
│ │ ├── test_build_context_validation.py
│ │ ├── test_chatgpt_tools_integration.py
│ │ ├── test_default_project_mode_integration.py
│ │ ├── test_delete_note_integration.py
│ │ ├── test_edit_note_integration.py
│ │ ├── test_list_directory_integration.py
│ │ ├── test_move_note_integration.py
│ │ ├── test_project_management_integration.py
│ │ ├── test_project_state_sync_integration.py
│ │ ├── test_read_content_integration.py
│ │ ├── test_read_note_integration.py
│ │ ├── test_search_integration.py
│ │ ├── test_single_project_mcp_integration.py
│ │ └── test_write_note_integration.py
│ ├── test_db_wal_mode.py
│ ├── test_disable_permalinks_integration.py
│ └── test_sync_performance_benchmark.py
├── tests
│ ├── __init__.py
│ ├── api
│ │ ├── conftest.py
│ │ ├── test_async_client.py
│ │ ├── test_continue_conversation_template.py
│ │ ├── test_directory_router.py
│ │ ├── test_importer_router.py
│ │ ├── test_knowledge_router.py
│ │ ├── test_management_router.py
│ │ ├── test_memory_router.py
│ │ ├── test_project_router_operations.py
│ │ ├── test_project_router.py
│ │ ├── test_prompt_router.py
│ │ ├── test_relation_background_resolution.py
│ │ ├── test_resource_router.py
│ │ ├── test_search_router.py
│ │ ├── test_search_template.py
│ │ ├── test_template_loader_helpers.py
│ │ └── test_template_loader.py
│ ├── cli
│ │ ├── conftest.py
│ │ ├── test_cli_tools.py
│ │ ├── test_cloud_authentication.py
│ │ ├── test_ignore_utils.py
│ │ ├── test_import_chatgpt.py
│ │ ├── test_import_claude_conversations.py
│ │ ├── test_import_claude_projects.py
│ │ ├── test_import_memory_json.py
│ │ ├── test_project_add_with_local_path.py
│ │ └── test_upload.py
│ ├── conftest.py
│ ├── db
│ │ └── test_issue_254_foreign_key_constraints.py
│ ├── importers
│ │ ├── test_importer_base.py
│ │ └── test_importer_utils.py
│ ├── markdown
│ │ ├── __init__.py
│ │ ├── test_date_frontmatter_parsing.py
│ │ ├── test_entity_parser_error_handling.py
│ │ ├── test_entity_parser.py
│ │ ├── test_markdown_plugins.py
│ │ ├── test_markdown_processor.py
│ │ ├── test_observation_edge_cases.py
│ │ ├── test_parser_edge_cases.py
│ │ ├── test_relation_edge_cases.py
│ │ └── test_task_detection.py
│ ├── mcp
│ │ ├── conftest.py
│ │ ├── test_obsidian_yaml_formatting.py
│ │ ├── test_permalink_collision_file_overwrite.py
│ │ ├── test_prompts.py
│ │ ├── test_resources.py
│ │ ├── test_tool_build_context.py
│ │ ├── test_tool_canvas.py
│ │ ├── test_tool_delete_note.py
│ │ ├── test_tool_edit_note.py
│ │ ├── test_tool_list_directory.py
│ │ ├── test_tool_move_note.py
│ │ ├── test_tool_read_content.py
│ │ ├── test_tool_read_note.py
│ │ ├── test_tool_recent_activity.py
│ │ ├── test_tool_resource.py
│ │ ├── test_tool_search.py
│ │ ├── test_tool_utils.py
│ │ ├── test_tool_view_note.py
│ │ ├── test_tool_write_note.py
│ │ └── tools
│ │ └── test_chatgpt_tools.py
│ ├── Non-MarkdownFileSupport.pdf
│ ├── repository
│ │ ├── test_entity_repository_upsert.py
│ │ ├── test_entity_repository.py
│ │ ├── test_entity_upsert_issue_187.py
│ │ ├── test_observation_repository.py
│ │ ├── test_project_info_repository.py
│ │ ├── test_project_repository.py
│ │ ├── test_relation_repository.py
│ │ ├── test_repository.py
│ │ ├── test_search_repository_edit_bug_fix.py
│ │ └── test_search_repository.py
│ ├── schemas
│ │ ├── test_base_timeframe_minimum.py
│ │ ├── test_memory_serialization.py
│ │ ├── test_memory_url_validation.py
│ │ ├── test_memory_url.py
│ │ ├── test_schemas.py
│ │ └── test_search.py
│ ├── Screenshot.png
│ ├── services
│ │ ├── test_context_service.py
│ │ ├── test_directory_service.py
│ │ ├── test_entity_service_disable_permalinks.py
│ │ ├── test_entity_service.py
│ │ ├── test_file_service.py
│ │ ├── test_initialization.py
│ │ ├── test_link_resolver.py
│ │ ├── test_project_removal_bug.py
│ │ ├── test_project_service_operations.py
│ │ ├── test_project_service.py
│ │ └── test_search_service.py
│ ├── sync
│ │ ├── test_character_conflicts.py
│ │ ├── test_sync_service_incremental.py
│ │ ├── test_sync_service.py
│ │ ├── test_sync_wikilink_issue.py
│ │ ├── test_tmp_files.py
│ │ ├── test_watch_service_edge_cases.py
│ │ ├── test_watch_service_reload.py
│ │ └── test_watch_service.py
│ ├── test_config.py
│ ├── test_db_migration_deduplication.py
│ ├── test_deps.py
│ ├── test_production_cascade_delete.py
│ ├── test_rclone_commands.py
│ └── utils
│ ├── test_file_utils.py
│ ├── test_frontmatter_obsidian_compatible.py
│ ├── test_parse_tags.py
│ ├── test_permalink_formatting.py
│ ├── test_utf8_handling.py
│ └── test_validate_project_path.py
├── uv.lock
├── v0.15.0-RELEASE-DOCS.md
└── v15-docs
├── api-performance.md
├── background-relations.md
├── basic-memory-home.md
├── bug-fixes.md
├── chatgpt-integration.md
├── cloud-authentication.md
├── cloud-bisync.md
├── cloud-mode-usage.md
├── cloud-mount.md
├── default-project-mode.md
├── env-file-removal.md
├── env-var-overrides.md
├── explicit-project-parameter.md
├── gitignore-integration.md
├── project-root-env-var.md
├── README.md
└── sqlite-performance.md
```
# Files
--------------------------------------------------------------------------------
/src/basic_memory/cli/auth.py:
--------------------------------------------------------------------------------
```python
1 | """WorkOS OAuth Device Authorization for CLI."""
2 |
3 | import base64
4 | import hashlib
5 | import json
6 | import os
7 | import secrets
8 | import time
9 | import webbrowser
10 |
11 | import httpx
12 | from rich.console import Console
13 |
14 | from basic_memory.config import ConfigManager
15 |
16 | console = Console()
17 |
18 |
19 | class CLIAuth:
20 | """Handles WorkOS OAuth Device Authorization for CLI tools."""
21 |
22 | def __init__(self, client_id: str, authkit_domain: str):
23 | self.client_id = client_id
24 | self.authkit_domain = authkit_domain
25 | app_config = ConfigManager().config
26 | # Store tokens in data dir
27 | self.token_file = app_config.data_dir_path / "basic-memory-cloud.json"
28 | # PKCE parameters
29 | self.code_verifier = None
30 | self.code_challenge = None
31 |
32 | def generate_pkce_pair(self) -> tuple[str, str]:
33 | """Generate PKCE code verifier and challenge."""
34 | # Generate code verifier (43-128 characters)
35 | code_verifier = base64.urlsafe_b64encode(secrets.token_bytes(32)).decode("utf-8")
36 | code_verifier = code_verifier.rstrip("=")
37 |
38 | # Generate code challenge (SHA256 hash of verifier)
39 | challenge_bytes = hashlib.sha256(code_verifier.encode("utf-8")).digest()
40 | code_challenge = base64.urlsafe_b64encode(challenge_bytes).decode("utf-8")
41 | code_challenge = code_challenge.rstrip("=")
42 |
43 | return code_verifier, code_challenge
44 |
45 | async def request_device_authorization(self) -> dict | None:
46 | """Request device authorization from WorkOS with PKCE."""
47 | device_auth_url = f"{self.authkit_domain}/oauth2/device_authorization"
48 |
49 | # Generate PKCE pair
50 | self.code_verifier, self.code_challenge = self.generate_pkce_pair()
51 |
52 | data = {
53 | "client_id": self.client_id,
54 | "scope": "openid profile email offline_access",
55 | "code_challenge": self.code_challenge,
56 | "code_challenge_method": "S256",
57 | }
58 |
59 | try:
60 | async with httpx.AsyncClient() as client:
61 | response = await client.post(device_auth_url, data=data)
62 |
63 | if response.status_code == 200:
64 | return response.json()
65 | else:
66 | console.print(
67 | f"[red]Device authorization failed: {response.status_code} - {response.text}[/red]"
68 | )
69 | return None
70 | except Exception as e:
71 | console.print(f"[red]Device authorization error: {e}[/red]")
72 | return None
73 |
74 | def display_user_instructions(self, device_response: dict) -> None:
75 | """Display user instructions for device authorization."""
76 | user_code = device_response["user_code"]
77 | verification_uri = device_response["verification_uri"]
78 | verification_uri_complete = device_response.get("verification_uri_complete")
79 |
80 | console.print("\n[bold blue]Authentication Required[/bold blue]")
81 | console.print("\nTo authenticate, please visit:")
82 | console.print(f"[bold cyan]{verification_uri}[/bold cyan]")
83 | console.print(f"\nAnd enter this code: [bold yellow]{user_code}[/bold yellow]")
84 |
85 | if verification_uri_complete:
86 | console.print("\nOr for one-click access, visit:")
87 | console.print(f"[bold green]{verification_uri_complete}[/bold green]")
88 |
89 | # Try to open browser automatically
90 | try:
91 | console.print("\n[dim]Opening browser automatically...[/dim]")
92 | webbrowser.open(verification_uri_complete)
93 | except Exception:
94 | pass # Silently fail if browser can't be opened
95 |
96 | console.print("\n[dim]Waiting for you to complete authentication in your browser...[/dim]")
97 |
98 | async def poll_for_token(self, device_code: str, interval: int = 5) -> dict | None:
99 | """Poll the token endpoint until user completes authentication."""
100 | token_url = f"{self.authkit_domain}/oauth2/token"
101 |
102 | data = {
103 | "client_id": self.client_id,
104 | "device_code": device_code,
105 | "grant_type": "urn:ietf:params:oauth:grant-type:device_code",
106 | "code_verifier": self.code_verifier,
107 | }
108 |
109 | max_attempts = 60 # 5 minutes with 5-second intervals
110 | current_interval = interval
111 |
112 | for _attempt in range(max_attempts):
113 | try:
114 | async with httpx.AsyncClient() as client:
115 | response = await client.post(token_url, data=data)
116 |
117 | if response.status_code == 200:
118 | return response.json()
119 |
120 | # Parse error response
121 | try:
122 | error_data = response.json()
123 | error = error_data.get("error")
124 | except Exception:
125 | error = "unknown_error"
126 |
127 | if error == "authorization_pending":
128 | # User hasn't completed auth yet, keep polling
129 | pass
130 | elif error == "slow_down":
131 | # Increase polling interval
132 | current_interval += 5
133 | console.print("[yellow]Slowing down polling rate...[/yellow]")
134 | elif error == "access_denied":
135 | console.print("[red]Authentication was denied by user[/red]")
136 | return None
137 | elif error == "expired_token":
138 | console.print("[red]Device code has expired. Please try again.[/red]")
139 | return None
140 | else:
141 | console.print(f"[red]Token polling error: {error}[/red]")
142 | return None
143 |
144 | except Exception as e:
145 | console.print(f"[red]Token polling request error: {e}[/red]")
146 |
147 | # Wait before next poll
148 | await self._async_sleep(current_interval)
149 |
150 | console.print("[red]Authentication timeout. Please try again.[/red]")
151 | return None
152 |
153 | async def _async_sleep(self, seconds: int) -> None:
154 | """Async sleep utility."""
155 | import asyncio
156 |
157 | await asyncio.sleep(seconds)
158 |
159 | def save_tokens(self, tokens: dict) -> None:
160 | """Save tokens to project root as .bm-auth.json."""
161 | token_data = {
162 | "access_token": tokens["access_token"],
163 | "refresh_token": tokens.get("refresh_token"),
164 | "expires_at": int(time.time()) + tokens.get("expires_in", 3600),
165 | "token_type": tokens.get("token_type", "Bearer"),
166 | }
167 |
168 | with open(self.token_file, "w") as f:
169 | json.dump(token_data, f, indent=2)
170 |
171 | # Secure the token file
172 | os.chmod(self.token_file, 0o600)
173 |
174 | console.print(f"[green]Tokens saved to {self.token_file}[/green]")
175 |
176 | def load_tokens(self) -> dict | None:
177 | """Load tokens from .bm-auth.json file."""
178 | if not self.token_file.exists():
179 | return None
180 |
181 | try:
182 | with open(self.token_file) as f:
183 | return json.load(f)
184 | except (OSError, json.JSONDecodeError):
185 | return None
186 |
187 | def is_token_valid(self, tokens: dict) -> bool:
188 | """Check if stored token is still valid."""
189 | expires_at = tokens.get("expires_at", 0)
190 | # Add 60 second buffer for clock skew
191 | return time.time() < (expires_at - 60)
192 |
193 | async def refresh_token(self, refresh_token: str) -> dict | None:
194 | """Refresh access token using refresh token."""
195 | token_url = f"{self.authkit_domain}/oauth2/token"
196 |
197 | data = {
198 | "client_id": self.client_id,
199 | "grant_type": "refresh_token",
200 | "refresh_token": refresh_token,
201 | }
202 |
203 | try:
204 | async with httpx.AsyncClient() as client:
205 | response = await client.post(token_url, data=data)
206 |
207 | if response.status_code == 200:
208 | return response.json()
209 | else:
210 | console.print(
211 | f"[red]Token refresh failed: {response.status_code} - {response.text}[/red]"
212 | )
213 | return None
214 | except Exception as e:
215 | console.print(f"[red]Token refresh error: {e}[/red]")
216 | return None
217 |
218 | async def get_valid_token(self) -> str | None:
219 | """Get valid access token, refresh if needed."""
220 | tokens = self.load_tokens()
221 | if not tokens:
222 | return None
223 |
224 | if self.is_token_valid(tokens):
225 | return tokens["access_token"]
226 |
227 | # Token expired - try to refresh if we have a refresh token
228 | refresh_token = tokens.get("refresh_token")
229 | if refresh_token:
230 | console.print("[yellow]Access token expired, refreshing...[/yellow]")
231 |
232 | new_tokens = await self.refresh_token(refresh_token)
233 | if new_tokens:
234 | # Save new tokens (may include rotated refresh token)
235 | self.save_tokens(new_tokens)
236 | console.print("[green]Token refreshed successfully[/green]")
237 | return new_tokens["access_token"]
238 | else:
239 | console.print("[yellow]Token refresh failed. Please run 'login' again.[/yellow]")
240 | return None
241 | else:
242 | console.print("[yellow]No refresh token available. Please run 'login' again.[/yellow]")
243 | return None
244 |
245 | async def login(self) -> bool:
246 | """Perform OAuth Device Authorization login flow."""
247 | console.print("[blue]Initiating authentication...[/blue]")
248 |
249 | # Step 1: Request device authorization
250 | device_response = await self.request_device_authorization()
251 | if not device_response:
252 | return False
253 |
254 | # Step 2: Display user instructions
255 | self.display_user_instructions(device_response)
256 |
257 | # Step 3: Poll for token
258 | device_code = device_response["device_code"]
259 | interval = device_response.get("interval", 5)
260 |
261 | tokens = await self.poll_for_token(device_code, interval)
262 | if not tokens:
263 | return False
264 |
265 | # Step 4: Save tokens
266 | self.save_tokens(tokens)
267 |
268 | console.print("\n[green]Successfully authenticated with Basic Memory Cloud![/green]")
269 | return True
270 |
271 | def logout(self) -> None:
272 | """Remove stored authentication tokens."""
273 | if self.token_file.exists():
274 | self.token_file.unlink()
275 | console.print("[green]Logged out successfully[/green]")
276 | else:
277 | console.print("[yellow]No stored authentication found[/yellow]")
278 |
```
--------------------------------------------------------------------------------
/v15-docs/gitignore-integration.md:
--------------------------------------------------------------------------------
```markdown
1 | # .gitignore Integration
2 |
3 | **Status**: New Feature
4 | **PR**: #314
5 | **Impact**: Improved security and reduced noise
6 |
7 | ## What's New
8 |
9 | v0.15.0 integrates `.gitignore` support into the sync process. Files matching patterns in `.gitignore` are automatically skipped during synchronization, preventing sensitive files and build artifacts from being indexed.
10 |
11 | ## How It Works
12 |
13 | ### Ignore Pattern Sources
14 |
15 | Basic Memory combines patterns from two sources:
16 |
17 | 1. **Global user patterns**: `~/.basic-memory/.bmignore`
18 | - User's personal ignore patterns
19 | - Applied to all projects
20 | - Useful for global exclusions (OS files, editor configs)
21 |
22 | 2. **Project-specific patterns**: `{project}/.gitignore`
23 | - Project's standard gitignore file
24 | - Applied to that project only
25 | - Follows standard gitignore syntax
26 |
27 | ### Automatic .gitignore Respect
28 |
29 | When syncing, Basic Memory:
30 | 1. Loads patterns from `~/.basic-memory/.bmignore` (if exists)
31 | 2. Loads patterns from `.gitignore` in project root (if exists)
32 | 3. Combines both pattern sets
33 | 4. Skips files matching any pattern
34 | 5. Does not index ignored files
35 |
36 | ### Pattern Matching
37 |
38 | Uses standard gitignore syntax:
39 | ```gitignore
40 | # Comments are ignored
41 | *.log # Ignore all .log files
42 | build/ # Ignore build directory
43 | node_modules/ # Ignore node_modules
44 | .env # Ignore .env files
45 | !important.log # Exception: don't ignore this file
46 | ```
47 |
48 | ## Benefits
49 |
50 | ### 1. Security
51 |
52 | **Prevents indexing sensitive files:**
53 | ```gitignore
54 | # Sensitive files automatically skipped
55 | .env
56 | .env.*
57 | secrets.json
58 | credentials/
59 | *.key
60 | *.pem
61 | cloud-auth.json
62 | ```
63 |
64 | **Result:** Secrets never indexed or synced
65 |
66 | ### 2. Performance
67 |
68 | **Skips unnecessary files:**
69 | ```gitignore
70 | # Build artifacts and caches
71 | node_modules/
72 | __pycache__/
73 | .pytest_cache/
74 | dist/
75 | build/
76 | *.pyc
77 | ```
78 |
79 | **Result:** Faster sync, smaller database
80 |
81 | ### 3. Reduced Noise
82 |
83 | **Ignores OS and editor files:**
84 | ```gitignore
85 | # macOS
86 | .DS_Store
87 | .AppleDouble
88 |
89 | # Linux
90 | *~
91 | .directory
92 |
93 | # Windows
94 | Thumbs.db
95 | desktop.ini
96 |
97 | # Editors
98 | .vscode/
99 | .idea/
100 | *.swp
101 | ```
102 |
103 | **Result:** Cleaner knowledge base
104 |
105 | ## Setup
106 |
107 | ### Default Behavior
108 |
109 | If no `.gitignore` exists, Basic Memory uses built-in patterns:
110 |
111 | ```gitignore
112 | # Default patterns
113 | .git
114 | .DS_Store
115 | node_modules
116 | __pycache__
117 | .pytest_cache
118 | .env
119 | ```
120 |
121 | ### Global .bmignore (Optional)
122 |
123 | Create global ignore patterns for all projects:
124 |
125 | ```bash
126 | # Create global ignore file
127 | cat > ~/.basic-memory/.bmignore <<'EOF'
128 | # OS files (apply to all projects)
129 | .DS_Store
130 | .AppleDouble
131 | Thumbs.db
132 | desktop.ini
133 | *~
134 |
135 | # Editor files (apply to all projects)
136 | .vscode/
137 | .idea/
138 | *.swp
139 | *.swo
140 |
141 | # Always ignore these
142 | .env
143 | .env.*
144 | *.secret
145 | EOF
146 | ```
147 |
148 | **Use cases:**
149 | - Personal preferences (editor configs)
150 | - OS-specific files
151 | - Global security rules
152 |
153 | ### Project-Specific .gitignore
154 |
155 | Create `.gitignore` in project root for project-specific patterns:
156 |
157 | ```bash
158 | # Create .gitignore
159 | cat > ~/basic-memory/.gitignore <<'EOF'
160 | # Project-specific secrets
161 | credentials.json
162 | *.key
163 |
164 | # Project build artifacts
165 | dist/
166 | build/
167 | *.pyc
168 | __pycache__/
169 | node_modules/
170 |
171 | # Project-specific temp files
172 | *.tmp
173 | *.cache
174 | EOF
175 | ```
176 |
177 | **Use cases:**
178 | - Build artifacts
179 | - Dependencies (node_modules, venv)
180 | - Project-specific secrets
181 |
182 | ### Sync with .gitignore and .bmignore
183 |
184 | ```bash
185 | # Sync respects both .bmignore and .gitignore
186 | bm sync
187 |
188 | # Ignored files are skipped
189 | # → ".DS_Store skipped (global .bmignore)"
190 | # → ".env skipped (gitignored)"
191 | # → "node_modules/ skipped (gitignored)"
192 | ```
193 |
194 | **Pattern precedence:**
195 | 1. Global `.bmignore` patterns checked first
196 | 2. Project `.gitignore` patterns checked second
197 | 3. If either matches, file is skipped
198 |
199 | ## Use Cases
200 |
201 | ### Git Repository as Knowledge Base
202 |
203 | Perfect synergy when using git for version control:
204 |
205 | ```bash
206 | # Project structure
207 | ~/my-knowledge/
208 | ├── .git/ # ← git repo
209 | ├── .gitignore # ← shared ignore rules
210 | ├── notes/
211 | │ ├── public.md # ← synced
212 | │ └── private.md # ← synced
213 | ├── .env # ← ignored by git AND sync
214 | └── build/ # ← ignored by git AND sync
215 | ```
216 |
217 | **Benefits:**
218 | - Same ignore rules for git and sync
219 | - Consistent behavior
220 | - No sensitive files in either system
221 |
222 | ### Sensitive Information
223 |
224 | ```gitignore
225 | # .gitignore
226 | *.key
227 | *.pem
228 | credentials.json
229 | secrets/
230 | .env*
231 | ```
232 |
233 | **Result:**
234 | ```bash
235 | $ bm sync
236 | Syncing...
237 | → Skipped: api-key.pem (gitignored)
238 | → Skipped: .env (gitignored)
239 | → Skipped: secrets/passwords.txt (gitignored)
240 | ✓ Synced 15 files (3 skipped)
241 | ```
242 |
243 | ### Development Environment
244 |
245 | ```gitignore
246 | # Project-specific
247 | node_modules/
248 | venv/
249 | .venv/
250 | __pycache__/
251 | *.pyc
252 | .pytest_cache/
253 | .coverage
254 | .tox/
255 | dist/
256 | build/
257 | *.egg-info/
258 | ```
259 |
260 | **Result:** Clean knowledge base without dev noise
261 |
262 | ## Pattern Examples
263 |
264 | ### Common Patterns
265 |
266 | **Secrets:**
267 | ```gitignore
268 | .env
269 | .env.*
270 | *.key
271 | *.pem
272 | *secret*
273 | *password*
274 | credentials.json
275 | auth.json
276 | ```
277 |
278 | **Build Artifacts:**
279 | ```gitignore
280 | dist/
281 | build/
282 | *.o
283 | *.pyc
284 | *.class
285 | *.jar
286 | node_modules/
287 | __pycache__/
288 | ```
289 |
290 | **OS Files:**
291 | ```gitignore
292 | .DS_Store
293 | .AppleDouble
294 | .LSOverride
295 | Thumbs.db
296 | desktop.ini
297 | *~
298 | ```
299 |
300 | **Editors:**
301 | ```gitignore
302 | .vscode/
303 | .idea/
304 | *.swp
305 | *.swo
306 | *~
307 | .project
308 | .settings/
309 | ```
310 |
311 | ### Advanced Patterns
312 |
313 | **Exceptions (!):**
314 | ```gitignore
315 | # Ignore all logs
316 | *.log
317 |
318 | # EXCEPT this one
319 | !important.log
320 | ```
321 |
322 | **Directory-specific:**
323 | ```gitignore
324 | # Ignore only in root
325 | /.env
326 |
327 | # Ignore everywhere
328 | **/.env
329 | ```
330 |
331 | **Wildcards:**
332 | ```gitignore
333 | # Multiple extensions
334 | *.{log,tmp,cache}
335 |
336 | # Specific patterns
337 | test_*.py
338 | *_backup.*
339 | ```
340 |
341 | ## Integration with Cloud Sync
342 |
343 | ### .bmignore Files Overview
344 |
345 | Basic Memory uses `.bmignore` in two contexts:
346 |
347 | 1. **Global user patterns**: `~/.basic-memory/.bmignore`
348 | - Used for **local sync**
349 | - Standard gitignore syntax
350 | - Applied to all projects
351 |
352 | 2. **Cloud bisync filters**: `.bmignore.rclone`
353 | - Used for **cloud sync**
354 | - rclone filter format
355 | - Auto-generated from .gitignore patterns
356 |
357 | ### Automatic Pattern Conversion
358 |
359 | Cloud bisync converts .gitignore to rclone filter format:
360 |
361 | ```bash
362 | # Source: .gitignore (standard gitignore syntax)
363 | node_modules/
364 | *.log
365 | .env
366 |
367 | # Generated: .bmignore.rclone (rclone filter format)
368 | - node_modules/**
369 | - *.log
370 | - .env
371 | ```
372 |
373 | **Automatic conversion:** Basic Memory handles conversion during cloud sync
374 |
375 | ### Sync Workflow
376 |
377 | 1. **Local sync** (respects .bmignore + .gitignore)
378 | ```bash
379 | bm sync
380 | # → Loads ~/.basic-memory/.bmignore (global)
381 | # → Loads {project}/.gitignore (project-specific)
382 | # → Skips files matching either
383 | ```
384 |
385 | 2. **Cloud bisync** (respects .bmignore.rclone)
386 | ```bash
387 | bm cloud bisync
388 | # → Generates .bmignore.rclone from .gitignore
389 | # → Uses rclone filters for cloud sync
390 | # → Skips same files as local sync
391 | ```
392 |
393 | **Result:** Consistent ignore behavior across local and cloud sync
394 |
395 | ## Verification
396 |
397 | ### Check What's Ignored
398 |
399 | ```bash
400 | # Dry-run sync to see what's skipped
401 | bm sync --dry-run
402 |
403 | # Output shows:
404 | # → Syncing: notes/ideas.md
405 | # → Skipped: .env (gitignored)
406 | # → Skipped: node_modules/package.json (gitignored)
407 | ```
408 |
409 | ### List Ignore Patterns
410 |
411 | ```bash
412 | # View .gitignore
413 | cat .gitignore
414 |
415 | # View effective patterns
416 | bm sync --show-patterns
417 | ```
418 |
419 | ### Test Pattern Matching
420 |
421 | ```bash
422 | # Check if file matches pattern
423 | git check-ignore -v path/to/file
424 |
425 | # Example:
426 | git check-ignore -v .env
427 | # → .gitignore:5:.env .env
428 | ```
429 |
430 | ## Migration
431 |
432 | ### From v0.14.x
433 |
434 | **Before v0.15.0:**
435 | - .gitignore patterns not respected
436 | - All files synced, including ignored ones
437 | - Manual exclude rules needed
438 |
439 | **v0.15.0+:**
440 | - .gitignore automatically respected
441 | - Ignored files skipped
442 | - No manual configuration needed
443 |
444 | **Action:** Just add/update .gitignore - next sync uses it
445 |
446 | ### Cleaning Up Already-Indexed Files
447 |
448 | If ignored files were previously synced:
449 |
450 | ```bash
451 | # Option 1: Re-sync (re-indexes from scratch)
452 | bm sync --force-resync
453 |
454 | # Option 2: Delete and re-sync specific project
455 | bm project remove old-project
456 | bm project add clean-project ~/basic-memory
457 | bm sync --project clean-project
458 | ```
459 |
460 | ## Troubleshooting
461 |
462 | ### File Not Being Ignored
463 |
464 | **Problem:** File still synced despite being in .gitignore
465 |
466 | **Check:**
467 | 1. Is .gitignore in project root?
468 | ```bash
469 | ls -la ~/basic-memory/.gitignore
470 | ```
471 |
472 | 2. Is pattern correct?
473 | ```bash
474 | # Test pattern
475 | git check-ignore -v path/to/file
476 | ```
477 |
478 | 3. Is file already indexed?
479 | ```bash
480 | # Force resync
481 | bm sync --force-resync
482 | ```
483 |
484 | ### Pattern Not Matching
485 |
486 | **Problem:** Pattern doesn't match expected files
487 |
488 | **Common issues:**
489 | ```gitignore
490 | # ✗ Wrong: Won't match subdirectories
491 | node_modules
492 |
493 | # ✓ Correct: Matches recursively
494 | node_modules/
495 | **/node_modules/
496 |
497 | # ✗ Wrong: Only matches in root
498 | /.env
499 |
500 | # ✓ Correct: Matches everywhere
501 | .env
502 | **/.env
503 | ```
504 |
505 | ### .gitignore Not Found
506 |
507 | **Problem:** No .gitignore file exists
508 |
509 | **Solution:**
510 | ```bash
511 | # Create default .gitignore
512 | cat > ~/basic-memory/.gitignore <<'EOF'
513 | .git
514 | .DS_Store
515 | .env
516 | node_modules/
517 | __pycache__/
518 | EOF
519 |
520 | # Re-sync
521 | bm sync
522 | ```
523 |
524 | ## Best Practices
525 |
526 | ### 1. Use Global .bmignore for Personal Preferences
527 |
528 | Set global patterns once, apply to all projects:
529 |
530 | ```bash
531 | # Create global ignore file
532 | cat > ~/.basic-memory/.bmignore <<'EOF'
533 | # Personal editor/OS preferences
534 | .DS_Store
535 | .vscode/
536 | .idea/
537 | *.swp
538 |
539 | # Never sync these anywhere
540 | .env
541 | .env.*
542 | EOF
543 | ```
544 |
545 | ### 2. Use .gitignore for Project-Specific Patterns
546 |
547 | Even if not using git, create .gitignore for project-specific sync:
548 |
549 | ```bash
550 | # Create project .gitignore
551 | cat > .gitignore <<'EOF'
552 | # Project build artifacts
553 | dist/
554 | node_modules/
555 | __pycache__/
556 |
557 | # Project secrets
558 | credentials.json
559 | *.key
560 | EOF
561 | ```
562 |
563 | ### 3. Ignore Secrets First
564 |
565 | Start with security (both global and project-specific):
566 | ```bash
567 | # Global: ~/.basic-memory/.bmignore
568 | .env*
569 | *.key
570 | *.pem
571 |
572 | # Project: .gitignore
573 | credentials.json
574 | secrets/
575 | api-keys.txt
576 | ```
577 |
578 | ### 4. Ignore Build Artifacts
579 |
580 | Reduce noise in project .gitignore:
581 | ```gitignore
582 | # Build outputs
583 | dist/
584 | build/
585 | node_modules/
586 | __pycache__/
587 | *.pyc
588 | ```
589 |
590 | ### 5. Use Standard Templates
591 |
592 | Start with community templates for .gitignore:
593 | - [GitHub .gitignore templates](https://github.com/github/gitignore)
594 | - Language-specific ignores (Python, Node, etc.)
595 | - Framework-specific ignores
596 |
597 | ### 6. Test Your Patterns
598 |
599 | ```bash
600 | # Verify pattern works
601 | git check-ignore -v file.log
602 |
603 | # Test sync
604 | bm sync --dry-run
605 | ```
606 |
607 | ## See Also
608 |
609 | - `cloud-bisync.md` - Cloud sync and .bmignore.rclone conversion
610 | - `env-file-removal.md` - Why .env files should be ignored
611 | - gitignore documentation: https://git-scm.com/docs/gitignore
612 | - GitHub gitignore templates: https://github.com/github/gitignore
613 |
614 | ## Summary
615 |
616 | Basic Memory provides flexible ignore patterns through:
617 | - **Global**: `~/.basic-memory/.bmignore` - personal preferences across all projects
618 | - **Project**: `.gitignore` - project-specific patterns
619 | - **Cloud**: `.bmignore.rclone` - auto-generated for cloud sync
620 |
621 | Use global .bmignore for OS/editor files, project .gitignore for build artifacts and secrets.
622 |
```
--------------------------------------------------------------------------------
/tests/cli/test_ignore_utils.py:
--------------------------------------------------------------------------------
```python
1 | """Tests for ignore_utils module."""
2 |
3 | import tempfile
4 | from pathlib import Path
5 |
6 | from basic_memory.ignore_utils import (
7 | DEFAULT_IGNORE_PATTERNS,
8 | load_gitignore_patterns,
9 | should_ignore_path,
10 | filter_files,
11 | )
12 |
13 |
14 | def test_load_default_patterns_only():
15 | """Test loading default patterns when no .gitignore exists."""
16 | with tempfile.TemporaryDirectory() as temp_dir:
17 | temp_path = Path(temp_dir)
18 | patterns = load_gitignore_patterns(temp_path)
19 |
20 | # Should include all default patterns
21 | assert DEFAULT_IGNORE_PATTERNS.issubset(patterns)
22 | # Should only have default patterns (no custom ones)
23 | assert patterns == DEFAULT_IGNORE_PATTERNS
24 |
25 |
26 | def test_load_patterns_with_gitignore():
27 | """Test loading patterns from .gitignore file."""
28 | with tempfile.TemporaryDirectory() as temp_dir:
29 | temp_path = Path(temp_dir)
30 |
31 | # Create a .gitignore file
32 | gitignore_content = """
33 | # Python
34 | *.pyc
35 | __pycache__/
36 |
37 | # Node
38 | node_modules/
39 | *.log
40 |
41 | # Custom
42 | secrets/
43 | temp_*
44 | """
45 | (temp_path / ".gitignore").write_text(gitignore_content)
46 |
47 | patterns = load_gitignore_patterns(temp_path)
48 |
49 | # Should include default patterns
50 | assert DEFAULT_IGNORE_PATTERNS.issubset(patterns)
51 |
52 | # Should include custom patterns from .gitignore
53 | assert "*.pyc" in patterns
54 | assert "__pycache__/" in patterns
55 | assert "node_modules/" in patterns
56 | assert "*.log" in patterns
57 | assert "secrets/" in patterns
58 | assert "temp_*" in patterns
59 |
60 | # Should skip comments and empty lines
61 | assert "# Python" not in patterns
62 | assert "# Node" not in patterns
63 | assert "# Custom" not in patterns
64 |
65 |
66 | def test_load_patterns_empty_gitignore():
67 | """Test loading patterns with empty .gitignore file."""
68 | with tempfile.TemporaryDirectory() as temp_dir:
69 | temp_path = Path(temp_dir)
70 |
71 | # Create empty .gitignore file
72 | (temp_path / ".gitignore").write_text("")
73 |
74 | patterns = load_gitignore_patterns(temp_path)
75 |
76 | # Should only have default patterns
77 | assert patterns == DEFAULT_IGNORE_PATTERNS
78 |
79 |
80 | def test_load_patterns_unreadable_gitignore():
81 | """Test graceful handling of unreadable .gitignore file."""
82 | with tempfile.TemporaryDirectory() as temp_dir:
83 | temp_path = Path(temp_dir)
84 |
85 | # Create .gitignore file with restricted permissions
86 | gitignore_file = temp_path / ".gitignore"
87 | gitignore_file.write_text("*.log")
88 | gitignore_file.chmod(0o000) # No read permissions
89 |
90 | try:
91 | patterns = load_gitignore_patterns(temp_path)
92 |
93 | # On Windows, chmod might not work as expected, so we need to check
94 | # if the file is actually unreadable
95 | try:
96 | with gitignore_file.open("r"):
97 | pass
98 | # If we can read it, the test environment doesn't support this scenario
99 | # In this case, the patterns should include *.log
100 | assert "*.log" in patterns
101 | except (PermissionError, OSError):
102 | # File is actually unreadable, should fallback to default patterns only
103 | assert patterns == DEFAULT_IGNORE_PATTERNS
104 | assert "*.log" not in patterns
105 | finally:
106 | # Restore permissions for cleanup
107 | gitignore_file.chmod(0o644)
108 |
109 |
110 | def test_should_ignore_default_patterns():
111 | """Test ignoring files matching default patterns."""
112 | with tempfile.TemporaryDirectory() as temp_dir:
113 | temp_path = Path(temp_dir)
114 |
115 | patterns = DEFAULT_IGNORE_PATTERNS
116 |
117 | test_cases = [
118 | # Git directory
119 | (temp_path / ".git" / "config", True),
120 | # Python artifacts
121 | (temp_path / "main.pyc", True),
122 | (temp_path / "__pycache__" / "main.cpython-39.pyc", True),
123 | (temp_path / "src" / "__pycache__" / "module.pyc", True),
124 | # Virtual environments
125 | (temp_path / ".venv" / "lib" / "python.so", True),
126 | (temp_path / "venv" / "bin" / "python", True),
127 | (temp_path / "env" / "lib" / "site-packages", True),
128 | # Node.js
129 | (temp_path / "node_modules" / "package" / "index.js", True),
130 | # IDE files
131 | (temp_path / ".idea" / "workspace.xml", True),
132 | (temp_path / ".vscode" / "settings.json", True),
133 | # OS files
134 | (temp_path / ".DS_Store", True),
135 | (temp_path / "Thumbs.db", True),
136 | # Valid files that should NOT be ignored
137 | (temp_path / "main.py", False),
138 | (temp_path / "README.md", False),
139 | (temp_path / "src" / "module.py", False),
140 | (temp_path / "package.json", False),
141 | ]
142 |
143 | for file_path, should_be_ignored in test_cases:
144 | result = should_ignore_path(file_path, temp_path, patterns)
145 | assert result == should_be_ignored, (
146 | f"Failed for {file_path}: expected {should_be_ignored}, got {result}"
147 | )
148 |
149 |
150 | def test_should_ignore_glob_patterns():
151 | """Test glob pattern matching."""
152 | with tempfile.TemporaryDirectory() as temp_dir:
153 | temp_path = Path(temp_dir)
154 |
155 | patterns = {"*.log", "temp_*", "test*.txt"}
156 |
157 | test_cases = [
158 | (temp_path / "debug.log", True),
159 | (temp_path / "app.log", True),
160 | (temp_path / "sub" / "error.log", True),
161 | (temp_path / "temp_file.txt", True),
162 | (temp_path / "temp_123", True),
163 | (temp_path / "test_data.txt", True),
164 | (temp_path / "testfile.txt", True),
165 | (temp_path / "app.txt", False),
166 | (temp_path / "file.py", False),
167 | (temp_path / "data.json", False),
168 | ]
169 |
170 | for file_path, should_be_ignored in test_cases:
171 | result = should_ignore_path(file_path, temp_path, patterns)
172 | assert result == should_be_ignored, f"Failed for {file_path}"
173 |
174 |
175 | def test_should_ignore_directory_patterns():
176 | """Test directory pattern matching (ending with /)."""
177 | with tempfile.TemporaryDirectory() as temp_dir:
178 | temp_path = Path(temp_dir)
179 |
180 | patterns = {"build/", "dist/", "logs/"}
181 |
182 | test_cases = [
183 | (temp_path / "build" / "output.js", True),
184 | (temp_path / "dist" / "main.css", True),
185 | (temp_path / "logs" / "app.log", True),
186 | (temp_path / "src" / "build" / "file.js", True), # Nested
187 | (temp_path / "build.py", False), # File with same name
188 | (temp_path / "build_script.sh", False), # Similar name
189 | (temp_path / "src" / "main.py", False), # Different directory
190 | ]
191 |
192 | for file_path, should_be_ignored in test_cases:
193 | result = should_ignore_path(file_path, temp_path, patterns)
194 | assert result == should_be_ignored, f"Failed for {file_path}"
195 |
196 |
197 | def test_should_ignore_root_relative_patterns():
198 | """Test patterns starting with / (root relative)."""
199 | with tempfile.TemporaryDirectory() as temp_dir:
200 | temp_path = Path(temp_dir)
201 |
202 | patterns = {"/config.txt", "/build/", "/tmp/*.log"}
203 |
204 | test_cases = [
205 | (temp_path / "config.txt", True), # Root level
206 | (temp_path / "build" / "app.js", True), # Root level directory
207 | (temp_path / "tmp" / "debug.log", True), # Root level with glob
208 | (temp_path / "src" / "config.txt", False), # Not at root
209 | (temp_path / "project" / "build" / "file.js", False), # Not at root
210 | (temp_path / "data" / "tmp" / "app.log", False), # Not at root
211 | ]
212 |
213 | for file_path, should_be_ignored in test_cases:
214 | result = should_ignore_path(file_path, temp_path, patterns)
215 | assert result == should_be_ignored, f"Failed for {file_path}"
216 |
217 |
218 | def test_should_ignore_invalid_relative_path():
219 | """Test handling of paths that cannot be made relative to base."""
220 | patterns = {"*.pyc"}
221 |
222 | # File outside of base path should not be ignored
223 | base_path = Path("/tmp/project")
224 | file_path = Path("/home/user/file.pyc")
225 |
226 | result = should_ignore_path(file_path, base_path, patterns)
227 | assert result is False
228 |
229 |
230 | def test_filter_files_with_patterns():
231 | """Test filtering files with given patterns."""
232 | with tempfile.TemporaryDirectory() as temp_dir:
233 | temp_path = Path(temp_dir)
234 |
235 | # Create test files
236 | files = [
237 | temp_path / "main.py",
238 | temp_path / "main.pyc",
239 | temp_path / "__pycache__" / "module.pyc",
240 | temp_path / "README.md",
241 | temp_path / ".git" / "config",
242 | temp_path / "package.json",
243 | ]
244 |
245 | # Ensure parent directories exist
246 | for file_path in files:
247 | file_path.parent.mkdir(parents=True, exist_ok=True)
248 | file_path.write_text("test content")
249 |
250 | patterns = {"*.pyc", "__pycache__", ".git"}
251 | filtered_files, ignored_count = filter_files(files, temp_path, patterns)
252 |
253 | # Should keep valid files
254 | expected_kept = [
255 | temp_path / "main.py",
256 | temp_path / "README.md",
257 | temp_path / "package.json",
258 | ]
259 |
260 | assert len(filtered_files) == 3
261 | assert set(filtered_files) == set(expected_kept)
262 | assert ignored_count == 3 # main.pyc, module.pyc, config
263 |
264 |
265 | def test_filter_files_no_patterns():
266 | """Test filtering with no patterns (should keep all files)."""
267 | with tempfile.TemporaryDirectory() as temp_dir:
268 | temp_path = Path(temp_dir)
269 |
270 | files = [
271 | temp_path / "main.py",
272 | temp_path / "main.pyc",
273 | temp_path / "README.md",
274 | ]
275 |
276 | patterns = set()
277 | filtered_files, ignored_count = filter_files(files, temp_path, patterns)
278 |
279 | assert len(filtered_files) == 3
280 | assert set(filtered_files) == set(files)
281 | assert ignored_count == 0
282 |
283 |
284 | def test_filter_files_with_gitignore_loading():
285 | """Test filtering with automatic .gitignore loading."""
286 | with tempfile.TemporaryDirectory() as temp_dir:
287 | temp_path = Path(temp_dir)
288 |
289 | # Create .gitignore
290 | gitignore_content = """
291 | *.log
292 | temp_*
293 | """
294 | (temp_path / ".gitignore").write_text(gitignore_content)
295 |
296 | # Create test files
297 | files = [
298 | temp_path / "app.py",
299 | temp_path / "debug.log",
300 | temp_path / "temp_file.txt",
301 | temp_path / "README.md",
302 | ]
303 |
304 | # Ensure files exist
305 | for file_path in files:
306 | file_path.write_text("test content")
307 |
308 | filtered_files, ignored_count = filter_files(files, temp_path) # patterns=None
309 |
310 | # Should ignore .log files and temp_* files, plus default patterns
311 | expected_kept = [temp_path / "app.py", temp_path / "README.md"]
312 |
313 | assert len(filtered_files) == 2
314 | assert set(filtered_files) == set(expected_kept)
315 | assert ignored_count == 2 # debug.log, temp_file.txt
316 |
```
--------------------------------------------------------------------------------
/tests/test_rclone_commands.py:
--------------------------------------------------------------------------------
```python
1 | """Test project-scoped rclone commands."""
2 |
3 | from pathlib import Path
4 | from unittest.mock import MagicMock, patch
5 |
6 | import pytest
7 |
8 | from basic_memory.cli.commands.cloud.rclone_commands import (
9 | RcloneError,
10 | SyncProject,
11 | bisync_initialized,
12 | get_project_bisync_state,
13 | get_project_remote,
14 | project_bisync,
15 | project_check,
16 | project_ls,
17 | project_sync,
18 | )
19 |
20 |
21 | def test_sync_project_dataclass():
22 | """Test SyncProject dataclass."""
23 | project = SyncProject(
24 | name="research",
25 | path="app/data/research",
26 | local_sync_path="/Users/test/research",
27 | )
28 |
29 | assert project.name == "research"
30 | assert project.path == "app/data/research"
31 | assert project.local_sync_path == "/Users/test/research"
32 |
33 |
34 | def test_sync_project_optional_local_path():
35 | """Test SyncProject with optional local_sync_path."""
36 | project = SyncProject(
37 | name="research",
38 | path="app/data/research",
39 | )
40 |
41 | assert project.name == "research"
42 | assert project.path == "app/data/research"
43 | assert project.local_sync_path is None
44 |
45 |
46 | def test_get_project_remote():
47 | """Test building rclone remote path with normalized path."""
48 | # Path comes from API already normalized (no /app/data/ prefix)
49 | project = SyncProject(name="research", path="/research")
50 |
51 | remote = get_project_remote(project, "my-bucket")
52 |
53 | assert remote == "basic-memory-cloud:my-bucket/research"
54 |
55 |
56 | def test_get_project_remote_strips_app_data_prefix():
57 | """Test that /app/data/ prefix is stripped from cloud path."""
58 | # If API returns path with /app/data/, it should be stripped
59 | project = SyncProject(name="research", path="/app/data/research")
60 |
61 | remote = get_project_remote(project, "my-bucket")
62 |
63 | # Should strip /app/data/ prefix to get actual S3 path
64 | assert remote == "basic-memory-cloud:my-bucket/research"
65 |
66 |
67 | def test_get_project_bisync_state():
68 | """Test getting bisync state directory path."""
69 | state_path = get_project_bisync_state("research")
70 |
71 | expected = Path.home() / ".basic-memory" / "bisync-state" / "research"
72 | assert state_path == expected
73 |
74 |
75 | def test_bisync_initialized_false_when_not_exists(tmp_path, monkeypatch):
76 | """Test bisync_initialized returns False when state doesn't exist."""
77 | # Patch to use tmp directory
78 | monkeypatch.setattr(
79 | "basic_memory.cli.commands.cloud.rclone_commands.get_project_bisync_state",
80 | lambda project_name: tmp_path / project_name,
81 | )
82 |
83 | assert bisync_initialized("research") is False
84 |
85 |
86 | def test_bisync_initialized_false_when_empty(tmp_path, monkeypatch):
87 | """Test bisync_initialized returns False when state directory is empty."""
88 | state_dir = tmp_path / "research"
89 | state_dir.mkdir()
90 |
91 | monkeypatch.setattr(
92 | "basic_memory.cli.commands.cloud.rclone_commands.get_project_bisync_state",
93 | lambda project_name: tmp_path / project_name,
94 | )
95 |
96 | assert bisync_initialized("research") is False
97 |
98 |
99 | def test_bisync_initialized_true_when_has_files(tmp_path, monkeypatch):
100 | """Test bisync_initialized returns True when state has files."""
101 | state_dir = tmp_path / "research"
102 | state_dir.mkdir()
103 | (state_dir / "state.lst").touch()
104 |
105 | monkeypatch.setattr(
106 | "basic_memory.cli.commands.cloud.rclone_commands.get_project_bisync_state",
107 | lambda project_name: tmp_path / project_name,
108 | )
109 |
110 | assert bisync_initialized("research") is True
111 |
112 |
113 | @patch("basic_memory.cli.commands.cloud.rclone_commands.subprocess.run")
114 | def test_project_sync_success(mock_run):
115 | """Test successful project sync."""
116 | mock_run.return_value = MagicMock(returncode=0)
117 |
118 | project = SyncProject(
119 | name="research",
120 | path="/research", # Normalized path from API
121 | local_sync_path="/tmp/research",
122 | )
123 |
124 | result = project_sync(project, "my-bucket", dry_run=True)
125 |
126 | assert result is True
127 | mock_run.assert_called_once()
128 |
129 | # Check command arguments
130 | cmd = mock_run.call_args[0][0]
131 | assert cmd[0] == "rclone"
132 | assert cmd[1] == "sync"
133 | # Use Path for cross-platform comparison (Windows uses backslashes)
134 | assert Path(cmd[2]) == Path("/tmp/research")
135 | assert cmd[3] == "basic-memory-cloud:my-bucket/research"
136 | assert "--dry-run" in cmd
137 |
138 |
139 | @patch("basic_memory.cli.commands.cloud.rclone_commands.subprocess.run")
140 | def test_project_sync_with_verbose(mock_run):
141 | """Test project sync with verbose flag."""
142 | mock_run.return_value = MagicMock(returncode=0)
143 |
144 | project = SyncProject(
145 | name="research",
146 | path="app/data/research",
147 | local_sync_path="/tmp/research",
148 | )
149 |
150 | project_sync(project, "my-bucket", verbose=True)
151 |
152 | cmd = mock_run.call_args[0][0]
153 | assert "--verbose" in cmd
154 | assert "--progress" not in cmd
155 |
156 |
157 | @patch("basic_memory.cli.commands.cloud.rclone_commands.subprocess.run")
158 | def test_project_sync_with_progress(mock_run):
159 | """Test project sync with progress (default)."""
160 | mock_run.return_value = MagicMock(returncode=0)
161 |
162 | project = SyncProject(
163 | name="research",
164 | path="app/data/research",
165 | local_sync_path="/tmp/research",
166 | )
167 |
168 | project_sync(project, "my-bucket")
169 |
170 | cmd = mock_run.call_args[0][0]
171 | assert "--progress" in cmd
172 | assert "--verbose" not in cmd
173 |
174 |
175 | def test_project_sync_no_local_path():
176 | """Test project sync raises error when local_sync_path not configured."""
177 | project = SyncProject(name="research", path="app/data/research")
178 |
179 | with pytest.raises(RcloneError) as exc_info:
180 | project_sync(project, "my-bucket")
181 |
182 | assert "no local_sync_path configured" in str(exc_info.value)
183 |
184 |
185 | @patch("basic_memory.cli.commands.cloud.rclone_commands.subprocess.run")
186 | @patch("basic_memory.cli.commands.cloud.rclone_commands.bisync_initialized")
187 | def test_project_bisync_success(mock_bisync_init, mock_run):
188 | """Test successful project bisync."""
189 | mock_bisync_init.return_value = True # Already initialized
190 | mock_run.return_value = MagicMock(returncode=0)
191 |
192 | project = SyncProject(
193 | name="research",
194 | path="app/data/research",
195 | local_sync_path="/tmp/research",
196 | )
197 |
198 | result = project_bisync(project, "my-bucket")
199 |
200 | assert result is True
201 | mock_run.assert_called_once()
202 |
203 | # Check command arguments
204 | cmd = mock_run.call_args[0][0]
205 | assert cmd[0] == "rclone"
206 | assert cmd[1] == "bisync"
207 | assert "--conflict-resolve=newer" in cmd
208 | assert "--max-delete=25" in cmd
209 | assert "--resilient" in cmd
210 |
211 |
212 | @patch("basic_memory.cli.commands.cloud.rclone_commands.subprocess.run")
213 | @patch("basic_memory.cli.commands.cloud.rclone_commands.bisync_initialized")
214 | def test_project_bisync_requires_resync_first_time(mock_bisync_init, mock_run):
215 | """Test that first bisync requires --resync flag."""
216 | mock_bisync_init.return_value = False # Not initialized
217 |
218 | project = SyncProject(
219 | name="research",
220 | path="app/data/research",
221 | local_sync_path="/tmp/research",
222 | )
223 |
224 | with pytest.raises(RcloneError) as exc_info:
225 | project_bisync(project, "my-bucket")
226 |
227 | assert "requires --resync" in str(exc_info.value)
228 | mock_run.assert_not_called()
229 |
230 |
231 | @patch("basic_memory.cli.commands.cloud.rclone_commands.subprocess.run")
232 | @patch("basic_memory.cli.commands.cloud.rclone_commands.bisync_initialized")
233 | def test_project_bisync_with_resync_flag(mock_bisync_init, mock_run):
234 | """Test bisync with --resync flag for first time."""
235 | mock_bisync_init.return_value = False # Not initialized
236 | mock_run.return_value = MagicMock(returncode=0)
237 |
238 | project = SyncProject(
239 | name="research",
240 | path="app/data/research",
241 | local_sync_path="/tmp/research",
242 | )
243 |
244 | result = project_bisync(project, "my-bucket", resync=True)
245 |
246 | assert result is True
247 | cmd = mock_run.call_args[0][0]
248 | assert "--resync" in cmd
249 |
250 |
251 | @patch("basic_memory.cli.commands.cloud.rclone_commands.subprocess.run")
252 | @patch("basic_memory.cli.commands.cloud.rclone_commands.bisync_initialized")
253 | def test_project_bisync_dry_run_skips_init_check(mock_bisync_init, mock_run):
254 | """Test that dry-run skips initialization check."""
255 | mock_bisync_init.return_value = False # Not initialized
256 | mock_run.return_value = MagicMock(returncode=0)
257 |
258 | project = SyncProject(
259 | name="research",
260 | path="app/data/research",
261 | local_sync_path="/tmp/research",
262 | )
263 |
264 | # Should not raise error even though not initialized
265 | result = project_bisync(project, "my-bucket", dry_run=True)
266 |
267 | assert result is True
268 | cmd = mock_run.call_args[0][0]
269 | assert "--dry-run" in cmd
270 |
271 |
272 | def test_project_bisync_no_local_path():
273 | """Test project bisync raises error when local_sync_path not configured."""
274 | project = SyncProject(name="research", path="app/data/research")
275 |
276 | with pytest.raises(RcloneError) as exc_info:
277 | project_bisync(project, "my-bucket")
278 |
279 | assert "no local_sync_path configured" in str(exc_info.value)
280 |
281 |
282 | @patch("basic_memory.cli.commands.cloud.rclone_commands.subprocess.run")
283 | def test_project_check_success(mock_run):
284 | """Test successful project check."""
285 | mock_run.return_value = MagicMock(returncode=0)
286 |
287 | project = SyncProject(
288 | name="research",
289 | path="app/data/research",
290 | local_sync_path="/tmp/research",
291 | )
292 |
293 | result = project_check(project, "my-bucket")
294 |
295 | assert result is True
296 | cmd = mock_run.call_args[0][0]
297 | assert cmd[0] == "rclone"
298 | assert cmd[1] == "check"
299 |
300 |
301 | @patch("basic_memory.cli.commands.cloud.rclone_commands.subprocess.run")
302 | def test_project_check_with_one_way(mock_run):
303 | """Test project check with one-way flag."""
304 | mock_run.return_value = MagicMock(returncode=0)
305 |
306 | project = SyncProject(
307 | name="research",
308 | path="app/data/research",
309 | local_sync_path="/tmp/research",
310 | )
311 |
312 | project_check(project, "my-bucket", one_way=True)
313 |
314 | cmd = mock_run.call_args[0][0]
315 | assert "--one-way" in cmd
316 |
317 |
318 | def test_project_check_no_local_path():
319 | """Test project check raises error when local_sync_path not configured."""
320 | project = SyncProject(name="research", path="app/data/research")
321 |
322 | with pytest.raises(RcloneError) as exc_info:
323 | project_check(project, "my-bucket")
324 |
325 | assert "no local_sync_path configured" in str(exc_info.value)
326 |
327 |
328 | @patch("basic_memory.cli.commands.cloud.rclone_commands.subprocess.run")
329 | def test_project_ls_success(mock_run):
330 | """Test successful project ls."""
331 | mock_run.return_value = MagicMock(returncode=0, stdout="file1.md\nfile2.md\nsubdir/file3.md\n")
332 |
333 | project = SyncProject(name="research", path="app/data/research")
334 |
335 | files = project_ls(project, "my-bucket")
336 |
337 | assert len(files) == 3
338 | assert "file1.md" in files
339 | assert "file2.md" in files
340 | assert "subdir/file3.md" in files
341 |
342 |
343 | @patch("basic_memory.cli.commands.cloud.rclone_commands.subprocess.run")
344 | def test_project_ls_with_subpath(mock_run):
345 | """Test project ls with subdirectory."""
346 | mock_run.return_value = MagicMock(returncode=0, stdout="")
347 |
348 | project = SyncProject(name="research", path="/research") # Normalized path
349 |
350 | project_ls(project, "my-bucket", path="subdir")
351 |
352 | cmd = mock_run.call_args[0][0]
353 | assert cmd[-1] == "basic-memory-cloud:my-bucket/research/subdir"
354 |
```
--------------------------------------------------------------------------------
/src/basic_memory/services/directory_service.py:
--------------------------------------------------------------------------------
```python
1 | """Directory service for managing file directories and tree structure."""
2 |
3 | import fnmatch
4 | import logging
5 | import os
6 | from typing import Dict, List, Optional, Sequence
7 |
8 | from basic_memory.models import Entity
9 | from basic_memory.repository import EntityRepository
10 | from basic_memory.schemas.directory import DirectoryNode
11 |
12 | logger = logging.getLogger(__name__)
13 |
14 |
15 | class DirectoryService:
16 | """Service for working with directory trees."""
17 |
18 | def __init__(self, entity_repository: EntityRepository):
19 | """Initialize the directory service.
20 |
21 | Args:
22 | entity_repository: Directory repository for data access.
23 | """
24 | self.entity_repository = entity_repository
25 |
26 | async def get_directory_tree(self) -> DirectoryNode:
27 | """Build a hierarchical directory tree from indexed files."""
28 |
29 | # Get all files from DB (flat list)
30 | entity_rows = await self.entity_repository.find_all()
31 |
32 | # Create a root directory node
33 | root_node = DirectoryNode(name="Root", directory_path="/", type="directory")
34 |
35 | # Map to store directory nodes by path for easy lookup
36 | dir_map: Dict[str, DirectoryNode] = {root_node.directory_path: root_node}
37 |
38 | # First pass: create all directory nodes
39 | for file in entity_rows:
40 | # Process directory path components
41 | parts = [p for p in file.file_path.split("/") if p]
42 |
43 | # Create directory structure
44 | current_path = "/"
45 | for i, part in enumerate(parts[:-1]): # Skip the filename
46 | parent_path = current_path
47 | # Build the directory path
48 | current_path = (
49 | f"{current_path}{part}" if current_path == "/" else f"{current_path}/{part}"
50 | )
51 |
52 | # Create directory node if it doesn't exist
53 | if current_path not in dir_map:
54 | dir_node = DirectoryNode(
55 | name=part, directory_path=current_path, type="directory"
56 | )
57 | dir_map[current_path] = dir_node
58 |
59 | # Add to parent's children
60 | if parent_path in dir_map:
61 | dir_map[parent_path].children.append(dir_node)
62 |
63 | # Second pass: add file nodes to their parent directories
64 | for file in entity_rows:
65 | file_name = os.path.basename(file.file_path)
66 | parent_dir = os.path.dirname(file.file_path)
67 | directory_path = "/" if parent_dir == "" else f"/{parent_dir}"
68 |
69 | # Create file node
70 | file_node = DirectoryNode(
71 | name=file_name,
72 | file_path=file.file_path, # Original path from DB (no leading slash)
73 | directory_path=f"/{file.file_path}", # Path with leading slash
74 | type="file",
75 | title=file.title,
76 | permalink=file.permalink,
77 | entity_id=file.id,
78 | entity_type=file.entity_type,
79 | content_type=file.content_type,
80 | updated_at=file.updated_at,
81 | )
82 |
83 | # Add to parent directory's children
84 | if directory_path in dir_map:
85 | dir_map[directory_path].children.append(file_node)
86 | else:
87 | # If parent directory doesn't exist (should be rare), add to root
88 | dir_map["/"].children.append(file_node) # pragma: no cover
89 |
90 | # Return the root node with its children
91 | return root_node
92 |
93 | async def get_directory_structure(self) -> DirectoryNode:
94 | """Build a hierarchical directory structure without file details.
95 |
96 | Optimized method for folder navigation that only returns directory nodes,
97 | no file metadata. Much faster than get_directory_tree() for large knowledge bases.
98 |
99 | Returns:
100 | DirectoryNode tree containing only folders (type="directory")
101 | """
102 | # Get unique directories without loading entities
103 | directories = await self.entity_repository.get_distinct_directories()
104 |
105 | # Create a root directory node
106 | root_node = DirectoryNode(name="Root", directory_path="/", type="directory")
107 |
108 | # Map to store directory nodes by path for easy lookup
109 | dir_map: Dict[str, DirectoryNode] = {"/": root_node}
110 |
111 | # Build tree with just folders
112 | for dir_path in directories:
113 | parts = [p for p in dir_path.split("/") if p]
114 | current_path = "/"
115 |
116 | for i, part in enumerate(parts):
117 | parent_path = current_path
118 | # Build the directory path
119 | current_path = (
120 | f"{current_path}{part}" if current_path == "/" else f"{current_path}/{part}"
121 | )
122 |
123 | # Create directory node if it doesn't exist
124 | if current_path not in dir_map:
125 | dir_node = DirectoryNode(
126 | name=part, directory_path=current_path, type="directory"
127 | )
128 | dir_map[current_path] = dir_node
129 |
130 | # Add to parent's children
131 | if parent_path in dir_map:
132 | dir_map[parent_path].children.append(dir_node)
133 |
134 | return root_node
135 |
136 | async def list_directory(
137 | self,
138 | dir_name: str = "/",
139 | depth: int = 1,
140 | file_name_glob: Optional[str] = None,
141 | ) -> List[DirectoryNode]:
142 | """List directory contents with filtering and depth control.
143 |
144 | Args:
145 | dir_name: Directory path to list (default: root "/")
146 | depth: Recursion depth (1 = immediate children only)
147 | file_name_glob: Glob pattern for filtering file names
148 |
149 | Returns:
150 | List of DirectoryNode objects matching the criteria
151 | """
152 | # Normalize directory path
153 | # Strip ./ prefix if present (handles relative path notation)
154 | if dir_name.startswith("./"):
155 | dir_name = dir_name[2:] # Remove "./" prefix
156 |
157 | # Ensure path starts with "/"
158 | if not dir_name.startswith("/"):
159 | dir_name = f"/{dir_name}"
160 |
161 | # Remove trailing slashes except for root
162 | if dir_name != "/" and dir_name.endswith("/"):
163 | dir_name = dir_name.rstrip("/")
164 |
165 | # Optimize: Query only entities in the target directory
166 | # instead of loading the entire tree
167 | dir_prefix = dir_name.lstrip("/")
168 | entity_rows = await self.entity_repository.find_by_directory_prefix(dir_prefix)
169 |
170 | # Build a partial tree from only the relevant entities
171 | root_tree = self._build_directory_tree_from_entities(entity_rows, dir_name)
172 |
173 | # Find the target directory node
174 | target_node = self._find_directory_node(root_tree, dir_name)
175 | if not target_node:
176 | return []
177 |
178 | # Collect nodes with depth and glob filtering
179 | result = []
180 | self._collect_nodes_recursive(target_node, result, depth, file_name_glob, 0)
181 |
182 | return result
183 |
184 | def _build_directory_tree_from_entities(
185 | self, entity_rows: Sequence[Entity], root_path: str
186 | ) -> DirectoryNode:
187 | """Build a directory tree from a subset of entities.
188 |
189 | Args:
190 | entity_rows: Sequence of entity objects to build tree from
191 | root_path: Root directory path for the tree
192 |
193 | Returns:
194 | DirectoryNode representing the tree root
195 | """
196 | # Create a root directory node
197 | root_node = DirectoryNode(name="Root", directory_path=root_path, type="directory")
198 |
199 | # Map to store directory nodes by path for easy lookup
200 | dir_map: Dict[str, DirectoryNode] = {root_path: root_node}
201 |
202 | # First pass: create all directory nodes
203 | for file in entity_rows:
204 | # Process directory path components
205 | parts = [p for p in file.file_path.split("/") if p]
206 |
207 | # Create directory structure
208 | current_path = "/"
209 | for i, part in enumerate(parts[:-1]): # Skip the filename
210 | parent_path = current_path
211 | # Build the directory path
212 | current_path = (
213 | f"{current_path}{part}" if current_path == "/" else f"{current_path}/{part}"
214 | )
215 |
216 | # Create directory node if it doesn't exist
217 | if current_path not in dir_map:
218 | dir_node = DirectoryNode(
219 | name=part, directory_path=current_path, type="directory"
220 | )
221 | dir_map[current_path] = dir_node
222 |
223 | # Add to parent's children
224 | if parent_path in dir_map:
225 | dir_map[parent_path].children.append(dir_node)
226 |
227 | # Second pass: add file nodes to their parent directories
228 | for file in entity_rows:
229 | file_name = os.path.basename(file.file_path)
230 | parent_dir = os.path.dirname(file.file_path)
231 | directory_path = "/" if parent_dir == "" else f"/{parent_dir}"
232 |
233 | # Create file node
234 | file_node = DirectoryNode(
235 | name=file_name,
236 | file_path=file.file_path,
237 | directory_path=f"/{file.file_path}",
238 | type="file",
239 | title=file.title,
240 | permalink=file.permalink,
241 | entity_id=file.id,
242 | entity_type=file.entity_type,
243 | content_type=file.content_type,
244 | updated_at=file.updated_at,
245 | )
246 |
247 | # Add to parent directory's children
248 | if directory_path in dir_map:
249 | dir_map[directory_path].children.append(file_node)
250 | elif root_path in dir_map:
251 | # Fallback to root if parent not found
252 | dir_map[root_path].children.append(file_node)
253 |
254 | return root_node
255 |
256 | def _find_directory_node(
257 | self, root: DirectoryNode, target_path: str
258 | ) -> Optional[DirectoryNode]:
259 | """Find a directory node by path in the tree."""
260 | if root.directory_path == target_path:
261 | return root
262 |
263 | for child in root.children:
264 | if child.type == "directory":
265 | found = self._find_directory_node(child, target_path)
266 | if found:
267 | return found
268 |
269 | return None
270 |
271 | def _collect_nodes_recursive(
272 | self,
273 | node: DirectoryNode,
274 | result: List[DirectoryNode],
275 | max_depth: int,
276 | file_name_glob: Optional[str],
277 | current_depth: int,
278 | ) -> None:
279 | """Recursively collect nodes with depth and glob filtering."""
280 | if current_depth >= max_depth:
281 | return
282 |
283 | for child in node.children:
284 | # Apply glob filtering
285 | if file_name_glob and not fnmatch.fnmatch(child.name, file_name_glob):
286 | continue
287 |
288 | # Add the child to results
289 | result.append(child)
290 |
291 | # Recurse into subdirectories if we haven't reached max depth
292 | if child.type == "directory" and current_depth < max_depth:
293 | self._collect_nodes_recursive(
294 | child, result, max_depth, file_name_glob, current_depth + 1
295 | )
296 |
```
--------------------------------------------------------------------------------
/v15-docs/cloud-bisync.md:
--------------------------------------------------------------------------------
```markdown
1 | # Cloud Bidirectional Sync (SPEC-9)
2 |
3 | **Status**: New Feature
4 | **PR**: #322
5 | **Requires**: Active subscription, rclone installation
6 |
7 | ## What's New
8 |
9 | v0.15.0 introduces **bidirectional cloud synchronization** using rclone bisync. Your local files sync automatically with the cloud, enabling multi-device workflows, backups, and collaboration.
10 |
11 | ## Quick Start
12 |
13 | ### One-Time Setup
14 |
15 | ```bash
16 | # Install and configure cloud sync
17 | bm cloud bisync-setup
18 |
19 | # What it does:
20 | # 1. Installs rclone
21 | # 2. Gets tenant credentials
22 | # 3. Configures rclone remote
23 | # 4. Creates sync directory
24 | # 5. Performs initial sync
25 | ```
26 |
27 | ### Regular Sync
28 |
29 | ```bash
30 | # Recommended: Use standard sync command
31 | bm sync # Syncs local → database
32 | bm cloud bisync # Syncs local ↔ cloud
33 |
34 | # Or: Use watch mode (auto-sync every 60 seconds)
35 | bm sync --watch
36 | ```
37 |
38 | ## How Bidirectional Sync Works
39 |
40 | ### Sync Architecture
41 |
42 | ```
43 | Local Files rclone bisync Cloud Storage
44 | ~/basic-memory- <─────────────> s3://bucket/
45 | cloud-sync/ (bidirectional) tenant-id/
46 | ├── project-a/ ├── project-a/
47 | ├── project-b/ ├── project-b/
48 | └── notes/ └── notes/
49 | ```
50 |
51 | ### Sync Profiles
52 |
53 | Three profiles optimize for different use cases:
54 |
55 | | Profile | Conflicts | Max Deletes | Speed | Use Case |
56 | |---------|-----------|-------------|-------|----------|
57 | | **safe** | Keep both versions | 10 | Slower | Preserve all changes, manual conflict resolution |
58 | | **balanced** | Use newer file | 25 | Medium | **Default** - auto-resolve most conflicts |
59 | | **fast** | Use newer file | 50 | Fastest | Rapid iteration, trust newer versions |
60 |
61 | ### Conflict Resolution
62 |
63 | **safe profile** (--conflict-resolve=none):
64 | - Conflicting files saved as `file.conflict1`, `file.conflict2`
65 | - Manual resolution required
66 | - No data loss
67 |
68 | **balanced/fast profiles** (--conflict-resolve=newer):
69 | - Automatically uses the newer file
70 | - Faster syncs
71 | - Good for single-user workflows
72 |
73 | ## Commands
74 |
75 | ### bm cloud bisync-setup
76 |
77 | One-time setup for cloud sync.
78 |
79 | ```bash
80 | bm cloud bisync-setup
81 |
82 | # Optional: Custom sync directory
83 | bm cloud bisync-setup --dir ~/my-sync-folder
84 | ```
85 |
86 | **What happens:**
87 | 1. Checks for/installs rclone
88 | 2. Generates scoped S3 credentials
89 | 3. Configures rclone remote
90 | 4. Creates local sync directory
91 | 5. Performs initial baseline sync (--resync)
92 |
93 | **Configuration saved to:**
94 | - `~/.basic-memory/config.json` - sync_dir path
95 | - `~/.config/rclone/rclone.conf` - remote credentials
96 | - `~/.basic-memory/bisync-state/{tenant_id}/` - sync state
97 |
98 | ### bm cloud bisync
99 |
100 | Manual bidirectional sync.
101 |
102 | ```bash
103 | # Basic sync (uses 'balanced' profile)
104 | bm cloud bisync
105 |
106 | # Choose sync profile
107 | bm cloud bisync --profile safe
108 | bm cloud bisync --profile balanced
109 | bm cloud bisync --profile fast
110 |
111 | # Dry run (preview changes)
112 | bm cloud bisync --dry-run
113 |
114 | # Force resync (rebuild baseline)
115 | bm cloud bisync --resync
116 |
117 | # Verbose output
118 | bm cloud bisync --verbose
119 | ```
120 |
121 | **Auto-registration:**
122 | - Scans local directory for new projects
123 | - Creates them on cloud before sync
124 | - Ensures cloud knows about all local projects
125 |
126 | ### bm sync (Recommended)
127 |
128 | The standard sync command now handles both local and cloud:
129 |
130 | ```bash
131 | # One command for everything
132 | bm sync # Local sync + cloud sync
133 | bm sync --watch # Continuous sync every 60s
134 | ```
135 |
136 | ## Sync Directory Structure
137 |
138 | ### Default Layout
139 |
140 | ```bash
141 | ~/basic-memory-cloud-sync/ # Configurable via --dir
142 | ├── project-a/ # Auto-created local projects
143 | │ ├── notes/
144 | │ ├── ideas/
145 | │ └── .bmignore # Respected during sync
146 | ├── project-b/
147 | │ └── documents/
148 | └── .basic-memory/ # Metadata (ignored in sync)
149 | ```
150 |
151 | ### Important Paths
152 |
153 | | Path | Purpose |
154 | |------|---------|
155 | | `~/basic-memory-cloud-sync/` | Default local sync directory |
156 | | `~/basic-memory-cloud/` | Mount point (DO NOT use for bisync) |
157 | | `~/.basic-memory/bisync-state/{tenant_id}/` | Sync state and history |
158 | | `~/.basic-memory/.bmignore` | Patterns to exclude from sync |
159 |
160 | **Critical:** Bisync and mount must use **different directories**
161 |
162 | ## File Filtering with .bmignore
163 |
164 | ### Default Patterns
165 |
166 | Basic Memory respects `.bmignore` patterns (gitignore format):
167 |
168 | ```bash
169 | # ~/.basic-memory/.bmignore (default)
170 | .git
171 | .DS_Store
172 | node_modules
173 | *.tmp
174 | .env
175 | __pycache__
176 | .pytest_cache
177 | .ruff_cache
178 | .vscode
179 | .idea
180 | ```
181 |
182 | ### How It Works
183 |
184 | 1. `.bmignore` patterns converted to rclone filter format
185 | 2. Auto-regenerated when `.bmignore` changes
186 | 3. Stored as `~/.basic-memory/.bmignore.rclone`
187 | 4. Applied to all bisync operations
188 |
189 | ### Custom Patterns
190 |
191 | Edit `~/.basic-memory/.bmignore`:
192 |
193 | ```bash
194 | # Your custom patterns
195 | .git
196 | *.log
197 | temp/
198 | *.backup
199 | ```
200 |
201 | Next sync will use updated filters.
202 |
203 | ## Project Management
204 |
205 | ### Auto-Registration
206 |
207 | Bisync automatically registers new local projects:
208 |
209 | ```bash
210 | # You create a new project locally
211 | mkdir ~/basic-memory-cloud-sync/new-project
212 | echo "# Hello" > ~/basic-memory-cloud-sync/new-project/README.md
213 |
214 | # Next sync auto-creates on cloud
215 | bm cloud bisync
216 | # → "Found 1 new local project, creating on cloud..."
217 | # → "✓ Created project: new-project"
218 | ```
219 |
220 | ### Project Discovery
221 |
222 | ```bash
223 | # List cloud projects
224 | bm cloud status
225 |
226 | # Shows:
227 | # - Total projects
228 | # - Last sync time
229 | # - Storage used
230 | ```
231 |
232 | ### Cloud Mode
233 |
234 | To work with cloud projects via CLI:
235 |
236 | ```bash
237 | # Set cloud API URL
238 | export BASIC_MEMORY_API_URL=https://api.basicmemory.cloud
239 |
240 | # Or in config.json:
241 | {
242 | "api_url": "https://api.basicmemory.cloud"
243 | }
244 |
245 | # Now CLI tools work against cloud
246 | bm sync --project new-project # Syncs cloud project
247 | bm tools continue-conversation --project new-project
248 | ```
249 |
250 | ## Sync Workflow Examples
251 |
252 | ### Daily Workflow
253 |
254 | ```bash
255 | # Morning: Start watch mode
256 | bm sync --watch &
257 |
258 | # Work in your sync directory
259 | cd ~/basic-memory-cloud-sync/work-notes
260 | vim ideas.md
261 |
262 | # Changes auto-sync every 60s
263 | # Watch output shows sync progress
264 | ```
265 |
266 | ### Multi-Device Workflow
267 |
268 | **Device A:**
269 | ```bash
270 | # Make changes
271 | echo "# New Idea" > ~/basic-memory-cloud-sync/ideas/innovation.md
272 |
273 | # Sync to cloud
274 | bm cloud bisync
275 | # → "✓ Sync completed - 1 file uploaded"
276 | ```
277 |
278 | **Device B:**
279 | ```bash
280 | # Pull changes from cloud
281 | bm cloud bisync
282 | # → "✓ Sync completed - 1 file downloaded"
283 |
284 | # See the new file
285 | cat ~/basic-memory-cloud-sync/ideas/innovation.md
286 | # → "# New Idea"
287 | ```
288 |
289 | ### Conflict Scenario
290 |
291 | **Using balanced profile (auto-resolve):**
292 |
293 | ```bash
294 | # Both devices edit same file
295 | # Device A: Updated at 10:00 AM
296 | # Device B: Updated at 10:05 AM
297 |
298 | # Device A syncs
299 | bm cloud bisync
300 | # → "✓ Sync completed"
301 |
302 | # Device B syncs
303 | bm cloud bisync
304 | # → "Resolving conflict: using newer version"
305 | # → "✓ Sync completed"
306 | # → Device B's version (10:05) wins
307 | ```
308 |
309 | **Using safe profile (manual resolution):**
310 |
311 | ```bash
312 | bm cloud bisync --profile safe
313 | # → "Conflict detected: ideas.md"
314 | # → "Saved as: ideas.md.conflict1 and ideas.md.conflict2"
315 | # → "Please resolve manually"
316 |
317 | # Review both versions
318 | diff ideas.md.conflict1 ideas.md.conflict2
319 |
320 | # Merge and cleanup
321 | vim ideas.md # Merge manually
322 | rm ideas.md.conflict*
323 | ```
324 |
325 | ## Monitoring and Status
326 |
327 | ### Check Sync Status
328 |
329 | ```bash
330 | bm cloud status
331 | ```
332 |
333 | **Shows:**
334 | ```
335 | Cloud Bisync Status
336 | ┏━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
337 | ┃ Property ┃ Value ┃
338 | ┡━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━┩
339 | │ Status │ ✓ Initialized │
340 | │ Local Directory │ ~/basic-memory-cloud-sync │
341 | │ Remote │ s3://bucket/tenant-id │
342 | │ Last Sync │ 2 minutes ago │
343 | │ Total Projects │ 5 │
344 | └─────────────────────┴────────────────────────────┘
345 | ```
346 |
347 | ### Verify Integrity
348 |
349 | ```bash
350 | bm cloud check
351 | ```
352 |
353 | Compares local and cloud file hashes to detect:
354 | - Corrupted files
355 | - Missing files
356 | - Sync drift
357 |
358 | ## Troubleshooting
359 |
360 | ### "First bisync requires --resync"
361 |
362 | **Problem:** Initial sync not established
363 |
364 | ```bash
365 | $ bm cloud bisync
366 | Error: First bisync requires --resync to establish baseline
367 | ```
368 |
369 | **Solution:**
370 | ```bash
371 | bm cloud bisync --resync
372 | ```
373 |
374 | ### "Cannot use mount directory for bisync"
375 |
376 | **Problem:** Trying to use mounted directory for sync
377 |
378 | ```bash
379 | $ bm cloud bisync --dir ~/basic-memory-cloud
380 | Error: Cannot use ~/basic-memory-cloud for bisync - it's the mount directory!
381 | ```
382 |
383 | **Solution:** Use different directory
384 | ```bash
385 | bm cloud bisync --dir ~/basic-memory-cloud-sync
386 | ```
387 |
388 | ### Sync Conflicts
389 |
390 | **Problem:** Files modified on both sides
391 |
392 | **Safe profile (manual):**
393 | ```bash
394 | # Find conflict files
395 | find ~/basic-memory-cloud-sync -name "*.conflict*"
396 |
397 | # Review and merge
398 | vimdiff file.conflict1 file.conflict2
399 |
400 | # Keep desired version
401 | mv file.conflict1 file
402 | rm file.conflict2
403 | ```
404 |
405 | **Balanced profile (auto):**
406 | ```bash
407 | # Already resolved to newer version
408 | # Check git history if needed
409 | cd ~/basic-memory-cloud-sync
410 | git log file.md
411 | ```
412 |
413 | ### Deleted Too Many Files
414 |
415 | **Problem:** Exceeds max_delete threshold
416 |
417 | ```bash
418 | $ bm cloud bisync
419 | Error: Deletion exceeds safety limit (26 > 25)
420 | ```
421 |
422 | **Solution:** Review deletions, then force if intentional
423 | ```bash
424 | # Preview what would be deleted
425 | bm cloud bisync --dry-run
426 |
427 | # If intentional, use higher threshold profile
428 | bm cloud bisync --profile fast # max_delete=50
429 |
430 | # Or resync to establish new baseline
431 | bm cloud bisync --resync
432 | ```
433 |
434 | ### rclone Not Found
435 |
436 | **Problem:** rclone not installed
437 |
438 | ```bash
439 | $ bm cloud bisync
440 | Error: rclone not found
441 | ```
442 |
443 | **Solution:**
444 | ```bash
445 | # Run setup again
446 | bm cloud bisync-setup
447 | # → Installs rclone automatically
448 | ```
449 |
450 | ## Configuration
451 |
452 | ### Bisync Config
453 |
454 | Edit `~/.basic-memory/config.json`:
455 |
456 | ```json
457 | {
458 | "bisync_config": {
459 | "sync_dir": "~/basic-memory-cloud-sync",
460 | "default_profile": "balanced",
461 | "auto_sync_interval": 60
462 | }
463 | }
464 | ```
465 |
466 | ### rclone Config
467 |
468 | Located at `~/.config/rclone/rclone.conf`:
469 |
470 | ```ini
471 | [basic-memory-{tenant_id}]
472 | type = s3
473 | provider = AWS
474 | env_auth = false
475 | access_key_id = AKIA...
476 | secret_access_key = ***
477 | region = us-east-1
478 | endpoint = https://fly.storage.tigris.dev
479 | ```
480 |
481 | **Security:** This file contains credentials - keep private (mode 600)
482 |
483 | ## Performance Tips
484 |
485 | 1. **Use balanced profile**: Best trade-off for most users
486 | 2. **Enable watch mode**: `bm sync --watch` for auto-sync
487 | 3. **Optimize .bmignore**: Exclude build artifacts and temp files
488 | 4. **Batch changes**: Group related edits before sync
489 | 5. **Use fast profile**: For rapid iteration on solo projects
490 |
491 | ## Migration from WebDAV
492 |
493 | If upgrading from v0.14.x WebDAV:
494 |
495 | 1. **Backup existing setup**
496 | ```bash
497 | cp -r ~/basic-memory ~/basic-memory.backup
498 | ```
499 |
500 | 2. **Run bisync setup**
501 | ```bash
502 | bm cloud bisync-setup
503 | ```
504 |
505 | 3. **Copy projects to sync directory**
506 | ```bash
507 | cp -r ~/basic-memory/* ~/basic-memory-cloud-sync/
508 | ```
509 |
510 | 4. **Initial sync**
511 | ```bash
512 | bm cloud bisync --resync
513 | ```
514 |
515 | 5. **Remove old WebDAV config** (if applicable)
516 |
517 | ## Security
518 |
519 | - **Scoped credentials**: S3 credentials only access your tenant
520 | - **Encrypted transport**: All traffic over HTTPS/TLS
521 | - **No plain text secrets**: Credentials stored securely in rclone config
522 | - **File permissions**: Config files restricted to user (600)
523 | - **.bmignore**: Prevents syncing sensitive files
524 |
525 | ## See Also
526 |
527 | - SPEC-9: Multi-Project Bidirectional Sync Architecture
528 | - `cloud-authentication.md` - Required for cloud access
529 | - `cloud-mount.md` - Alternative: mount cloud storage
530 | - `env-file-removal.md` - Why .env files aren't synced
531 | - `gitignore-integration.md` - File filtering patterns
532 |
```
--------------------------------------------------------------------------------
/v15-docs/api-performance.md:
--------------------------------------------------------------------------------
```markdown
1 | # API Performance Optimizations (SPEC-11)
2 |
3 | **Status**: Performance Enhancement
4 | **PR**: #315
5 | **Specification**: SPEC-11
6 | **Impact**: Faster API responses, reduced database queries
7 |
8 | ## What Changed
9 |
10 | v0.15.0 implements comprehensive API performance optimizations from SPEC-11, including query optimizations, reduced database round trips, and improved relation traversal.
11 |
12 | ## Key Optimizations
13 |
14 | ### 1. Query Optimization
15 |
16 | **Before:**
17 | ```python
18 | # Multiple separate queries
19 | entity = await get_entity(id) # Query 1
20 | observations = await get_observations(id) # Query 2
21 | relations = await get_relations(id) # Query 3
22 | tags = await get_tags(id) # Query 4
23 | ```
24 |
25 | **After:**
26 | ```python
27 | # Single optimized query with joins
28 | entity = await get_entity_with_details(id)
29 | # → One query returns everything
30 | ```
31 |
32 | **Result:** **75% fewer database queries**
33 |
34 | ### 2. Relation Traversal
35 |
36 | **Before:**
37 | ```python
38 | # Recursive queries for each relation
39 | for relation in entity.relations:
40 | target = await get_entity(relation.target_id) # N queries
41 | ```
42 |
43 | **After:**
44 | ```python
45 | # Batch load all related entities
46 | related_ids = [r.target_id for r in entity.relations]
47 | targets = await get_entities_batch(related_ids) # 1 query
48 | ```
49 |
50 | **Result:** **N+1 query problem eliminated**
51 |
52 | ### 3. Eager Loading
53 |
54 | **Before:**
55 | ```python
56 | # Lazy loading (multiple queries)
57 | entity = await get_entity(id)
58 | if need_relations:
59 | relations = await load_relations(id)
60 | if need_observations:
61 | observations = await load_observations(id)
62 | ```
63 |
64 | **After:**
65 | ```python
66 | # Eager loading (one query)
67 | entity = await get_entity(
68 | id,
69 | load_relations=True,
70 | load_observations=True
71 | ) # All data in one query
72 | ```
73 |
74 | **Result:** Configurable loading strategy
75 |
76 | ## Performance Impact
77 |
78 | ### API Response Times
79 |
80 | **read_note endpoint:**
81 | ```
82 | Before: 250ms average
83 | After: 75ms average (3.3x faster)
84 | ```
85 |
86 | **search_notes endpoint:**
87 | ```
88 | Before: 450ms average
89 | After: 150ms average (3x faster)
90 | ```
91 |
92 | **build_context endpoint (depth=2):**
93 | ```
94 | Before: 1200ms average
95 | After: 320ms average (3.8x faster)
96 | ```
97 |
98 | ### Database Queries
99 |
100 | **Typical MCP tool call:**
101 | ```
102 | Before: 15-20 queries
103 | After: 3-5 queries (75% reduction)
104 | ```
105 |
106 | **Context building (10 entities):**
107 | ```
108 | Before: 150+ queries (N+1 problem)
109 | After: 8 queries (batch loading)
110 | ```
111 |
112 | ## Optimization Techniques
113 |
114 | ### 1. SELECT Optimization
115 |
116 | **Specific column selection:**
117 | ```python
118 | # Before: SELECT *
119 | query = select(Entity)
120 |
121 | # After: SELECT only needed columns
122 | query = select(
123 | Entity.id,
124 | Entity.title,
125 | Entity.permalink,
126 | Entity.content
127 | )
128 | ```
129 |
130 | **Benefit:** Reduced data transfer
131 |
132 | ### 2. JOIN Optimization
133 |
134 | **Efficient joins:**
135 | ```python
136 | # Join related tables in one query
137 | query = (
138 | select(Entity, Observation, Relation)
139 | .join(Observation, Entity.id == Observation.entity_id)
140 | .join(Relation, Entity.id == Relation.from_id)
141 | )
142 | ```
143 |
144 | **Benefit:** Single query vs multiple
145 |
146 | ### 3. Index Usage
147 |
148 | **Optimized indexes:**
149 | ```sql
150 | -- Ensure indexes on frequently queried columns
151 | CREATE INDEX idx_entity_permalink ON entities(permalink);
152 | CREATE INDEX idx_relation_from_id ON relations(from_id);
153 | CREATE INDEX idx_relation_to_id ON relations(to_id);
154 | CREATE INDEX idx_observation_entity_id ON observations(entity_id);
155 | ```
156 |
157 | **Benefit:** Faster lookups
158 |
159 | ### 4. Query Caching
160 |
161 | **Result caching:**
162 | ```python
163 | from functools import lru_cache
164 |
165 | @lru_cache(maxsize=1000)
166 | async def get_entity_cached(entity_id: str):
167 | return await get_entity(entity_id)
168 | ```
169 |
170 | **Benefit:** Avoid redundant queries
171 |
172 | ### 5. Batch Loading
173 |
174 | **Load multiple entities:**
175 | ```python
176 | # Before: Load one at a time
177 | entities = []
178 | for id in entity_ids:
179 | entity = await get_entity(id) # N queries
180 | entities.append(entity)
181 |
182 | # After: Batch load
183 | query = select(Entity).where(Entity.id.in_(entity_ids))
184 | entities = await db.execute(query) # 1 query
185 | ```
186 |
187 | **Benefit:** Eliminates N+1 problem
188 |
189 | ## API-Specific Optimizations
190 |
191 | ### read_note
192 |
193 | **Optimizations:**
194 | - Single query with joins
195 | - Eager load observations and relations
196 | - Efficient permalink lookup
197 |
198 | ```python
199 | # Optimized query
200 | query = (
201 | select(Entity)
202 | .options(
203 | selectinload(Entity.observations),
204 | selectinload(Entity.relations)
205 | )
206 | .where(Entity.permalink == permalink)
207 | )
208 | ```
209 |
210 | **Performance:**
211 | - **Before:** 250ms (4 queries)
212 | - **After:** 75ms (1 query)
213 |
214 | ### search_notes
215 |
216 | **Optimizations:**
217 | - Full-text search index
218 | - Pagination optimization
219 | - Result limiting
220 |
221 | ```python
222 | # Optimized search
223 | query = (
224 | select(Entity)
225 | .where(Entity.content.match(search_query))
226 | .limit(page_size)
227 | .offset(page * page_size)
228 | )
229 | ```
230 |
231 | **Performance:**
232 | - **Before:** 450ms
233 | - **After:** 150ms (3x faster)
234 |
235 | ### build_context
236 |
237 | **Optimizations:**
238 | - Batch relation traversal
239 | - Depth-limited queries
240 | - Circular reference detection
241 |
242 | ```python
243 | # Optimized context building
244 | async def build_context(url: str, depth: int = 2):
245 | # Start entity
246 | entity = await get_entity_by_url(url)
247 |
248 | # Batch load all relations (depth levels)
249 | related_ids = collect_related_ids(entity, depth)
250 | related = await get_entities_batch(related_ids) # 1 query
251 |
252 | return build_graph(entity, related)
253 | ```
254 |
255 | **Performance:**
256 | - **Before:** 1200ms (150+ queries)
257 | - **After:** 320ms (8 queries)
258 |
259 | ### recent_activity
260 |
261 | **Optimizations:**
262 | - Time-indexed queries
263 | - Limit early in query
264 | - Efficient sorting
265 |
266 | ```python
267 | # Optimized recent query
268 | query = (
269 | select(Entity)
270 | .where(Entity.updated_at >= timeframe_start)
271 | .order_by(Entity.updated_at.desc())
272 | .limit(max_results)
273 | )
274 | ```
275 |
276 | **Performance:**
277 | - **Before:** 600ms
278 | - **After:** 180ms (3.3x faster)
279 |
280 | ## Configuration
281 |
282 | ### Query Optimization Settings
283 |
284 | No configuration needed - optimizations are automatic.
285 |
286 | ### Monitoring Query Performance
287 |
288 | **Enable query logging:**
289 | ```bash
290 | export BASIC_MEMORY_LOG_LEVEL=DEBUG
291 | ```
292 |
293 | **Log output:**
294 | ```
295 | [DEBUG] Query took 15ms: SELECT entity WHERE permalink=...
296 | [DEBUG] Query took 3ms: SELECT observations WHERE entity_id IN (...)
297 | ```
298 |
299 | ### Profiling
300 |
301 | ```python
302 | import time
303 | from loguru import logger
304 |
305 | async def profile_query(query_name: str):
306 | start = time.time()
307 | result = await execute_query()
308 | elapsed = (time.time() - start) * 1000
309 | logger.info(f"{query_name}: {elapsed:.2f}ms")
310 | return result
311 | ```
312 |
313 | ## Benchmarks
314 |
315 | ### Single Entity Retrieval
316 |
317 | ```
318 | Operation: get_entity_with_details(id)
319 |
320 | Before:
321 | - Queries: 4 (entity, observations, relations, tags)
322 | - Time: 45ms total
323 |
324 | After:
325 | - Queries: 1 (joined query)
326 | - Time: 12ms total (3.8x faster)
327 | ```
328 |
329 | ### Search Operations
330 |
331 | ```
332 | Operation: search_notes(query, limit=10)
333 |
334 | Before:
335 | - Queries: 1 search + 10 detail queries
336 | - Time: 450ms total
337 |
338 | After:
339 | - Queries: 1 optimized search with joins
340 | - Time: 150ms total (3x faster)
341 | ```
342 |
343 | ### Context Building
344 |
345 | ```
346 | Operation: build_context(url, depth=2)
347 |
348 | Scenario: 10 entities, 20 relations
349 |
350 | Before:
351 | - Queries: 1 root + 20 relations + 10 targets = 31 queries
352 | - Time: 620ms
353 |
354 | After:
355 | - Queries: 1 root + 1 batch relations + 1 batch targets = 3 queries
356 | - Time: 165ms (3.8x faster)
357 | ```
358 |
359 | ### Bulk Operations
360 |
361 | ```
362 | Operation: Import 100 notes
363 |
364 | Before:
365 | - Queries: 100 inserts + 300 relation queries = 400 queries
366 | - Time: 8.5 seconds
367 |
368 | After:
369 | - Queries: 1 bulk insert + 1 bulk relations = 2 queries
370 | - Time: 2.1 seconds (4x faster)
371 | ```
372 |
373 | ## Best Practices
374 |
375 | ### 1. Use Batch Operations
376 |
377 | ```python
378 | # ✓ Good: Batch load
379 | entity_ids = [1, 2, 3, 4, 5]
380 | entities = await get_entities_batch(entity_ids)
381 |
382 | # ✗ Bad: Load one at a time
383 | entities = []
384 | for id in entity_ids:
385 | entity = await get_entity(id)
386 | entities.append(entity)
387 | ```
388 |
389 | ### 2. Specify Required Data
390 |
391 | ```python
392 | # ✓ Good: Load what you need
393 | entity = await get_entity(
394 | id,
395 | load_relations=True,
396 | load_observations=False # Don't need these
397 | )
398 |
399 | # ✗ Bad: Load everything
400 | entity = await get_entity_full(id) # Loads unnecessary data
401 | ```
402 |
403 | ### 3. Use Pagination
404 |
405 | ```python
406 | # ✓ Good: Paginate results
407 | results = await search_notes(
408 | query="test",
409 | page=1,
410 | page_size=20
411 | )
412 |
413 | # ✗ Bad: Load all results
414 | results = await search_notes(query="test") # Could be thousands
415 | ```
416 |
417 | ### 4. Index Foreign Keys
418 |
419 | ```sql
420 | -- ✓ Good: Indexed joins
421 | CREATE INDEX idx_relation_from_id ON relations(from_id);
422 |
423 | -- ✗ Bad: No index
424 | -- Joins will be slow
425 | ```
426 |
427 | ### 5. Limit Depth
428 |
429 | ```python
430 | # ✓ Good: Reasonable depth
431 | context = await build_context(url, depth=2)
432 |
433 | # ✗ Bad: Excessive depth
434 | context = await build_context(url, depth=10) # Exponential growth
435 | ```
436 |
437 | ## Troubleshooting
438 |
439 | ### Slow Queries
440 |
441 | **Problem:** API responses still slow
442 |
443 | **Debug:**
444 | ```bash
445 | # Enable query logging
446 | export BASIC_MEMORY_LOG_LEVEL=DEBUG
447 |
448 | # Check for N+1 queries
449 | # Look for repeated similar queries
450 | ```
451 |
452 | **Solution:**
453 | ```python
454 | # Use batch loading
455 | ids = [1, 2, 3, 4, 5]
456 | entities = await get_entities_batch(ids) # Not in loop
457 | ```
458 |
459 | ### High Memory Usage
460 |
461 | **Problem:** Large result sets consume memory
462 |
463 | **Solution:**
464 | ```python
465 | # Use streaming/pagination
466 | async for batch in stream_entities(batch_size=100):
467 | process(batch)
468 | ```
469 |
470 | ### Database Locks
471 |
472 | **Problem:** Concurrent queries blocking
473 |
474 | **Solution:**
475 | - Ensure WAL mode enabled (see `sqlite-performance.md`)
476 | - Use read-only queries when possible
477 | - Reduce transaction size
478 |
479 | ## Implementation Details
480 |
481 | ### Optimized Query Builder
482 |
483 | ```python
484 | class OptimizedQueryBuilder:
485 | def __init__(self):
486 | self.query = select(Entity)
487 | self.joins = []
488 | self.options = []
489 |
490 | def with_observations(self):
491 | self.options.append(selectinload(Entity.observations))
492 | return self
493 |
494 | def with_relations(self):
495 | self.options.append(selectinload(Entity.relations))
496 | return self
497 |
498 | def build(self):
499 | if self.options:
500 | self.query = self.query.options(*self.options)
501 | return self.query
502 | ```
503 |
504 | ### Batch Loader
505 |
506 | ```python
507 | class BatchEntityLoader:
508 | def __init__(self, batch_size: int = 100):
509 | self.batch_size = batch_size
510 | self.pending = []
511 |
512 | async def load(self, entity_id: str):
513 | self.pending.append(entity_id)
514 |
515 | if len(self.pending) >= self.batch_size:
516 | return await self._flush()
517 |
518 | return None
519 |
520 | async def _flush(self):
521 | if not self.pending:
522 | return []
523 |
524 | ids = self.pending
525 | self.pending = []
526 |
527 | # Single batch query
528 | query = select(Entity).where(Entity.id.in_(ids))
529 | result = await db.execute(query)
530 | return result.scalars().all()
531 | ```
532 |
533 | ### Query Cache
534 |
535 | ```python
536 | from cachetools import TTLCache
537 |
538 | class QueryCache:
539 | def __init__(self, maxsize: int = 1000, ttl: int = 300):
540 | self.cache = TTLCache(maxsize=maxsize, ttl=ttl)
541 |
542 | async def get_or_query(self, key: str, query_func):
543 | if key in self.cache:
544 | return self.cache[key]
545 |
546 | result = await query_func()
547 | self.cache[key] = result
548 | return result
549 | ```
550 |
551 | ## Migration from v0.14.x
552 |
553 | ### Automatic Optimization
554 |
555 | **No action needed** - optimizations are automatic:
556 |
557 | ```bash
558 | # Upgrade and restart
559 | pip install --upgrade basic-memory
560 | bm mcp
561 |
562 | # Optimizations active immediately
563 | ```
564 |
565 | ### Verify Performance Improvement
566 |
567 | **Before upgrade:**
568 | ```bash
569 | time bm tools search --query "test"
570 | # → 450ms
571 | ```
572 |
573 | **After upgrade:**
574 | ```bash
575 | time bm tools search --query "test"
576 | # → 150ms (3x faster)
577 | ```
578 |
579 | ## See Also
580 |
581 | - SPEC-11: API Performance Optimization specification
582 | - `sqlite-performance.md` - Database-level optimizations
583 | - `background-relations.md` - Background processing optimizations
584 | - Database indexing guide
585 | - Query optimization patterns
586 |
```
--------------------------------------------------------------------------------
/src/basic_memory/db.py:
--------------------------------------------------------------------------------
```python
1 | import asyncio
2 | import os
3 | from contextlib import asynccontextmanager
4 | from enum import Enum, auto
5 | from pathlib import Path
6 | from typing import AsyncGenerator, Optional
7 |
8 | from basic_memory.config import BasicMemoryConfig, ConfigManager
9 | from alembic import command
10 | from alembic.config import Config
11 |
12 | from loguru import logger
13 | from sqlalchemy import text, event
14 | from sqlalchemy.ext.asyncio import (
15 | create_async_engine,
16 | async_sessionmaker,
17 | AsyncSession,
18 | AsyncEngine,
19 | async_scoped_session,
20 | )
21 | from sqlalchemy.pool import NullPool
22 |
23 | from basic_memory.repository.search_repository import SearchRepository
24 |
25 | # Module level state
26 | _engine: Optional[AsyncEngine] = None
27 | _session_maker: Optional[async_sessionmaker[AsyncSession]] = None
28 | _migrations_completed: bool = False
29 |
30 |
31 | class DatabaseType(Enum):
32 | """Types of supported databases."""
33 |
34 | MEMORY = auto()
35 | FILESYSTEM = auto()
36 |
37 | @classmethod
38 | def get_db_url(cls, db_path: Path, db_type: "DatabaseType") -> str:
39 | """Get SQLAlchemy URL for database path."""
40 | if db_type == cls.MEMORY:
41 | logger.info("Using in-memory SQLite database")
42 | return "sqlite+aiosqlite://"
43 |
44 | return f"sqlite+aiosqlite:///{db_path}" # pragma: no cover
45 |
46 |
47 | def get_scoped_session_factory(
48 | session_maker: async_sessionmaker[AsyncSession],
49 | ) -> async_scoped_session:
50 | """Create a scoped session factory scoped to current task."""
51 | return async_scoped_session(session_maker, scopefunc=asyncio.current_task)
52 |
53 |
54 | @asynccontextmanager
55 | async def scoped_session(
56 | session_maker: async_sessionmaker[AsyncSession],
57 | ) -> AsyncGenerator[AsyncSession, None]:
58 | """
59 | Get a scoped session with proper lifecycle management.
60 |
61 | Args:
62 | session_maker: Session maker to create scoped sessions from
63 | """
64 | factory = get_scoped_session_factory(session_maker)
65 | session = factory()
66 | try:
67 | await session.execute(text("PRAGMA foreign_keys=ON"))
68 | yield session
69 | await session.commit()
70 | except Exception:
71 | await session.rollback()
72 | raise
73 | finally:
74 | await session.close()
75 | await factory.remove()
76 |
77 |
78 | def _configure_sqlite_connection(dbapi_conn, enable_wal: bool = True) -> None:
79 | """Configure SQLite connection with WAL mode and optimizations.
80 |
81 | Args:
82 | dbapi_conn: Database API connection object
83 | enable_wal: Whether to enable WAL mode (should be False for in-memory databases)
84 | """
85 | cursor = dbapi_conn.cursor()
86 | try:
87 | # Enable WAL mode for better concurrency (not supported for in-memory databases)
88 | if enable_wal:
89 | cursor.execute("PRAGMA journal_mode=WAL")
90 | # Set busy timeout to handle locked databases
91 | cursor.execute("PRAGMA busy_timeout=10000") # 10 seconds
92 | # Optimize for performance
93 | cursor.execute("PRAGMA synchronous=NORMAL")
94 | cursor.execute("PRAGMA cache_size=-64000") # 64MB cache
95 | cursor.execute("PRAGMA temp_store=MEMORY")
96 | # Windows-specific optimizations
97 | if os.name == "nt":
98 | cursor.execute("PRAGMA locking_mode=NORMAL") # Ensure normal locking on Windows
99 | except Exception as e:
100 | # Log but don't fail - some PRAGMAs may not be supported
101 | logger.warning(f"Failed to configure SQLite connection: {e}")
102 | finally:
103 | cursor.close()
104 |
105 |
106 | def _create_engine_and_session(
107 | db_path: Path, db_type: DatabaseType = DatabaseType.FILESYSTEM
108 | ) -> tuple[AsyncEngine, async_sessionmaker[AsyncSession]]:
109 | """Internal helper to create engine and session maker."""
110 | db_url = DatabaseType.get_db_url(db_path, db_type)
111 | logger.debug(f"Creating engine for db_url: {db_url}")
112 |
113 | # Configure connection args with Windows-specific settings
114 | connect_args: dict[str, bool | float | None] = {"check_same_thread": False}
115 |
116 | # Add Windows-specific parameters to improve reliability
117 | if os.name == "nt": # Windows
118 | connect_args.update(
119 | {
120 | "timeout": 30.0, # Increase timeout to 30 seconds for Windows
121 | "isolation_level": None, # Use autocommit mode
122 | }
123 | )
124 | # Use NullPool for Windows filesystem databases to avoid connection pooling issues
125 | # Important: Do NOT use NullPool for in-memory databases as it will destroy the database
126 | # between connections
127 | if db_type == DatabaseType.FILESYSTEM:
128 | engine = create_async_engine(
129 | db_url,
130 | connect_args=connect_args,
131 | poolclass=NullPool, # Disable connection pooling on Windows
132 | echo=False,
133 | )
134 | else:
135 | # In-memory databases need connection pooling to maintain state
136 | engine = create_async_engine(db_url, connect_args=connect_args)
137 | else:
138 | engine = create_async_engine(db_url, connect_args=connect_args)
139 |
140 | # Enable WAL mode for better concurrency and reliability
141 | # Note: WAL mode is not supported for in-memory databases
142 | enable_wal = db_type != DatabaseType.MEMORY
143 |
144 | @event.listens_for(engine.sync_engine, "connect")
145 | def enable_wal_mode(dbapi_conn, connection_record):
146 | """Enable WAL mode on each connection."""
147 | _configure_sqlite_connection(dbapi_conn, enable_wal=enable_wal)
148 |
149 | session_maker = async_sessionmaker(engine, expire_on_commit=False)
150 | return engine, session_maker
151 |
152 |
153 | async def get_or_create_db(
154 | db_path: Path,
155 | db_type: DatabaseType = DatabaseType.FILESYSTEM,
156 | ensure_migrations: bool = True,
157 | ) -> tuple[AsyncEngine, async_sessionmaker[AsyncSession]]: # pragma: no cover
158 | """Get or create database engine and session maker."""
159 | global _engine, _session_maker
160 |
161 | if _engine is None:
162 | _engine, _session_maker = _create_engine_and_session(db_path, db_type)
163 |
164 | # Run migrations automatically unless explicitly disabled
165 | if ensure_migrations:
166 | app_config = ConfigManager().config
167 | await run_migrations(app_config, db_type)
168 |
169 | # These checks should never fail since we just created the engine and session maker
170 | # if they were None, but we'll check anyway for the type checker
171 | if _engine is None:
172 | logger.error("Failed to create database engine", db_path=str(db_path))
173 | raise RuntimeError("Database engine initialization failed")
174 |
175 | if _session_maker is None:
176 | logger.error("Failed to create session maker", db_path=str(db_path))
177 | raise RuntimeError("Session maker initialization failed")
178 |
179 | return _engine, _session_maker
180 |
181 |
182 | async def shutdown_db() -> None: # pragma: no cover
183 | """Clean up database connections."""
184 | global _engine, _session_maker, _migrations_completed
185 |
186 | if _engine:
187 | await _engine.dispose()
188 | _engine = None
189 | _session_maker = None
190 | _migrations_completed = False
191 |
192 |
193 | @asynccontextmanager
194 | async def engine_session_factory(
195 | db_path: Path,
196 | db_type: DatabaseType = DatabaseType.MEMORY,
197 | ) -> AsyncGenerator[tuple[AsyncEngine, async_sessionmaker[AsyncSession]], None]:
198 | """Create engine and session factory.
199 |
200 | Note: This is primarily used for testing where we want a fresh database
201 | for each test. For production use, use get_or_create_db() instead.
202 | """
203 |
204 | global _engine, _session_maker, _migrations_completed
205 |
206 | db_url = DatabaseType.get_db_url(db_path, db_type)
207 | logger.debug(f"Creating engine for db_url: {db_url}")
208 |
209 | # Configure connection args with Windows-specific settings
210 | connect_args: dict[str, bool | float | None] = {"check_same_thread": False}
211 |
212 | # Add Windows-specific parameters to improve reliability
213 | if os.name == "nt": # Windows
214 | connect_args.update(
215 | {
216 | "timeout": 30.0, # Increase timeout to 30 seconds for Windows
217 | "isolation_level": None, # Use autocommit mode
218 | }
219 | )
220 | # Use NullPool for Windows filesystem databases to avoid connection pooling issues
221 | # Important: Do NOT use NullPool for in-memory databases as it will destroy the database
222 | # between connections
223 | if db_type == DatabaseType.FILESYSTEM:
224 | _engine = create_async_engine(
225 | db_url,
226 | connect_args=connect_args,
227 | poolclass=NullPool, # Disable connection pooling on Windows
228 | echo=False,
229 | )
230 | else:
231 | # In-memory databases need connection pooling to maintain state
232 | _engine = create_async_engine(db_url, connect_args=connect_args)
233 | else:
234 | _engine = create_async_engine(db_url, connect_args=connect_args)
235 |
236 | # Enable WAL mode for better concurrency and reliability
237 | # Note: WAL mode is not supported for in-memory databases
238 | enable_wal = db_type != DatabaseType.MEMORY
239 |
240 | @event.listens_for(_engine.sync_engine, "connect")
241 | def enable_wal_mode(dbapi_conn, connection_record):
242 | """Enable WAL mode on each connection."""
243 | _configure_sqlite_connection(dbapi_conn, enable_wal=enable_wal)
244 |
245 | try:
246 | _session_maker = async_sessionmaker(_engine, expire_on_commit=False)
247 |
248 | # Verify that engine and session maker are initialized
249 | if _engine is None: # pragma: no cover
250 | logger.error("Database engine is None in engine_session_factory")
251 | raise RuntimeError("Database engine initialization failed")
252 |
253 | if _session_maker is None: # pragma: no cover
254 | logger.error("Session maker is None in engine_session_factory")
255 | raise RuntimeError("Session maker initialization failed")
256 |
257 | yield _engine, _session_maker
258 | finally:
259 | if _engine:
260 | await _engine.dispose()
261 | _engine = None
262 | _session_maker = None
263 | _migrations_completed = False
264 |
265 |
266 | async def run_migrations(
267 | app_config: BasicMemoryConfig, database_type=DatabaseType.FILESYSTEM, force: bool = False
268 | ): # pragma: no cover
269 | """Run any pending alembic migrations."""
270 | global _migrations_completed
271 |
272 | # Skip if migrations already completed unless forced
273 | if _migrations_completed and not force:
274 | logger.debug("Migrations already completed in this session, skipping")
275 | return
276 |
277 | logger.info("Running database migrations...")
278 | try:
279 | # Get the absolute path to the alembic directory relative to this file
280 | alembic_dir = Path(__file__).parent / "alembic"
281 | config = Config()
282 |
283 | # Set required Alembic config options programmatically
284 | config.set_main_option("script_location", str(alembic_dir))
285 | config.set_main_option(
286 | "file_template",
287 | "%%(year)d_%%(month).2d_%%(day).2d_%%(hour).2d%%(minute).2d-%%(rev)s_%%(slug)s",
288 | )
289 | config.set_main_option("timezone", "UTC")
290 | config.set_main_option("revision_environment", "false")
291 | config.set_main_option(
292 | "sqlalchemy.url", DatabaseType.get_db_url(app_config.database_path, database_type)
293 | )
294 |
295 | command.upgrade(config, "head")
296 | logger.info("Migrations completed successfully")
297 |
298 | # Get session maker - ensure we don't trigger recursive migration calls
299 | if _session_maker is None:
300 | _, session_maker = _create_engine_and_session(app_config.database_path, database_type)
301 | else:
302 | session_maker = _session_maker
303 |
304 | # initialize the search Index schema
305 | # the project_id is not used for init_search_index, so we pass a dummy value
306 | await SearchRepository(session_maker, 1).init_search_index()
307 |
308 | # Mark migrations as completed
309 | _migrations_completed = True
310 | except Exception as e: # pragma: no cover
311 | logger.error(f"Error running migrations: {e}")
312 | raise
313 |
```
--------------------------------------------------------------------------------
/tests/repository/test_observation_repository.py:
--------------------------------------------------------------------------------
```python
1 | """Tests for the ObservationRepository."""
2 |
3 | from datetime import datetime, timezone
4 |
5 | import pytest
6 | import pytest_asyncio
7 | import sqlalchemy
8 | from sqlalchemy.ext.asyncio import async_sessionmaker
9 |
10 | from basic_memory import db
11 | from basic_memory.models import Entity, Observation, Project
12 | from basic_memory.repository.observation_repository import ObservationRepository
13 |
14 |
15 | @pytest_asyncio.fixture(scope="function")
16 | async def repo(observation_repository):
17 | """Create an ObservationRepository instance"""
18 | return observation_repository
19 |
20 |
21 | @pytest_asyncio.fixture(scope="function")
22 | async def sample_observation(repo, sample_entity: Entity):
23 | """Create a sample observation for testing"""
24 | observation_data = {
25 | "entity_id": sample_entity.id,
26 | "content": "Test observation",
27 | "context": "test-context",
28 | }
29 | return await repo.create(observation_data)
30 |
31 |
32 | @pytest.mark.asyncio
33 | async def test_create_observation(
34 | observation_repository: ObservationRepository, sample_entity: Entity
35 | ):
36 | """Test creating a new observation"""
37 | observation_data = {
38 | "entity_id": sample_entity.id,
39 | "content": "Test content",
40 | "context": "test-context",
41 | }
42 | observation = await observation_repository.create(observation_data)
43 |
44 | assert observation.entity_id == sample_entity.id
45 | assert observation.content == "Test content"
46 | assert observation.id is not None # Should be auto-generated
47 |
48 |
49 | @pytest.mark.asyncio
50 | async def test_create_observation_entity_does_not_exist(
51 | observation_repository: ObservationRepository, sample_entity: Entity
52 | ):
53 | """Test creating a new observation"""
54 | observation_data = {
55 | "entity_id": "does-not-exist",
56 | "content": "Test content",
57 | "context": "test-context",
58 | }
59 | with pytest.raises(sqlalchemy.exc.IntegrityError):
60 | await observation_repository.create(observation_data)
61 |
62 |
63 | @pytest.mark.asyncio
64 | async def test_find_by_entity(
65 | observation_repository: ObservationRepository,
66 | sample_observation: Observation,
67 | sample_entity: Entity,
68 | ):
69 | """Test finding observations by entity"""
70 | observations = await observation_repository.find_by_entity(sample_entity.id)
71 | assert len(observations) == 1
72 | assert observations[0].id == sample_observation.id
73 | assert observations[0].content == sample_observation.content
74 |
75 |
76 | @pytest.mark.asyncio
77 | async def test_find_by_context(
78 | observation_repository: ObservationRepository, sample_observation: Observation
79 | ):
80 | """Test finding observations by context"""
81 | observations = await observation_repository.find_by_context("test-context")
82 | assert len(observations) == 1
83 | assert observations[0].id == sample_observation.id
84 | assert observations[0].content == sample_observation.content
85 |
86 |
87 | @pytest.mark.asyncio
88 | async def test_delete_observations(session_maker: async_sessionmaker, repo, test_project: Project):
89 | """Test deleting observations by entity_id."""
90 | # Create test entity
91 | async with db.scoped_session(session_maker) as session:
92 | entity = Entity(
93 | project_id=test_project.id,
94 | title="test_entity",
95 | entity_type="test",
96 | permalink="test/test-entity",
97 | file_path="test/test_entity.md",
98 | content_type="text/markdown",
99 | created_at=datetime.now(timezone.utc),
100 | updated_at=datetime.now(timezone.utc),
101 | )
102 | session.add(entity)
103 | await session.flush()
104 |
105 | # Create test observations
106 | obs1 = Observation(
107 | entity_id=entity.id,
108 | content="Test observation 1",
109 | )
110 | obs2 = Observation(
111 | entity_id=entity.id,
112 | content="Test observation 2",
113 | )
114 | session.add_all([obs1, obs2])
115 |
116 | # Test deletion by entity_id
117 | deleted = await repo.delete_by_fields(entity_id=entity.id)
118 | assert deleted is True
119 |
120 | # Verify observations were deleted
121 | remaining = await repo.find_by_entity(entity.id)
122 | assert len(remaining) == 0
123 |
124 |
125 | @pytest.mark.asyncio
126 | async def test_delete_observation_by_id(
127 | session_maker: async_sessionmaker, repo, test_project: Project
128 | ):
129 | """Test deleting a single observation by its ID."""
130 | # Create test entity
131 | async with db.scoped_session(session_maker) as session:
132 | entity = Entity(
133 | project_id=test_project.id,
134 | title="test_entity",
135 | entity_type="test",
136 | permalink="test/test-entity",
137 | file_path="test/test_entity.md",
138 | content_type="text/markdown",
139 | created_at=datetime.now(timezone.utc),
140 | updated_at=datetime.now(timezone.utc),
141 | )
142 | session.add(entity)
143 | await session.flush()
144 |
145 | # Create test observation
146 | obs = Observation(
147 | entity_id=entity.id,
148 | content="Test observation",
149 | )
150 | session.add(obs)
151 |
152 | # Test deletion by ID
153 | deleted = await repo.delete(obs.id)
154 | assert deleted is True
155 |
156 | # Verify observation was deleted
157 | remaining = await repo.find_by_id(obs.id)
158 | assert remaining is None
159 |
160 |
161 | @pytest.mark.asyncio
162 | async def test_delete_observation_by_content(
163 | session_maker: async_sessionmaker, repo, test_project: Project
164 | ):
165 | """Test deleting observations by content."""
166 | # Create test entity
167 | async with db.scoped_session(session_maker) as session:
168 | entity = Entity(
169 | project_id=test_project.id,
170 | title="test_entity",
171 | entity_type="test",
172 | permalink="test/test-entity",
173 | file_path="test/test_entity.md",
174 | content_type="text/markdown",
175 | created_at=datetime.now(timezone.utc),
176 | updated_at=datetime.now(timezone.utc),
177 | )
178 | session.add(entity)
179 | await session.flush()
180 |
181 | # Create test observations
182 | obs1 = Observation(
183 | entity_id=entity.id,
184 | content="Delete this observation",
185 | )
186 | obs2 = Observation(
187 | entity_id=entity.id,
188 | content="Keep this observation",
189 | )
190 | session.add_all([obs1, obs2])
191 |
192 | # Test deletion by content
193 | deleted = await repo.delete_by_fields(content="Delete this observation")
194 | assert deleted is True
195 |
196 | # Verify only matching observation was deleted
197 | remaining = await repo.find_by_entity(entity.id)
198 | assert len(remaining) == 1
199 | assert remaining[0].content == "Keep this observation"
200 |
201 |
202 | @pytest.mark.asyncio
203 | async def test_find_by_category(session_maker: async_sessionmaker, repo, test_project: Project):
204 | """Test finding observations by their category."""
205 | # Create test entity
206 | async with db.scoped_session(session_maker) as session:
207 | entity = Entity(
208 | project_id=test_project.id,
209 | title="test_entity",
210 | entity_type="test",
211 | permalink="test/test-entity",
212 | file_path="test/test_entity.md",
213 | content_type="text/markdown",
214 | created_at=datetime.now(timezone.utc),
215 | updated_at=datetime.now(timezone.utc),
216 | )
217 | session.add(entity)
218 | await session.flush()
219 |
220 | # Create test observations with different categories
221 | observations = [
222 | Observation(
223 | entity_id=entity.id,
224 | content="Tech observation",
225 | category="tech",
226 | ),
227 | Observation(
228 | entity_id=entity.id,
229 | content="Design observation",
230 | category="design",
231 | ),
232 | Observation(
233 | entity_id=entity.id,
234 | content="Another tech observation",
235 | category="tech",
236 | ),
237 | ]
238 | session.add_all(observations)
239 | await session.commit()
240 |
241 | # Find tech observations
242 | tech_obs = await repo.find_by_category("tech")
243 | assert len(tech_obs) == 2
244 | assert all(obs.category == "tech" for obs in tech_obs)
245 | assert set(obs.content for obs in tech_obs) == {"Tech observation", "Another tech observation"}
246 |
247 | # Find design observations
248 | design_obs = await repo.find_by_category("design")
249 | assert len(design_obs) == 1
250 | assert design_obs[0].category == "design"
251 | assert design_obs[0].content == "Design observation"
252 |
253 | # Search for non-existent category
254 | missing_obs = await repo.find_by_category("missing")
255 | assert len(missing_obs) == 0
256 |
257 |
258 | @pytest.mark.asyncio
259 | async def test_observation_categories(
260 | session_maker: async_sessionmaker, repo, test_project: Project
261 | ):
262 | """Test retrieving distinct observation categories."""
263 | # Create test entity
264 | async with db.scoped_session(session_maker) as session:
265 | entity = Entity(
266 | project_id=test_project.id,
267 | title="test_entity",
268 | entity_type="test",
269 | permalink="test/test-entity",
270 | file_path="test/test_entity.md",
271 | content_type="text/markdown",
272 | created_at=datetime.now(timezone.utc),
273 | updated_at=datetime.now(timezone.utc),
274 | )
275 | session.add(entity)
276 | await session.flush()
277 |
278 | # Create observations with various categories
279 | observations = [
280 | Observation(
281 | entity_id=entity.id,
282 | content="First tech note",
283 | category="tech",
284 | ),
285 | Observation(
286 | entity_id=entity.id,
287 | content="Second tech note",
288 | category="tech", # Duplicate category
289 | ),
290 | Observation(
291 | entity_id=entity.id,
292 | content="Design note",
293 | category="design",
294 | ),
295 | Observation(
296 | entity_id=entity.id,
297 | content="Feature note",
298 | category="feature",
299 | ),
300 | ]
301 | session.add_all(observations)
302 | await session.commit()
303 |
304 | # Get distinct categories
305 | categories = await repo.observation_categories()
306 |
307 | # Should have unique categories in a deterministic order
308 | assert set(categories) == {"tech", "design", "feature"}
309 |
310 |
311 | @pytest.mark.asyncio
312 | async def test_find_by_category_with_empty_db(repo):
313 | """Test category operations with an empty database."""
314 | # Find by category should return empty list
315 | obs = await repo.find_by_category("tech")
316 | assert len(obs) == 0
317 |
318 | # Get categories should return empty list
319 | categories = await repo.observation_categories()
320 | assert len(categories) == 0
321 |
322 |
323 | @pytest.mark.asyncio
324 | async def test_find_by_category_case_sensitivity(
325 | session_maker: async_sessionmaker, repo, test_project: Project
326 | ):
327 | """Test how category search handles case sensitivity."""
328 | async with db.scoped_session(session_maker) as session:
329 | entity = Entity(
330 | project_id=test_project.id,
331 | title="test_entity",
332 | entity_type="test",
333 | permalink="test/test-entity",
334 | file_path="test/test_entity.md",
335 | content_type="text/markdown",
336 | created_at=datetime.now(timezone.utc),
337 | updated_at=datetime.now(timezone.utc),
338 | )
339 | session.add(entity)
340 | await session.flush()
341 |
342 | # Create a test observation
343 | obs = Observation(
344 | entity_id=entity.id,
345 | content="Tech note",
346 | category="tech", # lowercase in database
347 | )
348 | session.add(obs)
349 | await session.commit()
350 |
351 | # Search should work regardless of case
352 | # Note: If we want case-insensitive search, we'll need to update the query
353 | # For now, this test documents the current behavior
354 | exact_match = await repo.find_by_category("tech")
355 | assert len(exact_match) == 1
356 |
357 | upper_case = await repo.find_by_category("TECH")
358 | assert len(upper_case) == 0 # Currently case-sensitive
359 |
```
--------------------------------------------------------------------------------
/tests/mcp/test_tool_search.py:
--------------------------------------------------------------------------------
```python
1 | """Tests for search MCP tools."""
2 |
3 | import pytest
4 | from datetime import datetime, timedelta
5 | from unittest.mock import patch
6 |
7 | from basic_memory.mcp.tools import write_note
8 | from basic_memory.mcp.tools.search import search_notes, _format_search_error_response
9 | from basic_memory.schemas.search import SearchResponse
10 |
11 |
12 | @pytest.mark.asyncio
13 | async def test_search_text(client, test_project):
14 | """Test basic search functionality."""
15 | # Create a test note
16 | result = await write_note.fn(
17 | project=test_project.name,
18 | title="Test Search Note",
19 | folder="test",
20 | content="# Test\nThis is a searchable test note",
21 | tags=["test", "search"],
22 | )
23 | assert result
24 |
25 | # Search for it
26 | response = await search_notes.fn(project=test_project.name, query="searchable")
27 |
28 | # Verify results - handle both success and error cases
29 | if isinstance(response, SearchResponse):
30 | # Success case - verify SearchResponse
31 | assert len(response.results) > 0
32 | assert any(r.permalink == "test/test-search-note" for r in response.results)
33 | else:
34 | # If search failed and returned error message, test should fail with informative message
35 | pytest.fail(f"Search failed with error: {response}")
36 |
37 |
38 | @pytest.mark.asyncio
39 | async def test_search_title(client, test_project):
40 | """Test basic search functionality."""
41 | # Create a test note
42 | result = await write_note.fn(
43 | project=test_project.name,
44 | title="Test Search Note",
45 | folder="test",
46 | content="# Test\nThis is a searchable test note",
47 | tags=["test", "search"],
48 | )
49 | assert result
50 |
51 | # Search for it
52 | response = await search_notes.fn(
53 | project=test_project.name, query="Search Note", search_type="title"
54 | )
55 |
56 | # Verify results - handle both success and error cases
57 | if isinstance(response, str):
58 | # If search failed and returned error message, test should fail with informative message
59 | pytest.fail(f"Search failed with error: {response}")
60 | else:
61 | # Success case - verify SearchResponse
62 | assert len(response.results) > 0
63 | assert any(r.permalink == "test/test-search-note" for r in response.results)
64 |
65 |
66 | @pytest.mark.asyncio
67 | async def test_search_permalink(client, test_project):
68 | """Test basic search functionality."""
69 | # Create a test note
70 | result = await write_note.fn(
71 | project=test_project.name,
72 | title="Test Search Note",
73 | folder="test",
74 | content="# Test\nThis is a searchable test note",
75 | tags=["test", "search"],
76 | )
77 | assert result
78 |
79 | # Search for it
80 | response = await search_notes.fn(
81 | project=test_project.name, query="test/test-search-note", search_type="permalink"
82 | )
83 |
84 | # Verify results - handle both success and error cases
85 | if isinstance(response, SearchResponse):
86 | # Success case - verify SearchResponse
87 | assert len(response.results) > 0
88 | assert any(r.permalink == "test/test-search-note" for r in response.results)
89 | else:
90 | # If search failed and returned error message, test should fail with informative message
91 | pytest.fail(f"Search failed with error: {response}")
92 |
93 |
94 | @pytest.mark.asyncio
95 | async def test_search_permalink_match(client, test_project):
96 | """Test basic search functionality."""
97 | # Create a test note
98 | result = await write_note.fn(
99 | project=test_project.name,
100 | title="Test Search Note",
101 | folder="test",
102 | content="# Test\nThis is a searchable test note",
103 | tags=["test", "search"],
104 | )
105 | assert result
106 |
107 | # Search for it
108 | response = await search_notes.fn(
109 | project=test_project.name, query="test/test-search-*", search_type="permalink"
110 | )
111 |
112 | # Verify results - handle both success and error cases
113 | if isinstance(response, SearchResponse):
114 | # Success case - verify SearchResponse
115 | assert len(response.results) > 0
116 | assert any(r.permalink == "test/test-search-note" for r in response.results)
117 | else:
118 | # If search failed and returned error message, test should fail with informative message
119 | pytest.fail(f"Search failed with error: {response}")
120 |
121 |
122 | @pytest.mark.asyncio
123 | async def test_search_pagination(client, test_project):
124 | """Test basic search functionality."""
125 | # Create a test note
126 | result = await write_note.fn(
127 | project=test_project.name,
128 | title="Test Search Note",
129 | folder="test",
130 | content="# Test\nThis is a searchable test note",
131 | tags=["test", "search"],
132 | )
133 | assert result
134 |
135 | # Search for it
136 | response = await search_notes.fn(
137 | project=test_project.name, query="searchable", page=1, page_size=1
138 | )
139 |
140 | # Verify results - handle both success and error cases
141 | if isinstance(response, SearchResponse):
142 | # Success case - verify SearchResponse
143 | assert len(response.results) == 1
144 | assert any(r.permalink == "test/test-search-note" for r in response.results)
145 | else:
146 | # If search failed and returned error message, test should fail with informative message
147 | pytest.fail(f"Search failed with error: {response}")
148 |
149 |
150 | @pytest.mark.asyncio
151 | async def test_search_with_type_filter(client, test_project):
152 | """Test search with entity type filter."""
153 | # Create test content
154 | await write_note.fn(
155 | project=test_project.name,
156 | title="Entity Type Test",
157 | folder="test",
158 | content="# Test\nFiltered by type",
159 | )
160 |
161 | # Search with type filter
162 | response = await search_notes.fn(project=test_project.name, query="type", types=["note"])
163 |
164 | # Verify results - handle both success and error cases
165 | if isinstance(response, SearchResponse):
166 | # Success case - verify all results are entities
167 | assert all(r.type == "entity" for r in response.results)
168 | else:
169 | # If search failed and returned error message, test should fail with informative message
170 | pytest.fail(f"Search failed with error: {response}")
171 |
172 |
173 | @pytest.mark.asyncio
174 | async def test_search_with_entity_type_filter(client, test_project):
175 | """Test search with entity type filter."""
176 | # Create test content
177 | await write_note.fn(
178 | project=test_project.name,
179 | title="Entity Type Test",
180 | folder="test",
181 | content="# Test\nFiltered by type",
182 | )
183 |
184 | # Search with entity type filter
185 | response = await search_notes.fn(
186 | project=test_project.name, query="type", entity_types=["entity"]
187 | )
188 |
189 | # Verify results - handle both success and error cases
190 | if isinstance(response, SearchResponse):
191 | # Success case - verify all results are entities
192 | assert all(r.type == "entity" for r in response.results)
193 | else:
194 | # If search failed and returned error message, test should fail with informative message
195 | pytest.fail(f"Search failed with error: {response}")
196 |
197 |
198 | @pytest.mark.asyncio
199 | async def test_search_with_date_filter(client, test_project):
200 | """Test search with date filter."""
201 | # Create test content
202 | await write_note.fn(
203 | project=test_project.name,
204 | title="Recent Note",
205 | folder="test",
206 | content="# Test\nRecent content",
207 | )
208 |
209 | # Search with date filter
210 | one_hour_ago = datetime.now() - timedelta(hours=1)
211 | response = await search_notes.fn(
212 | project=test_project.name, query="recent", after_date=one_hour_ago.isoformat()
213 | )
214 |
215 | # Verify results - handle both success and error cases
216 | if isinstance(response, SearchResponse):
217 | # Success case - verify we get results within timeframe
218 | assert len(response.results) > 0
219 | else:
220 | # If search failed and returned error message, test should fail with informative message
221 | pytest.fail(f"Search failed with error: {response}")
222 |
223 |
224 | class TestSearchErrorFormatting:
225 | """Test search error formatting for better user experience."""
226 |
227 | def test_format_search_error_fts5_syntax(self):
228 | """Test formatting for FTS5 syntax errors."""
229 | result = _format_search_error_response(
230 | "test-project", "syntax error in FTS5", "test query("
231 | )
232 |
233 | assert "# Search Failed - Invalid Syntax" in result
234 | assert "The search query 'test query(' contains invalid syntax" in result
235 | assert "Special characters" in result
236 | assert "test query" in result # Clean query without special chars
237 |
238 | def test_format_search_error_no_results(self):
239 | """Test formatting for no results found."""
240 | result = _format_search_error_response(
241 | "test-project", "no results found", "very specific query"
242 | )
243 |
244 | assert "# Search Complete - No Results Found" in result
245 | assert "No content found matching 'very specific query'" in result
246 | assert "Broaden your search" in result
247 | assert "very" in result # Simplified query
248 |
249 | def test_format_search_error_server_error(self):
250 | """Test formatting for server errors."""
251 | result = _format_search_error_response(
252 | "test-project", "internal server error", "test query"
253 | )
254 |
255 | assert "# Search Failed - Server Error" in result
256 | assert "The search service encountered an error while processing 'test query'" in result
257 | assert "Try again" in result
258 | assert "Check project status" in result
259 |
260 | def test_format_search_error_permission_denied(self):
261 | """Test formatting for permission errors."""
262 | result = _format_search_error_response("test-project", "permission denied", "test query")
263 |
264 | assert "# Search Failed - Access Error" in result
265 | assert "You don't have permission to search" in result
266 | assert "Check your project access" in result
267 |
268 | def test_format_search_error_project_not_found(self):
269 | """Test formatting for project not found errors."""
270 | result = _format_search_error_response(
271 | "test-project", "current project not found", "test query"
272 | )
273 |
274 | assert "# Search Failed - Project Not Found" in result
275 | assert "The current project is not accessible" in result
276 | assert "Check available projects" in result
277 |
278 | def test_format_search_error_generic(self):
279 | """Test formatting for generic errors."""
280 | result = _format_search_error_response("test-project", "unknown error", "test query")
281 |
282 | assert "# Search Failed" in result
283 | assert "Error searching for 'test query': unknown error" in result
284 | assert "## Troubleshooting steps:" in result
285 |
286 |
287 | class TestSearchToolErrorHandling:
288 | """Test search tool exception handling."""
289 |
290 | @pytest.mark.asyncio
291 | async def test_search_notes_exception_handling(self):
292 | """Test exception handling in search_notes."""
293 | with patch("basic_memory.mcp.tools.search.get_active_project") as mock_get_project:
294 | mock_get_project.return_value.project_url = "http://test"
295 |
296 | with patch(
297 | "basic_memory.mcp.tools.search.call_post", side_effect=Exception("syntax error")
298 | ):
299 | result = await search_notes.fn(project="test-project", query="test query")
300 |
301 | assert isinstance(result, str)
302 | assert "# Search Failed - Invalid Syntax" in result
303 |
304 | @pytest.mark.asyncio
305 | async def test_search_notes_permission_error(self):
306 | """Test search_notes with permission error."""
307 | with patch("basic_memory.mcp.tools.search.get_active_project") as mock_get_project:
308 | mock_get_project.return_value.project_url = "http://test"
309 |
310 | with patch(
311 | "basic_memory.mcp.tools.search.call_post",
312 | side_effect=Exception("permission denied"),
313 | ):
314 | result = await search_notes.fn(project="test-project", query="test query")
315 |
316 | assert isinstance(result, str)
317 | assert "# Search Failed - Access Error" in result
318 |
```
--------------------------------------------------------------------------------
/src/basic_memory/cli/commands/tool.py:
--------------------------------------------------------------------------------
```python
1 | """CLI tool commands for Basic Memory."""
2 |
3 | import asyncio
4 | import sys
5 | from typing import Annotated, List, Optional
6 |
7 | import typer
8 | from loguru import logger
9 | from rich import print as rprint
10 |
11 | from basic_memory.cli.app import app
12 | from basic_memory.config import ConfigManager
13 |
14 | # Import prompts
15 | from basic_memory.mcp.prompts.continue_conversation import (
16 | continue_conversation as mcp_continue_conversation,
17 | )
18 | from basic_memory.mcp.prompts.recent_activity import (
19 | recent_activity_prompt as recent_activity_prompt,
20 | )
21 | from basic_memory.mcp.tools import build_context as mcp_build_context
22 | from basic_memory.mcp.tools import read_note as mcp_read_note
23 | from basic_memory.mcp.tools import recent_activity as mcp_recent_activity
24 | from basic_memory.mcp.tools import search_notes as mcp_search
25 | from basic_memory.mcp.tools import write_note as mcp_write_note
26 | from basic_memory.schemas.base import TimeFrame
27 | from basic_memory.schemas.memory import MemoryUrl
28 | from basic_memory.schemas.search import SearchItemType
29 |
30 | tool_app = typer.Typer()
31 | app.add_typer(tool_app, name="tool", help="Access to MCP tools via CLI")
32 |
33 |
34 | @tool_app.command()
35 | def write_note(
36 | title: Annotated[str, typer.Option(help="The title of the note")],
37 | folder: Annotated[str, typer.Option(help="The folder to create the note in")],
38 | project: Annotated[
39 | Optional[str],
40 | typer.Option(
41 | help="The project to write to. If not provided, the default project will be used."
42 | ),
43 | ] = None,
44 | content: Annotated[
45 | Optional[str],
46 | typer.Option(
47 | help="The content of the note. If not provided, content will be read from stdin. This allows piping content from other commands, e.g.: cat file.md | basic-memory tools write-note"
48 | ),
49 | ] = None,
50 | tags: Annotated[
51 | Optional[List[str]], typer.Option(help="A list of tags to apply to the note")
52 | ] = None,
53 | ):
54 | """Create or update a markdown note. Content can be provided as an argument or read from stdin.
55 |
56 | Content can be provided in two ways:
57 | 1. Using the --content parameter
58 | 2. Piping content through stdin (if --content is not provided)
59 |
60 | Examples:
61 |
62 | # Using content parameter
63 | basic-memory tools write-note --title "My Note" --folder "notes" --content "Note content"
64 |
65 | # Using stdin pipe
66 | echo "# My Note Content" | basic-memory tools write-note --title "My Note" --folder "notes"
67 |
68 | # Using heredoc
69 | cat << EOF | basic-memory tools write-note --title "My Note" --folder "notes"
70 | # My Document
71 |
72 | This is my document content.
73 |
74 | - Point 1
75 | - Point 2
76 | EOF
77 |
78 | # Reading from a file
79 | cat document.md | basic-memory tools write-note --title "Document" --folder "docs"
80 | """
81 | try:
82 | # If content is not provided, read from stdin
83 | if content is None:
84 | # Check if we're getting data from a pipe or redirect
85 | if not sys.stdin.isatty():
86 | content = sys.stdin.read()
87 | else: # pragma: no cover
88 | # If stdin is a terminal (no pipe/redirect), inform the user
89 | typer.echo(
90 | "No content provided. Please provide content via --content or by piping to stdin.",
91 | err=True,
92 | )
93 | raise typer.Exit(1)
94 |
95 | # Also check for empty content
96 | if content is not None and not content.strip():
97 | typer.echo("Empty content provided. Please provide non-empty content.", err=True)
98 | raise typer.Exit(1)
99 |
100 | # look for the project in the config
101 | config_manager = ConfigManager()
102 | project_name = None
103 | if project is not None:
104 | project_name, _ = config_manager.get_project(project)
105 | if not project_name:
106 | typer.echo(f"No project found named: {project}", err=True)
107 | raise typer.Exit(1)
108 |
109 | # use the project name, or the default from the config
110 | project_name = project_name or config_manager.default_project
111 |
112 | note = asyncio.run(mcp_write_note.fn(title, content, folder, project_name, tags))
113 | rprint(note)
114 | except Exception as e: # pragma: no cover
115 | if not isinstance(e, typer.Exit):
116 | typer.echo(f"Error during write_note: {e}", err=True)
117 | raise typer.Exit(1)
118 | raise
119 |
120 |
121 | @tool_app.command()
122 | def read_note(
123 | identifier: str,
124 | project: Annotated[
125 | Optional[str],
126 | typer.Option(
127 | help="The project to use for the note. If not provided, the default project will be used."
128 | ),
129 | ] = None,
130 | page: int = 1,
131 | page_size: int = 10,
132 | ):
133 | """Read a markdown note from the knowledge base."""
134 |
135 | # look for the project in the config
136 | config_manager = ConfigManager()
137 | project_name = None
138 | if project is not None:
139 | project_name, _ = config_manager.get_project(project)
140 | if not project_name:
141 | typer.echo(f"No project found named: {project}", err=True)
142 | raise typer.Exit(1)
143 |
144 | # use the project name, or the default from the config
145 | project_name = project_name or config_manager.default_project
146 |
147 | try:
148 | note = asyncio.run(mcp_read_note.fn(identifier, project_name, page, page_size))
149 | rprint(note)
150 | except Exception as e: # pragma: no cover
151 | if not isinstance(e, typer.Exit):
152 | typer.echo(f"Error during read_note: {e}", err=True)
153 | raise typer.Exit(1)
154 | raise
155 |
156 |
157 | @tool_app.command()
158 | def build_context(
159 | url: MemoryUrl,
160 | project: Annotated[
161 | Optional[str],
162 | typer.Option(help="The project to use. If not provided, the default project will be used."),
163 | ] = None,
164 | depth: Optional[int] = 1,
165 | timeframe: Optional[TimeFrame] = "7d",
166 | page: int = 1,
167 | page_size: int = 10,
168 | max_related: int = 10,
169 | ):
170 | """Get context needed to continue a discussion."""
171 |
172 | # look for the project in the config
173 | config_manager = ConfigManager()
174 | project_name = None
175 | if project is not None:
176 | project_name, _ = config_manager.get_project(project)
177 | if not project_name:
178 | typer.echo(f"No project found named: {project}", err=True)
179 | raise typer.Exit(1)
180 |
181 | # use the project name, or the default from the config
182 | project_name = project_name or config_manager.default_project
183 |
184 | try:
185 | context = asyncio.run(
186 | mcp_build_context.fn(
187 | project=project_name,
188 | url=url,
189 | depth=depth,
190 | timeframe=timeframe,
191 | page=page,
192 | page_size=page_size,
193 | max_related=max_related,
194 | )
195 | )
196 | # Use json module for more controlled serialization
197 | import json
198 |
199 | context_dict = context.model_dump(exclude_none=True)
200 | print(json.dumps(context_dict, indent=2, ensure_ascii=True, default=str))
201 | except Exception as e: # pragma: no cover
202 | if not isinstance(e, typer.Exit):
203 | typer.echo(f"Error during build_context: {e}", err=True)
204 | raise typer.Exit(1)
205 | raise
206 |
207 |
208 | @tool_app.command()
209 | def recent_activity(
210 | type: Annotated[Optional[List[SearchItemType]], typer.Option()] = None,
211 | depth: Optional[int] = 1,
212 | timeframe: Optional[TimeFrame] = "7d",
213 | ):
214 | """Get recent activity across the knowledge base."""
215 | try:
216 | result = asyncio.run(
217 | mcp_recent_activity.fn(
218 | type=type, # pyright: ignore [reportArgumentType]
219 | depth=depth,
220 | timeframe=timeframe,
221 | )
222 | )
223 | # The tool now returns a formatted string directly
224 | print(result)
225 | except Exception as e: # pragma: no cover
226 | if not isinstance(e, typer.Exit):
227 | typer.echo(f"Error during recent_activity: {e}", err=True)
228 | raise typer.Exit(1)
229 | raise
230 |
231 |
232 | @tool_app.command("search-notes")
233 | def search_notes(
234 | query: str,
235 | permalink: Annotated[bool, typer.Option("--permalink", help="Search permalink values")] = False,
236 | title: Annotated[bool, typer.Option("--title", help="Search title values")] = False,
237 | project: Annotated[
238 | Optional[str],
239 | typer.Option(
240 | help="The project to use for the note. If not provided, the default project will be used."
241 | ),
242 | ] = None,
243 | after_date: Annotated[
244 | Optional[str],
245 | typer.Option("--after_date", help="Search results after date, eg. '2d', '1 week'"),
246 | ] = None,
247 | page: int = 1,
248 | page_size: int = 10,
249 | ):
250 | """Search across all content in the knowledge base."""
251 |
252 | # look for the project in the config
253 | config_manager = ConfigManager()
254 | project_name = None
255 | if project is not None:
256 | project_name, _ = config_manager.get_project(project)
257 | if not project_name:
258 | typer.echo(f"No project found named: {project}", err=True)
259 | raise typer.Exit(1)
260 |
261 | # use the project name, or the default from the config
262 | project_name = project_name or config_manager.default_project
263 |
264 | if permalink and title: # pragma: no cover
265 | print("Cannot search both permalink and title")
266 | raise typer.Abort()
267 |
268 | try:
269 | if permalink and title: # pragma: no cover
270 | typer.echo(
271 | "Use either --permalink or --title, not both. Exiting.",
272 | err=True,
273 | )
274 | raise typer.Exit(1)
275 |
276 | # set search type
277 | search_type = ("permalink" if permalink else None,)
278 | search_type = ("permalink_match" if permalink and "*" in query else None,)
279 | search_type = ("title" if title else None,)
280 | search_type = "text" if search_type is None else search_type
281 |
282 | results = asyncio.run(
283 | mcp_search.fn(
284 | query,
285 | project_name,
286 | search_type=search_type,
287 | page=page,
288 | after_date=after_date,
289 | page_size=page_size,
290 | )
291 | )
292 | # Use json module for more controlled serialization
293 | import json
294 |
295 | results_dict = results.model_dump(exclude_none=True)
296 | print(json.dumps(results_dict, indent=2, ensure_ascii=True, default=str))
297 | except Exception as e: # pragma: no cover
298 | if not isinstance(e, typer.Exit):
299 | logger.exception("Error during search", e)
300 | typer.echo(f"Error during search: {e}", err=True)
301 | raise typer.Exit(1)
302 | raise
303 |
304 |
305 | @tool_app.command(name="continue-conversation")
306 | def continue_conversation(
307 | topic: Annotated[Optional[str], typer.Option(help="Topic or keyword to search for")] = None,
308 | timeframe: Annotated[
309 | Optional[str], typer.Option(help="How far back to look for activity")
310 | ] = None,
311 | ):
312 | """Prompt to continue a previous conversation or work session."""
313 | try:
314 | # Prompt functions return formatted strings directly
315 | session = asyncio.run(mcp_continue_conversation.fn(topic=topic, timeframe=timeframe)) # type: ignore
316 | rprint(session)
317 | except Exception as e: # pragma: no cover
318 | if not isinstance(e, typer.Exit):
319 | logger.exception("Error continuing conversation", e)
320 | typer.echo(f"Error continuing conversation: {e}", err=True)
321 | raise typer.Exit(1)
322 | raise
323 |
324 |
325 | # @tool_app.command(name="show-recent-activity")
326 | # def show_recent_activity(
327 | # timeframe: Annotated[
328 | # str, typer.Option(help="How far back to look for activity")
329 | # ] = "7d",
330 | # ):
331 | # """Prompt to show recent activity."""
332 | # try:
333 | # # Prompt functions return formatted strings directly
334 | # session = asyncio.run(recent_activity_prompt(timeframe=timeframe))
335 | # rprint(session)
336 | # except Exception as e: # pragma: no cover
337 | # if not isinstance(e, typer.Exit):
338 | # logger.exception("Error continuing conversation", e)
339 | # typer.echo(f"Error continuing conversation: {e}", err=True)
340 | # raise typer.Exit(1)
341 | # raise
342 |
```
--------------------------------------------------------------------------------
/tests/repository/test_relation_repository.py:
--------------------------------------------------------------------------------
```python
1 | """Tests for the RelationRepository."""
2 |
3 | from datetime import datetime, timezone
4 |
5 | import pytest
6 | import pytest_asyncio
7 | import sqlalchemy
8 |
9 | from basic_memory import db
10 | from basic_memory.models import Entity, Relation, Project
11 | from basic_memory.repository.relation_repository import RelationRepository
12 |
13 |
14 | @pytest_asyncio.fixture
15 | async def source_entity(session_maker, test_project: Project):
16 | """Create a source entity for testing relations."""
17 | entity = Entity(
18 | project_id=test_project.id,
19 | title="test_source",
20 | entity_type="test",
21 | permalink="source/test-source",
22 | file_path="source/test_source.md",
23 | content_type="text/markdown",
24 | created_at=datetime.now(timezone.utc),
25 | updated_at=datetime.now(timezone.utc),
26 | )
27 | async with db.scoped_session(session_maker) as session:
28 | session.add(entity)
29 | await session.flush()
30 | return entity
31 |
32 |
33 | @pytest_asyncio.fixture
34 | async def target_entity(session_maker, test_project: Project):
35 | """Create a target entity for testing relations."""
36 | entity = Entity(
37 | project_id=test_project.id,
38 | title="test_target",
39 | entity_type="test",
40 | permalink="target/test-target",
41 | file_path="target/test_target.md",
42 | content_type="text/markdown",
43 | created_at=datetime.now(timezone.utc),
44 | updated_at=datetime.now(timezone.utc),
45 | )
46 | async with db.scoped_session(session_maker) as session:
47 | session.add(entity)
48 | await session.flush()
49 | return entity
50 |
51 |
52 | @pytest_asyncio.fixture
53 | async def test_relations(session_maker, source_entity, target_entity):
54 | """Create test relations."""
55 | relations = [
56 | Relation(
57 | from_id=source_entity.id,
58 | to_id=target_entity.id,
59 | to_name=target_entity.title,
60 | relation_type="connects_to",
61 | ),
62 | Relation(
63 | from_id=source_entity.id,
64 | to_id=target_entity.id,
65 | to_name=target_entity.title,
66 | relation_type="depends_on",
67 | ),
68 | ]
69 | async with db.scoped_session(session_maker) as session:
70 | session.add_all(relations)
71 | await session.flush()
72 | return relations
73 |
74 |
75 | @pytest_asyncio.fixture(scope="function")
76 | async def related_entity(entity_repository):
77 | """Create a second entity for testing relations"""
78 | entity_data = {
79 | "title": "Related Entity",
80 | "entity_type": "test",
81 | "permalink": "test/related-entity",
82 | "file_path": "test/related_entity.md",
83 | "summary": "A related test entity",
84 | "content_type": "text/markdown",
85 | "created_at": datetime.now(timezone.utc),
86 | "updated_at": datetime.now(timezone.utc),
87 | }
88 | return await entity_repository.create(entity_data)
89 |
90 |
91 | @pytest_asyncio.fixture(scope="function")
92 | async def sample_relation(
93 | relation_repository: RelationRepository, sample_entity: Entity, related_entity: Entity
94 | ):
95 | """Create a sample relation for testing"""
96 | relation_data = {
97 | "from_id": sample_entity.id,
98 | "to_id": related_entity.id,
99 | "to_name": related_entity.title,
100 | "relation_type": "test_relation",
101 | "context": "test-context",
102 | }
103 | return await relation_repository.create(relation_data)
104 |
105 |
106 | @pytest_asyncio.fixture(scope="function")
107 | async def multiple_relations(
108 | relation_repository: RelationRepository, sample_entity: Entity, related_entity: Entity
109 | ):
110 | """Create multiple relations for testing"""
111 | relations_data = [
112 | {
113 | "from_id": sample_entity.id,
114 | "to_id": related_entity.id,
115 | "to_name": related_entity.title,
116 | "relation_type": "relation_one",
117 | "context": "context_one",
118 | },
119 | {
120 | "from_id": sample_entity.id,
121 | "to_id": related_entity.id,
122 | "to_name": related_entity.title,
123 | "relation_type": "relation_two",
124 | "context": "context_two",
125 | },
126 | {
127 | "from_id": related_entity.id,
128 | "to_id": sample_entity.id,
129 | "to_name": related_entity.title,
130 | "relation_type": "relation_one",
131 | "context": "context_three",
132 | },
133 | ]
134 | return [await relation_repository.create(data) for data in relations_data]
135 |
136 |
137 | @pytest.mark.asyncio
138 | async def test_create_relation(
139 | relation_repository: RelationRepository, sample_entity: Entity, related_entity: Entity
140 | ):
141 | """Test creating a new relation"""
142 | relation_data = {
143 | "from_id": sample_entity.id,
144 | "to_id": related_entity.id,
145 | "to_name": related_entity.title,
146 | "relation_type": "test_relation",
147 | "context": "test-context",
148 | }
149 | relation = await relation_repository.create(relation_data)
150 |
151 | assert relation.from_id == sample_entity.id
152 | assert relation.to_id == related_entity.id
153 | assert relation.relation_type == "test_relation"
154 | assert relation.id is not None # Should be auto-generated
155 |
156 |
157 | @pytest.mark.asyncio
158 | async def test_create_relation_entity_does_not_exist(
159 | relation_repository: RelationRepository, sample_entity: Entity, related_entity: Entity
160 | ):
161 | """Test creating a new relation"""
162 | relation_data = {
163 | "from_id": "not_exist",
164 | "to_id": related_entity.id,
165 | "to_name": related_entity.title,
166 | "relation_type": "test_relation",
167 | "context": "test-context",
168 | }
169 | with pytest.raises(sqlalchemy.exc.IntegrityError):
170 | await relation_repository.create(relation_data)
171 |
172 |
173 | @pytest.mark.asyncio
174 | async def test_find_by_entities(
175 | relation_repository: RelationRepository,
176 | sample_relation: Relation,
177 | sample_entity: Entity,
178 | related_entity: Entity,
179 | ):
180 | """Test finding relations between specific entities"""
181 | relations = await relation_repository.find_by_entities(sample_entity.id, related_entity.id)
182 | assert len(relations) == 1
183 | assert relations[0].id == sample_relation.id
184 | assert relations[0].relation_type == sample_relation.relation_type
185 |
186 |
187 | @pytest.mark.asyncio
188 | async def test_find_relation(relation_repository: RelationRepository, sample_relation: Relation):
189 | """Test finding relations by type"""
190 | relation = await relation_repository.find_relation(
191 | from_permalink=sample_relation.from_entity.permalink,
192 | to_permalink=sample_relation.to_entity.permalink,
193 | relation_type=sample_relation.relation_type,
194 | )
195 | assert relation.id == sample_relation.id
196 |
197 |
198 | @pytest.mark.asyncio
199 | async def test_find_by_type(relation_repository: RelationRepository, sample_relation: Relation):
200 | """Test finding relations by type"""
201 | relations = await relation_repository.find_by_type("test_relation")
202 | assert len(relations) == 1
203 | assert relations[0].id == sample_relation.id
204 |
205 |
206 | @pytest.mark.asyncio
207 | async def test_find_unresolved_relations(
208 | relation_repository: RelationRepository, sample_entity: Entity, related_entity: Entity
209 | ):
210 | """Test creating a new relation"""
211 | relation_data = {
212 | "from_id": sample_entity.id,
213 | "to_id": None,
214 | "to_name": related_entity.title,
215 | "relation_type": "test_relation",
216 | "context": "test-context",
217 | }
218 | relation = await relation_repository.create(relation_data)
219 |
220 | assert relation.from_id == sample_entity.id
221 | assert relation.to_id is None
222 |
223 | unresolved = await relation_repository.find_unresolved_relations()
224 | assert len(unresolved) == 1
225 | assert unresolved[0].id == relation.id
226 |
227 |
228 | @pytest.mark.asyncio
229 | async def test_delete_by_fields_single_field(
230 | relation_repository: RelationRepository, multiple_relations: list[Relation]
231 | ):
232 | """Test deleting relations by a single field."""
233 | # Delete all relations of type 'relation_one'
234 | result = await relation_repository.delete_by_fields(relation_type="relation_one") # pyright: ignore [reportArgumentType]
235 | assert result is True
236 |
237 | # Verify deletion
238 | remaining = await relation_repository.find_by_type("relation_one")
239 | assert len(remaining) == 0
240 |
241 | # Other relations should still exist
242 | others = await relation_repository.find_by_type("relation_two")
243 | assert len(others) == 1
244 |
245 |
246 | @pytest.mark.asyncio
247 | async def test_delete_by_fields_multiple_fields(
248 | relation_repository: RelationRepository,
249 | multiple_relations: list[Relation],
250 | sample_entity: Entity,
251 | related_entity: Entity,
252 | ):
253 | """Test deleting relations by multiple fields."""
254 | # Delete specific relation matching both from_id and relation_type
255 | result = await relation_repository.delete_by_fields(
256 | from_id=sample_entity.id, # pyright: ignore [reportArgumentType]
257 | relation_type="relation_one", # pyright: ignore [reportArgumentType]
258 | )
259 | assert result is True
260 |
261 | # Verify correct relation was deleted
262 | remaining = await relation_repository.find_by_entities(sample_entity.id, related_entity.id)
263 | assert len(remaining) == 1 # Only relation_two should remain
264 | assert remaining[0].relation_type == "relation_two"
265 |
266 |
267 | @pytest.mark.asyncio
268 | async def test_delete_by_fields_no_match(
269 | relation_repository: RelationRepository, multiple_relations: list[Relation]
270 | ):
271 | """Test delete_by_fields when no relations match."""
272 | result = await relation_repository.delete_by_fields(
273 | relation_type="nonexistent_type" # pyright: ignore [reportArgumentType]
274 | )
275 | assert result is False
276 |
277 |
278 | @pytest.mark.asyncio
279 | async def test_delete_by_fields_all_fields(
280 | relation_repository: RelationRepository,
281 | multiple_relations: list[Relation],
282 | sample_entity: Entity,
283 | related_entity: Entity,
284 | ):
285 | """Test deleting relation by matching all fields."""
286 | # Get first relation's data
287 | relation = multiple_relations[0]
288 |
289 | # Delete using all fields
290 | result = await relation_repository.delete_by_fields(
291 | from_id=relation.from_id, # pyright: ignore [reportArgumentType]
292 | to_id=relation.to_id, # pyright: ignore [reportArgumentType]
293 | relation_type=relation.relation_type, # pyright: ignore [reportArgumentType]
294 | )
295 | assert result is True
296 |
297 | # Verify only exact match was deleted
298 | remaining = await relation_repository.find_by_type(relation.relation_type)
299 | assert len(remaining) == 1 # One other relation_one should remain
300 |
301 |
302 | @pytest.mark.asyncio
303 | async def test_delete_relation_by_id(relation_repository, test_relations):
304 | """Test deleting a relation by ID."""
305 | relation = test_relations[0]
306 |
307 | result = await relation_repository.delete(relation.id)
308 | assert result is True
309 |
310 | # Verify deletion
311 | remaining = await relation_repository.find_one(
312 | relation_repository.select(Relation).filter(Relation.id == relation.id)
313 | )
314 | assert remaining is None
315 |
316 |
317 | @pytest.mark.asyncio
318 | async def test_delete_relations_by_type(relation_repository, test_relations):
319 | """Test deleting relations by type."""
320 | result = await relation_repository.delete_by_fields(relation_type="connects_to")
321 | assert result is True
322 |
323 | # Verify specific type was deleted
324 | remaining = await relation_repository.find_by_type("connects_to")
325 | assert len(remaining) == 0
326 |
327 | # Verify other type still exists
328 | others = await relation_repository.find_by_type("depends_on")
329 | assert len(others) == 1
330 |
331 |
332 | @pytest.mark.asyncio
333 | async def test_delete_relations_by_entities(
334 | relation_repository, test_relations, source_entity, target_entity
335 | ):
336 | """Test deleting relations between specific entities."""
337 | result = await relation_repository.delete_by_fields(
338 | from_id=source_entity.id, to_id=target_entity.id
339 | )
340 | assert result is True
341 |
342 | # Verify all relations between entities were deleted
343 | remaining = await relation_repository.find_by_entities(source_entity.id, target_entity.id)
344 | assert len(remaining) == 0
345 |
346 |
347 | @pytest.mark.asyncio
348 | async def test_delete_nonexistent_relation(relation_repository):
349 | """Test deleting a relation that doesn't exist."""
350 | result = await relation_repository.delete_by_fields(relation_type="nonexistent")
351 | assert result is False
352 |
```
--------------------------------------------------------------------------------
/test-int/mcp/test_read_content_integration.py:
--------------------------------------------------------------------------------
```python
1 | """
2 | Integration tests for read_content MCP tool.
3 |
4 | Comprehensive tests covering text files, binary files, images, error cases,
5 | and memory:// URL handling via the complete MCP client-server flow.
6 | """
7 |
8 | import json
9 | import pytest
10 | from fastmcp import Client
11 | from fastmcp.exceptions import ToolError
12 |
13 |
14 | def parse_read_content_response(mcp_result):
15 | """Helper function to parse read_content MCP response."""
16 | assert len(mcp_result.content) == 1
17 | assert mcp_result.content[0].type == "text"
18 | return json.loads(mcp_result.content[0].text)
19 |
20 |
21 | @pytest.mark.asyncio
22 | async def test_read_content_markdown_file(mcp_server, app, test_project):
23 | """Test reading a markdown file created by write_note."""
24 |
25 | async with Client(mcp_server) as client:
26 | # First create a note
27 | await client.call_tool(
28 | "write_note",
29 | {
30 | "project": test_project.name,
31 | "title": "Content Test",
32 | "folder": "test",
33 | "content": "# Content Test\n\nThis is test content with **markdown**.",
34 | "tags": "test,content",
35 | },
36 | )
37 |
38 | # Then read the raw file content
39 | read_result = await client.call_tool(
40 | "read_content",
41 | {
42 | "project": test_project.name,
43 | "path": "test/Content Test.md",
44 | },
45 | )
46 |
47 | # Parse the response
48 | response_data = parse_read_content_response(read_result)
49 |
50 | assert response_data["type"] == "text"
51 | assert response_data["content_type"] == "text/markdown; charset=utf-8"
52 | assert response_data["encoding"] == "utf-8"
53 |
54 | content = response_data["text"]
55 |
56 | # Should contain the raw markdown with frontmatter
57 | assert "# Content Test" in content
58 | assert "This is test content with **markdown**." in content
59 | assert "tags:" in content # frontmatter
60 | assert "- test" in content # tags are in YAML list format
61 | assert "- content" in content
62 |
63 |
64 | @pytest.mark.asyncio
65 | async def test_read_content_by_permalink(mcp_server, app, test_project):
66 | """Test reading content using permalink instead of file path."""
67 |
68 | async with Client(mcp_server) as client:
69 | # Create a note
70 | await client.call_tool(
71 | "write_note",
72 | {
73 | "project": test_project.name,
74 | "title": "Permalink Test",
75 | "folder": "docs",
76 | "content": "# Permalink Test\n\nTesting permalink-based content reading.",
77 | },
78 | )
79 |
80 | # Read by permalink (without .md extension)
81 | read_result = await client.call_tool(
82 | "read_content",
83 | {
84 | "project": test_project.name,
85 | "path": "docs/permalink-test",
86 | },
87 | )
88 |
89 | # Parse the response
90 | response_data = parse_read_content_response(read_result)
91 | content = response_data["text"]
92 |
93 | assert "# Permalink Test" in content
94 | assert "Testing permalink-based content reading." in content
95 |
96 |
97 | @pytest.mark.asyncio
98 | async def test_read_content_memory_url(mcp_server, app, test_project):
99 | """Test reading content using memory:// URL format."""
100 |
101 | async with Client(mcp_server) as client:
102 | # Create a note
103 | await client.call_tool(
104 | "write_note",
105 | {
106 | "project": test_project.name,
107 | "title": "Memory URL Test",
108 | "folder": "test",
109 | "content": "# Memory URL Test\n\nTesting memory:// URL handling.",
110 | "tags": "memory,url",
111 | },
112 | )
113 |
114 | # Read using memory:// URL
115 | read_result = await client.call_tool(
116 | "read_content",
117 | {
118 | "project": test_project.name,
119 | "path": "memory://test/memory-url-test",
120 | },
121 | )
122 |
123 | # Parse the response
124 | response_data = parse_read_content_response(read_result)
125 | content = response_data["text"]
126 |
127 | assert "# Memory URL Test" in content
128 | assert "Testing memory:// URL handling." in content
129 |
130 |
131 | @pytest.mark.asyncio
132 | async def test_read_content_unicode_file(mcp_server, app, test_project):
133 | """Test reading content with unicode characters and emojis."""
134 |
135 | async with Client(mcp_server) as client:
136 | # Create a note with unicode content
137 | unicode_content = (
138 | "# Unicode Test 🚀\n\nThis note has emoji 🎉 and unicode ♠♣♥♦\n\n测试中文内容"
139 | )
140 |
141 | await client.call_tool(
142 | "write_note",
143 | {
144 | "project": test_project.name,
145 | "title": "Unicode Content Test",
146 | "folder": "test",
147 | "content": unicode_content,
148 | "tags": "unicode,emoji",
149 | },
150 | )
151 |
152 | # Read the content back
153 | read_result = await client.call_tool(
154 | "read_content",
155 | {
156 | "project": test_project.name,
157 | "path": "test/Unicode Content Test.md",
158 | },
159 | )
160 |
161 | # Parse the response
162 | response_data = parse_read_content_response(read_result)
163 | content = response_data["text"]
164 |
165 | # All unicode content should be preserved
166 | assert "🚀" in content
167 | assert "🎉" in content
168 | assert "♠♣♥♦" in content
169 | assert "测试中文内容" in content
170 |
171 |
172 | @pytest.mark.asyncio
173 | async def test_read_content_complex_frontmatter(mcp_server, app, test_project):
174 | """Test reading content with complex frontmatter and markdown."""
175 |
176 | async with Client(mcp_server) as client:
177 | # Create a note with complex content
178 | complex_content = """---
179 | title: Complex Note
180 | type: document
181 | version: 1.0
182 | author: Test Author
183 | metadata:
184 | status: draft
185 | priority: high
186 | ---
187 |
188 | # Complex Note
189 |
190 | This note has complex frontmatter and various markdown elements.
191 |
192 | ## Observations
193 | - [tech] Uses YAML frontmatter
194 | - [design] Structured content format
195 |
196 | ## Relations
197 | - related_to [[Other Note]]
198 | - depends_on [[Framework]]
199 |
200 | Regular markdown content continues here."""
201 |
202 | await client.call_tool(
203 | "write_note",
204 | {
205 | "project": test_project.name,
206 | "title": "Complex Note",
207 | "folder": "docs",
208 | "content": complex_content,
209 | "tags": "complex,frontmatter",
210 | },
211 | )
212 |
213 | # Read the content back
214 | read_result = await client.call_tool(
215 | "read_content",
216 | {
217 | "project": test_project.name,
218 | "path": "docs/Complex Note.md",
219 | },
220 | )
221 |
222 | # Parse the response
223 | response_data = parse_read_content_response(read_result)
224 | content = response_data["text"]
225 |
226 | # Should preserve all frontmatter and content structure
227 | assert "version: 1.0" in content
228 | assert "author: Test Author" in content
229 | assert "status: draft" in content
230 | assert "[tech] Uses YAML frontmatter" in content
231 | assert "[[Other Note]]" in content
232 |
233 |
234 | @pytest.mark.asyncio
235 | async def test_read_content_missing_file(mcp_server, app, test_project):
236 | """Test reading a file that doesn't exist."""
237 |
238 | async with Client(mcp_server) as client:
239 | try:
240 | await client.call_tool(
241 | "read_content",
242 | {
243 | "project": test_project.name,
244 | "path": "nonexistent/file.md",
245 | },
246 | )
247 | # Should not reach here - expecting an error
248 | assert False, "Expected error for missing file"
249 | except ToolError as e:
250 | # Should get an appropriate error message
251 | error_msg = str(e).lower()
252 | assert "not found" in error_msg or "does not exist" in error_msg
253 |
254 |
255 | @pytest.mark.asyncio
256 | async def test_read_content_empty_file(mcp_server, app, test_project):
257 | """Test reading an empty file."""
258 |
259 | async with Client(mcp_server) as client:
260 | # Create a note with minimal content
261 | await client.call_tool(
262 | "write_note",
263 | {
264 | "project": test_project.name,
265 | "title": "Empty Test",
266 | "folder": "test",
267 | "content": "", # Empty content
268 | },
269 | )
270 |
271 | # Read the content back
272 | read_result = await client.call_tool(
273 | "read_content",
274 | {
275 | "project": test_project.name,
276 | "path": "test/Empty Test.md",
277 | },
278 | )
279 |
280 | # Parse the response
281 | response_data = parse_read_content_response(read_result)
282 | content = response_data["text"]
283 |
284 | # Should still have frontmatter even with empty content
285 | assert "title: Empty Test" in content
286 | assert "permalink: test/empty-test" in content
287 |
288 |
289 | @pytest.mark.asyncio
290 | async def test_read_content_large_file(mcp_server, app, test_project):
291 | """Test reading a file with substantial content."""
292 |
293 | async with Client(mcp_server) as client:
294 | # Create a note with substantial content
295 | large_content = "# Large Content Test\n\n"
296 |
297 | # Add multiple sections with substantial text
298 | for i in range(10):
299 | large_content += f"""
300 | ## Section {i + 1}
301 |
302 | This is section {i + 1} with substantial content. Lorem ipsum dolor sit amet,
303 | consectetur adipiscing elit. Sed do eiusmod tempor incididunt ut labore et
304 | dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation.
305 |
306 | - [note] This is observation {i + 1}
307 | - related_to [[Section {i}]]
308 |
309 | Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore
310 | eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident.
311 |
312 | """
313 |
314 | await client.call_tool(
315 | "write_note",
316 | {
317 | "project": test_project.name,
318 | "title": "Large Content Note",
319 | "folder": "test",
320 | "content": large_content,
321 | "tags": "large,content,test",
322 | },
323 | )
324 |
325 | # Read the content back
326 | read_result = await client.call_tool(
327 | "read_content",
328 | {
329 | "project": test_project.name,
330 | "path": "test/Large Content Note.md",
331 | },
332 | )
333 |
334 | # Parse the response
335 | response_data = parse_read_content_response(read_result)
336 | content = response_data["text"]
337 |
338 | # Should contain all sections
339 | assert "Section 1" in content
340 | assert "Section 10" in content
341 | assert "Lorem ipsum" in content
342 | assert len(content) > 1000 # Should be substantial
343 |
344 |
345 | @pytest.mark.asyncio
346 | async def test_read_content_special_characters_in_filename(mcp_server, app, test_project):
347 | """Test reading files with special characters in the filename."""
348 |
349 | async with Client(mcp_server) as client:
350 | # Create notes with special characters in titles
351 | test_cases = [
352 | ("File with spaces", "test"),
353 | ("File-with-dashes", "test"),
354 | ("File_with_underscores", "test"),
355 | ("File (with parentheses)", "test"),
356 | ("File & Symbols!", "test"),
357 | ]
358 |
359 | for title, folder in test_cases:
360 | await client.call_tool(
361 | "write_note",
362 | {
363 | "project": test_project.name,
364 | "title": title,
365 | "folder": folder,
366 | "content": f"# {title}\n\nContent for {title}",
367 | },
368 | )
369 |
370 | # Read the content back using the exact filename
371 | read_result = await client.call_tool(
372 | "read_content",
373 | {
374 | "project": test_project.name,
375 | "path": f"{folder}/{title}.md",
376 | },
377 | )
378 |
379 | assert len(read_result.content) == 1
380 | assert read_result.content[0].type == "text"
381 | content = read_result.content[0].text
382 |
383 | assert f"# {title}" in content
384 | assert f"Content for {title}" in content
385 |
```